From 461b0f47fc921abc595971e1f8c597d85e0918af Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Fri, 29 May 2026 16:21:46 -0500 Subject: [PATCH 001/127] Add recommendation_v4 (HSTU/DLRM-v3 generative-recommenders fork @ d97e51c) Vendored snapshot of chriscai-amd/generative-recommenders branch chcai/dlrmv4 (HEAD d97e51c) as a sibling of recommendation_v2/torchrec_dlrm. The Python package generative_recommenders keeps its original name so all imports work unchanged from the new location. - recommendation_v4/generative_recommenders/: dlrm_v3, modules, ops, research, tests - recommendation_v4/configs/: research HSTU gins - recommendation_v4/scripts/launch_smoke_8gpu.sh: sanitized 8-GPU yambda-5b launcher (resolves package root from script path; AMD env defaults; pip_local override) - recommendation_v4/{setup.py,requirements.txt,main.py,...}: upstream entry points - .gitmodules: cutlass registered at parent repo level Co-Authored-By: Claude Opus 4.7 --- .gitmodules | 3 + recommendation_v4/.gitignore | 158 + recommendation_v4/LICENSE | 202 ++ recommendation_v4/README.MD | 135 + .../hstu-sampled-softmax-n512-final.gin | 49 + .../hstu-sampled-softmax-n512-large-final.gin | 49 + .../sasrec-sampled-softmax-n512-final.gin | 50 + .../ml-1m/hstu-sampled-softmax-n128-final.gin | 45 + .../hstu-sampled-softmax-n128-large-final.gin | 45 + .../sasrec-sampled-softmax-n128-final.gin | 44 + .../hstu-sampled-softmax-n128-final.gin | 45 + .../hstu-sampled-softmax-n128-large-final.gin | 45 + .../sasrec-sampled-softmax-n128-final.gin | 44 + ...tu-sampled-softmax-n96-seqlen500-final.gin | 42 + ...pled-softmax-n96-seqlen500-large-final.gin | 42 + ...ec-sampled-softmax-n96-seqlen500-final.gin | 42 + .../generative_recommenders/README.md | 0 .../generative_recommenders/common.py | 513 +++ .../dlrm_v3/checkpoint.py | 258 ++ .../dlrm_v3/configs.py | 704 ++++ .../dlrm_v3/datasets/dataset.py | 398 +++ .../dlrm_v3/datasets/kuairand.py | 163 + .../dlrm_v3/datasets/movie_lens.py | 177 + .../dlrm_v3/datasets/synthetic_movie_lens.py | 83 + .../dlrm_v3/datasets/synthetic_streaming.py | 400 +++ .../dlrm_v3/datasets/utils.py | 146 + .../dlrm_v3/datasets/yambda.py | 608 ++++ .../dlrm_v3/inference/README.md | 88 + .../dlrm_v3/inference/accuracy.py | 86 + .../dlrm_v3/inference/cpp/hstu_runner.cpp | 215 ++ .../dlrm_v3/inference/data_producer.py | 227 ++ .../dlrm_v3/inference/dense_predict_module.py | 96 + .../dlrm_v3/inference/end_to_end_test.py | 795 +++++ .../dlrm_v3/inference/gin/debug.gin | 13 + .../dlrm_v3/inference/gin/kuairand_1k.gin | 14 + .../dlrm_v3/inference/gin/movielens_13b.gin | 16 + .../dlrm_v3/inference/gin/streaming_100b.gin | 15 + .../dlrm_v3/inference/gin/streaming_400m.gin | 15 + .../dlrm_v3/inference/inference_modules.py | 253 ++ .../dlrm_v3/inference/main.py | 805 +++++ .../dlrm_v3/inference/mlperf.conf | 98 + .../dlrm_v3/inference/model_family.py | 705 ++++ .../inference/sparse_predict_module.py | 106 + .../dlrm_v3/inference/tests/inference_test.py | 39 + .../inference/tests/test_scripted_parity.py | 236 ++ .../thirdparty/loadgen/.clang-format | 2 + .../thirdparty/loadgen/CMakeLists.txt | 113 + .../inference/thirdparty/loadgen/MANIFEST.in | 2 + .../inference/thirdparty/loadgen/README.md | 223 ++ .../thirdparty/loadgen/README_BUILD.md | 47 + .../thirdparty/loadgen/README_FAQ.md | 78 + .../inference/thirdparty/loadgen/VERSION.txt | 1 + .../thirdparty/loadgen/benchmark/.gitignore | 2 + .../thirdparty/loadgen/benchmark/README.md | 10 + .../thirdparty/loadgen/benchmark/repro.cpp | 296 ++ .../thirdparty/loadgen/benchmark/run.sh | 21 + .../thirdparty/loadgen/benchmark/run_debug.sh | 21 + .../thirdparty/loadgen/bindings/c_api.cc | 176 + .../thirdparty/loadgen/bindings/c_api.h | 95 + .../thirdparty/loadgen/bindings/python_api.cc | 484 +++ .../thirdparty/loadgen/demos/lon/README.md | 67 + .../loadgen/demos/lon/py_demo_server_lon.py | 191 + .../demos/lon/sut_over_network_demo.py | 88 + .../loadgen/demos/py_demo_multi_stream.py | 86 + .../loadgen/demos/py_demo_offline.py | 81 + .../loadgen/demos/py_demo_server.py | 74 + .../loadgen/demos/py_demo_single_stream.py | 84 + .../token_metrics/py_demo_multi_stream.py | 142 + .../demos/token_metrics/py_demo_offline.py | 130 + .../token_metrics/py_demo_offline_inferred.py | 130 + .../demos/token_metrics/py_demo_server.py | 132 + .../token_metrics/py_demo_server_inferred.py | 125 + .../token_metrics/py_demo_single_stream.py | 129 + .../loadgen/diagram_network_submission.png | Bin 0 -> 51192 bytes .../thirdparty/loadgen/diagram_submission.png | Bin 0 -> 36510 bytes .../thirdparty/loadgen/docs/src/BUILD.gn | 33 + .../thirdparty/loadgen/docs/src/README.md | 34 + .../thirdparty/loadgen/docs/src/doxygen.cfg | 2495 +++++++++++++ .../loadgen/docs/src/doxygen_footer.html | 26 + .../loadgen/docs/src/doxygen_header.html | 49 + .../docs/src/doxygen_html_generator.py | 37 + .../loadgen/docs/src/doxygen_layout.xml | 211 ++ .../loadgen/docs/src/doxygen_stylesheet.css | 1629 +++++++++ .../docs/src/loadgen_integration_diagram.dia | Bin 0 -> 1943 bytes .../loadgen/docs/src/mlperf_icon.png | Bin 0 -> 4632 bytes .../docs/src/mlperf_logo_horizontal_color.svg | 55 + .../thirdparty/loadgen/early_stopping.cc | 117 + .../thirdparty/loadgen/early_stopping.h | 27 + .../loadgen/generated/version_generated.cc | 98 + .../loadgen/issue_query_controller.cc | 552 +++ .../loadgen/issue_query_controller.h | 215 ++ .../inference/thirdparty/loadgen/loadgen.cc | 1345 +++++++ .../inference/thirdparty/loadgen/loadgen.h | 103 + .../loadgen/loadgen_integration_diagram.svg | 85 + .../inference/thirdparty/loadgen/logging.cc | 1301 +++++++ .../inference/thirdparty/loadgen/logging.h | 816 +++++ .../inference/thirdparty/loadgen/mlperf.conf | 164 + .../thirdparty/loadgen/mlperf_conf.h | 167 + .../thirdparty/loadgen/pyproject.toml | 7 + .../loadgen/query_dispatch_library.h | 42 + .../thirdparty/loadgen/query_sample.h | 91 + .../thirdparty/loadgen/query_sample_library.h | 75 + .../thirdparty/loadgen/requirements.txt | 1 + .../inference/thirdparty/loadgen/results.cc | 856 +++++ .../inference/thirdparty/loadgen/results.h | 128 + .../inference/thirdparty/loadgen/setup.py | 136 + .../thirdparty/loadgen/system_under_test.h | 67 + .../thirdparty/loadgen/test_settings.h | 329 ++ .../loadgen/test_settings_internal.cc | 800 +++++ .../loadgen/test_settings_internal.h | 182 + .../thirdparty/loadgen/tests/BUILD.gn | 25 + .../thirdparty/loadgen/tests/README.md | 42 + .../thirdparty/loadgen/tests/basic.cc | 314 ++ .../thirdparty/loadgen/tests/loadgen_test.h | 198 ++ .../loadgen/tests/loadgen_test_main.cc | 33 + .../loadgen/tests/perftests_null_sut.cc | 230 ++ .../loadgen/tests/perftests_null_sut.py | 61 + .../loadgen/tools/mlperf-trace.ipynb | 441 +++ .../inference/thirdparty/loadgen/utils.cc | 124 + .../inference/thirdparty/loadgen/utils.h | 70 + .../inference/thirdparty/loadgen/version.cc | 85 + .../inference/thirdparty/loadgen/version.h | 39 + .../thirdparty/loadgen/version_generator.py | 141 + .../dlrm_v3/inference/ts_types.py | 70 + .../dlrm_v3/inference/user.conf | 5 + .../dlrm_v3/preprocess_public_data.py | 211 ++ .../dlrm_v3/streaming_synthetic_data.py | 664 ++++ .../dlrm_v3/train/gin/debug.gin | 35 + .../dlrm_v3/train/gin/kuairand_1k.gin | 41 + .../dlrm_v3/train/gin/movielens_13b.gin | 41 + .../dlrm_v3/train/gin/movielens_18b.gin | 56 + .../dlrm_v3/train/gin/movielens_1m.gin | 38 + .../dlrm_v3/train/gin/movielens_20m.gin | 56 + .../dlrm_v3/train/gin/streaming_100b.gin | 52 + .../dlrm_v3/train/gin/streaming_200b.gin | 63 + .../dlrm_v3/train/gin/streaming_400m.gin | 61 + .../dlrm_v3/train/gin/yambda_5b.gin | 50 + .../dlrm_v3/train/tests/train_test.py | 29 + .../dlrm_v3/train/train_ranker.py | 190 + .../dlrm_v3/train/utils.py | 902 +++++ .../generative_recommenders/dlrm_v3/utils.py | 652 ++++ .../modules/action_encoder.py | 112 + .../modules/content_encoder.py | 109 + .../contextual_interleave_preprocessor.py | 357 ++ .../modules/contextualize_mlps.py | 141 + .../modules/dlrm_hstu.py | 626 ++++ .../modules/dynamic_stu.py | 304 ++ .../modules/hstu_transducer.py | 330 ++ .../modules/multitask_module.py | 288 ++ .../modules/positional_encoder.py | 75 + .../modules/postprocessors.py | 182 + .../modules/preprocessors.py | 334 ++ .../generative_recommenders/modules/stu.py | 471 +++ .../modules/tests/action_encoder_test.py | 113 + .../modules/tests/content_encoder_test.py | 74 + ...contextual_interleave_preprocessor_test.py | 499 +++ .../modules/tests/dynamic_stu_test.py | 279 ++ .../modules/tests/multitask_module_test.py | 233 ++ .../modules/tests/stu_test.py | 453 +++ .../ops/benchmarks/addmm_bench.py | 174 + .../ops/benchmarks/hstu_attention_bench.py | 406 +++ .../ops/benchmarks/jagged_dense_bmm_bench.py | 199 ++ .../jagged_dense_bmm_broadcast_add_bench.py | 270 ++ .../jagged_dense_broadcast_add_bench.py | 205 ++ .../concat_1d_jagged_jagged_bench.py | 125 + .../benchmarks/jagged_transpose_1d_bench.py | 117 + .../replace_last_n_with_jagged_bench.py | 150 + .../split_1d_jagged_jagged_bench.py | 116 + .../generative_recommenders/ops/cpp/common.h | 60 + .../ops/cpp/complete_cumsum.cpp | 44 + .../ops/cpp/complete_cumsum.cu | 51 + .../ops/cpp/concat_1d_jagged_jagged.cpp | 111 + .../ops/cpp/concat_1d_jagged_jagged.cu | 130 + .../ops/cpp/cpp_ops.cpp | 207 ++ .../ops/cpp/cuda_hstu_attention.py | 193 + .../cpp/cuda_hstu_preprocess_and_attention.py | 668 ++++ .../ops/cpp/expand_1d_jagged_to_dense.cpp | 97 + .../ops/cpp/expand_1d_jagged_to_dense.cu | 103 + .../hstu_attention/copy_sm90_bulk_reduce.h | 66 + .../ops/cpp/hstu_attention/epilogue_bwd.h | 481 +++ .../ops/cpp/hstu_attention/epilogue_fwd.h | 550 +++ .../ops/cpp/hstu_attention/flash.h | 157 + .../ops/cpp/hstu_attention/flash_api.cpp | 322 ++ .../ops/cpp/hstu_attention/flash_api_cpu.cpp | 256 ++ .../hstu_attention/flash_bwd_kernel_sm90.h | 402 +++ .../flash_bwd_launch_template.h | 492 +++ .../flash_bwd_postprocess_kernel.h | 348 ++ .../flash_bwd_preprocess_kernel.h | 349 ++ .../ops/cpp/hstu_attention/flash_common.cpp | 1165 ++++++ .../ops/cpp/hstu_attention/flash_common.h | 149 + .../cpp/hstu_attention/flash_common_cpu.cpp | 172 + .../ops/cpp/hstu_attention/flash_common_cpu.h | 114 + .../hstu_attention/flash_fwd_kernel_sm90.h | 511 +++ .../flash_fwd_launch_template.h | 376 ++ .../cpp/hstu_attention/generate_kernels.py | 236 ++ ...lash_bwd_hdim128_bf16_softmaxfalse_sm90.cu | 33 + ...flash_bwd_hdim128_bf16_softmaxtrue_sm90.cu | 33 + ...lash_bwd_hdim128_fp16_softmaxfalse_sm90.cu | 33 + ...flash_bwd_hdim128_fp16_softmaxtrue_sm90.cu | 33 + ...lash_bwd_hdim192_bf16_softmaxfalse_sm90.cu | 33 + ...flash_bwd_hdim192_bf16_softmaxtrue_sm90.cu | 33 + ...lash_bwd_hdim192_fp16_softmaxfalse_sm90.cu | 33 + ...flash_bwd_hdim192_fp16_softmaxtrue_sm90.cu | 33 + ...lash_bwd_hdim256_bf16_softmaxfalse_sm90.cu | 33 + ...flash_bwd_hdim256_bf16_softmaxtrue_sm90.cu | 33 + ...lash_bwd_hdim256_fp16_softmaxfalse_sm90.cu | 33 + ...flash_bwd_hdim256_fp16_softmaxtrue_sm90.cu | 33 + ...flash_bwd_hdim64_bf16_softmaxfalse_sm90.cu | 33 + .../flash_bwd_hdim64_bf16_softmaxtrue_sm90.cu | 33 + ...flash_bwd_hdim64_fp16_softmaxfalse_sm90.cu | 33 + .../flash_bwd_hdim64_fp16_softmaxtrue_sm90.cu | 33 + ...flash_bwd_hdim96_bf16_softmaxfalse_sm90.cu | 33 + .../flash_bwd_hdim96_bf16_softmaxtrue_sm90.cu | 33 + ...flash_bwd_hdim96_fp16_softmaxfalse_sm90.cu | 33 + .../flash_bwd_hdim96_fp16_softmaxtrue_sm90.cu | 33 + ...lash_fwd_hdim128_bf16_softmaxfalse_sm90.cu | 33 + ...flash_fwd_hdim128_bf16_softmaxtrue_sm90.cu | 33 + ...lash_fwd_hdim128_e4m3_softmaxfalse_sm90.cu | 33 + ...flash_fwd_hdim128_e4m3_softmaxtrue_sm90.cu | 33 + ...lash_fwd_hdim128_fp16_softmaxfalse_sm90.cu | 33 + ...flash_fwd_hdim128_fp16_softmaxtrue_sm90.cu | 33 + ...lash_fwd_hdim192_bf16_softmaxfalse_sm90.cu | 33 + ...flash_fwd_hdim192_bf16_softmaxtrue_sm90.cu | 33 + ...lash_fwd_hdim192_e4m3_softmaxfalse_sm90.cu | 33 + ...flash_fwd_hdim192_e4m3_softmaxtrue_sm90.cu | 33 + ...lash_fwd_hdim192_fp16_softmaxfalse_sm90.cu | 33 + ...flash_fwd_hdim192_fp16_softmaxtrue_sm90.cu | 33 + ...lash_fwd_hdim256_bf16_softmaxfalse_sm90.cu | 33 + ...flash_fwd_hdim256_bf16_softmaxtrue_sm90.cu | 33 + ...lash_fwd_hdim256_e4m3_softmaxfalse_sm90.cu | 33 + ...flash_fwd_hdim256_e4m3_softmaxtrue_sm90.cu | 33 + ...lash_fwd_hdim256_fp16_softmaxfalse_sm90.cu | 33 + ...flash_fwd_hdim256_fp16_softmaxtrue_sm90.cu | 33 + ...flash_fwd_hdim64_bf16_softmaxfalse_sm90.cu | 33 + .../flash_fwd_hdim64_bf16_softmaxtrue_sm90.cu | 33 + ...flash_fwd_hdim64_e4m3_softmaxfalse_sm90.cu | 33 + .../flash_fwd_hdim64_e4m3_softmaxtrue_sm90.cu | 33 + ...flash_fwd_hdim64_fp16_softmaxfalse_sm90.cu | 33 + .../flash_fwd_hdim64_fp16_softmaxtrue_sm90.cu | 33 + ...flash_fwd_hdim96_bf16_softmaxfalse_sm90.cu | 33 + .../flash_fwd_hdim96_bf16_softmaxtrue_sm90.cu | 33 + ...flash_fwd_hdim96_e4m3_softmaxfalse_sm90.cu | 33 + .../flash_fwd_hdim96_e4m3_softmaxtrue_sm90.cu | 33 + ...flash_fwd_hdim96_fp16_softmaxfalse_sm90.cu | 33 + .../flash_fwd_hdim96_fp16_softmaxtrue_sm90.cu | 33 + .../mainloop_bwd_sm90_tma_gmma_ws.h | 3166 +++++++++++++++++ .../mainloop_fwd_sm90_tma_gmma_ws.h | 2180 ++++++++++++ .../ops/cpp/hstu_attention/mask.h | 396 +++ .../ops/cpp/hstu_attention/named_barrier.h | 101 + .../ops/cpp/hstu_attention/seqlen.h | 134 + .../hstu_attention/sm90_pipeline_no_cluster.h | 150 + .../ops/cpp/hstu_attention/softmax.h | 256 ++ .../ops/cpp/hstu_attention/static_switch.h | 135 + .../ops/cpp/hstu_attention/tile_scheduler.h | 616 ++++ .../ops/cpp/hstu_attention/tile_size.h | 220 ++ .../ops/cpp/hstu_attention/utils.h | 789 ++++ .../ops/cpp/hstu_attention/version.txt | 1 + .../ops/cpp/jagged_transpose_1d.cpp | 130 + .../ops/cpp/jagged_transpose_1d.cu | 127 + .../ops/cpp/replace_last_n_with_jagged.cpp | 139 + .../ops/cpp/replace_last_n_with_jagged.cu | 156 + .../generative_recommenders/ops/cpp/setup.py | 487 +++ .../ops/cpp/sort_kv_pairs_cuda.cpp | 40 + .../sort_kv_pairs_cuda_kernels_template.cu | 82 + .../cpp/sort_kv_pairs_cuda_kernels_template.h | 15 + .../ops/cpp/split_1d_jagged_jagged.cpp | 136 + .../ops/cpp/split_1d_jagged_jagged.cu | 147 + .../cpp/tests/concat_1d_jagged_jagged_test.py | 135 + .../ops/cpp/tests/hstu_mha_cpu_test.py | 39 + .../ops/cpp/tests/jagged_transpose_1d_test.py | 132 + .../tests/replace_last_n_with_jagged_test.py | 105 + .../cpp/tests/split_1d_jagged_jagged_test.py | 100 + .../ops/hstu_attention.py | 353 ++ .../ops/hstu_compute.py | 390 ++ .../ops/jagged_tensors.py | 451 +++ .../generative_recommenders/ops/layer_norm.py | 330 ++ .../generative_recommenders/ops/mm.py | 60 + .../generative_recommenders/ops/position.py | 147 + .../ops/pytorch/pt_hstu_attention.py | 251 ++ .../ops/pytorch/pt_hstu_linear.py | 130 + .../ops/pytorch/pt_jagged.py | 258 ++ .../ops/pytorch/pt_jagged_tensors.py | 246 ++ .../ops/pytorch/pt_layer_norm.py | 81 + .../ops/pytorch/pt_position.py | 142 + .../ops/tests/fake_signature_test.py | 162 + .../ops/tests/hstu_attention_test.py | 485 +++ .../ops/tests/hstu_attention_tma_test.py | 270 ++ .../ops/tests/hstu_compute_test.py | 503 +++ .../ops/tests/jagged_tensors_test.py | 963 +++++ .../ops/tests/layer_norm_test.py | 231 ++ .../ops/tests/mm_test.py | 156 + .../ops/tests/position_test.py | 234 ++ .../ops/tests/rms_norm_test.py | 229 ++ .../ops/triton/triton_addmm.py | 1706 +++++++++ .../ops/triton/triton_attention_utils.py | 64 + .../ops/triton/triton_hstu_attention.py | 3134 ++++++++++++++++ .../ops/triton/triton_hstu_linear.py | 3042 ++++++++++++++++ .../triton_hstu_preprocess_and_attention.py | 342 ++ .../ops/triton/triton_jagged.py | 2533 +++++++++++++ .../ops/triton/triton_jagged_tensors.py | 1067 ++++++ .../ops/triton/triton_layer_norm.py | 1327 +++++++ .../ops/triton/triton_position.py | 438 +++ .../ops/triton/triton_swiglu.py | 753 ++++ .../ops/triton_aot/README.md | 54 + .../ops/triton_aot/compile/arg_descriptor.py | 146 + .../ops/triton_aot/compile/codegen.py | 780 ++++ .../ops/triton_aot/compile/compile_state.py | 409 +++ .../ops/triton_aot/compile/pipeline.py | 300 ++ .../ops/triton_aot/compile/spec_processing.py | 593 +++ .../ops/triton_aot/compile/stable_types.py | 35 + .../triton_aot/compile/triton_aot_compile.py | 149 + .../ops/triton_aot/compile/utils.py | 47 + .../ops/triton_aot/preprocess.py | 76 + .../ops/triton_aot/shared/compat.py | 91 + .../ops/triton_aot/shared/spec_conversion.py | 389 ++ .../ops/triton_aot/shared/types.py | 58 + .../triton_aot/templates/embedded_cubins.cpp | 7 + .../ops/triton_aot/templates/kernel.cpp | 104 + .../ops/triton_aot/templates/kernel.h | 36 + .../triton_aot/templates/template_utils.py | 96 + .../ops/triton_aot/templates/torch_op.cpp | 22 + .../ops/triton_aot/transform/import_utils.py | 89 + .../transform/kernel_wrapper_codegen.py | 500 +++ .../triton_aot/transform/replace_kernels.py | 137 + .../triton_aot/transform/transform_kernels.py | 29 + .../ops/triton_aot/triton_addmm.py | 347 ++ .../ops/triton_aot/triton_concat_2d_jagged.py | 183 + .../triton_group_norm_mul_dropout.py | 124 + .../ops/triton_aot/triton_layer_norm.py | 119 + .../triton_layer_norm_mul_dropout.py | 162 + .../ops/triton_aot/triton_position.py | 176 + .../triton_ragged_hstu_attention.py | 366 ++ .../ops/triton_aot/triton_rms_norm.py | 114 + .../ops/triton_aot/triton_split_2d_jagged.py | 138 + .../ops/triton_aot/types.py | 181 + .../generative_recommenders/ops/utils.py | 93 + .../research/data/dataset.py | 248 ++ .../research/data/eval.py | 263 ++ .../research/data/item_features.py | 29 + .../research/data/preprocessor.py | 474 +++ .../research/data/reco_dataset.py | 176 + .../research/indexing/candidate_index.py | 179 + .../research/indexing/utils.py | 43 + .../research/modeling/initialization.py | 35 + .../sequential/autoregressive_losses.py | 477 +++ .../modeling/sequential/embedding_modules.py | 108 + .../modeling/sequential/encoder_utils.py | 150 + .../research/modeling/sequential/features.py | 94 + .../research/modeling/sequential/hstu.py | 808 +++++ .../input_features_preprocessors.py | 259 ++ .../sequential/losses/sampled_softmax.py | 193 + .../sequential/output_postprocessors.py | 82 + .../research/modeling/sequential/sasrec.py | 316 ++ .../research/modeling/sequential/utils.py | 129 + .../research/modeling/similarity_module.py | 68 + .../research/modeling/similarity_utils.py | 222 ++ .../rails/indexing/candidate_index.py | 41 + .../research/rails/indexing/mips_top_k.py | 80 + .../research/rails/indexing/mol_top_k.py | 132 + .../similarities/dot_product_similarity_fn.py | 68 + .../research/rails/similarities/layers.py | 82 + .../research/rails/similarities/module.py | 55 + .../rails/similarities/mol/embeddings_fn.py | 52 + .../similarities/mol/item_embeddings_fn.py | 99 + .../similarities/mol/query_embeddings_fn.py | 164 + .../rails/similarities/mol/similarity_fn.py | 388 ++ .../research/trainer/data_loader.py | 57 + .../research/trainer/train.py | 532 +++ .../tests/test_common.py | 41 + recommendation_v4/main.py | 82 + recommendation_v4/preprocess_public_data.py | 32 + recommendation_v4/requirements.txt | 7 + recommendation_v4/run_fractal_expansion.py | 588 +++ .../scripts/launch_smoke_8gpu.sh | 38 + recommendation_v4/setup.py | 41 + 375 files changed, 88510 insertions(+) create mode 100644 recommendation_v4/.gitignore create mode 100644 recommendation_v4/LICENSE create mode 100644 recommendation_v4/README.MD create mode 100644 recommendation_v4/configs/amzn-books/hstu-sampled-softmax-n512-final.gin create mode 100644 recommendation_v4/configs/amzn-books/hstu-sampled-softmax-n512-large-final.gin create mode 100644 recommendation_v4/configs/amzn-books/sasrec-sampled-softmax-n512-final.gin create mode 100644 recommendation_v4/configs/ml-1m/hstu-sampled-softmax-n128-final.gin create mode 100644 recommendation_v4/configs/ml-1m/hstu-sampled-softmax-n128-large-final.gin create mode 100644 recommendation_v4/configs/ml-1m/sasrec-sampled-softmax-n128-final.gin create mode 100644 recommendation_v4/configs/ml-20m/hstu-sampled-softmax-n128-final.gin create mode 100644 recommendation_v4/configs/ml-20m/hstu-sampled-softmax-n128-large-final.gin create mode 100644 recommendation_v4/configs/ml-20m/sasrec-sampled-softmax-n128-final.gin create mode 100644 recommendation_v4/configs/ml-3b/hstu-sampled-softmax-n96-seqlen500-final.gin create mode 100644 recommendation_v4/configs/ml-3b/hstu-sampled-softmax-n96-seqlen500-large-final.gin create mode 100644 recommendation_v4/configs/ml-3b/sasrec-sampled-softmax-n96-seqlen500-final.gin create mode 100644 recommendation_v4/generative_recommenders/README.md create mode 100644 recommendation_v4/generative_recommenders/common.py create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/checkpoint.py create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/configs.py create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/datasets/dataset.py create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/datasets/kuairand.py create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/datasets/movie_lens.py create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/datasets/synthetic_movie_lens.py create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/datasets/synthetic_streaming.py create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/datasets/utils.py create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/datasets/yambda.py create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/README.md create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/accuracy.py create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/cpp/hstu_runner.cpp create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/data_producer.py create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/dense_predict_module.py create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/end_to_end_test.py create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/gin/debug.gin create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/gin/kuairand_1k.gin create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/gin/movielens_13b.gin create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/gin/streaming_100b.gin create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/gin/streaming_400m.gin create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/inference_modules.py create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/main.py create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/mlperf.conf create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/model_family.py create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/sparse_predict_module.py create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/tests/inference_test.py create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/tests/test_scripted_parity.py create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/.clang-format create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/CMakeLists.txt create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/MANIFEST.in create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/README.md create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/README_BUILD.md create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/README_FAQ.md create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/VERSION.txt create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/benchmark/.gitignore create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/benchmark/README.md create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/benchmark/repro.cpp create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/benchmark/run.sh create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/benchmark/run_debug.sh create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/bindings/c_api.cc create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/bindings/c_api.h create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/bindings/python_api.cc create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/lon/README.md create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/lon/py_demo_server_lon.py create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/lon/sut_over_network_demo.py create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/py_demo_multi_stream.py create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/py_demo_offline.py create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/py_demo_server.py create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/py_demo_single_stream.py create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_multi_stream.py create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_offline.py create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_offline_inferred.py create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_server.py create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_server_inferred.py create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_single_stream.py create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/diagram_network_submission.png create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/diagram_submission.png create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/BUILD.gn create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/README.md create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/doxygen.cfg create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/doxygen_footer.html create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/doxygen_header.html create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/doxygen_html_generator.py create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/doxygen_layout.xml create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/doxygen_stylesheet.css create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/loadgen_integration_diagram.dia create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/mlperf_icon.png create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/mlperf_logo_horizontal_color.svg create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/early_stopping.cc create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/early_stopping.h create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/generated/version_generated.cc create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/issue_query_controller.cc create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/issue_query_controller.h create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/loadgen.cc create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/loadgen.h create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/loadgen_integration_diagram.svg create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/logging.cc create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/logging.h create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/mlperf.conf create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/mlperf_conf.h create mode 100755 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/pyproject.toml create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/query_dispatch_library.h create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/query_sample.h create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/query_sample_library.h create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/requirements.txt create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/results.cc create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/results.h create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/setup.py create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/system_under_test.h create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/test_settings.h create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/test_settings_internal.cc create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/test_settings_internal.h create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/BUILD.gn create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/README.md create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/basic.cc create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/loadgen_test.h create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/loadgen_test_main.cc create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/perftests_null_sut.cc create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/perftests_null_sut.py create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tools/mlperf-trace.ipynb create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/utils.cc create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/utils.h create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/version.cc create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/version.h create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/version_generator.py create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/ts_types.py create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/user.conf create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/preprocess_public_data.py create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/streaming_synthetic_data.py create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/train/gin/debug.gin create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/train/gin/kuairand_1k.gin create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/train/gin/movielens_13b.gin create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/train/gin/movielens_18b.gin create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/train/gin/movielens_1m.gin create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/train/gin/movielens_20m.gin create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/train/gin/streaming_100b.gin create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/train/gin/streaming_200b.gin create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/train/gin/streaming_400m.gin create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/train/tests/train_test.py create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/utils.py create mode 100644 recommendation_v4/generative_recommenders/modules/action_encoder.py create mode 100644 recommendation_v4/generative_recommenders/modules/content_encoder.py create mode 100644 recommendation_v4/generative_recommenders/modules/contextual_interleave_preprocessor.py create mode 100644 recommendation_v4/generative_recommenders/modules/contextualize_mlps.py create mode 100644 recommendation_v4/generative_recommenders/modules/dlrm_hstu.py create mode 100644 recommendation_v4/generative_recommenders/modules/dynamic_stu.py create mode 100644 recommendation_v4/generative_recommenders/modules/hstu_transducer.py create mode 100644 recommendation_v4/generative_recommenders/modules/multitask_module.py create mode 100644 recommendation_v4/generative_recommenders/modules/positional_encoder.py create mode 100644 recommendation_v4/generative_recommenders/modules/postprocessors.py create mode 100644 recommendation_v4/generative_recommenders/modules/preprocessors.py create mode 100644 recommendation_v4/generative_recommenders/modules/stu.py create mode 100644 recommendation_v4/generative_recommenders/modules/tests/action_encoder_test.py create mode 100644 recommendation_v4/generative_recommenders/modules/tests/content_encoder_test.py create mode 100644 recommendation_v4/generative_recommenders/modules/tests/contextual_interleave_preprocessor_test.py create mode 100644 recommendation_v4/generative_recommenders/modules/tests/dynamic_stu_test.py create mode 100644 recommendation_v4/generative_recommenders/modules/tests/multitask_module_test.py create mode 100644 recommendation_v4/generative_recommenders/modules/tests/stu_test.py create mode 100644 recommendation_v4/generative_recommenders/ops/benchmarks/addmm_bench.py create mode 100644 recommendation_v4/generative_recommenders/ops/benchmarks/hstu_attention_bench.py create mode 100644 recommendation_v4/generative_recommenders/ops/benchmarks/jagged_dense_bmm_bench.py create mode 100644 recommendation_v4/generative_recommenders/ops/benchmarks/jagged_dense_bmm_broadcast_add_bench.py create mode 100644 recommendation_v4/generative_recommenders/ops/benchmarks/jagged_dense_broadcast_add_bench.py create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/benchmarks/concat_1d_jagged_jagged_bench.py create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/benchmarks/jagged_transpose_1d_bench.py create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/benchmarks/replace_last_n_with_jagged_bench.py create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/benchmarks/split_1d_jagged_jagged_bench.py create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/common.h create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/complete_cumsum.cpp create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/complete_cumsum.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/concat_1d_jagged_jagged.cpp create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/concat_1d_jagged_jagged.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/cpp_ops.cpp create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/cuda_hstu_attention.py create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/cuda_hstu_preprocess_and_attention.py create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/expand_1d_jagged_to_dense.cpp create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/expand_1d_jagged_to_dense.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/copy_sm90_bulk_reduce.h create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/epilogue_bwd.h create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/epilogue_fwd.h create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash.h create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_api.cpp create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_api_cpu.cpp create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_bwd_kernel_sm90.h create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_bwd_launch_template.h create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_bwd_postprocess_kernel.h create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_bwd_preprocess_kernel.h create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_common.cpp create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_common.h create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_common_cpu.cpp create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_common_cpu.h create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_fwd_kernel_sm90.h create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_fwd_launch_template.h create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/generate_kernels.py create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim128_bf16_softmaxfalse_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim128_bf16_softmaxtrue_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim128_fp16_softmaxfalse_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim128_fp16_softmaxtrue_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim192_bf16_softmaxfalse_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim192_bf16_softmaxtrue_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim192_fp16_softmaxfalse_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim192_fp16_softmaxtrue_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim256_bf16_softmaxfalse_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim256_bf16_softmaxtrue_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim256_fp16_softmaxfalse_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim256_fp16_softmaxtrue_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim64_bf16_softmaxfalse_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim64_bf16_softmaxtrue_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim64_fp16_softmaxfalse_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim64_fp16_softmaxtrue_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim96_bf16_softmaxfalse_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim96_bf16_softmaxtrue_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim96_fp16_softmaxfalse_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim96_fp16_softmaxtrue_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_bf16_softmaxfalse_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_bf16_softmaxtrue_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_e4m3_softmaxfalse_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_e4m3_softmaxtrue_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_fp16_softmaxfalse_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_fp16_softmaxtrue_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_bf16_softmaxfalse_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_bf16_softmaxtrue_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_e4m3_softmaxfalse_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_e4m3_softmaxtrue_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_fp16_softmaxfalse_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_fp16_softmaxtrue_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_bf16_softmaxfalse_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_bf16_softmaxtrue_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_e4m3_softmaxfalse_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_e4m3_softmaxtrue_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_fp16_softmaxfalse_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_fp16_softmaxtrue_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_bf16_softmaxfalse_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_bf16_softmaxtrue_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_e4m3_softmaxfalse_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_e4m3_softmaxtrue_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_fp16_softmaxfalse_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_fp16_softmaxtrue_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_bf16_softmaxfalse_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_bf16_softmaxtrue_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_e4m3_softmaxfalse_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_e4m3_softmaxtrue_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_fp16_softmaxfalse_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_fp16_softmaxtrue_sm90.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/mainloop_bwd_sm90_tma_gmma_ws.h create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/mainloop_fwd_sm90_tma_gmma_ws.h create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/mask.h create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/named_barrier.h create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/seqlen.h create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/sm90_pipeline_no_cluster.h create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/softmax.h create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/static_switch.h create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/tile_scheduler.h create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/tile_size.h create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/utils.h create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/version.txt create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/jagged_transpose_1d.cpp create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/jagged_transpose_1d.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/replace_last_n_with_jagged.cpp create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/replace_last_n_with_jagged.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/setup.py create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/sort_kv_pairs_cuda.cpp create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/sort_kv_pairs_cuda_kernels_template.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/sort_kv_pairs_cuda_kernels_template.h create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/split_1d_jagged_jagged.cpp create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/split_1d_jagged_jagged.cu create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/tests/concat_1d_jagged_jagged_test.py create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/tests/hstu_mha_cpu_test.py create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/tests/jagged_transpose_1d_test.py create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/tests/replace_last_n_with_jagged_test.py create mode 100644 recommendation_v4/generative_recommenders/ops/cpp/tests/split_1d_jagged_jagged_test.py create mode 100644 recommendation_v4/generative_recommenders/ops/hstu_attention.py create mode 100644 recommendation_v4/generative_recommenders/ops/hstu_compute.py create mode 100644 recommendation_v4/generative_recommenders/ops/jagged_tensors.py create mode 100644 recommendation_v4/generative_recommenders/ops/layer_norm.py create mode 100644 recommendation_v4/generative_recommenders/ops/mm.py create mode 100644 recommendation_v4/generative_recommenders/ops/position.py create mode 100644 recommendation_v4/generative_recommenders/ops/pytorch/pt_hstu_attention.py create mode 100644 recommendation_v4/generative_recommenders/ops/pytorch/pt_hstu_linear.py create mode 100644 recommendation_v4/generative_recommenders/ops/pytorch/pt_jagged.py create mode 100644 recommendation_v4/generative_recommenders/ops/pytorch/pt_jagged_tensors.py create mode 100644 recommendation_v4/generative_recommenders/ops/pytorch/pt_layer_norm.py create mode 100644 recommendation_v4/generative_recommenders/ops/pytorch/pt_position.py create mode 100644 recommendation_v4/generative_recommenders/ops/tests/fake_signature_test.py create mode 100644 recommendation_v4/generative_recommenders/ops/tests/hstu_attention_test.py create mode 100644 recommendation_v4/generative_recommenders/ops/tests/hstu_attention_tma_test.py create mode 100644 recommendation_v4/generative_recommenders/ops/tests/hstu_compute_test.py create mode 100644 recommendation_v4/generative_recommenders/ops/tests/jagged_tensors_test.py create mode 100644 recommendation_v4/generative_recommenders/ops/tests/layer_norm_test.py create mode 100644 recommendation_v4/generative_recommenders/ops/tests/mm_test.py create mode 100644 recommendation_v4/generative_recommenders/ops/tests/position_test.py create mode 100644 recommendation_v4/generative_recommenders/ops/tests/rms_norm_test.py create mode 100644 recommendation_v4/generative_recommenders/ops/triton/triton_addmm.py create mode 100644 recommendation_v4/generative_recommenders/ops/triton/triton_attention_utils.py create mode 100644 recommendation_v4/generative_recommenders/ops/triton/triton_hstu_attention.py create mode 100644 recommendation_v4/generative_recommenders/ops/triton/triton_hstu_linear.py create mode 100644 recommendation_v4/generative_recommenders/ops/triton/triton_hstu_preprocess_and_attention.py create mode 100644 recommendation_v4/generative_recommenders/ops/triton/triton_jagged.py create mode 100644 recommendation_v4/generative_recommenders/ops/triton/triton_jagged_tensors.py create mode 100644 recommendation_v4/generative_recommenders/ops/triton/triton_layer_norm.py create mode 100644 recommendation_v4/generative_recommenders/ops/triton/triton_position.py create mode 100644 recommendation_v4/generative_recommenders/ops/triton/triton_swiglu.py create mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/README.md create mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/compile/arg_descriptor.py create mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/compile/codegen.py create mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/compile/compile_state.py create mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/compile/pipeline.py create mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/compile/spec_processing.py create mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/compile/stable_types.py create mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/compile/triton_aot_compile.py create mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/compile/utils.py create mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/preprocess.py create mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/shared/compat.py create mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/shared/spec_conversion.py create mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/shared/types.py create mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/templates/embedded_cubins.cpp create mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/templates/kernel.cpp create mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/templates/kernel.h create mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/templates/template_utils.py create mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/templates/torch_op.cpp create mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/transform/import_utils.py create mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/transform/kernel_wrapper_codegen.py create mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/transform/replace_kernels.py create mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/transform/transform_kernels.py create mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/triton_addmm.py create mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/triton_concat_2d_jagged.py create mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/triton_group_norm_mul_dropout.py create mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/triton_layer_norm.py create mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/triton_layer_norm_mul_dropout.py create mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/triton_position.py create mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/triton_ragged_hstu_attention.py create mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/triton_rms_norm.py create mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/triton_split_2d_jagged.py create mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/types.py create mode 100644 recommendation_v4/generative_recommenders/ops/utils.py create mode 100644 recommendation_v4/generative_recommenders/research/data/dataset.py create mode 100644 recommendation_v4/generative_recommenders/research/data/eval.py create mode 100644 recommendation_v4/generative_recommenders/research/data/item_features.py create mode 100644 recommendation_v4/generative_recommenders/research/data/preprocessor.py create mode 100644 recommendation_v4/generative_recommenders/research/data/reco_dataset.py create mode 100644 recommendation_v4/generative_recommenders/research/indexing/candidate_index.py create mode 100644 recommendation_v4/generative_recommenders/research/indexing/utils.py create mode 100644 recommendation_v4/generative_recommenders/research/modeling/initialization.py create mode 100644 recommendation_v4/generative_recommenders/research/modeling/sequential/autoregressive_losses.py create mode 100644 recommendation_v4/generative_recommenders/research/modeling/sequential/embedding_modules.py create mode 100644 recommendation_v4/generative_recommenders/research/modeling/sequential/encoder_utils.py create mode 100644 recommendation_v4/generative_recommenders/research/modeling/sequential/features.py create mode 100644 recommendation_v4/generative_recommenders/research/modeling/sequential/hstu.py create mode 100644 recommendation_v4/generative_recommenders/research/modeling/sequential/input_features_preprocessors.py create mode 100644 recommendation_v4/generative_recommenders/research/modeling/sequential/losses/sampled_softmax.py create mode 100644 recommendation_v4/generative_recommenders/research/modeling/sequential/output_postprocessors.py create mode 100644 recommendation_v4/generative_recommenders/research/modeling/sequential/sasrec.py create mode 100644 recommendation_v4/generative_recommenders/research/modeling/sequential/utils.py create mode 100644 recommendation_v4/generative_recommenders/research/modeling/similarity_module.py create mode 100644 recommendation_v4/generative_recommenders/research/modeling/similarity_utils.py create mode 100644 recommendation_v4/generative_recommenders/research/rails/indexing/candidate_index.py create mode 100644 recommendation_v4/generative_recommenders/research/rails/indexing/mips_top_k.py create mode 100644 recommendation_v4/generative_recommenders/research/rails/indexing/mol_top_k.py create mode 100644 recommendation_v4/generative_recommenders/research/rails/similarities/dot_product_similarity_fn.py create mode 100644 recommendation_v4/generative_recommenders/research/rails/similarities/layers.py create mode 100644 recommendation_v4/generative_recommenders/research/rails/similarities/module.py create mode 100644 recommendation_v4/generative_recommenders/research/rails/similarities/mol/embeddings_fn.py create mode 100644 recommendation_v4/generative_recommenders/research/rails/similarities/mol/item_embeddings_fn.py create mode 100644 recommendation_v4/generative_recommenders/research/rails/similarities/mol/query_embeddings_fn.py create mode 100644 recommendation_v4/generative_recommenders/research/rails/similarities/mol/similarity_fn.py create mode 100644 recommendation_v4/generative_recommenders/research/trainer/data_loader.py create mode 100644 recommendation_v4/generative_recommenders/research/trainer/train.py create mode 100644 recommendation_v4/generative_recommenders/tests/test_common.py create mode 100644 recommendation_v4/main.py create mode 100644 recommendation_v4/preprocess_public_data.py create mode 100644 recommendation_v4/requirements.txt create mode 100644 recommendation_v4/run_fractal_expansion.py create mode 100755 recommendation_v4/scripts/launch_smoke_8gpu.sh create mode 100644 recommendation_v4/setup.py diff --git a/.gitmodules b/.gitmodules index 51d8eac03..6ebcea592 100644 --- a/.gitmodules +++ b/.gitmodules @@ -6,3 +6,6 @@ path = text_to_image/torchtitan url = https://github.com/pytorch/torchtitan.git branch = mlperf-training-flux.1 +[submodule "recommendation_v4/cutlass"] + path = recommendation_v4/generative_recommenders/ops/cpp/cutlass + url = https://github.com/NVIDIA/cutlass.git diff --git a/recommendation_v4/.gitignore b/recommendation_v4/.gitignore new file mode 100644 index 000000000..560f823c4 --- /dev/null +++ b/recommendation_v4/.gitignore @@ -0,0 +1,158 @@ +# Don't check in parsed data files and other temporary files +tmp/ +exps/ +ckpts/ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ diff --git a/recommendation_v4/LICENSE b/recommendation_v4/LICENSE new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/recommendation_v4/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/recommendation_v4/README.MD b/recommendation_v4/README.MD new file mode 100644 index 000000000..a60d0d3d7 --- /dev/null +++ b/recommendation_v4/README.MD @@ -0,0 +1,135 @@ +# Generative Recommenders + +Repository hosting code for ``Actions Speak Louder than Words: Trillion-Parameter Sequential Transducers for Generative Recommendations`` ([ICML'24 paper](https://proceedings.mlr.press/v235/zhai24a.html)) and related code, where we demonstrate that the ubiquitously used classical deep learning recommendation paradigm (DLRMs) can be reformulated as a generative modeling problem (Generative Recommenders or GRs) to overcome known compute scaling bottlenecks, propose efficient algorithms such as HSTU and M-FALCON to accelerate training and inference for large-scale sequential models by 10x-1000x, and demonstrate scaling law for the first-time in deployed, billion-user scale recommendation systems. + +## Getting started + +We recommend using `requirements.txt`. This has been tested with Ubuntu 22.04, CUDA 12.4, and Python 3.10. + +```bash +pip3 install -r requirements.txt +``` + +Alternatively, you can manually install PyTorch based on official instructions. Then, + +```bash +pip3 install gin-config pandas fbgemm_gpu torchrec tensorboard +``` + +## Experiments + +### Public Experiments + +To reproduce the public experiments in our paper (traditional sequential recommender setting, Section 4.1.1) on MovieLens and Amazon Reviews in the paper, please follow these steps: + +#### Download and preprocess data. + +```bash +mkdir -p tmp/ && python3 preprocess_public_data.py +``` + +A GPU with 24GB or more HBM should work for most datasets. + +```bash +CUDA_VISIBLE_DEVICES=0 python3 main.py --gin_config_file=configs/ml-1m/hstu-sampled-softmax-n128-large-final.gin --master_port=12345 +``` + +Other configurations are included in configs/ml-1m, configs/ml-20m, and configs/amzn-books to make reproducing these experiments easier. + +#### Verify results. + +By default we write experimental logs to exps/. We can launch tensorboard with something like the following: + +```bash +tensorboard --logdir ~/generative-recommenders/exps/ml-1m-l200/ --port 24001 --bind_all +tensorboard --logdir ~/generative-recommenders/exps/ml-20m-l200/ --port 24001 --bind_all +tensorboard --logdir ~/generative-recommenders/exps/amzn-books-l50/ --port 24001 --bind_all +``` + +With the provided configuration (.gin) files, you should be able to reproduce the following results (verified as of 04/15/2024): + +**MovieLens-1M (ML-1M)**: + +| Method | HR@10 | NDCG@10 | HR@50 | NDCG@50 | HR@200 | NDCG@200 | +| ------------- | ---------------- | ----------------| --------------- | --------------- | --------------- | --------------- | +| SASRec | 0.2853 | 0.1603 | 0.5474 | 0.2185 | 0.7528 | 0.2498 | +| BERT4Rec | 0.2843 (-0.4%) | 0.1537 (-4.1%) | | | | | +| GRU4Rec | 0.2811 (-1.5%) | 0.1648 (+2.8%) | | | | | +| HSTU | 0.3097 (+8.6%) | 0.1720 (+7.3%) | 0.5754 (+5.1%) | 0.2307 (+5.6%) | 0.7716 (+2.5%) | 0.2606 (+4.3%) | +| HSTU-large | **0.3294 (+15.5%)** | **0.1893 (+18.1%)** | **0.5935 (+8.4%)** | **0.2481 (+13.5%)** | **0.7839 (+4.1%)** | **0.2771 (+10.9%)** | + +**MovieLens-20M (ML-20M)**: + +| Method | HR@10 | NDCG@10 | HR@50 | NDCG@50 | HR@200 | NDCG@200 | +| ------------- | ---------------- | --------------- | --------------- | --------------- | --------------- | --------------- | +| SASRec | 0.2889 | 0.1621 | 0.5503 | 0.2199 | 0.7661 | 0.2527 | +| BERT4Rec | 0.2816 (-2.5%) | 0.1703 (+5.1%) | | | | | +| GRU4Rec | 0.2813 (-2.6%) | 0.1730 (+6.7%) | | | | | +| HSTU | 0.3273 (+13.3%) | 0.1895 (+16.9%) | 0.5889 (+7.0%) | 0.2473 (+12.5%) | 0.7952 (+3.8%) | 0.2787 (+10.3%) | +| HSTU-large | **0.3556 (+23.1%)** | **0.2098 (+29.4%)** | **0.6143 (+11.6%)** | **0.2671 (+21.5%)** | **0.8074 (+5.4%)** | **0.2965 (+17.4%)** | + +**Amazon Reviews (Books)**: + +| Method | HR@10 | NDCG@10 | HR@50 | NDCG@50 | HR@200 | NDCG@200 | +| ------------- | ---------------- | ----------------|---------------- | --------------- | --------------- | --------------- | +| SASRec | 0.0306 | 0.0164 | 0.0754 | 0.0260 | 0.1431 | 0.0362 | +| HSTU | 0.0416 (+36.4%) | 0.0227 (+39.3%) | 0.0957 (+27.1%) | 0.0344 (+32.3%) | 0.1735 (+21.3%) | 0.0461 (+27.7%) | +| HSTU-large | **0.0478 (+56.7%)** | **0.0262 (+60.7%)** | **0.1082 (+43.7%)** | **0.0393 (+51.2%)** | **0.1908 (+33.4%)** | **0.0517 (+43.2%)** | + +for all three tables above, the ``SASRec`` rows are based on [Self-Attentive Sequential Recommendation](https://arxiv.org/abs/1808.09781) but with the original binary cross entropy loss +replaced with sampled softmax losses proposed in [Revisiting Neural Retrieval on Accelerators](https://arxiv.org/abs/2306.04039). These rows are reproducible with ``configs/*/sasrec-*-final.gin``. +The ``BERT4Rec`` and ``GRU4Rec`` rows are based on results reported by [Turning Dross Into Gold Loss: is BERT4Rec really better than SASRec?](https://arxiv.org/abs/2309.07602) - +note that the comparison slightly favors these two, due to them using full negatives whereas the other rows used 128/512 sampled negatives. The ``HSTU`` and ``HSTU-large`` rows are based on [Actions Speak Louder than Words: Trillion-Parameter Sequential Transducers for Generative Recommendations](https://arxiv.org/abs/2402.17152); in particular, HSTU rows utilize identical configurations as SASRec. ``HSTU`` and ``HSTU-large`` results can be reproduced with ``configs/*/hstu-*-final.gin``. + +### Synthetic Dataset / MovieLens-3B + +We support generating synthetic dataset with fractal expansion introduced in https://arxiv.org/abs/1901.08910. This allows us to expand the current 20 million real-world ratings in ML-20M to 3 billion. + +To download the pre-generated synthetic dataset: + +```bash +pip3 install gdown +mkdir -p tmp/ && cd tmp/ +gdown https://drive.google.com/uc?id=1-jZ6k0el7e7PyFnwqMLfqUTRh_Qdumt- +unzip ml-3b.zip && rm ml-3b.zip +``` + +To generate the synthetic dataset on your own: + +```bash +python3 run_fractal_expansion.py --input-csv-file tmp/ml-20m/ratings.csv --write-dataset True --output-prefix tmp/ml-3b/ +``` + +### Efficiency experiments + +``ops/triton`` contains triton kernels needed for efficiency experiments. ``ops/cpp`` contains efficient CUDA kernels. In particular, ``ops/cpp/hstu_attention`` contains the attention implementation based on [FlashAttention V3](https://github.com/Dao-AILab/flash-attention) with state-of-the-art efficiency on H100 GPUs. + +## DLRM-v3 + +We have created a DLRM model using HSTU and have developed benchmarks for both training and inference to faciliate production RecSys use cases. + +#### Run model training with 4 GPUs + +```bash +LOCAL_WORLD_SIZE=4 WORLD_SIZE=4 python3 generative_recommenders/dlrm_v3/train/train_ranker.py --dataset debug --mode train +``` + +#### Run model inference with 4 GPUs + +```bash +git clone --recurse-submodules https://github.com/mlcommons/inference.git mlperf_inference +cd mlperf_inference/loadgen +CFLAGS="-std=c++14 -O3" python -m pip install . + +LOCAL_WORLD_SIZE=4 WORLD_SIZE=4 python3 generative_recommenders/dlrm_v3/inference/main.py --dataset debug +``` + +## License +This codebase is Apache 2.0 licensed, as found in the [LICENSE](LICENSE) file. + +## Contributors +The overall project is made possible thanks to the joint work from many technical contributors (listed in alphabetical order): + +Adnan Akhundov, Bugra Akyildiz, Shabab Ayub, Alex Bao, Renqin Cai, Jennifer Cao, Xuan Cao, Guoqiang Jerry Chen, Lei Chen, Li Chen, Sean Chen, Xianjie Chen, Huihui Cheng, Weiwei Chu, Ted Cui, Shiyan Deng, Nimit Desai, Fei Ding, Shilin Ding, Francois Fagan, Lu Fang, Leon Gao, Zhaojie Gong, Fangda Gu, Liang Guo, Liz Guo, Jeevan Gyawali, Yuchen Hao, Daisy Shi He, Michael Jiayuan He, Yu He, Samuel Hsia, Jie Hua, Yanzun Huang, Hongyi Jia, Rui Jian, Jian Jin, Rafay Khurram, Rahul Kindi, Changkyu Kim, Yejin Lee, Fu Li, Han Li, Hong Li, Shen Li, Rui Li, Wei Li, Zhijing Li, Lucy Liao, Xueting Liao, Emma Lin, Hao Lin, Chloe Liu, Jingzhou Liu, Xing Liu, Xingyu Liu, Kai Londenberg, Yinghai Lu, Liang Luo, Linjian Ma, Matt Ma, Yun Mao, Bert Maher, Ajit Mathews, Matthew Murphy, Satish Nadathur, Min Ni, Jongsoo Park, Colin Peppler, Jing Qian, Lijing Qin, Jing Shan, Alex Singh, Timothy Shi, Yu Shi, Dennis van der Staay, Xiao Sun, Colin Taylor, Shin-Yeh Tsai, Rohan Varma, Omkar Vichare, Alyssa Wang, Pengchao Wang, Shengzhi Wang, Wenting Wang, Xiaolong Wang, Yueming Wang, Zhiyong Wang, Wei Wei, Bin Wen, Carole-Jean Wu, Yanhong Wu, Eric Xu, Bi Xue, Hong Yan, Zheng Yan, Chao Yang, Junjie Yang, Wen-Yun Yang, Ze Yang, Zimeng Yang, Yuanjun Yao, Chunxing Yin, Daniel Yin, Yiling You, Jiaqi Zhai, Keke Zhai, Yanli Zhao, Zhuoran Zhao, Hui Zhang, Jingjing Zhang, Lu Zhang, Lujia Zhang, Na Zhang, Rui Zhang, Xiong Zhang, Ying Zhang, Zhiyun Zhang, Charles Zheng, Erheng Zhong, Zhao Zhu, Xin Zhuang. + +For the initial paper describing the Generative Recommender problem formulation and the algorithms used, including HSTU and M-FALCON, please refer to ``Actions Speak Louder than Words: Trillion-Parameter Sequential Transducers for Generative Recommendations``([ICML'24 paper](https://dl.acm.org/doi/10.5555/3692070.3694484), [slides](https://icml.cc/media/icml-2024/Slides/32684.pdf)). diff --git a/recommendation_v4/configs/amzn-books/hstu-sampled-softmax-n512-final.gin b/recommendation_v4/configs/amzn-books/hstu-sampled-softmax-n512-final.gin new file mode 100644 index 000000000..8fb8b258c --- /dev/null +++ b/recommendation_v4/configs/amzn-books/hstu-sampled-softmax-n512-final.gin @@ -0,0 +1,49 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Frozen config, validated on 04/12/2024. +# Based on HSTU results (w/ identical configurations as a SotA Transformer baseline) in +# Actions Speak Louder than Words: Trillion-Parameter Sequential Transducers for Generative Recommendations (https://arxiv.org/abs/2402.17152). +# +# Run this as: +# mkdir -p logs/amzn-books-l50/ +# CUDA_VISIBLE_DEVICES=1 python3 main.py --gin_config_file=configs/amzn-books/hstu-sampled-softmax-n512-final.gin --master_port=12346 2>&1 | tee logs/amzn-books-l50/hstu-sampled-softmax-n512-final.log + +train_fn.dataset_name = "amzn-books" +train_fn.max_sequence_length = 50 +train_fn.local_batch_size = 128 +train_fn.eval_batch_size = 128 + +train_fn.main_module = "HSTU" +train_fn.dropout_rate = 0.5 +train_fn.user_embedding_norm = "l2_norm" +train_fn.item_embedding_dim = 64 + +hstu_encoder.num_blocks = 4 +hstu_encoder.num_heads = 4 +hstu_encoder.dv = 16 +hstu_encoder.dqk = 16 +hstu_encoder.linear_dropout_rate = 0.5 + +train_fn.eval_interval = 4000 +train_fn.num_epochs = 201 +train_fn.learning_rate = 1e-3 +train_fn.weight_decay = 0 +train_fn.num_warmup_steps = 0 + +train_fn.interaction_module_type = "DotProduct" +train_fn.top_k_method = "MIPSBruteForceTopK" + +train_fn.loss_module = "SampledSoftmaxLoss" +train_fn.num_negatives = 512 + +train_fn.sampling_strategy = "local" +train_fn.temperature = 0.05 +train_fn.item_l2_norm = True +train_fn.l2_norm_eps = 1e-6 + +train_fn.enable_tf32 = True +train_fn.full_eval_every_n = 5 +train_fn.partial_eval_num_iters = 64 + +create_data_loader.prefetch_factor = 1024 +create_data_loader.num_workers = 8 diff --git a/recommendation_v4/configs/amzn-books/hstu-sampled-softmax-n512-large-final.gin b/recommendation_v4/configs/amzn-books/hstu-sampled-softmax-n512-large-final.gin new file mode 100644 index 000000000..097d4cbc7 --- /dev/null +++ b/recommendation_v4/configs/amzn-books/hstu-sampled-softmax-n512-large-final.gin @@ -0,0 +1,49 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Frozen config, validated on 04/12/2024. +# Based on HSTU-large results in +# Actions Speak Louder than Words: Trillion-Parameter Sequential Transducers for Generative Recommendations (https://arxiv.org/abs/2402.17152). +# +# Run this as: +# mkdir -p logs/amzn-books-l50/ +# CUDA_VISIBLE_DEVICES=1 python3 main.py --gin_config_file=configs/amzn-books/hstu-sampled-softmax-n512-large-final.gin --master_port=12346 2>&1 | tee logs/amzn-books-l50/hstu-sampled-softmax-n512-large-final2.log + +train_fn.dataset_name = "amzn-books" +train_fn.max_sequence_length = 50 +train_fn.local_batch_size = 128 +train_fn.eval_batch_size = 128 + +train_fn.main_module = "HSTU" +train_fn.dropout_rate = 0.5 +train_fn.user_embedding_norm = "l2_norm" +train_fn.item_embedding_dim = 64 + +hstu_encoder.num_blocks = 16 +hstu_encoder.num_heads = 8 +hstu_encoder.dv = 8 +hstu_encoder.dqk = 8 +hstu_encoder.linear_dropout_rate = 0.5 + +train_fn.eval_interval = 4000 +train_fn.num_epochs = 201 +train_fn.learning_rate = 1e-3 +train_fn.weight_decay = 0 +train_fn.num_warmup_steps = 0 + +train_fn.interaction_module_type = "DotProduct" +train_fn.top_k_method = "MIPSBruteForceTopK" + +train_fn.loss_module = "SampledSoftmaxLoss" +train_fn.num_negatives = 512 + +train_fn.sampling_strategy = "local" +train_fn.temperature = 0.05 +train_fn.item_l2_norm = True +train_fn.l2_norm_eps = 1e-6 + +train_fn.enable_tf32 = True +train_fn.full_eval_every_n = 5 +train_fn.partial_eval_num_iters = 64 + +create_data_loader.prefetch_factor = 1024 +create_data_loader.num_workers = 8 diff --git a/recommendation_v4/configs/amzn-books/sasrec-sampled-softmax-n512-final.gin b/recommendation_v4/configs/amzn-books/sasrec-sampled-softmax-n512-final.gin new file mode 100644 index 000000000..bc899c9fb --- /dev/null +++ b/recommendation_v4/configs/amzn-books/sasrec-sampled-softmax-n512-final.gin @@ -0,0 +1,50 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Frozen config, validated on 04/12/2024. +# Based on baseline settings in Revisiting Neural Retrieval on Accelerators (https://arxiv.org/abs/2306.04039, KDD'23). +# +# Run this as: +# mkdir -p logs/amzn-books-l50/ +# CUDA_VISIBLE_DEVICES=1 python3 main.py --gin_config_file=configs/amzn-books/sasrec-sampled-softmax-n512-final.gin --master_port=12346 2>&1 | tee logs/amzn-books-l50/sasrec-sampled-softmax-n512-final.log + +train_fn.dataset_name = "amzn-books" +train_fn.max_sequence_length = 50 +train_fn.local_batch_size = 128 +train_fn.eval_batch_size = 128 + +train_fn.main_module = "SASRec" +train_fn.dropout_rate = 0.5 +train_fn.user_embedding_norm = "l2_norm" +train_fn.item_embedding_dim = 64 + +sasrec_encoder.num_blocks = 4 +sasrec_encoder.num_heads = 4 +sasrec_encoder.ffn_dropout_rate = 0.5 +sasrec_encoder.ffn_hidden_dim = 64 +sasrec_encoder.ffn_activation_fn = "relu" + +train_fn.eval_interval = 4000 +train_fn.num_epochs = 201 +train_fn.learning_rate = 1e-3 +train_fn.weight_decay = 0 +train_fn.num_warmup_steps = 0 + +train_fn.save_ckpt_every_n = 10 + +train_fn.interaction_module_type = "DotProduct" +train_fn.top_k_method = "MIPSBruteForceTopK" + +train_fn.loss_module = "SampledSoftmaxLoss" +train_fn.num_negatives = 512 + +train_fn.sampling_strategy = "local" +train_fn.temperature = 0.05 +train_fn.item_l2_norm = True +train_fn.l2_norm_eps = 1e-6 + +train_fn.enable_tf32 = True +train_fn.full_eval_every_n = 5 +train_fn.partial_eval_num_iters = 64 + +create_data_loader.prefetch_factor = 1024 +create_data_loader.num_workers = 8 diff --git a/recommendation_v4/configs/ml-1m/hstu-sampled-softmax-n128-final.gin b/recommendation_v4/configs/ml-1m/hstu-sampled-softmax-n128-final.gin new file mode 100644 index 000000000..841b1c80a --- /dev/null +++ b/recommendation_v4/configs/ml-1m/hstu-sampled-softmax-n128-final.gin @@ -0,0 +1,45 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Frozen config, validated on 04/11/2024. +# Based on HSTU results (w/ identical configurations as a SotA Transformer baseline) in +# Actions Speak Louder than Words: Trillion-Parameter Sequential Transducers for Generative Recommendations (https://arxiv.org/abs/2402.17152). +# +# Run this as: +# mkdir -p logs/ml-1m-l200/ +# CUDA_VISIBLE_DEVICES=0 python3 main.py --gin_config_file=configs/ml-1m/hstu-sampled-softmax-n128-final.gin --master_port=12345 2>&1 | tee logs/ml-1m-l200/hstu-sampled-softmax-n128-final.log + +train_fn.dataset_name = "ml-1m" +train_fn.max_sequence_length = 200 +train_fn.local_batch_size = 128 + +train_fn.main_module = "HSTU" +train_fn.dropout_rate = 0.2 +train_fn.user_embedding_norm = "l2_norm" +train_fn.num_epochs = 101 +train_fn.item_embedding_dim = 50 + +hstu_encoder.num_blocks = 2 +hstu_encoder.num_heads = 1 +hstu_encoder.dqk = 50 +hstu_encoder.dv = 50 +hstu_encoder.linear_dropout_rate = 0.2 + +train_fn.learning_rate = 1e-3 +train_fn.weight_decay = 0 +train_fn.num_warmup_steps = 0 + +train_fn.interaction_module_type = "DotProduct" +train_fn.top_k_method = "MIPSBruteForceTopK" + +train_fn.loss_module = "SampledSoftmaxLoss" +train_fn.num_negatives = 128 + +train_fn.sampling_strategy = "local" +train_fn.temperature = 0.05 +train_fn.item_l2_norm = True +train_fn.l2_norm_eps = 1e-6 + +train_fn.enable_tf32 = True + +create_data_loader.prefetch_factor = 128 +create_data_loader.num_workers = 8 diff --git a/recommendation_v4/configs/ml-1m/hstu-sampled-softmax-n128-large-final.gin b/recommendation_v4/configs/ml-1m/hstu-sampled-softmax-n128-large-final.gin new file mode 100644 index 000000000..7ffc7ef64 --- /dev/null +++ b/recommendation_v4/configs/ml-1m/hstu-sampled-softmax-n128-large-final.gin @@ -0,0 +1,45 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Frozen config, validated on 04/11/2024. +# Based on HSTU-large results in +# Actions Speak Louder than Words: Trillion-Parameter Sequential Transducers for Generative Recommendations (https://arxiv.org/abs/2402.17152). +# +# Run this as: +# mkdir -p logs/ml-1m-l200/ +# CUDA_VISIBLE_DEVICES=1 python3 main.py --gin_config_file=configs/ml-1m/hstu-sampled-softmax-n128-large-final.gin --master_port=12346 2>&1 | tee logs/ml-1m-l200/hstu-sampled-softmax-n128-large-final.log + +train_fn.dataset_name = "ml-1m" +train_fn.max_sequence_length = 200 +train_fn.local_batch_size = 128 + +train_fn.main_module = "HSTU" +train_fn.dropout_rate = 0.2 +train_fn.user_embedding_norm = "l2_norm" +train_fn.num_epochs = 101 +train_fn.item_embedding_dim = 50 + +hstu_encoder.num_blocks = 8 +hstu_encoder.num_heads = 2 +hstu_encoder.dqk = 25 +hstu_encoder.dv = 25 +hstu_encoder.linear_dropout_rate = 0.2 + +train_fn.learning_rate = 1e-3 +train_fn.weight_decay = 0 +train_fn.num_warmup_steps = 0 + +train_fn.interaction_module_type = "DotProduct" +train_fn.top_k_method = "MIPSBruteForceTopK" + +train_fn.loss_module = "SampledSoftmaxLoss" +train_fn.num_negatives = 128 + +train_fn.sampling_strategy = "local" +train_fn.temperature = 0.05 +train_fn.item_l2_norm = True +train_fn.l2_norm_eps = 1e-6 + +train_fn.enable_tf32 = True + +create_data_loader.prefetch_factor = 128 +create_data_loader.num_workers = 8 diff --git a/recommendation_v4/configs/ml-1m/sasrec-sampled-softmax-n128-final.gin b/recommendation_v4/configs/ml-1m/sasrec-sampled-softmax-n128-final.gin new file mode 100644 index 000000000..ead7bb21c --- /dev/null +++ b/recommendation_v4/configs/ml-1m/sasrec-sampled-softmax-n128-final.gin @@ -0,0 +1,44 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Frozen config, validated on 04/11/2024. +# Based on baseline settings in Revisiting Neural Retrieval on Accelerators (https://arxiv.org/abs/2306.04039, KDD'23). +# +# Run this as: +# mkdir -p logs/ml-1m-l200/ +# CUDA_VISIBLE_DEVICES=0 python3 main.py --gin_config_file=configs/ml-1m/sasrec-sampled-softmax-n128-final.gin --master_port=12345 2>&1 | tee logs/ml-1m-l200/sasrec-sampled-softmax-n128-final.log + +train_fn.dataset_name = "ml-1m" +train_fn.max_sequence_length = 200 +train_fn.local_batch_size = 128 + +train_fn.main_module = "SASRec" +train_fn.dropout_rate = 0.2 +train_fn.user_embedding_norm = "l2_norm" +train_fn.num_epochs = 101 +train_fn.item_embedding_dim = 50 + +sasrec_encoder.num_blocks = 2 +sasrec_encoder.num_heads = 1 +sasrec_encoder.ffn_dropout_rate = 0.2 +sasrec_encoder.ffn_hidden_dim = 50 +sasrec_encoder.ffn_activation_fn = "relu" + +train_fn.learning_rate = 1e-3 +train_fn.weight_decay = 0 +train_fn.num_warmup_steps = 0 + +train_fn.top_k_method = "MIPSBruteForceTopK" +train_fn.interaction_module_type = "DotProduct" + +train_fn.loss_module = "SampledSoftmaxLoss" +train_fn.num_negatives = 128 + +train_fn.sampling_strategy = "local" +train_fn.temperature = 0.05 +train_fn.item_l2_norm = True +train_fn.l2_norm_eps = 1e-6 + +train_fn.enable_tf32 = True + +create_data_loader.prefetch_factor = 128 +create_data_loader.num_workers = 8 diff --git a/recommendation_v4/configs/ml-20m/hstu-sampled-softmax-n128-final.gin b/recommendation_v4/configs/ml-20m/hstu-sampled-softmax-n128-final.gin new file mode 100644 index 000000000..5823ad5b6 --- /dev/null +++ b/recommendation_v4/configs/ml-20m/hstu-sampled-softmax-n128-final.gin @@ -0,0 +1,45 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Frozen config, validated on 04/12/2024. +# Based on HSTU results (w/ identical configurations as a SotA Transformer baseline) in +# Actions Speak Louder than Words: Trillion-Parameter Sequential Transducers for Generative Recommendations (https://arxiv.org/abs/2402.17152). +# +# Run this as: +# mkdir -p logs/ml-20m-l200/ +# CUDA_VISIBLE_DEVICES=0 python3 main.py --gin_config_file=configs/ml-20m/hstu-sampled-softmax-n128-final.gin --master_port=12345 2>&1 | tee logs/ml-20m-l200/hstu-sampled-softmax-n128-final.log + +train_fn.dataset_name = "ml-20m" +train_fn.max_sequence_length = 200 +train_fn.local_batch_size = 128 + +train_fn.main_module = "HSTU" +train_fn.dropout_rate = 0.2 +train_fn.user_embedding_norm = "l2_norm" +train_fn.num_epochs = 101 +train_fn.item_embedding_dim = 256 + +hstu_encoder.num_blocks = 4 +hstu_encoder.num_heads = 4 +hstu_encoder.dv = 64 +hstu_encoder.dqk = 64 +hstu_encoder.linear_dropout_rate = 0.2 + +train_fn.learning_rate = 1e-3 +train_fn.weight_decay = 0 +train_fn.num_warmup_steps = 0 + +train_fn.interaction_module_type = "DotProduct" +train_fn.top_k_method = "MIPSBruteForceTopK" + +train_fn.loss_module = "SampledSoftmaxLoss" +train_fn.num_negatives = 128 + +train_fn.sampling_strategy = "local" +train_fn.temperature = 0.05 +train_fn.item_l2_norm = True +train_fn.l2_norm_eps = 1e-6 + +train_fn.enable_tf32 = True + +create_data_loader.prefetch_factor = 128 +create_data_loader.num_workers = 8 diff --git a/recommendation_v4/configs/ml-20m/hstu-sampled-softmax-n128-large-final.gin b/recommendation_v4/configs/ml-20m/hstu-sampled-softmax-n128-large-final.gin new file mode 100644 index 000000000..0199afa24 --- /dev/null +++ b/recommendation_v4/configs/ml-20m/hstu-sampled-softmax-n128-large-final.gin @@ -0,0 +1,45 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Frozen config, validated on 04/12/2024. +# Based on HSTU-large results in +# Actions Speak Louder than Words: Trillion-Parameter Sequential Transducers for Generative Recommendations (https://arxiv.org/abs/2402.17152). +# +# Run this as: +# mkdir -p logs/ml-20m-l200/ +# CUDA_VISIBLE_DEVICES=0 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True python3 main.py --gin_config_file=configs/ml-20m/hstu-sampled-softmax-n128-large-final.gin --master_port=12345 2>&1 | tee logs/ml-20m-l200/hstu-sampled-softmax-n128-large-final.log + +train_fn.dataset_name = "ml-20m" +train_fn.max_sequence_length = 200 +train_fn.local_batch_size = 128 + +train_fn.main_module = "HSTU" +train_fn.dropout_rate = 0.2 +train_fn.user_embedding_norm = "l2_norm" +train_fn.num_epochs = 101 +train_fn.item_embedding_dim = 256 + +hstu_encoder.num_blocks = 16 +hstu_encoder.num_heads = 8 +hstu_encoder.dv = 32 +hstu_encoder.dqk = 32 +hstu_encoder.linear_dropout_rate = 0.2 + +train_fn.learning_rate = 1e-3 +train_fn.weight_decay = 0 +train_fn.num_warmup_steps = 0 + +train_fn.interaction_module_type = "DotProduct" +train_fn.top_k_method = "MIPSBruteForceTopK" + +train_fn.loss_module = "SampledSoftmaxLoss" +train_fn.num_negatives = 128 + +train_fn.sampling_strategy = "local" +train_fn.temperature = 0.05 +train_fn.item_l2_norm = True +train_fn.l2_norm_eps = 1e-6 + +train_fn.enable_tf32 = True + +create_data_loader.prefetch_factor = 128 +create_data_loader.num_workers = 8 diff --git a/recommendation_v4/configs/ml-20m/sasrec-sampled-softmax-n128-final.gin b/recommendation_v4/configs/ml-20m/sasrec-sampled-softmax-n128-final.gin new file mode 100644 index 000000000..3c666f802 --- /dev/null +++ b/recommendation_v4/configs/ml-20m/sasrec-sampled-softmax-n128-final.gin @@ -0,0 +1,44 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Frozen config, validated on 04/12/2024. +# Based on baseline settings in Revisiting Neural Retrieval on Accelerators (https://arxiv.org/abs/2306.04039, KDD'23). +# +# Run this as: +# mkdir -p logs/ml-20m-l200/ +# CUDA_VISIBLE_DEVICES=0 python3 main.py --gin_config_file=configs/ml-20m/sasrec-sampled-softmax-n128-final.gin --master_port=12345 2>&1 | tee logs/ml-20m-l200/sasrec-sampled-softmax-n128-final.log + +train_fn.dataset_name = "ml-20m" +train_fn.max_sequence_length = 200 +train_fn.local_batch_size = 128 + +train_fn.main_module = "SASRec" +train_fn.dropout_rate = 0.2 +train_fn.user_embedding_norm = "l2_norm" +train_fn.num_epochs = 101 +train_fn.item_embedding_dim = 256 + +sasrec_encoder.num_blocks = 4 +sasrec_encoder.num_heads = 4 +sasrec_encoder.ffn_dropout_rate = 0.2 +sasrec_encoder.ffn_hidden_dim = 256 +sasrec_encoder.ffn_activation_fn = "relu" + +train_fn.learning_rate = 1e-3 +train_fn.weight_decay = 0 +train_fn.num_warmup_steps = 0 + +train_fn.top_k_method = "MIPSBruteForceTopK" +train_fn.interaction_module_type = "DotProduct" + +train_fn.loss_module = "SampledSoftmaxLoss" +train_fn.num_negatives = 128 + +train_fn.sampling_strategy = "local" +train_fn.temperature = 0.05 +train_fn.item_l2_norm = True +train_fn.l2_norm_eps = 1e-6 + +train_fn.enable_tf32 = True + +create_data_loader.prefetch_factor = 128 +create_data_loader.num_workers = 8 diff --git a/recommendation_v4/configs/ml-3b/hstu-sampled-softmax-n96-seqlen500-final.gin b/recommendation_v4/configs/ml-3b/hstu-sampled-softmax-n96-seqlen500-final.gin new file mode 100644 index 000000000..ac7a85350 --- /dev/null +++ b/recommendation_v4/configs/ml-3b/hstu-sampled-softmax-n96-seqlen500-final.gin @@ -0,0 +1,42 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Run this as: +# mkdir -p logs/ml-3b-l500/ +# CUDA_VISIBLE_DEVICES=0 python3 main.py --gin_config_file=configs/ml-3b/hstu-sampled-softmax-n96-seqlen500-final.gin --master_port=12345 2>&1 | tee logs/ml-3b-l500/hstu-sampled-softmax-n96-seqlen500-final.log + +train_fn.dataset_name = "ml-3b" +train_fn.max_sequence_length = 500 +train_fn.local_batch_size = 96 +train_fn.eval_batch_size = 96 + +train_fn.main_module = "HSTU" +train_fn.dropout_rate = 0.2 +train_fn.user_embedding_norm = "l2_norm" +train_fn.num_epochs = 100 +train_fn.item_embedding_dim = 256 + +hstu_encoder.num_blocks = 4 +hstu_encoder.num_heads = 4 +hstu_encoder.dv = 64 +hstu_encoder.dqk = 64 +hstu_encoder.linear_dropout_rate = 0.2 + +train_fn.learning_rate = 1e-3 +train_fn.weight_decay = 0 +train_fn.num_warmup_steps = 0 + +train_fn.interaction_module_type = "DotProduct" +train_fn.top_k_method = "MIPSBruteForceTopK" + +train_fn.loss_module = "SampledSoftmaxLoss" +train_fn.num_negatives = 128 + +train_fn.sampling_strategy = "local" +train_fn.temperature = 0.05 +train_fn.item_l2_norm = True +train_fn.l2_norm_eps = 1e-6 + +train_fn.enable_tf32 = True + +create_data_loader.prefetch_factor = 128 +create_data_loader.num_workers = 8 diff --git a/recommendation_v4/configs/ml-3b/hstu-sampled-softmax-n96-seqlen500-large-final.gin b/recommendation_v4/configs/ml-3b/hstu-sampled-softmax-n96-seqlen500-large-final.gin new file mode 100644 index 000000000..a30ad3657 --- /dev/null +++ b/recommendation_v4/configs/ml-3b/hstu-sampled-softmax-n96-seqlen500-large-final.gin @@ -0,0 +1,42 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Run this as: +# mkdir -p logs/ml-3b-l500/ +# CUDA_VISIBLE_DEVICES=0 python3 main.py --gin_config_file=configs/ml-3b/hstu-sampled-softmax-n96-seqlen500-large-final.gin --master_port=12345 2>&1 | tee logs/ml-3b-l500/hstu-sampled-softmax-n96-seqlen500-large-final.log + +train_fn.dataset_name = "ml-3b" +train_fn.max_sequence_length = 500 +train_fn.local_batch_size = 96 +train_fn.eval_batch_size = 96 + +train_fn.main_module = "HSTU" +train_fn.dropout_rate = 0.2 +train_fn.user_embedding_norm = "l2_norm" +train_fn.num_epochs = 100 +train_fn.item_embedding_dim = 256 + +hstu_encoder.num_blocks = 16 +hstu_encoder.num_heads = 8 +hstu_encoder.dv = 32 +hstu_encoder.dqk = 32 +hstu_encoder.linear_dropout_rate = 0.2 + +train_fn.learning_rate = 1e-3 +train_fn.weight_decay = 0 +train_fn.num_warmup_steps = 0 + +train_fn.interaction_module_type = "DotProduct" +train_fn.top_k_method = "MIPSBruteForceTopK" + +train_fn.loss_module = "SampledSoftmaxLoss" +train_fn.num_negatives = 128 + +train_fn.sampling_strategy = "local" +train_fn.temperature = 0.05 +train_fn.item_l2_norm = True +train_fn.l2_norm_eps = 1e-6 + +train_fn.enable_tf32 = True + +create_data_loader.prefetch_factor = 128 +create_data_loader.num_workers = 8 diff --git a/recommendation_v4/configs/ml-3b/sasrec-sampled-softmax-n96-seqlen500-final.gin b/recommendation_v4/configs/ml-3b/sasrec-sampled-softmax-n96-seqlen500-final.gin new file mode 100644 index 000000000..034c478b4 --- /dev/null +++ b/recommendation_v4/configs/ml-3b/sasrec-sampled-softmax-n96-seqlen500-final.gin @@ -0,0 +1,42 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Run this as: +# mkdir -p logs/ml-3b-l500/ +# CUDA_VISIBLE_DEVICES=0 python3 main.py --gin_config_file=configs/ml-3b/sasrec-sampled-softmax-n96-seqlen500-final.gin --master_port=12345 2>&1 | tee logs/ml-3b-l500/sasrec-sampled-softmax-n96-seqlen500-final.log + +train_fn.dataset_name = "ml-3b" +train_fn.max_sequence_length = 500 +train_fn.local_batch_size = 96 +train_fn.eval_batch_size = 96 + +train_fn.main_module = "SASRec" +train_fn.dropout_rate = 0.2 +train_fn.user_embedding_norm = "l2_norm" +train_fn.num_epochs = 100 +train_fn.item_embedding_dim = 256 + +sasrec_encoder.num_blocks = 4 +sasrec_encoder.num_heads = 4 +sasrec_encoder.ffn_dropout_rate = 0.2 +sasrec_encoder.ffn_hidden_dim = 256 +sasrec_encoder.ffn_activation_fn = "relu" + +train_fn.learning_rate = 1e-3 +train_fn.weight_decay = 0 +train_fn.num_warmup_steps = 0 + +train_fn.top_k_method = "MIPSBruteForceTopK" +train_fn.interaction_module_type = "DotProduct" + +train_fn.loss_module = "SampledSoftmaxLoss" +train_fn.num_negatives = 128 + +train_fn.sampling_strategy = "local" +train_fn.temperature = 0.05 +train_fn.item_l2_norm = True +train_fn.l2_norm_eps = 1e-6 + +train_fn.enable_tf32 = True + +create_data_loader.prefetch_factor = 128 +create_data_loader.num_workers = 8 diff --git a/recommendation_v4/generative_recommenders/README.md b/recommendation_v4/generative_recommenders/README.md new file mode 100644 index 000000000..e69de29bb diff --git a/recommendation_v4/generative_recommenders/common.py b/recommendation_v4/generative_recommenders/common.py new file mode 100644 index 000000000..2ff8edf80 --- /dev/null +++ b/recommendation_v4/generative_recommenders/common.py @@ -0,0 +1,513 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +import abc +import copy +import os +from enum import Enum, unique +from typing import Any, Callable, List, Optional, Tuple + +import torch + +# @manual=//triton:triton +import triton +from generative_recommenders.ops.utils import is_sm100_plus, is_sm90_plus +from torch.fx._symbolic_trace import is_fx_tracing +from torch.utils._python_dispatch import _get_current_dispatch_mode_stack + +# @manual=//triton:triton +from triton.runtime.autotuner import Autotuner + +try: + torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops") + torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops_cpu") +except OSError: + pass + +try: + # @manual=//triton:triton + import triton.language.extra.tlx # type: ignore + + HAS_TLX = True +except ImportError: + HAS_TLX = False + +try: + from generative_recommenders.fb.triton_cc.utils import triton_cc + from hammer.ops.triton.utils import triton_autotune + from hammer.utils import is_dev_mode, set_dev_mode, set_verbose_level +except ImportError: + # pyre-ignore + def triton_cc(annotations): + # pyre-ignore + def decorator(fn): + return fn + + return decorator + + # pyre-ignore + def triton_autotune( + configs: List[triton.Config], + key: List[str], + # pyre-ignore + prune_configs_by=None, + # pyre-ignore + reset_to_zero=None, + # pyre-ignore + restore_value=None, + warmup: int = 25, + rep: int = 100, + ): + # pyre-ignore + def decorator(fn): + return Autotuner( + fn, + fn.arg_names, + configs, + key, + reset_to_zero, + restore_value, + pre_hook=None, + post_hook=None, + prune_configs_by=prune_configs_by, + warmup=warmup, + rep=rep, + ) + + return decorator + + DEV_MODE: bool = False + VERBOSE_LEVEL: int = 0 + + def set_dev_mode(val: bool) -> None: + global DEV_MODE + DEV_MODE = val + + def is_dev_mode() -> bool: + global DEV_MODE # noqa: F824 + return DEV_MODE + + def set_verbose_level(level: int) -> None: + global VERBOSE_LEVEL + VERBOSE_LEVEL = level + + def get_verbose_level() -> int: + global VERBOSE_LEVEL # noqa: F824 + return VERBOSE_LEVEL + + +@unique +class HammerKernel(Enum): + TRITON = "TRITON" + TLX = "TLX" + PYTORCH = "PYTORCH" + CUDA = "CUDA" + TRITON_CC = "TRITON_CC" + TRITON_INFERENCE = "TRITON_INFERENCE" + CUTEDSL = "CUTEDSL" + + +class HammerModule(torch.nn.Module, abc.ABC): + _is_inference: bool = False + _use_triton_cc: bool = True + _training_dtype: torch.dtype = torch.float32 + _hammer_kernel: Optional[HammerKernel] = None + + def __init__( + self, + is_inference: bool, + training_dytpe: torch.dtype = torch.float32, + use_triton_cc: bool = _use_triton_cc, + hammer_kernel: Optional[HammerKernel] = None, + ) -> None: + super().__init__() + self._is_inference = is_inference + self._training_dtype = training_dytpe + self._hammer_kernel = hammer_kernel + self._use_triton_cc = use_triton_cc + + def hammer_kernel(self) -> HammerKernel: + kernel = self._hammer_kernel + if kernel is not None: + return kernel + if self._is_inference and self._use_triton_cc: + return HammerKernel.TRITON_CC + else: + return HammerKernel.TRITON + + # pyre-ignore[2] + def recursive_setattr(self, name: str, value: Any) -> None: + for _, module in self.named_modules(): + if hasattr(module, name): + setattr(module, name, value) + + def set_use_triton_cc(self, use_triton_cc: bool) -> None: + self._use_triton_cc = use_triton_cc + self.recursive_setattr("_use_triton_cc", use_triton_cc) + + def set_is_inference(self, is_inference: bool) -> None: + self._is_inference = is_inference + self.recursive_setattr("_is_inference", is_inference) + + def set_training_dtype(self, training_dtype: torch.dtype) -> None: + self._training_dtype = training_dtype + self.recursive_setattr("_training_dtype", training_dtype) + + def set_hammer_kernel(self, hammer_kernel: HammerKernel) -> None: + self._hammer_kernel = hammer_kernel + self.recursive_setattr("_hammer_kernel", hammer_kernel) + + @property + def is_inference(self) -> bool: + return self._is_inference + + @property + def is_eval(self) -> bool: + return (not self._is_inference) and (not self.training) + + @property + def is_train(self) -> bool: + return (not self._is_inference) and self.training + + +def generate_sparse_seq_len( + size: int, + max_seq_len: int, + sparsity: float, + device: torch.device, +) -> torch.Tensor: + if sparsity == 0.0: + return torch.zeros(size=(size,), device=device, dtype=torch.int) + elif sparsity == 1.0: + return torch.ones(size=(size,), device=device, dtype=torch.int) * max_seq_len + elif sparsity >= 0.5: + min_seq_len: int = int((2 * sparsity - 1.0) * max_seq_len) + return torch.randint( + low=min_seq_len, + high=max_seq_len, + size=(size,), + device=device, + dtype=torch.int, + ) + else: + min_seq_len: int = 0 + max_seq_len: int = int(2 * sparsity * max_seq_len) + return torch.randint( + low=min_seq_len, + high=max_seq_len, + size=(size,), + device=device, + dtype=torch.int, + ) + + +def apply_sampling( + lengths: torch.Tensor, + alpha: float, + max_seq_len: int, +) -> torch.Tensor: + threshold = int(max_seq_len ** (alpha / 2)) + no_sample_prob = (max_seq_len**alpha) / torch.pow(lengths, 2) + users_to_sample = torch.logical_and( + lengths > threshold, + torch.rand_like(no_sample_prob) < 1 - no_sample_prob, + ) + lengths = torch.where(users_to_sample, threshold, lengths) + return lengths + + +nv_gpu_unavailable: Tuple[bool, str] = ( + not torch.cuda.is_available() or torch.cuda.device_count() == 0, + "CUDA is not available or no GPUs detected", +) +nv_gpu_available: bool = not nv_gpu_unavailable[0] + + +amd_gpu_unavailable: Tuple[bool, str] = ( + not torch.version.hip, + "AMD HIP not available or no GPUs detected", +) +amd_gpu_available: bool = not amd_gpu_unavailable[0] + +gpu_unavailable: Tuple[bool, str] = ( + not nv_gpu_available and not amd_gpu_available, + "CUDA/HIP is not available or no GPUs detected", +) + +gpu_available: bool = not gpu_unavailable[0] + +blackwell_tlx_unavailable: Tuple[bool, str] = ( + not is_sm100_plus() or not HAS_TLX, + "Skip TLX and blackwell only tests", +) + +tma_unavailable: Tuple[bool, str] = ( + not is_sm90_plus(), # noqa + "Skip TMA only tests", +) + + +def switch_to_contiguous_if_needed(x: torch.Tensor) -> torch.Tensor: + if torch.jit.is_scripting(): + if x.stride(-1) == 1: + return x + return x.contiguous() + if torch.compiler.is_compiling(): + # Tell Dynamo this data-dependent value is in the range (0, 10**9) + torch._check(x.size(0) > 0) + torch._check(x.size(0) < 10**9) + # FX cannot trace Python control flow over symbolic stride checks + # (`x.stride(-1) == 1`). For AOT-T lowering, conservatively emit the + # contiguous op instead of branching on a symbolic value. + if is_fx_tracing(): + return x.contiguous() + if x.stride(-1) == 1: + return x + return x.contiguous() + + +def cdiv(x: int, y: int) -> int: + return (x + y - 1) // y + + +def backend_allow_tf32() -> bool: + return True + + +BACKEND_ALLOW_TF32: bool = backend_allow_tf32() + + +def next_power_of_2(n: int) -> int: + """Return the smallest power of 2 greater than or equal to n""" + n -= 1 + n |= n >> 1 + n |= n >> 2 + n |= n >> 4 + n |= n >> 8 + n |= n >> 16 + n |= n >> 32 + n += 1 + return n + + +def _prev_power_of_2_bitwise(x: int) -> int: + """Return the largest power of 2 less than or equal to x.""" + x |= x >> 1 + x |= x >> 2 + x |= x >> 4 + x |= x >> 8 + x |= x >> 16 + x |= x >> 32 + return (x >> 1) + 1 + + +@torch.fx.wrap +def _prev_power_of_2_legacy(x: int) -> int: + if torch.compiler.is_compiling(): + # Re-write to make Dynamo happy + x_tensor = torch.scalar_tensor(x, dtype=torch.int64) # type: ignore[arg-type] + x_tensor_orig = x_tensor.clone() + out_val = next_power_of_2(int(x_tensor.item())) # type: ignore[arg-type] + out = torch.scalar_tensor(out_val, dtype=torch.int64) + return int(torch.where(torch.lt(x_tensor_orig, out), out // 2, out).item()) # type: ignore[return-value] + else: + out = next_power_of_2(x) + return out // 2 if out > x else out + + +prev_power_of_2: Callable[[int], int] = ( + _prev_power_of_2_legacy + if os.environ.get("PREV_POWER_OF_2_IMPL", "legacy") == "legacy" + else _prev_power_of_2_bitwise +) + + +STATIC_MAX_SEQ_LENS: List[int] = [] +USE_RUNTIME_MAX_SEQ_LEN: bool = False + + +def set_static_max_seq_lens(max_seq_lens: List[int]) -> None: + global STATIC_MAX_SEQ_LENS + STATIC_MAX_SEQ_LENS = copy.deepcopy(max_seq_lens) + STATIC_MAX_SEQ_LENS.sort() + + +def set_use_runtime_max_seq_len(use_runtime_max_seq_len: bool) -> None: + global USE_RUNTIME_MAX_SEQ_LEN + USE_RUNTIME_MAX_SEQ_LEN = use_runtime_max_seq_len + + +def autotune_max_seq_len(runtime_max_seq_len: int) -> int: + global USE_RUNTIME_MAX_SEQ_LEN # noqa: F824 + + if USE_RUNTIME_MAX_SEQ_LEN: + return prev_power_of_2(runtime_max_seq_len) + else: + if STATIC_MAX_SEQ_LENS == []: + return 1 + for max_len in STATIC_MAX_SEQ_LENS: + if max_len >= runtime_max_seq_len: + return max_len + return STATIC_MAX_SEQ_LENS[-1] + + +def fine_grained_autotune_max_seq_len(runtime_max_seq_len: int) -> int: + global USE_RUNTIME_MAX_SEQ_LEN # noqa: F824 + + if USE_RUNTIME_MAX_SEQ_LEN: + return _fine_grained_bucket_size(runtime_max_seq_len) + else: + if STATIC_MAX_SEQ_LENS == []: + return 1 + for max_len in STATIC_MAX_SEQ_LENS: + if max_len >= runtime_max_seq_len: + return max_len + return STATIC_MAX_SEQ_LENS[-1] + + +def _generate_fine_grained_buckets() -> List[int]: + buckets = [ + 1024, + 2048, + 4096, + 8192, + 12288, + 16384, + 24576, + 32768, + 40960, + 49152, + 65536, + 81920, + 98304, + ] + return buckets + + +@torch.fx.wrap +def _fine_grained_bucket_size(x: int) -> int: + if torch.compiler.is_compiling(): + x_tensor = torch.scalar_tensor(x, dtype=torch.int64) + buckets = torch.tensor(_generate_fine_grained_buckets(), dtype=torch.int64) + + mask = buckets >= x_tensor + valid_buckets = torch.where( + mask, buckets, torch.tensor(2**31 - 1, dtype=torch.int64) + ) + + result = torch.where(mask.any(), valid_buckets.min(), buckets[-1]) + + return int(result.item()) + else: + buckets = _generate_fine_grained_buckets() + + for bucket in buckets: + if x <= bucket: + return bucket + + return buckets[-1] + + +@torch.fx.wrap +def fx_unwrap_optional_tensor(optional: Optional[torch.Tensor]) -> torch.Tensor: + assert optional is not None, "Expected optional to be non-None Tensor" + return optional + + +@torch.fx.wrap +def fx_arange(len: int, device: torch.device) -> torch.Tensor: + return torch.arange(len, device=device) + + +@torch.fx.wrap +def fx_infer_max_len( + lengths: torch.Tensor, +) -> int: + # Do not call ".item()" to avoid unbacked symint problems for lowering + max_len = int(lengths.max()) + if not torch.jit.is_scripting() and torch.compiler.is_compiling(): + # Tell Dynamo this data-dependent value is in the range [0, 10**9) + torch._check_is_size(max_len) + torch._check(max_len < 10**9) + torch._check(max_len > 0) + return max_len + + +@torch.fx.wrap +def fx_mark_length_features(tensor: torch.Tensor) -> torch.Tensor: + return tensor + + +@torch.fx.wrap +def fx_torch_ones( + shape: List[int], + device: torch.device, + dtype: torch.dtype, +) -> torch.Tensor: + return torch.ones(shape, device=device, dtype=dtype) + + +@torch.fx.wrap +def fx_torch_zeros(shape: List[int], device: torch.device) -> torch.Tensor: + return torch.zeros(shape, device=device) + + +def _is_in_dispatch_modes(mode_names: List[str]) -> bool: + modes = _get_current_dispatch_mode_stack() + return any(mode.__class__.__name__ in mode_names for mode in modes) + + +def should_trigger_eager_impl() -> bool: + if torch.jit.is_scripting(): + return True + if torch.compiler.is_compiling(): + return False + return _is_in_dispatch_modes(["SplitDispatchMode", "FakeTensorMode"]) + + +@torch.fx.wrap +def jagged_to_padded_dense( + values: torch.Tensor, + offsets: List[torch.Tensor], + max_lengths: List[int], + padding_value: float, +) -> torch.Tensor: + return torch.ops.fbgemm.jagged_to_padded_dense( + values=values, + offsets=offsets, + max_lengths=max_lengths, + padding_value=padding_value, + ) + + +@torch.fx.wrap +def dense_to_jagged( + dense: torch.Tensor, + x_offsets: List[torch.Tensor], +) -> torch.Tensor: + return torch.ops.fbgemm.dense_to_jagged( + dense=dense, + x_offsets=x_offsets, + )[0] + + +def init_mlp_weights_optional_bias(m: torch.nn.Module) -> None: + if isinstance(m, torch.nn.Linear): + torch.nn.init.xavier_uniform_(m.weight) + if m.bias is not None: + m.bias.data.fill_(0.0) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/checkpoint.py b/recommendation_v4/generative_recommenders/dlrm_v3/checkpoint.py new file mode 100644 index 000000000..33445bce9 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/checkpoint.py @@ -0,0 +1,258 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-strict +""" +Checkpoint utilities for saving and loading DLRMv3 model checkpoints. + +This module provides functions for saving and loading distributed model checkpoints, +including both sparse (embedding) and dense (non-embedding) components. +""" + +import gc +import os +from datetime import datetime +from typing import Any, Dict, Optional, Set + +import gin +import torch +from generative_recommenders.dlrm_v3.utils import MetricsLogger +from torch.distributed.checkpoint.stateful import Stateful +from torch.optim.optimizer import Optimizer +from torchrec.distributed.types import ShardedTensor + + +class SparseState(Stateful): + """ + Stateful wrapper for sparse (embedding) tensors in a model. + + This class implements the Stateful interface for distributed checkpointing, + allowing sparse tensors to be saved and loaded separately from dense tensors. + + Args: + model: The PyTorch model containing sparse tensors. + sparse_tensor_keys: Set of keys identifying sparse tensors in the model's state dict. + """ + + def __init__(self, model: torch.nn.Module, sparse_tensor_keys: Set[str]) -> None: + self.model = model + self.sparse_tensor_keys = sparse_tensor_keys + + def state_dict(self) -> Dict[str, torch.Tensor]: + out_dict: Dict[str, torch.Tensor] = {} + is_sharded_tensor: Optional[bool] = None + for k, v in self.model.state_dict().items(): + if k in self.sparse_tensor_keys: + if is_sharded_tensor is None: + is_sharded_tensor = isinstance(v, ShardedTensor) + assert is_sharded_tensor == isinstance(v, ShardedTensor) + out_dict[k] = v + return out_dict + + def load_state_dict(self, state_dict: Dict[str, torch.Tensor]) -> None: + incompatible_keys = self.model.load_state_dict(state_dict, strict=False) + assert not incompatible_keys.unexpected_keys + + +def is_sparse_key(k: str, v: torch.Tensor) -> bool: + return isinstance(v, ShardedTensor) or "embedding_collection" in k + + +def load_dense_state_dict(model: torch.nn.Module, state_dict: Dict[str, Any]) -> None: + own_state = model.state_dict() + own_state_dense_keys = {k for k, v in own_state.items() if not is_sparse_key(k, v)} + state_dict_dense_keys = { + k for k, v in state_dict.items() if not is_sparse_key(k, v) + } + assert own_state_dense_keys == state_dict_dense_keys, ( + f"expects {own_state_dense_keys} but gets {state_dict_dense_keys}" + ) + for name in state_dict_dense_keys: + param = state_dict[name] + if isinstance(param, torch.nn.Parameter): + # backwards compatibility for serialized parameters + param = param.data + own_state[name].copy_(param) + + +@gin.configurable +def save_dmp_checkpoint( + model: torch.nn.Module, + optimizer: Optimizer, + metric_logger: MetricsLogger, + rank: int, + batch_idx: int, + path: str = "", +) -> None: + """ + Save a distributed model checkpoint including sparse and dense components. + + Saves the model's sparse tensors using distributed checkpointing and dense + tensors, optimizer state, and metrics using standard PyTorch serialization. + + Args: + model: The model to checkpoint. + optimizer: The optimizer whose state should be saved. + metric_logger: The metrics logger containing training/eval metrics. + rank: The current process rank in distributed training. + batch_idx: The current batch index (used for checkpoint naming). + path: Base path for saving the checkpoint. If empty, no checkpoint is saved. + """ + if path == "": + return + now = datetime.now() + formatted_datetime = now.strftime("%Y_%m_%d_%H_%M_%S") + path = f"{path}/{batch_idx}" + if not os.path.exists(path) and rank == 0: + os.makedirs(path) + sparse_path = f"{path}/sparse/" + if not os.path.exists(sparse_path) and rank == 0: + os.makedirs(sparse_path) + non_sparse_ckpt = f"{path}/non_sparse.ckpt" + + sparse_tensor_keys = { + k for k, v in model.state_dict().items() if isinstance(v, ShardedTensor) + } + if rank == 0: + dense_state_dict = { + k: v + for k, v in model.state_dict().items() + if not isinstance(v, ShardedTensor) + } + class_metric_state_dict = { + "train": [m.state_dict() for m in metric_logger.class_metrics["train"]], + "eval": [m.state_dict() for m in metric_logger.class_metrics["eval"]], + } + regression_metric_state_dict = { + "train": [ + m.state_dict() for m in metric_logger.regression_metrics["train"] + ], + "eval": [m.state_dict() for m in metric_logger.regression_metrics["eval"]], + } + torch.save( + { + "dense_dict": dense_state_dict, + "optimizer_dict": optimizer.state_dict(), + "class_metrics": class_metric_state_dict, + "reg_metrics": regression_metric_state_dict, + "global_step": metric_logger.global_step, + "sparse_tensor_keys": sparse_tensor_keys, + }, + non_sparse_ckpt, + ) + torch.distributed.barrier() + sparse_dict = {"sparse_dict": SparseState(model, sparse_tensor_keys)} + torch.distributed.checkpoint.save( + sparse_dict, + storage_writer=torch.distributed.checkpoint.FileSystemWriter(sparse_path), + ) + torch.distributed.barrier() + print("checkpoint successfully saved") + + +@gin.configurable +def load_sparse_checkpoint( + model: torch.nn.Module, + path: str = "", +) -> None: + if path == "": + return + sparse_path = f"{path}/sparse/" + + sparse_tensor_keys = { + k for k, v in model.state_dict().items() if is_sparse_key(k, v) + } + sparse_dict = {"sparse_dict": SparseState(model, sparse_tensor_keys)} + gc.collect() + torch.distributed.checkpoint.load( + sparse_dict, + storage_reader=torch.distributed.checkpoint.FileSystemReader(sparse_path), + ) + gc.collect() + print("sparse checkpoint successfully loaded") + + +@gin.configurable +def load_nonsparse_checkpoint( + model: torch.nn.Module, + device: torch.device, + optimizer: Optional[Optimizer] = None, + metric_logger: Optional[MetricsLogger] = None, + path: str = "", +) -> None: + """ + Load non-sparse (dense) components from a checkpoint. + + Loads dense model parameters, and optionally optimizer state and metrics. + + Args: + model: The model to load dense parameters into. + device: The device to load tensors onto. + optimizer: Optional optimizer to restore state for. + metric_logger: Optional metrics logger to restore state for. + path: Base path of the checkpoint. If empty, no loading is performed. + """ + if path == "": + return + non_sparse_ckpt = f"{path}/non_sparse.ckpt" + + non_sparse_state_dict = torch.load(non_sparse_ckpt, map_location=device) + load_dense_state_dict(model, non_sparse_state_dict["dense_dict"]) + print("dense checkpoint successfully loaded") + if optimizer is not None: + optimizer.load_state_dict(non_sparse_state_dict["optimizer_dict"]) + print("optimizer checkpoint successfully loaded") + if metric_logger is not None: + metric_logger.global_step = non_sparse_state_dict["global_step"] + class_metric_state_dict = non_sparse_state_dict["class_metrics"] + regression_metric_state_dict = non_sparse_state_dict["reg_metrics"] + for i, m in enumerate(metric_logger.class_metrics["train"]): + m.load_state_dict(class_metric_state_dict["train"][i]) + for i, m in enumerate(metric_logger.class_metrics["eval"]): + m.load_state_dict(class_metric_state_dict["eval"][i]) + for i, m in enumerate(metric_logger.regression_metrics["train"]): + m.load_state_dict(regression_metric_state_dict["train"][i]) + for i, m in enumerate(metric_logger.regression_metrics["eval"]): + m.load_state_dict(regression_metric_state_dict["eval"][i]) + + +@gin.configurable +def load_dmp_checkpoint( + model: torch.nn.Module, + optimizer: Optimizer, + metric_logger: MetricsLogger, + device: torch.device, + path: str = "", +) -> None: + """ + Load a complete distributed model checkpoint (both sparse and dense components). + + This is a convenience function that calls both load_sparse_checkpoint and + load_nonsparse_checkpoint. + + Args: + model: The model to load the checkpoint into. + optimizer: The optimizer to restore state for. + metric_logger: The metrics logger to restore state for. + device: The device to load tensors onto. + path: Base path of the checkpoint. If empty, no loading is performed. + """ + load_sparse_checkpoint(model=model, path=path) + load_nonsparse_checkpoint( + model=model, + optimizer=optimizer, + metric_logger=metric_logger, + path=path, + device=device, + ) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/configs.py b/recommendation_v4/generative_recommenders/dlrm_v3/configs.py new file mode 100644 index 000000000..2981f01e3 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/configs.py @@ -0,0 +1,704 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-strict +""" +Configuration module for DLRMv3 model. + +This module provides configuration functions for the HSTU model architecture and embedding table configurations. +""" + +from typing import Dict + +from generative_recommenders.modules.dlrm_hstu import DlrmHSTUConfig +from generative_recommenders.modules.multitask_module import ( + MultitaskTaskType, + TaskConfig, +) +from torchrec.modules.embedding_configs import DataType, EmbeddingConfig + +HSTU_EMBEDDING_DIM = 512 # final DLRMv3 model +HASH_SIZE = 10_000_000 +HASH_SIZE_1B = 1_000_000_000 + +YAMBDA_EMBEDDING_DIM = 512 + +# (name, keys, num_embeddings, salt) — single source of truth for both +# get_embedding_table_config("yambda-5b") and the dataset's cross-hash inputs. +# Sizes mirror Primus-DLRM/configs/bench_onetrans_large_5b_cross_feat_shampoo.yaml. +YAMBDA_5B_CROSS_SPECS = [ + ("user_x_artist", ("uid", "artist_id"), 100_000_000, 0), + ("user_x_album", ("uid", "album_id"), 40_000_000, 0), + ("user_x_hour", ("uid", "hour_of_day"), 24_000_000, 0), + ("item_x_hour", ("item_id", "hour_of_day"), 40_000_000, 0), + ("artist_x_hour", ("artist_id", "hour_of_day"), 32_000_000, 0), + ("user_x_is_organic", ("uid", "is_organic"), 2_000_000, 0), + ("user_x_artist_x_hour", ("uid", "artist_id", "hour_of_day"), 40_000_000, 0), +] + + +def get_hstu_configs(dataset: str = "debug") -> DlrmHSTUConfig: + """ + Create and return HSTU model configuration. + + Builds a complete DlrmHSTUConfig with default hyperparameters for the HSTU + architecture including attention settings, embedding dimensions, dropout rates, + and feature name mappings. + + Args: + dataset: Dataset identifier (currently unused, reserved for dataset-specific configs). + + Returns: + DlrmHSTUConfig: Complete configuration object for the HSTU model. + """ + hstu_config = DlrmHSTUConfig( + hstu_num_heads=4, + hstu_attn_linear_dim=128, + hstu_attn_qk_dim=128, + hstu_attn_num_layers=5, + hstu_embedding_table_dim=HSTU_EMBEDDING_DIM, + hstu_preprocessor_hidden_dim=256, + hstu_transducer_embedding_dim=512, + hstu_group_norm=False, + hstu_input_dropout_ratio=0.2, + hstu_linear_dropout_rate=0.1, + causal_multitask_weights=0.2, + ) + if "movielens" in dataset: + assert dataset in [ + "movielens-1m", + "movielens-20m", + "movielens-13b", + "movielens-18b", + ] + hstu_config.user_embedding_feature_names = ( + [ + "movie_id", + "user_id", + "sex", + "age_group", + "occupation", + "zip_code", + ] + if dataset == "movielens-1m" + else [ + "movie_id", + "user_id", + ] + ) + hstu_config.item_embedding_feature_names = [ + "item_movie_id", + ] + hstu_config.uih_post_id_feature_name = "movie_id" + hstu_config.uih_action_time_feature_name = "action_timestamp" + hstu_config.candidates_querytime_feature_name = "item_query_time" + hstu_config.candidates_weight_feature_name = "item_action_weights" + hstu_config.uih_weight_feature_name = "item_weights" + hstu_config.candidates_watchtime_feature_name = "item_movie_rating" + hstu_config.action_weights = [1, 2, 4, 8, 16] + hstu_config.contextual_feature_to_max_length = ( + { + "user_id": 1, + "sex": 1, + "age_group": 1, + "occupation": 1, + "zip_code": 1, + } + if dataset == "movielens-1m" + else { + "user_id": 1, + } + ) + hstu_config.contextual_feature_to_min_uih_length = ( + { + "user_id": 20, + "sex": 20, + "age_group": 20, + "occupation": 20, + "zip_code": 20, + } + if dataset == "movielens-1m" + else { + "user_id": 20, + } + ) + hstu_config.merge_uih_candidate_feature_mapping = [ + ("movie_id", "item_movie_id"), + ("movie_rating", "item_movie_rating"), + ("action_timestamp", "item_query_time"), + ("item_weights", "item_action_weights"), + ("dummy_watch_time", "item_dummy_watchtime"), + ] + hstu_config.hstu_uih_feature_names = ( + [ + "user_id", + "sex", + "age_group", + "occupation", + "zip_code", + "movie_id", + "movie_rating", + "action_timestamp", + "item_weights", + "dummy_watch_time", + ] + if dataset == "movielens-1m" + else [ + "user_id", + "movie_id", + "movie_rating", + "action_timestamp", + "item_weights", + "dummy_watch_time", + ] + ) + hstu_config.hstu_candidate_feature_names = [ + "item_movie_id", + "item_movie_rating", + "item_query_time", + "item_action_weights", + "item_dummy_watchtime", + ] + hstu_config.max_num_candidates = 10 + hstu_config.max_num_candidates_inference = ( + 5 if dataset not in ["movielens-13b", "movielens-18b"] else 2048 + ) + hstu_config.multitask_configs = [ + TaskConfig( + task_name="rating", + task_weight=1, + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ) + ] + elif "streaming" in dataset: + hstu_config.user_embedding_feature_names = [ + "item_id", + "user_id", + "item_category_id", + ] + hstu_config.item_embedding_feature_names = [ + "item_candidate_id", + "item_candidate_category_id", + ] + hstu_config.uih_post_id_feature_name = "item_id" + hstu_config.uih_action_time_feature_name = "action_timestamp" + hstu_config.candidates_querytime_feature_name = "item_query_time" + hstu_config.candidates_weight_feature_name = "item_action_weights" + hstu_config.uih_weight_feature_name = "item_weights" + hstu_config.candidates_watchtime_feature_name = "item_rating" + hstu_config.action_weights = [1, 2, 4, 8, 16] + hstu_config.action_embedding_init_std = 5.0 + hstu_config.contextual_feature_to_max_length = {"user_id": 1} + hstu_config.contextual_feature_to_min_uih_length = {"user_id": 20} + hstu_config.merge_uih_candidate_feature_mapping = [ + ("item_id", "item_candidate_id"), + ("item_rating", "item_candidate_rating"), + ("action_timestamp", "item_query_time"), + ("item_weights", "item_action_weights"), + ("dummy_watch_time", "item_dummy_watchtime"), + ("item_category_id", "item_candidate_category_id"), + ] + hstu_config.hstu_uih_feature_names = [ + "user_id", + "item_id", + "item_rating", + "action_timestamp", + "item_weights", + "dummy_watch_time", + "item_category_id", + ] + hstu_config.hstu_candidate_feature_names = [ + "item_candidate_id", + "item_candidate_rating", + "item_query_time", + "item_action_weights", + "item_dummy_watchtime", + "item_candidate_category_id", + ] + hstu_config.max_num_candidates = 32 + hstu_config.max_num_candidates_inference = 2048 + hstu_config.multitask_configs = [ + TaskConfig( + task_name="rating", + task_weight=1, + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ) + ] + elif "kuairand" in dataset: + hstu_config.user_embedding_feature_names = [ + "video_id", + "user_id", + "user_active_degree", + "follow_user_num_range", + "fans_user_num_range", + "friend_user_num_range", + "register_days_range", + ] + hstu_config.item_embedding_feature_names = [ + "item_video_id", + ] + hstu_config.uih_post_id_feature_name = "video_id" + hstu_config.uih_action_time_feature_name = "action_timestamp" + hstu_config.candidates_querytime_feature_name = "item_query_time" + hstu_config.uih_weight_feature_name = "action_weight" + hstu_config.candidates_weight_feature_name = "item_action_weight" + hstu_config.candidates_watchtime_feature_name = "item_target_watchtime" + # There are more contextual features in the dataset, see https://kuairand.com/ for details + hstu_config.contextual_feature_to_max_length = { + "user_id": 1, + "user_active_degree": 1, + "follow_user_num_range": 1, + "fans_user_num_range": 1, + "friend_user_num_range": 1, + "register_days_range": 1, + } + hstu_config.merge_uih_candidate_feature_mapping = [ + ("video_id", "item_video_id"), + ("action_timestamp", "item_query_time"), + ("action_weight", "item_action_weight"), + ("watch_time", "item_target_watchtime"), + ] + hstu_config.hstu_uih_feature_names = [ + "user_id", + "user_active_degree", + "follow_user_num_range", + "fans_user_num_range", + "friend_user_num_range", + "register_days_range", + "video_id", + "action_timestamp", + "action_weight", + "watch_time", + ] + hstu_config.hstu_candidate_feature_names = [ + "item_video_id", + "item_action_weight", + "item_target_watchtime", + "item_query_time", + ] + hstu_config.multitask_configs = [ + TaskConfig( + task_name="is_click", + task_weight=1, + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ), + TaskConfig( + task_name="is_like", + task_weight=2, + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ), + TaskConfig( + task_name="is_follow", + task_weight=4, + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ), + TaskConfig( + task_name="is_comment", + task_weight=8, + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ), + TaskConfig( + task_name="is_forward", + task_weight=16, + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ), + TaskConfig( + task_name="is_hate", + task_weight=32, + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ), + TaskConfig( + task_name="long_view", + task_weight=64, + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ), + TaskConfig( + task_name="is_profile_enter", + task_weight=128, + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ), + ] + hstu_config.action_weights = [1, 2, 4, 8, 16, 32, 64, 128] + elif "yambda" in dataset: + assert dataset in ["yambda-5b"] + cross_names = [name for (name, _k, _n, _s) in YAMBDA_5B_CROSS_SPECS] + # Smaller per-table dim for yambda (see YAMBDA_EMBEDDING_DIM); transducer + # still projects to 512. + hstu_config.hstu_embedding_table_dim = YAMBDA_EMBEDDING_DIM + hstu_config.hstu_transducer_embedding_dim = 512 + hstu_config.max_seq_len = 8192 + hstu_config.max_num_candidates = 1 + hstu_config.max_num_candidates_inference = 1 + # Per dlrm_hstu convention (see streaming-100b/movielens): + # - user_embedding_feature_names = UIH-side post-id features + contextual features. + # After main_forward merges UIH + candidate, only these entries hold the merged + # sequence (used by user-side transducer). + # - item_embedding_feature_names = candidate-side names only. _item_forward + # concats these along dim=-1 to feed the item MLP (per-candidate, not per-position). + hstu_config.user_embedding_feature_names = ( + ["uid"] + + cross_names + + ["item_id", "artist_id", "album_id"] + ) + hstu_config.item_embedding_feature_names = [ + "item_candidate_id", + "item_candidate_artist_id", + "item_candidate_album_id", + ] + hstu_config.uih_post_id_feature_name = "item_id" + hstu_config.uih_action_time_feature_name = "action_timestamp" + hstu_config.uih_weight_feature_name = "action_weight" + hstu_config.candidates_querytime_feature_name = "item_query_time" + hstu_config.candidates_weight_feature_name = "item_action_weight" + hstu_config.candidates_watchtime_feature_name = "item_dummy_watchtime" + hstu_config.action_weights = [1, 2, 4] # lp, like, skip bits + hstu_config.contextual_feature_to_max_length = { + "uid": 1, + **{name: 1 for name in cross_names}, + } + hstu_config.contextual_feature_to_min_uih_length = { + "uid": 0, + **{name: 0 for name in cross_names}, + } + # uih names map to candidate names (no name collisions allowed): + # item_id/artist_id/album_id appear with prefix "item_" on candidate side. + hstu_config.merge_uih_candidate_feature_mapping = [ + ("item_id", "item_candidate_id"), + ("artist_id", "item_candidate_artist_id"), + ("album_id", "item_candidate_album_id"), + ("action_weight", "item_action_weight"), + ("action_timestamp", "item_query_time"), + ("dummy_watch_time", "item_dummy_watchtime"), + ] + hstu_config.hstu_uih_feature_names = ( + ["uid"] + + cross_names + + [ + "item_id", + "artist_id", + "album_id", + "action_weight", + "action_timestamp", + "dummy_watch_time", + ] + ) + hstu_config.hstu_candidate_feature_names = [ + "item_candidate_id", + "item_candidate_artist_id", + "item_candidate_album_id", + "item_query_time", + "item_action_weight", + "item_dummy_watchtime", + ] + hstu_config.multitask_configs = [ + TaskConfig( + task_name="listen_plus", + task_weight=1, # matches action_weights[0] (lp bit) + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ) + ] + else: + hstu_config.user_embedding_feature_names = [ + "uih_post_id", + "uih_owner_id", + "viewer_id", + "dummy_contexual", + ] + hstu_config.item_embedding_feature_names = [ + "item_post_id", + "item_owner_id", + ] + hstu_config.uih_post_id_feature_name = "uih_post_id" + hstu_config.uih_action_time_feature_name = "uih_action_time" + hstu_config.candidates_querytime_feature_name = "item_query_time" + hstu_config.candidates_weight_feature_name = "item_action_weight" + hstu_config.candidates_watchtime_feature_name = "item_target_watchtime" + hstu_config.contextual_feature_to_max_length = { + "viewer_id": 1, + "dummy_contexual": 1, + } + hstu_config.contextual_feature_to_min_uih_length = { + "viewer_id": 128, + "dummy_contexual": 128, + } + hstu_config.merge_uih_candidate_feature_mapping = [ + ("uih_post_id", "item_post_id"), + ("uih_owner_id", "item_owner_id"), + ("uih_action_time", "item_query_time"), + ("uih_weight", "item_action_weight"), + ("uih_watchtime", "item_target_watchtime"), + ("uih_video_length", "item_video_length"), + ("uih_surface_type", "item_surface_type"), + ] + hstu_config.hstu_uih_feature_names = [ + "uih_post_id", + "uih_action_time", + "uih_weight", + "uih_owner_id", + "uih_watchtime", + "uih_surface_type", + "uih_video_length", + "viewer_id", + "dummy_contexual", + ] + hstu_config.hstu_candidate_feature_names = [ + "item_post_id", + "item_owner_id", + "item_surface_type", + "item_video_length", + "item_action_weight", + "item_target_watchtime", + "item_query_time", + ] + hstu_config.multitask_configs = [ + TaskConfig( + task_name="vvp100", + task_weight=1, + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ) + ] + return hstu_config + + +def get_embedding_table_config(dataset: str = "debug") -> Dict[str, EmbeddingConfig]: + """ + Create and return embedding table configurations. + + Defines the embedding table configurations for item IDs, category IDs, and user IDs + with their respective dimensions and data types. + + Args: + dataset: Dataset identifier (currently unused, reserved for dataset-specific configs). + + Returns: + Dict mapping table names to their EmbeddingConfig objects. + """ + if "movielens" in dataset: + assert dataset in [ + "movielens-1m", + "movielens-20m", + "movielens-13b", + "movielens-18b", + ] + return ( + { + "movie_id": EmbeddingConfig( + num_embeddings=HASH_SIZE, + embedding_dim=HSTU_EMBEDDING_DIM, + name="movie_id", + data_type=DataType.FP16, + feature_names=["movie_id", "item_movie_id"], + ), + "user_id": EmbeddingConfig( + num_embeddings=HASH_SIZE, + embedding_dim=HSTU_EMBEDDING_DIM, + name="user_id", + data_type=DataType.FP16, + feature_names=["user_id"], + ), + "sex": EmbeddingConfig( + num_embeddings=HASH_SIZE, + embedding_dim=HSTU_EMBEDDING_DIM, + name="sex", + data_type=DataType.FP16, + feature_names=["sex"], + ), + "age_group": EmbeddingConfig( + num_embeddings=HASH_SIZE, + embedding_dim=HSTU_EMBEDDING_DIM, + name="age_group", + data_type=DataType.FP16, + feature_names=["age_group"], + ), + "occupation": EmbeddingConfig( + num_embeddings=HASH_SIZE, + embedding_dim=HSTU_EMBEDDING_DIM, + name="occupation", + data_type=DataType.FP16, + feature_names=["occupation"], + ), + "zip_code": EmbeddingConfig( + num_embeddings=HASH_SIZE, + embedding_dim=HSTU_EMBEDDING_DIM, + name="zip_code", + data_type=DataType.FP16, + feature_names=["zip_code"], + ), + } + if dataset == "movielens-1m" + else { + "movie_id": EmbeddingConfig( + num_embeddings=HASH_SIZE_1B, + embedding_dim=HSTU_EMBEDDING_DIM, + name="movie_id", + data_type=DataType.FP16, + feature_names=["movie_id", "item_movie_id"], + ), + "user_id": EmbeddingConfig( + num_embeddings=3_000_000, + embedding_dim=HSTU_EMBEDDING_DIM, + name="user_id", + data_type=DataType.FP16, + feature_names=["user_id"], + ), + } + ) + elif "streaming" in dataset: + return { + "item_id": EmbeddingConfig( + num_embeddings=HASH_SIZE_1B, + embedding_dim=HSTU_EMBEDDING_DIM, + name="item_id", + data_type=DataType.FP16, + feature_names=["item_id", "item_candidate_id"], + ), + "item_category_id": EmbeddingConfig( + num_embeddings=128, + embedding_dim=HSTU_EMBEDDING_DIM, + name="item_category_id", + data_type=DataType.FP16, + weight_init_max=1.0, + weight_init_min=-1.0, + feature_names=["item_category_id", "item_candidate_category_id"], + ), + "user_id": EmbeddingConfig( + num_embeddings=10_000_000, + embedding_dim=HSTU_EMBEDDING_DIM, + name="user_id", + data_type=DataType.FP16, + feature_names=["user_id"], + ), + } + elif "kuairand" in dataset: + return { + "video_id": EmbeddingConfig( + num_embeddings=HASH_SIZE, + embedding_dim=HSTU_EMBEDDING_DIM, + name="video_id", + data_type=DataType.FP16, + feature_names=["video_id", "item_video_id"], + ), + "user_id": EmbeddingConfig( + num_embeddings=HASH_SIZE, + embedding_dim=HSTU_EMBEDDING_DIM, + name="user_id", + data_type=DataType.FP16, + feature_names=["user_id"], + ), + "user_active_degree": EmbeddingConfig( + num_embeddings=8, + embedding_dim=HSTU_EMBEDDING_DIM, + name="user_active_degree", + data_type=DataType.FP16, + feature_names=["user_active_degree"], + ), + "follow_user_num_range": EmbeddingConfig( + num_embeddings=9, + embedding_dim=HSTU_EMBEDDING_DIM, + name="follow_user_num_range", + data_type=DataType.FP16, + feature_names=["follow_user_num_range"], + ), + "fans_user_num_range": EmbeddingConfig( + num_embeddings=9, + embedding_dim=HSTU_EMBEDDING_DIM, + name="fans_user_num_range", + data_type=DataType.FP16, + feature_names=["fans_user_num_range"], + ), + "friend_user_num_range": EmbeddingConfig( + num_embeddings=8, + embedding_dim=HSTU_EMBEDDING_DIM, + name="friend_user_num_range", + data_type=DataType.FP16, + feature_names=["friend_user_num_range"], + ), + "register_days_range": EmbeddingConfig( + num_embeddings=8, + embedding_dim=HSTU_EMBEDDING_DIM, + name="register_days_range", + data_type=DataType.FP16, + feature_names=["register_days_range"], + ), + } + elif "yambda" in dataset: + assert dataset in ["yambda-5b"] + tables: Dict[str, EmbeddingConfig] = { + "item_id": EmbeddingConfig( + num_embeddings=9_390_000, + embedding_dim=YAMBDA_EMBEDDING_DIM, + name="item_id", + data_type=DataType.FP32, + feature_names=["item_id", "item_candidate_id"], + ), + "artist_id": EmbeddingConfig( + num_embeddings=1_290_000, + embedding_dim=YAMBDA_EMBEDDING_DIM, + name="artist_id", + data_type=DataType.FP32, + feature_names=["artist_id", "item_candidate_artist_id"], + ), + "album_id": EmbeddingConfig( + num_embeddings=3_370_000, + embedding_dim=YAMBDA_EMBEDDING_DIM, + name="album_id", + data_type=DataType.FP32, + feature_names=["album_id", "item_candidate_album_id"], + ), + "uid": EmbeddingConfig( + num_embeddings=1_000_000, + embedding_dim=YAMBDA_EMBEDDING_DIM, + name="uid", + data_type=DataType.FP32, + feature_names=["uid"], + ), + } + for name, _keys, num_embeddings, _salt in YAMBDA_5B_CROSS_SPECS: + tables[name] = EmbeddingConfig( + num_embeddings=num_embeddings, + embedding_dim=YAMBDA_EMBEDDING_DIM, + name=name, + data_type=DataType.FP32, + feature_names=[name], + ) + return tables + else: + return { + "post_id": EmbeddingConfig( + num_embeddings=HASH_SIZE, + embedding_dim=HSTU_EMBEDDING_DIM, + name="post_id", + data_type=DataType.FP16, + feature_names=[ + "uih_post_id", + "item_post_id", + "uih_owner_id", + "item_owner_id", + ], + ), + "viewer_id": EmbeddingConfig( + num_embeddings=HASH_SIZE, + embedding_dim=HSTU_EMBEDDING_DIM, + name="viewer_id", + data_type=DataType.FP16, + feature_names=["viewer_id"], + ), + "dummy_contexual": EmbeddingConfig( + num_embeddings=HASH_SIZE, + embedding_dim=HSTU_EMBEDDING_DIM, + name="dummy_contexual", + data_type=DataType.FP16, + feature_names=["dummy_contexual"], + ), + } diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/datasets/dataset.py b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/dataset.py new file mode 100644 index 000000000..a1cbb33fa --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/dataset.py @@ -0,0 +1,398 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe +""" +Dataset implementations for DLRMv3. + +This module provides dataset classes for loading and processing recommendation +data, including sample containers, collation functions, and random data generation. +""" + +import logging +import time +from dataclasses import dataclass +from typing import Dict, List, Tuple + +import torch +from generative_recommenders.modules.dlrm_hstu import DlrmHSTUConfig +from torchrec.sparse.jagged_tensor import KeyedJaggedTensor + + +logging.basicConfig(level=logging.INFO) +logger: logging.Logger = logging.getLogger("dlrmv3_dataset") + + +@dataclass +class Samples: + """ + Container for batched samples with user interaction history and candidate features. + + Attributes: + uih_features_kjt: User interaction history features as KeyedJaggedTensor. + candidates_features_kjt: Candidate item features as KeyedJaggedTensor. + """ + + uih_features_kjt: KeyedJaggedTensor + candidates_features_kjt: KeyedJaggedTensor + + def to(self, device: torch.device) -> None: + """ + Move all tensors to the specified device. + + Args: + device: Target device to move tensors to. + """ + for attr in vars(self): + setattr(self, attr, getattr(self, attr).to(device=device)) + + def batch_size(self) -> int: + """ + Get the batch size of the samples. + + Returns: + Number of samples in the batch. + """ + return self.uih_features_kjt.stride() + + +def collate_fn( + samples: List[Tuple[KeyedJaggedTensor, KeyedJaggedTensor]], +) -> Samples: + """ + Collate multiple samples into a batched Samples object. + + Args: + samples: List of (uih_features, candidates_features) tuples. + + Returns: + Batched Samples object with concatenated features. + """ + ( + uih_features_kjt_list, + candidates_features_kjt_list, + ) = list(zip(*samples)) + + return Samples( + uih_features_kjt=kjt_batch_func(uih_features_kjt_list), + candidates_features_kjt=kjt_batch_func(candidates_features_kjt_list), + ) + + +class Dataset: + """ + Base dataset class for DLRMv3. + + Provides the interface for loading, accessing, and managing samples + for recommendation model training and inference. + + Args: + hstu_config: HSTU model configuration. + **args: Additional arguments (unused in base class). + """ + + def __init__(self, hstu_config: DlrmHSTUConfig, **args): + self.arrival = None + self.image_list = [] + self.label_list = [] + self.image_list_inmemory = {} + self.last_loaded = -1.0 + + def preprocess(self, use_cache=True): + """ + Preprocess the dataset. + + Args: + use_cache: Whether to use cached preprocessed data. + + Raises: + NotImplementedError: Subclasses must implement this method. + """ + raise NotImplementedError("Dataset:preprocess") + + def get_item_count(self): + """ + Get the total number of items in the dataset. + + Returns: + Number of items. + """ + return len(self.image_list) + + def load_query_samples(self, sample_list): + """ + Load specified samples into memory. + + Args: + sample_list: List of sample indices to load. + + Raises: + NotImplementedError: Subclasses must implement this method. + """ + raise NotImplementedError("Dataset:load_query_samples") + + def unload_query_samples(self, sample_list): + """ + Unload specified samples from memory. + + Args: + sample_list: List of sample indices to unload. + + Raises: + NotImplementedError: Subclasses must implement this method. + """ + raise NotImplementedError("Dataset:unload_query_samples") + + def get_sample(self, id: int): + """ + Get a single sample by ID. + + Args: + id: Sample identifier. + + Raises: + NotImplementedError: Subclasses must implement this method. + """ + raise NotImplementedError("Dataset:get_sample") + + def get_samples(self, id_list: List[int]) -> Samples: + """ + Get multiple samples and collate them into a batch. + + Args: + id_list: List of sample identifiers. + + Returns: + Collated Samples object containing the batch. + """ + list_samples = [self.get_sample(ix) for ix in id_list] + return collate_fn(list_samples) + + +@torch.jit.script +def kjt_batch_func( + kjt_list: List[KeyedJaggedTensor], +) -> KeyedJaggedTensor: + """ + Batch multiple KeyedJaggedTensors into a single tensor. + + Uses FBGEMM operations for efficient batching and reordering of + jagged tensor data. + + Args: + kjt_list: List of KeyedJaggedTensors to batch. + + Returns: + Batched KeyedJaggedTensor with reordered indices and lengths. + """ + bs_list = [kjt.stride() for kjt in kjt_list] + bs = sum(bs_list) + batched_length = torch.cat([kjt.lengths() for kjt in kjt_list], dim=0) + batched_indices = torch.cat([kjt.values() for kjt in kjt_list], dim=0) + bs_offset = torch.ops.fbgemm.asynchronous_complete_cumsum( + torch.tensor(bs_list) + ).int() + batched_offset = torch.ops.fbgemm.asynchronous_complete_cumsum(batched_length) + reorder_length = torch.ops.fbgemm.reorder_batched_ad_lengths( + batched_length, bs_offset, bs + ) + reorder_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum(reorder_length) + reorder_indices = torch.ops.fbgemm.reorder_batched_ad_indices( + batched_offset, batched_indices, reorder_offsets, bs_offset, bs + ) + out = KeyedJaggedTensor( + keys=kjt_list[0].keys(), + lengths=reorder_length.long(), + values=reorder_indices.long(), + ) + return out + + +def get_random_data( + contexual_features: List[str], + hstu_uih_keys: List[str], + hstu_candidates_keys: List[str], + uih_max_seq_len: int, + max_num_candidates: int, + value_bound: int = 1000, +): + """ + Generate random sample data for testing and debugging. + + Creates synthetic user interaction history and candidate features + with random values. + + Args: + contexual_features: List of contextual feature names. + hstu_uih_keys: List of UIH feature keys. + hstu_candidates_keys: List of candidate feature keys. + uih_max_seq_len: Maximum sequence length for UIH. + max_num_candidates: Maximum number of candidates. + value_bound: Upper bound for random values. + + Returns: + Tuple of (uih_features_kjt, candidates_features_kjt). + """ + uih_non_seq_feature_keys = contexual_features + uih_seq_feature_keys = [ + k for k in hstu_uih_keys if k not in uih_non_seq_feature_keys + ] + uih_seq_len = torch.randint( + int(uih_max_seq_len * 0.8), + uih_max_seq_len + 1, + (1,), + ).item() + uih_lengths = torch.tensor( + [1 for _ in uih_non_seq_feature_keys] + + [uih_seq_len for _ in uih_seq_feature_keys] + ) + # logging.info(f"uih_lengths: {uih_lengths}") + uih_values = torch.randint( + 1, + value_bound, + # pyre-ignore[6] + (uih_seq_len * len(uih_seq_feature_keys) + len(uih_non_seq_feature_keys),), + ) + uih_features_kjt = KeyedJaggedTensor( + keys=uih_non_seq_feature_keys + uih_seq_feature_keys, + lengths=uih_lengths.long(), + values=uih_values.long(), + ) + num_candidates = torch.randint( + 1, + max_num_candidates + 1, + (1,), + ).item() + candidates_lengths = num_candidates * torch.ones(len(hstu_candidates_keys)) + candidates_values = torch.randint( + 1, + value_bound, + (num_candidates * len(hstu_candidates_keys),), # pyre-ignore[6] + ) + candidates_features_kjt = KeyedJaggedTensor( + keys=hstu_candidates_keys, + lengths=candidates_lengths.long(), + values=candidates_values.long(), + ) + return uih_features_kjt, candidates_features_kjt + + +class DLRMv3RandomDataset(Dataset): + """ + Dataset that generates random synthetic data for DLRMv3. + + Useful for testing and benchmarking without real data dependencies. + + Args: + hstu_config: HSTU model configuration. + num_aggregated_samples: Total number of samples to generate. + is_inference: Whether the dataset is used for inference mode. + *args: Additional positional arguments. + **kwargs: Additional keyword arguments. + """ + + def __init__( + self, + hstu_config: DlrmHSTUConfig, + num_aggregated_samples: int = 10000, + is_inference: bool = False, + *args, + **kwargs, + ): + super().__init__( + hstu_config=hstu_config, + ) + self.hstu_config: DlrmHSTUConfig = hstu_config + self._max_num_candidates: int = hstu_config.max_num_candidates + self._max_num_candidates_inference: int = ( + hstu_config.max_num_candidates_inference + ) + self._max_seq_len: int = hstu_config.max_seq_len + self._uih_keys: List[str] = hstu_config.hstu_uih_feature_names + self._candidates_keys: List[str] = hstu_config.hstu_candidate_feature_names + self._contextual_feature_to_max_length: Dict[str, int] = ( + hstu_config.contextual_feature_to_max_length + ) + self._max_uih_len: int = ( + self._max_seq_len + - self._max_num_candidates + - ( + len(self._contextual_feature_to_max_length) + if self._contextual_feature_to_max_length + else 0 + ) + ) + self._is_inference = is_inference + + self.contexual_features = [] + if hstu_config.contextual_feature_to_max_length is not None: + self.contexual_features = [ + p[0] for p in hstu_config.contextual_feature_to_max_length + ] + + self.num_aggregated_samples = num_aggregated_samples + self.items_in_memory = {} + + def get_sample(self, id: int) -> Tuple[KeyedJaggedTensor, KeyedJaggedTensor]: + """ + Get a sample by ID from in-memory storage. + + Args: + id: Sample identifier. + + Returns: + Tuple of (uih_features_kjt, candidates_features_kjt). + """ + return self.items_in_memory[id] + + def get_item_count(self): + """ + Get the total number of samples in the dataset. + + Returns: + Number of aggregated samples. + """ + return self.num_aggregated_samples + + def unload_query_samples(self, sample_list): + """ + Clear all samples from memory. + + Args: + sample_list: Ignored; clears all samples. + """ + self.items_in_memory = {} + + def load_query_samples(self, sample_list): + """ + Generate and load random samples into memory. + + Args: + sample_list: List of sample IDs to generate. + """ + max_num_candidates = ( + self._max_num_candidates_inference + if self._is_inference + else self._max_num_candidates + ) + self.items_in_memory = {} + for sample in sample_list: + self.items_in_memory[sample] = get_random_data( + contexual_features=self.contexual_features, + hstu_uih_keys=self.hstu_config.hstu_uih_feature_names, + hstu_candidates_keys=self.hstu_config.hstu_candidate_feature_names, + uih_max_seq_len=self._max_uih_len, + max_num_candidates=max_num_candidates, + ) + self.last_loaded = time.time() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/datasets/kuairand.py b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/kuairand.py new file mode 100644 index 000000000..f6cd9e672 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/kuairand.py @@ -0,0 +1,163 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe +import json +import time +from functools import partial +from typing import Any, Dict, List + +import pandas as pd +import torch +from generative_recommenders.dlrm_v3.datasets.dataset import DLRMv3RandomDataset +from generative_recommenders.dlrm_v3.datasets.utils import ( + maybe_truncate_seq, + separate_uih_candidates, +) +from generative_recommenders.modules.dlrm_hstu import DlrmHSTUConfig +from torchrec.sparse.jagged_tensor import KeyedJaggedTensor + + +def process_and_hash_x(x: Any, hash_size: int) -> Any: + if isinstance(x, str): + x = json.loads(x) + if isinstance(x, list): + return [x_i % hash_size for x_i in x] + else: + return x % hash_size + + +class DLRMv3KuaiRandDataset(DLRMv3RandomDataset): + def __init__( + self, + hstu_config: DlrmHSTUConfig, + embedding_config: Dict[str, Any], + seq_logs_file: str, + is_inference: bool, + **kwargs, + ) -> None: + super().__init__(hstu_config=hstu_config, is_inference=is_inference) + self.seq_logs_frame: pd.DataFrame = pd.read_csv(seq_logs_file, delimiter=",") + # apply hashing from embedding table config + for key, table in embedding_config.items(): + assert key in self.seq_logs_frame.columns, ( + "Rename key in embedding table configs!" + ) + hash_size = table.num_embeddings + self.seq_logs_frame[key] = self.seq_logs_frame[key].apply( + partial(process_and_hash_x, hash_size=hash_size) + ) + + def get_item_count(self): + return len(self.seq_logs_frame) + + def unload_query_samples(self, sample_list): + self.items_in_memory = {} + + def load_query_samples(self, sample_list): + max_num_candidates = ( + self._max_num_candidates_inference + if self._is_inference + else self._max_num_candidates + ) + self.items_in_memory = {} + for idx in sample_list: + data = self.seq_logs_frame.iloc[idx] + if len(data.video_id) <= max_num_candidates: + continue + sample = self.load_item(data, max_num_candidates) + self.items_in_memory[idx] = sample + + self.last_loaded = time.time() + + def load_item(self, data, max_num_candidates): + with torch.profiler.record_function("load_item"): + video_history_uih, video_history_candidates = separate_uih_candidates( + data.video_id, + candidates_max_seq_len=max_num_candidates, + ) + action_weights_uih, action_weights_candidates = separate_uih_candidates( + data.action_weights, + candidates_max_seq_len=max_num_candidates, + ) + timestamps_uih, _ = separate_uih_candidates( + data.time_ms, + candidates_max_seq_len=max_num_candidates, + ) + watch_time_uih, watch_time_candidates = separate_uih_candidates( + data.play_time_ms, + candidates_max_seq_len=max_num_candidates, + ) + + video_history_uih = maybe_truncate_seq(video_history_uih, self._max_uih_len) + action_weights_uih = maybe_truncate_seq( + action_weights_uih, self._max_uih_len + ) + timestamps_uih = maybe_truncate_seq(timestamps_uih, self._max_uih_len) + watch_time_uih = maybe_truncate_seq(watch_time_uih, self._max_uih_len) + + uih_seq_len = len(video_history_uih) + assert uih_seq_len == len(timestamps_uih), ( + "history len differs from timestamp len." + ) + assert uih_seq_len == len(action_weights_uih), ( + "history len differs from weights len." + ) + assert uih_seq_len == len(watch_time_uih), ( + "history len differs from watch time len." + ) + + uih_kjt_values: List[torch.Tensor] = [] + uih_kjt_lengths: List[torch.Tensor] = [] + for name, length in self._contextual_feature_to_max_length.items(): + uih_kjt_values.append(data[name]) + uih_kjt_lengths.append(length) + + uih_kjt_values.extend( + video_history_uih + timestamps_uih + action_weights_uih + watch_time_uih + ) + + uih_kjt_lengths.extend( + [ + uih_seq_len + for _ in range( + len(self._uih_keys) + - len(self._contextual_feature_to_max_length) + ) + ] + ) + + dummy_query_time = max(timestamps_uih) + uih_features_kjt = KeyedJaggedTensor( + keys=self._uih_keys, + lengths=torch.tensor(uih_kjt_lengths).long(), + values=torch.tensor(uih_kjt_values).long(), + ) + + candidates_kjt_lengths = max_num_candidates * torch.ones( + len(self._candidates_keys) + ) + candidates_kjt_values = ( + video_history_candidates + + action_weights_candidates + + watch_time_candidates + + [dummy_query_time] * max_num_candidates + ) + candidates_features_kjt = KeyedJaggedTensor( + keys=self._candidates_keys, + lengths=torch.tensor(candidates_kjt_lengths).long(), + values=torch.tensor(candidates_kjt_values).long(), + ) + + return uih_features_kjt, candidates_features_kjt diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/datasets/movie_lens.py b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/movie_lens.py new file mode 100644 index 000000000..d74fb575b --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/movie_lens.py @@ -0,0 +1,177 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe +import logging +import time +from typing import List, Optional + +import pandas as pd +import torch +from generative_recommenders.dlrm_v3.datasets.dataset import DLRMv3RandomDataset +from generative_recommenders.dlrm_v3.datasets.utils import ( + maybe_truncate_seq, + separate_uih_candidates, +) +from generative_recommenders.modules.dlrm_hstu import DlrmHSTUConfig +from torchrec.sparse.jagged_tensor import KeyedJaggedTensor + +logger = logging.getLogger(__name__) + + +class DLRMv3MovieLensDataset(DLRMv3RandomDataset): + def __init__( + self, + hstu_config: DlrmHSTUConfig, + ratings_file: str, + is_inference: bool, + *args, + **kwargs, + ): + super().__init__(hstu_config=hstu_config, is_inference=is_inference) + self.ratings_frame: Optional[pd.DataFrame] = None + if ratings_file != "": + self.ratings_frame = pd.read_csv( + ratings_file, + delimiter=",", + ) + assert hstu_config.action_weights is not None + self.action_weights: List[int] = hstu_config.action_weights + + def get_item_count(self): + assert self.ratings_frame is not None + return len(self.ratings_frame) + + def unload_query_samples(self, sample_list): + self.items_in_memory = {} + + def iloc(self, idx): + assert self.ratings_frame is not None + return self.ratings_frame.iloc[idx] + + def load_query_samples(self, sample_list): + max_num_candidates = ( + self._max_num_candidates_inference + if self._is_inference + else self._max_num_candidates + ) + self.items_in_memory = {} + for idx in sample_list: + data = self.iloc(idx) + if len(data.sequence_item_ids) <= max_num_candidates: + continue + sample = self.load_item(data, max_num_candidates) + self.items_in_memory[idx] = sample + + self.last_loaded = time.time() + + def get_timestamp_uih(self, data, max_num_candidates, size): + movie_timestamps_uih, _ = separate_uih_candidates( + data.sequence_timestamps, + candidates_max_seq_len=max_num_candidates, + ) + return movie_timestamps_uih + + def load_item(self, data, max_num_candidates): + movie_history_uih, movie_history_candidates = separate_uih_candidates( + data.sequence_item_ids, + candidates_max_seq_len=max_num_candidates, + ) + movie_history_ratings_uih, movie_history_ratings_candidates = ( + separate_uih_candidates( + data.sequence_ratings, + candidates_max_seq_len=max_num_candidates, + ) + ) + movie_timestamps_uih = self.get_timestamp_uih( + data=data, + max_num_candidates=max_num_candidates, + size=len(movie_history_uih), + ) + + assert len(movie_history_uih) == len(movie_timestamps_uih), ( + "history len differs from timestamp len." + ) + assert len(movie_history_uih) == len(movie_history_ratings_uih), ( + "history len differs from ratings len." + ) + + movie_history_uih = maybe_truncate_seq(movie_history_uih, self._max_uih_len) + movie_history_ratings_uih = maybe_truncate_seq( + movie_history_ratings_uih, self._max_uih_len + ) + movie_timestamps_uih = maybe_truncate_seq( + movie_timestamps_uih, self._max_uih_len + ) + + uih_kjt_values: List[torch.Tensor] = [] + uih_kjt_lengths: List[torch.Tensor] = [] + for name, length in self._contextual_feature_to_max_length.items(): + uih_kjt_values.append(data[name]) + uih_kjt_lengths.append(length) + + uih_seq_len = len(movie_history_uih) + movie_dummy_watch_times_uih = [0 for _ in range(uih_seq_len)] + action_weights_uih = [ + self.action_weights[int(rating) - 1] for rating in movie_history_ratings_uih + ] + uih_kjt_values.extend( + movie_history_uih + + movie_history_ratings_uih + + movie_timestamps_uih + + action_weights_uih + + movie_dummy_watch_times_uih + ) + uih_kjt_lengths.extend( + [ + uih_seq_len + for _ in range( + len(self._uih_keys) - len(self._contextual_feature_to_max_length) + ) + ] + ) + + dummy_query_time = ( + 0 if movie_timestamps_uih == [] else max(movie_timestamps_uih) + ) + uih_kjt_values.append(dummy_query_time) + uih_kjt_lengths.append(1) + uih_features_kjt = KeyedJaggedTensor( + keys=self._uih_keys + ["dummy_query_time"], + lengths=torch.tensor(uih_kjt_lengths).long(), + values=torch.tensor(uih_kjt_values).long(), + ) + + candidates_kjt_lengths = max_num_candidates * torch.ones( + len(self._candidates_keys) + ) + action_weights_candidates = [ + int(rating >= 3.5) for rating in movie_history_ratings_candidates + ] + candidates_kjt_values = ( + movie_history_candidates + + movie_history_ratings_candidates + + [dummy_query_time] * max_num_candidates # item_query_time + + action_weights_candidates + + [1] * max_num_candidates # item_dummy_watchtime + ) + candidates_features_kjt = KeyedJaggedTensor( + keys=self._candidates_keys, + lengths=candidates_kjt_lengths.detach().clone().long(), + values=torch.tensor(candidates_kjt_values).long(), + ) + return ( + uih_features_kjt, + candidates_features_kjt, + ) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/datasets/synthetic_movie_lens.py b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/synthetic_movie_lens.py new file mode 100644 index 000000000..6cf8a5f56 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/synthetic_movie_lens.py @@ -0,0 +1,83 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe +import csv +import linecache +import logging +import sys +from typing import List + +import numpy as np +import pandas as pd +from generative_recommenders.dlrm_v3.datasets.movie_lens import DLRMv3MovieLensDataset +from generative_recommenders.modules.dlrm_hstu import DlrmHSTUConfig + +csv.field_size_limit(sys.maxsize) +logger = logging.getLogger(__name__) + + +class DLRMv3SyntheticMovieLensDataset(DLRMv3MovieLensDataset): + def __init__( + self, + hstu_config: DlrmHSTUConfig, + ratings_file_prefix: str, + is_inference: bool, + *args, + **kwargs, + ): + super().__init__( + hstu_config=hstu_config, is_inference=is_inference, ratings_file="" + ) + self.ratings_file_prefix = ratings_file_prefix + with open(f"{self.ratings_file_prefix}_users.csv", "r") as file: + reader = csv.reader(file) + self.users_cumsum: List[int] = np.cumsum( + [int(row[1]) for row in reader] + ).tolist() + + def get_item_count(self): + return self.users_cumsum[-1] + + def _process_line(self, line: str) -> pd.Series: + reader = csv.reader([line]) + parsed_line = next(reader) + user_id = int(parsed_line[0]) + sequence_item_ids = parsed_line[1] + sequence_ratings = parsed_line[2] + return pd.Series( + data={ + "user_id": user_id, + "sequence_item_ids": sequence_item_ids, + "sequence_ratings": sequence_ratings, + } + ) + + def iloc(self, idx) -> pd.Series: + assert idx < self.users_cumsum[-1] + file_idx: int = 0 + while self.users_cumsum[file_idx] <= idx: + file_idx += 1 + if file_idx == 0: + local_idx = idx + else: + local_idx = idx - self.users_cumsum[file_idx - 1] + line = linecache.getline( + f"{self.ratings_file_prefix}_{file_idx}.csv", local_idx + 1 + ) + data = self._process_line(line) + return data + + def get_timestamp_uih(self, data, max_num_candidates, size): + return [1] * size diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/datasets/synthetic_streaming.py b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/synthetic_streaming.py new file mode 100644 index 000000000..437e5ae8e --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/synthetic_streaming.py @@ -0,0 +1,400 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-strict +""" +Synthetic streaming dataset for DLRMv3 inference benchmarking. + +This module provides a streaming dataset implementation that loads user interaction +data from pre-generated CSV files with temporal (timestamp) organization, suitable +for simulating real-time recommendation scenarios. +""" + +import csv +import logging +import sys +import time +from typing import Any, Dict, List, Set, Tuple + +import pandas as pd +import torch +from generative_recommenders.dlrm_v3.datasets.dataset import ( + collate_fn, + DLRMv3RandomDataset, + Samples, +) +from generative_recommenders.dlrm_v3.datasets.utils import ( + json_loads, + maybe_truncate_seq, +) +from generative_recommenders.modules.dlrm_hstu import DlrmHSTUConfig +from torchrec.sparse.jagged_tensor import KeyedJaggedTensor + +csv.field_size_limit(sys.maxsize) +logger: logging.Logger = logging.getLogger(__name__) + + +class DLRMv3SyntheticStreamingDataset(DLRMv3RandomDataset): + """ + Streaming dataset that loads pre-generated synthetic recommendation data. + + Supports timestamp-based data organization for simulating streaming scenarios + where user interaction histories evolve over time. + + Args: + hstu_config: HSTU model configuration. + ratings_file_prefix: Path prefix for rating data files. + is_inference: Whether dataset is used for inference. + train_ts: Number of timestamps used for training. + total_ts: Total number of timestamps in the data. + num_files: Number of data files (for parallelization). + num_users: Total number of users in the dataset. + num_items: Total number of items in the catalog. + num_categories: Number of item categories. + *args: Additional positional arguments. + **kwargs: Additional keyword arguments. + """ + + def __init__( + self, + hstu_config: DlrmHSTUConfig, + ratings_file_prefix: str, + is_inference: bool, + train_ts: int, + total_ts: int, + num_files: int, + num_users: int, + num_items: int, + num_categories: int, + *args: Any, + **kwargs: Any, + ) -> None: + super().__init__(hstu_config=hstu_config, is_inference=is_inference) + self.ratings_file_prefix = ratings_file_prefix + self.file_to_offsets: Dict[int, List[int]] = {} + with open(f"{self.ratings_file_prefix}offset.csv", "r") as file: + reader = csv.reader(file) + for size in range(num_files): + row = next(reader) + assert len(row) == 1 + offset = json_loads(row[0]) + assert len(offset) == num_users // num_files + self.file_to_offsets[size] = offset + self.ts_requests_offsets: List[int] = [] + with open(f"{self.ratings_file_prefix}requests_per_ts_offset.csv", "r") as file: + reader = csv.reader(file) + row = next(reader) + assert len(row) == 1 + self.ts_requests_offsets = json_loads(row[0]) + assert len(self.ts_requests_offsets) == total_ts + self.requests: List[int] = [] + self.ts_to_users_cumsum: Dict[int, List[int]] = {} + with open( + f"{self.ratings_file_prefix}users_cumsum_per_ts.csv", "r" + ) as cumsum_file: + reader = csv.reader(cumsum_file) + ts = 0 + for row in reader: + assert len(row) == 1 + cumsum = json_loads(row[0]) + self.ts_to_users_cumsum[ts] = cumsum + ts += 1 + self.train_ts = train_ts + self.total_ts = total_ts + self.num_files = num_files + self.ts: int = -1 + self.is_inference: bool = False + self.is_eval: bool = False + self.users_per_file: int = num_users // num_files + self.cached_files: Set[str] = set() + self.items_per_category: int = num_items // num_categories + assert hstu_config.action_weights is not None + self.action_weights: List[int] = hstu_config.action_weights + self.items_in_memory: Dict[ + int, Dict[int, Tuple[KeyedJaggedTensor, KeyedJaggedTensor]] + ] = {} + + def get_item_count(self) -> int: + return len(self.requests) + + def load_query_samples(self, sample_list: List[int]) -> None: + max_num_candidates = ( + self._max_num_candidates_inference + if self._is_inference + else self._max_num_candidates + ) + for idx in sample_list: + data = self.iloc(idx) + sample = self.load_item(data, max_num_candidates) + if self.ts not in self.items_in_memory: + self.items_in_memory[self.ts] = {} + self.items_in_memory[self.ts][idx] = sample + + self.last_loaded = time.time() + + def unload_query_samples(self, sample_list: List[int]) -> None: + self.items_in_memory = {} + + def get_sample(self, id: int) -> Tuple[KeyedJaggedTensor, KeyedJaggedTensor]: + return self.items_in_memory[self.ts][id] + + def get_sample_with_ts( + self, id: int, ts: int + ) -> Tuple[KeyedJaggedTensor, KeyedJaggedTensor]: + """ + Get a sample for a specific timestamp. + + Args: + id: Sample identifier. + ts: Timestamp index. + + Returns: + Tuple of (uih_features_kjt, candidates_features_kjt). + """ + return self.items_in_memory[ts][id] + + def get_samples_with_ts(self, id_list: List[int], ts: int) -> Samples: + """ + Get and collate multiple samples for a specific timestamp. + + Args: + id_list: List of sample identifiers. + ts: Timestamp index. + + Returns: + Collated Samples object. + """ + list_samples = [self.get_sample_with_ts(ix, ts) for ix in id_list] + return collate_fn(list_samples) + + def _process_line(self, line: str, user_id: int) -> pd.Series: + """ + Parse a CSV line into a pandas Series with user interaction data. + + Args: + line: CSV line containing user data. + user_id: User identifier. + + Returns: + pd.Series with parsed user interaction history and candidates. + """ + reader = csv.reader([line]) + parsed_line = next(reader) + # total ts + one more eval ts + one base ts so that uih won't be zero + # for each ts, ordered as candidate_ids, candidate_ratings, uih_ids, uih_ratings + assert len(parsed_line) == 4 * (self.total_ts + 2) + uih_item_ids_list = [] + uih_ratings_list = [] + candidate_item_ids = "" + candidate_ratings = "" + if (not self.is_eval) and (not self.is_inference): + assert self.ts < self.train_ts + for i in range(self.ts + 1): + if parsed_line[4 * i]: + uih_item_ids_list.append(parsed_line[2 + 4 * i]) + uih_ratings_list.append(parsed_line[3 + 4 * i]) + candidate_item_ids = parsed_line[4 * (self.ts + 1)] + candidate_ratings = parsed_line[1 + 4 * (self.ts + 1)] + elif self.is_eval: + for i in range(self.ts + 1): + if parsed_line[4 * i]: + uih_item_ids_list.append(parsed_line[2 + 4 * i]) + uih_ratings_list.append(parsed_line[3 + 4 * i]) + candidate_item_ids = parsed_line[4 * (self.ts + 1)] + candidate_ratings = parsed_line[1 + 4 * (self.ts + 1)] + else: + assert self.is_inference is True + assert self.ts >= self.train_ts + for i in range(self.train_ts + 1): + if parsed_line[4 * i]: + uih_item_ids_list.append(parsed_line[2 + 4 * i]) + uih_ratings_list.append(parsed_line[3 + 4 * i]) + for i in range(self.train_ts + 2, self.ts + 2): + if parsed_line[4 * i]: + uih_item_ids_list.append(parsed_line[2 + 4 * i]) + uih_ratings_list.append(parsed_line[3 + 4 * i]) + candidate_item_ids = parsed_line[4 * (self.ts + 2)] + candidate_ratings = parsed_line[1 + 4 * (self.ts + 2)] + uih_item_ids = ",".join(uih_item_ids_list) + uih_ratings = ",".join(uih_ratings_list) + assert candidate_item_ids != "" and candidate_ratings != "" + return pd.Series( + data={ + "user_id": user_id, + "uih_item_ids": uih_item_ids, + "uih_ratings": uih_ratings, + "candidate_item_ids": candidate_item_ids, + "candidate_ratings": candidate_ratings, + } + ) + + def iloc(self, idx: int) -> pd.Series: + """ + Get user data by request index using file offsets for efficient access. + + Args: + idx: Request index within the current timestamp. + + Returns: + pd.Series with parsed user interaction data. + """ + cumsum: List[int] = self.ts_to_users_cumsum[self.ts] + assert cumsum != [] + assert idx < cumsum[-1] + file_idx: int = 0 + while cumsum[file_idx] <= idx: + file_idx += 1 + user_idx = self.requests[idx] + filename = f"{self.ratings_file_prefix}{file_idx}.csv" + with open(filename, "r") as file: + idx = user_idx % self.users_per_file + file.seek(self.file_to_offsets[file_idx][idx]) + line = file.readline() + data = self._process_line(line=line, user_id=user_idx) + return data + + def get_timestamp_uih( + self, data: pd.Series, max_num_candidates: int, size: int + ) -> List[int]: + return [1] * size + + def set_ts(self, ts: int) -> None: + """ + Set the current timestamp and load associated request data. + + Args: + ts: Timestamp index to set. + """ + logger.warning(f"Streaming dataset ts set to {ts}") + if ts == self.ts: + return + self.ts = ts + with open( + f"{self.ratings_file_prefix}requests_per_ts.csv", "r" + ) as request_file: + request_file.seek(self.ts_requests_offsets[self.ts]) + line = request_file.readline() + reader = csv.reader([line]) + row = next(reader) + assert len(row) == 1 + requests = json_loads(row[0]) + self.requests = requests + logger.warning(f"DLRMv3SyntheticStreamingDataset: ts={ts} requests loaded") + assert self.ts_to_users_cumsum[self.ts][-1] == len(self.requests) + logger.warning( + f"DLRMv3SyntheticStreamingDataset: ts={ts} users_cumsum={self.ts_to_users_cumsum[self.ts]}" + ) + + def load_item( + self, data: pd.Series, max_num_candidates: int + ) -> Tuple[KeyedJaggedTensor, KeyedJaggedTensor]: + """ + Load and process a single user's data into KeyedJaggedTensors. + + Converts parsed user data into feature tensors suitable for model input, + including truncation to maximum sequence lengths. + + Args: + data: pd.Series with user interaction history and candidates. + max_num_candidates: Maximum number of candidates to include. + + Returns: + Tuple of (uih_features_kjt, candidates_features_kjt). + """ + ids_uih = json_loads(data.uih_item_ids) + ids_candidates = json_loads(data.candidate_item_ids) + ratings_uih = json_loads(data.uih_ratings) + ratings_candidates = json_loads(data.candidate_ratings) + timestamps_uih = self.get_timestamp_uih( + data=data, + max_num_candidates=max_num_candidates, + size=len(ids_uih), + ) + assert len(ids_uih) == len(timestamps_uih), ( + "history len differs from timestamp len." + ) + assert len(ids_uih) == len(ratings_uih), ( + f"history len {len(ids_uih)} differs from ratings len {len(ratings_uih)}." + ) + assert len(ids_candidates) == len(ratings_candidates), ( + f"candidates len {len(ids_candidates)} differs from ratings len {len(ratings_candidates)}." + ) + + ids_uih = maybe_truncate_seq(ids_uih, self._max_uih_len) + ratings_uih = maybe_truncate_seq(ratings_uih, self._max_uih_len) + timestamps_uih = maybe_truncate_seq(timestamps_uih, self._max_uih_len) + ids_candidates = maybe_truncate_seq(ids_candidates, max_num_candidates) + num_candidates = len(ids_candidates) + ratings_candidates = maybe_truncate_seq(ratings_candidates, max_num_candidates) + action_weights_uih = [ + self.action_weights[int(rating) - 1] for rating in ratings_uih + ] + action_weights_candidates = [ + int(rating >= 3.5) for rating in ratings_candidates + ] + + uih_kjt_values: List[int] = [] + uih_kjt_lengths: List[int] = [] + for name, length in self._contextual_feature_to_max_length.items(): + uih_kjt_values.append(data[name]) + uih_kjt_lengths.append(length) + + uih_seq_len = len(ids_uih) + dummy_watch_times_uih = [0 for _ in range(uih_seq_len)] + item_category_ids = [id // self.items_per_category for id in ids_uih] + extend_uih_kjt_values: List[int] = ( + ids_uih + + ratings_uih + + timestamps_uih + + action_weights_uih + + dummy_watch_times_uih + + item_category_ids + ) + uih_kjt_values.extend(extend_uih_kjt_values) + uih_kjt_lengths.extend( + [ + uih_seq_len + for _ in range( + len(self._uih_keys) - len(self._contextual_feature_to_max_length) + ) + ] + ) + + dummy_query_time = 0 if timestamps_uih == [] else max(timestamps_uih) + uih_kjt_values.append(dummy_query_time) + uih_kjt_lengths.append(1) + uih_features_kjt: KeyedJaggedTensor = KeyedJaggedTensor( + keys=self._uih_keys + ["dummy_query_time"], + lengths=torch.tensor(uih_kjt_lengths).long(), + values=torch.tensor(uih_kjt_values).long(), + ) + + candidates_kjt_lengths = num_candidates * torch.ones(len(self._candidates_keys)) + item_candidate_category_ids = [ + id // self.items_per_category for id in ids_candidates + ] + candidates_kjt_values = ( + ids_candidates + + ratings_candidates + + [dummy_query_time] * num_candidates # item_query_time + + action_weights_candidates + + [1] * num_candidates # item_dummy_watchtime + + item_candidate_category_ids + ) + candidates_features_kjt: KeyedJaggedTensor = KeyedJaggedTensor( + keys=self._candidates_keys, + lengths=candidates_kjt_lengths.detach().clone().long(), + values=torch.tensor(candidates_kjt_values).long(), + ) + return uih_features_kjt, candidates_features_kjt diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/datasets/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/utils.py new file mode 100644 index 000000000..aeca75d41 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/utils.py @@ -0,0 +1,146 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe +""" +Utility functions for dataset processing. + +This module provides helper functions for parsing and processing data +in the DLRMv3 dataset pipeline. +""" + +import json +import struct +from typing import Dict, List, Sequence, Tuple + +import numpy as np +import xxhash + + +def json_loads( + x: str | int | List[int], +) -> List[int]: + """ + Parse a JSON-like string into a list of integers. + + Handles multiple input formats including JSON arrays, comma-separated + strings, and single values. + + Args: + x: Input that can be a JSON array string, a single integer, + or already a list of integers. + + Returns: + List of integers parsed from the input. + """ + if isinstance(x, str): + if x[0] != "[" and x[-1] != "]": + x = "[" + x + "]" + y = json.loads(x) + else: + y = x + y_list = [y] if type(y) == int else list(y) + return y_list + + +def separate_uih_candidates( + x: str | int | List[int], + candidates_max_seq_len: int, +) -> Tuple[List[int], List[int]]: + """ + Separate a sequence into user interaction history (UIH) and candidates. + + Splits the input sequence such that the last `candidates_max_seq_len` + elements become candidates and the rest become UIH. + + Args: + x: Input sequence as JSON string, single int, or list of ints. + candidates_max_seq_len: Number of items at the end to use as candidates. + + Returns: + Tuple of (uih, candidates) where both are lists of integers. + """ + if isinstance(x, str): + if x[0] != "[" and x[-1] != "]": + x = "[" + x + "]" + y = json.loads(x) + else: + y = x + y_list = [y] if type(y) == int else list(y) + candidates, uih = ( + y_list[-candidates_max_seq_len:], + y_list[:-candidates_max_seq_len], + ) + return uih, candidates + + +def maybe_truncate_seq( + y: List[int], + max_seq_len: int, +) -> List[int]: + """ + Truncate a sequence if it exceeds the maximum length. + + Args: + y: Input sequence to potentially truncate. + max_seq_len: Maximum allowed sequence length. + + Returns: + The input sequence, truncated to max_seq_len if necessary. + """ + y_len = len(y) + if y_len > max_seq_len: + y = y[:max_seq_len] + return y + + +def xxhash_cross( + anchor: Dict[str, int], + keys: Sequence[str], + table_size: int, + salt: int = 0, +) -> int: + """xxhash64(seed=salt) over little-endian int64 concat(anchor[k] for k in keys), mod table_size. + + Bit-identical to primus_dlrm.data.hashing.cross_hash_nway — embedding rows + are interchangeable with Primus-trained ones. + """ + n = len(keys) + assert n >= 2, f"xxhash_cross needs >=2 keys, got {n}" + digest = xxhash.xxh64(seed=salt) + digest.update(struct.Struct(f"<{n}q").pack(*(int(anchor[k]) for k in keys))) + return digest.intdigest() % table_size + + +def xxhash_cross_batch( + arr_by_key: Dict[str, np.ndarray], + keys: Sequence[str], + table_size: int, + salt: int = 0, +) -> np.ndarray: + """Vectorised xxhash_cross over equal-length int64 arrays (one per key).""" + n = len(keys) + assert n >= 2 + cols = [np.asarray(arr_by_key[k], dtype=np.int64).ravel() for k in keys] + length = cols[0].shape[0] + for c in cols: + assert c.shape[0] == length + pack = struct.Struct(f"<{n}q").pack + digest_cls = xxhash.xxh64 + out = np.empty(length, dtype=np.int64) + for i in range(length): + d = digest_cls(seed=salt) + d.update(pack(*(int(c[i]) for c in cols))) + out[i] = d.intdigest() % table_size + return out diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/datasets/yambda.py b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/yambda.py new file mode 100644 index 000000000..fb8b212b1 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/yambda.py @@ -0,0 +1,608 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 + +# pyre-unsafe +""" +Yambda dataset for the DLRMv3 HSTU `modules/` path. + +Reads the same parquets produced by Primus-DLRM's preprocessing (no runtime +dep on Primus). Each sample is one anchor LISTEN event with: + * label = (played_ratio >= LISTEN_PLUS_THRESHOLD) — the listen_plus bit + * a chronologically interleaved 3-pool history (listen+/like/skip), with + pool identity tagged per-position in `action_weight` (bits 1/2/4) + * 7 pre-hashed cross-feature ids exposed as length-1 contextual entries + +Hash formula is byte-identical to `primus_dlrm.data.hashing.cross_hash_nway` +so embedding rows are interchangeable. +""" + +import logging +import mmap as _mmap_mod +import os +import time +from pathlib import Path +from typing import Dict, List, Optional, Sequence, Tuple, Union + +import numpy as np +import polars as pl +import torch +from generative_recommenders.dlrm_v3.datasets.dataset import DLRMv3RandomDataset +from generative_recommenders.dlrm_v3.datasets.utils import xxhash_cross +from generative_recommenders.modules.dlrm_hstu import DlrmHSTUConfig +from torchrec.sparse.jagged_tensor import KeyedJaggedTensor + +logger = logging.getLogger(__name__) + + +def _load_npy_readonly(path: Union[str, Path]) -> np.ndarray: + # MAP_SHARED + PROT_READ so the kernel does not charge the mapping against + # vm.overcommit_memory=2 limits. numpy's mmap_mode='r' uses MAP_PRIVATE and + # reserves per-process commit; at 8 ranks × ~190 GB store, that OOMs. + path = Path(path) + with open(path, "rb") as f: + version = np.lib.format.read_magic(f) + if version[0] == 1: + shape, _, dtype = np.lib.format.read_array_header_1_0(f) + else: + shape, _, dtype = np.lib.format.read_array_header_2_0(f) + offset = f.tell() + fd = os.open(str(path), os.O_RDONLY) + try: + buf = _mmap_mod.mmap(fd, 0, access=_mmap_mod.ACCESS_READ) + finally: + os.close(fd) + arr = np.ndarray(shape, dtype=dtype, buffer=buf, offset=offset) + arr.flags.writeable = False + return arr + +# Match primus_dlrm.data.preprocessing.EVENT_TYPE_MAP / dataset.LISTEN_PLUS_THRESHOLD +LISTEN_TYPE = 0 +LIKE_TYPE = 1 +LISTEN_PLUS_THRESHOLD = 50 + +# Action-weight bits (must match hstu_config.action_weights = [1, 2, 4]). +LP_BIT = 1 +LIKE_BIT = 2 +SKIP_BIT = 4 + + +class _FlatEventStore: + """Minimal port of Primus-DLRM's FlatEventStore. + + Reads `train_sessions.parquet` and explodes per-session arrays into flat + numpy columns + per-user `(start, end)` index arrays. Cache-compatible + layout, but writes nothing (rebuilds from parquet each construction). + """ + + # On-disk column layout written by Primus-DLRM's FlatEventStore.save_mmap. + # Bit-identical to that schema so the cache is interchangeable. + _MMAP_COLS = ( + "flat_uid", "flat_item_ids", "flat_timestamps", + "flat_event_types", "flat_played_ratio", + "flat_is_listen_plus", "flat_is_like", "flat_is_skip", + "flat_is_organic", + "user_start", "user_end", "unique_uids", + ) + + def __init__(self, sessions_df: pl.DataFrame) -> None: + logger.info("Building flat event store from sessions...") + sorted_sessions = sessions_df.sort(["uid", "session_id"]) + exploded = sorted_sessions.explode( + ["item_ids", "timestamps", "event_types", "is_organic", "played_ratio_pct"] + ) + + self.flat_uid: np.ndarray = exploded["uid"].to_numpy().astype(np.int64) + self.flat_item_ids: np.ndarray = exploded["item_ids"].to_numpy().astype(np.int64) + self.flat_timestamps: np.ndarray = exploded["timestamps"].to_numpy().astype(np.int64) + self.flat_event_types: np.ndarray = exploded["event_types"].to_numpy().astype(np.int64) + self.flat_played_ratio: np.ndarray = exploded["played_ratio_pct"].to_numpy().astype(np.float32) + self.flat_is_organic: np.ndarray = exploded["is_organic"].to_numpy().astype(np.int8) + np.nan_to_num(self.flat_played_ratio, copy=False, nan=0.0) + + is_listen = self.flat_event_types == LISTEN_TYPE + self.flat_is_listen_plus: np.ndarray = is_listen & ( + self.flat_played_ratio >= LISTEN_PLUS_THRESHOLD + ) + self.flat_is_like: np.ndarray = self.flat_event_types == LIKE_TYPE + self.flat_is_skip: np.ndarray = is_listen & ( + self.flat_played_ratio < LISTEN_PLUS_THRESHOLD + ) + + uid_changes = np.where(np.diff(self.flat_uid) != 0)[0] + 1 + starts = np.concatenate([[0], uid_changes]) + ends = np.concatenate([uid_changes, [len(self.flat_uid)]]) + uid_vals = self.flat_uid[starts] + max_uid = int(uid_vals.max()) + 1 + self.user_start: np.ndarray = np.full(max_uid, -1, dtype=np.int64) + self.user_end: np.ndarray = np.full(max_uid, -1, dtype=np.int64) + self.user_start[uid_vals] = starts + self.user_end[uid_vals] = ends + self.unique_uids: np.ndarray = uid_vals + self.num_users: int = len(uid_vals) + self.total_events: int = len(self.flat_item_ids) + logger.info( + f"FlatEventStore: {self.total_events:,} events, {self.num_users:,} users" + ) + + @classmethod + def load_mmap(cls, cache_dir: Union[str, Path]) -> "_FlatEventStore": + """Load flat columns by MAP_SHARED+PROT_READ from a prebuilt cache. + All ranks on a node share the same physical pages.""" + import json as _json + cache_dir = Path(cache_dir) + with open(cache_dir / "store_meta.json") as f: + meta = _json.load(f) + store = object.__new__(cls) + for name in cls._MMAP_COLS: + setattr(store, name, _load_npy_readonly(cache_dir / f"{name}.npy")) + store.num_users = int(meta["num_users"]) + store.total_events = int(meta["total_events"]) + logger.info( + f"FlatEventStore mmap from {cache_dir}: " + f"{store.total_events:,} events, {store.num_users:,} users" + ) + return store + + def save_mmap(self, cache_dir: Union[str, Path]) -> None: + """Persist flat columns to disk as .npy, then write a sentinel. + Subsequent runs (any rank, any node sharing the FS) load via mmap.""" + import json as _json + cache_dir = Path(cache_dir) + cache_dir.mkdir(parents=True, exist_ok=True) + for name in self._MMAP_COLS: + np.save(cache_dir / f"{name}.npy", getattr(self, name)) + with open(cache_dir / "store_meta.json", "w") as f: + _json.dump( + {"num_users": self.num_users, "total_events": self.total_events}, f + ) + # Sentinel — readers check this before mmap'ing to avoid partial files. + (cache_dir / "_READY").touch() + logger.info(f"FlatEventStore saved to {cache_dir}") + + +class DLRMv3YambdaDataset(DLRMv3RandomDataset): + """Yambda-5b dataset for the DLRMv3 HSTU modules/ path. + + Args: + hstu_config: DlrmHSTUConfig (must come from `get_hstu_configs("yambda-5b")`). + processed_dir: directory with `train_sessions.parquet` + `item_popularity.npy`. + metadata_dir: directory with `{artist,album}_item_mapping.parquet`. + history_length: per-pool truncation cap (total interleaved ≤ 3 * this). + scan_window: how far back to scan when filling each pool. + cross_specs: list of (name, keys, num_embeddings, salt). Source of truth + in `dlrm_v3/configs.py:YAMBDA_5B_CROSS_SPECS`. + is_inference: passed through to base class. + """ + + def __init__( + self, + hstu_config: DlrmHSTUConfig, + processed_dir: str, + metadata_dir: str, + history_length: int = 2048, + scan_window: int = 20000, + cross_specs: Optional[Sequence[Tuple[str, Sequence[str], int, int]]] = None, + cache_dir: Optional[str] = None, + is_inference: bool = False, + *args, + **kwargs, + ) -> None: + super().__init__(hstu_config=hstu_config, is_inference=is_inference) + self._processed_dir: str = processed_dir + self._metadata_dir: str = metadata_dir + self._history_length: int = history_length + self._scan_window: int = scan_window + self._cache_dir: Optional[str] = cache_dir + self._cross_specs: List[Tuple[str, Tuple[str, ...], int, int]] = [ + (name, tuple(keys), n, s) for (name, keys, n, s) in (cross_specs or []) + ] + assert hstu_config.action_weights is not None + self._action_weights: List[int] = hstu_config.action_weights + + self._load_metadata(metadata_dir) + # Build-once-mmap-many: first rank to arrive acquires the build lock + # and explodes the parquet (one ~190 GB in-memory pass), then writes + # flat .npy columns + _READY sentinel. All ranks (including the + # builder, after dropping its in-memory copy) reload via MAP_SHARED+ + # PROT_READ — kernel shares physical pages across ranks so the steady- + # state per-rank RSS for the dataset is ~0. + if cache_dir is None: + cache_dir = os.path.join(processed_dir, f"hstu_cache_L{history_length}") + self._cache_dir = cache_dir + self._ensure_cache_built(cache_dir, processed_dir, history_length) + self.store: _FlatEventStore = _FlatEventStore.load_mmap(cache_dir) + # Mmap the positions file built alongside the flat columns. + self._positions: np.ndarray = _load_npy_readonly( + os.path.join(cache_dir, f"positions_L{history_length}.npy") + ) + logger.info( + f"Yambda dataset ready: {self.store.total_events:,} events, " + f"{len(self._positions):,} training positions" + ) + + @staticmethod + def _ensure_cache_built( + cache_dir: str, processed_dir: str, history_length: int + ) -> None: + """File-locked one-shot build with column-at-a-time explode. + + A naive `pl.read_parquet(...).explode([5 list cols])` peaks at ~1.6 TB + on the 5b dataset (polars holds input list-columns + dense output + + parallel-worker scratch all together). Instead we: + 1) Read parquet + sort once (sorted list-column DF, ~80 GB). + 2) For each output column: select that single list, explode, write + .npy, drop. Bounds incremental peak to one column (~38 GB). + 3) Derive bool flags and indices from the on-disk mmaps. + + Peak RAM: ~150 GB. Steady state across all ranks afterward: ~0 + incremental thanks to MAP_SHARED in load_mmap. + """ + import fcntl + import gc + import json as _json + + ready = os.path.join(cache_dir, "_READY") + if os.path.exists(ready): + return + os.makedirs(cache_dir, exist_ok=True) + lock_path = os.path.join(cache_dir, "_lock") + with open(lock_path, "w") as lf: + logger.info(f"Acquiring build lock for {cache_dir}...") + fcntl.flock(lf, fcntl.LOCK_EX) + try: + if os.path.exists(ready): + return + parquet_path = os.path.join(processed_dir, "train_sessions.parquet") + logger.info( + f"Building flat-event cache from {parquet_path} " + f"(column-at-a-time, ~150 GB peak RAM)" + ) + + # Step 1: read + sort. List columns stay nested at this stage. + sessions = pl.read_parquet(parquet_path).sort(["uid", "session_id"]) + logger.info(f"Sessions sorted: {sessions.shape}") + + # Per-session lengths + uids — used to derive flat_uid via + # np.repeat (cheap) without exploding the whole DF at once. + lengths = ( + sessions.select(pl.col("item_ids").list.len()) + .to_numpy() + .reshape(-1) + .astype(np.int64) + ) + session_uids = sessions["uid"].to_numpy().astype(np.int64) + N = int(lengths.sum()) + num_users = int(np.unique(session_uids).shape[0]) + logger.info(f"Total events: {N:,}, users: {num_users:,}") + + # Step 2: column-at-a-time explode → save → drop. + # uid is per-session scalar; expand via np.repeat. + flat_uid = np.repeat(session_uids, lengths).astype(np.int64) + np.save(os.path.join(cache_dir, "flat_uid.npy"), flat_uid) + del flat_uid, session_uids, lengths + gc.collect() + logger.info("Wrote flat_uid.npy") + + # Derived columns flat_is_listen_plus/like/skip depend on + # event_types + played_ratio. Save those two first, then + # derive the bools from the mmaps. + _list_cols = [ + ("item_ids", "flat_item_ids", np.int64), + ("timestamps", "flat_timestamps", np.int64), + ("event_types", "flat_event_types", np.int64), + ("is_organic", "flat_is_organic", np.int8), + ("played_ratio_pct", "flat_played_ratio", np.float32), + ] + for src_col, dst_name, dtype in _list_cols: + exploded = sessions.select(pl.col(src_col).explode()) + arr = exploded[src_col].to_numpy().astype(dtype, copy=False) + if dtype == np.float32: + np.nan_to_num(arr, copy=False, nan=0.0) + np.save(os.path.join(cache_dir, f"{dst_name}.npy"), arr) + del exploded, arr + gc.collect() + logger.info(f"Wrote {dst_name}.npy") + + # Drop the sessions DF now that all source columns are on disk. + del sessions + gc.collect() + + # Step 3: derive bool flags from the just-written mmaps. + event_types = _load_npy_readonly( + os.path.join(cache_dir, "flat_event_types.npy") + ) + played_ratio = _load_npy_readonly( + os.path.join(cache_dir, "flat_played_ratio.npy") + ) + is_listen = event_types == LISTEN_TYPE + np.save( + os.path.join(cache_dir, "flat_is_listen_plus.npy"), + is_listen & (played_ratio >= LISTEN_PLUS_THRESHOLD), + ) + np.save( + os.path.join(cache_dir, "flat_is_like.npy"), + event_types == LIKE_TYPE, + ) + np.save( + os.path.join(cache_dir, "flat_is_skip.npy"), + is_listen & (played_ratio < LISTEN_PLUS_THRESHOLD), + ) + del is_listen, played_ratio + gc.collect() + logger.info("Wrote flat_is_listen_plus/like/skip.npy") + + # user_start / user_end / unique_uids from flat_uid mmap. + flat_uid = _load_npy_readonly( + os.path.join(cache_dir, "flat_uid.npy") + ) + uid_changes = np.where(np.diff(flat_uid) != 0)[0] + 1 + starts = np.concatenate([[0], uid_changes]) + ends = np.concatenate([uid_changes, [len(flat_uid)]]) + uid_vals = flat_uid[starts] + max_uid = int(uid_vals.max()) + 1 + user_start = np.full(max_uid, -1, dtype=np.int64) + user_end = np.full(max_uid, -1, dtype=np.int64) + user_start[uid_vals] = starts + user_end[uid_vals] = ends + np.save(os.path.join(cache_dir, "user_start.npy"), user_start) + np.save(os.path.join(cache_dir, "user_end.npy"), user_end) + np.save(os.path.join(cache_dir, "unique_uids.npy"), uid_vals) + logger.info("Wrote user_start/end/unique_uids.npy") + + # Positions: LISTEN events with ≥history_length prior history. + # Done now (before dropping user_start) so all sibling ranks + # just mmap the result instead of each running a 75 GB build. + user_start_per_event = user_start[flat_uid] + idx = np.arange(len(flat_uid), dtype=np.int64) + keep = (idx - user_start_per_event >= history_length) & ( + event_types == LISTEN_TYPE + ) + positions = np.where(keep)[0].astype(np.int64) + np.save( + os.path.join(cache_dir, f"positions_L{history_length}.npy"), + positions, + ) + logger.info( + f"Wrote positions_L{history_length}.npy: {len(positions):,}" + ) + del ( + flat_uid, event_types, user_start, user_end, uid_vals, + starts, ends, uid_changes, idx, user_start_per_event, + keep, positions, + ) + gc.collect() + + # Meta + sentinel — written last; readers gate on _READY. + with open(os.path.join(cache_dir, "store_meta.json"), "w") as f: + _json.dump( + {"num_users": num_users, "total_events": N}, f + ) + open(os.path.join(cache_dir, "_READY"), "w").close() + logger.info(f"Cache build complete: {cache_dir}") + finally: + fcntl.flock(lf, fcntl.LOCK_UN) + + def _load_metadata(self, metadata_dir: str) -> None: + item_pop_path = os.path.join(metadata_dir, "item_popularity.npy") + if os.path.exists(item_pop_path): + item_popularity = np.load(item_pop_path) + else: + # Fallback: derive vocab size from the artist+album maps. + item_popularity = None + + artist_map = pl.read_parquet(os.path.join(metadata_dir, "artist_item_mapping.parquet")) + album_map = pl.read_parquet(os.path.join(metadata_dir, "album_item_mapping.parquet")) + n_items = int( + max( + int(artist_map["item_id"].max()) + 1, + int(album_map["item_id"].max()) + 1, + len(item_popularity) if item_popularity is not None else 0, + ) + ) + self.item_to_artist: np.ndarray = np.zeros(n_items, dtype=np.int64) + valid = artist_map.filter(pl.col("item_id") < n_items) + self.item_to_artist[valid["item_id"].to_numpy()] = valid["artist_id"].to_numpy() + self.item_to_album: np.ndarray = np.zeros(n_items, dtype=np.int64) + valid = album_map.filter(pl.col("item_id") < n_items) + self.item_to_album[valid["item_id"].to_numpy()] = valid["album_id"].to_numpy() + self.num_items: int = n_items + + def get_item_count(self) -> int: + return int(len(self._positions)) + + def iloc(self, idx: int) -> int: + return int(self._positions[idx]) + + def load_query_samples(self, sample_list) -> None: + max_num_candidates = ( + self._max_num_candidates_inference + if self._is_inference + else self._max_num_candidates + ) + self.items_in_memory = {} + for idx in sample_list: + flat_pos = self.iloc(idx) + self.items_in_memory[idx] = self._build_sample(flat_pos, max_num_candidates) + self.last_loaded = time.time() + + def get_sample(self, idx: int) -> Tuple[KeyedJaggedTensor, KeyedJaggedTensor]: + if idx in self.items_in_memory: + return self.items_in_memory[idx] + max_num_candidates = ( + self._max_num_candidates_inference + if self._is_inference + else self._max_num_candidates + ) + flat_pos = self.iloc(idx) + return self._build_sample(flat_pos, max_num_candidates) + + def _gather_interleaved_history( + self, flat_pos: int, user_start: int + ) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + """Build a single chronologically-ordered history sequence from the 3 + behavior pools. Each event's `action_weight` carries the pool bitmask + (LP_BIT/LIKE_BIT/SKIP_BIT). Per-pool cap = history_length // 3.""" + L = self._history_length + per_pool = max(1, L // 3) + scan_start = max(int(user_start), int(flat_pos) - self._scan_window) + scan_end = int(flat_pos) + if scan_end <= scan_start: + empty = np.empty(0, dtype=np.int64) + return empty, empty, empty, empty, empty + + item_ids = self.store.flat_item_ids[scan_start:scan_end] + timestamps = self.store.flat_timestamps[scan_start:scan_end] + is_lp = self.store.flat_is_listen_plus[scan_start:scan_end] + is_like = self.store.flat_is_like[scan_start:scan_end] + is_skip = self.store.flat_is_skip[scan_start:scan_end] + + # Local indices into the scan window — preserves chronological order + # within each pool and lets us interleave by re-sorting. + idx_all = np.arange(item_ids.shape[0], dtype=np.int64) + lp_idx = idx_all[is_lp][-per_pool:] + like_idx = idx_all[is_like][-per_pool:] + skip_idx = idx_all[is_skip][-per_pool:] + + keep_local = np.concatenate([lp_idx, like_idx, skip_idx]) + if keep_local.size == 0: + empty = np.empty(0, dtype=np.int64) + return empty, empty, empty, empty, empty + + order = np.argsort(keep_local, kind="stable") + keep_local = keep_local[order] + + items = item_ids[keep_local] + ts = timestamps[keep_local] + artists = self.item_to_artist[np.clip(items, 0, self.item_to_artist.shape[0] - 1)] + albums = self.item_to_album[np.clip(items, 0, self.item_to_album.shape[0] - 1)] + + # Pool bitmask per kept event (LP/LIKE/SKIP are mutually exclusive in + # the source data, but OR is safe and forward-compatible). + weight = np.zeros(keep_local.shape[0], dtype=np.int64) + weight[is_lp[keep_local]] |= LP_BIT + weight[is_like[keep_local]] |= LIKE_BIT + weight[is_skip[keep_local]] |= SKIP_BIT + + return items, artists, albums, ts, weight + + def _build_sample( + self, flat_pos: int, max_num_candidates: int + ) -> Tuple[KeyedJaggedTensor, KeyedJaggedTensor]: + uid = int(self.store.flat_uid[flat_pos]) + user_start = int(self.store.user_start[uid]) + + items, artists, albums, ts, weight = self._gather_interleaved_history( + flat_pos, user_start + ) + + target_item = int(self.store.flat_item_ids[flat_pos]) + target_artist = int( + self.item_to_artist[target_item] + if target_item < self.item_to_artist.shape[0] + else 0 + ) + target_album = int( + self.item_to_album[target_item] + if target_item < self.item_to_album.shape[0] + else 0 + ) + target_ts = int(self.store.flat_timestamps[flat_pos]) + + played_ratio = float(self.store.flat_played_ratio[flat_pos]) + is_lp = ( + int(self.store.flat_event_types[flat_pos]) == LISTEN_TYPE + and played_ratio >= LISTEN_PLUS_THRESHOLD + ) + # Label encoded into the candidate's action_weight via the LP bit, so + # _get_supervision_labels_and_weights sees the right supervision. + candidate_action_weight = LP_BIT if is_lp else 0 + + cross_id_anchor: Dict[str, int] = { + "uid": uid, + "item_id": target_item, + "artist_id": target_artist, + "album_id": target_album, + "hour_of_day": int((target_ts // 3600) % 24), + "is_organic": int(self.store.flat_is_organic[flat_pos]), + } + cross_ids: Dict[str, int] = { + name: xxhash_cross(cross_id_anchor, list(keys), n, salt) + for (name, keys, n, salt) in self._cross_specs + } + + # ---- Truncate UIH to fit max_seq_len budget ---- + uih_seq_len_budget = ( + self._max_seq_len + - max_num_candidates + - len(self._contextual_feature_to_max_length or {}) + ) + if items.shape[0] > uih_seq_len_budget: + items = items[-uih_seq_len_budget:] + artists = artists[-uih_seq_len_budget:] + albums = albums[-uih_seq_len_budget:] + ts = ts[-uih_seq_len_budget:] + weight = weight[-uih_seq_len_budget:] + uih_seq_len = int(items.shape[0]) + dummy_watch_time = np.zeros(uih_seq_len, dtype=np.int64) + + # ---- Build UIH KJT ---- + # Contextual features (length-1 each) iterated in the same order as + # `_contextual_feature_to_max_length` (matches movielens reference). + uih_kjt_values: List[int] = [] + uih_kjt_lengths: List[int] = [] + for name, length in (self._contextual_feature_to_max_length or {}).items(): + assert length == 1, f"yambda contextuals are length-1, got {name}={length}" + if name == "uid": + uih_kjt_values.append(uid) + else: + uih_kjt_values.append(int(cross_ids[name])) + uih_kjt_lengths.append(1) + + # Sequential features — order must match the trailing entries of + # hstu_uih_feature_names in configs.py: + # item_id, artist_id, album_id, action_weight, action_timestamp, dummy_watch_time + uih_kjt_values.extend(items.tolist()) + uih_kjt_values.extend(artists.tolist()) + uih_kjt_values.extend(albums.tolist()) + uih_kjt_values.extend(weight.tolist()) + uih_kjt_values.extend(ts.tolist()) + uih_kjt_values.extend(dummy_watch_time.tolist()) + n_sequential = len(self._uih_keys) - len(self._contextual_feature_to_max_length or {}) + uih_kjt_lengths.extend([uih_seq_len] * n_sequential) + + dummy_query_time = int(ts[-1]) if uih_seq_len > 0 else target_ts + uih_kjt_values.append(dummy_query_time) + uih_kjt_lengths.append(1) + + uih_features_kjt = KeyedJaggedTensor( + keys=self._uih_keys + ["dummy_query_time"], + lengths=torch.tensor(uih_kjt_lengths, dtype=torch.long), + values=torch.tensor(uih_kjt_values, dtype=torch.long), + ) + + # ---- Build candidates KJT ---- + # Order must match configs.py:hstu_candidate_feature_names exactly: + # item_candidate_id, item_candidate_artist_id, item_candidate_album_id, + # item_query_time, item_action_weight, item_dummy_watchtime + candidates_kjt_lengths = max_num_candidates * torch.ones( + len(self._candidates_keys), dtype=torch.long + ) + candidates_kjt_values: List[int] = ( + [target_item] * max_num_candidates + + [target_artist] * max_num_candidates + + [target_album] * max_num_candidates + + [dummy_query_time] * max_num_candidates + + [candidate_action_weight] * max_num_candidates + + [1] * max_num_candidates # item_dummy_watchtime + ) + candidates_features_kjt = KeyedJaggedTensor( + keys=self._candidates_keys, + lengths=candidates_kjt_lengths, + values=torch.tensor(candidates_kjt_values, dtype=torch.long), + ) + return uih_features_kjt, candidates_features_kjt diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/README.md b/recommendation_v4/generative_recommenders/dlrm_v3/inference/README.md new file mode 100644 index 000000000..ef1c9686d --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/README.md @@ -0,0 +1,88 @@ +# MLPerf Inference reference implementation for DLRMv3 + +## Install dependencies + +The reference implementation has been tested on a single host, with x86_64 CPUs +and 8 NVIDIA H100/B200 GPUs. Dependencies can be installed below, + +``` +cd generative_recommenders/ +pip install -e . +``` + +## Build loadgen + +``` +cd generative_recommenders/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/ +CFLAGS="-std=c++14 -O3" python -m pip install . +``` + +## Dataset download + +DLRMv3 uses a synthetic dataset specifically designed to match the model and +system characteristics of large-scale sequential recommendation (large item set +and long average sequence length for each request). To generate the dataset used +for both training and inference, run + +``` +cd generative_recommenders/dlrm_v3/ +python streaming_synthetic_data.py +``` + +The generated dataset has 2TB size, and contains 5 million users interacting +with a billion items over 100 timestamps. + +Only 1% of the dataset is used in the inference benchmark. The sampled DLRMv3 +dataset and trained checkpoint are available at +https://inference.mlcommons-storage.org/. + +Script to download the sampled dataset used in inference benchmark: + +``` +bash <(curl -s https://raw.githubusercontent.com/mlcommons/r2-downloader/refs/heads/main/mlc-r2-downloader.sh) https://inference.mlcommons-storage.org/metadata/dlrm-v3-dataset.uri +``` + +Script to download the 1TB trained checkpoint: + +``` +bash <(curl -s https://raw.githubusercontent.com/mlcommons/r2-downloader/refs/heads/main/mlc-r2-downloader.sh) https://inference.mlcommons-storage.org/metadata/dlrm-v3-checkpoint.uri +``` + +## Inference benchmark + +``` +cd generative_recommenders/generative_recommenders/dlrm_v3/inference/ +WORLD_SIZE=8 python main.py --dataset sampled-streaming-100b +``` + +The config file is listed in `dlrm_v3/inference/gin/streaming_100b.gin`. +`WORLD_SIZE` is the number of GPUs used in the inference benchmark. + +To load checkpoint from training, modify `run.model_path` inside the inference +gin config file. (We will relase the checkpoint soon.) + +To achieve the best performance, tune `run.target_qps` and `run.batch_size` in +the config file. + +## Accuracy test + +Set `run.compute_eval` will run the accuracy test and dump prediction outputs in +`mlperf_log_accuracy.json`. To check the accuracy, run + +``` +python accuracy.py --path path/to/mlperf_log_accuracy.json +``` + +We use normalized entropy (NE), accuracy, and AUC as the metrics to evaluate the model quality. For accepted submissions, all three metrics (NE, Accuracy, AUC) must be within 99% of the reference implementation values. The accuracy for the reference implementation evaluated on 34,996 requests across 10 inference timestamps are listed below: + +``` +NE: 86.687% +Accuracy: 69.651% +AUC: 78.663% +``` + +## Run unit tests + +``` +python tests/inference_test.py +``` diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/accuracy.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/accuracy.py new file mode 100644 index 000000000..19242f7bd --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/accuracy.py @@ -0,0 +1,86 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-strict +""" +Tool to calculate accuracy for loadgen accuracy output found in mlperf_log_accuracy.json +""" + +import argparse +import json +import logging + +import numpy as np +import torch +from generative_recommenders.dlrm_v3.configs import get_hstu_configs +from generative_recommenders.dlrm_v3.utils import MetricsLogger + +logger: logging.Logger = logging.getLogger("main") + + +def get_args() -> argparse.Namespace: + """Parse commandline.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--path", + required=True, + help="path to mlperf_log_accuracy.json", + ) + args = parser.parse_args() + return args + + +def main() -> None: + """ + Main function to calculate accuracy metrics from loadgen output. + + Reads the mlperf_log_accuracy.json file, parses the results, and computes + accuracy metrics using the MetricsLogger. Each result entry contains + predictions, labels, and weights packed as float32 numpy arrays. + """ + args = get_args() + logger.warning("Parsing loadgen accuracy log...") + with open(args.path, "r") as f: + results = json.load(f) + hstu_config = get_hstu_configs(dataset="sampled-streaming-100b") + metrics = MetricsLogger( + multitask_configs=hstu_config.multitask_configs, + batch_size=1, + window_size=3000, + device=torch.device("cpu"), + rank=0, + ) + logger.warning(f"results have {len(results)} entries") + for result in results: + data = np.frombuffer(bytes.fromhex(result["data"]), np.float32) + num_candidates = data[-1].astype(int) + assert len(data) == 1 + num_candidates * 3 + mt_target_preds = torch.from_numpy(data[0:num_candidates]) + mt_target_labels = torch.from_numpy(data[num_candidates : num_candidates * 2]) + mt_target_weights = torch.from_numpy( + data[num_candidates * 2 : num_candidates * 3] + ) + num_candidates = torch.tensor([num_candidates]) + metrics.update( + predictions=mt_target_preds.view(1, -1), + labels=mt_target_labels.view(1, -1), + weights=mt_target_weights.view(1, -1), + num_candidates=num_candidates, + ) + for k, v in metrics.compute().items(): + logger.warning(f"{k}: {v}") + + +if __name__ == "__main__": + main() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/cpp/hstu_runner.cpp b/recommendation_v4/generative_recommenders/dlrm_v3/inference/cpp/hstu_runner.cpp new file mode 100644 index 000000000..d4d0d4082 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/cpp/hstu_runner.cpp @@ -0,0 +1,215 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. +// +// End-to-end runner for the HSTU torch.jit / torch.package artifacts produced +// by generative_recommenders/dlrm_v3/inference/packager.py and exercised by +// :end_to_end_test. +// +// CLI: +// hstu_runner [--aott_library ...] +// +// +// Where: +// sparse.pt ScriptModule whose forward(uih, candidates) returns +// Tuple[Dict[str,Tensor], Dict[str,Tensor], +// Dict[str,Tensor], Tensor, Tensor] +// dense.pt ScriptModule (cuda:0, bf16) whose forward(...) returns +// Tuple[Tensor, Optional[Tensor], Optional[Tensor]] +// inputs.pt ScriptModule whose forward() returns +// Tuple[KeyedJaggedTensor, KeyedJaggedTensor] +// output.pt torch::pickle_save destination for the predictions tensor; +// readable from Python as ``torch.load(output.pt)``. + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace { + +struct RunnerArgs { + std::vector aottLibraryPaths; + std::string sparsePath; + std::string densePath; + std::string inputsPath; + std::string outputPath; +}; + +RunnerArgs parseArgs(int argc, char** argv) { + RunnerArgs args; + std::vector positional; + for (int i = 1; i < argc; ++i) { + const std::string arg{argv[i]}; + if (arg == "--aott_library") { + if (++i >= argc) { + throw std::runtime_error("--aott_library requires a path"); + } + args.aottLibraryPaths.emplace_back(argv[i]); + } else { + positional.push_back(arg); + } + } + + if (positional.size() != 4) { + throw std::runtime_error( + "Usage: hstu_runner [--aott_library ...] " + " "); + } + args.sparsePath = positional[0]; + args.densePath = positional[1]; + args.inputsPath = positional[2]; + args.outputPath = positional[3]; + return args; +} + +void loadAottLibraries( + const std::vector& libraryPaths, + const std::function& log) { + for (const auto& path : libraryPaths) { + log("[runner] loading AOT-T library " + path); + void* handle = dlopen(path.c_str(), RTLD_GLOBAL | RTLD_NOW); + if (handle == nullptr) { + throw std::runtime_error( + "failed to dlopen AOT-T library " + path + ": " + dlerror()); + } + } +} + +torch::jit::Module loadModule(const std::string& path) { + // @patternlint-disable-next-line no-torch-low-level-api + auto m = torch::jit::load(path); + m.eval(); + return m; +} + +// Walk a Dict and replace every value with .to(device) (and +// optionally .to(bfloat16)). C++ analog of move_sparse_output_to_device. +void moveDictToDevice( + c10::impl::GenericDict& d, + const torch::Device& device, + bool toBfloat16) { + for (auto& kv : d) { + auto t = kv.value().toTensor().to(device); + if (toBfloat16) { + t = t.to(torch::kBFloat16); + } + d.insert_or_assign(kv.key(), t); + } +} + +void writePickle(const torch::Tensor& t, const std::string& path) { + // torch::pickle_save returns a byte buffer in the same wire format as + // ``torch.save(tensor, ...)``, so the Python side can read it with + // ``torch.load(path)``. + const auto data = torch::jit::pickle_save(c10::IValue(t)); + std::ofstream out(path, std::ios::binary); + if (!out) { + throw std::runtime_error("failed to open output: " + path); + } + out.write(data.data(), static_cast(data.size())); +} + +} // namespace + +int main(int argc, char** argv) { + RunnerArgs args; + try { + args = parseArgs(argc, argv); + } catch (const std::exception& e) { + std::cerr << e.what() << '\n'; + return 1; + } + + // Log to a file next to the output so we can inspect even if + // buck2 swallows stderr. + const std::string logPath = args.outputPath + ".log"; + std::ofstream logFile(logPath); + auto log = [&](const std::string& msg) { + logFile << msg << std::endl; + logFile.flush(); + std::cerr << msg << std::endl; + }; + + try { + log("[runner] step 0: loading AOT-T libraries"); + loadAottLibraries(args.aottLibraryPaths, log); + log("[runner] step 0 done: loaded " + + std::to_string(args.aottLibraryPaths.size()) + " AOT-T libraries"); + + log("[runner] step 1: loading sparse module from " + args.sparsePath); + auto sparse = loadModule(args.sparsePath); + + log("[runner] step 2: loading dense module from " + args.densePath); + auto dense = loadModule(args.densePath); + + log("[runner] step 3: loading inputs module from " + args.inputsPath); + auto inputs = loadModule(args.inputsPath); + + log("[runner] step 4: running inputs.forward()"); + auto inputsTuple = inputs.forward({}).toTuple(); + auto uihLengths = inputsTuple->elements()[0]; + auto uihValues = inputsTuple->elements()[1]; + auto candidatesLengths = inputsTuple->elements()[2]; + auto candidatesValues = inputsTuple->elements()[3]; + log("[runner] step 4 done: got 4 input tensors"); + + log("[runner] step 5: running sparse.forward()"); + std::vector sparseInputs{ + uihLengths, uihValues, candidatesLengths, candidatesValues}; + auto sparseOut = sparse.forward(sparseInputs).toTuple(); + log("[runner] step 5 done: sparse forward returned " + + std::to_string(sparseOut->elements().size()) + " elements"); + + log("[runner] step 6: unpacking sparse output dicts"); + auto seqEmbValues = sparseOut->elements()[0].toGenericDict(); + auto seqEmbLengths = sparseOut->elements()[1].toGenericDict(); + auto payloadFeatures = sparseOut->elements()[2].toGenericDict(); + auto uihSeqLengths = sparseOut->elements()[3].toTensor(); + auto numCandidates = sparseOut->elements()[4].toTensor(); + log("[runner] step 6 done: unpacked dicts"); + + log("[runner] step 7: moving dicts to cuda:0"); + const auto device = torch::Device(torch::kCUDA, 0); + moveDictToDevice(seqEmbValues, device, /*toBfloat16=*/true); + log("[runner] step 7a: seqEmbValues moved"); + moveDictToDevice(seqEmbLengths, device, /*toBfloat16=*/false); + log("[runner] step 7b: seqEmbLengths moved"); + moveDictToDevice(payloadFeatures, device, /*toBfloat16=*/false); + log("[runner] step 7c: payloadFeatures moved"); + uihSeqLengths = uihSeqLengths.to(device); + numCandidates = numCandidates.to(device); + log("[runner] step 7 done: all on cuda:0"); + + log("[runner] step 8: running dense.forward()"); + std::vector denseInputs{ + seqEmbValues, + seqEmbLengths, + payloadFeatures, + uihSeqLengths, + numCandidates, + }; + auto denseOut = dense.forward(denseInputs); + log("[runner] step 8 done: dense forward returned"); + + auto preds = denseOut.toTensor().detach().cpu(); + log("[runner] step 9: preds on cpu"); + + std::cout << "preds shape: " << preds.sizes() << '\n'; + std::cout << "preds sum: " + << preds.to(torch::kFloat32).sum().item() << '\n'; + + writePickle(preds, args.outputPath); + std::cout << "wrote " << args.outputPath << '\n'; + log("[runner] step 10: done, wrote output"); + return 0; + } catch (const std::exception& e) { + log(std::string("hstu_runner FAILED: ") + e.what()); + return 1; + } +} diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/data_producer.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/data_producer.py new file mode 100644 index 000000000..6a8db77c8 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/data_producer.py @@ -0,0 +1,227 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-strict +""" +Data producer module for DLRMv3 inference. + +This module provides classes for producing and managing query data during inference, +supporting both single-threaded and multi-threaded data production modes. +""" + +import logging +import threading +import time +from queue import Queue +from typing import List, Optional, Tuple, Union + +import torch +from generative_recommenders.dlrm_v3.datasets.dataset import Dataset, Samples + +logging.basicConfig(level=logging.INFO) +logger: logging.Logger = logging.getLogger("data_producer") + + +class QueryItem: + """ + Container for a query item to be processed by the inference thread pool. + + Attributes: + query_ids: List of unique identifiers for the queries in this batch. + samples: The sample data containing features for the queries. + start: Time when the query was first received. + dt_queue: Time spent in the queue before processing. + dt_batching: Time spent on batching the data. + """ + + def __init__( + self, + query_ids: List[int], + samples: Samples, + start: float, + dt_queue: float, + dt_batching: float, + ) -> None: + self.query_ids = query_ids + self.samples = samples + self.start: float = start + self.dt_queue: float = dt_queue + self.dt_batching: float = dt_batching + + +class SingleThreadDataProducer: + """ + Single-threaded data producer for synchronous query processing. + + This producer processes queries on the main thread without any parallelism, + suitable for debugging or low-throughput scenarios. + + Args: + ds: The dataset to fetch samples from. + run_one_item: Callback function to process a single QueryItem. + """ + + def __init__(self, ds: Dataset, run_one_item) -> None: # pyre-ignore [2] + self.ds = ds + self.run_one_item = run_one_item # pyre-ignore [4] + + def enqueue( + self, query_ids: List[int], content_ids: List[int], t0: float, dt_queue: float + ) -> None: + """ + Enqueue queries for immediate synchronous processing. + + Args: + query_ids: List of unique query identifiers. + content_ids: List of content/sample identifiers to fetch. + t0: Timestamp when the query batch was created. + dt_queue: Time spent waiting in the queue. + """ + with torch.profiler.record_function("data batching"): + t0_batching: float = time.time() + samples: Union[Samples, List[Samples]] = self.ds.get_samples(content_ids) + dt_batching: float = time.time() - t0_batching + if isinstance(samples, Samples): + query = QueryItem( + query_ids=query_ids, + samples=samples, + start=t0, + dt_queue=dt_queue, + dt_batching=dt_batching, + ) + self.run_one_item(query) + else: + start_idx = 0 + for sample in samples: + batch_size: int = sample.batch_size() + query = QueryItem( + query_ids=query_ids[start_idx : start_idx + batch_size], + samples=sample, + start=t0, + dt_queue=dt_queue, + dt_batching=dt_batching, + ) + start_idx += batch_size + self.run_one_item(query) + + def finish(self) -> None: + """Finalize the producer. No-op for single-threaded mode.""" + pass + + +class MultiThreadDataProducer: + """ + Multi-threaded data producer for parallel query processing. + + Uses a thread pool to fetch and batch data in parallel with model inference, + improving throughput for high-load scenarios. + + Args: + ds: The dataset to fetch samples from. + threads: Number of worker threads to use. + run_one_item: Callback function to process a single QueryItem. + """ + + def __init__( + self, + ds: Dataset, + threads: int, + run_one_item, # pyre-ignore [2] + ) -> None: + queue_size_multiplier = 4 + self.ds = ds + self.threads = threads + self.run_one_item = run_one_item # pyre-ignore [4] + self.tasks: Queue[Optional[Tuple[List[int], List[int], float, float]]] = Queue( + maxsize=threads * queue_size_multiplier + ) + self.workers: List[threading.Thread] = [] + for _ in range(self.threads): + worker = threading.Thread(target=self.handle_tasks, args=(self.tasks,)) + worker.daemon = True + self.workers.append(worker) + worker.start() + + def handle_tasks( + self, tasks_queue: Queue[Optional[Tuple[List[int], List[int], float, float]]] + ) -> None: + """ + Worker thread main loop to process tasks from the queue. + + Each worker maintains its own CUDA stream for parallel execution. + + Args: + tasks_queue: Queue containing task tuples or None for termination. + """ + stream = torch.cuda.Stream() + while True: + query_and_content_ids = tasks_queue.get() + if query_and_content_ids is None: + tasks_queue.task_done() + break + query_ids, content_ids, t0, dt_queue = query_and_content_ids + t0_batching: float = time.time() + samples: Union[Samples, List[Samples]] = self.ds.get_samples(content_ids) + dt_batching: float = time.time() - t0_batching + if isinstance(samples, Samples): + qitem = QueryItem( + query_ids=query_ids, + samples=samples, + start=t0, + dt_queue=dt_queue, + dt_batching=dt_batching, + ) + with torch.inference_mode(), torch.cuda.stream(stream): + self.run_one_item(qitem) + else: + start_idx = 0 + for sample in samples: + batch_size: int = sample.batch_size() + qitem = QueryItem( + query_ids=query_ids[start_idx : start_idx + batch_size], + samples=sample, + start=t0, + dt_queue=dt_queue, + dt_batching=dt_batching, + ) + start_idx += batch_size + with torch.inference_mode(), torch.cuda.stream(stream): + self.run_one_item(qitem) + tasks_queue.task_done() + + def enqueue( + self, query_ids: List[int], content_ids: List[int], t0: float, dt_queue: float + ) -> None: + """ + Enqueue queries for asynchronous processing by worker threads. + + Args: + query_ids: List of unique query identifiers. + content_ids: List of content/sample identifiers to fetch. + t0: Timestamp when the query batch was created. + dt_queue: Time spent waiting in the queue. + """ + with torch.profiler.record_function("data batching"): + self.tasks.put((query_ids, content_ids, t0, dt_queue)) + + def finish(self) -> None: + """ + Signal all worker threads to terminate and wait for completion. + + Sends None to each worker to trigger graceful shutdown. + """ + for _ in self.workers: + self.tasks.put(None) + for worker in self.workers: + worker.join() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/dense_predict_module.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/dense_predict_module.py new file mode 100644 index 000000000..add2781bc --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/dense_predict_module.py @@ -0,0 +1,96 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-strict + +""" +TorchScript-friendly wrapper for the HSTU dense path (GPU transformer). + +``HSTUDenseScriptModule`` accepts the *flattened* sparse-output dicts produced +by :class:`HSTUSparseScriptModule`, reconstructs ``Dict[str, +SequenceEmbedding]`` for the existing :meth:`DlrmHSTU.main_forward` and +returns a 3-tuple of ``(preds, labels, weights)`` -- the only fields the +predictor actually consumes. +""" + +from typing import Dict + +import torch +from generative_recommenders.dlrm_v3.inference.inference_modules import get_hstu_model +from generative_recommenders.dlrm_v3.inference.ts_types import ( + SeqEmbLengths, + SeqEmbValues, + unflatten_seq_embeddings, +) +from generative_recommenders.modules.dlrm_hstu import DlrmHSTU, DlrmHSTUConfig +from torchrec.modules.embedding_configs import EmbeddingConfig + + +class HSTUDenseScriptModule(torch.nn.Module): + """Script-friendly dense module. + + The wrapper owns a dense-only :class:`DlrmHSTU` (no + ``_embedding_collection``) and delegates to ``main_forward`` after + reconstructing the ``SequenceEmbedding`` NamedTuple form. + """ + + def __init__( + self, + hstu_config: DlrmHSTUConfig, + table_config: Dict[str, EmbeddingConfig], + ) -> None: + super().__init__() + self._hstu_model: DlrmHSTU = get_hstu_model( + table_config=table_config, + hstu_config=hstu_config, + table_device="cpu", + is_dense=True, + ) + + def forward( + self, + seq_emb_values: SeqEmbValues, + seq_emb_lengths: SeqEmbLengths, + payload_features: Dict[str, torch.Tensor], + uih_seq_lengths: torch.Tensor, + num_candidates: torch.Tensor, + ) -> torch.Tensor: + # TorchScript supports ``int(tensor.item())`` on a 0-d tensor. + max_uih_len: int = int(uih_seq_lengths.max().item()) + max_num_candidates: int = int(num_candidates.max().item()) + + seq_embeddings = unflatten_seq_embeddings(seq_emb_values, seq_emb_lengths) + + ( + _, + _, + _, + mt_target_preds, + _mt_target_labels, + _mt_target_weights, + ) = self._hstu_model.main_forward( + seq_embeddings=seq_embeddings, + payload_features=payload_features, + max_uih_len=max_uih_len, + uih_seq_lengths=uih_seq_lengths, + max_num_candidates=max_num_candidates, + num_candidates=num_candidates, + ) + assert mt_target_preds is not None + # Return just the predictions tensor; labels/weights are unused by + # the predictor at inference time and would force ``Optional[Tensor]`` + # in the return type, which torch.jit.trace rejects ("Only tensors, + # lists, tuples of tensors, or dictionary of tensors can be output + # from traced functions"). + return mt_target_preds diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/end_to_end_test.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/end_to_end_test.py new file mode 100644 index 000000000..f1b956d9c --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/end_to_end_test.py @@ -0,0 +1,795 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-strict + +""" +End-to-end smoke test for the HSTU TorchScript + C++ deployment pipeline. + +What this binary does, in order: + +1. Build a synthetic batch (uih_kjt, candidates_kjt) via :func:`get_random_data`. +2. Build the eager :class:`HSTUSparseScriptModule` and + :class:`HSTUDenseScriptModule`. +3. Run them eagerly to obtain the reference ``preds_eager``. +4. ``torch.jit.script`` + save: + - ``sparse.pt`` (CPU) + - ``dense.pt`` (cuda:0, bf16) + - ``inputs.pt`` (an :class:`InputsBundle` ScriptModule whose + ``forward()`` returns ``Tuple[KeyedJaggedTensor, KeyedJaggedTensor]``) +5. Run the C++ runner + ``hstu_runner [--aott_library ...] ``. +6. ``torch.load`` the runner's output and compare against ``preds_eager`` + with :func:`torch.testing.assert_close` (loose tolerance because the + scripted path may use either the PyTorch fallback trace or AOT-T-loaded + Triton inference kernels). + +Usage (manual override of the runner path): + + buck2 run @mode/opt //generative_recommenders/dlrm_v3/inference:end_to_end_test \\ + -- --cpp_runner /path/to/hstu_runner + +By default the binary locates the runner via ``libfb.py.parutil`` -- it ships +inside the par as a resource (see BUCK). +""" + +import argparse +import logging +import os +import shutil +import sys +import tempfile +from typing import Any, Dict, List, Tuple + +import torch +from generative_recommenders.dlrm_v3.configs import ( + get_embedding_table_config, + get_hstu_configs, +) +from generative_recommenders.dlrm_v3.datasets.dataset import get_random_data +from generative_recommenders.dlrm_v3.inference.dense_predict_module import ( + HSTUDenseScriptModule, +) +from generative_recommenders.dlrm_v3.inference.sparse_predict_module import ( + HSTUSparseScriptModule, +) +from generative_recommenders.dlrm_v3.inference.ts_types import ( + SeqEmbLengths, + SeqEmbValues, + unflatten_seq_embeddings, +) +from generative_recommenders.modules.dlrm_hstu import DlrmHSTUConfig +from security.frameworks.python.exec.subprocess import TrustedSubprocessWithList +from torchrec.modules.embedding_configs import EmbeddingConfig +from torchrec.sparse.jagged_tensor import KeyedJaggedTensor + + +logger: logging.Logger = logging.getLogger(__name__) + + +_DEFAULT_DATASET = "kuairand-1k" + + +class InputsBundle(torch.nn.Module): + """Scripted holder for the test inputs. + + Returns the constituent tensors of the two KJTs as a 4-tuple + ``(uih_lengths, uih_values, candidates_lengths, candidates_values)`` so + the traced sparse module can rebuild the KJTs inside its forward (KJT + instances themselves are not traceable inputs). + """ + + def __init__( + self, + uih_kjt: KeyedJaggedTensor, + candidates_kjt: KeyedJaggedTensor, + ) -> None: + super().__init__() + self.register_buffer("uih_lengths", uih_kjt.lengths()) + self.register_buffer("uih_values", uih_kjt.values()) + self.register_buffer("candidates_lengths", candidates_kjt.lengths()) + self.register_buffer("candidates_values", candidates_kjt.values()) + + def forward( + self, + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + return ( + self.uih_lengths, + self.uih_values, + self.candidates_lengths, + self.candidates_values, + ) + + +class _SparseTraceShim(torch.nn.Module): + """Adapter that takes raw tensors and rebuilds the KJTs inside forward. + + ``torch.jit.trace`` does not accept ``KeyedJaggedTensor`` (or any + non-Tensor / non-collection-of-Tensor type) as a top-level forward + input, so we make the traced boundary tensor-only and bake the + ``List[str]`` of feature keys in as Python constants captured by the + closure / module attribute. + """ + + def __init__( + self, + sparse_module: HSTUSparseScriptModule, + uih_keys: List[str], + candidates_keys: List[str], + ) -> None: + super().__init__() + self._sparse_module: HSTUSparseScriptModule = sparse_module + self._uih_keys: List[str] = uih_keys + self._candidates_keys: List[str] = candidates_keys + + def forward( + self, + uih_lengths: torch.Tensor, + uih_values: torch.Tensor, + candidates_lengths: torch.Tensor, + candidates_values: torch.Tensor, + ) -> Tuple[ + SeqEmbValues, + SeqEmbLengths, + Dict[str, torch.Tensor], + torch.Tensor, + torch.Tensor, + ]: + uih_kjt = KeyedJaggedTensor( + keys=self._uih_keys, + lengths=uih_lengths, + values=uih_values, + ) + candidates_kjt = KeyedJaggedTensor( + keys=self._candidates_keys, + lengths=candidates_lengths, + values=candidates_values, + ) + return self._sparse_module( + uih_features=uih_kjt, candidates_features=candidates_kjt + ) + + +class _DenseAottTraceShim(torch.nn.Module): + """FX-traceable dense adapter for the representative AOT-T shape.""" + + def __init__( + self, + dense_module: HSTUDenseScriptModule, + max_uih_len: int, + max_num_candidates: int, + total_uih_len: int, + total_targets: int, + ) -> None: + super().__init__() + self._dense_module: HSTUDenseScriptModule = dense_module + self._max_uih_len: int = max_uih_len + self._max_num_candidates: int = max_num_candidates + self._total_uih_len: int = total_uih_len + self._total_targets: int = total_targets + + def forward( + self, + seq_emb_values: SeqEmbValues, + seq_emb_lengths: SeqEmbLengths, + payload_features: Dict[str, torch.Tensor], + uih_seq_lengths: torch.Tensor, + num_candidates: torch.Tensor, + ) -> torch.Tensor: + seq_embeddings = unflatten_seq_embeddings(seq_emb_values, seq_emb_lengths) + + ( + _, + _, + _, + mt_target_preds, + _mt_target_labels, + _mt_target_weights, + ) = self._dense_module._hstu_model.main_forward( + seq_embeddings=seq_embeddings, + payload_features=payload_features, + max_uih_len=self._max_uih_len, + uih_seq_lengths=uih_seq_lengths, + max_num_candidates=self._max_num_candidates, + num_candidates=num_candidates, + total_uih_len=self._total_uih_len, + total_targets=self._total_targets, + ) + assert mt_target_preds is not None + return mt_target_preds + + +def _dense_aott_concrete_args( + dense_inputs: Tuple[ + Dict[str, torch.Tensor], + Dict[str, torch.Tensor], + Dict[str, torch.Tensor], + torch.Tensor, + torch.Tensor, + ], +) -> Dict[str, Any]: + from torch.fx._symbolic_trace import PH + + seq_emb_values, seq_emb_lengths, payload_features, _, _ = dense_inputs + return { + "seq_emb_values": {key: PH for key in seq_emb_values}, + "seq_emb_lengths": {key: PH for key in seq_emb_lengths}, + "payload_features": {key: PH for key in payload_features}, + } + + +def _find_cpp_runner() -> str: + """Locate the bundled hstu_runner binary. + + Tries ``importlib.resources`` (the canonical fbcode resource resolver, + works whether the binary is in a par or unpacked), and falls back to + looking next to ``sys.argv[0]``. + """ + try: + from importlib.resources import files + + path = files("generative_recommenders.dlrm_v3.inference.cpp").joinpath( + "hstu_runner" + ) + if path.is_file(): + return str(path) + except Exception as exc: + logger.debug("importlib.resources lookup failed: %s", exc) + + candidate = os.path.join( + os.path.dirname(os.path.abspath(sys.argv[0])), "hstu_runner" + ) + if os.path.exists(candidate): + return candidate + + raise RuntimeError( + "Could not find hstu_runner binary. " + "Pass --cpp_runner= or build the cpp_binary target first." + ) + + +def _eager_run( + sparse_module: HSTUSparseScriptModule, + dense_module: HSTUDenseScriptModule, + uih_kjt: KeyedJaggedTensor, + candidates_kjt: KeyedJaggedTensor, + device: torch.device, +) -> torch.Tensor: + """Reference path: sparse → device-move + bf16 → dense, all in Python.""" + with torch.no_grad(): + seq_emb_values, seq_emb_lengths, payload, uih_lens, num_cands = sparse_module( + uih_features=uih_kjt, candidates_features=candidates_kjt + ) + seq_emb_values = { + k: v.to(device).to(torch.bfloat16) for k, v in seq_emb_values.items() + } + seq_emb_lengths = {k: v.to(device) for k, v in seq_emb_lengths.items()} + payload = {k: v.to(device) for k, v in payload.items()} + uih_lens = uih_lens.to(device) + num_cands = num_cands.to(device) + preds = dense_module( + seq_emb_values, seq_emb_lengths, payload, uih_lens, num_cands + ) + return preds.detach().to(torch.float32).cpu() + + +def _find_aott_libraries() -> List[str]: + from generative_recommenders.ops.triton_aot.compile.compile_state import ( + get_aott_compile_path, + ) + + compile_path = get_aott_compile_path() + libraries: List[str] = [] + for root, _, files in os.walk(compile_path): + for filename in files: + if filename.endswith(".so"): + libraries.append(os.path.join(root, filename)) + return sorted(libraries) + + +def _copy_aott_libraries_to_workdir( + library_paths: List[str], workdir: str +) -> List[str]: + copied: List[str] = [] + for index, path in enumerate(library_paths): + dst = os.path.join(workdir, f"aott_{index}_{os.path.basename(path)}") + shutil.copy2(path, dst) + copied.append(dst) + return copied + + +def _load_aott_libraries_for_python(library_paths: List[str]) -> None: + for library_path in library_paths: + logger.info("Python roundtrip: loading AOT-T library %s", library_path) + torch.ops.load_library(library_path) + + +def _save_aott_dense_module( + dense_module: HSTUDenseScriptModule, + dense_inputs: Tuple[ + Dict[str, torch.Tensor], + Dict[str, torch.Tensor], + Dict[str, torch.Tensor], + torch.Tensor, + torch.Tensor, + ], + dense_path: str, + workdir: str, + atol: float, + rtol: float, +) -> List[str]: + """Lower the dense module with AOT-T and save a TorchScript artifact. + + This follows the AOT-T example flow: + + 1. FX trace the module. + 2. Unwrap outer `aot_triton_kernel_wrapper_*` nodes. + 3. Run representative CUDA inputs under `TritonAOTCompile`. + 4. `transform_kernels` to replace wrappers with `torch.ops.triton_aot.*`. + 5. Script and save the transformed dense module. + + The full HSTU dense wrapper has historically needed tracing rather than FX, + so failures here are reported with context and the default path remains the + D102 traced TorchScript fallback. + """ + from generative_recommenders.ops.triton_aot.compile.triton_aot_compile import ( + TritonAOTCompile, + ) + from generative_recommenders.ops.triton_aot.preprocess import ( + unwrap_aott_wrapper_nodes, + ) + from generative_recommenders.ops.triton_aot.transform.transform_kernels import ( + transform_kernels, + ) + from tgif.fx.tgif_tracer import TGIFTracer + + max_uih_len = int(dense_inputs[3].max().item()) + max_num_candidates = int(dense_inputs[4].max().item()) + total_uih_len = int(dense_inputs[3].sum().item()) + total_targets = int(dense_inputs[4].sum().item()) + trace_shim = _DenseAottTraceShim( + dense_module=dense_module, + max_uih_len=max_uih_len, + max_num_candidates=max_num_candidates, + total_uih_len=total_uih_len, + total_targets=total_targets, + ).eval() + + logger.info( + "AOT-T dense: FX tracing representative shape " + "(max_uih_len=%d, max_num_candidates=%d, " + "total_uih_len=%d, total_targets=%d)...", + max_uih_len, + max_num_candidates, + total_uih_len, + total_targets, + ) + try: + fx_dense = TGIFTracer().symbolic_trace( + trace_shim, + concrete_args=_dense_aott_concrete_args(dense_inputs), + ) + lowered_dense = unwrap_aott_wrapper_nodes(fx_dense, TGIFTracer()) + except Exception as exc: + raise RuntimeError( + "AOT-T dense lowering requires an FX-traceable dense entry point. " + "Use --dense_backend=torchscript to fall back to the D102 traced " + "TorchScript path." + ) from exc + + logger.info("AOT-T dense: compiling Triton kernels from sample inputs...") + with torch.no_grad(): + with TritonAOTCompile(): + ref_output = lowered_dense(*dense_inputs) + + original_code = lowered_dense.code + lowered_dense = transform_kernels(lowered_dense) + if lowered_dense.code == original_code: + logger.warning( + "AOT-T dense: transform_kernels did not change the FX graph. " + "This usually means no aot_triton_kernel_wrapper_* nodes were " + "present in the dense path." + ) + + libraries = _find_aott_libraries() + if not libraries: + raise RuntimeError( + "AOT-T dense lowering produced no .so files. Ensure the dense path " + "uses HammerKernel.TRITON_INFERENCE branches backed by triton_aot ops." + ) + + with torch.no_grad(): + lowered_output = lowered_dense(*dense_inputs) + torch.testing.assert_close(ref_output, lowered_output, atol=atol, rtol=rtol) + + logger.info("AOT-T dense: tracing transformed module...") + torch.jit.trace( + lowered_dense, + example_inputs=dense_inputs, + strict=False, + check_trace=False, + ).save(dense_path) + copied_libraries = _copy_aott_libraries_to_workdir(libraries, workdir) + logger.info("AOT-T dense: copied %d library file(s)", len(copied_libraries)) + return copied_libraries + + +def _build_synthetic_inputs( + hstu_config: DlrmHSTUConfig, + table_config: Dict[str, EmbeddingConfig], + uih_max_seq_len: int, +) -> Tuple[KeyedJaggedTensor, KeyedJaggedTensor]: + contextual: List[str] = list(hstu_config.contextual_feature_to_max_length.keys()) + # The kuairand-1k dataset has tiny embedding tables for some contextual + # features (e.g. user_active_degree has num_embeddings=8). Clamp the + # random value range so every index stays in range for every table. + min_rows = min(t.num_embeddings for t in table_config.values()) + value_bound = max(2, min_rows) + logger.info( + "synthetic value_bound=%d (min table rows=%d across %d tables)", + value_bound, + min_rows, + len(table_config), + ) + return get_random_data( + contexual_features=contextual, + hstu_uih_keys=hstu_config.hstu_uih_feature_names, + hstu_candidates_keys=hstu_config.hstu_candidate_feature_names, + uih_max_seq_len=uih_max_seq_len, + max_num_candidates=hstu_config.max_num_candidates_inference, + value_bound=value_bound, + ) + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--cpp_runner", + type=str, + default=None, + help="Path to the hstu_runner binary; default: bundled resource.", + ) + parser.add_argument( + "--dataset", + type=str, + default=_DEFAULT_DATASET, + help="Dataset key for HSTU/embedding configs.", + ) + parser.add_argument( + "--device", type=str, default="cuda:0", help="Dense-module device." + ) + parser.add_argument( + "--uih_max_seq_len", + type=int, + default=128, + help="Max UIH length for the synthetic batch.", + ) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--atol", type=float, default=1e-2) + parser.add_argument("--rtol", type=float, default=1e-2) + parser.add_argument( + "--dense_backend", + choices=("torchscript", "aott"), + default="torchscript", + help="Dense artifact backend. aott lowers TRITON_INFERENCE wrappers and passes compiled libraries to the C++ runner.", + ) + parser.add_argument( + "--aott_library", + action="append", + default=[], + help="Additional prebuilt AOT-T shared library to dlopen before loading dense.pt. May be repeated.", + ) + parser.add_argument( + "--keep_workdir", + action="store_true", + help="Do not delete the temp dir holding the saved artifacts.", + ) + return parser.parse_args() + + +def main() -> None: # noqa: C901 + logging.basicConfig(level=logging.INFO, format="[e2e] %(message)s", force=True) + logger.setLevel(logging.DEBUG) + args = _parse_args() + + if not torch.cuda.is_available(): + logger.error("CUDA is required; aborting.") + sys.exit(2) + + runner_path = args.cpp_runner or _find_cpp_runner() + logger.info("Using C++ runner: %s", runner_path) + + torch.manual_seed(args.seed) + device = torch.device(args.device) + torch.cuda.set_device(device) + + hstu_config = get_hstu_configs(args.dataset) + table_config = get_embedding_table_config(args.dataset) + + uih_kjt, candidates_kjt = _build_synthetic_inputs( + hstu_config, table_config, args.uih_max_seq_len + ) + + sparse_module = HSTUSparseScriptModule( + table_config=table_config, + hstu_config=hstu_config, + use_no_copy_embedding_collection=True, + ).eval() + dense_module = ( + HSTUDenseScriptModule(hstu_config=hstu_config, table_config=table_config) + .to(torch.bfloat16) + .to(device) + .eval() + ) + + from generative_recommenders.common import HammerKernel + + dense_kernel = ( + HammerKernel.TRITON_INFERENCE + if args.dense_backend == "aott" + else HammerKernel.PYTORCH + ) + sparse_module._sparse._hstu_model.set_hammer_kernel(HammerKernel.PYTORCH) + dense_module._hstu_model.set_hammer_kernel(dense_kernel) + + # Diagnostic: walk every HammerModule submodule and print its effective + # kernel selection, so any submodule that didn't pick up the override + # surfaces immediately. Triton/Triton-CC selections will fail at trace + # time, so this print is critical for triaging the next iteration if + # tracing fails. + from generative_recommenders.common import HammerModule as _HM + + for name, m in list(sparse_module.named_modules()) + list( + dense_module.named_modules() + ): + if isinstance(m, _HM): + logger.info( + "kernel-pin %-60s -> %s (is_inference=%s, use_triton_cc=%s)", + name or "", + m.hammer_kernel().value, + m._is_inference, + m._use_triton_cc, + ) + + # === 1. Eager reference === + logger.info("Running eager reference...") + preds_eager = _eager_run( + sparse_module, dense_module, uih_kjt, candidates_kjt, device + ) + logger.info( + "preds_eager shape=%s sum=%.6f", + tuple(preds_eager.shape), + preds_eager.sum().item(), + ) + + # === 2. Trace/lower + save === + # The default path keeps D102's trace-based TorchScript artifact. The + # AOT-T path follows ModelStore's compile/transform flow and saves a + # scripted FX module whose Triton kernels dispatch through torch.ops. + workdir = tempfile.mkdtemp(prefix="hstu_e2e_") + sparse_path = os.path.join(workdir, "sparse.pt") + dense_path = os.path.join(workdir, "dense.pt") + inputs_path = os.path.join(workdir, "inputs.pt") + cpp_out_path = os.path.join(workdir, "preds_cpp.pt") + eager_out_path = os.path.join(workdir, "preds_eager.pt") + aott_library_paths: List[str] = list(args.aott_library) + python_roundtrip_aott_library_paths: List[str] = list(args.aott_library) + logger.info("workdir: %s", workdir) + + # Re-run sparse eagerly to capture an example output that can drive the + # dense trace. + with torch.no_grad(): + sparse_out = sparse_module( + uih_features=uih_kjt, candidates_features=candidates_kjt + ) + seq_emb_values = { + k: v.to(device).to(torch.bfloat16) for k, v in sparse_out[0].items() + } + seq_emb_lengths = {k: v.to(device) for k, v in sparse_out[1].items()} + payload = {k: v.to(device) for k, v in sparse_out[2].items()} + uih_lens = sparse_out[3].to(device) + num_cands = sparse_out[4].to(device) + + logger.info("Tracing sparse module via raw-tensor shim (CPU)...") + sparse_shim = _SparseTraceShim( + sparse_module=sparse_module, + uih_keys=list(uih_kjt.keys()), + candidates_keys=list(candidates_kjt.keys()), + ) + traced_sparse = torch.jit.trace( + sparse_shim, + example_inputs=( + uih_kjt.lengths(), + uih_kjt.values(), + candidates_kjt.lengths(), + candidates_kjt.values(), + ), + strict=False, + check_trace=False, + ) + traced_sparse.save(sparse_path) + + dense_inputs = ( + seq_emb_values, + seq_emb_lengths, + payload, + uih_lens, + num_cands, + ) + if args.dense_backend == "aott": + logger.info("Lowering dense module with AOT-T...") + generated_aott_library_paths = _save_aott_dense_module( + dense_module, + dense_inputs, + dense_path, + workdir, + args.atol, + args.rtol, + ) + aott_library_paths.extend(generated_aott_library_paths) + else: + logger.info("Tracing dense module (cuda:0, bf16)...") + traced_dense = torch.jit.trace( + dense_module, + example_inputs=dense_inputs, + strict=False, + check_trace=False, + ) + traced_dense.save(dense_path) + + logger.info("Scripting + saving inputs bundle...") + torch.jit.script(InputsBundle(uih_kjt, candidates_kjt)).save(inputs_path) + torch.save(preds_eager, eager_out_path) + + # === 2.5. Python-side roundtrip verification === + # Load the saved traced artifacts back in Python and verify they produce + # the same results as the eager run. This proves the artifacts are correct + # independently of the C++ runner. + logger.info("Python roundtrip: loading traced artifacts back...") + if python_roundtrip_aott_library_paths: + _load_aott_libraries_for_python(python_roundtrip_aott_library_paths) + rt_inputs = torch.jit.load(inputs_path) + rt_sparse = torch.jit.load(sparse_path) + rt_dense = torch.jit.load(dense_path) + + with torch.no_grad(): + rt_uih_l, rt_uih_v, rt_cand_l, rt_cand_v = rt_inputs() + logger.info( + " rt inputs: uih_l=%s uih_v=%s cand_l=%s cand_v=%s", + rt_uih_l.shape, + rt_uih_v.shape, + rt_cand_l.shape, + rt_cand_v.shape, + ) + + rt_sparse_out = rt_sparse(rt_uih_l, rt_uih_v, rt_cand_l, rt_cand_v) + + for i, elem in enumerate(rt_sparse_out): + if isinstance(elem, dict): + for k, v in elem.items(): + has_nan = torch.isnan(v).any().item() + has_inf = torch.isinf(v).any().item() + logger.info( + " sparse_out[%d][%s] shape=%s dtype=%s nan=%s inf=%s", + i, + k, + tuple(v.shape), + v.dtype, + has_nan, + has_inf, + ) + elif isinstance(elem, torch.Tensor): + logger.info( + " sparse_out[%d] shape=%s dtype=%s nan=%s inf=%s", + i, + tuple(elem.shape), + elem.dtype, + torch.isnan(elem).any().item(), + torch.isinf(elem).any().item(), + ) + + rt_sev = { + k: v.to(device).to(torch.bfloat16) for k, v in rt_sparse_out[0].items() + } + rt_sel = {k: v.to(device) for k, v in rt_sparse_out[1].items()} + rt_pay = {k: v.to(device) for k, v in rt_sparse_out[2].items()} + rt_uih = rt_sparse_out[3].to(device) + rt_nc = rt_sparse_out[4].to(device) + + preds_rt = rt_dense(rt_sev, rt_sel, rt_pay, rt_uih, rt_nc) + + preds_rt_cpu = preds_rt.detach().to(torch.float32).cpu() + logger.info( + "preds_roundtrip shape=%s sum=%.6f nan=%s inf=%s", + tuple(preds_rt_cpu.shape), + preds_rt_cpu.sum().item(), + torch.isnan(preds_rt_cpu).any().item(), + torch.isinf(preds_rt_cpu).any().item(), + ) + + try: + torch.testing.assert_close( + preds_eager, preds_rt_cpu, atol=args.atol, rtol=args.rtol + ) + except AssertionError as e: + logger.error("PYTHON ROUNDTRIP PARITY FAILED: %s", e) + if not args.keep_workdir: + logger.info("(workdir kept for inspection: %s)", workdir) + sys.exit(1) + logger.info("PYTHON ROUNDTRIP PASSED (atol=%g rtol=%g)", args.atol, args.rtol) + + # === 3. Invoke C++ runner === + runner_args: List[str] = [] + for library_path in aott_library_paths: + runner_args.extend(["--aott_library", library_path]) + runner_args.extend([sparse_path, dense_path, inputs_path, cpp_out_path]) + + logger.info("Running C++: %s %s", runner_path, " ".join(runner_args)) + # pyre-fixme[6]: TrustedSubprocessWithList requires Literal[str] but this + # runner is resolved from a built resource or explicit test argument. + result = TrustedSubprocessWithList.run( + executable=runner_path, + cmd_args=runner_args, + capture_output=True, + text=True, + check=False, + ) + if result.stdout: + logger.info("--- runner stdout ---\n%s", result.stdout.rstrip()) + if result.stderr: + logger.info("--- runner stderr ---\n%s", result.stderr.rstrip()) + if result.returncode != 0: + if result.returncode == -11: + logger.warning( + "C++ runner SIGSEGV (exit -11). This is a known issue with " + "torch-cpp-cuda static initialization on some machines. " + "Python roundtrip verification passed above. " + "Artifacts in: %s", + workdir, + ) + args.keep_workdir = True + else: + logger.error("C++ runner exited with code %d", result.returncode) + if not args.keep_workdir: + shutil.rmtree(workdir, ignore_errors=True) + sys.exit(result.returncode) + + # === 4. Compare === + if not os.path.exists(cpp_out_path): + logger.error("C++ runner did not produce %s", cpp_out_path) + sys.exit(1) + preds_cpp = torch.load(cpp_out_path, weights_only=False).to(torch.float32).cpu() + logger.info( + "preds_cpp shape=%s sum=%.6f", + tuple(preds_cpp.shape), + preds_cpp.sum().item(), + ) + + try: + torch.testing.assert_close( + preds_eager, preds_cpp, atol=args.atol, rtol=args.rtol + ) + except AssertionError as e: + logger.error("PARITY FAILED: %s", e) + if not args.keep_workdir: + logger.info("(workdir kept for inspection: %s)", workdir) + sys.exit(1) + + logger.info("PASSED: eager and C++ agree (atol=%g rtol=%g)", args.atol, args.rtol) + if not args.keep_workdir: + shutil.rmtree(workdir, ignore_errors=True) + + +if __name__ == "__main__": + main() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/gin/debug.gin b/recommendation_v4/generative_recommenders/dlrm_v3/inference/gin/debug.gin new file mode 100644 index 000000000..e2025dee0 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/gin/debug.gin @@ -0,0 +1,13 @@ +run.model_path = "" +run.scenario_name = "Server" +run.batchsize = 16 +run.output_trace = False +run.data_producer_threads = 4 +run.compute_eval = False +run.find_peak_performance = False +run.train_split_percentage = 0.75 + +# below will override mlperf rules compliant settings - don't use for official submission +run.target_qps = 2000 +run.num_queries = 10000 +run.numpy_rand_seed = 123 diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/gin/kuairand_1k.gin b/recommendation_v4/generative_recommenders/dlrm_v3/inference/gin/kuairand_1k.gin new file mode 100644 index 000000000..a770aa014 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/gin/kuairand_1k.gin @@ -0,0 +1,14 @@ +# run.model_path = "/home/linjianma/ckpts/kuairand_1k/2025_01_12_17_56_43/" +run.scenario_name = "Server" +run.batchsize = 16 +run.output_trace = False +run.data_producer_threads = 4 +run.compute_eval = False +run.find_peak_performance = False +run.train_split_percentage = 0.75 + +# below will override mlperf rules compliant settings - don't use for official submission +run.target_qps = 2000 +run.num_queries = 10000 +run.numpy_rand_seed = 123 +run.dataset_path_prefix = "/home/linjianma" diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/gin/movielens_13b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/inference/gin/movielens_13b.gin new file mode 100644 index 000000000..3121ac0e7 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/gin/movielens_13b.gin @@ -0,0 +1,16 @@ +run.model_path = "" +run.scenario_name = "Server" +run.batchsize = 5 +run.output_trace = False +run.data_producer_threads = 8 +run.compute_eval = False +run.find_peak_performance = False +run.train_split_percentage = 0.75 +run.sparse_quant = False + +# below will override mlperf rules compliant settings - don't use for official submission +run.target_qps = 5000 +run.num_queries = 30000 +run.numpy_rand_seed = 123 +run.dataset_path_prefix = "/home/linjianma" +run.dataset_percentage = 0.0625 diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/gin/streaming_100b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/inference/gin/streaming_100b.gin new file mode 100644 index 000000000..0655734c2 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/gin/streaming_100b.gin @@ -0,0 +1,15 @@ +# run.model_path = "/home/linjianma/ckpts/streaming_100b/89/" +run.scenario_name = "Server" +run.batchsize = 10 +run.output_trace = False +run.data_producer_threads = 16 +run.compute_eval = False +run.find_peak_performance = False +run.sparse_quant = False +run.numpy_rand_seed = 123 +run.dataset_path_prefix = "/home/linjianma" +run.dataset_percentage = 0.001 +run.warmup_ratio = 0.3 +run.num_queries = 20000 +# Needs to be tuned for different implementations to balance latency and throughput +run.target_qps = 1000 diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/gin/streaming_400m.gin b/recommendation_v4/generative_recommenders/dlrm_v3/inference/gin/streaming_400m.gin new file mode 100644 index 000000000..eed13e0ff --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/gin/streaming_400m.gin @@ -0,0 +1,15 @@ +run.model_path = "" +run.scenario_name = "Server" +run.batchsize = 5 +run.output_trace = False +run.data_producer_threads = 8 +run.compute_eval = False +run.find_peak_performance = False +run.train_split_percentage = 0.75 +run.sparse_quant = False + +# below will override mlperf rules compliant settings - don't use for official submission +run.target_qps = 5000 +run.numpy_rand_seed = 123 +run.dataset_path_prefix = "/home/linjianma" +run.dataset_percentage = 0.00625 diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/inference_modules.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/inference_modules.py new file mode 100644 index 000000000..cb567df63 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/inference_modules.py @@ -0,0 +1,253 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe +""" +Inference modules for DLRMv3. + +This module provides inference-specific components for the HSTU model, +including sparse inference modules and utilities for moving tensors between devices. +""" + +from typing import Dict, Optional, Tuple + +import torch +import torchrec +from generative_recommenders.modules.dlrm_hstu import ( + DlrmHSTU, + DlrmHSTUConfig, + SequenceEmbedding, +) +from torchrec.modules.embedding_modules import ( + EmbeddingBagCollection, + EmbeddingCollection, +) +from torchrec.sparse.jagged_tensor import JaggedTensor, KeyedJaggedTensor +from torchrec.sparse.tensor_dict import maybe_td_to_kjt + + +IS_INFERENCE: bool = True + + +class _NoCopyEmbeddingCollection(torchrec.EmbeddingCollection): + """ + EmbeddingCollection variant that skips the dtype-cast copy in + ``EmbeddingCollection.forward`` and clamps indices into the hash-size + range. This is the script-mode replacement for the + ``functools.partial`` monkey-patch in + :func:`generative_recommenders.dlrm_v3.inference.model_family.ec_patched_forward_wo_embedding_copy`. + + The body mirrors that helper exactly so that the eager and scripted paths + produce the same embeddings. + """ + + def forward( + self, + features: KeyedJaggedTensor, + ) -> Dict[str, JaggedTensor]: + features = maybe_td_to_kjt(features, None) + feature_embeddings: Dict[str, JaggedTensor] = {} + jt_dict: Dict[str, JaggedTensor] = features.to_dict() + # Inline HASH_SIZE_1B - 1 as a literal so TorchScript can see it; the + # imported module-level constant is treated as an opaque "closed-over + # global" by jit.script and would fail with + # "python value of type 'int' cannot be used as a value". + max_index: int = 999_999_999 # HASH_SIZE_1B - 1 + for i, emb_module in enumerate(self.embeddings.values()): + feature_names = self._feature_names[i] + embedding_names = self._embedding_names_by_table[i] + for j, embedding_name in enumerate(embedding_names): + feature_name = feature_names[j] + f = jt_dict[feature_name] + indices = torch.clamp(f.values(), min=0, max=max_index) + lookup = emb_module(input=indices) + feature_embeddings[embedding_name] = JaggedTensor( + values=lookup, + lengths=f.lengths(), + weights=f.values() if self._need_indices else None, + ) + return feature_embeddings + + +def set_is_inference(is_inference: bool = False) -> None: + """ + Set the global inference mode flag. + + Args: + is_inference: If True, model operates in inference mode (no labels/weights). + If False, model operates in training/eval mode with labels. + """ + global IS_INFERENCE + IS_INFERENCE = is_inference + + +def get_hstu_model( + table_config, + hstu_config: DlrmHSTUConfig, + table_device: str = "meta", + max_hash_size: Optional[int] = None, + is_dense: bool = False, +) -> DlrmHSTU: + """ + Create and initialize an HSTU model for inference. + + Args: + table_config: Dictionary of embedding table configurations. + hstu_config: HSTU model configuration object. + table_device: Device to place embedding tables on ('meta', 'cpu', or 'cuda'). + max_hash_size: Optional maximum hash size to cap embedding table sizes. + is_dense: If True, creates model for dense-only operations. + + Returns: + Initialized DlrmHSTU model in eval mode. + """ + if max_hash_size is not None: + for t in table_config.values(): + t.num_embeddings = ( + max_hash_size if t.num_embeddings > max_hash_size else t.num_embeddings + ) + model = DlrmHSTU( + hstu_configs=hstu_config, + embedding_tables=table_config, + is_inference=IS_INFERENCE, + is_dense=is_dense, + ) + model.eval() + model.recursive_setattr("_use_triton_cc", False) + for _, module in model.named_modules(): + if isinstance(module, EmbeddingBagCollection) or isinstance( + module, EmbeddingCollection + ): + module.to_empty(device=table_device) + # to_empty leaves parameters uninitialized; fill with small random + # values so downstream bf16 ops don't produce NaN from + # uninitialized memory. + for p in module.parameters(): + if not p.is_meta: + torch.nn.init.uniform_(p, -0.01, 0.01) + return model + + +class HSTUSparseInferenceModule(torch.nn.Module): + """ + Module for sparse (embedding) inference operations. + + Handles embedding lookups and preprocessing for the HSTU model, + running on CPU to handle large embedding tables. + + Args: + table_config: Dictionary of embedding table configurations. + hstu_config: HSTU model configuration object. + """ + + def __init__( + self, + table_config, + hstu_config: DlrmHSTUConfig, + ) -> None: + super().__init__() + self._hstu_model: DlrmHSTU = get_hstu_model( + table_config, + hstu_config, + table_device="cpu", + ) + + def forward( + self, + uih_features: KeyedJaggedTensor, + candidates_features: KeyedJaggedTensor, + ) -> Tuple[ + Dict[str, SequenceEmbedding], + Dict[str, torch.Tensor], + int, + torch.Tensor, + int, + torch.Tensor, + ]: + """ + Run sparse preprocessing and embedding lookups. + + Args: + uih_features: User interaction history features as KeyedJaggedTensor. + candidates_features: Candidate item features as KeyedJaggedTensor. + + Returns: + Tuple containing: + - seq_embeddings: Dictionary of sequence embeddings per feature. + - payload_features: Dictionary of payload feature tensors. + - max_uih_len: Maximum user interaction history length. + - uih_seq_lengths: Tensor of UIH sequence lengths per batch item. + - max_num_candidates: Maximum number of candidates. + - num_candidates: Tensor of candidate counts per batch item. + """ + ( + seq_embeddings, + payload_features, + max_uih_len, + uih_seq_lengths, + max_num_candidates, + num_candidates, + ) = self._hstu_model.preprocess( + uih_features=uih_features, + candidates_features=candidates_features, + ) + return ( + seq_embeddings, + payload_features, + max_uih_len, + uih_seq_lengths, + max_num_candidates, + num_candidates, + ) + + +def move_sparse_output_to_device( + seq_embeddings: Dict[str, SequenceEmbedding], + payload_features: Dict[str, torch.Tensor], + uih_seq_lengths: torch.Tensor, + num_candidates: torch.Tensor, + device: torch.device, +) -> Tuple[ + Dict[str, SequenceEmbedding], + Dict[str, torch.Tensor], + torch.Tensor, + torch.Tensor, +]: + """ + Move sparse module outputs from CPU to the target device (typically GPU). + + Converts embeddings to bfloat16 for efficient GPU computation. + + Args: + seq_embeddings: Dictionary of sequence embeddings to move. + payload_features: Dictionary of payload features to move. + uih_seq_lengths: UIH sequence lengths tensor to move. + num_candidates: Number of candidates tensor to move. + device: Target device (e.g., torch.device('cuda:0')). + + Returns: + Tuple of moved tensors on the target device. + """ + num_candidates = num_candidates.to(device) + uih_seq_lengths = uih_seq_lengths.to(device) + seq_embeddings = { + k: SequenceEmbedding( + lengths=seq_embeddings[k].lengths.to(device), + embedding=seq_embeddings[k].embedding.to(device).to(torch.bfloat16), + ) + for k in seq_embeddings.keys() + } + for k, v in payload_features.items(): + payload_features[k] = v.to(device) + return seq_embeddings, payload_features, uih_seq_lengths, num_candidates diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/main.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/main.py new file mode 100644 index 000000000..00e334119 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/main.py @@ -0,0 +1,805 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-strict +""" +mlperf dlrm_v3 inference benchmarking tool. +""" + +import argparse +import array +import logging +import random +import threading + +logging.basicConfig(level=logging.INFO) +import os +import sys +import time +from typing import Any, Dict, List, Optional, Union + +import gin + +# pyre-ignore [21] +import mlperf_loadgen as lg # @manual +import numpy as np +import torch +from generative_recommenders.common import set_dev_mode, set_verbose_level +from generative_recommenders.dlrm_v3.configs import ( + get_embedding_table_config, + get_hstu_configs, +) +from generative_recommenders.dlrm_v3.datasets.dataset import Dataset, Samples +from generative_recommenders.dlrm_v3.datasets.synthetic_streaming import ( + DLRMv3SyntheticStreamingDataset, +) +from generative_recommenders.dlrm_v3.inference.data_producer import ( + MultiThreadDataProducer, + QueryItem, + SingleThreadDataProducer, +) +from generative_recommenders.dlrm_v3.inference.inference_modules import set_is_inference +from generative_recommenders.dlrm_v3.inference.model_family import HSTUModelFamily +from generative_recommenders.dlrm_v3.utils import ( + get_dataset, + profiler_or_nullcontext, + SUPPORTED_DATASETS, +) + + +logger: logging.Logger = logging.getLogger("main") + +torch.multiprocessing.set_start_method("spawn", force=True) + +USER_CONF = f"{os.path.dirname(__file__)}/user.conf" + +SUPPORTED_CONFIGS = { + "debug": "debug.gin", + "kuairand-1k": "kuairand_1k.gin", + "movielens-13b": "movielens_13b.gin", + "streaming-400m": "streaming_400m.gin", + "sampled-streaming-100b": "streaming_100b.gin", +} + + +SCENARIO_MAP = { # pyre-ignore [5] + "Server": lg.TestScenario.Server, + "Offline": lg.TestScenario.Offline, +} + + +def get_args(): # pyre-ignore [3] + """Parse commandline.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--dataset", default="debug", choices=SUPPORTED_DATASETS, help="dataset" + ) + args, unknown_args = parser.parse_known_args() + logger.warning(f"unknown_args: {unknown_args}") + return args + + +class Runner: + """ + Orchestrates inference benchmark execution. + + Manages data production, model inference, and result collection for + MLPerf LoadGen-based benchmarking. + + Args: + model: The HSTU model family instance for making predictions. + ds: Dataset to fetch samples from. + num_queries: Total number of queries to process. + data_producer_threads: Number of threads for data loading (default: 1). + batchsize: Batch size for inference (default: 128). + compute_eval: Whether to compute evaluation metrics (default: False). + """ + + def __init__( + self, + model: HSTUModelFamily, + ds: Dataset, + num_queries: int, + data_producer_threads: int = 1, + batchsize: int = 128, + compute_eval: bool = False, + ) -> None: + self.model = model + if data_producer_threads == 1: + self.data_producer: Union[ + MultiThreadDataProducer, SingleThreadDataProducer + ] = SingleThreadDataProducer(ds, self.run_one_item) + else: + self.data_producer = MultiThreadDataProducer( + ds, data_producer_threads, self.run_one_item + ) + self.batchsize = batchsize + self.compute_eval = compute_eval + self.reset_states(num_queries=num_queries) + + def reset_states(self, num_queries: int) -> None: + """ + Reset all internal state for a new benchmark run. + + Args: + num_queries: Number of queries expected in this run. + """ + self.result_timing: List[Dict[str, float]] = [] + self.result_batches: List[int] = [] + self.current_query_ids: List[int] = [] + self.current_content_ids: List[int] = [] + self.current_t0: List[float] = [] + self.num_queries: int = num_queries + self.processed_queries: int = 0 + + def run_one_item(self, qitem: QueryItem) -> None: + """ + Process a single query item through model inference. + + Runs prediction, records timing metrics, and sends results back to LoadGen. + + Args: + qitem: Query item containing batch of samples to process. + """ + try: + t0_prediction: float = time.time() + prediction_output = self.model.predict(qitem.samples) + dt_prediction: float = time.time() - t0_prediction + assert prediction_output is not None + ( + mt_target_preds, + mt_target_labels, + mt_target_weights, + dt_sparse, + dt_dense, + ) = prediction_output + if self.compute_eval: + assert mt_target_labels is not None + assert mt_target_weights is not None + self.result_timing.append( + { + "total": time.time() - qitem.start, + "prediction": dt_prediction, + "queue": qitem.dt_queue, + "batching": qitem.dt_batching, + "sparse": dt_sparse, + "dense": dt_dense, + } + ) + self.result_batches.append(len(qitem.query_ids)) + except Exception as ex: # pylint: disable=broad-except + logger.error("thread: failed, %s", ex) + finally: + candidate_size = mt_target_preds.size(1) // len(qitem.query_ids) + if not self.compute_eval: + for i, query_id in enumerate(qitem.query_ids): + query_mt_target_preds = ( + mt_target_preds[ # pyre-ignore [61] + 0, + candidate_size * i : candidate_size * (i + 1), + ] + .view(-1) + .float() + .numpy() + ) + response_array = array.array("B", query_mt_target_preds.tobytes()) + bi = response_array.buffer_info() + # since we send buffer to loadgen, needs `response_array` in memory during send + lg.QuerySamplesComplete( + [lg.QuerySampleResponse(query_id, bi[0], bi[1])] + ) + else: + for i, query_id in enumerate(qitem.query_ids): + query_mt_target_preds = ( + mt_target_preds[ # pyre-ignore [61] + 0, candidate_size * i : candidate_size * (i + 1) + ] + .view(-1) + .float() + .numpy() + ) + query_mt_target_labels = ( + mt_target_labels[ # pyre-ignore [16,61] + 0, candidate_size * i : candidate_size * (i + 1) + ] + .view(-1) + .float() + .numpy() + ) + query_mt_target_weights = ( + mt_target_weights[ # pyre-ignore [61] + 0, candidate_size * i : candidate_size * (i + 1) + ] + .view(-1) + .float() + .numpy() + ) + np_array = np.concatenate( + [ + query_mt_target_preds, + query_mt_target_labels, + query_mt_target_weights, + np.array([candidate_size]).astype(np.float32), + ] + ) + response_array = array.array("B", np_array.tobytes()) + bi = response_array.buffer_info() + # since we send buffer to loadgen, needs `response_array` in memory during send + lg.QuerySamplesComplete( + [lg.QuerySampleResponse(query_id, bi[0], bi[1])] + ) + + def enqueue(self, query_samples, t0: float) -> None: # pyre-ignore [2] + """ + Enqueue query samples for batch processing. + + Collects samples until batch size is reached, then dispatches to data producer. + + Args: + query_samples: List of LoadGen query sample objects. + t0: Timestamp when this batch started. + """ + self.current_query_ids.extend([q.id for q in query_samples]) + self.current_content_ids.extend([q.index for q in query_samples]) + self.current_t0.append(t0) + self.processed_queries += len(query_samples) + t0: float = min(self.current_t0) + dt_queue: float = max(self.current_t0) - min(self.current_t0) + if ( + self.processed_queries >= self.num_queries + or len(self.current_query_ids) >= self.batchsize + ): + for i in range(len(self.current_query_ids) // self.batchsize): + self.data_producer.enqueue( + query_ids=self.current_query_ids[ + i * self.batchsize : (i + 1) * self.batchsize + ], + content_ids=self.current_content_ids[ + i * self.batchsize : (i + 1) * self.batchsize + ], + t0=t0, + dt_queue=dt_queue, + ) + remaining_s: int = len(self.current_query_ids) % self.batchsize + if remaining_s > 0: + self.data_producer.enqueue( + query_ids=self.current_query_ids[-remaining_s:], + content_ids=self.current_content_ids[-remaining_s:], + t0=t0, + dt_queue=dt_queue, + ) + self.current_query_ids = [] + self.current_content_ids = [] + self.current_t0 = [] + + def finish(self) -> None: + """Signal data producer to finish and wait for completion.""" + self.data_producer.finish() + + +def add_results( + final_results: Dict[str, Any], + result_timing: List[Dict[str, float]], + result_batches: List[int], +) -> None: + """ + Aggregate and log benchmark results. + + Computes percentile statistics and QPS metrics from timing data. + + Args: + final_results: Dictionary to populate with aggregated results. + result_timing: List of timing dictionaries for each batch. + result_batches: List of batch sizes processed. + """ + percentiles: list[float] = [50.0, 80.0, 90.0, 95.0, 99.0, 99.9] + buckets_dict: Dict[str, List[float]] = {} + buckets_str_dict: Dict[str, str] = {} + total_timing: list[float] = [result["total"] for result in result_timing] + for key in ["total", "prediction", "queue", "batching", "sparse", "dense"]: + timing: list[float] = [result[key] for result in result_timing] + buckets: List[float] = np.percentile(timing, percentiles).tolist() + buckets_str: str = ",".join( + ["| {}:{:.4f}| ".format(p, b) for p, b in zip(percentiles, buckets)] + ) + buckets_dict[key] = buckets + buckets_str_dict[key] = buckets_str + total_batches = sum(result_batches) + + final_results["good"] = len(total_timing) + final_results["avg_time"] = np.mean(total_timing) + final_results["percentiles"] = { + str(k): v for k, v in zip(percentiles, buckets_dict["total"]) + } + final_results["qps"] = total_batches / final_results["took"] + final_results["count"] = total_batches + + for i, timing in enumerate(result_timing): + logger.warning(f"timing of {i}: {timing}") + + logger.warning( + "{} qps={:.2f}, avg_query_time={:.4f}, time={:.3f}, queries={}, tiles={}".format( + final_results["scenario"], + final_results["qps"], + final_results["avg_time"], + final_results["took"], + len(result_timing), + buckets_str_dict["total"], + ) + ) + for key in ["prediction", "queue", "batching", "sparse", "dense"]: + logger.warning(f"{key}: {buckets_str_dict[key]}") + + +def get_num_queries( + input_size: Optional[int], + one_pass_size: int, + scenario_name: str, + offline_target_qps: int, + target_duration: float, +) -> int: + """ + Determine the number of queries to run based on scenario and settings. + + Args: + input_size: User-specified query count (None to use defaults). + one_pass_size: Size of one complete pass through the dataset. + scenario_name: MLPerf scenario name ('Server' or 'Offline'). + offline_target_qps: Target QPS for offline scenario. + target_duration: Target duration in milliseconds. + + Returns: + Number of queries to execute in the benchmark run. + """ + if scenario_name == "Offline": + # consistent with https://github.com/mlcommons/inference/blob/8999c4d686f6e4a180da14597c97063fce7c9f33/loadgen/test_settings_internal.cc#L147 + return int(1.1 * target_duration / 1000 * offline_target_qps) + else: + if input_size is None: + return one_pass_size + return input_size + + +class StreamingQuerySampler: + """ + Sampler for streaming dataset + The execution order is determined by `StreamingQuerySampler.run_order`, not by the QSL or input query ID. + This ensures that queries are executed according to their timestamp constraints. + """ + + def __init__( + self, + ds: DLRMv3SyntheticStreamingDataset, + dataset_percentage: float, + scenario_name: str, + offline_target_qps: int, + target_duration: float, + input_queries: Optional[int] = None, + compute_eval: bool = False, + ) -> None: + self.ds: DLRMv3SyntheticStreamingDataset = ds + self.ds.is_inference = True + self.inference_ts: int = self.ds.total_ts - self.ds.train_ts + self.start_ts: int = self.ds.train_ts + self.dataset_percentage: float = dataset_percentage + self.num_unique_requests: List[int] = self.get_num_unique_requests( + warmup_ratio=1.0 + ) + self.num_unique_requests_cumsum: List[int] = np.cumsum( + self.num_unique_requests + ).tolist() + self.total_requests: int = sum(self.num_unique_requests) + self.run_order: List[List[int]] = self.build_random_exec_order() + self.ts_idx: int = 0 + self.ts_processed_cnt: int = 0 + self.last_loaded: float = -1.0 + num_queries: int = get_num_queries( + input_size=input_queries, + one_pass_size=self.total_requests, + scenario_name=scenario_name, + offline_target_qps=offline_target_qps, + target_duration=target_duration, + ) + logger.warning( + f"StreamingQuerySampler constructred to handle {num_queries} queries" + ) + self.num_repeats: int = ( + max(1, num_queries // self.total_requests) if not compute_eval else 1 + ) + self.remaining_queries: int = ( + num_queries % self.total_requests if not compute_eval else 0 + ) + self._lock = threading.Lock() + + def get_num_unique_requests(self, warmup_ratio: float) -> List[int]: + """ + Calculate number of unique requests per timestamp. + + Args: + warmup_ratio: Fraction of users to include in warmup. + + Returns: + List of request counts per timestamp. + """ + num_unique_requests = [ + int( + self.ds.ts_to_users_cumsum[t][-1] + * self.dataset_percentage + * warmup_ratio + ) + for t in range(self.start_ts, self.start_ts + self.inference_ts) + ] + return num_unique_requests + + def build_random_exec_order(self) -> List[List[int]]: + """ + Build randomized execution order for each timestamp. + + Returns: + List of shuffled index lists, one per timestamp. + """ + order = [] + for req_size in self.num_unique_requests: + within_ts_order = list(range(req_size)) + random.shuffle(within_ts_order) + order.append(within_ts_order) + return order + + def init_sut(self) -> None: + """Initialize System Under Test state for a new benchmark run.""" + self.ts_idx = 0 + self.ts_processed_cnt = 0 + self.ds.set_ts(self.start_ts) + + def load_query_samples(self, query_ids: List[Optional[int]]) -> None: + """ + Load query samples into memory for the benchmark. + + Args: + query_ids: List of query identifiers to load. + """ + length = len(query_ids) + ts_idx: int = 0 + while self.num_unique_requests_cumsum[ts_idx] < length: + ts_idx += 1 + for i in range(0, ts_idx): + self.ds.set_ts(i + self.start_ts) + self.ds.load_query_samples(self.run_order[i]) + self.ds.set_ts(ts_idx + self.start_ts) + delta_length = ( + length + if ts_idx == 0 + else length - self.num_unique_requests_cumsum[ts_idx - 1] + ) + self.ds.load_query_samples(self.run_order[ts_idx][:delta_length]) + self.init_sut() + self.last_loaded = time.time() + + def unload_query_samples(self, sample_list: List[int]) -> None: + """ + Unload query samples from memory. + + Args: + sample_list: List of sample identifiers to unload. + """ + self.ds.unload_query_samples(sample_list) + + def get_samples(self, id_list: List[int]) -> List[Samples]: + """ + Get samples for a batch of queries, handling timestamp boundaries. + + Args: + id_list: List of query identifiers. + + Returns: + List of Samples objects, potentially spanning multiple timestamps. + """ + batch_size: int = len(id_list) + with self._lock: + curr_ts_idx: int = self.ts_idx + curr_ts_unique_requests: int = self.num_unique_requests[curr_ts_idx] + curr_ts_queries: int = curr_ts_unique_requests * self.num_repeats + if curr_ts_idx == self.inference_ts - 1: + curr_ts_queries += self.remaining_queries + begin_query_idx: int = self.ts_processed_cnt + end_query_idx: int = min(begin_query_idx + batch_size, curr_ts_queries) + begin_request_idx: int = begin_query_idx % curr_ts_unique_requests + end_request_idx: int = end_query_idx % curr_ts_unique_requests + if begin_query_idx + batch_size >= curr_ts_queries: + self.ts_idx += 1 + self.ts_processed_cnt = begin_query_idx + batch_size - curr_ts_queries + else: + self.ts_processed_cnt = begin_query_idx + batch_size + # requests of current ts + outputs: List[Samples] = [] + if end_request_idx > begin_request_idx: + output: Samples = self.ds.get_samples_with_ts( + self.run_order[curr_ts_idx][begin_request_idx:end_request_idx], + curr_ts_idx + self.start_ts, + ) + outputs.append(output) + else: + if begin_request_idx < curr_ts_unique_requests: + output: Samples = self.ds.get_samples_with_ts( + self.run_order[curr_ts_idx][begin_request_idx:], + curr_ts_idx + self.start_ts, + ) + outputs.append(output) + if end_request_idx > 0: + output = self.ds.get_samples_with_ts( + self.run_order[curr_ts_idx][0:end_request_idx], + curr_ts_idx + self.start_ts, + ) + outputs.append(output) + # requests of next ts + if begin_query_idx + batch_size > curr_ts_queries: + output: Samples = self.ds.get_samples_with_ts( + self.run_order[curr_ts_idx + 1][ + : begin_query_idx + batch_size - curr_ts_queries + ], + curr_ts_idx + 1 + self.start_ts, + ) + outputs.append(output) + return outputs + + def get_item_count(self) -> int: + """ + Get total number of items in the dataset. + + Returns: + Total request count across all timestamps. + """ + return self.total_requests + + +@gin.configurable +def run( + dataset: str = "sampled-streaming-100b", + model_path: str = "", + scenario_name: str = "Server", + batchsize: int = 16, + output_trace: bool = False, + data_producer_threads: int = 4, + compute_eval: bool = False, + find_peak_performance: bool = False, + dataset_path_prefix: str = "", + train_split_percentage: float = 0.75, + warmup_ratio: float = 0.1, + target_qps: Optional[int] = None, + num_queries: Optional[int] = None, + numpy_rand_seed: int = 123, + sparse_quant: bool = False, + dataset_percentage: float = 1.0, +) -> None: + """ + Execute the MLPerf DLRMv3 inference benchmark. + + Sets up the model, dataset, and LoadGen infrastructure, then runs + warmup and official benchmark phases. + + Args: + dataset: Dataset identifier to use. + model_path: Path to model checkpoint directory. + scenario_name: MLPerf scenario ('Server' or 'Offline'). + batchsize: Batch size for inference. + output_trace: Whether to output profiling traces. + data_producer_threads: Number of data loading threads. + compute_eval: Whether to compute accuracy metrics. + find_peak_performance: Whether to run peak performance finding mode. + dataset_path_prefix: Prefix path for dataset files. + warmup_ratio: Fraction of data to use for warmup. + target_qps: Target queries per second. + num_queries: Number of queries to run (None for automatic). + numpy_rand_seed: Random seed for reproducibility. + sparse_quant: Whether to quantize sparse embeddings. + dataset_percentage: Fraction of dataset to use. + """ + set_dev_mode(False) + if scenario_name not in SCENARIO_MAP: + raise NotImplementedError("valid scanarios:" + str(list(SCENARIO_MAP.keys()))) + scenario = SCENARIO_MAP[scenario_name] + np.random.seed(numpy_rand_seed) + random.seed(numpy_rand_seed) + + hstu_config = get_hstu_configs(dataset) + hstu_config.max_num_candidates = hstu_config.max_num_candidates_inference + table_config = get_embedding_table_config(dataset) + set_is_inference(is_inference=not compute_eval) + + user_conf = os.path.abspath(USER_CONF) + if not os.path.exists(user_conf): + logger.error("{} not found".format(user_conf)) + sys.exit(1) + + settings = lg.TestSettings() + settings.FromConfig(user_conf, model_path, scenario_name) + settings.scenario = scenario + settings.mode = lg.TestMode.PerformanceOnly + if compute_eval: + settings.mode = lg.TestMode.AccuracyOnly + if find_peak_performance: + settings.mode = lg.TestMode.FindPeakPerformance + if target_qps: + settings.server_target_qps = float(target_qps) + settings.offline_expected_qps = float(target_qps) + + model_family = HSTUModelFamily( + hstu_config=hstu_config, + table_config=table_config, + sparse_quant=sparse_quant, + output_trace=output_trace, + compute_eval=compute_eval, + ) + is_streaming: bool = "streaming" in dataset + dataset, kwargs = get_dataset(dataset, dataset_path_prefix) + + ds: Dataset = dataset( + hstu_config=hstu_config, + embedding_config=table_config, + is_inference=not compute_eval, + **kwargs, + ) + if is_streaming: + ds = StreamingQuerySampler( # pyre-ignore + ds=ds, # pyre-ignore [6] + dataset_percentage=dataset_percentage, + input_queries=num_queries, + compute_eval=compute_eval, + scenario_name=scenario_name, + offline_target_qps=settings.offline_expected_qps, + target_duration=settings.min_duration_ms, + ) + model_family.load(model_path) + + # warmup + for autotune_bs in range(batchsize, 0, -1): + logger.warning(f"Autotune for batch size {autotune_bs}") + warmup_ids = list(range(autotune_bs)) + ds.load_query_samples(warmup_ids) + for _ in range(4 * int(os.environ.get("WORLD_SIZE", 1))): + if is_streaming: + ds.init_sut() # pyre-ignore [16] + sample: Union[Samples, List[Samples]] = ds.get_samples(warmup_ids) + if isinstance(sample, Samples): + model_family.predict(sample) + else: + for s in sample: + model_family.predict(s) + ds.unload_query_samples(None) + for h in logger.handlers: + h.flush() + logger.info("Model forward warmup done") + + count = int( + ds.get_item_count() * dataset_percentage + if not is_streaming + else ds.get_item_count() + ) + train_size: int = round(train_split_percentage * count) if not is_streaming else 0 + if compute_eval: + count = count - train_size + + runner: Runner = Runner( + model_family, + ds, + data_producer_threads=data_producer_threads, + batchsize=batchsize, + compute_eval=compute_eval, + num_queries=count, + ) + + def issue_queries(query_samples) -> None: # pyre-ignore [2] + if compute_eval: + for sample in query_samples: + sample.index = sample.index + train_size + runner.enqueue(query_samples, time.time()) + + def load_query_samples(query_ids: List[int]) -> None: + if compute_eval: + query_ids = [q + train_size for q in query_ids] + ds.load_query_samples(query_ids) + + def flush_queries() -> None: + pass + + if scenario == lg.TestScenario.Server: + # inference benchmark warmup + if is_streaming: + ds.init_sut() + warmup_count: int = sum( + ds.get_num_unique_requests( # pyre-ignore [16] + warmup_ratio=warmup_ratio + ) + ) + else: + warmup_count: int = int(count * warmup_ratio) + runner.reset_states(num_queries=warmup_count) + final_results = { + "runtime": model_family.name(), + "version": model_family.version(), + "time": int(time.time()), + "scenario": str(scenario), + } + settings.min_query_count = warmup_count + settings.max_query_count = warmup_count + sut = lg.ConstructSUT(issue_queries, flush_queries) + qsl = lg.ConstructQSL( + warmup_count, + warmup_count, + load_query_samples, + ds.unload_query_samples, + ) + with profiler_or_nullcontext(enabled=output_trace, with_stack=False): + logger.info(f"starting warmup {scenario} with {warmup_count} queries") + lg.StartTest(sut, qsl, settings) + lg.DestroyQSL(qsl) + lg.DestroySUT(sut) + + # official run + if is_streaming: + ds.init_sut() + final_results = { + "runtime": model_family.name(), + "version": model_family.version(), + "time": int(time.time()), + "scenario": str(scenario), + } + query_size: int = get_num_queries( + input_size=num_queries, + one_pass_size=count, + scenario_name=scenario_name, + offline_target_qps=settings.offline_expected_qps, + target_duration=settings.min_duration_ms, + ) + settings.min_query_count = query_size + settings.max_query_count = query_size + runner.reset_states(num_queries=query_size if not compute_eval else count) + sut = lg.ConstructSUT(issue_queries, flush_queries) + qsl = lg.ConstructQSL( + count, + count, + load_query_samples, + ds.unload_query_samples, + ) + with profiler_or_nullcontext(enabled=output_trace, with_stack=False): + logger.info( + f"starting {scenario} with {query_size} queries and {query_size // count} repeats" + ) + lg.StartTest(sut, qsl, settings) + runner.finish() + final_results["took"] = time.time() - ds.last_loaded + lg.DestroyQSL(qsl) + lg.DestroySUT(sut) + + add_results( + final_results, + runner.result_timing, + runner.result_batches, + ) + # If multiple subprocesses are running the model send a signal to stop them + if int(os.environ.get("WORLD_SIZE", 1)) > 1: + model_family.predict(None) + + +def main() -> None: + set_verbose_level(1) + args = get_args() + logger.info(args) + gin_path = f"{os.path.dirname(__file__)}/gin/{SUPPORTED_CONFIGS[args.dataset]}" + gin.parse_config_file(gin_path) + run(dataset=args.dataset) + + +if __name__ == "__main__": + main() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/mlperf.conf b/recommendation_v4/generative_recommenders/dlrm_v3/inference/mlperf.conf new file mode 100644 index 000000000..a2b4f6fff --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/mlperf.conf @@ -0,0 +1,98 @@ +# The format of this config file is 'key = value'. +# The key has the format 'model.scenario.key'. Value is mostly int64_t. +# Model maybe '*' as wildcard. In that case the value applies to all models. +# All times are in milli seconds + +# Set performance_sample_count for each model. +# User can optionally set this to higher values in user.conf. +resnet50.*.performance_sample_count_override = 1024 +ssd-mobilenet.*.performance_sample_count_override = 256 +retinanet.*.performance_sample_count_override = 64 +bert.*.performance_sample_count_override = 10833 +dlrm.*.performance_sample_count_override = 204800 +dlrm-v2.*.performance_sample_count_override = 204800 +rnnt.*.performance_sample_count_override = 2513 +gptj.*.performance_sample_count_override = 13368 +llama2-70b.*.performance_sample_count_override = 24576 +stable-diffusion-xl.*.performance_sample_count_override = 5000 +# set to 0 to let entire sample set to be performance sample +3d-unet.*.performance_sample_count_override = 0 + +# Set seeds. The seeds will be distributed two weeks before the submission. +*.*.qsl_rng_seed = 3066443479025735752 +*.*.sample_index_rng_seed = 10688027786191513374 +*.*.schedule_rng_seed = 14962580496156340209 +# Set seeds for TEST_05. The seeds will be distributed two weeks before the submission. +*.*.test05_qsl_rng_seed = 16799458546791641818 +*.*.test05_sample_index_rng_seed = 5453809927556429288 +*.*.test05_schedule_rng_seed = 5435552105434836064 + + +*.SingleStream.target_latency_percentile = 90 +*.SingleStream.min_duration = 600000 + +*.MultiStream.target_latency_percentile = 99 +*.MultiStream.samples_per_query = 8 +*.MultiStream.min_duration = 600000 +*.MultiStream.min_query_count = 662 +retinanet.MultiStream.target_latency = 528 + +# 3D-UNet uses equal issue mode because it has non-uniform inputs +3d-unet.*.sample_concatenate_permutation = 1 + +# LLM benchmarks have non-uniform inputs and outputs, and use equal issue mode for all latency scenario +gptj.*.sample_concatenate_permutation = 1 +llama2-70b.*.sample_concatenate_permutation = 1 +mixtral-8x7b.*.sample_concatenate_permutation = 1 + +*.Server.target_latency = 10 +*.Server.target_latency_percentile = 99 +*.Server.target_duration = 0 +*.Server.min_duration = 600000 +resnet50.Server.target_latency = 15 +retinanet.Server.target_latency = 100 +bert.Server.target_latency = 130 +dlrm.Server.target_latency = 60 +dlrm-v2.Server.target_latency = 60 +rnnt.Server.target_latency = 1000 +gptj.Server.target_latency = 20000 +stable-diffusion-xl.Server.target_latency = 20000 +# Llama2-70b benchmarks measures token latencies +llama2-70b.*.use_token_latencies = 1 +mixtral-8x7b.*.use_token_latencies = 1 +# gptj benchmark infers token latencies +gptj.*.infer_token_latencies = 1 +gptj.*.token_latency_scaling_factor = 69 +# Only ttft and tpot are tracked for the llama2-70b & mixtral-8x7B benchmark therefore target_latency = 0 +llama2-70b.Server.target_latency = 0 +llama2-70b.Server.ttft_latency = 2000 +llama2-70b.Server.tpot_latency = 200 + +mixtral-8x7b.Server.target_latency = 0 +mixtral-8x7b.Server.ttft_latency = 2000 +mixtral-8x7b.Server.tpot_latency = 200 + +*.Offline.target_latency_percentile = 90 +*.Offline.min_duration = 600000 + +# In Offline scenario, we always have one query. But LoadGen maps this to +# min_sample_count internally in Offline scenario. If the dataset size is larger +# than 24576 we limit the min_query_count to 24576 and otherwise we use +# the dataset size as the limit + +resnet50.Offline.min_query_count = 24576 +retinanet.Offline.min_query_count = 24576 +dlrm-v2.Offline.min_query_count = 24576 +bert.Offline.min_query_count = 10833 +gptj.Offline.min_query_count = 13368 +rnnt.Offline.min_query_count = 2513 +3d-unet.Offline.min_query_count = 43 +stable-diffusion-xl.Offline.min_query_count = 5000 +llama2-70b.Offline.min_query_count = 24576 +mixtral-8x7b.Offline.min_query_count = 15000 + +# These fields should be defined and overridden by user.conf. +*.SingleStream.target_latency = 10 +*.MultiStream.target_latency = 80 +*.Server.target_qps = 1.0 +*.Offline.target_qps = 1.0 diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/model_family.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/model_family.py new file mode 100644 index 000000000..1c8bcd237 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/model_family.py @@ -0,0 +1,705 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-strict +""" +model_family for dlrm_v3. +""" + +import copy +import functools +import logging +import os +import time +import uuid +from threading import Event +from typing import Dict, List, Optional, Tuple, Union + +import torch +import torch.multiprocessing as mp +import torchrec +from generative_recommenders.dlrm_v3.checkpoint import ( + load_nonsparse_checkpoint, + load_sparse_checkpoint, +) +from generative_recommenders.dlrm_v3.configs import HASH_SIZE_1B +from generative_recommenders.dlrm_v3.datasets.dataset import Samples +from generative_recommenders.dlrm_v3.inference.inference_modules import ( + get_hstu_model, + HSTUSparseInferenceModule, + move_sparse_output_to_device, + set_is_inference, +) +from generative_recommenders.dlrm_v3.utils import Profiler +from generative_recommenders.modules.dlrm_hstu import DlrmHSTUConfig, SequenceEmbedding +from pyre_extensions import none_throws +from torch import quantization as quant +from torchrec.distributed.quant_embedding import QuantEmbeddingCollection +from torchrec.modules.embedding_configs import EmbeddingConfig, QuantConfig +from torchrec.sparse.jagged_tensor import JaggedTensor, KeyedJaggedTensor +from torchrec.sparse.tensor_dict import maybe_td_to_kjt +from torchrec.test_utils import get_free_port + +logger: logging.Logger = logging.getLogger(__name__) + + +class HSTUModelFamily: + """ + High-level interface for the HSTU model family. + + Manages both sparse (embedding) and dense (transformer) components of the + HSTU model, supporting distributed inference across multiple GPUs. + + Args: + hstu_config: Configuration object for the HSTU model. + table_config: Dictionary of embedding table configurations. + output_trace: Whether to enable profiling trace output. + sparse_quant: Whether to quantize sparse embeddings. + compute_eval: Whether to compute evaluation metrics (includes labels). + """ + + def __init__( + self, + hstu_config: DlrmHSTUConfig, + table_config: Dict[str, EmbeddingConfig], + output_trace: bool = False, + sparse_quant: bool = False, + compute_eval: bool = False, + ) -> None: + self.hstu_config = hstu_config + self.table_config = table_config + self.sparse: ModelFamilySparseDist = ModelFamilySparseDist( + hstu_config=hstu_config, + table_config=table_config, + quant=sparse_quant, + ) + + assert torch.cuda.is_available(), "CUDA is required for this benchmark." + ngpus = torch.cuda.device_count() + self.world_size = int(os.environ.get("WORLD_SIZE", str(ngpus))) + logger.warning(f"Using {self.world_size} GPU(s)...") + dense_model_family_clazz = ( + ModelFamilyDenseDist + if self.world_size > 1 + else ModelFamilyDenseSingleWorker + ) + self.dense: Union[ModelFamilyDenseDist, ModelFamilyDenseSingleWorker] = ( + dense_model_family_clazz( + hstu_config=hstu_config, + table_config=table_config, + output_trace=output_trace, + compute_eval=compute_eval, + ) + ) + + def version(self) -> str: + """Return the PyTorch version string.""" + return torch.__version__ + + def name(self) -> str: + """Return the model family name identifier.""" + return "model-family-hstu" + + def load(self, model_path: str) -> None: + """ + Load model checkpoints from disk. + + Args: + model_path: Base path to the model checkpoint directory. + """ + self.sparse.load(model_path=model_path) + self.dense.load(model_path=model_path) + + def predict( + self, samples: Optional[Samples] + ) -> Optional[ + Tuple[ + torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor], float, float + ] + ]: + """ + Run inference on a batch of samples. + + Processes samples through sparse embeddings, then dense forward pass. + + Args: + samples: Input samples containing features. If None, signals shutdown. + + Returns: + Tuple of (predictions, labels, weights, sparse_time, dense_time) or None. + """ + with torch.no_grad(): + if samples is None: + self.dense.predict(None, None, 0, None, 0, None) + return None + ( + seq_embeddings, + payload_features, + max_uih_len, + uih_seq_lengths, + max_num_candidates, + num_candidates, + dt_sparse, + ) = self.sparse.predict(samples) + out = self.dense.predict( + seq_embeddings, + payload_features, + max_uih_len, + uih_seq_lengths, + max_num_candidates, + num_candidates, + ) + ( # pyre-ignore [23] + mt_target_preds, + mt_target_labels, + mt_target_weights, + dt_dense, + ) = out + return ( + mt_target_preds, + mt_target_labels, + mt_target_weights, + dt_sparse, + dt_dense, + ) + + +def ec_patched_forward_wo_embedding_copy( + ec_module: torchrec.EmbeddingCollection, + features: KeyedJaggedTensor, # can also take TensorDict as input +) -> Dict[str, JaggedTensor]: + """ + Run the EmbeddingBagCollection forward pass. This method takes in a `KeyedJaggedTensor` + and returns a `Dict[str, JaggedTensor]`, which is the result of the individual embeddings for each feature. + + Args: + features (KeyedJaggedTensor): KJT of form [F X B X L]. + + Returns: + Dict[str, JaggedTensor] + """ + features = maybe_td_to_kjt(features, None) + feature_embeddings: Dict[str, JaggedTensor] = {} + jt_dict: Dict[str, JaggedTensor] = features.to_dict() + for i, emb_module in enumerate(ec_module.embeddings.values()): + feature_names = ec_module._feature_names[i] + embedding_names = ec_module._embedding_names_by_table[i] + for j, embedding_name in enumerate(embedding_names): + feature_name = feature_names[j] + f = jt_dict[feature_name] + indices = torch.clamp(f.values(), min=0, max=HASH_SIZE_1B - 1) + lookup = emb_module( + input=indices + ) # remove the dtype cast at https://github.com/meta-pytorch/torchrec/blob/0a2cebd5472a7edc5072b3c912ad8aaa4179b9d9/torchrec/modules/embedding_modules.py#L486 + feature_embeddings[embedding_name] = JaggedTensor( + values=lookup, + lengths=f.lengths(), + weights=f.values() if ec_module._need_indices else None, + ) + return feature_embeddings + + +class ModelFamilySparseDist: + """ + Sparse Arch module manager. + + Handles loading and inference of sparse embedding lookups, optionally + with quantization for memory efficiency. + + Args: + hstu_config: HSTU model configuration. + table_config: Embedding table configurations. + quant: Whether to apply dynamic quantization to embeddings. + """ + + def __init__( + self, + hstu_config: DlrmHSTUConfig, + table_config: Dict[str, EmbeddingConfig], + quant: bool = False, + ) -> None: + super(ModelFamilySparseDist, self).__init__() + self.hstu_config = hstu_config + self.table_config = table_config + self.module: Optional[torch.nn.Module] = None + self.quant: bool = quant + + def load(self, model_path: str) -> None: + """ + Load sparse model checkpoint and optionally apply quantization. + + Args: + model_path: Path to the model checkpoint directory. + """ + logger.warning(f"Loading sparse module from {model_path}") + + sparse_arch: HSTUSparseInferenceModule = HSTUSparseInferenceModule( + table_config=self.table_config, + hstu_config=self.hstu_config, + ) + load_sparse_checkpoint(model=sparse_arch._hstu_model, path=model_path) + sparse_arch.eval() + if self.quant: + self.module = quant.quantize_dynamic( + sparse_arch, + qconfig_spec={ + torchrec.EmbeddingCollection: QuantConfig( + activation=quant.PlaceholderObserver.with_args( + dtype=torch.float + ), + weight=quant.PlaceholderObserver.with_args(dtype=torch.int8), + ), + }, + mapping={ + torchrec.EmbeddingCollection: QuantEmbeddingCollection, + }, + inplace=False, + ) + else: + sparse_arch._hstu_model._embedding_collection.forward = ( # pyre-ignore[8] + functools.partial( + ec_patched_forward_wo_embedding_copy, + sparse_arch._hstu_model._embedding_collection, + ) + ) + self.module = sparse_arch + logger.warning(f"sparse module is {self.module}") + + def predict( + self, samples: Samples + ) -> Tuple[ + Dict[str, SequenceEmbedding], + Dict[str, torch.Tensor], + int, + torch.Tensor, + int, + torch.Tensor, + float, + ]: + """ + Run sparse forward pass (embedding lookups). + + Args: + samples: Input samples with feature tensors. + + Returns: + Tuple of (seq_embeddings, payload_features, max_uih_len, uih_seq_lengths, + max_num_candidates, num_candidates, elapsed_time). + """ + with torch.profiler.record_function("sparse forward"): + module: torch.nn.Module = none_throws(self.module) + assert self.module is not None + uih_features = samples.uih_features_kjt + candidates_features = samples.candidates_features_kjt + t0: float = time.time() + ( + seq_embeddings, + payload_features, + max_uih_len, + uih_seq_lengths, + max_num_candidates, + num_candidates, + ) = module( + uih_features=uih_features, + candidates_features=candidates_features, + ) + dt_sparse: float = time.time() - t0 + return ( + seq_embeddings, + payload_features, + max_uih_len, + uih_seq_lengths, + max_num_candidates, + num_candidates, + dt_sparse, + ) + + +class ModelFamilyDenseDist: + """ + Distributed dense module manager for multi-GPU inference. + + Spawns worker processes for each GPU to run dense forward passes in parallel, + with samples distributed via inter-process queues. + + Args: + hstu_config: HSTU model configuration. + table_config: Embedding table configurations. + output_trace: Whether to enable profiling traces. + compute_eval: Whether to compute evaluation metrics. + """ + + def __init__( + self, + hstu_config: DlrmHSTUConfig, + table_config: Dict[str, EmbeddingConfig], + output_trace: bool = False, + compute_eval: bool = False, + ) -> None: + super(ModelFamilyDenseDist, self).__init__() + self.hstu_config = hstu_config + self.table_config = table_config + self.output_trace = output_trace + self.compute_eval = compute_eval + + ngpus = torch.cuda.device_count() + self.world_size = int(os.environ.get("WORLD_SIZE", str(ngpus))) + self.rank = 0 + os.environ["MASTER_ADDR"] = "localhost" + os.environ["MASTER_PORT"] = str(get_free_port()) + self.dist_backend = "nccl" + + ctx = mp.get_context("spawn") + self.samples_q: List[mp.Queue] = [ctx.Queue() for _ in range(self.world_size)] + self.result_q: List[mp.Queue] = [ctx.Queue() for _ in range(self.world_size)] + + def load(self, model_path: str) -> None: + """ + Load dense model and spawn worker processes for distributed inference. + + Args: + model_path: Path to the model checkpoint directory. + """ + logger.warning(f"Loading dense module from {model_path}") + + ctx = mp.get_context("spawn") + processes = [] + for rank in range(self.world_size): + p = ctx.Process( + target=self.distributed_setup, + args=( + rank, + self.world_size, + model_path, + ), + ) + p.start() + processes.append(p) + + def distributed_setup(self, rank: int, world_size: int, model_path: str) -> None: + """ + Initialize and run a dense worker process. + + Each worker loads the model, processes samples from its queue, and + returns results. + + Args: + rank: Process rank (GPU index). + world_size: Total number of worker processes. + model_path: Path to model checkpoint. + """ + nprocs_per_rank = 16 + start_core: int = nprocs_per_rank * rank + cores: set[int] = set([start_core + i for i in range(nprocs_per_rank)]) + os.sched_setaffinity(0, cores) + set_is_inference(is_inference=not self.compute_eval) + model = get_hstu_model( + table_config=self.table_config, + hstu_config=self.hstu_config, + table_device="cpu", + max_hash_size=100, + is_dense=True, + ).to(torch.bfloat16) + model.set_training_dtype(torch.bfloat16) + device = torch.device(f"cuda:{rank}") + torch.cuda.set_device(f"cuda:{rank}") + load_nonsparse_checkpoint( + model=model, device=device, optimizer=None, path=model_path + ) + model = model.to(device) + model.eval() + profiler = Profiler(rank) if self.output_trace else None + + with torch.no_grad(): + while True: + item = self.samples_q[rank].get() + # If -1 is received terminate all subprocesses + if item == -1: + break + if self.output_trace: + assert profiler is not None + profiler.step() + with torch.profiler.record_function("get_item_from_queue"): + # Copy here to release data in the producer to avoid invalid cuda caching allocator release. + item = copy.deepcopy(item) + ( + id, + seq_embeddings, + payload_features, + max_uih_len, + uih_seq_lengths, + max_num_candidates, + num_candidates, + ) = item + assert seq_embeddings is not None + with torch.profiler.record_function("dense forward"): + ( + _, + _, + _, + mt_target_preds, + mt_target_labels, + mt_target_weights, + ) = model.main_forward( + seq_embeddings=seq_embeddings, + payload_features=payload_features, + max_uih_len=max_uih_len, + uih_seq_lengths=uih_seq_lengths, + max_num_candidates=max_num_candidates, + num_candidates=num_candidates, + ) + # mt_target_preds = torch.empty(1, 2048 * 20).to(device="cpu") + # mt_target_labels = None + # mt_target_weights = None + assert mt_target_preds is not None + mt_target_preds = mt_target_preds.detach().to(device="cpu") + if mt_target_labels is not None: + mt_target_labels = mt_target_labels.detach().to(device="cpu") + if mt_target_weights is not None: + mt_target_weights = mt_target_weights.detach().to(device="cpu") + self.result_q[rank].put( + (id, mt_target_preds, mt_target_labels, mt_target_weights) + ) + + def capture_output( + self, id: uuid.UUID, rank: int + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor]]: + """ + Retrieve inference results from a worker process. + + Args: + id: Unique identifier for the request. + rank: Worker rank to retrieve from. + + Returns: + Tuple of (predictions, labels, weights). + """ + while True: + recv_id, preds, labels, weights = self.result_q[rank].get() + assert recv_id == id + return preds, labels, weights + + def get_rank(self) -> int: + """ + Get the next worker rank for load balancing. + + Returns: + Rank index, cycling through available workers. + """ + rank = self.rank + self.rank = (self.rank + 1) % self.world_size + return rank + + def predict( + self, + seq_embeddings: Optional[Dict[str, SequenceEmbedding]], + payload_features: Optional[Dict[str, torch.Tensor]], + max_uih_len: int, + uih_seq_lengths: Optional[torch.Tensor], + max_num_candidates: int, + num_candidates: Optional[torch.Tensor], + ) -> Optional[ + Tuple[torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor], float] + ]: + """ + Run distributed dense forward pass. + + Dispatches work to a worker process and collects results. + + Args: + seq_embeddings: Sequence embeddings from sparse module. + payload_features: Additional feature tensors. + max_uih_len: Maximum UIH sequence length. + uih_seq_lengths: Per-sample UIH lengths. + max_num_candidates: Maximum candidates per sample. + num_candidates: Per-sample candidate counts. + + Returns: + Tuple of (predictions, labels, weights, elapsed_time) or None if shutdown. + """ + id = uuid.uuid4() + # If none is received terminate all subprocesses + if seq_embeddings is None: + for rank in range(self.world_size): + self.samples_q[rank].put(-1) + return None + rank = self.get_rank() + device = torch.device(f"cuda:{rank}") + assert ( + payload_features is not None + and num_candidates is not None + and uih_seq_lengths is not None + ) + t0: float = time.time() + seq_embeddings, payload_features, uih_seq_lengths, num_candidates = ( + move_sparse_output_to_device( + seq_embeddings=seq_embeddings, + payload_features=payload_features, + uih_seq_lengths=uih_seq_lengths, + num_candidates=num_candidates, + device=device, + ) + ) + self.samples_q[rank].put( + ( + id, + seq_embeddings, + payload_features, + max_uih_len, + uih_seq_lengths, + max_num_candidates, + num_candidates, + ) + ) + (mt_target_preds, mt_target_labels, mt_target_weights) = self.capture_output( + id, rank + ) + dt_dense = time.time() - t0 + return ( + mt_target_preds, + mt_target_labels, + mt_target_weights, + dt_dense, + ) + + +class ModelFamilyDenseSingleWorker: + """ + Single-worker dense module manager for single-GPU inference. + + Simpler alternative to ModelFamilyDenseDist for single-GPU setups. + + Args: + hstu_config: HSTU model configuration. + table_config: Embedding table configurations. + output_trace: Whether to enable profiling traces. + compute_eval: Whether to compute evaluation metrics. + """ + + def __init__( + self, + hstu_config: DlrmHSTUConfig, + table_config: Dict[str, EmbeddingConfig], + output_trace: bool = False, + compute_eval: bool = False, + ) -> None: + self.model: Optional[torch.nn.Module] = None + self.hstu_config = hstu_config + self.table_config = table_config + self.output_trace = output_trace + self.device: torch.device = torch.device("cuda:0") + torch.cuda.set_device(self.device) + self.profiler: Optional[Profiler] = ( + Profiler(rank=0) if self.output_trace else None + ) + + def load(self, model_path: str) -> None: + """ + Load dense model for single-GPU inference. + + Args: + model_path: Path to the model checkpoint directory. + """ + logger.warning(f"Loading dense module from {model_path}") + self.model = ( + get_hstu_model( + table_config=self.table_config, + hstu_config=self.hstu_config, + table_device="cpu", + is_dense=True, + ) + .to(self.device) + .to(torch.bfloat16) + ) + self.model.set_training_dtype(torch.bfloat16) + load_nonsparse_checkpoint( + model=self.model, device=self.device, optimizer=None, path=model_path + ) + assert self.model is not None + self.model.eval() + + def predict( + self, + seq_embeddings: Optional[Dict[str, SequenceEmbedding]], + payload_features: Optional[Dict[str, torch.Tensor]], + max_uih_len: int, + uih_seq_lengths: Optional[torch.Tensor], + max_num_candidates: int, + num_candidates: Optional[torch.Tensor], + ) -> Optional[ + Tuple[ + torch.Tensor, + Optional[torch.Tensor], + Optional[torch.Tensor], + float, + ] + ]: + """ + Run dense forward pass on single GPU. + + Args: + seq_embeddings: Sequence embeddings from sparse module. + payload_features: Additional feature tensors. + max_uih_len: Maximum UIH sequence length. + uih_seq_lengths: Per-sample UIH lengths. + max_num_candidates: Maximum candidates per sample. + num_candidates: Per-sample candidate counts. + + Returns: + Tuple of (predictions, labels, weights, elapsed_time). + """ + if self.output_trace: + assert self.profiler is not None + self.profiler.step() + assert ( + payload_features is not None + and uih_seq_lengths is not None + and num_candidates is not None + and seq_embeddings is not None + ) + t0: float = time.time() + with torch.profiler.record_function("dense forward"): + seq_embeddings, payload_features, uih_seq_lengths, num_candidates = ( + move_sparse_output_to_device( + seq_embeddings=seq_embeddings, + payload_features=payload_features, + uih_seq_lengths=uih_seq_lengths, + num_candidates=num_candidates, + device=self.device, + ) + ) + assert self.model is not None + ( + _, + _, + _, + mt_target_preds, + mt_target_labels, + mt_target_weights, + ) = self.model.main_forward( # pyre-ignore [29] + seq_embeddings=seq_embeddings, + payload_features=payload_features, + max_uih_len=max_uih_len, + uih_seq_lengths=uih_seq_lengths, + max_num_candidates=max_num_candidates, + num_candidates=num_candidates, + ) + assert mt_target_preds is not None + mt_target_preds = mt_target_preds.detach().to(device="cpu") + if mt_target_labels is not None: + mt_target_labels = mt_target_labels.detach().to(device="cpu") + if mt_target_weights is not None: + mt_target_weights = mt_target_weights.detach().to(device="cpu") + dt_dense: float = time.time() - t0 + return mt_target_preds, mt_target_labels, mt_target_weights, dt_dense diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/sparse_predict_module.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/sparse_predict_module.py new file mode 100644 index 000000000..e3ec10415 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/sparse_predict_module.py @@ -0,0 +1,106 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-strict + +""" +TorchScript-friendly wrapper for the HSTU sparse path (CPU embedding lookup). + +``HSTUSparseScriptModule`` wraps :class:`HSTUSparseInferenceModule` and +flattens the ``Dict[str, SequenceEmbedding]`` output into the parallel +value/length dicts defined in :mod:`ts_types` so the boundary is composed +entirely of TorchScript-supported types. +""" + +from typing import Dict, Tuple + +import torch +from generative_recommenders.dlrm_v3.inference.inference_modules import ( + _NoCopyEmbeddingCollection, + HSTUSparseInferenceModule, +) +from generative_recommenders.dlrm_v3.inference.ts_types import ( + flatten_seq_embeddings, + SeqEmbLengths, + SeqEmbValues, +) +from generative_recommenders.modules.dlrm_hstu import DlrmHSTUConfig +from torchrec.modules.embedding_configs import EmbeddingConfig +from torchrec.sparse.jagged_tensor import KeyedJaggedTensor + + +class HSTUSparseScriptModule(torch.nn.Module): + """Script-friendly sparse module. + + ``forward`` returns 5 tensors / dicts (no Python ``int`` scalars): + + 1. ``seq_emb_values`` ``Dict[str, Tensor]`` -- jagged embedding values. + 2. ``seq_emb_lengths`` ``Dict[str, Tensor]`` -- per-feature lengths. + 3. ``payload_features`` ``Dict[str, Tensor]`` -- side features. + 4. ``uih_seq_lengths`` ``Tensor[B]`` -- UIH lengths. + 5. ``num_candidates`` ``Tensor[B]`` -- candidate counts. + + The dense module (or the C++ glue) recovers the ``int`` ``max_uih_len`` / + ``max_num_candidates`` values from these tensors via ``.max().item()``. + """ + + def __init__( + self, + table_config: Dict[str, EmbeddingConfig], + hstu_config: DlrmHSTUConfig, + use_no_copy_embedding_collection: bool = True, + ) -> None: + super().__init__() + self._sparse: HSTUSparseInferenceModule = HSTUSparseInferenceModule( + table_config=table_config, + hstu_config=hstu_config, + ) + if use_no_copy_embedding_collection: + # Re-class the existing EmbeddingCollection so TorchScript picks up + # the no-copy ``forward`` override (matches the eager-only + # ``ec_patched_forward_wo_embedding_copy`` monkey-patch). + self._sparse._hstu_model._embedding_collection.__class__ = ( + _NoCopyEmbeddingCollection + ) + + def forward( + self, + uih_features: KeyedJaggedTensor, + candidates_features: KeyedJaggedTensor, + ) -> Tuple[ + SeqEmbValues, + SeqEmbLengths, + Dict[str, torch.Tensor], + torch.Tensor, + torch.Tensor, + ]: + ( + seq_embeddings, + payload_features, + _max_uih_len, + uih_seq_lengths, + _max_num_candidates, + num_candidates, + ) = self._sparse( + uih_features=uih_features, + candidates_features=candidates_features, + ) + seq_emb_values, seq_emb_lengths = flatten_seq_embeddings(seq_embeddings) + return ( + seq_emb_values, + seq_emb_lengths, + payload_features, + uih_seq_lengths, + num_candidates, + ) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/tests/inference_test.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/tests/inference_test.py new file mode 100644 index 000000000..948f10618 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/tests/inference_test.py @@ -0,0 +1,39 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import unittest + +from generative_recommenders.common import gpu_unavailable +from generative_recommenders.dlrm_v3.inference.main import main +from hypothesis import given, settings, strategies as st, Verbosity + + +class DLRMV3InferenceTest(unittest.TestCase): + @unittest.skipIf(*gpu_unavailable) + @given( + world_size=st.sampled_from([1]), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=1, + deadline=None, + ) + def test_e2e(self, world_size: int) -> None: + os.environ["WORLD_SIZE"] = str(world_size) + main() + + +if __name__ == "__main__": + unittest.main() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/tests/test_scripted_parity.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/tests/test_scripted_parity.py new file mode 100644 index 000000000..34d0388ea --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/tests/test_scripted_parity.py @@ -0,0 +1,236 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-strict + +""" +Numerical parity test: eager HSTU vs traced (sparse + dense) on a synthetic +batch. + +The production deployment path (see ``end_to_end_test.py``) uses +``torch.jit.trace``, not ``torch.jit.script``, for the HSTU sparse/dense +wrappers. Tracing records the actual tensor ops executed during a forward +pass and ignores source-level dispatch logic (HammerKernel enum, +``is_fx_tracing()``, ``torch.autocast``, IntEnum branches) that scripting +cannot compile. This unit test mirrors that path. + +Tolerances are deliberately loose because the traced path replaces the +Triton fused kernels with PyTorch fallbacks and skips ``torch.autocast`` in +the user-forward block; both can perturb low-order bits in bf16. +""" + +import unittest +from typing import Dict, List, Tuple + +import torch +from generative_recommenders.common import gpu_unavailable, HammerKernel +from generative_recommenders.dlrm_v3.configs import ( + get_embedding_table_config, + get_hstu_configs, +) +from generative_recommenders.dlrm_v3.datasets.dataset import get_random_data +from generative_recommenders.dlrm_v3.inference.dense_predict_module import ( + HSTUDenseScriptModule, +) +from generative_recommenders.dlrm_v3.inference.sparse_predict_module import ( + HSTUSparseScriptModule, +) +from generative_recommenders.dlrm_v3.inference.ts_types import ( + SeqEmbLengths, + SeqEmbValues, +) +from torchrec.sparse.jagged_tensor import KeyedJaggedTensor + + +_DATASET = "kuairand-1k" + + +def _move_dense_inputs( + seq_emb_values: Dict[str, torch.Tensor], + seq_emb_lengths: Dict[str, torch.Tensor], + payload_features: Dict[str, torch.Tensor], + uih_seq_lengths: torch.Tensor, + num_candidates: torch.Tensor, + device: torch.device, +) -> Tuple[ + Dict[str, torch.Tensor], + Dict[str, torch.Tensor], + Dict[str, torch.Tensor], + torch.Tensor, + torch.Tensor, +]: + """C++-side ``move_sparse_output_to_device`` analog for the test.""" + return ( + {k: v.to(device).to(torch.bfloat16) for k, v in seq_emb_values.items()}, + {k: v.to(device) for k, v in seq_emb_lengths.items()}, + {k: v.to(device) for k, v in payload_features.items()}, + uih_seq_lengths.to(device), + num_candidates.to(device), + ) + + +class _SparseTraceShim(torch.nn.Module): + """Adapter that takes raw tensors and rebuilds the KJTs inside forward. + + ``torch.jit.trace`` does not accept ``KeyedJaggedTensor`` (or any + non-Tensor / non-collection-of-Tensor type) as a top-level forward + input, so we make the traced boundary tensor-only and bake the + ``List[str]`` of feature keys in as module attributes. + """ + + def __init__( + self, + sparse_module: HSTUSparseScriptModule, + uih_keys: List[str], + candidates_keys: List[str], + ) -> None: + super().__init__() + self._sparse_module: HSTUSparseScriptModule = sparse_module + self._uih_keys: List[str] = uih_keys + self._candidates_keys: List[str] = candidates_keys + + def forward( + self, + uih_lengths: torch.Tensor, + uih_values: torch.Tensor, + candidates_lengths: torch.Tensor, + candidates_values: torch.Tensor, + ) -> Tuple[ + SeqEmbValues, + SeqEmbLengths, + Dict[str, torch.Tensor], + torch.Tensor, + torch.Tensor, + ]: + uih_kjt = KeyedJaggedTensor( + keys=self._uih_keys, + lengths=uih_lengths, + values=uih_values, + ) + candidates_kjt = KeyedJaggedTensor( + keys=self._candidates_keys, + lengths=candidates_lengths, + values=candidates_values, + ) + return self._sparse_module( + uih_features=uih_kjt, candidates_features=candidates_kjt + ) + + +class HSTUScriptedParityTest(unittest.TestCase): + @unittest.skipIf(*gpu_unavailable) + def test_scripted_matches_eager(self) -> None: + torch.manual_seed(0) + device = torch.device("cuda:0") + torch.cuda.set_device(device) + + hstu_config = get_hstu_configs(_DATASET) + table_config = get_embedding_table_config(_DATASET) + + # Some embedding tables in kuairand-1k are tiny (e.g. + # user_active_degree has num_embeddings=8). Clamp the random value + # range so every index stays in range for every table; otherwise the + # default value_bound=1000 triggers an out-of-range embedding lookup. + min_rows = min(t.num_embeddings for t in table_config.values()) + value_bound = max(2, min_rows) + + uih_kjt, candidates_kjt = get_random_data( + contexual_features=list( + hstu_config.contextual_feature_to_max_length.keys() + ), + hstu_uih_keys=hstu_config.hstu_uih_feature_names, + hstu_candidates_keys=hstu_config.hstu_candidate_feature_names, + uih_max_seq_len=128, + max_num_candidates=hstu_config.max_num_candidates_inference, + value_bound=value_bound, + ) + + sparse_module = HSTUSparseScriptModule( + table_config=table_config, + hstu_config=hstu_config, + use_no_copy_embedding_collection=True, + ).eval() + dense_module = ( + HSTUDenseScriptModule( + hstu_config=hstu_config, + table_config=table_config, + ) + .to(torch.bfloat16) + .to(device) + .eval() + ) + + # Pin the HammerKernel to PyTorch on both wrappers. The Triton + # kernels use Python-level dispatch (autotune, constexpr arguments) + # that interacts badly with torch.jit.trace's recording pass. The + # eager reference run uses the same setting so the comparison is + # apples-to-apples. + sparse_module._sparse._hstu_model.set_hammer_kernel(HammerKernel.PYTORCH) + dense_module._hstu_model.set_hammer_kernel(HammerKernel.PYTORCH) + + # === Eager reference path === + with torch.no_grad(): + sparse_out_e = sparse_module( + uih_features=uih_kjt, candidates_features=candidates_kjt + ) + dense_inputs_e = _move_dense_inputs(*sparse_out_e, device=device) + preds_eager = dense_module(*dense_inputs_e) + + # === Traced path === + # Sparse is traced via a raw-tensor shim because KJT is not a valid + # traced input. Dense is traced directly with the eager sparse + # output as the example. + sparse_shim = _SparseTraceShim( + sparse_module=sparse_module, + uih_keys=list(uih_kjt.keys()), + candidates_keys=list(candidates_kjt.keys()), + ) + traced_sparse = torch.jit.trace( + sparse_shim, + example_inputs=( + uih_kjt.lengths(), + uih_kjt.values(), + candidates_kjt.lengths(), + candidates_kjt.values(), + ), + strict=False, + check_trace=False, + ) + traced_dense = torch.jit.trace( + dense_module, + example_inputs=tuple(dense_inputs_e), + strict=False, + check_trace=False, + ) + + with torch.no_grad(): + sparse_out_t = traced_sparse( + uih_kjt.lengths(), + uih_kjt.values(), + candidates_kjt.lengths(), + candidates_kjt.values(), + ) + dense_inputs_t = _move_dense_inputs(*sparse_out_t, device=device) + preds_traced = traced_dense(*dense_inputs_t) + + torch.testing.assert_close( + preds_eager.float(), + preds_traced.float(), + atol=1e-2, + rtol=1e-2, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/.clang-format b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/.clang-format new file mode 100644 index 000000000..f08c9c2c8 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/.clang-format @@ -0,0 +1,2 @@ +BasedOnStyle: Google +Standard: Cpp11 diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/CMakeLists.txt b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/CMakeLists.txt new file mode 100644 index 000000000..4fec0e44f --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/CMakeLists.txt @@ -0,0 +1,113 @@ +cmake_minimum_required(VERSION 3.12) + +project(mlperf_loadgen) + +# Read the version file +file(READ "${CMAKE_SOURCE_DIR}/VERSION.txt" VERSION_CONTENTS) + +# Extract the major, minor, and patch versions from the VERSION file (assuming "MAJOR.MINOR.PATCH" format) +string(REGEX MATCH "^([0-9]+)\\.([0-9]+)\\.([0-9]+)" VERSION_MATCH ${VERSION_CONTENTS}) + +# Set the variables for the major, minor, and patch versions +set(mlperf_loadgen_VERSION_MAJOR "${CMAKE_MATCH_1}") +set(mlperf_loadgen_VERSION_MINOR "${CMAKE_MATCH_2}") +set(mlperf_loadgen_VERSION_PATCH "${CMAKE_MATCH_3}") + +# Check if the version format was parsed correctly +if(NOT DEFINED mlperf_loadgen_VERSION_MAJOR OR NOT DEFINED mlperf_loadgen_VERSION_MINOR OR NOT DEFINED mlperf_loadgen_VERSION_PATCH) + message(FATAL_ERROR "Version format in VERSION.txt is incorrect. Expected format: MAJOR.MINOR.PATCH") +endif() + +# Print out the version +message("mlperf_loadgen v${mlperf_loadgen_VERSION_MAJOR}.${mlperf_loadgen_VERSION_MINOR}.${mlperf_loadgen_VERSION_PATCH}") + +# Set build options. NB: CXX_STANDARD is supported since CMake 3.1. +if (NOT MSVC) +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -W -Wall") +endif() +# Extra build options can be specified by setting the MLPERF_LOADGEN_CXX_FLAGS variable +if (MLPERF_LOADGEN_CXX_FLAGS) +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${MLPERF_LOADGEN_CXX_FLAGS}") +endif() +message(STATUS "Using C++ compiler flags: ${CMAKE_CXX_FLAGS}") +set(CMAKE_CXX_STANDARD "14") +message(STATUS "Using C++ standard: ${CMAKE_CXX_STANDARD}") +message(STATUS "Using static linker flags: ${CMAKE_STATIC_LINKER_FLAGS}") +message(STATUS "Using shared linker flags: ${CMAKE_SHARED_LINKER_FLAGS}") + +# Output directory for libraries. +set(LIBRARY_OUTPUT_PATH ${CMAKE_BINARY_DIR}) +message(STATUS "Using output path: ${LIBRARY_OUTPUT_PATH}") + +# Detect Python to use for generating source file with version info. +# NB: PythonInterp has been deprecated since CMake 3.12 +# but it works with earlier versions of CMake. +find_package(PythonInterp) +message(STATUS "Using Python interpreter: ${PYTHON_EXECUTABLE}") + +# Specify the source and destination files +set(CONF_FILE "mlperf.conf") +set(HEADER_FILE "mlperf_conf.h") + +# Read the content of the configuration file +file(READ ${CONF_FILE} CONF_CONTENTS) + +# Escape all double quotes and backslashes +string(REPLACE "\\" "\\\\" CONF_CONTENTS "${CONF_CONTENTS}") +string(REPLACE "\"" "\\\"" CONF_CONTENTS "${CONF_CONTENTS}") + +# Handle new lines +string(REPLACE "\n" "\\n\"\n\"" CONF_CONTENTS "${CONF_CONTENTS}") + +# Wrap the content in a C++ string declaration +set(FORMATTED_CONTENT "const char* mlperf_conf =\n\"${CONF_CONTENTS}\";\n") + +# Write the formatted content to the header file +file(WRITE ${HEADER_FILE} "${FORMATTED_CONTENT}") + +message(STATUS "Output config: ${CMAKE_BINARY_DIR}/mlperf_conf.h") + +# Generate source file with version info. +execute_process(COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/version_generator.py ${CMAKE_BINARY_DIR}/version_generated.cc ${CMAKE_CURRENT_SOURCE_DIR}) + +# Add source files. +set(SOURCE + ${CMAKE_CURRENT_SOURCE_DIR}/bindings/c_api.h + ${CMAKE_CURRENT_SOURCE_DIR}/bindings/c_api.cc + ${CMAKE_CURRENT_SOURCE_DIR}/early_stopping.cc + ${CMAKE_CURRENT_SOURCE_DIR}/issue_query_controller.cc + ${CMAKE_CURRENT_SOURCE_DIR}/loadgen.cc + ${CMAKE_CURRENT_SOURCE_DIR}/logging.cc + ${CMAKE_CURRENT_SOURCE_DIR}/logging.h + ${CMAKE_CURRENT_SOURCE_DIR}/test_settings_internal.cc + ${CMAKE_CURRENT_SOURCE_DIR}/test_settings_internal.h + ${CMAKE_CURRENT_SOURCE_DIR}/utils.cc + ${CMAKE_CURRENT_SOURCE_DIR}/utils.h + ${CMAKE_CURRENT_SOURCE_DIR}/results.h + ${CMAKE_CURRENT_SOURCE_DIR}/results.cc + ${CMAKE_CURRENT_SOURCE_DIR}/version.cc + ${CMAKE_CURRENT_SOURCE_DIR}/version.h + ${CMAKE_CURRENT_SOURCE_DIR}/mlperf_conf.h + ${CMAKE_CURRENT_SOURCE_DIR}/VERSION.txt + ${CMAKE_BINARY_DIR}/version_generated.cc +) + +include_directories(${CMAKE_CURRENT_SOURCE_DIR}) + +add_library(mlperf_loadgen STATIC ${SOURCE}) +target_link_libraries(mlperf_loadgen) + +if(WIN32) +set (LIBS "") +else() +set (LIBS pthread) +endif() + +add_executable(benchmark benchmark/repro.cpp) +target_link_libraries(benchmark PUBLIC mlperf_loadgen ${LIBS}) + +# Install library and headers. +install(TARGETS mlperf_loadgen + DESTINATION ${CMAKE_INSTALL_PREFIX}/lib) +install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/ + DESTINATION ${CMAKE_INSTALL_PREFIX}/include FILES_MATCHING PATTERN "*.h") diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/MANIFEST.in b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/MANIFEST.in new file mode 100644 index 000000000..152b53111 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/MANIFEST.in @@ -0,0 +1,2 @@ +include VERSION.txt +include mlperf.conf diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/README.md b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/README.md new file mode 100644 index 000000000..212c8a53c --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/README.md @@ -0,0 +1,223 @@ +# Overview {#mainpage} + +## Introduction + +* The LoadGen is a *reusable* module that *efficiently* and *fairly* measures + the performance of inference systems. +* It generates traffic for scenarios as formulated by a diverse set of experts + in the [MLCommons working group](https://mlcommons.org/). +* The scenarios emulate the workloads seen in mobile devices, + autonomous vehicles, robotics, and cloud-based setups. +* Although the LoadGen is not model or dataset aware, its strength is in its + reusability with logic that is. + +## Integration Example and Flow +The following is an diagram of how the LoadGen can be integrated into an +inference system, resembling how some of the MLPerf reference models are +implemented. +
+ +
    +
  1. Benchmark knows the model, dataset, and preprocessing.
    2. +
  2. Benchmark hands dataset sample IDs to LoadGen.
    3. +
  3. LoadGen starts generating queries of sample IDs.
    4. +
  4. Benchmark creates requests to backend.
    5. +
  5. Result is post processed and forwarded to LoadGen.
    6. +
  6. LoadGen outputs logs for analysis.
    +
+
+ +## Useful Links +* [FAQ](README_FAQ.md) +* [LoadGen Build Instructions](README_BUILD.md) +* [LoadGen API](loadgen.h) +* [Test Settings](test_settings.h) - + A good description of available scenarios, modes, and knobs. +* [MLPerf Inference Code](https://github.com/mlcommons/inference) - + Includes source for the LoadGen and reference models that use the LoadGen. +* [MLPerf Inference Rules](https://github.com/mlcommons/inference_policies) - + Any mismatch with this is a bug in the LoadGen. + +## Scope of the LoadGen's Responsibilities + +### In Scope +* **Provide a reusable** C++ library with python bindings. +* **Implement** the traffic patterns of the MLPerf Inference scenarios and + modes. +* **Record** all traffic generated and received for later analysis and + verification. +* **Summarize** the results and whether performance constraints were met. +* **Target high-performance** systems with efficient multi-thread friendly + logging utilities. +* **Generate trust** via a shared, well-tested, and community-hardened + code base. + +### Out of Scope +The LoadGen is: +* **NOT** aware of the ML model it is running against. +* **NOT** aware of the data formats of the model's inputs and outputs. +* **NOT** aware of how to score the accuracy of a model's outputs. +* **NOT** aware of MLPerf rules regarding scenario-specific constraints. + +Limitting the scope of the LoadGen in this way keeps it reusable across +different models and datasets without modification. Using composition and +dependency injection, the user can define their own model, datasets, and +metrics. + +Additionally, not hardcoding MLPerf-specific test constraints, like test +duration and performance targets, allows users to use the LoadGen unmodified +for custom testing and continuous integration purposes. + +## Submission Considerations + +### Upstream all local modifications +* As a rule, no local modifications to the LoadGen's C++ library are allowed +for submission. +* Please upstream early and often to keep the playing field level. + +### Choose your TestSettings carefully! +* Since the LoadGen is oblivious to the model, it can't enforce the MLPerf +requirements for submission. *e.g.:* target percentiles and latencies. +* For verification, the values in TestSettings are logged. +* To help make sure your settings are spec compliant, use +TestSettings::FromConfig in conjunction with the relevant config file provided +with the reference models. + +## Responsibilities of a LoadGen User + +### Implement the Interfaces +* Implement the SystemUnderTest and QuerySampleLibrary interfaces and pass + them to the StartTest function. +* Call QuerySampleComplete for every sample received by + SystemUnderTest::IssueQuery. + +### Assess Accuracy +* Process the *mlperf_log_accuracy.json* output by the LoadGen to determine + the accuracy of your system. +* For the official models, Python scripts will be provided by the MLPerf model + owners for you to do this automatically. + +For templates of how to do the above in detail, refer to code for the demos, +tests, and reference models. + + +## LoadGen over the Network + +For reference, on a high level a submission looks like this: + +
+ +
+ +The LoadGen implementation is common to all submissions, while the QSL (“Query Sample Library”) and SUT (“System Under Test”) are implemented by submitters. QSL is responsible for loading the data and includes untimed preprocessing. + +A submission over the network introduces a new component “QDL” (query dispatch library) that is added to the system as presented in the following diagram: + +
+ +
+ +QDL is a proxy for a load-balancer, that dispatches queries to SUT over a physical network, receives the responses and passes them back to LoadGen. It is implemented by the submitter. The interface of the QDL is the same as the API to SUT. + +In scenarios using QDL, data may be compressed in QSL at the choice of the submitter in order to reduce network transmission time. Decompression is part of the timed processing in SUT. A set of approved standard compression schemes will be specified for each benchmark; additional compression schemes must be approved in advance by the Working Group. + +All communication between LoadGen/QSL and SUT is via QDL, and all communication between QDL and SUT must pass over a physical network. + +QDL implements the protocol to transmit queries over the network and receive responses. It also implements decompression of any response returned by the SUT, where compression of responses is allowed. Performing any part of the timed preprocessing or inference in QDL is specifically disallowed. Currently no batching is allowed in QDL, although this may be revisited in future. + +The MLperf over the Network will run in Server mode and Offline mode. All LoadGen modes are expected to work as is with insignificant changes. These include running the test in performance mode, accuracy mode, find peak performance mode and compliance mode. The same applies for power measurements. + +### QDL details +The Query Dispatch Library is implemented by the submitter and interfaces with LoadGen using the same SUT API. All MLPerf Inference SUTs implement the `mlperf::SystemUnderTest` class which is defined in system_under_test.h. The QDL implements `mlperf::QueryDispatchLibrary` class which inherits the `mlperf::SystemUnderTest` class and has the same API and support all existing `mlperf::SystemUnderTest` methods. It has a separate header file query_dispatch_library.h. Using sut with `mlperf::SystemUnderTest` class in LoadGen StartTest is natively upcasting `mlperf::QueryDispatchLibrary` class. + +#### QDL Query issue and response over the network + +The QDL gets the queries from the LoadGen through +```CPP +void IssueQuery(const std::vector& samples) +``` + +The QDL dispatches the queries to the SUT over the physical media. The exact method and implementation for it are submitter specific and would not be specified at MLCommons. Submitter implementation includes all methods required to serialize the query, load balance, drive it to the Operating system and network interface card and send to the SUT. + +The QDL receives the query responses over the network from the SUT. The exact method and implementation for it are submitter specific and would not be specified at MLCommons. The submitter implementation includes all methods required to receive the network data from the Network Interface card, go through the Operating system, deserialize the query response, and provide it back to the LoadGen through query completion by: + +```CPP +struct QuerySampleResponse { + ResponseId id; + uintptr_t data; + size_t size; +}; +void QuerySamplesComplete(QuerySampleResponse* responses, + size_t response_count); + +``` + +#### QDL Additional Methods + +In addition to that the QDL needs to implement the following methods that are provided by the SUT interface to the LoadGen: +```CPP +const std::string& Name(); +``` +The `Name` function returns a known string for over the Network SUTs to identify it as over the network benchmark. +```CPP +void FlushQueries(); +``` + +It is not specified here how the QDL would query and configure the SUT to execute the above methods. The QDL responds to the LoadGen after receiving its own response from the SUT. + +### Example + +Refer to [LON demo](demos/lon) for a reference example illustrating usage of Loadgen over the network. + +## Find Peak Performance Mode + +The Find Peak Performance mode can be used to find the optimal queries per second (QPS) for the server scenario. + +### Setup + +You can setup loadgen to run this mode by setting the `mode` variable in the `test_settings` used to run the test. Using the Python API: + +```python +settings = mlperf_loadgen.TestSettings() +settings.server_target_qps = 100 +settings.scenario = mlperf_loadgen.TestScenario.Server +settings.mode = mlperf_loadgen.TestMode.FindPeakPerformance +... + +mlperf_loadgen.StartTest(sut, qsl, settings) +``` + +Using the C/C++ API: +```CPP +mlperf::TestSettings settings; +setting.server_target_qps = 100; +settings.scenario = mlperf::TestScenario::Server; +settings.mode = mlperf::TestMode::FindPeakPerformance; +mlperf::LogSettings log_settings; +/* +Construct QSL and SUT +*/ +mlperf::StartTest(&sut, &qsl, settings, log_settings); +``` + +**Note:** Make sure you are setting the TestScenario to server and you are providing an initial target QPS. + +### Description + +The Find Peak Performance mode works by finding a lower and upper boundary for the optimal QPS. Then performing a binary search between the lower and upper bound to find the optimal QPS. + +#### Finding lower and upper boundary + +LoadGen begins by running performance mode at the specified target QPS. If the test passes, this value is used as the lower bound; otherwise, an error is raised. The algorithm then guesses the upper bound as twice the target QPS. + +Then LoadGen will run performance mode using the upper bound guess. If the test is successful, both the lower bound and upper bound will be doubled. This repeats until the upper bound guess fails the test. + +``` +[initial_target_qps, 2*initial_target_qps] -> [2*initial_target_qps, 4*initial_target_qps] -> [4*initial_target_qps, 8*initial_target_qps]... +``` + +Finally, the final lower bound and upper bound are set to their current values. This process assures that the lower bound passes the performance mode, but the upper bound doesn’t. + +#### Binary Search + +Once the lower and upper bounds are set, binary search can be performed over the range `[lower, upper]`` to find the optimal QPS. If a given QPS fails in performance mode, the optimal value lies below it; if it passes, the optimal is higher. diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/README_BUILD.md b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/README_BUILD.md new file mode 100644 index 000000000..499cc360a --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/README_BUILD.md @@ -0,0 +1,47 @@ +# Building the LoadGen {#ReadmeBuild} + +## Prerequisites + + sudo apt-get install libglib2.0-dev python-pip python3-pip + pip2 install absl-py numpy + pip3 install absl-py numpy + +## Quick Start +### Installation - Python + + pip install absl-py numpy + git clone --recurse-submodules https://github.com/mlcommons/inference.git mlperf_inference + cd mlperf_inference/loadgen + CFLAGS="-std=c++14 -O3" python -m pip install . + +This will fetch the loadgen source, build and install the loadgen as a python module, and run a simple end-to-end demo. + +Alternatively, we provide wheels for several python versions and operating system that can be installed using pip directly. + + pip install mlperf-loadgen + +**NOTE:** Take into account that we only update the published wheels after an official release, they may not include the latest changes. + +### Testing your Installation +The following command will run a simple end-to-end demo: + + python mlperf_inference/loadgen/demos/py_demo_single_stream.py + +A summary of the test results can be found in the *"mlperf_log_summary.txt"* logfile. + +For a timeline visualization of what happened during the test, open the *"mlperf_log_trace.json"* file in Chrome: +* Type “chrome://tracing” in the address bar, then drag-n-drop the json. +* This may be useful for SUT performance tuning and understanding + debugging the loadgen. + +### Installation - C++ +To build the loadgen as a C++ library, rather than a python module: + + git clone https://github.com/mlcommons/inference.git mlperf_inference + cd mlperf_inference + mkdir loadgen/build/ && cd loadgen/build/ + cmake .. && cmake --build . + cp libmlperf_loadgen.a .. + +## Quick start: Loadgen Over the Network + +Refer to [LON demo](demos/lon/README.md) for a basic example. diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/README_FAQ.md b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/README_FAQ.md new file mode 100644 index 000000000..ab4e0c75d --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/README_FAQ.md @@ -0,0 +1,78 @@ +# LoadGen FAQ {#ReadmeFAQ} + +## Q: The LoadGen does not match the MLPerf specification. Who is right? +**A:** +The MLPerf spec is *always* right. +Please file a LoadGen bug so it may be resolved. + +## Q: How can I file a bug? +**A:** +On GitHub: https://github.com/mlcommons/inference/issues/new + +## Q: Can I make local modifications to the LoadGen for submission? +**A:** +No. To keep the playing field level, please upstream any local +modificiations you need to make. Ideally upstream such changes behind a runtime +flag or via an abstract interface the client can implement. This will help +with testability. + +## Q: Where can I find the results of a test? +**A:** +By default, the loadgen will output an *mlperf_log_summary.txt* file +that summarizes the target metrics and constraints of the test, along with +other stats about the run. + +*Note:* LogSettings also has a flag to forward the results to stdout and +there's an outstanding TODO to make this more programmable. + +## Q: The reference implementation for \<*some_model*\> prints out results of its own. Are those for submission? +**A:** +They are not. The LoadGen results are the ground truth for submission +results since they will work even for systems that forgo the python bindings. +If you notice a bug in the LoadGen's results, please file a bug or submit a +patch. + +## Q: I'm getting linker errors for LoadgenVersion definitions. Where is *version_generated.cc*? +**A:** +If you have a custom build setup, make sure you run the *version_generator.py* +script, which will create the cc file you are looking for. The official build +files that come with the LoadGen do this for you out of the box. + +## Q: What is this *version_generator.py* script? +**A:** +The LoadGen records git stats (if available) and the SHA1 of all its +source files (always) at build time for verification purposes. This is easy +to circumvent, but try your best to run *version_generator.py* correctly; +ideally integrated with your build system if you have a custom build. +The intention is more to help with debugging efforts and detect accidental +version missmatches than to detect bad actors. + +## Q: How do I view the *mlperf_log_trace.json* file? +**A:** +This file uses the [Trace Event Format] +(https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/edit) +to record a timeline of all the threads involved. +You can view the file by typing [chrome://tracing](chrome://tracing) into +Chrome's address bar and dragging the json file there. +This file zips well and you can drag the zip file directly into +[chrome://tracing](chrome://tracing) too. +Please include zipped traces (and the other logs) when filing bug reports. + +## Q: Why is the code littered with so many lambdas? My eyes hurt. +**A:** +Lambdas are a convenient and efficient way to ship arbitrary data + deferred +logic over to the logging thread without much boilerplate. +Much of the loadgen is built on top of the logging utilities. +Thus the lambdas. (Sorry about the eyes.) + +## Q: What C++ version does the LoadGen target? +**A:** +It currently targets and requires C++14. It should compile with recent +versions of clang, gcc, and msvc. + +## Q: What dependencies does the LoadGen code have? +**A:** +The C++ code has no external dependencies. The loadgen itself, logging +utilities, and unit test utilities are built solely on the C++ Standard Library. +The python bindings, however, do require +[pybind11](https://github.com/pybind/pybind11). diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/VERSION.txt b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/VERSION.txt new file mode 100644 index 000000000..ac14c3dfa --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/VERSION.txt @@ -0,0 +1 @@ +5.1.1 diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/benchmark/.gitignore b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/benchmark/.gitignore new file mode 100644 index 000000000..e792c8e55 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/benchmark/.gitignore @@ -0,0 +1,2 @@ +loadgen_build +build \ No newline at end of file diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/benchmark/README.md b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/benchmark/README.md new file mode 100644 index 000000000..24e872983 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/benchmark/README.md @@ -0,0 +1,10 @@ +Note: please install jemalloc first. See: http://jemalloc.net/ +Command: bash run.sh <0=Basic,1=Queue> + +Experiments: +- On Intel(R) Xeon(R) CPU E5-1650 v4 @ 3.60GHz +- Basic SUT : 500-600k i/s +- Basic SUT + jemalloc: 800-900k i/s (`bash run.sh 800000 0`) +- Queued SUT (2 complete threads) + jemalloc: 1.2-1.3M i/s (`bash run.sh 1200000 1 2 2048`) +- Queued SUT (2 complete threads) + jemalloc + server_coalesce_queries: 1.4-1.5M is/ (`bash run.sh 1400000 1 2 512 1`) +- Basic SUT + jemalloc + server_coalesce_queries + 4 IssueQueryThreads: 2.4-2.5M is/ (`bash run.sh 2400000 0 2 512 1 4`) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/benchmark/repro.cpp b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/benchmark/repro.cpp new file mode 100644 index 000000000..44ff53efa --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/benchmark/repro.cpp @@ -0,0 +1,296 @@ +/* + * Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "loadgen.h" +#include "query_sample_library.h" +#include "system_under_test.h" +#include "test_settings.h" + +class QSL : public mlperf::QuerySampleLibrary { + public: + ~QSL() override{}; + const std::string& Name() override { return mName; } + size_t TotalSampleCount() override { return 1000000; } + size_t PerformanceSampleCount() override { return TotalSampleCount(); } + void LoadSamplesToRam(const std::vector&) override { + } + void UnloadSamplesFromRam( + const std::vector&) override {} + + private: + std::string mName{"Dummy QSL"}; +}; + +class BasicSUT : public mlperf::SystemUnderTest { + public: + BasicSUT() { + // Start with some large value so that we don't reallocate memory. + initResponse(10000); + } + ~BasicSUT() override {} + const std::string& Name() override { return mName; } + void IssueQuery(const std::vector& samples) override { + size_t n = samples.size(); + if (n > mResponses.size()) { + std::cerr << "Warning: reallocating response buffer in BasicSUT. Maybe " + "you should initResponse with larger value!?" + << std::endl; + initResponse(samples.size()); + } + for (size_t i = 0; i < n; i++) { + mResponses[i].id = samples[i].id; + } + mlperf::QuerySamplesComplete(mResponses.data(), n); + } + void FlushQueries() override {} + + private: + void initResponse(int size) { + mResponses.resize(size, + {0, reinterpret_cast(&mBuf), sizeof(int)}); + } + int mBuf{0}; + std::string mName{"BasicSUT"}; + std::vector mResponses; +}; + +class QueueSUT : public mlperf::SystemUnderTest { + public: + QueueSUT(int numCompleteThreads, int maxSize) { + // Each thread handle at most maxSize at a time. + std::cout << "QueueSUT: maxSize = " << maxSize << std::endl; + initResponse(numCompleteThreads, maxSize); + // Launch complete threads + for (int i = 0; i < numCompleteThreads; i++) { + mThreads.emplace_back(&QueueSUT::CompleteThread, this, i); + } + } + ~QueueSUT() override { + { + std::unique_lock lck(mMtx); + mDone = true; + mCondVar.notify_all(); + } + for (auto& thread : mThreads) { + thread.join(); + } + } + const std::string& Name() override { return mName; } + void IssueQuery(const std::vector& samples) override { + std::unique_lock lck(mMtx); + for (const auto& sample : samples) { + mIdQueue.push_back(sample.id); + } + // Let some worker thread to consume tasks + mCondVar.notify_one(); + } + void FlushQueries() override {} + + private: + void CompleteThread(int threadIdx) { + auto& responses = mResponses[threadIdx]; + size_t maxSize{responses.size()}; + size_t actualSize{0}; + while (true) { + { + std::unique_lock lck(mMtx); + mCondVar.wait(lck, [&]() { return !mIdQueue.empty() || mDone; }); + + if (mDone) { + break; + } + + actualSize = std::min(maxSize, mIdQueue.size()); + for (size_t i = 0; i < actualSize; i++) { + responses[i].id = mIdQueue.front(); + mIdQueue.pop_front(); + } + mCondVar.notify_one(); + } + mlperf::QuerySamplesComplete(responses.data(), actualSize); + } + } + void initResponse(int numCompleteThreads, int size) { + mResponses.resize(numCompleteThreads); + for (auto& responses : mResponses) { + responses.resize(size, + {0, reinterpret_cast(&mBuf), sizeof(int)}); + } + } + int mBuf{0}; + std::string mName{"QueueSUT"}; + std::vector> mResponses; + std::vector mThreads; + std::deque mIdQueue; + std::mutex mMtx; + std::condition_variable mCondVar; + bool mDone{false}; +}; + +class MultiBasicSUT : public mlperf::SystemUnderTest { + public: + MultiBasicSUT(int numThreads) + : mNumThreads(numThreads), mResponses(numThreads) { + // Start with some large value so that we don't reallocate memory. + initResponse(10000); + for (int i = 0; i < mNumThreads; ++i) { + mThreads.emplace_back(&MultiBasicSUT::startIssueThread, this, i); + } + } + ~MultiBasicSUT() override { + for (auto& thread : mThreads) { + thread.join(); + } + } + const std::string& Name() override { return mName; } + void IssueQuery(const std::vector& samples) override { + int thread_idx = mThreadMap[std::this_thread::get_id()]; + size_t n = samples.size(); + auto& reponses = mResponses[thread_idx]; + if (n > reponses.size()) { + std::cout + << "Warning: reallocating response buffer in MultiBasicSUT. Maybe " + "you should initResponse with larger value!?" + << std::endl; + initResponse(samples.size()); + } + for (size_t i = 0; i < n; i++) { + reponses[i].id = samples[i].id; + } + mlperf::QuerySamplesComplete(reponses.data(), n); + } + void FlushQueries() override {} + + private: + void initResponse(int size) { + for (auto& responses : mResponses) { + responses.resize(size, + {0, reinterpret_cast(&mBuf), sizeof(int)}); + } + } + void startIssueThread(int thread_idx) { + { + std::lock_guard lock(mMtx); + mThreadMap[std::this_thread::get_id()] = thread_idx; + } + mlperf::RegisterIssueQueryThread(); + } + int mBuf{0}; + int mNumThreads{0}; + std::string mName{"MultiBasicSUT"}; + std::vector> mResponses; + std::mutex mMtx; + std::vector mThreads; + std::map mThreadMap; +}; + +int main(int argc, char** argv) { + assert(argc >= 2 && "Need to pass in at least one argument: target_qps"); + int target_qps = std::stoi(argv[1]); + std::cout << "target_qps = " << target_qps << std::endl; + + bool useQueue{false}; + int numCompleteThreads{4}; + int maxSize{1}; + bool server_coalesce_queries{false}; + int num_issue_threads{0}; + if (argc >= 3) { + useQueue = std::stoi(argv[2]) != 0; + } + if (argc >= 4) { + numCompleteThreads = std::stoi(argv[3]); + } + if (argc >= 5) { + maxSize = std::stoi(argv[4]); + } + if (argc >= 6) { + server_coalesce_queries = std::stoi(argv[5]) != 0; + } + if (argc >= 7) { + num_issue_threads = std::stoi(argv[6]); + } + + QSL qsl; + std::unique_ptr sut; + + // Configure the test settings + mlperf::TestSettings testSettings; + testSettings.scenario = mlperf::TestScenario::Server; + testSettings.mode = mlperf::TestMode::PerformanceOnly; + testSettings.server_target_qps = target_qps; + testSettings.server_target_latency_ns = 10000000; // 10ms + testSettings.server_target_latency_percentile = 0.99; + testSettings.min_duration_ms = 60000; + testSettings.min_query_count = 270000; + testSettings.server_coalesce_queries = server_coalesce_queries; + std::cout << "testSettings.server_coalesce_queries = " + << (server_coalesce_queries ? "True" : "False") << std::endl; + testSettings.server_num_issue_query_threads = num_issue_threads; + std::cout << "num_issue_threads = " << num_issue_threads << std::endl; + + // Configure the logging settings + mlperf::LogSettings logSettings; + logSettings.log_output.outdir = "build"; + logSettings.log_output.prefix = "mlperf_log_"; + logSettings.log_output.suffix = ""; + logSettings.log_output.prefix_with_datetime = false; + logSettings.log_output.copy_detail_to_stdout = false; + logSettings.log_output.copy_summary_to_stdout = true; + logSettings.log_mode = mlperf::LoggingMode::AsyncPoll; + logSettings.log_mode_async_poll_interval_ms = 1000; + logSettings.enable_trace = false; + + // Choose SUT + if (num_issue_threads == 0) { + if (useQueue) { + std::cout << "Using QueueSUT with " << numCompleteThreads + << " complete threads" << std::endl; + sut.reset(new QueueSUT(numCompleteThreads, maxSize)); + } else { + std::cout << "Using BasicSUT" << std::endl; + sut.reset(new BasicSUT()); + } + } else { + if (useQueue) { + std::cout << "Using MultiQueueSUT with " << numCompleteThreads + << " complete threads" << std::endl; + std::cerr << "!!!! MultiQueueSUT is NOT implemented yet !!!!" + << std::endl; + return 1; + // sut.reset(new MultiQueueSUT(num_issue_threads, numCompleteThreads, + // maxSize)); + } else { + std::cout << "Using MultiBasicSUT" << std::endl; + sut.reset(new MultiBasicSUT(num_issue_threads)); + } + } + + // Start test + std::cout << "Start test..." << std::endl; + mlperf::StartTest(sut.get(), &qsl, testSettings, logSettings); + std::cout << "Test done. Clean up SUT..." << std::endl; + sut.reset(); + std::cout << "Done!" << std::endl; + return 0; +} diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/benchmark/run.sh b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/benchmark/run.sh new file mode 100644 index 000000000..62559c1a8 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/benchmark/run.sh @@ -0,0 +1,21 @@ +#!/usr/bin/bash +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +echo "Building loadgen..." +if [ ! -e loadgen_build ]; then mkdir loadgen_build; fi; +cd loadgen_build && cmake ../.. && make -j && cd .. +echo "Building test program..." +if [ ! -e build ]; then mkdir build; fi; +g++ --std=c++11 -O3 -I.. -o build/repro.exe repro.cpp -Lloadgen_build -lmlperf_loadgen -lpthread && \ +LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.2 build/repro.exe $1 $2 $3 $4 $5 $6 diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/benchmark/run_debug.sh b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/benchmark/run_debug.sh new file mode 100644 index 000000000..ba63727c8 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/benchmark/run_debug.sh @@ -0,0 +1,21 @@ +#!/usr/bin/bash +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +echo "Building loadgen in Debug mode..." +if [ ! -e loadgen_build ]; then mkdir loadgen_build; fi; +cd loadgen_build && cmake -DCMAKE_BUILD_TYPE=Debug ../.. && make -j && cd .. +echo "Building test program in Debug mode..." +if [ ! -e build ]; then mkdir build; fi; +g++ --std=c++11 -O0 -g -I.. -o build/repro.exe repro.cpp -Lloadgen_build -lmlperf_loadgen -lpthread && \ +gdb --args build/repro.exe $1 $2 $3 $4 $5 $6 diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/bindings/c_api.cc b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/bindings/c_api.cc new file mode 100644 index 000000000..0248a1c16 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/bindings/c_api.cc @@ -0,0 +1,176 @@ +/* Copyright 2019 The MLPerf Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "c_api.h" + +#include + +#include "../loadgen.h" +#include "../query_sample.h" +#include "../query_sample_library.h" +#include "../system_under_test.h" +#include "../test_settings.h" + +namespace mlperf { +namespace c { +namespace { + +// Forwards SystemUnderTest calls to relevant callbacks. +class SystemUnderTestTrampoline : public SystemUnderTest { + public: + SystemUnderTestTrampoline(ClientData client_data, std::string name, + IssueQueryCallback issue_cb, + FlushQueriesCallback flush_queries_cb) + : client_data_(client_data), + name_(std::move(name)), + issue_cb_(issue_cb), + flush_queries_cb_(flush_queries_cb) {} + ~SystemUnderTestTrampoline() override = default; + + const std::string& Name() override { return name_; } + + void IssueQuery(const std::vector& samples) override { + (*issue_cb_)(client_data_, samples.data(), samples.size()); + } + + void FlushQueries() override { (*flush_queries_cb_)(); } + + private: + ClientData client_data_; + std::string name_; + IssueQueryCallback issue_cb_; + FlushQueriesCallback flush_queries_cb_; +}; + +} // namespace + +void* ConstructSUT(ClientData client_data, const char* name, size_t name_length, + IssueQueryCallback issue_cb, + FlushQueriesCallback flush_queries_cb) { + SystemUnderTestTrampoline* sut = new SystemUnderTestTrampoline( + client_data, std::string(name, name_length), issue_cb, flush_queries_cb); + return reinterpret_cast(sut); +} + +void DestroySUT(void* sut) { + SystemUnderTestTrampoline* sut_cast = + reinterpret_cast(sut); + delete sut_cast; +} + +namespace { + +// Forwards QuerySampleLibrary calls to relevant callbacks. +class QuerySampleLibraryTrampoline : public QuerySampleLibrary { + public: + QuerySampleLibraryTrampoline( + ClientData client_data, std::string name, size_t total_sample_count, + size_t performance_sample_count, + LoadSamplesToRamCallback load_samples_to_ram_cb, + UnloadSamplesFromRamCallback unload_samples_from_ram_cb) + : client_data_(client_data), + name_(std::move(name)), + total_sample_count_(total_sample_count), + performance_sample_count_(performance_sample_count), + load_samples_to_ram_cb_(load_samples_to_ram_cb), + unload_samples_from_ram_cb_(unload_samples_from_ram_cb) {} + ~QuerySampleLibraryTrampoline() override = default; + + const std::string& Name() override { return name_; } + size_t TotalSampleCount() override { return total_sample_count_; } + size_t PerformanceSampleCount() override { return performance_sample_count_; } + + void LoadSamplesToRam(const std::vector& samples) override { + (*load_samples_to_ram_cb_)(client_data_, samples.data(), samples.size()); + } + void UnloadSamplesFromRam( + const std::vector& samples) override { + (*unload_samples_from_ram_cb_)(client_data_, samples.data(), + samples.size()); + } + + private: + ClientData client_data_; + std::string name_; + size_t total_sample_count_; + size_t performance_sample_count_; + LoadSamplesToRamCallback load_samples_to_ram_cb_; + UnloadSamplesFromRamCallback unload_samples_from_ram_cb_; +}; + +} // namespace + +void* ConstructQSL(ClientData client_data, const char* name, size_t name_length, + size_t total_sample_count, size_t performance_sample_count, + LoadSamplesToRamCallback load_samples_to_ram_cb, + UnloadSamplesFromRamCallback unload_samples_from_ram_cb) { + QuerySampleLibraryTrampoline* qsl = new QuerySampleLibraryTrampoline( + client_data, std::string(name, name_length), total_sample_count, + performance_sample_count, load_samples_to_ram_cb, + unload_samples_from_ram_cb); + return reinterpret_cast(qsl); +} + +void DestroyQSL(void* qsl) { + QuerySampleLibraryTrampoline* qsl_cast = + reinterpret_cast(qsl); + delete qsl_cast; +} + +// mlperf::c::StartTest just forwards to mlperf::StartTest after doing the +// proper cast. +void StartTest(void* sut, void* qsl, const TestSettings& settings, + const std::string& audit_config_filename = "audit.config") { + SystemUnderTestTrampoline* sut_cast = + reinterpret_cast(sut); + QuerySampleLibraryTrampoline* qsl_cast = + reinterpret_cast(qsl); + LogSettings default_log_settings; + mlperf::StartTest(sut_cast, qsl_cast, settings, default_log_settings, + audit_config_filename); +} + +void QuerySamplesComplete(QuerySampleResponse* responses, + size_t response_count) { + mlperf::QuerySamplesComplete(responses, response_count); +} + +void QuerySamplesCompleteResponseCb(QuerySampleResponse* responses, + size_t response_count, + ResponseCallback response_cb, + ClientData client_data) { + mlperf::QuerySamplesComplete( + responses, response_count, + [client_data, response_cb](QuerySampleResponse* response) { + response_cb(client_data, response); + }); +} + +void FirstTokenComplete(QuerySampleResponse* responses, size_t response_count) { + mlperf::FirstTokenComplete(responses, response_count); +} + +void FirstTokenCompleteResponseCb(QuerySampleResponse* responses, + size_t response_count, + ResponseCallback response_cb, + ClientData client_data) { + mlperf::FirstTokenComplete( + responses, response_count, + [client_data, response_cb](QuerySampleResponse* response) { + response_cb(client_data, response); + }); +} + +void RegisterIssueQueryThread() { mlperf::RegisterIssueQueryThread(); } + +} // namespace c +} // namespace mlperf diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/bindings/c_api.h b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/bindings/c_api.h new file mode 100644 index 000000000..0ee44fb71 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/bindings/c_api.h @@ -0,0 +1,95 @@ +/* Copyright 2019 The MLPerf Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +/// \file +/// \brief A C API wrapping the C++ loadgen. Not tested. Needs work. +/// \details The C API allows a C or Python client to easily create +/// a SystemUnderTest without having to expose the SystemUnderTest class +/// directly. +/// ConstructSUT works with a bunch of function poitners instead that are +/// called from an underlying trampoline class. + +#ifndef SYSTEM_UNDER_TEST_C_API_H_ +#define SYSTEM_UNDER_TEST_C_API_H_ + +#include +#include + +#include "../query_sample.h" +#include "../test_settings.h" + +namespace mlperf { + +namespace c { + +/// \brief Optional opaque client data that creators of SUTs and QSLs can have +/// the loadgen pass back to their callback invocations. +/// Helps avoids global variables. +typedef uintptr_t ClientData; + +typedef void (*IssueQueryCallback)(ClientData, const QuerySample*, size_t); +typedef void (*FlushQueriesCallback)(); +typedef void (*ResponseCallback)(ClientData, QuerySampleResponse*); + +/// \brief SUT calls this function to report query result back to loadgen +void QuerySamplesComplete(QuerySampleResponse* responses, + size_t response_count); + +void QuerySamplesCompleteResponseCb(QuerySampleResponse* responses, + size_t response_count, + ResponseCallback response_cb, + ClientData client_data); + +void FirstTokenComplete(QuerySampleResponse* responses, size_t response_count); + +void FirstTokenCompleteResponseCb(QuerySampleResponse* responses, + size_t response_count, + ResponseCallback response_cb, + ClientData client_data); + +/// \brief Create an opaque SUT pointer based on C callbacks. +void* ConstructSUT(ClientData client_data, const char* name, size_t name_length, + IssueQueryCallback issue_cb, + FlushQueriesCallback flush_queries_cb); +/// \brief Destroys the SUT created by ConstructSUT. +void DestroySUT(void* sut); + +typedef void (*LoadSamplesToRamCallback)(ClientData, const QuerySampleIndex*, + size_t); +typedef void (*UnloadSamplesFromRamCallback)(ClientData, + const QuerySampleIndex*, size_t); + +/// \brief Create an opaque QSL pointer based on C callbacks. +void* ConstructQSL(ClientData client_data, const char* name, size_t name_length, + size_t total_sample_count, size_t performance_sample_count, + LoadSamplesToRamCallback load_samples_to_ram_cb, + UnloadSamplesFromRamCallback unload_samples_from_ram_cb); +/// \brief Destroys the QSL created by ConsructQSL. +void DestroyQSL(void* qsl); + +/// \brief Run tests on a SUT created by ConstructSUT(). +/// \details This is the C entry point. See mlperf::StartTest for the C++ entry +/// point. +void StartTest(void* sut, void* qsl, const TestSettings& settings, + const std::string& audit_config_filename); + +/// +/// \brief Register a thread for query issuing in Server scenario. +/// \details This is the C entry point. See mlperf::RegisterIssueQueryThread for +/// the C++ entry point. +/// +void RegisterIssueQueryThread(); + +} // namespace c +} // namespace mlperf + +#endif // SYSTEM_UNDER_TEST_C_API_H_ diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/bindings/python_api.cc b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/bindings/python_api.cc new file mode 100644 index 000000000..96396dab9 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/bindings/python_api.cc @@ -0,0 +1,484 @@ +/* Copyright 2019 The MLPerf Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +/// \file +/// \brief Python bindings for the loadgen using pybind11. + +#ifndef PYTHON_BINDINGS_H +#define PYTHON_BINDINGS_H + +#include + +#include "../loadgen.h" +#include "../query_dispatch_library.h" +#include "../query_sample.h" +#include "../query_sample_library.h" +#include "../system_under_test.h" +#include "../test_settings.h" +#include "pybind11/functional.h" +#include "pybind11/pybind11.h" +#include "pybind11/stl.h" +#include "pybind11/stl_bind.h" + +namespace mlperf { + +namespace { + +using IssueQueryCallback = std::function)>; +using FastIssueQueriesCallback = + std::function, std::vector)>; +using FlushQueriesCallback = std::function; +using NameCallback = std::function; + +// Forwards SystemUnderTest calls to relevant callbacks. +class SystemUnderTestTrampoline : public SystemUnderTest { + public: + SystemUnderTestTrampoline(std::string name, IssueQueryCallback issue_cb, + FlushQueriesCallback flush_queries_cb) + : name_(std::move(name)), + issue_cb_(issue_cb), + flush_queries_cb_(flush_queries_cb) {} + ~SystemUnderTestTrampoline() override = default; + + const std::string& Name() override { return name_; } + + void IssueQuery(const std::vector& samples) override { + pybind11::gil_scoped_acquire gil_acquirer; + issue_cb_(samples); + } + + void FlushQueries() override { flush_queries_cb_(); } + + protected: + std::string name_; + IssueQueryCallback issue_cb_; + FlushQueriesCallback flush_queries_cb_; +}; + +class FastSystemUnderTestTrampoline : public SystemUnderTestTrampoline { + public: + FastSystemUnderTestTrampoline(std::string name, + FastIssueQueriesCallback fast_issue_cb, + FlushQueriesCallback flush_queries_cb) + : SystemUnderTestTrampoline(name, nullptr, flush_queries_cb), + fast_issue_cb_(fast_issue_cb) {} + ~FastSystemUnderTestTrampoline() override = default; + + void IssueQuery(const std::vector& samples) override { + pybind11::gil_scoped_acquire gil_acquirer; + std::vector responseIds; + std::vector querySampleIndices; + for (auto& s : samples) { + responseIds.push_back(s.id); + querySampleIndices.push_back(s.index); + } + fast_issue_cb_(responseIds, querySampleIndices); + } + + private: + FastIssueQueriesCallback fast_issue_cb_; +}; + +using LoadSamplesToRamCallback = + std::function)>; +using UnloadSamplesFromRamCallback = + std::function)>; + +// Forwards QuerySampleLibrary calls to relevant callbacks. +class QuerySampleLibraryTrampoline : public QuerySampleLibrary { + public: + QuerySampleLibraryTrampoline( + std::string name, size_t total_sample_count, + size_t performance_sample_count, + LoadSamplesToRamCallback load_samples_to_ram_cb, + UnloadSamplesFromRamCallback unload_samples_from_ram_cb) + : name_(std::move(name)), + total_sample_count_(total_sample_count), + performance_sample_count_(performance_sample_count), + load_samples_to_ram_cb_(load_samples_to_ram_cb), + unload_samples_from_ram_cb_(unload_samples_from_ram_cb) {} + ~QuerySampleLibraryTrampoline() override = default; + + const std::string& Name() override { return name_; } + size_t TotalSampleCount() { return total_sample_count_; } + size_t PerformanceSampleCount() { return performance_sample_count_; } + + void LoadSamplesToRam(const std::vector& samples) override { + pybind11::gil_scoped_acquire gil_acquirer; + load_samples_to_ram_cb_(samples); + } + void UnloadSamplesFromRam( + const std::vector& samples) override { + pybind11::gil_scoped_acquire gil_acquirer; + unload_samples_from_ram_cb_(samples); + } + + private: + std::string name_; + size_t total_sample_count_; + size_t performance_sample_count_; + LoadSamplesToRamCallback load_samples_to_ram_cb_; + UnloadSamplesFromRamCallback unload_samples_from_ram_cb_; +}; + +// A QDL that allows defining callbacks for +// IssueQuery, FlushQueries, and Name methods. +class QueryDispatchLibraryTrampoline : public QueryDispatchLibrary { + public: + QueryDispatchLibraryTrampoline(IssueQueryCallback issue_query_callback, + FlushQueriesCallback flush_queries_callback, + NameCallback name_callback) + : issue_query_callback_(issue_query_callback), + flush_queries_callback_(flush_queries_callback), + name_callback_(name_callback) {} + + // Returns the name of the SUT. Name shall be returned over the network + // TODO: other bindings should also be fixed eventually to be used over the + // network + const std::string& Name() override { + static std::string name; // HACK: avoid returning a reference to temporary. + pybind11::gil_scoped_acquire gil_acquirer; + name = name_callback_(); // name_callback_() shall returned name over the + // network. + return name; + } + + void IssueQuery(const std::vector& samples) override { + pybind11::gil_scoped_acquire gil_acquirer; + issue_query_callback_(samples); + } + + void FlushQueries() override { flush_queries_callback_(); } + + protected: + IssueQueryCallback issue_query_callback_; + FlushQueriesCallback flush_queries_callback_; + NameCallback name_callback_; +}; + +} // namespace + +/// \brief Python bindings. +namespace py { + +uintptr_t ConstructSUT(IssueQueryCallback issue_cb, + FlushQueriesCallback flush_queries_cb) { + SystemUnderTestTrampoline* sut = + new SystemUnderTestTrampoline("PySUT", issue_cb, flush_queries_cb); + return reinterpret_cast(sut); +} + +void DestroySUT(uintptr_t sut) { + SystemUnderTestTrampoline* sut_cast = + reinterpret_cast(sut); + delete sut_cast; +} + +uintptr_t ConstructFastSUT(FastIssueQueriesCallback fast_issue_cb, + FlushQueriesCallback flush_queries_cb) { + FastSystemUnderTestTrampoline* sut = new FastSystemUnderTestTrampoline( + "PyFastSUT", fast_issue_cb, flush_queries_cb); + return reinterpret_cast(sut); +} + +void DestroyFastSUT(uintptr_t sut) { + FastSystemUnderTestTrampoline* sut_cast = + reinterpret_cast(sut); + delete sut_cast; +} + +uintptr_t ConstructQSL( + size_t total_sample_count, size_t performance_sample_count, + LoadSamplesToRamCallback load_samples_to_ram_cb, + UnloadSamplesFromRamCallback unload_samples_from_ram_cb) { + QuerySampleLibraryTrampoline* qsl = new QuerySampleLibraryTrampoline( + "PyQSL", total_sample_count, performance_sample_count, + load_samples_to_ram_cb, unload_samples_from_ram_cb); + return reinterpret_cast(qsl); +} + +void DestroyQSL(uintptr_t qsl) { + QuerySampleLibraryTrampoline* qsl_cast = + reinterpret_cast(qsl); + delete qsl_cast; +} + +uintptr_t ConstructQDL(IssueQueryCallback issue_cb, + FlushQueriesCallback flush_queries_cb, + NameCallback name_callback) { + QueryDispatchLibraryTrampoline* qdl = new QueryDispatchLibraryTrampoline( + issue_cb, flush_queries_cb, name_callback); + return reinterpret_cast(qdl); +} + +void DestroyQDL(uintptr_t qdl) { + QueryDispatchLibraryTrampoline* qdl_cast = + reinterpret_cast(qdl); + delete qdl_cast; +} + +void StartTest(uintptr_t sut, uintptr_t qsl, mlperf::TestSettings test_settings, + const std::string& audit_config_filename) { + pybind11::gil_scoped_release gil_releaser; + SystemUnderTestTrampoline* sut_cast = + reinterpret_cast(sut); + QuerySampleLibraryTrampoline* qsl_cast = + reinterpret_cast(qsl); + LogSettings default_log_settings; + mlperf::StartTest(sut_cast, qsl_cast, test_settings, default_log_settings, + audit_config_filename); +} + +void StartTestWithLogSettings(uintptr_t sut, uintptr_t qsl, + mlperf::TestSettings test_settings, + mlperf::LogSettings log_settings, + const std::string& audit_config_filename) { + pybind11::gil_scoped_release gil_releaser; + SystemUnderTestTrampoline* sut_cast = + reinterpret_cast(sut); + QuerySampleLibraryTrampoline* qsl_cast = + reinterpret_cast(qsl); + mlperf::StartTest(sut_cast, qsl_cast, test_settings, log_settings, + audit_config_filename); +} + +using ResponseCallback = std::function; + +/// TODO: Get rid of copies. +void QuerySamplesComplete(std::vector responses, + ResponseCallback response_cb = {}) { + pybind11::gil_scoped_release gil_releaser; + mlperf::QuerySamplesComplete(responses.data(), responses.size(), response_cb); +} + +void FirstTokenComplete(std::vector responses, + ResponseCallback response_cb = {}) { + pybind11::gil_scoped_release gil_releaser; + mlperf::FirstTokenComplete(responses.data(), responses.size(), response_cb); +} + +PYBIND11_MODULE(mlperf_loadgen, m) { + m.doc() = "MLPerf Inference load generator."; + + pybind11::enum_(m, "TestScenario") + .value("SingleStream", TestScenario::SingleStream) + .value("MultiStream", TestScenario::MultiStream) + .value("Server", TestScenario::Server) + .value("Offline", TestScenario::Offline); + + pybind11::enum_(m, "TestMode") + .value("SubmissionRun", TestMode::SubmissionRun) + .value("AccuracyOnly", TestMode::AccuracyOnly) + .value("PerformanceOnly", TestMode::PerformanceOnly) + .value("FindPeakPerformance", TestMode::FindPeakPerformance); + + pybind11::class_(m, "TestSettings") + .def(pybind11::init<>()) + .def_readwrite("scenario", &TestSettings::scenario) + .def_readwrite("mode", &TestSettings::mode) + .def_readwrite("single_stream_expected_latency_ns", + &TestSettings::single_stream_expected_latency_ns) + .def_readwrite("single_stream_target_latency_percentile", + &TestSettings::single_stream_target_latency_percentile) + .def_readwrite("multi_stream_expected_latency_ns", + &TestSettings::multi_stream_expected_latency_ns) + .def_readwrite("multi_stream_target_latency_percentile", + &TestSettings::multi_stream_target_latency_percentile) + .def_readwrite("multi_stream_samples_per_query", + &TestSettings::multi_stream_samples_per_query) + .def_readwrite("server_target_qps", &TestSettings::server_target_qps) + .def_readwrite("server_target_latency_ns", + &TestSettings::server_target_latency_ns) + .def_readwrite("server_target_latency_percentile", + &TestSettings::server_target_latency_percentile) + .def_readwrite("server_coalesce_queries", + &TestSettings::server_coalesce_queries) + .def_readwrite("server_find_peak_qps_decimals_of_precision", + &TestSettings::server_find_peak_qps_decimals_of_precision) + .def_readwrite("server_find_peak_qps_boundary_step_size", + &TestSettings::server_find_peak_qps_boundary_step_size) + .def_readwrite("server_max_async_queries", + &TestSettings::server_max_async_queries) + .def_readwrite("server_num_issue_query_threads", + &TestSettings::server_num_issue_query_threads) + .def_readwrite("offline_expected_qps", + &TestSettings::offline_expected_qps) + .def_readwrite("min_duration_ms", &TestSettings::min_duration_ms) + .def_readwrite("max_duration_ms", &TestSettings::max_duration_ms) + .def_readwrite("min_query_count", &TestSettings::min_query_count) + .def_readwrite("max_query_count", &TestSettings::max_query_count) + .def_readwrite("qsl_rng_seed", &TestSettings::qsl_rng_seed) + .def_readwrite("sample_index_rng_seed", + &TestSettings::sample_index_rng_seed) + .def_readwrite("schedule_rng_seed", &TestSettings::schedule_rng_seed) + .def_readwrite("accuracy_log_rng_seed", + &TestSettings::accuracy_log_rng_seed) + .def_readwrite("accuracy_log_probability", + &TestSettings::accuracy_log_probability) + .def_readwrite("print_timestamps", &TestSettings::print_timestamps) + .def_readwrite("performance_issue_unique", + &TestSettings::performance_issue_unique) + .def_readwrite("performance_issue_same", + &TestSettings::performance_issue_same) + .def_readwrite("performance_issue_same_index", + &TestSettings::performance_issue_same_index) + .def_readwrite("performance_sample_count_override", + &TestSettings::performance_sample_count_override) + .def_readwrite("test05", &TestSettings::test05) + .def_readwrite("test05_qsl_rng_seed", &TestSettings::test05_qsl_rng_seed) + .def_readwrite("test05_sample_index_rng_seed", + &TestSettings::test05_sample_index_rng_seed) + .def_readwrite("test05_schedule_rng_seed", + &TestSettings::test05_schedule_rng_seed) + .def_readwrite("use_token_latencies", &TestSettings::use_token_latencies) + .def_readwrite("ttft_latency", &TestSettings::server_ttft_latency) + .def_readwrite("tpot_latency", &TestSettings::server_tpot_latency) + .def_readwrite("infer_token_latencies", + &TestSettings::infer_token_latencies) + .def_readwrite("token_latency_scaling_factor", + &TestSettings::token_latency_scaling_factor) + .def("FromConfig", &TestSettings::FromConfig, pybind11::arg("path"), + pybind11::arg("model"), pybind11::arg("scenario"), + pybind11::arg("conf_type") = 1, + "This function configures settings from the given user " + "configuration file, model, and scenario. The conf_type flag " + "should be set to 1 for loading user.conf or else only the default " + "mlperf_conf file " + "will be loaded by the loadgen."); + + pybind11::enum_(m, "LoggingMode") + .value("AsyncPoll", LoggingMode::AsyncPoll) + .value("EndOfTestOnly", LoggingMode::EndOfTestOnly) + .value("Synchronous", LoggingMode::Synchronous); + + pybind11::class_(m, "LogOutputSettings") + .def(pybind11::init<>()) + .def_readwrite("outdir", &LogOutputSettings::outdir) + .def_readwrite("prefix", &LogOutputSettings::prefix) + .def_readwrite("suffix", &LogOutputSettings::suffix) + .def_readwrite("prefix_with_datetime", + &LogOutputSettings::prefix_with_datetime) + .def_readwrite("copy_detail_to_stdout", + &LogOutputSettings::copy_detail_to_stdout) + .def_readwrite("copy_summary_to_stdout", + &LogOutputSettings::copy_summary_to_stdout); + + pybind11::class_(m, "LogSettings") + .def(pybind11::init<>()) + .def_readwrite("log_output", &LogSettings::log_output) + .def_readwrite("log_mode", &LogSettings::log_mode) + .def_readwrite("log_mode_async_poll_interval_ms", + &LogSettings::log_mode_async_poll_interval_ms) + .def_readwrite("enable_trace", &LogSettings::enable_trace); + + pybind11::class_(m, "QuerySample") + .def(pybind11::init<>()) + .def(pybind11::init()) + .def_readwrite("id", &QuerySample::id) + .def_readwrite("index", &QuerySample::index) + .def(pybind11::pickle( + [](const QuerySample& qs) { // __getstate__ + /*Return a tuple that fully encodes state of object*/ + return pybind11::make_tuple(qs.id, qs.index); + }, + [](pybind11::tuple t) { // __setstate__ + if (t.size() != 2) + throw std::runtime_error("Invalid state for QuerySample"); + /* Create a new C++ instance*/ + QuerySample q; + q.id = t[0].cast(); + q.index = t[1].cast(); + return q; + })); + + pybind11::class_(m, "QuerySampleResponse") + .def(pybind11::init<>()) + .def(pybind11::init()) + .def(pybind11::init()) + .def_readwrite("id", &QuerySampleResponse::id) + .def_readwrite("data", &QuerySampleResponse::data) + .def_readwrite("size", &QuerySampleResponse::size) + .def_readwrite("n_tokens", &QuerySampleResponse::n_tokens) + .def(pybind11::pickle( + [](const QuerySampleResponse& qsr) { // __getstate__ + /* Return a tuple that fully encodes state of object*/ + return pybind11::make_tuple(qsr.id, qsr.data, qsr.size); + }, + [](pybind11::tuple t) { // __setstate__ + if ((t.size() != 3) || (t.size() != 4)) + throw std::runtime_error("Invalid state for QuerySampleResponse"); + /* Create a new C++ instance*/ + QuerySampleResponse q; + q.id = t[0].cast(); + q.data = t[1].cast(); + q.size = t[2].cast(); + if (t.size() == 4) { + q.n_tokens = t[3].cast(); + } else { + q.n_tokens = 0; + } + return q; + })); + + // TODO: Use PYBIND11_MAKE_OPAQUE for the following vector types. + pybind11::bind_vector>(m, "VectorQuerySample"); + pybind11::bind_vector>( + m, "VectorQuerySampleResponse"); + + m.def("ConstructSUT", &py::ConstructSUT, "Construct the system under test."); + m.def("DestroySUT", &py::DestroySUT, + "Destroy the object created by ConstructSUT."); + + m.def("ConstructFastSUT", &py::ConstructFastSUT, + "Construct the system under test, fast issue query"); + m.def("DestroyFastSUT", &py::DestroyFastSUT, + "Destroy the object created by ConstructFastSUT."); + + m.def("ConstructQSL", &py::ConstructQSL, + "Construct the query sample library."); + m.def("DestroyQSL", &py::DestroyQSL, + "Destroy the object created by ConstructQSL."); + + m.def("ConstructQDL", &py::ConstructQDL, + "Construct the query sample library, communicating with the SUT over " + "the network."); + m.def("DestroyQDL", &py::DestroyQDL, + "Destroy the object created by ConstructQDL."); + + m.def("StartTest", &py::StartTest, + "Run tests on a SUT created by ConstructSUT() with the provided QSL. " + "Uses default log settings.", + pybind11::arg("sut"), pybind11::arg("qsl"), + pybind11::arg("test_settings"), + pybind11::arg("audit_config_filename") = "audit.config"); + m.def("StartTestWithLogSettings", &py::StartTestWithLogSettings, + "Run tests on a SUT created by ConstructSUT() with the provided QSL. " + "Accepts custom log settings.", + pybind11::arg("sut"), pybind11::arg("qsl"), + pybind11::arg("test_settings"), pybind11::arg("log_settings"), + pybind11::arg("audit_config_filename") = "audit.config"); + m.def("QuerySamplesComplete", &py::QuerySamplesComplete, + "Called by the SUT to indicate that samples from some combination of" + "IssueQuery calls have finished.", + pybind11::arg("responses"), + pybind11::arg("response_cb") = ResponseCallback{}); + m.def("FirstTokenComplete", &py::FirstTokenComplete, + "Called by the SUT to indicate that tokens from some combination of" + "IssueQuery calls have finished.", + pybind11::arg("responses"), + pybind11::arg("response_cb") = ResponseCallback{}); +} + +} // namespace py +} // namespace mlperf + +#endif // PYTHON_BINDINGS_H diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/lon/README.md b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/lon/README.md new file mode 100644 index 000000000..f46e22a65 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/lon/README.md @@ -0,0 +1,67 @@ +# Demo + +## Loadgen Over the Network + +### Overview + + +This folder provides a demo implementation for LoadGen over the network.\ +Two sides are implemented: + +1. The SUT side which is implemented in [sut_over_network_demo.py](sut_over_network_demo.py). Each Node should run it for multiple Nodes operation. +2. The LoadGen node running the LoadGen, QSL and QDL instances, implemented in [py_demo_server_lon.py](py_demo_server_lon.py) + +The demo SUT is implemented with a Flask server. the LON node implements a Flask client for network operation. + +The test runs in MLPerf Server mode. the SUT is not implementing a benchmark but contains dummy interface to preprocessing, postprocessing and model calling functions. + +### Setup + +Install python packages: + +```sh +pip install absl-py numpy wheel flask requests +``` + +Clone: + +```sh +git clone --recurse-submodules https://github.com/mlcommons/inference.git mlperf_inference +``` + +Build: + +```sh +cd mlperf_inference/loadgen +CFLAGS="-std=c++14 -O3" python setup.py bdist_wheel +cd ..; pip install --force-reinstall loadgen/dist/`ls -r loadgen/dist/ | head -n1` ; cd - +``` + +### Run the demo (single machine) + +Start the demo SUT server (run this at a separate terminal): + +```sh +python demos/lon/sut_over_network_demo.py --port 8000 +``` + +Start the test: + +```sh +python demos/lon/py_demo_server_lon.py --sut_server http://localhost:8000 +``` + +### Run the demo (over the network) + +To run over a network - simply run the demo SUT over on a different machine. For multiple Nodes run the demo SUT on each machine specifying the node number.\ + +```sh +python demos/lon/sut_over_network_demo.py --port 8000 --node N1 +``` + +Then, when running the client, replace `localhost` with the correct IP. + + +```sh +python demos/lon/py_demo_server_lon.py --sut_server IP1:8000,IP2:8000,IP3:8000 +``` diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/lon/py_demo_server_lon.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/lon/py_demo_server_lon.py new file mode 100644 index 000000000..1248215db --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/lon/py_demo_server_lon.py @@ -0,0 +1,191 @@ +# Copyright 2019 The MLPerf Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +""" +Python demo showing how to use the MLPerf Inference LoadGen over the Network bindings. +This programs runs in the LON Node side. +It runs the demo in MLPerf server mode over the network. +It communicates over the network with a Network SUT node, +which is running the Network SUT demo based on a flask server, implemented in SUT_over_network.py +""" + +import threading +import requests +import array +import time + +from absl import app +from absl import flags +import mlperf_loadgen + +FLAGS = flags.FLAGS + +flags.DEFINE_list( + "sut_server", "http://localhost:8000", "Address of the server(s) under test." +) + + +class QSL: + """Demo QuerySampleLibrary with dummy features.""" + + def __init__(self, total_sample_count, performance_sample_count): + self.eval_features = { + i: f"what_is_my_dummy_feature_{i}?" for i in range(total_sample_count) + } + self.qsl = mlperf_loadgen.ConstructQSL( + total_sample_count, + performance_sample_count, + self.load_samples_to_ram, + self.unload_samples_from_ram, + ) + + def get_features(self, sample_id): + """Returns the feature for a given sample id.""" + return self.eval_features[sample_id] + + def load_samples_to_ram(self, query_samples): + """Loads the features for the given query samples into RAM.""" + # Current implementation is not using this functionality. + del query_samples + return + + def unload_samples_from_ram(self, query_samples): + """Unloads the features for the given query samples from RAM.""" + # Current implementation is not using this functionality. + del query_samples + return + + def __del__(self): + mlperf_loadgen.DestroyQSL(self.qsl) + + +class QDL: + """QDL acting as a proxy to the SUT. + This QDL communicates with the SUT via HTTP. + It uses two endpoints to communicate with the SUT: + - /predict/ : Send a query to the SUT and get a response. + - /getname/ : Get the name of the SUT. Send a getname to the SUT and get a response. + """ + + def __init__(self, qsl: QSL, sut_server_addr: list): + """ + Constructor for the QDL. + Args: + qsl: The QSL to use. + sut_server_addr: A list of addresses of the SUT. + """ + self.qsl = qsl + + # Construct QDL from the python binding + self.qdl = mlperf_loadgen.ConstructQDL( + self.issue_query, self.flush_queries, self.client_get_name + ) + self.sut_server_addr = sut_server_addr + self.num_nodes = len(sut_server_addr) + + # For round robin between the SUTs: + self.next_sut_id = 0 + self.lock = threading.Lock() + + def issue_query(self, query_samples): + """Process the query to send to the SUT""" + threading.Thread( + target=self.process_query_async, + args=[query_samples]).start() + + def flush_queries(self): + """Flush the queries. Dummy implementation.""" + pass + + def process_query_async(self, query_samples): + """ + This function is called by the Loadgen in a separate thread. + It is responsible for + 1. Creating a query for the SUT, by reading the features from the QSL. + 2. Sending the query to the SUT. + 3. Waiting for the response from the SUT. + 4. Deserializing the response. + 5. Calling mlperf_loadgen.QuerySamplesComplete(query_samples, response) + Args: + query_samples: A list of QuerySample objects. + """ + responses = [] + for s in query_samples: + # Overall process: + # QDL builds a real-world query and sends to SUT --> SUT processes --> SUT sends back to QDL + # Read features from the QSL + features = self.qsl.get_features(s.index) + + time.sleep(0.001) # Ensure a maximal rate of queries to the SUT + + # Send the query to SUT in round robin + # Wait for a response + sut_result = self.client_predict(features, s.index) + response_array = array.array("B", sut_result.encode("utf-8")) + bi = response_array.buffer_info() + responses.append( + mlperf_loadgen.QuerySampleResponse( + s.id, bi[0], bi[1])) + mlperf_loadgen.QuerySamplesComplete(responses) + + def get_sut_id_round_robin(self): + """Get the SUT id in round robin.""" + with self.lock: + res = self.next_sut_id + self.next_sut_id = (self.next_sut_id + 1) % self.num_nodes + return res + + def client_predict(self, query, id): + """Serialize the query, send it to the SUT in round robin, and return the deserialized response.""" + url = "{}/predict/".format( + self.sut_server_addr[self.get_sut_id_round_robin()]) + response = requests.post(url, json={"query": query, id: id}) + return response.json()["result"] + + def client_get_name(self): + """Get the name of the SUT from ALL the SUTS.""" + if len(self.sut_server_addr) == 1: + return requests.post( + f"{self.sut_server_addr[0]}/getname/").json()["name"] + + sut_names = [ + requests.post(f"{addr}/getname/").json()["name"] + for addr in self.sut_server_addr + ] + return "Multi-node SUT: " + ", ".join(sut_names) + + def __del__(self): + mlperf_loadgen.DestroyQDL(self.qdl) + + +def main(argv): + del argv + settings = mlperf_loadgen.TestSettings() + settings.scenario = mlperf_loadgen.TestScenario.Server + settings.mode = mlperf_loadgen.TestMode.PerformanceOnly + settings.server_target_qps = 100 + settings.server_target_latency_ns = 100000000 + settings.min_query_count = 100 + settings.min_duration_ms = 10000 + + # QDL and QSL + qsl = QSL(1024, 128) + qdl = QDL(qsl, sut_server_addr=FLAGS.sut_server) + + mlperf_loadgen.StartTest(qdl.qdl, qsl.qsl, settings) + + +if __name__ == "__main__": + app.run(main) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/lon/sut_over_network_demo.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/lon/sut_over_network_demo.py new file mode 100644 index 000000000..55e5e038d --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/lon/sut_over_network_demo.py @@ -0,0 +1,88 @@ +# Copyright 2019 The MLPerf Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + + +""" +Python demo showing how to use the MLPerf Inference load generator bindings over the network. +This part of the demo runs the "demo SUT" which is connected over the network to the LON node. +A corresponding "demo LON node" with the demo test is implemented in py_demo_server_lon.py. + +The SUT is implemented using a Flask server, with dummy implementation of the inference processing. +Two endpoints are exposed: +- /predict/ : Receives a query (e.g., a text) runs inference, and returns a prediction. +- /getname/ : Get the name of the SUT. + +The current implementation is a dummy implementation, which does not use +a real DNN model, batching, or pre/postprocessing code, +but rather just returns subset of the input query as a response, +Yet, it illustrates the basic structure of a SUT server. +""" + +import argparse +from flask import Flask, request, jsonify + + +app = Flask(__name__) + + +node = "" + + +def preprocess(query): + """[SUT Node] A dummy preprocess.""" + # Here may come for example batching, tokenization, resizing, + # normalization, etc. + response = query + return response + + +def dnn_model(query): + """[SUT Node] A dummy DNN model.""" + # Here may come for example a call to a dnn model such as resnet, bert, + # etc. + response = query + return response + + +def postprocess(query): + """[SUT Node] A dummy postprocess.""" + # Here may come for example a postprocessing call, e.g., NMS, + # detokenization, etc. + response = query + return response + + +@app.route("/predict/", methods=["POST"]) +def predict(): + """Receives a query (e.g., a text) runs inference, and returns a prediction.""" + query = request.get_json(force=True)["query"] + result = postprocess(dnn_model(preprocess(query))) + return jsonify(result=result) + + +@app.route("/getname/", methods=["POST", "GET"]) +def getname(): + """Returns the name of the SUT.""" + return jsonify(name=f"Demo SUT (Network SUT) node" + + (" " + node) if node else "") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--port", type=int, default=8000) + parser.add_argument("--node", type=str, default="") + args = parser.parse_args() + node = args.node + app.run(debug=False, port=args.port) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/py_demo_multi_stream.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/py_demo_multi_stream.py new file mode 100644 index 000000000..f6082cad6 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/py_demo_multi_stream.py @@ -0,0 +1,86 @@ +# Copyright 2019 The MLPerf Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +"""Python demo showing how to use the MLPerf Inference load generator bindings. +""" + +from __future__ import print_function + +import threading +import time + +from absl import app +import mlperf_loadgen + +from datetime import datetime + +# Global var +NUM_AGENTS = 8 +LOOPBACK_LATENCY_S = 0.001 + + +def load_samples_to_ram(query_samples): + del query_samples + return + + +def unload_samples_from_ram(query_samples): + del query_samples + return + + +# Processes queries in NUM_AGENTS slices that complete at different times. +def process_query_async(query_samples, i_slice): + time.sleep(LOOPBACK_LATENCY_S * (i_slice + 1)) + responses = [] + samples_to_complete = query_samples[i_slice: len( + query_samples): NUM_AGENTS] + for j, s in enumerate(samples_to_complete): + responses.append(mlperf_loadgen.QuerySampleResponse(s.id, 0, 0)) + mlperf_loadgen.QuerySamplesComplete(responses) + + +def issue_query(query_samples): + for i in range(8): + threading.Thread( + target=process_query_async, args=( + query_samples, i)).start() + + +def flush_queries(): + pass + + +def main(argv): + del argv + settings = mlperf_loadgen.TestSettings() + settings.scenario = mlperf_loadgen.TestScenario.MultiStream + settings.mode = mlperf_loadgen.TestMode.PerformanceOnly + settings.multi_stream_expected_latency_ns = 8000000 + settings.multi_stream_samples_per_query = 8 + settings.min_query_count = 100 + settings.min_duration_ms = 10000 + + sut = mlperf_loadgen.ConstructSUT(issue_query, flush_queries) + qsl = mlperf_loadgen.ConstructQSL( + 1024, 128, load_samples_to_ram, unload_samples_from_ram + ) + mlperf_loadgen.StartTest(sut, qsl, settings) + mlperf_loadgen.DestroyQSL(qsl) + mlperf_loadgen.DestroySUT(sut) + + +if __name__ == "__main__": + app.run(main) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/py_demo_offline.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/py_demo_offline.py new file mode 100644 index 000000000..909585edc --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/py_demo_offline.py @@ -0,0 +1,81 @@ +# Copyright 2019 The MLPerf Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +"""Python demo showing how to use the MLPerf Inference load generator bindings. +""" + +from __future__ import print_function + +import threading +import time + +from absl import app +import mlperf_loadgen + + +def load_samples_to_ram(query_samples): + del query_samples + return + + +def unload_samples_from_ram(query_samples): + del query_samples + return + + +# Processes queries in 3 slices that complete at different times. +def process_query_async(query_samples, i_slice): + time.sleep(3 * (i_slice + 1)) + responses = [] + samples_to_complete = query_samples[i_slice: len(query_samples): 3] + for s in samples_to_complete: + responses.append(mlperf_loadgen.QuerySampleResponse(s.id, 0, 0)) + mlperf_loadgen.QuerySamplesComplete(responses) + + +def issue_query(query_samples): + threading.Thread( + target=process_query_async, args=( + query_samples, 0)).start() + threading.Thread( + target=process_query_async, args=( + query_samples, 1)).start() + threading.Thread( + target=process_query_async, args=( + query_samples, 2)).start() + + +def flush_queries(): + pass + + +def main(argv): + del argv + settings = mlperf_loadgen.TestSettings() + settings.scenario = mlperf_loadgen.TestScenario.Offline + settings.mode = mlperf_loadgen.TestMode.PerformanceOnly + settings.offline_expected_qps = 1000 + + sut = mlperf_loadgen.ConstructSUT(issue_query, flush_queries) + qsl = mlperf_loadgen.ConstructQSL( + 1024, 128, load_samples_to_ram, unload_samples_from_ram + ) + mlperf_loadgen.StartTest(sut, qsl, settings) + mlperf_loadgen.DestroyQSL(qsl) + mlperf_loadgen.DestroySUT(sut) + + +if __name__ == "__main__": + app.run(main) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/py_demo_server.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/py_demo_server.py new file mode 100644 index 000000000..8b6f2b826 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/py_demo_server.py @@ -0,0 +1,74 @@ +# Copyright 2019 The MLPerf Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +"""Python demo showing how to use the MLPerf Inference load generator bindings. +""" + +from __future__ import print_function + +import threading +import time + +from absl import app +import mlperf_loadgen + + +def load_samples_to_ram(query_samples): + del query_samples + return + + +def unload_samples_from_ram(query_samples): + del query_samples + return + + +def process_query_async(query_samples): + time.sleep(0.001) + responses = [] + for s in query_samples: + responses.append(mlperf_loadgen.QuerySampleResponse(s.id, 0, 0)) + mlperf_loadgen.QuerySamplesComplete(responses) + + +def issue_query(query_samples): + threading.Thread(target=process_query_async, args=[query_samples]).start() + + +def flush_queries(): + pass + + +def main(argv): + del argv + settings = mlperf_loadgen.TestSettings() + settings.scenario = mlperf_loadgen.TestScenario.Server + settings.mode = mlperf_loadgen.TestMode.PerformanceOnly + settings.server_target_qps = 100 + settings.server_target_latency_ns = 100000000 + settings.min_query_count = 100 + settings.min_duration_ms = 10000 + + sut = mlperf_loadgen.ConstructSUT(issue_query, flush_queries) + qsl = mlperf_loadgen.ConstructQSL( + 1024, 128, load_samples_to_ram, unload_samples_from_ram + ) + mlperf_loadgen.StartTest(sut, qsl, settings) + mlperf_loadgen.DestroyQSL(qsl) + mlperf_loadgen.DestroySUT(sut) + + +if __name__ == "__main__": + app.run(main) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/py_demo_single_stream.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/py_demo_single_stream.py new file mode 100644 index 000000000..8806271bd --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/py_demo_single_stream.py @@ -0,0 +1,84 @@ +# Copyright 2019 The MLPerf Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +"""Python demo showing how to use the MLPerf Inference load generator bindings. +""" + +from __future__ import print_function + +import array +import threading +import time + +from absl import app +import mlperf_loadgen + + +def load_samples_to_ram(query_samples): + del query_samples + return + + +def unload_samples_from_ram(query_samples): + del query_samples + return + + +def process_query_async(query_samples): + """Processes the list of queries.""" + time.sleep(0.001) + responses = [] + response_array = array.array( + "f", [0, 1, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128, 254, 255] + ) + response_info = response_array.buffer_info() + response_data = response_info[0] + response_size = response_info[1] * response_array.itemsize + for s in query_samples: + responses.append( + mlperf_loadgen.QuerySampleResponse( + s.id, response_data, response_size) + ) + mlperf_loadgen.QuerySamplesComplete(responses) + + +def issue_query(query_samples): + threading.Thread(target=process_query_async, args=[query_samples]).start() + + +def flush_queries(): + pass + + +def main(argv): + del argv + settings = mlperf_loadgen.TestSettings() + settings.scenario = mlperf_loadgen.TestScenario.SingleStream + settings.mode = mlperf_loadgen.TestMode.PerformanceOnly + settings.single_stream_expected_latency_ns = 1000000 + settings.min_query_count = 100 + settings.min_duration_ms = 10000 + + sut = mlperf_loadgen.ConstructSUT(issue_query, flush_queries) + qsl = mlperf_loadgen.ConstructQSL( + 1024, 128, load_samples_to_ram, unload_samples_from_ram + ) + mlperf_loadgen.StartTest(sut, qsl, settings) + mlperf_loadgen.DestroyQSL(qsl) + mlperf_loadgen.DestroySUT(sut) + + +if __name__ == "__main__": + app.run(main) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_multi_stream.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_multi_stream.py new file mode 100644 index 000000000..e4b083853 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_multi_stream.py @@ -0,0 +1,142 @@ +# Copyright 2019 The MLPerf Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +"""Python demo showing how to use the MLPerf Inference load generator bindings. +""" + +from __future__ import print_function + +import argparse +import threading +import time +import numpy as np +import array + +import mlperf_loadgen + +from datetime import datetime + +# Global var +NUM_AGENTS = 8 +LOOPBACK_LATENCY_S = 0.001 + + +def f(x, y): + return 4 + 3 * x * y + x**3 + y**2 + + +def create_responses(n, m, mod=4): + r = [] + for i in range(n): + r.append([f(i, j) for j in range(m + (i % mod))]) + return r + + +responses = create_responses(1024, 20) + + +def load_samples_to_ram(query_samples): + del query_samples + return + + +def unload_samples_from_ram(query_samples): + del query_samples + return + + +# Processes queries in NUM_AGENTS slices that complete at different times. +def process_query_async(query_samples, i_slice): + time.sleep(LOOPBACK_LATENCY_S * (i_slice + 1)) + query_responses = [] + samples_to_complete = query_samples[i_slice: len( + query_samples): NUM_AGENTS] + for j, s in enumerate(samples_to_complete): + response_array = np.array(responses[s.index], np.int32) + token = response_array[0] + time.sleep(0.0002) + response_token = array.array("B", token.tobytes()) + response_token_info = response_token.buffer_info() + response_token_data = response_token_info[0] + response_token_size = response_token_info[1] * response_token.itemsize + mlperf_loadgen.FirstTokenComplete( + [ + mlperf_loadgen.QuerySampleResponse( + s.id, response_token_data, response_token_size + ) + ] + ) + time.sleep(0.02) + n_tokens = len(response_array) + response_array = array.array("B", response_array.tobytes()) + response_info = response_array.buffer_info() + response_data = response_info[0] + response_size = response_info[1] * response_array.itemsize + query_responses.append( + mlperf_loadgen.QuerySampleResponse( + s.id, response_data, response_size, n_tokens + ) + ) + mlperf_loadgen.QuerySamplesComplete(query_responses) + + +def issue_query(query_samples): + for i in range(8): + threading.Thread( + target=process_query_async, args=( + query_samples, i)).start() + + +def flush_queries(): + pass + + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--mode", choices=["performance", "accuracy"], default="performance" + ) + parser.add_argument("--expected-latency", type=int, default=8000000) + parser.add_argument("--samples-per-query", type=int, default=8) + parser.add_argument("--min-query-count", type=int, default=100) + parser.add_argument("--min-duration-ms", type=int, default=30000) + return parser.parse_args() + + +def main(): + args = get_args() + settings = mlperf_loadgen.TestSettings() + settings.scenario = mlperf_loadgen.TestScenario.MultiStream + if args.mode == "performance": + settings.mode = mlperf_loadgen.TestMode.PerformanceOnly + else: + settings.mode = mlperf_loadgen.TestMode.AccuracyOnly + settings.multi_stream_expected_latency_ns = args.expected_latency + settings.multi_stream_samples_per_query = args.samples_per_query + settings.min_query_count = args.min_query_count + settings.min_duration_ms = args.min_duration_ms + settings.use_token_latencies = True + + sut = mlperf_loadgen.ConstructSUT(issue_query, flush_queries) + qsl = mlperf_loadgen.ConstructQSL( + 1024, 128, load_samples_to_ram, unload_samples_from_ram + ) + mlperf_loadgen.StartTest(sut, qsl, settings) + mlperf_loadgen.DestroyQSL(qsl) + mlperf_loadgen.DestroySUT(sut) + + +if __name__ == "__main__": + main() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_offline.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_offline.py new file mode 100644 index 000000000..2e190cdd5 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_offline.py @@ -0,0 +1,130 @@ +# Copyright 2019 The MLPerf Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +"""Python demo showing how to use the MLPerf Inference load generator bindings. +""" + +from __future__ import print_function + +import argparse +import threading +import time +import numpy as np +import array + +import mlperf_loadgen + + +def f(x, y): + return 4 + 3 * x * y + x**3 + y**2 + + +def create_responses(n, m, mod=4): + r = [] + for i in range(n): + r.append([f(i, j) for j in range(m + (i % mod))]) + return r + + +responses = create_responses(1024, 20) + + +def load_samples_to_ram(query_samples): + del query_samples + return + + +def unload_samples_from_ram(query_samples): + del query_samples + return + + +# Processes queries in 3 slices that complete at different times. +def process_query_async(query_samples, i_slice): + time.sleep(3 * (i_slice + 1)) + query_responses = [] + samples_to_complete = query_samples[i_slice: len(query_samples): 3] + for s in samples_to_complete: + response_array = np.array(responses[s.index], np.int32) + token = response_array[0] + time.sleep(0.0002) + response_token = array.array("B", token.tobytes()) + response_token_info = response_token.buffer_info() + response_token_data = response_token_info[0] + response_token_size = response_token_info[1] * response_token.itemsize + # mlperf_loadgen.FirstTokenComplete([mlperf_loadgen.QuerySampleResponse(s.id, response_token_data, response_token_size)]) + time.sleep(0.02) + n_tokens = len(response_array) + response_array = array.array("B", response_array.tobytes()) + response_info = response_array.buffer_info() + response_data = response_info[0] + response_size = response_info[1] * response_array.itemsize + query_responses.append( + mlperf_loadgen.QuerySampleResponse( + s.id, response_data, response_size, n_tokens + ) + ) + mlperf_loadgen.QuerySamplesComplete(query_responses) + + +def issue_query(query_samples): + threading.Thread( + target=process_query_async, args=( + query_samples, 0)).start() + threading.Thread( + target=process_query_async, args=( + query_samples, 1)).start() + threading.Thread( + target=process_query_async, args=( + query_samples, 2)).start() + + +def flush_queries(): + pass + + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--mode", choices=["performance", "accuracy"], default="performance" + ) + parser.add_argument("--expected-qps", type=int, default=1000) + parser.add_argument("--min-duration-ms", type=int, default=30000) + return parser.parse_args() + + +def main(): + args = get_args() + settings = mlperf_loadgen.TestSettings() + settings.scenario = mlperf_loadgen.TestScenario.Offline + if args.mode == "performance": + settings.mode = mlperf_loadgen.TestMode.PerformanceOnly + else: + settings.mode = mlperf_loadgen.TestMode.AccuracyOnly + settings.offline_expected_qps = args.expected_qps + settings.min_duration_ms = args.min_duration_ms + settings.use_token_latencies = True + + sut = mlperf_loadgen.ConstructSUT(issue_query, flush_queries) + qsl = mlperf_loadgen.ConstructQSL( + 1024, 128, load_samples_to_ram, unload_samples_from_ram + ) + mlperf_loadgen.StartTest(sut, qsl, settings) + mlperf_loadgen.DestroyQSL(qsl) + mlperf_loadgen.DestroySUT(sut) + + +if __name__ == "__main__": + main() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_offline_inferred.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_offline_inferred.py new file mode 100644 index 000000000..9325b8410 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_offline_inferred.py @@ -0,0 +1,130 @@ +# Copyright 2019 The MLPerf Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +"""Python demo showing how to use the MLPerf Inference load generator bindings. +""" + +from __future__ import print_function + +import argparse +import threading +import time +import numpy as np +import array + +import mlperf_loadgen + + +def f(x, y): + return 4 + 3 * x * y + x**3 + y**2 + + +def create_responses(n, m, mod=4): + r = [] + for i in range(n): + r.append([f(i, j) for j in range(m + (i % mod))]) + return r + + +responses = create_responses(1024, 20, mod=3) + + +def load_samples_to_ram(query_samples): + del query_samples + return + + +def unload_samples_from_ram(query_samples): + del query_samples + return + + +# Processes queries in 3 slices that complete at different times. +def process_query_async(query_samples, i_slice): + time.sleep(3 * (i_slice + 1)) + query_responses = [] + samples_to_complete = query_samples[i_slice: len(query_samples): 3] + for s in samples_to_complete: + response_array = np.array(responses[s.index], np.int32) + token = response_array[0] + time.sleep(0.0002) + response_token = array.array("B", token.tobytes()) + response_token_info = response_token.buffer_info() + response_token_data = response_token_info[0] + response_token_size = response_token_info[1] * response_token.itemsize + # mlperf_loadgen.FirstTokenComplete([mlperf_loadgen.QuerySampleResponse(s.id, response_token_data, response_token_size)]) + time.sleep(0.02) + n_tokens = len(response_array) + response_array = array.array("B", response_array.tobytes()) + response_info = response_array.buffer_info() + response_data = response_info[0] + response_size = response_info[1] * response_array.itemsize + query_responses.append( + mlperf_loadgen.QuerySampleResponse( + s.id, response_data, response_size) + ) + mlperf_loadgen.QuerySamplesComplete(query_responses) + + +def issue_query(query_samples): + threading.Thread( + target=process_query_async, args=( + query_samples, 0)).start() + threading.Thread( + target=process_query_async, args=( + query_samples, 1)).start() + threading.Thread( + target=process_query_async, args=( + query_samples, 2)).start() + + +def flush_queries(): + pass + + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--mode", choices=["performance", "accuracy"], default="performance" + ) + parser.add_argument("--expected-qps", type=int, default=1000) + parser.add_argument("--min-duration-ms", type=int, default=30000) + return parser.parse_args() + + +def main(): + args = get_args() + settings = mlperf_loadgen.TestSettings() + settings.scenario = mlperf_loadgen.TestScenario.Offline + if args.mode == "performance": + settings.mode = mlperf_loadgen.TestMode.PerformanceOnly + else: + settings.mode = mlperf_loadgen.TestMode.AccuracyOnly + settings.offline_expected_qps = args.expected_qps + settings.min_duration_ms = args.min_duration_ms + settings.infer_token_latencies = 1 + settings.token_latency_scaling_factor = 21 + + sut = mlperf_loadgen.ConstructSUT(issue_query, flush_queries) + qsl = mlperf_loadgen.ConstructQSL( + 1024, 128, load_samples_to_ram, unload_samples_from_ram + ) + mlperf_loadgen.StartTest(sut, qsl, settings) + mlperf_loadgen.DestroyQSL(qsl) + mlperf_loadgen.DestroySUT(sut) + + +if __name__ == "__main__": + main() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_server.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_server.py new file mode 100644 index 000000000..b564543cd --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_server.py @@ -0,0 +1,132 @@ +# Copyright 2019 The MLPerf Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +"""Python demo showing how to use the MLPerf Inference load generator bindings. +""" + +from __future__ import print_function + +import argparse +import array +import threading +import time +import numpy as np + +from absl import app +import mlperf_loadgen + + +def f(x, y): + return 4 + 3 * x * y + x**3 + y**2 + + +def create_responses(n, m, mod=4): + r = [] + for i in range(n): + r.append([f(i, j) for j in range(m + (i % mod))]) + return r + + +responses = create_responses(1024, 20) + + +def load_samples_to_ram(query_samples): + del query_samples + return + + +def unload_samples_from_ram(query_samples): + del query_samples + return + + +def process_query_async(query_samples): + """Processes the list of queries.""" + query_responses = [] + for s in query_samples: + response_array = np.array(responses[s.index], np.int32) + token = response_array[0] + time.sleep(0.0002) + response_token = array.array("B", token.tobytes()) + response_token_info = response_token.buffer_info() + response_token_data = response_token_info[0] + response_token_size = response_token_info[1] * response_token.itemsize + mlperf_loadgen.FirstTokenComplete( + [ + mlperf_loadgen.QuerySampleResponse( + s.id, response_token_data, response_token_size + ) + ] + ) + time.sleep(0.02) + n_tokens = len(response_array) + response_array = array.array("B", response_array.tobytes()) + response_info = response_array.buffer_info() + response_data = response_info[0] + response_size = response_info[1] * response_array.itemsize + # print(f"Reported size python: {n_tokens}") + query_responses.append( + mlperf_loadgen.QuerySampleResponse( + s.id, response_data, response_size, n_tokens + ) + ) + mlperf_loadgen.QuerySamplesComplete(query_responses) + + +def issue_query(query_samples): + threading.Thread(target=process_query_async, args=[query_samples]).start() + + +def flush_queries(): + pass + + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--mode", choices=["performance", "accuracy"], default="performance" + ) + parser.add_argument("--target-qps", type=int, default=100) + parser.add_argument("--target-latency-ns", type=int, default=100000000) + parser.add_argument("--min-query-count", type=int, default=100) + parser.add_argument("--min-duration-ms", type=int, default=30000) + return parser.parse_args() + + +def main(): + args = get_args() + settings = mlperf_loadgen.TestSettings() + settings.scenario = mlperf_loadgen.TestScenario.Server + if args.mode == "performance": + settings.mode = mlperf_loadgen.TestMode.PerformanceOnly + else: + settings.mode = mlperf_loadgen.TestMode.AccuracyOnly + settings.server_target_qps = args.target_qps + settings.server_target_latency_ns = args.target_latency_ns + settings.min_query_count = args.min_query_count + settings.min_duration_ms = args.min_duration_ms + settings.use_token_latencies = True + + sut = mlperf_loadgen.ConstructSUT(issue_query, flush_queries) + qsl = mlperf_loadgen.ConstructQSL( + 1024, 128, load_samples_to_ram, unload_samples_from_ram + ) + mlperf_loadgen.StartTest(sut, qsl, settings) + mlperf_loadgen.DestroyQSL(qsl) + mlperf_loadgen.DestroySUT(sut) + + +if __name__ == "__main__": + main() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_server_inferred.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_server_inferred.py new file mode 100644 index 000000000..76461a75d --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_server_inferred.py @@ -0,0 +1,125 @@ +# Copyright 2019 The MLPerf Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +"""Python demo showing how to use the MLPerf Inference load generator bindings. +""" + +from __future__ import print_function + +import argparse +import array +import threading +import time +import numpy as np + +from absl import app +import mlperf_loadgen + + +def f(x, y): + return 4 + 3 * x * y + x**3 + y**2 + + +def create_responses(n, m, mod=4): + r = [] + for i in range(n): + r.append([f(i, j) for j in range(m + (i % mod))]) + return r + + +responses = create_responses(1024, 20, mod=3) + + +def load_samples_to_ram(query_samples): + del query_samples + return + + +def unload_samples_from_ram(query_samples): + del query_samples + return + + +def process_query_async(query_samples): + """Processes the list of queries.""" + query_responses = [] + for s in query_samples: + response_array = np.array(responses[s.index], np.int32) + token = response_array[0] + time.sleep(0.0002) + response_token = array.array("B", token.tobytes()) + response_token_info = response_token.buffer_info() + response_token_data = response_token_info[0] + response_token_size = response_token_info[1] * response_token.itemsize + time.sleep(0.02) + n_tokens = len(response_array) + response_array = array.array("B", response_array.tobytes()) + response_info = response_array.buffer_info() + response_data = response_info[0] + response_size = response_info[1] * response_array.itemsize + # print(f"Reported size python: {n_tokens}") + query_responses.append( + mlperf_loadgen.QuerySampleResponse( + s.id, response_data, response_size) + ) + mlperf_loadgen.QuerySamplesComplete(query_responses) + + +def issue_query(query_samples): + threading.Thread(target=process_query_async, args=[query_samples]).start() + + +def flush_queries(): + pass + + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--mode", choices=["performance", "accuracy"], default="performance" + ) + parser.add_argument("--target-qps", type=int, default=100) + parser.add_argument("--target-latency-ns", type=int, default=100000000) + parser.add_argument("--min-query-count", type=int, default=100) + parser.add_argument("--min-duration-ms", type=int, default=30000) + return parser.parse_args() + + +def main(): + args = get_args() + settings = mlperf_loadgen.TestSettings() + settings.scenario = mlperf_loadgen.TestScenario.Server + if args.mode == "performance": + settings.mode = mlperf_loadgen.TestMode.PerformanceOnly + else: + settings.mode = mlperf_loadgen.TestMode.AccuracyOnly + settings.server_target_qps = args.target_qps + settings.server_target_latency_ns = args.target_latency_ns + settings.min_query_count = args.min_query_count + settings.min_duration_ms = args.min_duration_ms + settings.infer_token_latencies = 1 + settings.token_latency_scaling_factor = 21 + + sut = mlperf_loadgen.ConstructSUT(issue_query, flush_queries) + qsl = mlperf_loadgen.ConstructQSL( + 1024, 128, load_samples_to_ram, unload_samples_from_ram + ) + mlperf_loadgen.StartTest(sut, qsl, settings) + mlperf_loadgen.DestroyQSL(qsl) + mlperf_loadgen.DestroySUT(sut) + + +if __name__ == "__main__": + main() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_single_stream.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_single_stream.py new file mode 100644 index 000000000..ca8d84591 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_single_stream.py @@ -0,0 +1,129 @@ +# Copyright 2019 The MLPerf Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +"""Python demo showing how to use the MLPerf Inference load generator bindings. +""" + +from __future__ import print_function + +import argparse +import array +import threading +import time +import numpy as np + +from absl import app +import mlperf_loadgen + + +def f(x, y): + return 4 + 3 * x * y + x**3 + y**2 + + +def create_responses(n, m, mod=4): + r = [] + for i in range(n): + r.append([f(i, j) for j in range(m + (i % mod))]) + return r + + +responses = create_responses(1024, 20) + + +def load_samples_to_ram(query_samples): + del query_samples + return + + +def unload_samples_from_ram(query_samples): + del query_samples + return + + +def process_query_async(query_samples): + """Processes the list of queries.""" + query_responses = [] + for s in query_samples: + response_array = np.array(responses[s.index], np.int32) + time.sleep(0.0002) + token = response_array[:1] + response_token = array.array("B", token.tobytes()) + response_token_info = response_token.buffer_info() + response_token_data = response_token_info[0] + response_token_size = response_token_info[1] * response_token.itemsize + mlperf_loadgen.FirstTokenComplete( + [ + mlperf_loadgen.QuerySampleResponse( + s.id, response_token_data, response_token_size + ) + ] + ) + time.sleep(0.02) + n_tokens = len(response_array) + response_array = array.array("B", response_array.tobytes()) + response_info = response_array.buffer_info() + response_data = response_info[0] + response_size = response_info[1] * response_array.itemsize + query_responses.append( + mlperf_loadgen.QuerySampleResponse( + s.id, response_data, response_size, n_tokens + ) + ) + mlperf_loadgen.QuerySamplesComplete(query_responses) + + +def issue_query(query_samples): + threading.Thread(target=process_query_async, args=[query_samples]).start() + + +def flush_queries(): + pass + + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--mode", choices=["performance", "accuracy"], default="performance" + ) + parser.add_argument("--expected-latency", type=int, default=2050000) + parser.add_argument("--min-query-count", type=int, default=100) + parser.add_argument("--min-duration-ms", type=int, default=30000) + return parser.parse_args() + + +def main(): + args = get_args() + settings = mlperf_loadgen.TestSettings() + settings.scenario = mlperf_loadgen.TestScenario.SingleStream + if args.mode == "performance": + settings.mode = mlperf_loadgen.TestMode.PerformanceOnly + else: + settings.mode = mlperf_loadgen.TestMode.AccuracyOnly + settings.single_stream_expected_latency_ns = args.expected_latency + settings.min_query_count = args.min_query_count + settings.min_duration_ms = args.min_duration_ms + settings.use_token_latencies = True + + sut = mlperf_loadgen.ConstructSUT(issue_query, flush_queries) + qsl = mlperf_loadgen.ConstructQSL( + 1024, 128, load_samples_to_ram, unload_samples_from_ram + ) + mlperf_loadgen.StartTest(sut, qsl, settings) + mlperf_loadgen.DestroyQSL(qsl) + mlperf_loadgen.DestroySUT(sut) + + +if __name__ == "__main__": + main() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/diagram_network_submission.png b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/diagram_network_submission.png new file mode 100644 index 0000000000000000000000000000000000000000..35663b97fe3ad1453c431cbf5c6155ab37c78851 GIT binary patch literal 51192 zcmeFYby$?o_cu(f4<-E_5A++{qsD}Ue|ri+%t2|nK}2FIdkSU8wpoeyhlh&h=GA|PgzM$3j+hQ z3IhWx34n(#sqKi1!oVQ5wUL#DE6d74;I2-VHue@67)p_^b@36}{Z!fdNeKyKSb0hW z&8**|Q-CjUFt?0Z*@95yW-xelK{R27fn0Td8B0_}MIhG$)Al;(SqlQLIU#QVl3#nO zBt7Z3=Cg6VK9VC~b*s(qwoi=l#_we$8|e=$h|x<(Y&^iEsOX)5vIZuWb1t@0V8>cN zTX}uGHHJmWU1L)t=7+qO9y-o9ZogZjNQ@vWn1O16!pS@ zDV0D-#;z|~U`Ur8NLWA8^PzJQxzN2Cww_w%7UDqK*MqMuv2$HWw=+VU?E4B12~2hQ zjM(s`zY{D=XD1AXBgh)H{cl0I6@gPEGpdlVLyVjuC*j(=o*a>((KSm_GYi9ze5waY z%18AAx1-;FJN1tylU`N@#Xd+;=y`Y)F4**1j9n$D6zCZ(U}j4BDXd23hh%>0#Y9`oRl4$N@u_A(mHGNjVtP1~6xLg|t9oo9dugg4%Ay`a$ixM_ukc_e zBx~6Peo&Z+K!VjOi(pO&e#Fd| zoyH}eH6`IiEY3bWti|$g#KTH7^kkh@SWv70X*P_SB`@ZX0xUS-G$;JiK(Rm{Ej>4V zW+G8GER!@}n(;R`Qzkp(St@b1bnGV6d_&ctag$dSXU|=;ztZ*d&fquJuN5R$6H7(R zm5Euy$XO$oys;$wkT$_Qv zm)^fH_tz(y!*lgsW=3L|hYty33t^4apTm+WH?{ozgS;@PbV7m zL-lg#)#|SOi^4*}Hh81+Cl)3YY3$aWo2NTj7p|=WwEZM^3tB#RQJIBT>FnaXBC+e( z#-tF76Evo-E+76GopX z^RQoIv2NrPp;U~E$v z7A5Rtj1@G3?s$s&MV`8`~wFf#V_fov|`8soN}^8RkOEiYc0K zBo-Mq6@e(!7*NzSBAt06f^|9U0urPICXel6wNb)Lij5!>;V04yOkpldxYzfjkoJ;B zBsf!ggn6&eVU_t?Xp@X&K3GNGLz7}0k0J6Zm+LX%Q#p?O>CI;!0R|C|WFL+_aOZgu zo-NIjJNcF5gncF&Q;osQcs*TUJ*2uC!NoAxQ8-0k4tm)8zUoCe+3l!qHAy+$bs&X{ z=tx)A5@B3dvNNM3WljiBx6oqaq3CarKV?9~`JxfvaWG$d;-carP>Wz1WrHGziI{LK z@h&}{Q_Vto%CRKMd}R0TY))4gVjf|8#O}!Qi{lB!Y-D_7GaGq-!m1J>q&um;5KD`| zM#6??OL&X0G(wF`)0^;X&kKbn3Fra_Y5Bxs5+Q`d;eU zo%D}9QTX&Y!7~vL8V~AE8QyQNQWXlO$AWFPK4Q)AzYyGG+r#{z5mK>~-~M&cZsZd8vooPLWRK69b==SLLr- z&b0O?F9`yLY%>h!T0!?PQL7sr7EAx+|YyBc!qP)$>=AyOC#-XANg> zcX`BeNwP^`;VR*(aU}(51${2bW{ni>{gcYO&(_?(XKW9-K60;c@!#xlj(o-Ws?^g< z@SCmqRo`n?eMNtlQ}$ z7ptdNQaOIQm2a?&QjLxp&Kqh~zNmcU^l+beKXzWo&)sj?H}xv>l>H*?V&v4}=w@+a zO?T0AY+ob5>h9&8#O}lwD)BJRb)j1*_D~0a573b;o57G`o??lH zPat0;2}ncPz<+eV7uC7)%+0$88J{7s{tP7efZ0}dyudo#+Qxka;l7(*UDi7vT^&c( zJKh_NP~Y92x^}5^(O&UqP^708s&JkAexZ3-yR1*KNsvr&#@3`tto$+?Db)W+IeR>* zJVTthLG;p3#d_d^g1m+ij=e?O85G=9CP zrz1V_zI!Y|^h3=9zI z>S$U+>gzPW7X$jH>8mbmtH>A6?HM>^+w+W2|Is{Ni}e z?djH=)(Le(pNhEx-$Bs<(aSG!QJYa=r#d^j<#!)t>X*DNl-6!zd+5^aeOn`2w+7Z1 zhd=47)ulLG)<6GvI#Czl;PL#)^R?N?S)$KUKNFh@PC0UN-h0ySam`+Or6F~De_i}6 ze&1YHb`IXxGYpyNv-@(UKIP+OwAm8&Vq?bPn&x_tBoO-uo5z3T;CIqI=Sk-~52c?mU$VVT{1jzh$X+M%DZZe+p(DN9KH90ce_S+B;H{X(R6wtfv|mJi zJ)E~xb~}&UrHI$R|9MU#!~1Ba)TMYvT`^12=l<`ZejS~i!&*$Mw|-`euns@|Xo;A-+Nq)JRdY$`_qaw^u!Y$Tquj4d*XHO`}aqf%A zl4URV?T3xe$BwNn+Q{O&UrjzJpLgewN^hye`X7LgnAnI^WDmCsP+%u{z{VicsF%1@aqRp0N$NfHOHso z<4-52ZExQ>-COu|HMmllI$uAJzkjrF+RC#huseIxz9GBQn-V{IRqV&&_x{5EuHjI1 zJk1CD5*wSmB1H@dYBQFtt-(On5@Ecc#c)T4<;LjF+txjymgULX4KDi`}dd8{OK9fpjytgPFp!rYOJVUr6o~ntOIXOHsF?TYxc!YFt{=)}D94U& z`41mC3pX=Y8)tVLCr8L1z9yzl9_|t>EPn+3*YBTxS|DxyTau&Ozr#WgkmpYg58oqR zp8w*FCKdlvDhjtjTG;E!**KtehL$1uL{M1#Z~OmG&3{Y$AC!pyrhF{G|34}JN6r6E z`P|LIRo2M?Ez(``zdzT%#Q(GMUqW%7KO_Gin)pY}e@oGNmLwGC`LAy#Nf?Zr{E7aI z^fq!D&(JB_$o~8=|JVlfpY%`OeF|F7_XGn&8beu5<{1)mx7pL(pf~ZCexEUhG7tw4 zD5A)lOwO*~K9FQFow^2{uCb5bJB+cK|3Yz>qi5G)ke%VN;FnD^y8_X#qi35{(W!A| zW53U9MUJM0q+_rg|EMI<7sY752gh+9X)j&J(wOq`)D8IA*z$gR+mC<5hKwgJkaZCt zj1?yx#hkE&lWNMkzATFMw-Ah_ZY)VgkEz&3=8U67j>%D#LjQz%2L1Ae?@Xe22vnYNZu0d{gIDm zhO_SWM--Yngq_S8OSRv`Esxyx56tbcF#VKO{=;J>f52Q#R{ihLqP>Ng| z0T+AtaOkVQeMB=k7Wq56K-PpLfN&7|*Q&$Wd;>SKPVJ(;i^COm9v%%y?Gk(>TjF7CY^>|{IH!%RZ4w!)Lf>kCvc*t3 zUuT}r%`a?W-ig!GQ!RacJLJdDpJz{jIx;drAZqH?b0ORD6936P4pFSFV6=*oVji!0 ztPjylw+6H}U9P{id|jzmt6({jX*0k;PDg@!lpp`&#}C)tDLzgvu9P2Tx~g7Ym(H7p zIeq}fSF*ApOI22*j7y{}W+ZT4ov5fN^Q)6Be!B^-3^C6NGCg(mIIl9lC(9hdST=v9 z5xXC#&HJ4jc&#QX#xzItqnf)ilrCfS7vJ#=Zj;MctH2WJ|RoYb6}}6{9Spu z+OMW#^$a1W`L*YnEV{Cb*Jpdl4zqOxW(K;t96$;@!T=N@b4_Y0)is@vV>z8CfDwj5EiZZD0swp=+cwqeq(Od9$Oi|*~zI2}wmE2{O`I~}J>jmDS91ENoz^pck5vM0ae zIXr*^UX4pw~^NxH6$KF>4yxX8M)jaAvt+g6WC7`(P zu5F*GXoT2d5eb@HU$<;btz*XC4(lt)VSA0llD*BIJkEfDU$5eUgjJHbBx{ZlQY$`$ zQRFmAQMq@DarD*YiY+T|vTrs7*cW>|km7y)SM&?Dmfh!zQRE!sitFY|6}g60kq=}l z(hR+oZ!dH1rpXbKXVvljpC%3M7^k`-DH;t%Xs046I0|cOw6guLY*=5@Io8Wi@fg~t zhYNm98155h6><$0B4kJ^FoY!-Qo%cal6?B43074wisGT! zzaYQrPOqs@#-y%z$9)hIlNf&(mlj_goyId?8dR%+>C5;1%ph)YPq9;;PI zl0;b1uuhvHU=_!y(3g0oDQ*rfjnw3D$SsHGkB20!idT^0P*o^SlbNn~H$4W4 zOiF9@4GNCq6C~OC4|0D-}pKM=qN&^7P>t75S zU-g$&w%BfbR|Fk=@z_zjgzbJLqH(@CoiuVV+*Z6@S zx<(EjLHGQ2w=@IE_Cln(r0+E6PjH>^-jVbyK1Ch{CV`RK)P>kyiE9ncOE3XMeQGsH z|KoAeUoShq(8+ZnWfQ=73B>iryL0~6dtnS7+W72X@tDcuQ7NZ|1>fW0SKJ=FOuTFf z@vB)7rb+zEzPeC)8*Kh%u&cA=(N~RCv!=^z$>dKeVr(Ko${Mmi?d!@lCCxvz+#3~cC~KGqbP>uJU=DSnk&n^~=}i8Zw{Fygq=xPXG>N-pams8ISeuF5$&983NXYtJ9q|e5s2dPWK*+ zcf;U6j)hlFr{`e;%Mt9!$5<|3rg-+QgAO8|^!(!Z-+Y#!o@5bB^^>(8Iw#+FXJ$O- zj!q&&euS+0BC0qvT=tqR6SA`*c`ou+AV}HHHp!BvU0s8{^}T~a2C}1II8eYX-n+Sw3GJPg`Pb7_jiH!In3XHn3wJH_ za8>%9w{-ORGM6%jh=Eaw^Gv-SM6<4;*E_u$M0Eniz;^51b?eho1D9^fqL&26OCotg zSnV>wP==9)or*>^Of!e?w*j{s0WYq{HLepVm>Ic#55E<9OO4gA=#h(p@Q%nXxk&7> zN-;_+F>QE@%Yq(lKdAI-F>_8yJ#4{ z%5s?WGF_~%_ocL?N|SHGncw-(^K9RQZ3+!>lN(vdJ}E;<02Aq(NhIu3)J^Nhzvdlw zs9`@P{&rQ^CxUoQOWC34jD^S=fXf#IFLtl0D2zSn%HAW(BOa zuEdF{-8xGYQUKC*w>z zn>Z2Z(UQ;=faV*Ac+=t%EFH+x6-YgigLN24AzW~fh`)k8y(utHxf;lU9my=lHe}aS zjOma3!aNk_EA-PnxnI_ZnfD++izCPP+k6NvBTwP5Z%7-MCjxIY{sswr!jDE@7|o87 zbAERLC~DSUY#TRm#&j0MOWIa0cAn;3ba4#nn=&#p`N_S4HPaUd^ii+NCXf=B(iH$Y z%(9Kkd39tY4-_QV+2rW^q8nD(Z~i(vzSfsmveo48O`37u@f&>b;xYm8MukeLKt#o3 zjqa`{i!g4tD{luP@Iu3>gJGSGCi`Sv!5(oPOZ$P|3aD3oEGS<1Zy<1hN|niV70g^F zb@N?HFqC?x$PZ=+B;K(@#z5IB6!o-tRy#!c?oKNMegYoddZ$Q5#lC_sLp=e+A`Mib z`0cs8t*0NHGQvSHjgO>!n)rsDP65~YcISVV-q=k5H_Uk9dj7n;vd<&yS7*VmMVB$F z5zGy4{krsCIjQyrDO}=qiW(g3s^Ia%-2+iDb{qwHND9PKTsq>-xi9`#8Db!}S+^(j zPT#5dM-QtZ$vPI@iU=ty%9$M}q$9koEwO>Z8I`+ySQJkxagtzySGv4p0$$?#M$L=Q zqeQq*7=q)~e)MOm#$aU7(X;L(?JN_mqI`kuD=6Qb->V$MizbKi%k2P#y-O&n}<#dm#2wDv$3HQIYVn6S5-$s?rw>XBYtD#(X0%q_vYl8!kv3 z+k8c|+!REajZFw9CGPGIvjd6e34-_sh~(v!0Ns=el2NpzB)G&hC@whwMQCUT@zKh3 z9`A}OKre3!Q5g(n=50~~&|}G#l6!YzhZt*4$lzTC`a;zK)DLfG7jEmwaBjLnWSPHv zw)7H(pj{IAO|PBI;djaGuQ@l1p$t&+&ha+jrM zikeQd?gI>-Dv0@zPXvBJxLbPs-67MR$trRC5yNWN^+ks`Owy2swMlL}ye)$?L0nj- zid6g{kmkkV{6Qbgbfv--+XG2n#3x@tR?6*a@)V|2gGHhxap#kSHHT< z81?O$$c50sLhdf&@2qi9#Ps1)34$#sFI*EXz?T1Mnb9X8Gm_6qI9#-pX>NsRHH>)i zYFt}`COhCVEx;y>pvDxZYF)+XcA4k4W0e)ve*t0Y3Qrtkx@^O{43+^A)52o=&ZkHH z&#*3+El$t3!v97AUhv1zHPRy}C+O>Fz^$9MFVsjpZ61Ijul$O3@AqKq?`Se`st_X> zh3bIm<;hjwGP5?(0Pr+08GVzIAkaS1mm7Ov80q~xR2#{+)Ean~_P7NPb0@%)Ai1Z3uGwA}xbhpHG+uny8 zL(uSzT73F~JR-5Wi=tb=@ax%rYo(K4%uD@%z-oke|c9u%8~tbBsHhTD|J6 z6qCT+r38K50E>9i7fvScN|Df6!KfkHuOz{vEp! z9F%wBe>rGGiwUy?B9LcHr=c(?@p6bXqF0u4850*PDuR?lioV16zmYhGdEg_VRoVWh^SEIumI_?RkFvov%cjk0w z3FfOzCmOV-qqythpbAA7TZQ zK%w9Vzn3uchTy(X9uN}_o=L(@M8`XSkboTx()tkho#HGCsF!KwfVQwF08JV7k7NOU zO|Id>daWVS!qkI|SSAjku^)kSHF?IeN?h5HK6M;BapsN;^I4hszIc`%H!v|)2=6A( z&Ha8HF3k(5=M_oAiB8*buheZd8Q7dC=n#k0{AR?n#ms~J4BKj(WQ6FsSJ}gsCvZ7wDjI2_WSs>FX87 zy-b)T7|K1lV?le)XbdYT=_wm;xgki*PJ8YI+AvQjBoW04ebt0r?on<46EzPFWy4F+ zoRR{aZRFPLeGkM?st9ZYkiuS~qTo><$+#>g>Zf{9EbHPa-*bNV=S-af>2_pM_=jp2 z*$L!JLfdU`tQ^8W+JFQMarVYECY<2jKnk3JQ{YN>MKQxw%m!vqo}4U#lpIEulnL7w zS_O(jLF_on`~-g8`fHcLsJ6B+=P6?#7!~pvWz|u8uN^1Ystg7;>fuXSWQNg8T~Cj4 zX(vd^7@HrP6egf#vtk5SBIR5(F`H=|I{j zf;E+hiq9SA(0d)yP!J%HE~i8C0U&UUoJtC<9cG>;5P%ISih6Zm3>QZdLghhRI1xfU zv?K!wE90bOn!@T|B%>&z>B(6n7zXL(qkqJvb%~PeNHMZZuI%Mi36lgZMJ{C};`W_> z#P?d^26muO^^92Tv%qep0Ba_!5Wtoc+f|v-%|M0KhwzmXvVGjp$WIzf^^6^`CllN7 z8mJe*k{3YyE}j~%|LC!IML?CE zkhsT^1AjJh7f5dwdl>YYQYm}|1i)z$;Xn!x;m6>v>p9t_nv8s9TnRrsW_`LIh+x*= z?hN4|0K^1SNB+)4Pf8lQVj0P+VBgFT4)skM`-8EP_y+WD?C}v_#p3r_T7!g9qwf^x zqg^%{&lDj>Oe#@A5+|HW4AZF5;2q9K^91O?A)S|1g@_#+bFn&i3L*s{e3(7@F_bbU zU%I`w(gqz|fN-L$@V)Y%g^(*Y;5lJg#BJbW=7A3B6W;RD52W!@L`CmtzWT^ivy7Ct zNi1()7SEGz=@gJ@kbMRfZVCCFkQ2<0K2>o=K2CUiU3KtLIPFaH4OFtCS2bj_KKtw< zhQG+DofQ**X3-arVh#oz(=A+tkx|s(V_{Q+1mt3lx@(P%C~x-5#v~n_C6-j1jIV;f z#orMvP4K>op`0gR!^{ieu=7d~zQJ*FE`XEPgo%QAoP&lsYq1kE{Bbl=zobh6>D}@^ zlTVN^J0$@$=s%+?yv@ z#drdBZ&K?^J$!d}h+AHY-N`EXctAXt) z!Wr0XHFw`PUTE|}j?z37#S2KMIC<;kUd>i;J3wu|`&5!Q@AkHjn!S;U?oZ%zua)m< zKch@4kg+hLH>|_dFG{dz@h0~-GmrQ)ajq2JRhrg2D7)DHA84fwAJ#^&i`~9XeeW?A z5X{KB#7oWzMU5bm2@F|uq`KrINHwsJShy7GSmslx(Rhmz58AhUvujGO`;4mT%QSD# zxmOFMjlL?sshWTR;;SG#hCuqIhuMJkFZbjWB8H>UfJz}X`V(yl&MVLAPXruPDaApu zay+0)(Vw8V$k7+~Oymf6U~G8_K=gr!oxb58sHXW(bX<9;el?%O)+S1w2jiavJPcz4 zPILn(Yn~|FXP-tTr2XxV#-HxE%_Y}OG7q8;p!(@iRQ-@HY*y^}Q(pcF0L>AYi+``_ z4;JP92T{si`PKXv$APTVQ~4JhQu0Ex)$#{4|AQ+Pa{L*9 z$gg?3e{d_tKe!bWZ2upyYU>Z?;|{ujmyN^J5FWA@C)0|VyWQIuz*x#UQTe^Izjte`hBjv}quVAb z4D4dO_nXg2L<>l4otj<$!SI4)WQ~v4hmXc&NtMy)k@!Iyc8%}lk&In*R8|`vrD5gV zpWXzW{5g#B>zVP{{sZr|RiTa79&xYOv*-WS{$HYv^7!gsySpi8bx*d&bVYjm&y1LK zFHQcLmXtxn{R0C@b4@ROo~|1T2c7bX{!o0smgTYYZhO(h#>pZ1QN+-n1xXmI6C)@l zCdRRO(G3mOs=aw4KT+NcyZl z^RI;$La~4^EW@}Anqj~0u5`_(2iBB-l5tc?hFN_Ni#=K-4dA+dvY(>twNePNlVj4o z__u=;K@=aXRu@ZOWa#l3FeZMl8x@moyKq#=gib#6M7L!_PaZCBUUZ~zcv$dvSZu`Tb;IMfS=j%N(#H3=wvotNF;Dd`yOpZ^Cq_D!spgjEa;&dS6}QYb8XoP~g`LF)oFtxi zHlq$>Pg~1!e#V+$E9lW=*aT@y1u6G`H+9QZJt5(}^pNwoEuOCz4U*Sr{m5E15%)^u z$2T?({ShUzW1IfLkwlvnRSWyuP3uCf-OY@;TGx{k7m?Zd2VQz2{Gy&HP#jkRnTQwX zM6xQy=YleA@Ct=!onSHDs=J+A>w`WW*84`)ob-GzRC+-)hBsfvG*nb@-puM3-VPDi z*xK=3Dy*%m9&Wnu^K(dXt~IB_)s3lT@1Ob2kWFn()M~lCmwB%~(cj)f%h^ZAsjF_l zmfZXL?A!bsVARlC7J!&pD88O4t*M>8w z&n1t4B+wL0snO`O5OJsrSxnYC-ermc9zV`WGB^zn)m8c0$H)H}8P!~*&LfAOLW&Rf z0K0|WMru(vDuzCB3_9-wRV4ZC-VK3`sA8W*F@zQ_h}3Au?c7si8YGM|A3^o=8#=8n zy?h}$v${Hce|pkcZoXpXQSWHB);mERh0?XBB^Rd5<4xuE>{p3eqTWqI58jy1Z#(zc z)SrL&-Wkwn#|ENX4`5mTws8DJ&D+aciO|HhtnP}a> z-tgLHuG&Hc9}3j!0^L&P&>YBua~{`Ci60wkG@*F#UB!9w^rW7XwFSY#X9#+tgXMg_k4an5a{}89=&D4PMvBgY%WL*!g=BW-qVBNfX zqMh*ZD&h30j&yp9mT5oV*;()%p0~J9T<^7OmwEecWKwZ)AU2<UN9YLkQnb%UhSEU)q|PCu2Y&NPY= zA7>8P*PdubQ4iV@=_R}z^Z_~eSXu3QKQByxOlsj9Vpkrr&mAmz&v5LJglfRBs7gY| z9;2q-;!jMTexz1K#FVP1n2GcCUIn!5Z(M6IK|eB?kBBaWOIt5}0qafQy}Ad6NL6N< z)!Rr+Q*X5zj1LA#Nbs89mlLl>17pL7Bh!^lSc^Bhwqq^4Epu{;4;|dZCrEo9KPld; z*P%7f%bL3B{4G*k-?N(cv|`I)KJ?72a{LE7!xg^dN%8Fo!Su6QMcYyO;^=SRv7`GL zIyK%_?_=PxDR;9Pz-yNGJ4b9!*d~+%!GieAg zNW77wdIp8oe>W$1(C@q#SzG*0ulC){=tF+Tso%RV^^-GrT{G{egXjocWbxOeQ<+_E zz3-7}DJJ{>(jp(!95pGg@T#VAzc6o3GHnFKf?2i4u_H;Dld?{4Ppj>@L8f&BapmTA ziOotz5Y{Av`&2OjJYtpanulQ?A7hh*-$hb5%uxBs2E~jpl=?9}?B2L$s%eF!uh1@w z{hTqMC-=kqSj+Ii6(%67etqqHl;e{Afu8 zcGi1t3MU`%)t0>Md04mjy)bWJTf5hO_?~?`*1I93jJp}Ab@^*(m~!GWtUd{rg&#L* z-TWcy5Fiji>+v(rynw1(F0aZ365>8E6&a;kH=L%BC9r6{0S#On^nLjC$#N~Uo!sK- zI`V9m7G4nIY?+Y)6O(lMQv)_pQb}4SNi#msuF~3b?Bj|7+ z7jSr4Iwr*L!>k<@y89_$yw?{>uW9TdEnGgzXeVmp2ELpb$B;9uupp^^L`qi;$VC3bB z1+NiCG; z1XD_LZ#+VaaDZtVE(zt2*eHL%SrH^xM43v9KBcX{O*aToI@>x8cW>U68?Dke9Lb~SO!73(y~)9=ZZ|`esX<{{IBD~Q z{9u`EU4xj>)B+z$t6|geCw#ndu<`c?@E!|X3Ulu1GL;9=#;bLTC)rT>I_r<`O@BBd z7I>95i6UX;y~C0@YMmZ+I$l-ZYSwt&A(X7J=f9wv>6%I0Ycum&2A#|_@2*bx#Cq!p zO2l_xJ%D$MS*S(+Xhtrp2|gkGut&nKx%xr#=)L$C4U)|(9tBdQfD!K#g>6K^2{o0* zq!+(tuWjb3XZQ$Dohh%v5Zy!qM3M0L)0KJ1aLxzO_zWw%`#j^ful2tHk0Y)~1t$i| z@P+h!+&I*#eN2PTlI_3xdGFH;8#?d{mN1W7;z6=sq|%rqdvLp2nq+v7R)5kLXCr3X zF5Djyi6Fj&@chCg{viLRy!|0gzi%!MR>)%eO2l&YVV`3si$*NgRcoPp-I>IW*dm(%HMcc5*92_dvl2fqNSZHOIwd^=vw($y~ z?DY9Qt+9-$P13gdCeB*ykyzg3@^o$$RVnOE9u6va5N;9i+~>ml1ACsc?8-go8^H4` zP`THLi0gYpryeysy6#B)9b`@?{MxcL&`lZ9 zGNpBxd{*jdWpnlZvBLJJzR1@xH3~Ah2h8$2tq&e0DfN_52h~g$7sj%LUL3wS zmY+KZ5!G5ud^mAeLnN*z*3cW!r+x<`$5&{7-(Mc|`i*A72STwH9M!jVy`{b~VO3$S zN-p=wT7;!t5~>UnVKCV{fe&LY=&>rE4r<<;ZOIPa1@Fy~eAeVt~dkI8WU zwKmmsD#3_MQAn@d@ND3P=wr25-^HjrT*)Ek9<}g!@p{56r^Pm~z$iRF<~qEH=aY9k z@u=PBqEfT2y&o|e;_GV`ZZXozDQy@9>T%6YP6cGguV?mM+1b|-S+wxk*B9wlCte>1 zI>kE@+8S4GMM{tH=`G`O%-z6R_qn2b3D@z}=|43@O?G*mSd|YDOn=9#(UyAmthxT( z113D0Fx}VlbYRVwA&Izp;o>2Q5 z0zXmngNgZ^^cYmy%t4>^nWfI7YB|*S-_>b+Q0HfEAZzfNEMb=_C>c@;q$6Ekxtb(( zOuYZ8qxxlOjwnCBz|f8?U)3n?yU(^qShcmas7rdR66{P9 zHU4di)JcF}9+S+m^Nrs;q9IAC@b=i%k*n@o5= z;nlR2(p0p9Gfj@gHrIe$=E^|MC!ue}wC2t9V8_#0_#9^nD)`4Ru-$gdd;Xv+eKJZP ztS|p9asWn$q}hMA$E!j2OGsJ?*_-;#8$nD> z8NTd;m$nc-jr3wxk8B{GMO;fwKml3zu-hcma%FF(dB#ZmWlsnjT^&*yj-0By2aB(q z^p~x$*k)LYq`(YxZi~8Xe!3yVxMT0&lr-|yG?s+dzra-i>_V0Y>3SM8rChM#kNj*~ z?Bz`C+u(h+`1B&-%a%SrB}difS7)q7Ijrb+8^8E>#+&f(edf~a?ztD}nwKl58K|Fv z0k=oEqo6uFvHX@@Q0)lleXe6NTQb*9iZ`-+`D6Mia&Yg;GAzYNprou`H5mnwyY{eQ z?Ns$I0_sPcZ!Jm}5W@_%i8=O;{GsP{cD5qEUrL9n?030Sx_qV3 z1*gGzp!5XOw29qgpuj42P@eIWzq8T@BhZ5#hr!ovfV3pGpx*Fe8E|=x!gb}fGy1|W zeZH&>otG&_3HgAo>J8y37*g!Qsz`^KM2sETQaXzP!4TIFFTs7^hGhEVf%svMfr?Ji zCv`I>QTD<8)ZfangG{V*<)Rg0Uca_2sZy6yO62O+Z15_w_q3G?<`_Xv(WaQn{6t~u z+0B)rhLO%c6)it#VZAji3{1lgTH;=5H|65Z8SoGnPU8i4g}0;lYh%y1`=vJ*aWOMB zlX~d9T^1VG+gCVc0K0Z#?^h0L$N_Hmc)+Gza44-{59pPCns?(DMjXy2KV;*FS?(W~ zT@6E4J%^O1K+Rns-B+B6{d!VEv8w4Nh!UznF+IiQJv7S(r$+>=ihOah_a$SH>!otQ z`;Ra6jVdNrhNlfupQeVFPN$N4%~kZQAFIJ|`5NmnmAhQsvyV;`N3UM>rG z&*wQjzS||v3%}#n;MSM`EJueT#viO(b`Gp7V+fU45Pvnv$ab*%B*^MmSFK_@7r|Uh z7}|F4HuZsclgPHE1Oe{I@b9PxBvdPUWs5#s3)y?MBPZW(?@_Jf^l1y9;BlZLVC?4I zw2X`tgjS9O8qm#0QbHt*kR3@BzGgWP5uQZU)j^A6QsDZWtDNCv4iZ%y_4?_8h-dH_ z6RuJT`XO`z9qN8QOE`$hf0~1jLA{2 zu%Vb)1}Aca03A+IDPVV5+8MsLmX+b7l_^b=wI*?;$tw(FCZ7;Hv_5nNe3HSLNvW%y z6X&gx&VU)0t5#3b%hTuBW5?BP8CG(b8>AaSJ`XjstPg8vP_0gtqAwwyO{!&o(xxu! z73&AYc%AJ2`Yf0^NhJV61ZWTOWiAiO2!ai@@;i-Tc(9%X2F4+Q#`z~2nH&>~H)Mnr z6MTM-lcGeOl%<-OEo|Cj;SW=c+k>E37bfLj?@tQ!9wG=DlBPPEI%>PGqj<`#$hK!= ztxDt+tPgbdgpNTggs|dK;Nl0j%4Z!Dn_XVwB^;L4zjSOSq$qXylRJ8mKjdv%7e>6|mJU6IVXr}JV4b{vd zUS?h%P#ly(I}nMG$_V!DWqGoLhel`F>MsFqY7fS`?H$5_gd2A*`(l=w1?JXt7V;Oo zExGj_3YHxY5O1Ig;x!v*<)3k#U_2C=76r>uvl# zdT3{Pi!wpZmzjusA5&f~jKz*vXjIQ3lG9k}h)|^#PD$6L4iIb^yA{3X_f<0EH8k^` zMJ-Q88-ny@F0&jLGR1xrXbSf>Mo|YUDx<&i7qXi!6(bRwY%hgF*B4K+&*JTy1-{J7 z0nVGNpAokONu>yE@*@u&-dz(Qm0W<{>pI#8ti0d`2vIu%39$KiaM1Lxnzi%F)DWJN6n`f|2XNO=@ z9lMX})aHh~+_sP${(A5@TSPBlLQ3$Nb;#p0QHgjnun+109B+(4PWu=e-uxvS541N? zau{T#Yz^9vxyCPueFbnZrBB*)@7#U??Ywb!b@WZb z1&ns5x!IyO>yOI~@7clTK5x2Is_?{Bm7=32xJJh-X_{ zK{5y?b0&l}6Df#?25To)nosngqmNL&I)93QgM5M&m+8o~3`2oHC8gN63t%4=tM@5v z`k;+g&?LyE2eQNiW^1QlKE$1ayZ|tT+;3UTl6W-ACW(F#lR1yL=;2B-(S-x1+ZB z*9AEs<`aKMZH#6Mx#%nqQH*N`h}i)z%r_lhT#=v76L8Ub{hf_T)Y&toIQeq0Ix3+h z5MZ}yjd8JupM5x;Zu94M7M0=E=I=5tQ1`juhIStF!w&Ab+}-vL(JoU92Eu5DZY-}V zM=}5xG|!u%OT@$RR|j9vFamLT9-=`{ z$ZB*=fk6$RQ?k-o!jXvPHT21CqHjAbqmJtXGQ<$}&=c&yPkF$OLZcJXWtN}x9pU8= z_bTI?2ig>lrgjf#YzwzRRKF{r|<@Uk1e$bZww$m`rejCAe#Fhu{R4 z-~k4=!GZ;M3-0a~g1bX-cXx;2?iT!R-uL^eZr%Iy-cxl>o%179X3wo^x|R=!erDO1}Y%6i)X!DK=IcDe3HclD8$xKOfQaaMlEBMyy(G z8c)6~UJKU-k@oK17Jw8&SE?SSUBm0=yMAo0luXex&JMz#Mq7SFBIRr$2F+Zfbw(P) z+zNZ6F@@lw!rv)~I-@fc{1(2!PziAU(BIKo{trX>tdd&bjQ_z;@@*fl6hCOHRzUl0 zmmvzcS6pkhcMrg}Wzep(RXtJ_!gaxF#eBlijS+zGdU&3?4y6DnKbD9NBqYQqUspY4 z4$)pBCeG@%B2wMKKofcnf6Ij=p1kv(jFYTYR`4+7{D)!lC%%C$yaWCMs6K4`4L&L| zQ-CD~8vFn^G{RQ*Y_L_V*Xt?zSVO){tDK0u7wORI6ZFggm1JG=erZW0?_6ggG!RDe z0!h5X@)E%CZ|>vy6gruK^UM>mFWWyy{(x3utbLd5>>zPhnTuMe4_1WRDPc&Q$BO?) z0_Ecgb~^=Qg|}eNQ)1w7p{i>`Rue?CR>wMHaIv9%=vd2z3$7))QO03nGJ9~Sp;7Ni z1wJ5fz1KktVUo`z?sY}oWwwzLq+0kb+CnTD5cOWwQ*w#QLNY-AQMpbX^@|J1Zt(-h zf(V0GiPZ-@aBnQ*s-|fDVH4ADI|eC=68Y$E#e5>1RHzh|PkTiH!pCTLp}lvM?Y9(u z-HB+bNb-AnUMOu%y}irpLVC^w6q{&D#%c5Hqp${HxXrR*urXTj*%)$E``!vT;4l4g zmM9dbXa|U--98k|&3K)&&V$YXC~@E?rJK!9W0D93;h>Mr&-+^Nm^pur-gsA!svEEt>@1?E0S{ku`b%B7ib!OX9LDi z%Y<=@vGEE^wFyqC8yMcOKdHv$3FoxDw;wK3uUlH-!_G(->T*u21A>tg-WASaz#5fE zW>Y7>J&oBM`1b0JCqG6&#t1sLcJWed=IV!UsK^k8_j{sQ#_{Wwd|%Pyp|bUf8j+%X zq$$|mfxld(y1GJRz`$^1t6}BPas>$~t~RUT03nP0)AGNx>f3pA7b$*13{pvNcykQM zpx@1WklBOlbQP2s&+M#59)c93KaqKee|(Q|Swq77f!5aMvU*fV(s(BB-HudcyX;fx z;`6iNjQF&lE<9!Q0jlDJn5PL+Xc{xM&U}2z>Q@l>5T4`>V~NAB-U*- z>g^K+^K25$az3Aw-`5E|l*m{#5kDWdlpO44Aq*3eN+_+I-b8lvEHIrVshA`8^uwu( zqP})0gU?9%EjC~`;vs^w>_qVRpJ+b8%*rWfK)fufC=8>*hnM%9h5Y_ijGve799|y{ zO+0WWp#56s{xGOh%hy082J(p#A4TcS$+J`nQxyJG%(|!+ir>?551Pxihfj|32-rsP z**3%34GQ2qcIB7*&ayx0YzFB@`y>yu-rF}wX_--hiyMkI-`~vVHpx#qPMAX4yNEyp zh4;D0K`L?nfR?~Eh)S0(<{k$&`izDn4r@4oWB7@A$GQKL7X~3o5!T4OUq1^LB(#V} zh1&2^{LnKGF*)BbfXj!P6^951KZGU3d6sN|4@YTOyK~z4k}7*t@V>+SON0o|>TtOc z9tH(38fvsBbD;ClRA}QE@+sU;eK|B93ishfEO}&~ACf2Bzn{7ler_>0I}1xVgFi}1 zJ}QAOO%m}wf7p4aUIyTh>oF+X&tNpeh{Gr(6Yq-mu@c4f^nR8sw^69A`}fpk$7v*3 zADMAg#>-$bcD!)=h@lECqT`q&9z}P8$m7O~vWPqNAjXQ63Ej*~v#yXeVT88aEcSy< zW5!ug*iKwD#I@+!9-ZXM?<{4isDZVt^ajG@_(e_&D59SO#GV%JOWGleSKQTc!AqDw z`HHb<-9rBC!y~MBpk4fhg`$2W6__k<=->_v6SyIdzF+x8;VeDXsP4-GQWop-WZ{#e z-||4=z#E+K&?)+(pm9YU`37X@fpm&!LH{-9$*}1S7;CW)(NjBfy{}<9_z}|Zi4TeR z;I`!92mjocRp+dJ2}qED!HBPVvp-Sqn=x=2Gq@8^>k;qU=ki@TMAR@+YvgyxVTZ;g z6)1o7!r|sKg&$78KcKM~zWEYcgL%``<2Fda^n>V7XZYQ{R4;y{4H;M~_o+Mf9X7K6 zOsu^?!Q{>Wy2Jgny^P%)w=XG+M_QS$SwXzL3nC&`Ijpa*hl9IU3nhd7nxjzT9cBul z%taU82x50Z*7K}m1~<--o*!Biz9C|6ct>aUu;eGQul>!xGNHSbr7$JTXP+KBIh=pe znd^o*oP92&zPb9qQFQfBb%j%y!kdEJEhH?0|1toi&+66n2!q{u=*JOyMYIuo<7b?# zh9+o;m0|CTy_WZ+uS1~HH0STyf2W}e45RFUVTo)5qZ2g zUu_;|oTfn)b_I&^ydc9b*ty|wejgoG;}PZMs9QLTD&W5^ z@^FzI5`UtyqTr;z#0_Qh6DkNyebdu-_bH4_g(?a`8nhjB(MxN5?1xN;)g-IGhw^o= zN7%=;IyUcX%coWsN@dyg&3F#e04!A&)M!e8Z!a|8xv$J$tTN`jj0BCd4Fk3tX~RZ> zg80|gdBuTZNeDs+%Egy;yUVJ~pdYAf*%S(B7N57jl+3oeVtrDX1t|_Yd!nJtO>+8( z9zkMwEB>8#N^Z21XTP9f4k8`7E%i(9#i3EG!a#l+S}%O(?+dGJXA^EX4{g?~|B4WY zvh{aujU%E0J^k$C+_-@GH+}6OfuAT(CL#<^q%x>)19soQ(s%gT9a3D?g;SSaByuHfbO+nW|f&D0Nqc0m$!AA7q9aC6kw`sent91q~ zXF_w^e9>{OtDg9Z-{qV80tRiH&9;X)Vm5bkGj z_FJxi64_n~lR5Lo=?zRWihiwvrMW}H%EHHBQb>b_qR(!MOXyd1e4qraS8&2%K?;}0 z``;uRpo`QTb!1rv1@@_NmOLUNXc`Fl#G|Ln6;coMz{_8|O&h8f3doLCQ61B<70b(R6Yt~AR4HcsNlnC;J zLz`|-*JrM2d+4xTav}Yo1E+eg=PiQf{Y`@1oG$U8tM`3aYVVsSdZjni&ZBY>P z6C6lgEmMbaz`nEdKM&*j)J|wv@&yH7I4FsE7&j=h0coHId#mP~%TDB|uB6{xyF(PA zCL1J>Jg2Y4`)i!{?Z7OXqrhkVr0X<6pMPEa1CJd+LT@@&Rn#jt_SgNMmoKZ|>;i+i>`C0ri&VT~$|W6@6G6ZH)3 z&R0KFm0HDNg&_MItCr1%Hm8MJIO~0SH%kJE%eRK)@-=wx{`IF81(OAI3%`g6n#S@# zrv9$C<+aC2(y;stwwxAsxjjlu&v|+G6IK}+mJiB|g2ut z>v!369MaO*zBqhV1al_I;{2&aAy?KQ8sCDsUuTYoz$UH%#nuiqm9M{hHwotLd0z%I zu{=qZXGJ)=Tpo=%D`XStsi!!vb>i-k^-CDLX(A{h%;hMOdoIrqJrr-8=&-(j0P^Mg zsf*bzzWy4vAtC1i+k3yX{kN2VN4Y7=6r6Eb`bf)30tv@oHg-{;`ZfcXX{N&ohMgoo zJ698guAe_YewqFE@YC*7R@-_;j@(HovsV;$c_bR`rQcW-0j!q0Tje6gZ5t+p{l-i} zZ{i2?OEzSJ_qtN#sB7K0cLUl%6d5eskI~~OG&eub*zO9(wW~a;MdJF3>ebyG7$LOr z8lTepdZx|PcZ$14V;$%ZG1PvClrT;~sLjp8{by|xM!duPd-=*=TOn6tJ^3g~HvZ6UFupASPw>P4eGsRWTV z+nWs|gg~!N!LI!072VjO_0cE`(|AK~$dM*kvDobFk7cUiXWo*>Mhv$+Z=axO{&_7U z{8}d9wG6S~f6$vko&S0ADyaYGNyPt2r6E9xaMUOGFKhbb^Cvp$;g~AL&1|ijw}wYZ3dN8ZlJTbGaRgy38b#$Rrc@>FN6%LJ4Y8o%I@H-4{$qox6*IOX-(pCA7NcE%O~5cCH9|8D>1vs5^M!TkS21!Ln#PAALU>*dwe z$~-(gU%iixESA!(Qwbchv#}-gK9i#DxVgDCIfU*W7#SOfjNNV#8&Owm5oeReN2UI0 zXizUIE;dRP5HFSqwir$0R?^X71PD~(o12?Sj1qEZa>^u2PzpwT{lVa+bM?ENY}cPtw$@gu3WLv6&JkCzX)O zD^&mXpi6RqA#0TOk~!=3ZBOY0=B9BCp$VS~(_D;#c@Dn&=DC3+@-ha717;SXTYc0(~C7^tIqA+sZwlN{yQwbwf?9}h?PoO}Xg zrHA5^E-2}Ehb9l?)A?vPRN!*!b}9&p`N@tMMp(%N<*DKmrNqSijc9gub~O5Jn73~l z34MmX{GHyPsynH58`D{+X;5lpz3RhHEQu?W*7FzZ!Ll*6w2W@do~bf-%bh0X-^iHX zek0I7$K!*-+CHG+(a*X3$1IlbZcBMGPmtZ?IJ|!7XU|Yh!5>lVG6@IFbbD9HQ;2r> z(L$yA&CxQIo6~U=f|$SU1EG_oe_3cW0CdY-JLY)-z`k!ml{w*d35;#}o8l;hCy9*# zm{_!J2Pyc_!J6ciZVKKB{z|JTifuURiX4K34(t#Q9Voule1)RqvgzSoOtRh1P-loy zmHV{@4PEB4Y|#B;c9p{rEcYL;NP_b68IDSgLN!)=HrE<@?YeIX3^Q#;L-Y;VOa9Lt z=#n6?hK5FblpJEl9KLlcO@8X=;B@jOz5QpS17ut_M=dTILn)$@V28Jq2!ep^S^tIX zFWKaP^0zSm`Imki!++Sqin*MPRqKZ5YRe-%H`wsRwWejmw3q4!HzNh;o1CoTEmq)W zb03T}*u#=C6V25%WD@D40Xha1rP(}pN>4Lp? zXQbuf24MkWGFI2WUPOD{%Q-NAPI*x&@1Ya>nYIyO2<5R!SCAqfKXRZ*g)YZ_Lv&?p z*~%Q6eHHvJ2X8EiMFbHEsY!d67}%i*{gU3j1w=y$&{-v{G&zMrjm;-}g_FyZ149!^ zqBeq!R=SbV15?NYbFVF814bhUpIe$JxTuF_-4o4g^tT7&>u#2ggK$jf9A_W4{X7+={Gvl9k-g^) z`uuo6%5Aribn;e{blV8eW?VGyD@tbMxM=RihCwp7U7iF1tf{+zDTkaM@rMR|$8^kj zW_*QHISSD5E!>y{!N~!`h5|0O&A;Cg1Y{jw|9+va0RPPVD$DCbIc55X-WOP*=Jm+= z*spngP*6st)&NA!Bx*F$XYW_>kR%}hI2?MPztqZ8fL&w9V;s^0c=R~u;ZbFufYJ4} zo2=geCtQhaf!>nf!hW7p$)On8u{H4-Q?utm|VfDL;#@Ne7gZpfe$*rd|Ga>tIDlXBAUHhwTZrK zO}{H^6Us=Rpvg38WtzP!n+t~S5saPPWl+350(igPvP>IH&&){Wu6CmsjjYue*NO}HG@wb zi>5F>F0Na5=akSVN67oG6<9?H3I)ha#xlZ3Zu#AEOg8|W?5TY#u}SM#CY{(OQf_v( zNL6)}(R4wyz*?-qH?db*Ly`cEFks#e#mWnq2^Q;UYSLLArziPEe`O5Zu3NH-U8pv3Y8n;%3>xrm zFVk+?I`7Mjl1=3ts0@JJ-#dz%$nNX^*(0-^$1C76ao1|HtGa(+kytm2#ddXJYxins z)axvgUkSiiO-9q8dpe7QKww~=2r*v;3O?5$Kt+~YdYKk8$EyCvxq*IsZX;&Ws-K<6 zI_k;INopqRNl8vFCHk%M>Vm2hh12hKpXAF)2^0 zyydJ*kAWCOlx%EiPfyQBqi4?`Aea*0pglI1d*dz&;^0urXlflVq9U=im0ZnQR{^0- zD2=(GnSyo)MaKZ4iCngu=Z00*w{o?jr~?DNt^>1L;gcI=$vj3?!zmn^A)(aH7!p;; zWkWVFAb<`N3$$Dh&{ST}@|c39 z!p&<|$1C_`9+wzO`?$nJd>0!pXDI@}yaaa%&}c+lbzkN6@>tGsl#9UK3@nwmQPj=^ zGK8_3?0ju_zCVQahT$ee``7OPJ3vRU@SgkJNsFyQcpc043T=;LSyqQ$i`2gU(+st$ z$c={!Zl9J&F^$%yBx+9AVP~1#vC&b*i$%I{l$Na_0rF8Pd6@B*yyrPbe zwfWSJjfEyk+kz3?g*i11OD9M-;reUI+YB9(TR3fk* zLdW?4hU^|px9Ba^S;k5bxrl!MerR+*+)yWIk#D>Z!9=jR0MIWywyPe1IV+s=+$}pW z0eZ|NjRLsgx2Yw`0X>av=2v@)F{!Dk7ap(SyQH!ivX6Yx#;2_w_i+&#sRtx;3TT_2Kf8|hXQ-z^;ilc@mKiu?AYDiX7k_QOTqx3mGq*tME@u16$zL(O=y@e z|A}7(;-23DE|LH3?`wsOfJf*^`0FG7|NQ=gP&-0NQcB(#jrtcT|;r@1eC37{~&8{9yt_Jo3*w-(742 zN4=EE7JfXgHr_yI&j;&YRBO|fcj?hIM%zXPmrd*qyY-k5V!Hy{Lsgf5>w!jxv&S)6 zAwym+e^A>4?(~T`re_u?L$Mg5#yoe}ZrbO*om|XbUN%kcI?Y(W#x9qsb%z( zQlL%Ql^IUxA9EY^!DB1eyPen0__mr&8Q%@~3dXBygC73npHAsoYA*tg0O!x-6^4gY zE?4VtluVjizy^*^CXrEU|HKL~z(jn4ZKsWWyH3)mJ8$6ITKXP!61M)+xi3zj!?(ZK zd4V<4_-%&qiUZ(6J3|hy%m@__gB~x|DrIFQyPdYECD2Qi_#c92=S*|wfEdavyi@zb zQUG%l#0x|=Y)ZreB*YaI#3b0KRsV5njs3@|m9q8fD!6;o;GF0W_ypM9flJYU|0XD2 z1H=d5`g@w_0d@GjZU?vv`y#2h0UISHAjG(81RS}sP%?o60-o#xuVLyJFt+E6NI(;& z|6bj6=h;#bKz%W&QK3%iKoeqJy8y%>l33Ojs4vFdTdR5KHMSE^;MEuSXc`0T-V(On zZf$5|0ZmMt0e6D`x9oDPoh{%`**`dl86A~#SwW!2Zsj-(>zUEX|0ZC_*p-U#KrTV#`|k`u5Pn|U;FLk4i(Lg9>+Zk`gDQa&3(o@ zeNU(K%r)&@dpQ>3&G-lAW#nO7l#Oho4io6GR}W%9Jmk88e%J)ih9T zXA7M!m{5YSe~})d8gTX?vyn;AUc?8|A`4_mof?qB|u zT^oTwnX9D5s`w&ZI{FVWbu=wuF?+`LVhh2{+PwObT0Wi8us0%9kl}a@&}Mm<{;bkc z6}b%4xNM#_{D_%>{aET;v0XlBTzovk9l$s}Z?kD+px*F7_ESQoArd8%V4G%-z7DZ< zK_WkIJyLlK7I_!)z{t?lL!MACRzy9wK%-vR)?SNW2bQi}CJzmV#QP=wzT0>240A0e zKhWT?Q9akxcmN<7{wp&Xz2&Ioc3D|jM>bVL9+D;}i})guhij_x`qQ!fk<8RYrp+rV z`b<`pol#v1<@~Xj;9MhC4eOd!zTcyx@69YNIarrf8>DWI7bs^`;I`v1&4A5f7B|c9SUq+ulG%+?Ff_78mEoy(5d0sE z)=ttd9@!#?d_0;)%n$cVS=t(myZaTDAP^|Dn;^5`8rzf?O|zK`w$RG;*0M?W2VAt_ ze#nfY=PAETxixSx+sA8(P}0yK6)`in5DsZLCoEL7F3fHj=_Dk--s!sB$U8Q*=A!h@ z&vVLfk-5ZXVA)s~CuTFY;PN~FlVtDaXf@S%`w2aR1@%207Z=yuL3fim_mV{hG0{w# z3lj9E*&i4a?2nor67PxQ>ermbZcbNd^5xTGq@=aDmTmI^e^tGv+x~M0Ldh-e_~N30 z_*B;0hcFW6@gR^8MkTy(*~o50#CtO{Gft*Vw9P-~VyteXpkxlGjFI!sbIcMSG=uWx ztBA*gEaJgcZQii00wimcZ2Au;sdA;*B8F5bK>#umk?yF@`-Y24L`&W8Vb zp1ev-s`A{~S3MmGn~4K8f*Da~4dLnetmCQ5;S#oGKofdUlU!xL{qZ6?^)w2%dS@{_ z%>SXYTK)iU%$R#&{?nAnNOa`dPm-XQYanA=5W@hUm6N=Dy#l6wg?nI8=lBF(^_gT$qpiKK#p2BcJYpm%15?IHM#-6iol)jf zcIueqa&6`hs+ZG{Oap$zP9FsbTmRrFtGwl9@AXALljm>J=SVyZ*MgGz(?8qlKAqkhPjV9 zC+GvcPL*)pXp+(4A5mQ9=0%RqcK5R4(=lQJH6p#%F9Ifb81H@%uhiaoY8$q-{5 zWj-JDN=PB;u?rHGjQSlEmvFWuZch?o5PrOcK&gnLZng4Q=HfiX;i`~z`4CcER(lt* zC8xL2SH4S4U6B!F|B~hio}KIrpZRp!`qaqxIQ0TJ_x{R?m9nf?uA`KC9dfvwH{{a? zZ$?CVY5Xw`i6P3dyMSh^7YIJAdui;QV`etUtLom7h8}Ku^dySQ;&o)Z?H+S&-3jth zWq-?Zzmd_6{a9SiT3&kJPfA_k+|GEhicrGKZ2ZeC$hC{tdVtCEk)JZ>X{Uurvr@e5 zuD`Ud_hTFE$1>yeu!448>ozL+r;|D3EsuZqsqBM4YUxWL#;_1RpY(CJy@W{5&Yjy5 z?n??kJQ0tQ@-Gp*b(VGa3f)nQ&Hf*4$97K>#ia8};@1cGuan&LVCpl<<@ET`_n*0e z2z2KzmW3v_CV$lm1v_?eufz1rny6qS!E2I~H*@ORY~&4wmjMnJIjeuE@gfhyTfPBD zQp+sSst=^S4%V$?s845Y?3cx|0*<%4(0Gh_=OFYTv175 zoUBZ#*!+c`P8r7E^-*|zdVuVBZoqr}fnY=%m$z1*IJg}5>~bvCO1A!+u}(G_YYvgG z+w(L!nrh`wF)Bd!Db?gW&^Tt1#Y>^iREi9CiEzT6ZNI;07ha)YnVBQ1J@kAUkF;kf zl@Yj0W*RK2t?UP90hnj+S$|;ounjO?WPxdGc04Bdl;qDjnkK-&HbeU~pD)04$x$x)gD~`CD7*aJDukVycL;&rv*_meRcRTGf3=)fQKBEu z%xpNNQ>wI3DPIz3)Mco%d>hlLNj4Bfk^2z$hUJX8ig{deV~@$FG7ueOvk|WC#cMmP ztbWD*K^x1J`bv_j0vIjD|M$_d=&NJ6Xws$YD9GiOmV!QaQYK^wC}Sr*D9o|Mj` z=rkx=|G56!`gkokcT_}obfMSL+H`^}YtGQ|QQunEMU8W#P6bZ!bC}++T%Y zNg2z{0Oq3$mVuXeSL|ok-ye|ahjeCw-spG9yWg*Rnv@HFqVN}0rzs?n25W-oW9ILqqA*iIEW{}QN`?8hPnPL>Fq*=w*a9EFXQ?Wz-(a| zkH5_&_)-}?1Rmkj`h_$-xAG&E!3s{{w~iEdFAwFNLCfk^?sn`Jz{aAB_gmdO5go(N z{p{-IWHoC&7EiO;_)Dt9zY6X+WabQZX1PqyH_6dtsFmj*z!8dR?Z^QFFLtOUyrRPm zCK=A(qzaT4?TKfBX}>)x?>S2bUKu-jQqB%Qxjis8f`Bc5^LQEhz;w;n=VVeMRH zn7j~#_t77J`up+_k$0q~xzQKGVeZg%fv4$Y2Rihi|2KP60cOiIFr&TzSh?vp{^dR_pU|TV<5wAEMCMGqM4)lK9H~SPv+N zv}ZoMa&!eE5q7Fj{yyxOikgn){$=$pV;n+H58mK#obSV}Gd#;1yz>L_EFpo6%p3)| zPI92#4pZC`aOApCa}cb)mB-P8b1;V+kffl~S3nomD)j3zev~~eYl4EZb5T;hUHHB{ za&f!nNGI$Q4SuXkv@PXF?*_IYm47vVQ6R`cj-?&TGt!aY&%mi_mP_Z+KgGw7loIi1F{Z0CmG*3P z(I;*^5m#o1(pUXCHrMlr1g?9X!iL|vT{zv(7ckuWg;%qney!GDD4)s17@AzspYS)y z_*d$pckA`U6xKfccB;XPh6v{tN@drKQjqZ)M4xTHLYF^fuwBBNuV*0Wbj6mRqx#lA z)gl!hX+$9Wy)xe_c4wI94Zdu><-)H43Qg!(gk@dy{b-Y*d8){OIFNrq+W~awCYtuT z-U^Q`(QzGeh8r3uRT4+jcRo|lTrZ#YKbMueU%pJBZD=%QDE!SugIE|k1fp-;KA$?V zvnuAJN2W3ubWrVMQb+LmI1qPQWdfCT*mMNekTX}k)pfo>=)g22#yXVGe5_ghdl1~S z_7@V4hT;E1_jB(QV!r{!$REYqxM*Yd$UI#6seH>Q#<2RC9EzT7TZqu^f^D2$@CEb- z(WuOzhp$z{j!%n4nAO~=OI3%G$MkzH&d7f@AhtcS7eHIz%hM0+aV(T%32>rc#)XY} zo}1rzYi01gw0@a+c#Popy)@z&Uriw7;ToD2YC821j7J<1?uM_1hRO5`Bcs08Yx$Q& z{8BGQW%V$>QYRR*OMWr_{kFX^QvVUy4>+?kx@S6qdr?MZ`!Oq|{4*h(>H_8R?RsAp zGGNQ0Vx;>UM+;Qsl?1ZQ@ajL02p^uvoqsg3`~HZdAEy`3kZiBUn+)cO`Ud zf$;5`5t57l`fMP-_Wti?G`h~h?=iW@27!yf+S&#r?4cMVx)Drr-(dv$Uk?9W_bYZo zxMe204I>|g!z+yM#%sA2yYkiHQWf|n7)jy+JNzqWmX|Feg*R{8N!k-L9(?(MT^w=f z)q+0-kTu~>hO~mY&E@y7W_cTi+Yj$mVhVl_>04P0Gm?w zdc9~|>vOPp0YO*f-B{O~&T=jiS@^a~Wzil0=C*10FU-0E)mR>v z$Ha-rjROt!oqY8{y>6F}oqYz^j@m_@fCGr%7J#d*!%?H|Y0MY9zP%(*@Z%<-$ms3x zbwT2IoGZfBI@;c7ew#A6nj?Qq5)QE|iqlHD29h;eddd2`e$^T>bd*eAt6G0CIX&{s zF8RsqX*v~>ccECMkVxz29f$R!h5cOI8u@#L8aO=VN>`+gMJEm)A;nbqE4!$yy^n@~ zs7%|NCX{)Q;?lky4){(6M z2MZC>jlnmY(J&kF?w?26;^KN~_2`rC-_4!Ap0z&|a|GFsH~LE=s9Y>4hx3W*wc4jf zw6lIT-t-6J9arW|I`_xqdX5SpEM2T`&TCAge@d`f-l|H%2Q8Cccj#U6DsF-29!8u`=y&vC$qjoW0M6(HxB3JT=N)t%*ReL@F2=(&Mq5cs55MLqCN);JHWvtH`4~KOl_Cl1{lJ4nif*+<#$GC4fHP$!jteSsrRE#5trD1g{~sMmXlI46l4X-U2C3|mghnCtQkYdF&!zSR`rp+M z{tz@@7h8>V5|bj}#MeRh=x}{N?M5!gl0ocHqV$topbfIHef)maTOq#i*UzrVCsE0$S@4bwJ2`9E zuS={VDzlBo{hc&_WbW+Fv-OQfRiu5JWUmR~bsT#%Vpu}aFM&|$GKZu}u`2W|rw~~; z-)_ogJ~GrM$6F0WbMugfus>B}w%2|R;ZN=hH!CPIpBmC-XCCDxuENFj?4AFT)xeif zeSKL9-L~b9lg>@5;r#p=C!}X1r7U+JAuP#bW~dQ&EZy{|4|ndTi$Sk^o_ofYBkx zt91WC9pfP!_con+@-GT9A97209peqkEd#zw|J%#lr7JMtqx(w_?k2`}wl$zP7PUNG zKp!~na5{{y6&C{6al&J&cAJya_|c1xrfUpKy3Tv_)~frrs(^P6z2S%Z@^|RLH>oRA zDfM07S^4LeA^CWbL@?;{i+ZE{qA@HuqKM#0S(RP?ku?7 z-JL!4Gl;CB`(4iVwy+S;|Np1L|KDY(<^@(|3=jMmtI-M_;@~{ z*}Gi>>!@9u$EE8nPHPBos6+*aYY+VQPaWq!=B^&b_4l2$k?cNVMS6NYCVIB%!rg0g zJ~6~g173LsK4y{7iH&Fw(6O-zpt{yA`B zf|-wxKwMnBbEDG$0$|eCF;lFdui!Lcb93|gdTU)A931$T7PrIuUFo(%4fo>Kf38#3 z=<|rQ3WeV~dt?%Yjf|;c=33n7+=Zr4!rL1iNGk^FE%p(Ci|)FY>*HvdHhc~3m+Q!4 zt|!lF@C`MPy7c9RN`hbgfAf_dr#mGP_QcnZ4V!-W0s|vW68y`%3LF!x)85P3%Q;V# zg3LL@xVmRD*Yt2IFt+p8+9@*S2~e? zvY*V$=27f5!nT~OIir%lI>pl_Ff+5F68L*WsjhmIXQ(%V(EK$YWH>$-V{ToDm=IW0 z2pl4*x7zG-^Vb(u!_u;J#de5+(`KWavqymv#$37eyVmP(5JiMQsyJ%qzsN{HWN270 z2oT8gv?=g~=r3k4g`U4nEi5ev_)Z@^%G-ZiK3x3&;6m-$kf%*ZaZyniv1Pmr(6@%x z*lnPFT!f>wwl#sfEzEkgNFx3cyWLS-O@(Vdpj+Dm_*G&+e`OJlmK&ykJLA9iILvz9 zgdSHDmW!Eq)|w!Uin{IMSwJYZ!rp~lD#fY`=H)Fr^!GsSM^ljivW$l5=U=<#D^U%f<<|G;$*6dtB=bNg!x_it5o1-{9g~RKCCBF2}WVi*Yo{CGwd9vq07~Aqk?J3JN{_2eoWW81x|cqxs4% zS>je8Y3WTrdE{p*HEoCZbh}Y*E4)gl)76b{K7imq?y)D+gUF&@Rq*iWxPFZjY_V8l zW;J;B6N$0HZj679Ni%pjFaf2Z96~-NT;IiNXN1s9T;hRAC+pL#2raHDB>Lw);y34?4ZoYaLZYT6{Awx)(Tn-F6ktovIFFwBLBDD=WU(at(9h*XG)dqT0702%By zZZ6IN&OxnNG0xdyH7169YdVP<#U_jjPEfHi&>l}lJAll#Tx<2{rN+mdH;m-p@w}aD z{g*cAe@UXmXwKp7JEy8$491hSj z2Q+5~IuP0TGcY*EW6cb1ao)9*+O-V87|{U=rqz%=(<|Ao5w3s|#bCHXq zbbCslR4t%WAfOeb_TWlV$#w(gt71TN=7E~@@6C^u=QhTBdxot_1`xv$P{>GPF$+byo##Uv4>k63s7)v3t@qzIa}yS66W^5gxjdft ztrY*TYS_lQ4LGhNkX5--Y7j9aO!Ioaa~x*rn?GCcDz#o~HJedkd^IB@WbLWR0BCLh zKrWfJrzN;SowrDd5y4`qZsawgtS=mD-Ro4N`kFMJ=g2A3&e(LsdV4c{fA6{}0n9dU z_&DG8K4LG6!&x8;Eealczw6DB`hSP0pscKJG@ZBUwz0M^OuHIK=3$%eS}VF>5hwLjz^V!b%m58fC|+L zg@<~>e@JLQTBtrgX*zY>Y`dNX6f;Fi{|g+?4+O}sh?xq}Ul{O`?X(>-ceRS31l_fM zR&@9bU^Y5q*YCff)Uymxd#!{57)Z(qMRhzt^wW;J>L^45D2a6~ceieLwiP z1H#{)Vv0A22bU=EQSyqB2mu58CVeqNB*wdz+`@qpME2t|48r_6-<29C@RddLH%&kz z!d^APPrW^lLhqe@V*owKg_HzlD3qFZ>OBI)=w4PeR0s|D$(KKsU9hAuhs;+85Wt3m z@(h1*0FO(v@;!iwq9G;(nx%F3UsDIE{u2>;Bl>F-crBg?4|sl=cHy(%y(FNI(ttJ2 ziWmxBz^F+3ENmPkgbqy4kiF}RaG;Pt!gndKC()0&4CCyjLyt z@!5ThxBsqkxFnc1z#Cj*pcOA8LaEna9~y??V}KUr{jqDzl0zB+q8Y8y1PoUp(6_>% zZ=^neEMX!-01aMD0E$`O1*|rL>FrU?-N~X10U;3Z=7YY*daHQHLVv^i0bRr@GFdXw z0%pW|i4IUo8mKzn`*(SYEbxLB--^MepzaAD)#^&MXxx=!|B4< zaALt#AoZx)ZwHT$Jb)1Ph-UwZlJtSA;Bk`YPF9*M0gbH#GP=5L$E%e9CeHD0J8`K% zG250;I(b$Vcv(v^)ni~*=3tnXd2_I{<1EY-sT5lPNvS!3C=1P2_XZSP)^4lDeX3Mu zec|uAMQB7oE6xHlL88A5;MvvMZ;qEtc%4pa8tir#mF4-j1)Po-xARa>|7^Zb<#Vy^ z2~!V9hy*0)PCpX^jEXqO4AfggDri1eR_nCt(r2Wn^>ztJchCgOX}_oN3j02~nvx%z zuPzD{0)+Y#n*$cTK;gfI7Sy!0)A{eVVi}QmFNfJy>^Ex6rb>XN%ki+Zei29UDlesdMuQ_Qu;E4ZxHJ{9pay%%^U$p5XG?7W5kM^#G zbtid{ZU@-_ykVet$zPS;Hp;B>3wncRKmz1y4)fVR)Rs{m>40!JozL{RhI=3O@gS70 zZP$QV(il@<0E|&hl#n4Q&vu;ha)?M3)5*1g;ouLz9pTuCKn)qo;9qeHzm4_gruToy z1eg3^obwhV5d=4*ResCBk9oK~wFYuXWTf&y3m*ZMJC2cItOW9g6C2m8MKU>{95J)| z-;01$KRKCKkVP?pbb_2kr*ji^zDp1~#5Jvjy0!dS72D|o3K49HXP zl59uZwn9+_Ux;~IgwFW!t(~RWjaI(eMWaiPhZ~2n&jkFgHExr9^68P*7!{wxH=Y?` z=*mtxY|;0AfU1p0s+vw3hh%Z96gImtX3Z-u$x9Q*#XMwpOF%mAY z^I$KKO2ZmRlm4Sz6qn67%&m$ql*?Jl2#Cp&(xlX*to(2yg8*6ADuW3X@`OKA8Z^mp z>E%B?d%PU02U5k)Pe>43LUpB5!Sum$HiHy;J?FLJw4es^c1$kkMOZE-yQk2ZfFSGz zwMi-%ncvE3tOyVr_tNMPq1U}^$Db;dx4^g~W;$ZQpzpy0Rv_Y6Mwql z5}4;QFyEpK5FlR&O7%m8B!N<-F4fzHK&iC!n-4%K5}?$3P-e9ir5?9k&@ehsiW(?o z^34wqqk_y?-SC4?J776McBV*ypO2}4O@j?k5n-QzJYY;Owon$xfJgK|MaKGw ztue1_g}FD`|IBAg;HvU}x1l_7zxRpL ze^LP3ipkPsI7*nIu&=zYOZDrzF5yN(`~)<%4yR7~|7!0$qnc{kb|FSc1cVT20@90s z(xeEQfKsKYC`DQj5R@Y2p$LTDJA%@s2^M-sN(e=|6qQ~=6-263frK->&-y|UI*bx zk3Z5G695&V)Q*FLpT@v*ly9A@s)mX@=znI+O(zBl#`k>@y9PI)F!%`4f;d|65Z(vb zxkVs8s-*nh1Pn_Bj13slArQfX&aa3&Yo-oiY>1RnK)m;c(9qDF+D@4`z-oHXtir5DzHDHiYEczz^K5r`S5 zri%S}x`>TC7!^0w;`66|{SM~T@mO}z%LE^QRb&K0;kDCOKd<#l_S-#87%vF`g$s?o zcp=t^!%bhY|Dl@(tCS*-sQ|{}lYrBWgPB0!1}$U=-W3j(4z`zea0LyxBRe%>0&E{hyTvOV7Sk7$VDq8S7~0t~*RNtJbkArJ$ywn8 ztgLYeaiyRn|GXOHb%YwjOPFKn*4>8zU9KZ0V5j&nKV(f2ucLot%fB}G?$E{v0*2Z} zQf~69xJ7%^_r8DE1mXsCks+(RAdNKBMYasj>F0LI) zfk8pE!C2|FEh7szAbE2~DrMb2?dBmDhG#XrN-XI{wyPaXVDtCMP!ZCgg`1&%WMK-YE3iA{K8;i;Fa*~VAE*NNb{*XICHt+M6%Qx5FzX#DUPv%7 zK*Gmd12{5}ne-SnhAVV}Yq-2*LM;;f76&d%s6&NLEy*o*{SQwxPFqoK1pTcF@&R-v z)IJ_6Vzd9uk$a#R+_V;9l%bCA_M+ZQrJHvB{7xFN3H3111n7a+nW~ekZ8~s+Fw-wv z$-J0F*Yg_>({6fI7sXCTJ}~p`JQlv)JrVh+-~uEQmtEUUah`4S=h5Ebe#wPzMwaiO zh03Fqx*DJp9^EP4@!4h)DFwp$HbxN-LSbF|$DET&h9oipF&dg(Y0sUfZeJeik0p!e z28M8vQ;j$NVNHh?HlSveeQ0kS#Mw_a45`G*PsMDYOB>kVS4D7_KArb0NZT&nMdM0L z+}EU}uWmB6-@yy5Y!-T3@L?8rXD64VEB6WLp*`g!{ME)ePuA<6-Y?|u+A=855YM`- zeNbJmxth`I&_I$)6Z0`v9?KjufE(yN{zwo4OG$55{;@TD6rpyG7z`B|BR(8t`<1;Q zB*fsKf4VTAe0xa`H_&(=TptVmt)VZqLhm>3;*KjL^iloHe(FG%YpUg_+wVxG~(ZdyPP#y?w@_vwd3k(xP(4q_~ z!c&Sb(7?ZB7a`a9k^A)mXxIe>mPkGfi~m_u6UggT4BSA1<5fPGoq1&41|sA=FpBi`)1UX(g24nnd$%3H%nImy-4o_I4wP)JL_0xEBY+t;CHRkN30ht6?)!M6+OFD4W4gwD+Ow6h$;_H{xVS17J-%L^xgyBA zGF(*8?&XbU_5Aul+PT+59{bx++GF|haFLgL(jw+?d`Q^7o%P#lAYWt-)_oKn7&@*m z`lmWW!rOALlkrNjb(7~RImlTuGQ3w{U`3}#XUdLAV{4l)9e&oaC#CP7pUBjVE-cEP zy}N@={Hbh+(Mn|SvWbBp~og&udxw~RvQuo)cvZBu&@DxJ-cnTw7jOMoZ z!&N%PXzYT&SWG#uCEoGht8K%vS}+aX$QCB!2`%94_=?iWC zzaD{MtmfMgEv?C+w(ZHMEdmL}=qUW%#Aia?4f>41t5lCZc|YZq6&=kv>soy?Rh!qS zel^0@MCH}B$`=*ZV`uhBXqktybCu%K(kouhyf;j8o;Kx0GS#h$;ZA;Nv91ySZ0CqL zRUscr+zyKP&~h+bpp&Vlq~rSSrKZ2L$(02yV7W~GVYwzBkAU?o?!ZmzP=Jd*g|KEUYc{VNZ_` z277pk)ymt*PRG*Pdp>nkUeeS(K!Y*F5&P4YP@0hQv*Tb`Dpg{2tUk7$6zoy9%=>?C zibZ@|of`EM<&nlWF~y3mKgYi~95>UNNVirRFWDT5;jon!^+gf+aDvy;m7}$79oBEo zg!-S-GE*Iub9GV$-9ngXNulRG-HSarK0fx5jC~brMm6ID328|`{}68z6aDzS^!{Oo zYxMA)wFZJ)eMTkq)KOMDgu`_yheL;f@Y^x9`E7TUm8b`eev`bDGPG{YYqI_|hpCCP z*zNxP1Oose${^rOlK$aP9E$wb#wsSRbbGk}_{+;~<>ubKO5hh=L`&iEf@KPUpIL?m zD_IMUfVn!t`=HOd&)|I~*sqjX?-B-8_bdBM>STL*ur!*G*K2+N!`goN4@*?UAm)2A zG8gQ+H|L`BX>mh4F^3`S3?+?Q&krwEDlX4m6#wDRHI&!>Pg|4R&kT0P`VpcrAbk1@ zlXB|I{K1m@FV{3Td@gyzVQT!LwB(};;$A&wSk0AiEx2`w6^m~!RzI_=iVc=N*n~wy>NZ#FNb-npf{j=br zFh(`+TJwzhj)=?cft0zbH-@&=KHMIBU|#uAv~J|%i63%x!JNmgQ~lj2yv)`Lc& zs0ya&-G59|&59&I&m=yRzQoH6p`7=6EtDuriGQ|8-ZrX{}|ff?cQ&rV)s zEDapg`eJEPpA!`|6f$X{>{h)!zihpvTJ5^c?_RML!#=T!ClLp8-)dJ=;eNZyU-8^Q zj&yZS8cdUXc5DPMX-V-unkKCYm$)yg-=0_b@nL`RXHY_QyH@J)2QiiY+ags@S+JLd z7@ab0b#6L0x&E|to8KK73)-5$qffms@nZYO*!K^*rzzNz-sE^WA-I7M_d_*qkoV>! zG!|At+AZE5Uzp9diHH;V`2E}buV1tmV_(j3HErjHuJ^^PpGmt;tXjUzuBzOBu|epO zo3HkcBOxJ&A$nr9D#!|xPn%nY2*J;Co)8#lRd^kT(xK1^ozfasM|gbrSnzyGwzZ)>uZVes2hjZz9j zKl?%#Yu=knu>(%A%>A57$v<#1-%LFYohFb|Om2HGGDYhJywbJ~uL(F8=!v-Jxb5x| zV&ANae^}u3VB|qpST|bjlbTNE*VWU!hhIovrY|X**Kya$6RpaH7G7Ym!sLDMCKW+f zT^J9G{>~U*mulFa?M2_5zS$;|%4n(BuAlR+BUA3UQfGr0(fEr>vSH@?mlP}7X?OiZ z@gf)6>BX^Nq|Qlz*TRJoG*>P+6;EK@-j50Uf$5%wj+tH5n9O02%1ply$6~gdEqdR4}@}8vWIt2MGU3UgHpofm-Eju4S#WE z8U37O3Vg2Vpd_?jCH8g1^BV{5{cm(~qJG+Y9VDS+ixccCDw;gZc}=b5nzOH#Jf0nk zc&>RAy}Rs`h!se^m40op&_P`e}+KX86=hJikxEsdqef5%SKRY<0iZj`xruSV_knfuZsjhwsX+PtO zcnbu_qXrj#=S4Ox>Is)G;M^+*<|~r8Rdqwta^H=MMmXfWL27Z~7#jP|4mPYD8s+Zw zk#>kCgV-l|(r|;zLiD9yAD~Y3AbZ!Th{F&NN;4{GjOJBAeq7?Ka*e!_j_&Z;pQ~`_ zx?xBwBUBNT!J5+5LdV~>G9T++`?U27heOosiRaF`K}^aJ>1Vw`YL1@u9a`}ZuN0

K;A2e}UxagM1m(RY!FHun5{%{hzHyl^CGGxv{k;Ce=Jgy$#I*2_ z6{63sxvSPY!EY-U!xzc+`Ja?m`kbK{yEj;1{8Cz}#G$Xgn%^^Fr+E32ltUp(_z_Ll zeG)eSGc7m%k?PDj!wecUPSh@&GI!NC(v?pQe!Je8{4AGAZ!i4Pg1pV*9yA zW%t$7e0`Y<*kzPbCf>;?Y(y%Rb;I=u7O|1}j4#-R?(X+0=d-3&TyKtszI+$5d0%^# z8RZPpduNwD+Nv0kGFAK|WvZQov|pPo2B6ePajeOc`OM(XYZh9G*sT_g{G(&qkqrXEYMoNB?t`<#AEqoqUUT5>`f|Z#`y=&}r@}#^cpGL`Y{xXkQ*4TrI%AOus z!v`%@*H<}(R~f7H{}roX|LKn$$a_W`KQ2XqJa3HaAMN>{EcE}cL>gK8fp+u_ad&T5 zqomxNN7dR+ap*R8spTNVYwv?X8>fE28b3< zH>)ofxvi#>uQl-CJwd`%&L&gdUYpsWc3+0_4IaI@4yAoo~EX^s6!8tVfU1=@r)VuWHUz>4CHL z3JMDI^V?NoFIo|ZeOA&dmGKN#zIp(XF%19UQve{3N5LCm86c5-rt$h}6@YCYfuru& zU;3~&zgIJR@Gu2r)e~hMdsT?mP(uRF+cc zy8ofM&st?eMFicSiE$5P1E&j`)Fga18qb_}Ruz=UcPUnAl`?7`qe=F{+p4b00R#*9Z6Qs zvOldqq~MW#04NwPcK8VJwvwRaCzj*c|BVu!10^|KJZ$^|`eB@{OpeaK1=PmSKJZZwae z<3XTWt-whX8>pfRBX_-17DK#4sNE)4HQ4F$q3 z00^s4@J(NrEZ|<8f_mxv$v_DZ$H#m$iV!{Z>6b;^anFIb^ga@h{RQZrPY=$}`=1BJ z0M7gKssTh#v&r^5_W&b!E_=m?_R^;V!+VhdP$rguMCpNP4jBPZbXd-39MI}BE}*1- z!{r*c(*TCDbLAHF9KhoA=v0@ilIcsq;pT2(3X8}%zlS(l;b&0T=*|5}i=(}@3gKe# z=A7t@5AW^u7=TecrZcE@%)WWJoZG&%^Yfc7O@Z76`wvwQ6rrh{V(#n4fW$nq`;saJ zggIirfQkqywOe(nbKpge*0~m3F{%50kS=Zi$a(SogMWk9A_k@S3~?Z?83WRmqU>z0 z4|S)ov{IHj1pGMsL04dYd#i)vdHEaBIKLP4hjs{mu(C#Tlq2l`u&p8?N^Xl#S>pvg zjClX?qcb@D$STR?b+^*Lia#v~h(*Ahd97N~$&UbH5<33}peq2<4}fm)iBkpNZg?t7 zhx?tj^DP*;Jns8v|Gx6ZsfJ-txEx3UKtY_ZfG=W*7Xp`o&q6%-7Hb->by~{*CioTq zzB2Z7+U`Wy0TvEasm2FV3+1=x{=V|X>FDI@TPB>A5~d&o<{U^rPFn^R&Z*$wQX_ZT z-wi)S+3zDh1pIyFCGhR{GQ${n*3*Uqbp90nzr}-(l=)*m|M|tSO%Dh82G7Y);sScx zx=iHR>Fn9#FDA(){+DdXKYk<$Q0YThxA*2N$6}D(-CZ}pP3*UB*a3;adPkEpsg!+3 zl?&8$?&sby%|%6Ux>GEY0m7gN;Ej;FM8Vyu1=k{H;>iOF+%SYUUYeH1d-D=TqY0O~1?aIejzi5Q(&LGA->eX+t>vA+ttHnxVC834#wd z`JR-I*4qqn9}!N7W?sbI5qm(JRa^Bi3g`h&H=no}imn8bw+<`w40-pf8otkXS^ZB$_ICmxL__*6gUn<`^J`%yPfxCCh}0n} zn-R~ULL3L)EQHU#-q^Ot&txTyd#S`$^l{*+dJeZ+^nC2aQpQr>H2h_BN$x=Yb3)DS z39vddtso>XR3lAv(>5Wgyk~zFJtkusO8^H0HntNaRm@%kunJ8mIe9Bmg?;p zJ2Vm9fbQcr!sl)5>(Or1)|Z;urtS1O-y;I6Z~MM|1BWa21>@_dds43H($!ajn)onW z9^!uD*Z43%U)4}g(c((S=(!o2@D$}^Elxp&KIrIo*A#V%Cs5I!3>Z3fx=KM;(I?yJ zDHKpteai4!|J1%KIv#6wlw-1IUPg)D(eSFA0Gd2~nV2-(hxxT!9lM3p^F;c}29vEJ zm&7CGb2}X&>^rRiFWYrfPy^p+S=Syj&8;%!ZL&wQF30x2Rd`+HJMwe~-A}nb+E0k0 zs>e3r99&R_h5kg%wQ113)fOG+j}Awg95mO?bD$l7ul)T5#qO85k#&6T-|&{>Jy_&ztrnLNaI5-Pfa-~`s)VPu#=-*^_n`z-{jKx#(D*@U8CIi%1H!QAs} z`|DFj6+jol{r>aXwqneTcDq7cb|wycREJm3;vwU5&YOpM-^OaD+saJUA(}v~xlMZ{lwMYA#T$?P!-M zLuzv<_QC;CpBvB#f5NTO2};_mt+`328)o#Rl|g|wq*fYye& z;A^e+tr-E2P1+=d+hn|@o3!bezkYWKOc+{cEGnJhEzmDy8>$ZG87J5dOOf873s61MFonNs`;-55)GVGSsu3v-8l>(U-Oz(n*1~S93n#g zia0BF?c9@5IL9it!6E5sBngf~wKkjT9eh3t<(UsayH&hQXfkhaPxfxwTuSA(n2{I7 z7e5X^dC`nNIoUNkS%h4V^hlnwIu3I(#%!LIwvZp`8QtU%fJX&7(NA9X1r&Aqmvptw z4AD?YNIUZ%7HPX=3l*bP8H;x6Q=#mykt3!-1^ig7h2Ok6;PV|A7Lsr~eEaN2M_)wm z__MI^wEP|_@fj-1V=taJ=Ma9}h-IT3<*1>ZIG7!^Xhr24fgeu1m(~K&kw&u9%Sid# z)29q^O+fo_q939GUaQICG^|CY^4&X&o9vK-G+R-5IBr`~6^<0B^KB?Zc`wAPrh6a+ zVRO(kjAT(jgk>WCX|sjOgzRK$!)GagiHNABqYXeSwq)8PMVgMLHslaIM%<2cYK>SJ zKWjBzTjx34X0T{&6s=-*kr{DCzZj)R?FIW9g>2@TK}LPbclnu@B$urW(<^$ksJCR! z%5oMmx0aKR*Y@>(EXnP>{Y7R1D5{M)xBcT8l5xHPoUB9(O5bm2NJvqViCc<=5Tvn>xC=aI;8@F0`{hH(;o9Q0vKVvLdTkWVm?aLWmz<$Eu$R< z$F!4ktR0XsYH8F~8&cZ3kIkB+I-Xoy`@kv3uP6$6f;@Q2YKGdxpIHZE9HH|R882Y5 zFK`2PeUCVnNle)m-EF+N(;RW9?Z+#@`XKRL*{e^H^$pmAvN9~&{7&lwA;{0QR^QPM z0qO=UTj*#ll`LdR!0Iq#`5l@Ks(xvWR8@DolAe{;-=-d*qgEj zRNQXliDaMA&q!rFK468WQ*AC~`r?du6|(leY4;?F)>vm^hf&9yw`;=45L;_JDq+#+ z`6ZMTUH@vJ4fGs)5 zCAXJ|@`iN3S|Cy!5oHTeVPRxpa^n<=3Y1%oC2NN0IdZacmhvd!d{^K4+F7hRD(#f9 zaJ8sNm*01|Jp+YE7tXNBI%csl9V5^OoOG+L+|){8p6aZ-2=PPjA1T(x7=a3|tO`-? zu7l%P>uy52!Zk*<)fr337U#D4?@7ko8NP4Fy|n=IHD6>o%hJ4KEhDAGqyPPavgI4z$mWKNYO$ADb}6oxpG-1=eh6E zLOL;Hk3RD^z^;rm6}=!2>T-R(cb>Zxvp#Qm{HnX;JqiUjAf=ha(iNI0{yh< z@{d8N3BRp5x{!0Bi+uqIY>a-zd&c5QyY z7V9`8ix;ST7^DT!gNo|onZH4!@LCjj76I!9s6*!1fk-4?VXp;knNJVL8_%oJixpqt zLZw&nh*Z7+TG~3lz|sN6HOXkD0Uj}DH?7jgyR0nK{!B;7sXQ~J8A`O{o>Tt%*-|Gz zflXtu0phCPASFZv&yVCnE@-lfD+?m*imeW)qpCY6VTGFF%|ZBX670f$y%UAbz@Dy? zrLm@=xJvu&8D6oMR|e7CL4MqomX1)*!zZcd@?mEbS(GohP?o6g z0=7Z4y2+KZ?}k(5F2Gkcv%j7f;M!DC=(K_FjJ6)o(K5F(hLA4u`&bws#1BOo8?~sy zDCchFcjMBd#I%n)ze}`ql?5o=ixpw^$FS{Ca!3dml%nS`HxgZP{Acd9lQdPGzqW3L z;LRx?zZg_z`h+}d7c5FRvUs0}jba+kTjlxHg6eKmu;9S8qhii@w`?en@?w-XozF&z zYvJi=^U2`GzDG5_zxveX72~s1ulv9Lz!AV|T@78~CFi;O(vWTd_B(xS+valL?|D}L z#Ghyh!Qxl(h~qDW1aOl~yFII#{G3 z$HWa^f{D&8{Bq_mbEq|L>5;BCPw5mmhpeq;tYtS1ZYvs3t07^Yf4|GY_jsP89tHadC05dq=a?M_m(J)0g=NLEA8+1N zWJ9#7S|(fG4Lie{LX{$|8)Phh%V0+9$pwNefLxa&70vgBo(nwI?>b-wgH=4d-3tJl&cnp;y5QrAQmRK_d=fT;d2h^ zWB7U0l{paq<&3z<{Q1JPNJ$^`9%ON({>kXnd6AGQbOt;3&nfJTisb9kk$y3DU$X)e`#JR zdn=FQ!uc3VHJ>A&%em1^OMW{G)(0Eal9^iQoxJ5&**hYu!TVP?;4YZUFqtty6m7ytuRkvOU@%=5}J0}~bPQQ>KEuj!_ zGt+L=4=wA2&paML%_|5ZF8Rok&$R#59u5=pk+qPK7Q-D>U49;j=JFRu-BzO$62-9T zgk6Ft=1&@wA{b-T(R`ZdgK#9ITsY<(a7fdCk}lhhThGfLjQA2qH20heOVKH{Ok%6^ z7dcz>x>4x90W@%SUbVN{llU>PHehLunCyIZaydTzOg$KYUA&R)PaTCHF($V_OE5yH zn&Zj(5{<`yTR`|n0_flYaPmw3T%(EQvD9lECtDm-J8ax|o%@)jixiLVP&k3?vcAG` zTtHVz2_c3P@cS&&jSy1`%DVGJkw2i+@k>bI`tkB1*EbUpIxgLa=P)Mj7J)g9A5wso z=m^Eav%bs*nv8}LqjEpkO6(}7!6>=a@>SXgA7JynyF6#qfAr!e5284a62dc&7X)IE zEy_kPCtVg!0XoZ9qu`x2IW2G+mWwz`Hcj%cZr(f^dFQ!9*u%Qajz62#R?~?x+8n%# z5F6@wz$+l2sejVpd)#3yAn~P)7X6qbMKr1bYiy>BF_b2Z;%bYyPOg$ujny+;tBwJ= z?6ca@z7+Yn&ttNRnYJvzl^VQPh_jXp&kDwG63s}9W|z21p>@80l6;+-lPX?7`UY(3 z!h1h{Z{OkZXYx0%i=Sa)65DH9ultpH1!Ji*X{}7r?kZU>8h|pyc+Wq&P(X&DPRAF-t4E$1tLMo~xkB$F&d{KO^M4W8ZZL>S6G<{-7AVvIyFdC=+yMqEAu}dhwmBb9~IhSs@$J zFUygsj_92C{+fRt-e#<8`Sf~Bdv-*uDU$FIdYZmP9HvH z`W7`TEZrR0dp%G96VDZI=(sn1k=(++`!?;(g>q#(mG{Is%0vD*wI`LD$VJ2<ciA;w5rpKwfi^G z-Eb)Jqw!IeHSfd~KCxm;e=GJ-t;sTGw4?0D+I9giHtl;LPYRpvE6dPQJgTXeV2z;{ zYsk`q$|WnP!kNh_n>qbnK6RXZb5xN0*kUDwtPhRI|5{7Yk%Ddx#{52AVpp?dRPnHN zkyuqh_$^BsQaBfkfm>xJ2kh4ygXtO>+aC3~ zmt;LeRT1ZzK~>93mT!Q|HivKokvl0m&fMm2qxp(Tcw850eX&Jc9Y3R^cW1bzN*;Q) zdG;dv7*%s75zDHX*Ai=+Gb!!ATwdF(!^CY2LdndY2g$D^t-p(foJpZ^li2MDheS0F zaK)fn)sbSPE=LhDhvSGaaYiKu^Vw8pH}O|63u#~O+x$ZZN-YziYA#ws1lq9JnNB#e zp#j^C?@jdo)OzRSmkorS4+AxWaFa*d-t_T|A=9Z=q_dQZ&eGXja_{a`NPaZeTX!8C z3V~FN#0TBwX4KSidZHz*ny^eS(0WT$Y z43nDQgbcEg=c4UY?tYr`n-4&&dK+T`1T_H+uV4U+^b_} zn!`Q_UY$eoUT+03+KD@pogqVeB@I(MJO(NS5wzZ!4E%h7%euqT5PBcNM?iQdJIqC8 zJ{5+XQ)Do2K?RW?8KC-DT)cS%k?#=#kT}|6=v{~uM1=z-X0gAI+rK$-{+p9yHg@&x zCI7Ue$xg93*0rywTS0N0FPBtxUmVBSd_@HaS0eP=3$u11O^@1IhQ;Lv`O+}%pKDe#R)gAe&u6bfTS zxE*rowlT2@-(Wezm0xu5{Sq~Tw>_L=>vB#&Z9H7kRD!Q9jfn9Vo&9B&%u+LQa$Yq5 zk1Wn36-8a^)4la@g(SY(n&foEta15>S;yk2bK%pINHUkXK1=0kHXiNMFM2Vnfz}X&#@}{OFj!g8j|@ZJ{5$>_{Z*j8cZhq zq11`v5<9$nD}cM1it8;J!k2UF1bQ=oo&43_G0V*qsyVexr=zR5gE^knTpFdQnSR;Y z5XAujGWzr}@E%_BdS-cDBh74t0sLhcvKrwHFNQvE6U8AE;Q3LZ!Q_>JFa#%yJn`Fk z@;JGj*toJmbU<^tGn8B1Ua^*G()*rMUUaCA`n_*v=han+7?sKeGA(=pEG5@cj&Mo-^Y zdaat)?A|mW{&BaDO(2Xp`ofNyP-=+$w|h`UlHy@jn=1k$$Q>L;$99pR22Sy&^|Yjb z(nD#tvasbaAA!w5lwvQXRGfH5*j>6wxES<70Fql3tCV0DEL|2z_b1}2FJa3 zzR6qzVtkwu+CEG2qHL>F9Co|jbS&j^K~yM5fS`DyR+FqO?810(D!m77HKvn|f-X^F zSM$X*>`e_i+h?Lg&BF#-qL61~d4kx^M7@gM8WeS9Ipew&yrln86xVu^PUdt2AAk(u zCD*Bl-7*K;`CD=MtHnDTe3)13%?hgmbT1JH3Vq+-&;~%Iy5Zf&7W}ew>GQ1Cqfqvh zuM~<5;0!{14=J4;iHDbx>LO^3E0u3T=*TlT@AyJ523*=5i&Y3wdoi1vjzY+-=+-1+ zr}Yl;6sw9N@V?dl{dDthDF0YI8?7;_BI_#laLZi7dwWkea}T3tgzAahr==yJZ+&J) z`1A{0HO0{qP~J0IDu|_4k`v0O>4(ppd^?Qem`t}FB6( z2=>GpG_w`Yf?b|r3W9*}%J#TFe@bKd2Y~It485}#i%)s0AV4$z&rSc2cErI6RY^#_ Vh{?rhaPSA2y0W%X@hwci{{qr(cGlY(vd^2MGx z9eoHG*Al2+nID5%W+GFWS3(ih|@!;FkZ+!}e%4 zkL|M-x5o)O+~+{!D0-|Zcxp3a>bL}Ci^9UMJc?>B;9YYN6yA4j4bYc1G}ysee}gu+ zG{5+fYwV@#`sDH09*t>6z47AxoA-hrIX1SW7t!I`l9-YR;Uu+p*@_4h@IjP4O`62i zJr4MSMo~U2u7$V)81wYK)iiDB?@2!meX>Ewal_ir2y1cd&p$(XtH)(V|4Q;V%DN;Z zaVSC`r&%lL86UCi{Ve9ZGIjVFT=uYwU^TQiTWEM}%LdER+B7te@O84{c>~Y$*umaKO;f<7(5A>UG9PJJLSiuv zkchFH&l__!u*sy!*+*ldboY3wW!A_e*WO7L#h?mCVUtO==u@L9tIs)pQjOBP10GZr zIHoEEr?OC7mYiNmLNa?a&)RWN1z!if$yZ2Ehv!0kTLRQ@NADrUZti!-%hTr1j{zU1 z(~uR5n&Pw*^qj%F$wVeou?#1N_mvzfUsy}uy)_D>RgFcZV#Q{Zdwmf4W!(*FBcjnf z4e_n+1a^XCDydisni)1};V^|iJ2n%;H_$yjBF))gqo8auQBhSN+e=>@)Ka{PQCYb9j|iVqij}Ke_*CYikxpcXkO#kQV?8) zXYPLmzgjwhKB7QEj?--m4}cGrq=&0o^Q8&RhgV0w%8sZVEaK@WC1)egN&AMb=S3B&E;%0^Wc6&vAF3Ds7V31AzrtTrA zFg!GThchN`dSzOk#9`aDb*`Ik<-s;sD^UC}zpcK9&@!Sz_wc1RrbE~M3p|l{K69eV zd{ohvCJ1_28I8+X=-DQa@|VP;n~pfar(yfC7Hp^4851gaHQ*4|_)aJb%9g@HGOz;DCD1}k$pw^?AF~VCZS_OoU=AXh` zzbJR|a-t0*G320^!Z~!lsz7u^YVH(w#hwghbcN4)LDXUGO0gJLQiRv?Qf!rCR~{P- z9>*4b8<|cs9Z9zyeoGyqK#UvLLuaq>Dmg9^M~EBE;C%{BNg_r+e*x(oiO`2E$x)i4 zey2^EiLe$a@jPNBSuYK|$yXFn_c^S*s46l{d2_qkHOMBB?9wdbuRS>oBOsC-IWxa7 zFB#@zUc8~OG~Z6=*$%C&)Mup_>MEEeFU4o+`(9yKit{|CSBY6l_V6ChO?b5D%NlBY z&qr5Eaf0knj-LXn&1b@o_(250kvFSm$h;r8I+IrASFtrw=D_w~T%eEz(;DX*@1pV- zu#XIFl2i?YUw3P|+%U~3Jv)Ol?J*NS-a=GDR4YC1K;otXD)rCgh5~p^6nimyjy=IW z)Z&kmMTKQHh1-RMWvOM1x-7aZdJFm9Z;qnpd1`~I2F%$#1_K^3Ie# zmfHUqn8+$HC@?7VRk%pDsV6iTl8gEAbLw-xPO+EFL7{Frzlnc}cd2*VwdRr7{7y~w zL87OlXP4)#Fq&80LDr=2o4(ytPyR3RW|AvX5-KN?4Eyu@?w%f=L!LK#b$i#ld%M27 z3xeQ#~Ckhr)EJz(cTQ>DxrrW2HEU}Kiv8wAmY?wnUr2P1E*ix5Mjmv*H zZiH@RYkOgMVfcZ~ku8}un(f+XvA#h6Uf;%`tkSXau#&r))PZm4d+&Zc-`sqkpSdb>s`Q3#1GF-n`A!&Bfk2-ZO_$*W%ZW*O`YLA~~24Okjjk zgmV11{IvXjw~v<1c%1_?iig@;p1(8phuzsd%iMx?yIiBZnZ1jBeEBBcS>5-4qBD{Y za=Y>z@mN~(^GO-~#+4r}&lm5vvAx>sQR3OWt?Gg0lYOwf`aIwLOM10(ZX=ax$)j|K zc8qXr%yijQv)r(p-G$`@{UmN#AkZ^#Js|Zy>x$tv{C4!p>HKMRXG?F@XZ%Dh*cNIG z6?-0&Ux+7+UFN!XdoK$oU~VKSPtC(g=jvcCpXiG%@a^@kMujz}1jzQ3Csm9kqBia7 zbq}8msk`8&*g}F><`Q{ym>f7Yye@1H%n;^;?2qk?1EDa*TgF=>;o`{?O2#H3XyiU; z>;rdiXnXkef)X;swzcv3Uemmjp3Jw4u(S8v(Dyt{uPo_Xj;V~t>6`5Pps#wkKl|WT z=cct0L?KU3Bv9tQ`1@Alta{xDZx`hw-Zg!TGPb%puY@2l+UmhcyjIqxW2WyVghfW@eLKz;0+lHeig7qGzJuOQY%oiRzP~AhrmhZRa zh|FlDM4wr+>QB$Ga3m&E>{1kzo!QULYOJ~1^lKWNq8^g3l-p9ymB-Q=Q$M8z8V(xy zJ@;uy=spTousRqlxR-Bb>S60e>MiP%)VSZgKGKmbp)99tZgR%(P}%$3KCNo%U$&SZ zFeE%EeAg5oy&D~VrF)=P3ayc9So5=1*m{oZB};P*Xpd^&8{A$UsWp02m*RBSpi^@- zT^H%(rNghYwGg#{Rxh!X)RKS2l%4(EhxCYb;m$V=q}O+RyHxbOwWQ>R_{6|8bh_W6 z=~{Kx-`8xnE!=Qt-syqlVF>d*!Z`v*(CF=5Z|ApAwuk`Cspq#(L%rRX-Cw;FmSUSA zeini9(uUFxNnG-Vq)%j6(Eam+GRBL-!F)gYG^%`ZBaq`N=-1h@jiSd*)FEDi5o7(L zScc#Ee6d^6ysG>caev0g;Q?LUgR|-vwwa@MUqsxOcLJUe%h9}u#5gFQM=t8SolKp) z(pot9jIUc4XT_VIW4UzMgt7agK-$3Xw~o-pGv&!Ne}p>(1l+O|5fFZdxpZ|E z9H<%%&X5$&6BM2ktGA4Zgs8D>hJtBMAYA5%Xg`MkPaJVB z;+*5-ZWV?qRv!TdB7$Wwav%bNiwYbyAN;9-Fz0x?z4__|Ch?&Y+~=a#CpoAzT_WAn zaEMLL)ctTPQRhMTD)B2fk~kqqw{1|OFDi+#n$9~A*yAJGT3^vdMFs9PERPKL@&zdz zA}sd;mc(C>{U4l;u{V9z8 zw7s^zr@qP?VM`Y$b_*+)x7O?+C)Yo8;6y>fu&k4{rv)|0$}dgFbM~P7hsb}?k+Jr$bhmf)w0Ci){zKQ| zt&5kZ7%lCej{fueXPnj``~U9A+2h}7!6wM@r-g%yos;7~v|*^Ce`vp%>U8y|HRbsuy&Vrae{T}DgNKf^)K@O zZ2T8dl;h9L{|6`jVe{Ww7@x&aMLGVnX5y&2_Mi;dGLqZNsA%248VpfN0VKa2rIvb$OPz$56jC1S!L@p}4yUg^yR{^Xb?`QA z2g0+q!t-=;yeA~GZJxrYOq+;I4<98-%h`oP{1MFG#gM+F_7XQlh8hQeOrN_Yqwr6G zm;hFaL*?40{@bSTGI`t`^Dkl|ax5yWBo{{PfeoYchn%9F5YoS3a)_`}J}_1&5=QY4 zE&7@)gnz;E;bA2WnqG35e+PkUIQpmKKVTXmlDPBVuq`oZ{}yGaQP_6=As*H7O_CbL zNjs7YDe0e~CxTNw{=t$l*QQPc^S|y%5B+!C3cw+u|C-@zYy}`0!P{Y}f5#1*9{K;@ z>7l|SVNp6_k_)=t^4JWg+Z@hTj@TD{nZ6ZMhSH9wepw-KFR4<#yDC4dvSlKI>c- zPUH`#ljYVCWscU#!^OtxxmpK3n(DpZMFJn0)SP?c$n;V+A}JOD$AWZLD{X-z$4kwb zM+^07D>!pKQFxiZ)g)RoI4!y?-je;~+#JUZ`5=jFn5_zadblBC2r~}=DOcN!ykk`P z?5XMhhU9zjP%7IEEZwg=NcMz%*XGsXO`QyROlGlV>Bc%0^IXqBO6aUb=T-NiGIF9l$$8Tgv2rz+s? ze7mxb(t$GIv=eBva_zJ-Z5(j+lc~0kqwV(XML%6a%2U zsy1oa0p|nT!O)xFOpd@kHtu}R0phD!bOG!t)wi;U}cX9>YAck?rC)1`*B-$8bfgGI;r_7&4jA(h5AWiYdpeOCu(!l31&AUCZ*yw?lYvQxyHG)WF ztbQDLCI3-O;iW1&Yzdf%#y;vUoFT^%EZ6n$8UOI0DZc;Wz-LssB($OxVD`99l)3WI zy24^nlIh%3+;}x@lqt}%hA}enRaxXCifDe21i>2HmEK9tQuvC4#FMAmOzk~vMaLy- zgXfE#rrgl$e$1Qx=vJGVzp7a{yf=~92OSWqzFdFzvU>Rh|C0D5W#qhq_qC0%MxCrvD?fg?YKGur(0*1iic*{0lIdf0% z!SQP*Vbhxjbk|wno;ju}{J}yCPW+b4lk+=<& z08F6Yoo|gJ8TwN@_doqdsZnLT(dH)S_rshN%#mo17Z7??n zv!z$_SX<%~jaO~V{gjc!v4HoHuPJ<3n}b?#*wW-$n>zzu-4Asqn zL`%P7=KUC6&{swNo0Z_;3z%%K0FzJs%naHQSuqHr<+;)oQpClPBh)mlb2PX+{$5h{ zt_}Lo-ro3lJ|q%h(pR!_UoJktAm8nkQb1=Z(4@dQB=#4Y&&sQV0Gp0sBP32TpW((FaJ?k9R zf!DZ-i7$TwLlk%uLyCi-<#9^*kY7|*AIt%)Gnd&PG}`M#%b{(0_0RS(nsR`BnAq^v zD8pynGFhMnZk8?IyQu_EA~6-Li+~H-PbD378mFJTdU@^(sURO+bFt<*U#*Ee9r;n} zT8&eN?wHKjj@2woQ=OB2dwWcyFgG!{#D| zS)m1wp*nPT`XKcEz}iMbA3?f6Fopb9yHF6T%?_Gj6smhL*pd6>@MGqpmVM^Z%*c!K zv*or6g&B#}GtXPt)p4MuL6I=7TW)I7r5a>dqm!%Efh)o~{rx?&Yif(@dRG`)%-ci? zra2h}B}N8B)nB)>2~v)%!8audVZmc!H@st_7jJ0Dyc}lMi#C&zdk~ZeLo~6KbiQ4? z17uKTbs@r28r94kt(Oyd4VPvj2@=mBMeiK_h3JH?U*Tt`+%>Yyv=TIePZzo004hg6 zj2o*0)P8+Ot3+;81Xb!OtW7^RWmjoO6p%m7Fx>(m^1ypGFq6MJ)nqX>0ty)?0i`GY zhb~!i)MT-P(&|AE$PP+&Bz&#Ubz1OU+DJE+gz^5mTi`=|u>XME*KbLZ&BQTW)Lxe6 z=({{y<%uDU>y%c^Z=&9>nU{+1^D@d%M>5*_I_Xd#^lS z*9#=a_seH5grPGuQ}YQ9jHlQ;94ET@Qzj2}_#%%9V;wLi)nB-qp?3?M1!UwdewZ6^Y$4kW*tqguJ3dOtMcl_M92?%8ZCX zjafrdA)R}I5mAu_&EwoJfJtUsDEgNeKGnoIx-vP5hM|l9!ouJM@>FX|Ymnw}O19f> z_9pV6&KmRH$;2$(W4Oz0IJfHHRI5T!a=oAURk0eu+uz<4)jLzxEx;Ouq`abUaTqk9 z(F^3b4+73(4r#295)4mqE4Oi52J|EO#BZdmptlK7eTD4B#282I^b$R>O<-@(z6g3v zLS*dR3jO*TgA>-UP%YVcN_ zN&06CXr_5tryn6|w>poaim~~o^(4@mG9B;QYed$aN)Rhfg(`OH;#9V!Jn+atSz-XO zm?VY{lj^nuX;4(g{kXPwU({-LHU{5azQZcndXwhwg+|7jt8z5!q4;vFb;w> zr(R8%@A+Sgv~7q)cNJpeFah&dnQ=B{e0TH3H6-=S0=@P)4i+*5t-_6CBi3cA@ClRi z0+NKjT8WA(K4}Ylbrg+E1M;b^Xk@WEdnvi?e9~+wqg!>kFLZfrn$2&1z5;z(`MQGs z5|!jm`)8!!7c_7Te9X!7@d-Pmw8=UyXdN#b5V)nd-)`$mdmmZBtmmY0nZo~J6$PvC zl+)ZEJ>F2gzqhl58n5S#oH%D;`dS2be~f!MA|*0$(%wg|P6DCi7$HOi4~_Of+IB(J1Sk}G$w6%Lk%0AGi31mprhgNOcYJN}<%T93VCbJZDJJi28#ytq&9w0VYgi9*>oQoCs4Pa&!jvkw# z3AFR+X%vh8II$sxLRUG<97qX=wex4*$|AiHr9taVL3xR<$F4_3PFObrVOk%WL2-Ml zysUk9EWG^m{*@@=C_=GJXhZNRQg8*pYMC6dILPW-U_tD7Er(*anq2Q0YF>C_9Wm*? zhl&X@dTs_9K5z2nFL~mN4vTm={8!O#)YyXb}AUe1otOUDXmoWh2v zlB82Uz-LNQj))V$S#_e{j!zpFN^Ee29JkX&xB5$+9$!%QD@EYcSo>{EJAg(dl_xAx zkeM}?dyMv_(#Nl=Qm;!rDRE1ha7n}DqS_voPT0Y68g!~&73*Oy*KEp_pW08++ST`S zvzU)zx^O$Q(;gGIC)Q!gchFQm7CzHDm~RZzq6^&Fp(VR`|27Nx(?s~E>o}6R$4fP+ zr^ceIAcR&H$hlofP4gs)o5K6{H-A?DE8h$>WK6eHxe+{5CWKB?5_oE=O;hO#PFcB4 zOXJ?*YOaKu?tK2SetmyrjHH9U3F>I7=QS$!b7_20tqt7obppDrMo)K1RCh_5X>Fgo zIg6Ta^U)k%+7>4EnejYbzf9ir0O z9?`@E(iL*9#(MknLA$Bw==Xw2Edr*WKJp-smLjvePiD+76R2Zv{>)Ly40eGAH)7NN zEX71*-YakA56rzy4ab##?v2{4QPF6-8*yQ$wD&z7m*^@ymW4x3l+O=@ST%pS<}qp4 zTTHCrEOup^7-t$Zf*#t_xD~)NL4@Y0@yVxvgwo}KpL^+gP<+5PVw{|%c(0UAkgo0?g{pA=XgYTosp9~+ z=$P{&IKYH=ZsfajWb4DWYoIu3Zt=%@V3N8dtJfg*3-xd(wA6`$3IJXht&~kiK5!UZ6XPGI#_Zl+a*`Ibmxk)5o__C$rHp(! z6Ld7ABr*T=^>B&l(yCm=?pQDQN-st}j9yI5?OYu@&aDPzQGe~*N(-sr=|rpv)EDkP zHZn{vNxB&n@4nTDDsuiD^t=vyC&(Q5>mRZ#l^Ti8*H0#|+^$GPW828?CqaE65dPRc%XLEPLqx?J9*DC}I$amk-bQ4T}tPpVQ24jMk0-vA~uvbnX& zI-rzICW*TNZpI$@sQEPaC!XxPVT(Mn+kj^F%|dEE(~XnI(G&bqY?nN4Xf zt*T8%ZFCNL!Nd*+|4$vU6jAmci*ke79{~58rY}G6Ce?U)o$u5lGtsJV@CjUI7Detl z*y)PZinp_DV{-HG&(ae3mmTx8gAL;b9;^Bazj&XC(kc2!nnA%+?5@l^vb(G|hM6na;0^1W*svf>+?Lg?^xz`#4E)o?uva-QmTs5c8P-?)MT&4Bjp z56$gHk=?TaiM^6oBToc$&|%qev&`$TIJ`uda5;TOvCz6fK5v0QyT(>}1(PhtaZH0pH2 zo`)^`=YL-v8L+bm;{7yEFQ3JVh4HH*n)zL{X+v}#PU=4DmAXx-eOT8=w(X*49bkR{ z&b-m`d1^Pf_noa58uQZ`71RyIXwGt@^0W!`?g+?y{u00zO@^{d7Spa`my(*eCW&=m zM@jnAMgq3gcnaR&eBMMDz|k4d1IiL6V?VZI6RJh-gt!(2&+r%_|6yua8WuP3UBv{Z zV>7B_m_Zy)NJBzIkMz{@FVW`xVlLwTPLC??BouDV>_g3?h0G5knp0`sA@xGJ(EB%S z{qrD}+(x8aFB08a_J_b@Tf1Vkz1Vs(zLPX*G>_ZAe1QNEEW16%U2{e#xlml<)t zYl&%(WkJi!EoFD4Aa!K+Og{H_qdb~A0ZXQNt!YK#&hH1Z9|v%0-}&R}bRKXxWKV2- zq2`DVg%={swJ$oJd3rv925HQT3ZlmC&<@YB<7bP}JV-}kkWd?J9nkP9>6qb^6kuDH z`WxZD#h}n#FQlf>GJkVw9M{aKvWpkSyik2iFwI2`?0LySl7P>SyVk0?Do;iTtc?I+~5uCbyZQ9GAoQT9mA`MNXNOtWoZI-yUl<=PA;ePhEKt3%4MfK{5QMn7^ z05>vp=%R2WpOdvgL8qK9`;_FGVFU@2%Vg5hfM25c$&>PwnQtlYzqLd7@Kn^Z+oG}Q zb88udg@e^#qPekc9xRuke%iuo|M0v*MhX4^l412TMGKS3P*%lhT31xBYQZeO{g5%3)gkyyU8_!y9axW&G}HUc zq#VG!#j9MEmo4-&=C2TBdbL6jzQ%BZ=#k&g3~E)n^lZ2~$JhqKi1%=a{5GY_0!b4v z>DP;~%yta58DI#_~eGzwlMHDjQQE4;cc=B=#1zb8eVq zBj+}nt;B`CFo$##VWK1XyYOAw+c%lzMfyf24#2H(f%Z*F9S=l#b^C<-B^3ytuLtI7 z`YvInjlC~mXNKG_FR>%?qV+Pzu?acecRiSAaai8z$}&3k?{C#s%vy8O@}^b>a^$Kh zB19$#z<3c#v88OG8QTJv%Q?18xHFzeR+ZBa(6XXdd=I~^qcN&{YQa_S`Lm`$wQ;m#6>)n?p)L{RtWCtP{`To0&T}dtYF* z?1MhQ$w^rK2gRm>YGl|%;i@m^zO3$M#I3L3J%%lKjhqOmna#=amQmVN&?QQ$OMUgl z8N~Y2d0`@|d+M7I*2LW?Kf9^lGIl4y&Gh4);*4zv2}eM#CB03yvx`H*u}{fU&9)@q6i z@_G0MOZ=zIG7xgy9LKAKE-P!a5{)*K%zLkvE)9I9@(yu)Tt4qGBx_M%X1}&wvV5*; zpe!$>o$fjY{5trID7X@^!%vCd{M?k#w=QlEO=-w$zfO5wd9M9V0efwiY3n;j3lPlU zfM3Z4oIF^h;)q|L3a-jE54#2E4z~EYRisSqEO(|ve-w_zd+wF2{1_z$u!=?wh=>LF z737xp&Z!v(~PRgz;blP zJJSC7m}B8Ly5QH z?#=g~XZ|%w@Ss*3W?oBxSS2oaB((Zv(%6jccktuSqiBSWcAhEd58_i;?=E}i z3f67~fjg^nQkW%{)4K9kJGEolC@7`t?jt8bpCZ$CdNLh}3Y70?Su;F0YER|BAuodOBh0(D=M-VLU&#aNL>P#m~Y^U#1bM3%9K4q(Ihh5S&s<4yCksT-}2 zB_CEz_s0pjr-t^ow%PG%$Y|i-jOa9CUfPG)ZF!U%x)EF*i|Jx&L85}(^66DfS4iw- z^*egSjvW%)1hF_YAGl{MsC$MYf0-Rktp~Z&;XQfEbfk~8-`kH#Z#axednH96!(ojX zp0`$Pb@=Kp3}@5f{Z)yKba)B55`co=obIY(4H_dqOjrn2m$Gep7w0P97+jWWp%yR8 zHyZ7hn#x$^mXsLwE4-|%SoWpDNBv5j^;O&y)g_+kgZZNjZpxQ+AsS$hBu&*fif1`N zldre{?%amU>?&Pw>gC7<&{n(_(li6q*r#i02{R3zmu4@$$M;Ar+(ilYElu@GK80!|iOiI(N zeEagWuD>~%ef{e&i;%-udMpXoUU}1N#pG7M*g3%wEU!I}FI1v(a!FjxP`eA))!BD- z%y~4H%cf$kE5~Mms#G{7ZpmmQA2nbvN%fuFI~B+_4Lbf97kUztB}V7P#A=fD;YQ$p{U}Js!`3z@AGubeRqRPOb|C!#w6t*pGtGC)KlFEj=D{!J^$(C12ga-SWehj_GcQD2 z9NbmXa9wWvS5Lgi=huEDqIBoK{i;U5fx0N4dHSy2x<%YeoP@*iyHDv@Z|T^53FP`w zaQTsR_T(PA9qRi7x^E80<^7#}Kk%wZoVK>k7-puO^;;ID_p$u3-0BY#`2zc*rQv!` zOu_Mve>{tIk5xDSx(p*=8qrYk|2KzT7IyAbcSio(&Hwlw+vFtSKhkG+rsC1^2Ez`P zZtB0@Mj7*OIEmH=SXY>JFs!?wa!t@z)fvy(PX3e|hCklXKRt#@bu_B<36Y1MZRh+U zQR9a_d63kCLi-;ds%sU_>G`X2S&!9~OrKXqs*&R@!hH{`EhS@Ptp5^;>KKsAY2#f` zD+iG|Dgwz^HGR$g@qTtx{Bd2fa@x}U3o0E4bAhVUWnD4-GXiupm|?UV-h)5+Uyto1 zOcBE1>oN%bzXbjd<3SN??$KOz(Og@g|AwuHtA53(++Tk?u`hD0{bv+BMqaHVRW^wS zUpN@#510${?mX?-QTK=O9QkZ;j!=5R^w?^mBYhZ_Qu+)lKC4}+nud%ry9A~ z6S5gzzP@~WssgD3U#}uy@lrKCNsrKBiB7(;>*X)vvk13R{3j|vFjg}Cegt9@>~W`~ z#}7R*k(e|lk$7m#ssekjwBGPhfuLprER=!Y8_jz^*jyLbKd}}a8vWg_-SHj0OADW^ zjiz~(*_(5GZ~TL58t>;F!dtPJCRUl1?QWI>)h?cIo@A?!qdtxNur-@zg>XS-T!;Me z{sajSUm=&yH5EStfYv5kXq%0Ef4~Fe%Shc42Ve8e5yhke4l#nc;VPf) zeyq*lZB(cqy9+4^vUX^KILKn5@a^R=gu69^-_4!J;_#N!O0IA%4Lr>Sm7~kmaAZ7* zSr8Byb2j{)MVNYn=t0PLQ8ePn*!$AvOI#t)&1J5ZuD7|^pmpn~&BQFqh_x-`_i(1L z2QS7&gQc(iNtw;lH5+0??ScTm$o1jt!F>>a!yTuPf}=CB*v_zUNd}+wyxYPcnH(m3 z9#P9RQRpzGznd%<+<@!9d02MYXZp9Cvkw@=s1T= zjvt_J7N@3O(|02i`cynjYEF!CnVhup>ELOax?c)*90Z8%Uwz*`4Q>yyP_a|ak&U(1 zHM%aZZh1w*=;;M|4jx# zYb-~78_W-k_y3G2rFkl-Yq9!oO!Q}nqMR5Q4t35l{C z{?5SzfKs(NbxIK%M~*rApc5qlp)5CpsuRDl5*~LhWp9eG`I0_018@0h&4WZZT94|R z04?rwP{B1?_Hu);Xbn@sey(q!xC^V-=l*AM5&rqOH&djc@s2*RFM^ znKg(Kdaou#1T6O;bj0FC^>J<j~kdj+xtUU{d@#NFtxZc%UGHAo za?70AaYzK$l&|Ii;3?W~(dewip|lLgX=WYbeVylbWpVOs$F#q@w9p(pcRw}K2N7+G z(0$k`+IbB@=+jsTstqoTn5Ox2zo@@5{_gK?UK-7GF6ff&pK|&Ohsv}`JF{uDZ zCuvv1{E9c!Si7pN^HIC5mvU^|WI>5=8KSUR5$V5H79|BNk{2r?!m==bldV}jUN3K_ zu^!P%8%6y&gV*`30jMoF=-_@kginp{cRr*nabvW#S$TSAn>eCvH%(TAq07mma2OEC zr8|_J#x2x4z3fp?fjvIG+jmcqsZ4QPBxw~h@I6I8H*icVUMW!Znr1@7CR@B!h?ux6 z=FD-aHGj@p^lYWz3=b7z`{v8kn=iayu|vc$3$s{(3wR_|Ev6ay4}h85Ly6fTdOmMO zxoRdg7n|@!+c)B^i3_6*K1SwEG`%|pUepvy`pKhvNgB40VU*-AxDBMghTNCtO3_3u zfilRBHl5Y0z)_8avusN4&qBV2idGdnNJDH%zupZeTTJTU86BdbR3|GoCx+hF9SPI7 zcPxnlQSq#OdwD#Hh8+w0(!pEI?wfkJrIM(*PGti_K6+~$tdk7BO@xWHyD^2$Xwj!6B04~A zH2|Hmihtcl;0QfS;?gt*xu@K2|c~y6Z!$T|EfQ>dnur}KjT zHWAQuEF%PJ@Oy_J&1TrCurF($L{ID_S*m*PE97yyCxD;uTnn4Di_H@Ck%as{Zg@b2 zc*akk`(ijznb?J`kuzS`#gyo2&UkyThEs+e(%eu?`7n=cEd>rCk!XDy z3AIK)@OF%9UG=iScZ_Nzl(Sq#atP0Bv@<243LR)#gK`U+KZC3|xOqn;97cs{QP2!*D7e{gZXhwusK+N9LVzQ5RjuO1YWD*8b(OMlt@V-&jFw; zoaP!1=@ySz3baT4w7*!x*le5K4SyUt_uutpwM z$3b^LK#KA<8*_nNZi7p(wsWUAsUoIm#TU2ix}t~)Vj`~!Z#Ju#v>jOz^=K#HhK?u^ zYM-F4vu`f;Z_iZcNwqc2@cRyBZbqf>bxo|Fk-Tgm!edL0Qe?y#k)cq}qEn@Av|m`! zsB)#7F_z>IuW2Ol(qn>%D%E;$SGG`by`D}W$0jDn@_Yj_j8cqk)-Pk;U=0mc`Lv0l z71+7|NYgAD@84nc1)m%%&Pz(_f_)UzK&yhMi;PxJ5r9moapr7Z!YKzLJI+o28bdZ? zz_X2iN9)%R;5B8r<*THf4CK{{MK0*q0#j|ZncrvS71SU57S9n`@P)t3o2&G(#A{j92wLT_E z^GgJ{+{booob!xfkQ~~Hn*!sVu$=rBo6E6tu3qG$(6poNu)p>%_jpV38TzCT+YJTD zU~hT=UrMFTQ26#UpIYi=@ZexiVWLgMgar+4(v%b^Ww}hh0>5Jh zDW%R--sXs__D9S7`jd{>EMBUgy5*s%>Py)0i-jiCFe zTmJ9~;J%OCJTEXyypkx$@`z5|G(Wha=9a6#M{kU^70xK6;0%cpx?eT zVWNcO7r!8BkM5BvOj4anPi7Pi$r}Y7Z+2sRB-gaQ-8&t28Qr{MKV}&eZ+RI_aQCUZ zTj(MAJTHA$n%B>^jP4eUd0;u`HBb@3`SH(pr)Yxb?%lZn1b3wPS|f$hkUv;V{~i z(?2F?7W*9hZmu=VbzpO`1Fv!f^JMAKk3VwXW9WqCoR2bmi44KVppM~?_$Cn+G&DNK z*4|F6vAM2~0yHGxZg-f<|4Gaf6v*vj{yYJJ*qWV%mS2;+p(R1SjUv)ZRi`yLQ?hs& z{3^G0`1JivDs_!_>V_{@`@|}BD9PYgzBmS>Y2M9Z=#*=7~{l~5k7;3#2 z4q?W?7v-oYXlJ@uaT74cs|!ZTq`wOSaQ6&#=VTE~pUfkhR90liCMI&`dg~V>$N8LZ z#*`=t_Uf~qEAM1$RqPtvS`M;=(HDX!O;ZNuc((cUBHk95qvp!a*LX-SbTz-n*?Or) ziKUh_k+1GsBYy3%><)`e_4OEI5B$DXLdP)Oe9oPFHE|u550Fqg*GCjGBYkb;Ot5Y5Y;Bg$~Hv51hlM&)?MN0UdMU*YQy|kjH~X4;PFKP_>&Xs%K%nm> zMI~x0-JDUXv8gD4@=!}E<+@jfd)VPAl;#ot6`a~bTiV&2;%cXR&cFt{Q zkxR4J`_;jJEENBU6Yc)I$F69kL79`Ei~egIs3theZsxx~v~6TOyj^o*Su42MW{p&z zTX#>GZNr|vRky;02fuNgHxeam5`2`!BJh=99r_==%=6hQOKx>63b;!s!4L0RTUf_ftc)iyRG zlN%l7b}=K{>6T#{KHC2@Gv0pZ1fiP};PHG*k&ZV_1>}h!u^_qwTUaEha@oc(`RtOD zW|d1kBy*oY5?Ylm!fzbfmK#o!C))Vp)gKz^bG;Op{zwh|Z}Z_HSsMm$N`gJW z3gA-^rB~j}_~v(NGD246fb#~$$|TYDMw?E=pIy>6rM74Z+S z(0@udGhR!w?jv>aNNctT$aoE{59K{0cYZ=#IcVljgdjq9V}>#fqymsJ3={-ipN^*z zKGlW!fj`WnO*1yNmRlb{9K2!T?mQ39HV-C^O@l@bFn=nsqFiBazPl!SOJo4o%4sDgATY7H080@mg@vQz z7AG)s{Lq36AKTDBA$sRb0UMQst{#%>YQp6hy`IqcUT}?t&17=5>TZXBC&JdDY*z2- zQVSN_?P6rM9j<^-#iceM7Mn;o!J9TxUk`pfq&quKyy01^q&1{W^{wDawg6S$Ag!Hu z6}_6{R8JJ)i0DHlld1832nV)6EGxKP;rtNl-e5amEW02dIWYt;UBy{k-ys{29{j+G zCNzw=8QdiU@&wDgWRJdcTBJo7lmu=hu|@Iq=G(-k>n-DMjL^^mu3dhdz22Em2zr1$ zJZO^WYv`nkUskf>lWy6qwQuBQMt_hvKQ6>hzA#InV(5dH7bWyGd6P0VI*V1|3!OM# zvMxu>P(;mf)h|Cz$_0FkLmcsMW=Te5y>+YH#qRd1%y90kN2|);)!)4lz0sdDlClH7 z?oZ{el{XK$0I{bxpF#n#hH0%$YT{l#C?tlnU0%o-0|A!<<>aUcqX2*Ml<(T--cr$9 zc2WS82ij{O_UDn{Xn%uCgj@|mU{s`1fAZI|Q+I$Ycw3z2TfK>^<=Hqf{8XY{{oM1D zHPFZeXbWk-K17SDdbc8Q>hl2L{ItBp$b0f4ti-M5Y|#Og@W{3#jqQ+;wFoK#?8S9_ zWM{#j<3c`k>I}~_C0(nS36uXe!k>FqW$H6JB}-FpHlU3{SWGSOjDT?(xX%NWB7omFGU|oA zabVpE*G|W@KkdbCjH=HsB($;tDJo7|HEerQoOZw2)y>j+d`@95zM-V{8U7*+kIL zY-HdNAAhd~YJ9KozAEm%{W;8e{2SM(iKaAc%zdQ&jnkJz;>YmEBumsmN?}+IfjWv1|xcpKO@HXAk_N8Q~Xx? z3L4f8dNwoz!X;iK?BRF}AwTFCv|J|Gu|>aF4-|YuNgN3?0KpyX7{93(Ls*y^)eUXk z_Ag-`NJZ+SzW1;V`X_8-!LElV3Mgav_K?W^lfzxj86OwPj}e2O5O&d&bxJ$wvM|CYv7Z|I))-# z6mM2k9;@;96=aJqh6WN67AVZgc)hTw9;D(qtpd$#NG!O7y$3CXQPZbfUm2K_`K#t% zD`0JLb03r>vMDlLz-p@FfzJ&R>aQ091GJh(qo29p{gG)^e5x{r)G(np;32Ly1uzNP z`6#awVEVrq#-@!)^cI^SS_liEjur`UQ^)i-hZc~PZb_Yv5GA?<5z(;J_?k2Rh z5#hOf{$BD#sF9H}NGlZRO%Ma?HLrI>+V=)haM;IkBxG*HLPMUDtJ_MFQb zlalCLPx^yMGnZ%jLJVCn2)t9mS|W+-C*s4!;VKrMY@~MhBWjbhE2~s1% zMMG`Qxs95(T9M|gHMd0;-v7lDFntPLS9C%kq=L=9Bv5;Lo)=3A?TNw5D)o{t|5?MD zHqBVE?2Jp=?9)$y3dt|`gUkcvF81>M{a+dycy${QB0k+MpJP~{y1Xsw7OR+ zQNw!^r0X#K?ufopRGVS3Qdr=pOZdBB|Y^-pB zB3((D$KETLvFF(40wPJz;Je*Xg}`PtPhP8#nd%5Dxe@%kJW-r=3{TsG0dD4tpuV$- zblrBhE$dvxtzH))P&r+nrJ z_A?}W(R#0l6W%N0TOwva&9W@hDtx?*KghxXxF)P$`1w}hZ(?eLA;6QmblP6-KDB*cg21@UBbaPlcHjtDq@1mW(I^kl?o+|F`y(5QZcr=#0 z2B;7HME+Lb)NT%<0EmdDDV-zu=}^|D>~U(sPCS(J0HB7p4{GCag`v*(VXekdB?a!( z-Mw~)H&L1KnnMO_IMWEe22GXkZh7;r$M&Eb6u>S>VsBM0$n*lbV$-aIJv;6yVwoR^ z0n81$gLdcvSdfk4lnui*EjC-C1Am1AV6zVf1QU3T2;3IeQ;pDei-}kzi5C%VZS^-@ zRZfkW`t{Th#T`e|Ma*)1_h0Lpp3@~(o3c+mH3fk08V}>H^WN%{P9UQN)DZ7CaYfzW zf3JnH&T2exJ95i;P5>thq3_88mHFvqI1xb%A0gH@Vgx-BbXUn~gvwasc`oe0@0R$z z)_lsJ9)B@Vi5ezQCu?*`%d08lVO`Zp6F==;xF21G#teipx)}i3fx*C>yK7L*DXPS( zxWo=n;ae)!Zr>KwVKs+;O(N>o0b<&H-UaX>tz5vt5M{$a?%^?4R(uxrdf`DAR``cRAB$OyMl6iqwJK}ma9UY1-;Lw0oMaSV6V-e~sa7B_>Sk$cDkO2KkNU>dF518>z}65o1p zZ%JHIm^cH@1khOr|Pl^l_w07_nJ&mLZ{gNKSP4iV4Aus>B+^XTLa>f&r;$uKX& z@MC?O>fiIokS0%x3o`P97Nv)D_Wr5Z?5I)F^*8W{1>w65QW>kB*8C=zd%$z_kf`2g z0bou4{X5zA(C#n_xH9<&>VGsj1JH!0KQw#VUhu{AO48cd-gh#;+Wm0J#Jmbdx`?hP zipxDYj!UnWI5TkvRCuZh=+J7~Lg+6%)>Eu(>BD(@phE|_I-i>VvZd4?<3tbg0`Xq> z`e=ZDW;4LYbA(PC#lzu02p8){ev9KHkfcC@g{R_STa2Q9i41Wot$R!I`)>mwuPF5?E! z7`JnV9&a;hoeog)2B+L{{%LP)q1dGvoBmEQ{t=4EiFX%-Wvj7_U*|TQVPKH$aZq4{ zi24C4(9bQQ!+qY9CAvb>ae~)Ca3oRLVz*bwm)K#aDr9NHXDbSIR7ZW-UPT1njTbgl zeDrTT`P7c>8-$8uK+m9h{S>(V2cYiX!KigShWD5tat0Ge{7U1_o;#WBXuXklQ+Bxs zL(`NbfYWe4DB;eI`Kd|5yeJS;f3J3~>ocGRksI?W3GVg}mI|6-G%9;+K@*7neE!tC zRall-(`3$x*XF%6=_cHFRTrz?!kbsW&ExrbC*ax5VI5=ZPaOzJN_Ken^cxgb!MC}T zUND=qhP0zx!Gl>oqEKh903TM=qi!n}ZnlAIBH*A$0p)&xrWxd+-kO~Pw|{??d=HB{ z%4*2unISMs3&H}45KE-&7b;Y=!x4*tIH@qAYN9#(hh@-Pwa}%)Aq~>TIKO{YEqg(d zf@ZpcD!(7-6OM(I9qqiM67B(nArJavEVvqRzYcE*ql+aBB7NC1-Vw2Y{rHKI$(9u4 zrWENpoZz+Y)pILK+@`AQ%|=R~iSqR!mgh<@_0NjC?xV)K2l{$Ji-gbmSmNg_h+9kayMV{CMb5zCg&f=y+`O@>)Kmr zO8eW>F}+#3+{5<^YYLo5E=n=9H?<|3Oj#2^*maEY{U~wMrcXSzU>cZ@qc96HAe%y$a!2w1BQ5XL zN4Xk>lkGZKH^l)z;Wj@RQ@GXmv^KL zfkPZNg||IDEbpqS=WAY>XS|9&KYn%$M=|!s)|0{1phml4(${o{`G5iB{H9FT7k#|w zTdOCYE!M@>E=Ze=k)Gp>UPH0jvu*y;(}j9bz5UpfLk}u?<2JfT0CX_pJwo77<8>9i zL!IuPdp3Fz2$;Pc%L3T3>I)cFjd_w~BMRs9gO z<#k5f0h^7)q}QE`H$m);vkvCN$`@5P`+bTBH`jcwv)={YTm-tZsF1 zT!6?K|8>u^;<%^h@ZztJsQPvnMX6njlND6^u2Xt*41}I`YN8TL&GS2FJa^V3o6&CQ z(au|Xt+`#&LFuhnJ~IMT5{%1_TEy7*F~UPB$B*h;l<-5ulA>p_wH^q~ePVYAp# zAhEdldJXpNfd?2R_4vB30&|`rU~ZYXnTQxJ&8h*)pb`p-G3_M1)7F~P8MGpj?2xqn zS=4e$O-K+FEfR>br#~-$^UUy1=5Lb9bj9l{wSnzfm~|p~xHM((!G)SQ1hFMID8$>AtV_VoA$+-d)=IrZ7IX%xX#!w0;@_78JflWdv+65f6JUlquP z{%n;z4Jj?(L4V0gwl`cxU|9E|}JgMxW)F zbGc~{}prg-r_J5??bIz)68etBWqIF0FT(stYKwS6>}pOTI?6~o;R z;j*ljC@Oq%<2zRhe+EkI=f<|X522%dB}|1XBoG&IGv{@-{M5M*=b((|MSkMUl%u!v z*PoF~xEec<_*zp0MelTPWS{V=(Z(*seTT50e7nce5u@K8yE*+yLiKy}z+e33Bp2Y_ zzF{d~W!m=s^6P}BI6=kr-NqBTxsU~KL-r6~Dd_iBF8$f964;f-4w{iRSi~*oGfT!z z>xD~7!-elPP4Q`bXfx;<_m^Y%`*V{PB|_V+0FRX(%Xuo^y2-dlu|eVkF@SdISYJ-W zjBwPTVM)lHSh7F#bcLN58Wc%MzHBy7AJGY5@fNI+80yWZrD7K8zrLyvC^3(j?i`AC zqacilzAChnLk=Roxvu4`EwNtAsXiEEb0DI$s@G#J$|0{NgML7 z=2t_;7Q%Q48jme`IIJY!ReHpAU?&`TQva;Q^ZKLMJ*atP275${%8_d^Q=sN(f@&oP zXV**{pT-Hsk#nV7nDUtFn%b`7#R6B*nRjIw@1>RiTlddNJHvj^Q)J3a@y40G=3JzH zNPiF!rh0dsx9&q8@;@=H*Nl`EeYIfE{2Kt{TyGTY4|^HGrwH21fThl9r@a=TL}d&F zB2$nT9hdpkfBda$d$Y1;zI3thEZEj+Dr4zZM|~```K;4X)g*n}Fi)b;7vK^j^05Gf zi{HvyLEMGWZf6^(($ z)6>oLj{EtAeIp0|V7%R6jNJF9oH!gQSEPvE{5!3jc=VleBDPExhKCeA`kJss`Cwr3 z9L1n7sZZ-EcSq-D5|2rW$LY(?2xpMk6W=EZCB_Qo9OG|GgP9A)iQ09`^kKp->9pcc z#=xkbiQeIV#tV{pq$x00HF!WNWMBg1<2%tG=^bSDW0qoApyf<+sT+2)7?!!iv6Tr{ zKTnHN+&ZWc!+*akK=z-z>Xd2Nu}RMoWVF*Aoz;;Vhwf!BQ)2ocS=}B4hZ6sXcuFW<@*0Qi64z*q-wBwy*YAyPkM5r0F}1n%nGv zXIXWiM)RG){iEV+8+Myt(V=(-QtSckEF7ebR#h^>XolClS!ah~UmZ#AE+6KkX3} ze1T7*)ZM@IVv$V{m~zacKYaB+gVKCNHj~g+-oO(H-`uVIRJ)u!oM$DQ zYFW6@UGRp&YSDKZqk2qY1zGY<;FzS$1g}>&P*sMTm7S@QO1S4;u|%~OI_`fvcGlm& zOfH+|cmv2;!}^qUbm)zZjq5Cz=&#TIuOKf)wdyf>cpaxq3XQhVP`Ip~2ty zA_R1d#AG0a6!C*a@#pdT0C$GuH!h9>0T6@=?LH+D1ZH?gK0v+y_W`x}BW&#S`G?`$ z8(10d=%}cXKYwZv5Rk$TH|*^%(a_OJW@Z%7aBy&>28=aAyYYqe%5)Hb46J0Nq}Uyy zp`j>VsX#&qG|j^%1wm<8z{$x;Lh_<&7AK$0aAk9W2-d;Al%_>^|IiT3jr({c>@U9$ zH`^;e`OQ^2N~)@oOG+gCS-w{ti)v|Qf(d@sC1Sr);y_sBfX!q7ySqCnPtUgb#f35d zKp0~uCz-|M>ZI*$z{vM@bxArE14BdQ-27fB995P~RJ$WldHL#wQl0ry9XxKQZ+R-O z7j3p}iOI=QVq(EKmG+5PkMTet{0sYM2{>(%`g+r;Q6aLr$?Gqq?8K5aciSGi)ua>h ziM_phiZxE9y9axc4i1Wf*<&)Hf|MkiU(>R<998*woFsTWv(Pa4hEg{U%yMibeX_0A zT2-!;DfFT#N`o9M!|j)fSAEQyW+J5yj{RH3OxS)pUmf{R85!{c7p;F&$0^J>Z2!`4 z;kT0&H{+`F((tdC)iHN*X&leOwUwb$-o{~rutgGr((>5)nzk@6b3Cv)9hz97NfO6x zx~T4|rI>Iqh8`neB4%;DKf&T?L9C9Cb@T}mFL!M(&+cTn;%OT*EhU~FM}sTA%c*5t z!8G>#I%{@qfqpH^nwg267RS7Fg=HcoiAGJzWFL4ZbK!skH*~zl&D>^pSvM1LGj269 zGLevb5~!%FL>9PICG{A9L8V?N5(unN@E%4GZ>}h1ftyG#YV8PU_4v^gJx)q3cJ95Y zprO;Q7$2`lDt0iodHZW|mQ)w-S$!pAYV_}64ejV;;sh>-lu(bN&=dyx4tA-1gNx@` zceyY;M2gODZsv&4=p<|8^VmFGkC|eVze>BcaW&riGD5ulB~AqPdbC z$8#4d&Szt%e}DU}4Ko)t&g^()q@iD1U}{92HgJ4MBgJ=5F@IgFCJruU7G|jNttXg& z@XZ~D!jU!7YBW^^tn0X@xPh##i*VD7g&4xa7a)uj8N`CccaM{#XT9zc@dR8X$i+e; zsXA-ymZP}#cWVY16KUjt@Oz7CLdzc2+6hRvE8Inw<9BDCy1_%_Mkg;kPCh@h>gpUx z?bp3xT3nqQ$1d}8qz%vP&AI2yv>pi8YK0doH0X|wjs{o9cA}o+U_p$Ka^fbn7GbIB zE1HudJ@WTsK~?OMQmN%de+P;JQ|c*0PS}efF#$uQ4JhWJf?qzKnDZ;y4vvmemsZ=x zZq3+hWa<);m5&+qC7*H2$6j2<$(qWIydtkBwH{NqIBm6)mKaKyOV(D6%sjDM#b##W z#oS+t15id`Iq}m(!4uEtohqBed4lOyT6m}^79>fD>BSnYmT5(Ug0z~rFiTLl$a8}v zgrQooO(!$syf1uzoJ<_=9xo6a5n@jt5|LP_QY97VcuIy#sk(a^8EXG*e_7Q%d=`qT z8VFMcX3&&UjSLDI4MmGP$yIjmc)u^18M32esi5*KcwqbG-q=XS=;ULJL@9E_z@O<@ z`sVYv;1%?%lzSfrk|h|U>wVcelUr(yqSn~59@tbM%)57K5EYc3otq<}qpNgyxH86> zrG!10WJ?R054%>{tA2g_uGsDnHFUsMQhHgb`y%q_K|ez7{CPypc8D{}Qd((hXrT2E zH&xil953V$q3~A-!`ibx%oZ&AwR_v41n1VY>}3J2h#K=MDTPvuo8OyA2co^5Pb@22 z=>SmN3Qp2|(4aPNzHPwZ$9Fys=KSU206@B&^||h`wRdo^Y|m&M4s3}Ui|-F z_WuEu3>D(z;KaT9L*Nj6jlaCKFDDp7#Oo*xX zQ6Fw~SP4IGlVdSB*h&}`91B}jkd9|bd&9=vIb#Nb_JP~weSG#G|zQU@BRd?z;?=GNi<5{M4C7qOmLydX%%W!+Y$jH=GLU{P+k) zsd`~UW7F95bm(|S4r!s~;;5lKAH`UN6ff4>c_bmtVWOeZsUxLYnUoi=Qb)wsw==h@ znZA4d@d>(UD0U@R*eb-H@rd4r53z0CLBNJF8Nc3iWdWA1_b3kn%3 zDk?J*6LEfizQfC-pOdTe@X_E`b3RwiR-RQ-77}d&pTGK4s(X8QE2mAkIgC88&HK~G zC%8r%PZ!c^_o5@iqjA5VA*r-#6`7gq$>@EmM8w70oDICZWGXQ_96HOjo%2+wyEuv9 z?3u0(qs5K(Mw8;>;{o>e_Vr47`^R4(Nm;w};9oKXzX>0s%q}dD-Eu4PEX@P5%KF13 z%@sR+5_b3LR05AjsmXv|IvaA%vz2&WN00SVUW0OqY-75_-j5abg9rI$7?WGMvMn|o|he%zz zJW^3I5El<&ZE758ah!0tl!}RrmX%N(g``r~DIoug+Vi#H8}T1)PXWqH^Xbm%&fHz> zRq5Z<>lEja7!cW#jxmykxmY4VejpPC!-;~Drs(-ei|U4sllJ2?kP)o}Th#-!bV{m% zhNfm2xU87kwQo;>Rrl}i-u$7}>ZGxs34WlWz~C_Rs^h@Z!4 zY9i{$tQd+vfzhnkDrRAa@nH8RKGDVy4Et63%lalB^>TwwZk*EG&#Ex95*H2kGeB+H`omF3jq# zfE65~JpwYpqn75Xrn4_8Cc^c`DLibB-WSLO&z`W|{oPUU&}|tnfi{cHmWJ3jVWnL{ zPY@HT|98|H+1I}0X*p%Ju_@onU{+SvW2B#9%oHY*mDVF&VSbSaF~ggJIT+x+cZ#lu zf@-Cvv)T_uZ{gPysyA0r&q-A>tAw{2vWBvqp}pIjXJvY=R_YUWn}pNMfqF9Z@g)N& z?p5$Cg774d5K?c~CS zfT@m#?^3EsZMbnTBHclP+RlZi-i9&#VqvH2FX4$PNiA|3FWMR^WS%y|ZC_5)kV`lx+ys^4mUou*l{^rfAT|v?mULKuKFjAmpCe`Ez~KE8 z;P>xG?x?2Ob+WlxjoANt#s80Vl|cr?B_vF_ykvTj36(j7p#lH_BqU_i)QZR?6uxay zZxpOKa1*ZMN!2oD{HWv%yAjB*cZuU2MR@5USTHHM2y@yOR55nX6JTRn5vvz9i><$8 zN_q;jPf|%@qeCHO?j80{94&dUTro(0QW=aRH^#j>(YI*)af6Be-V(8C3QF0TKG9B4f9FF{POb-B(%KR$1^x|4Gec|q)RAn zpqYvo!cUf9_B3s)5fHUG7$Qp!&B7ugSRMw4D26k+9C`O6JjKPtSYs3t%RuSHSt<&At36pVNne;Aqv%QO)h=2QgJ}Od!gSwRifxKOPom zmUK`;iIRJdv{`ce>&Nj|?)ZKFB+FrT?xero%Gb9Pq80}%nzvYlhso6rf3G9Ibq@Wr zjL>O8>~EX7CrfJuj(b<4M{0_=I5_Og@S@Vn%+lXu>96-GaM8k}!q}vEvhw`=fsBR#Qu0710y?SBLe(BRpVgaHhOTZlBdPzQGH^JUQXI< zr=LtpItg#={n=yLO3AF)4?29BTwE~rML%f-;>@blOQ)66d&xuA6bXcei3u*s!orG2_cgLZ7&KR-mDh3a)tNCGF0CAMsdJJ?iv%pH(vF?WN{q8gNHB z-lhplMQ7n#2VB1vU@yogLS#+5U&oV@voGdYRdoE{Yi6~klb!wTq?$gN9eJeTmWTor zc;wTTx#}uPM6OckS-Ab8vC+g-4>7V8aNah-L-UDs5|Y5Fab2&0Mq&oVwXN3up62go z_}1DBWLvnKLN}E(+V7n}2dsm!pmTcnx3!ItG$lk1hR&$``35H{N-h0S|GI)Ay5^|1 zv#$AftC)zL{?Wkr=G^deG^~t7QO!|Hj$?6T;QYEMB}@ZUGEB*m!dR70H%{qiREI^E zykNN`T}n+DN=HKCI9p(!KlB+|oB@*|oFs=Z)?%Wqwra4QR*ycind!^cF>#xiFLc>( zMw0;L0L_ww$vyzTRyMj@6O3v3zdc4|mhb{h1Xv)WyR;!)s)jzIctWaV2 z0~#zl_-1F;0Y53qcDu!W+3TPah-u;J=Kw~^X%mNnkG(Qf+S*k{D#Vt`ul4ng&?b&I zlX^D1UWZB#mtC*AEnY_fzu$(@>wI?Dspz$hx%K825Fd6r37@;(=)O$4jyA%$47VbF zZ!lCN*FLkps#o*<;o$pMx(OG(J)Wt88e1DofU^{h)MPIz3et!$k{$5Kr?6aHCrNvB zl6s1KT|I;&J#EXyWjByd?BYKpM~_KgLf*Ju(eKKTE-gc2(>HW5<$cQYLoY z8_`zzzMm&yl9fY^2I|nF$%He~EiNd7A*WX}F)#BYocLUm-th`6vNBQ@?m#Z|jx8Jf zmMp#nv+)d#2!}!uIP0;y5F`6n#RNjQiEg{qx8TZ?pVY3;B|E zmyDLW!j)1;dwm-YMIJ|@7=onx#zk^Ns_sKeg>!?Jw=Igq3RP(-i*%+v==kYH+us(U z==gm*L!FF|ESR|J%G?e0q0BvX%<|4Iefwu7${e@cPO!NtM9f}{#wctAJjteNIb>9Y?qLx2(OtE{@i<>pQQNL#ci*`kzPY*B( zDkq_DQ8~}*JW7c1MOn?5Qs3^^-$F4kDj7|j#&i5j^!BTkCYNjbVD_dd2up<(GkG6( z>lN*%u-H7mm<=&d{UBjQ`6`nxPoT{tV3eh2_)dS7U;hAMR?c~zZCtVwHJAmR1SHr|g>uR0X0x}v! zIxzTwuRw~+Gl#l#kBK!etABOS(9;8WwTNv~tcQ{sH+$@290kM0t1Y+UY6;YrYN&bg5dO7lM^kZUo$Ugn@`RylhPS8+>hBy}OD++( zEFQUbA`rPR5AwDrY)n`4*9tBu19UG}`SQq||PO*@&*3*j0hKTdO zDf>JUzL~U}627EIH?!Nskx8eV3{y_DKiiV|B_RI1I=}F#beyE)rXFR-=Zq}U%SLV( zK_y{LJ>(;E;|~~)_PI>MTi`mTbyj_7dwcCNI^)aW(?@e_z{7&bQL4yR2YQ&u==om!sEh zlc=>9;uxH6QfPIb%Da-8@$LO-w?6T)>C+>U%Z?j1loXSBBuYCchPLoF|8Z?#OPDT# z-2d|n{8rWJ(x`HydU2!*X9d)07w~(?HN4dgO*UP?w(MYtfGC@T7lCuxLZy0qSo_rY{^;0=gZM4&* zvGl9j^N`5I=Xp+|mVB*bn2`lu8$6t?+30&2=UB@!_^#fRJBIX_mz!^WsI-;nA0D9N z^UnL+kg#HUr4csv)qRDl|1M@TWdXH@H`bfLIedc)__YjEj7KvPzbYO~Z(<>Txi=Oa zaF{CcB(ptYuJ=LMcS^}@HCjmU5$xmwI<9!Qo^!uD+I@;#ed}Kogx)_Aw#B20mkhM! z@3v_+6wq~p;=?Xt6^la3&tDl`Pap1mnMlef?)lArN2w*Lq&jn84=kvvjfOm}vsunC zQe~B#*w--Pp6DwoUP>kyy=)o>xU*C&31P%bsvTwo$OGHL2F~!_{ptz363~%L^k>P? zxoImNl;yNtq%rN0g}xtcs}V*tPOQ=}CAYy}+&JMPa0$jzhhIssX+5X>e!VJX`O7+)} zu@w8fvG+x2iw0pFKdAeKa|L3s?dZuy9pPuN@f#$6o*uh&(KNZz1P4fSL{llVssn8F z?Up%{s{73$!cndyHIc?1yei^aX!wSY!F%LsCJN4~E>%@M<4jphsW@AFp-4ml$aY{b zkoI>8q$JqHvV`sr!idyKZ=d}+j91l8>>$TkgnqKPD$k?QpT`Bh`>I3p^9DHF(U zwQRFbG!4&wV(bLjp?%Q3;fnB3+v>>M|q1d0xJt#HCT zI`&EHGz5qY64mh!PhME;bbFHq>qfU6Yi0WkxqmJwhdtx$(r9i3q8GONN%XcBs@%a6Nx%>?vJm_!hj%x-BQ5t6>NwiS@^3( z(~}ltEO!p=w_FKwfc&qB;vX^YOT5aUA^dA*Py@#eF5jS&WSdV&=*=6dj;QCOmn`rL z*H~*~z8XBv+`lR8-3D0s{$+ARQm~z6W6dmvVh*JDpZ59N@s^L@e(g{_NGY0~FlKt# zcmc;z{6Ah05fzB!RZ)w|RxxSryaD6yWq9v8F-&=t44s-FReX7BL+VBZSj6|J_~YPp zCceQ@d(yamKV6^{Du&P_0nY$Mn;Fakt97BT+P;9evcNJJkq1VN(QCmV#K%Fz$2f{? zriY&PIoHY%(7uJ=>fc6iq*t{rB4HgSlL>>Bi=4#TMU$2!Dpo=_A~fQ_xvpK{jM0nJ! zFo#OKlq+jpcog&B4*A3)_0z)ze-3pqS6+Y_eUkye91>lJhGz4L%ejRkCh90Sr0lod zfjAU2?YLM2B<1$^uE$cQCf?VZ>pD5lWLRiW_Ssfo&SAulCEC`>#ggZdi@K+AiH*Kl%pKbol4N{ zZ3H+vl=D|zgX4qMm|{6W=bx*}CHXRXGIud?%~Tk%5JdPVKC&-y$^=|yzy|-4&X=_} z^F`b3$5E0TTmxXY9u&QpI_*+s9YqUF8u2g$Eq>SYh_4xTvyi(CPcS2w0Sy@~>Yqfd zJ-q@H`NQ8}aNyr0_Rl$x;iuFNUFXACPI}ly@3tb#SKz!2r(rk3$L#=;AudGSpi1NE ze35qX$O;q*>`DC4UoqYze}I$=c6{pj9~nkBF1-6vSx|rCc}np8Jp^+E^~rUyb=P~o zn0)DX0ws)0>AvKKeF>vu9bFiK`reO#>FeEAk$ZcW-1eM>WCQLF$J|8 z9F!|V2f5B3&9>gixNs6+bqIC$F%vJhl0~$dPl2lI^?@KC_qriLt*%bS#SF#J?Yy_u zo~G1)eDg;TiWZ?6imjWB!2^O!(mLQSb~;ls{2eO0PM9V?_%v9cBTLcscE-RO{}+x0 z<^%fZN1kj}N6252U(vVi7X2W7>)yN#KRZ!TkPivj9Tpmp-VUS7zvZbiBc+6IEQ09S=DKKq2!8d}=J?|^?#_jiTJd6oDvi)zmc;n6-HF)y9)|JiX4PU3$zF3EdE&i#SaHp*CNlcTT#kw zvY*7n@zT#Sl`-=ba3d~Sj(6-#|8g?@GO3p&IIaYBFnEr|;B{>)D?T}wne86>Dh_lc z-g;U6&ko8I#Y$@eS@TGh%bQ3VXFtU49uXK}=L+GVqdDWc;dALHs^(`n^+w?+0ISMN zp`bvLmZ{1p7z;nW5kwG}aBxp7qv`+>_BoSPn0Nazp-uK^t}lF z1KtlyatlOp_v`L+R;OW;$TBg~@Si6%St_^!$NjvtdfK;Sg5-#NXs9*J1FZY%m-ZG` zE62VuN(Jui{K=I!`uiB0KB<=GZVbf|OhqUXMn+AfQZXfBT6b`4yki7u;Pv;?)M@TF zns||XI$no#`4M`T@rfv8!$YyC^?$=tA zl4p|LonIT1Ow=l&$9yXE7a9$JuNv`{!{L&f8?CsnD(Cm>%jK&R^Irj(#i&3*3pe}b zUYMS(k37$fhbuY6Wb;Pe>{`kCWmL)ogYX|5EkcmIjC3$kV~2jP8$`i{Ie18i@(0{n zN#BsMEi4bL^104&y=&#}T`P__1nsavkLBFjA8;=1swzx?ISgw4K$%j_XkMth}VSTf1U+V|#MDq?!F6)uOu~O9SV3@h! zL6ogc3EanAY$dkjdOtUm2n?RmKCfu}Rn#IxawpOyW(=pt8e+=vtLb9P*3*8{q!;@$ z*xs8cu48O`h~9=SiEDphsm@QL!Sr0qXOQ2QGb^f@gt`@qC~~}oK07~rn3IK)FypAT zBTPz3kxuJ`bvaux?p&h?aQmjAK~{YZrgoS}m{n0XXEJY2`f6tJ)of^lw6Ypr{dVIh zgCr-F8`K@27+W%f3mZHQwbZMdpKDz&>P^+FVTb9^B3%lw`1Oo?B9~gJPw(P7rovWB zw2P$}5Kk!Pa(HWiguEUV8q2ug-v84Y#$}wH0aiQnj5N3Sd!tJr4yHjH@-7Y-t^mWp z?EU~&USxb`PWCL7D|3F&6_L+9!*J5#V!^>ZCMMdX<4cTveIQsG;dGB{U=c6bk_`r< zLhqwLT^!04qMYh9+%Pfw{J`*M(%?zl#qR(g@cGZPG`%X&0u(_K?6Ea)vQ#wSh#V4Z z65?hM09Li#7=M2V^nDmUHBJL9HOEbCF~zcZb?hh<0Y+Q{kswrDF|9L;;C?f!&U29t zMn^>DvcT5O7EYQ%J%O$}c)~t--UCbKc zwi7EfSAf?xZtI7%t3<6f1wV}1&a1In>7gh*z+V?$Bb^aghn&O$M;DI$B)&baTaHMg z+?-*;85g}pk;oAdd^+Ye5pdH7jE09!+k|(Y;rE<|GOD^k=z%BdeWCp%iUD1;Nurkl z!^)wO<#LHqxBpwFk~aKVR44q|B%AB4Jmj4Y+7AwZ(Ln*X!6d;a z0~of)l^UrU=CDZq_lkFg{GX*Oa3k~EC0}(1y>}qV)i0%~vAR4ohKW(7t?=%n1m~;b zjW;p0orB?X?Y4r7iqk5;-r=_srQNRBD_=EMD?b`A;1)uco5g7l^m(k$kyet#cG_~a>Q3A;GMfimm4~U~=MBs#Zuq2T58>Bq zy14hA=-{B3`%{){LG4ySq`XY%p#VAH4<-~q1V92X8VCovACFKJmXsXQr2P8(ac^T# z6&{#oJM0@b`Zk9(?E4FC1xyOv1OGCv!`?9_L=L!FdxY|hKOu1!L!-_pte}8Y-Zf37 zOX;=JqUs0MBKdm*6}}AgXX~HNH2}p9>`Ea*5^>P};TQ`tf?L_pAD#IVz-?_X%@{KJ z103CEEVJ;xlw;UWU}lNlk4e-2q`OhU(s+s{68{pKdj!Ee3559vng66Y!P0)hqX=RD zu$_sp!F=Taf9@Q~f2ASB!P0gTX~h5b)S*6t>4J>k4p{z^-ULfSB2lsYM~2oT5(L73 zI`$&}PZ|#_-A!y9{NFMVz{}_n%$oX7+7C>XhUY98kNtNU0si1+V3%B{{qLgxPk7UI z;;@x@c|a*CDWikQEJZCXyhx9$|K6Ql3duKJT?Wtl^Q6{RZZ40TlARSX88Ziu+hn@$ zq-<;|$@JRPR8&-QwmZ)MzHLD$JQBOTQA{q+y9%vNFAeZ7lyCm+Tm>{dU3BSo`|j-Q zL=$$y|7&%Au)pac+`#||M$^gvQ+G}35?pl)W0V0iVcCbg8wNqhU6dOi2pK} zN5an2%&hkBhH35ti=Q)Ydg1+R!oGaUKLtLh*l}Rv*~?2BY5np_P1yWbFg^Pf}D)qzb4P@P7d!Kz)G# literal 0 HcmV?d00001 diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/BUILD.gn b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/BUILD.gn new file mode 100644 index 000000000..865bc4d3b --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/BUILD.gn @@ -0,0 +1,33 @@ +generated_doxygen_out_dir = + get_path_info(".", "gen_dir") + "/.." + +loadgen_doxygen_sources = [ + "doxygen.cfg", + "doxygen_footer.html", + "doxygen_header.html", + "doxygen_layout.xml", + "doxygen_stylesheet.css", + "loadgen-integration_diagram.dia", + "mlperf_icon.png", + "mlperf_logo_horizontal_color.svg", + "README.md" +] + +source_set("loadgen_doxygen_sources") { + sources = loadgen_doxygen_sources +} + +source_set("doxygen_html_generator_script") { + sources = [ "doxygen_html_generator.py" ] +} + +action("generate_doxygen_html") { + script = "doxygen_html_generator.py" + args = [ rebase_path(generated_doxygen_out_dir, root_build_dir), + rebase_path("../..") ] + outputs = [ generated_doxygen_out_dir ] + deps = [ ":loadgen_doxygen_sources", + ":doxygen_html_generator_script", + "../..:mlperf_loadgen_sources_no_gen", + "../..:docs" ] +} diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/README.md b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/README.md new file mode 100644 index 000000000..d5cf5fe18 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/README.md @@ -0,0 +1,34 @@ +# Generating the HTML docs {#ReadmeHtmlDocs} + +This document is generated from inline docstrings in the source and +various markdown files checked into the git repository. If you've +checked out the code, you can generate this documentation. + +*Prerequisite:* You must have [doxygen](http://www.doxygen.nl) installed +on your system: + +## With gn / ninja + +If you are using the gn build flow, you may run: + + ninja -C out/Release generate_doxygen_html + +* This will output the documentation to out/Release/gen/loadgen/docs/gen and +avoid poluting the source directory. + +## Manually + +Alternatively, you can manually run: + + python docs/src/doxygen_html_generator.py + +* If is omitted, it will default to ".". +* If is also omitted, it will default to "./docs/gen". + +## Hosting + +A version of this doc is currently hosted online at +https://mlperf.github.io/inference/loadgen/index.html + +To update the hosted version, submit a PR to the +[mlperf.github.io](https://github.com/mlperf/mlperf.github.io) repository. diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/doxygen.cfg b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/doxygen.cfg new file mode 100644 index 000000000..fc05853d1 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/doxygen.cfg @@ -0,0 +1,2495 @@ +# Doxyfile 1.8.13 + +# This file describes the settings to be used by the documentation system +# doxygen (www.doxygen.org) for a project. +# +# All text after a double hash (##) is considered a comment and is placed in +# front of the TAG it is preceding. +# +# All text after a single hash (#) is considered a comment and will be ignored. +# The format is: +# TAG = value [value, ...] +# For lists, items can also be appended using: +# TAG += value [value, ...] +# Values that contain spaces should be placed between quotes (\" \"). + +#--------------------------------------------------------------------------- +# Project related configuration options +#--------------------------------------------------------------------------- + +# This tag specifies the encoding used for all characters in the config file +# that follow. The default is UTF-8 which is also the encoding used for all text +# before the first occurrence of this tag. Doxygen uses libiconv (or the iconv +# built into libc) for the transcoding. See http://www.gnu.org/software/libiconv +# for the list of possible encodings. +# The default value is: UTF-8. + +DOXYFILE_ENCODING = UTF-8 + +# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by +# double-quotes, unless you are using Doxywizard) that should identify the +# project for which the documentation is generated. This name is used in the +# title of most generated pages and in a few other places. +# The default value is: My Project. + +PROJECT_NAME = "LoadGen Guide" + +# The PROJECT_NUMBER tag can be used to enter a project or revision number. This +# could be handy for archiving the generated documentation or if some version +# control system is used. + +PROJECT_NUMBER = + +# Using the PROJECT_BRIEF tag one can provide an optional one line description +# for a project that appears at the top of each page and should give viewer a +# quick idea about the purpose of the project. Keep the description short. + +PROJECT_BRIEF = + +# With the PROJECT_LOGO tag one can specify a logo or an icon that is included +# in the documentation. The maximum height of the logo should not exceed 55 +# pixels and the maximum width should not exceed 200 pixels. Doxygen will copy +# the logo to the output directory. + +PROJECT_LOGO = $(MLPERF_LOADGEN_SRC_PATH)/docs/src/mlperf_logo_horizontal_color.svg + +# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path +# into which the generated documentation will be written. If a relative path is +# entered, it will be relative to the location where doxygen was started. If +# left blank the current directory will be used. + +OUTPUT_DIRECTORY = $(MLPERF_DOXYGEN_OUT_PATH) + +# If the CREATE_SUBDIRS tag is set to YES then doxygen will create 4096 sub- +# directories (in 2 levels) under the output directory of each output format and +# will distribute the generated files over these directories. Enabling this +# option can be useful when feeding doxygen a huge amount of source files, where +# putting all generated files in the same directory would otherwise causes +# performance problems for the file system. +# The default value is: NO. + +CREATE_SUBDIRS = NO + +# If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII +# characters to appear in the names of generated files. If set to NO, non-ASCII +# characters will be escaped, for example _xE3_x81_x84 will be used for Unicode +# U+3044. +# The default value is: NO. + +ALLOW_UNICODE_NAMES = NO + +# The OUTPUT_LANGUAGE tag is used to specify the language in which all +# documentation generated by doxygen is written. Doxygen will use this +# information to generate all constant output in the proper language. +# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Catalan, Chinese, +# Chinese-Traditional, Croatian, Czech, Danish, Dutch, English (United States), +# Esperanto, Farsi (Persian), Finnish, French, German, Greek, Hungarian, +# Indonesian, Italian, Japanese, Japanese-en (Japanese with English messages), +# Korean, Korean-en (Korean with English messages), Latvian, Lithuanian, +# Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, Romanian, Russian, +# Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, Swedish, Turkish, +# Ukrainian and Vietnamese. +# The default value is: English. + +OUTPUT_LANGUAGE = English + +# If the BRIEF_MEMBER_DESC tag is set to YES, doxygen will include brief member +# descriptions after the members that are listed in the file and class +# documentation (similar to Javadoc). Set to NO to disable this. +# The default value is: YES. + +BRIEF_MEMBER_DESC = YES + +# If the REPEAT_BRIEF tag is set to YES, doxygen will prepend the brief +# description of a member or function before the detailed description +# +# Note: If both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the +# brief descriptions will be completely suppressed. +# The default value is: YES. + +REPEAT_BRIEF = YES + +# This tag implements a quasi-intelligent brief description abbreviator that is +# used to form the text in various listings. Each string in this list, if found +# as the leading text of the brief description, will be stripped from the text +# and the result, after processing the whole list, is used as the annotated +# text. Otherwise, the brief description is used as-is. If left blank, the +# following values are used ($name is automatically replaced with the name of +# the entity):The $name class, The $name widget, The $name file, is, provides, +# specifies, contains, represents, a, an and the. + +ABBREVIATE_BRIEF = "The $name class" \ + "The $name widget" \ + "The $name file" \ + is \ + provides \ + specifies \ + contains \ + represents \ + a \ + an \ + the + +# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then +# doxygen will generate a detailed section even if there is only a brief +# description. +# The default value is: NO. + +ALWAYS_DETAILED_SEC = YES + +# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all +# inherited members of a class in the documentation of that class as if those +# members were ordinary class members. Constructors, destructors and assignment +# operators of the base classes will not be shown. +# The default value is: NO. + +INLINE_INHERITED_MEMB = NO + +# If the FULL_PATH_NAMES tag is set to YES, doxygen will prepend the full path +# before files name in the file list and in the header files. If set to NO the +# shortest path that makes the file name unique will be used +# The default value is: YES. + +FULL_PATH_NAMES = YES + +# The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path. +# Stripping is only done if one of the specified strings matches the left-hand +# part of the path. The tag can be used to show relative paths in the file list. +# If left blank the directory from which doxygen is run is used as the path to +# strip. +# +# Note that you can specify absolute paths here, but also relative paths, which +# will be relative from the directory where doxygen is started. +# This tag requires that the tag FULL_PATH_NAMES is set to YES. + +STRIP_FROM_PATH = + +# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the +# path mentioned in the documentation of a class, which tells the reader which +# header file to include in order to use a class. If left blank only the name of +# the header file containing the class definition is used. Otherwise one should +# specify the list of include paths that are normally passed to the compiler +# using the -I flag. + +STRIP_FROM_INC_PATH = + +# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but +# less readable) file names. This can be useful is your file systems doesn't +# support long names like on DOS, Mac, or CD-ROM. +# The default value is: NO. + +SHORT_NAMES = NO + +# If the JAVADOC_AUTOBRIEF tag is set to YES then doxygen will interpret the +# first line (until the first dot) of a Javadoc-style comment as the brief +# description. If set to NO, the Javadoc-style will behave just like regular Qt- +# style comments (thus requiring an explicit @brief command for a brief +# description.) +# The default value is: NO. + +JAVADOC_AUTOBRIEF = NO + +# If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first +# line (until the first dot) of a Qt-style comment as the brief description. If +# set to NO, the Qt-style will behave just like regular Qt-style comments (thus +# requiring an explicit \brief command for a brief description.) +# The default value is: NO. + +QT_AUTOBRIEF = NO + +# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make doxygen treat a +# multi-line C++ special comment block (i.e. a block of //! or /// comments) as +# a brief description. This used to be the default behavior. The new default is +# to treat a multi-line C++ comment block as a detailed description. Set this +# tag to YES if you prefer the old behavior instead. +# +# Note that setting this tag to YES also means that rational rose comments are +# not recognized any more. +# The default value is: NO. + +MULTILINE_CPP_IS_BRIEF = NO + +# If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the +# documentation from any documented member that it re-implements. +# The default value is: YES. + +INHERIT_DOCS = YES + +# If the SEPARATE_MEMBER_PAGES tag is set to YES then doxygen will produce a new +# page for each member. If set to NO, the documentation of a member will be part +# of the file/class/namespace that contains it. +# The default value is: NO. + +SEPARATE_MEMBER_PAGES = NO + +# The TAB_SIZE tag can be used to set the number of spaces in a tab. Doxygen +# uses this value to replace tabs by spaces in code fragments. +# Minimum value: 1, maximum value: 16, default value: 4. + +TAB_SIZE = 4 + +# This tag can be used to specify a number of aliases that act as commands in +# the documentation. An alias has the form: +# name=value +# For example adding +# "sideeffect=@par Side Effects:\n" +# will allow you to put the command \sideeffect (or @sideeffect) in the +# documentation, which will result in a user-defined paragraph with heading +# "Side Effects:". You can put \n's in the value part of an alias to insert +# newlines. + +ALIASES = + +# This tag can be used to specify a number of word-keyword mappings (TCL only). +# A mapping has the form "name=value". For example adding "class=itcl::class" +# will allow you to use the command class in the itcl::class meaning. + +TCL_SUBST = + +# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources +# only. Doxygen will then generate output that is more tailored for C. For +# instance, some of the names that are used will be different. The list of all +# members will be omitted, etc. +# The default value is: NO. + +OPTIMIZE_OUTPUT_FOR_C = NO + +# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java or +# Python sources only. Doxygen will then generate output that is more tailored +# for that language. For instance, namespaces will be presented as packages, +# qualified scopes will look different, etc. +# The default value is: NO. + +OPTIMIZE_OUTPUT_JAVA = NO + +# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran +# sources. Doxygen will then generate output that is tailored for Fortran. +# The default value is: NO. + +OPTIMIZE_FOR_FORTRAN = NO + +# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL +# sources. Doxygen will then generate output that is tailored for VHDL. +# The default value is: NO. + +OPTIMIZE_OUTPUT_VHDL = NO + +# Doxygen selects the parser to use depending on the extension of the files it +# parses. With this tag you can assign which parser to use for a given +# extension. Doxygen has a built-in mapping, but you can override or extend it +# using this tag. The format is ext=language, where ext is a file extension, and +# language is one of the parsers supported by doxygen: IDL, Java, Javascript, +# C#, C, C++, D, PHP, Objective-C, Python, Fortran (fixed format Fortran: +# FortranFixed, free formatted Fortran: FortranFree, unknown formatted Fortran: +# Fortran. In the later case the parser tries to guess whether the code is fixed +# or free formatted code, this is the default for Fortran type files), VHDL. For +# instance to make doxygen treat .inc files as Fortran files (default is PHP), +# and .f files as C (default is Fortran), use: inc=Fortran f=C. +# +# Note: For files without extension you can use no_extension as a placeholder. +# +# Note that for custom extensions you also need to set FILE_PATTERNS otherwise +# the files are not read by doxygen. + +EXTENSION_MAPPING = + +# If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments +# according to the Markdown format, which allows for more readable +# documentation. See http://daringfireball.net/projects/markdown/ for details. +# The output of markdown processing is further processed by doxygen, so you can +# mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in +# case of backward compatibilities issues. +# The default value is: YES. + +MARKDOWN_SUPPORT = YES + +# When the TOC_INCLUDE_HEADINGS tag is set to a non-zero value, all headings up +# to that level are automatically included in the table of contents, even if +# they do not have an id attribute. +# Note: This feature currently applies only to Markdown headings. +# Minimum value: 0, maximum value: 99, default value: 0. +# This tag requires that the tag MARKDOWN_SUPPORT is set to YES. + +TOC_INCLUDE_HEADINGS = 1 + +# When enabled doxygen tries to link words that correspond to documented +# classes, or namespaces to their corresponding documentation. Such a link can +# be prevented in individual cases by putting a % sign in front of the word or +# globally by setting AUTOLINK_SUPPORT to NO. +# The default value is: YES. + +AUTOLINK_SUPPORT = YES + +# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want +# to include (a tag file for) the STL sources as input, then you should set this +# tag to YES in order to let doxygen match functions declarations and +# definitions whose arguments contain STL classes (e.g. func(std::string); +# versus func(std::string) {}). This also make the inheritance and collaboration +# diagrams that involve STL classes more complete and accurate. +# The default value is: NO. + +BUILTIN_STL_SUPPORT = NO + +# If you use Microsoft's C++/CLI language, you should set this option to YES to +# enable parsing support. +# The default value is: NO. + +CPP_CLI_SUPPORT = NO + +# Set the SIP_SUPPORT tag to YES if your project consists of sip (see: +# http://www.riverbankcomputing.co.uk/software/sip/intro) sources only. Doxygen +# will parse them like normal C++ but will assume all classes use public instead +# of private inheritance when no explicit protection keyword is present. +# The default value is: NO. + +SIP_SUPPORT = NO + +# For Microsoft's IDL there are propget and propput attributes to indicate +# getter and setter methods for a property. Setting this option to YES will make +# doxygen to replace the get and set methods by a property in the documentation. +# This will only work if the methods are indeed getting or setting a simple +# type. If this is not the case, or you want to show the methods anyway, you +# should set this option to NO. +# The default value is: YES. + +IDL_PROPERTY_SUPPORT = YES + +# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC +# tag is set to YES then doxygen will reuse the documentation of the first +# member in the group (if any) for the other members of the group. By default +# all members of a group must be documented explicitly. +# The default value is: NO. + +DISTRIBUTE_GROUP_DOC = NO + +# If one adds a struct or class to a group and this option is enabled, then also +# any nested class or struct is added to the same group. By default this option +# is disabled and one has to add nested compounds explicitly via \ingroup. +# The default value is: NO. + +GROUP_NESTED_COMPOUNDS = NO + +# Set the SUBGROUPING tag to YES to allow class member groups of the same type +# (for instance a group of public functions) to be put as a subgroup of that +# type (e.g. under the Public Functions section). Set it to NO to prevent +# subgrouping. Alternatively, this can be done per class using the +# \nosubgrouping command. +# The default value is: YES. + +SUBGROUPING = YES + +# When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and unions +# are shown inside the group in which they are included (e.g. using \ingroup) +# instead of on a separate page (for HTML and Man pages) or section (for LaTeX +# and RTF). +# +# Note that this feature does not work in combination with +# SEPARATE_MEMBER_PAGES. +# The default value is: NO. + +INLINE_GROUPED_CLASSES = NO + +# When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and unions +# with only public data fields or simple typedef fields will be shown inline in +# the documentation of the scope in which they are defined (i.e. file, +# namespace, or group documentation), provided this scope is documented. If set +# to NO, structs, classes, and unions are shown on a separate page (for HTML and +# Man pages) or section (for LaTeX and RTF). +# The default value is: NO. + +INLINE_SIMPLE_STRUCTS = NO + +# When TYPEDEF_HIDES_STRUCT tag is enabled, a typedef of a struct, union, or +# enum is documented as struct, union, or enum with the name of the typedef. So +# typedef struct TypeS {} TypeT, will appear in the documentation as a struct +# with name TypeT. When disabled the typedef will appear as a member of a file, +# namespace, or class. And the struct will be named TypeS. This can typically be +# useful for C code in case the coding convention dictates that all compound +# types are typedef'ed and only the typedef is referenced, never the tag name. +# The default value is: NO. + +TYPEDEF_HIDES_STRUCT = NO + +# The size of the symbol lookup cache can be set using LOOKUP_CACHE_SIZE. This +# cache is used to resolve symbols given their name and scope. Since this can be +# an expensive process and often the same symbol appears multiple times in the +# code, doxygen keeps a cache of pre-resolved symbols. If the cache is too small +# doxygen will become slower. If the cache is too large, memory is wasted. The +# cache size is given by this formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range +# is 0..9, the default is 0, corresponding to a cache size of 2^16=65536 +# symbols. At the end of a run doxygen will report the cache usage and suggest +# the optimal cache size from a speed point of view. +# Minimum value: 0, maximum value: 9, default value: 0. + +LOOKUP_CACHE_SIZE = 0 + +#--------------------------------------------------------------------------- +# Build related configuration options +#--------------------------------------------------------------------------- + +# If the EXTRACT_ALL tag is set to YES, doxygen will assume all entities in +# documentation are documented, even if no documentation was available. Private +# class members and static file members will be hidden unless the +# EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES. +# Note: This will also disable the warnings about undocumented members that are +# normally produced when WARNINGS is set to YES. +# The default value is: NO. + +EXTRACT_ALL = NO + +# If the EXTRACT_PRIVATE tag is set to YES, all private members of a class will +# be included in the documentation. +# The default value is: NO. + +EXTRACT_PRIVATE = YES + +# If the EXTRACT_PACKAGE tag is set to YES, all members with package or internal +# scope will be included in the documentation. +# The default value is: NO. + +EXTRACT_PACKAGE = YES + +# If the EXTRACT_STATIC tag is set to YES, all static members of a file will be +# included in the documentation. +# The default value is: NO. + +EXTRACT_STATIC = YES + +# If the EXTRACT_LOCAL_CLASSES tag is set to YES, classes (and structs) defined +# locally in source files will be included in the documentation. If set to NO, +# only classes defined in header files are included. Does not have any effect +# for Java sources. +# The default value is: YES. + +EXTRACT_LOCAL_CLASSES = YES + +# This flag is only useful for Objective-C code. If set to YES, local methods, +# which are defined in the implementation section but not in the interface are +# included in the documentation. If set to NO, only methods in the interface are +# included. +# The default value is: NO. + +EXTRACT_LOCAL_METHODS = NO + +# If this flag is set to YES, the members of anonymous namespaces will be +# extracted and appear in the documentation as a namespace called +# 'anonymous_namespace{file}', where file will be replaced with the base name of +# the file that contains the anonymous namespace. By default anonymous namespace +# are hidden. +# The default value is: NO. + +EXTRACT_ANON_NSPACES = NO + +# If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all +# undocumented members inside documented classes or files. If set to NO these +# members will be included in the various overviews, but no documentation +# section is generated. This option has no effect if EXTRACT_ALL is enabled. +# The default value is: NO. + +HIDE_UNDOC_MEMBERS = NO + +# If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all +# undocumented classes that are normally visible in the class hierarchy. If set +# to NO, these classes will be included in the various overviews. This option +# has no effect if EXTRACT_ALL is enabled. +# The default value is: NO. + +HIDE_UNDOC_CLASSES = NO + +# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend +# (class|struct|union) declarations. If set to NO, these declarations will be +# included in the documentation. +# The default value is: NO. + +HIDE_FRIEND_COMPOUNDS = NO + +# If the HIDE_IN_BODY_DOCS tag is set to YES, doxygen will hide any +# documentation blocks found inside the body of a function. If set to NO, these +# blocks will be appended to the function's detailed documentation block. +# The default value is: NO. + +HIDE_IN_BODY_DOCS = NO + +# The INTERNAL_DOCS tag determines if documentation that is typed after a +# \internal command is included. If the tag is set to NO then the documentation +# will be excluded. Set it to YES to include the internal documentation. +# The default value is: NO. + +INTERNAL_DOCS = NO + +# If the CASE_SENSE_NAMES tag is set to NO then doxygen will only generate file +# names in lower-case letters. If set to YES, upper-case letters are also +# allowed. This is useful if you have classes or files whose names only differ +# in case and if your file system supports case sensitive file names. Windows +# and Mac users are advised to set this option to NO. +# The default value is: system dependent. + +CASE_SENSE_NAMES = YES + +# If the HIDE_SCOPE_NAMES tag is set to NO then doxygen will show members with +# their full class and namespace scopes in the documentation. If set to YES, the +# scope will be hidden. +# The default value is: NO. + +HIDE_SCOPE_NAMES = NO + +# If the HIDE_COMPOUND_REFERENCE tag is set to NO (default) then doxygen will +# append additional text to a page's title, such as Class Reference. If set to +# YES the compound reference will be hidden. +# The default value is: NO. + +HIDE_COMPOUND_REFERENCE= NO + +# If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of +# the files that are included by a file in the documentation of that file. +# The default value is: YES. + +SHOW_INCLUDE_FILES = YES + +# If the SHOW_GROUPED_MEMB_INC tag is set to YES then Doxygen will add for each +# grouped member an include statement to the documentation, telling the reader +# which file to include in order to use the member. +# The default value is: NO. + +SHOW_GROUPED_MEMB_INC = NO + +# If the FORCE_LOCAL_INCLUDES tag is set to YES then doxygen will list include +# files with double quotes in the documentation rather than with sharp brackets. +# The default value is: NO. + +FORCE_LOCAL_INCLUDES = NO + +# If the INLINE_INFO tag is set to YES then a tag [inline] is inserted in the +# documentation for inline members. +# The default value is: YES. + +INLINE_INFO = YES + +# If the SORT_MEMBER_DOCS tag is set to YES then doxygen will sort the +# (detailed) documentation of file and class members alphabetically by member +# name. If set to NO, the members will appear in declaration order. +# The default value is: YES. + +SORT_MEMBER_DOCS = YES + +# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the brief +# descriptions of file, namespace and class members alphabetically by member +# name. If set to NO, the members will appear in declaration order. Note that +# this will also influence the order of the classes in the class list. +# The default value is: NO. + +SORT_BRIEF_DOCS = NO + +# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen will sort the +# (brief and detailed) documentation of class members so that constructors and +# destructors are listed first. If set to NO the constructors will appear in the +# respective orders defined by SORT_BRIEF_DOCS and SORT_MEMBER_DOCS. +# Note: If SORT_BRIEF_DOCS is set to NO this option is ignored for sorting brief +# member documentation. +# Note: If SORT_MEMBER_DOCS is set to NO this option is ignored for sorting +# detailed member documentation. +# The default value is: NO. + +SORT_MEMBERS_CTORS_1ST = NO + +# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the hierarchy +# of group names into alphabetical order. If set to NO the group names will +# appear in their defined order. +# The default value is: NO. + +SORT_GROUP_NAMES = NO + +# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by +# fully-qualified names, including namespaces. If set to NO, the class list will +# be sorted only by class name, not including the namespace part. +# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. +# Note: This option applies only to the class list, not to the alphabetical +# list. +# The default value is: NO. + +SORT_BY_SCOPE_NAME = NO + +# If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to do proper +# type resolution of all parameters of a function it will reject a match between +# the prototype and the implementation of a member function even if there is +# only one candidate or it is obvious which candidate to choose by doing a +# simple string match. By disabling STRICT_PROTO_MATCHING doxygen will still +# accept a match between prototype and implementation in such cases. +# The default value is: NO. + +STRICT_PROTO_MATCHING = NO + +# The GENERATE_TODOLIST tag can be used to enable (YES) or disable (NO) the todo +# list. This list is created by putting \todo commands in the documentation. +# The default value is: YES. + +GENERATE_TODOLIST = YES + +# The GENERATE_TESTLIST tag can be used to enable (YES) or disable (NO) the test +# list. This list is created by putting \test commands in the documentation. +# The default value is: YES. + +GENERATE_TESTLIST = YES + +# The GENERATE_BUGLIST tag can be used to enable (YES) or disable (NO) the bug +# list. This list is created by putting \bug commands in the documentation. +# The default value is: YES. + +GENERATE_BUGLIST = YES + +# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or disable (NO) +# the deprecated list. This list is created by putting \deprecated commands in +# the documentation. +# The default value is: YES. + +GENERATE_DEPRECATEDLIST= YES + +# The ENABLED_SECTIONS tag can be used to enable conditional documentation +# sections, marked by \if ... \endif and \cond +# ... \endcond blocks. + +ENABLED_SECTIONS = + +# The MAX_INITIALIZER_LINES tag determines the maximum number of lines that the +# initial value of a variable or macro / define can have for it to appear in the +# documentation. If the initializer consists of more lines than specified here +# it will be hidden. Use a value of 0 to hide initializers completely. The +# appearance of the value of individual variables and macros / defines can be +# controlled using \showinitializer or \hideinitializer command in the +# documentation regardless of this setting. +# Minimum value: 0, maximum value: 10000, default value: 30. + +MAX_INITIALIZER_LINES = 30 + +# Set the SHOW_USED_FILES tag to NO to disable the list of files generated at +# the bottom of the documentation of classes and structs. If set to YES, the +# list will mention the files that were used to generate the documentation. +# The default value is: YES. + +SHOW_USED_FILES = YES + +# Set the SHOW_FILES tag to NO to disable the generation of the Files page. This +# will remove the Files entry from the Quick Index and from the Folder Tree View +# (if specified). +# The default value is: YES. + +SHOW_FILES = YES + +# Set the SHOW_NAMESPACES tag to NO to disable the generation of the Namespaces +# page. This will remove the Namespaces entry from the Quick Index and from the +# Folder Tree View (if specified). +# The default value is: YES. + +SHOW_NAMESPACES = YES + +# The FILE_VERSION_FILTER tag can be used to specify a program or script that +# doxygen should invoke to get the current version for each file (typically from +# the version control system). Doxygen will invoke the program by executing (via +# popen()) the command command input-file, where command is the value of the +# FILE_VERSION_FILTER tag, and input-file is the name of an input file provided +# by doxygen. Whatever the program writes to standard output is used as the file +# version. For an example see the documentation. + +FILE_VERSION_FILTER = + +# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed +# by doxygen. The layout file controls the global structure of the generated +# output files in an output format independent way. To create the layout file +# that represents doxygen's defaults, run doxygen with the -l option. You can +# optionally specify a file name after the option, if omitted DoxygenLayout.xml +# will be used as the name of the layout file. +# +# Note that if you run doxygen from a directory containing a file called +# DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE +# tag is left empty. + +LAYOUT_FILE = $(MLPERF_LOADGEN_SRC_PATH)/docs/src/doxygen_layout.xml + +# The CITE_BIB_FILES tag can be used to specify one or more bib files containing +# the reference definitions. This must be a list of .bib files. The .bib +# extension is automatically appended if omitted. This requires the bibtex tool +# to be installed. See also http://en.wikipedia.org/wiki/BibTeX for more info. +# For LaTeX the style of the bibliography can be controlled using +# LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the +# search path. See also \cite for info how to create references. + +CITE_BIB_FILES = + +#--------------------------------------------------------------------------- +# Configuration options related to warning and progress messages +#--------------------------------------------------------------------------- + +# The QUIET tag can be used to turn on/off the messages that are generated to +# standard output by doxygen. If QUIET is set to YES this implies that the +# messages are off. +# The default value is: NO. + +QUIET = NO + +# The WARNINGS tag can be used to turn on/off the warning messages that are +# generated to standard error (stderr) by doxygen. If WARNINGS is set to YES +# this implies that the warnings are on. +# +# Tip: Turn warnings on while writing the documentation. +# The default value is: YES. + +WARNINGS = YES + +# If the WARN_IF_UNDOCUMENTED tag is set to YES then doxygen will generate +# warnings for undocumented members. If EXTRACT_ALL is set to YES then this flag +# will automatically be disabled. +# The default value is: YES. + +WARN_IF_UNDOCUMENTED = NO + +# If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for +# potential errors in the documentation, such as not documenting some parameters +# in a documented function, or documenting parameters that don't exist or using +# markup commands wrongly. +# The default value is: YES. + +WARN_IF_DOC_ERROR = YES + +# This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that +# are documented, but have no documentation for their parameters or return +# value. If set to NO, doxygen will only warn about wrong or incomplete +# parameter documentation, but not about the absence of documentation. +# The default value is: NO. + +WARN_NO_PARAMDOC = NO + +# If the WARN_AS_ERROR tag is set to YES then doxygen will immediately stop when +# a warning is encountered. +# The default value is: NO. + +WARN_AS_ERROR = NO + +# The WARN_FORMAT tag determines the format of the warning messages that doxygen +# can produce. The string should contain the $file, $line, and $text tags, which +# will be replaced by the file and line number from which the warning originated +# and the warning text. Optionally the format may contain $version, which will +# be replaced by the version of the file (if it could be obtained via +# FILE_VERSION_FILTER) +# The default value is: $file:$line: $text. + +WARN_FORMAT = "$file:$line: $text" + +# The WARN_LOGFILE tag can be used to specify a file to which warning and error +# messages should be written. If left blank the output is written to standard +# error (stderr). + +WARN_LOGFILE = + +#--------------------------------------------------------------------------- +# Configuration options related to the input files +#--------------------------------------------------------------------------- + +# The INPUT tag is used to specify the files and/or directories that contain +# documented source files. You may enter file names like myfile.cpp or +# directories like /usr/src/myproject. Separate the files or directories with +# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING +# Note: If this tag is empty the current directory is searched. + +INPUT = $(MLPERF_LOADGEN_SRC_PATH) + +# This tag can be used to specify the character encoding of the source files +# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses +# libiconv (or the iconv built into libc) for the transcoding. See the libiconv +# documentation (see: http://www.gnu.org/software/libiconv) for the list of +# possible encodings. +# The default value is: UTF-8. + +INPUT_ENCODING = UTF-8 + +# If the value of the INPUT tag contains directories, you can use the +# FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and +# *.h) to filter out the source-files in the directories. +# +# Note that for custom extensions or not directly supported extensions you also +# need to set EXTENSION_MAPPING for the extension otherwise the files are not +# read by doxygen. +# +# If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp, +# *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, +# *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc, +# *.m, *.markdown, *.md, *.mm, *.dox, *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, +# *.f, *.for, *.tcl, *.vhd, *.vhdl, *.ucf and *.qsf. + +FILE_PATTERNS = *.c \ + *.cc \ + *.cxx \ + *.cpp \ + *.c++ \ + *.java \ + *.ii \ + *.ixx \ + *.ipp \ + *.i++ \ + *.inl \ + *.idl \ + *.ddl \ + *.odl \ + *.h \ + *.hh \ + *.hxx \ + *.hpp \ + *.h++ \ + *.cs \ + *.d \ + *.php \ + *.php4 \ + *.php5 \ + *.phtml \ + *.inc \ + *.m \ + *.markdown \ + *.md \ + *.mm \ + *.dox \ + *.py \ + *.pyw \ + *.f90 \ + *.f95 \ + *.f03 \ + *.f08 \ + *.f \ + *.for \ + *.tcl \ + *.vhd \ + *.vhdl \ + *.ucf \ + *.qsf + +# The RECURSIVE tag can be used to specify whether or not subdirectories should +# be searched for input files as well. +# The default value is: NO. + +RECURSIVE = YES + +# The EXCLUDE tag can be used to specify files and/or directories that should be +# excluded from the INPUT source files. This way you can easily exclude a +# subdirectory from a directory tree whose root is specified with the INPUT tag. +# +# Note that relative paths are relative to the directory from which doxygen is +# run. + +EXCLUDE = depot_tools + +# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or +# directories that are symbolic links (a Unix file system feature) are excluded +# from the input. +# The default value is: NO. + +EXCLUDE_SYMLINKS = NO + +# If the value of the INPUT tag contains directories, you can use the +# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude +# certain files from those directories. +# +# Note that the wildcards are matched against the file with absolute path, so to +# exclude all test directories for example use the pattern */test/* + +EXCLUDE_PATTERNS = + +# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names +# (namespaces, classes, functions, etc.) that should be excluded from the +# output. The symbol name can be a fully qualified name, a word, or if the +# wildcard * is used, a substring. Examples: ANamespace, AClass, +# AClass::ANamespace, ANamespace::*Test +# +# Note that the wildcards are matched against the file with absolute path, so to +# exclude all test directories use the pattern */test/* + +EXCLUDE_SYMBOLS = + +# The EXAMPLE_PATH tag can be used to specify one or more files or directories +# that contain example code fragments that are included (see the \include +# command). + +EXAMPLE_PATH = + +# If the value of the EXAMPLE_PATH tag contains directories, you can use the +# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and +# *.h) to filter out the source-files in the directories. If left blank all +# files are included. + +EXAMPLE_PATTERNS = * + +# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be +# searched for input files to be used with the \include or \dontinclude commands +# irrespective of the value of the RECURSIVE tag. +# The default value is: NO. + +EXAMPLE_RECURSIVE = NO + +# The IMAGE_PATH tag can be used to specify one or more files or directories +# that contain images that are to be included in the documentation (see the +# \image command). + +IMAGE_PATH = $(MLPERF_LOADGEN_SRC_PATH)/docs/src + +# The INPUT_FILTER tag can be used to specify a program that doxygen should +# invoke to filter for each input file. Doxygen will invoke the filter program +# by executing (via popen()) the command: +# +# +# +# where is the value of the INPUT_FILTER tag, and is the +# name of an input file. Doxygen will then use the output that the filter +# program writes to standard output. If FILTER_PATTERNS is specified, this tag +# will be ignored. +# +# Note that the filter must not add or remove lines; it is applied before the +# code is scanned, but not when the output code is generated. If lines are added +# or removed, the anchors will not be placed correctly. +# +# Note that for custom extensions or not directly supported extensions you also +# need to set EXTENSION_MAPPING for the extension otherwise the files are not +# properly processed by doxygen. + +INPUT_FILTER = + +# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern +# basis. Doxygen will compare the file name with each pattern and apply the +# filter if there is a match. The filters are a list of the form: pattern=filter +# (like *.cpp=my_cpp_filter). See INPUT_FILTER for further information on how +# filters are used. If the FILTER_PATTERNS tag is empty or if none of the +# patterns match the file name, INPUT_FILTER is applied. +# +# Note that for custom extensions or not directly supported extensions you also +# need to set EXTENSION_MAPPING for the extension otherwise the files are not +# properly processed by doxygen. + +FILTER_PATTERNS = + +# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using +# INPUT_FILTER) will also be used to filter the input files that are used for +# producing the source files to browse (i.e. when SOURCE_BROWSER is set to YES). +# The default value is: NO. + +FILTER_SOURCE_FILES = NO + +# The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file +# pattern. A pattern will override the setting for FILTER_PATTERN (if any) and +# it is also possible to disable source filtering for a specific pattern using +# *.ext= (so without naming a filter). +# This tag requires that the tag FILTER_SOURCE_FILES is set to YES. + +FILTER_SOURCE_PATTERNS = + +# If the USE_MDFILE_AS_MAINPAGE tag refers to the name of a markdown file that +# is part of the input, its contents will be placed on the main page +# (index.html). This can be useful if you have a project on for instance GitHub +# and want to reuse the introduction page also for the doxygen output. + +USE_MDFILE_AS_MAINPAGE = + +#--------------------------------------------------------------------------- +# Configuration options related to source browsing +#--------------------------------------------------------------------------- + +# If the SOURCE_BROWSER tag is set to YES then a list of source files will be +# generated. Documented entities will be cross-referenced with these sources. +# +# Note: To get rid of all source code in the generated output, make sure that +# also VERBATIM_HEADERS is set to NO. +# The default value is: NO. + +SOURCE_BROWSER = YES + +# Setting the INLINE_SOURCES tag to YES will include the body of functions, +# classes and enums directly into the documentation. +# The default value is: NO. + +INLINE_SOURCES = NO + +# Setting the STRIP_CODE_COMMENTS tag to YES will instruct doxygen to hide any +# special comment blocks from generated source code fragments. Normal C, C++ and +# Fortran comments will always remain visible. +# The default value is: YES. + +STRIP_CODE_COMMENTS = YES + +# If the REFERENCED_BY_RELATION tag is set to YES then for each documented +# function all documented functions referencing it will be listed. +# The default value is: NO. + +REFERENCED_BY_RELATION = NO + +# If the REFERENCES_RELATION tag is set to YES then for each documented function +# all documented entities called/used by that function will be listed. +# The default value is: NO. + +REFERENCES_RELATION = NO + +# If the REFERENCES_LINK_SOURCE tag is set to YES and SOURCE_BROWSER tag is set +# to YES then the hyperlinks from functions in REFERENCES_RELATION and +# REFERENCED_BY_RELATION lists will link to the source code. Otherwise they will +# link to the documentation. +# The default value is: YES. + +REFERENCES_LINK_SOURCE = YES + +# If SOURCE_TOOLTIPS is enabled (the default) then hovering a hyperlink in the +# source code will show a tooltip with additional information such as prototype, +# brief description and links to the definition and documentation. Since this +# will make the HTML file larger and loading of large files a bit slower, you +# can opt to disable this feature. +# The default value is: YES. +# This tag requires that the tag SOURCE_BROWSER is set to YES. + +SOURCE_TOOLTIPS = YES + +# If the USE_HTAGS tag is set to YES then the references to source code will +# point to the HTML generated by the htags(1) tool instead of doxygen built-in +# source browser. The htags tool is part of GNU's global source tagging system +# (see http://www.gnu.org/software/global/global.html). You will need version +# 4.8.6 or higher. +# +# To use it do the following: +# - Install the latest version of global +# - Enable SOURCE_BROWSER and USE_HTAGS in the config file +# - Make sure the INPUT points to the root of the source tree +# - Run doxygen as normal +# +# Doxygen will invoke htags (and that will in turn invoke gtags), so these +# tools must be available from the command line (i.e. in the search path). +# +# The result: instead of the source browser generated by doxygen, the links to +# source code will now point to the output of htags. +# The default value is: NO. +# This tag requires that the tag SOURCE_BROWSER is set to YES. + +USE_HTAGS = NO + +# If the VERBATIM_HEADERS tag is set the YES then doxygen will generate a +# verbatim copy of the header file for each class for which an include is +# specified. Set to NO to disable this. +# See also: Section \class. +# The default value is: YES. + +VERBATIM_HEADERS = YES + +# If the CLANG_ASSISTED_PARSING tag is set to YES then doxygen will use the +# clang parser (see: http://clang.llvm.org/) for more accurate parsing at the +# cost of reduced performance. This can be particularly helpful with template +# rich C++ code for which doxygen's built-in parser lacks the necessary type +# information. +# Note: The availability of this option depends on whether or not doxygen was +# generated with the -Duse-libclang=ON option for CMake. +# The default value is: NO. + +CLANG_ASSISTED_PARSING = YES + +# If clang assisted parsing is enabled you can provide the compiler with command +# line options that you would normally use when invoking the compiler. Note that +# the include paths will already be set by doxygen for the files and directories +# specified with INPUT and INCLUDE_PATH. +# This tag requires that the tag CLANG_ASSISTED_PARSING is set to YES. + +CLANG_OPTIONS = -I ../third_party/pybind/include --std=c++14 + +#--------------------------------------------------------------------------- +# Configuration options related to the alphabetical class index +#--------------------------------------------------------------------------- + +# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index of all +# compounds will be generated. Enable this if the project contains a lot o= +# classes, structs, unions or interfaces. +# The default value is: YES. + +ALPHABETICAL_INDEX = YES + +# The COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns in +# which the alphabetical index list will be split. +# Minimum value: 1, maximum value: 20, default value: 5. +# This tag requires that the tag ALPHABETICAL_INDEX is set to YES. + +COLS_IN_ALPHA_INDEX = 5 + +# In case all classes in a project start with a common prefix, all classes will +# be put under the same header in the alphabetical index. The IGNORE_PREFIX tag +# can be used to specify a prefix (or a list of prefixes) that should be ignored +# while generating the index headers. +# This tag requires that the tag ALPHABETICAL_INDEX is set to YES. + +IGNORE_PREFIX = + +#--------------------------------------------------------------------------- +# Configuration options related to the HTML output +#--------------------------------------------------------------------------- + +# If the GENERATE_HTML tag is set to YES, doxygen will generate HTML output +# The default value is: YES. + +GENERATE_HTML = YES + +# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a +# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of +# it. +# The default directory is: html. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_OUTPUT = html + +# The HTML_FILE_EXTENSION tag can be used to specify the file extension for each +# generated HTML page (for example: .htm, .php, .asp). +# The default value is: .html. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_FILE_EXTENSION = .html + +# The HTML_HEADER tag can be used to specify a user-defined HTML header file for +# each generated HTML page. If the tag is left blank doxygen will generate a +# standard header. +# +# To get valid HTML the header file that includes any scripts and style sheets +# that doxygen needs, which is dependent on the configuration options used (e.g. +# the setting GENERATE_TREEVIEW). It is highly recommended to start with a +# default header using +# doxygen -w html new_header.html new_footer.html new_stylesheet.css +# YourConfigFile +# and then modify the file new_header.html. See also section "Doxygen usage" +# for information on how to generate the default header that doxygen normally +# uses. +# Note: The header is subject to change so you typically have to regenerate the +# default header when upgrading to a newer version of doxygen. For a description +# of the possible markers and block names see the documentation. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_HEADER = $(MLPERF_LOADGEN_SRC_PATH)/docs/src/doxygen_header.html + +# The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each +# generated HTML page. If the tag is left blank doxygen will generate a standard +# footer. See HTML_HEADER for more information on how to generate a default +# footer and what special commands can be used inside the footer. See also +# section "Doxygen usage" for information on how to generate the default footer +# that doxygen normally uses. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_FOOTER = $(MLPERF_LOADGEN_SRC_PATH)/docs/src/doxygen_footer.html + +# The HTML_STYLESHEET tag can be used to specify a user-defined cascading style +# sheet that is used by each HTML page. It can be used to fine-tune the look of +# the HTML output. If left blank doxygen will generate a default style sheet. +# See also section "Doxygen usage" for information on how to generate the style +# sheet that doxygen normally uses. +# Note: It is recommended to use HTML_EXTRA_STYLESHEET instead of this tag, as +# it is more robust and this tag (HTML_STYLESHEET) will in the future become +# obsolete. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_STYLESHEET = + +# The HTML_EXTRA_STYLESHEET tag can be used to specify additional user-defined +# cascading style sheets that are included after the standard style sheets +# created by doxygen. Using this option one can overrule certain style aspects. +# This is preferred over using HTML_STYLESHEET since it does not replace the +# standard style sheet and is therefore more robust against future updates. +# Doxygen will copy the style sheet files to the output directory. +# Note: The order of the extra style sheet files is of importance (e.g. the last +# style sheet in the list overrules the setting of the previous ones in the +# list). For an example see the documentation. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_EXTRA_STYLESHEET = $(MLPERF_LOADGEN_SRC_PATH)/docs/src/doxygen_stylesheet.css + +# The HTML_EXTRA_FILES tag can be used to specify one or more extra images or +# other source files which should be copied to the HTML output directory. Note +# that these files will be copied to the base HTML output directory. Use the +# $relpath^ marker in the HTML_HEADER and/or HTML_FOOTER files to load these +# files. In the HTML_STYLESHEET file, use the file name only. Also note that the +# files will be copied as-is; there are no commands or markers available. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_EXTRA_FILES = $(MLPERF_LOADGEN_SRC_PATH)/docs/src/mlperf_icon.png \ + $(MLPERF_LOADGEN_SRC_PATH)/loadgen_integration_diagram.svg + +# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen +# will adjust the colors in the style sheet and background images according to +# this color. Hue is specified as an angle on a colorwheel, see +# http://en.wikipedia.org/wiki/Hue for more information. For instance the value +# 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300 +# purple, and 360 is red again. +# Minimum value: 0, maximum value: 359, default value: 220. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_COLORSTYLE_HUE = 220 + +# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors +# in the HTML output. For a value of 0 the output will use grayscales only. A +# value of 255 will produce the most vivid colors. +# Minimum value: 0, maximum value: 255, default value: 100. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_COLORSTYLE_SAT = 127 + +# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to the +# luminance component of the colors in the HTML output. Values below 100 +# gradually make the output lighter, whereas values above 100 make the output +# darker. The value divided by 100 is the actual gamma applied, so 80 represents +# a gamma of 0.8, The value 220 represents a gamma of 2.2, and 100 does not +# change the gamma. +# Minimum value: 40, maximum value: 240, default value: 80. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_COLORSTYLE_GAMMA = 80 + +# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML +# page will contain the date and time when the page was generated. Setting this +# to YES can help to show when doxygen was last run and thus if the +# documentation is up to date. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_TIMESTAMP = NO + +# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML +# documentation will contain sections that can be hidden and shown after the +# page has loaded. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_DYNAMIC_SECTIONS = YES + +# With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries +# shown in the various tree structured indices initially; the user can expand +# and collapse entries dynamically later on. Doxygen will expand the tree to +# such a level that at most the specified number of entries are visible (unless +# a fully collapsed tree already exceeds this amount). So setting the number of +# entries 1 will produce a full collapsed tree by default. 0 is a special value +# representing an infinite number of entries and will result in a full expanded +# tree by default. +# Minimum value: 0, maximum value: 9999, default value: 100. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_INDEX_NUM_ENTRIES = 50 + +# If the GENERATE_DOCSET tag is set to YES, additional index files will be +# generated that can be used as input for Apple's Xcode 3 integrated development +# environment (see: http://developer.apple.com/tools/xcode/), introduced with +# OSX 10.5 (Leopard). To create a documentation set, doxygen will generate a +# Makefile in the HTML output directory. Running make will produce the docset in +# that directory and running make install will install the docset in +# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at +# startup. See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html +# for more information. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_DOCSET = NO + +# This tag determines the name of the docset feed. A documentation feed provides +# an umbrella under which multiple documentation sets from a single provider +# (such as a company or product suite) can be grouped. +# The default value is: Doxygen generated docs. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_FEEDNAME = "Doxygen generated docs" + +# This tag specifies a string that should uniquely identify the documentation +# set bundle. This should be a reverse domain-name style string, e.g. +# com.mycompany.MyDocSet. Doxygen will append .docset to the name. +# The default value is: org.doxygen.Project. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_BUNDLE_ID = org.doxygen.Project + +# The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify +# the documentation publisher. This should be a reverse domain-name style +# string, e.g. com.mycompany.MyDocSet.documentation. +# The default value is: org.doxygen.Publisher. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_PUBLISHER_ID = org.doxygen.Publisher + +# The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher. +# The default value is: Publisher. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_PUBLISHER_NAME = Publisher + +# If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three +# additional HTML index files: index.hhp, index.hhc, and index.hhk. The +# index.hhp is a project file that can be read by Microsoft's HTML Help Workshop +# (see: http://www.microsoft.com/en-us/download/details.aspx?id=21138) on +# Windows. +# +# The HTML Help Workshop contains a compiler that can convert all HTML output +# generated by doxygen into a single compiled HTML file (.chm). Compiled HTML +# files are now used as the Windows 98 help format, and will replace the old +# Windows help format (.hlp) on all Windows platforms in the future. Compressed +# HTML files also contain an index, a table of contents, and you can search for +# words in the documentation. The HTML workshop also contains a viewer for +# compressed HTML files. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_HTMLHELP = NO + +# The CHM_FILE tag can be used to specify the file name of the resulting .chm +# file. You can add a path in front of the file if the result should not be +# written to the html output directory. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +CHM_FILE = + +# The HHC_LOCATION tag can be used to specify the location (absolute path +# including file name) of the HTML help compiler (hhc.exe). If non-empty, +# doxygen will try to run the HTML help compiler on the generated index.hhp. +# The file has to be specified with full path. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +HHC_LOCATION = + +# The GENERATE_CHI flag controls if a separate .chi index file is generated +# (YES) or that it should be included in the master .chm file (NO). +# The default value is: NO. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +GENERATE_CHI = NO + +# The CHM_INDEX_ENCODING is used to encode HtmlHelp index (hhk), content (hhc) +# and project file content. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +CHM_INDEX_ENCODING = + +# The BINARY_TOC flag controls whether a binary table of contents is generated +# (YES) or a normal table of contents (NO) in the .chm file. Furthermore it +# enables the Previous and Next buttons. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +BINARY_TOC = NO + +# The TOC_EXPAND flag can be set to YES to add extra items for group members to +# the table of contents of the HTML help documentation and to the tree view. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +TOC_EXPAND = NO + +# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and +# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that +# can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help +# (.qch) of the generated HTML documentation. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_QHP = NO + +# If the QHG_LOCATION tag is specified, the QCH_FILE tag can be used to specify +# the file name of the resulting .qch file. The path specified is relative to +# the HTML output folder. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QCH_FILE = + +# The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help +# Project output. For more information please see Qt Help Project / Namespace +# (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#namespace). +# The default value is: org.doxygen.Project. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_NAMESPACE = org.doxygen.Project + +# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt +# Help Project output. For more information please see Qt Help Project / Virtual +# Folders (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#virtual- +# folders). +# The default value is: doc. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_VIRTUAL_FOLDER = doc + +# If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom +# filter to add. For more information please see Qt Help Project / Custom +# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom- +# filters). +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_CUST_FILTER_NAME = + +# The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the +# custom filter to add. For more information please see Qt Help Project / Custom +# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom- +# filters). +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_CUST_FILTER_ATTRS = + +# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this +# project's filter section matches. Qt Help Project / Filter Attributes (see: +# http://qt-project.org/doc/qt-4.8/qthelpproject.html#filter-attributes). +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_SECT_FILTER_ATTRS = + +# The QHG_LOCATION tag can be used to specify the location of Qt's +# qhelpgenerator. If non-empty doxygen will try to run qhelpgenerator on the +# generated .qhp file. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHG_LOCATION = + +# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files will be +# generated, together with the HTML files, they form an Eclipse help plugin. To +# install this plugin and make it available under the help contents menu in +# Eclipse, the contents of the directory containing the HTML and XML files needs +# to be copied into the plugins directory of eclipse. The name of the directory +# within the plugins directory should be the same as the ECLIPSE_DOC_ID value. +# After copying Eclipse needs to be restarted before the help appears. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_ECLIPSEHELP = NO + +# A unique identifier for the Eclipse help plugin. When installing the plugin +# the directory name containing the HTML and XML files should also have this +# name. Each documentation set should have its own identifier. +# The default value is: org.doxygen.Project. +# This tag requires that the tag GENERATE_ECLIPSEHELP is set to YES. + +ECLIPSE_DOC_ID = org.doxygen.Project + +# If you want full control over the layout of the generated HTML pages it might +# be necessary to disable the index and replace it with your own. The +# DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) at top +# of each HTML page. A value of NO enables the index and the value YES disables +# it. Since the tabs in the index contain the same information as the navigation +# tree, you can set this option to YES if you also set GENERATE_TREEVIEW to YES. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +DISABLE_INDEX = NO + +# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index +# structure should be generated to display hierarchical information. If the tag +# value is set to YES, a side panel will be generated containing a tree-like +# index structure (just like the one that is generated for HTML Help). For this +# to work a browser that supports JavaScript, DHTML, CSS and frames is required +# (i.e. any modern browser). Windows users are probably better off using the +# HTML help feature. Via custom style sheets (see HTML_EXTRA_STYLESHEET) one can +# further fine-tune the look of the index. As an example, the default style +# sheet generated by doxygen has an example that shows how to put an image at +# the root of the tree instead of the PROJECT_NAME. Since the tree basically has +# the same information as the tab index, you could consider setting +# DISABLE_INDEX to YES when enabling this option. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_TREEVIEW = YES + +# The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that +# doxygen will group on one line in the generated HTML documentation. +# +# Note that a value of 0 will completely suppress the enum values from appearing +# in the overview section. +# Minimum value: 0, maximum value: 20, default value: 4. +# This tag requires that the tag GENERATE_HTML is set to YES. + +ENUM_VALUES_PER_LINE = 4 + +# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be used +# to set the initial width (in pixels) of the frame in which the tree is shown. +# Minimum value: 0, maximum value: 1500, default value: 250. +# This tag requires that the tag GENERATE_HTML is set to YES. + +TREEVIEW_WIDTH = 250 + +# If the EXT_LINKS_IN_WINDOW option is set to YES, doxygen will open links to +# external symbols imported via tag files in a separate window. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +EXT_LINKS_IN_WINDOW = NO + +# Use this tag to change the font size of LaTeX formulas included as images in +# the HTML documentation. When you change the font size after a successful +# doxygen run you need to manually remove any form_*.png images from the HTML +# output directory to force them to be regenerated. +# Minimum value: 8, maximum value: 50, default value: 10. +# This tag requires that the tag GENERATE_HTML is set to YES. + +FORMULA_FONTSIZE = 10 + +# Use the FORMULA_TRANPARENT tag to determine whether or not the images +# generated for formulas are transparent PNGs. Transparent PNGs are not +# supported properly for IE 6.0, but are supported on all modern browsers. +# +# Note that when changing this option you need to delete any form_*.png files in +# the HTML output directory before the changes have effect. +# The default value is: YES. +# This tag requires that the tag GENERATE_HTML is set to YES. + +FORMULA_TRANSPARENT = YES + +# Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see +# http://www.mathjax.org) which uses client side Javascript for the rendering +# instead of using pre-rendered bitmaps. Use this if you do not have LaTeX +# installed or if you want to formulas look prettier in the HTML output. When +# enabled you may also need to install MathJax separately and configure the path +# to it using the MATHJAX_RELPATH option. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +USE_MATHJAX = NO + +# When MathJax is enabled you can set the default output format to be used for +# the MathJax output. See the MathJax site (see: +# http://docs.mathjax.org/en/latest/output.html) for more details. +# Possible values are: HTML-CSS (which is slower, but has the best +# compatibility), NativeMML (i.e. MathML) and SVG. +# The default value is: HTML-CSS. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_FORMAT = HTML-CSS + +# When MathJax is enabled you need to specify the location relative to the HTML +# output directory using the MATHJAX_RELPATH option. The destination directory +# should contain the MathJax.js script. For instance, if the mathjax directory +# is located at the same level as the HTML output directory, then +# MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax +# Content Delivery Network so you can quickly see the result without installing +# MathJax. However, it is strongly recommended to install a local copy of +# MathJax from http://www.mathjax.org before deployment. +# The default value is: http://cdn.mathjax.org/mathjax/latest. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest + +# The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax +# extension names that should be enabled during MathJax rendering. For example +# MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_EXTENSIONS = + +# The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces +# of code that will be used on startup of the MathJax code. See the MathJax site +# (see: http://docs.mathjax.org/en/latest/output.html) for more details. For an +# example see the documentation. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_CODEFILE = + +# When the SEARCHENGINE tag is enabled doxygen will generate a search box for +# the HTML output. The underlying search engine uses javascript and DHTML and +# should work on any modern browser. Note that when using HTML help +# (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets (GENERATE_DOCSET) +# there is already a search function so this one should typically be disabled. +# For large projects the javascript based search engine can be slow, then +# enabling SERVER_BASED_SEARCH may provide a better solution. It is possible to +# search using the keyboard; to jump to the search box use + S +# (what the is depends on the OS and browser, but it is typically +# , /

+ + +
+$generatedby   +doxygen + $doxygenversion +
+ + + diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/doxygen_header.html b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/doxygen_header.html new file mode 100644 index 000000000..91d214b95 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/doxygen_header.html @@ -0,0 +1,49 @@ + + + + + + + + + +LoadGen: $title +$title + + + +$treeview +$search +$mathjax + +$extrastylesheet + + +
+ + +
+ + MLPerf + + +
+
$projectname +  $projectnumber +
+
$projectbrief
+
+ + + +
$projectbrief
+ + + + +
$searchbox
+ + +
+ + diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/doxygen_html_generator.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/doxygen_html_generator.py new file mode 100644 index 000000000..4065d7bd0 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/doxygen_html_generator.py @@ -0,0 +1,37 @@ +# Copyright 2019 The MLPerf Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +# \file +# \brief A script that sets the environment variables expected by doxygen.cfg. +# \details This can be run manually without any arguments, but also allows a +# build system to customize the output directory. + +import os +import sys + + +def generate_doxygen_html(doxygen_out_dir, loadgen_root): + os.environ["MLPERF_LOADGEN_SRC_PATH"] = loadgen_root + os.environ["MLPERF_DOXYGEN_OUT_PATH"] = doxygen_out_dir + os.popen("doxygen " + loadgen_root + "/docs/src/doxygen.cfg") + + +def main(argv): + doxygen_out_dir = "./docs/gen" if len(argv) < 2 else argv[1] + loadgen_root = "." if len(argv) < 3 else argv[2] + generate_doxygen_html(doxygen_out_dir, loadgen_root) + + +main(sys.argv) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/doxygen_layout.xml b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/doxygen_layout.xml new file mode 100644 index 000000000..1fc5a9cb4 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/doxygen_layout.xml @@ -0,0 +1,211 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/doxygen_stylesheet.css b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/doxygen_stylesheet.css new file mode 100644 index 000000000..3bd61261c --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/doxygen_stylesheet.css @@ -0,0 +1,1629 @@ +/* The standard CSS for doxygen 1.8.13 */ + +body, table, div, p, dl { + font: 400 14px/22px Roboto,sans-serif; +} + +p.reference, p.definition { + font: 400 14px/22px Roboto,sans-serif; +} + +/* @group Heading Levels */ + +h1.groupheader { + font-size: 150%; +} + +.title { + font: 400 14px/28px Roboto,sans-serif; + font-size: 175%; + font-weight: bold; + margin: 10px 2px; + color: #135384; +} + +h2.groupheader { + border-bottom: 1px solid #879ECB; + color: #354C7B; + font-size: 150%; + font-weight: normal; + margin-top: 1.75em; + padding-top: 8px; + padding-bottom: 4px; + width: 100%; +} + +h3.groupheader { + font-size: 100%; +} + +h1, h2, h3, h4, h5, h6 { + -webkit-transition: text-shadow 0.5s linear; + -moz-transition: text-shadow 0.5s linear; + -ms-transition: text-shadow 0.5s linear; + -o-transition: text-shadow 0.5s linear; + transition: text-shadow 0.5s linear; + margin-right: 15px; + color: #135384; + +} + +h1.glow, h2.glow, h3.glow, h4.glow, h5.glow, h6.glow { + text-shadow: 0 0 15px cyan; +} + +dt { + font-weight: bold; +} + +div.multicol { + -moz-column-gap: 1em; + -webkit-column-gap: 1em; + -moz-column-count: 3; + -webkit-column-count: 3; +} + +p.startli, p.startdd { + margin-top: 2px; +} + +p.starttd { + margin-top: 0px; +} + +p.endli { + margin-bottom: 0px; +} + +p.enddd { + margin-bottom: 4px; +} + +p.endtd { + margin-bottom: 2px; +} + +/* @end */ + +caption { + font-weight: bold; +} + +span.legend { + font-size: 70%; + text-align: center; +} + +h3.version { + font-size: 90%; + text-align: center; +} + +div.qindex, div.navtab{ + background-color: #EBEFF6; + border: 1px solid #A3B4D7; + text-align: center; +} + +div.qindex, div.navpath { + width: 100%; + line-height: 140%; +} + +div.navtab { + margin-right: 15px; +} + +/* @group Link Styling */ + +a { + color: #3D578C; + font-weight: normal; + text-decoration: none; +} + +.contents a:visited { + color: #4665A2; +} + +a:hover { + text-decoration: underline; +} + +a.qindex { + font-weight: bold; +} + +a.qindexHL { + font-weight: bold; + background-color: #9CAFD4; + color: #ffffff; + border: 1px double #869DCA; +} + +.contents a.qindexHL:visited { + color: #ffffff; +} + +a.el { + font-weight: bold; +} + +a.elRef { +} + +a.code, a.code:visited, a.line, a.line:visited { + color: #4665A2; +} + +a.codeRef, a.codeRef:visited, a.lineRef, a.lineRef:visited { + color: #4665A2; +} + +/* @end */ + +dl.el { + margin-left: -1cm; +} + +pre.fragment { + border: 1px solid #C4CFE5; + background-color: #FBFCFD; + padding: 4px 6px; + margin: 4px 8px 4px 2px; + overflow: auto; + word-wrap: break-word; + font-size: 9pt; + line-height: 125%; + font-family: monospace, fixed; + font-size: 105%; +} + +div.fragment { + padding: 0px; + margin: 4px 8px 4px 2px; + background-color: #FBFCFD; + border: 1px solid #C4CFE5; +} + +div.line { + font-family: monospace, fixed; + font-size: 13px; + min-height: 13px; + line-height: 1.0; + text-wrap: unrestricted; + white-space: -moz-pre-wrap; /* Moz */ + white-space: -pre-wrap; /* Opera 4-6 */ + white-space: -o-pre-wrap; /* Opera 7 */ + white-space: pre-wrap; /* CSS3 */ + word-wrap: break-word; /* IE 5.5+ */ + text-indent: -53px; + padding-left: 53px; + padding-bottom: 0px; + margin: 0px; + -webkit-transition-property: background-color, box-shadow; + -webkit-transition-duration: 0.5s; + -moz-transition-property: background-color, box-shadow; + -moz-transition-duration: 0.5s; + -ms-transition-property: background-color, box-shadow; + -ms-transition-duration: 0.5s; + -o-transition-property: background-color, box-shadow; + -o-transition-duration: 0.5s; + transition-property: background-color, box-shadow; + transition-duration: 0.5s; +} + +div.line:after { + content:"\000A"; + white-space: pre; +} + +div.line.glow { + background-color: cyan; + box-shadow: 0 0 10px cyan; +} + + +span.lineno { + padding-right: 4px; + text-align: right; + border-right: 2px solid #0F0; + background-color: #E8E8E8; + white-space: pre; +} +span.lineno a { + background-color: #D8D8D8; +} + +span.lineno a:hover { + background-color: #C8C8C8; +} + +.lineno { + -webkit-touch-callout: none; + -webkit-user-select: none; + -khtml-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + user-select: none; +} + +div.ah, span.ah { + background-color: black; + font-weight: bold; + color: #ffffff; + margin-bottom: 3px; + margin-top: 3px; + padding: 0.2em; + border: solid thin #333; + border-radius: 0.5em; + -webkit-border-radius: .5em; + -moz-border-radius: .5em; + box-shadow: 2px 2px 3px #999; + -webkit-box-shadow: 2px 2px 3px #999; + -moz-box-shadow: rgba(0, 0, 0, 0.15) 2px 2px 2px; + background-image: -webkit-gradient(linear, left top, left bottom, from(#eee), to(#000),color-stop(0.3, #444)); + background-image: -moz-linear-gradient(center top, #eee 0%, #444 40%, #000 110%); +} + +div.classindex ul { + list-style: none; + padding-left: 0; +} + +div.classindex span.ai { + display: inline-block; +} + +div.groupHeader { + margin-left: 16px; + margin-top: 12px; + font-weight: bold; +} + +div.groupText { + margin-left: 16px; + font-style: italic; +} + +body { + background-color: white; + color: black; + margin: 0; +} + +div.contents { + margin-top: 10px; + margin-left: 12px; + margin-right: 8px; +} + +td.indexkey { + background-color: #EBEFF6; + font-weight: bold; + border: 1px solid #C4CFE5; + margin: 2px 0px 2px 0; + padding: 2px 10px; + white-space: nowrap; + vertical-align: top; +} + +td.indexvalue { + background-color: #EBEFF6; + border: 1px solid #C4CFE5; + padding: 2px 10px; + margin: 2px 0px; +} + +tr.memlist { + background-color: #EEF1F7; +} + +p.formulaDsp { + text-align: center; +} + +img.formulaDsp { + +} + +img.formulaInl { + vertical-align: middle; +} + +div.center { + text-align: center; + margin-top: 0px; + margin-bottom: 0px; + padding: 0px; +} + +div.center img { + border: 0px; +} + +address.footer { + text-align: right; + padding-right: 12px; +} + +img.footer { + border: 0px; + vertical-align: middle; +} + +/* @group Code Colorization */ + +span.keyword { + color: #008000 +} + +span.keywordtype { + color: #604020 +} + +span.keywordflow { + color: #e08000 +} + +span.comment { + color: #800000 +} + +span.preprocessor { + color: #806020 +} + +span.stringliteral { + color: #002080 +} + +span.charliteral { + color: #008080 +} + +span.vhdldigit { + color: #ff00ff +} + +span.vhdlchar { + color: #000000 +} + +span.vhdlkeyword { + color: #700070 +} + +span.vhdllogic { + color: #ff0000 +} + +blockquote { + background-color: #F7F8FB; + border-left: 2px solid #9CAFD4; + margin: 0 24px 0 4px; + padding: 0 12px 0 16px; +} + +/* @end */ + +/* +.search { + color: #003399; + font-weight: bold; +} + +form.search { + margin-bottom: 0px; + margin-top: 0px; +} + +input.search { + font-size: 75%; + color: #000080; + font-weight: normal; + background-color: #e8eef2; +} +*/ + +td.tiny { + font-size: 75%; +} + +.dirtab { + padding: 4px; + border-collapse: collapse; + border: 1px solid #A3B4D7; +} + +th.dirtab { + background: #EBEFF6; + font-weight: bold; +} + +hr { + height: 0px; + border: none; + border-top: 1px solid #4A6AAA; +} + +hr.footer { + height: 1px; +} + +/* @group Member Descriptions */ + +table.memberdecls { + border-spacing: 0px; + padding: 0px; +} + +.memberdecls td, .fieldtable tr { + -webkit-transition-property: background-color, box-shadow; + -webkit-transition-duration: 0.5s; + -moz-transition-property: background-color, box-shadow; + -moz-transition-duration: 0.5s; + -ms-transition-property: background-color, box-shadow; + -ms-transition-duration: 0.5s; + -o-transition-property: background-color, box-shadow; + -o-transition-duration: 0.5s; + transition-property: background-color, box-shadow; + transition-duration: 0.5s; +} + +.memberdecls td.glow, .fieldtable tr.glow { + background-color: cyan; + box-shadow: 0 0 15px cyan; +} + +.mdescLeft, .mdescRight, +.memItemLeft, .memItemRight, +.memTemplItemLeft, .memTemplItemRight, .memTemplParams { + background-color: #F9FAFC; + border: none; + margin: 4px; + padding: 1px 0 0 8px; +} + +.mdescLeft, .mdescRight { + padding: 0px 8px 4px 8px; + color: #555; +} + +.memSeparator { + border-bottom: 1px solid #DEE4F0; + line-height: 1px; + margin: 0px; + padding: 0px; +} + +.memItemLeft, .memTemplItemLeft { + white-space: nowrap; +} + +.memItemRight { + width: 100%; +} + +.memTemplParams { + color: #4665A2; + white-space: nowrap; + font-size: 80%; +} + +/* @end */ + +/* @group Member Details */ + +/* Styles for detailed member documentation */ + +.memtitle { + padding: 8px; + border-top: 1px solid #A8B8D9; + border-left: 1px solid #A8B8D9; + border-right: 1px solid #A8B8D9; + border-top-right-radius: 4px; + border-top-left-radius: 4px; + margin-bottom: -1px; + background-image: url('nav_f.png'); + background-repeat: repeat-x; + background-color: #E2E8F2; + line-height: 1.25; + font-weight: 300; + float:left; +} + +.permalink +{ + font-size: 65%; + display: inline-block; + vertical-align: middle; +} + +.memtemplate { + font-size: 80%; + color: #4665A2; + font-weight: normal; + margin-left: 9px; +} + +.memnav { + background-color: #EBEFF6; + border: 1px solid #A3B4D7; + text-align: center; + margin: 2px; + margin-right: 15px; + padding: 2px; +} + +.mempage { + width: 100%; +} + +.memitem { + padding: 0; + margin-bottom: 10px; + margin-right: 5px; + -webkit-transition: box-shadow 0.5s linear; + -moz-transition: box-shadow 0.5s linear; + -ms-transition: box-shadow 0.5s linear; + -o-transition: box-shadow 0.5s linear; + transition: box-shadow 0.5s linear; + display: table !important; + width: 100%; +} + +.memitem.glow { + box-shadow: 0 0 15px cyan; +} + +.memname { + font-weight: 400; + margin-left: 6px; +} + +.memname td { + vertical-align: bottom; +} + +.memproto, dl.reflist dt { + border-top: 1px solid #A8B8D9; + border-left: 1px solid #A8B8D9; + border-right: 1px solid #A8B8D9; + padding: 6px 0px 6px 0px; + color: #253555; + font-weight: bold; + text-shadow: 0px 1px 1px rgba(255, 255, 255, 0.9); + background-color: #DFE5F1; + /* opera specific markup */ + box-shadow: 5px 5px 5px rgba(0, 0, 0, 0.15); + border-top-right-radius: 4px; + /* firefox specific markup */ + -moz-box-shadow: rgba(0, 0, 0, 0.15) 5px 5px 5px; + -moz-border-radius-topright: 4px; + /* webkit specific markup */ + -webkit-box-shadow: 5px 5px 5px rgba(0, 0, 0, 0.15); + -webkit-border-top-right-radius: 4px; + +} + +.overload { + font-family: "courier new",courier,monospace; + font-size: 65%; +} + +.memdoc, dl.reflist dd { + border-bottom: 1px solid #A8B8D9; + border-left: 1px solid #A8B8D9; + border-right: 1px solid #A8B8D9; + padding: 6px 10px 2px 10px; + background-color: #FBFCFD; + border-top-width: 0; + background-image:url('nav_g.png'); + background-repeat:repeat-x; + background-color: #FFFFFF; + /* opera specific markup */ + border-bottom-left-radius: 4px; + border-bottom-right-radius: 4px; + box-shadow: 5px 5px 5px rgba(0, 0, 0, 0.15); + /* firefox specific markup */ + -moz-border-radius-bottomleft: 4px; + -moz-border-radius-bottomright: 4px; + -moz-box-shadow: rgba(0, 0, 0, 0.15) 5px 5px 5px; + /* webkit specific markup */ + -webkit-border-bottom-left-radius: 4px; + -webkit-border-bottom-right-radius: 4px; + -webkit-box-shadow: 5px 5px 5px rgba(0, 0, 0, 0.15); +} + +dl.reflist dt { + padding: 5px; +} + +dl.reflist dd { + margin: 0px 0px 10px 0px; + padding: 5px; +} + +.paramkey { + text-align: right; +} + +.paramtype { + white-space: nowrap; +} + +.paramname { + color: #602020; + white-space: nowrap; +} +.paramname em { + font-style: normal; +} +.paramname code { + line-height: 14px; +} + +.params, .retval, .exception, .tparams { + margin-left: 0px; + padding-left: 0px; +} + +.params .paramname, .retval .paramname { + font-weight: bold; + vertical-align: top; +} + +.params .paramtype { + font-style: italic; + vertical-align: top; +} + +.params .paramdir { + font-family: "courier new",courier,monospace; + vertical-align: top; +} + +table.mlabels { + border-spacing: 0px; +} + +td.mlabels-left { + width: 100%; + padding: 0px; +} + +td.mlabels-right { + vertical-align: bottom; + padding: 0px; + white-space: nowrap; +} + +span.mlabels { + margin-left: 8px; +} + +span.mlabel { + background-color: #728DC1; + border-top:1px solid #5373B4; + border-left:1px solid #5373B4; + border-right:1px solid #C4CFE5; + border-bottom:1px solid #C4CFE5; + text-shadow: none; + color: white; + margin-right: 4px; + padding: 2px 3px; + border-radius: 3px; + font-size: 7pt; + white-space: nowrap; + vertical-align: middle; +} + + + +/* @end */ + +/* these are for tree view inside a (index) page */ + +div.directory { + margin: 10px 0px; + border-top: 1px solid #9CAFD4; + border-bottom: 1px solid #9CAFD4; + width: 100%; +} + +.directory table { + border-collapse:collapse; +} + +.directory td { + margin: 0px; + padding: 0px; + vertical-align: top; +} + +.directory td.entry { + white-space: nowrap; + padding-right: 6px; + padding-top: 3px; +} + +.directory td.entry a { + outline:none; +} + +.directory td.entry a img { + border: none; +} + +.directory td.desc { + width: 100%; + padding-left: 6px; + padding-right: 6px; + padding-top: 3px; + border-left: 1px solid rgba(0,0,0,0.05); +} + +.directory tr.even { + padding-left: 6px; + background-color: #F7F8FB; +} + +.directory img { + vertical-align: -30%; +} + +.directory .levels { + white-space: nowrap; + width: 100%; + text-align: right; + font-size: 9pt; +} + +.directory .levels span { + cursor: pointer; + padding-left: 2px; + padding-right: 2px; + color: #3D578C; +} + +.arrow { + color: #9CAFD4; + -webkit-user-select: none; + -khtml-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + user-select: none; + cursor: pointer; + font-size: 80%; + display: inline-block; + width: 16px; + height: 22px; +} + +.icon { + font-family: Arial, Helvetica; + font-weight: bold; + font-size: 12px; + height: 14px; + width: 16px; + display: inline-block; + background-color: #728DC1; + color: white; + text-align: center; + border-radius: 4px; + margin-left: 2px; + margin-right: 2px; +} + +.icona { + width: 24px; + height: 22px; + display: inline-block; +} + +.iconfopen { + width: 24px; + height: 18px; + margin-bottom: 4px; + background-image:url('folderopen.png'); + background-position: 0px -4px; + background-repeat: repeat-y; + vertical-align:top; + display: inline-block; +} + +.iconfclosed { + width: 24px; + height: 18px; + margin-bottom: 4px; + background-image:url('folderclosed.png'); + background-position: 0px -4px; + background-repeat: repeat-y; + vertical-align:top; + display: inline-block; +} + +.icondoc { + width: 24px; + height: 18px; + margin-bottom: 4px; + background-image:url('doc.png'); + background-position: 0px -4px; + background-repeat: repeat-y; + vertical-align:top; + display: inline-block; +} + +table.directory { + font: 400 14px Roboto,sans-serif; +} + +/* @end */ + +div.dynheader { + margin-top: 8px; + -webkit-touch-callout: none; + -webkit-user-select: none; + -khtml-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + user-select: none; +} + +address { + font-style: normal; + color: #2A3D61; +} + +table.doxtable caption { + caption-side: top; +} + +table.doxtable { + border-collapse:collapse; + margin-top: 4px; + margin-bottom: 4px; +} + +table.doxtable td, table.doxtable th { + border: 1px solid #2D4068; + padding: 3px 7px 2px; +} + +table.doxtable th { + background-color: #374F7F; + color: #FFFFFF; + font-size: 110%; + padding-bottom: 4px; + padding-top: 5px; +} + +table.fieldtable { + /*width: 100%;*/ + margin-bottom: 10px; + border: 1px solid #A8B8D9; + border-spacing: 0px; + -moz-border-radius: 4px; + -webkit-border-radius: 4px; + border-radius: 4px; + -moz-box-shadow: rgba(0, 0, 0, 0.15) 2px 2px 2px; + -webkit-box-shadow: 2px 2px 2px rgba(0, 0, 0, 0.15); + box-shadow: 2px 2px 2px rgba(0, 0, 0, 0.15); +} + +.fieldtable td, .fieldtable th { + padding: 3px 7px 2px; +} + +.fieldtable td.fieldtype, .fieldtable td.fieldname { + white-space: nowrap; + border-right: 1px solid #A8B8D9; + border-bottom: 1px solid #A8B8D9; + vertical-align: top; +} + +.fieldtable td.fieldname { + padding-top: 3px; +} + +.fieldtable td.fielddoc { + border-bottom: 1px solid #A8B8D9; + /*width: 100%;*/ +} + +.fieldtable td.fielddoc p:first-child { + margin-top: 0px; +} + +.fieldtable td.fielddoc p:last-child { + margin-bottom: 2px; +} + +.fieldtable tr:last-child td { + border-bottom: none; +} + +.fieldtable th { + background-image:url('nav_f.png'); + background-repeat:repeat-x; + background-color: #E2E8F2; + font-size: 90%; + color: #253555; + padding-bottom: 4px; + padding-top: 5px; + text-align:left; + font-weight: 400; + -moz-border-radius-topleft: 4px; + -moz-border-radius-topright: 4px; + -webkit-border-top-left-radius: 4px; + -webkit-border-top-right-radius: 4px; + border-top-left-radius: 4px; + border-top-right-radius: 4px; + border-bottom: 1px solid #A8B8D9; +} + + +.tabsearch { + top: 0px; + left: 10px; + height: 36px; + background-image: url('tab_b.png'); + z-index: 101; + overflow: hidden; + font-size: 13px; +} + +.navpath ul { + display: flex; + flex-flow: row wrap; + justify-content: flex-start; + align-items: center; + font-size: 11px; + background-image:none; + background-repeat:repeat-x; + background-position: 0 -5px; + height:auto; + line-height:30px; + color:#8AA0CC; + border:solid 1px #C2CDE4; + overflow:hidden; + margin:0px; + padding:0px; +} + +.navpath li +{ + list-style-type:none; + float:left; + padding-left:10px; + padding-right:15px; + background-image:url('bc_s.png'); + background-repeat:no-repeat; + background-position:right; + color:#364D7C; +} + +.navpath li.navelem a +{ + height:32px; + display:block; + text-decoration: none; + outline: none; + color: #283A5D; + font-family: 'Lucida Grande',Geneva,Helvetica,Arial,sans-serif; + text-shadow: 0px 1px 1px rgba(255, 255, 255, 0.9); + text-decoration: none; +} + +.navpath li.navelem a:hover +{ + color:#6884BD; +} + +.navpath li.footer +{ + display: flex; + flex-flow: row wrap; + justify-content: flex-start; + align-items: center; + flex-grow: 1; + list-style-type:none; + float:none; + padding-left:10px; + padding-right:15px; + background-image:none; + background-repeat:no-repeat; + background-position:right; + color:#364D7C; + font-size: 8pt; +} + +div.summary +{ + float: right; + font-size: 8pt; + padding-right: 5px; + width: 50%; + text-align: right; +} + +div.summary a +{ + white-space: nowrap; +} + +table.classindex +{ + margin: 10px; + white-space: nowrap; + margin-left: 3%; + margin-right: 3%; + width: 94%; + border: 0; + border-spacing: 0; + padding: 0; +} + +div.ingroups +{ + font-size: 8pt; + width: 50%; + text-align: left; +} + +div.ingroups a +{ + white-space: nowrap; +} + +div.header +{ + background-image:url('nav_h.png'); + background-repeat:repeat-x; + background-color: #F9FAFC; + margin: 0px; + border-bottom: 1px solid #C4CFE5; +} + +div.headertitle +{ + padding: 5px 5px 5px 10px; + color: #135384; +} + +dl +{ + padding: 0 0 0 10px; +} + +/* dl.note, dl.warning, dl.attention, dl.pre, dl.post, dl.invariant, dl.deprecated, dl.todo, dl.test, dl.bug */ +dl.section +{ + margin-left: 0px; + padding-left: 0px; +} + +dl.note +{ + margin-left:-7px; + padding-left: 3px; + border-left:4px solid; + border-color: #D0C000; +} + +dl.warning, dl.attention +{ + margin-left:-7px; + padding-left: 3px; + border-left:4px solid; + border-color: #FF0000; +} + +dl.pre, dl.post, dl.invariant +{ + margin-left:-7px; + padding-left: 3px; + border-left:4px solid; + border-color: #00D000; +} + +dl.deprecated +{ + margin-left:-7px; + padding-left: 3px; + border-left:4px solid; + border-color: #505050; +} + +dl.todo +{ + margin-left:-7px; + padding-left: 3px; + border-left:4px solid; + border-color: #00C0E0; +} + +dl.test +{ + margin-left:-7px; + padding-left: 3px; + border-left:4px solid; + border-color: #3030E0; +} + +dl.bug +{ + margin-left:-7px; + padding-left: 3px; + border-left:4px solid; + border-color: #C08050; +} + +dl.section dd { + margin-bottom: 6px; +} + + +#projectlogo +{ + text-align: center; + vertical-align: bottom; + border-collapse: separate; +} + +#projectlogo img +{ + border: 0px none; +} + +#projectalign +{ + vertical-align: middle; +} + +#projectname +{ + font: 200% Tahoma, Arial,sans-serif; + margin: 0px; + padding: 2px 0px; +} + +#projectbrief +{ + font: 120% Tahoma, Arial,sans-serif; + margin: 0px; + padding: 0px; +} + +#projectnumber +{ + font: 50% Tahoma, Arial,sans-serif; + margin: 0px; + padding: 0px; +} + +#top { + border-bottom: 1px solid #5373B4; +} + +#titlearea +{ + flex-grow: 1; + padding: 0px; + margin: 0px; + width: auto; + border-bottom: none; +} + +#main-nav { +} + +#main-menu { + display: flex; + flex-flow: row wrap; + justify-content: flex-start; + align-items: center; + background-image: none; + min-width: 770px; +} + +.ui-resizable-e { + height: 100%; + background-repeat: repeat-y; +} + +.image +{ + text-align: center; +} + +.dotgraph +{ + text-align: center; +} + +.mscgraph +{ + text-align: center; +} + +.plantumlgraph +{ + text-align: center; +} + +.diagraph +{ + text-align: center; +} + +.caption +{ + font-weight: bold; +} + +div.zoom +{ + border: 1px solid #90A5CE; +} + +dl.citelist { + margin-bottom:50px; +} + +dl.citelist dt { + color:#334975; + float:left; + font-weight:bold; + margin-right:10px; + padding:5px; +} + +dl.citelist dd { + margin:2px 0; + padding:5px 0; +} + +div.toc { + padding: 14px 25px; + background-color: #F4F6FA; + border: 1px solid #D8DFEE; + border-radius: 7px 7px 7px 7px; + float: right; + height: auto; + margin: 0 8px 10px 10px; + width: 200px; +} + +div.toc li { + background: url("bdwn.png") no-repeat scroll 0 5px transparent; + font: 10px/1.2 Verdana,DejaVu Sans,Geneva,sans-serif; + margin-top: 5px; + padding-left: 10px; + padding-top: 2px; +} + +div.toc h3 { + font: bold 12px/1.2 Arial,FreeSans,sans-serif; + color: #4665A2; + border-bottom: 0 none; + margin: 0; +} + +div.toc ul { + list-style: none outside none; + border: medium none; + padding: 0px; +} + +div.toc li.level1 { + margin-left: 0px; +} + +div.toc li.level2 { + margin-left: 15px; +} + +div.toc li.level3 { + margin-left: 30px; +} + +div.toc li.level4 { + margin-left: 45px; +} + +.inherit_header { + font-weight: bold; + color: gray; + cursor: pointer; + -webkit-touch-callout: none; + -webkit-user-select: none; + -khtml-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + user-select: none; +} + +.inherit_header td { + padding: 6px 0px 2px 5px; +} + +.inherit { + display: none; +} + +tr.heading h2 { + margin-top: 12px; + margin-bottom: 4px; +} + +/* tooltip related style info */ + +.ttc { + position: absolute; + display: none; +} + +#powerTip { + cursor: default; + white-space: nowrap; + background-color: white; + border: 1px solid gray; + border-radius: 4px 4px 4px 4px; + box-shadow: 1px 1px 7px gray; + display: none; + font-size: smaller; + max-width: 80%; + opacity: 0.9; + padding: 1ex 1em 1em; + position: absolute; + z-index: 2147483647; +} + +#powerTip div.ttdoc { + color: grey; + font-style: italic; +} + +#powerTip div.ttname a { + font-weight: bold; +} + +#powerTip div.ttname { + font-weight: bold; +} + +#powerTip div.ttdeci { + color: #006318; +} + +#powerTip div { + margin: 0px; + padding: 0px; + font: 12px/16px Roboto,sans-serif; +} + +#powerTip:before, #powerTip:after { + content: ""; + position: absolute; + margin: 0px; +} + +#powerTip.n:after, #powerTip.n:before, +#powerTip.s:after, #powerTip.s:before, +#powerTip.w:after, #powerTip.w:before, +#powerTip.e:after, #powerTip.e:before, +#powerTip.ne:after, #powerTip.ne:before, +#powerTip.se:after, #powerTip.se:before, +#powerTip.nw:after, #powerTip.nw:before, +#powerTip.sw:after, #powerTip.sw:before { + border: solid transparent; + content: " "; + height: 0; + width: 0; + position: absolute; +} + +#powerTip.n:after, #powerTip.s:after, +#powerTip.w:after, #powerTip.e:after, +#powerTip.nw:after, #powerTip.ne:after, +#powerTip.sw:after, #powerTip.se:after { + border-color: rgba(255, 255, 255, 0); +} + +#powerTip.n:before, #powerTip.s:before, +#powerTip.w:before, #powerTip.e:before, +#powerTip.nw:before, #powerTip.ne:before, +#powerTip.sw:before, #powerTip.se:before { + border-color: rgba(128, 128, 128, 0); +} + +#powerTip.n:after, #powerTip.n:before, +#powerTip.ne:after, #powerTip.ne:before, +#powerTip.nw:after, #powerTip.nw:before { + top: 100%; +} + +#powerTip.n:after, #powerTip.ne:after, #powerTip.nw:after { + border-top-color: #ffffff; + border-width: 10px; + margin: 0px -10px; +} +#powerTip.n:before { + border-top-color: #808080; + border-width: 11px; + margin: 0px -11px; +} +#powerTip.n:after, #powerTip.n:before { + left: 50%; +} + +#powerTip.nw:after, #powerTip.nw:before { + right: 14px; +} + +#powerTip.ne:after, #powerTip.ne:before { + left: 14px; +} + +#powerTip.s:after, #powerTip.s:before, +#powerTip.se:after, #powerTip.se:before, +#powerTip.sw:after, #powerTip.sw:before { + bottom: 100%; +} + +#powerTip.s:after, #powerTip.se:after, #powerTip.sw:after { + border-bottom-color: #ffffff; + border-width: 10px; + margin: 0px -10px; +} + +#powerTip.s:before, #powerTip.se:before, #powerTip.sw:before { + border-bottom-color: #808080; + border-width: 11px; + margin: 0px -11px; +} + +#powerTip.s:after, #powerTip.s:before { + left: 50%; +} + +#powerTip.sw:after, #powerTip.sw:before { + right: 14px; +} + +#powerTip.se:after, #powerTip.se:before { + left: 14px; +} + +#powerTip.e:after, #powerTip.e:before { + left: 100%; +} +#powerTip.e:after { + border-left-color: #ffffff; + border-width: 10px; + top: 50%; + margin-top: -10px; +} +#powerTip.e:before { + border-left-color: #808080; + border-width: 11px; + top: 50%; + margin-top: -11px; +} + +#powerTip.w:after, #powerTip.w:before { + right: 100%; +} +#powerTip.w:after { + border-right-color: #ffffff; + border-width: 10px; + top: 50%; + margin-top: -10px; +} +#powerTip.w:before { + border-right-color: #808080; + border-width: 11px; + top: 50%; + margin-top: -11px; +} + +@media print +{ + #top { display: none; } + #side-nav { display: none; } + #nav-path { display: none; } + body { overflow:visible; } + h1, h2, h3, h4, h5, h6 { page-break-after: avoid; } + .summary { display: none; } + .memitem { page-break-inside: avoid; } + #doc-content + { + margin-left:0 !important; + height:auto !important; + width:auto !important; + overflow:inherit; + display:inline; + } +} + +/* @group Markdown */ + +/* +table.markdownTable { + border-collapse:collapse; + margin-top: 4px; + margin-bottom: 4px; +} + +table.markdownTable td, table.markdownTable th { + border: 1px solid #2D4068; + padding: 3px 7px 2px; +} + +table.markdownTableHead tr { +} + +table.markdownTableBodyLeft td, table.markdownTable th { + border: 1px solid #2D4068; + padding: 3px 7px 2px; +} + +th.markdownTableHeadLeft th.markdownTableHeadRight th.markdownTableHeadCenter th.markdownTableHeadNone { + background-color: #374F7F; + color: #FFFFFF; + font-size: 110%; + padding-bottom: 4px; + padding-top: 5px; +} + +th.markdownTableHeadLeft { + text-align: left +} + +th.markdownTableHeadRight { + text-align: right +} + +th.markdownTableHeadCenter { + text-align: center +} +*/ + +table.markdownTable { + border-collapse:collapse; + margin-top: 4px; + margin-bottom: 4px; +} + +table.markdownTable td, table.markdownTable th { + border: 1px solid #2D4068; + padding: 3px 7px 2px; +} + +table.markdownTable tr { +} + +th.markdownTableHeadLeft, th.markdownTableHeadRight, th.markdownTableHeadCenter, th.markdownTableHeadNone { + background-color: #374F7F; + color: #FFFFFF; + font-size: 110%; + padding-bottom: 4px; + padding-top: 5px; +} + +th.markdownTableHeadLeft, td.markdownTableBodyLeft { + text-align: left +} + +th.markdownTableHeadRight, td.markdownTableBodyRight { + text-align: right +} + +th.markdownTableHeadCenter, td.markdownTableBodyCenter { + text-align: center +} + + +/* @end */ diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/loadgen_integration_diagram.dia b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/loadgen_integration_diagram.dia new file mode 100644 index 0000000000000000000000000000000000000000..569089f243e4584e12134caf36d078248cb50af1 GIT binary patch literal 1943 zcmV;I2Wa>oiwFP!000021MOW~kJ>mCexF|v(Z^*%?A+VVRH|0H`_NS@ZTA@&;(=^K z%qE@-FZ1`E8Yu$2geet%`O_5|KoELebsi z%|r2;%nC!ZlBE;Yw`heI2}a-AjT`Lc``wIhaZ)wB*^)G5P3Uf0Ytmwe|2=9`v`Sf{ zcy<5g6Q&d=Z}M&x2M_j|I@{`qZcwrcrY`CE+X92`!J@1ncod# zzB&Ukoj9D{bH?S?T7@X^u#N35LbQ4e1Du|j%;h#MmexhH*}3apZP)YC1Yx?3(C+jQ zs*PG~p_qn@!%&I?u}m4G?JXs@6`By|h%ElWOF9sHw)t9<=huvA2scX-$810>6usRN z2G?|$tN7Zfm>;S{shL+c$#7Ei^y48u)e5#LdZHEmM@NSc8_v`I-O+r{zq(=`{}z$w zE)yKktB{8}`&cl=fv|WiT-zl}Lq(^^}07u#{+OG`2 zDW)xX(g}WQi&RNWoBZ)x`@4$)=r0D2Xfd!4V&DlE14oI05`#uD_>y4lOG1`7O-Z7pK)e*VN(y930qiXW zmQXE#=65kLffO;YM5=+K`=yr$l{}1E9!N@+KFHGt=<2qT2)Pnr_LT@=PJzgD0dk2j zxJ1BAB7DIT(S1BWI$`jXB8XOmk2qegHz@h0#6Ywd0G9=ZN(v-Ofi*z30EA}?#vxm< z#b*lwrN9*^1(uQm;ZndZ6OIEV1X8jBwTB>83hW`W0`X}9ILZrvDIqZcYAZp>L37zH zLBa1hR&pRAFJRzTYA~1@{Ka@RKuN&F;8Pgm1~qVctAQ~!1&V>swE*gl>A`oI46ty) zbU;T&SdKxc7{HduXN6dUXQ`jxEeq2nl>pA{;VK0 z%OC`>HUkJ@zRyhlGlHa zx7dUg$xC}Dd~F*-#!n+!+qM)hNxT2FVJ<3S8YeLgQUfaVQOUd%Ghf}#?{ibj+iH2c zKJOI&NamjbD{X-zsl)&pLp;z$B-=qt=XZzb4s^_S(AN#Xb3+!2xu!RoTfVhVWR)rg z6ds0Ua|P-U0#gy8Fe2;-_u7wJU0lO8Olcz=qP$ov=Vi5xob(RGjhynNR2Vnhs5K1T ztNO-~+~?!T3O?WvYd~2tBzg^KnpLmmYe3oJG#JUInWGkgWG(_#C!>lzM!5(C`&VAu zJQQ_xp_^uz4@dWn>~{XlpQ9FoR4UXw{}dOElZ$#6TT7s#BPLQrq=@L`+C7LLNhg22mH|rR`$qOSNHg%6&&$TR;UiFrG=x`O|EO*#A<)W$@v?*Ve7+H dUb*O!zt4VH`2ME!%ft6K{{zrB$(5Fa002n3teyY> literal 0 HcmV?d00001 diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/mlperf_icon.png b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/mlperf_icon.png new file mode 100644 index 0000000000000000000000000000000000000000..95321896d3e467b923909c3654a4260346df5b9f GIT binary patch literal 4632 zcmb_gXHXMNyG2n1{1g!clqO9CBp9Uz2+B(h0@6F7BQRAaZ%{ulwU0f8Ck0`^@Z~bDr7Rvwt?}v4Qp_)*Gx03=Efabu^4m z>+!#Xndx*kspxh;E!TrIErU#8Zb6}r0j>-WU0_bGH+6j+-Cd1c9bFLq-L5JO4D484 zjfbZ2@zs3&08VYLr}rz)yNSCgFmgQeeHw9Q1eeE<(8)3h7k6`{og2>kvKC*(FKO^J z5eV$fjChu2;2`1j#U2~VGnny)Xm)IQvn=Xf#l6~A8TAK>pG-{`EH+$Q!`>Q8h%vkq z6rDWF5O?A4cmQYkJN~~7kgq0V{Ag5f`8x#@9Wx!a1SkQI%6R2@OxAA2Vq7bSB2YpbCM3ct@U0s2<-Lx1%cM>r zwzF{wFH}(Yx}S{v)uUm5#9$ZXMUlI5f;lo0Tt}hWMgM@!;u4VO6+CD^V}JOla{PpX zwtMN%ELZ8yvLK3y_CZ1Mq8jX$wO>9fYz*a|&=U~pAT!MNR7NXwpXU;@VAWjMJFl;u zZgEB}y1V3IwpP_S?E_{}g=C*@p6=dW8BKXt5_0qFCW`32_A=9!RcRfH^_5v8v^_HH z6vnCDp!d@sz)?jsi^)qBDn`P35;UO}ZWR<=WsEUExnU9_G^X4ym6;+U@Tq-eYo%Ir*L*Cs;M!{r^~um8-8JDSIRo*AwoK6t1Vi(M%Ozy zBCY{!XrUFtH8)1_m1`6%sIirN;1oQ!ZTJMXVJ;k$eJ*Rc^OBh+uk<@e5q;gredU zu@fU{oHe`eOw{cbgxuZjF^pQ0+j#9FCowRQgP%C+d0Rbs9;iQacd*Uf$-h$Vl{2r@ zcbZ#{lQetm=SQV-O)3RFd}f`D)7uwH;JaEbCX~&GKh{@f@`y=hZ#}*{5*9w;R4YCz z_IT=GPL!{=z^Ys?%aDSb)(&qs8gx@Rs((?N`g7adRCvmd8!N!;@6`w>!W?Nam1=~e zwjYUwN2R<@z3TCU_bz8{bV_&t=hzz){?UIB4XuRsBcX7<*Hpu(TCp)FzVRq3?iz`a zi1J_=0_P{0=OGowqL1&*?b=D*r#saFS+-`vB_gUHP{{zM{UxNk_JlnP`PnL{uy?Ih5N7)PST#s)OaJ(n zeC_JXP1;ILY)2+nYUYlLa9AUI+smA=K}=e~vWL3O><4blSZlNb3AbA93!7x>RLxx+ z@EUiV9jZY`HN@v&5sgda*1j$@;ten{IOWPvXY9z{?<6`y>_@bNv>;{BTA$8wv8yVG zsEHa{ps&tAKZ>X`!>oX9YN0$0_9oF_^IiPj?|_+&Lf|(41=CD(yi^4 zZP_sp%min08u!cIQK!|gq%3yQv~WIQf_pCsWb=wx{%(@*R*fBa2!JX!~o9gBOw(P@8bM;c45q;pU& zyzy>t@0KgISap7uY9^-KwfQHc4TwV0^7{K$P;XDhO!wPTL&to>BG$|bCwUb^**%way6-jA@9mHwhlASDol51mA~L2Rg6yf4`zTDJE3Mbl*IPAP2Rr*~gbH)f1**^toGpVIi5 z+!L`Af3>(Pp&;7OTh99}aJ_?lG#|YyMzU59X?(@zwJfG}RFs8mfZ(V2*UCrWqu|B# z)^9d?KQ1qzHw4E<^!$v?_l0S}Tem2l1#*e~Tw5T$rJXTdDcG=3_`qGc;2ntfd7`GV zP)36t5cFm|+9ZW;uz0t90v-ckqfcl(i*oQ>W|UYCnp7x&)sC8RIE^A@-L zL%rY;exe>M6)NO3jX%~Yx&OwkCRzJEKij#{M%rF~qlP~2n?pCGWE-u7UMs50!T!c= z(g3KJzRR_bk<2ky3z5;53NHbWB)h=~?L7G$=a}T}D*SXZfiri~bZ4lYM|$QF87fYw z|GZc)FXdCE^xZfe<|8?19oHtw^s=8{VJC!ZX=c_(-|lv@GKbXX$h=Iu(pn=Rt9VZX z*K}^5#iOtgBqH22J10yyOKQ~Iu&|26S2Xl-Lo?T?HG`S#?P_0;9ohF-w6N5EWF(&M zT;e6$b>PQkAO7z)W_W828wsL3J!4E$d}3`P!%iw`Tl3}5x~P77p5uxpY0G=QB0b6* znH4p|3nj(~Zsln8=9V=RZeOUUQdm>7*&+WP%Iyf$@v0#bYv_8pdNayUSvq{m)YU*z z5Ip9&oOQ{?to3XJa~eU7m{Jz?>|On=xgIJMSJ*Om$C zRp`2-5k8!fI=KXB=Wa$+nxFjVeyZO-U+wMCov;0Mx+v^6WOu-CE+T z23%sqZSSP3Zz^dWyi<=R(Re){%5~K(JkFAcoKJRjB|%9(0%a{%7OaTQg=fW6n4vQH zS>mfsv+jrXve@Gp-Sj;7a%-Zb8p^KV9Q-K%!(I{{)~Ir(&J$!9L`?yBE{ies>^9Wd z2KHT#vE?nFHC}0#5~=`aP%xcNY@NrtWSGh?q7>JBgrqaSnX(c^wZthc$DeE5-Ke8D zBwk@2dE;}x>Nu4CZaC-AmCUNdVMO(aV#1V{m5P1DQ7+% ztVB6}^0wGm6dJ!zoHHQizP2=2s_0jmV<~Q2-~51%T1v-20{aki$?ZY_5;4DPqM*Y9 zt0=)4NKL&hk5t+BB4in+q_N_GX{ico@iGSADlJocjYqt=Yj3;nM-gtRUK40YqQtz2 zOV2r+!1Y{I0CkFDm37F5^PhF$JHu*5O-g8cYndQx zLGxbkb<055Rb}p?F-cNDT*HqEt*kGu+>mlzJcWn-=%8XfdqOFqSo1t(gUjTA3hS9- zJw^rWuFNJ>wYc#w9@N(Oxr-HBWB|1I-QvNyL>ox9Y0n9X6Oc~IQIcA=vv#B+TsMcc!% znDl&;FjyVGND3qNNc878JAHrdRfXPdF}6Xfy&ib9+-!DJBdYnv~^k>f3=bK+-uQw!8)~jT@z9|dj3WiR3LJTjpYK_1B;V8>q z{+Yt_y@X|sUw|lpltZmJh;x4$KQ;Xg}D=}RWz`iS@@GRQG=^d8E z8@v#NQJ}lZ)#T&coew~O<9MBM&sqIA|EuIAm4G3?%li|A|cTI7STR z9vwx5{9LBvly87=ZRWx)S8Lka)t_(HFYH35xPA2?;uQB?&HdWIFx6H&GP#L%9Y9$% zh?60m#qK`O4*Im5CkW`OB(2DXula*s9BE|!R;M+wU*FG z^%k=-em^e{xYq`YT@~5!>(n3YgvRDi)Yn1ky!QbRxj2 z78AZZTw~?#dDhiTFx@zc8Fnz>U0;8tTjOM_;=W;Z8N!B{IMJD>=bl + + + + + + + + + + + + + + + + + + + + diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/early_stopping.cc b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/early_stopping.cc new file mode 100644 index 000000000..41f74b803 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/early_stopping.cc @@ -0,0 +1,117 @@ +#include "early_stopping.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace mlperf { + +namespace loadgen { + +double lbeta(int64_t x, int64_t y) { + return std::lgamma(x) + std::lgamma(y) - std::lgamma(x + y); +} + +// The Gaussian Hypergeometric function specialized for a = 1. +// Based on http://dlmf.nist.gov/15.2.E1. +// Converges if c > 0 and (b <= 0 or x < 1). +// TODO(ckstanton): http://dlmf.nist.gov/15.2.E1 says there are transformations +// to replace x with with a value less than 0.5, for faster convergence. +// Presently, this function can take up to 200,000 iterations to converge. +double hypergeometric_2F1_A1(int64_t b, int64_t c, double x) { + // TODO(ckstanton): Is there a more principled way to pick kTolerance? + constexpr double kTolerance = 1.0 / (1LL << 33); + double term = 1.0; + double result = 1.0; + for (int64_t i = 0; std::abs(term) > kTolerance; ++i) { + term *= (b + i) * x / (c + i); + result += term; + } + return result; +} + +// BetaRegularized[x, a, b] = +// Beta[x, a, b]/Beta[a, b] = +// x^a/a Hypergeometric2F1[a, 1-b, 1+a, x]/Beta[a, b] = +// (http://dlmf.nist.gov/15.8.E1.) +// x^a/a (1-x)^(b-1) Hypergeometric2F1[1, 1-b, 1+a, x/(x-1)]/Beta[a, b] +double beta_regularized(double x, int64_t a, int64_t b) { + return std::exp(a * std::log(x) + (b - 1) * std::log(1 - x) - lbeta(a, b)) / + a * hypergeometric_2F1_A1(1 - b, 1 + a, x / (x - 1)); +} + +// Compute the odds of t or fewer overlatency queries in h + t total queries. +// The binomial distribution is the discrete probability distribution for +// independent boolean experiments. The CDF of the binomial distribution is: +// BetaRegularized[q, n - k, 1 + k] where 1 - q is the probability of an event +// per experiment, n is the total number of experiments, and k is the number of +// events. An even in our case is an overlatency query, so q = p - d, n = h + t, +// and k = t. +// Sum[Binomial[h + t, x] (p - d)^(h + t - x) (1 - p + d)^x, {x, 0, t}] = +// BetaRegularized[p - d, h, 1 + t] +double odds(int64_t h, int64_t t, double p, double d) { + return beta_regularized(p - d, h, 1 + t); +} + +// Binary search to find the minimum value h such that: +// odds(h, t, p, d) <= 1 - c on the range [min_h, max_h] given t, p, d, and c. +int64_t find_min_passing(int64_t min_h, int64_t max_h, int64_t t, double p, + double d, double c) { + int64_t count = max_h - min_h; + while (count > 0) { + int64_t step = count / 2; + int64_t h = min_h + step; + double prob = odds(h, t, p, d); + if (prob < 1 - c) { + count = step; + } else { + min_h = h + 1; + count -= step + 1; + } + } + return min_h; +} + +int64_t MinPassingQueriesFinder::operator()(int64_t t, double p, double d, + double c) { + // Given t, p, d, and c, return the minimum h such that odds(h, t, p, d) <= 1 + // - c + + auto &cache = caches_[std::make_tuple(p, d, c)]; + auto it = cache.lower_bound(t); + if (it != cache.end() && it->first == t) { + return it->second; + } + + int64_t x0 = -1; + int64_t y0 = 0; + int64_t x1 = 0; + int64_t y1 = std::ceil(std::log(1 - c) / std::log(p - d)); + + if (it != cache.begin()) { + --it; + x1 = it->first; + y1 = it->second; + } + + if (it != cache.begin()) { + --it; + x0 = it->first; + y0 = it->second; + } + + double min_slope = (p - d) / (1 - p + d); + double max_slope = (y1 - y0) * (x1 - x0); + int64_t min_h = (t - x1) * min_slope + y1; + int64_t max_h = (t - x1) * max_slope + y1 + 1; + int64_t h = find_min_passing(min_h, max_h, t, p, d, c); + cache[t] = h; + return h; +} + +} // namespace loadgen +} // namespace mlperf diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/early_stopping.h b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/early_stopping.h new file mode 100644 index 000000000..49b7a901e --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/early_stopping.h @@ -0,0 +1,27 @@ +#ifndef MLPERF_LOADGEN_EARLYSTOPPING_H_ +#define MLPERF_LOADGEN_EARLYSTOPPING_H_ + +#include +#include + +namespace mlperf { +namespace loadgen { + +class MinPassingQueriesFinder { + public: + int64_t operator()(int64_t t, double p, double d, double c); + + private: + // Memoize prior computations results and use them to bound the binary search + // range for subsequent computations. + + // TODO: Is there something more efficient to use besides std::map for + // caches_? + std::map, std::map> + caches_; +}; + +} // namespace loadgen +} // namespace mlperf + +#endif // MLPERF_LOADGEN_EARLYSTOPPING_H_ diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/generated/version_generated.cc b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/generated/version_generated.cc new file mode 100644 index 000000000..75fdc9519 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/generated/version_generated.cc @@ -0,0 +1,98 @@ +// DO NOT EDIT: Autogenerated by version_generator.py. + +#include + +namespace mlperf { + +const std::string& LoadgenVersion() { + static const std::string str = "4.1"; + return str; +} + +const std::string& LoadgenBuildDateLocal() { + static const std::string str = "2024-10-18T23:12:51.002440"; + return str; +} + +const std::string& LoadgenBuildDateUtc() { + static const std::string str = "2024-10-19T06:12:51.002446"; + return str; +} + +const std::string& LoadgenGitRevision() { + static const std::string str = "f5c8f17583"; + return str; +} + +const std::string& LoadgenGitCommitDate() { + static const std::string str = "2024-10-08T18:30:16+01:00"; + return str; +} + +const std::string& LoadgenGitStatus() { + static const std::string str = R"LGVG_RSLD()LGVG_RSLD"; + return str; +} + +const std::string& LoadgenGitLog() { + static const std::string str = R"LGVG_RSLD(f5c8f1758374aeaba26b2e84d31690111cfdf054 Fix bug: Loadgen ignoring token latency targets in user conf (#1874) +976bb1ad9c7946be79507f3ff67955c27426af52 Set correct remote repo (#1871) +41fa8aadd1ba0ecc97f6a519d8b42b04278e5f24 Add format files github action (#1682) +518b454fd8647bfbd23a074e875e87353f33393e Tflite tpu (#1449) +e0fdec1c7a75c98cfc194f13d62ac4388d419c8a Fix link in GettingStarted.ipynb (#1512) +92bd8198d15411d7fb7d7c27f8904bc5a0bcfe7a Fix warning in the submission checker (#1808) +224cfbf5c0e82cae6d48620025b7e1258ae3666a Fix typo in reference datatype (#1851) +3ef1249b7f50a250c02c568342e0aea6638fc5a7 Fix docs (#1853) +a0874c100c54cbc54fb743ac8bf9fb5fadc64135 Update build_wheels.yml (#1758) +6eff09986e337ccf03f675c9f244d8ee93644e16 Extend the final report generation script to output a json file of results (#1825) +54f3f93a73cc8ca5e3319ad87fb325e510574f56 Add binding for server_num_issue_query_threads parameter (#1862) +c4d0b3ea98e6fe7252e50cb573f0d523da7979df Update docs: SCC24, fix broken redirect (#1843) +7d2f0c41e5cd79c9178702867392e38f57953338 Update DLRM readme (#1811) +cf5fddc5d0746bf3820eb0ab7294bbf709d788ab Enable systems to be marked as power only (#1850) +81c2de69de4af90410cd1ba000fc5bd731bf6dee Documentation updates (#1821) +73b02798219c794a735a7f2ddabbc3df9173352d Fix error with generate_final_report.py when the input CSV file is empty (#1827))LGVG_RSLD"; + return str; +} + +const std::string& LoadgenSha1OfFiles() { + static const std::string str = R"LGVG_RSLD(012aad77e5206c89d50718c46c119d1f3cb056b2 /.clang-format +e173f4513f3c5dac1f0bea1473bb0a058e23f190 /=42 +d5274ff0b56e8d3cdb273174628a4461fca6f02a /CMakeLists.txt +20a55bb946c2c0bbb564ced2af1e48efd096b3a8 /README.md +5f6c6a784e9cd6995db47f9b9f70b1769909c9d8 /README_BUILD.md +01f9ae9887f50bc030dc6107e740f40c43ca388f /README_FAQ.md +32181da9e161c285f8fe46ddaa49e6cba2f9f918 /bindings/c_api.cc +91f58bd79b83b278f3240174a9af747fc38aff74 /bindings/c_api.h +ea4c89decad19eaf3217bfa2fb757d3b83a561d6 /bindings/python_api.cc +53dba8ad4272190ceb6335c12fd25e53dc02a8cb /diagram_network_submission.png +84c2f79309b237cef652aef6a187ba8e875a3952 /diagram_submission.png +0cd7b546a389deac73f7955cd39255ed76557d62 /early_stopping.cc +158fcae6a5f47e82150d6416fa1f7bcef37e77fe /early_stopping.h +126e952d00f4ea9efd12405fb209aa3ed585e4b2 /issue_query_controller.cc +923d9d5cdf598e3ec33d7a1110a31f7e11527ec7 /issue_query_controller.h +6650091ba7a918f343b06eb7a5aa540eae87275f /loadgen.cc +e00fdc6dbc85a8c9a8485dbcbfe2944f81251c4e /loadgen.h +47f748307536f80cfc606947b440dd732afc2637 /loadgen_integration_diagram.svg +197efc96d178e5d33a750d07fa7b2966417506ea /logging.cc +ddb961df7bcc145bcd7cce8c21f7cf075350dcbe /logging.h +ca17720f9c8246e821331946d893e830fc88f8bd /pyproject.toml +13ad6d842200cb161d6927eb74a3fafd79c46c75 /query_dispatch_library.h +e9187c8612bbdc972305b789feb6e15c26e96cfe /query_sample.h +8323a2225be1dff31f08ecc86b76eb3de06568bc /query_sample_library.h +a5ff7e77caa6e9e22ada90f0de0c865c987bf167 /requirements.txt +34e2d2a44324cb07c884f92146ecbb8ef9d704e2 /results.cc +d82500c326c2de83db411f1146882aa4692b419c /results.h +13c49b028b22749b5f3c44f3d9bb489e8c0574e9 /setup.py +18d4809589dae33317d88d9beeb5491a6e1ccdec /system_under_test.h +c15c3e150030089a8d634bd2ad6d4b644002e613 /test_settings.h +e21febd60f9b5bedd1fc81bb990f09c34b32043c /test_settings_internal.cc +f1d5335b53ca610c30e0edc5d07999a27b5b4b9a /test_settings_internal.h +3df8fdabf6eaea4697cf25d1dcb89cae88e36efd /utils.cc +40775e32d619ea6356826ae5ea4174c7911f6894 /utils.h +cbec2a5f98f9786c8c3d8b06b3d12df0b6550fa0 /version.cc +9d574baa64424e9c708fcfedd3dbb0b518a65fcc /version.h +eea9b9cb1a06cd1abe1bbdaee82f9af31527fedb /version_generator.py)LGVG_RSLD"; + return str; +} + +} // namespace mlperf diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/issue_query_controller.cc b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/issue_query_controller.cc new file mode 100644 index 000000000..c1abea9d1 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/issue_query_controller.cc @@ -0,0 +1,552 @@ +/* Copyright 2019 The MLPerf Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +/// \file +/// \brief Implements IssueQueryController and other helper classes for +/// query issuing. + +#include "issue_query_controller.h" + +#include + +namespace mlperf { + +void RegisterIssueQueryThread() { + loadgen::IssueQueryController::GetInstance().RegisterThread(); +} + +/// \brief Loadgen implementation details. +namespace loadgen { + +QueryMetadata::QueryMetadata( + const std::vector& query_sample_indices, + std::chrono::nanoseconds scheduled_delta, + ResponseDelegate* response_delegate, SequenceGen* sequence_gen) + : scheduled_delta(scheduled_delta), + response_delegate(response_delegate), + sequence_id(sequence_gen->NextQueryId()), + wait_count_(query_sample_indices.size()) { + samples_.reserve(query_sample_indices.size()); + for (QuerySampleIndex qsi : query_sample_indices) { + samples_.push_back({this, sequence_gen->NextSampleId(), qsi, + sequence_gen->NextAccLogRng()}); + } + query_to_send.reserve(query_sample_indices.size()); + for (auto& s : samples_) { + query_to_send.push_back({reinterpret_cast(&s), s.sample_index}); + } +} + +QueryMetadata::QueryMetadata(QueryMetadata&& src) + : query_to_send(std::move(src.query_to_send)), + scheduled_delta(src.scheduled_delta), + response_delegate(src.response_delegate), + sequence_id(src.sequence_id), + wait_count_(src.samples_.size()), + samples_(std::move(src.samples_)) { + // The move constructor should only be called while generating a + // vector of QueryMetadata, before it's been used. + // Assert that wait_count_ is in its initial state. + assert(src.wait_count_.load() == samples_.size()); + // Update the "parent" of each sample to be this query; the old query + // address will no longer be valid. + // TODO: Only set up the sample parenting once after all the queries have + // been created, rather than re-parenting on move here. + for (size_t i = 0; i < samples_.size(); i++) { + SampleMetadata* s = &samples_[i]; + s->query_metadata = this; + query_to_send[i].id = reinterpret_cast(s); + } +} + +void QueryMetadata::NotifyOneSampleCompleted(PerfClock::time_point timestamp) { + size_t old_count = wait_count_.fetch_sub(1, std::memory_order_relaxed); + if (old_count == 1) { + all_samples_done_time = timestamp; + all_samples_done_.set_value(); + response_delegate->QueryComplete(); + } +} + +void QueryMetadata::WaitForAllSamplesCompleted() { + all_samples_done_.get_future().wait(); +} + +PerfClock::time_point QueryMetadata::WaitForAllSamplesCompletedWithTimestamp() { + all_samples_done_.get_future().wait(); + return all_samples_done_time; +} + +// When server_coalesce_queries is set to true in Server scenario, we +// sometimes coalesce multiple queries into one query. This is done by moving +// the other query's sample into current query, while maintaining their +// original scheduled_time. +void QueryMetadata::CoalesceQueries(QueryMetadata* queries, size_t first, + size_t last, size_t stride) { + // Copy sample data over to current query, boldly assuming that each query + // only has one sample. + query_to_send.reserve((last - first) / stride + + 2); // Extra one for the current query. + for (size_t i = first; i <= last; i += stride) { + auto& q = queries[i]; + auto& s = q.samples_[0]; + query_to_send.push_back({reinterpret_cast(&s), s.sample_index}); + q.scheduled_time = scheduled_time + q.scheduled_delta - scheduled_delta; + q.issued_start_time = issued_start_time; + } +} + +void QueryMetadata::Decoalesce() { query_to_send.resize(1); } + +/// \brief A base template that should never be used since each scenario has +/// its own specialization. +template +struct QueryScheduler { + static_assert(scenario != scenario, "Unhandled TestScenario"); +}; + +/// \brief Schedules queries for issuance in the single stream scenario. +template <> +struct QueryScheduler { + QueryScheduler(const TestSettingsInternal& /*settings*/, + const PerfClock::time_point) {} + + PerfClock::time_point Wait(QueryMetadata* next_query) { + auto tracer = MakeScopedTracer([](AsyncTrace& trace) { trace("Waiting"); }); + if (prev_query != nullptr) { + prev_query->WaitForAllSamplesCompleted(); + } + prev_query = next_query; + + auto now = PerfClock::now(); + next_query->scheduled_time = now; + next_query->issued_start_time = now; + return now; + } + + QueryMetadata* prev_query = nullptr; +}; + +/// \brief Schedules queries for issuance in the multi stream scenario. +template <> +struct QueryScheduler { + QueryScheduler(const TestSettingsInternal& /*settings*/, + const PerfClock::time_point) {} + + PerfClock::time_point Wait(QueryMetadata* next_query) { + auto tracer = MakeScopedTracer([](AsyncTrace& trace) { trace("Waiting"); }); + if (prev_query != nullptr) { + prev_query->WaitForAllSamplesCompleted(); + } + prev_query = next_query; + + auto now = PerfClock::now(); + next_query->scheduled_time = now; + next_query->issued_start_time = now; + return now; + } + + QueryMetadata* prev_query = nullptr; +}; + +/// \brief Schedules queries for issuance in the server scenario. +template <> +struct QueryScheduler { + QueryScheduler(const TestSettingsInternal& /*settings*/, + const PerfClock::time_point start) + : start(start) {} + + PerfClock::time_point Wait(QueryMetadata* next_query) { + auto tracer = + MakeScopedTracer([](AsyncTrace& trace) { trace("Scheduling"); }); + + auto scheduled_time = start + next_query->scheduled_delta; + next_query->scheduled_time = scheduled_time; + + auto now = PerfClock::now(); + if (now < scheduled_time) { + std::this_thread::sleep_until(scheduled_time); + now = PerfClock::now(); + } + next_query->issued_start_time = now; + return now; + } + + const PerfClock::time_point start; +}; + +/// \brief Schedules queries for issuance in the offline scenario. +template <> +struct QueryScheduler { + QueryScheduler(const TestSettingsInternal& /*settings*/, + const PerfClock::time_point start) + : start(start) {} + + PerfClock::time_point Wait(QueryMetadata* next_query) { + next_query->scheduled_time = start; + auto now = PerfClock::now(); + next_query->issued_start_time = now; + return now; + } + + const PerfClock::time_point start; +}; + +IssueQueryController& IssueQueryController::GetInstance() { + // The singleton. + static IssueQueryController instance; + return instance; +} + +void IssueQueryController::RegisterThread() { + // Push this thread to thread queue. + auto thread_id = std::this_thread::get_id(); + size_t thread_idx{0}; + { + std::lock_guard lock(mtx); + thread_idx = thread_ids.size(); + thread_ids.emplace_back(thread_id); + } + + LogDetail([thread_id, thread_idx](AsyncDetail& detail) { +#if USE_NEW_LOGGING_FORMAT + std::stringstream ss; + ss << "Registered IssueQueryThread[" << thread_idx + << "]. thread ID : " << std::hash()(thread_id); + MLPERF_LOG(detail, "generic_message", ss.str()); +#else + detail("Registered IssueQueryThread[" + std::to_string(thread_idx) + + "]. thread ID : ", + std::to_string(std::hash()(thread_id))); +#endif + }); + + // Start test. + while (true) { + // Wait until the main thread signals a start or the end. + { + std::unique_lock lock(mtx); + cond_var.wait(lock, [this]() { return issuing || end_test; }); + // The test has ended. + if (end_test) { + break; + } + } + + // Start issuing queries. + if (thread_idx <= num_threads) { + IssueQueriesInternal(num_threads, thread_idx); + { + std::lock_guard lock(mtx); + thread_complete[thread_idx] = true; + } + cond_var.notify_all(); + } + + // Wait until all issue threads complete. + { + std::unique_lock lock(mtx); + cond_var.wait(lock, [this]() { return !issuing; }); + } + } +} + +void IssueQueryController::SetNumThreads(size_t n) { + // Try waiting for IssueQueryThreads() to registered themselves. + std::unique_lock lock(mtx); + const std::chrono::seconds timeout(10); + num_threads = n; + cond_var.wait_for(lock, timeout, + [this]() { return thread_ids.size() >= num_threads; }); + // If the number of registered threads do not match the settings, report an + // error. + if (num_threads != thread_ids.size()) { + LogDetail([this](AsyncDetail& detail) { +#if USE_NEW_LOGGING_FORMAT + std::stringstream ss; + ss << "Mismatch between settings and number of registered " + << "IssueQueryThreads! settings.server_num_issue_query_threads = " + << num_threads << " but " << thread_ids.size() + << " threads registered."; + MLPERF_LOG_ERROR(detail, "error_runtime", ss.str()); +#else + detail.Error( + "Mismatch between settings and number of registered ", + "IssueQueryThreads! settings.server_num_issue_query_threads = ", + num_threads, " but ", thread_ids.size(), " threads registered."); +#endif + }); + } +} + +template +void IssueQueryController::StartIssueQueries(IssueQueryState* s) { + // Get the state. + state = s; + state->start_for_power = std::chrono::system_clock::now(); + state->start_time = PerfClock::now(); + + if (scenario != TestScenario::Server || num_threads == 0) { + // Usually, we just use the same thread to issue queries. + IssueQueriesInternal(1, 0); + } else { + // If server_num_issue_query_threads is non-zero, issue queries on the + // registered threads. + // Tell all threads to start issuing queries. + { + std::unique_lock lock(mtx); + issuing = true; + thread_complete.assign(num_threads, false); + } + cond_var.notify_all(); + // Wait until all issue threads complete. + { + std::unique_lock lock(mtx); + cond_var.wait(lock, [this]() { + return std::all_of(thread_complete.begin(), thread_complete.end(), + [](bool in) { return in; }); + }); + issuing = false; + } + cond_var.notify_all(); + } +} + +template void IssueQueryController::StartIssueQueries< + TestScenario::MultiStream>(IssueQueryState* s); +template void IssueQueryController::StartIssueQueries( + IssueQueryState* s); +template void IssueQueryController::StartIssueQueries( + IssueQueryState* s); +template void IssueQueryController::StartIssueQueries< + TestScenario::SingleStream>(IssueQueryState* s); + +void IssueQueryController::EndThreads() { + // Tell all the issue threads to end. + { + std::lock_guard lock(mtx); + end_test = true; + } + cond_var.notify_all(); +} + +template +void IssueQueryController::IssueQueriesInternal(size_t query_stride, + size_t thread_idx) { + // Get all the needed information. + auto sut = state->sut; + auto& queries = *state->queries; + auto& response_logger = *state->response_delegate; + + // Some book-keeping about the number of queries issued. + size_t queries_issued = 0; + size_t queries_issued_per_iter = 0; + size_t queries_count = queries.size(); + + // Calculate the min/max queries per issue thread. + const auto& settings = *state->settings; + const size_t min_query_count = settings.min_query_count; + const size_t min_query_count_for_thread = + (thread_idx < (min_query_count % query_stride)) + ? (min_query_count / query_stride + 1) + : (min_query_count / query_stride); + const size_t max_query_count = settings.max_query_count; + const size_t max_query_count_for_thread = + (thread_idx < (max_query_count % query_stride)) + ? (max_query_count / query_stride + 1) + : (max_query_count / query_stride); + + // Create query scheduler. + const auto start = state->start_time; + QueryScheduler query_scheduler(settings, start); + auto last_now = start; + + // We can never run out of generated queries in the server scenario, + // since the duration depends on the scheduled query time and not + // the actual issue time. + bool ran_out_of_generated_queries = scenario != TestScenario::Server; + // This is equal to the sum of numbers of samples issued. + size_t expected_latencies = 0; + + for (size_t queries_idx = thread_idx; queries_idx < queries_count; + queries_idx += query_stride) { + queries_issued_per_iter = 0; + auto& query = queries[queries_idx]; + auto tracer1 = + MakeScopedTracer([](AsyncTrace& trace) { trace("SampleLoop"); }); + last_now = query_scheduler.Wait(&query); + + // If in Server scenario and server_coalesce_queries is enabled, multiple + // queries are coalesed into one big query if the current time has already + // passed the scheduled time of multiple queries. + if (scenario == TestScenario::Server && + settings.requested.server_coalesce_queries) { + auto current_query_idx = queries_idx; + for (; queries_idx + query_stride < queries_count; + queries_idx += query_stride) { + auto next_scheduled_time = + start + queries[queries_idx + query_stride].scheduled_delta; + // If current time hasn't reached the next query's scheduled time yet, + // don't include next query. + if (last_now < next_scheduled_time) { + break; + } + queries_issued_per_iter++; + } + if (queries_idx > current_query_idx) { + // Coalesced all the pass due queries. + query.CoalesceQueries(queries.data(), current_query_idx + query_stride, + queries_idx, query_stride); + } + } + + // Issue the query to the SUT. + { + auto tracer3 = + MakeScopedTracer([](AsyncTrace& trace) { trace("IssueQuery"); }); + sut->IssueQuery(query.query_to_send); + } + + // Increment the counter. + expected_latencies += query.query_to_send.size(); + queries_issued_per_iter++; + queries_issued += queries_issued_per_iter; + + if (scenario == TestScenario::Server && + settings.requested.server_coalesce_queries) { + // Set the query back to its clean state. + query.Decoalesce(); + } + + if (state->mode == TestMode::AccuracyOnly) { + // TODO: Rate limit in accuracy mode so accuracy mode works even + // if the expected/target performance is way off. + continue; + } + + auto duration = (last_now - start); + if (scenario == TestScenario::Server) { + if (settings.max_async_queries != 0) { + // Checks if there are too many outstanding queries. + size_t queries_issued_total{0}; + if (multi_thread) { + // To check actual number of async queries in multi-thread case, + // we would have to combine the number of queries_issued from all + // issue threads. + { + std::lock_guard lock(state->mtx); + state->queries_issued += queries_issued_per_iter; + queries_issued_total = state->queries_issued; + } + } else { + queries_issued_total = queries_issued; + } + size_t queries_outstanding = + queries_issued_total - + response_logger.queries_completed.load(std::memory_order_relaxed); + if (queries_outstanding > settings.max_async_queries) { + LogDetail([thread_idx, queries_issued_total, + queries_outstanding](AsyncDetail& detail) { +#if USE_NEW_LOGGING_FORMAT + std::stringstream ss; + ss << "IssueQueryThread " << thread_idx + << " Ending early: Too many outstanding queries." << " issued " + << queries_issued_total << " outstanding " + << queries_outstanding; + MLPERF_LOG_ERROR(detail, "error_runtime", ss.str()); +#else + detail.Error("IssueQueryThread ", std::to_string(thread_idx), + " Ending early: Too many outstanding queries.", + "issued", std::to_string(queries_issued_total), + "outstanding", std::to_string(queries_outstanding)); +#endif + }); + break; + } + } + } else { + // Checks if we end normally. + if (queries_issued >= min_query_count_for_thread && + duration >= settings.target_duration) { + LogDetail([thread_idx](AsyncDetail& detail) { +#if USE_NEW_LOGGING_FORMAT + MLPERF_LOG( + detail, "generic_message", + "Ending naturally: Minimum query count and test duration met."); +#else + detail( + " Ending naturally: Minimum query count and test duration met."); +#endif + }); + ran_out_of_generated_queries = false; + break; + } + } + + // Checks if we have exceeded max_query_count for this thread. + if (settings.max_query_count != 0 && + queries_issued >= max_query_count_for_thread) { + LogDetail([thread_idx, queries_issued](AsyncDetail& detail) { +#if USE_NEW_LOGGING_FORMAT + std::stringstream ss; + ss << "IssueQueryThread " << thread_idx + << " Ending early: Max query count reached." << " query_count " + << queries_issued; + MLPERF_LOG_ERROR(detail, "error_runtime", ss.str()); +#else + detail.Error("IssueQueryThread ", std::to_string(thread_idx), + " Ending early: Max query count reached.", "query_count", + std::to_string(queries_issued)); +#endif + }); + ran_out_of_generated_queries = false; + break; + } + + // Checks if we have exceeded max_duration. + if (settings.max_duration.count() != 0 && + duration > settings.max_duration) { + LogDetail([thread_idx, duration](AsyncDetail& detail) { +#if USE_NEW_LOGGING_FORMAT + std::stringstream ss; + ss << "IssueQueryThread " << thread_idx + << " Ending early: Max test duration reached." << " duration_ns " + << duration.count(); + MLPERF_LOG_ERROR(detail, "error_runtime", ss.str()); +#else + detail.Error("IssueQueryThread ", std::to_string(thread_idx), + " Ending early: Max test duration reached.", "duration_ns", + std::to_string(duration.count())); +#endif + }); + ran_out_of_generated_queries = false; + break; + } + } + + // Combine the issuing statistics from multiple issue threads. + { + std::lock_guard lock(state->mtx); + state->ran_out_of_generated_queries |= ran_out_of_generated_queries; + // In Server scenario and when max_async_queries != 0, we would have set + // state->queries_issued when we check max_async_queries in the loop. + if (!(scenario == TestScenario::Server && settings.max_async_queries != 0 && + multi_thread)) { + state->queries_issued += queries_issued; + } + state->expected_latencies += expected_latencies; + } +} + +} // namespace loadgen + +} // namespace mlperf diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/issue_query_controller.h b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/issue_query_controller.h new file mode 100644 index 000000000..5668c574e --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/issue_query_controller.h @@ -0,0 +1,215 @@ +/* Copyright 2019 The MLPerf Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +/// \file +/// \brief Declare IssueQueryController and other helper classes for +/// query issuing. + +#ifndef MLPERF_LOADGEN_ISSUE_QUERY_CONTROLLER_H_ +#define MLPERF_LOADGEN_ISSUE_QUERY_CONTROLLER_H_ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "loadgen.h" +#include "logging.h" +#include "query_sample.h" +#include "system_under_test.h" +#include "test_settings_internal.h" +#include "utils.h" + +namespace mlperf { + +namespace loadgen { + +struct SampleMetadata; +class QueryMetadata; + +/// \brief Every query and sample within a call to StartTest gets a unique +/// sequence id for easy cross reference, and a random number which is used to +/// determine accuracy logging when it is enabled. +struct SequenceGen { + uint64_t NextQueryId() { return query_id++; } + uint64_t NextSampleId() { return sample_id++; } + uint64_t CurrentSampleId() { return sample_id; } + double NextAccLogRng() { return accuracy_log_dist(accuracy_log_rng); } + void InitAccLogRng(uint64_t accuracy_log_rng_seed) { + accuracy_log_rng = std::mt19937(accuracy_log_rng_seed); + } + + private: + uint64_t query_id = 0; + uint64_t sample_id = 0; + std::mt19937 accuracy_log_rng; + std::uniform_real_distribution accuracy_log_dist = + std::uniform_real_distribution(0, 1); +}; + +/// \brief An interface for a particular scenario + mode to implement for +/// extended hanlding of sample completion. +struct ResponseDelegate { + virtual ~ResponseDelegate() = default; + virtual void SampleComplete(SampleMetadata*, QuerySampleResponse*, + PerfClock::time_point, + const ResponseCallback&) = 0; + virtual void TokenComplete(SampleMetadata*, QuerySampleResponse*, + PerfClock::time_point, + const ResponseCallback&) = 0; + virtual void QueryComplete() = 0; + std::atomic queries_completed{0}; +}; + +/// \brief Used by the loadgen to coordinate response data and completion. +struct SampleMetadata { + QueryMetadata* query_metadata; + uint64_t sequence_id; + QuerySampleIndex sample_index; + double accuracy_log_val; +}; + +/// \brief Maintains data and timing info for a query and all its samples. +class QueryMetadata { + public: + QueryMetadata(const std::vector& query_sample_indices, + std::chrono::nanoseconds scheduled_delta, + ResponseDelegate* response_delegate, SequenceGen* sequence_gen); + QueryMetadata(QueryMetadata&& src); + + void NotifyOneSampleCompleted(PerfClock::time_point timestamp); + + void WaitForAllSamplesCompleted(); + + PerfClock::time_point WaitForAllSamplesCompletedWithTimestamp(); + + /// \brief Coalesce multiple queries into one query. + /// When server_coalesce_queries is set to true in Server scenario, we + /// sometimes coalesce multiple queries into one query. This is done by moving + /// the other query's sample into current query, while maintaining their + /// original scheduled_time. + void CoalesceQueries(QueryMetadata* queries, size_t first, size_t last, + size_t stride); + + /// \brief Set a coalesced query back to its original state. + void Decoalesce(); + + public: + std::vector query_to_send; + const std::chrono::nanoseconds scheduled_delta; + ResponseDelegate* const response_delegate; + const uint64_t sequence_id; + + // Performance information. + + size_t scheduled_intervals = 0; // Number of intervals between queries, as + // actually scheduled during the run. + // For the MultiStream scenario only. + PerfClock::time_point scheduled_time; + PerfClock::time_point issued_start_time; + PerfClock::time_point all_samples_done_time; + + private: + std::atomic wait_count_; + std::promise all_samples_done_; + std::vector samples_; +}; + +/// \brief A state object for communications between the controller and its +/// caller. +struct IssueQueryState { + // Information from caller to controller. + SystemUnderTest* sut; + std::vector* queries; + ResponseDelegate* response_delegate; + const TestSettingsInternal* settings; + TestMode mode; + // Information from controller to caller. + std::chrono::system_clock::time_point start_for_power; + PerfClock::time_point start_time; + bool ran_out_of_generated_queries; + size_t queries_issued; + size_t expected_latencies; + // The lock to modify this state (in multi-thread case). + std::mutex mtx; +}; + +/// \brief Controls the query issuing part. +/// This controller handles both the cases if the user registers or does not +/// register IssueQueryThreads. It is implemented as a singleton, and is NOT +/// thread-safe (i.e. users should not call StartTest() on multiple threads). +/// It is thread-safe with regard to IssueQueryThreads. +class IssueQueryController { + public: + /// \brief Get the controller instance singleton. + static IssueQueryController& GetInstance(); + + /// \brief Don't allow copy. This is a singleton. + IssueQueryController(IssueQueryController const&) = delete; + void operator=(IssueQueryController const&) = delete; + + /// \brief Register an IssueQueryThread. + /// It is blocking until the entire test ends. + void RegisterThread(); + + /// \brief Set number of IssueQueryThreads and wait for thread registration. + /// If for any reason the number of registered threads do not match the + /// specified number, it prints out an error. + void SetNumThreads(size_t n); + + /// \brief Kick off the query issuing. + /// The query issuing will be done on the current thread if there is no + /// registered IssueQueryThreads or if it is not in Server scenario. + template + void StartIssueQueries(IssueQueryState* s); + + /// \brief Notify the IssueQueryThreads to end. + void EndThreads(); + + private: + /// \brief Hide constructor. This is a singleton. + IssueQueryController() {} + + /// \brief The internal helper which actually issues queries. + /// This should be called by the thread(s) which issues queries. + template + void IssueQueriesInternal(size_t query_stride, size_t thread_idx); + + /// \brief The issue query state. + IssueQueryState* state; + /// \brief Locks for communications across IssueQueryThreads and the main + /// thread. + std::mutex mtx; + std::condition_variable cond_var; + /// \brief Thread ids of the registered IssueQueryThreads. + std::vector thread_ids; + size_t num_threads{0}; + /// \brief Whether the threads should be actively issuing queries. + bool issuing{false}; + /// \brief Flags for each IssueQueryThread to mark that it is done. + std::vector thread_complete; + /// \brief Whether the threads can end now. + bool end_test{false}; +}; + +} // namespace loadgen + +} // namespace mlperf + +#endif // MLPERF_LOADGEN_ISSUE_QUERY_CONTROLLER_H_ diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/loadgen.cc b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/loadgen.cc new file mode 100644 index 000000000..42b2140de --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/loadgen.cc @@ -0,0 +1,1345 @@ +/* Copyright 2019 The MLPerf Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "loadgen.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "early_stopping.h" +#include "issue_query_controller.h" +#include "logging.h" +#include "query_sample.h" +#include "query_sample_library.h" +#include "results.h" +#include "system_under_test.h" +#include "test_settings.h" +#include "test_settings_internal.h" +#include "utils.h" +#include "version.h" + +namespace mlperf { + +/// \brief Loadgen implementation details. +namespace loadgen { + +/// \brief A random set of samples in the QSL that should fit in RAM when +/// loaded together. +struct LoadableSampleSet { + std::vector set; + const size_t sample_distribution_end; // Excludes padding in MultiStream. +}; + +/// \brief Generates nanoseconds from a start time to multiple end times. +/// TODO: This isn't very useful anymore. Remove it. +struct DurationGeneratorNs { + const PerfClock::time_point start; + int64_t delta(PerfClock::time_point end) const { + return std::chrono::duration_cast(end - start) + .count(); + } +}; + +/// \brief ResponseDelegate implementation templated by scenario and mode. +template +struct ResponseDelegateDetailed : public ResponseDelegate { + double accuracy_log_offset = 0.0f; + double accuracy_log_prob = 0.0f; + + void SampleComplete(SampleMetadata* sample, QuerySampleResponse* response, + PerfClock::time_point complete_begin_time, + const ResponseCallback& response_cb) override { + // Using a raw pointer here should help us hit the std::function + // small buffer optimization code path when we aren't copying data. + // For some reason, using std::unique_ptr wasn't moving + // into the lambda; even with C++14. + std::vector* sample_data_copy = nullptr; + double accuracy_log_val = + sample->accuracy_log_val + accuracy_log_offset < 1.0 + ? sample->accuracy_log_val + accuracy_log_offset + : sample->accuracy_log_val + accuracy_log_offset - 1.0; + if (mode == TestMode::AccuracyOnly || + accuracy_log_val <= accuracy_log_prob) { + // if a response_cb callback is provided, data only needs to reside on the + // host *after* calling it note that the callback is blocking and will + // likely involve a memcpy from accelerator to host + if (response_cb) { + response_cb(response); + } + // TODO: Verify accuracy with the data copied here. + uint8_t* src_begin = reinterpret_cast(response->data); + uint8_t* src_end = src_begin + response->size; + sample_data_copy = new std::vector(src_begin, src_end); + } + int64_t n_tokens = response->n_tokens; + Log([sample, complete_begin_time, sample_data_copy, + n_tokens](AsyncLog& log) { + QueryMetadata* query = sample->query_metadata; + DurationGeneratorNs sched{query->scheduled_time}; + if (scenario == TestScenario::Server) { + // Trace the server scenario as a stacked graph via counter events. + DurationGeneratorNs issued{query->issued_start_time}; + log.TraceCounterEvent("Latency", query->scheduled_time, "issue_delay", + sched.delta(query->issued_start_time), + "issue_to_done", + issued.delta(complete_begin_time)); + } + + // While visualizing overlapping samples in offline mode is not + // practical, sample completion is still recorded for auditing purposes. + log.TraceSample("Sample", sample->sequence_id, query->scheduled_time, + complete_begin_time, "sample_seq", sample->sequence_id, + "query_seq", query->sequence_id, "sample_idx", + sample->sample_index, "issue_start_ns", + sched.delta(query->issued_start_time), "complete_ns", + sched.delta(complete_begin_time)); + + if (sample_data_copy) { + log.LogAccuracy(sample->sequence_id, sample->sample_index, + LogBinaryAsHexString{sample_data_copy}, n_tokens); + delete sample_data_copy; + } + + // Record the latency at the end, since it will unblock the issuing + // thread and potentially destroy the metadata being used above. + QuerySampleLatency latency = sched.delta(complete_begin_time); + log.RecordSampleCompletion(sample->sequence_id, complete_begin_time, + latency, n_tokens); + }); + } + + void TokenComplete(SampleMetadata* sample, QuerySampleResponse* response, + PerfClock::time_point complete_begin_time, + const ResponseCallback& response_cb) override { + // Using a raw pointer here should help us hit the std::function + // small buffer optimization code path when we aren't copying data. + // For some reason, using std::unique_ptr wasn't moving + // into the lambda; even with C++14. + std::vector* token_data_copy = nullptr; + double accuracy_log_val = + sample->accuracy_log_val + accuracy_log_offset < 1.0 + ? sample->accuracy_log_val + accuracy_log_offset + : sample->accuracy_log_val + accuracy_log_offset - 1.0; + if (mode == TestMode::AccuracyOnly || + accuracy_log_val <= accuracy_log_prob) { + uint8_t* src_begin = reinterpret_cast(response->data); + uint8_t* src_end = src_begin + response->size; + token_data_copy = new std::vector(src_begin, src_end); + } + Log([sample, complete_begin_time, token_data_copy](AsyncLog& log) { + QueryMetadata* query = sample->query_metadata; + DurationGeneratorNs sched{query->scheduled_time}; + if (scenario == TestScenario::Server) { + DurationGeneratorNs issued{query->issued_start_time}; + log.TraceCounterEvent( + "Token_Latency", query->scheduled_time, "issue_delay", + sched.delta(query->issued_start_time), "issue_to_done", + issued.delta(complete_begin_time)); + } else { + log.TraceSample("Token", sample->sequence_id, query->scheduled_time, + complete_begin_time, "sample_seq", sample->sequence_id, + "query_seq", query->sequence_id, "sample_idx", + sample->sample_index, "issue_start_ns", + sched.delta(query->issued_start_time), "complete_ns", + sched.delta(complete_begin_time)); + } + if (token_data_copy) { + log.CacheToken(sample->sequence_id, + LogBinaryAsHexString{token_data_copy}); + } + QuerySampleLatency latency = sched.delta(complete_begin_time); + log.RecordTokenCompletion(sample->sequence_id, complete_begin_time, + latency); + }); + } + + void QueryComplete() override { + // We only need to track outstanding queries in the server scenario to + // detect when the SUT has fallen too far behind. + if (scenario == TestScenario::Server) { + queries_completed.fetch_add(1, std::memory_order_relaxed); + } + } +}; + +/// \brief Selects the query timestamps for all scenarios except Server. +template +auto ScheduleDistribution(double qps) { + return [period = std::chrono::duration_cast( + std::chrono::duration(1.0 / qps))](auto& /*gen*/) { + return period; + }; +} + +/// \brief Selects the query timestamps for the Server scenario. +template <> +auto ScheduleDistribution(double qps) { + // Poisson arrival process corresponds to exponentially distributed + // interarrival times. + return [dist = std::exponential_distribution<>(qps)](auto& gen) mutable { + return std::chrono::duration_cast( + std::chrono::duration(dist(gen))); + }; +} + +/// \brief Selects samples for the accuracy mode. +template +auto SampleDistribution(size_t sample_count, size_t stride, std::mt19937* rng) { + std::vector indices; + for (size_t i = 0; i < sample_count; i += stride) { + indices.push_back(i); + } + std::shuffle(indices.begin(), indices.end(), *rng); + return [indices = std::move(indices), i = size_t(0)](auto& /*gen*/) mutable { + return indices.at(i++); + }; +} + +/// \brief Selects samples for the performance mode. +template <> +auto SampleDistribution(size_t sample_count, + size_t /*stride*/, + std::mt19937* /*rng*/) { + return [dist = std::uniform_int_distribution<>(0, sample_count - 1)]( + auto& gen) mutable { return dist(gen); }; +} + +/// \brief Sample across the dataset, and ensure coverage of each of the +/// samples. +// Useful for non-uniform dataset (e.g. Llama2, GPTJ, 3d-unet) +auto SampleDistributionEqualIssue(size_t sample_count, size_t set_size, + std::mt19937* rng) { + std::vector indices; + std::vector shuffle_indices(set_size); + std::iota(shuffle_indices.begin(), shuffle_indices.end(), 0); + for (size_t j = 0; j < sample_count; j += set_size) { + std::shuffle(shuffle_indices.begin(), shuffle_indices.end(), *rng); + indices.insert(indices.end(), shuffle_indices.begin(), + shuffle_indices.end()); + } + return [indices = std::move(indices), i = size_t(0)](auto& /*gen*/) mutable { + return indices.at((i++) % indices.size()); + }; +} + +/// \brief Generates queries for the requested settings, templated by +/// scenario and mode. +/// \todo Make GenerateQueries faster. +/// QueryMetadata is expensive to move; either reserve queries in advance +/// so the queries vector doesn't need to grow. And/or parent samples to their +/// queries only after all queries have been generated. +/// \todo For the server scenario only, scale the query timeline at the end so +/// the QPS as scheduled is equal to the QPS as requested. +template +std::vector GenerateQueries( + const TestSettingsInternal& settings, + const LoadableSampleSet& loaded_sample_set, SequenceGen* sequence_gen, + ResponseDelegate* response_delegate) { + auto tracer = + MakeScopedTracer([](AsyncTrace& trace) { trace("GenerateQueries"); }); + + auto& loaded_samples = loaded_sample_set.set; + + // Generate 2x more samples than we think we'll need given the expected + // QPS in case the SUT is faster than expected. + // We should exit before issuing all queries. + // Does not apply to the server scenario since the duration only + // depends on the ideal scheduled time, not the actual issue time. + const int duration_multiplier = scenario == TestScenario::Server ? 1 : 2; + std::chrono::microseconds gen_duration = + duration_multiplier * settings.target_duration; + size_t min_queries = settings.min_query_count; + + size_t samples_per_query = settings.samples_per_query; + if (mode == TestMode::AccuracyOnly && scenario == TestScenario::Offline) { + samples_per_query = loaded_sample_set.sample_distribution_end; + } + + // We should not exit early in accuracy mode. + if (mode == TestMode::AccuracyOnly || settings.performance_issue_unique) { + gen_duration = std::chrono::microseconds(0); + // Integer truncation here is intentional. + // For MultiStream, loaded samples is properly padded. + // For Offline, we create a 'remainder' query at the end of this function. + min_queries = loaded_samples.size() / samples_per_query; + } + + std::vector queries; + + // Using the std::mt19937 pseudo-random number generator ensures a modicum of + // cross platform reproducibility for trace generation. + std::mt19937 sample_rng(settings.sample_index_rng_seed); + std::mt19937 schedule_rng(settings.schedule_rng_seed); + + constexpr bool kIsMultiStream = scenario == TestScenario::MultiStream; + const size_t sample_stride = kIsMultiStream ? samples_per_query : 1; + + auto sample_distribution = SampleDistribution( + loaded_sample_set.sample_distribution_end, sample_stride, &sample_rng); + // Use the unique sample distribution same as in AccuracyMode to + // to choose samples when either flag performance_issue_unique + // or performance_issue_same is set. + auto sample_distribution_unique = SampleDistribution( + loaded_sample_set.sample_distribution_end, sample_stride, &sample_rng); + + auto sample_distribution_equal_issue = SampleDistributionEqualIssue( + min_queries, loaded_samples.size(), &sample_rng); + + auto schedule_distribution = + ScheduleDistribution(settings.target_qps); + + // When sample_concatenate_permutation is turned on, pad to a multiple of the + // complete dataset to ensure fairness. + auto enable_equal_issue = settings.sample_concatenate_permutation; + if (mode != TestMode::AccuracyOnly && enable_equal_issue) { + if (scenario == TestScenario::Offline && + samples_per_query % loaded_samples.size() != 0) { + // In offline mode, we pad samples_per_query + size_t pad_size = + (loaded_samples.size() - samples_per_query % loaded_samples.size()); + samples_per_query += pad_size; + } else if ((scenario != TestScenario::Offline) && + (min_queries % loaded_samples.size() != 0)) { + // In Server, SingleStream, MultiStream mode, the min_queries should be + // padded + size_t pad_size = + (loaded_samples.size() - min_queries % loaded_samples.size()); + min_queries += pad_size; + } + } + + std::vector samples(samples_per_query); + std::chrono::nanoseconds timestamp(0); + std::chrono::nanoseconds prev_timestamp(0); + // Choose a single sample to repeat when in performance_issue_same mode + QuerySampleIndex same_sample = settings.performance_issue_same_index; + + while (prev_timestamp < gen_duration || queries.size() < min_queries) { + if (kIsMultiStream) { + QuerySampleIndex sample_i = settings.performance_issue_unique + ? sample_distribution_unique(sample_rng) + : settings.performance_issue_same + ? same_sample + : sample_distribution(sample_rng); + for (auto& s : samples) { + // Select contiguous samples in the MultiStream scenario. + // This will not overflow, since GenerateLoadableSets adds padding at + // the end of the loadable sets in the MultiStream scenario. + // The padding allows the starting samples to be the same for each + // query with respect to samples_per_query. + s = loaded_samples[sample_i++]; + } + } else if (scenario == TestScenario::Offline) { + // For the Offline + Performance scenario, we also want to support + // contiguous samples. In this scenario the query can be much larger than + // what fits into memory. We simply repeat loaded_samples N times, plus a + // remainder to ensure we fill up samples. Note that this eliminates + // randomization. + size_t num_loaded_samples = loaded_samples.size(); + size_t num_full_repeats = samples_per_query / num_loaded_samples; + uint64_t remainder = samples_per_query % (num_loaded_samples); + if (settings.performance_issue_same) { + std::fill(samples.begin(), samples.begin() + samples_per_query, + loaded_samples[same_sample]); + } else { + for (size_t i = 0; i < num_full_repeats; ++i) { + std::copy(loaded_samples.begin(), loaded_samples.end(), + samples.begin() + i * num_loaded_samples); + + if (settings.sample_concatenate_permutation) { + std::shuffle(samples.begin() + i * num_loaded_samples, + samples.begin() + (i + 1) * num_loaded_samples, + sample_rng); + } + } + + std::copy(loaded_samples.begin(), loaded_samples.begin() + remainder, + samples.begin() + num_full_repeats * num_loaded_samples); + + if (settings.sample_concatenate_permutation) { + assert(remainder == 0); + } + } + } else { + for (auto& s : samples) { + s = loaded_samples[settings.performance_issue_unique + ? sample_distribution_unique(sample_rng) + : settings.performance_issue_same ? same_sample + : enable_equal_issue + ? sample_distribution_equal_issue(sample_rng) + : sample_distribution(sample_rng)]; + } + } + queries.emplace_back(samples, timestamp, response_delegate, sequence_gen); + prev_timestamp = timestamp; + timestamp += schedule_distribution(schedule_rng); + // In equal_issue mode, the min_queries will be bumped up by a multiple of + // the dataset size if the test time has not met the threshold. + if (enable_equal_issue && (queries.size() >= min_queries) && + (prev_timestamp < gen_duration) && + (scenario != TestScenario::Offline)) { + min_queries += loaded_samples.size(); + } + } + + // See if we need to create a "remainder" query for offline+accuracy to + // ensure we issue all samples in loaded_samples. Offline doesn't pad + // loaded_samples like MultiStream does. + if (scenario == TestScenario::Offline && mode == TestMode::AccuracyOnly) { + size_t remaining_samples = loaded_samples.size() % samples_per_query; + if (remaining_samples != 0) { + samples.resize(remaining_samples); + for (auto& s : samples) { + s = loaded_samples[sample_distribution(sample_rng)]; + } + queries.emplace_back(samples, timestamp, response_delegate, sequence_gen); + } + } + + LogDetail([count = queries.size(), spq = samples_per_query, + duration = timestamp.count()](AsyncDetail& detail) { +#if USE_NEW_LOGGING_FORMAT + MLPERF_LOG(detail, "generated_query_count", count); + MLPERF_LOG(detail, "generated_samples_per_query", spq); + MLPERF_LOG(detail, "generated_query_duration", duration); +#else + detail("GeneratedQueries: ", "queries", count, "samples per query", spq, + "duration", duration); +#endif + }); + + return queries; +} + +/// \brief Issues a series of pre-generated queries. +// TODO: Templates for scenario and mode are overused, given the loadgen +// no longer generates queries on the fly. Should we reduce the +// use of templates? +template +PerformanceResult IssueQueries(SystemUnderTest* sut, + const TestSettingsInternal& settings, + const LoadableSampleSet& loaded_sample_set, + SequenceGen* sequence_gen) { + // Create reponse handler. + ResponseDelegateDetailed response_logger; + std::uniform_real_distribution accuracy_log_offset_dist = + std::uniform_real_distribution(0.0, 1.0); + std::mt19937 accuracy_log_offset_rng(settings.accuracy_log_rng_seed); + response_logger.accuracy_log_offset = + accuracy_log_offset_dist(accuracy_log_offset_rng); + response_logger.accuracy_log_prob = settings.accuracy_log_probability; + + // Generate queries. + auto sequence_id_start = sequence_gen->CurrentSampleId(); + std::vector queries = GenerateQueries( + settings, loaded_sample_set, sequence_gen, &response_logger); + + // Calculated expected number of queries + uint64_t expected_queries = + settings.target_qps * settings.min_duration.count() / 1000; + uint64_t minimum_queries = + settings.min_query_count * settings.samples_per_query; + + if (scenario != TestScenario::Offline) { + expected_queries *= settings.samples_per_query; + } else { + minimum_queries = settings.min_sample_count; + } + + expected_queries = + expected_queries < minimum_queries ? minimum_queries : expected_queries; + + if (settings.accuracy_log_sampling_target > 0) { + response_logger.accuracy_log_prob = + (double)settings.accuracy_log_sampling_target / expected_queries; + } + auto sequence_id_end = sequence_gen->CurrentSampleId(); + size_t max_latencies_to_record = sequence_id_end - sequence_id_start; + + // Initialize logger for latency recording. + GlobalLogger().RestartLatencyRecording(sequence_id_start, + max_latencies_to_record); + + // Create and initialize an IssueQueryState. + IssueQueryState state{ + sut, &queries, &response_logger, &settings, mode, {}, {}, false, 0, + 0, {}}; + auto& controller = IssueQueryController::GetInstance(); + + // Set number of IssueQueryThreads and wait for the threads to register. + controller.SetNumThreads(settings.requested.server_num_issue_query_threads); + + // Start issuing the queries. + controller.StartIssueQueries(&state); + + // Gather query issuing statistics. + const auto start_for_power = state.start_for_power; + const auto start = state.start_time; + const auto ran_out_of_generated_queries = state.ran_out_of_generated_queries; + const auto queries_issued = state.queries_issued; + const auto expected_latencies = state.expected_latencies; + + // Let the SUT know it should not expect any more queries. + sut->FlushQueries(); + + if (mode == TestMode::PerformanceOnly && ran_out_of_generated_queries) { + LogDetail([](AsyncDetail& detail) { +#if USE_NEW_LOGGING_FORMAT + MLPERF_LOG_ERROR( + detail, "error_runtime", + "Ending early: Ran out of generated queries to issue before the " + "minimum query count and test duration were reached. " + "Please update the relevant expected latency or target qps in the " + "TestSettings so they are more accurate."); +#else + detail.Error( + "Ending early: Ran out of generated queries to issue before the " + "minimum query count and test duration were reached."); + detail( + "Please update the relevant expected latency or target qps in the " + "TestSettings so they are more accurate."); +#endif + }); + } + + // Wait for tail queries to complete and collect all the latencies. + // We have to keep the synchronization primitives alive until the SUT + // is done with them. + auto& final_query = queries[queries_issued - 1]; + std::vector sample_latencies( + GlobalLogger().GetLatenciesBlocking(expected_latencies)); + + std::vector first_token_latencies( + GlobalLogger().GetTokenLatencies(expected_latencies)); + + std::vector time_per_output_token_arr( + GlobalLogger().GetTimePerOutputToken(expected_latencies)); + + std::vector tokens_per_sample( + GlobalLogger().GetTokensPerSample(expected_latencies)); + + // Log contention counters after every test as a sanity check. + GlobalLogger().LogContentionAndAllocations(); + + // This properly accounts for the fact that the max completion time may not + // belong to the final query. It also excludes any time spent postprocessing + // in the loadgen itself after final completion, which may be significant + // in the offline scenario. + PerfClock::time_point max_completion_time = + GlobalLogger().GetMaxCompletionTime(); + auto sut_active_duration = max_completion_time - start; + LogDetail([start_for_power, sut_active_duration](AsyncDetail& detail) { + auto end_for_power = + start_for_power + + std::chrono::duration_cast( + sut_active_duration); +#if USE_NEW_LOGGING_FORMAT + MLPERF_LOG_INTERVAL_START(detail, "power_begin", + DateTimeStringForPower(start_for_power)); + MLPERF_LOG_INTERVAL_END(detail, "power_end", + DateTimeStringForPower(end_for_power)); +#else + detail("POWER_BEGIN: ", "mode", ToString(mode), "time", + DateTimeStringForPower(start_for_power)); + detail("POWER_END: ", "mode", ToString(mode), "time", + DateTimeStringForPower(end_for_power)); +#endif + }); + + double max_latency = + QuerySampleLatencyToSeconds(GlobalLogger().GetMaxLatencySoFar()); + double final_query_scheduled_time = + DurationToSeconds(final_query.scheduled_delta); + double final_query_issued_time = + DurationToSeconds(final_query.issued_start_time - start); + double final_query_all_samples_done_time = + DurationToSeconds(final_query.all_samples_done_time - start); + + std::vector query_latencies; + if (scenario == TestScenario::MultiStream) { + query_latencies.resize(queries_issued); + for (size_t i = 0; i < queries_issued; i++) { + query_latencies[i] = DurationGeneratorNs{queries[i].scheduled_time}.delta( + queries[i].all_samples_done_time); + } + } + + return PerformanceResult{ + std::move(sample_latencies), + std::move(query_latencies), + queries_issued, + max_latency, + final_query_scheduled_time, + final_query_issued_time, + final_query_all_samples_done_time, + TokenPerformanceResults{first_token_latencies, time_per_output_token_arr, + tokens_per_sample}}; +} + +void LoadSamplesToRam(QuerySampleLibrary* qsl, + const std::vector& samples) { + LogDetail([&samples](AsyncDetail& detail) { +#if USE_NEW_LOGGING_FORMAT + MLPERF_LOG(detail, "loaded_qsl_set", samples); +#else + std::string set("\"["); + for (auto i : samples) { + set += std::to_string(i) + ","; + } + set.resize(set.size() - 1); + set += "]\""; + detail("Loading QSL : ", "set", set); +#endif + }); + qsl->LoadSamplesToRam(samples); +} + +/// \brief Generates random sets of samples in the QSL that we can load into +/// RAM at the same time. +std::vector GenerateLoadableSets( + QuerySampleLibrary* qsl, const TestSettingsInternal& settings) { + auto tracer = MakeScopedTracer( + [](AsyncTrace& trace) { trace("GenerateLoadableSets"); }); + + std::vector result; + std::mt19937 qsl_rng(settings.qsl_rng_seed); + + // Generate indices for all available samples in the QSL. + const size_t qsl_total_count = qsl->TotalSampleCount(); + std::vector samples(qsl_total_count); + for (size_t i = 0; i < qsl_total_count; i++) { + samples[i] = static_cast(i); + } + + // Randomize the order of the samples. + std::shuffle(samples.begin(), samples.end(), qsl_rng); + + // Partition the samples into loadable sets. + const size_t set_size = settings.performance_sample_count; + const size_t set_padding = (settings.scenario == TestScenario::MultiStream) + ? settings.samples_per_query - 1 + : 0; + std::vector loadable_set; + loadable_set.reserve(set_size + set_padding); + + for (auto s : samples) { + loadable_set.push_back(s); + if (loadable_set.size() == set_size) { + result.push_back({std::move(loadable_set), set_size}); + loadable_set.clear(); + loadable_set.reserve(set_size + set_padding); + } + } + + if (!loadable_set.empty()) { + // Copy the size since it will become invalid after the move. + size_t loadable_set_size = loadable_set.size(); + result.push_back({std::move(loadable_set), loadable_set_size}); + } + + // Add padding for the multi stream scenario. Padding allows the + // starting sample to be the same for all SUTs, independent of the value + // of samples_per_query, while enabling samples in a query to be contiguous. + for (auto& loadable_set : result) { + auto& set = loadable_set.set; + for (size_t i = 0; i < set_padding; i++) { + // It's not clear in the spec if the STL deallocates the old container + // before assigning, which would invalidate the source before the + // assignment happens. Even though we should have reserved enough + // elements above, copy the source first anyway since we are just moving + // integers around. + QuerySampleIndex p = set[i]; + set.push_back(p); + } + } + + return result; +} + +/// \brief Opens and owns handles to all of the log files. +struct LogOutputs { + LogOutputs(const LogOutputSettings& output_settings, + const std::string& test_date_time) { + std::string prefix = output_settings.outdir; + prefix += "/" + output_settings.prefix; + if (output_settings.prefix_with_datetime) { + prefix += test_date_time + "_"; + } + const std::string& suffix = output_settings.suffix; + + summary_out.open(prefix + "summary" + suffix + ".txt"); + detail_out.open(prefix + "detail" + suffix + ".txt"); + accuracy_out.open(prefix + "accuracy" + suffix + ".json"); + trace_out.open(prefix + "trace" + suffix + ".json"); + } + + bool CheckOutputs() { + bool all_ofstreams_good = true; + if (!summary_out.good()) { + all_ofstreams_good = false; + std::cerr << "LoadGen: Failed to open summary file."; + } + if (!detail_out.good()) { + all_ofstreams_good = false; + std::cerr << "LoadGen: Failed to open detailed log file."; + } + if (!accuracy_out.good()) { + all_ofstreams_good = false; + std::cerr << "LoadGen: Failed to open accuracy log file."; + } + if (!trace_out.good()) { + all_ofstreams_good = false; + std::cerr << "LoadGen: Failed to open trace file."; + } + return all_ofstreams_good; + } + + std::ofstream summary_out; + std::ofstream detail_out; + std::ofstream accuracy_out; + std::ofstream trace_out; +}; + +/// \brief Find boundaries of performance settings by widening bounds +/// exponentially. +/// \details To find an upper bound of performance, widen an +/// upper bound exponentially until finding a bound that can't satisfy +/// performance constraints. i.e. [1, 2) -> [2, 4) -> [4, 8) -> ... +template +std::pair FindBoundaries( + SystemUnderTest* sut, QuerySampleLibrary* qsl, SequenceGen* sequence_gen, + PerformanceSummary l_perf_summary) { + // Get upper bound + TestSettingsInternal u_settings = l_perf_summary.settings; + find_peak_performance::WidenPerformanceField(&u_settings); + + LogDetail( + [l_field = find_peak_performance::ToStringPerformanceField( + l_perf_summary.settings), + u_field = find_peak_performance::ToStringPerformanceField( + u_settings)](AsyncDetail& detail) { +#if USE_NEW_LOGGING_FORMAT + MLPERF_LOG(detail, "generic_message", + "FindBoundaries: Checking fields [" + l_field + ", " + + u_field + ")"); +#else + detail("FindBoundaries: Checking fields [" + l_field + ", " + u_field + + ")"); +#endif + }); + + std::vector loadable_sets( + loadgen::GenerateLoadableSets(qsl, u_settings)); + const LoadableSampleSet& performance_set = loadable_sets.front(); + LoadSamplesToRam(qsl, performance_set.set); + + PerformanceResult u_pr(IssueQueries( + sut, u_settings, performance_set, sequence_gen)); + PerformanceSummary u_perf_summary{sut->Name(), u_settings, std::move(u_pr)}; + + qsl->UnloadSamplesFromRam(performance_set.set); + + std::string tmp; + if (!u_perf_summary.PerfConstraintsMet(&tmp)) { + return std::make_pair(l_perf_summary, u_perf_summary); + } else { + return FindBoundaries(sut, qsl, sequence_gen, u_perf_summary); + } +} + +/// \brief Find peak performance by binary search. +/// \details The found lower & upper bounds by the function 'FindBoundaries' are +/// used as initial bounds of binary search +template +PerformanceSummary FindPeakPerformanceBinarySearch( + SystemUnderTest* sut, QuerySampleLibrary* qsl, SequenceGen* sequence_gen, + const LoadableSampleSet& performance_set, PerformanceSummary l_perf_summary, + PerformanceSummary u_perf_summary) { + if (find_peak_performance::IsFinished(l_perf_summary.settings, + u_perf_summary.settings)) { + return l_perf_summary; + } + + const TestSettingsInternal m_settings = + find_peak_performance::MidOfBoundaries(l_perf_summary.settings, + u_perf_summary.settings); + + LogDetail([l_field = + find_peak_performance::ToStringPerformanceField( + l_perf_summary.settings), + u_field = + find_peak_performance::ToStringPerformanceField( + u_perf_summary.settings), + m_field = + find_peak_performance::ToStringPerformanceField( + m_settings)](AsyncDetail& detail) { +#if USE_NEW_LOGGING_FORMAT + MLPERF_LOG( + detail, "generic_message", + "FindPeakPerformanceBinarySearch: Testing the mid value of bounds [" + + l_field + ", " + u_field + "): " + m_field); +#else + detail( + "FindPeakPerformanceBinarySearch: Testing the mid value of bounds [" + + l_field + ", " + u_field + "): " + m_field); +#endif + }); + + PerformanceResult m_pr(IssueQueries( + sut, m_settings, performance_set, sequence_gen)); + PerformanceSummary m_perf_summary{sut->Name(), m_settings, std::move(m_pr)}; + + std::string tmp; + if (m_perf_summary.PerfConstraintsMet(&tmp)) { + return FindPeakPerformanceBinarySearch( + sut, qsl, sequence_gen, performance_set, m_perf_summary, + u_perf_summary); + } else { + return FindPeakPerformanceBinarySearch( + sut, qsl, sequence_gen, performance_set, l_perf_summary, + m_perf_summary); + } +} + +/// \brief Runs the performance mode, templated by scenario. +template +void RunPerformanceMode(SystemUnderTest* sut, QuerySampleLibrary* qsl, + const TestSettingsInternal& settings, + SequenceGen* sequence_gen) { + LogDetail([](AsyncDetail& detail) { +#if USE_NEW_LOGGING_FORMAT + MLPERF_LOG(detail, "generic_message", "Starting performance mode"); +#else + detail("Starting performance mode:"); +#endif + }); + + // Use first loadable set as the performance set. + std::vector loadable_sets( + loadgen::GenerateLoadableSets(qsl, settings)); + const LoadableSampleSet& performance_set = loadable_sets.front(); + LoadSamplesToRam(qsl, performance_set.set); + + // Start PerfClock/system_clock timers for measuring performance interval + // for comparison vs external timer. + auto pc_start_ts = PerfClock::now(); + auto sc_start_ts = std::chrono::system_clock::now(); + if (settings.print_timestamps) { + std::cout << "Loadgen :: Perf mode start. system_clock Timestamp = " + << std::chrono::system_clock::to_time_t(sc_start_ts) << "\n" + << std::flush; + } + + PerformanceResult pr(IssueQueries( + sut, settings, performance_set, sequence_gen)); + + // Measure PerfClock/system_clock timer durations for comparison vs + // external timer. + auto pc_stop_ts = PerfClock::now(); + auto sc_stop_ts = std::chrono::system_clock::now(); + auto pc_duration = std::chrono::duration_cast( + pc_stop_ts - pc_start_ts) + .count(); + auto sc_duration = std::chrono::duration_cast( + sc_stop_ts - sc_start_ts) + .count(); + float pc_sc_ratio = static_cast(pc_duration) / sc_duration; + if (settings.print_timestamps) { + std::cout << "Loadgen :: Perf mode stop. systme_clock Timestamp = " + << std::chrono::system_clock::to_time_t(sc_stop_ts) << "\n" + << std::flush; + std::cout << "Loadgen :: PerfClock Perf duration = " << pc_duration + << "ms\n" + << std::flush; + std::cout << "Loadgen :: system_clock Perf duration = " << sc_duration + << "ms\n" + << std::flush; + std::cout << "Loadgen :: PerfClock/system_clock ratio = " << std::fixed + << std::setprecision(4) << pc_sc_ratio << "\n" + << std::flush; + } + + if (pc_sc_ratio > 1.01 || pc_sc_ratio < 0.99) { + LogDetail([pc_sc_ratio](AsyncDetail& detail) { +#if USE_NEW_LOGGING_FORMAT + std::stringstream ss; + ss << "PerfClock and system_clock differ by more than 1%! " + << " pc_sc_ratio: " << pc_sc_ratio; + MLPERF_LOG_ERROR(detail, "error_runtime", ss.str()); +#else + detail.Error("PerfClock and system_clock differ by more than 1\%! ", + "pc_sc_ratio", pc_sc_ratio); +#endif + }); + } else if (pc_sc_ratio > 1.001 || pc_sc_ratio < 0.999) { + LogDetail([pc_sc_ratio](AsyncDetail& detail) { +#if USE_NEW_LOGGING_FORMAT + std::stringstream ss; + ss << "PerfClock and system_clock differ by more than 0.1%! " + << " pc_sc_ratio: " << pc_sc_ratio; + MLPERF_LOG_WARNING(detail, "warning_generic_message", ss.str()); +#else + detail.Warning("PerfClock and system_clock differ by more than 0.1\%. ", + "pc_sc_ratio", pc_sc_ratio); +#endif + }); + } + + PerformanceSummary perf_summary{sut->Name(), settings, std::move(pr)}; + LogSummary([perf_summary](AsyncSummary& summary) mutable { + perf_summary.LogSummary(summary); + }); + // Create a copy to prevent thread hazard between LogSummary and LogDetail. + PerformanceSummary perf_summary_detail{perf_summary}; + LogDetail([perf_summary_detail](AsyncDetail& detail) mutable { + perf_summary_detail.LogDetail(detail); + }); + + qsl->UnloadSamplesFromRam(performance_set.set); +} + +/// \brief Runs the binary search mode, templated by scenario. +/// \details 1. Check whether lower bound from user satisfies the performance +/// constraints, 2. Find an upper bound using the function 'FindBoundaries' +/// based on the lower bound, 3. Find peak performance settings using the +/// function 'FindPeakPerformanceBinarySearch'. note: Since we can't find a +/// lower bound programmatically because of the monotonicity issue of Server +/// scenario, rely on user's settings. After resolving this issue, we can +/// make the function 'FindBoundaries' find a lower bound as well from some +/// random initial settings. +template +void FindPeakPerformanceMode(SystemUnderTest* sut, QuerySampleLibrary* qsl, + const TestSettingsInternal& base_settings, + SequenceGen* sequence_gen) { + LogDetail([](AsyncDetail& detail) { +#if USE_NEW_LOGGING_FORMAT + MLPERF_LOG(detail, "generic_message", "Starting FindPeakPerformance mode"); +#else + detail("Starting FindPeakPerformance mode:"); +#endif + }); + + if (scenario != TestScenario::Server) { + LogDetail([unsupported_scenario = ToString(scenario)](AsyncDetail& detail) { +#if USE_NEW_LOGGING_FORMAT + MLPERF_LOG_ERROR(detail, "error_invalid_config", + find_peak_performance::kNotSupportedMsg); +#else + detail.Error(find_peak_performance::kNotSupportedMsg); +#endif + }); + return; + } + + LogDetail( + [base_field = find_peak_performance::ToStringPerformanceField( + base_settings)](AsyncDetail& detail) { +#if USE_NEW_LOGGING_FORMAT + MLPERF_LOG( + detail, "generic_message", + "FindPeakPerformance: Check validity of the base settings field: " + + base_field); +#else + detail( + "FindPeakPerformance: Check validity of the base settings field: " + + base_field); +#endif + }); + + // 1. Check whether the lower bound came from user satisfy performance + // constraints or not. + std::vector base_loadable_sets( + loadgen::GenerateLoadableSets(qsl, base_settings)); + const LoadableSampleSet& base_performance_set = base_loadable_sets.front(); + LoadSamplesToRam(qsl, base_performance_set.set); + + PerformanceResult base_pr(IssueQueries( + sut, base_settings, base_performance_set, sequence_gen)); + PerformanceSummary base_perf_summary{sut->Name(), base_settings, + std::move(base_pr)}; + + // We can also use all_constraints_met to check performance constraints, + // but to reduce searching time, leave it up to whether the settings satisfy + // min duration & min queries or not to users. + std::string msg; + if (!base_perf_summary.PerfConstraintsMet(&msg)) { + LogDetail([msg](AsyncDetail& detail) { +#if USE_NEW_LOGGING_FORMAT + std::stringstream ss; + ss << "FindPeakPerformance: Initial lower bound does not satisfy " + << "performance constraints, msg: " << msg; + MLPERF_LOG_ERROR(detail, "error_runtime", ss.str()); +#else + detail.Error( + "FindPeakPerformance: Initial lower bound does not satisfy " + "performance constraints, msg: " + + msg); +#endif + }); + + PerformanceSummary perf_summary{sut->Name(), base_settings, + std::move(base_perf_summary.pr)}; + LogSummary([perf_summary](AsyncSummary& summary) mutable { + perf_summary.LogSummary(summary); + }); + // Create a copy to prevent thread hazard between LogSummary and LogDetail. + PerformanceSummary perf_summary_detail{perf_summary}; + LogDetail([perf_summary_detail](AsyncDetail& detail) mutable { + perf_summary_detail.LogDetail(detail); + }); + + qsl->UnloadSamplesFromRam(base_performance_set.set); + + return; + } + + // Clear loaded samples. + qsl->UnloadSamplesFromRam(base_performance_set.set); + + // 2. Find an upper bound based on the lower bound. + std::pair boundaries = + FindBoundaries(sut, qsl, sequence_gen, base_perf_summary); + PerformanceSummary l_perf_summary = boundaries.first; + PerformanceSummary u_perf_summary = boundaries.second; + + LogDetail( + [l_field = find_peak_performance::ToStringPerformanceField( + l_perf_summary.settings), + u_field = find_peak_performance::ToStringPerformanceField( + u_perf_summary.settings)](AsyncDetail& detail) { +#if USE_NEW_LOGGING_FORMAT + MLPERF_LOG(detail, "generic_message", + "FindPeakPerformance: Found boundaries: [" + l_field + ", " + + u_field + ")"); +#else + detail("FindPeakPerformance: Found boundaries: [" + l_field + ", " + + u_field + ")"); +#endif + }); + + // Reuse performance_set, u_perf_summary has the largest 'samples_per_query'. + std::vector loadable_sets( + loadgen::GenerateLoadableSets(qsl, u_perf_summary.settings)); + const LoadableSampleSet& performance_set = loadable_sets.front(); + LoadSamplesToRam(qsl, performance_set.set); + + // 3. Find peak performance settings using the found boundaries + PerformanceSummary perf_summary = FindPeakPerformanceBinarySearch( + sut, qsl, sequence_gen, performance_set, l_perf_summary, u_perf_summary); + + // Print-out the peak performance test setting. + LogDetail([field = find_peak_performance::ToStringPerformanceField( + perf_summary.settings)](AsyncDetail& detail) { +#if USE_NEW_LOGGING_FORMAT + MLPERF_LOG(detail, "generic_message", + "FindPeakPerformance: Found peak performance field: " + field); +#else + detail("FindPeakPerformance: Found peak performance field: " + field); +#endif + }); + + LogSummary([perf_summary](AsyncSummary& summary) mutable { + perf_summary.LogSummary(summary); + }); + // Create a copy to prevent thread hazard between LogSummary and LogDetail. + PerformanceSummary perf_summary_detail{perf_summary}; + LogDetail([perf_summary_detail](AsyncDetail& detail) mutable { + perf_summary_detail.LogDetail(detail); + }); + + qsl->UnloadSamplesFromRam(performance_set.set); +} + +/// \brief Runs the accuracy mode, templated by scenario. +template +void RunAccuracyMode(SystemUnderTest* sut, QuerySampleLibrary* qsl, + const TestSettingsInternal& settings, + SequenceGen* sequence_gen) { + LogDetail([](AsyncDetail& detail) { +#if USE_NEW_LOGGING_FORMAT + MLPERF_LOG(detail, "generic_message", "Starting accuracy mode"); +#else + detail("Starting accuracy mode:"); +#endif + }); + + std::vector loadable_sets( + loadgen::GenerateLoadableSets(qsl, settings)); + + for (auto& loadable_set : loadable_sets) { + { + auto tracer = MakeScopedTracer( + [count = loadable_set.set.size()](AsyncTrace& trace) { + trace("LoadSamples", "count", count); + }); + LoadSamplesToRam(qsl, loadable_set.set); + } + + PerformanceResult pr(IssueQueries( + sut, settings, loadable_set, sequence_gen)); + + { + auto tracer = MakeScopedTracer( + [count = loadable_set.set.size()](AsyncTrace& trace) { + trace("UnloadSampes", "count", count); + }); + qsl->UnloadSamplesFromRam(loadable_set.set); + } + } +} + +/// \brief Routes runtime scenario requests to the corresponding instances +/// of its templated mode functions. +struct RunFunctions { + using Signature = void(SystemUnderTest* sut, QuerySampleLibrary* qsl, + const TestSettingsInternal& settings, + SequenceGen* sequence_gen); + + template + static RunFunctions GetCompileTime() { + return {(RunAccuracyMode), + (RunPerformanceMode), + (FindPeakPerformanceMode)}; + } + + static RunFunctions Get(TestScenario run_time_scenario) { + switch (run_time_scenario) { + case TestScenario::SingleStream: + return GetCompileTime(); + case TestScenario::MultiStream: + return GetCompileTime(); + case TestScenario::Server: + return GetCompileTime(); + case TestScenario::Offline: + return GetCompileTime(); + } + // We should not reach this point. + assert(false); + return GetCompileTime(); + } + + Signature& accuracy; + Signature& performance; + Signature& find_peak_performance; +}; + +} // namespace loadgen + +void StartTest(SystemUnderTest* sut, QuerySampleLibrary* qsl, + const TestSettings& requested_settings, + const LogSettings& log_settings, + const std::string audit_config_filename) { + GlobalLogger().StartIOThread(); + + const std::string test_date_time = CurrentDateTimeISO8601(); + + loadgen::LogOutputs log_outputs(log_settings.log_output, test_date_time); + if (!log_outputs.CheckOutputs()) { + return; + } + + GlobalLogger().StartLogging(&log_outputs.summary_out, &log_outputs.detail_out, + &log_outputs.accuracy_out, + log_settings.log_output.copy_detail_to_stdout, + log_settings.log_output.copy_summary_to_stdout); + + GlobalLogger().SetUseTokens(requested_settings.use_token_latencies); + bool needs_first_token = + (requested_settings.scenario != TestScenario::Offline); + GlobalLogger().SetNeedsFirstToken(needs_first_token); + + if (log_settings.enable_trace) { + GlobalLogger().StartNewTrace(&log_outputs.trace_out, PerfClock::now()); + } + + // measure sut->Name() response time + PerfClock::time_point pre_get_sut_name_ts = PerfClock::now(); + const std::string& sut_name = sut->Name(); + PerfClock::time_point post_get_sut_name_ts = PerfClock::now(); + + auto get_sut_name_duration_ns = + std::chrono::duration_cast( + post_get_sut_name_ts - pre_get_sut_name_ts) + .count(); + + LogLoadgenVersion(); + LogDetail([sut, qsl, test_date_time, &sut_name, + &get_sut_name_duration_ns](AsyncDetail& detail) { +#if USE_NEW_LOGGING_FORMAT + MLPERF_LOG(detail, "test_datetime", test_date_time); + MLPERF_LOG(detail, "sut_name", sut_name); + MLPERF_LOG(detail, "get_sut_name_duration_ns", get_sut_name_duration_ns); + MLPERF_LOG(detail, "qsl_name", qsl->Name()); + MLPERF_LOG(detail, "qsl_reported_total_count", qsl->TotalSampleCount()); + MLPERF_LOG(detail, "qsl_reported_performance_count", + qsl->PerformanceSampleCount()); +#else + detail("Date + time of test: ", test_date_time); + detail("System Under Test (SUT) name: ", sut_name); + detail("Get SUT name time [ns]: ", get_sut_name_duration_ns); + detail("Query Sample Library (QSL) name: ", qsl->Name()); + detail("QSL total size: ", qsl->TotalSampleCount()); + detail("QSL performance size*: ", qsl->PerformanceSampleCount()); + detail("*TestSettings (performance_sample_count_override) can override"); + detail("*Refer to Effective Settings for actual value"); +#endif + }); + + TestSettings test_settings = requested_settings; + // Look for Audit Config file to override TestSettings during audit + if (FileExists(audit_config_filename)) { + LogDetail([](AsyncDetail& detail) { +#if USE_NEW_LOGGING_FORMAT + MLPERF_LOG_WARNING(detail, "warning_generic_message", + "Found Audit Config file (audit.config)." + " Overriding TestSettings from audit.config file."); +#else + detail( + "Found Audit Config file (audit.config)." + " Overriding TestSettings from audit.config file."); +#endif + }); + std::string audit_scenario = loadgen::ToString(test_settings.scenario); + // Remove Spaces from the string + RemoveValue(&audit_scenario, ' '); + const std::string generic_model = "*"; + test_settings.FromConfig(audit_config_filename, generic_model, + audit_scenario, 2); + } + if (test_settings.test05) { + // If the configuration indicates we are running test05, + // random seeds + LogDetail([](AsyncDetail& detail) { +#if USE_NEW_LOGGING_FORMAT + MLPERF_LOG_WARNING(detail, "warning_generic_message", + "Test05 flag detected" + " Overriding random seeds"); +#else + detail( + "Test05 flag detected" + " Overriding random seeds"); +#endif + }); + test_settings.mode = TestMode::PerformanceOnly; + test_settings.qsl_rng_seed = requested_settings.test05_qsl_rng_seed; + test_settings.sample_index_rng_seed = + requested_settings.test05_sample_index_rng_seed; + test_settings.schedule_rng_seed = + requested_settings.test05_schedule_rng_seed; + } + + loadgen::TestSettingsInternal sanitized_settings( + test_settings, qsl->PerformanceSampleCount()); + sanitized_settings.LogAllSettings(); + + auto run_funcs = loadgen::RunFunctions::Get(sanitized_settings.scenario); + + loadgen::SequenceGen sequence_gen; + switch (sanitized_settings.mode) { + case TestMode::SubmissionRun: + run_funcs.accuracy(sut, qsl, sanitized_settings, &sequence_gen); + run_funcs.performance(sut, qsl, sanitized_settings, &sequence_gen); + break; + case TestMode::AccuracyOnly: + run_funcs.accuracy(sut, qsl, sanitized_settings, &sequence_gen); + break; + case TestMode::PerformanceOnly: + run_funcs.performance(sut, qsl, sanitized_settings, &sequence_gen); + break; + case TestMode::FindPeakPerformance: + run_funcs.find_peak_performance(sut, qsl, sanitized_settings, + &sequence_gen); + break; + } + + loadgen::IssueQueryController::GetInstance().EndThreads(); + + // Stop tracing after logging so all logs are captured in the trace. + GlobalLogger().StopLogging(); + GlobalLogger().StopTracing(); + GlobalLogger().StopIOThread(); +} + +void AbortTest() { + loadgen::IssueQueryController::GetInstance().EndThreads(); + GlobalLogger().StopLogging(); + GlobalLogger().StopTracing(); + GlobalLogger().StopIOThread(); +} + +void QuerySamplesComplete(QuerySampleResponse* responses, size_t response_count, + const ResponseCallback& response_cb) { + PerfClock::time_point timestamp = PerfClock::now(); + + auto tracer = MakeScopedTracer( + [](AsyncTrace& trace) { trace("QuerySamplesComplete"); }); + + const QuerySampleResponse* end = responses + response_count; + + // Notify first to unblock loadgen production ASAP. + for (QuerySampleResponse* response = responses; response < end; response++) { + loadgen::SampleMetadata* sample = + reinterpret_cast(response->id); + loadgen::QueryMetadata* query = sample->query_metadata; + query->NotifyOneSampleCompleted(timestamp); + } + + // Log samples. + for (QuerySampleResponse* response = responses; response < end; response++) { + loadgen::SampleMetadata* sample = + reinterpret_cast(response->id); + loadgen::QueryMetadata* query = sample->query_metadata; + query->response_delegate->SampleComplete(sample, response, timestamp, + response_cb); + } + // PerfClock::time_point end_timestamp = PerfClock::now(); + // mlperf::samples_overhead_acum += (end_timestamp - timestamp).count(); +} + +void FirstTokenComplete(QuerySampleResponse* responses, size_t response_count, + const ResponseCallback& response_cb) { + PerfClock::time_point timestamp = PerfClock::now(); + + auto tracer = + MakeScopedTracer([](AsyncTrace& trace) { trace("FirstTokenComplete"); }); + + const QuerySampleResponse* end = responses + response_count; + + // Log samples. + for (QuerySampleResponse* response = responses; response < end; response++) { + loadgen::SampleMetadata* sample = + reinterpret_cast(response->id); + loadgen::QueryMetadata* query = sample->query_metadata; + query->response_delegate->TokenComplete(sample, response, timestamp, + response_cb); + } + // PerfClock::time_point end_timestamp = PerfClock::now(); + // mlperf::tokens_overhead_acum += (end_timestamp - timestamp).count(); +} + +} // namespace mlperf diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/loadgen.h b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/loadgen.h new file mode 100644 index 000000000..84e02656c --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/loadgen.h @@ -0,0 +1,103 @@ +/* Copyright 2019 The MLPerf Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +/// \file +/// \brief Provides the entry points for a SUT to start a test and respond +/// to issued queries. + +#ifndef MLPERF_LOADGEN_LOADGEN_H_ +#define MLPERF_LOADGEN_LOADGEN_H_ + +#include +#include +#include +#include + +/// \brief Contains the loadgen API. +namespace mlperf { + +struct QuerySampleResponse; +class QuerySampleLibrary; +class SystemUnderTest; +struct TestSettings; +struct LogSettings; + +using ResponseCallback = std::function; + +/// \addtogroup LoadgenAPI Loadgen API +/// @{ + +/// +/// \brief SUT calls this to notify loadgen of completed samples. +/// \details +/// * The samples may be from any combination of queries or partial queries as +/// issued by \link mlperf::SystemUnderTest::IssueQuery +/// +/// SystemUnderTest::IssueQuery \endlink. +/// * The SUT is responsible for owning and allocating the reponse data. The +/// loadgen will copy the response data if needed (e.g. for accuracy mode). +/// + If no response callback is provided, the response data must remain valid +/// for the entire duration of this call. +/// + The response callback is untimed; it is called for each response in +/// responses after the loadgen records the completion time and before the +/// loadgen copies the response data. The response callback enables the +/// loadgen to simulate response data being stored in accelerator DRAM. +/// After the response callback is called, response data must reside on the +/// host so that the loadgen can copy it. Submitters must seek prior +/// approval to use this feature of loadgen (refer to +/// https://github.com/mlcommons/inference_policies/blob/master/inference_rules.adoc#5-load-generator). +/// * All calls to QuerySampleComplete are thread-safe and wait-free bounded. +/// + Any number of threads can call QuerySampleComplete simultaneously. +/// + Regardless of where any other thread stalls, the current thread will +/// finish QuerySampleComplete in a bounded number of cycles. +/// + Note: If a callback is provided, the SUT must ensure that the callback +/// is also thread-safe and wait-free bounded for the above to hold. +void QuerySamplesComplete(QuerySampleResponse* responses, size_t response_count, + const ResponseCallback& response_cb = {}); + +void FirstTokenComplete(QuerySampleResponse* responses, size_t response_count, + const ResponseCallback& response_cb = {}); + +/// +/// \brief Starts the test against SUT with the specified settings. +/// \details This is the C++ entry point. See mlperf::c::StartTest for the +/// C entry point. +/// +void StartTest(SystemUnderTest* sut, QuerySampleLibrary* qsl, + const TestSettings& requested_settings, + const LogSettings& log_settings, + const std::string audit_config_filename = "audit.config"); + +/// +/// \brief Aborts the running test. +/// \details This function will stop issueing new samples to the SUT. StartTest +/// will return after the current inference finishes. Since StartTest is a +/// blocking function, this function can only be called in another thread. +void AbortTest(); + +/// +/// \brief Register a thread for query issuing in Server scenario. +/// \details If a thread registers itself, the thread(s) is used to call SUT's +/// IssueQuery(). This function is blocking until the entire test is done. The +/// number of registered threads must match server_num_issue_query_threads in +/// TestSettings. This function only has effect in Server scenario. +/// This is the C++ entry point. See mlperf::c::RegisterIssueQueryThread for the +/// C entry point. +/// +void RegisterIssueQueryThread(); +// inline long long samples_overhead_acum; +// inline long long tokens_overhead_acum; +/// @} + +} // namespace mlperf + +#endif // MLPERF_LOADGEN_LOADGEN_H_ diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/loadgen_integration_diagram.svg b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/loadgen_integration_diagram.svg new file mode 100644 index 000000000..17dd1b481 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/loadgen_integration_diagram.svg @@ -0,0 +1,85 @@ + + + + + + + + +Model + Dataset + + + +Pre Processor + + + +Post Processor + + + +Benchmark + + + +Backend + + + +LoadGen + + + + + + + + + + + + + + + + + + + + + + + +1 + + + +2 + + +3 + + +5 + + +4 + + + +LoadGen Logs + + + + + +6 + + + + + + + \ No newline at end of file diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/logging.cc b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/logging.cc new file mode 100644 index 000000000..807c1954a --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/logging.cc @@ -0,0 +1,1301 @@ +/* Copyright 2019 The MLPerf Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +/// \file +/// \brief Implements a logging system with a central IO thread that handles +/// all stringification and IO. +/// \details Log-producing threads only submit lambdas to be executed on the +/// IO thread. +/// All producers and consumers use lock-free operations that guarantee +/// forward progress independent of a) other stalled threads and b) where +/// those threads are stalled. +/// Each thread uses a double-buffering scheme to queue its logs. One buffer +/// is always reserved for writes and the other is reserved for reads. +/// A producing thread sends requests to the IOThread to swap the buffers +/// and the IOThread does the actual read/write swap after it has finished +/// reading the buffer it was working on. + +#include "logging.h" + +#include +#include +#include +#include +#include +#include +#include + +#if defined(_WIN32) || defined(WIN32) || defined(_WIN64) || defined(WIN64) +#define WIN32_LEAN_AND_MEAN +#define NOMINMAX +#include +#include +#define MLPERF_GET_PID() _getpid() +#else +#include +#define MLPERF_GET_PID() getpid() +#endif + +// Use system-level TID for tracing. This enables correlation with other +// performance tools that are not aware of C++ std::thread::id. +#if defined(__linux__) +#include +#define MLPERF_GET_TID() syscall(SYS_gettid) +#elif defined(_WIN32) || defined(WIN32) || defined(_WIN64) || defined(WIN64) +#define MLPERF_GET_TID() GetCurrentThreadId() +#elif defined(__APPLE__) +#define MLPERF_GET_TID() \ + std::hash{}(std::this_thread::get_id()) +#else +// TODO: std::this_thread::id is a class but MLPERF_GET_TID() assigned to +// uint64_t +#define MLPERF_GET_TID() std::this_thread::get_id() +#endif + +#include "utils.h" + +namespace mlperf { +namespace logging { + +namespace { + +uintptr_t SwapRequestSlotIsWritableValue(size_t id) { + // LSB of 1 indicates that this isn't a pointer. + // MSBs encode the id to detect collisions when a slot in + // |thread_swap_request_slots_| is reused for a different id and the request + // for the previous id is very slow. + return (id << 1) | 0x1; +} + +bool SwapRequestSlotIsReadable(uintptr_t value) { + // Valid pointers will not have their lsb set. + return (value & 0x1) != 0x1; +} + +constexpr size_t kMaxThreadsToLog = 1024; +constexpr std::chrono::milliseconds kLogPollPeriod(10); + +/// \brief How many log entries to pre-allocate per thread to help avoid +/// runtime allocation. +constexpr size_t kTlsLogReservedEntryCount = 1024; + +constexpr auto kInvalidLatency = std::numeric_limits::min(); +constexpr auto nTokenInvalid = std::numeric_limits::min(); + +} // namespace + +const std::string& ArgValueTransform(const bool& value) { + static const std::string v_true("true"); + static const std::string v_false("false"); + return value ? v_true : v_false; +} + +char Bin2Hex(uint8_t four_bits) { + char number = '0' + four_bits; + char letter = ('A' - 10) + four_bits; + return four_bits < 10 ? number : letter; +} + +const std::string ArgValueTransform(const LogBinaryAsHexString& value) { + if (value.data == nullptr) { + return "\"\""; + } + std::string hex; + hex.reserve(value.data->size() + 2); + hex.push_back('"'); + for (auto b : *value.data) { + hex.push_back(Bin2Hex(b >> 4)); + hex.push_back(Bin2Hex(b & 0x0F)); + } + hex.push_back('"'); + return hex; +} + +#if USE_NEW_LOGGING_FORMAT +const std::string ArgValueTransform(const std::string& value) { + return std::string("\"") + value + std::string("\""); +} + +const std::string ArgValueTransform(const char* value) { + return std::string("\"") + std::string(value) + std::string("\""); +} + +const std::string ArgValueTransform(const std::vector& value) { + std::string s("["); + for (auto i : value) { + s += std::to_string(i) + ","; + } + s.resize(s.size() - 1); + s += "]"; + return s; +} + +const std::string ArgValueTransform( + const std::map& value) { + std::string s("{"); + for (const auto& i : value) { + s += "\""; + s += i.first; + s += "\":\""; + s += i.second; + s += "\","; + } + s.resize(s.size() - 1); + s += "}"; + return s; +} + +const std::string ArgValueTransform(const float value) { + if (value == std::numeric_limits::infinity()) { + return "Infinity"; + } else if (value == -std::numeric_limits::infinity()) { + return "-Infinity"; + } else if (std::isnan(value)) { + return "NaN"; + } + return std::to_string(value); +} + +const std::string ArgValueTransform(const double value) { + if (value == std::numeric_limits::infinity()) { + return "Infinity"; + } else if (value == -std::numeric_limits::infinity()) { + return "-Infinity"; + } else if (std::isnan(value)) { + return "NaN"; + } + return std::to_string(value); +} +#endif + +ChromeTracer::ChromeTracer(std::ostream* out, PerfClock::time_point origin) + : out_(out), origin_(origin) { + WriteTraceEventHeader(); +} + +ChromeTracer::~ChromeTracer() { + WriteTraceEventFooter(); + out_->flush(); +} + +void ChromeTracer::WriteTraceEventHeader() { + // Times and durations are converted from nanoseconds to microseconds, use + // 3 decimal digits to preserve precision. + *out_ << std::fixed << std::setprecision(3) << "{\"traceEvents\":[\n"; +} + +void ChromeTracer::WriteTraceEventFooter() { + *out_ << "{\"name\":\"LastTrace\"}\n" + << "],\n" + << "\"displayTimeUnit\":\"ns\",\n" + << "\"otherData\":{\n" + << "\"ts\":" << Micros(origin_.time_since_epoch()).count() << ",\n" + << "\"version\":\"MLPerf LoadGen v1.0\"\n" + << "}\n" + << "}\n"; +} + +void AsyncLog::SetCurrentPidTid(uint64_t pid, uint64_t tid) { + current_pid_ = pid; + current_tid_ = tid; +} + +void AsyncLog::SetLogFiles(std::ostream* summary, std::ostream* detail, + std::ostream* accuracy, bool copy_detail_to_stdout, + bool copy_summary_to_stdout, + PerfClock::time_point log_origin) { + std::unique_lock lock(log_mutex_); + if (summary_out_ != &std::cerr) { + std::string warning_summary; + if (log_warning_count_ == 0) { + warning_summary = "\nNo warnings encountered during test.\n"; + } else if (log_warning_count_ == 1) { + warning_summary = "\n1 warning encountered. See detailed log.\n"; + } else if (log_warning_count_ != 0) { + warning_summary = "\n" + std::to_string(log_warning_count_) + + " warnings encountered. See detailed log.\n"; + } + + std::string error_summary; + if (log_error_count_ == 0) { + error_summary = "\nNo errors encountered during test.\n"; + } else if (log_error_count_ == 1) { + error_summary = "\n1 ERROR encountered. See detailed log.\n"; + } else if (log_error_count_ != 0) { + error_summary = "\n" + std::to_string(log_error_count_) + + " ERRORS encountered. See detailed log.\n"; + } + + *summary_out_ << warning_summary << error_summary; + if (copy_summary_to_stdout_) { + std::cout << warning_summary << error_summary; + } + } + if (summary_out_) { + summary_out_->flush(); + } + if (detail_out_) { + detail_out_->flush(); + } + if (accuracy_out_ != &std::cerr) { + WriteAccuracyFooterLocked(); + accuracy_out_->flush(); + } + summary_out_ = summary; + detail_out_ = detail; + accuracy_out_ = accuracy; + if (accuracy_out_ != &std::cerr) { + WriteAccuracyHeaderLocked(); + } + copy_detail_to_stdout_ = copy_detail_to_stdout; + copy_summary_to_stdout_ = copy_summary_to_stdout; + log_origin_ = log_origin; + log_error_count_ = 0; + log_warning_count_ = 0; +} + +void AsyncLog::StartNewTrace(std::ostream* trace_out, + PerfClock::time_point origin) { + std::unique_lock lock(trace_mutex_); + if (trace_out) { + tracer_ = std::make_unique(trace_out, origin); + } else { + tracer_.reset(); + } +} + +void AsyncLog::StopTrace() { + std::unique_lock lock(trace_mutex_); + tracer_.reset(); +} + +void AsyncLog::LogAccuracy(uint64_t seq_id, const QuerySampleIndex qsl_idx, + const LogBinaryAsHexString& response, + int64_t n_tokens = 0) { + std::unique_lock lock(log_mutex_); + if (!accuracy_out_) { + return; + } + *accuracy_out_ << (accuracy_needs_comma_ ? ",\n{ " : "\n{ "); + if (!use_tokens_) { + LogArgs(accuracy_out_, "seq_id", seq_id, "qsl_idx", qsl_idx, "data", + response); + } else if (!needs_first_token_) { + LogArgs(accuracy_out_, "seq_id", seq_id, "qsl_idx", qsl_idx, "data", + response, "token_count", n_tokens); + } else { + const size_t i = seq_id - latencies_first_sample_sequence_id_; + LogArgs(accuracy_out_, "seq_id", seq_id, "qsl_idx", qsl_idx, "data", + response, "token_data", token_records_[i], "token_count", n_tokens); + } + + *accuracy_out_ << " }"; + accuracy_needs_comma_ = true; +} + +void AsyncLog::CacheToken(uint64_t seq_id, + const LogBinaryAsHexString& response) { + std::unique_lock lock(token_record_mutex_); + const size_t i = seq_id - latencies_first_sample_sequence_id_; + if (token_records_.size() <= i) { + token_records_.resize(i + 1); + } + token_records_[i] = response; +} + +void AsyncLog::Flush() { + { + std::unique_lock lock(log_mutex_); + if (summary_out_) { + summary_out_->flush(); + } + if (detail_out_) { + detail_out_->flush(); + } + if (accuracy_out_) { + accuracy_out_->flush(); + } + } + + { + std::unique_lock lock(trace_mutex_); + if (tracer_) { + tracer_->Flush(); + } + } +} + +void AsyncLog::WriteAccuracyHeaderLocked() { + *accuracy_out_ << "["; + accuracy_needs_comma_ = false; +} + +void AsyncLog::WriteAccuracyFooterLocked() { *accuracy_out_ << "\n]\n"; } + +void AsyncLog::RestartLatencyRecording(uint64_t first_sample_sequence_id, + size_t latencies_to_reserve) { + std::unique_lock lock(latencies_mutex_); + assert(latencies_.empty()); + assert(latencies_recorded_ == latencies_expected_); + latencies_recorded_ = 0; + latencies_expected_ = 0; + max_latency_ = 0; + max_completion_timstamp_ = PerfClock::now(); + latencies_first_sample_sequence_id_ = first_sample_sequence_id; + latencies_.reserve(latencies_to_reserve); + token_latencies_.reserve(latencies_to_reserve); + tokens_per_sample_.reserve(latencies_to_reserve); + time_per_output_token_.reserve(latencies_to_reserve); +} + +void AsyncLog::RecordSampleCompletion(uint64_t sample_sequence_id, + PerfClock::time_point completion_time, + QuerySampleLatency latency, + int64_t n_tokens = 0) { + std::unique_lock lock(latencies_mutex_); + + max_latency_ = std::max(max_latency_, latency); + + max_completion_timstamp_ = + std::max(max_completion_timstamp_, completion_time); + + if (sample_sequence_id < latencies_first_sample_sequence_id_) { + // Call LogErrorSync here since this kind of error could result in a + // segfault in the near future. +#if USE_NEW_LOGGING_FORMAT + std::stringstream ss; + ss << "Received completion for an old sample." + << " Min expected id: " << latencies_first_sample_sequence_id_ + << " Actual id: " << sample_sequence_id; + MLPERF_LOG_ERROR_SYNC(GlobalLogger(), "error_runtime", ss.str()); +#else + GlobalLogger().LogErrorSync( + "Received completion for an old sample.", "Min expected id", + latencies_first_sample_sequence_id_, "Actual id", sample_sequence_id); +#endif + return; + } + + const size_t i = sample_sequence_id - latencies_first_sample_sequence_id_; + + if (latencies_.size() <= i) { + // TODO: Reserve in advance. + latencies_.resize(i + 1, kInvalidLatency); + } else if (latencies_[i] != kInvalidLatency) { + // Call LogErrorSync here since this kind of error could result in a + // segfault in the near future. +#if USE_NEW_LOGGING_FORMAT + MLPERF_LOG_ERROR_SYNC(GlobalLogger(), "error_runtime", + "Attempted to complete a sample twice."); +#else + GlobalLogger().LogErrorSync("Attempted to complete a sample twice."); +#endif + + // Return without recording the latency again to avoid potentially + // ending the test before the SUT is actually done, which could result + // in a segfault. + // If the SUT recorded the wrong sample, the test will hang and see + // the error above. + return; + } + + if (use_tokens_) { + if (needs_first_token_ && (token_latencies_.size() <= i)) { + MLPERF_LOG_ERROR_SYNC(GlobalLogger(), "error_runtime", + "Attempted to record a sample latency before it's " + "first token latency"); + } else if (needs_first_token_ && (token_latencies_[i] == kInvalidLatency)) { + MLPERF_LOG_ERROR_SYNC(GlobalLogger(), "error_runtime", + "Attempted to record a sample latency before it's " + "first token latency"); + } + + if (tokens_per_sample_.size() <= i) { + // TODO: Reserve in advance. + tokens_per_sample_.resize(i + 1, nTokenInvalid); + } else if (tokens_per_sample_[i] != nTokenInvalid) { + // Call LogErrorSync here since this kind of error could result in a + // segfault in the near future. +#if USE_NEW_LOGGING_FORMAT + MLPERF_LOG_ERROR_SYNC(GlobalLogger(), "error_runtime", + "Attempted to complete a sample twice."); +#else + GlobalLogger().LogErrorSync("Attempted to complete a sample twice."); +#endif + + // Return without recording the latency again to avoid potentially + // ending the test before the SUT is actually done, which could result + // in a segfault. + // If the SUT recorded the wrong sample, the test will hang and see + // the error above. + return; + } + if (n_tokens == 0) { + MLPERF_LOG_ERROR_SYNC(GlobalLogger(), "error_runtime", + "n_tokens argument missing or attempted to record " + "0 as number of tokens"); + } else if (n_tokens < 0) { + MLPERF_LOG_ERROR_SYNC(GlobalLogger(), "error_runtime", + "Attempted to record a negative number of tokens"); + n_tokens = 0; + } else if (n_tokens == 1) { + MLPERF_LOG_ERROR_SYNC(GlobalLogger(), "error_runtime", + "Number of tokens need to be greater than 1"); + n_tokens = 0; + } + if (time_per_output_token_.size() <= i) { + time_per_output_token_.resize(i + 1, kInvalidLatency); + } else if (time_per_output_token_[i] != kInvalidLatency) { + // Call LogErrorSync here since this kind of error could result in a + // segfault in the near future. +#if USE_NEW_LOGGING_FORMAT + MLPERF_LOG_ERROR_SYNC(GlobalLogger(), "error_runtime", + "Attempted to complete a sample twice."); +#else + GlobalLogger().LogErrorSync("Attempted to complete a sample twice."); +#endif + + // Return without recording the latency again to avoid potentially + // ending the test before the SUT is actually done, which could result + // in a segfault. + // If the SUT recorded the wrong sample, the test will hang and see + // the error above. + return; + } + tokens_per_sample_[i] = n_tokens; + time_per_output_token_[i] = + (latency - token_latencies_[i]) / (n_tokens - 1); + } + latencies_[i] = latency; + latencies_recorded_++; + if (AllLatenciesRecorded()) { + all_latencies_recorded_.notify_all(); + } +} + +void AsyncLog::RecordTokenCompletion(uint64_t sample_sequence_id, + PerfClock::time_point completion_time, + QuerySampleLatency latency) { + std::unique_lock lock(token_latencies_mutex_); + // std::unique_lock lock(latencies_mutex_); + // max_latency_ = std::max(max_latency_, latency); + + // max_completion_timstamp_ = + // std::max(max_completion_timstamp_, completion_time); + + if (sample_sequence_id < latencies_first_sample_sequence_id_) { + // Call LogErrorSync here since this kind of error could result in a + // segfault in the near future. +#if USE_NEW_LOGGING_FORMAT + std::stringstream ss; + ss << "Received completion for an old sample." + << " Min expected id: " << latencies_first_sample_sequence_id_ + << " Actual id: " << sample_sequence_id; + MLPERF_LOG_ERROR_SYNC(GlobalLogger(), "error_runtime", ss.str()); +#else + GlobalLogger().LogErrorSync( + "Received completion for an old sample.", "Min expected id", + latencies_first_sample_sequence_id_, "Actual id", sample_sequence_id); +#endif + return; + } + + const size_t i = sample_sequence_id - latencies_first_sample_sequence_id_; + + if (latencies_.size() > i) { + if (latencies_[i] != kInvalidLatency) { +#if USE_NEW_LOGGING_FORMAT + MLPERF_LOG_ERROR_SYNC( + GlobalLogger(), "error_runtime", + "Attempted to record token latency after sample was completed"); +#else + GlobalLogger().LogErrorSync( + "Attempted to record token latency after sample was completed"); +#endif + + // Return without recording the latency again to avoid potentially + // ending the test before the SUT is actually done, which could result + // in a segfault. + // If the SUT recorded the wrong sample, the test will hang and see + // the error above. + return; + } + } + if (token_latencies_.size() <= i) { + // TODO: Reserve in advance. + token_latencies_.resize(i + 1, kInvalidLatency); + } else if (token_latencies_[i] != kInvalidLatency) { + // Call LogErrorSync here since this kind of error could result in a + // segfault in the near future. +#if USE_NEW_LOGGING_FORMAT + MLPERF_LOG_ERROR_SYNC(GlobalLogger(), "error_runtime", + "Attempted to complete a sample twice."); +#else + GlobalLogger().LogErrorSync("Attempted to complete a sample twice."); +#endif + + // Return without recording the latency again to avoid potentially + // ending the test before the SUT is actually done, which could result + // in a segfault. + // If the SUT recorded the wrong sample, the test will hang and see + // the error above. + return; + } + token_latencies_[i] = latency; +} + +std::vector AsyncLog::GetLatenciesBlocking( + size_t expected_count) { + std::vector latencies; + { + std::unique_lock lock(latencies_mutex_); + latencies_expected_ = expected_count; + all_latencies_recorded_.wait(lock, [&] { return AllLatenciesRecorded(); }); + latencies.swap(latencies_); + } + + if (latencies.size() != expected_count) { + // Call LogErrorSync here since this kind of error could result in a + // segfault in the near future. +#if USE_NEW_LOGGING_FORMAT + std::stringstream ss; + ss << "Received SequenceId that was too large." + << " expected_size: " << expected_count + << " actual_size: " << latencies.size(); + MLPERF_LOG_ERROR_SYNC(GlobalLogger(), "error_runtime", ss.str()); +#else + GlobalLogger().LogErrorSync("Received SequenceId that was too large.", + "expected_size", expected_count, "actual_size", + latencies.size()); +#endif + } + + size_t invalid_latency_count = 0; + for (auto l : latencies) { + if (l == kInvalidLatency) { + invalid_latency_count++; + } + } + if (invalid_latency_count != 0) { + // Call LogErrorSync here since this kind of error could result in a + // segfault in the near future. +#if USE_NEW_LOGGING_FORMAT + std::stringstream ss; + ss << "Encountered incomplete samples at the end of a series of queries." + << " count: " << invalid_latency_count; + MLPERF_LOG_ERROR_SYNC(GlobalLogger(), "error_runtime", ss.str()); +#else + GlobalLogger().LogErrorSync( + "Encountered incomplete samples at the end of a series of queries.", + "count", invalid_latency_count); +#endif + } + + return latencies; +} + +std::vector AsyncLog::GetTokenLatencies( + size_t expected_count) { + std::vector token_latencies; + token_latencies.swap(token_latencies_); + return token_latencies; +} + +std::vector AsyncLog::GetTimePerOutputToken( + size_t expected_count) { + std::vector tpot_latencies; + tpot_latencies.swap(time_per_output_token_); + return tpot_latencies; +} + +std::vector AsyncLog::GetTokensPerSample(size_t expected_count) { + std::vector tokens_per_sample; + tokens_per_sample.swap(tokens_per_sample_); + return tokens_per_sample; +} + +PerfClock::time_point AsyncLog::GetMaxCompletionTime() { + return max_completion_timstamp_; +} + +QuerySampleLatency AsyncLog::GetMaxLatencySoFar() { + std::unique_lock lock(latencies_mutex_); + return max_latency_; +} + +void AsyncLog::SetUseTokens(bool use_tokens) { use_tokens_ = use_tokens; } + +void AsyncLog::SetNeedsFirstToken(bool needs_first_token) { + needs_first_token_ = needs_first_token; +} + +/// \brief Records a single thread using thread-local storage and submits +/// entries to the central Logger. +/// +/// \details This setup allows for each log entry to be added: +/// * With forward-progress guarantees. (i.e.: no locking or blocking +/// operations even if other threads have stalled.) +/// * Without expensive syscalls or I/O operations, which are deferred to +/// the central Logger. +class TlsLogger { + public: + TlsLogger(std::function forced_detatch); + ~TlsLogger(); + void ForcedDetatchFromThread() { forced_detatch_(); } + + void Log(AsyncLogEntry&& entry); + void SwapBuffers(); + + std::vector* StartReadingEntries(); + void FinishReadingEntries(); + bool ReadBufferHasBeenConsumed(); + size_t MaxEntryVectorSize() { return max_entry_size_; } + + uint64_t Pid() const { return pid_; } + uint64_t Tid() const { return tid_; } + + void RequestSwapBuffersSlotRetried() { + swap_buffers_slot_retry_count_.fetch_add(1, std::memory_order_relaxed); + } + + size_t ReportLogCasFailCount() { + size_t c = log_cas_fail_count_.load(std::memory_order_relaxed); + log_cas_fail_count_.fetch_sub(c, std::memory_order_relaxed); + return c; + } + + size_t ReportSwapBuffersSlotRetryCount() { + size_t c = swap_buffers_slot_retry_count_.load(std::memory_order_relaxed); + swap_buffers_slot_retry_count_.fetch_sub(c, std::memory_order_relaxed); + return c; + } + + void TraceCounters(); + + private: + using EntryVector = std::vector; + enum class EntryState { Unlocked, ReadLock, WriteLock }; + + // Accessed by producer only. + size_t i_read_ = 0; + + // Accessed by producer and consumer atomically. + EntryVector entries_[2]; + std::atomic entry_states_[2]{{EntryState::ReadLock}, + {EntryState::Unlocked}}; + std::atomic i_write_{1}; + + std::atomic log_cas_fail_count_{0}; + std::atomic swap_buffers_slot_retry_count_{0}; + + // Accessed by consumer only. + size_t unread_swaps_ = 0; + size_t i_write_prev_ = 0; + uint64_t pid_; + uint64_t tid_; + size_t max_entry_size_ = kTlsLogReservedEntryCount; + + std::function forced_detatch_; +}; + +Logger::Logger(std::chrono::duration poll_period, + size_t max_threads_to_log) + : poll_period_(poll_period), + max_threads_to_log_(max_threads_to_log), + thread_swap_request_slots_(max_threads_to_log * 2) { + const size_t kSlotCount = max_threads_to_log * 2; + for (size_t i = 0; i < kSlotCount; i++) { + std::atomic_init(&thread_swap_request_slots_[i], + SwapRequestSlotIsWritableValue(i)); + } +} + +Logger::~Logger() { + // TlsLoggers might outlive this Logger when loaded as a python module. + // Forcefully make all currently registered TlsLoggers orphans. + std::unique_lock lock(tls_loggers_registerd_mutex_); + TlsLogger* tls_logger_prev = nullptr; + (void)tls_logger_prev; // Avoid unused error in release builds. + while (!tls_loggers_registerd_.empty()) { + TlsLogger* tls_logger = *tls_loggers_registerd_.begin(); + // Otherwise, this is an infinite loop. + assert(tls_logger != tls_logger_prev); + tls_loggers_registerd_mutex_.unlock(); + tls_logger->ForcedDetatchFromThread(); + tls_loggers_registerd_mutex_.lock(); + tls_logger_prev = tls_logger; + } +} + +void Logger::RequestSwapBuffers(TlsLogger* tls_logger) { + auto tls_logger_as_uint = reinterpret_cast(tls_logger); + assert(SwapRequestSlotIsReadable(tls_logger_as_uint)); + size_t id, slot; + uintptr_t slot_is_writeable_value; + // The compare_exchange below should almost always succeed. + // The compare_exchange may fail if a recycled slot is still actively used + // by another thread, so we retry with subsequent slots here if needed. + // Since the slot count is 2x the expected number of threads to log, + // the CAS should only fail at most 50% of the time when all logging threads + // happen to be descheduled between the fetch_add and CAS below, which is + // very unlikely. + id = swap_request_id_.fetch_add(1, std::memory_order_relaxed); + slot = id % thread_swap_request_slots_.size(); + slot_is_writeable_value = SwapRequestSlotIsWritableValue(id); + while (!thread_swap_request_slots_[slot].compare_exchange_strong( + slot_is_writeable_value, tls_logger_as_uint, std::memory_order_release)) { + id = swap_request_id_.fetch_add(1, std::memory_order_relaxed); + slot = id % thread_swap_request_slots_.size(); + slot_is_writeable_value = SwapRequestSlotIsWritableValue(id); + tls_logger->RequestSwapBuffersSlotRetried(); + } +} + +void Logger::RegisterTlsLogger(TlsLogger* tls_logger) { + std::unique_lock lock(tls_loggers_registerd_mutex_); + if (tls_loggers_registerd_.size() >= max_threads_to_log_) { +#if USE_NEW_LOGGING_FORMAT + MLPERF_LOG_ERROR_SYNC((*this), "error_runtime", + "Warning: More TLS loggers registerd than can be " + "active simultaneously."); +#else + LogErrorSync( + "Warning: More TLS loggers registerd than can " + "be active simultaneously.\n"); +#endif + } + tls_loggers_registerd_.insert(tls_logger); +} + +// This moves ownership of the tls_logger data to Logger so the +// exiting thread can exit immediately, even if all the logs of the +// exiting thread haven't been processed. +void Logger::UnRegisterTlsLogger(std::unique_ptr tls_logger) { + OrphanContainer::iterator orphan; + { + std::unique_lock lock(tls_logger_orphans_mutex_); + tls_logger_orphans_.emplace_front(std::move(tls_logger)); + orphan = tls_logger_orphans_.begin(); + } + + // Only remove the TlsLogger from the registry after adding to orphans so + // CollectTlsLoggerStats doesn't have any gaps in coverage. + { + std::unique_lock lock(tls_loggers_registerd_mutex_); + tls_loggers_registerd_.erase(orphan->get()); + } + + // This will flush the logs of |tls_logger| and mark it for destruction. + // Deferring destruction via orphans_to_destroy helps avoid use-after-frees + // when the IOThread calls FinishReadingEntries. + (*orphan)->Log([this, orphan](AsyncLog&) { + CollectTlsLoggerStats(orphan->get()); + orphans_to_destroy_.push_back(orphan); + }); +} + +void Logger::CollectTlsLoggerStats(TlsLogger* tls_logger) { + tls_total_log_cas_fail_count_ += tls_logger->ReportLogCasFailCount(); + tls_total_swap_buffers_slot_retry_count_ += + tls_logger->ReportSwapBuffersSlotRetryCount(); + + size_t max_entry_vector_size = tls_logger->MaxEntryVectorSize(); + if (max_entry_vector_size > kTlsLogReservedEntryCount) { +#if USE_NEW_LOGGING_FORMAT + std::stringstream msg; + msg << "Logging allocation detected:" << " tid: " << tls_logger->Tid() + << " reserved_entries: " << kTlsLogReservedEntryCount + << " max_entries: " << max_entry_vector_size; + MLPERF_LOG_WARNING((*this), "warning_generic_message", msg.str()); +#else + async_logger_.FlagWarning(); + async_logger_.LogDetail("Logging allocation detected: ", "tid", + tls_logger->Tid(), "reserved_entries", + kTlsLogReservedEntryCount, "max_entries", + max_entry_vector_size); +#endif + } +} + +void Logger::StartIOThread() { + { + std::unique_lock lock(io_thread_mutex_); + keep_io_thread_alive_ = true; + } + io_thread_ = std::thread(&Logger::IOThread, this); +} + +void Logger::StopIOThread() { + { + std::unique_lock lock(io_thread_mutex_); + keep_io_thread_alive_ = false; + io_thread_cv_.notify_all(); + } + io_thread_.join(); +} + +void Logger::StartLogging(std::ostream* summary, std::ostream* detail, + std::ostream* accuracy, bool copy_detail_to_stdout, + bool copy_summary_to_stdout) { + async_logger_.SetLogFiles(summary, detail, accuracy, copy_detail_to_stdout, + copy_summary_to_stdout, PerfClock::now()); +} + +void Logger::StopLogging() { + if (std::this_thread::get_id() == io_thread_.get_id()) { +#if USE_NEW_LOGGING_FORMAT + MLPERF_LOG_ERROR_SYNC((*this), "error_runtime", + "StopLogging() not supported from IO thread."); +#else + LogErrorSync("StopLogging() not supported from IO thread."); +#endif + return; + } + + // Flush logs from this thread. + std::promise io_thread_flushed_this_thread; + Log([&](AsyncLog&) { io_thread_flushed_this_thread.set_value(); }); + io_thread_flushed_this_thread.get_future().wait(); + async_logger_.SetLogFiles(&std::cerr, &std::cerr, &std::cerr, false, false, + PerfClock::now()); +} + +void Logger::StartNewTrace(std::ostream* trace_out, + PerfClock::time_point origin) { + async_logger_.StartNewTrace(trace_out, origin); +} + +void Logger::StopTracing() { + // Flush traces from this thread. + std::promise io_thread_flushed_this_thread; + Log([&](AsyncLog&) { io_thread_flushed_this_thread.set_value(); }); + io_thread_flushed_this_thread.get_future().wait(); + async_logger_.StopTrace(); +} + +void Logger::LogContentionAndAllocations() { + LogDetail([&](AsyncDetail& detail) { + { + std::unique_lock lock(tls_loggers_registerd_mutex_); + for (auto tls_logger : tls_loggers_registerd_) { + CollectTlsLoggerStats(tls_logger); + } + } + + { + std::unique_lock lock(tls_logger_orphans_mutex_); + for (auto& orphan : tls_logger_orphans_) { + CollectTlsLoggerStats(orphan.get()); + } + } + +#if USE_NEW_LOGGING_FORMAT + MLPERF_LOG(detail, "logger_swap_request_slots_retry_count", + swap_request_slots_retry_count_); + MLPERF_LOG(detail, "logger_swap_request_slots_retry_retry_count", + swap_request_slots_retry_retry_count_); + MLPERF_LOG(detail, "logger_swap_request_slots_retry_reencounter_count", + swap_request_slots_retry_reencounter_count_); + MLPERF_LOG(detail, "logger_start_reading_entries_retry_count", + start_reading_entries_retry_count_); + MLPERF_LOG(detail, "logger_tls_total_log_cas_fail_count", + tls_total_log_cas_fail_count_); + MLPERF_LOG(detail, "logger_tls_total_swap_buffers_slot_retry_count", + tls_total_swap_buffers_slot_retry_count_); +#else + detail("Log Contention Counters:"); + detail(std::to_string(swap_request_slots_retry_count_) + + " : swap_request_slots_retry_count"); + detail(std::to_string(swap_request_slots_retry_retry_count_) + + " : swap_request_slots_retry_retry_count"); + detail(std::to_string(swap_request_slots_retry_reencounter_count_) + + " : swap_request_slots_retry_reencounter_count"); + detail(std::to_string(start_reading_entries_retry_count_) + + " : start_reading_entries_retry_count"); + detail(std::to_string(tls_total_log_cas_fail_count_) + + " : tls_total_log_cas_fail_count"); + detail(std::to_string(tls_total_swap_buffers_slot_retry_count_) + + " : tls_total_swap_buffers_slot_retry_count"); +#endif + + swap_request_slots_retry_count_ = 0; + swap_request_slots_retry_retry_count_ = 0; + swap_request_slots_retry_reencounter_count_ = 0; + start_reading_entries_retry_count_ = 0; + tls_total_log_cas_fail_count_ = 0; + tls_total_swap_buffers_slot_retry_count_ = 0; + }); +} + +void Logger::RestartLatencyRecording(uint64_t first_sample_sequence_id, + size_t latencies_to_reserve) { + async_logger_.RestartLatencyRecording(first_sample_sequence_id, + latencies_to_reserve); +} + +std::vector Logger::GetLatenciesBlocking( + size_t expected_count) { + return async_logger_.GetLatenciesBlocking(expected_count); +} +std::vector Logger::GetTokenLatencies( + size_t expected_count) { + return async_logger_.GetTokenLatencies(expected_count); +} +std::vector Logger::GetTimePerOutputToken( + size_t expected_count) { + return async_logger_.GetTimePerOutputToken(expected_count); +} +std::vector Logger::GetTokensPerSample( + size_t expected_count) { + return async_logger_.GetTokensPerSample(expected_count); +} + +PerfClock::time_point Logger::GetMaxCompletionTime() { + return async_logger_.GetMaxCompletionTime(); +} + +QuerySampleLatency Logger::GetMaxLatencySoFar() { + return async_logger_.GetMaxLatencySoFar(); +} + +void Logger::SetUseTokens(bool use_tokens) { + async_logger_.SetUseTokens(use_tokens); +} + +void Logger::SetNeedsFirstToken(bool needs_first_token) { + async_logger_.SetNeedsFirstToken(needs_first_token); +} + +TlsLogger* Logger::GetTlsLoggerThatRequestedSwap(size_t slot, size_t next_id) { + uintptr_t slot_value = thread_swap_request_slots_[slot].load(); + if (SwapRequestSlotIsReadable(slot_value)) { + // TODO: Convert this block to a simple write once we are confidient + // that we don't need to check for success. + bool success = thread_swap_request_slots_[slot].compare_exchange_strong( + slot_value, SwapRequestSlotIsWritableValue(next_id)); + if (!success) { +#if USE_NEW_LOGGING_FORMAT + MLPERF_LOG_WARNING((*this), "warning_generic_message", "CAS failed."); +#else + LogErrorSync("CAS failed.", "line", __LINE__); +#endif + assert(success); + } + return reinterpret_cast(slot_value); + } + return nullptr; +} + +void Logger::GatherRetrySwapRequests(std::vector* threads_to_swap) { + if (swap_request_slots_to_retry_.empty()) { + return; + } + + std::vector retry_slots; + retry_slots.swap(swap_request_slots_to_retry_); + for (auto& slot_retry : retry_slots) { + TlsLogger* tls_logger = + GetTlsLoggerThatRequestedSwap(slot_retry.slot, slot_retry.next_id); + if (tls_logger) { + threads_to_swap->push_back(tls_logger); + } else { + swap_request_slots_to_retry_.push_back(slot_retry); + swap_request_slots_retry_retry_count_++; + } + } +} + +void Logger::GatherNewSwapRequests(std::vector* threads_to_swap) { + auto swap_request_end = swap_request_id_.load(std::memory_order_acquire); + for (; swap_request_id_read_ < swap_request_end; swap_request_id_read_++) { + size_t slot = swap_request_id_read_ % thread_swap_request_slots_.size(); + size_t next_id = swap_request_id_read_ + thread_swap_request_slots_.size(); + TlsLogger* tls_logger = GetTlsLoggerThatRequestedSwap(slot, next_id); + if (tls_logger) { + threads_to_swap->push_back(tls_logger); + } else { + swap_request_slots_retry_count_++; + // A thread is in the middle of its call to RequestSwapBuffers. + // Retry later once it's done. + auto it = std::find_if(swap_request_slots_to_retry_.begin(), + swap_request_slots_to_retry_.end(), + [=](SlotRetry& s) { return s.slot == slot; }); + if (it == swap_request_slots_to_retry_.end()) { + // This is the first time we are retrying the slot. + swap_request_slots_to_retry_.push_back({slot, next_id}); + } else { + // Whoa. We've been retrying this slot since the last time it was + // encountered. Just update the next_id. + it->next_id = next_id; + swap_request_slots_retry_reencounter_count_++; + } + } + } +} + +void Logger::IOThread() { + while (keep_io_thread_alive_) { + auto tracer1 = + MakeScopedTracer([](AsyncTrace& trace) { trace("IOThreadLoop"); }); + { + auto tracer2 = MakeScopedTracer([](AsyncTrace& trace) { trace("Wait"); }); + std::unique_lock lock(io_thread_mutex_); + io_thread_cv_.wait_for(lock, poll_period_, + [&] { return !keep_io_thread_alive_; }); + } + + { + auto tracer3 = + MakeScopedTracer([](AsyncTrace& trace) { trace("Gather"); }); + std::vector threads_to_swap; + threads_to_swap.swap(threads_to_swap_deferred_); + GatherRetrySwapRequests(&threads_to_swap); + GatherNewSwapRequests(&threads_to_swap); + for (TlsLogger* thread : threads_to_swap) { + if (thread->ReadBufferHasBeenConsumed()) { + thread->SwapBuffers(); + // After swapping a thread, it's ready to be read. + threads_to_read_.push_back(thread); + } else { + // Don't swap buffers again until we've finish reading the + // previous swap. + threads_to_swap_deferred_.push_back(thread); + } + } + } + + { + auto tracer4 = + MakeScopedTracer([](AsyncTrace& trace) { trace("Process"); }); + // Read from the threads we are confident have activity. + for (std::vector::iterator thread = threads_to_read_.begin(); + thread != threads_to_read_.end(); thread++) { + auto tracer5 = + MakeScopedTracer([tid = (*thread)->Tid()](AsyncTrace& trace) { + trace("Thread", "tid", tid); + }); + std::vector* entries = (*thread)->StartReadingEntries(); + if (!entries) { + start_reading_entries_retry_count_++; + continue; + } + + async_logger_.SetCurrentPidTid((*thread)->Pid(), (*thread)->Tid()); + for (auto& entry : *entries) { + // Execute the entry to perform the serialization and I/O. + entry(async_logger_); + } + (*thread)->FinishReadingEntries(); + // Mark for removal by the call to RemoveValue below. + *thread = nullptr; + } + + // Only remove threads where reading succeeded so we retry the failed + // threads the next time around. + RemoveValue(&threads_to_read_, nullptr); + } + + // Explicitly flush every time we wake up. The goal being minimization + // of large implicit flushes which could affect tail latency measurements, + // especially at percentiles closer to 100%. + /// \todo Determine if explicitly flushing logs every wake up is better + /// than relying on implicit flushing. + { + auto tracer6 = + MakeScopedTracer([](AsyncTrace& trace) { trace("FlushAll"); }); + async_logger_.Flush(); + } + + if (!orphans_to_destroy_.empty()) { + auto tracer7 = MakeScopedTracer( + [](AsyncTrace& trace) { trace("Abandoning Orphans"); }); + std::unique_lock lock(tls_logger_orphans_mutex_); + for (auto orphan : orphans_to_destroy_) { + tls_logger_orphans_.erase(orphan); + } + orphans_to_destroy_.clear(); + } + } +} + +TlsLogger::TlsLogger(std::function forced_detatch) + : pid_(MLPERF_GET_PID()), + tid_(MLPERF_GET_TID()), + forced_detatch_(std::move(forced_detatch)) { + for (auto& entry : entries_) { + entry.reserve(kTlsLogReservedEntryCount); + } +} + +TlsLogger::~TlsLogger() {} + +// Log always makes forward progress since it can unconditionally obtain a +// "lock" on at least one of the buffers for writing. +// Notificiation is also lock free. +void TlsLogger::Log(AsyncLogEntry&& entry) { + size_t cas_fail_count = 0; + auto unlocked = EntryState::Unlocked; + size_t i_write = i_write_.load(std::memory_order_relaxed); + while (!entry_states_[i_write].compare_exchange_strong( + unlocked, EntryState::WriteLock, std::memory_order_acquire, + std::memory_order_relaxed)) { + unlocked = EntryState::Unlocked; + i_write ^= 1; + // We may need to try 3 times, since there could be a race with a + // previous SwapBuffers request and we use memory_order_relaxed when + // loading i_write_ above. + cas_fail_count++; + if (cas_fail_count >= 3) { +#if USE_NEW_LOGGING_FORMAT + MLPERF_LOG_WARNING(GlobalLogger(), "warning_generic_message", + "CAS failed."); +#else + GlobalLogger().LogErrorSync("CAS failed.", "times", cas_fail_count, + "line", __LINE__); +#endif + } + log_cas_fail_count_.fetch_add(1, std::memory_order_relaxed); + } + entries_[i_write].emplace_back(std::forward(entry)); + + // TODO: Convert this block to a simple write once we are confidient + // that we don't need to check for success. + auto write_lock = EntryState::WriteLock; + bool success = entry_states_[i_write].compare_exchange_strong( + write_lock, EntryState::Unlocked, std::memory_order_release); + if (!success) { +#if USE_NEW_LOGGING_FORMAT + MLPERF_LOG_WARNING(GlobalLogger(), "warning_generic_message", + "CAS failed."); +#else + GlobalLogger().LogErrorSync("CAS failed.", "line", __LINE__); +#endif + assert(success); + } + + bool write_buffer_swapped = i_write_prev_ != i_write; + if (write_buffer_swapped) { + GlobalLogger().RequestSwapBuffers(this); + i_write_prev_ = i_write; + } +} + +void TlsLogger::SwapBuffers() { + // TODO: Convert this block to a simple write once we are confidient + // that we don't need to check for success. + auto read_lock = EntryState::ReadLock; + bool success = entry_states_[i_read_].compare_exchange_strong( + read_lock, EntryState::Unlocked, std::memory_order_release); + if (!success) { +#if USE_NEW_LOGGING_FORMAT + MLPERF_LOG_WARNING(GlobalLogger(), "warning_generic_message", + "CAS failed."); +#else + GlobalLogger().LogErrorSync("CAS failed.", "line", __LINE__); +#endif + assert(success); + } + + i_write_.store(i_read_, std::memory_order_relaxed); + i_read_ ^= 1; + unread_swaps_++; +} + +// Returns nullptr if read lock fails. +std::vector* TlsLogger::StartReadingEntries() { + auto unlocked = EntryState::Unlocked; + if (entry_states_[i_read_].compare_exchange_strong( + unlocked, EntryState::ReadLock, std::memory_order_acquire, + std::memory_order_relaxed)) { + return &entries_[i_read_]; + } + return nullptr; +} + +void TlsLogger::FinishReadingEntries() { + // Detect first logging allocation and track max allocated size. + size_t new_size = entries_[i_read_].size(); + if (new_size > max_entry_size_) { + if (max_entry_size_ == kTlsLogReservedEntryCount) { + Log([ts = PerfClock::now()](AsyncLog& log) { + log.TraceAsyncInstant("FirstAllocation", 0, ts); + }); + } + max_entry_size_ = new_size; + } + + entries_[i_read_].clear(); + unread_swaps_--; +} + +bool TlsLogger::ReadBufferHasBeenConsumed() { return unread_swaps_ == 0; } + +void TlsLogger::TraceCounters() { + auto tracer = MakeScopedTracer( + [lcfc = log_cas_fail_count_.load(std::memory_order_relaxed), + sbsrc = swap_buffers_slot_retry_count_.load(std::memory_order_relaxed)]( + AsyncTrace& trace) { + trace("TlsLogger:ContentionCounters", "log_cas_fail_count", lcfc, + "swap_buffers_slot_retry_count", sbsrc); + }); +} + +Logger& GlobalLogger() { + static Logger g_logger(kLogPollPeriod, kMaxThreadsToLog); + return g_logger; +} + +/// \brief Moves ownership of the TlsLogger to Logger on thread exit +/// so no round-trip synchronization with the IO thread is required. +struct TlsLoggerWrapper { + TlsLoggerWrapper(std::function forced_detatch) + : tls_logger(std::make_unique(std::move(forced_detatch))) { + GlobalLogger().RegisterTlsLogger(tls_logger.get()); + } + ~TlsLoggerWrapper() { + tls_logger->TraceCounters(); + GlobalLogger().UnRegisterTlsLogger(std::move(tls_logger)); + } + std::unique_ptr tls_logger; +}; + +TlsLoggerWrapper* InitializeMyTlsLoggerWrapper() { + thread_local std::unique_ptr tls_logger_wrapper; + // forced_detatch lets the global Logger forcefully detatch TlsLoggers + // from the thread in the Logger's destructor, which may run before + // thread-local variables are destroyed when the loadgen is used as a python + // module and dynamically unloaded. + // Note: We capture a pointer to the tls_logger_wrapper since variables of + // the thread-local storage class aren't actually captured. C++ spec says + // only variables of the automatic storage class are captured. + /// \todo There is a race where the same TlsLoggerWrapper might be + /// destroyed both naturally and via forced_detatch. Destruction of + /// the TlsLoggerWrapper should be locked. + auto forced_detatch = [tls_logger_wrapper = &tls_logger_wrapper]() { + tls_logger_wrapper->reset(); + }; + tls_logger_wrapper = std::make_unique(forced_detatch); + return tls_logger_wrapper.get(); +} + +TlsLogger* InitializeMyTlsLogger() { + thread_local TlsLoggerWrapper* wrapper = InitializeMyTlsLoggerWrapper(); + return wrapper->tls_logger.get(); +} + +void Log(AsyncLogEntry&& entry) { + thread_local TlsLogger* const tls_logger = InitializeMyTlsLogger(); + tls_logger->Log(std::forward(entry)); +} + +} // namespace logging +} // namespace mlperf diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/logging.h b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/logging.h new file mode 100644 index 000000000..8f1a398e9 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/logging.h @@ -0,0 +1,816 @@ +/* Copyright 2019 The MLPerf Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +/// \file +/// \brief Internal logging implementation details. + +#ifndef MLPERF_LOADGEN_LOGGING_H_ +#define MLPERF_LOADGEN_LOGGING_H_ + +#define USE_NEW_LOGGING_FORMAT 1 +#define MLPERF_LOG(logger, key, value) \ + logger.Log((key), (value), __FILE__, __LINE__) +#define MLPERF_LOG_ERROR(logger, key, value) \ + logger.LogError((key), (value), __FILE__, __LINE__) +#define MLPERF_LOG_ERROR_SYNC(logger, key, value) \ + logger.LogErrorSync((key), (value), __FILE__, __LINE__) +#define MLPERF_LOG_WARNING(logger, key, value) \ + logger.LogWarning((key), (value), __FILE__, __LINE__) +#define MLPERF_LOG_INTERVAL_START(logger, key, value) \ + logger.LogIntervalStart((key), (value), __FILE__, __LINE__) +#define MLPERF_LOG_INTERVAL_END(logger, key, value) \ + logger.LogIntervalEnd((key), (value), __FILE__, __LINE__) + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "query_sample.h" + +namespace mlperf { + +/// \brief Wait-free logging utilities that defer stringification +/// and syscalls to a worker thread. +namespace logging { + +class AsyncLog; +class Logger; +class TlsLogger; +struct TlsLoggerWrapper; + +/// \todo Verify lambas are not allocating when bounded to a std::function. +using AsyncLogEntry = std::function; +using PerfClock = std::chrono::high_resolution_clock; + +/// \brief Logs the raw bytes as a hexadecimal ascii string. +struct LogBinaryAsHexString { + std::vector* data; +}; + +/// \brief By default, print out the value directly. +template +const T& ArgValueTransform(const T& value) { + return value; +} + +/// \brief Print out True/False. +const std::string& ArgValueTransform(const bool& value); +/// \brief Print out binary day as hex string. +const std::string ArgValueTransform(const LogBinaryAsHexString& value); +#if USE_NEW_LOGGING_FORMAT +/// \brief Print out a string in JSON format (with quotes). +const std::string ArgValueTransform(const std::string& value); +const std::string ArgValueTransform(const char* value); +/// \brief Prints a list of int in JSON format. +const std::string ArgValueTransform(const std::vector& value); +/// \brief Prints a dict in JSON format. +const std::string ArgValueTransform( + const std::map& value); +#endif + +/// \brief Helper to print out values without quotes when value is a string. +template +const T& ArgValueTransformWithoutQuote(const T& value) { + return ArgValueTransform(value); +} +inline const std::string ArgValueTransformWithoutQuote( + const LogBinaryAsHexString& value) { + return ArgValueTransform(value); +} +/// \brief Helper to print out a string without the quotes. +inline const std::string ArgValueTransformWithoutQuote( + const std::string& value) { + return value; +} + +/// \brief Outputs a trace that can be uploaded to chrome://tracing for +/// visualization. +/// \details Trace event format definition: +/// https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/edit?usp=sharing +class ChromeTracer { + public: + ChromeTracer(std::ostream* trace_out, PerfClock::time_point origin); + ~ChromeTracer(); + + template + void AddCompleteEvent(const std::string& name, uint64_t pid, uint64_t tid, + PerfClock::time_point start, PerfClock::time_point end, + const Args... args) { + *out_ << "{\"name\":\"" << name << "\"," << "\"ph\":\"X\"," + << "\"pid\":" << pid << "," << "\"tid\":" << tid << "," + << "\"ts\":" << Micros(start - origin_).count() << "," + << "\"dur\":" << Micros(end - start).count() << "," << "\"args\":{"; + AddArgs(args...); + *out_ << "}},\n"; + } + + template + void AddAsyncBeginEvent(const std::string& name, uint64_t pid, uint64_t id, + PerfClock::time_point time, const Args... args) { + *out_ << "{\"name\":\"" << name << "\"," << "\"cat\":\"default\"," + << "\"ph\":\"b\"," << "\"pid\":" << pid << "," << "\"id\":" << id + << "," << "\"ts\":" << Micros(time - origin_).count() << "," + << "\"args\":{"; + AddArgs(args...); + *out_ << "}},\n"; + } + + template + void AddAsyncInstantEvent(const std::string& name, uint64_t pid, uint64_t id, + PerfClock::time_point time, const Args... args) { + *out_ << "{\"name\":\"" << name << "\"," << "\"cat\":\"default\"," + << "\"ph\":\"n\"," << "\"pid\":" << pid << "," << "\"id\":" << id + << "," << "\"ts\":" << Micros(time - origin_).count() << "," + << "\"args\":{"; + AddArgs(args...); + *out_ << "}},\n"; + } + + template + void AddAsyncEndEvent(const std::string& name, uint64_t pid, uint64_t id, + PerfClock::time_point time) { + *out_ << "{\"name\":\"" << name << "\"," << "\"cat\":\"default\"," + << "\"ph\":\"e\", " << "\"pid\":" << pid << "," << "\"id\":" << id + << "," << "\"ts\":" << Micros(time - origin_).count() << "},\n"; + } + + template + void AddCounterEvent(const std::string& name, uint64_t pid, + PerfClock::time_point time, const Args... args) { + *out_ << "{\"name\":\"" << name << "\"," << "\"ph\": \"C\"," + << "\"pid\":" << pid << "," + << "\"ts\":" << Micros(time - origin_).count() << "," + << "\"args\":{ "; + AddArgs(args...); + *out_ << "}},\n"; + } + + void Flush() { out_->flush(); } + + private: + using Micros = std::chrono::duration; + + void WriteTraceEventHeader(); + void WriteTraceEventFooter(); + + void AddArgs() {} + + template + void AddArgs(const std::string& arg_name, const T& arg_value) { + *out_ << "\"" << arg_name << "\":" << ArgValueTransform(arg_value); + } + + template + void AddArgs(const std::string& arg_name, const T& arg_value, + const Args... args) { + *out_ << "\"" << arg_name << "\":" << ArgValueTransform(arg_value) << ","; + AddArgs(args...); + } + + std::ostream* out_; + PerfClock::time_point origin_; +}; + +/// \brief The proxy all logging lambdas ultimately use to write any log type. +/// \details Passed as an argument to the log lambda on the +/// recording thread to serialize the data captured by the lambda and +/// forward it to the output stream. +/// \todo Make summary_out_, detail_out_, accuracy_out_, and trace_out_ +/// instances of a new LogOutput interface that the client may override. +class AsyncLog { + public: + void SetLogFiles(std::ostream* summary, std::ostream* detail, + std::ostream* accuracy, bool copy_detail_to_stdout, + bool copy_summary_to_stdout, + PerfClock::time_point log_origin); + void StartNewTrace(std::ostream* trace_out, PerfClock::time_point origin); + void StopTrace(); + void Flush(); + + void SetCurrentPidTid(uint64_t pid, uint64_t tid); + + void LogAccuracy(uint64_t seq_id, const QuerySampleIndex qsl_idx, + const LogBinaryAsHexString& response, int64_t n_tokens); + void CacheToken(uint64_t seq_id, const LogBinaryAsHexString& response); + + template + void LogSummary(const std::string& message, const Args... args); + + void SetLogDetailTime(PerfClock::time_point time) { log_detail_time_ = time; } + + void FlagError() { + std::unique_lock lock(log_mutex_); + log_error_count_++; + error_flagged_ = true; + } + + void FlagWarning() { + std::unique_lock lock(log_mutex_); + log_warning_count_++; + warning_flagged_ = true; + } + +#if USE_NEW_LOGGING_FORMAT + template + void LogDetail(const std::string& key, const T& value, + const std::string file_name, const unsigned int line_no); +#else + template + void LogDetail(const std::string& message, const Args... args); +#endif + + template + void Trace(const std::string& trace_name, PerfClock::time_point start, + PerfClock::time_point end, const Args... args) { + std::unique_lock lock(trace_mutex_); + if (tracer_) { + tracer_->AddCompleteEvent(trace_name, current_pid_, current_tid_, start, + end, args...); + } + } + + template + void TraceAsyncInstant(const std::string& trace_name, uint64_t id, + PerfClock::time_point instant_time, + const Args... args) { + std::unique_lock lock(trace_mutex_); + if (tracer_) { + tracer_->AddAsyncInstantEvent(trace_name, current_pid_, id, instant_time, + args...); + } + } + + void SetScopedTraceTimes(PerfClock::time_point start, + PerfClock::time_point end) { + scoped_start_ = start; + scoped_end_ = end; + } + + template + void ScopedTrace(const std::string& trace_name, const Args... args) { + std::unique_lock lock(trace_mutex_); + if (tracer_) { + tracer_->AddCompleteEvent(trace_name, current_pid_, current_tid_, + scoped_start_, scoped_end_, args...); + } + } + + template + void TraceSample(const std::string& trace_name, uint64_t id, + PerfClock::time_point start, PerfClock::time_point end, + const Args... args) { + std::unique_lock lock(trace_mutex_); + if (tracer_) { + tracer_->AddAsyncBeginEvent(trace_name, current_pid_, id, start, args...); + tracer_->AddAsyncEndEvent(trace_name, current_pid_, id, end); + } + } + + template + void TraceCounterEvent(const std::string& trace_name, + PerfClock::time_point time, const Args... args) { + std::unique_lock lock(trace_mutex_); + if (tracer_) { + tracer_->AddCounterEvent(trace_name, current_pid_, time, args...); + } + } + + void RestartLatencyRecording(uint64_t first_sample_sequence_id, + size_t latencies_to_reserve); + void RecordSampleCompletion(uint64_t sample_sequence_id, + PerfClock::time_point completion_time, + QuerySampleLatency latency, int64_t n_tokens); + void RecordTokenCompletion(uint64_t sample_sequence_id, + PerfClock::time_point completion_time, + QuerySampleLatency latency); + std::vector GetLatenciesBlocking(size_t expected_count); + std::vector GetTokenLatencies(size_t expected_count); + std::vector GetTimePerOutputToken(size_t expected_count); + std::vector GetTokensPerSample(size_t expected_count); + PerfClock::time_point GetMaxCompletionTime(); + QuerySampleLatency GetMaxLatencySoFar(); + void SetUseTokens(bool use_tokens); + void SetNeedsFirstToken(bool needs_first_token); + size_t GetErrorCount() { return log_error_count_; }; + + private: + void WriteAccuracyHeaderLocked(); + void WriteAccuracyFooterLocked(); + + void LogArgs(std::ostream*) {} + + template + void LogArgs(std::ostream* out, const T& value_only) { + *out << ArgValueTransformWithoutQuote(value_only); + } + + template + void LogArgs(std::ostream* out, const std::string& arg_name, + const T& arg_value) { + *out << "\"" << arg_name + << "\" : " << ArgValueTransformWithoutQuote(arg_value); + } + + template + void LogArgs(std::ostream* out, const std::string& arg_name, + const T& arg_value, const Args... args) { + *out << "\"" << arg_name + << "\" : " << ArgValueTransformWithoutQuote(arg_value) << ", "; + LogArgs(out, args...); + } + + std::mutex log_mutex_; + std::ostream* summary_out_ = &std::cerr; + std::ostream* detail_out_ = &std::cerr; + std::ostream* accuracy_out_ = &std::cerr; + // TODO: Instead of these bools, use a class that forwards to two streams. + bool copy_detail_to_stdout_ = false; + bool copy_summary_to_stdout_ = false; + bool accuracy_needs_comma_ = false; + PerfClock::time_point log_origin_; + size_t log_error_count_ = 0; + bool error_flagged_ = false; + size_t log_warning_count_ = 0; + bool warning_flagged_ = false; + bool use_tokens_ = false; + bool needs_first_token_ = false; + + std::mutex trace_mutex_; + std::unique_ptr tracer_; + + uint64_t current_pid_; + uint64_t current_tid_; + PerfClock::time_point log_detail_time_; + PerfClock::time_point scoped_start_; + PerfClock::time_point scoped_end_; + + std::mutex latencies_mutex_; + std::mutex token_latencies_mutex_; + std::mutex token_record_mutex_; + std::condition_variable all_latencies_recorded_; + uint64_t latencies_first_sample_sequence_id_ = 0; + std::vector latencies_; + std::vector token_latencies_; + std::vector time_per_output_token_; + std::vector token_records_; + std::vector tokens_per_sample_; + QuerySampleLatency max_latency_ = 0; + PerfClock::time_point max_completion_timstamp_; + size_t latencies_recorded_ = 0; + size_t latencies_expected_ = 0; + // Must be called with latencies_mutex_ held. + bool AllLatenciesRecorded() { + return latencies_recorded_ == latencies_expected_; + } +}; + +/// \brief The central logger that logs all threads belonging to a run. +class Logger { + public: + Logger(std::chrono::duration poll_period, size_t max_threads_to_log); + ~Logger(); + + void StartIOThread(); + void StopIOThread(); + + void StartLogging(std::ostream* summary, std::ostream* detail, + std::ostream* accuracy, bool copy_detail_to_stdout, + bool copy_summary_to_stdout); + void StopLogging(); + + void StartNewTrace(std::ostream* trace_out, PerfClock::time_point origin); + void StopTracing(); + + void LogContentionAndAllocations(); + + void RestartLatencyRecording(uint64_t first_sample_sequence_id, + size_t latencies_to_reserve); + std::vector GetLatenciesBlocking(size_t expected_count); + std::vector GetTokenLatencies(size_t expected_count); + std::vector GetTimePerOutputToken(size_t expected_count); + std::vector GetTokensPerSample(size_t expected_count); + PerfClock::time_point GetMaxCompletionTime(); + QuerySampleLatency GetMaxLatencySoFar(); + void SetUseTokens(bool use_tokens); + void SetNeedsFirstToken(bool needs_first_token); + + private: + friend AsyncLog; + friend TlsLogger; + friend TlsLoggerWrapper; + + void RegisterTlsLogger(TlsLogger* tls_logger); + void UnRegisterTlsLogger(std::unique_ptr tls_logger); + void RequestSwapBuffers(TlsLogger* tls_logger); + void CollectTlsLoggerStats(TlsLogger* tls_logger); + + TlsLogger* GetTlsLoggerThatRequestedSwap(size_t slot, size_t next_id); + void GatherRetrySwapRequests(std::vector* threads_to_swap); + void GatherNewSwapRequests(std::vector* threads_to_swap); + + /// \brief The main logging thread function that handles the serialization + /// and I/O to the stream or file. + /// + /// \todo Provide client hook to set logging thead affinity and priority. + void IOThread(); + +// Slow synchronous error logging for internals that may prevent +// async logging from working. +#if USE_NEW_LOGGING_FORMAT + template + void LogErrorSync(const std::string& key, const T& value, + const std::string file_name, const unsigned int line_no) { + /// \todo Acquire mutex once for FlagError + LogDetail to avoid + /// races. Better yet, switch to a non-stateful error API. + // This is better than nothing though. + async_logger_.FlagError(); + async_logger_.LogDetail(key, value, file_name, line_no); + } + template + void LogWarning(const std::string& key, const T& value, + const std::string file_name, const unsigned int line_no) { + async_logger_.FlagWarning(); + async_logger_.LogDetail(key, value, file_name, line_no); + } +#else + template + void LogErrorSync(const std::string& message, Args&&... args) { + /// \todo Acquire mutex once for FlagError + LogDetail to avoid + /// races. Better yet, switch to a non-stateful error API. + // This is better than nothing though. + async_logger_.FlagError(); + async_logger_.LogDetail(message, std::forward(args)...); + } +#endif + + // Accessed by IOThead only. + const std::chrono::duration poll_period_; + AsyncLog async_logger_; + + const size_t max_threads_to_log_; + std::thread io_thread_; + + // Accessed by producers and IOThead during thread registration and + // destruction. Protected by io_thread_mutex_. + std::mutex io_thread_mutex_; + std::condition_variable io_thread_cv_; + bool keep_io_thread_alive_ = false; + + std::mutex tls_loggers_registerd_mutex_; + std::unordered_set tls_loggers_registerd_; + + // Temporarily stores TlsLogger data for threads that have exited until + // all their log entries have been processed. + // Accessed by IOThread and producers as their threads exit. + std::mutex tls_logger_orphans_mutex_; + using OrphanContainer = std::list>; + OrphanContainer tls_logger_orphans_; + + // Accessed by producers and IOThead atomically. + std::atomic swap_request_id_{0}; + std::vector> thread_swap_request_slots_; + + // Accessed by IOThead only. + size_t swap_request_id_read_{0}; + struct SlotRetry { + size_t slot; + uintptr_t next_id; + }; + std::vector swap_request_slots_to_retry_; + std::vector threads_to_swap_deferred_; + std::vector threads_to_read_; + std::vector orphans_to_destroy_; + + // Counts for retries related to the lock-free scheme. + // Abnormally high counts could be an indicator of contention. + // Access on IOThread only. + size_t swap_request_slots_retry_count_ = 0; + size_t swap_request_slots_retry_retry_count_ = 0; + size_t swap_request_slots_retry_reencounter_count_ = 0; + size_t start_reading_entries_retry_count_ = 0; + size_t tls_total_log_cas_fail_count_ = 0; + size_t tls_total_swap_buffers_slot_retry_count_ = 0; +}; + +Logger& GlobalLogger(); + +/// \brief The generic way to add a log entry. +/// \details Supports all types of logs, which is useful for complex +/// lambdas that may wish to log in multiple places or log something other +/// than a simple summary, detail, or trace entry. +void Log(AsyncLogEntry&& entry); + +/// \brief The convenience proxy a LogSummary lambda uses to write to the +/// summary log. +class AsyncSummary { + public: + explicit AsyncSummary(AsyncLog& async_log) : async_log_(async_log) {} + AsyncLog& async_log() { return async_log_; } + + template + AsyncLog& operator()(Args&&... args) { + async_log_.LogSummary(std::forward(args)...); + return async_log_; + } + + private: + AsyncLog& async_log_; +}; + +/// \brief A helper to simplify adding a summary log entry. +template +void LogSummary(LambdaT&& lambda) { + Log([lambda = std::forward(lambda)](AsyncLog& log) mutable { + AsyncSummary async_summary(log); + lambda(async_summary); + }); +} + +/// \brief The convenience proxy a LogDetail lambda uses to write to the detail +/// log. +class AsyncDetail { + public: + explicit AsyncDetail(AsyncLog& async_log) : async_log_(async_log) {} + AsyncLog& async_log() { return async_log_; } + +#if USE_NEW_LOGGING_FORMAT + template + AsyncLog& Log(const std::string& key, const T& value, + const std::string file_name, const unsigned int line_no) { + async_log_.LogDetail(key, value, file_name, line_no); + return async_log_; + } + + template + AsyncLog& LogError(const std::string& key, const T& value, + const std::string file_name, const unsigned int line_no) { + async_log_.FlagError(); + async_log_.LogDetail(key, value, file_name, line_no); + return async_log_; + } + + template + AsyncLog& LogWarning(const std::string& key, const T& value, + const std::string file_name, + const unsigned int line_no) { + async_log_.FlagWarning(); + async_log_.LogDetail(key, value, file_name, line_no); + return async_log_; + } + + template + AsyncLog& LogIntervalStart(const std::string& key, const T& value, + const std::string file_name, + const unsigned int line_no) { + async_log_.LogDetail(key, value, file_name, line_no); + return async_log_; + } + + template + AsyncLog& LogIntervalEnd(const std::string& key, const T& value, + const std::string file_name, + const unsigned int line_no) { + async_log_.LogDetail(key, value, file_name, line_no); + return async_log_; + } +#else + template + AsyncLog& operator()(Args&&... args) { + async_log_.LogDetail(std::forward(args)...); + return async_log_; + } + + template + AsyncLog& Error(Args&&... args) { + async_log_.FlagError(); + async_log_.LogDetail(std::forward(args)...); + return async_log_; + } + + template + AsyncLog& Warning(Args&&... args) { + async_log_.FlagWarning(); + async_log_.LogDetail(std::forward(args)...); + return async_log_; + } +#endif + + private: + AsyncLog& async_log_; +}; + +/// \brief A helper to simplify adding a detail log entry. +template +void LogDetail(LambdaT&& lambda) { + Log([lambda = std::forward(lambda), + timestamp = PerfClock::now()](AsyncLog& log) mutable { + log.SetLogDetailTime(timestamp); + AsyncDetail async_detail(log); + lambda(async_detail); + }); +} + +/// \brief The convenience proxy a ScopedTracer lambda uses to write to the +/// detail log. +class AsyncTrace { + public: + explicit AsyncTrace(AsyncLog& async_log) : async_log_(async_log) {} + AsyncLog& async_log() { return async_log_; } + + template + AsyncLog& operator()(Args&&... args) { + async_log_.ScopedTrace(std::forward(args)...); + return async_log_; + } + + private: + AsyncLog& async_log_; +}; + +/// \brief ScopedTracer is an RAII object that traces the start and end +/// of its lifetime. +template +class ScopedTracer { + public: + ScopedTracer(LambdaT&& lambda) + : start_(PerfClock::now()), lambda_(std::forward(lambda)) {} + + ~ScopedTracer() { + Log([start = start_, lambda = std::move(lambda_), + end = PerfClock::now()](AsyncLog& log) { + log.SetScopedTraceTimes(start, end); + AsyncTrace async_trace(log); + lambda(async_trace); + }); + } + + private: + PerfClock::time_point start_; + LambdaT lambda_; +}; + +/// \brief Helper that creates a ScopeTracer with automatic type deduction. +/// \details Helps with automatic template type deduction, which has been +/// supported for functions for a long time. +/// C++17 will support deduction for classes, which will neutralize the utility +/// of a helper function like this. +/// \todo Determine which traces to keep for submission purposes. +template +auto MakeScopedTracer(LambdaT&& lambda) -> ScopedTracer { + return ScopedTracer(std::forward(lambda)); +} + +template +void AsyncLog::LogSummary(const std::string& message, const Args... args) { + auto tracer = MakeScopedTracer([message](AsyncTrace& trace) { + std::string sanitized_message = message; + std::replace(sanitized_message.begin(), sanitized_message.end(), '"', '\''); + std::replace(sanitized_message.begin(), sanitized_message.end(), '\n', ';'); + trace("LogSummary", "message", "\"" + sanitized_message + "\""); + }); + std::unique_lock lock(log_mutex_); + *summary_out_ << message; + LogArgs(summary_out_, args...); + *summary_out_ << "\n"; + + if (copy_summary_to_stdout_) { + std::cout << message; + LogArgs(&std::cout, args...); + std::cout << "\n"; + } +} + +#if USE_NEW_LOGGING_FORMAT +template +void AsyncLog::LogDetail(const std::string& key, const T& value, + const std::string file_name, + const unsigned int line_no) { + auto tracer = MakeScopedTracer([key](AsyncTrace& trace) { + std::string sanitized_key = key; + std::replace(sanitized_key.begin(), sanitized_key.end(), '"', '\''); + std::replace(sanitized_key.begin(), sanitized_key.end(), '\n', ';'); + trace("LogDetail", "key", "\"" + sanitized_key + "\""); + }); + std::unique_lock lock(log_mutex_); + std::vector detail_streams{detail_out_, &std::cout}; + if (!copy_detail_to_stdout_) { + detail_streams.pop_back(); + } + auto time_ns = (log_detail_time_ - log_origin_).count(); + for (auto os : detail_streams) { + *os << ":::MLLOG {" << "\"key\": " << ArgValueTransform(key) << ", " + << "\"value\": " << ArgValueTransform(value) << ", " + << "\"time_ms\": " << ArgValueTransform(time_ns / 1000000ULL) << "." + << std::setfill('0') << std::setw(6) + << ArgValueTransform(time_ns % 1000000ULL) << ", " + << "\"namespace\": \"mlperf::logging\", " + << "\"event_type\": \"POINT_IN_TIME\", " << "\"metadata\": {" + << "\"is_error\": " << ArgValueTransform(error_flagged_) << ", " + << "\"is_warning\": " << ArgValueTransform(warning_flagged_) << ", " + << "\"file\": \"" << file_name << "\", " + << "\"line_no\": " << ArgValueTransform(line_no) << ", " + << "\"pid\": " << ArgValueTransform(current_pid_) << ", " + << "\"tid\": " << ArgValueTransform(current_tid_) << "}}\n"; + if (error_flagged_) { + os->flush(); + } + } + error_flagged_ = false; + warning_flagged_ = false; +} +#else +template +void AsyncLog::LogDetail(const std::string& message, const Args... args) { + auto tracer = MakeScopedTracer([message](AsyncTrace& trace) { + std::string sanitized_message = message; + std::replace(sanitized_message.begin(), sanitized_message.end(), '"', '\''); + std::replace(sanitized_message.begin(), sanitized_message.end(), '\n', ';'); + trace("LogDetail", "message", "\"" + sanitized_message + "\""); + }); + std::unique_lock lock(log_mutex_); + std::vector detail_streams{detail_out_, &std::cout}; + if (!copy_detail_to_stdout_) { + detail_streams.pop_back(); + } + for (auto os : detail_streams) { + *os << "\"pid\": " << current_pid_ << ", " << "\"tid\": " << current_tid_ + << ", " << "\"ts\": " << (log_detail_time_ - log_origin_).count() + << "ns : "; + if (error_flagged_) { + *os << "ERROR : "; + } else if (warning_flagged_) { + *os << "WARNING : "; + } + *os << message; + LogArgs(os, args...); + *os << "\n"; + if (error_flagged_) { + os->flush(); + } + } + error_flagged_ = false; + warning_flagged_ = false; +} +#endif + +} // namespace logging + +// Export some things out of the logging namespace to simplify call sites. + +const auto GlobalLogger = logging::GlobalLogger; +const auto Log = logging::Log; + +using PerfClock = logging::PerfClock; + +using LogBinaryAsHexString = logging::LogBinaryAsHexString; + +using AsyncLog = logging::AsyncLog; + +using AsyncSummary = logging::AsyncSummary; +template +void LogSummary(LambdaT&& lambda) { + logging::LogSummary(std::forward(lambda)); +} + +using AsyncDetail = logging::AsyncDetail; +template +void LogDetail(LambdaT&& lambda) { + logging::LogDetail(std::forward(lambda)); +} + +using AsyncTrace = logging::AsyncTrace; + +template +using ScopedTracer = logging::ScopedTracer; + +template +auto MakeScopedTracer(LambdaT&& lambda) -> ScopedTracer { + return ScopedTracer(std::forward(lambda)); +} + +} // namespace mlperf + +#endif // MLPERF_LOADGEN_LOGGING_H_ diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/mlperf.conf b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/mlperf.conf new file mode 100644 index 000000000..1b825514b --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/mlperf.conf @@ -0,0 +1,164 @@ +# The format of this config file is 'key = value'. +# The key has the format 'model.scenario.key'. Value is mostly int64_t. +# Model maybe '*' as wildcard. In that case the value applies to all models. +# All times are in milli seconds + +# Set performance_sample_count for each model. +# User can optionally set this to higher values in user.conf. +resnet50.*.performance_sample_count_override = 1024 +ssd-mobilenet.*.performance_sample_count_override = 256 +retinanet.*.performance_sample_count_override = 64 +bert.*.performance_sample_count_override = 10833 +dlrm.*.performance_sample_count_override = 204800 +dlrm-v2.*.performance_sample_count_override = 204800 +rnnt.*.performance_sample_count_override = 2513 +gptj.*.performance_sample_count_override = 13368 +mixtral-8x7b.*.performance_sample_count_override = 15000 +llama2-70b.*.performance_sample_count_override = 24576 +llama2-70b-interactive.*.performance_sample_count_override = 24576 +llama3_1-405b.*.performance_sample_count_override = 8313 +llama3_1-405b-interactive.*.performance_sample_count_override = 8313 +llama3_1-8b.*.performance_sample_count_override = 13368 +llama3_1-8b-edge.*.performance_sample_count_override = 5000 +llama3_1-8b-interactive.*.performance_sample_count_override = 13368 +stable-diffusion-xl.*.performance_sample_count_override = 5000 +rgat.*.performance_sample_count_override = 788379 +pointpainting.*.performance_sample_count_override = 1024 +deepseek-r1.*.performance_sample_count_override = 4388 +whisper.*.performance_sample_count_override = 1633 +# set to 0 to let entire sample set to be performance sample +3d-unet.*.performance_sample_count_override = 0 + +# Set seeds. +*.*.qsl_rng_seed = 1780908523862526354 +*.*.sample_index_rng_seed = 14771362308971278857 +*.*.schedule_rng_seed = 18209322760996052031 + +# Set seeds for TEST_05 (not needed from v5.0 onwards) +*.*.test05_qsl_rng_seed = 7975553102935885558 +*.*.test05_sample_index_rng_seed = 11403566307062068064 +*.*.test05_schedule_rng_seed = 15816800565822761601 + +*.SingleStream.target_latency_percentile = 90 +pointpainting.SingleStream.target_latency_percentile = 99.9 +*.SingleStream.min_duration = 600000 + +*.MultiStream.target_latency_percentile = 99 +*.MultiStream.samples_per_query = 8 +*.MultiStream.min_duration = 600000 +*.MultiStream.min_query_count = 662 +retinanet.MultiStream.target_latency = 528 + +# 3D-UNet uses equal issue mode because it has non-uniform inputs +3d-unet.*.sample_concatenate_permutation = 1 + +# R-GAT uses equal issue mode because it may have non-uniform inputs +rgat.*.sample_concatenate_permutation = 1 + +# LLM benchmarks have non-uniform inputs and outputs, and use equal issue mode for all latency scenario +gptj.*.sample_concatenate_permutation = 1 +llama2-70b.*.sample_concatenate_permutation = 1 +llama2-70b-interactive.*.sample_concatenate_permutation = 1 +mixtral-8x7b.*.sample_concatenate_permutation = 1 +llama3_1-405b.*.sample_concatenate_permutation = 1 +llama3_1-405b-interactive.*.sample_concatenate_permutation = 1 +llama3_1-8b.*.sample_concatenate_permutation = 1 +llama3_1-8b-edge.*.sample_concatenate_permutation = 1 +llama3_1-8b-interactive.*.sample_concatenate_permutation = 1 +deepseek-r1.*.sample_concatenate_permutation = 1 +whisper.*.sample_concatenate_permutation = 1 + +*.Server.target_latency = 10 +*.Server.target_latency_percentile = 99 +*.Server.target_duration = 0 +*.Server.min_duration = 600000 +resnet50.Server.target_latency = 15 +retinanet.Server.target_latency = 100 +bert.Server.target_latency = 130 +dlrm.Server.target_latency = 60 +dlrm-v2.Server.target_latency = 60 +rnnt.Server.target_latency = 1000 +gptj.Server.target_latency = 20000 +stable-diffusion-xl.Server.target_latency = 20000 +# Benchmarks that measure token latencies +llama2-70b.*.use_token_latencies = 1 +llama2-70b-interactive.*.use_token_latencies = 1 +mixtral-8x7b.*.use_token_latencies = 1 +llama3_1-405b.*.use_token_latencies = 1 +llama3_1-405b-interactive.*.use_token_latencies = 1 +llama3_1-8b.*.use_token_latencies = 1 +llama3_1-8b-edge.*.use_token_latencies = 1 +llama3_1-8b-interactive.*.use_token_latencies = 1 +deepseek-r1.*.use_token_latencies = 1 +whisper.*.use_token_latencies = 1 + +# gptj benchmark infers token latencies +gptj.*.infer_token_latencies = 1 +gptj.*.token_latency_scaling_factor = 69 +# Only ttft and tpot are tracked for the llama2-70b, mixtral-8x7B & llama3_1-405b benchmark therefore target_latency = 0 +llama2-70b.Server.target_latency = 0 +llama2-70b.Server.ttft_latency = 2000 +llama2-70b.Server.tpot_latency = 200 + +# Target Latencies for interactive setting +llama2-70b-interactive.Server.target_latency = 0 +llama2-70b-interactive.Server.ttft_latency = 450 +llama2-70b-interactive.Server.tpot_latency = 40 + +mixtral-8x7b.Server.target_latency = 0 +mixtral-8x7b.Server.ttft_latency = 2000 +mixtral-8x7b.Server.tpot_latency = 200 + +llama3_1-405b.Server.target_latency = 0 +llama3_1-405b.Server.ttft_latency = 6000 +llama3_1-405b.Server.tpot_latency = 175 + +# Target Latencies for interactive setting +llama3_1-405b-interactive.Server.target_latency = 0 +llama3_1-405b-interactive.Server.ttft_latency = 4500 +llama3_1-405b-interactive.Server.tpot_latency = 80 + + +llama3_1-8b.Server.target_latency = 0 +llama3_1-8b.Server.ttft_latency = 2000 +llama3_1-8b.Server.tpot_latency = 100 + +# Target Latencies for interactive setting +llama3_1-8b-interactive.Server.target_latency = 0 +llama3_1-8b-interactive.Server.ttft_latency = 500 +llama3_1-8b-interactive.Server.tpot_latency = 30 + +deepseek-r1.Server.target_latency = 0 +deepseek-r1.Server.ttft_latency = 2000 +deepseek-r1.Server.tpot_latency = 80 + +*.Offline.target_latency_percentile = 90 +*.Offline.min_duration = 600000 + +# In Offline scenario, we always have one query. But LoadGen maps this to +# min_sample_count internally in Offline scenario. If the dataset size is larger +# than 24576 we limit the min_query_count to 24576 and otherwise we use +# the dataset size as the limit + +resnet50.Offline.min_query_count = 24576 +retinanet.Offline.min_query_count = 24576 +dlrm-v2.Offline.min_query_count = 24576 +bert.Offline.min_query_count = 10833 +gptj.Offline.min_query_count = 13368 +rnnt.Offline.min_query_count = 2513 +3d-unet.Offline.min_query_count = 43 +stable-diffusion-xl.Offline.min_query_count = 5000 +llama2-70b.Offline.min_query_count = 24576 +llama3_1-405b.Offline.min_query_count = 8313 +llama3_1-8b.Offline.min_query_count = 13368 +llama3_1-8b-edge.Offline.min_query_count = 5000 +mixtral-8x7b.Offline.min_query_count = 15000 +rgat.Offline.min_query_count = 788379 +deepseek-r1.Offline.min_query_count = 4388 +whisper.Offline.min_query_count = 1633 + +# These fields should be defined and overridden by user.conf. +*.SingleStream.target_latency = 10 +*.MultiStream.target_latency = 80 +*.Server.target_qps = 1.0 +*.Offline.target_qps = 1.0 diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/mlperf_conf.h b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/mlperf_conf.h new file mode 100644 index 000000000..7859e0139 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/mlperf_conf.h @@ -0,0 +1,167 @@ +const char* mlperf_conf = +"# The format of this config file is 'key = value'.\n" +"# The key has the format 'model.scenario.key'. Value is mostly int64_t.\n" +"# Model maybe '*' as wildcard. In that case the value applies to all models.\n" +"# All times are in milli seconds\n" +"\n" +"# Set performance_sample_count for each model.\n" +"# User can optionally set this to higher values in user.conf.\n" +"resnet50.*.performance_sample_count_override = 1024\n" +"ssd-mobilenet.*.performance_sample_count_override = 256\n" +"retinanet.*.performance_sample_count_override = 64\n" +"bert.*.performance_sample_count_override = 10833\n" +"dlrm.*.performance_sample_count_override = 204800\n" +"dlrm-v2.*.performance_sample_count_override = 204800\n" +"rnnt.*.performance_sample_count_override = 2513\n" +"gptj.*.performance_sample_count_override = 13368\n" +"mixtral-8x7b.*.performance_sample_count_override = 15000\n" +"llama2-70b.*.performance_sample_count_override = 24576\n" +"llama2-70b-interactive.*.performance_sample_count_override = 24576\n" +"llama3_1-405b.*.performance_sample_count_override = 8313\n" +"llama3_1-405b-interactive.*.performance_sample_count_override = 8313\n" +"llama3_1-8b.*.performance_sample_count_override = 13368\n" +"llama3_1-8b-edge.*.performance_sample_count_override = 5000\n" +"llama3_1-8b-interactive.*.performance_sample_count_override = 13368\n" +"stable-diffusion-xl.*.performance_sample_count_override = 5000\n" +"rgat.*.performance_sample_count_override = 788379\n" +"pointpainting.*.performance_sample_count_override = 1024\n" +"deepseek-r1.*.performance_sample_count_override = 4388\n" +"whisper.*.performance_sample_count_override = 1633\n" +"# set to 0 to let entire sample set to be performance sample\n" +"3d-unet.*.performance_sample_count_override = 0\n" +"\n" +"# Set seeds.\n" +"*.*.qsl_rng_seed = 1780908523862526354\n" +"*.*.sample_index_rng_seed = 14771362308971278857\n" +"*.*.schedule_rng_seed = 18209322760996052031\n" +"\n" +"# Set seeds for TEST_05 (not needed from v5.0 onwards)\n" +"*.*.test05_qsl_rng_seed = 7975553102935885558\n" +"*.*.test05_sample_index_rng_seed = 11403566307062068064\n" +"*.*.test05_schedule_rng_seed = 15816800565822761601\n" +"\n" +"*.SingleStream.target_latency_percentile = 90\n" +"pointpainting.SingleStream.target_latency_percentile = 99.9\n" +"*.SingleStream.min_duration = 600000\n" +"\n" +"*.MultiStream.target_latency_percentile = 99\n" +"*.MultiStream.samples_per_query = 8\n" +"*.MultiStream.min_duration = 600000\n" +"*.MultiStream.min_query_count = 662\n" +"retinanet.MultiStream.target_latency = 528\n" +"\n" +"# 3D-UNet uses equal issue mode because it has non-uniform inputs\n" +"3d-unet.*.sample_concatenate_permutation = 1\n" +"\n" +"# R-GAT uses equal issue mode because it may have non-uniform inputs\n" +"rgat.*.sample_concatenate_permutation = 1\n" +"\n" +"# LLM benchmarks have non-uniform inputs and outputs, and use equal issue mode for all latency scenario\n" +"gptj.*.sample_concatenate_permutation = 1\n" +"llama2-70b.*.sample_concatenate_permutation = 1\n" +"llama2-70b-interactive.*.sample_concatenate_permutation = 1\n" +"mixtral-8x7b.*.sample_concatenate_permutation = 1\n" +"llama3_1-405b.*.sample_concatenate_permutation = 1\n" +"llama3_1-405b-interactive.*.sample_concatenate_permutation = 1\n" +"llama3_1-8b.*.sample_concatenate_permutation = 1\n" +"llama3_1-8b-edge.*.sample_concatenate_permutation = 1\n" +"llama3_1-8b-interactive.*.sample_concatenate_permutation = 1\n" +"deepseek-r1.*.sample_concatenate_permutation = 1\n" +"whisper.*.sample_concatenate_permutation = 1\n" +"\n" +"*.Server.target_latency = 10\n" +"*.Server.target_latency_percentile = 99\n" +"*.Server.target_duration = 0\n" +"*.Server.min_duration = 600000\n" +"resnet50.Server.target_latency = 15\n" +"retinanet.Server.target_latency = 100\n" +"bert.Server.target_latency = 130\n" +"dlrm.Server.target_latency = 60\n" +"dlrm-v2.Server.target_latency = 60\n" +"rnnt.Server.target_latency = 1000\n" +"gptj.Server.target_latency = 20000\n" +"stable-diffusion-xl.Server.target_latency = 20000\n" +"# Benchmarks that measure token latencies\n" +"llama2-70b.*.use_token_latencies = 1\n" +"llama2-70b-interactive.*.use_token_latencies = 1\n" +"mixtral-8x7b.*.use_token_latencies = 1\n" +"llama3_1-405b.*.use_token_latencies = 1\n" +"llama3_1-405b-interactive.*.use_token_latencies = 1\n" +"llama3_1-8b.*.use_token_latencies = 1\n" +"llama3_1-8b-edge.*.use_token_latencies = 1\n" +"llama3_1-8b-interactive.*.use_token_latencies = 1\n" +"deepseek-r1.*.use_token_latencies = 1\n" +"whisper.*.use_token_latencies = 1\n" +"\n" +"# gptj benchmark infers token latencies\n" +"gptj.*.infer_token_latencies = 1\n" +"gptj.*.token_latency_scaling_factor = 69\n" +"# Only ttft and tpot are tracked for the llama2-70b, mixtral-8x7B & llama3_1-405b benchmark therefore target_latency = 0\n" +"llama2-70b.Server.target_latency = 0\n" +"llama2-70b.Server.ttft_latency = 2000\n" +"llama2-70b.Server.tpot_latency = 200\n" +"\n" +"# Target Latencies for interactive setting\n" +"llama2-70b-interactive.Server.target_latency = 0\n" +"llama2-70b-interactive.Server.ttft_latency = 450\n" +"llama2-70b-interactive.Server.tpot_latency = 40\n" +"\n" +"mixtral-8x7b.Server.target_latency = 0\n" +"mixtral-8x7b.Server.ttft_latency = 2000\n" +"mixtral-8x7b.Server.tpot_latency = 200\n" +"\n" +"llama3_1-405b.Server.target_latency = 0\n" +"llama3_1-405b.Server.ttft_latency = 6000\n" +"llama3_1-405b.Server.tpot_latency = 175\n" +"\n" +"# Target Latencies for interactive setting\n" +"llama3_1-405b-interactive.Server.target_latency = 0\n" +"llama3_1-405b-interactive.Server.ttft_latency = 4500\n" +"llama3_1-405b-interactive.Server.tpot_latency = 80\n" +"\n" +"\n" +"llama3_1-8b.Server.target_latency = 0\n" +"llama3_1-8b.Server.ttft_latency = 2000\n" +"llama3_1-8b.Server.tpot_latency = 100\n" +"\n" +"# Target Latencies for interactive setting\n" +"llama3_1-8b-interactive.Server.target_latency = 0\n" +"llama3_1-8b-interactive.Server.ttft_latency = 500\n" +"llama3_1-8b-interactive.Server.tpot_latency = 30\n" +"\n" +"deepseek-r1.Server.target_latency = 0\n" +"deepseek-r1.Server.ttft_latency = 2000\n" +"deepseek-r1.Server.tpot_latency = 80\n" +"\n" +"*.Offline.target_latency_percentile = 90\n" +"*.Offline.min_duration = 600000\n" +"\n" +"# In Offline scenario, we always have one query. But LoadGen maps this to\n" +"# min_sample_count internally in Offline scenario. If the dataset size is larger\n" +"# than 24576 we limit the min_query_count to 24576 and otherwise we use\n" +"# the dataset size as the limit\n" +"\n" +"resnet50.Offline.min_query_count = 24576\n" +"retinanet.Offline.min_query_count = 24576\n" +"dlrm-v2.Offline.min_query_count = 24576\n" +"bert.Offline.min_query_count = 10833\n" +"gptj.Offline.min_query_count = 13368\n" +"rnnt.Offline.min_query_count = 2513\n" +"3d-unet.Offline.min_query_count = 43\n" +"stable-diffusion-xl.Offline.min_query_count = 5000\n" +"llama2-70b.Offline.min_query_count = 24576\n" +"llama3_1-405b.Offline.min_query_count = 8313\n" +"llama3_1-8b.Offline.min_query_count = 13368\n" +"llama3_1-8b-edge.Offline.min_query_count = 5000\n" +"mixtral-8x7b.Offline.min_query_count = 15000\n" +"rgat.Offline.min_query_count = 788379\n" +"deepseek-r1.Offline.min_query_count = 4388\n" +"whisper.Offline.min_query_count = 1633\n" +"\n" +"# These fields should be defined and overridden by user.conf.\n" +"*.SingleStream.target_latency = 10\n" +"*.MultiStream.target_latency = 80\n" +"*.Server.target_qps = 1.0\n" +"*.Offline.target_qps = 1.0\n" +"\n" +""; diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/pyproject.toml b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/pyproject.toml new file mode 100755 index 000000000..6f0ae06f0 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/pyproject.toml @@ -0,0 +1,7 @@ +[build-system] +requires = ["setuptools>=42", "wheel", "pybind11==2.11.1"] +build-backend = "setuptools.build_meta:__legacy__" + +[tool.cibuildwheel] +environment = "CFLAGS='-std=c++14'" +build = "cp3{7,8,9,10,11,12,13}-*" diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/query_dispatch_library.h b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/query_dispatch_library.h new file mode 100644 index 000000000..6c594efe0 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/query_dispatch_library.h @@ -0,0 +1,42 @@ +/* Copyright 2019 The MLPerf Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +/// \file +/// \brief Defines the QueryDispatchLibrary interface. + +#ifndef MLPERF_LOADGEN_QUERY_DISPATCH_LIBRARY_H +#define MLPERF_LOADGEN_QUERY_DISPATCH_LIBRARY_H + +#include + +#include "system_under_test.h" + +namespace mlperf { + +/// \addtogroup LoadgenAPI +/// @{ + +/// \brief The interface a client implements for the LoadGen over the network to +/// test. The API inherits the System_under_test.h API When working in LON mode +/// the QueryDispatchLibrary class is used and natively Upcasted to the +/// QueryDispatchLibrary class. + +class QueryDispatchLibrary : public SystemUnderTest { + public: + virtual ~QueryDispatchLibrary() = default; +}; + +/// @} + +} // namespace mlperf + +#endif // MLPERF_LOADGEN_QUERY_DISPATCH_LIBRARY_H diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/query_sample.h b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/query_sample.h new file mode 100644 index 000000000..e740be99e --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/query_sample.h @@ -0,0 +1,91 @@ +/* Copyright 2019 The MLPerf Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +/// \file +/// \brief Defines the structs involved in issuing a query and responding to +/// a query. +/// \details These are broken out into their own files since they are exposed +/// as part of the C API and we want to avoid C clients including C++ code. + +#ifndef MLPERF_LOADGEN_QUERY_SAMPLE_H_ +#define MLPERF_LOADGEN_QUERY_SAMPLE_H_ + +#include +#include + +#include + +namespace mlperf { + +/// \addtogroup LoadgenAPI +/// @{ + +/// \brief Represents a unique identifier for a sample of an issued query. +/// \details As currently implemented, the id is a pointer to an internal +/// loadgen struct whose value will never be zero/null. +typedef uintptr_t ResponseId; +constexpr ResponseId kResponseIdReserved = 0; + +/// \brief An index into the QuerySampleLibrary corresponding to a +/// single sample. +typedef size_t QuerySampleIndex; + +/// \brief Represents the smallest unit of input inference can run on. +/// A query consists of one or more samples. +struct QuerySample { + ResponseId id; + QuerySampleIndex index; +}; + +/// \brief Represents a single response to QuerySample +struct QuerySampleResponse { + ResponseId id; + uintptr_t data; + size_t size; ///< Size in bytes. + int64_t n_tokens; + + public: + QuerySampleResponse(ResponseId id, uintptr_t data, size_t size, + int64_t n_tokens) + : id(id), + data(data), + size(size), + n_tokens(n_tokens){ + // std::cout << "Initialized with 4 arguments, n_tokens: " << + // n_tokens <<"\n"; + }; + QuerySampleResponse(ResponseId id, uintptr_t data, size_t size) + : id(id), + data(data), + size(size), + n_tokens(0){ + // std::cout << "Initialized with 3 arguments, n_tokens: " << + // n_tokens <<"\n"; + }; + QuerySampleResponse() + : id(0), + data(0), + size(0), + n_tokens(0){ + // std::cout << "Initialized with 0 arguments, n_tokens: " << + // n_tokens <<"\n"; + }; +}; + +/// \brief A latency in nanoseconds, as recorded by the loadgen. +typedef int64_t QuerySampleLatency; + +/// @} + +} // namespace mlperf + +#endif // MLPERF_LOADGEN_QUERY_SAMPLE_H_ diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/query_sample_library.h b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/query_sample_library.h new file mode 100644 index 000000000..7258068cb --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/query_sample_library.h @@ -0,0 +1,75 @@ +/* Copyright 2019 The MLPerf Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +/// \file +/// \brief Defines the QuerySampleLibrary interface. + +#ifndef MLPERF_LOADGEN_QUERY_SAMPLE_LIBRARY_H +#define MLPERF_LOADGEN_QUERY_SAMPLE_LIBRARY_H + +#include +#include +#include + +#include "query_sample.h" + +namespace mlperf { + +/// \addtogroup LoadgenAPI +/// @{ + +/// \brief The interface a client implements to coordinate with the loadgen +/// which samples should be loaded. +class QuerySampleLibrary { + public: + virtual ~QuerySampleLibrary() {} + + /// \brief A human readable name for the model. + virtual const std::string& Name() = 0; + + /// \brief Total number of samples in library. + virtual size_t TotalSampleCount() = 0; + + /// \brief The number of samples that are guaranteed to fit in RAM. + virtual size_t PerformanceSampleCount() = 0; + + /// \brief Loads the requested query samples into memory. + /// \details Paired with calls to UnloadSamplesFromRam. + /// In the MultiStream scenarios: + /// * Samples will appear more than once. + /// * SystemUnderTest::IssueQuery will only be called with a set of samples + /// that are neighbors in the vector of samples here, which helps + /// SUTs that need the queries to be contiguous. + /// In all other scenarios: + /// * A previously loaded sample will not be loaded again. + virtual void LoadSamplesToRam( + const std::vector& samples) = 0; + + /// \brief Unloads the requested query samples from memory. + /// \details In the MultiStream scenarios: + /// * Samples may be unloaded the same number of times they were loaded; + /// however, if the implementation de-dups loaded samples rather than + /// loading samples into contiguous memory, it may unload a sample the + /// first time they see it unloaded without a refcounting scheme, ignoring + /// subsequent unloads. A refcounting scheme would also work, but is not + /// a requirement. + /// In all other scenarios: + /// * A previously unloaded sample will not be unloaded again. + virtual void UnloadSamplesFromRam( + const std::vector& samples) = 0; +}; + +/// @} + +} // namespace mlperf + +#endif // MLPERF_LOADGEN_QUERY_SAMPLE_LIBRARY_H diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/requirements.txt b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/requirements.txt new file mode 100644 index 000000000..e47c59fd7 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/requirements.txt @@ -0,0 +1 @@ +pybind11 diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/results.cc b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/results.cc new file mode 100644 index 000000000..f7c61af43 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/results.cc @@ -0,0 +1,856 @@ +/* Copyright 2019 The MLPerf Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "results.h" + +#include "early_stopping.h" +#include "utils.h" + +namespace mlperf { +namespace loadgen { + +void PerformanceSummary::ProcessLatencies() { + if (pr.sample_latencies.empty()) { + return; + } + + sample_count = pr.sample_latencies.size(); + + QuerySampleLatency accumulated_sample_latency = 0; + for (auto latency : pr.sample_latencies) { + accumulated_sample_latency += latency; + } + sample_latency_mean = accumulated_sample_latency / sample_count; + + std::sort(pr.sample_latencies.begin(), pr.sample_latencies.end()); + + target_latency_percentile.sample_latency = + pr.sample_latencies[sample_count * target_latency_percentile.percentile]; + sample_latency_min = pr.sample_latencies.front(); + sample_latency_max = pr.sample_latencies.back(); + for (auto& lp : latency_percentiles) { + assert(lp.percentile >= 0.0); + assert(lp.percentile < 1.0); + lp.sample_latency = pr.sample_latencies[sample_count * lp.percentile]; + } + + query_count = pr.queries_issued; + + // Count the number of overlatency queries. Only for Server scenario. Since in + // this scenario the number of samples per query is 1, sample_latencies are + // used. + if (settings.scenario == TestScenario::Server) { + QuerySampleLatency max_latency = settings.target_latency.count() + 1; + overlatency_query_count = + pr.sample_latencies.end() - + std::lower_bound(pr.sample_latencies.begin(), pr.sample_latencies.end(), + max_latency); + } + + if (settings.use_token_latencies) { + ProcessTokenLatencies(); + } + + // MultiStream only after this point. + if (settings.scenario != TestScenario::MultiStream) { + return; + } + + // Calculate per-query stats. + size_t query_count = pr.queries_issued; + assert(pr.query_latencies.size() == query_count); + std::sort(pr.query_latencies.begin(), pr.query_latencies.end()); + QuerySampleLatency accumulated_query_latency = 0; + for (auto latency : pr.query_latencies) { + accumulated_query_latency += latency; + } + query_latency_mean = accumulated_query_latency / query_count; + query_latency_min = pr.query_latencies.front(); + query_latency_max = pr.query_latencies.back(); + target_latency_percentile.query_latency = + pr.query_latencies[query_count * target_latency_percentile.percentile]; + for (auto& lp : latency_percentiles) { + lp.query_latency = pr.query_latencies[query_count * lp.percentile]; + } +} + +void PerformanceSummary::ProcessTokenLatencies() { + constexpr auto nTokenInvalid = std::numeric_limits::min(); + token_count = 0; + for (auto n_tokens : pr.token_results.tokens_per_sample) { + if (n_tokens != nTokenInvalid) token_count += n_tokens; + } + if (pr.token_results.first_token_latencies.empty()) { + return; + } + QuerySampleLatency accumulated_first_token_latency = 0; + for (auto latency : pr.token_results.first_token_latencies) { + accumulated_first_token_latency += latency; + } + first_token_latency_mean = accumulated_first_token_latency / sample_count; + QuerySampleLatency accumulated_tpot = 0; + for (auto latency : pr.token_results.time_per_output_token_arr) { + accumulated_tpot += latency; + } + time_per_output_token_mean = accumulated_tpot / sample_count; + std::sort(pr.token_results.first_token_latencies.begin(), + pr.token_results.first_token_latencies.end()); + std::sort(pr.token_results.time_per_output_token_arr.begin(), + pr.token_results.time_per_output_token_arr.end()); + + token_target_latency_percentile.sample_latency = + pr.token_results + .first_token_latencies[sample_count * + token_target_latency_percentile.percentile]; + first_token_latency_min = pr.token_results.first_token_latencies.front(); + first_token_latency_max = pr.token_results.first_token_latencies.back(); + for (auto& lp : token_latency_percentiles) { + assert(lp.percentile >= 0.0); + assert(lp.percentile < 1.0); + lp.sample_latency = + pr.token_results.first_token_latencies[sample_count * lp.percentile]; + } + + target_tpot_percentile.sample_latency = + pr.token_results + .time_per_output_token_arr[sample_count * + target_tpot_percentile.percentile]; + time_per_output_token_min = + pr.token_results.time_per_output_token_arr.front(); + time_per_output_token_max = pr.token_results.time_per_output_token_arr.back(); + for (auto& lp : tpot_percentiles) { + assert(lp.percentile >= 0.0); + assert(lp.percentile < 1.0); + lp.sample_latency = + pr.token_results + .time_per_output_token_arr[sample_count * lp.percentile]; + } + + if (settings.scenario == TestScenario::Server) { + // TODO: Maybe another target latency needs to be added? + QuerySampleLatency max_latency = settings.target_latency.count() + 1; + overlatency_first_token_count = + pr.token_results.first_token_latencies.end() - + std::lower_bound(pr.token_results.first_token_latencies.begin(), + pr.token_results.first_token_latencies.end(), + max_latency); + } +} + +bool PerformanceSummary::EarlyStopping( + std::string* recommendation, int64_t queries_issued, + std::vector* sample_latencies, + std::vector* query_latencies, + std::chrono::nanoseconds target_latency) { + recommendation->clear(); + + MinPassingQueriesFinder find_min_passing; + double confidence = 0.99; + double tolerance = 0.0; + + ProcessLatencies(); + switch (settings.scenario) { + case TestScenario::SingleStream: { + // TODO: Grab multistream percentile from settings, instead of hardcoding. + double multi_stream_percentile = 0.99; + int64_t t = 1; + int64_t h_min = find_min_passing(1, target_latency_percentile.percentile, + tolerance, confidence); + int64_t h = h_min; + if (queries_issued < h_min + 1) { + *recommendation = + " * Only processed " + std::to_string(queries_issued) + + " queries.\n * Need to process at least " + + std::to_string(h_min + 1) + " queries for early stopping."; + return false; + } else { + for (int64_t i = 2; i < queries_issued + 1; ++i) { + h = find_min_passing(i, target_latency_percentile.percentile, + tolerance, confidence); + if (queries_issued < h + i) { + t = i - 1; + break; + } + } + } + QuerySampleLatency percentile_estimate = + (*sample_latencies)[queries_issued - t]; + *recommendation = + " * Processed at least " + std::to_string(h_min + 1) + " queries (" + + std::to_string(queries_issued) + ").\n" + " * Would discard " + + std::to_string(t - 1) + " highest latency queries.\n" + + " * Early stopping " + + DoubleToString(target_latency_percentile.percentile * 100, 1) + + "th percentile estimate: " + std::to_string(percentile_estimate); + early_stopping_latency_ss = percentile_estimate; + + // Early stopping estimate for 99%ile (used for infering multi-stream from + // single-stream) + t = 1; + h_min = + find_min_passing(1, multi_stream_percentile, tolerance, confidence); + h = h_min; + if (queries_issued < h_min + 1) { + *recommendation += + "\n * Not enough queries processed for " + + DoubleToString(multi_stream_percentile * 100, 1) + + "th percentile\n" + + " early stopping estimate (would need to process at\n least " + + std::to_string(h_min + 1) + " total queries)."; + } else { + for (int64_t i = 2; i < queries_issued + 1; ++i) { + h = find_min_passing(i, multi_stream_percentile, tolerance, + confidence); + if (queries_issued < h + i) { + t = i - 1; + break; + } + } + percentile_estimate = (*sample_latencies)[queries_issued - t]; + *recommendation += + "\n * Early stopping " + + DoubleToString(multi_stream_percentile * 100, 1) + + "th percentile estimate: " + std::to_string(percentile_estimate); + early_stopping_latency_ms = percentile_estimate; + } + break; + } + case TestScenario::Server: { + int64_t t = + std::count_if((*sample_latencies).begin(), (*sample_latencies).end(), + [=](auto const& latency) { + return latency > target_latency.count(); + }); + int64_t h = find_min_passing(t, target_latency_percentile.percentile, + tolerance, confidence); + if (queries_issued >= h + t) { + *recommendation = " * Run successful."; + } else { + *recommendation = " * Run unsuccessful.\n * Processed " + + std::to_string(queries_issued) + " queries.\n" + + " * Would need to run at least " + + std::to_string(h + t - queries_issued) + + " more queries,\n with the run being successful if " + "every additional\n query were under latency."; + return false; + } + break; + } + case TestScenario::MultiStream: { + int64_t t = 1; + int64_t h_min = find_min_passing(1, target_latency_percentile.percentile, + tolerance, confidence); + int64_t h = h_min; + if (queries_issued < h_min + 1) { + *recommendation = + " * Only processed " + std::to_string(queries_issued) + + " queries.\n * Need to process at least " + + std::to_string(h_min + 1) + " queries for early stopping."; + return false; + } else { + for (int64_t i = 2; i < queries_issued + 1; ++i) { + h = find_min_passing(i, target_latency_percentile.percentile, + tolerance, confidence); + if (queries_issued < h + i) { + t = i - 1; + break; + } + } + } + QuerySampleLatency percentile_estimate = + (*query_latencies)[queries_issued - t]; + *recommendation = + " * Processed at least " + std::to_string(h_min + 1) + " queries (" + + std::to_string(queries_issued) + ").\n" + " * Would discard " + + std::to_string(t - 1) + " highest latency queries.\n" + + " * Early stopping " + + DoubleToString(target_latency_percentile.percentile * 100, 1) + + "th percentile estimate: " + std::to_string(percentile_estimate); + early_stopping_latency_ms = percentile_estimate; + break; + } + case TestScenario::Offline: + break; + } + return true; +} + +bool PerformanceSummary::MinDurationMet(std::string* recommendation) { + recommendation->clear(); + const double min_duration = DurationToSeconds(settings.min_duration); + bool min_duration_met = false; + switch (settings.scenario) { + case TestScenario::Offline: + min_duration_met = pr.max_latency >= min_duration; + break; + case TestScenario::Server: + min_duration_met = pr.final_query_scheduled_time >= min_duration; + break; + case TestScenario::SingleStream: + case TestScenario::MultiStream: + min_duration_met = pr.final_query_issued_time >= min_duration; + break; + } + if (min_duration_met) { + return true; + } + + switch (settings.scenario) { + case TestScenario::SingleStream: + case TestScenario::MultiStream: + *recommendation = + "Decrease the expected latency so the loadgen pre-generates more " + "queries."; + break; + case TestScenario::Server: + *recommendation = + "Increase the target QPS so the loadgen pre-generates more queries."; + break; + case TestScenario::Offline: + *recommendation = + "Increase expected QPS so the loadgen pre-generates a larger " + "(coalesced) query."; + break; + } + return false; +} + +bool PerformanceSummary::MinQueriesMet() { + return pr.queries_issued >= settings.min_query_count; +} + +bool PerformanceSummary::MinSamplesMet() { + return sample_count >= settings.min_sample_count; +} + +bool PerformanceSummary::HasPerfConstraints() { + return settings.scenario == TestScenario::Server; +} + +bool PerformanceSummary::PerfConstraintsMet(std::string* recommendation) { + recommendation->clear(); + bool perf_constraints_met = true; + switch (settings.scenario) { + case TestScenario::SingleStream: + case TestScenario::MultiStream: + break; + case TestScenario::Server: + ProcessLatencies(); + if (!settings.use_token_latencies) { + if (target_latency_percentile.sample_latency > + settings.target_latency.count()) { + *recommendation = "Reduce target QPS to improve latency."; + perf_constraints_met = false; + } + } else { + if (token_target_latency_percentile.sample_latency > + settings.server_ttft_latency) { + *recommendation = + "TTFT constrain not met: Reduce target QPS to improve latency."; + perf_constraints_met = false; + } + + if (target_tpot_percentile.sample_latency > + settings.server_tpot_latency) { + if (recommendation->empty()) { + *recommendation = + "TPOT constrain not met: Reduce target QPS to improve latency."; + } else { + recommendation->append( + "\n * TPOT constrain not met: Reduce target QPS to improve " + "latency."); + } + perf_constraints_met = false; + } + } + break; + case TestScenario::Offline: + break; + } + return perf_constraints_met; +} + +void PerformanceSummary::LogSummary(AsyncSummary& summary) { + ProcessLatencies(); + + summary( + "================================================\n" + "MLPerf Results Summary\n" + "================================================"); + summary("SUT name : ", sut_name); + summary("Scenario : ", ToString(settings.scenario)); + summary("Mode : ", ToString(settings.mode)); + + switch (settings.scenario) { + case TestScenario::SingleStream: { + summary(DoubleToString(target_latency_percentile.percentile * 100, 1) + + "th percentile latency (ns) : ", + target_latency_percentile.sample_latency); + break; + } + case TestScenario::MultiStream: { + summary(DoubleToString(target_latency_percentile.percentile * 100, 1) + + "th percentile latency (ns) : ", + target_latency_percentile.query_latency); + break; + } + case TestScenario::Server: { + // Subtract 1 from sample count since the start of the final sample + // represents the open end of the time range: i.e. [begin, end). + // This makes sense since: + // a) QPS doesn't apply if there's only one sample; it's pure latency. + // b) If you have precisely 1k QPS, there will be a sample exactly on + // the 1 second time point; but that would be the 1001th sample in + // the stream. Given the first 1001 queries, the QPS is + // 1000 queries / 1 second. + // TODO: make a more permanent solution + double qps_as_completed = + (sample_count - 1) / pr.final_query_all_samples_done_time; + summary("Completed samples per second : ", + DoubleToString(qps_as_completed)); + break; + } + case TestScenario::Offline: { + double samples_per_second = sample_count / pr.max_latency; + summary("Samples per second: ", samples_per_second); + break; + } + } + + if (settings.use_token_latencies) { + switch (settings.scenario) { + case TestScenario::SingleStream: { + summary(DoubleToString(token_target_latency_percentile.percentile * 100, + 1) + + "th first token percentile latency (ns) : ", + token_target_latency_percentile.sample_latency); + break; + } + case TestScenario::MultiStream: { + summary(DoubleToString(token_target_latency_percentile.percentile * 100, + 1) + + "th first token percentile latency (ns) : ", + token_target_latency_percentile.sample_latency); + break; + } + case TestScenario::Offline: { + double tokens_per_second = token_count / pr.max_latency; + summary("Tokens per second: ", tokens_per_second); + break; + } + case TestScenario::Server: + double tps_as_completed = + token_count / pr.final_query_all_samples_done_time; + summary("Completed tokens per second: ", + DoubleToString(tps_as_completed)); + break; + } + } + + if (settings.infer_token_latencies) { + switch (settings.scenario) { + case TestScenario::SingleStream: { + break; + } + case TestScenario::MultiStream: { + break; + } + case TestScenario::Offline: { + double tokens_per_second = settings.token_latency_scaling_factor * + sample_count / pr.max_latency; + summary("Tokens per second (inferred): ", tokens_per_second); + break; + } + case TestScenario::Server: + double tps_as_completed = settings.token_latency_scaling_factor * + (sample_count - 1) / + pr.final_query_all_samples_done_time; + summary("Completed tokens per second (inferred): ", + DoubleToString(tps_as_completed)); + break; + } + } + + std::string min_duration_recommendation; + std::string perf_constraints_recommendation; + std::string early_stopping_recommendation; + std::string early_stopping_ttft_recommendation; + std::string early_stopping_tpot_recommendation; + + bool min_duration_met = MinDurationMet(&min_duration_recommendation); + bool min_queries_met = MinQueriesMet() && MinSamplesMet(); + bool early_stopping_met = true; + if (!settings.use_token_latencies) { + early_stopping_met = EarlyStopping( + &early_stopping_recommendation, pr.queries_issued, &pr.sample_latencies, + &pr.query_latencies, settings.target_latency); + } else { + early_stopping_met = + EarlyStopping(&early_stopping_tpot_recommendation, pr.queries_issued, + &pr.token_results.time_per_output_token_arr, + &pr.query_latencies, + std::chrono::nanoseconds(settings.server_tpot_latency)) && + EarlyStopping(&early_stopping_ttft_recommendation, pr.queries_issued, + &pr.token_results.first_token_latencies, + &pr.query_latencies, + std::chrono::nanoseconds(settings.server_ttft_latency)); + } + bool perf_constraints_met = + PerfConstraintsMet(&perf_constraints_recommendation); + bool all_constraints_met = min_duration_met && min_queries_met && + perf_constraints_met && early_stopping_met; + summary("Result is : ", all_constraints_met ? "VALID" : "INVALID"); + if (HasPerfConstraints()) { + summary(" Performance constraints satisfied : ", + perf_constraints_met ? "Yes" : "NO"); + } + summary(" Min duration satisfied : ", min_duration_met ? "Yes" : "NO"); + summary(" Min queries satisfied : ", min_queries_met ? "Yes" : "NO"); + summary(" Early stopping satisfied: ", early_stopping_met ? "Yes" : "NO"); + + if (!all_constraints_met) { + summary("Recommendations:"); + if (!perf_constraints_met) { + summary(" * " + perf_constraints_recommendation); + } + if (!min_duration_met) { + summary(" * " + min_duration_recommendation); + } + if (!min_queries_met) { + summary( + " * The test exited early, before enough queries were issued.\n" + " See the detailed log for why this may have occurred."); + } + } + // Early stopping results + if (settings.scenario == TestScenario::SingleStream || + settings.scenario == TestScenario::Server || + settings.scenario == TestScenario::MultiStream) { + if (!settings.use_token_latencies) { + summary("Early Stopping Result:"); + summary(early_stopping_recommendation); + } else { + summary("TTFT Early Stopping Result:"); + summary(early_stopping_ttft_recommendation); + summary("TPOT Early Stopping Result:"); + summary(early_stopping_tpot_recommendation); + } + } + + summary( + "\n" + "================================================\n" + "Additional Stats\n" + "================================================"); + + if (settings.scenario == TestScenario::SingleStream) { + double qps_w_lg = (sample_count - 1) / pr.final_query_issued_time; + double qps_wo_lg = 1 / QuerySampleLatencyToSeconds(sample_latency_mean); + summary("QPS w/ loadgen overhead : " + DoubleToString(qps_w_lg)); + summary("QPS w/o loadgen overhead : " + DoubleToString(qps_wo_lg)); + summary(""); + } else if (settings.scenario == TestScenario::Server) { + // Scheduled samples per second as an additional stat + double qps_as_scheduled = + (sample_count - 1) / pr.final_query_scheduled_time; + summary("Scheduled samples per second : ", + DoubleToString(qps_as_scheduled)); + } else if (settings.scenario == TestScenario::MultiStream) { + summary("Per-query latency: "); + summary("Min latency (ns) : ", query_latency_min); + summary("Max latency (ns) : ", query_latency_max); + summary("Mean latency (ns) : ", query_latency_mean); + for (auto& lp : latency_percentiles) { + summary( + DoubleToString(lp.percentile * 100) + " percentile latency (ns) : ", + lp.query_latency); + } + } + + if (settings.scenario != TestScenario::MultiStream) { + summary("Min latency (ns) : ", sample_latency_min); + summary("Max latency (ns) : ", sample_latency_max); + summary("Mean latency (ns) : ", sample_latency_mean); + for (auto& lp : latency_percentiles) { + summary( + DoubleToString(lp.percentile * 100) + " percentile latency (ns) : ", + lp.sample_latency); + } + } + if (settings.use_token_latencies) { + summary(""); + if (settings.scenario == TestScenario::SingleStream) { + double tps_w_lg = token_count / pr.final_query_issued_time; + double tps_wo_lg = + ((double)token_count) / + (QuerySampleLatencyToSeconds(sample_latency_mean) * sample_count); + summary("TPS w/ loadgen overhead : " + DoubleToString(tps_w_lg)); + summary("TPS w/o loadgen overhead : " + DoubleToString(tps_wo_lg)); + + } else if (settings.scenario == TestScenario::Server) { + double tps_as_completed = + token_count / pr.final_query_all_samples_done_time; + summary("Completed tokens per second : ", + DoubleToString(tps_as_completed)); + } + + if (settings.scenario != TestScenario::Offline) { + summary("Min First Token latency (ns) : ", + first_token_latency_min); + summary("Max First Token latency (ns) : ", + first_token_latency_max); + summary("Mean First Token latency (ns) : ", + first_token_latency_mean); + for (auto& lp : token_latency_percentiles) { + summary(DoubleToString(lp.percentile * 100) + + " percentile first token latency (ns) : ", + lp.sample_latency); + } + summary(""); + summary("Min Time per Output Token (ns) : ", + time_per_output_token_min); + summary("Max Time per Output Token (ns) : ", + time_per_output_token_max); + summary("Mean Time per Output Token (ns) : ", + time_per_output_token_mean); + for (auto& lp : tpot_percentiles) { + summary(DoubleToString(lp.percentile * 100) + + " percentile time to output token (ns) : ", + lp.sample_latency); + } + } + } + + summary( + "\n" + "================================================\n" + "Test Parameters Used\n" + "================================================"); + settings.LogSummary(summary); +} + +void PerformanceSummary::LogDetail(AsyncDetail& detail) { +#if USE_NEW_LOGGING_FORMAT + ProcessLatencies(); + + // General validity checking + std::string min_duration_recommendation; + std::string perf_constraints_recommendation; + std::string early_stopping_recommendation; + std::string early_stopping_ttft_recommendation; + std::string early_stopping_tpot_recommendation; + bool min_duration_met = MinDurationMet(&min_duration_recommendation); + bool min_queries_met = MinQueriesMet() && MinSamplesMet(); + bool perf_constraints_met = + PerfConstraintsMet(&perf_constraints_recommendation); + bool early_stopping_met = true; + if (!settings.use_token_latencies) { + early_stopping_met = EarlyStopping( + &early_stopping_recommendation, pr.queries_issued, &pr.sample_latencies, + &pr.query_latencies, settings.target_latency); + } else { + early_stopping_met = + EarlyStopping(&early_stopping_tpot_recommendation, pr.queries_issued, + &pr.token_results.time_per_output_token_arr, + &pr.query_latencies, + std::chrono::nanoseconds(settings.server_tpot_latency)) && + EarlyStopping(&early_stopping_ttft_recommendation, pr.queries_issued, + &pr.token_results.first_token_latencies, + &pr.query_latencies, + std::chrono::nanoseconds(settings.server_ttft_latency)); + } + bool all_constraints_met = min_duration_met && min_queries_met && + perf_constraints_met && early_stopping_met; + + MLPERF_LOG(detail, "result_validity", + all_constraints_met ? "VALID" : "INVALID"); + if (HasPerfConstraints()) { + MLPERF_LOG(detail, "result_perf_constraints_met", perf_constraints_met); + } + MLPERF_LOG(detail, "result_min_duration_met", min_duration_met); + MLPERF_LOG(detail, "result_min_queries_met", min_queries_met); + MLPERF_LOG(detail, "early_stopping_met", early_stopping_met); + if (!all_constraints_met) { + std::string recommendation; + if (!perf_constraints_met) { + recommendation += perf_constraints_recommendation + " "; + } + if (!min_duration_met) { + recommendation += min_duration_recommendation + " "; + } + if (!min_queries_met) { + recommendation += + "The test exited early, before enough queries were issued."; + } + std::replace(recommendation.begin(), recommendation.end(), '\n', ' '); + MLPERF_LOG(detail, "result_invalid_reason", recommendation); + } + std::replace(early_stopping_recommendation.begin(), + early_stopping_recommendation.end(), '\n', ' '); + if (!settings.use_token_latencies) { + MLPERF_LOG(detail, "early_stopping_result", early_stopping_recommendation); + } else { + std::replace(early_stopping_ttft_recommendation.begin(), + early_stopping_ttft_recommendation.end(), '\n', ' '); + std::replace(early_stopping_tpot_recommendation.begin(), + early_stopping_tpot_recommendation.end(), '\n', ' '); + MLPERF_LOG(detail, "early_stopping_ttft_result", + early_stopping_ttft_recommendation); + MLPERF_LOG(detail, "early_stopping_tpot_result", + early_stopping_tpot_recommendation); + } + // Report number of queries + MLPERF_LOG(detail, "result_query_count", query_count); + if (settings.scenario == TestScenario::Server) { + MLPERF_LOG(detail, "result_overlatency_query_count", + overlatency_query_count); + } + + auto reportPerQueryLatencies = [&]() { + MLPERF_LOG(detail, "result_min_query_latency_ns", query_latency_min); + MLPERF_LOG(detail, "result_max_query_latency_ns", query_latency_max); + MLPERF_LOG(detail, "result_mean_query_latency_ns", query_latency_mean); + for (auto& lp : latency_percentiles) { + std::string percentile = DoubleToString(lp.percentile * 100); + MLPERF_LOG(detail, + "result_" + percentile + "_percentile_per_query_latency_ns", + lp.query_latency); + } + }; + + // Per-scenario performance results. + switch (settings.scenario) { + case TestScenario::SingleStream: { + double qps_w_lg = (sample_count - 1) / pr.final_query_issued_time; + double qps_wo_lg = 1 / QuerySampleLatencyToSeconds(sample_latency_mean); + MLPERF_LOG(detail, "result_qps_with_loadgen_overhead", qps_w_lg); + MLPERF_LOG(detail, "result_qps_without_loadgen_overhead", qps_wo_lg); + MLPERF_LOG(detail, "early_stopping_latency_ss", + early_stopping_latency_ss); + MLPERF_LOG(detail, "early_stopping_latency_ms", + early_stopping_latency_ms); + break; + } + case TestScenario::MultiStream: { + reportPerQueryLatencies(); + MLPERF_LOG(detail, "early_stopping_latency_ms", + early_stopping_latency_ms); + break; + } + case TestScenario::Server: { + // Subtract 1 from sample count since the start of the final sample + // represents the open end of the time range: i.e. [begin, end). + // This makes sense since: + // a) QPS doesn't apply if there's only one sample; it's pure latency. + // b) If you have precisely 1k QPS, there will be a sample exactly on + // the 1 second time point; but that would be the 1001th sample in + // the stream. Given the first 1001 queries, the QPS is + // 1000 queries / 1 second. + double qps_as_scheduled = + (sample_count - 1) / pr.final_query_scheduled_time; + MLPERF_LOG(detail, "result_scheduled_samples_per_sec", qps_as_scheduled); + double qps_as_completed = + (sample_count - 1) / pr.final_query_all_samples_done_time; + MLPERF_LOG(detail, "result_completed_samples_per_sec", qps_as_completed); + break; + } + case TestScenario::Offline: { + double samples_per_second = sample_count / pr.max_latency; + MLPERF_LOG(detail, "result_samples_per_second", samples_per_second); + break; + } + } + + // Detailed latencies + MLPERF_LOG(detail, "result_min_latency_ns", sample_latency_min); + MLPERF_LOG(detail, "result_max_latency_ns", sample_latency_max); + MLPERF_LOG(detail, "result_mean_latency_ns", sample_latency_mean); + for (auto& lp : latency_percentiles) { + MLPERF_LOG(detail, + "result_" + DoubleToString(lp.percentile * 100) + + "_percentile_latency_ns", + lp.sample_latency); + } + // Detailed first token latencies + if (settings.use_token_latencies) { + if (settings.scenario != TestScenario::Offline) { + MLPERF_LOG(detail, "result_first_token_min_latency_ns", + first_token_latency_min); + MLPERF_LOG(detail, "result_first_token_max_latency_ns", + first_token_latency_max); + MLPERF_LOG(detail, "result_first_token_mean_latency_ns", + first_token_latency_mean); + for (auto& lp : token_latency_percentiles) { + MLPERF_LOG(detail, + "result_first_token_" + DoubleToString(lp.percentile * 100) + + "_percentile_latency_ns", + lp.sample_latency); + } + double tps_w_lg = ((double)token_count) / pr.final_query_issued_time; + double tps_wo_lg = + ((double)token_count) / (sample_latency_mean * sample_count); + MLPERF_LOG(detail, "result_token_throughput_with_loadgen_overhead", + tps_w_lg); + MLPERF_LOG(detail, "result_token_throughput", tps_wo_lg); + for (auto& lp : tpot_percentiles) { + MLPERF_LOG(detail, + "result_time_per_output_token_" + + DoubleToString(lp.percentile * 100) + "_percentile_ns", + lp.sample_latency); + } + MLPERF_LOG(detail, "result_time_to_output_token_min", + time_per_output_token_min); + MLPERF_LOG(detail, "result_time_to_output_token_max", + time_per_output_token_max); + MLPERF_LOG(detail, "result_time_to_output_token_mean", + time_per_output_token_mean); + double tps_as_completed = + token_count / pr.final_query_all_samples_done_time; + MLPERF_LOG(detail, "result_completed_tokens_per_second", + tps_as_completed); + } else { + double tokens_per_second = token_count / pr.max_latency; + MLPERF_LOG(detail, "result_tokens_per_second", tokens_per_second); + } + } + + if (settings.infer_token_latencies) { + switch (settings.scenario) { + case TestScenario::Server: { + double completed_tokens_per_second = + (sample_count - 1) * settings.token_latency_scaling_factor / + pr.final_query_all_samples_done_time; + MLPERF_LOG(detail, "result_inferred_completed_tokens_per_second", + completed_tokens_per_second); + break; + } + case TestScenario::Offline: { + double tokens_per_second = sample_count * + settings.token_latency_scaling_factor / + pr.max_latency; + MLPERF_LOG(detail, "result_inferred_tokens_per_second", + tokens_per_second); + break; + } + case TestScenario::SingleStream: { + break; + } + case TestScenario::MultiStream: { + break; + } + } + } + MLPERF_LOG(detail, "num_errors", detail.async_log().GetErrorCount()); +#endif +} +} // namespace loadgen +} // namespace mlperf diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/results.h b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/results.h new file mode 100644 index 000000000..6befea2c0 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/results.h @@ -0,0 +1,128 @@ +/* Copyright 2019 The MLPerf Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +/// \file +/// \brief Defines PerformanceResult and PerformanceSummary. + +#ifndef MLPERF_LOADGEN_RESULTS_H_ +#define MLPERF_LOADGEN_RESULTS_H_ + +#include +#include + +#include "query_sample.h" +#include "test_settings_internal.h" + +namespace mlperf { +namespace loadgen { + +/// \brief Contains the performance results for benchmarks that have +/// token based metrics +struct TokenPerformanceResults { + std::vector first_token_latencies; + std::vector time_per_output_token_arr; + std::vector tokens_per_sample; +}; + +/// \brief Provides performance results that are independent of scenario +/// and other context. +struct PerformanceResult { + std::vector sample_latencies; + std::vector query_latencies; + size_t queries_issued; + double max_latency; + double final_query_scheduled_time; // seconds from start. + double final_query_issued_time; // seconds from start. + double final_query_all_samples_done_time; // seconds from start. + TokenPerformanceResults token_results; +}; + +/// \brief Wraps PerformanceResult with relevant context to change how +/// it's interpreted and reported. +struct PerformanceSummary { + std::string sut_name; + TestSettingsInternal settings; + PerformanceResult pr; + + // Set by ProcessLatencies. + size_t sample_count = 0; + size_t query_count = 0; + size_t overlatency_query_count = 0; + QuerySampleLatency sample_latency_min = 0; + QuerySampleLatency sample_latency_max = 0; + QuerySampleLatency sample_latency_mean = 0; + QuerySampleLatency query_latency_min = 0; + QuerySampleLatency query_latency_max = 0; + QuerySampleLatency query_latency_mean = 0; + + /// \brief The latency at a given percentile. + struct PercentileEntry { + const double percentile; + QuerySampleLatency sample_latency = 0; + QuerySampleLatency query_latency = 0; // MultiStream only. + }; + + // Latency target percentile + PercentileEntry target_latency_percentile{settings.target_latency_percentile}; + PercentileEntry latency_percentiles[6] = {{.50}, {.90}, {.95}, + {.97}, {.99}, {.999}}; + + // Early stopping percentile estimates for SingleStream and MultiStream + QuerySampleLatency early_stopping_latency_ss = 0; + QuerySampleLatency early_stopping_latency_ms = 0; + + // Set by ProcessTokenLatencies + size_t token_count = 0; + size_t overlatency_first_token_count = 0; + QuerySampleLatency first_token_latency_min = 0; + QuerySampleLatency first_token_latency_max = 0; + QuerySampleLatency first_token_latency_mean = 0; + QuerySampleLatency time_per_output_token_min = 0; + QuerySampleLatency time_per_output_token_max = 0; + QuerySampleLatency time_per_output_token_mean = 0; + + // Latency token target percentile + PercentileEntry token_target_latency_percentile{ + settings.target_latency_percentile}; + PercentileEntry token_latency_percentiles[6] = {{.50}, {.90}, {.95}, + {.97}, {.99}, {.999}}; + PercentileEntry target_tpot_percentile{settings.target_latency_percentile}; + PercentileEntry tpot_percentiles[6] = {{.50}, {.90}, {.95}, + {.97}, {.99}, {.999}}; + +#if defined(_WIN32) || defined(WIN32) || defined(_WIN64) || defined(WIN64) + // MSVC complains if there is no explicit constructor. + // (target_latency_percentile above depends on construction with settings) + PerformanceSummary(const std::string& sut_name_arg, + const TestSettingsInternal& settings_arg, + const PerformanceResult& pr_arg) + : sut_name(sut_name_arg), settings(settings_arg), pr(pr_arg){}; +#endif + void ProcessLatencies(); + void ProcessTokenLatencies(); + + bool MinDurationMet(std::string* recommendation); + bool EarlyStopping(std::string* recommendation, int64_t queries_issued, + std::vector* sample_latencies, + std::vector* query_latencies, + std::chrono::nanoseconds target_latency); + bool MinQueriesMet(); + bool MinSamplesMet(); + bool HasPerfConstraints(); + bool PerfConstraintsMet(std::string* recommendation); + void LogSummary(AsyncSummary& summary); + void LogDetail(AsyncDetail& detail); +}; +} // namespace loadgen +} // namespace mlperf + +#endif diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/setup.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/setup.py new file mode 100644 index 000000000..6254eea17 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/setup.py @@ -0,0 +1,136 @@ +# Copyright 2019 The MLPerf Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +# \file +# \brief MLPerf Inference LoadGen python module setup. +# \details Creates a module that python can import. +# All source files are compiled by python"s C++ toolchain without depending +# on a loadgen lib. +# +# This setup.py can be used stand-alone, without the use of an external +# build system. This will polute your source tree with output files +# and binaries. Use one of the gn build targets instead if you want +# to avoid poluting the source tree. + +from setuptools import Extension, setup +from pathlib import Path +from pybind11 import get_include +from pybind11.setup_helpers import Pybind11Extension, build_ext +from version_generator import generate_loadgen_version_definitions +import subprocess + +generated_version_source_filename = "generated/version_generated.cc" +generate_loadgen_version_definitions(generated_version_source_filename, ".") + +public_headers = [ + "loadgen.h", + "query_sample.h", + "query_sample_library.h", + "system_under_test.h", + "test_settings.h", + "issue_query_controller.h", + "early_stopping.h", + "query_dispatch_library.h" +] + +lib_headers = [ + "logging.h", + "test_settings_internal.h", + "trace_generator.h", + "utils.h", + "version.h", + "results.h", + "bindings/c_api.h", + "version_generator.py", + "mlperf_conf.h" +] + +lib_sources = [ + "early_stopping.cc", + "issue_query_controller.cc", + "loadgen.cc", + "logging.cc", + "test_settings_internal.cc", + "utils.cc", + "version.cc", + "results.cc", +] + +lib_bindings = [ + "bindings/c_api.cc", + "bindings/python_api.cc", +] + +this_directory = Path(__file__).parent +mlperf_loadgen_headers = public_headers + lib_headers +mlperf_loadgen_sources_no_gen = lib_sources + lib_bindings +mlperf_loadgen_sources = mlperf_loadgen_sources_no_gen + [ + generated_version_source_filename +] +mlperf_long_description = ( + this_directory / + "README.md").read_text( + encoding="utf-8") + +with open("VERSION.txt", "r") as f: + version = f.read() +version_split = version.split(".") + +if len(version_split) < 2: + print("Version is incomplete. Needs a format like 4.1.1 in VERSION file") + + +try: + with open("mlperf.conf", 'r') as file: + conf_contents = file.read() + + # Escape backslashes and double quotes + conf_contents = conf_contents.replace('\\', '\\\\').replace('"', '\\"') + + # Convert newlines + conf_contents = conf_contents.replace('\n', '\\n"\n"') + + formatted_content = f'const char* mlperf_conf =\n"{conf_contents}";\n' + + with open("mlperf_conf.h", 'w') as header_file: + header_file.write(formatted_content) + +except IOError as e: + raise RuntimeError(f"Failed to generate header file: {e}") + +mlperf_loadgen_module = Pybind11Extension( + "mlperf_loadgen", + define_macros=[ + ("MAJOR_VERSION", + version_split[0]), + ("MINOR_VERSION", + version_split[1]) + ], + include_dirs=[".", get_include()], + sources=mlperf_loadgen_sources, + depends=mlperf_loadgen_headers, +) + +setup(name="mlcommons_loadgen", + version=version, + description="MLPerf Inference LoadGen python bindings", + url="https://mlcommons.org/", + cmdclass={"build_ext": build_ext}, + ext_modules=[mlperf_loadgen_module], + packages=['mlcommons_loadgen'], + package_dir={'mlcommons_loadgen': '.'}, + include_package_data=True, + long_description=mlperf_long_description, + long_description_content_type='text/markdown') diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/system_under_test.h b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/system_under_test.h new file mode 100644 index 000000000..843453962 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/system_under_test.h @@ -0,0 +1,67 @@ +/* Copyright 2019 The MLPerf Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +/// \file +/// \brief Defines the SystemUnderTest interface. + +#ifndef MLPERF_LOADGEN_SYSTEM_UNDER_TEST_H +#define MLPERF_LOADGEN_SYSTEM_UNDER_TEST_H + +#include +#include + +#include "query_sample.h" + +namespace mlperf { + +/// \addtogroup LoadgenAPI +/// @{ + +/// \brief The interface a client implements for the loadgen to test. +/// \todo Add hook for an untimed warm up period for the SUT. +/// \todo Add hook for an untimed warm up period for the loadgen logic. +/// \todo Support power hooks for cool-down period before runing performance +/// traffic. +/// \todo Support power hooks for correlating test timeline with power +/// measurment timeline. +class SystemUnderTest { + public: + virtual ~SystemUnderTest() {} + + /// \brief A human-readable string for logging purposes. + virtual const std::string& Name() = 0; + + /// \brief Lets the loadgen issue N samples to the SUT. + /// \details The SUT may either a) return immediately and signal completion + /// at a later time on another thread or b) it may block and signal + /// completion on the current stack. The load generator will handle both + /// cases properly. + /// Note: The data for neighboring samples may or may not be contiguous + /// depending on the scenario. + virtual void IssueQuery(const std::vector& samples) = 0; + + /// \brief Called immediately after the last call to IssueQuery + /// in a series is made. + /// \details This doesn't necessarily signify the end of the + /// test since there may be multiple series involved during a test; for + /// example in accuracy mode. + /// Clients can use this to flush any deferred queries immediately, rather + /// than waiting for some timeout. + /// This is especially useful in the server scenario. + virtual void FlushQueries() = 0; +}; + +/// @} + +} // namespace mlperf + +#endif // MLPERF_LOADGEN_SYSTEM_UNDER_TEST_H diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/test_settings.h b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/test_settings.h new file mode 100644 index 000000000..584d073bb --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/test_settings.h @@ -0,0 +1,329 @@ +/* Copyright 2019 The MLPerf Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +/// \file +/// \brief Provides ways for a client to change the behavior and +/// constraints of the load generator. +/// \details Note: The MLPerf specification takes precedent over any of the +/// comments in this file if there are inconsistencies in regards to how the +/// loadgen *should* work. +/// The comments in this file are indicative of the loadgen implementation. + +#ifndef MLPERF_LOADGEN_TEST_SETTINGS_H +#define MLPERF_LOADGEN_TEST_SETTINGS_H + +#include +#include + +namespace mlperf { + +/// \addtogroup LoadgenAPI +/// @{ + +/// \addtogroup LoadgenAPITestSettings Test Settings +/// \brief This page contains a description of all the scenarios, modes, +/// and log settings as implemented by the LoadGen. +/// @{ + +/// +/// \enum TestScenario +/// * **SingleStream** +/// + Issues queries containing a single sample. +/// + The next query is only issued once the previous one has completed. +/// + Internal LoadGen latency between queries is not included in the +/// latency results. +/// + **Final performance result is:** a percentile of the latency. +/// * **MultiStream** +/// + Issues queries containing N samples. +/// - N is specified by \link +/// mlperf::TestSettings::multi_stream_samples_per_query +/// multi_stream_samples_per_query \endlink. +/// + The next query is only issued once the previous one has completed. +/// + The samples of each query are guaranteed to be contiguous with respect +/// to the order they were loaded in the QuerySampleLibrary. +/// + Latency is tracked and reported on a per-query and per-sample basis. +/// + The latency of a query is the maximum latency of its samples, including +/// any cross-thread communication within the loadgen. +/// + Internal LoadGen latency between queries is not included in the +/// latency results. +/// + **Final performance result is:** a percentile of the query latency. +/// * **Server** +/// + Sends queries with a single sample. +/// + Queries have a random poisson (non-uniform) arrival rate that, when +/// averaged, hits the target QPS. +/// + There is no limit on the number of outstanding queries, as long as +/// the latency constraints are met. +/// + **Final performance result is:** PASS if the a percentile of the latency +/// is under a given threshold. FAIL otherwise. +/// - Threshold is specified by \link +/// mlperf::TestSettings::server_target_latency_ns server_target_latency_ns +/// \endlink. +/// * **Offline** +/// + Sends all N samples to the SUT inside of a single query. +/// + The samples of the query are guaranteed to be contiguous with respect +/// to the order they were loaded in the QuerySampleLibrary. +/// + **Final performance result is:** samples per second. +/// +enum class TestScenario { + SingleStream, + MultiStream, + Server, + Offline, +}; + +/// +/// \enum TestMode +/// * **SubmissionRun** +/// + Runs accuracy mode followed by performance mode. +/// + TODO: Implement further requirements as decided by MLPerf. +/// * **AccuracyOnly** +/// + Runs each sample from the QSL through the SUT a least once. +/// + Outputs responses to an accuracy json that can be parsed by a model + +/// sample library specific script. +/// * **PerformanceOnly** +/// + Runs the performance traffic for the given scenario, as described in +/// the comments for TestScenario. +/// * **FindPeakPerformance** +/// + Determines the maximumum QPS for the Server scenario. +/// + Not applicable for SingleStream, MultiStream or Offline scenarios. +/// +enum class TestMode { + SubmissionRun, + AccuracyOnly, + PerformanceOnly, + FindPeakPerformance, +}; + +/// +/// \brief Top-level struct specifing the modes and parameters of the test. +/// +struct TestSettings { + TestScenario scenario = TestScenario::SingleStream; + TestMode mode = TestMode::PerformanceOnly; + + // ================================== + /// \name SingleStream-specific + /**@{*/ + /// \brief A hint used by the loadgen to pre-generate enough samples to + /// meet the minimum test duration. + double single_stream_expected_latency_ns = 1000000; + /// \brief The latency percentile reported as the final result. + double single_stream_target_latency_percentile = 0.90; + /**@}*/ + + // ================================== + /// \name MultiStream-specific + /**@{*/ + /// \brief A hint used by the loadgen to pre-generate enough samples to + /// meet the minimum test duration. + /// \brief MultiStream latency is for query (not sample) latency + double multi_stream_expected_latency_ns = 8000000; + /// \brief The latency percentile for MultiStream mode. + double multi_stream_target_latency_percentile = 0.99; + /// \brief The number of samples in each query. + /// \details How many samples are bundled in a query + uint64_t multi_stream_samples_per_query = 8; + /**@}*/ + + // ================================== + /// \name Server-specific + /**@{*/ + /// \brief The average QPS of the poisson distribution. + /// \details note: This field is used as a FindPeakPerformance's lower bound. + /// When you run FindPeakPerformanceMode, you should make sure that this value + /// satisfies performance constraints. + double server_target_qps = 1; + /// \brief The latency constraint for the Server scenario. + uint64_t server_target_latency_ns = 100000000; + /// \brief The latency percentile for server mode. This value is combined with + /// server_target_latency_ns to determine if a run is valid. + /// \details 99% is the default value, which is correct for image models. GNMT + /// should be set to 0.97 (97%) in v0.5.(As always, check the policy page for + /// updated values for the benchmark you are running.) + double server_target_latency_percentile = 0.99; + /// \brief If this flag is set to true, LoadGen will combine samples from + /// multiple queries into a single query if their scheduled issue times have + /// passed. + bool server_coalesce_queries = false; + /// \brief The decimal places of QPS precision used to terminate + /// FindPeakPerformance mode. + int server_find_peak_qps_decimals_of_precision = 1; + /// \brief A step size (as a fraction of the QPS) used to widen the lower and + /// upper bounds to find the initial boundaries of binary search. + double server_find_peak_qps_boundary_step_size = 1; + /// \brief The maximum number of outstanding queries to allow before earlying + /// out from a performance run. Useful for performance tuning and speeding up + /// the FindPeakPerformance mode. + uint64_t server_max_async_queries = 0; ///< 0: Infinity. + /// \brief The number of issue query threads that will be registered and used + /// to call SUT's IssueQuery(). If this is 0, the same thread calling + /// StartTest() will be used to call IssueQuery(). See also + /// mlperf::RegisterIssueQueryThread(). + uint64_t server_num_issue_query_threads = 0; + /**@}*/ + + // ================================== + /// \name Offline-specific + /**@{*/ + /// \brief Specifies the QPS the SUT expects to hit for the offline load. + /// The loadgen generates 10% more queries than it thinks it needs to meet + /// the minimum test duration. + double offline_expected_qps = 1; + /// \brief Affects the order in which the samples of the dataset are chosen. + /// If false it concatenates a single permutation of the dataset (or part + /// of it depending on QSL->PerformanceSampleCount()) several times up to the + /// number of samples requested. + /// If true it concatenates a multiple permutation of the dataset (or a + /// part of it depending on QSL->PerformanceSampleCount()) several times + /// up to the number of samples requested. + bool sample_concatenate_permutation = false; + /**@}*/ + + // ================================== + /// \name Test duration + /// The test runs until **both** min duration and min query count have been + /// met. However, it will exit before that point if **either** max duration or + /// max query count have been reached. + /**@{*/ + uint64_t min_duration_ms = 10000; + uint64_t max_duration_ms = 0; ///< 0: Infinity. + uint64_t min_query_count = 100; + uint64_t max_query_count = 0; ///< 0: Infinity. + /**@}*/ + + // ================================== + /// \name Random number generation + /// There are 4 separate seeds, so each dimension can be changed + /// independently. + /**@{*/ + /// \brief Affects which subset of samples from the QSL are chosen for + /// the performance sample set and accuracy sample sets. + uint64_t qsl_rng_seed = 0; + /// \brief Affects the order in which samples from the performance set will + /// be included in queries. + uint64_t sample_index_rng_seed = 0; + /// \brief Affects the poisson arrival process of the Server scenario. + /// \details Different seeds will appear to "jitter" the queries + /// differently in time, but should not affect the average issued QPS. + uint64_t schedule_rng_seed = 0; + /// \brief Affects which samples have their query returns logged to the + /// accuracy log in performance mode. + uint64_t accuracy_log_rng_seed = 0; + + /// \brief Probability of the query response of a sample being logged to the + /// accuracy log in performance mode + double accuracy_log_probability = 0.0; + + /// \brief Target number of samples that will have their results printed to + /// accuracy log in performance mode for compliance testing + uint64_t accuracy_log_sampling_target = 0; + + /// \brief Variables for running test05 from native config. A boolean that + /// determines whether or not to run test05 and three random seed to run the + /// test + bool test05 = false; + uint64_t test05_qsl_rng_seed = 0; + uint64_t test05_sample_index_rng_seed = 0; + uint64_t test05_schedule_rng_seed = 0; + + /// \brief Load mlperf parameter config from file. + int FromConfig(const std::string &path, const std::string &model, + const std::string &scenario, int conf_type = 1); + /**@}*/ + + // ================================== + /// \name Performance Sample modifiers + /// \details These settings can be used to Audit Performance mode runs. + /// In order to detect sample caching by SUT, performance of runs when only + /// unique queries (with non-repeated samples) are issued can be compared with + /// that when the same query is repeatedly issued. + /**@{*/ + /// \brief Prints measurement interval start and stop timestamps to std::cout + /// for the purpose of comparison against an external timer + bool print_timestamps = false; + /// \brief Allows issuing only unique queries in Performance mode of any + /// scenario \details This can be used to send non-repeat & hence unique + /// samples to SUT + bool performance_issue_unique = false; + /// \brief If true, the same query is chosen repeatedley for Inference. + /// In offline scenario, the query is filled with the same sample. + bool performance_issue_same = false; + /// \brief Offset to control which sample is repeated in + /// performance_issue_same mode. + /// Value should be within [0, performance_sample_count) + uint64_t performance_issue_same_index = 0; + /// \brief Overrides QSL->PerformanceSampleCount() when non-zero + uint64_t performance_sample_count_override = 0; + /// \brief Measure token latencies + bool use_token_latencies = false; + /// Token latency parameters + uint64_t server_ttft_latency = 100000000; + uint64_t server_tpot_latency = 100000000; + /// \brief Infer token latencies + bool infer_token_latencies = false; + uint64_t token_latency_scaling_factor; + /**@}*/ +}; + +/// +/// \enum LoggingMode +/// Specifies how and when logging should be sampled and stringified at +/// runtime. +/// * **AsyncPoll** +/// + Logs are serialized and output on an IOThread that polls for new logs at +/// a fixed interval. This is the only mode currently implemented. +/// * **EndOfTestOnly** +/// + TODO: Logs are serialzied and output only at the end of the test. +/// * **Synchronous** +/// + TODO: Logs are serialized and output inline. +enum class LoggingMode { + AsyncPoll, + EndOfTestOnly, + Synchronous, +}; + +/// +/// \brief Specifies where log outputs should go. +/// +/// By default, the loadgen outputs its log files to outdir and +/// modifies the filenames of its logs with a prefix and suffix. +/// Filenames will take the form: +/// "/summary.txt" +/// +/// Affordances for outputing logs to stdout are also provided. +/// +struct LogOutputSettings { + std::string outdir = "."; + std::string prefix = "mlperf_log_"; + std::string suffix = ""; + bool prefix_with_datetime = false; + bool copy_detail_to_stdout = false; + bool copy_summary_to_stdout = false; +}; + +/// +/// \brief Top-level log settings. +/// +struct LogSettings { + LogOutputSettings log_output; + LoggingMode log_mode = LoggingMode::AsyncPoll; + uint64_t log_mode_async_poll_interval_ms = 1000; ///< TODO: Implement this. + bool enable_trace = true; +}; + +/// @} + +/// @} + +} // namespace mlperf + +#endif // MLPERF_LOADGEN_TEST_SETTINGS_H diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/test_settings_internal.cc b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/test_settings_internal.cc new file mode 100644 index 000000000..3f2cd8847 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/test_settings_internal.cc @@ -0,0 +1,800 @@ +/* Copyright 2019 The MLPerf Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "test_settings_internal.h" + +#include +#include +#include +#include + +#include "logging.h" +#include "mlperf_conf.h" +#include "utils.h" + +namespace mlperf { +namespace loadgen { + +TestSettingsInternal::TestSettingsInternal( + const TestSettings &requested_settings, size_t qsl_performance_sample_count) + : requested(requested_settings), + scenario(requested.scenario), + mode(requested.mode), + samples_per_query(1), + target_qps(1), + max_async_queries(0), + target_duration(std::chrono::milliseconds(requested.min_duration_ms)), + min_duration(std::chrono::milliseconds(requested.min_duration_ms)), + max_duration(std::chrono::milliseconds(requested.max_duration_ms)), + min_query_count(requested.min_query_count), + max_query_count(requested.max_query_count), + min_sample_count(0), + qsl_rng_seed(requested.qsl_rng_seed), + sample_index_rng_seed(requested.sample_index_rng_seed), + schedule_rng_seed(requested.schedule_rng_seed), + accuracy_log_rng_seed(requested.accuracy_log_rng_seed), + accuracy_log_probability(requested.accuracy_log_probability), + accuracy_log_sampling_target(requested.accuracy_log_sampling_target), + print_timestamps(requested.print_timestamps), + performance_issue_unique(requested.performance_issue_unique), + performance_issue_same(requested.performance_issue_same), + performance_issue_same_index(requested.performance_issue_same_index), + performance_sample_count(0), + sample_concatenate_permutation(false), + use_token_latencies(requested.use_token_latencies), + server_ttft_latency(requested.server_ttft_latency), + server_tpot_latency(requested.server_tpot_latency), + infer_token_latencies(requested.infer_token_latencies), + token_latency_scaling_factor(requested.token_latency_scaling_factor) { + // Target QPS, target latency, and max_async_queries. + switch (requested.scenario) { + case TestScenario::SingleStream: + target_qps = static_cast(std::nano::den) / + requested.single_stream_expected_latency_ns; + max_async_queries = 1; + target_latency_percentile = + requested.single_stream_target_latency_percentile; + break; + case TestScenario::MultiStream: + target_qps = static_cast(std::nano::den) / + requested.multi_stream_expected_latency_ns; + max_async_queries = 1; + target_latency_percentile = + requested.multi_stream_target_latency_percentile; + break; + case TestScenario::Server: + if (requested.server_target_qps >= 0.0) { + target_qps = requested.server_target_qps; + } else { + LogDetail([server_target_qps = requested.server_target_qps, + target_qps = target_qps](AsyncDetail &detail) { +#if USE_NEW_LOGGING_FORMAT + std::stringstream ss; + ss << "Invalid value for server_target_qps requested." + << " requested: " << server_target_qps << " using: " << target_qps; + MLPERF_LOG_ERROR(detail, "error_invalid_test_settings", ss.str()); +#else + detail.Error("Invalid value for server_target_qps requested.", + "requested", server_target_qps, "using", target_qps); +#endif + }); + } + target_latency = + std::chrono::nanoseconds(requested.server_target_latency_ns); + target_latency_percentile = requested.server_target_latency_percentile; + max_async_queries = requested.server_max_async_queries; + break; + case TestScenario::Offline: + // target_latency_percentile is not used in Offline, but set it to + // 0.99 anyway to avoid garbage value. + target_latency_percentile = 0.99; + if (requested.offline_expected_qps >= 0.0) { + target_qps = requested.offline_expected_qps; + } else { + LogDetail([offline_expected_qps = requested.offline_expected_qps, + target_qps = target_qps](AsyncDetail &detail) { +#if USE_NEW_LOGGING_FORMAT + std::stringstream ss; + ss << "Invalid value for offline_expected_qps requested." + << " requested: " << offline_expected_qps + << " using: " << target_qps; + MLPERF_LOG_ERROR(detail, "error_invalid_test_settings", ss.str()); +#else + detail.Error("Invalid value for offline_expected_qps requested.", + "requested", offline_expected_qps, "using", target_qps); +#endif + }); + } + max_async_queries = 1; + break; + } + + // Performance Sample Count: TestSettings override QSL -> + // PerformanceSampleCount + performance_sample_count = (requested.performance_sample_count_override == 0) + ? qsl_performance_sample_count + : requested.performance_sample_count_override; + + // Sample by concatentating several permutations of the dataset + // sample_concatenate_permutation + sample_concatenate_permutation = + (requested.sample_concatenate_permutation == 0) + ? false + : requested.sample_concatenate_permutation; + + // Samples per query. + if (requested.scenario == TestScenario::MultiStream) { + samples_per_query = requested.multi_stream_samples_per_query; + } + + // In the offline scenario, coalesce all queries into a single query. + if (requested.scenario == TestScenario::Offline) { + // TODO: Should the spec require a max duration for large query counts? + // kSlack is used to make sure we generate enough samples for the SUT + // to take longer than than the minimum test duration required by the + // MLPerf spec. + constexpr double kSlack = 1.1; + uint64_t target_sample_count = + kSlack * DurationToSeconds(target_duration) * target_qps; + samples_per_query = + (requested.performance_issue_unique) + ? performance_sample_count + : std::max(min_query_count, target_sample_count); + min_query_count = 1; + target_duration = std::chrono::milliseconds(0); + } + + // FIXME: Only do this for 3D-UNet SingleStream, for v2.0 + // TODO: consolidate after v2.0 + // make min_queries to be multiple of performance_sample_count + // performance_sample_count == 0 makes it to be equal to loaded_samples.size() + if (sample_concatenate_permutation && + requested.scenario == TestScenario::SingleStream) { + // set slack larger for 3D-UNet KiTS19 distribution, i.e. 50% latency << 90% + // latency + constexpr double kSlack = 2.0; + uint64_t expected_queries = + kSlack * DurationToSeconds(target_duration) * target_qps; + min_query_count = + min_query_count > expected_queries ? min_query_count : expected_queries; + min_query_count += qsl_performance_sample_count - + (min_query_count % qsl_performance_sample_count); + } + + min_sample_count = min_query_count * samples_per_query; + + // Validate TestSettings + if (requested.performance_issue_same && + (requested.performance_issue_same_index >= performance_sample_count)) { + LogDetail([performance_issue_same_index = + requested.performance_issue_same_index, + performance_sample_count = + performance_sample_count](AsyncDetail &detail) { +#if USE_NEW_LOGGING_FORMAT + std::stringstream ss; + ss << "Sample Idx to be repeated in performance_issue_same mode" + << " cannot be greater than loaded performance_sample_count." + << " performance_issue_same_index: " << performance_issue_same_index + << " performance_sample_count: " << performance_sample_count; + MLPERF_LOG_ERROR(detail, "error_invalid_test_settings", ss.str()); +#else + detail.Error( + "Sample Idx to be repeated in performance_issue_same mode" + " cannot be greater than loaded performance_sample_count.", + "performance_issue_same_index", performance_issue_same_index, + "performance_sample_count", performance_sample_count); +#endif + }); + } + + if (requested.performance_issue_unique && requested.performance_issue_same) { + LogDetail([performance_issue_unique = requested.performance_issue_unique, + performance_issue_same = + requested.performance_issue_same](AsyncDetail &detail) { +#if USE_NEW_LOGGING_FORMAT + std::stringstream ss; + ss << "Performance_issue_unique and performance_issue_same, both" + << " cannot be true at the same time." + << " performance_issue_unique: " << performance_issue_unique + << " performance_issue_same: " << performance_issue_same; + MLPERF_LOG_ERROR(detail, "error_invalid_test_settings", ss.str()); +#else + detail.Error( + "Performance_issue_unique and performance_issue_same, both" + " cannot be true at the same time.", + "performance_issue_unique", performance_issue_unique, + "performance_issue_same", performance_issue_same); +#endif + }); + } +} + +std::string ToString(TestScenario scenario) { + switch (scenario) { +#if USE_NEW_LOGGING_FORMAT + case TestScenario::SingleStream: + return "SingleStream"; + case TestScenario::MultiStream: + return "MultiStream"; +#else + case TestScenario::SingleStream: + return "Single Stream"; + case TestScenario::MultiStream: + return "Multi Stream"; +#endif + case TestScenario::Server: + return "Server"; + case TestScenario::Offline: + return "Offline"; + } + assert(false); + return "InvalidScenario"; +} + +std::string ToString(TestMode mode) { + switch (mode) { +#if USE_NEW_LOGGING_FORMAT + case TestMode::SubmissionRun: + return "SubmissionRun"; + case TestMode::AccuracyOnly: + return "AccuracyOnly"; + case TestMode::PerformanceOnly: + return "PerformanceOnly"; + case TestMode::FindPeakPerformance: + return "FindPeakPerformance"; +#else + case TestMode::SubmissionRun: + return "Submission"; + case TestMode::AccuracyOnly: + return "Accuracy"; + case TestMode::PerformanceOnly: + return "Performance"; + case TestMode::FindPeakPerformance: + return "Find Peak Performance"; +#endif + } + assert(false); + return "InvalidMode"; +} + +void LogRequestedTestSettings(const TestSettings &s) { + LogDetail([s](AsyncDetail &detail) { +#if USE_NEW_LOGGING_FORMAT + MLPERF_LOG(detail, "requested_scenario", ToString(s.scenario)); + MLPERF_LOG(detail, "requested_test_mode", ToString(s.mode)); + + // Scenario-specific + switch (s.scenario) { + case TestScenario::SingleStream: + MLPERF_LOG(detail, "requested_single_stream_expected_latency_ns", + s.single_stream_expected_latency_ns); + MLPERF_LOG(detail, "requested_single_stream_target_latency_percentile", + s.single_stream_target_latency_percentile); + break; + case TestScenario::MultiStream: + MLPERF_LOG(detail, "requested_multi_stream_expected_latency_ns", + s.multi_stream_expected_latency_ns); + MLPERF_LOG(detail, "requested_multi_stream_target_latency_percentile", + s.multi_stream_target_latency_percentile); + MLPERF_LOG(detail, "requested_multi_stream_samples_per_query", + s.multi_stream_samples_per_query); + break; + case TestScenario::Server: + MLPERF_LOG(detail, "requested_server_target_qps", s.server_target_qps); + MLPERF_LOG(detail, "requested_server_target_latency_ns", + s.server_target_latency_ns); + MLPERF_LOG(detail, "requested_server_target_latency_percentile", + s.server_target_latency_percentile); + MLPERF_LOG(detail, "requested_server_coalesce_queries", + s.server_coalesce_queries); + MLPERF_LOG(detail, + "requested_server_find_peak_qps_decimals_of_precision", + s.server_find_peak_qps_decimals_of_precision); + MLPERF_LOG(detail, "requested_server_find_peak_qps_boundary_step_size", + s.server_find_peak_qps_boundary_step_size); + MLPERF_LOG(detail, "requested_server_max_async_queries", + s.server_max_async_queries); + MLPERF_LOG(detail, "requested_server_num_issue_query_threads", + s.server_num_issue_query_threads); + break; + case TestScenario::Offline: + MLPERF_LOG(detail, "requested_offline_expected_qps", + s.offline_expected_qps); + break; + } + + // Overrides + MLPERF_LOG(detail, "requested_min_duration_ms", s.min_duration_ms); + MLPERF_LOG(detail, "requested_max_duration_ms", s.max_duration_ms); + MLPERF_LOG(detail, "requested_min_query_count", s.min_query_count); + MLPERF_LOG(detail, "requested_max_query_count", s.max_query_count); + MLPERF_LOG(detail, "requested_qsl_rng_seed", s.qsl_rng_seed); + MLPERF_LOG(detail, "requested_sample_index_rng_seed", + s.sample_index_rng_seed); + MLPERF_LOG(detail, "requested_schedule_rng_seed", s.schedule_rng_seed); + MLPERF_LOG(detail, "requested_accuracy_log_rng_seed", + s.accuracy_log_rng_seed); + MLPERF_LOG(detail, "requested_accuracy_log_probability", + s.accuracy_log_probability); + MLPERF_LOG(detail, "requested_accuracy_log_sampling_target", + s.accuracy_log_sampling_target); + MLPERF_LOG(detail, "requested_print_timestamps", s.print_timestamps); + MLPERF_LOG(detail, "requested_performance_issue_unique", + s.performance_issue_unique); + MLPERF_LOG(detail, "requested_performance_issue_same", + s.performance_issue_same); + MLPERF_LOG(detail, "requested_performance_issue_same_index", + s.performance_issue_same_index); + MLPERF_LOG(detail, "requested_performance_sample_count_override", + s.performance_sample_count_override); + MLPERF_LOG(detail, "requested_sample_concatenate_permutation", + s.sample_concatenate_permutation); + // Token latencies specific values + if (s.use_token_latencies) { + MLPERF_LOG(detail, "requested_use_token_latencies", + s.use_token_latencies); + if (s.scenario != TestScenario::Offline) { + MLPERF_LOG(detail, "requested_server_ttft_latency", + s.server_ttft_latency); + MLPERF_LOG(detail, "requested_server_tpot_latency", + s.server_tpot_latency); + } + } +#else + detail(""); + detail("Requested Settings:"); + detail("Scenario : " + ToString(s.scenario)); + detail("Test mode : " + ToString(s.mode)); + + // Scenario-specific + switch (s.scenario) { + case TestScenario::SingleStream: + detail("single_stream_expected_latency_ns : ", + s.single_stream_expected_latency_ns); + detail("single_stream_target_latency_percentile : ", + s.single_stream_target_latency_percentile); + break; + case TestScenario::MultiStream: + detail("multi_stream_expected_latency_ns : ", + s.multi_stream_expected_latency_ns); + detail("multi_stream_target_latency_percentile : ", + s.multi_stream_target_latency_percentile); + detail("multi_stream_samples_per_query : ", + s.multi_stream_samples_per_query); + break; + case TestScenario::Server: + detail("server_target_qps : ", s.server_target_qps); + detail("server_target_latency_ns : ", s.server_target_latency_ns); + detail("server_target_latency_percentile : ", + s.server_target_latency_percentile); + detail("server_coalesce_queries : ", s.server_coalesce_queries); + detail("server_find_peak_qps_decimals_of_precision : ", + s.server_find_peak_qps_decimals_of_precision); + detail("server_find_peak_qps_boundary_step_size : ", + s.server_find_peak_qps_boundary_step_size); + detail("server_max_async_queries : ", s.server_max_async_queries); + detail("server_num_issue_query_threads : ", + s.server_num_issue_query_threads); + break; + case TestScenario::Offline: + detail("offline_expected_qps : ", s.offline_expected_qps); + break; + } + + // Overrides + detail("min_duration_ms : ", s.min_duration_ms); + detail("max_duration_ms : ", s.max_duration_ms); + detail("min_query_count : ", s.min_query_count); + detail("max_query_count : ", s.max_query_count); + detail("qsl_rng_seed : ", s.qsl_rng_seed); + detail("sample_index_rng_seed : ", s.sample_index_rng_seed); + detail("schedule_rng_seed : ", s.schedule_rng_seed); + detail("accuracy_log_rng_seed : ", s.accuracy_log_rng_seed); + detail("accuracy_log_probability : ", s.accuracy_log_probability); + detail("accuracy_log_sampling_target : ", s.accuracy_log_sampling_target); + detail("print_timestamps : ", s.print_timestamps); + detail("performance_issue_unique : ", s.performance_issue_unique); + detail("performance_issue_same : ", s.performance_issue_same); + detail("performance_issue_same_index : ", s.performance_issue_same_index); + detail("performance_sample_count_override : ", + s.performance_sample_count_override); + detail(""); +#endif + }); +} + +void TestSettingsInternal::LogEffectiveSettings() const { + LogDetail([s = *this](AsyncDetail &detail) { +#if USE_NEW_LOGGING_FORMAT + MLPERF_LOG(detail, "effective_scenario", ToString(s.scenario)); + MLPERF_LOG(detail, "effective_test_mode", ToString(s.mode)); + + MLPERF_LOG(detail, "effective_samples_per_query", s.samples_per_query); + MLPERF_LOG(detail, "effective_target_qps", s.target_qps); + MLPERF_LOG(detail, "effective_target_latency_ns", s.target_latency.count()); + MLPERF_LOG(detail, "effective_target_latency_percentile", + s.target_latency_percentile); + MLPERF_LOG(detail, "effective_max_async_queries", s.max_async_queries); + MLPERF_LOG(detail, "effective_target_duration_ms", + s.target_duration.count()); + MLPERF_LOG(detail, "effective_min_duration_ms", s.min_duration.count()); + MLPERF_LOG(detail, "effective_max_duration_ms", s.max_duration.count()); + MLPERF_LOG(detail, "effective_min_query_count", s.min_query_count); + MLPERF_LOG(detail, "effective_max_query_count", s.max_query_count); + MLPERF_LOG(detail, "effective_min_sample_count", s.min_sample_count); + MLPERF_LOG(detail, "effective_qsl_rng_seed", s.qsl_rng_seed); + MLPERF_LOG(detail, "effective_sample_index_rng_seed", + s.sample_index_rng_seed); + MLPERF_LOG(detail, "effective_schedule_rng_seed", s.schedule_rng_seed); + MLPERF_LOG(detail, "effective_accuracy_log_rng_seed", + s.accuracy_log_rng_seed); + MLPERF_LOG(detail, "effective_accuracy_log_probability", + s.accuracy_log_probability); + MLPERF_LOG(detail, "effective_accuracy_log_sampling_target", + s.accuracy_log_sampling_target); + MLPERF_LOG(detail, "effective_print_timestamps", s.print_timestamps); + MLPERF_LOG(detail, "effective_performance_issue_unique", + s.performance_issue_unique); + MLPERF_LOG(detail, "effective_performance_issue_same", + s.performance_issue_same); + MLPERF_LOG(detail, "effective_performance_issue_same_index", + s.performance_issue_same_index); + MLPERF_LOG(detail, "effective_performance_sample_count", + s.performance_sample_count); + MLPERF_LOG(detail, "effective_sample_concatenate_permutation", + s.sample_concatenate_permutation); +#else + detail(""); + detail("Effective Settings:"); + + detail("Scenario : " + ToString(s.scenario)); + detail("Test mode : " + ToString(s.mode)); + + detail("samples_per_query : ", s.samples_per_query); + detail("target_qps : ", s.target_qps); + detail("target_latency (ns): ", s.target_latency.count()); + detail("target_latency_percentile : ", s.target_latency_percentile); + detail("max_async_queries : ", s.max_async_queries); + detail("target_duration (ms): ", s.target_duration.count()); + detail("min_duration (ms): ", s.min_duration.count()); + detail("max_duration (ms): ", s.max_duration.count()); + detail("min_query_count : ", s.min_query_count); + detail("max_query_count : ", s.max_query_count); + detail("min_sample_count : ", s.min_sample_count); + detail("qsl_rng_seed : ", s.qsl_rng_seed); + detail("sample_index_rng_seed : ", s.sample_index_rng_seed); + detail("schedule_rng_seed : ", s.schedule_rng_seed); + detail("accuracy_log_rng_seed : ", s.accuracy_log_rng_seed); + detail("accuracy_log_probability : ", s.accuracy_log_probability); + detail("accuracy_log_sampling_target : ", s.accuracy_log_sampling_target); + detail("print_timestamps : ", s.print_timestamps); + detail("performance_issue_unique : ", s.performance_issue_unique); + detail("performance_issue_same : ", s.performance_issue_same); + detail("performance_issue_same_index : ", s.performance_issue_same_index); + detail("performance_sample_count : ", s.performance_sample_count); +#endif + }); +} + +void TestSettingsInternal::LogAllSettings() const { + LogRequestedTestSettings(requested); + LogEffectiveSettings(); +} + +void TestSettingsInternal::LogSummary(AsyncSummary &summary) const { + summary("samples_per_query : ", samples_per_query); + summary("target_qps : ", target_qps); + if (!use_token_latencies) { + summary("target_latency (ns): ", target_latency.count()); + } else { + summary("ttft_latency (ns): ", server_ttft_latency); + summary("tpot_latency (ns): ", server_tpot_latency); + } + summary("max_async_queries : ", max_async_queries); + summary("min_duration (ms): ", min_duration.count()); + summary("max_duration (ms): ", max_duration.count()); + summary("min_query_count : ", min_query_count); + summary("max_query_count : ", max_query_count); + summary("qsl_rng_seed : ", qsl_rng_seed); + summary("sample_index_rng_seed : ", sample_index_rng_seed); + summary("schedule_rng_seed : ", schedule_rng_seed); + summary("accuracy_log_rng_seed : ", accuracy_log_rng_seed); + summary("accuracy_log_probability : ", accuracy_log_probability); + summary("accuracy_log_sampling_target : ", accuracy_log_sampling_target); + summary("print_timestamps : ", print_timestamps); + summary("performance_issue_unique : ", performance_issue_unique); + summary("performance_issue_same : ", performance_issue_same); + summary("performance_issue_same_index : ", performance_issue_same_index); + summary("performance_sample_count : ", performance_sample_count); + if (sample_concatenate_permutation) { + summary( + "WARNING: sample_concatenate_permutation was set to true. \n" + "Generated samples per query might be different as the one in the " + "setting.\n" + "Check the generated_samples_per_query line in the detailed log for " + "the real\n" + "samples_per_query value"); + } +} + +} // namespace loadgen + +int TestSettings::FromConfig(const std::string &path, const std::string &model, + const std::string &scenario, int conf_type) { + std::map kv; + static int configCount = 0; + + if (conf_type == 1) { + if (configCount == 0) { + // Only allow userConf as the single configFile and loadgen loads the + // mlperfConf automatically for perf and accuracy runs + FromConfig("", model, scenario, 0); + } + + else { + LogDetail([](AsyncDetail &detail) { + std::stringstream ss; + ss << "Multiple conf files are used. This is not valid for official " + "submission."; + MLPERF_LOG_ERROR(detail, "error_invalid_config", ss.str()); + }); + } + configCount++; + } + + // lookup key/value pairs from config + auto lookupkv = [&](const std::string &model, const std::string &scenario, + const std::string &key, uint64_t *val_l, double *val_d, + double multiplier = 1.0) { + std::map::iterator it; + std::string found; + // lookup exact key first + it = kv.find(model + "." + scenario + "." + key); + if (it != kv.end()) { + found = it->second; + } else { + // lookup key with model wildcard + it = kv.find("*." + scenario + "." + key); + if (it != kv.end()) { + found = it->second; + } else { + it = kv.find(model + ".*." + key); + if (it != kv.end()) { + found = it->second; + } else { + it = kv.find("*.*." + key); + if (it != kv.end()) { + found = it->second; + } else { + return false; + } + } + } + } + // if we get here, found will be set + if (val_l) { + *val_l = strtoull(found.c_str(), nullptr, 0) * + static_cast(multiplier); + } + if (val_d) *val_d = strtod(found.c_str(), nullptr) * multiplier; + return true; + }; + + int line_nr = 0; + int errors = 0; + // Declare the input stream before the if-else block + std::unique_ptr fss; + std::string line; + + if (conf_type != 0) { + // dirt simple config parser + fss = std::make_unique(path); + if (!static_cast(fss.get())->is_open()) { + LogDetail([p = path](AsyncDetail &detail) { +#if USE_NEW_LOGGING_FORMAT + std::stringstream ss; + ss << "can't open file " << p; + MLPERF_LOG_ERROR(detail, "error_invalid_config", ss.str()); +#else + detail.Error("can't open file ", p); +#endif + }); + return -ENOENT; + } + } else { + // Convert unsigned char array to std::string + std::string config_str(mlperf_conf); + fss = std::make_unique(config_str); + } + while (std::getline(*fss, line)) { + line_nr++; + std::istringstream iss(line); + std::string s, k; + int looking_for = 0; // 0=key, 1=equal, 2=value + while (iss >> s) { + if (s == "#" && looking_for != 2) { + // done with this line + break; + } + if (looking_for == 2) { + // got key and value + const char *start = s.c_str(); + char *stop; + (void)strtoul(start, &stop, 0); + if (start + s.size() == stop) { + kv[k] = s; + continue; + } + (void)strtod(start, &stop); + if (start + s.size() == stop) { + kv[k] = s; + continue; + } + errors++; + LogDetail([l = line_nr](AsyncDetail &detail) { +#if USE_NEW_LOGGING_FORMAT + std::stringstream ss; + ss << "value needs to be integer or double, line=" << l; + MLPERF_LOG_ERROR(detail, "error_invalid_config", ss.str()); +#else + detail.Error("value needs to be integer or double, line=", l); +#endif + }); + break; + } + if (looking_for == 1 && s != "=") { + errors++; + LogDetail([l = line_nr](AsyncDetail &detail) { +#if USE_NEW_LOGGING_FORMAT + std::stringstream ss; + ss << "expected 'key=value', line=" << l; + MLPERF_LOG_ERROR(detail, "error_invalid_config", ss.str()); +#else + detail.Error("expected 'key=value', line=", l); +#endif + }); + break; + } + if (looking_for == 0) k = s; + looking_for++; + } + } + if (errors != 0) return -EINVAL; + + uint64_t val; + + // keys that apply to all scenarios + if (lookupkv(model, scenario, "mode", &val, nullptr)) { + switch (val) { + case 0: + mode = TestMode::SubmissionRun; + break; + case 1: + mode = TestMode::AccuracyOnly; + break; + case 2: + mode = TestMode::PerformanceOnly; + break; + case 3: + mode = TestMode::FindPeakPerformance; + break; + default: + LogDetail([](AsyncDetail &detail) { +#if USE_NEW_LOGGING_FORMAT + std::stringstream ss; + ss << "Invalid value passed to Mode key in config."; + MLPERF_LOG_ERROR(detail, "error_invalid_config", ss.str()); +#else + detail.Error("Invalid value passed to Mode key in config."); +#endif + }); + break; + } + } + + if (conf_type == 0) { + lookupkv(model, scenario, "qsl_rng_seed", &qsl_rng_seed, nullptr); + lookupkv(model, scenario, "sample_index_rng_seed", &sample_index_rng_seed, + nullptr); + lookupkv(model, scenario, "schedule_rng_seed", &schedule_rng_seed, nullptr); + lookupkv(model, scenario, "accuracy_log_probability", nullptr, + &accuracy_log_probability, 0.01); + if (lookupkv(model, scenario, "test05", &val, nullptr)) + test05 = (val == 1) ? true : false; + lookupkv(model, scenario, "test05_qsl_rng_seed", &test05_qsl_rng_seed, + nullptr); + lookupkv(model, scenario, "test05_sample_index_rng_seed", + &test05_sample_index_rng_seed, nullptr); + lookupkv(model, scenario, "test05_schedule_rng_seed", + &test05_schedule_rng_seed, nullptr); + } + + // keys that can be overriden in user.conf but will make the results eligible + // only for open submissions + + // keys to measure token metrics + if (lookupkv(model, scenario, "use_token_latencies", &val, nullptr)) { + use_token_latencies = (val == 1) ? true : false; + } + if (use_token_latencies) { + lookupkv(model, "Server", "ttft_latency", &server_ttft_latency, nullptr, + 1000 * 1000); + lookupkv(model, "Server", "tpot_latency", &server_tpot_latency, nullptr, + 1000 * 1000); + } + + // keys to infer token metrics + if (lookupkv(model, scenario, "infer_token_latencies", &val, nullptr)) { + infer_token_latencies = (val == 1) ? true : false; + } + if (infer_token_latencies) { + lookupkv(model, scenario, "token_latency_scaling_factor", + &token_latency_scaling_factor, nullptr, 1); + } + // keys that apply to SingleStream + lookupkv(model, "SingleStream", "target_latency_percentile", nullptr, + &single_stream_target_latency_percentile, 0.01); + + // keys that apply to MultiStream + lookupkv(model, "MultiStream", "target_latency_percentile", nullptr, + &multi_stream_target_latency_percentile, 0.01); + lookupkv(model, "MultiStream", "samples_per_query", + &multi_stream_samples_per_query, nullptr, 1); + + // keys that apply to Server + lookupkv(model, "Server", "target_latency_percentile", nullptr, + &server_target_latency_percentile, 0.01); + lookupkv(model, "Server", "target_latency", &server_target_latency_ns, + nullptr, 1000 * 1000); + + // keys that can be overriden in user.conf (the provided values still need to + // pass the submission checker rules) + if (lookupkv(model, scenario, "performance_issue_unique", &val, nullptr)) + performance_issue_unique = (val == 0) ? false : true; + if (lookupkv(model, scenario, "performance_issue_same", &val, nullptr)) + performance_issue_same = (val == 0) ? false : true; + lookupkv(model, scenario, "performance_issue_same_index", + &performance_issue_same_index, nullptr); + + if (lookupkv(model, scenario, "sample_concatenate_permutation", &val, + nullptr)) + sample_concatenate_permutation = (val == 1) ? true : false; + if (lookupkv(model, "Server", "coalesce_queries", &val, nullptr)) + server_coalesce_queries = (val == 0) ? false : true; + if (lookupkv(model, "Server", "max_async_queries", &val, nullptr)) + server_max_async_queries = int(val); + + lookupkv(model, scenario, "min_duration", &min_duration_ms, nullptr); + lookupkv(model, scenario, "max_duration", &max_duration_ms, nullptr); + lookupkv(model, scenario, "min_query_count", &min_query_count, nullptr); + lookupkv(model, scenario, "max_query_count", &max_query_count, nullptr); + lookupkv(model, scenario, "performance_sample_count_override", + &performance_sample_count_override, nullptr); + lookupkv(model, "SingleStream", "target_latency", nullptr, + &single_stream_expected_latency_ns, 1000 * 1000); + lookupkv(model, "MultiStream", "target_latency", nullptr, + &multi_stream_expected_latency_ns, 1000 * 1000); + lookupkv(model, "Server", "target_qps", nullptr, &server_target_qps); + lookupkv(model, "Offline", "target_qps", 0, &offline_expected_qps); + + if (lookupkv(model, scenario, "print_timestamps", &val, nullptr)) + print_timestamps = (val == 0) ? false : true; + + // keys that are used in audit.conf + lookupkv(model, scenario, "accuracy_log_rng_seed", &accuracy_log_rng_seed, + nullptr); + lookupkv(model, scenario, "accuracy_log_sampling_target", + &accuracy_log_sampling_target, nullptr); + return 0; +} + +} // namespace mlperf diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/test_settings_internal.h b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/test_settings_internal.h new file mode 100644 index 000000000..ab2773bd1 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/test_settings_internal.h @@ -0,0 +1,182 @@ +/* Copyright 2019 The MLPerf Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +/// \file +/// \brief The internal representation of user-provided settings. + +#ifndef MLPERF_LOADGEN_TEST_SETTINGS_INTERNAL_H +#define MLPERF_LOADGEN_TEST_SETTINGS_INTERNAL_H + +#include +#include +#include + +#include "logging.h" +#include "test_settings.h" + +namespace mlperf { + +namespace logging { +class AsyncSummary; +} + +namespace loadgen { + +using AsyncSummary = logging::AsyncSummary; + +std::string ToString(TestScenario scenario); +std::string ToString(TestMode mode); + +/// \brief takes the user-friendly TestSettings and normalizes it +/// for consumption by the loadgen. +/// \details It does things like remove scenario-specific naming and introduce +/// the concept of target_duration used to pre-generate queries. +struct TestSettingsInternal { + explicit TestSettingsInternal(const TestSettings &requested_settings, + size_t qsl_performance_sample_count); + void LogEffectiveSettings() const; + void LogAllSettings() const; + void LogSummary(AsyncSummary &summary) const; + + const TestSettings requested; + const TestScenario scenario; // Copied here for convenience. + const TestMode mode; // Copied here for convenience. + + uint64_t samples_per_query; + double target_qps; + std::chrono::nanoseconds target_latency{0}; + double target_latency_percentile; // Single, multistream, and server modes. + uint64_t max_async_queries; + + // Target duration is used to generate queries of a minimum duration before + // the test run. + std::chrono::milliseconds target_duration{0}; + + // Min duration/query_count/sample_count are used to validate the test + // duration at the end of the run. + std::chrono::milliseconds min_duration{0}; + std::chrono::milliseconds max_duration{0}; + uint64_t min_query_count; + uint64_t max_query_count; + uint64_t min_sample_count; // Offline only. + + uint64_t qsl_rng_seed; + uint64_t sample_index_rng_seed; + uint64_t schedule_rng_seed; + uint64_t accuracy_log_rng_seed; + double accuracy_log_probability; + uint64_t accuracy_log_sampling_target; + bool print_timestamps; + bool performance_issue_unique; + bool performance_issue_same; + uint64_t performance_issue_same_index; + uint64_t performance_sample_count; + + bool sample_concatenate_permutation; + bool use_token_latencies = false; + int64_t server_ttft_latency; + int64_t server_tpot_latency; + + bool infer_token_latencies = false; + int64_t token_latency_scaling_factor; +}; + +/// \brief A namespace of collections of FindPeakPerformance helper functions, +/// mainly about binary search. +namespace find_peak_performance { + +constexpr char const *kNotSupportedMsg = + "Finding peak performance is only supported in Server scenarios."; + +template +TestSettingsInternal MidOfBoundaries( + const TestSettingsInternal &lower_bound_settings, + const TestSettingsInternal &upper_bound_settings) { + TestSettingsInternal mid_settings = lower_bound_settings; + if (scenario == TestScenario::Server) { + assert(lower_bound_settings.target_qps < upper_bound_settings.target_qps); + mid_settings.target_qps = + lower_bound_settings.target_qps + + (upper_bound_settings.target_qps - lower_bound_settings.target_qps) / 2; + } else { + LogDetail([](AsyncDetail &detail) { +#if USE_NEW_LOGGING_FORMAT + MLPERF_LOG_ERROR(detail, "error_invalid_test_settings", kNotSupportedMsg); +#else + detail(kNotSupportedMsg); +#endif + }); + } + return mid_settings; +} + +template +bool IsFinished(const TestSettingsInternal &lower_bound_settings, + const TestSettingsInternal &upper_bound_settings) { + if (scenario == TestScenario::Server) { + uint8_t precision = lower_bound_settings.requested + .server_find_peak_qps_decimals_of_precision; + double l = + std::floor(lower_bound_settings.target_qps * std::pow(10, precision)); + double u = + std::floor(upper_bound_settings.target_qps * std::pow(10, precision)); + return l + 1 >= u; + } else { + LogDetail([](AsyncDetail &detail) { +#if USE_NEW_LOGGING_FORMAT + MLPERF_LOG_ERROR(detail, "error_invalid_test_settings", kNotSupportedMsg); +#else + detail(kNotSupportedMsg); +#endif + }); + return true; + } +} + +template +std::string ToStringPerformanceField(const TestSettingsInternal &settings) { + if (scenario == TestScenario::Server) { + return std::to_string(settings.target_qps); + } else { + LogDetail([](AsyncDetail &detail) { +#if USE_NEW_LOGGING_FORMAT + MLPERF_LOG_ERROR(detail, "error_invalid_test_settings", kNotSupportedMsg); +#else + detail(kNotSupportedMsg); +#endif + }); + return ToString(settings.scenario); + } +} + +template +void WidenPerformanceField(TestSettingsInternal *settings) { + if (scenario == TestScenario::Server) { + settings->target_qps = + settings->target_qps * + (1 + settings->requested.server_find_peak_qps_boundary_step_size); + } else { + LogDetail([](AsyncDetail &detail) { +#if USE_NEW_LOGGING_FORMAT + MLPERF_LOG_ERROR(detail, "error_invalid_test_settings", kNotSupportedMsg); +#else + detail(kNotSupportedMsg); +#endif + }); + } +} + +} // namespace find_peak_performance +} // namespace loadgen +} // namespace mlperf + +#endif // MLPERF_LOADGEN_TEST_SETTINGS_INTERNAL_H diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/BUILD.gn b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/BUILD.gn new file mode 100644 index 000000000..d73bf831a --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/BUILD.gn @@ -0,0 +1,25 @@ +static_library("mlperf_loadgen_tests_loadgen_test_main") { + sources = [ "loadgen_test.h", "loadgen_test_main.cc" ] + configs += [ "//build/config/compiler:exceptions" ] +} + +executable("mlperf_loadgen_perftests") { + sources = [ "perftests_null_sut.cc" ] + deps = [ "..:mlperf_loadgen" ] +} + +executable("mlperf_loadgen_tests_basic") { + sources = [ "basic.cc" ] + deps = [ "..:mlperf_loadgen", + ":mlperf_loadgen_tests_loadgen_test_main" ] + configs += [ "//build/config/compiler:exceptions" ] +} + +source_set("mlperf_loadgen_perftests_py") { + sources = [ "perftests_null_sut.py" ] + deps = [ "../..:loadgen_pymodule_wheel_lib" ] +} + +source_set("docs") { + sources = [ "README.md" ] +} diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/README.md b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/README.md new file mode 100644 index 000000000..41056b457 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/README.md @@ -0,0 +1,42 @@ +# Building and Running the Tests {#ReadmeTests} + +The unit and performance tests are only supported via gn/ninja at the moment. + +See the [top-level build readme](@ref ReadmeBuild) for details but, from a clean checkout, you must first run: + + make bootstrap_gn_ninja + third_party/gn/gn gen out/Release --args="is_debug=false" + +This will build the gn and ninja build tools and create a release project. + +## Unit Tests + +To build: + + third_party/ninja/ninja -C out/Release mlperf_loadgen_tests_basic + +To run all tests: + + out/Release/mlperf_loadgen_tests_basic . + +To run specific tests: + + out/Release/mlperf_loadgen_tests_basic + e.g.: + out/Release/mlperf_loadgen_tests_basic SingleStream + +## Performance Tests + +To build: + + third_party/ninja/ninja -C out/Release mlperf_loadgen_perftests + +To run all tests: + + out/Release/mlperf_loadgen_perftests . + +To run specific tests: + + out/Release/mlperf_loadgen_perftests + e.g.: + out/Release/mlperf_loadgen_tests_basic ServerPool diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/basic.cc b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/basic.cc new file mode 100644 index 000000000..97c6a0bb1 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/basic.cc @@ -0,0 +1,314 @@ +/* Copyright 2019 The MLPerf Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +/// \file +/// \brief Basic functionality unit tests. + +#include +#include +#include +#include +#include +#include +#include + +#include "../loadgen.h" +#include "../query_sample_library.h" +#include "../system_under_test.h" +#include "../test_settings.h" +#include "loadgen_test.h" + +/// \brief Correctness unit tests. +namespace unit_tests { + +/// \defgroup LoadgenTestsBasic Test Coverage: Basic + +/// \brief Implements the client interfaces of the loadgen and +/// has some basic sanity checks that are enabled for all tests. +/// \details It also forwards calls to overrideable *Ext methods and implements +/// the TestProxy concept. +struct SystemUnderTestBasic : public mlperf::QuerySampleLibrary, + public mlperf::SystemUnderTest { + const std::string& Name() const override { return name_; } + + size_t TotalSampleCount() override { return total_sample_count_; } + size_t PerformanceSampleCount() override { return performance_sample_count_; } + + void LoadSamplesToRam( + const std::vector& samples) override { + for (auto s : samples) { + samples_load_count_.at(s)++; + loaded_samples_.push_back(s); + } + LoadSamplesToRamExt(samples); + } + virtual void LoadSamplesToRamExt( + const std::vector& samples) {} + + void UnloadSamplesFromRam( + const std::vector& samples) override { + for (auto s : samples) { + FAIL_IF(loaded_samples_.front() != s) && + FAIL_EXP(loaded_samples_.front()) && FAIL_EXP(s); + loaded_samples_.pop_front(); + size_t prev_load_count = samples_load_count_.at(s)--; + FAIL_IF(prev_load_count == 0) && FAIL_EXP(prev_load_count); + } + UnloadSamplesFromRamExt(samples); + } + virtual void UnloadSamplesFromRamExt( + const std::vector& samples) {} + + void IssueQuery(const std::vector& samples) override { + std::vector responses; + query_sizes_.push_back(samples.size()); + samples_between_flushes_.back() += samples.size(); + responses.reserve(samples.size()); + for (auto s : samples) { + FAIL_IF(samples_load_count_.at(s.index) == 0) && + FAIL_MSG("Issued unloaded sample:") && FAIL_EXP(s.index); + samples_issue_count_.at(s.index)++; + issued_samples_.push_back(s.index); + responses.push_back({s.id, 0, 0}); + } + mlperf::QuerySamplesComplete(responses.data(), responses.size()); + IssueQueryExt(samples); + } + virtual void IssueQueryExt(const std::vector& samples) {} + + void FlushQueries() override { + samples_between_flushes_.push_back(0); + FlushQueriesExt(); + } + virtual void FlushQueriesExt() {} + + virtual void RunTest() { + samples_load_count_.resize(total_sample_count_, 0); + samples_issue_count_.resize(total_sample_count_, 0); + samples_between_flushes_.resize(1, 0); + mlperf::StartTest(this, this, test_settings_, log_settings_); + } + + virtual void EndTest() {} + + protected: + mlperf::TestSettings test_settings_; + mlperf::LogSettings log_settings_; + + std::string name_{"BasicSUT"}; + size_t total_sample_count_; + size_t performance_sample_count_; + std::vector issued_samples_; + std::deque loaded_samples_; + std::vector samples_load_count_; + std::vector samples_issue_count_; + + std::vector query_sizes_; + std::vector samples_between_flushes_; +}; + +/// \brief Provides common test set up logic. +struct SystemUnderTestAccuracy : public SystemUnderTestBasic { + virtual void SetUpTest(size_t samples_per_query, + size_t samples_per_query_remainder, + size_t accuracy_remainder, + mlperf::TestScenario scenario) { + performance_sample_count_ = + samples_per_query * 16 + samples_per_query_remainder; + total_sample_count_ = performance_sample_count_ * 32 + accuracy_remainder; + + log_settings_.log_output.prefix_with_datetime = false; + + test_settings_.scenario = scenario; + test_settings_.mode = mlperf::TestMode::AccuracyOnly; + test_settings_.multi_stream_samples_per_query = samples_per_query; + + double qps = 1e3; + test_settings_.server_target_qps = qps; + } +}; + +/// \brief Verifies all samples from the QSL are included at least once +/// in accuracy mode. +/// \ingroup LoadgenTestsBasic +struct TestAccuracyIncludesAllSamples : public SystemUnderTestAccuracy { + void EndTest() override { + std::sort(issued_samples_.begin(), issued_samples_.end()); + + FAIL_IF(issued_samples_.size() < total_sample_count_) && + FAIL_EXP(issued_samples_.size()) && FAIL_EXP(total_sample_count_); + FAIL_IF(issued_samples_.front() != 0) && FAIL_EXP(issued_samples_.front()); + FAIL_IF(issued_samples_.back() != total_sample_count_ - 1) && + FAIL_EXP(issued_samples_.back()) && FAIL_EXP(total_sample_count_); + + mlperf::QuerySampleIndex prev = -1; + size_t discontinuities = 0; + size_t dupes = 0; + for (auto s : issued_samples_) { + if (s == prev) { + dupes++; + } else if (s - prev > 1) { + discontinuities++; + } + prev = s; + } + + FAIL_IF(discontinuities != 0) && FAIL_EXP(discontinuities); + FAIL_IF(dupes != 0) && FAIL_EXP(dupes); + } +}; + +REGISTER_TEST_ALL_SCENARIOS(AccuracyIncludesAllSamples, + TestProxy(), 4, 0, + 0); + +/// \brief Verifies samples from the QSL aren't included too many times. +/// \details This is a regression test for: +/// https://github.com/mlperf/inference/pull/386 +/// The root cause was using different values for samples_per_query while +/// generating queries for the GNMT dataset. +/// \ingroup LoadgenTestsBasic +struct TestAccuracyDupesAreLimitted : public SystemUnderTestAccuracy { + void SetUpTest(bool, mlperf::TestScenario scenario) { + SystemUnderTestAccuracy::SetUpTest(4, 0, 0, scenario); + total_sample_count_ = 3003; + performance_sample_count_ = 1001; + } + + void EndTest() override { + std::sort(issued_samples_.begin(), issued_samples_.end()); + + FAIL_IF(issued_samples_.size() < total_sample_count_) && + FAIL_EXP(issued_samples_.size()) && FAIL_EXP(total_sample_count_); + FAIL_IF(issued_samples_.front() != 0) && FAIL_EXP(issued_samples_.front()); + FAIL_IF(issued_samples_.back() != total_sample_count_ - 1) && + FAIL_EXP(issued_samples_.back()) && FAIL_EXP(total_sample_count_); + + std::vector issue_counts(total_sample_count_, 0); + for (auto s : issued_samples_) { + issue_counts.at(s)++; + } + + const size_t max_count = 1; + for (size_t i = 0; i < issue_counts.size(); i++) { + FAIL_IF(issue_counts[i] > max_count) && FAIL_EXP(i) && + FAIL_EXP(max_count) && FAIL_EXP(issue_counts[i]); + } + } +}; + +REGISTER_TEST_ALL_SCENARIOS(TestAccuracyDupesAreLimitted, + TestProxy(), true); + +/// \brief Verifies offline + accuracy doesn't hang if the last set +/// in the accuracy series is smaller than others. +/// \ingroup LoadgenTestsBasic +struct TestOfflineRemainderAccuracySet : public SystemUnderTestAccuracy { + void SetUpTest() { + SystemUnderTestAccuracy::SetUpTest(4, 0, 7, mlperf::TestScenario::Offline); + } + + void EndTest() override { + auto& flush_samples = samples_between_flushes_; + + FAIL_IF(flush_samples.size() < 3) && FAIL_EXP(flush_samples.size()) && + BAD_TEST_MSG("Test should generate multiple query sets.") && ABORT_TEST; + + // The last counter will be 0, since a test ends with a call to + // FlushQuery. + FAIL_IF(flush_samples.back() != 0) && FAIL_EXP(flush_samples.back()) && + FAIL_MSG( + "Detected stray calls to IssueQuery after the last call to " + "FlushQuery."); + flush_samples.pop_back(); + + // Verify the test ran with a smaller last accuracy set. + size_t first_size = flush_samples.front(); + size_t last_size = flush_samples.back(); + FAIL_IF(first_size <= last_size) && FAIL_EXP(first_size) && + FAIL_EXP(last_size) && BAD_TEST_MSG(); + + flush_samples.pop_back(); // Don't check the last set for equality. + for (size_t query_size : flush_samples) { + FAIL_IF(query_size != first_size) && FAIL_EXP(query_size) && + FAIL_EXP(first_size); + } + } +}; + +REGISTER_TEST(Offline_RemainderAccuracySets, + TestProxy()); + +/// \brief Verifies all queries only contain samples that are contiguous, +/// even if the set size is not a multiple of samples_per_query. +/// \ingroup LoadgenTestsBasic +struct TestMultiStreamContiguousRemainderQuery + : public SystemUnderTestAccuracy { + void SetUpTest(mlperf::TestScenario scenario) { + SystemUnderTestAccuracy::SetUpTest(4, 1, 0, scenario); + first_qsl_offsets_.resize(total_sample_count_, kBadQslOffset); + + auto spq = test_settings_.multi_stream_samples_per_query; + FAIL_IF(performance_sample_count_ % spq == 0) && + FAIL_EXP(performance_sample_count_) && FAIL_EXP(spq) && + BAD_TEST_MSG("There is no remainder."); + } + + void LoadSamplesToRamExt( + const std::vector& samples) override { + FAIL_IF(loaded_samples_.size() != samples.size()) && + FAIL_MSG("Contiguous sample order is likely ambiguous."); + for (size_t i = 0; i < samples.size(); i++) { + auto& offset = first_qsl_offsets_.at(samples.at(i)); + // Samples may be loaded into multiple slots for padding purposes, + // so make sure to only index the first time a sample appears in a + // loaded set. + if (offset == kBadQslOffset) { + offset = i; + } + } + } + + void UnloadSamplesFromRamExt( + const std::vector& samples) override { + FAIL_IF(!loaded_samples_.empty()) && + FAIL_MSG("Contiguous sample order is likely ambiguous."); + for (size_t i = 0; i < samples.size(); i++) { + first_qsl_offsets_.at(samples.at(i)) = kBadQslOffset; + } + } + + void IssueQueryExt(const std::vector& samples) override { + size_t expected_offset = first_qsl_offsets_[samples[0].index]; + for (auto s : samples) { + FAIL_IF(loaded_samples_[expected_offset] != s.index) && + FAIL_MSG("Samples are not contiguous."); + expected_offset++; + } + } + + void FlushQueriesExt() override {} + + void EndTest() override {} + + private: + static const size_t kBadQslOffset; + std::vector first_qsl_offsets_; +}; + +constexpr size_t TestMultiStreamContiguousRemainderQuery::kBadQslOffset = + std::numeric_limits::max(); + +REGISTER_TEST(MultiStream_RemainderQueryContiguous, + TestProxy(), + mlperf::TestScenario::MultiStream); +} // namespace unit_tests diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/loadgen_test.h b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/loadgen_test.h new file mode 100644 index 000000000..777029b99 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/loadgen_test.h @@ -0,0 +1,198 @@ +/* Copyright 2019 The MLPerf Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +/// \file +/// \brief A minimal test framework. + +#ifndef MLPERF_LOADGEN_TESTS_LOADGEN_TEST_H_ +#define MLPERF_LOADGEN_TESTS_LOADGEN_TEST_H_ + +#include +#include +#include +#include +#include +#include + +#define REGISTER_TEST(name, ...) \ + static Test::StaticRegistrant test##name(#name, __VA_ARGS__); + +#define REGISTER_TEST_SCENARIO(name, scenario, test, ...) \ + static Test::StaticRegistrant t##name##scenario( \ + #name "_" #scenario, test, __VA_ARGS__, mlperf::TestScenario::scenario) + +#define REGISTER_TEST_ALL_SCENARIOS(name, test, ...) \ + REGISTER_TEST_SCENARIO(name, SingleStream, test, __VA_ARGS__); \ + REGISTER_TEST_SCENARIO(name, MultiStream, test, __VA_ARGS__); \ + REGISTER_TEST_SCENARIO(name, Server, test, __VA_ARGS__); \ + REGISTER_TEST_SCENARIO(name, Offline, test, __VA_ARGS__); + +#define FAIL_IF(exp) \ + [&]() { \ + const bool v = exp; \ + if (v) { \ + std::cerr << "\n ERROR: (" << __FILE__ << "@" << __LINE__ \ + << ") : " #exp; \ + Test::AddFailure(); \ + } \ + return v; \ + }() + +#define FAIL_MSG(...) \ + [&]() { \ + std::cerr << "\n Info: (" << __FILE__ << "@" << __LINE__ << ") : "; \ + Test::Log(__VA_ARGS__); \ + return true; \ + }() + +#define FAIL_EXP(exp) \ + [&]() { \ + std::cerr << "\n Info: (" << __FILE__ << "@" << __LINE__ << ") : "; \ + std::cerr << #exp << " is " << (exp); \ + return true; \ + }() + +#define BAD_TEST_MSG(...) \ + [&]() { \ + FAIL_MSG("The test isn't testing what it claims to test. "); \ + Test::Log(__VA_ARGS__); \ + return true; \ + }() + +#define ABORT_TEST \ + [&]() { \ + FAIL_MSG("ABORTING"); \ + throw std::logic_error("ABORT_TEST encountered."); \ + return false; \ + }(); + +/// \brief Testing utilities. +namespace testing { + +/// \brief Wraps a test class as a functor for easy registration. +/// Forwards registration args to a SetUpTest method. +/// \details Calls SetUpTest, RunTest, and EndTest. +template +struct TestProxy { + template + void operator()(Args&&... args) { + TestT test; + test.SetUpTest(std::forward(args)...); + test.RunTest(); + test.EndTest(); + } +}; + +/// \brief A collection of methods for registering and running tests. +class Test { + /// \brief Maps registered test names to a callback. + using TestMap = std::multimap>; + + /// \brief The registered tests. + /// \details Wraps a static local to avoid undefined initialization order + /// and guarantee it is initialized before the first test registers itself. + static TestMap& tests() { + static TestMap tests_; + return tests_; + } + + /// \brief The number of errors the current test has encountered. + static size_t& test_fails() { + static size_t test_fails_ = 0; + return test_fails_; + } + + public: + /// \brief Registers a test before main() starts during static initialization. + struct StaticRegistrant { + template + StaticRegistrant(Args&&... args) { + Test::Register(std::forward(args)...); + } + }; + + /// \brief Registers a test at runtime. + template + static void Register(const char* name, TestF test, Args&&... args) { + std::function test_closure = + std::bind(test, std::forward(args)...); + tests().insert({std::move(name), std::move(test_closure)}); + } + + /// \brief Runs all currently registered tests that match the given filter. + static int Run(std::function filter) { + // Determine which tests are enabled. + std::vector enabled_tests; + for (auto& test : tests()) { + if (filter(test.first)) { + enabled_tests.push_back(&test); + } + } + const size_t enabled = enabled_tests.size(); + std::cout << enabled << " of " << tests().size() << " tests enabled.\n"; + + // Run the tests. + std::vector failures; + for (size_t i = 0; i < enabled; i++) { + const char* name = enabled_tests[i]->first; + std::cout << "[" << (i + 1) << "/" << enabled << "] : " << name << " : "; + std::cout.flush(); + test_fails() = 0; + try { + enabled_tests[i]->second(); // Run the test. + } catch (std::exception& e) { + constexpr bool TestThrewException = true; + FAIL_IF(TestThrewException) && FAIL_EXP(e.what()); + } + if (test_fails() > 0) { + failures.push_back(name); + std::cerr << "\n FAILED: " << name << "\n"; + } else { + std::cout << "SUCCESS\n"; + } + } + + // Summarize. + if (enabled_tests.empty()) { + std::cerr << "Check your test filter.\n"; + } else if (failures.empty()) { + std::cout << "All " << enabled << " tests passed! \\o/\n"; + } else { + std::cout << failures.size() << " of " << enabled << " tests failed:\n"; + for (auto failed_test_name : failures) { + std::cout << " " << failed_test_name << "\n"; + } + } + return failures.size(); + } + + /// \brief Used by test macros to flag test failure. + static void AddFailure() { test_fails()++; } + + /// \brief Base case for the variadic version of Log. + static void Log() {} + + /// \brief Used by test macros to log an arbitrary list of args. + template + static void Log(T&& v, Args&&... args) { + std::cerr << v; + Log(std::forward(args)...); + } +}; + +} // namespace testing + +// The testing namespace exists for documentation purposes. +// Export the testing namespace for all files that define tests. +using namespace testing; + +#endif // MLPERF_LOADGEN_TESTS_LOADGEN_TEST_H_ diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/loadgen_test_main.cc b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/loadgen_test_main.cc new file mode 100644 index 000000000..3dc5afa80 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/loadgen_test_main.cc @@ -0,0 +1,33 @@ +/* Copyright 2019 The MLPerf Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +/// \file +/// \brief A main entry point a test binary can use if it just wants to execute +/// Test::Run on all statically registered tests. + +#include + +#include "loadgen_test.h" + +int main(int argc, char* argv[]) { + if (argc <= 1) { + std::cerr << "Usage: " << argv[0] << " \n"; + return -1; + } + std::regex include_regex(argc >= 2 ? argv[1] : ".*"); + std::regex exclude_regex(argc >= 3 ? std::regex(argv[2]) : std::regex()); + auto test_filter = [&](const char* test_name) { + return (std::regex_search(test_name, include_regex) && + !std::regex_search(test_name, exclude_regex)); + }; + return Test::Run(test_filter); +} diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/perftests_null_sut.cc b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/perftests_null_sut.cc new file mode 100644 index 000000000..56d562c3e --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/perftests_null_sut.cc @@ -0,0 +1,230 @@ +/* Copyright 2019 The MLPerf Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +/// \file +/// \brief Performance tests using a null backend. + +#include + +#include "../loadgen.h" +#include "../query_sample_library.h" +#include "../system_under_test.h" +#include "../test_settings.h" + +/// \brief Performance unit tests. +namespace perf_tests { + +/// \defgroup LoadgenTestsPerformance Test Coverage: Performance + +/// \brief A simple SUT implemenatation that immediately completes +/// issued queries sychronously ASAP. +class SystemUnderTestNull : public mlperf::SystemUnderTest { + public: + SystemUnderTestNull() = default; + ~SystemUnderTestNull() override = default; + const std::string& Name() override { return name_; } + void IssueQuery(const std::vector& samples) override { + std::vector responses; + responses.reserve(samples.size()); + for (auto s : samples) { + responses.push_back({s.id, 0, 0}); + } + mlperf::QuerySamplesComplete(responses.data(), responses.size()); + } + + void FlushQueries() override {} + + private: + std::string name_{"NullSUT"}; +}; + +/// \brief A stub implementation of QuerySampleLibrary. +class QuerySampleLibraryNull : public mlperf::QuerySampleLibrary { + public: + QuerySampleLibraryNull() = default; + ~QuerySampleLibraryNull() = default; + const std::string& Name() const override { return name_; } + + size_t TotalSampleCount() override { return 1024 * 1024; } + + size_t PerformanceSampleCount() override { return 1024; } + + void LoadSamplesToRam( + const std::vector& samples) override { + return; + } + + void UnloadSamplesFromRam( + const std::vector& samples) override { + return; + } + + private: + std::string name_{"NullQSL"}; +}; + +/// \brief Runs single stream traffic. +/// \ingroup LoadgenTestsPerformance +void TestSingleStream() { + SystemUnderTestNull null_sut; + QuerySampleLibraryNull null_qsl; + + mlperf::LogSettings log_settings; + log_settings.log_output.prefix_with_datetime = true; + + mlperf::TestSettings ts; + + mlperf::StartTest(&null_sut, &null_qsl, ts, log_settings); +} + +/// \brief A SUT implementation that completes queries asynchronously using +/// std::async. +class SystemUnderTestNullStdAsync : public mlperf::SystemUnderTest { + public: + SystemUnderTestNullStdAsync() { futures_.reserve(1000000); } + ~SystemUnderTestNullStdAsync() override = default; + const std::string& Name() const override { return name_; } + void IssueQuery(const std::vector& samples) override { + futures_.emplace_back(std::async(std::launch::async, [samples] { + std::vector responses; + responses.reserve(samples.size()); + for (auto s : samples) { + responses.push_back({s.id, 0, 0}); + } + mlperf::QuerySamplesComplete(responses.data(), responses.size()); + })); + } + + void FlushQueries() override {} + + private: + std::string name_{"NullStdAsync"}; + std::vector> futures_; +}; + +/// \brief Tests server traffic using SystemUnderTestNullStdAsync. +/// \ingroup LoadgenTestsPerformance +void TestServerStdAsync() { + SystemUnderTestNullStdAsync null_std_async_sut; + QuerySampleLibraryNull null_qsl; + + mlperf::LogSettings log_settings; + log_settings.log_output.prefix_with_datetime = true; + log_settings.log_output.copy_summary_to_stdout = true; + + mlperf::TestSettings ts; + ts.scenario = mlperf::TestScenario::Server; + ts.server_target_qps = 2000000; + ts.min_duration_ms = 100; + + mlperf::StartTest(&null_std_async_sut, &null_qsl, ts, log_settings); +} + +/// \brief A SUT implementation that completes queries asynchronously using +/// an explicitly managed thread pool. +class SystemUnderTestNullPool : public mlperf::SystemUnderTest { + public: + SystemUnderTestNullPool() { + samples_.reserve(kReserveSampleSize); + next_poll_time_ = std::chrono::high_resolution_clock::now() + poll_period_; + for (size_t i = 0; i < thread_count_; i++) { + threads_.emplace_back(&SystemUnderTestNullPool::WorkerThread, this); + } + } + + ~SystemUnderTestNullPool() override { + { + std::unique_lock lock(mutex_); + keep_workers_alive_ = false; + } + cv_.notify_all(); + for (auto& thread : threads_) { + thread.join(); + } + } + + const std::string& Name() const override { return name_; } + + void IssueQuery(const std::vector& samples) override { + std::unique_lock lock(mutex_); + samples_.insert(samples_.end(), samples.begin(), samples.end()); + } + + void FlushQueries() override {} + + private: + void WorkerThread() { + std::vector my_samples; + my_samples.reserve(kReserveSampleSize); + std::unique_lock lock(mutex_); + while (keep_workers_alive_) { + next_poll_time_ += poll_period_; + auto my_wakeup_time = next_poll_time_; + cv_.wait_until(lock, my_wakeup_time, + [&]() { return !keep_workers_alive_; }); + my_samples.swap(samples_); + lock.unlock(); + + std::vector responses; + responses.reserve(my_samples.size()); + for (auto s : my_samples) { + responses.push_back({s.id, 0, 0}); + } + mlperf::QuerySamplesComplete(responses.data(), responses.size()); + + lock.lock(); + my_samples.clear(); + } + } + + static constexpr size_t kReserveSampleSize = 1024 * 1024; + const std::string name_{"NullPool"}; + const size_t thread_count_ = 4; + const std::chrono::milliseconds poll_period_{1}; + std::chrono::high_resolution_clock::time_point next_poll_time_; + + std::mutex mutex_; + std::condition_variable cv_; + bool keep_workers_alive_ = true; + std::vector threads_; + + std::vector samples_; +}; + +/// \brief Tests server traffic using SystemUnderTestNullPool. +/// \ingroup LoadgenTestsPerformance +void TestServerPool() { + SystemUnderTestNullPool null_pool; + QuerySampleLibraryNull null_qsl; + + mlperf::LogSettings log_settings; + log_settings.log_output.prefix_with_datetime = true; + log_settings.log_output.copy_summary_to_stdout = true; + + mlperf::TestSettings ts; + ts.scenario = mlperf::TestScenario::Server; + ts.server_target_qps = 2000000; + ts.min_duration_ms = 100; + + mlperf::StartTest(&null_pool, &null_qsl, ts, log_settings); +} + +/// @} + +} // namespace perf_tests + +int main(int argc, char* argv[]) { + perf_tests::TestSingleStream(); + perf_tests::TestServerStdAsync(); + perf_tests::TestServerPool(); + return 0; +} diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/perftests_null_sut.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/perftests_null_sut.py new file mode 100644 index 000000000..115372e18 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/perftests_null_sut.py @@ -0,0 +1,61 @@ +# Copyright 2019 The MLPerf Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +"""Python version of perftests_null_sut.cc. +""" + +from __future__ import print_function +from absl import app +import mlperf_loadgen + + +def load_samples_to_ram(query_samples): + del query_samples + return + + +def unload_samples_from_ram(query_samples): + del query_samples + return + + +def issue_query(query_samples): + responses = [] + for s in query_samples: + responses.append(mlperf_loadgen.QuerySampleResponse(s.id, 0, 0)) + mlperf_loadgen.QuerySamplesComplete(responses) + + +def flush_queries(): + pass + + +def main(argv): + del argv + settings = mlperf_loadgen.TestSettings() + settings.scenario = mlperf_loadgen.TestScenario.SingleStream + settings.mode = mlperf_loadgen.TestMode.PerformanceOnly + + sut = mlperf_loadgen.ConstructSUT(issue_query, flush_queries) + qsl = mlperf_loadgen.ConstructQSL( + 1024 * 1024, 1024, load_samples_to_ram, unload_samples_from_ram + ) + mlperf_loadgen.StartTest(sut, qsl, settings) + mlperf_loadgen.DestroyQSL(qsl) + mlperf_loadgen.DestroySUT(sut) + + +if __name__ == "__main__": + app.run(main) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tools/mlperf-trace.ipynb b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tools/mlperf-trace.ipynb new file mode 100644 index 000000000..ab834d17a --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tools/mlperf-trace.ipynb @@ -0,0 +1,441 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Tool to extract usefull information from mlperf trace" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "%matplotlib inline\n", + "# Ignore warnings\n", + "import warnings\n", + "warnings.filterwarnings('ignore')\n", + "\n", + "import json\n", + "import os\n", + "import seaborn as sns\n", + "from operator import itemgetter\n", + "import pandas as pd\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "\n", + "figsize=(10, 5)\n", + "font=10\n", + "\n", + "plt.figure(dpi=600)\n", + "plt.rc('xtick', labelsize=font) \n", + "plt.rc('font', size=font)\n", + "sns.set(font_scale=1.4, style=\"whitegrid\");" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "def trace_to_df(fname):\n", + " with open(fname, \"r\") as f:\n", + " j = json.load(f)\n", + " if type(j) == dict:\n", + " j = j['traceEvents']\n", + " \n", + " result = []\n", + " for item in j:\n", + " name = item['name']\n", + " if name not in [\"Latency\", \"Sample\", \"QuerySamplesComplete\", \"IssueQuery\"]:\n", + " continue\n", + "\n", + " args = item.get('args')\n", + " d = {\"ts\": item['ts'], \"name\": name, \"dur\": item.get(\"dur\")}\n", + "\n", + " if name == \"Latency\":\n", + " d[\"issue_delay\"] = args[\"issue_delay\"]\n", + " d[\"issue_to_done\"] = args[\"issue_to_done\"] / 1e3\n", + " result.append(d)\n", + " elif name == \"Sample\":\n", + " if args:\n", + " d[\"issue_start_ns\"] = args[\"issue_start_ns\"]\n", + " d[\"complete_ns\"] = args[\"complete_ns\"]\n", + " d[\"issue_to_done\"] = (args[\"complete_ns\"] - args[\"issue_start_ns\"]) / 1e3\n", + " result.append(d)\n", + " elif name == \"QuerySamplesComplete\":\n", + " result.append(d)\n", + " elif name == \"IssueQuery\":\n", + " result.append(d)\n", + "\n", + " df = pd.DataFrame(result)\n", + " df = df.sort_values(by=[\"ts\"])\n", + " return df\n", + "\n", + "BINS = 10" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
tsdurissue_delayissue_to_doneissue_start_nscomplete_ns
count2.000000e+0410000.0000005.000000e+0310000.0000005.000000e+035.000000e+03
mean4.894584e+0717.7316827.001508e+046112.5544917.001508e+046.182570e+06
std2.839099e+0725.5786399.666462e+042254.0772359.666462e+042.263719e+06
min4.102560e+031.1520008.810000e+022754.9670008.810000e+022.780383e+06
25%2.463025e+073.9747505.806250e+044100.4730005.806250e+044.166623e+06
50%4.881766e+077.3640006.159800e+046089.8800006.159800e+046.155939e+06
75%7.373552e+0727.4410006.835175e+047337.2570006.835175e+047.408272e+06
max9.832065e+07508.5520006.522433e+0622234.1010006.522433e+062.414005e+07
\n", + "
" + ], + "text/plain": [ + " ts dur issue_delay issue_to_done \\\n", + "count 2.000000e+04 10000.000000 5.000000e+03 10000.000000 \n", + "mean 4.894584e+07 17.731682 7.001508e+04 6112.554491 \n", + "std 2.839099e+07 25.578639 9.666462e+04 2254.077235 \n", + "min 4.102560e+03 1.152000 8.810000e+02 2754.967000 \n", + "25% 2.463025e+07 3.974750 5.806250e+04 4100.473000 \n", + "50% 4.881766e+07 7.364000 6.159800e+04 6089.880000 \n", + "75% 7.373552e+07 27.441000 6.835175e+04 7337.257000 \n", + "max 9.832065e+07 508.552000 6.522433e+06 22234.101000 \n", + "\n", + " issue_start_ns complete_ns \n", + "count 5.000000e+03 5.000000e+03 \n", + "mean 7.001508e+04 6.182570e+06 \n", + "std 9.666462e+04 2.263719e+06 \n", + "min 8.810000e+02 2.780383e+06 \n", + "25% 5.806250e+04 4.166623e+06 \n", + "50% 6.159800e+04 6.155939e+06 \n", + "75% 6.835175e+04 7.408272e+06 \n", + "max 6.522433e+06 2.414005e+07 " + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df = trace_to_df('/tmp/mlperf_log_trace.json')\n", + "df.describe()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAoIAAAFKCAYAAACJoz5RAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAAAstklEQVR4nO3deZxcVZ338Q8ECIQASVgNyjIIP8AoPkRGUfRBkSXgOiIMjAsoKIroIAgzKItsowODKCIoI4IzqICODgjigiwqKtCAEiA/MLI8EEFIAoJgB0ieP84tUhSVdHelu6vS9/N+vfp1U/eeunWqTi/fnHvOuSssWrQISZIk1c+K3a6AJEmSusMgKEmSVFMGQUmSpJoyCEqSJNWUQVCSJKmmDIKSJEk1tVK3KyBJdRIRmwB3A/tn5nndrc3z9ULdIuJdwDeAjTLz0W7UYbAiYhpwC/DKzJzZ5epIHTEISmNIROxH+SO6fWb+psvVGZSIWAc4Eng7sBHwJHAD8MXMvLybdRurIuKjwJM9GETHAScAZ/V6CATIzJkRcTlwPPAP3a6P1AkvDUvqmogISo/KIcCVwMeAfwPWAy6LiM91r3Zj2keB/drsvxdYDfivUa3NYm8BtgK+2qXX78TZwDsj4qXdrojUCXsEJXVFRKwMfBeYDLwhM69vOnYacAFwZET0ZebFo1y31TPzr6P5mp2qetFWysz+ZT1XZi4C/rbsterYB4AbM/OPXazDUP0UmE8J1p/pblWkoTMISmNcRKwPnATsSulpexS4CTg8M2+rymwLnAhsB6wBPARcC3woM5+KiB2Bq4A3ZubVTefehDZjyiJii+p8OwGrA3cAJ2Xmd5uq9i5gGnBMcwgEyMxnI+LDVZ0/C1xcnXfY69F0OX0n4B3A3sB61XPvBA7LzNNaPtNXAL8DPpqZZ7EEETEJOB14J7AI+F/gC23KXV297x1b9p8H7JiZm7S8z38FngA+AWwKvBm4OiIOq15rS2AicBflEvt/Np3zHmDj6t+Ne4zem5mbLOVz3IbSU7sDMI5y6f7ozPxFU5n9KJ/jjpTL/O8FJgA/oXwfPbykz6l6/qrAbsB/tDm2CPhsZh7Xsv8e4OrM3K96vBLwL8D7gJcAT1Wfwecz83+anjeY708iYi1KuHsXsCHwCHAN8KnMfAAgM5+u2u+dGAS1HPLSsDT2fRfYEzifckmwEUS2AIiIdSm9GpsB/065PHseJaStPtQXi4itgN8CL6/OdxgwF7g4It7TVPSt1fab7c6TmY9RgtNWEbHZCNaj4QxgW0poPiYz7wJ+DbQr+x5gAXDhUl5/har+76X0bn4GmEpph2X1XuBwSvA6FPhTtf9QYCZlzNqnKIH+nIg4qOm5/wzcD8yqzvPeat+S3sdWwC+A/wOcAhxXvY+fRcQb2jzldGAbSoA/i9LOXx7Ee5oOrALcOIiyS3Is5b1fA3y8+vcs4O8bBQb7fRERq1fnORT4OSV0f4USolsvA/dRvk8nL0Pdpa6wR1Aaw6oeqR0oPRinNh1qHnv3WmAKsGtmNv8RPrbDl/0iMAd4VWY+Ve07MyJ+AnwuIi6oLkFuDTyWmfcu5Vy/q7ZbA7NHqB4NT1B6355p2vdN4KyI2DozbweIiBWBfYDLMnPeUl7/bcAbgCMz89+r554F/GyI76OdjYHNM/NPLfu3yMwnmx6fUb3fwylj2cjMH0TEicAjmfnfg3itk4BVgelVOCYivkEJWKcBr2opPxfYufHZVp/XxyNirSrcL8mW1XZZLgu/Bbg8Mw9cSpnBfl98ihJo92oZmnBSFfKb/RFYgTK+8bplqL806uwRlMa2pyg9VztGxJQllGn8cX5LNW6vY9VrvBm4CFg9ItZpfAFXUC6vbVEVXwN4fIBTNo6vMYL1aDinJQRC6fHrp/SaNewIvJiBJ1TsDiyk9IoB5ZI3cOZQ3ssS/KBNCKQRAiNi5YiYUr3fq4DNqsucQ1KNP9wVuLQRAqvXeYTSazy9GnrQ7OstAfsXlMvJGw/wcmtX2/lDrWeTx4CXVZd+X2CI3xd7Are1G5/a8v6a67zOMtRd6gqDoDSGVRMIjqSMvXooIn4ZEUdFxEuail1DuXx8LDA3Ii6NiAOrS2ND9VJKz8hxwMMtX42xX+tV28cZOOA1jv95BOvR8IIex8ycD1wC7NvUC/QeYB5w2QB12Bh4MDNbw+6dg3sLS9W2dzQi3h4RN1L+AzCX8n5Prg4POQgC61LG+WWbY3dU201a9t/X8rgRkgZ72bS1t20ojqG8z4yI2yLitIho7rEcyvfFZpTL7EOpc2tAlHqeQVAa4zLzdGBzyqWux4CjgTuqiRdk5qLMfDfwasr4rnWArwG3RkTjj+KS/sCNa3nc+J3yBWDnJXw1/rjeDqwVERstpfqvqLaNy4UjUY+Gp2jvm5T1Dd9QTWh4F3BRZi5YSr2HarDvq+EFdY2IHYDvU9ZhPAjYg/I+G2NCR+v3/bNL2D9QwHuk2g5lnN3zPp/MvJYS4N4P3EyZNHJ9RBxRFenk+2IwGnV+ZKmlpB7kGEGpBjLzbkrIOz0iXkxZu+/TwNVNZa4HrgeOiYgZwOXAgZQxYo1enUktp2693NcIbM9k5kBj4S4F9qX8sT6x9WBErEmZfXpT03IiI1GPgVxB6ZF8L7A+sCaDW2fvXmDniFijpVew3WXL+cDftdk/0OXUZntSln7ZJTOfWwImIt7Ypuxge64epgTLaHOsMabvniHUcWkaPYybUkJcs/m0tHlErAK8qPUkVS/uN4FvRsRqlO/jz0bEfzC074vZlAlTg7Ep5TOdNcjyUs+wR1AawyJiQvXH8DmZeT8l2EyqykxuM/j9pmo7qdreS+npaZ0l+tGWc/+ZMibtwIjYsE191m16+D3gNuBfWi7fNcamnUXpaTmp6dBI1GOpqnGDF1CC1geBP2TmYCYEXE75HfuRptddETi4TdnZwJbN9aqWbHndYOtJ+VwW0fR7vZrF+oE2Zf/KIHreqjGNVwBvbZ65XY21ez9lzb+HhlDHpemjBNnWySdQPp/WNv8QLT2CEbF28+NqMsgsymSX1Yb4ffFdynjDd7cp1/rzMh2YVYVQablij6A0tm0B/DwiLqaErn7KJIatKDNJofxBPzgivk/5g7sasD8lWHwXylIu1TkOqdZ0m02Zodk6zg5K8PkV8PuIOKcqux7l0vPWVEtvVOuvvYuyNMcvI+JcShiYTOkp/D/Aic3rv41EPQbpm5RlRHahjC8bjEur1/+3an2+2yjrFLabtHMu8EngxxHx9aqeB1XPWXMIr/dJ4KcR8V/V6xwIPAhs0FL2RuCjEXEsZcziE5l56RLO+xnK+/5lRJxJCWsHUv6TsOcg6zagzFwQEVdQLs8e1XL4P4GzI+J7lKWOtqFMYmm9FHtHRFxLWefwkarcAcAPM/OJqsxgvy9OoQwD+HZE7EL53pwEzKCMRbwGnlsY/f+yfN0NRXqOPYLS2Pb/KL1Zr6f0rJ1CmRn5wcxsDI6/hnJJeC/K0hpHUcLDmzLzt03nOoSyLt5BlEu591FC5PNkZlJ6dS6hXPY9k9JjtxJlfGJr2W0o68ztTFmn7RRKCHx/Zj6v/EjVYyCZeQvw++rhYJZcITMXUpaQuQD4J8rn/6cl1PWOqo5rUZZkeRvlUvRNrWWX8npXV+eeQhkG8AHK2ohfalP8eBYHx29V5ZZ03jsoSxDdTJl49FnK98ebqzF5w+lc4FURsWnL/nOAz1N6Bf+Dcil2Z0rPZrPTKTO6j6R8T+1GWSppn0aBwX5fVHeWeUN1fDfK5/gxyhqMz82gpsxCnkKZRS0td1ZYtMhJTpJ6R0S8nLLkyL2UW88tbe25URMRNwALMnMol2s1BNWl85mU5WqO7HZ9BiMiLgEWZuY7ul0XqRP2CErqKZl5K2WSSADfryYFdFVEvJLSizQcdwXRElS9qEcDH6kWQ+9pETGNMtTCW8tpuWWPoCQtQfWHfjplfOCLgE1b7twhScs1ewQlacn2pNzPdzXgHw2BksYaewQlSZJqyuVjOtDX1zce2I4yA3BJq+hLkiT1gnGU4S03TJ8+vb/5gEGwM9tRZjVKkiQtL14P/LJ5h0GwM38C2GKLLVhlleGd0Dhz5kymTRvsXY00GmyT3mJ79Bbbo/fYJr2lF9pjwYIF3HnnnVDll2YGwc48C7DKKqswfvz4YT/5SJxTy8Y26S22R2+xPXqPbdJbeqg9XjCczVnDkiRJNWUQlCRJqimDoCRJUk0ZBCVJkmrKIChJklRTBkFJkqSaMghKkiTVlEFQkiSppgyCkiRJNeWdRXrY439dwJP9z3S7GsNiwviVWGP14b0dnyRJWjYGwR72ZP8zXHnDfd2uxrDYabuNDIKSJPUYLw1LkiTVlEFQkiSppgyCkiRJNWUQlCRJqimDoCRJUk0ZBCVJkmrKIChJklRTBkFJkqSaMghKkiTVlEFQkiSppgyCkiRJNWUQlCRJqimDoCRJUk0ZBCVJkmrKIChJklRTBkFJkqSaMghKkiTVlEFQkiSppgyCkiRJNWUQlCRJqimDoCRJUk0ZBCVJkmrKIChJklRTBkFJkqSaMghKkiTVlEFQkiSppgyCkiRJNWUQlCRJqimDoCRJUk0ZBCVJkmrKIChJklRTBkFJkqSaMghKkiTVlEFQkiSpplbqdgUaImIiMAvYENguM29sOvY+4ChgE2A2cHxmXtjy/JWB44H3A5OAG4BPZOYtLeU2AL4I7AYsAn4I/HNmPjIS70uSJKlX9VKP4HG0CaYRsSdwPvB9YAbwM+DbETGjpegXgIOBY4G3AwuAKyNiatO5VgKuAF4OvA84AHgtcElErDDM70eSJKmn9USPYERMAw4CPgl8teXwCcDFmfmv1eOrImIr4LPAj6rnb1g9/+OZeU617zfA3cA/A0dUz30XsA0wLTNvq8rNAX5FCZmXj8T7kyRJ6kW90iN4JvBl4M7mnRGxKbAl8J2W8t8CtouIdavHuwDjgOcuF2fm45TLvrs3PW934NZGCKzKXQfc21JOkiRpzOt6EIyI9wIvBU5sc3irant7y/5GkIumcg9l5tw25baIiBWbyrWeq1Fuy6HUW5IkaXnX1SAYEWsBpwBHZOYTbYpMrraPtuyfX22nNJVrLdMotzIwcRDlprTZL0mSNGZ1e4zgicBdmXlBl+vRkZkzZ47Iefv6+gBYbc11mTNnzoi8xmibO3cC99/9cLer0bFGm6g32B69xfboPbZJb+nl9uhaEIyIl1EmeOwcEZOq3Y2eu4kRsQaLe/4mAQ82Pb3RUziv2s6vyrSaDDwNPDGIcvPa7F+qadOmMX78+KE+ban6+vqYPn06AA/Ne5KpU58c1vN3y9prr8P6m2/U7Wp0pLlN1H22R2+xPXqPbdJbeqE9+vv7l9h51c1Lw5tTguhVlIA2H7i0OnYV8AvgjurxVi3P3braZrW9A1gvIlov724N3JmZC5vKtZ6rUW5WB+9BkiRpudXNIPhL4I0tX4dWxw4CDsjMuykBbe+W5+4D3JCZjWuNPwEWAns1ClQLVL+V5y8Jcznw8mr5mUa511AWqnbpGEmSVCtduzRc3cnj6uZ9EY1JwPQ13VnkGODCiJgN/JSyWPQuwB5N53ogIs4GPh8Rz1CWgzkcWAE4veklvgf8HvhuRPwr5f2fAvyaak1CSZKkuuj68jEDycyLgf2BPYEfA7sC+2Zma3A7FDiLMgHlEmA14M2ZOafpXM9Qbi03E/hv4BvAb4C3ZeaiEX4rkiRJPaXbs4afJzOvpvTite4/n3KbuaU992ngX6qvpZV7kBdeapYkSaqdnu8RlCRJ0sgwCEqSJNWUQVCSJKmmDIKSJEk1ZRCUJEmqKYOgJElSTRkEJUmSasogKEmSVFMGQUmSpJoyCEqSJNWUQVCSJKmmDIKSJEk1ZRCUJEmqKYOgJElSTRkEJUmSasogKEmSVFMGQUmSpJoyCEqSJNWUQVCSJKmmDIKSJEk1ZRCUJEmqKYOgJElSTRkEJUmSasogKEmSVFMGQUmSpJoyCEqSJNWUQVCSJKmmDIKSJEk1ZRCUJEmqKYOgJElSTRkEJUmSasogKEmSVFMGQUmSpJoyCEqSJNWUQVCSJKmmhhwEI2LXiFhhJCojSZKk0dNJj+CPgPsj4pSI2Ga4KyRJkqTR0UkQfAfwK+Bg4KaI+H1EHB4RU4e1ZpIkSRpRQw6CmXlJZu4FrA8cCDwMfA64NyJ+EhHviYgJw1xPSZIkDbOOJ4tk5uOZeW5m7gRsDBwFrAecDzwUEd+MiJ2GqZ6SJEkaZsM1a3gcsDIwHlgBeAp4M/DTiLg5IqYN0+tIkiRpmKzU6RMjYi1gL+A9wOuAZ4DLgH+ptguBtwFfAL4BbLeslZUkSdLwGXIQjIh3UMLf7sCqwA3AJ4BvZ+a8luI/iIh1gK+0Oc8/AJ8EtgQmAg8A3wdOyMzHmsrNAE4Ctq7KnJ6ZZ7Q53+GUCSwbALcBR2bmlS1l1gBOAfas6n4VcEhm3jOkD0GSJGkM6OTS8P8Arwa+CGydma/OzDPbhMCG3wMXtNk/BbgW+BCwW3W+DwAXNwpExPbAJcDNwAxKz+LpEXFQ84mqEHgycCawB3AXcFmb5W2+TemlPATYG5gKXOnkFkmSVEedXBreBbgyMxcNpnBmXg9c32b/f7bsujoi/gZ8NSKmZuYc4Bjgpsz8YFXmqojYCDg2Ir6WmQsjYjzwGUpP4akAEXENcCvwacrlayLi1ZSQuEdmXl7tuxWYDexHm15LSZKksayT5WN+NtgQ2IFHqu0qVcB7E3BhS5lvUS7/bls9fi2wFvCdpjo+C1wEzGi6C8ruwGPAFU3l7qOsibj78L4NSZKk3tfJLea+EBF3LeX4nRFxyhDONy4iVo2I6ZQewEuqMXubAasAt7c85bZqu2W13ara3tGm3ERgw6ZyszJzYZtyWyJJklQznVwa3oMX9tI1uxB4N/CpQZ5vLqVHD0pv3b7VvydX20dbys+vtlOayvVn5lNLKXd/Va71XI1yU9rsH9DMmTM7edqA+vr6AFhtzXWZM2fOiLzGaJs7dwL33/1wt6vRsUabqDfYHr3F9ug9tklv6eX26CQIvgS4ZynH763KDNaOwARgGmWs36URsXMH9Rp106ZNY/z48cN6zr6+PqZPnw7AQ/OeZOrUJ4f1/N2y9trrsP7mG3W7Gh1pbhN1n+3RW2yP3mOb9JZeaI/+/v4ldl51EgT/Amy6lON/R1lQelAy85bqn9dFRB9wI/BOFl8SntTylEZPYWOW8nxgfESsmpl/G6BcuyQyuamMJElSbXSyfMzPgQ9Xs3efJyI2AT5clenELZSFqF9Kmc27gMVjABu2rrazqm1jbGC7co9T1h5slIumySPN5WYhSZJUM50EwWMoPYkzI+KLEfGh6utLlDUDVwSO7rA+21fP/2Nm9lMC5V4tZfYBHgRuqh5fR5kNvHejQESMq553RdMM58spvYu7NpV7CbBDdUySJKlWhnxpODPviojXURZvPqTl8DWUO3XkQOeJiB8DV1Jm7f4NeCVlgsnvgR9UxY4Hro2IcyiLUr8OOBA4uDH7NzP7I+JE4OSIeJgSEA+gzDpuTDwhM38bEZcBX4+IwyiXuI8H7gPOG9qnIEmStPzr6F7DmXkbsGN1+7i/q3bPzsy5QzjN9ZRb1TXGG94DnA2clpkLqtf5dUS8nXLXkPcBc4BDM/PslvqcGhEAHwfWp4TLPTLzdy2vuQ9wKmXx6PGUW8y9OzPHxowMSZKkIegoCDZk5iMsXgR6qM89mkFcQq7uAjLgpdvqriKnDlDmccoYxg8PspqSJEljVkdBsBqDtyulN3Ay0DoBY1FmnrCMdZMkSdIIGnIQjIhXAd8DXswLA2DDIsAgKEmS1MM66RH8CrAa8A7gF5n56HBWSJIkSaOjkyD4CuDTmXnpcFdGkiRJo6eTdQTvZ8mXhCVJkrSc6CQIfg44MCLWHO7KSJIkafR0cml4CvBX4A8R8V3g/wHPtpRZlJmnLGvlJEmSNHI6CYKfa/r3QUsoswgwCEqSJPWwToLgpgMXkSRJUq/r5F7D945ERSRJkjS6Or7FXERsDuwIrAdckJn3RMQqwAbAg437BUuSJKk3dXJnkRWBs4EPUpaRWQT8GrgHWAW4FTge+I9hq6UkSZKGXSfLxxwFfAA4GtiepjUFM/MJyu3n/mFYaidJkqQR00kQ3B84NzNPBv7Q5vitwObLVCtJkiSNuE6C4IuB65dy/Clgjc6qI0mSpNHSSRB8ENh4KcenA84sliRJ6nGdBMHvAR+pZg03LAKIiBnA+4CLhqFukiRJGkGdBMHjgPuAm4ELKCHwqIj4DfBD4HfAvw1XBSVJkjQyhhwEM/MvwGuBk4H1gb8BOwATKSHxDZn51DDWUZIkSSOgowWlM/NvlCB48vBWR5IkSaOlk0vDkiRJGgM6ubPIuYMotigzP9hBfSRJkjRKOrk0/CaqWcJNxgEvqrYPA39dxnpJkiRphA05CGbmJu32R8TKwIeBfwZ2XqZaSZIkacQN2xjBzHw6M78M/AT48nCdV5IkSSNjJCaL/A54wwicV5IkScNoJILgzsCTI3BeSZIkDaNOZg0fs4RDkyg9gdsCn1uGOkmSJGkUdDJr+Lgl7J8PzAYOAs7ptEKSJEkaHZ3MGnYRakmSpDHAUCdJklRTnYwR3KiTF8rM+zp5niRJkkZGJ2ME7+GFdxYZjHEdPEeSJEkjpJMgeADwceAlwLeAO6v9AewD3Ad8CVg4HBWUJEnSyOgkCL4IGA+8NDPnNx+IiGOBXwEbZOa/DUP9JEmSNEI6mSxyEPC11hAIkJlzKUvHfGRZKyZJkqSR1UkQXBuYuJTjq1dlJEmS1MM6CYK/AT4REdNbD0TEq4BPAL9d1opJkiRpZHUyRvBjwNXA9RFxA3BXtX9zYDtgHnDIsNROkiRJI2bIPYKZeTvwcsrM4EnAntXXJOCLwMsz87bhq6IkSZJGQic9gmTmQ8Ch1ZckSZKWQx0FwYaI2BxYD5iZmY8N8bnvBv4JmA5MAWYDZwFfzcyFTeVmACcBWwMPAKdn5hltznc4cDCwAXAbcGRmXtlSZg3gFEoP5qrAVcAhmXnPUOouSZI0FnR0r+GI2Dci7gNmAddSwhwRsU5E3BkRew3iNIcB/cCngLcAP6Bcbv580+tsD1wC3AzMAL4BnB4RB7XU53DgZOBMYA/KuMXLImKbltf8NvA2yhjGvYGpwJURMWHQb16SJGmM6ORew+8C/hv4KXA6cGrjWGY+EhF3AO8DLhrgVG/NzIebHl8VEROBj0XEZzKzHzgGuCkzP9hUZiPg2Ij4WmYujIjxwGcoPYWnVnW8BrgV+DSwV7Xv1ZSQuEdmXl7tu5XSE7kf8JWhfhaSJEnLs056BD8N/CwzdwXOb3P8t0BrT9wLtITAhpspl2ynVAHvTcCFLWW+Rbn8u231+LXAWsB3ms79LCWIzoiIFarduwOPAVc0lbuPcieU3QeqryRJ0ljTSRDcCvj+Uo7/GVi3s+rwesryM38GNgNWAW5vKdOYkbxlU30A7mhTbiKwYVO5Wc3jD5vKbYkkSVLNdDJZ5K8s/c4imwGPDPWk1WLU+wOfzcxnI2JydejRlqKNW9tNqbaTgf7MfGop5e6vyrWeq1FuSpv9A5o5c2YnTxtQX18fAKutuS5z5swZkdcYbXPnTuD+u9t1Ai8fGm2i3mB79Bbbo/fYJr2ll9ujkyD4c2C/iPhi64GImAocCPzvUE4YERsA3wOup2mySK+bNm0a48ePH9Zz9vX1MX16uWnLQ/OeZOrUJ4f1/N2y9trrsP7mG3W7Gh1pbhN1n+3RW2yP3mOb9JZeaI/+/v4ldl51OkbwRcCNwEeBRcDuEfE5ygSNhcBnB3uyiFgL+BHwJPC2zHy6OtTo0ZvU8pRGT+G8pnLjI2LVQZRrPVej3Lw2+yVJksa0Tu4schfwOuBB4DhgBeCTwBHALcAO1SSMAVXh7RLKWoS7ZebcpsOzgQUsHgPYsHW1nVVtG2MD25V7nLL2YKNcNE0eaS43C0mSpJoZUhCMiHHV8i0PZeYuwDrAq4HtgfUzc6fMvHOQ51qJMrP3FcCMzLy3+Xi1fMzPqZZ/abIPJYTeVD2+jjIbeO/melbPuyIzF1W7L6f0CO7aVO4lwA7VMUmSpFoZ6hjBFSk9dUcCp2XmfOCGDl/7TOCtlJ7ECRHxmqZjt2fmX4DjgWsj4hzgAkpP5IHAwY3Zv5nZHxEnAidHxMOUgHgAZdLKvo0TZuZvI+Iy4OsRcRjQOP99wHkdvgdJkqTl1pB6BKvxe3Mo4wKXVaNn7t+BX7d8bVu93q+BtwPbAT+mBLxDM/PslnqdChwFfJwy3nBLysLRv2t5zX2AH1IWj76Y0rP45swcGzMyJEmShqCTWcPfoMwaPisz/9bpC2fmJoMsdzmDuHRbhcFTByjzOPDh6kuSJKnWOgmCdwLjgFkRcT7wR6B1DT8yc6BbzEmSJKmLOgmC/93076OXUGYRA99rWJIkSV00qCAYEV8Czs/MPuCN1e6JlJ7AZ0eobpIkSRpBg+0R/BjwG6AvM6+JiLUp9wPeOTOvGbHaSZIkacR0cmeRhtaFmSVJkrQcWZYgKEmSpOWYQVCSJKmmhjJr+O8i4u+rf69VbbeMiCfaFc7M65epZpIkSRpRQwmCn62+mp3RptwKlOVjxnVaKUmSJI28wQbB/Ue0FpIkSRp1gwqCmXn+SFdEkiRJo8vJIpIkSTVlEJQkSaopg6AkSVJNGQQlSZJqyiAoSZJUUwZBSZKkmjIISpIk1ZRBUJIkqaYMgpIkSTVlEJQkSaopg6AkSVJNGQQlSZJqyiAoSZJUUwZBSZKkmjIISpIk1ZRBUJIkqaYMgpIkSTVlEJQkSaopg6AkSVJNGQQlSZJqyiAoSZJUUwZBSZKkmjIISpIk1ZRBUJIkqaYMgpIkSTVlEJQkSaopg6AkSVJNGQQlSZJqyiAoSZJUUwZBSZKkmjIISpIk1dRK3XzxiHgpcDjwGmAaMCszp7UpNwM4CdgaeAA4PTPPaFPucOBgYAPgNuDIzLyypcwawCnAnsCqwFXAIZl5z/C9M0mSpN7X7R7BlwF7AH8Abm9XICK2By4BbgZmAN8ATo+Ig1rKHQ6cDJxZnfMu4LKI2KbllN8G3gYcAuwNTAWujIgJw/SeJEmSlgtd7REELs3M/wWIiPOAV7UpcwxwU2Z+sHp8VURsBBwbEV/LzIURMR74DKWn8NTqfNcAtwKfBvaq9r2aEhL3yMzLq323ArOB/YCvjMi7lCRJ6kFd7RHMzIVLO14FvDcBF7Yc+hbl8u+21ePXAmsB32k697PARcCMiFih2r078BhwRVO5+4BfVcckSZJqo9uXhgeyGbAKL7xsfFu13bLablVt72hTbiKwYVO5WW0C6G1N55IkSaqFbl8aHsjkavtoy/751XZKU7n+zHxqKeXur8q1nqtRbkqb/Us1c+bMoT5lUPr6+gBYbc11mTNnzoi8xmibO3cC99/9cLer0bFGm6g32B69xfboPbZJb+nl9uj1INjTpk2bxvjx44f1nH19fUyfPh2Ah+Y9ydSpTw7r+btl7bXXYf3NN+p2NTrS3CbqPtujt9gevcc26S290B79/f1L7Lzq9UvDjR69SS37Gz2F85rKjY+IVQdRrvVcjXLz2uyXJEkas3o9CM4GFrB4DGDD1tV2VrVtjA1sV+5xytqDjXLRNHmkudwsJEmSaqSng2Bm9gM/p1r+pck+wIPATdXj6yizgfduFIiIcdXzrsjMRdXuyyk9grs2lXsJsEN1TJIkqTa6fWeRCSxetmVjYM2I2LN6fENm3gscD1wbEecAFwCvAw4EDm7M/s3M/og4ETg5Ih6mBMQDKLOO9228Xmb+NiIuA74eEYcBf6nOfx9w3oi+WUmSpB7T7cki6wEXt+xrPN4fOC8zfx0Rb6fcNeR9wBzg0Mw8u/lJmXlqRAB8HFifsiTMHpn5u5bz7wOcSlk8ejzlFnPvzsyxMStDkiRpkLoaBKv7+7aO12tX7nIGcem2uqvIqQOUeRz4cPUlSZJUWz09RlCSJEkjxyAoSZJUUwZBSZKkmjIISpIk1ZRBUJIkqaYMgpIkSTVlEJQkSaopg6AkSVJNGQQlSZJqyiAoSZJUUwZBSZKkmjIISpIk1ZRBUJIkqaYMgpIkSTVlEJQkSaopg6AkSVJNGQQlSZJqyiAoSZJUUwZBSZKkmjIISpIk1ZRBUJIkqaYMgpIkSTVlEJQkSaopg6AkSVJNGQQlSZJqyiAoSZJUUwZBSZKkmjIISpIk1ZRBUJIkqaYMgpIkSTVlEJQkSaopg6AkSVJNGQQlSZJqyiAoSZJUUwZBSZKkmjIISpIk1ZRBUJIkqaZW6nYFVA8LFy7ioXlPdrsaHVltzXWfV/cJ41dijdVX6WKNJEkaHgZBjYr+p5/lut/P6XY1OjJnzhymTl0cBHfabiODoCRpTPDSsCRJUk0ZBCVJkmqqdpeGI2Jz4AxgB+Ap4DvAkZm5fA5gkyRJ6lCtgmBETAKuAu4F9gTWA04D1gX+sXs1kyRJGn21CoLAh4HJwCsz8xGAiHgGuCAiTsjM27paO0mSpFFUtzGCuwNXNkJg5XtAPzCjO1WSJEnqjrr1CG4FnNu8IzP7I2I2sOUQzjMOYMGCBcNYtcX6+/sBeObpBay04sIReY3R9uwzTy+372XVlVd4Xt0XLOjngYdGpu1H07gVV+DZhYu6XY0hGz9xCg889Ojz9q22ykqsPmHl7lRIz/3OUu+wTXpLt9ujKa+Maz1WtyA4GXi0zf75wJQhnOdFAHfeeecwVOmFZs6c+dy/NxtKrXrYo3++Z7l9L5tNWZMyr6j48wN/7F5lBMCDT8zrdhXUpPl3lnqDbdJbeqg9XgTMbt5RtyA4XG4AXg/8CXi2y3WRJElamnGUEHhD64G6BcH5wKQ2+ycDswZ7kunTp/cDvxymOkmSJI202e121m2yyB2UcYLPiYjxwGYMIQhKkiSNBXULgpcDO0XE2k373gmMr45JkiTVxgqLFi1/swY7VS0oPRO4BziBxQtKX5mZLigtSZJqpVY9gpn5KPAm4Angf4AvABcCH+hitSRJkrqiVj2CkiRJWqxWPYKSJElazCAoSZJUUwZBSZKkmqrbgtI9KyI2B84AdqDcz+w7wJGZ+WRXKzbGRMRLgcOB1wDTgFmZOa1NuRnAScDWwAPA6Zl5RptyhwMHAxsAt1Ha7MqRewdjR0S8G/gnYDrlFo+zgbOAr2bmwqZytsUoiYh/AD5Juff6RMrn/X3ghMx8rKmcbdIFETGRsubthsB2mXlj07H3AUcBm1B+lo7PzAtbnr8ycDzwfsrNFW4APpGZt4xC9Zd7EbEf8I02h87MzI81lVuufj7sEewB1bI2VwFrAHsChwH7AOd2sVpj1cuAPYA/ALe3KxAR2wOXADcDMyg/+KdHxEEt5Q4HTgbOrM55F3BZRGwzYrUfWw4D+oFPAW8BfgB8Cfh8o4BtMeqmANcCHwJ2A75IWVXh4kYB26SrjqNNB05E7AmcTwntM4CfAd+uAkmzL1CCx7HA24EFwJURMXUE6zwW7QZs3/R1auPA8vjz4azhHhARRwLHABtn5iPVvn2BC4BpmXlbN+s3lkTEio3epog4D3hVa49gRPwImJKZr27a9zXgrcCGmbmwuiPNQ8DXMvOIqsw44FZgZmbuNSpvaDkWEetm5sMt+04DPgJMysx+26L7IuJDwFcpn/cc26Q7ImIa8BtKj+1XaeoRjIg7gFubP9eI+Anl5+jvq8cbAvcCH8/Mr1T71gDuBs5ttJOWrKlHcN3G3+o2ZZa7nw97BHvD7pRFrZu/sb5H6S1p/R+dlkHzJcd2qh/QN1HWl2z2LUr3/bbV49cCa1Eu4TfO/SxwETAjIlYYrjqPVa0hsHIzsCowxbboGY3fS6vYJl11JvBl4M7mnRGxKeVS/ndayn8L2C4i1q0e7wKMo6ntMvNx4IeUv0FaRsvrz4dBsDdsRctlyszsp4zz2LIrNaqvzYBVeOFl40avbKM9GvesvqNNuYmUMTwautcD84A/Y1t0TUSMi4hVI2I65WrFJZl5D7ZJV0TEe4GXAie2Odz4rJfUJtFU7qHMnNum3BYRYR4YvJkR8WxE3B0Rx0ZE43L9cvnzYcP3hsnAo232z6eM2dHomVxtH23ZP7/aTmkq15+ZTw1QToMUEa8C9ge+UP3v2LbonrmUSWs3An8C9q322yajLCLWAk4BjsjMJ9oUGUqbtJZplFuZEkC0dH+ijK/cjzJO8PvA0cB/VseXy58PZw1L6rqI2IAyHOJ6miaLqGt2BCZQZtZ/Brg0Inbuao3q60Tgrsy8oNsVqbvM/DHw46ZdP42Ix4DjIuKELlVrmdkj2BvmU6byt5pMuUym0dP4H9mklv2N/+nNayo3PiJWHaCcBlD1ePwIeBJ4W2Y+XR2yLbokM2/JzOsy82vAO4E3VlvbZBRFxMuAg4CjI2JStcJEo+duYjXZYyht0lqmUe5poF1vowZ2UbXdluX058Mg2BvuYPGYAeC5QaebUdaM0uiZTVlSYauW/VtX20Z7NMZ2tCv3OGXtKA2g+kV4CbAesFvL+CXbojfcAiykjFGzTUbX5pQrd1dRwsN84NLq2FXAL1j6Zw2Q1fYOYL2IaL3suDVw50AT6TQoy+XPh0GwN1wO7BQRazfteycwvjqmUVJN0vk50Dp9fx/gQeCm6vF1wGPA3o0C1fT/vYArMtN1mQZQDbC+CHgFMCMz720+blv0jO0pfyv+aJuMul9SemObvw6tjh0EHJCZd1MCxt4tz90HuKFpdv5PKIG+eYmZiZRlTfw707l/BBYBfcvrz4djBHvDV4FDgP+txhmsB5wGXJiZbRc9VmciYgKLl0rYGFizWowVyi/Neykr718bEedQ1nJ8HXAgcHDjf83VGncnAidHxMOUH/ADKL24+6LBOJPyR+gIYEJEvKbp2O2Z+Rdsi1EVET8GrqTMXvwb8ErKgt+/pyz4DbbJqKmWFLu6eV9EYxIwfU13FjkGuDAiZgM/pSwWvQtloeLGuR6IiLOBz0fEM5Q1BQ8HVgBOH7l3MXZUPx8/B2ZSQvUM4KPA1zPzj1Wx5e7nwwWle0REbEG5q8LrWXyLuSO8xdzwiohNKAuotrN/Zp5Xldudsur7VsAcykzWL7U53+GUEL8+5Y/nEd5Ca3Ai4h5KGG/njZl5dVXOthgl1X9E3w5sWu26hzKJ57QqmDfK2SZdEhE7Ui4Lt95i7v288BZz32l57srACZRZr2ux+BZzN49G3Zd3EXE6Jfy9mNKRdhfVnUOqlQ4a5Zarnw+DoCRJUk05RlCSJKmmDIKSJEk1ZRCUJEmqKYOgJElSTRkEJUmSasogKEmSVFMGQUmSpJoyCEqSJNXU/we2dyrPLzUr8gAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAApsAAAFKCAYAAABSGJRzAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAAA3OklEQVR4nO3debgcVZn48W8EuRDZQtiMIy4sr2DUGQIjICiKigFFHFnEUQSF0ZHFBRQXREDEhYyiAu4KjiCL/lQQxAUQHRDBwAAB8oIIZDAjsoRFYS6Y5PfHqSZFc29yb9+u23f5fp4nT6er3q46dbq679unzjk1ZenSpUiSJElNeEqvCyBJkqSJy2RTkiRJjTHZlCRJUmNMNiVJktQYk01JkiQ1xmRTkiRJjTHZlMagiDg6IsbUvGQRsTQivtLrckgjERFTIuLaiPhEr8syFBFxTkSc3etySCOxcq8LIE0Ww0ge92+0IGNEREwFPgj8KjN/1ePiNCIidgQuAfbJzDN7XJwhiYinAe8H9gI2Bh4DrgW+BpyemWPqR1AH9gE2Ab7Q64IM0aeA30fEizLz2l4XRuqEyaY0et7a9vzfgG2At7ctvxz4LvDp0ShUD00FPl79/1c9LIcqEbEBcBGwOfA94EvAasC/AP8JzI6It2bmkt6VcsQ+AHw/M+/pdUGGIjOvjojfA4fz5O8QaVww2ZRGSWZ+t/48Il4J/HP78pq/N18q6QlOoySab8jMc2vLvxARJ1ASnv8GThjNQkXEKsCSzBzRZyIi/gn4R+DIbpRrFJ0FHBsRB2Xmg70ujDRcJpvSGBQRRwMfz8wptWW3A/MpLZ5zgOcDtwKHZubFEbE7cCywGXAjcGBmzm3b7mbAccBOwNOAm4BPZub3h1G2vSktks8FEjgiMy9si1mritkD2BC4E/g2cHxmLo6IZwO3VeEfj4hWC+dpwH8A1wF7ZOYPqu1Fdex/yMxNa/v5T+Clmfms2rKtgWOAlwCrAHOBj2XmJW1lfDrwCeC1wDTgj8AXM/PLtZgdKZfB3ww8B3g3sC5wGfDOzPzDUOutts3VgaOBNwIzgAeBG4CjMvPXVcwmwPHADsA6wL3Ab4GDM/N/a/W3f2ae2rb9pcAxmXn0MI91G2Bn4FttiWbLh4HXAx+KiJMy85GGyrEjpc7fAmxKafl/BvBPEXEF8I3MPLRtX9OB/wU+n5lHDFD2lt2BxcDFba8/mrbPW7V8P8p5+5zMvL1atiXlM7Q1sAZwF/Br4N8y85EqZgpwMOXqxaaU9/g8ymflnrZ9vIpSt1sBU4CbgS9n5jdqYb+gfOZ3Bs5ZzvFJY5IDhKTx5bmUy5vnAx8C1gbOjYg3A18EzgCOquLOiYiVWi+MiM2B3wEvAD4LHEZJYs6JiLcMcf8vAb4MnA18FFgVOC8itq/tZzVKsrAfpTvAwZQ/7kcDX63C7gb+vfr/DymXB99arZ8HLAJeWtvvS4ElwCZVwtKyA+UPfWvfLwN+Q0nQjgWOAPqAn1dJTCtufeAK4DXAKcB7qv2eEhEDtXp9kHIpeQ6lD902wOmD1tLyfRk4hHLc7wY+Q6mPF1VleyrwM2B74OQq5hRgA0pyOizDONbXVY/fGWg7VaviGZS63a7BcrR8hJKQf4HyPi4EfgTsHRHtDSV7A08drOw12wE3tpLCDo5hPUritzHlM3QwcCowk/LjreXLwOcon7f3UPq77gFcEhGr1rb3Vsp7vUG1vQ8CVwK7tu36RuARyudPGnds2ZTGl00pLXm/AYiImyh/rL4FbJ6Zt1XL76ckbi8Hflm99guUP9hb1f7YnhwRPwc+HRFDGfwxE9guM39b7edU4BZKa2sr4Xwf8Dxgy8ycXy37WkTcBhwXESdkZkbE9yl/lK8boIvBZTwx2dwB+CmwY7X8rIh4JvAs4JPVa6ZUx/xfwKtax1KNoL+G0lLYSpKOoyShL8jMu6tlX4mIrwMfqVru7q/tf1XgRZn5aLXNRZRLyzMzc94K6qzda4GvZ+b7B1m/BeXHwp5tLc7HDXM/9dcN5Vi3qNYtbxBKa90WlL6dTZSjZQ3KOf231oKI+A5lgM+rgQtqsW8BrsnMG1ZQhudRWro7tR0l2d45M39fW95qmScitgPeCbwtM79TW34h5YfQvpTPw5rAScDVwA71BLg6lx+XmX+PiP9h2XskjSu2bErjy82tRLPyu+rxV61Es235cwEiYh3glZQWyadFxLqtf8CFlMuUmw1h/79vJZoAmXkvpbXrJRExrVq8FyXhu6dtP62kd8ch7Oc3wAury/FQEsyLKS1jrSR0h1oslJbBqMozvbbfNSmtUS+OiKnVH/I9KK3DS9vK+HPKgJgXt5XnO61Es22fzx3CsbR7oCrLMwZZ3+qTt3M1MrxjwzzWNarHh5azyda6NZYTM9JytHynnmhWfkG5XP74QJmIeC6wLWUA04pMp7Sad+qB6vG1VQv0QPYC/gpc2Hac8ymX3F9exb2acm5+ur2ldZAffYsoXTikcceWTWl8WVB/kpkPlO6M/E9bXOuPYisB3ITSH+zo6t9A1qf0wVyeWwZYdnP1+CzKH8TNKInf3QPEtvazIr+h/BjePiKuq7b9a2B1YM8qZgfgL7XW01ay/M3lbHc60E+pl7fz5JkABivjgrbnrYRlGsP3AUrf1AURcQ0l2f/PzEyAzLwtIj5HmX7oLVUr73nAd6vkfjjWY+jHWk8k7x8ktpVk/qXBcrTc2h5Q9ff9LnBQRKyRmQ9RWjUXU7qXDMWUFYcM6lLg+5SWzPdHxKXAucAZtcR4M8p5etcg22gd58bV41BbxqcA433aKU1SJpvS+LJ4mMtbf1hbVzE+zxMvP9YN93LwYJ5CaYX81CDr/ziEbfye0kftpZR+qQ9RLoWvARxdtdTuQGlBre8XSl/WwS6V3l1tD0py8q1B4tovx66ofocsM8+JiN9QBtu8GjgU+GBE7JeZZ1Qxh0XEt4Ddqpj/AI6MiJdl5o0MknTU++hWWnUylGO9kTKA5oXU+sG2eWH12HoPmyhHy2D9Kr9DSdj/hZK0/yvwi8z88yDxdfcw8A+EwZK4JxxH1eK4Z0T8M6U7xKso/TE/HBHbZOZfKMd6L/CmQbbZacvqNJYNqpPGFZNNaXJoJQd/z8xfLjdy+TYdYFmrRfGO6vFWYI0h7GfQVprMfKwaefxSYC3g8qpV6wrKlFCvp/Rf+3rtZa2WsIeWt++IuJuSvK48wrroWJUYfRX4akSsTekecAylC0Ar5gZKAvapiHghJYF+H3AgyxKWtds2/ay258M51vMog3L2ZYBks0og38yy0dc0VI7lysx5EXE18Naqz/JmlLobipsoswq0WwQQEWu39RttP45WGa6kDOQ5KiJmU37AHUjpP3wrJQm9IjP/upyytM7XmZRL7IOqBkQ9k8F/KEpjmn02pUmganG5BDhwoL6C1SjbodgqIratvW46JQG5PDNbicdZwNYRscsA+1kjIvqqpw9Xj4Ndiv4NMIvyh/vX1XE8Qmn1PILSqlhPiuYCf6Bc3nxSn8LWMWbmYsql0N0j4kWDxTUhIlaq9UOlKs/9lBartauYNQcYbX0TpaVv7eo1D1Ja6V7aFvfutm0P+Vgz8wpK/8n9I6J9NDSURGoz4DOt+S6bKMcQnUbp+/hBShL7wyG+7jJgi2rGhLpW4vf4cVT9Zd/WVs5p7YN3KAN8YFnCfRblb+tR7Tuv3v/W+f5zSv/cD7WXZ4B9bEEZpHb5wIcljW22bEqTx79T/theV40AvpXSf+zFlD9mmwxhG/OAn0TElyh/5P+Ncmn7w7WYEyjT6Pw4Ik6jJIGrUVpw9qRMvXR7NU/jDcCbIuJmyqXH2zKzNbjpNyybxqmeVP6akmw+SG3kdGYuiYh3UPpA3lhdhr6TMl3QyyjJaWtwxocoA5V+W9XFDZSk9x+BN1D+sDdhDeBPEfGDquwPUqazeQ1lZDLAKyizBHyf0od2CmVqnzUoiUzLNyiJyjcoCfhLGXiQ13COdV9KF4hzI+IMynuwKuWS9csog3Y+37b9JsqxIt+jTEP1RuDUYUxl9GNKK+grKIOVWn5O6Zf7zSiT1y+m9C29G9ioFvc2Sn/RH1I+P6tRbi/bSqbJzF9HxMnAB6oW6Z9R+glvQhkkdVRV5gcj4j2UbgW/r+r7Xsr8uc+g1HnLqyg/Nn42xOOUxhRbNqVJohqAshVlQMO+LJvDcWXgY0PczGXVa/amTCXUD+zemoy82s8jlKTiM5TE40TK5dnNKRN61/vWvQO4ndIn8Xssm3sTyiTmfwf+j3LJsqU1Evyy9tsmVuXYhnJZ+t2UBO7twH1VeVpxf6Ek2d+g9FM8iXKJekPK/KNNeZhS7y+g1PmJlPfk8Gr/UJLQnwK7UBKqT1ASzt3bpkI6ljIYag/KHI0rAbPbdzicY83Mu6rYY4B/okyX9QVKovnxzHxCS19T5ViRauqkn1ZPhzIKvfW6ayktkXu1LX+MkvDeSqnvQ6tyntS2iUsp5+JelHr5COV8fkXtRxKZeTDl3F6H0iL8aUrf27OpTShfTYT/Wsr5+RFK/W1L6dJQtxfww8x8AGkcmrJ0qYPbJEkDq7pdXE5pnNg2M+/scZEAiIhzKD8sntX+o2MFr9uH0tf3WR2M7h91Ue5Y9HtgVmZe0+vySJ2wZVOSNKjM/BPlMv9UytyRa/e2RI/fjWg3ypRRQ040K2dSWjDf2+1yNeTDwPdNNDWe2bIpSRoXIuI5lD6ub6dcbt50rLS0ShqcA4QkSePFy4BvU25isJ+JpjQ+2LIpSZKkxtiy2YG5c+f2AVtT7tE72J1FJEmSxoKVgKcDV82aNat/tHdustmZrVk2/YokSdJ40H6b31FhstmZ/wXYbLPNWGWVVRrbybx585g5c2Zj2x8vrIfCeiish8J6KKyHwnoorIeivR4effRRbr75Zqjyl9FmstmZxQCrrLIKfX19K4odkaa3P15YD4X1UFgPhfVQWA+F9VBYD8Ug9dCTrn/OsylJkqTGmGxKkiSpMSabkiRJaozJpiRJkhpjsilJkqTGmGxKkiSpMSabkiRJaozJpiRJkhpjsilJkqTGeAehMWz1tdblrvse7nUxumJq38qs8bTmbu0pSZLGJpPNMWzx0ilcdNWCXhejK3baeiOTTUmSJiEvo0uSJKkxJpuSJElqjMmmJEmSGmOyKUmSpMaYbEqSJKkxJpuSJElqjMmmJEmSGmOyKUmSpMaYbEqSJKkxJpuSJElqjMmmJEmSGmOyKUmSpMaYbEqSJKkxJpuSJElqjMmmJEmSGrNyr3YcEXsC/wrMAtYBbgW+DHw1M5dUMacCbxvg5Xtm5vfbtnc4cBCwIXADcERmXtQWswZwArAHsCpwCXBIZt7etQOTJEnS43rZsnkY0A98AHgt8CPgi8Bn2uL+CGzb9u/iekCVaB4PnAzsCtwCnB8RL2rb1veA3YBDgL2BGcBFETG1WwclSZKkZXrWsgm8LjPvrj2/JCJWBw6OiCMzs79a/khmXjHYRiKiDzgSODEz51TLLgWuBz4K7FUtezElEd01My+oll1PaVHdDzilmwcnSZKkHrZstiWaLddQLm+vM4xNbQesBZxZ2/Zi4GxgdkRMqRbvAjwAXFiLWwBcVq2TJElSl/WyZXMgOwD3AX+pLds4Iu4HngbMAz6dmWfV1m9ePd7Utq0bgNWBZwB3VnHzW/1B2+J27krpJUmS9ARjJtmMiK2A/YFjqpZJKC2dV1ESwrWAA4AzI2K1zDy1ipkG9GfmI22bXFQ9rkNJNqcB9w+w60UMryX1cfPmzevkZUO22prrsXDhwkb3MVruvXcqd942UGP20MydO7eLpRm/rIfCeiish8J6KKyHwnooxlI9jIlkMyI2BH4AXEltgFBmfqEt9McRcTFwDHDqqBVwEDNnzqSvr6+x7d94ywJmzJjR2PZH0/Tp67LBpht19Nq5c+cya9asLpdo/LEeCuuhsB4K66GwHgrroWivh/7+/sYbyJan5/NsRsRawE+Bh4HdMvOxFbzkHGCjiFiver4I6IuIVdviplWP99Xi1h5ge9NqMZIkSeqiniabVYJ4LrA+8JrMvLeDzbT6am7etnwL4CHgT7W4qA0YqsfN72C/kiRJWoGeJZsRsTJlxPgLgdmZeccQXjOFMpXRHbXR7JdTRpnvXYtbqYq7MDOXVosvoLRs7lyLeyawfbVOkiRJXdbLPpsnA68DPghMjYhtautupFzePo0yEfsfKIniAcCOwFtbgZnZHxHHAcdHxN3A1VXcxsCba3G/i4jzgW9GxGHAg8CxwALGQP9PSZKkiaiXyWarhfGzA6x7OXAdpcXySMpl9scoieRumXlePTgz50QEwKHABpTR67tm5rVt290HmEOZwL2PcrvKPTPz4W4ckCRJkp6oZ8lmZj57CGGvH8b25lASyeXFPAS8s/onSZKkhvV8NLokSZImLpNNSZIkNcZkU5IkSY0x2ZQkSVJjTDYlSZLUGJNNSZIkNcZkU5IkSY0x2ZQkSVJjTDYlSZLUGJNNSZIkNcZkU5IkSY0x2ZQkSVJjTDYlSZLUGJNNSZIkNcZkU5IkSY0x2ZQkSVJjTDYlSZLUGJNNSZIkNcZkU5IkSY0x2ZQkSVJjTDYlSZLUGJNNSZIkNcZkU5IkSY0x2ZQkSVJjTDYlSZLUGJNNSZIkNcZkU5IkSY0x2ZQkSVJjTDYlSZLUGJNNSZIkNcZkU5IkSY0x2ZQkSVJjTDYlSZLUGJNNSZIkNWblXu04IvYE/hWYBawD3Ap8GfhqZi6pxc0GPglsAfwJODEzvzTA9g4HDgI2BG4AjsjMi9pi1gBOAPYAVgUuAQ7JzNu7fXySJEnqbcvmYUA/8AHgtcCPgC8Cn2kFRMS2wLnANcBs4NvAiRHxrvqGqkTzeOBkYFfgFuD8iHhR2z6/B+wGHALsDcwALoqIqV0+NkmSJNHDlk3gdZl5d+35JRGxOnBwRByZmf3AUcDVmfmOWsxGwMcj4muZuSQi+oAjKS2ecwAi4lLgeuCjwF7VshdTEtFdM/OCatn1lBbV/YBTGj5eSZKkSadnLZttiWbLNZTL2+tUSeQrgLPaYs6gXCrfsnq+HbAWcGZt24uBs4HZETGlWrwL8ABwYS1uAXBZtU6SJEldNtYGCO0A3Af8BdgYWAW4sS3mhurxedXj5tXjTQPErQ48oxY3v94ftBb3PCRJktR1vbyM/gQRsRWwP3BMZi6OiGnVqvvbQhdVj+tUj9OA/sx8ZDlxd1Zx7dtqxa0zwPIVmjdvXicvG7LV1lyPhQsXNrqP0XLvvVO587aBGrOHZu7cuV0szfhlPRTWQ2E9FNZDYT0U1kMxluphTCSbEbEh8APgSmoDhMa6mTNn0tfX19j2b7xlATNmzGhs+6Np+vR12WDTjTp67dy5c5k1a1aXSzT+WA+F9VBYD4X1UFgPhfVQtNdDf39/4w1ky9Pzy+gRsRbwU+BhYLfMfKxa1WqZXLvtJa0Wz/tqcX0RseoQ4tq31Yq7b4DlkiRJGqGeJptVgngusD7wmsy8t7b6VuBRlvXJbNmiepxfPbb6ag4U9xBlbs5WXNQGDNXj5iNJkqSu61myGRErU0aMvxCYnZl31NdXUx9dTDV1Uc0+wJ+Bq6vnl1NGme9d2/ZK1esuzMyl1eILKC2bO9finglsX62TJElSl/Wyz+bJwOuADwJTI2Kb2robM/NB4Fjg1xHxdeB04CXAgcBBrVHlmdkfEccBx0fE3ZQk9ADKaPY3tzaYmb+LiPOBb0bEYUBr+wuAUxs9UkmSpEmql8lmq4XxswOseznwq8z8bUS8nnJ3oH2BhcD7MvMr9eDMnBMRAIcCG1CmM9o1M69t2+4+wBzKBO59lNtV7pmZD3fnkCRJklTXs2QzM589xLgLGMJl7uruQXNWEPMQ8M7qnyRJkhrW89HokiRJmrhMNiVJktQYk01JkiQ1xmRTkiRJjTHZlCRJUmNMNiVJktQYk01JkiQ1ZtjJZkTsPMD9xSVJkqQn6aRl86fAnRFxQkS8qNsFkiRJ0sTRSbK5O3AZcBBwdURcFxGHR8SMrpZMkiRJ496wk83MPDcz96Lcg/xA4G7g08AdEfHziHhLREztcjklSZI0DnU8QCgzH8rMb2XmTsCzgI8A6wOnAXdFxHciYqculVOSJEnjULdGo68EPBXoA6YAjwCvBH4REddExMwu7UeSJEnjyMqdvjAi1gL2At4CvAT4O3A+8KHqcQmwG/B54NvA1iMtrCRJksaXYSebEbE7JcHcBVgVuAp4D/C9zLyvLfxHEbEucMoIyylJkqRxqJOWzf8H/An4AnBaZs5fQfx1wOkd7EeSJEnjXCfJ5quBizJz6VCCM/NK4MoO9iNJkqRxbtjJZmb+somCSJIkaeLp5HaVn4+IW5az/uaIOGFkxZIkSdJE0MnUR7sCZy1n/VnA6zorjiRJkiaSTpLNZwK3L2f9HVWMJEmSJrlOks0HgecsZ/1zKZO6S5IkaZLrJNm8GHhnRGzUviIing28s4qRJEnSJNfJ1EdHAbOBeRHxbeCGavlMYD9gMfCxrpROkiRJ41onUx/dEhEvAU4GDmlbfSlwSGZmNwonSZKk8a2je6Nn5g3AjtWtKJ9bLb41M+/tWskkSZI07nWUbLZk5j3APV0qiyRJkiaYjpLNiFgJ2JnSqjkNmNIWsjQzPzHCskmSJGmcG3ayGRFbAT8A/oEnJ5ktSwGTTUmSpEmuk5bNU4DVgN2B32Tm/d0skCRJkiaOTpLNFwIfzczzul0YSZIkTSydTOp+J4NfPpckSZIe10my+WngwIhYs9uFkSRJ0sTSyWX0dYC/AX+IiO8D/0O5a1Dd0sw8YaSFkyRJ0vjWSbL56dr/3zVIzFLAZFOSJGmS6yTZfE63dh4RmwCHA9tQ7q0+PzNntsWcCrxtgJfvmZnfb4s9HDgI2JByz/YjMvOitpg1KInwHsCqwCWUW2ze3oVDkiRJUk0n90a/o4v7fz6wK/A7Sv/RwfqQ/hH417ZlN9efVInm8cBHgKuBA4HzI+LFmXltLfR7wJaU+7o/CBwLXBQRL8jMh0d2OJIkSarr+HaVEbEpsCOwPnB6Zt4eEatQWhX/nJmPDmEz52Xmj6vtnQpsNUjcI5l5xXLK0gccCZyYmXOqZZcC1wMfBfaqlr2YktzumpkXVMuuB24F9qPMISpJkqQuGfZo9Ih4SkR8DZgPfJXSMvjcavUqlATvkKFsKzOXDHf/g9gOWAs4s7btxcDZwOyIaE3VtAvwAHBhLW4BcFm1TpIkSV3UydRHHwHeDnwM2JbanJuZ+VfKrSz/pSulW2bjiLg/Ih6LiGsiYu+29ZtXjze1Lb8BWB14Ri1u/gBJ7g3A87paYkmSJHV0GX1/4FuZeXxETB9g/fXAa0dWrCe4BriKkhCuBRwAnBkRq2XmqVXMNKA/Mx9pe+2i6nEdymT004D7B9jHoipmWObNmzfclwzLamuux8KFCxvdx2i5996p3Hnb3R2/fu7cuV0szfhlPRTWQ2E9FNZDYT0U1kMxluqhk2TzH4Arl7P+EWCNzorzZJn5hbZFP46Ii4FjgFO7tZ9OzJw5k76+vsa2f+MtC5gxY0Zj2x9N06evywabbtTRa+fOncusWbO6XKLxx3oorIfCeiish8J6KKyHor0e+vv7G28gW55OLqP/GXjWctbPAro5Yn0g5wAbRcR61fNFQF9ErNoWN616vK8Wt/YA25tWi5EkSVKXdJJs/gD492o0estSgIiYDexLGZgzmlp9NTdvW74F8BDwp1pc1AYM1ePmN1c8SZKkyamTZPNoYAGlL+XplETzIxFxBfAT4FrgU90qYLsqUdwLuCMzW50AL6eMMt+7FrdSFXdhZi6tFl9AadncuRb3TGD7ap0kSZK6qJNJ3R+MiO2A9wN7Av9HSdZupSSiJ2Tm/w1lWxExlWVTDj0LWDMi9qieX1U9nkaZiP0PlETxAMr8nm+tlak/Io4Djo+IuymTuh8AbAy8uRb3u4g4H/hmRBzGskndF9Dj/p+SJEkTUUeTulfJ5PHVv5FYn9L/sq71fH/gXEqL5ZFV7GOURHK3zDyvrUxzIgLgUGADyuj1XdvuHgSwDzCHMoF7H+V2lXt69yBJkqTu6/gOQt1Q3Y+8vf9ku9cPY3tzKInk8mIeAt5Z/ZMkSVKDhp1sRsS3hhC2NDPf0UF5JEmSNIF00rL5CqrR5zUrAU+vHu8G/jbCckmSJGkC6GSA0LMHWh4RT6Vcmn4v8KoRlUqSJEkTQidTHw0oMx/LzJOAnwMndWu7kiRJGr+6lmzWXAu8tIHtSpIkaZxpItl8FeA0QpIkSepoNPpRg6xam9KiuSXw6RGUSZIkSRNEJ6PRjx5k+SLKXYTeBXy90wJJkiRp4uhkNHoTl94lSZI0AZk4SpIkqTGd9NncqJMdZeaCTl4nSZKk8auTPpu38+Q7CA3FSh28RpIkSeNYJ8nmAcChwDOBM4Cbq+UB7AMsAL4ILOlGASVJkjR+dZJsPh3oAzbJzEX1FRHxceAyYMPM/FQXyidJkqRxrJMBQu8CvtaeaAJk5r2UaY/+faQFkyRJ0vjXSbI5HVh9OeufVsVIkiRpkusk2bwCeE9EzGpfERFbAe8BfjfSgkmSJGn866TP5sHAr4ArI+Iq4JZq+abA1sB9wCFdKZ0mjCVLlnLXfQ939NrV1lyv49c2YWrfyqzxtFV6XQxJksaFTu4gdGNEvAD4EDAb2KNadQfwBeCzmfnn7hVRE0H/Y4u5/LqFHb124cKFzJgxdpLNnbbeyGRTkqQh6qRlk8y8C3hf9U+SJEkaUEfJZktEbAqsD8zLzAe6UyRJkiRNFB3dGz0i3hwRC4D5wK+BWdXydSPi5ojYq4tllCRJ0jg17GQzIt4IfBe4CfgAMKW1LjPvqZbv260CSpIkafzqpGXzo8AvM3Nn4LQB1v8OeNGISiVJkqQJoZNkc3Pgh8tZ/xdgvc6KI0mSpImkk2Tzbyz/DkIbA/d0VhxJkiRNJJ0kmxcD+0XEkyYajIgZwIHAz0ZaMEmSJI1/nfbZfDrwe+DdwFJgl4j4NHA9sAQ4pmsllCRJ0rg17GQzM28BXgL8GTiaMhr9/cAHgf8Gts/MBd0roiRJksarYU3qHhErAc8A7srMV0fENGATStL6x8y8u4EySpIkaZwa7h2EngLcChwBfC4zFwFXdb1UkiRJmhCGdRk9Mx8DFlL6aUqSJEnL1ckAoW9TRqOv2u3CSJIkaWIZ7mV0gJuBlYD5EXEa8EfgkfagzDx7hGWTJEnSONdJsvnd2v8/NkjMUmCFyWZEbAIcDmwDzATmZ+bMAeJmA58EtgD+BJyYmV8aIO5w4CBgQ+AG4IjMvKgtZg3gBGAPYFXgEuCQzLx9ReWVJEnS8AzpMnpEfDEiZlVPX179ex3wytrz+r9XDHH/zwd2Bf4A3DjIvrcFzgWuAWZTLuOfGBHvaos7HDgeOLna5i3A+RHRfp/27wG7AYcAewMzgIsiYuoQyyxJkqQhGmrL5sHAFcDczLw0IqZT7oH+qsy8dAT7Py8zfwwQEacCWw0QcxRwdWa+o3p+SURsBHw8Ir6WmUsiog84ktLiOafa3qWUSeY/CuxVLXsxJRHdNTMvqJZdTxlhvx9wygiORZIkSW06GSDUMmWkO8/MJctbXyWRrwDOalt1BuVS+ZbV8+2AtYAza9teTLmUPzsiWmXdBXgAuLAWtwC4rFonSZKkLhpJsjkaNgZW4cmX2G+oHp9XPW5ePd40QNzqlInoW3HzB0hyb6htS5IkSV3SyQCh0TStery/bfmi6nGdWlx/ZraPiq/H3VnFtW+rFbfOAMuXa968ecN9ybCstuZ6LFy4sNF9jJb+WGdExzKW6uHee6dy5229uVnW3Llze7LfscZ6KKyHwnoorIfCeijGUj0MJ9l8bkT8c/X/tarH50XEXwcKzswrR1SycWDmzJn09fU1tv0bb1nAjBkzGtv+aOrrW7XjY1m4cOGYqofp09dlg003GvX9zp07l1mzZq04cIKzHgrrobAeCuuhsB6K9nro7+9vvIFseYaTbB5T/at70vRDlL6cSylzcY5Uq2Vy7bblrRbP+2pxfRGxamb+3wriBsoSptViJEmS1CVDTTb3b7QUg7sVeJTS1/LC2vItqsf51WOrr+bmlCmS6nEPUebmbMW9KiKmZObStrj5SJIkqauGlGxm5mlNF2SQ/fZHxMWUqYs+X1u1D/Bn4Orq+eWUUeZ7UyWbEbFS9boLa4nlBZSplHamSl4j4pnA9sB7Gj0YSZKkSainA4SqidRbUw49C1gzIvaonl+VmXcAxwK/joivA6cDLwEOBA5qjSqvktLjgOMj4m5KEnoAZTT7m1v7y8zfRcT5wDcj4jDgwWr7C4BTGz1YSZKkSajXo9HXB85pW9Z6vj9wamb+NiJeT7k70L7AQuB9mfmV+osyc05EABwKbECZzmjXzLy2bfv7AHMoE7j3UW5XuWdmPty1o5IkSRLQ42Szuh/5CieHr+72c8EQ4uZQEsnlxTwEvLP6J0mSpAaN9UndJUmSNI6ZbEqSJKkxJpuSJElqjMmmJEmSGmOyKUmSpMaYbEqSJKkxJpuSJElqjMmmJEmSGmOyKUmSpMaYbEqSJKkxJpuSJElqjMmmJEmSGmOyKUmSpMaYbEqSJKkxJpuSJElqjMmmJEmSGmOyKUmSpMaYbEqSJKkxJpuSJElqjMmmJEmSGmOyKUmSpMaYbEqSJKkxJpuSJElqjMmmJEmSGmOyKUmSpMaYbEqSJKkxJpuSJElqjMmmJEmSGmOyKUmSpMaYbEqSJKkxJpuSJElqjMmmJEmSGmOyKUmSpMaYbEqSJKkxJpuSJElqzMq9LsCKRMR+wLcHWHVyZh5ci5sNfBLYAvgTcGJmfmmA7R0OHARsCNwAHJGZFzVQdE1QS5Ys5a77Hh71/a625npd3e/UvpVZ42mrdG17kiQNZMwnmzWvAR6oPf9z6z8RsS1wLvAd4DDgJcCJEfFYZn6lFnc4cDzwEeBq4EDg/Ih4cWZe2/whaCLof2wxl1+3cNT3u3DhQmbM6F6yudPWG5lsSpIaN56SzbmZec8g644Crs7Md1TPL4mIjYCPR8TXMnNJRPQBR1JaPOcARMSlwPXAR4G9Gi6/JEnSpDPu+2xWSeQrgLPaVp1BuVS+ZfV8O2At4MxWQGYuBs4GZkfElOZLK0mSNLmMp5bNeRGxHrAAOBX4ZGb+HdgYWAW4sS3+hurxecDvgc2r5zcNELc68Azgzu4XW5IkafIaD8nm/wIfB64EFgOzgY8BzwH2A6ZVcfe3vW5R9bhO9TgN6M/MR5YTN6xkc968ecMJH7bV1lyPhQtHv29gE/pjnREdy1iqh5Eey0h0c7/33juVO2+7u2vbG01z587tdRHGBOuhsB4K66GwHoqxVA9jPtnMzJ8BP6st+kVEPAAcHRGf6FGxAJg5cyZ9fX2Nbf/GWxYwY8aMxrY/mvr6Vu34WMrAmLFTDyM5lpHodj1Mn74uG2y6Ude2N1rmzp3LrFmzel2MnrMeCuuhsB4K66For4f+/v7GG8iWZ7z22Ty7etySZS2Ta7fFtFo876seFwF9EbHqCuIkSZLUJeM12ay7FXiUZX0yW7aoHudXj62+mgPFPUSZm1OSJEldNF6TzTcBSynTIfUDF/PkqYv2oczFeXX1/HLKPJ17twIiYqXqdRdm5tKmCy1JkjTZjPk+mxHxM0oyOQ9YQhkg9G7gm5n5xyrsWODXEfF14HTKpO4HAgdl5hKAzOyPiOOA4yPibkoSegBlNPubR/GQJEmSJo0xn2xSLn+/HfgHSnlvAY4ATmwFZOZvI+L1lLsD7QssBN5Xv3tQFTcnIgAOBTagTHu0q3cPkiRJasaYTzYz873Ae4cQdwFwwRDi5gBzRlwwSZIkrdB47bMpSZKkccBkU5IkSY0x2ZQkSVJjTDYlSZLUGJNNSZIkNcZkU5IkSY0x2ZQkSVJjTDYlSZLUGJNNSZIkNcZkU5IkSY0x2ZQkSVJjTDYlSZLUGJNNSZIkNcZkU5IkSY0x2ZQkSVJjTDYlSZLUmJV7XQBJvbFkyVLuuu/hXhdj2FZbc70nlXtq38qs8bRVelQiSdLymGxKk1T/Y4u5/LqFvS7GsC1cuJAZM56YbO609UYmm5I0RnkZXZIkSY0x2ZQkSVJjTDYlSZLUGJNNSZIkNcZkU5IkSY0x2ZQkSVJjTDYlSZLUGJNNSZIkNcZkU5IkSY0x2ZQkSVJjTDYlSZLUGO+NLmncW7JkKXfd9/CKA8eBqX0re593SROKyaakca//scVcft3CXhejK3baeiOTTUkTipfRJUmS1BhbNiVpDBlql4DV1lxvTHcdsDuApBaTTUkaQ4baJWDhwoXMmDF2k027A0hqmXTJZkRsCnwJ2B54BDgTOCIzx+63tiRJ0jg1qZLNiFgbuAS4A9gDWB/4HLAe8KbelUySJGlimlTJJvBOYBrwj5l5D0BE/B04PSI+kZk39LR0kjRBjNZ0VKPRd9X+p9LITLZkcxfgolaiWfkB8C1gNmCyKUldMFrTUY1G31X7n0ojM9mSzc0pieXjMrM/Im4FnjeM7awE8Oijj3axaE+2ZMliVn7Kkkb3MVoW//2xjo9l1adOGVP1MJJjGYlu10OvjmOkBqqH8XosAxnqsYy1z0W70XpPRqMe/v7Yo/T3r9ToPrqhv7+/10UYE6yHol4PtXylJyfylKVLl/Zivz0REY8BH8vMT7ct/y/gL5n5L0PZzty5c7cHftNAESVJkpqyw6xZs/5rtHc62Vo2u+UqYAfgf4HFPS6LJEnS8qwEPJ2Sv4y6yZZsLgLWHmD5NGD+UDcya9asfmDUfxlIkiR16NZe7Xiy3a7yJkq/zcdFRB+wMcNINiVJkjQ0ky3ZvADYKSKm15a9Aeir1kmSJKmLJtsAobWBecDtwCdYNqn7RZnppO6SJEldNqlaNjPzfuAVwF+B/wd8HjgLeHsPiyVJkjRhTaqWTUmSJI2uSdWyKUmSpNFlsilJkqTGmGxKkiSpMZNtUvcxLyI2Bb4EbA88ApwJHJGZD/e0YCsQEXsC/wrMAtahTB77ZeCrmbmkFjcb+CSwBfAn4MTM/NIA2zscOAjYELiBUgcXtcWsAZwA7AGsClwCHJKZt7fF9aROI2J1yvytzwC2zszf19btC3wEeDalro7NzLPaXv9U4FjgbZSbEVwFvCcz/7stbkPgC8BrgKXAT4D3ZuY9bXH/TJl9YRZwH/CNar+N3QUrIt4KvJfyfj8MXA3s0yrbZDgfImJ3ynu9OfA34DLgQ5l5S1vchDknImIT4HBgG2AmMD8zZw4QN2bf/6GWbST1EBErAYcBu1b7WRm4Hjim/fgmcj0MED8LuBJ4JDNXb1vXk8/AUD6fKzKMz8WqwIeAtwL/ANwDXJCZB7bFjZvzwZbNMaSamukSYA3KiXEYsA/wrR4Wa6gOA/qBDwCvBX4EfBH4TCsgIrYFzgWuAWYD3wZOjIh31TdUfYCOB06mfAnfApwfES9q2+f3gN2AQ4C9gRnARRExtbatteldnR7NAD/oImIP4DTgh5R6+CXwvepDXPd5yhfJx4HXA49Sjm9GbVsrAxcCLwD2BQ4AtgPOjYgptbjnVvu5j/L+HE95rz7ZheMcUER8lPKD4/9RjvMdlC/Evmr9hD8fImInyvHPB/6lKtvzgF9GxJq1uIl2Tjyf8l79AbhxoICx/P4PtWxDsKJ6WI2SwPw3sD/wJsof8F9ExGvbyjSR66G+z6dQvjfuHiRk1D8Dw/h8rshQPhdPofz93Lcqz6uBD1Jm0anHjavzwZbNseWdlFtn/mOt5efvwOkR8YnMvKGnpVu+12Vm/cvhkqpl7+CIODIz+4GjgKsz8x21mI2Aj0fE1zJzSXVHpyMpv5bmAETEpZRf+x8F9qqWvZjyAds1My+oll1P+cW5H3BKtY+e1GlEzATeBbwf+Grb6k8A52Tmh6vnl0TE5sAxwE+r1z+jev2hmfn1atkVwG2UlsIPVq99I/AiYGbrWCJiIaX1bDbLblbwAeB+YM/qvbgoItYCjoqIz2bmfd07eoiIoCTbb8jMn9RW/aj2/8lwPuwD3AG8LTOXVvu7A/gd8BKq95uJd06cl5k/rvZ9KrDVADFj+f1fYdm6VA+PAM/JzEWtBRHxc2Azyh/8n1TLJno91B0IrEVJdA6tr+jhZ2CFn88u1sP+wLbAFpn5p9ry02v1MO7OB1s2x5ZdKBPM15v5f0BpMRzuL6hR1ZZotlxDabZfp/pwvIIyr2ndGZRLAFtWz7ejfNGcWdv2YuBsYHbtV+kuwAOUX6+tuAWUL5NdatvvVZ2eDJwE3FxfGBHPobRsndkWfwawdUSsVz1/NbAStfrKzIcof3zaj+/6epKUmZdTEpz2uB9VX6j1fbbel27bH7ijLdF83CQ6H54KPNRKNCv3V49TYGKeEytKQsby+z+Msq3QiuohMxfXE81q2VJKS+eM2uIJXQ8tEbEupbXuPZQWy3aj/hkYxudzhYZYDwdSEts/LSdm3J0PJptjy+a0Na1XH4JbKSf7eLMD5fLEXyj3n1+FJ186aH0ZtI6vde/6mwaIW53S/7EVN3+AD+8NPLGuRr1Oo/RT3AQ4boDVreMbrB6iFndXZt47QNxm1aWWVtxAl2Mer4eIeBqwUXtc1WfnYZqph22A6yLiyIj4c0Q8FhFXRsTLqvWT5Xw4Fdg8Ig6JiLUj4tnAHMrxtPpWTZZzom4sv/9DLVsjqvdxO554zJOlHj4D/FdmXjjI+l58Bob6+RyxKP1RtwRuj4jTIuKvEfG3iPhR1YLYMu7OB5PNsWUay1o96hZRBt2MGxGxFaV16/PVL65p1ar720Jbv+pbxzcN6M/MR4YQ176tVly9rka1TqtLMCcAH8zMvw4QMpx6aI9pxT2V8oWyorjWttYeZJ/tcd20IfAqyjlwKPA64EHgwirhmhTnQ2ZeQumr+clqH7cBzwFeVWtNmSznRN1Yfv+HWramHEJJYP6jtmzC10PVH3Af4H3LCevFZ2A062E65TiOoHyHvpHS1/1FwAVR+qK2yjSuzgeTTXVdlFGAP6CMJvzMCsInmuOAWzLz9BVGTmxPoXzxvzEzz65aKnajJJwf6GnJRlFEbAd8B/gm5RLUnsASykCF1XpZNo09Vcv/Z4E5mfmbXpdntEQZlX8K8LnM/GOvy9NDrZzsr8DumfmzzDyT8r3xfOANPSvZCJlsji2LWPZrq24a5XL0mFe17P2Uchlit8x8rFrV+vWzdttLWr+W7qvF9UWZ+mFFce3basXV62rU6jQink/pvP6x6pLp2iz7pb16lCkohlMP7TGtuMdYNjJxKMd3/yD7bI/rpkXAvVmbjiTLFBpXUKb8mPDnQ+WLwCWZ+b7MvCQzv0/psP9PlGlNWmVigHJNtHOibiy//0MtW1dFxAuBH1MG0R3Rtnqi18OBwNOBU2rfnatCGSld+2HWi8/AaNbD/ZRpmi6rt1pmmTbvQcp3Z6tM4+p8MNkcW25iWV8M4PFOuRtTpk4Z06oT/1xgfeA1bf1qbqV0+N687WVbVI+t42v1QRko7iHKtCCtuKh1hK7H1etqNOt0U8oMD5dQPpiLgPOqdZcAv2H5xweQ1eNNwPoR0X5pYgvg5lofnCcdXy1uPkBm/g1Y0B4XEc8CptLMubW8Ud2rMjnOh9b+/7u+IDPvpMybt3GtTLSXi4l3TtSN5fd/qGXrmojYGPgZZR7at7YNKIOJXw/PAzagHEfru/MI4GnV/z9VK/dofwaG+vkcseoH+e2DrF5KlYCvoExj8nww2RxbLgB2iojptWVvoIyMu2Dgl4wNVV+Ss4EXArMz8476+qp/2sVUUzLU7AP8mfIlC3A5ZfTc3rVtr1S97sLal/AFlF9YO9finkmZkLZeV6NZp/8FvLztX6v/0buAAzLzNsqHcu+21+4DXFUb1f9zyuXWx+srylRSr+PJx/eCahqOVtw2lImH2+N2j4hV2vbZz7KBKt30E2B6RDw+QrHqkL8tMHeSnA9QRr/Oqi+o/pitS/VHZRKdE48by+//MMrWFVW3o59X2949MwcahT3R6+EknvzdeRrwf9X/T6riRv0zMIzPZ7f8BNi+3s0mysTzawFzq0Xj7nyYsnRp+w8o9Up16WAe5Y/QJygthJ+jTEnwpt6VbMUi4qvAv1HmOWvva3RjZj5YdQD/NWWE7umUeQaPBQ7KzK/UttWarPbDlBP4AEpH6Rdn5rW1uJ9QLkceRrnEcCylSf8F1S/EntdpROxIadV8/A5CUe62dBbl1/ovKBMTv4cyF9pPa689iXKp9TBK0nI4ZV62F2TmwipmZeD3lE7lH6a0rJ4A3AW8JJfN7fhcSgvbxZS7REQV96XM/FADx/0U4LfAepR53x6qjmNrylxuf5gM50NEHEyp75Mol0inU+bHWw94fqv1f6KdE1EmjG5NrXIQpYXk/dXzqzLzjrH8/g+1bCOtB8pMHb+tlr+F8h49LjOvmAz10N44Ub3maODwfPIdhEb9MzDUz2c36qFKBq+lvMefpySLx1Peyy1bXdPG2/lgy+YYkpn3UwYR/JVy15HPU07wt/ewWEPV+uX0WcqXZ/3flgCZ+VvKh3RryiWjA4D3tZ+kWSap/QhlFPNPKZdYdq1/gCr7UH4FngKcQ/l19cqs3V5rLNZpZp5DGaW9B6UedgbePMCX1vsod9I4jtI9YTXK8S2sbevvlNuxzQO+S7mjwxWU/rJLa3F/BF5JSXLOpyQ8/0FJBLuuupy1K+WLqfX+AOyYmX+oYibD+XAyZcLkHSh98U6k3D3k5fVuJhPwnFif8h6cA+wIPLP2/OXV/sfs+z/UsnWhHjagjDRenXJ+tH93TpZ6GI5R/wwM4/O5IkP5XPxP9f8p1fKTKA04r8xlYyDG3flgy6YkSZIaY8umJEmSGmOyKUmSpMaYbEqSJKkxJpuSJElqjMmmJEmSGmOyKUmSpMaYbEqSJKkxJpuSJElqzP8HVvNfDeBqWn8AAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "df1 = df[df[\"name\"].isin([\"IssueQuery\"])]\n", + "df1['delta'] = df1['ts'].diff()\n", + "ax = df1['dur'].plot.hist(bins=BINS, alpha=0.5, figsize=figsize)\n", + "ax.set_title('IssueQuery duration (usec)');\n", + "plt.show()\n", + "ax = df1['delta'].plot.hist(bins=BINS, alpha=0.5, figsize=figsize)\n", + "ax.set_title('Time between IssueQuery (usec)');\n", + "\n", + "# df1['delta'].describe()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "# for SingleStream\n", + "if False:\n", + " df1 = df[df[\"name\"].isin([\"QuerySamplesComplete\"])]\n", + " ax = df1['dur'].plot.hist(bins=BINS, alpha=0.5, figsize=figsize)\n", + " ax.set_title('Inference time (usec)');\n", + " plt.show()\n", + " ax = df1['dur'].plot(figsize=figsize)\n", + " ax.set(ylim=(0, 100))\n", + " ax.set_title('Individual inference time (usec)');" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAoMAAAFtCAYAAAB8yGDhAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAABA20lEQVR4nO3deZwcZbX/8U8WMkmAkIWwBAkgwgGM4DXyExQF5SKyCFxlEQQFRUAQRETZd81FiQsoKKAsV5TVK8tlEWVfJQ4IJJAjW4hhIIQkLJIwZJnfH+fppNL0bD29Tdf3/Xrl1emq6uqnz9TUnH7qeU4N6OjoQERERETyaWC9GyAiIiIi9aNkUERERCTHlAyKiIiI5JiSQREREZEcUzIoIiIikmNKBkVERERybHC9GyAi+WFmw4GfArsDawHnuvvRdW1UAzKz9YEXgIPc/bL6tmZFZvZd4AhgY3dfXO/2dMXMdgGuBjZw9zn1bo9Io1LPoIj0mJkdaGYdZrZVmbs4BjgEuBg4APhdxRrXD5nZ4WZ2YL3b0VNmtipwAnBOoyeCAO5+M/Ac0WYR6YR6BkWklrYDHnf3U+vdkAZxOPAacFnR8heBYcCiWjeoG18j2vU/9W5IL1wInGNmp7v7m/VujEgjUjIoIrW0BjCvUjszswHAUHdfWKl9NgJ37wDeqXc7SvgacIu7v13vhvTCdcB5wN7Ab+rcFpGGpGRQRPrEzC4DvgRsCJwP/CewELgcOM7dl5jZdsBdmdcU7oO5gbvPMLMW4Hhgf2A80Vt2DXCSuy8oet2FaV8nA0Zcdr7MzFYDTgP2JMYjzgIuBSa5+5L0+vWJsXgnAHPTe74PeAI43N2nFH22jYEzgO2BEcC/gJuz4xzNbG3gLGBXYBTwPHCeu/+qm7jNANYriseL7r5+qTGDZnZ6+nybAScBnyd6Di9Kz8cR8f8MEf/J7n5O0Xv2KM6dtHcDYPP0Htnl72lrZl0HcIa7n56erwKcDnwxtfdNYBpwqrvfm3ndlkTcPwEMAVqBU9z9rqL9r532twswFngZ+AtwjLu/BeDur5rZE8B/oWRQpCSNGRSRShgI3EYkWMcC9wDfJRI1gKeJMYKzgOnp/wcAc1Lv3p+A7wM3A0cSCcrhwPVpfdaniITkj8BRwHQzG0YkiAcCVwDfAu4kEoULS7R3n/R+FxJJ5frA/5rZSoUNzOyDwCPAjsAl6b2uJZKwwjZrAA8DnwMuAL4NTAUuMLOTu4nZ0SXicXQ3rwG4kvgifzzwEJHYfpdIgl4BjgOeAX5sZp/JtLW3cS728fT49x60sTO/Su/7p/S+PwLmAFtk2rktcB8wGjgzfZ4W4Pb0paKw3VrEz+crxLFwJHG5/f8BY4retxXYugefUSSX1DMoIpWwEnCtu5+Znv/azB4Fvg78yt1nA1eY2fHAa+5+ReGFZrYfkUx92t3vySz/O5HY7QDcnnmvTYCPuPs/MtuemFk+PS2+yMxeAH5gZue4u2f2sS6wkbvPT6934AYi8fu/tM35xDnyQ+7+Qua9Tsrs5wdEovKhzGzVX5vZxcCJZvZLd3+9VMDc/Xoz+0FxPHqg1d2/ntpyETAD+DHRc/bDtPxKoI24rHtnet2+9C7OxTZJj8/3oq3FdgUudvdjSq1MydqFwP3ADulyOWb2a+AxYBLLk9Kzid7Fj7v73zK7Ob1E0vc80Wu7NhEXEclQMigilXJx0fP7iN6u7uwN/BOYZmarZ5bfA3QAn2bFJOXBbCKY2cf9wGtF+/grkbBtB2STwT8WEsFMWwHeD2BmY4FtgfOziSAsG89XSFz2JHq5Oore93bgYOBjwJ87++BlWnapM12C/ztxqfu3meWvpwT3/ZnX9TbOxcYAS4E3+tD2N4CPmdk67v5SifVbEJf+fwyMMbPsur8AR6byRO8Ql31vLUoEgeU/o4zCz3p1lAyKvIeSQRGphEXu/nLRsvlEb0x3NiYSgM7qwK1R9Py5TvaxRS/2MTP7xN3np8Sj0N5CEjW1k/1BjFEbRfS+fa2H71sJM4uev0HE/5USy9fMPO9tnEsZkP4VJ1s99T1iLOlMM3uMGFrwu0yv7cbp8belXpyMAd4lxnB29fPJKvQUlttukaamZFBEKmFpH147EHiKGG9XSnFPTqmZwwOJy6H/3ck+ii9tLulku96MKSuMub6SGFNYyrRe7K+nSrW9s/hnP09v41zstbS/1Vje0wadJFhmNqh4mbtfa2b3EUXHP0uMw/y+mR3o7n9geUyPJ8b5lTIntaE3Ckn+a718nUguKBkUkXp7DpgI3FHi8l5v9rGqu/+1gm0CmNDFNnOAt4DBfXjfWvZU9TXOT6fHDVgxGSz8f2TR9uuV2knqwbwQuNDMRhITcM4A/sDyuL/VVUzN7F1iJnJXP5+sQpuLe09FBM0mFpH6u5q4nPnN4hVm1pLuetGTfWxpZjuX2MeqqaRKj7n7a8RYugNTSZXs/gakbZYQNez2MLMtiveRxh125216dim9Evoa5wfS40ezC1Mh59eIWd5Zhxe9x6BU/if72teJsjQj06JW4FngmFLtKcTU3ZcSYzV3MrOPldiuuId3IvBwH75siDQ19QyKSL1dQUzEOD+VFbmfuBxpxKSHvYC7u9nHOUTJlxvM7HIiqRhG9BztBXyImHXbG0emtrSa2YXEpebxRE3FjdI2xxOTUx5KM4inEcndh4kJDkO7eY+/A4eb2WnE5I5/u/tNvWxnT/Upzu4+08z+Qcw6vqho9W+A483sN8Rn+hTLx/8VrAq8ZGZ/BB4nevY+Qcxw/mV6j6Vm9nViLOFTZnYJUX5nHDGhZwAx0QWipM4OwN3p5/MUkex+gYj9DFhW/mdz4Nfdh0gkn5QMikhdpQTgC0SNva8S48kWEsnXBURB6O72sTDVoDuBSGwOIC7hPkMUhO715UF3fzLdg/ks4FAiufwXcFNmm1dTz9QpwB5Er9s84pLqd3vwNmcSZW6OISZEvJjdfyVVIs7E2MizzWzloruQnElMqNmTiP+twE7Aq5ltFhDlenYAdiPKEb1A1KU8N9POe1PcTyF6F0cQP78prDiT+uUU+7OIsjkjiXGPt7Pi2MAvEhNOru7B5xPJpQEdHeo1FxGR7qVLt88Tdwzp8g4rjSL1Zt6dvWuMiKxIYwZFRKRH0i3ezga+Z2YNf2XJzHYBPkAUqxaRTqhnUERERCTH1DMoIiIikmNKBkVERERyrOHHfDSi1tbWFmBL4GU6v5OBiIiISCMYBKwNTJk4cWJ78Uolg+XZkuU3thcRERHpDz5J1BhdgZLB8rwMsPHGGzNkyJA+7Wjq1KlMmNDTOyo1L8VBMQDFABQDUAxAMQDFACoXg3fffZd//vOfkPKXYkoGy7MEYMiQIbS09OouVyVVYh/NQHFQDEAxAMUAFANQDEAxgIrHoOTQNk0gEREREckxJYMiIiIiOaZkUERERCTHlAyKiIiI5JiSQREREZEcUzIoIiIikmNKBkVERERyTMmgiIiISI4pGRQRERHJMSWDIiIiIjmm29E1sLfefpcF7Yvr3YyKGN4ymFVX7tt9nEVERKTylAw2sAXti7ljysx6N6Mitt9yvJJBERGRBqTLxCIiIiI5pmRQREREJMeUDIqIiIjkmJJBERERkRxTMigiIiKSY0oGRURERHJMyaCIiIhIjikZFBEREckxJYMiIiIiOaZkUERERCTHlAyKiIiI5JiSQREREZEcUzIoIiIikmNKBkVERERyTMmgiIiISI4Nruebm9kHgGOBrYAJwHR3n1C0zWXAV0u8fC93v65o22OBI4C1gGnAce5+R9E2qwLnAHsCQ4G7gCPdfUYFPpKIiIhIv1LvnsEPArsAzwJPdbHd88DWRf/uzG6QEsFJwPlpn88AN5vZFkX7uhLYDTgS2AcYB9xhZsP7+mFERERE+pu69gwCN7n7DbCsB/CjnWy30N0f7mwnZtYCnAz83N0np2X3AE8CJwF7p2UfIxLFXdz9lrTsSeA54EDggr5/JBEREZH+o649g+6+tEK7+jiwGnBVZt9LgGuAncxsQFq8M/AGcFtmu5nAA2mdiIiISK7Uu2ewpzY0s9eBlYGpwNnufnVm/abp8emi100DVgHWAWal7aaXSEKnATtWutEiIiIija7eYwZ74jFikskexKSPWcBVZnZgZptRQLu7Lyx67fz0ODqz3esl3mN+ZhsRERGR3Gj4nkF3P7do0Q1mdidwBnBZ7Vu03NSpUyuyn9bW1pLLh40YS1tbW0Xeo97mzh3OrBfmdLlNZ3HIE8VAMQDFABQDUAxAMYDaxKDhk8FOXAtcYGZj3X0O0bPXYmZD3f2dzHaj0uO89DgfGF9if6My2/TYhAkTaGlp6e3LVtDa2srEiRNLrps9bwHjxi3o0/4bxZgxq7PmRqVCH7qKQ14oBooBKAagGIBiAIoBVC4G7e3tXXZg9YfLxD1RGCu4adHyzYC3gJcy21lmQkl2u+nVa56IiIhIY+p3yWBK5PYGXky9ggAPErOE98lsNyhtd5u7d6TFtwAjyUwWMbN1gW3SOhEREZFcqfcdSIazvKTLesAIM9szPZ+SHi8nCkU/SyRyBwPbAQcU9uPu7Wb2A2CSmc0BHk3bbQjsl9nub2Z2M/BbM/su8CZwJjCTOo8/FBEREamHeo8ZXIMY/5dVeH4QcCPR43dy2nYRkejt5u43ZV/k7pPNDOAoYE2iXMwu7v540f73BSYTBaZbiNvR7eXuzTE4T0RERKQX6poMpvsBF4/fK7Z7L/Y3mUj0utrmLeDQ9E9EREQk1/rdmEERERERqRwlgyIiIiI5pmRQREREJMeUDIqIiIjkmJJBERERkRxTMigiIiKSY0oGRURERHJMyaCIiIhIjikZFBEREckxJYMiIiIiOaZkUERERCTH6npvYpH+6K2332VB++KK73fYiLHMnreg4vvtzPCWway68pCavZ+IiDQmJYMivbSgfTF3TJlZ8f22tbUxblztksHttxyvZFBERHSZWERERCTPlAyKiIiI5JiSQREREZEcUzIoIiIikmNKBkVERERyTLOJpSaWLu3osmxKrcuq9MWixUvq3QQREZGKUTIoNdG+aAkPPtHW6fpal1Xpi49vPq7eTRAREakYXSYWERERyTElgyIiIiI5pmRQREREJMeUDIqIiIjkmJJBERERkRxTMigiIiKSY0oGRURERHKsrnUGzewDwLHAVsAEYLq7T8isHwR8F9gF2Ixo75PAGe5+R9G+ZgDrlXibse7+Wma7VYFzgD2BocBdwJHuPqNiH0xERESkn6h3z+AHiUTvWeCpEuuHAScC/wAOAr4EvAT8xcx2LbH9dcDWRf9eL9rmSmA34EhgH2AccIeZDe/bRxERERHpf+p9B5Kb3P0GADO7DPho0fqFwAbuPr+wwMxuBzYmegz/r2j72e7+cGdvZmYfI5LPXdz9lrTsSeA54EDggr58GBEREZH+pq49g+6+tJv1S7KJYFrWQfQUlnNPsJ2BN4DbMvubCTyQ1omIiIjkSr17BnvNzAYCHweeLrH6y2Z2MLAEuB84wd0fzazflBiXWJyETgN2rEZ7RURERBpZv0sGibF+BhxStPxG4G/ATGIiyQnAfWa2pbsXxiOO4r1jCAHmA6N725CpU6f29iUltba2llw+bMRY2traKvIe9dZuo7v9LP3ls/bks5SrljGYO3c4s16YU7P366nOfh/yRDFQDEAxAMUAahODfpUMmtm2wI+Bye5+X3adux+VeXqfmd0KTAeOB75SjfZMmDCBlpaWPu2jtbWViRMnllw3e94Cxo1b0Kf9N4qWlqGMG9f5lf22trYu1zeS7j5LuWodgzFjVmfNjcbX7P16oqvfh7xQDBQDUAxAMYDKxaC9vb3LDqx6zybuMTPbHLgBuB44rrvt3X0ucCeQjeJ8YGSJzUcB8/rcSBEREZF+pl8kg2a2IfBn4FHggDSJpBxPx+5sQNHyzYheRBEREZFcafhk0MzWAm4HXgH2cPd3e/i61YHtgSmZxbcQPYM7ZrZbF9gmrRMRERHJlXrfgWQ4y0u6rAeMMLM90/MpwKtEGZg1gGOAzcxs2esLNQXNbF9gV+BWoij1+sSl5Bbg7Mz2fzOzm4Hfmtl3gTeBM4lJJ5dV4zOKiIiINLJ6TyBZA7i2aFnh+UHA3cAW6fn1JV5fuNz7AlF38KfE+L83gHuAPd29+PLvvsBkosB0C3E7ur3cvTlmaoiIiIj0Ql2TwXQ/4OLxe8W6W1/oIfx0D9/zLeDQ9E9EREQk1xp+zKCIiIiIVI+SQREREZEcUzIoIiIikmNKBkVERERyTMmgiIiISI4pGRQRERHJMSWDIiIiIjmmZFBEREQkx5QMioiIiOSYkkERERGRHFMyKCIiIpJjSgZFREREckzJoIiIiEiO9ToZNLMdzWxANRojIiIiIrVVTs/grcAsMzvHzLaodINEREREpHbKSQb3AB4AjgAeNbMnzOxYMxtX0ZaJiIiISNX1Ohl09xvdfW9gTeAbwBzgbOBFM7vdzPY3s+EVbqeIiIiIVEHZE0jc/S13v8TdtwfWA04E1gAuB2ab2f+Y2fYVaqeIiIiIVEGlZhMPAlYCWoABwELgP4G/mNljZjahQu8jIiIiIhU0uNwXmtlqwN7A/sAngMXAzcDx6XEpsBvwM+BSYMu+NlZEREREKqvXyaCZ7UEkgDsDQ4EpwLeBK919XtHm15vZ6sAFfWyniIiIiFRBOT2D/wu8BJwLXO7u07vZ/gng92W8j4iIiIhUWTnJ4GeBO9y9oycbu/sjwCNlvI+IiIiIVFmvk0F3/2s1GiIiIiIitVfO7eh+ZmbPdLH+n2Z2Tt+aJSIiIiK1UE5pmV2Aq7tYfzXw+fKaIyIiIiK1VM6YwXWBGV2sfzFt0y0z+wBwLLAVMAGY7u7vqUloZjsBPwQ2Iyav/Nzdf1Fiu2OJ2+StBUwDjnP3O4q2WRU4B9iTmA19F3Cku3f1mURERESaUjk9g28CG3Sx/v1E0eme+CDR0/gs8FSpDcxsa+BG4DFgJ6Jm4c/N7LCi7Y4FJgHnp30+A9xsZlsU7fJKov7hkcA+wDjgDt1CT0RERPKonGTwTuBQMxtfvMLM1gcOTdv0xE3uvq677wk82sk2pwKPuvvX3f0ud/8B8FvgNDMbmN63BTiZ6DGc7O53ErUQnwdOyrTvY0SieLC7X+nuNwP/BYwHDuxhm0VERESaRjnJ4KnE5eWpZnaumR2S/p1H1BQcCJzSkx25+9Ku1qck7zO8d4ziH4hLwR9Jzz8OrAZcldn3EuAaYCczG5AW7wy8AdyW2W4m8EBaJyIiIpIrvU4G3f0Z4vZzjxKXWn+d/n0LaAU+6e5eofZtCAzhvZeQp6XHTdLjpunx6RLbrQKsk9lueokkdFpmXyIiIiK5Uda9id19GrBdutXc+9Pi59x9bsVaFkalx9eLls9Pj6Mz27W7e/FYxex2s9J2xfsqbDe6xPIuTZ06tbcvKam1tbXk8mEjxtLW1laR96i3dhvd7WfpL5+1J5+lXLWMwdy5w5n1wpyavV9Pdfb7kCeKgWIAigEoBlCbGJSVDBa4+2vAaxVqS78zYcIEWlpa+rSP1tZWJk6cWHLd7HkLGDduQZ/23yhaWoYybty4Tte3tbV1ub6RdPdZylXrGIwZszprbvSeob911dXvQ14oBooBKAagGEDlYtDe3t5lB1ZZyaCZDQJ2JHoFRwEDijbpcPezytl3kULP3sii5YUew3mZ7VrMbKi7v9PNdqX++o3KbCMiIiKSG71OBs3so8Afgffx3iSwoAOoRDL4HPAuMdbvtszyzdLj9PRYGCu4KVGCJrvdW0RtwsJ2O5jZgKJ7K2+W2ZeIiIhIbpTTM3gBMAzYA7jP3V+vZIOy3L3dzO4E9gZ+llm1L/AKy8vRPEjMEt6HlAym3su9gdsyid8txGzoHUnJpZmtC2wDfLtan0NERESkUZWTDG4OnOTuN/X1zVOh50JJl/WAEWa2Z3o+xd1fBM4E7jWzi4HfEzOZvwEcUZgVnJLGHwCTzGwOkSQeTMxG3q/wfu7+NzO7GfitmX2XKKB9JjATuKyvn0dERESkvyknGZxF55eHe2sN4NqiZYXnBwGXuftDZrY7cXeRrwBtwHfc/dfZF7n7ZDMDOApYkygXs4u7P160/32ByUQPZwtxO7q93L05ZmqIiIiI9EI5yeDZwPfM7CJ3f7Mvb57uB9xtYunutxCXeLvbbjKR6HW1zVvEXVIO7VkrRURERJpXOcngaOBt4Fkzuw74F7CkaJsOdz+nr40TERERkeoqt2ew4LBOtukAlAyKiIiINLhyksENKt4KEREREamLXieDaYaviIiIiDSBsm9HZ2YbAdsRM4J/7+4zzGwIsBbwiru/W5kmioiIiEi1lHMHkoHAr4GvEzOBO4CHgBnAEOBJonbfTyrWShERERGpioFlvOZE4GvAKcDWZErDuPu/iVvVfaEirRMRERGRqionGTwIuMTdJwHPllj/JLBRn1olIiIiIjVRTjL4PuCRLtYvBFYtrzkiIiIiUkvlJIOvEPcR7sxEQDOORURERPqBcpLBPwLfTLOJCzoAzGwn4v7B11SgbSIiIiJSZeUkg6cDM4HHgN8TieCJZvYw8H/A48B/V6qBIiIiIlI9vU4G3f1N4OPAJGBN4B1gG2AVIlH8lLsvrGAbRURERKRKyio67e7vEMngpMo2R0RERERqqZzLxCIiIiLSJMq5A8klPdisw92/XkZ7RERERKSGyrlM/BnS7OGMQcDa6XEO8HYf2yUiIiIiNdDrZNDd1y+13MxWAg4FjgZ26FOrRERERKQmKjZm0N0XufsvgduBX1ZqvyIiIiJSPdWYQPI48Kkq7FdEREREKqwayeAOwIIq7FdEREREKqyc2cSndrJqJNEj+BHg7D60SURERERqpJzZxKd3snw+8BxwGHBxuQ0SERERkdopZzaxClWLiIiINAkldiIiIiI5Vs6YwfHlvJG7zyzndSIiIiJSPeWMGZzBe+9A0hODyngNZnY3sG0nq09w97PN7HTgtBLrv+fuk4v29xXgRGB9Yozjme5+dTltExEREenvykkGDwaOAtYF/gD8My03YF9gJnAesLQSDQQOB0YULTsgLb8ls2whcau8rBezT8xsT+ByYrbz7cAewJVm9qa731qh9oqIiIj0G+Ukg2sDLcAH3H1+doWZnQY8AKzl7v9dgfbh7k8VLzOz84An3f2JzOKl7v5wN7s7C7jW3U9Iz+8ys02BMwAlgyIiIpI75UwgOQy4qDgRBHD3uURZmW/2tWGdMbONgC2BK3r5ug2ATYCrilb9AdjSzMZWpoUiIiIi/Uc5PYNjgFW6WL9y2qZa9icuQf+haPkwM3sVGA08C/zC3c/PrN80PRb3NE5LjwbMqXBbRURERBpaOT2DDwPfNrOJxSvM7KPAt4G/9bVhXfgycI+7z8osexY4jhizuBvwEPDLNLGkYFR6fL1of4UeztEVb6mIiIhIgyunZ/BbwN3AI2Y2BXgmLS9cvp0HHFmR1hUxs62ADYFJ2eXuXnzJ+BYzAzjOzM5x97er0Z6pU6dWZD+tra0llw8bMZa2traKvEe9tdvobj9Lf/msPfks5aplDObOHc6sFxqvM7yz34c8UQwUA1AMQDGA2sSgnDuQPGVmHwKOB3YC9kyrXgTOBX7s7q9Urokr2B94B7iuB9teAxwIbAZMYXkP4Egg275Cj+G83jZmwoQJtLS09PZlK2htbWXixPd0sgIwe94Cxo1b0Kf9N4qWlqGMGzeu0/VtbW1drm8k3X2WctU6BmPGrM6aG5VVNrRquvp9yAvFQDEAxQAUA6hcDNrb27vswCqnZxB3nw18J/2rCTMbDOwD3OTub5axi6fT46bA9MzyzdKj96F5IiIiIv1Sn25HZ2YbmdknzGy1SjWoCzsCq9PzWcRfImoPTgNw9xeIJHCfou32Baa4e+NdLxMRERGpsrJ6Bs1sP6Jw8zpp0Q7AnWa2OvAgcLK7X1OZJi6zPzCXEvUAzayVKCbtwBAi4ftyakf2OuupwNVm9hzwF2B34LPALhVuq4iIiEi/0OueQTP7ItE79zTwPWBAYZ27v5aWf6VSDUzvuQoxS/gad19UYpNngaOB64mxgpsAX3P3H2Y3cvdrgYOIcY5/Jnob99PdR0RERCSvyukZPAn4q7vvaGZjgMlF6/9GhYtOu/u/ifqFna0vvvTb1b4uJ3oRRURERHKvnDGDmwJ/6mL9q4Du5iEiIiLSD5STDL5N13cg2RB4rbzmiIiIiEgtlZMM3gkcaGZDileY2TjgG8R4PBERERFpcOUkgycBawN/Bw4HOoCdzexs4EnivsFnVKyFIiIiIlI1vU4G3f0Z4BPEXTxOJ2YTHwN8H/gHsI27z6xcE0VERESkWno1m9jMBhG1BWe7+2fNbBTwASKpfF6Fm0VERET6l96WlhkIPAccB/zU3ecT9/0VERERkX6oV5eJU8HnNmKcoIiIiIj0c+VMILmUmE08tNKNEREREZHaKucOJP8EBgHTzexy4HlgYfFGVbg3sYiIiIhUWDnJ4BWZ/5/SyTYdxD2CRURERKSB9SgZNLPzgMvdvRX4dFq8CtEjuKRKbRMRERGRKutpz+C3gIeBVne/x8zGEPcg3sHd76la60RERESkqsqZQFIwoGKtEBEREZG66EsyKCIiIiL9nJJBERERkRzrzWzi95vZ/0v/Xy09bmJm/y61sbs/0qeWiYiIiEjV9SYZPCP9y/pFie0GEKVlBpXbKBERERGpjZ4mgwdVtRUiIiIiUhc9Sgbd/fJqN0REREREak8TSERERERyTMmgiIiISI4pGRQRERHJMSWDIiIiIjmmZFBEREQkx5QMioiIiORYb4pO14WZHQhcWmLV+e7+rcx2OwE/BDYDXgJ+7u7vKYptZscCRwBrAdOA49z9jio0XURERKTh9aeewc8BW2f+TS6sMLOtgRuBx4CdiOTx52Z2WHYHKRGcBJwP7AI8A9xsZlvU4gOIiIiINJqG7xnMaHX31zpZdyrwqLt/PT2/y8zGA6eZ2UXuvtTMWoCTiR7DyQBmdg/wJHASsHeV2y8iIiLScPpTz2BJKcn7DHB10ao/EJeCP5KefxxYDbiqsIG7LwGuAXYyswHVb62IiIhIY+lPPYNTzWwsMBO4DPihuy8GNgSGAE8VbT8tPW4C/B3YND1/usR2qwDrALMq32wRERGRxtUfksGXgdOAR4AlxJjAU4ANgAOBUWm714teNz89jk6Po4B2d1/YxXZKBkVERCRXGj4ZdPc/A3/OLPqLmb0BnG5mZ9WpWQBMnTq1IvtpbW0tuXzYiLG0tbVV5D3qrd1Gd/tZ+stn7clnKVctYzB37nBmvTCnZu/XU539PuSJYqAYgGIAigHUJgYNnwx24hrgdGI8YOFy8MiibQo9hvPS43ygxcyGuvs7XWzXYxMmTKClpaW3L1tBa2srEydOLLlu9rwFjBu3oE/7bxQtLUMZN25cp+vb2tq6XN9Iuvss5ap1DMaMWZ01Nxpfs/fria5+H/JCMVAMQDEAxQAqF4P29vYuO7D6/QQS4DngXZaPCSzYLD1OT4+FsYKltnuLqE0oIiIikiv9NRn8EtBBlJtpB+7kvaVh9gVeAR5Nzx8E3gD2KWxgZoPS625z945qN1pERESk0TT8ZWIz+zOR7E0FlhITSA4Hfuvuz6fNzgTuNbOLgd8DnwC+ARzh7ksB3L3dzH4ATDKzOUSSeDAxG3m/Gn4kERERkYbR8MkgcXn3a8D7iPY+AxwH/Lywgbs/ZGa7E3cX+QrQBnzH3X+d3ZG7TzYzgKOANYnxhru4++PV/xgiIiIijafhk0F3Pxo4ugfb3QLc0oPtJpO5lZ2IiIhInvXXMYMiIiIiUgEN3zMoItWxdGkHs+c1VumiYSPGltWm4S2DWXXlIVVokYhI81MyKJJT7YuW8OATjVXoO2ot9j4Z3H7L8UoGRUTKpMvEIiIiIjmmZFBEREQkx5QMioiIiOSYkkERERGRHFMyKCIiIpJjSgZFREREckzJoIiIiEiOKRkUERERyTElgyIiIiI5pmRQREREJMeUDIqIiIjkmJJBERERkRxTMigiIiKSY0oGRURERHJMyaCIiIhIjikZFBEREckxJYMiIiIiOaZkUERERCTHlAyKiIiI5JiSQREREZEcUzIoIiIikmNKBkVERERyTMmgiIiISI4pGRQRERHJscH1bkB3zGwv4MvARGA08BzwK+BCd1+atrkM+GqJl+/l7tcV7e9Y4AhgLWAacJy731G1DyAiIiLSwPpDz+B3gXbge8CuwPXAecCPirZ7Hti66N+d2Q1SIjgJOB/YBXgGuNnMtqhe80VEREQaV8P3DAKfd/c5med3mdkqwLfM7GR3b0/LF7r7w53txMxagJOBn7v75LTsHuBJ4CRg7+o0X0RERKRxNXzPYFEiWPAYMJS4bNxTHwdWA67K7HsJcA2wk5kN6Es7RURERPqj/tAzWMongXnAq5llG5rZ68DKwFTgbHe/OrN+0/T4dNG+pgGrAOsAs6rSWhEREZEG1e+SQTP7KHAQcEbq2YPoKZxCJHarAQcDV5nZMHe/LG0zCmh394VFu5yfHkfTy2Rw6tSpvf8AJbS2tpZcPmzEWNra2iryHvXWbqO7/Sz95bP25LOUq5YxqObn6Ity2jR37nBmvVDqIkL/1Nk5IU8UA8UAFAOoTQz6VTJoZmsBfwQeITOBxN3PLdr0BjO7EzgDuKxa7ZkwYQItLS192kdraysTJ04suW72vAWMG7egT/tvFC0tQxk3blyn69va2rpc30i6+yzlqnUMqvU5+qLcGIwZszprbjS+Ci2qva7OCXmhGCgGoBhA5WLQ3t7eZQdWw48ZLDCz1YBbgQXAbu6+qJuXXAuMN7Ox6fl8oMXMhhZtNyo9zqtYY0VERET6iX6RDKYE7kZgDeBz7j63jN0UxgpuWrR8M+At4KXyWygiIiLSPzV8Mmhmg4kZv5sDO7n7iz14zQCiVMyLmdnIDwJvAPtkthuUtrvN3Tsq3XYRERGRRtcfxgyeD3we+D4w3My2yqx7irjMezlwJfAsMJKYQLIdcEBhQ3dvN7MfAJPMbA7waNpuQ2C/qn8KERERkQbUH5LBHdPjj0us+zTwBNHjdzJxGXkRkejt5u43ZTd298lmBnAUsCYx+3gXd3+8Ok0XERERaWwNnwy6+/o92Gz3XuxvMjC57AaJiIiINJGGHzMoIiIiItWjZFBEREQkx5QMioiIiOSYkkERERGRHFMyKCIiIpJjSgZFREREckzJoIiIiEiOKRkUERERyTElgyIiIiI5pmRQREREJMeUDIqIiIjkWMPfm1hEpDtLl3Ywe96CejejIlZZbfV6N0FEckbJoIj0e+2LlvDgE231bkZFfPj9w+vdBBHJGV0mFhEREckxJYMiIiIiOaZkUERERCTHlAyKiIiI5JiSQREREZEcUzIoIiIikmNKBkVERERyTMmgiIiISI4pGRQRERHJMSWDIiIiIjmmZFBEREQkx5QMioiIiOSYkkERERGRHBtc7wbUmpltBPwC2AZYCFwFHOfuC+raMBEREZE6yFUyaGYjgbuAF4E9gTWAnwJjgS/Vr2UiImHo0GHMntf/v5sObxnMqisPqXczRKQHcpUMAocCo4APu/trAGa2GPi9mZ3l7tPq2joRyb1FSzq4Y8rMejejz7bfcrySQZF+Im9jBncG7igkgskfgXZgp/o0SURERKR+8tYzuClwSXaBu7eb2XPAJr3YzyCAd999tyKNam9vL7l88aJ3GTxwaUXeo96WLF7U5WcZutKAfvNZu/ss5ap1DKr1Ofqi3Bg04mcp19Ili5vis7z7bjsvzS7vHNmyymhemv16ZRvUB8OGDGbl4SvV/H07+9uQJ4pBZWKQyVcGlVo/oKOjo89v0l+Y2SLgFHc/u2j5/cCr7v6FnuyntbV1G+C+KjRRREREpFo+OXHixPuLF+atZ7BSpgCfBF4GltS5LSIiIiJdGQSsTeQv75G3ZHA+MLLE8lHA9J7uZOLEie3AezJrERERkQb1XGcr8jaB5Gli3OAyZtYCbEgvkkERERGRZpG3ZPAWYHszG5NZ9l9AS1onIiIikit5m0AyEpgKzADOYnnR6TvcXUWnRUREJHdy1TPo7q8DnwH+Dfwv8DPgauBrdWyWiIiISN3kqmdQRERERFaUq55BEREREVmRkkERERGRHFMyKCIiIpJjSgZFREREckzJoIiIiEiOKRlsEGY2oOgxdz8bxeC9FIOgOCgGoBiAYgCKAVQ+BrkPaKNw945UFHukma3s7ksBzGxQfVtWO4oBmNkQM1vXzDY2s5GZGAyod9tqLfuZC3EoXt7sFAPFoFheY6DjoLoxUJ3BBmBmewN7AbsALwLPAH919/My2wzM/vCbjWIAZnYgsC+wA/Av4A3g/4AfuPuCtE1TxyDLzFYHNgG2BZyIyTPuPi+tH+DuTX0CUwwUAwAzGwdMBHYG/gnMAh5095fS+qY/L+g4qG4MlAzWmZltADwG3AHcBGwBbAZsTtwp5SR3v6Z+Law+xWBZDJ4AriHujmPAB4E9gAHAKe5+ftq26U96AGZ2K/EHcAEwHngZeJBIkK9y9/Y6Nq8mFAPFAMDM7gPeD8wF3kecF9uAG4BzM18Wm/bcoOOgujFQMlhnZnYpsDqwT+YXeh1gO2A/YGvgT0Qy0FavdlaTYgBmdgGwAfAFd1+Ylo0EPgR8FTgQuAc41N2frVMza8bMziJ6io8A/pEWHwHsDqwK3A9c4O5/r0sDa0AxUAwAzOwMYB/gQHd/2MyGAfsDOwH/AcwAfuTut9WvldWl46D6MVAyWEdpLNylwEjgi8BSoCMzTmwjIhHYB7jJ3Y+pU1OrRjFYNt7jXOIb3w7uviB72cfMxgK7AUcTl4gOKCTNzcjMhhKJ71/c/eSidesC3wb2JGJxuLs/22w9IoqBYgAxhhi4GZjq7t8pWjcW+BKRGA4krqDc3oQx0HFQgxgoGawzMzsJOAyY6O6vpmUrufui9P9BwPeAScBe7v7HujW2ShQDMLOvAj8Gdnb31rSsOAZfBi4DvunuF9arrbVgZtcBS9197/R8JWBJJkHeAbgCeA7YtTBmppkoBooBgJn9BtjQ3T+dnhfH4GPARcBQYDt3f7luja0SHQfVj4FmE9ff5cT4j/vN7FMA7r7IzAaa2RB3X+LuZwN/Bz5az4ZWkWIQYz6eA243s8/DshgMSknhEnf/H+DPwEfq2dAauRf4opkdYmaD3H2Ruy9NJ0Dc/S/A54jB1J+pZ0OrSDFQDADuBLY1s9PTFYPiGPwN+E9gFHF1pRnpOKhyDJQM1lH6xZ4FHAq8Bvwq/cKv4+5L3f3dwnbAC8AH6tjcqlAMgrvPJWYSPwhcZmaXmNmGKQks9A4OAF4F1qtjU2vlUuBq4vLH0WY2HpZ/SUjbzCAS6C3q0sLqUwwUA9z9D8CPiOEy55nZh9PywpfFgcSEgieATaw5S63k/jigyjFQMlhHhe5dd78XOAmYDnwFuMrMTjCzUWkixReJwcJX1a2xVaIYLOfuLwLfJ37pPwHcbWbnmtkmZvYfwEHE7OLf1K+V1WOZYuPu/hZwKnFimwSca2ZfMLO1fXkJjSHEwOnX69HealAMFIOsTGL3S+LqwR5EDI4xsw+kL4tLgRHAOkBbs4yV03FQ2xhozGCNmdmqwJZEnaABwAvufmla10LMGv08MUtsTaInaBFwu7t/ox5trjTFYNlM4c8S3fmrAE8CV7j7S2l84GeJGOxE9ATOBd4CbnT3o+vR5lpIcRkILEonP8zsIOC/gcHAQ0R9rZeBXYH13X2D+rS2OhQDxcBi4siawHBgji+vI7cH8YVxHWAe8eV5FnEuHenuG9elwVWS9+MAahcDJYM1ZmbXELNGVyH+wI8n/sj/DPiFuy80szWJy6FrpvW3AS96KjnS3ykGYGY3ABsTyXAb8GFgEPA/wPnuPt3MVgFGEyf+dYCHiT8MTVdPy0oXHb/H3X+S2eY44FPE8TAO+D3wB3d/uPYtrjzFQDGATovP3wqc4ctLbx1EnEM3B9YCrgSud/fH6tHmStNxUPsYKBmsITPbnZgs8WV3v9nM3kckBHsRl0afB45Il0yLX9sUU+UVAzCzXYlf2t3c/R4zWw0YC+wNHAu8DZzo7r8r8dqmiEGWdV10/G3gdHe/Im27CpFAd7j7v+vT4spTDBQD6Lb4/EAiIfx52naou79jmaoDzUDHQX1ioGSwhszsZ8Qlvz19xfsKDiUum55GFFo+AZgMDHT3JXVoatUoBmBmpxDf5nYpTJDJrFuX6P7fD/ipux9bhybWlPWs6PiNRNHxf1nMpGu2Y0IxUAywnhWfvw84zN29SWOg46AOMdAEkhrIDAKeSSQB66XlgwHc/R13vw84BLiEqLk3oZkOcMVgBf8CPk1841+Bu/+LmFl9KvBlM9upxm2rqTQ+chDQASybHenuL7n774lC2xcQE2q+A9Bsx4RioBjAsnPkYmIySHbiwOvp3HgCcW5YHZhkZsObMAY6DuoUAyWDNZC5rHcv8ct+RFq+2MwGpx8+7v48cDwxI+g7pfbVXykGK/gr8BRwnJmtBSsky7j728BPgFeAb1pzlooAlp3EnJgsNKowO9KW1856hugt/i1RTqHp6qgpBooBLDtHthJjpTdNy7IxmEMUnT8H+C/ggPq0tHp0HNQvBkoGa8jjzhJnAUeZ2S1pSvhid1+S+UHPA64FVre4B2VTyXsM0pi/WcAPiZlf15vZR4vHAaZLRFcTZQJG1L6lNaWi44oBKAag4vOg4wDqEAONGawyMxvs7ouLlu0PnA6sDfwUOMuXF1dehRg8vMDd96xxc2smxeBMYrZw08egMNi7aNnWxGf/f0TtwJ8Ar7j7m2a2OjGzeJG7717zBtdIuvyxNJ3wzgZWI74IXOzuL2W3I2ZMDnT3verT2spLvb4DUgy2IW5JOAK4jpzEAHQcZFkUE/4lcRnwBuCH7v5cZv0AoodwTXf/XF0aWSV5Pw7qeT5QMlhlFvfdvdbd/5lZthIxAHTf9K8D+BPwDjF77EPAVumSab9nUTtwbWCEuz+Rlg0gkqCvETdbX0pzx+BnxHHwYGbZAGAjoqD2YcAawN3EZfS1iJh90t1fqHmD68DMPg18i7g88hJwC/Brotbax4nLIgd5k9yb2sxGe9H9Q81sW+AootRQG00eg1JyeBwY8LxnZgSb2QeJIvO7ECW4/hf4FTCMiMvPgK+7+3W1b3Ft5PA4qOv5QMlgFZnZN4ALiZuMv2BFZUHMbCwxXXxbYE+i1t6TRNJwRz3aXGlmth1x+5ztiGEJc0k1sYjxMS1EvaztiaSoGWNwMPELbO7+XInjYDjRQ7odkRi/QVwq+l93n1KHJleVqeg4FneUuRfYvPjcYDGz/mvAzsQfgbVozhjoOIhbyz0ATPSoLZo9DoYQyeAONHHxeR0HjXE+UDJYRWY2B/g5cHYaEzeA6PIdD8zwVE08s/1q7v5G7VtaPWb2EnGj9b8RB/A2xIG9gPim+xuPGbSF7ZsxBnOAc4H/TsfBYOKm8psD09z9laLtm6puWDFT0XHM7AHiXtz7FA8fyGyzFrAhzRsDHQdxHMwBvpQ9DrLDiyxKy6xGFBVuuuLzOg4a43wwuBI7kfcys58Ss0Ev8eXTvo8Gvk5k9kMsagkdX/hhFpKg4p6j/srMDgPaiQLKhYTvGjM7mZgRdwrwSTM7uHA5uAlj8FPiNkG/zRwHpwH7E7/UQ83sWuBUd/e0fvF799QcLIqOf5bSRcdPAw4ws0LR8dlFr22KWdUWdxb4MHGZpz0tW5vo/RhJ9I5c6e4ziXNI9rXNEgMdB8uPg0+w/DjYmJgpvE4aE/Zzd3+WuNfsi5nXNksMdBw0yPlAPYNVkC7/zgYmufvJadmpRDJY6CV7H/Bl4lvhbh7TxZuKmR0AnAx8xN3fTgfusiLSaQLFJcT4h8+5+9P1a23lpUkgrwLnuvt30rIziLI61wH3EOMCv0l889/Z3f9ep+bWhKnoOGY2E/i9u5+Qnm9L9BxvDrwJLCTuMvAjd7/YmrOoro4Ds1nA7zLHwWeIqyXrE+PDWog7E51BTCJpuj/WOg4a53yg0jLVsQ3xLW4/MzsvjQv5NlFIeF93P4c4yA8nuv73rldDq+xZ4hf9MDMb5u4d6TJpoYjmQ0Q19QVEdf1mY8AjwDfM7EYz+xjwDaJH9Ah3vxL4BdFL2A4cXLeWVpmp6DgAZvZN4ovgSma2WVp8EfACMWZqDaK+5kzgRDPbuJlioOMgmNmXiXP/+y1mjQKcDzwO/D933wDYB7gKOIY4NpqGjoPQSOcDJYNV4O5/Iga93kp0gT9AFBm+pjAWzKOq/DXEfSi3slR0uclMAX4HfBfYM02UwFMRzfT/x4iBs582s6app2dmK7v7A8RxcDox1uMh4u4jNxR+od19kbv/jYjVZmY2rFkuf2S5io4X3E3MCNwfmJyGCKxE/I7c7+7vuvtVxHCStYk7TjQNHQfLPETcReKDwM/M7G6iJ/B4YgIdKRk6iojTV+rTzOrQcbDM3TTI+UDJYJW4+z3EN7pTiUKidxBdvsu+FaXZYs8TYzebJhksfMNJA6CPJu6leSnwSzPbxFJx6YwnieLKzfSt7ydm9ll3n050+R9CXOa4iRj/kz0OBgOzCi9sxstBBZ7zouPu/rS770oMDVgD+AIxZOAld+/I9Iy8QMy2X7lJvxzk9jhI46Gfd/dvAUcCM4irCNcTdUaXZpKh+cTVhSElzpv9Xp6PA2is84HGDFaQxRT5CcAniVmiN6fl44CR7v5Uujy6NC1fj0gSL3f3s+rV7kpKk0Z2JC77/jvzWQ8jLo8OIwqm3k3ccmdz4EdEr+nxdWhyxZnZocTYn+eJYQFT0vKRxO2FXig6DjYgblH3O3c/vT6tri3LWdHxzqTfi8Xu/pui5WOI5OAxdz+qHm2rNFMB/kJB6TW9qGSUme1DjIe7smj5KOI4cHc/pGYNrTGdD0I9zwdKBiskfZO7lKgF1AGMIS4Pf8lXrBo+KH3r2ZAYB7Gvu7+vHm2utPQt5m3gJHefXGL9eOKy6beIntAWooTAXe6+Xw2bWjUpBnOJX9ytidlfX/WiwtGZ42ATotfwi+6+Xq3bWwumouPAshP6Ku7+YtHyAakXYCWPW04NIeqOXkbUpmyKouOmAvyY2WPEbNnt3f3h7GSAEsdBC3EcXEpzHQc6H9B45wMlgxViZucRs57OIsYB/gfR43Wfu+9ftO0goqL8x4DD3P36mja2SszsQmAr4kT3Wlq2CZEgrwrcBTzh7q+b2W7AfKLA8j+9k9pK/U0mBtsSJSIuJi5xHFCiV2Qg0SO4CXCou99U4+ZWnanoOGa2BTFR6gBiqMhjwMleogB52v7bxBem6939ezVvcBWYCvBjZl8lErvZxFWRA9z9X6V6TNP2pxBfnq9z9+Nq2tgq0fmgcc8HSgYrwKI20hTgSE+3B0o9RMcAk4BPufuDtmJV8S2A8c2SAJjZB4DpxCXie9JA4G8RA6DXJWYMjyQGyx6W7S1tFma2EXGS/0IhwU9/BM8l/gic6O5vFF0iHgP8h7v/tU7NripT0XHM7ElgHvAoMUxie+AfwFe8qGCsma0LfI8oSv6VZhk/airAj5nNI+41+wJwBXAjUV/vPV+EzWw0kTSNAw5pouNA54MGPR9oAkllfJYoJfMMLOvmXUyMfZhB3FKITCL4PqKCfFMkgsmxxPE0OiWCo4kK8jcR3/Y3I7r/twJusKim3mwuAm4mZpEXXAdcTcwG+yJAJhEc5O5zmzgRzBYd/6W7X5PGu4wjLgGdAlxuZu8vvMYzRcfr0eZKs6gruRT4hketycOBnxDHwr5pm2WTx9IfwnOAo5ooAeisAP8DxNWCl8zs3OzkgCY8Dn5K9IL9xt2vJnp6diQmmg1L2yz7e5wmTZwNfLuJjgOdDxr4fKBksDL+TdxKZwZE0pd6fxYTycCyQa8W5VNuIsaJNYV0EruR+FxXm9mfiG++twI/cPdHgFfd/Vric38I+Ei92lsNaRzM08SJbtltotx9vrsfRCSEF5rZ18xsYPrC0Eyzp0t5m7h/5jyIE3pKgN9090OJOy+sDdxlZptmX9gMfwDT7/qOwP8Qk4lw96Xu/mvij9+X07JlY8bS8395zCLt99Ll36OJckovp2WnEn/4pxNDaS4lYvFo6l1fpkmOg3WJ0iknFYbPEJdGryBmkR4Iy78kFrj7QndfUMOmVpvOBw18PlAyWAEe9QJ3TZcACz/Awi/2bcAHzGyr9HwvorbUebVvaXWkA/oWYgzEgcAGRG/gLaQyKhlPEL0E69awiVXn7u3ufri7P5ldnvmWdw4x9uVE4IPNcHLrgbwXHV83/Vvsy+8zWzjn/gH4qJl9MLP9p8zsZmuu8hkqwB+fdRpRYqxw5ehNdz+MSIQnm9kh6UtiM/9N1vmggc8HzXzg1ZS7z0iPxX/kHwJeArYzszVYfmuhebVtYfW5+5tEkendiMvGj6Re0sLsqAHERJJBvDdJbEq+vLj0VOIP3WLgT2b2H3VtWG3ktug4gLtPI7703QHLkoDCl8T7ibqan0vrRhC9ZaOKxw31Zx4F+A8gxwX43f0bxK0mF6TnHZkk4GdED+nxwCbFvYNNppXoDc3z+eB8GvR8oGSwitIPu52YMfp54DhgkLufUd+WVU/6tjcTuMjdH02LC8fZCKLncEkaN5Mb6XLIs8SkonWJnpCmlOkdLxQdv5+cFR235UWDf5S+CKzwRdHdZwN/BnZNi3YnZlh+sbYtrR5bXjD3fuLuESeTvwL8QwDc/ZXs8kzyM5X4mS8E/s/ilpVNyaNu4NHAg+TsfFDg7pPcfWrqCW2o84FmE1dRpkfs88TU+QFEPbk/1bdltZe+BR5J/FE4xN1vrHOT6sbMjgGedPe/1Lst1ZKSoYGF3h8zO4T4pjucqJd1D01adLwgJUNLOhsSYGb/RfSUfAr4PXCTN0kpmYJMQli4LLYGMMbdn7YmL8BfkH4XlpY6DgoxMLPdiQlojwL7NdGY0c5uxPBN4CSW34Sgac8HRTF43N3/nJYvO/7T87qeD5QM1oDFnSceB55298/VuTl1YWYbE5cM72qWX/LeshI1pJpJ+qN3EnBxZrLASpmE8P1Ez/A3iS9GQ2m+ouOlYrCssHBmu4HEDeqnECU21nL3sbVubzV0dhwQiXH2j18zF+Dv0XFQ9JpvAke7u9WomVVlnd+IYW93fzn93L/K8qskzXo+KI7BvcSx/nJmu7qfD5QM1kg6Ga7s7q/Xuy31kn4xBntmtq00DzM7G/g+cZnnUuBcX15OqbjI8OeJcaNvAdO9eYqOdxWDUknhX4HPAHu5+x9r3d5q6CYGKxRYtuUF+LciCq9fX/sWV15vjoPMFaSViF7TV0rutJ+x0jdi+DFwr7/3RgyF88GbxK33muV80OObUaTt63Y+UDIoIn2WSojcS5SNmE9c7pkBTC4MCUjjw4Y065eBXsRgoC8vH7EJccvK0+vR5krrbQzS/ycQBfhvrk+rK6uc46DZWPc3YtjW3R8o/nLQTHoQg1I3o9iU6DU8tdbtHVzrNxSRpvQRYC1i9uyVwMHAHsAFZrYvMMmj7E67mQ0lak22AA800aXznsZgiUVdyg8Tl46aaYxcb2IwhEiUBhFlqJpFOTEYSnP9LpS8EYNF8e1DiMumD2TGkq5LJMcvdrbDfqi7GOwCPJhJBNcjekdPq0djNZtYRCphOjEQ/JpUKmQyMWHoSqLn5wYzOzuVTBhO1NXauYn++EHvYrAyMVh8jybrHepNDFZJy3fP8XFQiEGz/S709kYMN5KKbzeR3sbgemKoRF2OA10mFpGKMLMh7v5udtJIWv45opjsNsAbRN3NHYhB0k0xa7JAMVAMQDEAMLP13X1GifHC2xEzxz/h7g+b2deJ+xKv5U1Wf7c/xUDJoIhURVHpkBHAF4ib0m8DHOvuP61n+2pBMVAMQDHISkMkngEuAC4hyulc7E1cf7dYI8ZAyaCIVFWmhMhg4CpgC3ffqLvXNRPFQDEAxSAzc/oSwICHibqKa9e5aTXTqDHQBBIRqar0x28g8J9Ej8ie3byk6SgGigEoBhl/IsbIbU0T3XWnlxoqBuoZFJGaMLPxwP7uPqnebakXxUAxAMXAdCOGhouBkkERERGpKd2IobFioGRQREREJMdUZ1BEREQkx5QMioiIiOSYkkERERGRHFMyKCIiIpJjSgZFREREckzJoIiIiEiO/X/icxdMgxke1QAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAnYAAAFKCAYAAACQBBKyAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAACHR0lEQVR4nO2dd5wURdrHf7MLLDkHEUkCFiKY1hzOHFAx5/CennreneE8MZzhxHzmnD3jmdBTCZJEohIUlpyKuOSwwO4Cm0O/f1T3THdPdZye6ZnZ5/v56DLd1RW6u6qefup5noooigKCIAiCIAgi88kJuwIEQRAEQRBEMJBgRxAEQRAEkSWQYEcQBEEQBJElkGBHEARBEASRJZBgRxAEQRAEkSWQYEcQBEEQBJElkGBHEBkAY6wXY0xhjN2oO/YYY8xXvCLGWCFj7BMX6U5Vyz3VTzlJqM9UxtjUBMo5gzE2jzFWobarrd+8shnG2CeMscKw62GGMdaCMbaNMXZz2HVxA2NsDmPs+bDrQTQsSLAjiIBhjN2oCg3HhV0XIgZjrBWAbwHUA7gDwA0AykKtVIgwxgaoHwe9wq6LB/4OoAbAf8OuiEv+DeB2xth+YVeEaDg0CrsCBEH45ikAz/q8lkEIOJnG2QlcexiAdgCe5JyPDKg+mcwAAMMATAVQaDp3K9Lsw58x1hjA3QDe4ZxXh1wdt4wAsAfA7QD+FW5ViIYCCXYEkaFwzmsB1Pq8tirg6qSEBCf0zurf0iDqAoilQc551mn9OOc1YddBwgUAOgH4JuyKuIVzXs8Y+x+APzLGhnHOM/FjisgwSLAjiBSg2o9dDaAPgLcAnAmgAsCnAB7gnNfp0rYF8CqASwAoAEYCeEWS52MAhnHOI+rvHwEcCqAn51wxpf0ZQB/OeW/1dyGAqZzzG3VpDgDwJoCzIJYovwAwXlJu3LXq8akAwDk/Vf3dBMDDAM4D0BdAUwCLATzDOR8hv1P2SMroBWAdgAcB7ALwTwAHAFgE4G+c8zm6605Rs5nCGAOAT7U2MMaOBvA4gBMBNAFQAOBfnPMpurIfg9BwDQLwAIDzIYRE7Z6erbY3X73kVwD/5Jwv0OXxCdy/BxEAf4PQnjGIZzIfwBOc81906a4F8A8AAwFUAvgZwP2c83U29/FGAB+b7gcA3MQ5/0St56mc8166axQA76n5Pw7gQIj7fBvnfAFj7FYA9wPoDuA3Na+1pnId77MNFwPYyjlfasozrq7q8ceg6x/qsTMQe4Z5ALYCGMc5v0OXJg/iPboeQA8AOyGEyYc55+WmMq6G0CIOglgiXgLgBZNGeCLE0n8+gDku2kkQCZFWqnaCyHJyIASlXQDuBTANwFAAf9YSqJP5SAj7ry8APAJgf4iJ34mvISbV4/UHGWOdAZwKYLjVhYyxZgAmATgHQrh7GmLyTcTwuzWA2wDMgBB4Hoa4Bz8wxgYnkK+MqyCEivcg7lkvAN+ry3eAaM/r6r+fgbi/7wEAY+wUAL8AaA/gCQihLQ/ATxZOI8MhlnQfAfCamse1EM+2EkLIfAxC8PmFMdbfdL3je6DyPsSz2Kbm+TSAEgB/0BIwxv4J4HMI4XYogBcBnARgBmOsk/xWAQCmS+7HDepxO06A+Mj4r9pGBuBHxthfIITLdyDemeMAfKK/0Md9lpU910U6KYyxAQDGAGim1v3vEDaXJ+rSRAD8APEujQFwJ4RQ9zcAI9TzWtpHAHwF8fH1OMRS62qIPqSnQP17IggiBZDGjiBSR2MA33LOn1B/v8sYmwfgZogJEQAuhJi4H+CcPw8AjLF3ILQkToyE0P5cBWCm7vjlAHIhBD8r/gzgIABXcc6/Uct9H0JD5JdiCO1hdNmXMfYmgHkQQsi4BPI20x1AP855sVoOh7gf5wD4kXM+kTHWBsBdACZyzqeq6SIQAt6vAM7SNJ2MsXch2v4MhEChZwXn/DJdm1pACGCfcM7/pDv+IQAO4FEA1+qud3wPVEHnFgBvc85v1137iiZcMMZ6AHgSwGO6vMAY+xrAUghB6yHZzeKcr2WM/WK+Hy7oD+BgzvkataxiiPv3BMT9L1WPNwLwIGOsL+d8tc/7HEXNrw+EsOWXsyAEycGc85264//U/fsaAOcCOI1zPk1X/lwIAfosCEG0D4QwNwrApRJNaxTO+WbGWDWETSNBJB3S2BFEavnA9PsXCM2OxnkQTg2aoAd10njLKWPO+V4AYwFcwRjT9+2rIISRBTaXnwdgO4D/6fKrAPAfp3Jt6lOnCXWMsSaMsfYQWrzpiC1XBsV3mlCnoi1VHihLrOMwCK3TlwA6MMY6MsY6qvWcCOBYxlhz0zXvmH6fBaHB+1K7Xs0jV63HaZJynd6Dy9W/w8wX6pbZL4X4OB9uKrcUYslbVm6iTNGEOpXf1L/fa0Kd6bjWJj/3WU97ABGIjwW/aPW72NQ/9FwJYCWApaZ7Og1CM6fd00sg5s8n9UIdYHg+eooBdEyg7gThGtLYEUTqqOGcbzUdK4YQCjR6AtimCml6Vros42sAl0Fo/aYyxvaHWJp70uG6ngDWSIy73ZYrhTF2C4Tm6GCIiVnDV/w9Gzbof3DOi1W7sXby5FEOUv9+aJOmAwC9bdUa03ktj4kW15vvqZv3oA+A7SbNkhmt3BUW59daHE+EDabfmrC00eK41iY/91lGxOG8HcMhtKIfAHiWMTYZwmv1G9URSasnA1BkkYfmgNNH/bvUIp2ZCIJ/5wlCCgl2BJE6UuERNwbAXggt3VQIDUQO7JdhvWI1QeUC0C9JXQcxiY4G8ByAHRBevDfBuDQZBHUWx50EAU1z80/EbKHMmCf5Cos8bgSw2aE8ILj3QCt3MOTe0eZ6BoHVfXa6/37us55dEO+dTFC3ex+jcM4rVDu/P0BoqM+BsGO9hzF2sqqhzgGwDML+TsYWmzra0RbCCYMgkg4JdgSRXqwHcBZjrJVJa3eQ1QV61MlrFIDLGGN3QAh4CznnVlodfbmHMcZyTFo7WbnFEBOVmZ4waomuUH9fpF+eYozd5NySlKFp3/Zyzt3YMdrlUZRAHrI8z2WMdeKcWwk8WrkbOOfLfJSRSg1SQveZc17HGFsF1QPZhN37aM6nHuKDZyqA+xljfwXwNsSy9hdqPfMBTLJYUtXQ2nMIHBw6GGPdIDyAl9ulI4igIBs7gkgvxkL0y79qB1R7oNstr4jna4h4XzdBeCe60daNBdAFMdsuzVP2FknaNQCOU8OZaGkvgHBg0KNpcfSehAdC2CelCwUQnoz3qDtTGHDwLNWYAOGt+pD+nnjMw4xm6/iYJD/tfn4HcY8fNRvsq+mcbLq0+HtOy9VBEMR9ngHgKMnxNQDaMMYO1eXXFab3jDHWQXLtPPVvW/XvcIh+8FdzQsZYnq7uP0BoXh9ljOWa0pmfhWZPOhMEkQJIY0cQ6cVoiAns32qMtqUQ8bvae8hjAoQW42X1txvB7gOIWFufMsbyIZYUrwcgC2T8HwgBcDxj7BsIe6PrEW97NgpCEzJK1SJ2gwgbwQEc7qE9SUMNIHszRPiRZYyxjwBsgggxcwqEUGrrhMA536OG+/gCwHzG2FcQjig9IDwsl0Is03qp11Q1PtvfVA9MzYP4eIjYcc+onq3/BPACgJ6MsREQAmZvABdBCCmP2RQzH0IwfJCJ2IkVAH6zi3/nlyDuM4SX802MsUNMsey+hljq/4Ex9jqA5hCC2UoAR+rS/Uv1Nh4DsdNGOwB/gRBwf1TTfA7xbr+lLtv+qtaNQZg1XAERw3ENY+wJiPv7K2Psewj7wCMhQt7oP8TOUtvqO1QLQXiBNHYEkUaoS0UXQggJ10HELtsK4I8e8qgB8D2AVhATdaGLa8oBnAHgJwgB718QGob7JWknQIQrOQgikPLxELsCbDKl+1S9fgBEzLTLIBwp0mo7L875dAjN5mwIwfNNAH8CsBtCYHCTx3AIwWQDxL15HcKOcDnUeHk+uBnAPRCa0Ochnkl7CA9NrdwXIQT/aoi4ei9DaKqmQsRos6vzdojgx+0gBPuvEAviHDgB3OcxEHaaV5ry3QXR5nKI+/RHiLh/o03Xj4QwDfijWvbdEMLtiZzz9Wpe9RAfI/dBvLcvQIQ1OQ5iyXaRrtzH1bwaQ4R7eQpCAJygpVG17ZcD+Ix2nSBSRURRyFGHIAiCSH8YYw9CaOP6pOm2ZwYYY5dCaAH7SDyhCSIpkMaOIAiCyBReg3BE+L+wK+KSBwG8SUIdkUpIY0cQBEEQBJElkMaOIAiCIAgiS2jwXrEFBQWNABwAYFN+fr4syCdBEARBEERa4CS3NHjBDiKI5WoAJxcUFGxySkwQBEEQBBEiB0DsL90X8WGmSLAD0FX9+4ttKoIgCIIgiPShK0iwk7IVAA466CA0aRIXND5QlixZgoEDBya1jHSlIbcdaNjtp7Y3zLYDDbv9DbntQMNuf7LbXl1djZUrVwKq/GKGBDt126MmTZogLy8v6YWloox0pSG3HWjY7ae2N1wacvsbctuBht3+FLW9TnaQvGIJgiAIgiCyBBLsCIIgCIIgsgQS7AiCIAiCILIEEuwIgiAIgiCyBBLsCIIgCIIgsgQS7AiCIAiCILIEEuwIgiAIgiCyBBLsCIIgCIIgsgQS7AiCIAgiC9hStA/XDxuHouKKsKtChAgJdgRBEASRBYyfvR6l+6rxy4JNYVeFCBES7AiCIAiCILIEEuwIgiAIgiCyBBLsCIIgCCILUBQl7CoQaQAJdgRBEASRVUTCrgARIiTYEQRBEARBZAkk2BEEQRAEQWQJJNgRBEEQBEFkCSTYEQRBEARBZAkk2BEEQRAEQWQJJNgRBEEQBEFkCSTYEQRBEEQWEaFoJw0aEuwIgiAIgiCyBBLsCIIgCIIgsgQS7AiCIAiCILIEEuwIgiAIgiCyBBLsCIIgCIIgsgQS7AiCIAgiC1CUsGtApAMk2BEEQRBEFkHhTho2JNgRBEEQBEFkCSTYEQRBEARBZAkk2BEEQRAEQWQJJNgRBEEQBEFkCY2cEjDGrgBwHYB8AO0BrAHwDoD3OOf1unSDATwNYACAzQBe5Zy/IcnvXgC3A9gPwFIAD3DOJ5nStALwAoDLATQFMAXAnZzzQlO6fgDeAHASgAoAX6v5lbtoO0EQBEEQRFbhRmM3FEAVgPsAXABgBIDXATynJWCMHQ9gFID5AAYD+BjAq4yxv+gzUoW6ZwC8BeB8AKsAjGGMHWYq8ysAFwK4E8BVAPYHMIkx1lyXV1sIga8VhAA4FMA1AD5y0SaCIAiCyCoUULwTwoXGDsAQznmR7vcUxlhLAHcwxh7hnFcBeBTAPM75zbo0PQAMY4y9zzmvZ4zlAXgEQpP3IgAwxqYBWAzgYQBXqseOhRD6zuecj1WPLYbQFN4I4G21jNsAtANwOOd8p5quFsAXjLEnOedL/dwQgiAIgshsKN5JQ8ZRY2cS6jTmQyyRtlcFttMBDDel+RJiufVI9fcJANpALJdqedcB+AbAYMaY9iaeB6AUwHhdug0AZqjnoEs3SRPqVL6D0C4OdmoXQRAEQRBEtuHXeeJkALsB7ADQB0ATAMtMaTSNWX/178Hq3+WSdC0BdNOlW6G339Ol66/7fbC5TFV7uMaUjiAIgiAIokHgZinWAGPsKAA3AXicc17HGGunnioxJS1W/7ZX/7YDUMU5r7BJt0lNZ85LS9de99ttOlcsWbLE6yW+KCgoSEk56UhDbjvQsNtPbW+4NOT2p7rtO7aXAAA2bdqIgoKSlJYtg559OHgS7Bhj+0Esd/4OnfNENjBw4EDk5eUltYyCggLk5+cntYx0pSG3HWjY7ae2N8y2Aw27/WG0fd6mxQDfhwMO6I78/D4pLdsMPfvktb2qqspWGeV6KZYx1gbAOADlAC7knNeopzSNW1vTJZomb7cuXR5jrKmLdOa8tHS7db/dpiMIgiCI7IecYgm4FOxUYWwUgM4AzuWc79KdXgOgGjEbOo0B6t8V6l/Ntk6Wbi9E7DstHdM5U+jTrdD9Xm7OS3Xk6GNKRxAEQRANhgg5xTZoHAU7xlgjCM/VQwEM5pyv159XHRYmQw1XouMaANsAzFN/z4Twdr1Kl3euet14zrn2rTEWQhN3ji5dd4ggxGN1+Y8FcAZjrIPu2CUA8kzpCIIgCIIgGgRubOzeAjAEwP0AmjPGjtOdW8Y53wPgCQDTGWMfAPgCwIkAbgVwu+bdyjmvYow9BeAZxlgRhMB3C4SG7VotQ875b4yxMQA+ZIwNBaDlvwHAJ7qy34MIYDySMfYkhDbxZQDDOedmD12CIAiCIIisx81SrKY5ex7ALNN/RwIA53wWgIsAHA1gAoTA9g/O+bv6jNTAxA8BuAvCXq8/RCDihaYyrwHwI0Qw4m8hNH9n6rcK45yXQMTP2wfgewCvQMTS+5OLNhEEQRAEQWQdjho7znkvNxmpu0Q4LoGqwt2LDmn2QuwscZtDupUAznVTP4IgCIIgiGzHb4BigiAIgiAIIs0gwY4gCIIgsgCKdkIAJNgRBEEQRFZB0U4aNiTYEQRBEARBZAkk2BEEQRAEQWQJJNgRBEEQBEFkCSTYEQRBEARBZAkk2BEEQRAEQWQJJNgRBEEQRBagKBTwhCDBjiAIgiCyC4p30qAhwY4gCIIgCCJLIMGOIAiCyDh2FJfT0iNBSCDBjiAIgsgo+PrduPmpifjpt/VhV4Ug0g4S7AiCIIiMYuP2fQCAZet2h1wTgkg/SLAjCIIgMgxagiUIK0iwIwiCIDKSCHl/GiF5lwAJdgRBEESGQT4T9kQo3kmDhgQ7giAMlFXU4LclW8OuBkE4QgIMQcRDgh1BEAZe/KIAT338O7bvLg+7KkSCjPplDdZv3RN2NQiCSCGNwq4AQRDpxdadZQCA6pq6kGtCJMoHI5YgJwKMfPGisKsSKLQSSxDWkMaOIAgTYtokw/TsoD4LpSDNxo7eUYKIhzR2BEEYiE2aNGv6pbK6Ftt30VJ28shCaZUgAoI0dgRBGNCmTBLr/PP8f+fijhenoLq2PuyqZDX08WGExF0CIMGOIAgT2v6bNGn6Z8maXQCycxk0HaBwJ/ZQ123YkGBHEIQBsl8KApI8UgG9owQRDwl2BEEYIJEkcaLCcbjVyFroHbWHNJoNGxLsCIIwQkuxBEEQGQsJdgRBGCDnicRJB4WJks1qG/r4sIVuS8OGBDtCypChI/HcZ3PCrgYRAgpJdglDdorJhV5RgrCGBDvCkl8Xbgm7CkQYaNoQmjb9kwbasjSoApFislpLS7iGBDuCIAxEtSEk1/kmplGim5gMSH6xh966hg0JdgRBGKBlxAAJ8R42CNmH3lGCiIMEO4IgTDQIkSCpkEYpydANJghLSLAjCMIA7RUbBJqdYphVyH7hh95QgoiHBDuCIAyQx2HipINMlQZVSBrZ3DaCSBQS7AiCMEKSXcKQA0pyIa0yQVhDgh1BEAbqKdxJVpAOWsNkoaTDUncaksWPnPAACXZE4IybVYiSvVVhV4PwCXnFBkA2S1XpBL2jcqjzNmhIsCMCZUvRPrz9v4V4lnatyGBIKAGAD0YuxvT5mxLO59IHRuPOF6cEUCOv0HMkiIZIo7ArQGQXNbX1AIA9ZdUh14TwC9kvCUZNXwsA+MMRB3i+Vi9S1dTWo3DrnoBqRQAgmZUgbCCNHREoZDSe+dAzTJx0WIlNhzokm4b+8UEQMkiwIwjCiEKG6dlANst12dw2gkgUEuwIgjAQ09iRaEekJ1FzgXCrQRBpCQl2DZAxM9bhq594UvJWSNuT8ZBXbHagZPVaLEl2UrL5kROuIeeJBsi73y8CAFxzNktaGaTtyWRodiAyA4q1KIeG34YNaewIgjBAXrFZQhbL51mtjCSIBCHBjiAIA7SjWHAk6x5WVtVixLQ1qK9PLwnn2c/m4JWv5qWsPPr2IIh4SLAjAoW+pDMfhSS7tOfTscvw4aglmLVkq2WaMLrijIVbMHnuxqSXQ+MMQVhDgh2RFOhLOoPRHGDoIaYt+8prAABV1XWWabLbeYIgCCtIsCMIwgAp7DIHkr0JgjDjyiuWMdYXwL0AjgMwEMAKzvlAU5pPAPxRcvkVnPP/mdLeC+B2APsBWArgAc75JFOaVgBeAHA5gKYApgC4k3NeaErXD8AbAE4CUAHgazW/cjdtI4KFtASZD4U7IdKf7NAq7yguBxSgc/vmgeRHoy8BuNfYHQLgfACrASyzSbcWwPGm/ybrE6hC3TMA3lLzXAVgDGPsMFNeXwG4EMCdAK4CsD+ASYyx5rq82kIIfK0gBMChAK4B8JHLdhFJgsIQZC4KGdkFB93CpJAtAYpvfmoibn56YuD5Zvp9IRLDbRy70ZzzkUBUM3eURboKzvlsq0wYY3kAHgHwKuf8RfXYNACLATwM4Er12LEQQt/5nPOx6rHFANYAuBHA22qWtwFoB+BwzvlONV0tgC8YY09yzpe6bB8REPTFmPnQXrHpjxvFeDYrz7O4aQSRMK40dpzz+oDKOwFAG4jlUi3vOgDfABjMGNOmkvMAlAIYr0u3AcAM9Rx06SZpQp3KdwCqAAwOqM4E0bCg3UOyimwW0LO5bQThl6CdJ/owxkoYYzWMsfmMsatM5w9W/y43HV8KoCWAbrp0KyQC5VIA/U35GZaGOedVEJo9fToi1dCAm7FEtSE0a6aMEdPWYN2W0kDzzGatFtnyEoQ1QQp28yEcLC6GsHfbBOBrxtiNujTtAFRxzitM1xarf9vr0pVIyijWpfGSjkgVNN5mPNliv5RJfDhqCe56aarn62yfEWleCaJBEthesZzz10yHRjLGJgN4HMAnQZWTLJYsWZKScgoKClJSjhvc1MVrfbfsrgYAVJSXx12bTm0Pg0xr/7x5BYF5HWZa2/UEVfcg+9uu3bsAAOvWFaIVdkjTVFSLBQ/FQ75BoS8vGWVv3rwHALBt+3YUFFQGnn9QuG17UPeoqEjoSNZv2ICCvN2B5JkImdzvEyXMtgcm2FnwLYC3GWOdOOdFEJq0PMZYU865vje2U/9qb2IxgB6S/Nrp0mjp2lqkW+GlogMHDkReXp6XSzxTUFCA/Pz8pJbhii83AYB9XdykkdBmYwkwfgeat2huuDZt2h4SGdV+9dkfdZSVj5Q3Mqrtenz2Af21erE4yP42eflcYP1m9O7dC/n53aVp9pZXA//bgoiHfBPG1I5kPfs1xSuBhXvQdb/9kJ8/IPD8g8BV2xN5xyTMWrsAWF2Gnj16ID+/dyB5+iVj+30AJLvtVVVVtsqoVAco1mzrDjYdHwBgL4DNunRM50yhT6cX2Jab81I9b/vAo2BHBIOShLXY14fPx0+/rQ88X4JoEGShraQSjWMXckXSFRc3ZmdJBd75biHq6oLyjSTShaQJdqpQdiWA9aq2DgBmQni7XqVLl6umG88516SCsRCauHN06bpDBCEeqytmLIAzGGMddMcuAZBnSkekiGTYZ038fQPe+GZBgDkSRPqQkCOAzQRO/gWEHW9+uwBjZxZi4aqdzomJjMLtzhPNEQsz0hNAa8bY5ervOerfTyGCCq+GEMpuAXAqgBu0fDjnVYyxpwA8wxgrAjBPTdcHwLW6dL8xxsYA+JAxNhTAHgBPANgAo73eexABjEcyxp4E0BnAywCGc87tAikTyYY+pQnCFb4EMBLaiATR3rt6+gLIOtza2HWGsJfTo/2+CcAoCE3cI2raGgih7ULO+Wj9RZzzFxljAHAXgC4QIUzO55wvNOV/DYAXIYIR50HsMHGFfqswznkJY+x0AK8D+B6xLcXud9kugiCIUElW6I6sDgmSxU1LGfTtnbW4EuzU/VmdXoOL3Baq7jrxokOavRA7S9zmkG4lgHPdlk0QBJFOJFtGyeb5O9P3ig0T7c5l9QdAAyXVzhNElqNQ7CyCiOJG8Ij2GR+dpqH2MxJFEkd7N+leZh8k2BEEQYSIH4cjN5NxQ1DENFTBNlAawHvS0CDBjkg7vp20EisKww+uaWbx6p0YMnQkdu+pRHllDW4YNh6LVhc5X0gQNmjzarKWFbNxtTIqtGZh24LAzW3J0TR2DeELoIFBgh0RKLFJyn8en41djvve+CWQ+gTJ6F/XAgCWF+7G+q17UbKvCv8da972mCC8odRrS7HBSinJiCmZNkRNPkiy84v2umXxW9JgIcGOSAo04DYctu0qo6/+BNDuXI6HLuPKLi/6SLKvL5rftlUbi1FUbN6CnHADdd3sgwQ7gnCJzDGkoY+JfP1u3PrMzxg3qzDsqmQsUaE4G9dMk4x2y+55dTr+9NRP4VYmY2noo1j2QYIdESxZPEbo59/oHJzF7XXD5qIyAGJ5mvBHsuS6bH4100nLdNXDYzD6l7VhV8Mz0aXYNLqXRDCQYEcESsMwao5E25fVdkyuaOjtTxw/S7EabkweslkRGHbTFEVBeWUt3h+xOOSaCLwIaRTuJHshwY5ICmEPuMkm29vnFbof/lF8fA25C3dCU3ayqa9Pz3vsSZhPzyYQCUCCHUF4RD9o0tzpTF29gro0nQDTAe0d8qOxSwd2labeaSGqKQ9ZHants5qTgQ8vQqsOWQsJdkSgNJRBgpYx3PPk15vxwJvpF74mXYh5uGaecLBi/W7c+MRPmDx3Q2oLTg+5DnV1qmAXdkV8EB3DaBDLOkiwI5JCKiepyupaLFu3K+nl0ADoH76+OOwqpC0JOU/YXONnRwuvrN+6BwCwdG1qnWfSpStmtMZO/UtL9tkHCXZEoIQxRrz5zUI88Oav2FFcntRyNG2kYQhv4INitjY/lZNd9L3yItml2X0PSzgIW5zSbOwyUK7LSA0x4Q4S7IiMZ+2WUgBARWVtUsuJaVYiFLXdRLZNEqmUU5IW7iQlbQjnuaeLlimInXbCJk1uJREgJNgRhFcitLOGGW2iLa+swazFW0KuTeKkcq5Luo1dJksdToTcNP3HXjrgReClUJzZCwl2hC3fTV6F8sqasKvhDoex9bOxyzBu5rpAi8zmr10/WpHXhs/HM5/Mwcbte5NQo+wkEa9YO3mioTgyeWXj9r0YPpEHkpdsNxonxs8qxCX3j0qyp7iLGsWM7JJYDyIMSLAjbPlkzDJ8NHqp6/Su9rB0cb2/i+1PfztpFd7+bpH//PXQ564BTWOxY7ewc6yoSu6yeNJJpY2djz1d00VoC3v3Aj+a84fenoHPx69AWUXiH6yKj7XYD0YuQW2dgpqauoTLNzPxd8072fmBaPcuPd4kIkhIsEszdu+pxJChI7Fg5Y7Q6mAWrvzYrvldmsiEj8cI9HJdBlTYJ26eRVyaNFmSSpSULsUiSQb4KWhEWE87EbvE6trgBCo//V97zvVJHOw87UCRtFoQYUGCXZrB14uwAWNmBLtkmCmks5ikF3jTxaYmmfh5FtkSQiEM54mgheJsMOy3IpH3K9Bn6yOkjBYaJZnvmKudSdJ6tCUSgQS7NCXMeTGJq6GW/Hfccnw1YYWvwlM1ccUmykjoS1DpSjYKEclGE1KStles92w901CFBD/CcywwcPw9W72pJJCPIld5pJnjhx8mz92A1ZtKwq5G2kGCHeFMCvYd/ObnlfjyJ57e00NaVy4J+BGys8ZuJ/U2dnYT7NadZfhs7DJPk75fAWHx6p2uPZszUSYIss5+9vnVBHiz88TiNTvxj1emYeT0tQHUyzmNthScLs+wdF8VKqvdm/3U1Nbjla/m44E3aFcbMyTYEXGEOSmTBizD0TSZ9eFWI1FSuhQL5wn2yY9+w7eTVmHrzrKk1+ehd2bgmU/mJL2cIPCjbUr2s527fDse+2CWpWBttRSrOR6t3VySzOpF0a9ApAPXDxuP+153L6Rp0RqqazN8sEkCJNilHenRycIjsyS7TLcls8Ndy4ypssWpJKW1d6Gxq60zTl7p9tqluj5BlDfs/Vn4+ff1CeVRrz6WHN1M+sSHs1GwYoflOxTVapsaEajtnYtMEo1gkAwK1S3q3FCjCnRNGucmqzoZS6OwK0CkIYkYJrvQPrgpOp0GGw39QBi1kwmzQmkIbSzuneiSmIu0cbc19H4SbgUSKZ1vKAbfUIwzj+npO4/YNoOSmlj0AW1sM3vFan0nCG9ZNyHyYnsJh/4S+ULzbm7ahAQ7M6SxIxxJZcdPt+UBGRHdHclmASab2+ZESveKdaGx89Mbsvn5pU3TJCZ2Th99Vpq5aBiUAAIXe9KYBzTUVlTVYsjQkZixKDU7z9TUCI1dHgl2cZBgR8QRro2d90ju0WuDrYp9/ukrd4aCJpPEvIXTZur1R0rDnThrub1UR1EUzFm2Le2M44MkXZYRZa4TTlWKaubq5Ro7P11nReFuY5+T5PH9lFVYu7k0liSBsVbGtl3C/vOrCSsCytEeTWOXR0uxcZBgRwRKwur9dHbBlzrAZbgAY4uf0DPZsUSd2gDFgqDe+ZmLt+KJD3/DiGlrtJwDyTc9Cbdtsg8Yp1BImsbOvORqddyJ35dtw31v/IKxutinMqXfxz8uw99fnhr9HfQ+tzkp7vuaYNwol8QYM3RH0ox0kGeCULb4trFL4Ppk37qYPU2srCCWTTIZy3elYd8WT7jZK9bqlOz4rtIKAMCO4vKE6uWGsDW06TBeAuZ62As4VmYcfpdiNU3Zph37dEfdOE+o9QnqHqb4XdBKyQl8y5bMhwQ7Iq2ILbH4CGMQdGVsmMeLAACbi5IffiIs/IzP0Yk+wyU7N5NTRVVtdFINoiw373yllz14XTyCRAPihiVYpctKv2yfX6d7EtVsxe/H57MS6tUeBRz9h2oQ5ESXmAPK0CXpItwDQE1tHYqKK8KuBgl2hIwERs2ABtx06qwydu+p9H3tnGXbUhKPLAxioRwSy+fbSSt1S4npycPvzMCtz/yccD5uNCfaubtfmaZek3hH+3XhZvzjlWmYNn9zwnmFhb9xIjipULbPbyR2UooWGiWovWJlFiJe9nn2KhBakXLtbZoI93pe/nIe/vTUT6itC7dyFO6EcMTL4JmopiZmo5d+6CfgROr3xIe/IRIBRr14USD1ShZenqQm0EXflQTHtc/GLgcAXHxKn8Qy8okbYWvVxpJAyoqFO7EuzGqulNZPcZEGsaW7Tdv3OlXRkTScY1OCdJ/fqK2Z/K4EHRJIlo8b4Spo54lU29hphKW9raiqxZ6yanRp3zx67Lel20SdwqlSFNLYEXGkh42d+wxSLQRGEElYoxj0YLRwZRFqVC+xdCDjl2LVvyl5t3wUlsrJrK6u3marp2Du0LotpRijM/53Il3eLz/OE1b4H1Piv4Y9bBUbnKNaqm3sQn4FHnz7V9zy9ETDMc0+MmzFBAl2aUrYL21opEkYA1siAQ6GAbBmUwkeeW8mPhy1NNB8/QzQfie1sCneU4lp8zbFn0jBc9Y0drbOEz6qEdQE+8RHv+GKB8c4FJZYGXe9NBXvfr/I/QUS27YwkGl2E62R38em1/h21mmRRJ4ytZ6/cqzICVgTme6s2VQad0zb/zfs6YEEOyKORPplop068K/IANG3LZ2qV1pWDQDYXLTPIaVH/DhPOHgEmqmuqcO6LfEDZKp57D+z8eIXBdhbLu4lAl6msmPDNrGNUrLe+URznbdih3XeIfWD6apdYIUXZ5IkYohjF3UgkhO04KPP77B+HQEATRrlWKaJHUuOEJI6r9j0lSDDnh5IsCOSgt84dmHb2G3cvhfPfTYnbm9OPZE009hp6v+cEOqkDa0/z9kg/uHRxu6t/y3EXS9NRXECzihO1NbVY/Qva1Fn80x3lghPtjrV6DmRsDteeW34AgBOz89LwBPB/JVFvuuU7pTsqwIAbN0ZzMfM356fhP+MXOL5OkWmsoODZBcwbuxBZVUJ+iM66hWbciO7FJeXAZBgl2akg7iQyBdXon0skXAnQfDa8Pn4deEWrHYwjE+H56SRtC9vH9fE5Dp3V/P1xQCAfRU1Pkpzx8hpa/D+iMUYP6vQMo05TIssjEXSsS0qvWevsJbfWjVvEkg+G7fvw8jp9l7YKwp346mPfosutwHyp+KksbPCa9+J1kE3ZlrHlYw/YXae2FFcjh9/XeupbD2xrdIaho2dLbQUS6QdCYRX0jq1X1f+wINmeq6A3Sn9Wmzyq+KWoCPI+ylbI2KxD6YV0Wj7SfzML6usUf9aL9tZaZhTeUuDKipZd1LqKJCkspw48dD9AQAH9WiXsjL//ekc/LZ0m1G7rN4SQ7iTkAYvu1KlGjv14MzFWwEAw96fhfd+WIxSVRvqufxUhzuJFhx/qHRfVUIhqTIdEuzSlHS2H7Bjwuz1AIBFq3f6uj4aNDMkY3G+QWiQ6iSChn67NN9bpiUBTYjODTgCuy/nCY/pc31uo+SWhSuL8OtCsSm5l/YEHQrCDfYCQQI1CUjQCDs2lx7tgyC1MlT82CQLVRPVvKVqKdZNGqmNnfg7flYh1m/bg33l4gPIb18MOoyLW2rr6qO7rWhcP2w8/vj4hJTVYduuMkNg4rBnBxLsiDgS6ZfGbW38Fx62DVtVtU3okEh6OU/Uh+iJZVWmWyFKuz5ZGrtH3psZDQbtqoQ4bXXqbmpOTgQjp6/BkKEjsU9z4nAgmdUbY1qWs7M7TfWHqNX7Vbh1TxLLtDkpfQ6W66LyLHw+TJkphuSz1DYP2/HOJSkXaNWCNmzbixuf+ClUR5pbn/kZf3rqp+jvsOcvEuyynJraOluj8aBxOyFZUR+CpkSG7S4ACL/j6kmrpViPQUqTrbHT42Zi1pK42b81aCIRYMLsQgDArgSWkeKeic983v1hseG3dBzxa1CWIFHDf1PrErERc1um3slFu9dbd5bpQl2401zpu+u+8mp8MHKxdeIEkUc7UWx/e0Vrd6J9ma/f7es6T9vtZTkk2KUZQU/Olz7wIx58e4anaxJZ0pQtYfqheG8VZqhLaE4kQ6Bx2lg6jeQ6XRy0NKqUy9cgOhmkwpXO5r229DlNqY1d/HJeWpGO75epSl620vJcpMRlXz9WLlgpwsL4uU2fjl2ObbvKfdVP/2Fnda2djZ0Zv2YmQdnY3fv6L67SmUtJxcdhpkCCXQNgeaG/L6Aw0G8g/exnc0Krh0xI0g9Y6TTHRcOdBG5j5/0a6w3O7UmFttGdLZIW7iT1k8TiNTtdC7h2t3fR6uSEOQnahjMRErHF9V2mzsY2Vg/JeR9KzERWVfT3Qvv3JC38ULRu1l6xcf/OUPvuoJQK2QAJdoQjqRzOUzGoTJ8v2WHAjN1SbCS9nCeUNNLY+V2ZS4UnnV0Rcbcumja193RzkbAHNAu6+p/3v/ELqtXt42S1m7Nsu/Tawq17ErJDapQbX1pozusSIUscT957JPXYlxQXDdLtoSpePmw2bt8r/QD4fsrqqNZv9pJtsWDbFvWMX7JP7Glq5bUMKASNI6b6p0TrnyGQYJemNFStsrndyRioX/i8wDGNbIhL150ntPEsEnBvTuTOu31sKdW4eGhRzJ4qOXVJhOWFuz05CeyrqMGu0grc+eKUhLTgOTlpOF2kVGNntKEDjO9U9HAS67R6Uwn+9vxkfD91ta5i4k9dvRINtG1GvhQbrI2dFouyZbPG0WOP/2c2PhzlPfCzHxrqnCkjDXsqEQZlFTWoqhFagDA7SNxgE1JdFABTCzZG70k6E67GzviAYlVIfEkxcGzLstD8pJME7xNFAW58Qnjs8QTMMuyE3OkLNvvON12prK5FTW2s/0fN+gw2drF/axovpzdG9s67fc22qxq5lRuKUbBiO1ZuKLZ8rY1OHrKlWN2/9XXxK5lGHY5i189dvh0jptkHft5TVo3rh43zUZyxTWRjF4MEOwIAcPUjY/H3l6aEXY20+epatGonXvpyHj4ZvTR6TB9WINHQBEESRLiTrTvL4u18/MSxS+ONwN3Z2Bl/p6tY5/c9atI4N+CahINVnMFE37tq3YfcFQ+Owd+enxyXecRKYFIPx5wI7MvyM4ZEHaVyInjsg9kY+tp0y3KCtrl1jcdil67dhdJ9iUVTAGgpVg8JdkQUzb4nFUtwltfHueCHgxa2pXhvfBT2CCK+l+iSIfDUS76UvVBUXIE///tnfDJmWWB1ctvM1Bq/23jFupyMg6S6pg41tXKjeaf74ree2SLYaZjvU6LLieZYfZrNGqAzeTCUp6uL6V/JsBfWhJdcK7WhDr1g5yrcSVDV9ZhPUGOpX+eJiqpaTJhdmPodM5IICXYhs2nHXrzw+Vzb4J9mvKTNONKkb9U57ubgU2Pnsz62eeq+4v2gbSG0eI1xtxA/dY3ONy4vTpex1Erzk0zB87J//oi/PjdJXh9dwdt3l8cF/vYbL1Im2A3/eaWra1P5rGYu2hK3m4C8Lql17QKsnSe0Dyu3fcDoZW+/bKqxUt0ZxyC0WaQ1LMVKzhvkoDTph4ngV2P37veL8Oa3C7FsXeZEj3CCBLuQefXr+Zg+fzNWbSgRBxzGqR27y3HJ/aPx02/rk1cp08CSysC3cXYSIc380RAiufLB0fctScZSrGSJyA9ubn3pvipTmASLvNJwpvBy64MSHJy0ANt3l0uP19crUW3eUx/9Fn/e5+3Naxwb8s1V27G7PC44bBjLW+WVNfj3p3Pw1Ee/YUvRPmwpit/NxkrwTuZwYbctl6iM+schSLesm7p9y0b9sjYuD8t4dAYB1NktNrBhPuDporqmTq6NM3vFenj4C1buwIX3jsTe8upoH5Rd/9VPHA+/4y0ObDpAgl3INMoVj2BywUZX6bUv918CMFYeN3MdPhq9NNStWMzECRfhVCMm2ElGO2Fj5y9fd3Zeird9TaN19Vcnt2zbVYbrh43HD1OtjaG92tily1JsNE3Ab5zf3F77ej4ufWA0AATaPxs3sl6KvfnpiXHBYYdP5NF/p6ovllWI9hbvrcJtz07Cbc/KtZpAMDKEWwcprf1GTVj8XfG1jO6xIW608w5RWQwfB0EIxF7z2FK0D9U1dY4fpJf980e8/EV8JINE+ur/Jq+CogBrNpVEn3+eRJv95YQVvvc9DxMS7EJGW+obP6sQG7fvdb4gwInw7e8W4Yepqw2DN5DoAJ7g1QmMMEF+rWtfb4alWL2dtG/nCec0F947ylNYCm3ZOFGv2Hjth7Gy2ibXc5Zvs84jenFCVUk9pgB8YWscubrk9sPU1ZZaPSD+Njttx6QXCNy8Lobg5gF0sJra+ujSv4w9ZdW6gLvWFbR6PvX1CuYu3+5pHNm+q8xVOu0D6relsfffLgSSF+cJJyHMjJNwaa6Qmxh8svqWVdRgyNCR+H7K6viTCVBRVYvbnp2E14cvcPWs3Hhda0oSN8S2P4vtk5vXJHvsTxu5ScQY6wvgXgDHARgIYAXnfKAk3WAATwMYAGAzgFc5529I0t0L4HYA+wFYCuABzvkkU5pWAF4AcDmApgCmALiTc15oStcPwBsATgJQAeBrNT/r0TCN0A+0ldW1+Fj1wmyortvmVptvw7R5m9Bjv1bovX8bd/kpCr4YvwInH9HNUz3qJLs5RAfQiH/52u2EM3PRVg95ir+RJKvstHtRV2fnhJA8w/FE8RegONFCE7v8I51XthsKt7r4OPRA0MPQc5/NwW9Lt2H0SxdJz1/36Di88o9TXNfLLPxNnbcJU+dtwiM3HYNjB3aVX2v67dY2VbvuxS8KcMqRB8Sdj777AXipO2Fot5Vcp/u3NNyJxYX647vVPYt/+m09Lj2tr+d6WlGiOqWtWL8bJx++v688zE2SBdC2Qr9DjhbSprEHwTDdcduSQwCcD2A1AKnrHGPseACjAMwHMBjAxwBeZYz9xZTuXgDPAHhLzXMVgDGMscNMWX4F4EIAdwK4CsD+ACYxxprr8moLIfC1ghAAhwK4BsBHLtsVOmYziKiRtNOAGuCA20IXUFKrh18S9op1uP7FLwpw10tTMWp6/HKgbPAqq6zF8J9X4sG3PNpJqFnJNvyORP9nzZaifRgydGRcINlkiDv6MCx6Zi7agiFDR0YDhzrnY3/eabuw9Vv3RG9LOn6XuFoGN/1NdHJO9m0w3+fGjYKVJvSTfBBt0Wu7rNijhr5wde8t0ui92Z3sBN1qumXvtP4D3Owz4eXd8boC4MZ5wsmWz7KPGpZovQupbpJqu1S0at44MBtuJ43dtl1lKNlbhbf/txDzuNjXV1F0S9LpGtvIB24Fu9Gc8+6c88sBzLNI8yiAeZzzmznnUzjnTwH4EMAwxlgOADDG8gA8AqHJe5FzPhnA9QDWAnhYy4gxdiyE0HcL5/wrzvkYAJcA6AHgRl2ZtwFoB+Aizvl4zvlnAO4CcBVj7BCXbQsV/UvtRkuXjHcvyMC2xw8SX8n7dWjukFJOvNAgvycfjIyPZm5nuuLbk9ji1jgF8dSWDsy2kMmJYyf+mp+jtnyyyWmJX3eZnb2RtvGA/j3Vt2brrrJoXmHLdcV7KqPaBjdky5ie67A7hNeubghi6+Gh7iguxw6bJWTbMrWlWF9XC8ora7CvvBrT5m3CRfeNwtad1sutrjV2uhsQFRalPgla/d23QJ9y3oodlmFwNAwWIm5sRx2kTQWKdDtAL0KqFy19eaX42GzetLFDSrvyjNi9+/P4Dtz6zM+44bHxGDerMJaHR3vmTMGVYMc5t33LVIHtdADDTae+hFhuPVL9fQKANhDLpVredQC+ATCYMaa9PucBKAUwXpduA4AZ6jno0k3inOutG78DUAWhNUx/9B00pCgm8cuf/l/0owd0AQD06NI6gRrp65LY9f9861ctJ28XOgxkTgOdFui0SWNTF0vCGBINWmqqVKNGomynSULPJz/qAzIbz0WXYi2srhVFv09muIPl/z0+AX98fILn67R6h11/vzi9l153FfBrEnLzUxNx89MTLc8nen+tAhRrfPzjMlzzr3HRDyu7LdicNEa/q1pGfY21PmDcUszoOJTI9/KE2YW25105T6gVGPPrWtzw2Pi48/Wmvit9JBZL3l6pqa3D4/+ZHX0OicbelGGXVeGWUulxRdEtnWfN511wzhN9ADRB/DKtNkv0V/8erP5dLknXEkA3XboVEoFyqS4vLZ2hTM55FYA1pnQZgWEQdfKs8ighvPPdQgwZOlJ6LkjTLD/j9QqdgXZC4RUkl2oDSSBeX+rfSCSCH39da5/WYuBK5lKsebBvogp21bUOXn+6ShXvsTZsd/J4/XTM0szdgStJFU++fGgswG95Uyy88o35xX7sKq3Au98vQl29v69Ru27upF3S1ySIZTwn4eJJLdyM7mZou7TInCcMJhsSpE03Ja528NT1IhB9OjY23bpZijVoJh0EaLes2liCucu3461vF6iFSCrkFVP9/bz79Uri1sDzVuxIMIfgceU84YJ26t8S0/Fi9W97Xboqzrk58qQ+3SY1nTkvLV173W+36RxZsiQ1GxUXFBjdtveUxr4kVvCYd2ppaWlcWgBYs00sL+3Zs1d63oqxMwul5QPApk2x5cKCggKUVRoHlV27drkua80GsfxSWlpiuKa8qh4zZs9BU7MGC8BjX26K/nvZcqPMP2/efEPd9Gi/KyoqoteW7mgirZd5uyyn9uzeJYTNoh1F0bTlZWI5Z8WKFdFdOqzy2rpNPNfNmzejoGBvNF1VTb3tdV7qqLFpkxBet2/fhoKCWNcqKxPlruCrgLJN0msBYMtuYe9SXl6O4pxY4NuFCxeiZbNcaTqtbus3xGKMbS4qQ4cWYphct24dWsM44MnaU6a7p/t25sWd9/KOO7Ft2zYUFMiXZ6ur1CDNS5Zgc8tG2L1XhNyora0N7DkF2RaNNWvWokl1zNFmXaH98ufefbFxY8vmmBbr5S/lFjb79sWW8QvmzUdj1UD9q2k7wTdXYsOmZrHzFu2THZ87t8DS2H3VqlUAYs9ElkepOm6uXr0KkXLrd7u4pAQAsHbtGjSp3gIAqDeNBYsXL4r+e/6CBdIxqqCgwCCMzps/H3mNc7B2W+x94pyjYnceqmtUm9aIvO1VlaJd+jmnaEeRIc3GTZtRUGCtZSwqivWtrVvldota2fX1uv1uFSVu3NTqXlsr3vlFixajdXPR77cWiz5fWVnp+P5uKBLtKisri0vL1bmtZM8+fPbDL9GPzr179mD16niPW+16vZBpznP1FmNfXrR4EVo3z8WCteVx16xcUyKt8+rVa1BVLdq4ZMkSbGklF4ns2v7GcHkEg2T0d7cEJdhlPAMHDkReXvzEEiQFBQXIz883HBs1bxawVXTSfv36AT+LDt6mdZu4tACQu3IHMHknWrduFXe+oqoWNbX1aN1CJ9x8aRz0Duh9MITsrDt2wAHAAjFQ5ufni3AE38cmiw4dOiA//0i4obLRFuDX3Wjbtm20fuWVNbjq4bEAIPeG09Wxf//+wPjYoHXEkUcA32yO1k2fVsu/2ZQpQEkNDj64P/p1bwcDanrxZa/EXWtVl44dOwCF5ejUqRPy84VfT/PpU4HdpaKOE2J1lOW1eOtSYNledOvWDfn5B0WffXllDfDtFld1kJ2fPHcjunVqAdazPX78dS2OOWQ/7LdrI7BoD/bv2hX5+QdjyZqdqKtX0KlDPVZu3oIePXsj/3Brr+C534uJbVtxDfr06ARsFAP+oYcdinatmkbTtdlYAozfgebNm0frtq1yHTCnJJqmQ4f2QGE5evXqhfz8Ho7t+fLXacCuEvTv3x+V1XWYUrARd199pO01GhNmF6J1i7yoXacx4/jJvkuXLsjPjznzb99djsqqWvTs2hp54ycCZeUYeMhAdO3YAlt27gNGb0Pjxo18PSdZ+XHpJGm8cuCBByL/0JhX4b7IJmCmdciT1q1i48aq3RxYbC08AECrVq2AHbsAAEcecUR054qxC34DNm9D23btgfXy/qlhaLd6/vAjjjDGDdNd17dvX2DaLjRtmgeUlcfnAWDk3JnA1iL069cP+f27WN7LNm3aAJsr0a9vX+Qfsh8AIOd/W4G6mLAzcOAgYKQQjo44/PCY3Zd5rNH9PuKII9Asr1F0PAaAg/v3R/9e7ZE7YgeAakQk9QaAvJ9+BvbV4pBDDgF+3A4A6NKlM7Ay9pEkxo1+8Q1S69ClSxdgxb7Yv5fF29FqZTf6YTuqVGEzJycSPZ43cRIAkQdjDI1m/Q5UVWPQoEHo2FYI7Ks3lQDjdqCFrs9b0XTtLmBiEVq2bIn8/HyDYMMYAyYWYVtxDb79dTeuOZsB2Ik2bdqgb9/ewPRd0rrX1yvAV5sNx6K02A5MjVlhDRo4CEvW7sKo3+bF5fPYl/LVqt4HHohGCxYBqMKgQQOxX4cWxgQu+njTpk2BPfFBtJ3uVyJUVVXZKqOCWorVNG5tTce1WXa3Ll0eY6ypi3TmvLR0+lHLbbq0xVIT7bQUK9Ef3/7CZFz36Djb62SBjZ3iL3lZ7YjFoIod21Pmfvuj+N0PnBXlm9XI9HZJfavbJVEFEvF2S3Rp7pWv5uHe139B6b4qvPfDYjz63kyd55oo68G3Z+CRd2eisUsbu+3FFloeta7bd5dbb2FlsUuJUzt37C7Hq1/PQ3llbbSoR96diUlz3AXqBoA3v12IZz753XV6M7c8PRF3vDgFQPJCU2SmpV4Mvd2Rvi2xZUd/LVRs1mK95OhkF+WmegZnAZeF3/3yVDzzye/SnSfsPEm37y6POnIYHFPMdXKoiBdnMGtTGw83J8H+YX5OZRUxraZd3lb3YcvOfVHbx2haeJtrohdleieVEJRgtwZANWI2dBoD1L8r1L/aOpss3V6I2HdaOqZzptCnW6H7vdycl+rI0ceULm0xBql0YRBrk0YLIGuHrJMHOalpDiD6dskielte72PrCTfOAUHaOrl5Tpb2KwHVQQucvHtPVTSERPFe49JETLBzF1kfEN6kZm55eiJuf2GKp8HdaWL6fupqTJqzMW4P1LQhQwb7OIHA5XVT523CF+Odh0jj1lWx3DV7Tr/96t7Xp1ufjNqoyV+48soaVFa7f6cBYzvMdrx+hNMtO8swa/FWSOQ6W/u/WYu3eC5Lhj6WpJdAyPpObCVbGwVOzTHLaw3tcWu7J6tiXV09bvv3pKh5UTStoqDeo82nAiUrY8YGItipDguTAVxpOnUNgG2IhUiZCeHtepWWgDGWq143nnOu3eGxEJq4c3TpukMEIR6ry38sgDMYYx10xy4BkGdKR6jIjY3NBv7+X3Sna52MgpPXxVLbebXSauvqsWh1zH4mKG9LbWKtqKrF2s1iGX3C7PWGNFpcJ+k+izr0E6h+I2z9VVahQ8w5u/1IKHMZWy/VaO9vTFGR4IyW6jnD4f3Sns9Lki2avKAZ7/udFNdvsw7BE+0jFrf+qofHxnbEcFzZMGqzAW9e4k7o+3NMW219T+yCexvztT+v19g55WjlaKGv56qNJVFtlyF2oWmZoqKqVpgpmFi5oTjuw9IOqwDTVnXUJ7Mazwwx6TyQqR7wdrjdeaI5YmFGegJozRi7XP09h3O+HsATAKYzxj4A8AWAEwHcCuB2zbuVc17FGHsKwDOMsSIIge8WCA3btVp5nPPfGGNjAHzIGBsKYI+a/wYAn+iq9h5EAOORjLEnAXQG8DKA4ZxzaSDlho5TRyreW5nQZCTrI/pDl/3zR3z37AVRWx3bxPE/Hcq2Th3EZuZW2a/bUop1W/bg9KO6x53TtCJDL+lqm0cyiC6X2bT9X+/OxIJVRZbnnTC3p0Z1DvGyvY8XXvqiAHx9Md5/6EzP19ruPGEKwBfUYJ/0HThM2e+xWjJX8SqoWm1BpR0Pol+Z0bKU1VSLfxath0Nebmone9SV1e7255WOd3aCXb3iKp3Te+P0sablH4lEDM/QqIGN/fvDUXJ7rajHvXrdYx/MwrJ1u+NspYe+ZqOBRfwHn9MHgVZ3swC4cFURtu2Sm44oUHy9j4m+wukYCcCt80RnAN+ajmm/bwLwCed8FmPsIohdJf4PwBYA/+Ccv6u/iHP+ImMMEIGEu0CEMDmfc77QlP81AF4E8DaEBm4KgCv0W4VxzksYY6cDeB3A94htKXa/y3aFRl1dPdZsNsbWWVa4yyK1Po34UvW7MbEshqNezb5qYwn6HdDWV96AOzu06po6S8HO3OG9TLB2KQOZfiwyueulqQDEzgs3DRFxsc31rlW/1AMTGFxko32p19kkthPqXAU+Nd2U0jLhGde6pX9HJKtyd5ZUYOo8/w4HtpOlUa5LCgtW7sDhB3VOYgnAByMcvPs9TkJmQbCouAJzV2wPNP6YZdmSIsxbrDlWIyoY2CXRCVvq33Wb7Z1KbIu0eYkMgp1tJvZlGDR2LrW0gPM2Y+ayY8pTcaVeow+IUFV2MQItcXgudfUKGuVGdDE6xfFH3p1pm6cbgddwidKANXbq/qyOPZlzPhYulkA55y9CCG12afZC7Cxxm0O6lQDOdSoz3fh64kp8PZGjTcuYB+vn4+JtXvaUVRu8XFeqm4P7JddhJDT6jvpAEnXd3HFsN/cOsI/py/War1bFcTMLcfOFA5HXONfxK/r7qaujgp1Vfma27NyHXSWVGNS3Y1ydE8Zs9JMElq41foxUqxo7za7Sysjb1tHF4ty/3rMZ1AMmGWP9v96bhe+fuwCNGwW32Xgq9+RVAAz7YCY2bt+HI5kQUL1OpO5LAmRTTkWlUZPm7Dwh8pq3YgeaNmmEQw7sIEnkq5KG/PX/jm5KIcnXEPfPptzPx6/A2cf2RLvWZj9DgVGws6qbNu5YLMValK0/rlg/CgDAfW/8YpGLPdruD3OWbceh6thnqINWsMslWy2pZ42dTrALamuzdCB7dr3NMAq3Cm1d6T7rpZPl63bjukfHYcbCmMFtohO/dKN4g6o+/ryX5Zto7Wwusf16TqR9pkuDmnNGTI2Ps+QXc/Nu+/ckPPRObB9bfZ2HDB1psM8z5OOiLG3SzQ3Y8nnt5lK8/d1CrN5YgpmLttqmfVMLSGpi5UbrDxSrtun3/0wZiZrYmRrj1sbKLc99NtdTes/NMTlP7C0TS6H1USEmeMHObueGOO1zxGlJUzDql7W6XWjkafSFu53jFdm/XZqEON25d39YZHnO4DxhkZP0qGTv67jrlPi8E9bQ2lz+4aillueiGjsXY5ii+HOESPwVTj+BkAS7kHDzdbBqk5j8lqyJLbt6+SKRaUucBLecSCQh4Upm92zOzm6QSKSP+QmVomfbrljQYf09qZFEmfdal5jyzNm2RM+YGevcF2pCe1dydPZupfuqXGtZ7No7bmahq9ACMxfFewHW1yu2e3daEbRnnp7o81GCXTI3E7YHnletxHrzMlvE8Md1hx0ydCS+nGBckVhRuBtDho7Ehm3GMuzMOfxoZByTSJ6J68ckkez0VTSPwUZvVqexwN25UdMtdsFxaoSLRu4scY60YCYIxVf8nAHsdbAfVRR/Np9JUTqHDAl2IbGv3L1XYPHeKjz2wSzsq6jxJFy8+338F59juJOENRTOam2p1tB0fex34nVxi9Wku7u0Eu/9sEi6P6QVI6evsaiU/LBsiyLZb+cTMbRJpZF6v8sqanD9sPH4yMJQ2isO+80DkA+ac1dst7/Ism2Je6iWV9Zg8Zp4+9RUrcKEPYlUVLlzCtAwa0m1rqv1by8T6Vc/ccNvLabmPO7svDNj4RaU7quSmHU4CEAu+qoxvIc35EKhOLZrTw0uuX+0wS7Uy9L1rMX22nDHukmOeX3PX/hceE+70Zglg6gjTSTiausu2f3dUmQdUkmBEpuz/FVRBPRPM2jniZCoqHIh2Knv6AxV6zF57gZPX/wFko4g05ZFDOddZy/FjcbOrog4wcZL2abUTmPoisLdKNlXheMGCo9VK03ixN83eKiFVd2Mf83U1NYjNzdHItj6lwSiS7Hq1k1lqkfhzEVbcOvFgxLSGAAul0ckD6HWIdzEtPnxQbSBxIUvBcBz/52LeSt24PPH5Wa5Wps320wGXsvUkwwvUi8sL9ztnMgCYbNlfAiJeFRrxD1XiT3Hs5/NwSEHdkALbVcIl/jtPm5tF/WPM2oWpv5jW4nob7MWb8GpRx7gKd8gcFTY+bzOD167bnS81CkKnMcrRZpm2AezHK9LBM9BkVMAaexSyIjZu1Ggaiscl0Qs3jUvgp1sEnGajO3s6eYu347Pxxn3ct22q8zUMSRfsKZjtk1PSENn+u0wid73xi94+uPYzgX6e+P0eJo28WcAbzmIaOEjXGos3dym+jrNPkV0c01wjQ2aLjKxIddGZRcNtSK9zv7mvvKVfN/SILRqhVuEbWu8mUIs8+XrduOZT+aYjvrEdAPqPAZQTTeC1NtEl1xNx7UlN/PzLiqpkI5/Mg2sp3q4cLKynvx116r/NodrsRoeEzMndnNxfBo3z0+Wd1h+BVFzkoiLECmQz3nVNfZ9LmwtejIgwS6FLFhbjsc+mA3AnTFq3PumuBsMFq/ZiXkrdkgHI6lAqd/9Ise6jMf/MxvDf14Z/b1uSylufeZnjNTZeEi9qDx0nLjO6+IrzXVeDrg1EJ7Pi3zLn1ZV0oKmBvm1bHae8Bp7zGnycOOUIXs+fpd1UuW1tlVna1m8twoLA9BKadTXK/j59w3YtMM6QG86MaB3++i/FejMKAKxo5Ko9wG89T9z5CtBbk5E+u7ahcBw059ciUiWAp+Liy3zjF08+hcLOzkL3PThAEzsosjGRqtdNKR23D77bsyRJgLHbyLFaqnbuqGGcCcRYaqxw2qLxQyCBLuQcPOeyzqeVYf+8dfYwPDQ2zMw7INZUsFm9C/xtl/GOEeyTimvn7ZctVwXfy8m1/nsyB7T241vdgPXlIL4PUndChz/HbccVR63NIot08jPX/uvcep583JyIkuxYiTUBLCoxk6ri++cBW7ul+z5+BbsfF0Vw+4jQP+Om6v3no13omOZpt/19cBrw+dHYx+GwYr17pdju3bUbYquKNFnEIiI7fACmsvIicgFOztc9R+DFk1R/xqTrN5U4nSpZyEvkf4nM7Oxyt8qQLFVDYZPXGm504yeNZtKHdP4xezEtKes2nkpFvL50YuAe8+r03HzUxNd1y9dIcEuJJwmt+27y7F6Y4nhmAL5QFVbV4/3flgcd1z2kq+WdEaD74SDMbIsf/3XnOxaLwsCcTZmTpWwC3Vgc+7lL+OX+1KhD7LTgr357QLXzhNunpH29bpx+148/p/Z8c4ffo1wVGy9m6NFyJ5+/HWlLkKZJMPbTkN7l/eV10j6pvj90eilGDJ0ZELla8J2kNtaeeW+1/3FHgOAHep+1EFoT50+As1l5OoC1kbTBNBr3Qx3ljsr6C6urK7F356fHDugmSOEJATIy41gzaYSWyeayXM3xm05J+vrqWiV/nk7yfSKooCvLzYcq6mt91RPt7a1aS7XkWCXKtbqdplYt6U0LoK3mc1F+zBtvjHKvkFtrCMuJIGK649b/VKshwE7KtgZJkJxLKJ7szztHuFtJda206aj7QQvLLY8N2H2+kAnAS2rz8evwNzl27FWtS/TljTMJR3er5On/GUfJ369mp/S2TomDV1dzO/5FjX8ysTf18ed037+EEA8w7DDnSSCf/MD+ZWyfUDtyInEC3ZBoK+fV0N4vSZv1cYSbNzuYYk92a+CJP+a2jrc/co0/PuT3237ZpVpT+/q2jrsM4UbSarAKvn2dN6GDNExTmNL0T7beurP2H0kmDcGII0dAQCGWE1O++rZIRNWrF54t8sWcRo71x5hIp3e1spun0c3eO0wxk24E8vLD7Ko6XY8/7l9QFnzI3OaFO0wp3n1q/mG4+YsWjY3ehwqANbr3luzHZDX+7t1Zxk+HRPuFs5WNT7q4C4AgEP7dYoTWBMJzmq+RWF7xYaBkwOQ27urKArWb/Vmm+j1HTVo3Fzwv8mrdGVZ1UH3b/1xj5Kd112HZLlrWntH72jTxcvW7cY1qrkIIByMUiHbGDR2Dn1H+qw9rEDZMfS16SjRrSqkey8mwS4E/C/DyDc5tnrh3Q5qTjZ2VtRFvS511+jc092UZ8Zc40RCcnj9uvfTWc3CEAAMn8jjjrnNO8g4fmbMAT65ydZK9szueGFK9N+T5xrtEu2qJntuT340G/+bvMrgnJAuaF7OOZEAouwbMN6H5GzBlRr0j9TLLXJssUVm5sPrt+11DFIbV3Yqb3fcMrGn5FIWrirC3OUikoJXhUAyP2zvf/OXlHw4G4RiFzZ2ZiJwuM8e2lBZHVu+dhN6JUxIsEsRQYzniiIX4qwmCz/agR9/XetaColp7GKvkexSu3d84m/rjWm9buKs+7dZKE1F35JtqfX5+Pg9f93eU3Od5/EdWGdaXkgUrYgH355hn87hBsrerxUmGxc9idqVWVVnV2kFvp200tPAv2N3zPPt20krDfZe5hXmSASGJbZEBm3zXqeZhO/JzEllZ0EQ9nNu4vYFNU7EZ2N0VhL/9lbYI+/OxOP/mZ1QveJrJOpqK+8koJOSjQtev5W0HPR5Oc1n1TXxDm3iY9W7gkN+3t6WXE/Yn28k2GUQVjZ2Vi+8exkp9sJOXyAPDivNX2JjJ3eesK7IhNlGwc6rLGq2+9DjeTAxLOsG70pR6SLqv+z56pd7AiGgmcyt7Yq52KCD2L/weQE+G7schRa2psZKiD/6zcs/G7vcaH9nqmAEEdslOr39bFxxphvxwJv+HRfSHa/ym3b8s7EWy/Mpip0WlHZlwUqrfZ1F/jW1ddiwLZgPBN/ox7UEi7d674O0g9TfI6e5QbYvdSQSnK21/nV0bGPIkh0Jdikj8Se9bVeZ9IWy1Nj5WIoF3NdUE+zGzyrEtf8aq16sLcVaZ6g/xePsRrwtRf770zmxfE3tSGRJLWi5ToG75+H2mXmM4mCgzEJr5LXJTnGlVm00PttYvKhgb265uqOGc1gDFzdNshQbMY2S5mze/k4ed02WNoNXYg3ItGmypq3cXGH5YaI9j/IUajFlzhFBPRIn7eCb3y7EotWJBVP2gqKIuWFXaSx0SUR3LtFdMOZbCbIBCHZaHvrx0CnfjdvjPVpzHMI8eKqph9isYXdzEuxSRBAB58fNKpTmY6mxczmLFBV72+hZ29dUL1DuVfe+1S9p+cHrxKd3b99ZWoE/PfUTtqk2XInsbxj4x3QiApvfuiTYBqcq2wqhiogJJatOsDZs3rCazPRtiRPsHOpr18/CHuCDxOgEIFPNxx/7ctoufCEzTXCBq9fE46t03aPj4g8m+SFpt2W5KRKCl2LNoa9clQsbT24fzldukXUH33FNdXn5cjyKONkC65IGaTcaMiTYZRiyzpaoQfbXJmN/pw7960IRcVw2sWvHbDX+kQjKK2uwq1QiUCrmn+7bNnnuRhQVV+An1W7PbT9Nxq4PfnFbh1TsOblT9nz0dbD9Eo4/p9lPBr0U6/aejZ1ZaHlOmzQikfgB3qm6dgJuOrxTiaC3i3TaeivopoYn/idnmTRuBz4PRfzj1WneC1SUuPArsViWri73RZD3zhjHznu+W3eWBRYzUi+cJrrPdrJpFG7xDYlgnrRVgOJkYtZYaDGONK9YDUXo9+OQdYLbX5iCnSXxgkP8lmLu6xlb6Yt4urRkbxU6tWtmOJaMpVhX6ZJkn+IKU5udYnr59Tr2vb2Qw3k32Vo5L+g/jsyaXidTBVuNXdgjfIJMnx+zuXWOI+bxfQjigQZAKj6SgNiezdFyk60phP+PDouh3F25sunI56M0Op54v17bwjMIDMqKNO/WJNiliKBsa2T5pDqEgjaRyTasl64kSg7KhDqR2Pjz9W8WWNbDvMSjD3g6duY6adgRGbm58d5rqUC2J6ZsIA5rDHGMG2XzPSHV6MjsLwMkEbtD7eNIeMXGO0/4LTfNx39PGJbFZO+px8ami9CbdAErGj3A5LWf5LdDUaz7qHDEc5GBD6TCpI+s1m/bg607Y3Zzyblf3vOsrqlDmWrXG1yuwUKCXYoIavCQhjtJssYurg6aYaupLgqMWjNFUfDpmGXo172dIZ3dNGnuvFoMJxnmJeQN6rJDTiSCd75zv79nWDZf42cVxh1z/Z4kIMS4vcBp4q3zKNEkyXfCm22MRZXrPWjsRCaxg3YfVmu3OW+VlikYDNlle3IGXF6oS7EB5KG9N1peidj8+kFRFEthyM0qj2+NnaLgv+OWo7kSe/e368ILucvDGEMzWejHg6kFm6wTIvY8//HqNIN3szzjBCuWICTYpYxgnrR+sl29sQR9u7dFbV2wb5Fs8tMHs9UG9TghU1GgtTMSEQ4V302JN961+9pJxB6iyOc+llGNXRJ7YyK2c9ozX7p2l8dCE0vuZm9GK2RCX+xIsBNcEB9N+onO3C7n98m6AttL7L/sMwmj84TsfLD9J1XfW+Z6+3FUcINZsKupSe4H+YZtex0c45JjJ1avAN/8vNJw7NnP5liklrNph0RwSrKw9IluV5y5y7dbxg90FOqQuuV9K8h5IkUkQ2O3bXeZeiz5Grt7dRuHaxoKs6ZCr7EDrAd6u44RhKGr1wkhSI2dfgsuP1i9J1U1dfjnW7/G0iVUijuclmLtzg97f1b8wQQ0dmN+XWvY0kdGIo8x+i5H4peSnGzs7Pp28b7MDUhsxhBTTKax8/hSWoXLSDU7SyoNv305Ktih3hfzUuxz/7XfXjBRHnpnhqsAzTJK91V53uVDIwgBXz/fhMFHo5fEHUtGbNNkQYJdighqItZPOtqLFrSNnflrwxxYVyvOzs4mWQbybvAaXiH6JR1A4XbLB27Gu0S3hzNck2CDXvlqnn3+njWCmjbX+7vx7g+LPV9jVQsZeo2d2SnIqb6KAqzZVILvJdrplZsrJVdkJkE7T1ja2aoEsfOEG8wmHYDYPD5RamrF/dDui/kD0q/gFBR2j2tHcQX++PgEX/kmYz/kJo1zUZMEk6PPxy2XHs+Nc2H2RtjmoyTYpYqAnrR+0smJAKs3ldhGvg8C824U9RZLscIgVxwbP6vQ4FHnljA2SU/F9OG2VTLbl1Qse/mhzqOmWEueYlMjA1a3LfpBAucPpTjHbUXB3a9Mw8c/LgUgdhhIJTMWbUlJOfp2m7VPycCN/F9Vnfi9lu1YsqKwOOF8xxWUJJxHJpKMoam6pg7DJ650TuiRnaXyDy+ZPWTm6OtIsEsZydDY1dYp+Mcr02zjc/nCobJaHeInQKOO6P0Rci2L3YBtJyzMWhy/L2sQvPntQtTXh20VIbjrpalxx+rrFXwwIn5pwIlky37vfu/eQQWIaSh27wnWocDVVmIqbm6J+ePCrGkxC9XmPC994EfX9QmCZz+dgwoX29Uliv6+VEtMJoJ+3xo1cp6eRk1fE2yhKhXVwd1PBcLBze+yaLJI1vCQHiNpYsg+XGYvcT//hH0HyHkiRSTDxu5Vh6UyvzhVdcS01dhZUhF1OtAoKq5IWJNkpwR65pPfE8rbimnzN+Gc43vGBfMMlARuy4rC3Sh2sC8LAq/Pzq+A9uEo90JqbV09GuXGT/DJCvETicTnbf4QeX34Agzq2zF2QJd8nMTTORVc+dAYjH7poqSWoX8/ZI48QT8RN7avyfp4mbEwOC3ogpVFqE3HveSSdPPCWHUJGplg9/Z3i/DLApfvBS3FNhAC6kR6D1jZV3MqKN1XjTEz1mGvKYDt8J/dqcr//Ym1h9SYGWsTqptfHnp7hqcQKV6prKnHGzYx+dKBdDQOvuT+0dLjK/xqPyzXYmP/NE9M5tsybf4mw4bjeg3F25LYhKliyNCRSc3faQgL2jygsQuNXU5u+r2zMtLBdCJVZIFch1zJxyQALF7jbq/fsG8BaexSRFAPOtm7TADuB6Epkrg/bi4t2Wet6dm2y1u8o2QRtJDz0/xSbNrpz1haZu9hdZ9vfCJm8OxVq5WK5Tw/yN5H2Rd1MM8sgrr6OtMR/wGKswkn54myimBDu7Rv3dQxTSps/QKhgbwjQHYIsam2kw0a0tiliCx4113xXwsvo0wj6MHJr1AHeBNYdumMgb22wS4YdJisksQVK97rz9vUjaw7ae5G4wGH299AurbjGHbrMz+npiI6UuU5myhrkuzg5oek2dhlQYdYuaEkoevDvgck2KWIVi2ahF0F1/h9KSebJ8QMZsLs9WFXIYpMKeFq+6wsGGABYOhr0+OOPWOznO+HtVvExBuJAPNW7DCccwxPnC032oHXhs9PaXnutojLjHuvj0GZ7TSU/mDHO2PD/UimpdgU0altM+dEBCEjDW3f0pXaunpUJhAC4+mP4x10Ig7LffbR/bOHZIdV8kO6mg9kAuWVybl3Tkv2DYG9FeEu5ZJgRxBpzg6P+yxqaFooGXrj/2whAmtni0TzJVKPG23ckjUet9kjkk7pvnADLxO0FJsyzHvnEUQiuJn07IK3ptNSc1AkS0+wbVdZknImCCIb6dEpXNMrEuxSxDy+wzlRmkA2EkQmkqz3toQ0EKFAwxCRqcgiGaS0/FBLJwjCHzTppQzzXskEQRB2hB2FhwQ7Ig6SGYhMJFkanmTtckEQRHYStl0uCXZEHJPmZE/YkmyFRI14GooJwdnH9gy7CgRBpDEk2BFxLF3rbtsUgkgnGoZY13Ci3zQUQZ0ggoYEOyIOWnlKf2jSk9BAbkk67ulLEIQOsrEj0o3Vki2cCCLdyZRdCBKloch1DeNpEkTwkGBHEERW0FCUmDkNRbIjiAwl7B5Kgh1BZCD1tF7eYAl70kgVm3fsC7sKBOETimNHEIRHvvqJh12FtGPrzoaxQ4TT3rXZwioyCSEIX5BgRxAZyKLVRWFXIe148YuCsKuQEpK9EjvsluOSWwBBZDlhW0uQYEcQGUhDsScj4km2jV2fA9okNX+CIJILCXYEkYGQXEckDXq5CCKjIcGOIHzQvUurUMsv2VsVavlEeJBXLEEQdpBgRxA+oLmVCIuDe7dPav6ksCOIxAh7eiDBjiB8EHbHJRouzfIaJTV/2tWEIBKEnCcIgiAIt5C2mCAIO0iwIzKesO3dCCKVkEKNINKbsL+9AtPpM8ZuBPCx5NRbnPM7dOkGA3gawAAAmwG8yjl/Q5LfvQBuB7AfgKUAHuCcTzKlaQXgBQCXA2gKYAqAOznnhQE0icgQBh/fC++PWJzSMmkjdiJbIcGRIDKbZGjszgVwvO6/F7UTjLHjAYwCMB/AYAhB8FXG2F/0GahC3TMA3gJwPoBVAMYwxg4zlfUVgAsB3AngKgD7A5jEGGsefLOIdCW/f2fb8+ed0Cs1FSEIgiCIkL/7k2GFW8A532lx7lEA8zjnN6u/pzDGegAYxhh7n3NezxjLA/AIhCbvRQBgjE0DsBjAwwCuVI8dCyH0nc85H6seWwxgDYAbAbydhLYRaYiT9oz1bIexMwtTUxmCyHBIY0f4pUv75miUG8HmooaxvV+6kjIbO1VgOx3AcNOpLyGWW49Uf58AoA2Ar7UEnPM6AN8AGMwY02bx8wCUAhivS7cBwAz1HBECRw/okvIynVdFadmUINyiUMATwidiLKbxNuw7kAyN3RLGWCcAGwB8AuBpznktgD4AmgBYZkq/VP3bH8BcAAerv5dL0rUE0A3AJjXdCs55vSTdOYk3g/BDJIRX2ilgazL2TCcTO4IgCCMRRJBDLpmhE6RgtxXAMAC/A6iDsKH7F4DeEEuj7dR0JabritW/WtTNdgCqOOcVNuk2qenMeWnpPEfwXLJkiddLCAklpSUpL3PxEnvHicLCwsDLLC83v54EkRpWrlyZ1PwXL06tIxKRPVRVVaG2NuxapAcFBQWhlR2YYMc5nwBggu7QRMZYKYDHGGNPBlVOshg4cCDy8vKSV8CXm5KXdxrRrm1bYPO2lJZ52KGHAiOtyzzwwN7AzN2BltmieXOgpDTQPAnCiUgEOOigg4DJVmbMiTNw4CDb/kQQVjRtmofGjXKA0r1hVyVUIpEI8vPzk5Z/VVWVrTIq2UrTb9S/RyKmcWtrSqNp8rSZtxhAHmOsqYt05ry0dMHO4kRa47QsSqFJiGwhAiR9zy/aeYLwiwIab9OBVK6GrwFQjZgNncYA9e8K9a9mWydLtxci9p2WjumcKfTpVoBoMDjZ2CVlnKGxiwiDBF/mRrlkAEUkF5LrwifZvfxqCCG+gHNeBWAy1HAlOq4BsA3APPX3TAhv16u0BIyxXPW68Zxz7XNyLITG7hxduu4ATlLPESEQRqd2+kJMhkNHTjI8MggiydCkSySbMBzoCCNB7jwxAUJwWwKgHsJ54m8APuScr1WTPQFgOmPsAwBfADgRwK0Abte8WznnVYyxpwA8wxgrghD4boHwqr1WK49z/htjbAyADxljQwHsUfPXvHGJBoLjZJWEcSaXZkgiBBJ969xc77QSmxMB6jNgtbZ1iybYU1YddjUaHBFSCodOkI9gOYA/QdjVjYCIWfcAgOiuEpzzWQAuAnA0hKPFLQD+wTl/V5+RGpj4IQB3ARgHEQrlfM75QlOZ1wD4ESIY8bcQmr8zOeflAbaL8ICd9uy7Zy9IeZkAhTshsodE37tIEJ0hQ17+p/5yQthVaJBkxtuRXHbtDdc1OEiv2LsB3O0i3Vi4WCpVhbsXHdLsBXCb+h+R5jRpnJuUfJ3nKlqKJRoOZx7dAz/P2SA950pj5+CdkSlvPhnxhwPdd2BHSU2o5ZPSNA0ZfHwvtGjWOOxqZAxhaOxIsCPSFTvBzNWc67DMmm7zdqPcNKtQAyfd3o+GCAl2aQjr2Q4XnXxg2NXwRTjOE07nk6Cxo9GLCIVIgh99Qby39O4T1pDzRPiQYJeG1NaZd0nLHFo0Tb2m0VFwo3GGyCL6dm+Lf/3pWMvzds4PkQhwzID9bPPPAL8IV2Rqt7//+qPCroJ/FKC2PnPnr2yBBLs0RFGAfZXhrtH75eqzWMrLJLmOaCho7/oxh9gLZ1YoCvDIn45xSGMv2qWbFUK2xVPusV8r2/OXnto3RTXxx5pN2bsjT9Mm7uzEe3dJ4i5WLiDBLk058+gentIf3Eu+Pe5RB3cJojquadw49a+Uc4Bi+/MtyZ4xaeQ1ycURBzYPuxopp1Xz5LxTbmSqehtJp6a23rE/OApKSTZDGP3SRUnN3yu3XjTQdVrWs51zIiciwBv3nmZ5Ol3se3NyIvjb5YcZBc0UVu2vlx2KT4ed45wwQE4+vJvhd67Fs7jipA6pqI4lJNilKb33bxNIPq1bNAkkH7eEISQ5Bih2GGwOOTDcTpjNRADkSYT9Vs3Fe3nWMd4+YIJk6HXJ28vxlCMPsDyXbMeo+jprySwIMw9zf/IqaNh9tJ53Qi/P9bESZP0o8i6RaMMuOOlAPHv7SfjqqfPw5n3WAhcAdOvU0kepwEmH7R/9dwTBfxhce07/QPMDgMP6dsTg43vhpiGH4NGbVdMABdivQ2o+5Nq0yEP71uadR/3hVng/oLPx+V5xxkEAgL9ceqjhePO8cEUrEuyyHNkXResWTZL2Vdy4UXJCmtjh1nmiRbPGGPXihbjstL6m8+7KCUoA7Nw+uQNf144tkpq/mTuuOMzyXCQSwWmHto473rpFE/zw/BDcccXhSayZNa1bNMGpRx6Ac47rKT3vdsnFEhupwosc9Pb9p3suWm/j1Lypc0SrCxN01BpyUnCOXieZNCJu0Mt1+mVMP3vetpF8COfkRHDIgR3Qsllj9Nwv/l3W41eZuW7LHtdp3bSrp2k5d0Bv+YpOIug/qLvpBJ5zj+sVeFnyCgSXlVkT55brzu2PkS9ciIN6tA2uMgFAgl2Wk8q9If9whL/OkSjOW4qpKAoikUhc1Hyn6x+5SdgklVXE7B4Tses5bqA/+yi3NG6U2m7dsW0zw2/9pBKJyDV27VrnoVFujqW257LT+qZEk9rfYunsDI+mEGYmWsSREyQwI+neVav7U1RcEf13n25tHbO88syDDL+dBAdz7d0IM6ccEdNg2i0VD0zwmV+XgGaqzwFtEn7ufj1CNxftw/+dJ7ZHb9e6KTq0aeZwhT3P/O0kw++ke/HrHmluBu5H3LpFE5x+VHfpuUF9Otpem5MTCUWhYUfmPYEsxEpr4IVO7eQDQa4kxpPbPj7sluM81eE+G2+uls3Eq3bCoV095ekGxx3F1ASN1QDJ5onLadBrp6r795RV4e6rj/BVRz11NktlAHD94P5o18q/8W2TFAt25nn6jisPtzzndLxX19b4+F9n44bzBqBZXmDx0y2pqpEvTd584SGJ5VtdZ3kuJ4HHo39VrQS7VRtLov92CjYsw6uJ3fkn9o5LY/640NfDTnDUPrL8yiF6odHp48u8JH7PNUeiaV5iE3Qi8tPlp/fDd89eEDVTkHH4QZ1c5WX+Xkq2XV5NrehHrVo0lq4S+f1Is4tRaNeiPw1x339vvXigrTAaXWa2oVdXe01uqiHBLg2444rDceP5AxLK44bBB0uPd2lvvSz3wUNn2uZ5aF/7LxUv/HWwcOK4/4aj0UKyPGSeSJ+87XjXeTsNpi3VgVLTKNabVHZO17dtKYSsft3bRQVoBYrtFmmP/9m6/v17tTdoMMyclt8dHdp4sx3RL7/Kvh69LMV4tQFrqbMHatsqD/17xsqqqva2tc4RrDM6tm2G3JyIr6U0r5xpYeOX6Be4+X0+UWdDVesg2Gv03j9+sjj72NhH4CkmDfkFJwkBy24s6XuAs+1uba1R2L3+XKMWTK/hHv3SRehk0tge1q8j/nHNkZb561tvXjKUlaHxxRODLfPUaK4Lt+T0/uT37xxXrya65965fXN0tOmH+mdhxklr/rfLDo07FolEHHfnefgma49mg1mD6f7ZjXFu7qsezVGvdcuYANpjv1a46syD8OAfj5EKY3q7Oy9C0N1Xx96jl/7+B9fXebLzc/yScZ9VukCCXQg8+Mejo//+6knRqS47vV/0mJ/pzGoguegPMfsXs+Hnfh1aYPRLF1l6zgYZ2LdFUzFg5eZEcLtuALrijH4Y/dJFuPgUo93b4QcZB10Zrw89FW/ee5pjPVs0bYzPhp2DP10gJluzYOeksWvcKAdv3Xca7jUZ2+sH4SNZZ3z/XEzQO8Lmy/rUIw/AvdcHa7j/xr2n4cBuYtKWaUW9LMmfeXQP9OraGh89crb0/BCTTVb/nu2jE5V2J6PLswm8Q7J+cCRzfi/MPP3XE3DxKX2k5/Ia5+ICicYJAIY/fR7+drm1/aAd5vdZb8Mj0+Y9d8dJUcFM4/iBxuf4+ePnGoy8e+zX2jCWdGrbHKNfusgwlui1wx8+fBae/uuJjnWvNmkxI5EIjuxvfd/N/e/YQ7rGPzzdb0WX/aH9OuGea+OFQJlQZuUINuyW4/DS3/+A+284Ckcc1AlfPDEYnw47x3Ycvf+Go+K174rQbGmCyYcPn4WPH7X2upRpzzTNmJOzwgCfGqymTeRa7Iv+0MdoAmFRLzOd2zf37GD31F9OwJCTD8SfLx4UKy8SwfWDD0bHts2QI1FJ33JRLK2dTa6ZZnmNog4pzfIaGYTQPNUO9o+SDxmZTD/yhQt1dTjcdR3080OmhNYhwS4E9J0sL1EjbQcikQi6dhDaHNmSif21yaiRe0NVq2XPvt3bYvRLF6H3/m3Q0+XXX7vWTaPqdvOA7GaZosd+rdFUtzRotqVp3CjHoOVJSCi2GDxusfHcymucG71uoMQmxMotX0b71k3xxr2nWS7v5/fvjC4mBxAtrprW7DvVgVOmdQKME/eb950WtW8xTOiy++DjtuZEInHPQ++1etul8doTQGh/nDSnfsItyDxTB/TugM7t7LUMbVrmeX6v9GV1bt/coNGyorrWKHial3PNH5E5ORHDx6ECJc6OTjH8O/YrAqC5ZMndbgI1a/laNmuMg3q0w8mHd0MkEkHrFk2Et6RNHicf3s1SK/vds0MwQicEWGH3KGSrEgYCFhBYj3aGvmHO3urj1esKbf+e7dCkcS7+fPEgyyVjmcauWZNc9OveFoC3sdEs4Ldu0QTfPHM+brtkUPQj73Ldh0z0Okle+nHeSzdKl/AyXiDBLgSSYchq11natRZLiVYvqNWSRdivs6Uhs8fPJvOtOdakCXF6HLLSzJOd287vOuyApFJul2dlNfFm0Ox8f51sfZxslfSPsOd+raXehjIjez/vZCRiXNZ9959n4OYLXcYmc7gVXgRmDbNXts+i49IcNyjeKafO7CkkwTx21JiWYs2PQeYgoF/6UpT4utfbCOzNPSz9f/jIWXj+zpNt6xcrxp/0lJMTcfVcZfdBG9sVAJedELwnql+sBTtv768bMw2re+dXltWqqPXhZnmNcMFJB9oLiA6Feem2+nu0dN0u9xeGCAl2KeLlu2P2Afr3MTcRS2odbt5Tq45g2QdSsB/qhm17PV8TtDZcJpRpHmpmrLze3C51XnO2y505ZEKNU2BZmzuT6o3SA9kvUtIcP5pQ80TTspncwFuGnQenLG83yJaOgPjupijOXVCzAbv2bIb9O8bHUDObHcho1byxQZtfXSNx/FCzufAPB8afg/F17dCmafzHou6n+Z568YTt3C5e62gZxy6MZTNNCKlXMKiXtQY2kapJ2xUxLnGbE1l9eHrtT27SS+e0SCRaJy9FKroyvTxPZ6Fet7zqkFJ/7+Ys2+6+EiFCgl2K6Nc9FlZB3zlSqeb1WpIsvdfwKUf274y7dF6SGprn5m9Lt3msVfCYv1pbNGuMP9g4NwDxg4yfCd4re8qqXaWTDb5BhyAwt79tyzwc3Kt91Ng5kiNPZ4XmZJGns1v0q3Exk5NjXIpNZClIlneiHJhAMHLWsz1e+vsfcKXFVn49usidE/ReipFIxBBgNW4ZVffzSNZZOjBo1xx7yH448dD94+6b0Ss2drxZXiPp8zDfk2NttlCzekZ2nsmJcPygeBvW5+44CQ/deLRBY2eHH693W6cDxeQRbDptZYMdkF7BgCwSAxCrk/6jz2kVomPbZjGNnYc6OI07siHAzbBgDkScrpBgFwJBLcUathFzkaVlsWonMAdZlKV32sfQzOO3Ho+zJN5jiUyIijxChW+0ieWmCw7Ba/eciq+fOg9d2jePDr6GicOi2vrBTLMB8rIVkRsqKr15mOppFOAIrijxk2lubg6ev/NkHKHavTg9XfP1eapRePcu+gCz8df56Trma7zk4aTwct2X1Xw6t2sWJ8g8pHo6mrVr/Xu1QycHuzsAOKhHO8sPi7uvOQK9urbGq/84xXDcSuADgOMGdsWlp/aNOXPEhQeKv+bKMw7CoX074u6rj1CXvo3njSuxsR9XmGLoaTz11xMMv+3scq0eUXebNnrJxxL1PuREhI3k8YP2j773ToJFm5Z5+N+zF3iysTZvM2YW1vTvj7l8q3vhdcXITRgR2cd/BLoxISKCYv/9qsOjbXj/QXmEhr4HtNXdU+ub+sD/GUNt1TnssCL7mHDySAaAwcf3wu2qQ1UqwjH5hQS7EAhqhbONzt1c/xVk9orU+kOOhUpbC/kwoLdxSUSq2ZD0rYF9OuCMo+XBHa3QjLplAp75i9gcly0S8Fur1aFZXm7UsxSQPyfN/qixafDSBsjXh56KZ28XwUEv/EMfTx5gg4/vBUDsFnDpqRKDYIfZwu50x7ZN8dCN9pu/a+zvYlskrSwr7zKvSzxDTu6N+284yhDkOqiltJyI/9Ap+usO72ftBelIREw+z90h7MNOPfIAHHFQJ1x6al90Vp1UzLZt+f27RDWYXiLb6+vcuFEu3rj3NPQ5wHi9zBGne5eWuOy0vmiUm4ObhhyC6889GGcc3R0XmTyKZRNgx7bN8PRfT4yGFiq3+QjR97E8i8m0VfMmeEUnjPoZM9u2yjPssHPducFsq6V5abZtmYerzjoIL98dq6dmL+hG25zXOBeKzZeDlXOCdoXZrEO/y4hM6JCZY1hp1/QxTD9THYT269DclbDc0mI7tPtvOBoXnNQbB+7fBrdePAhnHhP/wS/DzVhi9haukpkT6Ni/Uyw8lNZdrDyO9eTkRHDu8b3w2K3H4am/nOCYPizSV+TMYpzsdtyiKMDVZzGUV9YYjj/4x2MwZOjIuPTN1I5vnkBOPrwbTj68G0ZMW+1Y5p8vGYSnP/4de8vFsqDZ7d0tfQ5oC76+2BBC5Lk7TkIL07Y9r9x9Ctq1zsONT/wEQNgnnXjo/nH5aYx4fgguvn+04ZjT5KuNG27skbTlHfPkpmlMzHv86uPLmXdoMPPnSwbhijMOQsvmTXDyEd3w/OdzDeed3ptjB+6Hwq174pZ5br7wEJx/Ym80bpSL9q2bYveeyrhr+x7QBqs3lQIQGhuNa85m+OonHpdem7isbm1UuLA0bDfSuFFunFbmxEO7YvGanYZjfmz3zM/fS/fTP897rhPLzDW19aisqsVvS7cZ3oPjB3XFrMVbo7+bNMpBta6vnXRYrH2yfWpr1I+dS07ta3Cw+Pqp89BEsnuHFW7aJxPO3r7/DMPvFs0aG+KIaWzbVe6Y/wmHdsX7Ixajc9vG2FFSA0VRMPzp8zB/ZRGOH9gVn49bEXeNef7ue0Bb5PfvjIIVO+LSfjrsHPzx8QkAgEN6u7PR269DMFvtXXdufxzatyMOObBDXODd0/IPAF+/Gycd1g3rVpWgSeNcg83iFWf0M3jj2g05Hzx0Jq5+ZGzcce1DwKzN1QvvjRvlYOQLF+Ki+0ZFj33++OC4/Mwa57OO6YH8g7sYvJzbtW6KP188KOr57kT/nu3xt8sPQ5tIEf797RYA4tl27dgCt13ifynT7r3O798Z3Tq1xOaifQBEyJ03v11oW0cz2kfGgd3aYO3mUtu65PePDxF24P5tsHZLKbp1Su2WjjJIYxcC7VoJuwLN/duMl6Xa687tj1svHhQdFO1svbR4RVZBWbUtJo8ZsF80eLGmPTvq4C74/PFzcciBHQyBIv0IdUBMXd9VN9gO6N0hzjuyb/e26NCmGW66YADOOa4nLj+9n+VeqAN6t5faksnCSHymC1Oh3e9mLkJBaLEATzzMqFU8V9W2mTm0b0zL06dbG2kajUa5OYYQI6NfusjTnr7Xnt0fnz9+Ltq1bmrYTP3iU/pGJxOzkKPFaTvCIj6clW2TNshavaqa8NhetaH5dNg5+ORRXVw8F8LH+ScdiM8fP9dwTF/e3686wjkTyMOduKVbp5bRD4m8xrlo16opOrdrjh77tY5uAP72/afjnmuPjAvo7dWuUQsM3LpFE7RpGRPOWzRr7ClgctSWyaHJ157TH6fl29uSRlHzcuNlCwAd2jTD6JcuwmmDRH9WFBE+5sRD95d+aN115eF46774fXG1pTrzvdRv/u7FrMPOVk+2T6yMRrk5lv3lgM6t8NRfTozWz7yd3v+dN8CgadPGk1YSLVeLZo2lQeLPO6E3ht1yXFyAajPm+yLzaNXmi/Zq5ISuHVtIP5yHnHxgXIgju3IHH99LupWgH2IrTdbvXiQSMezg1LaVWOo2k5sTwUV/kMe01PaHlsUgffimY3C/zc5KAPDEbcfjyduOx7v/tA/8nwpIsEshD1/VDV89ORi992+N68/tj3/qgosCwH3X56Nj22bRmF53Xnk4/mWznYl56RSwF+xyc3Iw/OnzLEM9aB2nW+eW0a/bv191BG67ZBAevfnY6GQThG3BXy87FENOPhC9HYQdjUtP62cbVPLjf50d3e3BTYTydq2b4tC+HTH0unxce05/XH56v7i9bo8fJAY4fXu7d2mFb545H6cfZRSOD3TRjjtNTiSP3XpcNHq9m51HmMW+pgP7iPcgJycSfUZ/vUy+BHymLoRMs7xcnH1MD/xpyCG4+iyG14eeahS+ABzQpRW6d2mJq3XG+U0a50S1glbhD/Iai3umCbPt1f0vtWUst4o3vYCjp1O7ZoYPFP1+p41ycww7HzRpnIuObWOCQFMHuybz7gz3XHsk3nngdMsYcN27tMJp+d3jtDf6pa+uLrRFJ6gT6jED5AHD3ZITEfa3D5rGFzPXnM1wz7XuAmVfqgZc7tOtDf6qBqN20880ecy85PfYrccZTAPOOrandJlP26VDFuvODw/deIwhRp32AXTHFYdFtw4MEm3pHRABr81oO0nceaX8I0Ubq1vq+pkWN9DNx8pNFxyCoyXvk7abiGZ28e/bT0LblnmG+I5B8PwdJxs+Ms1o5bXSCdUnH94Npx55QOzjOxruxL4sTbB77FaxjCxb5h/xwoVxZghHMCHInXdib5xxdHdcdno/dO9iNEc5bmBXnOwgSLdpmecqsH5KUBSlQf83d+7cXnPnzlUqKyuVZDN37lxf123cvkeZt2K7UlZRrfy2ZKuyt7xaWbu5RKmvr4+mqa6pVW56coLy/ZRViqIoSsneSqWqulZRFEV545v5ygX3jFDWbSm1Laesolp5+csCZW9ZlWOdtu0qU2pr66TnVm0sVh59b6ZywT0josf8tt0vF9wzQrnv9enK+z8s8p1HbW2dsntPhW2aucu3KRu27bFNc9u/fzbcCz319fWO1/++dKvyn5GLFUVRlMWri5QL7hkR/a++vt7wHuhZs6lEWbu5xHCstq5e2bRjr3LL0z8pMxZuti3XjL7M6ppaZeJvhZZl19fXKzMXbVFqa+sMz762rl559/uFytad+1yXO3nuhmjZS9fuVPaWVysVVTWKoijKzEWblS/GL1dqa+uiaTRWFO5Svpu8UlEURamrq1cmzC5UyitrpGUMfXWa8ta3C5QL7hmhfDlhheu6mVm3pVTZXLRXURRF+e/YZXF1amjMnP278spXBcrOknJf1+/YXa68/8Mi6VizZM1OZcmanY55LFmzU7nt3z8re8ur485VVNYoo39ZY/keJ4L23tfV1Su1df7znzJ3g+U49Mg7M5QL7hmhlOwV89cF94xQbn16omVeKzfsVkr3ifF99uIt0X4k4+ff1ysvf1ngu95uxvy6unqlQu2T2jxlfk7T5m2UHnfDlqJ9yoVD5X2wYMV227HXbmx1ItnzXWVlpTJ37lxl7ty5vRSJXBNRQgn2kz4UFBT0ArBu4MCByMvzv/G6y7KQnx/sVlJuqK6pw9K1uyyXD1JBWG1PB8orazDr93k44w/Om0m7oaKqFne/PBWXntYX5xzXK5A83TBq+hoM7NPRlXZSTxDPvrKqFjuKy9FDEshY45f5m3FQz3aul4xkbC7ah64dWgQSxkRRFMz+fS76DxgUNb9oaDTkfh9G20v3VaFJ49y08NhMl2dfXVOH+nrFsHNQskl226uqqrBkyRIA6J2fn19oPh/+0yeSTpPGuaEKdQ2d5k0bo22L4Lpas7xGeM8iPEAyudDCNiUVNM1rZCvUAXBcKnFDNxcewW6JRCJo0iinwQp1ROqxMl1oyLgJY5JtkI0dQRAEQRBElkCCHUEQBEEQRJZAgh1BEARBEESWQIIdQRAEQRBElkCCHUEQBEEQRJZAgh1BEARBEESWQIIdQRAEQRBElkCCHUEQBEEQRJZAgh1BEARBEESWQDtPALkAUF1dnZLCqqqqUlJOOtKQ2w407PZT2xsuDbn9DbntQMNufzLbrpNXpNtq0F6xBQUnAfgl7HoQBEEQBEF44OT8/PxfzQdJYwfMAXAygK0A6kKuC0EQBEEQhB25ALpCyC9xNHiNHUEQBEEQRLZAzhMEQRAEQRBZAgl2BEEQBEEQWQIJdgRBEARBEFkCCXYEQRAEQRBZAgl2BEEQBEEQWQIJdgRBEARBEFkCCXYEQRAEQRBZAgl2BEEQBEEQWQLtPJFkGGP9ALwB4CQAFQC+BvAA57w81Iq5hDF2BYDrAOQDaA9gDYB3ALzHOa9X03wC4I+Sy6/gnP/PlN+9AG4HsB+ApRD3YpIpTSsALwC4HEBTAFMA3Mk5LwysYS5gjN0I4GPJqbc453fo0g0G8DSAAQA2A3iVc/6GJL+Mabtal6kATrE4/SDn/FnG2GMAhknO38c5f9GU3/8BeAhAL4j36AnO+XBTmsYAnoB4n9pCRFb/O+d8gd92uIEx1hfAvQCOAzAQwArO+UBJupQ/62SPIU5tZ4zlAhgK4HyIdjcCsBjA45I2FQLoKSmmE+d8py5dWrRdLcPx2Yc1xoX97NU0drsYHM85n62mmwr5eHE053yuLj9XfZwxth+A1wCcC0AB8COAu/XvUSK4mdvUdBnX50ljl0QYY20hHlwriAc5FMA1AD4KsVpeGQqgCsB9AC4AMALA6wCeM6VbC+B403+T9QnUF/8ZAG9BTBKrAIxhjB1myusrABcCuBPAVQD2BzCJMdY8qEZ55FwY2xUVWBhjxwMYBWA+gMEQguCrjLG/6DPI0Lb/DfHP9G313FhdugpJui/0GTHGLgfwKYAfIO7TzwC+UgdNPa9ADI7DAFwEoBqi/fsH1io5h0A8l9UAlskShPGsUzSGOLW9GYRAvgDATQCuhpjgJjLGLpCk/x/i34cSU5p0aTvg4tmrpHSMS5NnD8S3+XgAswHsADDXlHaGJO1yUxrHPs4YawRgPIBBAP4PwC0ATgAwijEW8dFOGY5zW6b2edLYJZfbALQDcLj2lcEYqwXwBWPsSc750lBr544hnPMi3e8pjLGWAO5gjD3COa9Sj1doX24yGGN5AB6B+Np5UT02DeLL/2EAV6rHjoXoGOdzzseqxxZDfE3diJhgkUoKbL4SHwUwj3N+s/p7CmOsB4BhjLH3Oef1mdp2znncQM8Yex3AYs75It3hertnr/IkgG855w+qv6cwxg4G8DiAcWre3QD8BcBdnPMP1GOzAawDcDeA+xNojhOjOecj1TI/AXCUJE0YzzoVY4hT2ysA9OacF2sHGGM/ATgIYtL50ZR+u8NYkE5tB9w9eyD1Y1w6PHuY26wKHkcAeJ9zXmtKXuJwj9z28csAHAZgoNZOxtgWCMFxMIwfln5xM7dlZJ8njV1yOQ/AJJNQ8B3EV4JZU5GWmF58jfkQauT2HrI6AUAbCJWylncdgG8ADNZ9hZ0HoBTia01LtwGiQ5/nqfJJRu3QpwMYbjr1JYQ6/kj1d1a0XV0eOBrA5x6v6w2gP3TtV/kSwNGMsU7q77MhNreO3k/O+V4IwSGp7dcvvcgI8VknfQxxajvnvE4v1KnHFAgNnh9Natq0XS3btv0eyLpnb8EVAPLgcRxQcdvHz4P4gFyqSzcTwHoENBY4zW2Z3OdJsEsuB8Ok3la/AtZATHSZyskAdkOo4jX6MMZKGGM1jLH5jLGrTNccrP41q+WXAmgJoJsu3QrJgLMU4d2zJYyxOsbYOsbYMHWZAAD6AGiC+CUMbTDS6pvJbddzPYB6iIFNTzPG2A7GWC1jbAVj7HbTea39VveJ6dJt55zvkqQ7iDEW5ngV1rNOyzFEfRYnIL6dAHAdY6ySMVbGGJvAGDvSdD5T257qMS7d2q9xPYCVnPPfJedOYYztU5//r4yxM0zn3fbxuLbr0iWz7fq5LWP7PAl2yaUd4m1LAKAY3rRdaQNj7CgIO5tX1K8SQHzl3AvgYgibgE0AvmbC+UCjHYAqznmFKUtNE9Bel65EUnQY92wrhB3IjRB2dj8A+BeA/6jn26l/S0zXydqUaW2XcR2AaZzzTbpjqwE8AGEDciGAWQDeZMKpQsPLfTKn0dI1hhgkwyKsZ52u78SdEAL5S6bjowDcAeAsiCWlAwD8whgboEuTiW0PY4xLp/YDANRlyJNhsqFVmQaxnHoegBsARAD8xBg7XZfGbR9Pedslc1vG9nmysSNcw4SX0ncAfofOwJRz/pop6UjG2GQI+6lPUlbBgOGcTwAwQXdoImOsFMBjjLEnQ6pWKDDGjoP4gn1Gf5xzbl6OGcsYA4AHGGMvcM7LUlRFIkUwxk4B8DyAFznnv+jPcc7v0v38hTE2DsAKAP+EMILPSLJ1jPPBtRACW9wyLOfc4B3PGBsFYCGAx2ByMkk3rOa2TIU0dsmlGMKd20w7CHVvxsAYawNh5F4O4ELOeY3DJd8C6KGznyoGkMcYa2pKp30V7dalayvJL13u2Tfq3yMR+yJra0oja1Omt/16AJUQHo9OfANhp6JpabzcJ3MaLV0NgH3uqpoUwnrWafVOMMYOBTASwoPwAaf06pLbZIiQEhoZ2XYJyR7j0rH91wGYxTlf65RQXT4cCffPXt/HU9Z2m7ktY/s8CXbJZTli6+8AokbYfSC+YjMC9YUdBaAzgHMl9hFu0OwPDjYdHwBgL0T4BC0dY/Eu7QOQfvdsDYSrvqxNQKy+Gd121abwKggPuj0+srBrPwBwXbrOjDHzcsMACJueoIzc/RDWs06bMYQx1gdCgz0PwA2qA4UfMq7tLsnaZ6+WfThErDs/ThMabvt4XNt16QJru8PclrF9ngS75DIWwBmMsQ66Y5dAeBQF4a6ddNRJ/RsAhwIYzDlf7+KaCISL93qd59FMCI+gq3TpctV043WTxFiIr5VzdOm6QwRqTId7djVEsMwC9Yt0MlR3dh3XANgGMQECmd/2cwB0hPsB/WqIEBlLAYBzvg5iQDIbm18DYI7uHfkJwjkjej/V8ANDEPKzD/FZp8UYoi5V/QTR1os559Uur+sI4AyIILQaGdV2GSka49Kt/ddBaNXMXqJSVGHkYhifvds+PhbAICZCImnpjoMIbh5I253mtkzu8xFF8fvRRTjBRLyfJQAKIeJ4dQbwMoQr89Xh1cw9jLH3APwZIr7QL6bTyyDUw59CBF5cDfHi3gLhbHCD3gaLxYI4PgjRKW6BiFd0LOd8oS7djxBxkoYC2AMRpbwdgEE8hTt2MMYmQHTsJRCD0WCIoL0fc85vVdMcD2A6hJ3NFwBOVOt7O+f8XV1eGdV2PYyxryCM4bual+AZYwUQz59DeJBdBTEBPMI5f1qX7gqICeHfACZCBCb9O0RMp3G6dG9CGF4PhQhtcC9EbK1BnPMtSWxjc8RCDtwO8XV8j/p7Dud8fRjPOhVjiFPbITwEZ6nHrwewXX89j+08cA1EoNdxEFqKXhDLtd0gdh+IahvSpe1qOU7tB0IY49Lh2WvCjuqtugEiptuFknxOhgj0+4Na3/0g+nc+gLM451N1aR37uCp0zYVwqHgQwh/gBYh378QEtMX6OtvObZzzPZna58l5IolwzktUj6DXAXyP2NYgyQy0GjTa18XzknOnAVgE8bXyCMQLWAPxYl/IOR+tT8w5f1E1rL8LQBcIjc75+hdf5RqI3R3ehvhKmQKxdU+qBZvlAP4E4dnXCCKa+AMAXtUScM5nMcYugujU/wdgC4B/6Du9mi7T2g4g+jV9IYBPLewqV0N4wnVVfy8F8CfOuWErNs75t+ok8hDEQL4GwLV6oU7lHxB2Nk9BxIaaA+DMZAp1Kp0hbKb0aL9vAvBJGM86RWOIU9unQgSLBYRtnRltaWkdRFy7lyEmq1IIT8nL9UKdSrq0HXBu/yiEMMalybP/RP33qRAC+j2QsxXiw+4ZAB0g7NVmAziVcz7DlNaxj3POaxlj50JsKfY5YluK/T0IoU7FaW6bmql9njR2BEEQBEEQWQLZ2BEEQRAEQWQJJNgRBEEQBEFkCSTYEQRBEARBZAkk2BEEQRAEQWQJJNgRBEEQBEFkCSTYEQRBEARBZAkk2BEEQRAEQWQJJNgRBEEQBEFkCf8Pn/9H/koRoC8AAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "df1 = df[df[\"name\"].isin([\"Latency\"])]\n", + "ax = df1['issue_to_done'].plot.hist(bins=BINS, alpha=0.5, figsize=figsize)\n", + "ax.set_title('Inference time (usec)');\n", + "#ax.set(xlim=(0, 25000))\n", + "plt.xticks(rotation=60)\n", + "plt.show()\n", + "\n", + "ax = df1['issue_to_done'].plot(figsize=figsize)\n", + "ax.set_title('Individual inference time (usec)');\n", + "#ax.set(ylim=(0, 200))\n", + "plt.show()\n", + "\n", + "\n", + "# df1['issue_to_done'].describe()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAApwAAAFKCAYAAACwxI8KAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAAA8y0lEQVR4nO3deZgkRZn48e8A0oBcw+24ooL4Ao6iDKyKuIsiIqDIKoegIigs7iIeC4oiIiIiyqh4gNeq4G9REF1PEA9AVJDDwUWG4wURmMVZEbkVbJCZ3x+RNZMkNdPdNZVd3T3fz/P0U12ZUZmRkUe9FRkROW3hwoVIkiRJbVlh0BmQJEnS1GbAKUmSpFYZcEqSJKlVBpySJElqlQGnJEmSWmXAKUmSpFYZcGpSi4hjI2JCje0VEQsj4nODzofUi4g4LSJuGXQ+JqqIeEp1jh8w6Lw0RcS0iLgqIj446LyMRkScHRHfGHQ+ND5WGnQGpKYxBJAHtpqRCSIiVgPeBfwsM3824Oy0KiLWA44EXglsDDwAXAF8MjPPHWTexiIitgSOAZ4LPAG4C7gRuDAzjx1g1iaMiHg58GbgH4G1gbuBy4CvZOa3B5i11kTEDOBfge9k5v+0sIp9gacBn2xh2W34MPDriNgqM68adGbULgNOTUSvb7z/V+B5wBsb0y8B/gs4cTwyNUCrAe+v/v/ZAPPRqogI4HxgPeArwBxgOrAfcE5EfCQz3z3ALI5KRDwfuBCYD5wG/AGYAcwC3g0cO6i8TQQRMQ34PHAw8Fvg05QyWh/YFfjviHhtZn5tcLlszQzKuXwL8D8tLP+dwDcz888tLLvvMvPKiPg1cASPve5rijHg1ISTmf9Vfx8RLwH+sTm95u/t50ptiojHAd+kBJj/lJmX1+Z9HDgDODIi5mTm2eOct8dn5l/H8JGjgb8C22bmnY1lbdjXzE1O76AEm58B3paZC2rzPhIRLwMeN5CcTWIR8Rzg2ZTjbzI5CzguIg7NzPsGnRm1x4BTk1pEHAu8PzOn1abdAlxPqfmcDTwDuAl4a2ZeEBF7AMcBTweuBQ7OzDmN5T4dOB7YEXg8cB3wocz85hjytg+lNmMTIIEjM/O8Rpq1qjR7AhsBt1Fq907IzEci4inAzVXy90dEp6bzdOBjlBqiPTPzW9Xyotr232XmZrX1/D9KIPfk2rRtgQ8ALwBWptQovi8zL2zk8QnAB4GXUwLC3wOfyszP1tLsQKnV2w94KvDvlJrKi4FDMvN3IxTXq4GZwDH1YBOgKodDgJ2r/J7dWOeL6k0NamV2YGaeVps+4j6t2uV9pUqzB7APsEH12RuAwzPz443yeRZwFfDvVZlsClzbDDarbbm98dndKcHXcyg1fH8EvkHZD3+rpTsNeA3lmD0V2AG4DzgxMz8VEc8APkW5E3An8N7M/H9dtuvFlLJ+DTAE/BA4rJmvbiJiP0qwOBP4G/BT4F2ZeXMtzdOAE4AXAutUefkV8JbM/L+IWBU4inI+vKMRbHbKqHmOrFctc3fKrfffUZpYfLGW5imUff6eqlyOoJxPlwBvAuZV8/6Nclz+BHhjvSawdt34KHAS5boxDzg+M786ivJZ6nlSO14BvhIRX6n+/0CnmcUyXnf2AB4BLmjk61ga18hq+gGUY+KpmXlLNW3rav3bAmsAtwM/B/41Mx+s0kwD3kK587QZpby/T7m+/bmxjp0o5b4NMI1yDn02M/+zluwnlOv0zlTntqYmOw1pqtoE+DpwDuU25trA96ovzU8BX6O0sdsEODsiVux8MCK2oLQleybly+dwyhfn2RHxulGu/wXAZynBw3uBVYDvR8T2tfWsSvkCOoDSNOAtlC+LYym3HAHuoHxJAnybctvp9dX8uZR2b/9UW+8/AQuAp1VfgB0vpHxxdNb9z8AvKEHBcZR2k0PAj6svxk66DYBLgZdRAp23Ves9NSK61aS8C3gV5Qvkw5QA6IwlltJir6heu36xZ+a9wHeBLSJi01Es71F62KefBrYGPkQJgm+kBE7d0r4OeIhSUwPldulzImKrUWTtQGCYcky+lbL/30G5Fd+0AnAu5fbzOykBzScj4kDgR8CVlP14H3BaFfw1fZJya/844AuUIOXHEbHy0jIZEe+mHKM3U8puNrA9cHFErF+leVyVj+2BUyg/Ok4FNqTcSoZyXqwLfC0zR7wzERGrUM6RA4Ezq+2+HfhClaem1wCHUWpPP0Y57s+m/FB5BWXff54SFH68y+c3Ab5FadpxJHAPcHr143Fp+RzNeXId5ZoDpew75/J/V8tY1uvOdpQfOg+OIm23bVifEvxtWq3/LZTjcCYl+O34LKXsLqu28wuUH8wXVvurs7zXU46HDavlvQu4HNitseprgQcpx4amMGs4NVVtRqnR+wVARFxHufh9GdiiUysTEfdQvoBeRKmxgfKlPB/YpnbxPiUifgycGBFnZOZIHZtmAttl5q+q9ZxG6TRyIuULGUpgsTmwdWZeX037QkTcDBwfESdlZkbENykX+d92aW5wMY8OOF9IqbXaoZp+VkQ8CXgyJXiqt6H7JbBTZ1uqnvW/odQmbVct73hKIPrMzLyjmva5iPgicFREfCYz76mtfxVgq8x8qFrm3ZSgaGZmzl1KeW0J3JuZty4lzVW1tDctJV03Y92nfwF2aARFXwU+GxFbZua1ABGxAqWjxjmZeVeV7qPATkCnfdovKEHT+fVay8prM/OB2vvPR8SNlP3/zsz839q8xwFnZeYHq3V/vdqmLwGvz8wzquk/odTUHUD326s7ZOZwlfaa6vP7A//ZJS0RsTGl5u7YzDyuNv1M4BrKcXwUZb9sAuzVqJE7vvb/ltXrb7utq4t/pZxLB2Tm6dV6T6Wcy8dGxBcbNcn/ADytc0xWPyTfQ2kH/ZzMfLiavgHwmog4pBGgbQbsl5lfr9J9gXJOnBQRZ3erka1t40jnye0R8UNKsP+rLk2ElvW6sznlLkWvtqP8AN05M39dm965q0JEbAccAryhXusbEedRjvP9KdewNSlB/5XAC+tlXF1/FsnMv0fE/7L42NAUZQ2npqobOsFm5bLq9Wf1W4C16ZsARMQ6wEsoNZOPj4j1On/AecATKbc1R/LrTrAJUH0pfg14QURMrybvTQn6/txYTyfw3WEU6/kF8Kzq1jyUIPMCSm1LJxB9YS0twFZAVPlZt7beNSk1HM+NiNWqL4Y9KbXECxt5/DGwKqUXdt1XO8FmY52bjLAdawD3j5CmM3+NEdI9So/79ItdauDOotRG1js37EAJchbdvs7MCyhl/gNKsHR49f/tVW0ktbQPVHlcISLWqvL0S8rtx627bM5/1j57D+XW9N8otfmd6UmpmetW5p/vBJuVr1ZpX94lbcerKJUTZzXK7l7gasqPNSg1qwA7R8TjuywHyjEGI+/rjt0otfyLgrPMfAQ4mRLgvaSR/luNH0Cd8/u/OsFmbfrjgCc1Pv8nFtdUUwVK/1mle1a3DPZ4njSX0Y/rzrqUOx69urd6fXlVW93N3pQfY+c18ng9pea5cyy8lLKvT2zWuC4haL6b0tRBU5gBp6aqefU31S1ZgP9tpOtM7wSBT6N82R9L+aKr/32sSrPBKNZ/Y5dpN1SvnXaUT6fUhDXXc+kY1vMLynm8fa0m8+fVXz3g/FOtFrXzxfWlLut+W7W8dSltCqdTRgdopuuMndfM47zG+84X4HSW7n5GDiQ78/80QrqmXvbpY2pQM/Nu4HvAfrVamtdRhjw6p5H2ksx8JaUpx7MpNY0LgS9HxIs76SJiZkScS/kSv6fK00XV7LV4tIcz8/8a0+4F/tCl5u1eupf5o47LKqi+GXhKl7QdnePleh5bfttQlV31Q+7jwEGUH1E/jYi3RcS6tWV1gtLR/mh4MqU98iON6ddVr818N4+/0Z73HTd1KcvOedtcV0cv50lTv64700ZOskQXUTruvR+4MyK+HxEHN348PB1YnRJcNvO5YS2PnWYvS7ur0cz3hBpPWf3nLXVNVc0vqJGmdy7UnR9hn6C0l+tmtBfRkaxAqY388BLm/34Uy/g1pf3TP1GCm/sptwDXoNxyXIcScP6ysV4obVuXdAvujmp5UGrPvryEdNc03o9UvktyLfDsiNg4M5tBQ0enhqlTLkv6glqx8b6XfbqkdnBfBfYC/ikiLqN0wPlao1Z3kapW7Srgqoj4FaVt4OuAC6pa6QspPdrfS+kM8yClNus0HlshsKTbub2W+Wh18rEL3UeEWFRWmXl4RHyZ0sHnpZRg6eiI+OeqGUInUHwm8J0+5a+u1/N+WXTKZyznyZKWsSzXnT/T/UfGqM6TquZxr4j4R0qN906U9pnviYjnZeafqnzeSWkr202vNazTWdw5UlOUAaf0aJ1g5u+Z+dOlply6zbpM69QUddop3gSsMYr1LPGXf2Y+HBGd2+drAZdUvbovpQQHr6S0jfpi7WOd2rv7l7buiLiDEsCutIxlMRrfp/Rw359Ht/nr5GVNyrZcmZmdfdT5clu7kfzJjff92qdQbm/+iXJbfUPKbcP/t9RPLNbpfd/pQPMiym3EPTOzU6vZ6dnbls0ot3k761qJMqrARUv8xOLjZV6n7erSZOY1lADrw1F68M9h8VBIv6Tst/0i4oQuNZdNt1I6YK3YSLt59XrLSPkZo00jYoVGLWfnvF3SusZynizpXO7HMXodZV823Q0QEWs3mhs0zxMAsowScTlwTETsQgmAD6a0Ab+JEohempl/WUpeOsfMTErN+BJVx+CTWHKgrSnCW+pSTfUr/kLg4Ih4YnN+1ZNzNLaJMgB453PrUgKqS6pbs1Daim0bEbt2Wc8aETFUve10KlnSbelfUHoe70TVE71qN/VrSk/badR6qFMCgN8B/xERj7m12dnG6gv+m8Ae0aXH9RjKYjS+RQlS3h0R2zTWsyKl09R0qo5PlVspNVf1TlNQekcv0sd92rkFfQalzd6bKLd7L2ks78VVZ6Kmzn7ufAF3Aqj6kF4rAP8x2vz04JDacQUlwF+bRpOAhm9R8npMs8MHLBq2iIhYswoe6q6j1ICuDYuOyw9TAsaPLWF5L43yFCIobV/Xp5w7nfkrUJp+DLO4vXO/bEAZCquzrlUpTQRuYwkdncZ4nnTGc33UudynY/RiYMsqz3Wd4G/ReVLdJn9DYx3Tu+yPK6vXtavXsyhxwzGNdETEirX26T+mNJ94dzM/XdaxJaWz4SVoSrOGU3qsf6NcvH9b9TK9ifJF9FzKxbHbcDNNc4EfRMSnKbUf/0q5zf2eWpqTKEO1fDciTqcEgqtSagX2otx2vCUzH4zSm/g1EXED5ZbWzZnZ6RDxCxYP8VQPLH/O4mFyFj02LjMXRMSbKLV111a3QG+j1Lz9MyUA6jT+fzelY8yvqrK4hvJl+WzgXyhfFMusqql9NaWJwS+rPNWfNPQcyniI/137zL0RcTZwWJTHod5EuRXYra1bP/Zpx1cpNXYvpftTgz4FrB4R36YEXCtQOgC9nrLvTq7SXVy9P706Th6mBLKrjyEvvbgwSg/3p1CGEJpLGde1q8z8fZQhiE4CnhwR36G0N30qpdb5LEo5vJjSq/qblM5M0yjB2xrUOuJQhlTanBI07lDtw/mUwHLnajmdAPOLlHPnS1EGNv89ZSinHYH3ZJexTpfRjZShjJ5DOSdeR+lg99ql9FCH0Z8nN1FqHP8tIv5CuTbMrUZwWNZj9LuU4Z9ezKN/QPyY0rb1SxFxEuXHQ6e96ca1dG8ADq2O25so16IDq/TfBMjMn0fEKcA7q9rrH1EC/6dRjt1jgNMy876IeBulicGvI+JrlGP9GZQmI6+qrXcnyo+SH42wfZrkrOGUGqpevttQOojsz+IxBVcC3jfKxVxcfWYfyjBDw8AembkoIKxqe3YAPkKpfTiZMrzMFpRhaP5YW96bKLf0PkZpK/ZvtXm/otw+/xuLb9vC4h7iFze/LKt8PI/SQenfKUOYvJHSAeYjtXR/onzh/Sfli/4zlGBrI0rv676pyn2rah07UcYzPIkSbL4hM7uV/WGUL9o3U27Fz6NRc1Nb9rLu086y/ofFtV3dnn51BKW3/86U4OqT1facQXn60C3Vcu6i9ML+X0qgcBSl1/f+Y8nPGL2N0sb3/ZThbb4HvHRJbVA7MnM2Zf8/ROkA9XFKIPUzFg/WfRVlSK5dKdv9QUrQuUd9mKTMXJiZb6IEq/OBt1PaCh5O6Ty1e2dYomoYqRdRAuLXUo7/J1AGIm/jkba/p7TL3ZEyvNV0ygMElvqYzdGeJ1Wb3tdTztVTKOfyntW8ZTpGszyL/EpKT/L69Icp++omyj55a5XPzzQWcRHl+rE35Zg9inINenHtxy2Z+RbK9Wgdyh2HEyk/vr5BbdD5LA9deDnlmnIUpTyfT2k+U7c38O1ax05NUdMWLrRjmKSJKSKeSQmcb6WMqzohvpQi4grgocycFINVx+Knyjw/My8dIflyKaonDWXmywadl15FxL6UWuEnt1D723dRnmz0a2BWZv5m0PlRu6zhlDRhZebVlJqwAL4dIzwRZzxExLMpNVFLvA0tDciZlJrMtw84H6P1HuCbBpvLB9twSprQqh7cfWkruiwiYialc9Y7KD3Vu91OlwamGtpoNI9UnRAyc69B50HjxxpOSRqdPSm3pVcFXpOPfiSlJGkpbMMpSZKkVnlLvQdz5swZArYF/o8lP8FCkiRpIliRMsLDFbNmzRoeRAYMOHuzLYuHnJEkSZoMmo86HjcGnL35P4CnP/3prLxye51m586dy8yZM1tb/mRhORSWQ2E5FJZDYTkUlsNilkVRL4eHHnqIG264Aar4ZRAMOHvzCMDKK6/M0NDQSGmXSdvLnywsh8JyKCyHwnIoLIfCcljMsii6lMPAmgHaS12SJEmtMuCUJElSqww4JUmS1CoDTkmSJLXKgFOSJEmtMuCUJElSqww4JUmS1CoDTkmSJLXKgFOSJEmt8klDE9jqa63H7Xc9MOhs9MVqQyuxxuPbewyoJEmauAw4J7BHFk7j/CvmDTobfbHjthsbcEqStJzylrokSZJaZcApSZKkVhlwSpIkqVUGnJIkSWqVAackSZJaZcApSZKkVhlwSpIkqVUGnJIkSWqVAackSZJaZcApSZKkVhlwSpIkqVUGnJIkSWqVAackSZJaZcApSZKkVhlwSpIkqVUGnJIkSWrVSoNacUTsBbwWmAWsA9wEfBb4fGYuqNKcBryhy8f3ysxvNpZ3BHAosBFwDXBkZp7fSLMGcBKwJ7AKcCFwWGbe0rcNkyRJ0qMMsobzcGAYeCfwcuA7wKeAjzTS/R54fuPvgnqCKtg8ATgF2A24ETgnIrZqLOvrwO7AYcA+wAzg/IhYrV8bJUmSpEcbWA0n8IrMvKP2/sKIWB14S0QcnZnD1fQHM/PSJS0kIoaAo4GTM3N2Ne0i4GrgvcDe1bTnUoLR3TLz3Gra1ZSa1QOAU/u5cZIkSSoGVsPZCDY7fkO51b3OGBa1HbAWcGZt2Y8A3wB2iYhp1eRdgXuB82rp5gEXV/MkSZLUgkHWcHbzQuAu4E+1aZtGxD3A44G5wImZeVZt/hbV63WNZV0DrA48EbitSnd9p31oI93Ofcm9JEmSHmPCBJwRsQ1wIPCBqoYSSo3nFZSgcC3gIODMiFg1M0+r0kwHhjPzwcYi765e16EEnNOBe7qs+m7GVqO6yNy5c3v52Kituub6zJ8/v9V1jJc771yN227uVqk9OnPmzOljbiYvy6GwHArLobAcCsthMcuimEjlMCECzojYCPgWcDm1TkOZ+clG0u9GxAXAB4DTxi2DSzBz5kyGhoZaW/61N85jxowZrS1/PK277npsuNnGPX12zpw5zJo1q885mnwsh8JyKCyHwnIoLIfFLIuiXg7Dw8OtV5KNZODjcEbEWsAPgQeA3TPz4RE+cjawcUSsX72/GxiKiFUa6aZXr3fV0q3dZXnTa2kkSZLUZwMNOKsg8XvABsDLMvPOHhbTabu5RWP6lsD9wB9q6aLWiaie7voe1itJkqRRGFjAGRErUXqSPwvYJTNvHcVnplGGObq11sv9Ekrv831q6Vas0p2XmQuryedSajh3rqV7ErB9NU+SJEktGGQbzlOAVwDvAlaLiOfV5l1LudV9OmWw9t9RgsWDgB2A13cSZuZwRBwPnBARdwBXVuk2BfarpbssIs4BvhQRhwP3AccB85gA7UElSZKmqkEGnJ2axo92mfci4LeUmsujKbfcH6YEk7tn5vfriTNzdkQAvBXYkNKrfbfMvKqx3H2B2ZRB3ocoj7bcKzMf6McGSZIk6bEGFnBm5lNGkeyVY1jebEowubQ09wOHVH+SJEkaBwPvpS5JkqSpzYBTkiRJrTLglCRJUqsMOCVJktQqA05JkiS1yoBTkiRJrTLglCRJUqsMOCVJktQqA05JkiS1yoBTkiRJrTLglCRJUqsMOCVJktQqA05JkiS1yoBTkiRJrTLglCRJUqsMOCVJktQqA05JkiS1yoBTkiRJrTLglCRJUqsMOCVJktQqA05JkiS1yoBTkiRJrTLglCRJUqsMOCVJktQqA05JkiS1yoBTkiRJrTLglCRJUqsMOCVJktQqA05JkiS1yoBTkiRJrTLglCRJUqsMOCVJktQqA05JkiS1yoBTkiRJrVppUCuOiL2A1wKzgHWAm4DPAp/PzAW1dLsAHwK2BP4AnJyZn+6yvCOAQ4GNgGuAIzPz/EaaNYCTgD2BVYALgcMy85Z+b58kSZKKQdZwHg4MA+8EXg58B/gU8JFOgoh4PvA94DfALsBXgJMj4s31BVXB5gnAKcBuwI3AORGxVWOdXwd2Bw4D9gFmAOdHxGp93jZJkiRVBlbDCbwiM++ovb8wIlYH3hIRR2fmMHAMcGVmvqmWZmPg/RHxhcxcEBFDwNGUms/ZABFxEXA18F5g72racynB6G6ZeW417WpKzeoBwKktb68kSdJyaWA1nI1gs+M3lFvd61SB5IuBsxppvka5bb519X47YC3gzNqyHwG+AewSEdOqybsC9wLn1dLNAy6u5kmSJKkFE63T0AuBu4A/AZsCKwPXNtJcU71uXr1uUb1e1yXd6sATa+mur7cPraXbHEmSJLViwgScEbENcCDwiaqGcno1655G0rur13Wq1+nAcGY+OIp0zWV10q3TZbokSZL6YJBtOBeJiI2AbwGXU+s0NNHNnTu31eWvuub6zJ8/v9V1jJc771yN227u1opidObMmdPH3ExelkNhORSWQ2E5FJbDYpZFMZHKYeABZ0SsBfwQeADYPTMfrmZ1aijXbnykU/N5Vy3dUESskpl/GyHdxl2yML2WZkxmzpzJ0NBQLx8dlWtvnMeMGTNaW/54Wnfd9dhws27FP7I5c+Ywa9asPudo8rEcCsuhsBwKy6GwHBazLIp6OQwPD7deSTaSgd5Sj4hVKMMebQC8LDPvrM2+CXiIxW00O7asXq+vXjttN7ulu58ydmcnXdQ6EdXTXY8kSZJaMbCAMyJWovQkfxawS2beWp9fDYt0AdWwRjX7An8ErqzeX0Lpfb5PbdkrVp87LzMXVpPPpdSW7lxL9yRg+2qeJEmSWjDIW+qnAK8A3gWsFhHPq827NjPvA44Dfh4RXwTOAF4AHAwc2ultnpnDEXE8cEJE3EEJRA+i9HLfr7PAzLwsIs4BvhQRhwOd5c8DTmt1SyVJkpZjgww4OzWNH+0y70XAzzLzVxHxSspThPYH5gPvyMzP1RNn5uyIAHgrsCFlqKPdMvOqxnL3BWZTBnkfojzacq/MfKA/myRJkqSmgQWcmfmUUaY7l1Hc8q6eMjR7hDT3A4dUf5IkSRoHE2YcTkmSJE1NBpySJElqlQGnJEmSWmXAKUmSpFYZcEqSJKlVBpySJElq1ZgDzojYucvjISVJkqSueqnh/CFwW0ScFBFb9TtDkiRJmlp6CTj3AC4GDgWujIjfRsQRETGjrzmTJEnSlDDmgDMzv5eZe1MeIXkwcAdwInBrRPw4Il4XEav1OZ+SJEmapHruNJSZ92fmlzNzR+DJwFHABsDpwO0R8dWI2LFP+ZQkSdIk1a9e6isCjwOGgGnAg8BLgJ9ExG8iYmaf1iNJkqRJZqVePxgRawF7A68DXgD8HTgHeHf1ugDYHfgE8BVg22XNrCRJkiafMQecEbEHJcjcFVgFuAJ4G/D1zLyrkfw7EbEecOoy5lOSJEmTVC81nP8N/AH4JHB6Zl4/QvrfAmf0sB5JkiRNAb0EnC8Fzs/MhaNJnJmXA5f3sB5JkiRNAWMOODPzp21kRJIkSVNTL4+2/ERE3LiU+TdExEnLli1JkiRNFb0Mi7QbcNZS5p8FvKK37EiSJGmq6SXgfBJwy1Lm31qlkSRJknoKOO8DnrqU+ZtQBn6XJEmSego4LwAOiYiNmzMi4inAIVUaSZIkqadhkY4BdgHmRsRXgGuq6TOBA4BHgPf1JXeSJEma9HoZFunGiHgBcApwWGP2RcBhmZn9yJwkSZImv56epZ6Z1wA7VI+t3KSafFNm3tm3nEmSJGlK6Cng7MjMPwN/7lNeJEmSNAX1FHBGxIrAzpTazenAtEaShZn5wWXMmyRJkqaAMQecEbEN8C3gH3hsoNmxEDDglCRJUk81nKcCqwJ7AL/IzHv6mSFJkiRNLb0EnM8C3puZ3+93ZiRJkjT19DLw+20s+Va6JEmS9Ci9BJwnAgdHxJr9zowkSZKmnl5uqa8D/BX4XUR8E/hfytOF6hZm5knLmjlJkiRNfr0EnCfW/n/zEtIsBAw4JUmS1FPA+dR+rTwingYcATyP8iz26zNzZiPNacAbunx8r8z8ZiPtEcChwEaUZ7wfmZnnN9KsQQmG9wRWAS6kPI7zlj5skiRJkhp6eZb6rX1c/zOA3YDLKO1Jl9Sm9PfAaxvTbqi/qYLNE4CjgCuBg4FzIuK5mXlVLenXga0pz4G/DzgOOD8inpmZDyzb5kiSJKmp50dbRsRmwA7ABsAZmXlLRKxMqV38Y2Y+NIrFfD8zv1st7zRgmyWkezAzL11KXoaAo4GTM3N2Ne0i4GrgvcDe1bTnUgLc3TLz3Gra1cBNwAGUMUYlSZLUR2PupR4RK0TEF4Drgc9Tagg3qWavTAnyDhvNsjJzwVjXvwTbAWsBZ9aW/QjwDWCXiOgM47QrcC9wXi3dPODiap4kSZL6rJdhkY4C3gi8D3g+tTE5M/MvlMdevqovuVts04i4JyIejojfRMQ+jflbVK/XNaZfA6wOPLGW7vouge41wOZ9zbEkSZKA3gLOA4EvZ+YJwO+6zL8a2GyZcvVov6F0LNqD0tHnNuDMiDiglmY6MJyZDzY+e3f1uk4t3T1d1nF3LY0kSZL6qJc2nP8AXL6U+Q8Ca/SWncfKzE82Jn03Ii4APgCc1q/19GLu3LmtLn/VNddn/vz5ra5jvNx552rcdvMdPX9+zpw5fczN5GU5FJZDYTkUlkNhOSxmWRQTqRx6CTj/CDx5KfNnAf3syd7N2cCpEbF+Zt5BqaEciohVMvNvtXTTq9e7qte7gY27LG96Lc2ozZw5k6GhobF+bNSuvXEeM2bMaG3542nddddjw826Ff3I5syZw6xZs/qco8nHcigsh8JyKCyHwnJYzLIo6uUwPDzceiXZSHq5pf4t4N+qXuodCwEiYhdgf0pnnfHUabu5RWP6lsD9wB9q6aLWiaie7vr2sidJkrT86iXgPBaYR2lbeQYl2DwqIi4FfgBcBXy4XxlsqoLFvYFbq9pNgEsovc/3qaVbsUp3XmYurCafC6wN7FxL9yRg+2qeJEmS+qyXgd/vi4jtgP8A9gL+RgnYbqIEoyc1bmsvUUSsxuLhiJ4MrBkRe1bvr6heT6cM1v47SrB4EGX8z9fX8jQcEccDJ0TEHZSB3w8CNgX2q6W7LCLOAb4UEYezeOD3eQy4PagkSdJU1dPA71VAeUL1tyw2oLTHrOu8PxD4HqXm8ugq7cOUYHL3zPx+I0+zIwLgrcCGlKGOdms8ZQhgX2A2ZZD3IcqjLffyKUOSJEnt6PlJQ/1QPb+82Z6y6ZVjWN5sSjC5tDT3A4dUf5IkSWrZmAPOiPjyKJItzMw39ZAfSZIkTTG91HC+mKpXes2KwBOq1zuAvy5jviRJkjRF9NJp6CndpkfE4yi3qd8O7LRMuZIkSdKU0cuwSF1l5sOZ+Rngx8Bn+rVcSZIkTW59CzhrrgL+qYXlSpIkaRJqI+DcCXCIIUmSJAG99VI/Zgmz1qbUbG4NnLgMeZIkSdIU0ksv9WOXMP1uytOG3gx8sdcMSZIkaWrppZd6G7fhJUmSNEUZPEqSJKlVvbTh3LiXFWXmvF4+J0mSpMmtlzact/DYJw2Nxoo9fEaSJEmTXC8B50HAW4EnAV8DbqimB7AvMA/4FLCgHxmUJEnS5NZLwPkEYAh4WmbeXZ8REe8HLgY2yswP9yF/kiRJmuR66TT0ZuALzWATIDPvpAyJ9G/LmjFJkiRNDb0EnOsCqy9l/uOrNJIkSVJPAeelwNsiYlZzRkRsA7wNuGxZMyZJkqSpoZc2nG8BfgZcHhFXADdW0zcDtgXuAg7rS+4kSZI06Y25hjMzrwWeSemJvjawZ/W3NvBJ4JmZeU3/sihJkqTJrJcaTjLzduAd1Z8kSZK0RD0FnB0RsRmwATA3M+/tT5YkSZI0lfQUcEbEfsCJwBOrSTsBF0TEesAlwNGZ+Y3+ZFFTwYIFC7n9rgd6+uyqa67f82fbsNrQSqzx+JUHnQ1JkiaNXp6l/mrgv4CfACcDszvzMvPPEXEdsD9gwKlFhh9+hEt+O7+nz86fP58ZMyZOwLnjthsbcEqSNAa9DIv0XuCnmbkzcHqX+ZcBWy1TriRJkjRl9BJwbgF8eynz/wSs31t2JEmSNNX0EnD+laU/aWhT4M+9ZUeSJElTTS8B5wXAARHxmEZsETEDOBj40bJmTJIkSVNDr204nwD8Gvh3YCGwa0ScCFwNLAA+0LccSpIkaVLr5UlDNwIvAP4IHAtMA/4DeBfwP8D2mTmvf1mUJEnSZDamYZEiYkXK2Ju3Z+ZLI2I68DRK4Pr7zLyjhTxKkiRpEhvrOJwrADcBRwIfz8y7gSv6nitJkiRNGWO6pZ6ZDwPzKe02JUmSpBH10mnoK5Re6qv0OzOSJEmaenp5lvoNwIrA9RFxOvB74MFmIp+lLkmSJOgt4Pyv2v/vW0KahfgsdUmSJDHKgDMiPgWcnplzgBdVk1en1Gw+0uvKI+JpwBHA84CZwPWZObNLul2ADwFbAn8ATs7MT3dJdwRwKLARcA1wZGae30izBnASsCewCnAhcFhm3tLrdkiSJGnJRlvD+RbgUmBOZl4UEetSnpm+U2ZetAzrfwawG3AZpT3pY9qURsTzge8BXwUOp4wBenJEPJyZn6ulOwI4ATgKuJLyxKNzIuK5mXlVbZFfB7YGDgPuA44Dzo+IZ2bmA8uwLZIkSeqil1vqHdP6sP7vZ+Z3ASLiNGCbLmmOAa7MzDdV7y+MiI2B90fEFzJzQUQMAUdTaj5nV8u7iPLko/cCe1fTnksJcHfLzHOraVdThno6ADi1D9skSZKkml56qfdNZi5Y2vwqkHwxcFZj1tcot823rt5vB6wFnFlb9iOUdqS7REQnON4VuBc4r5ZuHnBxNU+SJEl9NtCAcxQ2BVYGrm1Mv6Z63bx63aJ6va5LutUpT0fqpLu+S6B7TW1ZkiRJ6qOx3FLfJCL+sfp/rep184j4S7fEmXn5MuWsmF693tOYfnf1uk4t3XBmNodnqqe7rUrXXFYn3Tpdpi/V3Llzx/qRMVl1zfWZP39+q+sYL8OxzjJty0QqhzvvXI3bbh7MU1znzJkzkPVONJZDYTkUlkNhOSxmWRQTqRzGEnB+oPqre0xPcUrbzoWUsTqntJkzZzI0NNTa8q+9cR4zZsxobfnjaWholZ63Zf78+ROqHNZddz023GzjcV/vnDlzmDVr1rivd6KxHArLobAcCsthMcuiqJfD8PBw65VkIxltwHlgq7lYsk4N5dqN6Z2az7tq6YYiYpXM/NsI6bpFCtNraSRJktRHowo4M/P0tjOyBDcBD1HaXp5Xm75l9Xp99dppu7kF8JtGuvspY3d20u0UEdMyc2Ej3fVIkiSp7yZ0p6HMHAYuoBrWqGZf4I+U8TYBLqH0Pt+nkyAiVqw+d14tuDyXUlu6cy3dk4Dtq3mSJEnqs2UZh3OZRcRqLB6O6MnAmhGxZ/X+isy8lTIw+88j4ovAGZSB3w8GDu30Ns/M4Yg4HjghIu6gBKIHUXq579dZX2ZeFhHnAF+KiMNZPPD7POC0VjdWkiRpOTXQgBPYADi7Ma3z/kDgtMz8VUS8kvIUof2B+cA76k8ZAsjM2REB8FZgQ8pQR7s1njIEpXZ0NmWQ9yHKoy338ilDkiRJ7RhowFk9v3zEJxZVTwUa8ZZ39ZSh2SOkuR84pPqTJElSyyZ0G05JkiRNfgackiRJapUBpyRJklplwClJkqRWGXBKkiSpVQackiRJapUBpyRJklplwClJkqRWGXBKkiSpVQackiRJapUBpyRJklplwClJkqRWGXBKkiSpVQackiRJapUBpyRJklplwClJkqRWGXBKkiSpVQackiRJapUBpyRJklplwClJkqRWGXBKkiSpVQackiRJapUBpyRJklplwClJkqRWGXBKkiSpVQackiRJapUBpyRJklplwClJkqRWGXBKkiSpVQackiRJapUBpyRJklplwClJkqRWGXBKkiSpVSsNOgPSZLNgwUJuv+uBcV/vqmuu39f1rja0Ems8fuW+LU+SpCWZ8AFnRBwAfKXLrFMy8y21dLsAHwK2BP4AnJyZn+6yvCOAQ4GNgGuAIzPz/Bayrilq+OFHuOS388d9vfPnz2fGjP4FnDtuu7EBpyRpXEymW+ovA55f+5vdmRERzwe+B/wG2IUSoJ4cEW+uL6AKNk8ATgF2A24EzomIrcZjAyRJkpZHE76Gs2ZOZv55CfOOAa7MzDdV7y+MiI2B90fEFzJzQUQMAUdTaj5nA0TERcDVwHuBvVvOvyRJ0nJpMtVwdlUFki8GzmrM+hrltvnW1fvtgLWAMzsJMvMR4BvALhExrf3cSpIkLX8mUw3n3IhYH5gHnAZ8KDP/DmwKrAxc20h/TfW6OfBrYIvq/XVd0q0OPBG4rf/ZliRJWr5NhoDz/4D3A5cDj1DaaL4PeCpwADC9SndP43N3V6/rVK/TgeHMfHAp6cYUcM6dO3csycds1TXXZ/788e+c0obhWGeZtmUilcOybsuy6Od677xzNW67+Y6+LW88zZkzZ9BZmBAsh8JyKCyHxSyLYiKVw4QPODPzR8CPapN+EhH3AsdGxAcHlC0AZs6cydDQUGvLv/bGecyYMaO15Y+noaFVet6W0jt74pTDsmzLsuh3Oay77npsuNnGfVveeJkzZw6zZs0adDYGznIoLIfCcljMsijq5TA8PNx6JdlIJmsbzm9Ur1uzuIZy7UaaTs3nXdXr3cBQRKwyQjpJkiT10WQNOOtuAh5icRvNji2r1+ur107bzW7p7qeM3SlJkqQ+m6wB52uAhZShkoaBC3jssEb7An8ErqzeXwLcC+zTSRARK1afOy8zF7adaUmSpOXRhG/DGRE/ogSUc4EFlE5D/w58KTN/XyU7Dvh5RHwROAN4AXAwcGhmLgDIzOGIOB44ISLuoASiB1F6ue83jpskSZK0XJnwASflVvgbgX+g5PdG4Ejg5E6CzPxVRLyS8hSh/YH5wDsy83P1BWXm7IgAeCuwIWVIpN0y86r2N0OSJGn5NOEDzsx8O/D2UaQ7Fzh3FOlmU3sspiRJkto1WdtwSpIkaZIw4JQkSVKrDDglSZLUKgNOSZIktcqAU5IkSa0y4JQkSVKrDDglSZLUKgNOSZIktcqAU5IkSa0y4JQkSVKrDDglSZLUKgNOSZIktcqAU5IkSa0y4JQkSVKrDDglSZLUKgNOSZIktWqlQWdA0mAsWLCQ2+96YNDZGLNV11z/MflebWgl1nj8ygPKkSRpJAac0nJq+OFHuOS38wedjTGbP38+M2Y8OuDccduNDTglaQLzlrokSZJaZcApSZKkVhlwSpIkqVUGnJIkSWqVAackSZJaZcApSZKkVhlwSpIkqVUGnJIkSWqVAackSZJaZcApSZKkVhlwSpIkqVU+S13SpLdgwUJuv+uBkRNOAqsNreRz4SVNOQackia94Ycf4ZLfzh90Nvpix203NuCUNOV4S12SJEmtMuCUJElSq7ylLkkTyGjbo6665voTut2qbVEl1S13AWdEbAZ8GtgeeBA4EzgyMyfulVvScmO07VHnz5/PjBkT97JlW1RJdctVwBkRawMXArcCewIbAB8H1gdeM7icSZIkTV3LVcAJHAJMB56dmX8GiIi/A2dExAcz85qB5k6SpojxGqpqPJoW2DxAWnbLW8C5K3B+J9isfAv4MrALYMApSX0wXkNVjUfTApsHSMtueQs4t6AEl4tk5nBE3ARsPoblrAjw0EMP9TFrj7VgwSOstMKCVtcxXh75+8M9b8sqj5s2ocphWbZlWfS7HAa1HcuqWzlM1m3pZrTbMtHOi6bx2ifjUQ5/f/ghhodXbHUd/TA8PDzoLEwYlkXRKYdavDKwA3nawoULB7XucRcRDwPvy8wTG9N/CfwpM181muXMmTNne+AXLWRRkiSpLS+cNWvWLwex4uWthrNfrgBeCPwf8MiA8yJJkrQ0KwJPoMQvA7G8BZx3A2t3mT4duH60C5k1a9YwMJBfCJIkST24aZArX96eNHQdpR3nIhExBGzKGAJOSZIkjd7yFnCeC+wYEevWpv0LMFTNkyRJUp8tb52G1gbmArcAH2TxwO/nZ6YDv0uSJLVguarhzMx7gBcDfwH+G/gEcBbwxgFmS5IkaUpbrmo4JUmSNP6WqxpOSZIkjT8DTkmSJLXKgFOSJEmtWt4Gfp/wImIz4NPA9sCDwJnAkZn5wEAzNoKI2At4LTALWIcywOxngc9n5oJaul2ADwFbAn8ATs7MT3dZ3hHAocBGwDWUMji/kWYN4CRgT2AV4ELgsMy8pZFuYGUaEatTxnh9IrBtZv66Nm9/4CjgKZTyOi4zz2p8/nHAccAbKA8tuAJ4W2b+TyPdRsAngZcBC4EfAG/PzD830v0jZWSGWcBdwH9W623liVkR8Xrg7ZT9/QBwJbBvJ1/Lw/EQEXtQ9vMWwF+Bi4F3Z+aNjXRT5niIiKcBRwDPA2YC12fmzC7pJuz+H23elqUcImJF4HBgt2o9KwFXAx9obt9ULocu6WcBlwMPZubqjXkDOQdGc36OZAznxSrAu4HXA/8A/Bk4NzMPbqSbVMeDNZwTSDVs04XAGpSD43BgX+DLA8zWaB0ODAPvBF4OfAf4FPCRToKIeD7wPeA3wC7AV4CTI+LN9QVVJ9EJwCmUC/GNwDkRsVVjnV8HdgcOA/YBZgDnR8RqtWWtzWDL9Fi6/LCLiD2B04FvU8rip8DXq5O57hOUC8r7gVcCD1G2cUZtWSsB5wHPBPYHDgK2A74XEdNq6Tap1nMXZR+dQNlfH+rDdj5GRLyX8qPjvynb+CbKRXGomj/lj4eI2JGy/dcDr6rytjnw04hYs5Zuqh0Pz6Dsq98B13ZLMJH3/2jzNgojlcOqlCDmf4ADgddQvsR/EhEvb+RpKpdDfZ0rUK4bdywhybifA2M4P0cymvNiBcr35/5Vfl4KvIsyuk493aQ7HqzhnFgOoTxm89m1GqC/A2dExAcz85qB5m7pXpGZ9QvEhVXt3lsi4ujMHAaOAa7MzDfV0mwMvD8ivpCZC6onPx1N+dU0GyAiLqL86n8vsHc17bmUk2y3zDy3mnY15ZfnAcCp1ToGVqYRMRN4M/AfwOcbsz8InJ2Z76neXxgRWwAfAH5Yff6J1effmplfrKZdCtxMqTV8V/XZVwNbATM72xMR8yk1abuw+KEG7wTuAfaq9sf5EbEWcExEfDQz7+rjtgcl2P6XzPxBbdZ3av8vD8fDvsCtwBsyc2G1vluBy4AXUO1rpt7x8P3M/G617tOAbbqkmcj7f8S89akcHgSempl3dyZExI+Bp1O+9H9QTZvq5VB3MLAWJdh5a33GAM+BEc/PPpbDgcDzgS0z8w+16WfUymFSHg/WcE4su1IGoa9X+X+LUnM41l9S46oRbHb8hlKFv051gryYMu5p3dcotwO2rt5vR7nYnFlb9iPAN4Bdar9OdwXupfyK7aSbR7mg7Fpb/iDL9BTgM8AN9YkR8VRKLdeZjfRfA7aNiPWr9y8FVqRWZpl5P+VLqLmNV9eDpcy8hBLoNNN9p7qw1tfZ2Tf9dCBwayPYXGQ5Oh4eB9zfCTYr91Sv02BqHg8jffFM5P0/hryNaKRyyMxH6sFmNW0hpcZzRm3ylC6HjohYj1Jr9zZKzWXTuJ8DYzg/RzTKcjiYEtz+YSlpJuXxYMA5sWxBo5q9OhFuohzwk80LKbcq/kR5Xv3KPPY2QueC0Nm+zrPur+uSbnVKW8hOuuu7nMDX8OiyGkiZRmm7+DTg+C6zO9u4pLKIWrrbM/POLumeXt166aTrdntmUVlExOOBjZvpqnY8D9D/snge8NuIODoi/hgRD0fE5RHxz9X85eV4OA3YIiIOi4i1I+IpwGzK9nTaWi0Px0PTRN7/o81bK6r9uB2P3ublpRw+AvwyM89bwvxBnAOjPT+XWZT2qVsDt0TE6RHxl4j4a0R8p6pJ7JiUx4MB58QyncW1H3V3UzriTBoRsQ2llusT1S+v6dWsexpJO7/uO9s3HRjOzAdHka65rE66elmNe5lWt2ROAt6VmX/pkmQsZdFM00n3OMqFZaR0nWWtvYR1NtP1y0bATpRj4K3AK4D7gPOqoGu5OB4y80JK280PVeu4GXgqsFOtVmV5OB6aJvL+H23e2nIYJYj5WG3alC+Hqn3gvsA7lpJsEOfAeJbDupTtOJJyDX01pe37VsC5UdqmdvI06Y4HA071XZTegd+i9DL8yAjJp6LjgRsz84wRU05dK1Au/q/OzG9UNRa7U4LOdw40Z+MoIrYDvgp8iXI7ai9gAaXzwqqDzJsmnuoOwEeB2Zn5i0HnZ7xE6a1/KvDxzPz9oPMzQJ2Y7C/AHpn5o8w8k3LdeAbwLwPLWR8YcE4sd7P4V1fddMqt6Qmvqt37IeWWxO6Z+XA1q/MraO3GRzq/mu6qpRuKMizESOmay+qkq5fVuJZpRDyD0qj9fdUt1LVZ/Kt79ShDVIylLJppOukeZnGvxdFs4z1LWGczXb/cDdyZtaFKsgyvcSllOJDl4nigjNRwYWa+IzMvzMxvUhrxP4cy5EknT3TJ11Q6Hpom8v4fbd76KiKeBXyX0rHuyMbsqV4OBwNPAE6tXTdXgdKDuvbjbBDnwHiWwz2UIZwurtdeZhlO7z7KtbOTp0l3PBhwTizXsbhtBrCooe6mlGFVJrTq4P8esAHwskY7m5sojcC3aHxsy+q1s32dNind0t1PGTKkky5qjaPr6eplNd5luhll9IcLKSfo3cD3q3kXAr9g6dsIkNXrdcAGEdG8VbElcEOtXc5jtrGW7nqAzPwrMK+ZLiKeDKxG/8tiab29V2H5OR62pHQAWSQzb6OMq7dpLU8088XUOh6aJvL+H23e+iYiNgV+RBmn9vWNTmYw9cthc2BDynZ0rptHAo+v/v9wLd/jfQ6M9vxcZtWP8luWMHshVRA+Qp4m7PFgwDmxnAvsGBHr1qb9C6XH3LndPzIxVG1LvgE8C9glM2+tz6/aq11ANVxDzb7AHykXWoBLKL3q9qkte8Xqc+fVLsTnUn5p7VxL9yTKoLX1shrvMv0l8KLGX6dN0puBgzLzZsrJuU/js/sCV9R6/P+Ycvt1UZlFGWrqFTx2G59ZDdPRSfc8ygDFzXR7RMTKjXUOs7gDS7/8AFg3Ihb1XKwa6T8fmLMcHQ+3UgaUXqT6QluP6otlOTkeHmUi7/8x5K0vqiZIP66WvUdmduudPdXL4TM89rp5OvC36v/PVOnG/RwYw/nZLz8Atq83uYkyOP1awJxq0qQ8HqYtXNj8IaVBqW4jzKV8EX2QUlP4ccpwBa8ZXM5GFhGfB/6VMg5as+3RtZl5X9Uo/OeUnrtnUMYhPA44NDM/V1tWZ0Db91AO4oMojaefm5lX1dL9gHJr8nDK7YbjKNX7z6x+KU6IMo2IHSi1m4ueNBTlyUxnUX65/4QygPHbKOOl/bD22c9Qbr0eTglejqCM3fbMzJxfpVkJ+DWlsfl7KDWsJwG3Ay/IxeM/bkKpbbuA8kSJqNJ9OjPf3edtXgH4FbA+ZVy4+6tt2JYy1tvvlofjISLeQinrz1Bul65LGT9vfeAZnbsAU+14iDKodGfYlUMpNSX/Ub2/IjNvncj7f7R5W9ZyoIzg8atq+uso+2iRzLx0eSiHZgVF9ZljgSPysU8aGvdzYLTnZz/KoQoIr6Ls409QAsYTKPty604ztcl4PFjDOYFk5j2UjgV/oTyd5BOUg/yNA8zWaHV+QX2UcgGt/20NkJm/opyo21JuHx0EvKN5oGYZyPYoSu/mH1Jut+xWP4kq+1J+DZ4KnE35lfWSrD2Ka6KWaWaeTenBvSelLHYG9uty8XoH5akbx1OaK6xK2cb5tWX9nfL4trnAf1Ge/nAppQ3twlq63wMvoQQ751ACn49RAsJ+b98CSlvFn7N4/wDskJm/q9IsD8fDKZRBlV9IaZt3MuUpIy+qNzmZgsfDBpR9cDawA/Ck2vsXVeufsPt/tHnrQzlsSOmBvDrl+GheO5eXchiLcT8HxnB+jmQ058X/Vv9Pq6Z/hlKJ85Jc3CdiUh4P1nBKkiSpVdZwSpIkqVUGnJIkSWqVAackSZJaZcApSZKkVhlwSpIkqVUGnJIkSWqVAackSZJaZcApSZKkVv1/xVCuu0pwNnUAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAoIAAAFKCAYAAACJoz5RAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAAAw2ElEQVR4nO3debxtc/348deNXMksEpUy9Ea3+HWpDPVrki6l4WuIypeifJMkU5Mh5Kf4lpKiCRWRVF8iDcZKGU4D1/CmK3y5kXmI7sW9vz8+a3eWbZ9pn2Gfe9br+Xicx7p7rc9a+7M++3PPfp/PtKYtXLgQSZIkNc8zep0BSZIk9YaBoCRJUkMZCEqSJDWUgaAkSVJDGQhKkiQ1lIGgJElSQxkISuMgIg6NCNdm0tNExOsiYmFEvK7XeZmsIuLiiLi41/noJCL2jYibI2LxXudlKBGxdUQ8EhEr9zovmrwmfUWWJosRBHa7jmtGRiEipgHvA3YDNgCeCcwBzgC+mJmP9jB7wxYRzwb2AbYH1gQeB+4Afgd8KTNv6GH2JoWIWAPYH9gSeD7wJHA98FPg+Mx8oGeZG0cR8WHg0cw8eRyuvQzwSeCgzHxirK8/1jLz3IiYQ8nzx3udH01OBoLS8L2v7fUHgVcD72/bfxnwfeCoicjUcEXEYsBplODpN8DBwGPAa4HPAttHxJsy8x+9y+XQIuKZwCXADOB7wNeAZwHrAlsDvwcaHQhGxJbAWZTg73vA1ZTf9xtRgoL/C7y5ZxkcXx8G7gFOHodrv59S1747DtceLycCR0fEoZn5UK8zo8nHQFAapsz8fv11RLwJeGX7/prJ1mJwACUIPCYz96/t/0ZE/BD4CXASJZiaMFUr5ZKZ+dgwT3kHMBPYtb3Vp+quW25MM7iIiYgXAWdSWkjfkJl3tB3/FKVFWCP3fuC8zPxnrzMyAj8CvkL5v/+tHudFk5CBoDQOIuJQ4JDMnFbbdwulpeoo4BjgpZRu2Y9m5oUR8Q7gMOAlwHXA7pnZ13bdlwBHAG8Enk3p6vtcZv5oiPw8i9JNeCOlRegpMvPsiDgF2DUiXpmZV9TyfHFm7tJ2vYur815X2zcd+ATwXuCFlFaZHwKfrnc5V13sJwIXAZ8BAvhgROwOLJOZL++Q/z8Cj2fmq4C1qt2/6XAfTwD31s5bgxIAvwFYA5gP/Bb4ZGZeU0v3uio/7wHWobT2Lgf8EvgA8E/K5/YeSrmfBexRD17b7usQSpd1Agdm5vntee1wj0N+tlWg+wlgZ+AFlBbdm4DPZ+aPq2QHAMsAW7UHgVUZ3Vm9T/299wA+AqwNPAD8T1VG99XSXAysCmwHfBXYGLizSndGRGxOqdcbALdR6vUvaucfWpXLSymf+9bAAkod2TczHxmifKZVefwg5TN6CDiHUr73VGluoXzO9aEct2bmi6p9w6qjA7z/i4GXA8e37X8R8Dc6/2GyEPhsZh5avV4aOBT4D2C16h6uBQ7OzEtr521MaaXfDFgC6KN0R1/Udv3nVdfbGlgZ+DvwK+DjmfkwQGb+IyKuBt6JgaA6cLKINLHWBH4AnEv5QloeODsidqL81X4apct2TeDMqjsXgIhYD7gceBnwBWBfStBzZkS8d4j33RxYAThtkLFNre6ut430pqov6Z9QgpBzgb0oX7AfBn5aHa97LeUL9Szgo5QA+RTgZRHxlECwuu//U8vfLdV25w7Xbbdx9V4/AvYGvgS8Arik+hJtdwAwixL0fZvS+vgN4JvA+pQv57MogdjTAmrKF/fXq3v/NLAkcE4VJA1oBJ/tIZQ/Fi6hlNthlLJ7ZS3NNsDfMvO3g71n7b0/U+X5LsofC6dTWr4urAKnuuUon++VlLJ6FDg1It5NKePzKfV6qSrvnVpnT6fUxU9V53yQUl5D+TrwRUo57U35XLYFLoqIJas0HwNup5TJ+6qfj1X3OdI62m7TanvVMPI62D3sVeXjw8DngbspwTNVPv8v5Y+cFSmf74HAdOCX9QlGEbEqcAWlLp5VXfdkSl1Yqe19+4BNhnGPaiBbBKWJtQ7w2sz8DUBEXA/8AvgOsF5m/q3a/wClden1wK+rc78MzAU2qrVEHR8RvwSOiohTM3OgCS3rV9u/DJK31rH1B0kzkB2BtwCvz8xLWjsj4irKeMktKK1rLesCr8jMP9fS3kC5x/dSvqxb3keZDHJG9fqnlC/6gyktmBdTWvnO7dACdm57a2lEfI/S4voB2lrGKK0vr8zM+VXalYF3U1pZ3lKV79ciYh1KsHRw2/kzgE0z8/fV+SdTWuyOogTjAxnuZ/tWStfk7p0uEhHLAqtTWvSGVN3fQcAFwJaZ+WS1/8+UYQK7U1r/WlYFds7M71XpfkX5LE4DXpOZv6v2t+r1djy9FeoOSmvlwirt34GDqvGpv6aDiNgU+BDwn5n53dr+8ylB087ANzLzpxFxBHBPhyEbI62j7dattjcPkmYobwW+mZkdJ25UgdqJlPq8Ra2MTgD+BBxJf0B6FKVVcdPMvLx2mUM7BHw3U4Lv51HqmfRvtghKE+vGVhBYaf0Cv7gVBLbtXxMgIlYE3kRpwXh2RDyn9UNphVmd0qU8kGWq7cODpGkdW2aQNAPZntLtfG1b3i4BFlIC2rrL6kEgQDWL9Wxgp4h4Bvz7i3En4Oet7r/M/BclqDoGaM2CPhH434j4fhUMta5Z77pdKiJWonTHJWWcYbvvtYLAyuXVe5zUFmRfDqzWocXsqlYQWL3/vZQgabOIWKHD+430s30QeGnVjdxJ694H+5zr3kQJfr/cCgIr36O0ELaPF30MOLV2f0npSr6xFQRWnlJ/23y1rSy/Um3fOkg+twceAc5vK58bqny216+BrjGSOtpuJUpX9oPDeK+BPAi8KiJWH+D4BpShEqcBK9XyuCzlj5FXVfX4GZSu3p+3BYEAdPiD8P5q+5xR5F1TlC2C0sS6rf4iMx+MCID/bUvX+rJpBQ9rUwKSQ6ufTlahBDidDCfIax3rZtbwSyhfYHcPcHyVttdzBkh3CuUL+/WUVqrXUMZ87VdPVAVY+wP7R8TzKd2/e1PG8C2gtBBRdRkeRmllbO8Kvpenu63tdetz6PT5TKN07d9V239Th2veWG3XoP8LuW4kn+3BlBbRjIjrKK1up2Vmq7uyNSt0uMH8GtX2KfUmM5+MiJuAF7WlvyMzF7Tte5C28qnV607B701tae+JiPs7vFfdS4CleWpZ17XXr4GuMZI62sm06qfbNUL3p9Tx2yLiT5RA/3tVQN3KI5RhCQNZiTLWdVlg9jDft9VC6NqmehoDQWliPTnC/a1f4K3W+y8B5w2QdrAvheuq7cspgUQnrbF59a6vgb44FuOpeX5G9R57D5C+vTtqoBnCv6B82b+XEgi+l9LidM4A6cnM24HTIuJHlIH3746I91djIY+jdOEeR1nW5wFKoHgsnXtEuv18RmPYn21mXhoRa1HGcb6ZEvB+LCI+kZlfyMyHImIuZazheOhF+UApo3sp3fSddAqwO11jJHW03T2U+1mu7f06/h+pj+9tycwzI+I3wNspn99HgQMiYpfMPI3+uvAJyri+Tu5m5DPjWwH5PSM8Tw1gICgtGlrB2RMDjaMawu8oQdBOEfG5tm7Alp2r7Zm1ffdTWr3arcFTA8Y5lK7WCwYZpzikqiXqVGD3iNiHMhngzMycN4xz50fEXygtbM+hzGjdDvhuZn6snrbqph2PL8V1OuxrtfLcOsA5I/psM/N+ysSZ70aZDX4e8NmI+O/qcz0b2CMiNmvrru2klaegv+WSqutxHcq4tLG2Ttt7PYcSqNwyyDlzKGP4/jDU7GIG/uNltHX0+mr7Yp4aCLb+vXxb+jXooJq1fSJwYkQsD/yBMgnpNPpbyh8erC5ExHxK6++MYea9lec7h5leDeIYQWkRUC3yfBElQHra+KIY4hFS1dIYX6B84X+uw/lbA7sAZ9eXVaF8Mb06IpaopX0rZemSujOA5wL/1eHa06M8kWG4TqF0bZ5ICRCesnhvRGzQ6X6rL9VNgPvo7/57krZWqYjYkTLIfjxsFBGb1N5rJcoYx8uqAO5pRvLZVtern/sYZZzckpSFjgGOpoyn+3ZEPO0+I+K51UxhKOPO5gMfbY3LrLyH8nn+bPDb7cpH2iYzfLTanjvIOWdQvq/aJ+cQEYu1jb/8J527pEdbR1tB9Ub1ndUizfdQhifUfbhDPp/SkleNi/0b/UFkH/BX4OOd8tOqC1X3/E+AWRHxqg7p2ltiZ1KCaLuG9TS2CEqLjv+ifBldHRHfpARpqwCvosz0XXuI878AbAgcGBGvBn4MtCZevAe4hhIM1n2L0ip3fpRFp9eidNe2j/H7fpXu+CjLX/yWEoAFZczfdsDFw7nJzLy6atnbnvIl2d6qtQVweEScQ3mKyIOUwHRnSoC3V63F82zKMjMPUbpXNwR2YHQzPwczG/hZRBxHGZf5QUpQ22mpmbrhfrbXR8SllOVb7qFMLtgN+FmrpSwzb46IHSgtu9dVs6RbTxZ5BaV79bIq7T0RcThwOGV5kp9SJnh8hDKLfDzWnVsdOC8iflblf3fgl5n5q4FOqLrEj6eMCX05ZQjBPEq5bEsJEE+ukl8FfDgiDqG0PD6SmecwyjqambdVs6m3oCxdU/ct4BMR8a3q/V/L0ydvLQPcERFnUcr2IcpyQ2+hmpmdmQsi4gOUsYPXRcR3KMvhrEZ5Gsw0+ie1fLLKy8URcSKl2/u5wLsoE0luAYiIVSjDPk4Y6N7UbLYISouIakD5RlTBDWUdvg9TvuAPGsb5T1KCgF2qc46gtLq9j9Iy9Mr2VqtqQeB9KV9qx1Ja3N5K+XKqp1tA+QLanxK4HE3p7no15RFwV4/wdk+ptt/v0IpxFmXpjNUpa6ydSGlVuhl4Z2bWlzvZmzLwfgfKGLyXUr542yd/jJXfUT6THShLfcwD3lFfLLiTEXy2x1KeG3wgJXh4C6Usdmy73nmUbsPTgK0oYySPpgTCn6MEPa20R1AC0ecB/035o+Bk4I3D6ZLvwo6Ubsoj6X/axXaDnlHy+RHKkj8rUu7hKMo4ux8CF9aSHkYZU/pxyv0fV50/FnX0O8BWUZ51XXcYpZ5tS/mDazHKepR1j1I+15dRPtNjKZ/5fpTnZrfu89IqT3+g1IGvUsa53kdZd7CV7u+UPxROp5RpazzsFTx12MN/UFp9z0DqYNrChbYUS00V5bm951CevLH1YK0yEyki9qR8AUZm3jhU+skgqieLZOYevc7LZBT9TxZ5XjVObpFTddfeTHkSyNd7nZ/hqFoxL24fJyu12CIoNVhmPk5pMfgLcFZEvKLHWWrZDfj9ohIEqhmqx7YdReminvRDq6qxv2tTWl+ljiZ9RZY0vjLzn5RHsfVU1d22DWUs1IaUbjZpUsnM/6Z0oU96mXkuZf1FaUAGgpImi5UpY7oeAL6QmWf1NjuSNPU5RlCSJKmhbBHsQl9f33RKV9rfGXhFfUmSpMlgMcrKAFfOnDnzKasBGAh2Z2PgN73OhCRJ0gi8hrKG5r8ZCHbn7wAveclLWGKJJQZNOHv2bGbMGO5TgKYuy6GwHArLoZ9lUVgOheVQWA79xqIs5s+fz4033ghV/FJnINidJwGWWGIJpk+fPmTi4aRpAsuhsBwKy6GfZVFYDoXlUFgO/cawLJ42nM11BCVJkhrKQFCSJKmhDAQlSZIaykBQkiSpoQwEJUmSGspAUJIkqaEMBCVJkhrKQFCSJKmhDAQlSZIayieLTGIP/3M+j857otfZGBNLL/ecXmdBkiS1mTSBYEQsDdwArA5snJlX1Y7tDHwKeBEwBzgsM89oO/+ZwGHAfwLLA1cCe2fmn9vSrQp8GXgLsBD4GfCxzLxnPO5rNB6d9wQXXHlbr7MxJjZcc6leZ0GSJLWZTF3Dh9IhMI2IbYFTgJ8As4BfAz+IiFltSb8E7AkcArwdmA9cEBGr1a61OHA+8DJgZ2A3YFPg7IiYNsb3I0mSNKlNihbBiJgB7AF8HDix7fDhwJmZ+cnq9UURsR7wWeDn1fmrV+d/NDO/We37A/A34GPAAdW5/wFsAMzIzGurdHOB31GCzPPG4/4kSZImo8nSIng88FXgxvrOiHgxsC5welv604CNI2Ll6vWbgcWAf3cXZ+bDlG7frWrnbQVc0woCq3SXAbe2pZMkSZryeh4IRsT7gLWBIzocXq/aXte2vxXIRS3dXZl5b4d0L4mIZ9TStV+rlW7dkeRbkiRpUdfTruGIWA44Gtg3Mx+JiPYkK1TbB9r2319tV6yla0/TSvdMYGngoSHSrT/8nBezZ88eVrq+vr6RXhqAZy27MnPnzu3q3MlmwzXX7rocphrLobAc+lkWheVQWA6F5dBvPMui12MEjwBuysxTe5yPrsyYMYPp06cPmqavr4+ZM2d2df277nuU1VZ7tKtzJ6Nuy2EqGU19mEosh36WRWE5FJZDYTn0G4uymDdv3oCNVz0LBCPipZQJHltExPLV7qVb24hYhv6Wv+WBO2unt1oK76u291dp2q0APA48Mox093XYL0mSNGX1cozgOpRA9CJKgHY/cE517CLgN8D11ev12s5tdeNmtb0eWCUiVuyQ7sbMXFBL136tVroburgHSZKkRVYvA8HfAq9v+9mnOrYHsFtm/o0SoO3Qdu6OwJWZeXf1+pfAAmD7VoJqgeq38dQlYc4DXlYtP9NK92rKQtUuHSNJkhqlZ13D1ZM8Lq7vq00W6as9WeRg4IyImAP8irJY9JuBrWvXuiMiTgA+HxFPUJaD2Q+YBhxbe4uzgKuBH0XEJyn3fzTwe6o1CSVJkpqi58vHDCUzzwR2BbYFfgFsCeyUme2B2z7A1ykTUM4GngW8KTPn1q71BOXRcrOB7wMnAX8AtsnMheN8K5IkSZNKr2cNP0VmXkxpxWvffwrlMXODnfs48InqZ7B0d/L0rmZJkqTGmfQtgpIkSRofBoKSJEkNZSAoSZLUUAaCkiRJDWUgKEmS1FAGgpIkSQ1lIChJktRQBoKSJEkNZSAoSZLUUAaCkiRJDWUgKEmS1FAGgpIkSQ1lIChJktRQBoKSJEkNZSAoSZLUUAaCkiRJDWUgKEmS1FAGgpIkSQ1lIChJktRQBoKSJEkNZSAoSZLUUAaCkiRJDWUgKEmS1FAGgpIkSQ1lIChJktRQBoKSJEkNZSAoSZLUUAaCkiRJDWUgKEmS1FAGgpIkSQ1lIChJktRQBoKSJEkNZSAoSZLUUAaCkiRJDWUgKEmS1FAGgpIkSQ1lIChJktRQBoKSJEkNZSAoSZLUUAaCkiRJDWUgKEmS1FAGgpIkSQ1lIChJktRQBoKSJEkNZSAoSZLUUAaCkiRJDWUgKEmS1FAGgpIkSQ1lIChJktRQBoKSJEkNtXiv3jgi3gV8HFgXWBq4A/gJcHhmPlhLNwv4HLB+lebYzDyuw/X2A/YEVgWuBQ7MzAva0iwDHA1sCywJXATslZm3jPX9SZIkTXa9bBFcEbgU+CDwFuDLwPuBM1sJImIT4GzgT8As4CTg2IjYo36hKgg8Ejge2Bq4CTg3IjZoe88fANsAewE7AKsBF0TEUmN9c5IkSZNdz1oEM/Nbbbsujoh/ASdGxGqZORc4GPhjZn6gSnNRRLwQOCQivpGZCyJiOvAZSkvhMQARcQlwDfBpYPtq36soQeLWmXlete8aYA6wC/C1cbxdSZKkSWeyjRG8p9ouUQV4bwDOaEtzGqX79xXV602B5YDTWwky80ngh8CsiJhW7d4KeBA4v5buNuB31TFJkqRG6VmLYEtELAY8E3gppQXw7My8JSLWB5YArms75dpquy5wFbBe9fr6DumWBlYHbq/S3ZCZCzqk23IMbkWSJGmR0vNAELiX0qIHpbVup+rfK1TbB9rS319tV6ylm5eZjw2S7vYqXfu1WulW7LB/SLNnzx5Wur6+vm4uz7OWXZm5c+d2de5ks+Gaa3ddDlON5VBYDv0si8JyKCyHwnLoN55lMRkCwdcBSwEzKGP9zomILXqao2GaMWMG06dPHzRNX18fM2fO7Or6d933KKut9mhX505G3ZbDVDKa+jCVWA79LIvCcigsh8Jy6DcWZTFv3rwBG696Hghm5p+rf14WEX2U7t530t8lvHzbKa2Wwvuq7f3A9IhYMjP/NUS6F3bIwgq1NJIkSY0x2SaL/BlYAKxNmc07n/4xgC3rV9sbqm1rbGCndA9T1h5spYva5JF6uhuQJElqmMkWCG5CydPNmTkPuJBq+ZeaHYE7gT9Wry+jzAbeoZWgmoCyPXB+Zi6sdp9HaV3cspbuBcDm1TFJkqRG6eWTRX4BXECZtfsvYENgf+Bq4KdVssOASyPim8CpwGbA7sCerdm/mTkvIo4AjoyIuykB4m7AWvRPPCEzL4+Ic4FvR8S+wEPV9W8DTh7Pe5UkSZqMejlG8ArgvcCLq9e3ACcAX8zM+QCZ+fuIeDvlqSE7A3OBfTLzhPqFMvOYiAD4KPBcSnC5dWb+pe09dwSOoSwePZ3yiLntMnPqzMiQJEkapl4+WeQg4KBhpDuPYXTdVk8VOWaINA8DH6p+JEmSGm2yjRGUJEnSBDEQlCRJaigDQUmSpIYyEJQkSWooA0FJkqSGMhCUJElqKANBSZKkhhpxIBgRW3Z4Xq8kSZIWMd20CP4cuD0ijo6IDcY6Q5IkSZoY3QSC7wB+B+wJ/DEiro6I/SJitTHNmSRJksbViAPBzDw7M7enPNN3d+Bu4Cjg1oj4ZUS8NyKWGuN8SpIkaYx1PVkkMx/OzO9k5huBNYBPAasApwB3RcR3I+KNY5RPSZIkjbGxmjW8GPBMYDowDXgMeBPwq4j4U0TMGKP3kSRJ0hhZvNsTI2I5YHvgvcBmwBPAucAnqu0CYBvgS8BJwMajzawkSZLGzogDwYh4ByX42wpYErgS2Bv4QWbe15b8pxHxHOBro8ynJEmSxlg3LYI/Bu4Avgyckpk3DJH+auDULt5HkiRJ46ibQPDNwAWZuXA4iTPzCuCKLt5HkiRJ42jEgWBm/no8MiJJkqSJ1c0j5r4UETcNcvzGiDh6dNmSJEnSeOtm+ZitgTMGOX4G8LbusiNJkqSJ0k0g+ALglkGO31qlkSRJ0iTWTSD4EPDiQY6vSVlQWpIkSZNYN4HghcCHIuKF7Qci4kXAh6o0kiRJmsS6WT7mYGAWMDsiTgKurfbPAHYBngQOGpPcSZIkadx0s3zMTRGxGXA8sFfb4UuAvTIzxyJzkiRJGj9dPWs4M68FXlc9Pm7NaveczLx3zHImSZKkcdVVINiSmfcA94xRXiRJkjSBugoEI2IxYEtKa+AKwLS2JAsz8/BR5k2SJEnjaMSBYERsBJwFPJ+nB4AtCwEDQUmSpEmsmxbBrwHPAt4B/CYzHxjLDEmSJGlidBMIvhz4dGaeM9aZkSRJ0sTpZkHp2xm4S1iSJEmLiG4CwaOA3SNi2bHOjCRJkiZON13DKwL/BP4aET8C/pfyNJG6hZl59GgzJ0mSpPHTTSB4VO3fewyQZiFgIChJkjSJdRMIvnjMcyFJkqQJ182zhm8dj4xIkiRpYnX9iLmIWAd4HbAKcGpm3hIRSwCrAndm5vyxyaIkSZLGQzdPFnkGcALwAcoyMguB3wO3AEsA1wCHAf89ZrmUJEnSmOtm+ZhPAe8HDgI2obamYGY+Qnn83LvGJHeSJEkaN90EgrsC38nMI4G/djh+DbDOqHIlSZKkcddNIPh84IpBjj8GLNNddiRJkjRRugkE7wTWGOT4TMCZxZIkSZNcN4HgWcB/VbOGWxYCRMQsYGfgh2OQN0mSJI2jbgLBQ4HbgD8Bp1KCwE9FxB+AnwF/Af7fWGVQkiRJ42PEgWBmPgRsChwJPBf4F7A5sDQlSHxtZj42hnmUJEnSOOhqQenM/BclEDxybLMjSZKkidJN17AkSZKmgG6eLPKdYSRbmJkf6CI/kiRJmiDddA2/gWqWcM1iwPOq7d3AP0eZL0mSJI2zEQeCmfmiTvsj4pnAh4CPAVsMdZ2I2A54D2XdwRWBOcDXgRMzc0Et3Szgc8D6wB3AsZl5XIfr7QfsCawKXAscmJkXtKVZBjga2BZYErgI2Cszbxkqv5IkSVPNmI0RzMzHM/OrwC+Brw7jlH2BecD+wFuBnwJfAT7fShARmwBnU5aqmQWcBBwbEXvUL1QFgUcCxwNbAzcB50bEBm3v+QNgG2AvYAdgNeCCiFhqJPcqSZI0FXQ1a3gIfwHeN4x0b8vMu2uvL4qIpYGPRMRnMnMecDDwx9p4w4si4oXAIRHxjcxcEBHTgc9QWgqPAYiISyjPPP40sH2171WUIHHrzDyv2ncNpSVyF+Bro7prSZKkRcx4zBreAnh0qERtQWDLnyhdtitWAd4bgDPa0pxG6f59RfV6U2A54PTatZ+kPN1kVkRMq3ZvBTwInF9Ldxvwu+qYJElSo3Qza/jgAQ4tD7yWEqAd1WV+XgPcB/wDCGAJ4Lq2NNdW23WBq4D1qtfXd0i3NLA6cHuV7ob6+MNaui27zK8kSdIiq5uu4UMH2H8/pZt1D+CbI71oRGwE7Ap8NjOfjIgVqkMPdHgfKBNMAFYA5nV4mkk93e1VuvZrtdKt2GG/JEnSlNbNrOEx706OiFWBs4ArqE0Wmexmz549rHR9fX1dXf9Zy67M3Llzuzp3stlwzbW7LoepxnIoLId+lkVhORSWQ2E59BvPshiPySIjEhHLAT+njCvcJjMfrw61WvSWbzul1VJ4Xy3d9IhYsnr03WDpXtghCyvU0ozIjBkzmD59+qBp+vr6mDlzZjeX5677HmW11YYcbrnI6LYcppLR1IepxHLoZ1kUlkNhORSWQ7+xKIt58+YN2HjVzRjBTsHUkKqJGe3XWpKyPMwqwKaZeW/t8BxgPmVs3/m1/etX2xuqbWts4HqUySb1dA9T1h5spdsiIqZl5sK2dDcgSZLUMN10894C/K2Ln6eIiMUpM3tfDszKzFvrx6vlYy6kWv6lZkfgTuCP1evLKLOBd6hde7HqvPNrQd95lNbFLWvpXgBsXh2TJElqlG66hncDPgq8gLKUy43V/qAEabdRFoZun53b7njgbcABwFIR8erasesy8yHgMODSiPgmcCqwGbA7sGdr9m9mzouII4AjI+JuSoC4G7AWsFPrgpl5eUScC3w7IvYFWte/DTi5i3KQJElapHUTCD4PmA6snZn31w9ExCGUdflWzcz/N8R1Wi1zX+hw7PXAxZn5+4h4O+WpITsDc4F9MvOEeuLMPCYioASoz6UsCbN1Zv6l7bo7AsdQFo+eTnnE3HaZOXUG4kmSJA1TN4HgHsAX24NAgMy8t2q92xsYNBAc6JnFHdKdxzC6bqunihwzRJqHKc9D/tBw3luSJGkq62aM4EqUhZoH8uwqjSRJkiaxbgLBPwB7R8TT5jJXi0LvDVw+2oxJkiRpfHXTNfwR4GLgioi4Erip2r8OsDFlTb69xiR3kiRJGjcjbhHMzOuAl1FmBi8PbFv9LA98GXhZZl470PmSJEmaHLp6skhm3gXsU/1IkiRpETSqR8xFxDqUp4LMzswHxyZLkiRJmgjdTBYhInaKiNsoj2a7FJhZ7X9ORNwYEe1PA5EkSdIkM+JAMCL+A/g+5dm9+wPTWscy855q/85jlUFJkiSNj25aBD8N/DoztwRO6XD8cmCDUeVKkiRJ466bQHA94CeDHP8HsHJ32ZEkSdJE6SYQ/CeDP1lkLeCe7rIjSZKkidJNIHghsEtELNF+ICJWA3YHfjHajEmSJGl8dTtG8HnAVcCHgYXAVhFxFHANsAD47JjlUJIkSeOimyeL3ARsBtwJHEqZNfxx4ADgz8DmmXnb2GVRkiRJ42FEC0pHxGLA6sBdmfnmiFgBWJsSUN6cmXePQx4lSZI0Dkb6ZJFnAHOAA4EvZub9wJVjnitJkiSNuxF1DWfm48BcyrhASZIkLcK6mSxyEmXW8JJjnRlJkiRNnJF2DQPcCCwG3BARpwA3A4+1J8rMH44yb5IkSRpH3QSC36/9+6AB0iwEDAQlSZImsWEFghHxFeCUzOwDXl/tXprSEvjkOOVNkiRJ42i4LYIfAf4A9GXmJRGxEuWZwltk5iXjljtJkiSNm24mi7RMG7NcSJIkacKNJhCUJEnSIsxAUJIkqaFGMmt4zYh4ZfXv5artuhHxSKfEmXnFqHImSZKkcTWSQPCz1U/dcR3STaMsH7NYt5mSJEnS+BtuILjruOZCkiRJE25YgWBmnjLeGZEkSdLEcrKIJElSQxkISpIkNZSBoCRJUkMZCEqSJDWUgaAkSVJDGQhKkiQ1lIGgJElSQxkISpIkNZSBoCRJUkMZCEqSJDWUgaAkSVJDGQhKkiQ1lIGgJElSQxkISpIkNZSBoCRJUkMZCEqSJDWUgaAkSVJDGQhKkiQ1lIGgJElSQxkISpIkNZSBoCRJUkMZCEqSJDXU4r1884hYG9gPeDUwA7ghM2d0SDcL+BywPnAHcGxmHtch3X7AnsCqwLXAgZl5QVuaZYCjgW2BJYGLgL0y85axuzNJkqTJr9ctgi8Ftgb+ClzXKUFEbAKcDfwJmAWcBBwbEXu0pdsPOBI4vrrmTcC5EbFB2yV/AGwD7AXsAKwGXBARS43RPUmSJC0SetoiCJyTmf8DEBEnAxt1SHMw8MfM/ED1+qKIeCFwSER8IzMXRMR04DOUlsJjqutdAlwDfBrYvtr3KkqQuHVmnlftuwaYA+wCfG1c7lKSJGkS6mmLYGYuGOx4FeC9ATij7dBplO7fV1SvNwWWA06vXftJ4IfArIiYVu3eCngQOL+W7jbgd9UxSZKkxuh11/BQ1gKW4OndxtdW23Wr7XrV9voO6ZYGVq+lu6FDAHpt7VqSJEmN0Ouu4aGsUG0faNt/f7VdsZZuXmY+Nki626t07ddqpVuxw/5BzZ49e1jp+vr6RnppAJ617MrMnTu3q3Mnmw3XXLvrcphqLIfCcuhnWRSWQ2E5FJZDv/Esi8keCE5qM2bMYPr06YOm6evrY+bMmV1d/677HmW11R7t6tzJqNtymEpGUx+mEsuhn2VRWA6F5VBYDv3GoizmzZs3YOPVZO8abrXoLd+2v9VSeF8t3fSIWHIY6dqv1Up3X4f9kiRJU9ZkDwTnAPPpHwPYsn61vaHatsYGdkr3MGXtwVa6qE0eqae7AUmSpAaZ1IFgZs4DLqRa/qVmR+BO4I/V68sos4F3aCWIiMWq887PzIXV7vMoLYJb1tK9ANi8OiZJktQYvX6yyFL0L9uyBrBsRGxbvb4yM28FDgMujYhvAqcCmwG7A3u2Zv9m5ryIOAI4MiLupgSIu1FmHe/Uer/MvDwizgW+HRH7Ag9V178NOHlcb1aSJGmS6fVkkVWAM9v2tV7vCpycmb+PiLdTnhqyMzAX2CczT6iflJnHRATAR4HnUpaE2Toz/9J2/R2BYyiLR0+nPGJuu8ycOrMyJqEll3wWd903NYp4qemLs8yzl+h1NiRJGrWeBoLV833bx+t1Sncew+i6rZ4qcswQaR4GPlT9aII8/uRCLrjytl5nY0y8ceMXGghKkqaEST1GUJIkSePHQFCSJKmhDAQlSZIaykBQkiSpoQwEJUmSGspAUJIkqaEMBCVJkhrKQFCSJKmhDAQlSZIaykBQkiSpoQwEJUmSGspAUJIkqaEMBCVJkhrKQFCSJKmhDAQlSZIaykBQkiSpoQwEJUmSGspAUJIkqaEMBCVJkhrKQFCSJKmhDAQlSZIaykBQkiSpoQwEJUmSGspAUJIkqaEMBCVJkhrKQFCSJKmhDAQlSZIaykBQkiSpoQwEJUmSGspAUJIkqaEMBCVJkhrKQFCSJKmhDAQlSZIaykBQkiSpoQwEJUmSGspAUJIkqaEMBCVJkhrKQFCSJKmhDAQlSZIaykBQkiSpoQwEJUmSGspAUJIkqaEMBCVJkhrKQFCSJKmhDAQlSZIaykBQkiSpoQwEJUmSGspAUJIkqaEMBCVJkhrKQFCSJKmhFu91BiZaRKwDHAdsDjwGnA4cmJmP9jRjkiRJE6xRgWBELA9cBNwKbAusAnwRWBl4d+9yJkmSNPEaFQgCHwJWADbMzHsAIuIJ4NSIODwzr+1p7iRJkiZQ08YIbgVc0AoCK2cB84BZvcmSJElSbzStRXA94Dv1HZk5LyLmAOuO4DqLAcyfP39YiefNmzeCS/d74vH5LP6MBV2dO9ksePKJKXMvTzw+n3nzFuv6/G7rw1RjOfSzLArLobAcCsuh32jLohavPO3La9rChQtHdfFFSUQ8DhyUmUe17f8t8I/MfNdwrtPX17c58JtxyKIkSdJ4ec3MmTN/W9/RtBbBsXIl8Brg78CTPc6LJEnSYBYDnkeJX56iaYHg/cDyHfavANww3IvMnDlzHvDbIRNKkiRNDnM67WzaZJHrKeME/y0ipgNrMYJAUJIkaSpoWiB4HvDGiFiptu+dwPTqmCRJUmM0bbLI8sBs4BbgcPoXlL4gM11QWpIkNUqjWgQz8wHgDcAjwI+BLwFnAO/vYbYkSZJ6olEtgpIkSerXqBZBSZIk9TMQlCRJaigDQUmSpIZq2oLSEyIi1gGOAzYHHgNOBw7MzEd7mrEJFBG7ACd1OHR8Zn5kgrMzYSJibWA/4NXADOCGzJzRId0s4HPA+sAdwLGZedxE5nU8DaccIuJk4D87nL5dZv5o3DM5ASJiO+A9wExgRcqCrl8HTszMBbV0U70+DFkOTagPABHxLuDjlOfbL035vH8CHJ6ZD9bSTfU6MWQ5NKVO1EXE0pR1jVcHNs7Mq2rHdgY+BbyI8n/osMw8Y7TvaSA4xqolai4CbgW2pX+JmpWBJi5R8xbgwdrrO3uVkQnyUmBr4HJKi/vTWt0jYhPgbOC7wL7AZsCxEfF4Zp4wgXkdT0OWQ+VmSoBQd+M45mui7Uv5XbA/cBfweuArwJrVvqbUhyHLoTLV6wOUQPhSyvfCfcDLgUOr7ZuhMXViyHKoNKFO1B1Kh9gsIrYFTgGOAn4JvAP4QUQ8lJk/H80bOmt4jEXEgcDBwBqZeU+1byfgVGBGZl7by/xNlFqL4MqtcmiCiHhGWwvHRh1awn4OrJiZr6rt+wbwNmD1ekvRomqY5dBx/1QSEStn5t1t+74I/BewfGbOa0h9GE45nMwUrw8DiYgPAidSPu+5TagTnXQoh5NpUJ2IiBnAHygtpSdSaxGMiOuBazJz+1r6X1L+/7xyNO/rGMGxtxVlgep68HMWMA+Y1ZssaaIM9Qu6eqThGyjrV9adBqwKvGKcsjahpuoX1Ui1Bz+VPwFLAis2qD4MWg4TnJ3JqPV9sURT6sQA/l0OPc1F7xwPfJW2Fs+IeDGlC/30tvSnARtHxMqjeVO7hsfeesB36juqv3bnUD7IppldVdLbgJOBz2XmE73NUk+tRfkld13b/lZL8brAVTTHWhHxAPBsylN/jhqLMS+T3GsoXWH/AILm1od6ObQ0pj5ExGLAMynDKA4Gzs7MWyJifRpUJwYqh1qSRtSJiHgfsDZlSM1GbYfXq7YD1YkAOv2xNSy2CI69FYAHOuy/n2b95ft34BBgF8o4wZ8ABwHf6mGeJoMVqu0Dbfvvr7ZNqiN/okwoeQdlPO3twOnVsIIpKSI2AnYFvpSZT9LQ+tChHKB59eFeymTCqyi/L3eq9jetTgxUDtCQOhERywFHAwdk5iMdkoxrnbBFUOMiM38B/KK261cR8SBwaEQcnplzepQ1TRKZ+eW2Xf8TERcCn6W0Hk8pEbEqZZjIFcDne5ydnhmoHJpWH4DXAUtRZtV/BjgnIrboaY5643V0KIfMfLJBdeII4KbMPLUXb26L4Ni7H1i+w/4VKN0gTfbDajuVx7gMpfUX3PJt+1t/8TW9jpwJvHC0Y14mm+ov/p8DjwLbZObj1aFG1YdBymEgU7I+AGTmnzPzssz8BvBOykzqd9KwOjFIOQxkStWJiHgpsAdwUEQsX608snR1eOmIWIZxrhMGgmPvevr784F/TxBYi7I2kJptDjCftjpCWSsMrCNTTkQsSVkKZBXgLZl5b+1wY+rDEOXQdH8GFlDGiDWmTnTwZ/rLoSnWofTOXkQJ+O4HzqmOXQT8hhJXwMB1IkeTAQPBsXce8MaIWKm2753A9OpYk70bWAj09TojvZKZ84ALge3bDu1IWWPxjxOeqUkiIqZRyuXWAWaZLnIiYnFKS/jLgVmZeWv9eFPqw1DlMMA5U64+DGITyvfxzU2pEwP4dzl0OjhF68RvKa2g9Z99qmN7ALtl5t8ofwDs0HbujsCVoy0LxwiOvROBvShjGQ6nf0HpMzKzfcbPlBURv6D8MptN+QtvFvBh4NuZ2fE/+VQQEUtRlhACWANYtloIFMp/2FuBw4BLI+KblPUlNwN2B/acKsuuDFUO1fYU4AfAXyldHrtRxgu9b8IyOv6Op6z9dgCwVES8unbsusx8iAbUB4YoB0oXVxPqQ+t34wWUGZ//AjakLKp9NfDTKtmUrxNDlUNErEED6kS11NzF9X0R0fpnX+3JIgcDZ1QrkPwKeDtl4e2tR5sHA8ExlpkPRMQbKKvm/5j+R8wd0NOMTbzrgfcDz6fUs5uAA4Fje5inibAKZQxLXev1rsDJmfn7iHg7cCSwMzAX2GcKPTEAhi6HsylPnPlMlfZxSkvHNpl5DlPHltX2Cx2OvR64uCH1YahyuJpm1Acok2TeC7y4en0LcALwxcycD9CQOjFoOUTEwzSnTgwpM8+s/sD+FGUm9Rxgp9E+VQR8sogkSVJjOUZQkiSpoQwEJUmSGspAUJIkqaEMBCVJkhrKQFCSJKmhDAQlSZIaykBQkiSpoQwEJUmSGur/A0w6fGYym9kTAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "df1 = df[df[\"name\"].isin([\"QuerySamplesComplete\"])]\n", + "df1['delta'] = df1['ts'].diff()\n", + "ax = df1['delta'].plot.hist(bins=BINS, alpha=0.5, figsize=figsize)\n", + "ax.set_title('Time between QuerySamplesComplete (usec)');\n", + "plt.show()\n", + "\n", + "ax = df1['dur'].plot.hist(bins=BINS, alpha=0.5, figsize=figsize)\n", + "ax.set_title('Time QuerySamplesComplete (usec)');" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.3" + }, + "varInspector": { + "cols": { + "lenName": 16, + "lenType": 16, + "lenVar": 40 + }, + "kernels_config": { + "python": { + "delete_cmd_postfix": "", + "delete_cmd_prefix": "del ", + "library": "var_list.py", + "varRefreshCmd": "print(var_dic_list())" + }, + "r": { + "delete_cmd_postfix": ") ", + "delete_cmd_prefix": "rm(", + "library": "var_list.r", + "varRefreshCmd": "cat(var_dic_list()) " + } + }, + "types_to_exclude": [ + "module", + "function", + "builtin_function_or_method", + "instance", + "_Feature" + ], + "window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/utils.cc b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/utils.cc new file mode 100644 index 000000000..de74eb820 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/utils.cc @@ -0,0 +1,124 @@ +/* Copyright 2019 The MLPerf Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "utils.h" + +#include +#include +#include +#include +#include + +#include "logging.h" + +namespace mlperf { + +std::string DoubleToString(double value, int precision) { + std::stringstream ss; + ss.precision(precision); + ss << std::fixed << value; + return ss.str(); +} + +bool FileExists(const std::string filename) { + std::ifstream file_object(filename); + return file_object.good(); +} + +namespace { + +std::string DateTimeString(const char* format, + std::chrono::system_clock::time_point tp, + bool append_ms, bool utc) { + std::time_t tp_time_t = std::chrono::system_clock::to_time_t(tp); + std::tm date_time = + utc ? *std::gmtime(&tp_time_t) : *std::localtime(&tp_time_t); + constexpr size_t kDateTimeMaxSize = 256; + char date_time_cstring[kDateTimeMaxSize]; + std::strftime(date_time_cstring, kDateTimeMaxSize, format, &date_time); + std::string date_time_string(date_time_cstring); + if (!append_ms) { + return date_time_string; + } + + auto tp_time_t_part = std::chrono::system_clock::from_time_t(tp_time_t); + auto tp_remainder = tp - tp_time_t_part; + auto ms = std::chrono::duration_cast(tp_remainder) + .count(); + if (ms < 0 || ms >= 1000) { + LogDetail([ms](AsyncDetail& detail) { +#if USE_NEW_LOGGING_FORMAT + std::stringstream ss; + ss << "WARNING: Unexpected milliseconds getting date and time." + << " ms: " << ms; + MLPERF_LOG_WARNING(detail, "warning_generic_message", ss.str()); +#else + detail("WARNING: Unexpected milliseconds getting date and time.", "ms", + ms); +#endif + }); + } + std::string ms_string = std::to_string(ms); + // Prefix with zeros so length is always 3. + ms_string.insert(0, std::min(2, 3 - ms_string.length()), '0'); + return date_time_string + "." + ms_string; +} + +} // namespace + +std::string CurrentDateTimeISO8601() { + return DateTimeString("%FT%TZ", std::chrono::system_clock::now(), false, + false); +} + +std::string DateTimeStringForPower(std::chrono::system_clock::time_point tp) { + return DateTimeString("%m-%d-%Y %T", tp, true, true); +} + +std::string EscapeStringJson(const std::string& in) { + std::stringstream ss; + for (auto c = in.cbegin(); c != in.cend(); c++) { + int c_val = static_cast(*c); + switch (*c) { + case '"': + ss << "\\\""; + break; + case '\\': + ss << "\\\\"; + break; + case '\b': + ss << "\\b"; + break; + case '\f': + ss << "\\f"; + break; + case '\n': + ss << "\\n"; + break; + case '\r': + ss << "\\r"; + break; + case '\t': + ss << "\\t"; + break; + default: + if (c_val >= 0x00 && c_val < 0x20) { + ss << "\\u" << std::hex << std::setw(4) << std::setfill('0') << c_val; + } else { + ss << *c; + } + } + } + return ss.str(); +} + +} // namespace mlperf diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/utils.h b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/utils.h new file mode 100644 index 000000000..c587e0cbe --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/utils.h @@ -0,0 +1,70 @@ +/* Copyright 2019 The MLPerf Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +/// \file +/// \brief Various shared utility functions. + +#ifndef MLPERF_LOADGEN_UTILS_H +#define MLPERF_LOADGEN_UTILS_H + +#include +#include +#include + +#include "query_sample.h" + +namespace mlperf { + +template +void RemoveValue(T* container, const typename T::value_type& value_to_remove) { + container->erase(std::remove_if(container->begin(), container->end(), + [&](typename T::value_type v) { + return v == value_to_remove; + }), + container->end()); +} + +template +double DurationToSeconds( + const std::chrono::duration& chrono_duration) { + return std::chrono::duration_cast>( + chrono_duration) + .count(); +} + +inline double QuerySampleLatencyToSeconds(QuerySampleLatency qsl) { + return static_cast(qsl) / std::nano::den; +} + +template +inline DurationT SecondsToDuration(double seconds) { + return std::chrono::duration_cast( + std::chrono::duration(seconds)); +} + +std::string CurrentDateTimeISO8601(); + +/// \brief Uses a format that matches the one used by SPEC power +/// measurement logging. +std::string DateTimeStringForPower(std::chrono::system_clock::time_point tp); + +std::string DoubleToString(double value, int precision = 2); + +bool FileExists(const std::string filename); + +// \brief Escape special characters in a string for JSON. +// Don't use this in performance critical path. +std::string EscapeStringJson(const std::string& in); + +} // namespace mlperf + +#endif // MLPERF_LOADGEN_UTILS_H diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/version.cc b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/version.cc new file mode 100644 index 000000000..3216c9d72 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/version.cc @@ -0,0 +1,85 @@ +/* Copyright 2019 The MLPerf Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +/// \file +/// \brief Non-generated version logic. + +#include "version.h" + +#include "logging.h" +#include "utils.h" + +namespace mlperf { + +/// Helper function to split a string based on a delimiting character. +std::vector splitString(const std::string& input, + const std::string& delimiter) { + std::vector result; + size_t start = 0; + size_t next = 0; + while (next != std::string::npos) { + next = input.find(delimiter, start); + result.emplace_back(input, start, next - start); + start = next + 1; + } + return result; +} + +/// Converts the hash-filename pairs to a dict. +std::map LoadgenSha1OfFilesToDict( + const std::string& in) { + std::map result; + auto files = splitString(in, "\n"); + for (const auto& file : files) { + auto hash_and_name = splitString(file, " "); + assert(hash_and_name.size() > 1); + result[hash_and_name[1]] = hash_and_name[0]; + } + return result; +} + +void LogLoadgenVersion() { + LogDetail([](AsyncDetail& detail) { +#if USE_NEW_LOGGING_FORMAT + MLPERF_LOG(detail, "loadgen_version", + LoadgenVersion() + " @ " + LoadgenGitRevision()); + MLPERF_LOG(detail, "loadgen_build_date_local", LoadgenBuildDateLocal()); + MLPERF_LOG(detail, "loadgen_build_date_utc", LoadgenBuildDateUtc()); + MLPERF_LOG(detail, "loadgen_git_commit_date", LoadgenGitCommitDate()); + MLPERF_LOG(detail, "loadgen_git_log_message", + EscapeStringJson(LoadgenGitLog())); + MLPERF_LOG(detail, "loadgen_git_status_message", + EscapeStringJson(LoadgenGitStatus())); + if (!LoadgenGitStatus().empty() && LoadgenGitStatus() != "NA") { + MLPERF_LOG_ERROR(detail, "error_uncommitted_loadgen_changes", + "Loadgen built with uncommitted changes!"); + ; + } + MLPERF_LOG(detail, "loadgen_file_sha1", + LoadgenSha1OfFilesToDict(LoadgenSha1OfFiles())); +#else + detail("LoadgenVersionInfo:"); + detail("version : " + LoadgenVersion() + " @ " + LoadgenGitRevision()); + detail("build_date_local : " + LoadgenBuildDateLocal()); + detail("build_date_utc : " + LoadgenBuildDateUtc()); + detail("git_commit_date : " + LoadgenGitCommitDate()); + detail("git_log :\n\n" + LoadgenGitLog() + "\n"); + detail("git_status :\n\n" + LoadgenGitStatus() + "\n"); + if (!LoadgenGitStatus().empty() && LoadgenGitStatus() != "NA") { + detail.Error("Loadgen built with uncommitted changes!"); + } + detail("SHA1 of files :\n\n" + LoadgenSha1OfFiles() + "\n"); +#endif + }); +} + +} // namespace mlperf diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/version.h b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/version.h new file mode 100644 index 000000000..87c3409aa --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/version.h @@ -0,0 +1,39 @@ +/* Copyright 2019 The MLPerf Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +/// \file +/// \brief Declares the version-related strings that will be defined in +/// a version_generated.cc as created by version_generator.py. + +#ifndef MLPERF_LOADGEN_VERSION_H +#define MLPERF_LOADGEN_VERSION_H + +#include + +namespace mlperf { + +// Non-generated. +void LogLoadgenVersion(); + +// Definitions generated at compile time. +const std::string& LoadgenVersion(); +const std::string& LoadgenGitRevision(); +const std::string& LoadgenBuildDateLocal(); +const std::string& LoadgenBuildDateUtc(); +const std::string& LoadgenGitCommitDate(); +const std::string& LoadgenGitStatus(); +const std::string& LoadgenGitLog(); +const std::string& LoadgenSha1OfFiles(); + +} // namespace mlperf + +#endif // MLPERF_LOADGEN_VERSION_H diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/version_generator.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/version_generator.py new file mode 100644 index 000000000..2e7524330 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/version_generator.py @@ -0,0 +1,141 @@ +# Copyright 2019 The MLPerf Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +# \file +# \brief A script run by the build to generate the version definitions +# expected at link time. + +import datetime +import errno +import hashlib +import os +import sys +import subprocess + + +# Creates a C++ raw string literal using a delimiter that is very +# unlikely to show up in a git stats. +def make_raw_string(str): + delimeter = "LGVG_RSLD" + return 'R"' + delimeter + "(" + str + ")" + delimeter + '"' + + +def func_def(name, string): + return ( + "const std::string& Loadgen" + + name + + "() {\n" + + " static const std::string str = " + + string + + ";\n" + + " return str;\n" + + "}\n\n" + ) + + +# For clients that build the loadgen from the git respository without +# any modifications. +def generate_loadgen_version_definitions_git(ofile, git_command): + git_rev = os.popen(git_command + "rev-parse --short=10 HEAD").read() + git_commit_date = os.popen( + git_command + + "log --format=\"%cI\" -n 1").read() + git_status = os.popen(git_command + "status -s -uno .").read() + git_log = subprocess.Popen( + git_command + "log --pretty=oneline -n 16 --no-decorate", stdout=subprocess.PIPE, shell=True, encoding='ascii', errors="ignore").stdout.read() + ofile.write(func_def("GitRevision", "\"" + git_rev[0:-1] + "\"")) + ofile.write(func_def("GitCommitDate", "\"" + git_commit_date[0:-1] + "\"")) + ofile.write(func_def("GitStatus", make_raw_string(git_status[0:-1]))) + ofile.write(func_def("GitLog", make_raw_string(git_log[0:-1]))) + + +# For clients that might not import the loadgen code as the original git +# repository. +def generate_loadgen_verstion_definitions_git_stubs(ofile): + na = '"NA"' + ofile.write(func_def("GitRevision", na)) + ofile.write(func_def("GitCommitDate", na)) + ofile.write(func_def("GitStatus", na)) + ofile.write(func_def("GitLog", na)) + + +# Always log the sha1 of the loadgen files, regardless of whether we are +# in the original git repository or not. +def generate_loadgen_version_definitions_sha1(ofile, loadgen_root): + """Writes definition for Sha1OfFiles.""" + sha1s = "" + loadgen_files = [ + "/bindings/" + s for s in os.listdir(loadgen_root + "/bindings") + ] + ["/" + s for s in os.listdir(loadgen_root)] + for fn in sorted(loadgen_files): + full_fn = loadgen_root + fn + if not os.path.isfile(full_fn): + continue + file_data = open(full_fn, "rb").read() + sha1s += hashlib.sha1(file_data).hexdigest() + " " + fn + "\n" + + ofile.write(func_def("Sha1OfFiles", make_raw_string(sha1s[0:-1]))) + + +# Outputs version function definitions to cc_filename. +# Includes SHA1's of the relevant dirs in the loadgen_root directory. +def generate_loadgen_version_definitions(cc_filename, loadgen_root): + """Generates the C++ source file with the loadgen version info.""" + try: + os.makedirs(os.path.dirname(cc_filename)) + except OSError as exc: + if exc.errno != errno.EEXIST: + raise + ofile = open(cc_filename, "w") + ofile.write("// DO NOT EDIT: Autogenerated by version_generator.py.\n\n") + ofile.write("#include \n\n") + ofile.write("namespace mlperf {\n\n") + # Open and read the VERSION.txt file + with open(os.path.join(loadgen_root, "VERSION.txt"), "r") as version_file: + # Read and strip any extra whitespace/newlines + version_contents = version_file.read().strip() + + # Write the version into the function definition + ofile.write(func_def("Version", f"\"{version_contents}\"")) + + date_time_now_local = datetime.datetime.now().isoformat() + date_time_now_utc = datetime.datetime.utcnow().isoformat() + ofile.write(func_def("BuildDateLocal", '"' + date_time_now_local + '"')) + ofile.write(func_def("BuildDateUtc", '"' + date_time_now_utc + '"')) + + git_dir = '--git-dir="' + loadgen_root + '/../.git" ' + git_work_tree = '--work-tree="' + loadgen_root + '/.." ' + git_command = "git " + git_dir + git_work_tree + git_status = os.popen(git_command + "status") + git_status.read() + is_git_repo = git_status.close() is None + if is_git_repo: + generate_loadgen_version_definitions_git(ofile, git_command) + else: + generate_loadgen_verstion_definitions_git_stubs(ofile) + generate_loadgen_version_definitions_sha1(ofile, loadgen_root) + + ofile.write("} // namespace mlperf\n") + ofile.close() + + +def main(): + if len(sys.argv) != 3: + raise ValueError("Incorrect command-line arguments.") + generate_loadgen_version_definitions(sys.argv[1], sys.argv[2]) + + +if __name__ == "__main__": + main() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/ts_types.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/ts_types.py new file mode 100644 index 000000000..cb558726b --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/ts_types.py @@ -0,0 +1,70 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-strict + +""" +TorchScript-friendly boundary types for the HSTU sparse <-> dense interface. + +The eager path uses ``Dict[str, SequenceEmbedding]`` (a NamedTuple of +``lengths`` and ``embedding`` tensors). TorchScript supports ``NamedTuple`` but +does not script cleanly through ``Dict[str, NamedTuple]`` once the dict crosses +device boundaries. The packaged sparse / dense modules instead exchange two +parallel ``Dict[str, Tensor]`` dicts -- one of jagged values, one of lengths. + +These helpers convert between the two representations so we can keep the +existing eager code unchanged while the scripted modules use only TS-friendly +types at their boundaries. +""" + +from typing import Dict, Tuple + +import torch +from generative_recommenders.modules.dlrm_hstu import SequenceEmbedding + + +# Per-feature jagged values (concatenated across batch, [L_total, table_dim]). +SeqEmbValues = Dict[str, torch.Tensor] +# Per-feature per-batch lengths ([B]). +SeqEmbLengths = Dict[str, torch.Tensor] + + +def flatten_seq_embeddings( + seq_embeddings: Dict[str, SequenceEmbedding], +) -> Tuple[SeqEmbValues, SeqEmbLengths]: + """Split ``Dict[str, SequenceEmbedding]`` into parallel value/length dicts. + + Lossless and zero-copy -- the returned tensors alias the inputs. + """ + values: Dict[str, torch.Tensor] = {} + lengths: Dict[str, torch.Tensor] = {} + for k, v in seq_embeddings.items(): + values[k] = v.embedding + lengths[k] = v.lengths + return values, lengths + + +def unflatten_seq_embeddings( + values: SeqEmbValues, + lengths: SeqEmbLengths, +) -> Dict[str, SequenceEmbedding]: + """Inverse of :func:`flatten_seq_embeddings`. + + Reconstructs ``Dict[str, SequenceEmbedding]`` for code paths (e.g. + ``DlrmHSTU.main_forward``) that still consume the NamedTuple form. + """ + out: Dict[str, SequenceEmbedding] = {} + for k, val in values.items(): + out[k] = SequenceEmbedding(lengths=lengths[k], embedding=val) + return out diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/user.conf b/recommendation_v4/generative_recommenders/dlrm_v3/inference/user.conf new file mode 100644 index 000000000..c6ca854f9 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/inference/user.conf @@ -0,0 +1,5 @@ +# Please set these fields depending on the performance of your system to +# override default LoadGen settings. +*.Server.target_latency = 80 +# *.Server.min_duration = 20000 +# *.Offline.min_duration = 20000 diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/preprocess_public_data.py b/recommendation_v4/generative_recommenders/dlrm_v3/preprocess_public_data.py new file mode 100644 index 000000000..488af712d --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/preprocess_public_data.py @@ -0,0 +1,211 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe +import argparse +import logging +import os +import tarfile +from typing import Dict, List +from urllib.request import urlretrieve + +import numpy as np +import pandas as pd + + +logging.basicConfig(level=logging.INFO) +log = logging.getLogger("main") + +""" +Usage: mkdir -p data/ && python3 preprocess_public_data.py --dataset kuairand-1k +""" + +SUPPORTED_DATASETS = ["kuairand-1k", "kuairand-27k"] + + +def get_feature_merge_weights(dataset: str = "debug") -> Dict[str, int]: + if "kuairand" in dataset: + return { + "is_click": 1, + "is_like": 2, + "is_follow": 4, + "is_comment": 8, + "is_forward": 16, + "is_hate": 32, + "long_view": 64, + "is_profile_enter": 128, + } + else: + return {"dummy": 1} + + +class DataProcessor: + def __init__( + self, + download_url: str, + data_path: str, + file_name: str, + prefix: str, + ) -> None: + self._download_url = download_url + self._data_path = data_path + self._file_name = file_name + self._prefix = prefix + + def download(self) -> None: + return + + def preprocess(self) -> None: + return + + def file_exists(self, name: str) -> bool: + return os.path.isfile("%s/%s" % (os.getcwd(), name)) + + +class DLRMKuaiRandProcessor(DataProcessor): + def __init__( + self, + download_url: str, + data_path: str, + file_name: str, + prefix: str, + ) -> None: + super().__init__(download_url, data_path, file_name, prefix) + if prefix == "KuaiRand-1K": + self._log_files: List[str] = [ + f"{data_path}{prefix}/data/log_standard_4_08_to_4_21_1k.csv", + f"{data_path}{prefix}/data/log_standard_4_22_to_5_08_1k.csv", + ] + self._user_features_file: str = ( + f"{data_path}{prefix}/data/user_features_1k.csv" + ) + elif prefix == "KuaiRand-27K": + self._log_files: List[str] = [ + f"{data_path}{prefix}/data/log_standard_4_08_to_4_21_27k_part1.csv", + f"{data_path}{prefix}/data/log_standard_4_08_to_4_21_27k_part2.csv", + f"{data_path}{prefix}/data/log_standard_4_22_to_5_08_27k_part1.csv", + f"{data_path}{prefix}/data/log_standard_4_22_to_5_08_27k_part2.csv", + ] + self._user_features_file: str = ( + f"{data_path}{prefix}/data/user_features_27k.csv" + ) + self._output_file: str = f"{data_path}{prefix}/data/processed_seqs.csv" + self._event_merge_weight: Dict[str, int] = get_feature_merge_weights( + prefix.lower() + ) + + def download(self) -> None: + file_path = f"{self._data_path}{self._file_name}" + if not self.file_exists(file_path): + log.info(f"Downloading {self._download_url}") + urlretrieve(self._download_url, file_path) + log.info(f"Downloaded to {file_path}") + with tarfile.open(file_path, "r:*") as tar_ref: + tar_ref.extractall(path=self._data_path) + log.info("Data files extracted") + os.remove(file_path) + log.info("Tar file removed") + + def preprocess(self) -> None: + self.download() + log.info("Preprocessing data...") + seq_cols = [ + "video_id", + "time_ms", + "action_weights", + "play_time_ms", + "duration_ms", + ] + df = None + for idx, log_file in enumerate(self._log_files): + log.info(f"Processing {log_file}...") + log_df = pd.read_csv( + log_file, + delimiter=",", + ) + df_grouped_by_user = log_df.groupby("user_id").agg(list).reset_index() + + for event, weight in self._event_merge_weight.items(): + df_grouped_by_user[event] = df_grouped_by_user[event].apply( + lambda seq: np.where(np.array(seq) == 0, 0, weight) + ) + + events = list(self._event_merge_weight.keys()) + df_grouped_by_user["action_weights"] = df_grouped_by_user.apply( + lambda row: [int(sum(x)) for x in zip(*[row[col] for col in events])], + axis=1, + ) + df_grouped_by_user = df_grouped_by_user[["user_id"] + seq_cols] + + if idx == 0: + df = df_grouped_by_user + else: + df = df.merge(df_grouped_by_user, on="user_id", suffixes=("_x", "_y")) + for col in seq_cols: + df[col] = df.apply( + lambda row: row[col + "_x"] + row[col + "_y"], axis=1 + ) + df = df.drop(columns=[col + "_x", col + "_y"]) + + max_seq_len = df["video_id"].apply(len).max() + min_seq_len = df["video_id"].apply(len).min() + average_seq_len = df["video_id"].apply(len).mean() + log.info(f"{max_seq_len=}, {min_seq_len=}, {average_seq_len=}") + + log.info("Merging user features...") + user_features_df = pd.read_csv(self._user_features_file, delimiter=",") + + def _one_hot_encode(row): + mapping = {category: i + 1 for i, category in enumerate(row.unique())} + row = row.map(mapping) + return row + + for col in [ + "user_active_degree", + "follow_user_num_range", + "fans_user_num_range", + "friend_user_num_range", + "register_days_range", + ]: + user_features_df[col] = _one_hot_encode(user_features_df[col]) + + final_df = pd.merge(df, user_features_df, on="user_id") + final_df.to_csv(self._output_file, index=False, sep=",") + log.info(f"Processed file saved to {self._output_file}") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--dataset", choices=SUPPORTED_DATASETS, help="dataset") + args = parser.parse_args() + if args.dataset == "kuairand-1k": + kuairand_processor = DLRMKuaiRandProcessor( + download_url="https://zenodo.org/records/10439422/files/KuaiRand-1K.tar.gz", + data_path="data/", + file_name="KuaiRand-1K.tar.gz", + prefix="KuaiRand-1K", + ) + kuairand_processor.preprocess() + elif args.dataset == "kuairand-27k": + kuairand_processor = DLRMKuaiRandProcessor( + download_url="https://zenodo.org/records/10439422/files/KuaiRand-27K.tar.gz", + data_path="data/", + file_name="KuaiRand-27K.tar.gz", + prefix="KuaiRand-27K", + ) + kuairand_processor.preprocess() + + +if __name__ == "__main__": + main() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/streaming_synthetic_data.py b/recommendation_v4/generative_recommenders/dlrm_v3/streaming_synthetic_data.py new file mode 100644 index 000000000..bb9e508af --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/streaming_synthetic_data.py @@ -0,0 +1,664 @@ +# pyre-strict +""" +Streaming synthetic data generator for DLRMv3. + +This module generates synthetic streaming recommendation data for benchmarking +and testing purposes. It creates user-item interaction histories with timestamps, +ratings, and category-based item distributions. +""" + +import csv +import logging +import math +import multiprocessing +import os +import random +import shutil +import time +from typing import Dict, List, Tuple + +import numpy as np + +logger: logging.Logger = logging.getLogger(__name__) + + +class StreamingSyntheticDataGenerator: + """ + Generator for streaming synthetic recommendation data. + + Creates realistic user-item interaction data with temporal dynamics, + category preferences, and rating distributions for benchmarking + recommendation systems. + + Args: + num_categories: Number of item categories. + categories_per_user: Number of categories each user is interested in. + num_users: Total number of users to generate. + num_items: Total number of items in the catalog. + num_timestamps: Number of time periods in the streaming data. + avg_samples_per_item: Average number of interactions per item. + train_ratio: Fraction of timestamps used for training. + user_sampling_ratio: Probability of sampling a user at each timestamp. + num_eval_candidates: Number of candidates for evaluation. + num_inference_candidates: Number of candidates for inference. + debug: If True, use deterministic ratings for debugging. + rank: Process rank for distributed generation. + """ + + def __init__( + self, + num_categories: int, + categories_per_user: int, + num_users: int, + num_items: int, + num_timestamps: int, + avg_samples_per_item: int, + train_ratio: float, + user_sampling_ratio: float, + num_eval_candidates: int, + num_inference_candidates: int, + debug: bool = False, + rank: int = 0, + ) -> None: + self.num_categories = num_categories + self.categories_per_user = categories_per_user + self.num_users = num_users + self.num_items = num_items + self.num_timestamps = num_timestamps + self.avg_samples_per_item = avg_samples_per_item + self.avg_seq_len_per_timestamp = int( + num_items * avg_samples_per_item / num_users / num_timestamps + ) + self.items_per_category: int = num_items // num_categories + self.category_to_start_end_item_idx: Dict[int, Tuple[int, int]] = {} + for i in range(num_categories): + start_idx = i * self.items_per_category + end_idx = (i + 1) * self.items_per_category + self.category_to_start_end_item_idx[i] = (start_idx, end_idx) + self.alpha_range = (1, 500) + self.min_seq_len: int = num_eval_candidates + 1 + self.train_ratio = train_ratio + self.num_eval_candidates = num_eval_candidates + self.num_inference_candidates = num_inference_candidates + self.debug = debug + self.total_cnt = 0 + self.rank = rank + logger.warning(f"rank {self.rank}: start generating item rating") + np.random.seed(1001) + self.item_rating = np.random.choice( # pyre-ignore [4] + [5.0, 4.0, 3.0, 2.0, 1.0], size=num_items, p=[0.2, 0.25, 0.25, 0.2, 0.1] + ) + logger.warning(f"rank {self.rank}: finish generating item rating") + self.user_sampling_ratio = user_sampling_ratio + + def generate_one_timestamp( + self, + category_to_cnt: Dict[int, int], + categories: List[int], + t: int, + id: int, + output_folder: str, + uih_seq_len: int, + eval: bool, + inference: bool, + file_idx: int, + ts_buffers: Dict[int, List[int]], + ) -> Tuple[List[int], List[float], List[int], List[float], Dict[int, int]]: + """ + Generate interaction data for a single user at one timestamp. + + Args: + category_to_cnt: Running count of interactions per category. + categories: Categories this user is interested in. + t: Current timestamp index. + id: User ID. + output_folder: Output directory for files. + uih_seq_len: Length of user interaction history to generate. + eval: Whether this is for evaluation. + inference: Whether this is for inference. + file_idx: File index for output. + ts_buffers: Buffer for timestamp data. + + Returns: + Tuple of (uih_item_ids, uih_ratings, candidate_ids, candidate_ratings, + updated_category_counts). + """ + if t >= 0 and (not eval): + if t not in ts_buffers: + ts_buffers[t] = [] + ts_buffers[t].append(id) + seq_len: int = self.num_inference_candidates if inference else uih_seq_len + self.total_cnt += seq_len + alpha = random.randint(self.alpha_range[0], self.alpha_range[1]) + total_cnt = sum(category_to_cnt.values()) + p = np.array( + [ + (alpha / len(categories) + category_to_cnt[c]) / (alpha + total_cnt) + for c in categories + ] + ) + item_categories = np.random.choice(categories, size=seq_len, p=p) + unique, counts = np.unique(item_categories, return_counts=True) + for cat, cnt in zip(unique, counts): + category_to_cnt[cat] += int(cnt) + sample_end_idx = int( + self.items_per_category * max((t + 1), 1) / self.num_timestamps + ) + sample_inds = np.random.randint(0, sample_end_idx, size=seq_len) + offsets = np.array( + [self.category_to_start_end_item_idx[cat][0] for cat in item_categories] + ) + sample_inds = sample_inds + offsets + num_categories = len(categories) + quarter = num_categories // 4 + half = num_categories // 2 + three_quarter = num_categories // 4 * 3 + category_to_ratings = {} + cos1 = math.cos(t * math.pi / 4) + cos2 = math.cos((t + 2) * math.pi / 4) + cos3 = math.cos((t + 4) * math.pi / 4) + for i, cat in enumerate(categories): + if i < quarter: + if self.debug: + ratings = np.full(seq_len, 5.0) + else: + ratings = np.random.choice( + [4.5 + 0.5 * cos1, 4.0 + 0.5 * cos2], + size=seq_len, + p=[0.8, 0.2], + ) + elif i < half: + if self.debug: + ratings = np.full(seq_len, 4.0) + else: + ratings = np.random.choice( + [4.5 + 0.5 * cos1, 4.0 + 0.5 * cos2, 3.5 + 0.5 * cos3], + size=seq_len, + p=[0.1, 0.8, 0.1], + ) + elif i < three_quarter: + if self.debug: + ratings = np.full(seq_len, 3.0) + else: + ratings = np.random.choice( + [3.5 + 0.5 * cos1, 3.0 + 0.5 * cos2, 2.5 + 0.5 * cos3], + size=seq_len, + p=[0.1, 0.8, 0.1], + ) + else: + if self.debug: + ratings = np.full(seq_len, 2.0) + else: + ratings = np.random.choice( + [2.5 + 0.5 * cos1, 2.0 + 0.5 * cos2, 1.5 + 0.5 * cos3], + size=seq_len, + p=[0.1, 0.8, 0.1], + ) + category_to_ratings[cat] = ratings + sample_inds = sample_inds.tolist() + sample_ratings = [ + ( + category_to_ratings[item_categories[i]][i] + + self.item_rating[sample_inds[i]] + ) + / 2 + for i in range(seq_len) + ] + if not inference: + sub_indices = random.sample(range(seq_len), self.num_eval_candidates) + sample_candidate_inds = [sample_inds[i] for i in sub_indices] + sample_candidate_ratings = [sample_ratings[i] for i in sub_indices] + sample_uih_inds = sample_inds + sample_uih_ratings = sample_ratings + else: + sub_indices = random.sample(range(seq_len), uih_seq_len) + sample_uih_inds = [sample_inds[i] for i in sub_indices] + sample_uih_ratings = [sample_ratings[i] for i in sub_indices] + sample_candidate_inds = sample_inds + sample_candidate_ratings = sample_ratings + return ( + sample_uih_inds, + sample_uih_ratings, + sample_candidate_inds, + sample_candidate_ratings, + category_to_cnt, + ) + + def gen_rand_seq_len(self) -> int: + """ + Generate a random sequence length from a Gaussian distribution. + + Returns: + Sequence length, guaranteed to be at least min_seq_len. + """ + seq_len = round( + random.gauss( + self.avg_seq_len_per_timestamp, self.avg_seq_len_per_timestamp // 4 + ) + ) + seq_len = self.min_seq_len if seq_len < self.min_seq_len else seq_len + return seq_len + + def get_timestamp_sample(self, t: int) -> int: + """ + Determine if a user should be sampled at this timestamp. + + Args: + t: Timestamp index. Base timestamp (-1) is always sampled. + + Returns: + 1 if the user should be sampled, 0 otherwise. + """ + if t == -1: + sample = 1 + else: + sample = np.random.choice( + [1, 0], + size=1, + p=[self.user_sampling_ratio, 1 - self.user_sampling_ratio], + )[0] + return sample + + def generate_one_user( + self, + id: int, + output_folder: str, + file_idx: int, + ts_buffers: Dict[int, List[int]], + ) -> List[str]: + """ + Generate complete interaction history for one user. + + Creates training, evaluation, and inference data for a single user + across all timestamps. + + Args: + id: User ID. + output_folder: Output directory. + file_idx: File index for output. + ts_buffers: Buffer for timestamp metadata. + + Returns: + List of CSV row values for this user's data. + """ + categories = random.sample(range(self.num_categories), self.categories_per_user) + category_to_cnt = {c: 0 for c in categories} + out_list: List[str] = [] + # t = -1 as base UIH + ( + sample_inds, + sample_ratings, + sample_candidate_inds, + sample_candidate_ratings, + category_to_cnt, + ) = self.generate_one_timestamp( + category_to_cnt=category_to_cnt, + categories=categories, + t=-1, + id=id, + output_folder=output_folder, + uih_seq_len=self.gen_rand_seq_len(), + eval=False, + inference=False, + file_idx=file_idx, + ts_buffers=ts_buffers, + ) + out_list.append(",".join([str(ind) for ind in sample_candidate_inds])) + out_list.append(",".join([str(rat) for rat in sample_candidate_ratings])) + out_list.append(",".join([str(ind) for ind in sample_inds])) + out_list.append(",".join([str(rat) for rat in sample_ratings])) + # train + for t in range(int(self.num_timestamps * self.train_ratio)): + if self.get_timestamp_sample(t): + ( + sample_inds, + sample_ratings, + sample_candidate_inds, + sample_candidate_ratings, + category_to_cnt, + ) = self.generate_one_timestamp( + category_to_cnt=category_to_cnt, + categories=categories, + t=t, + id=id, + output_folder=output_folder, + uih_seq_len=self.gen_rand_seq_len(), + eval=False, + inference=False, + file_idx=file_idx, + ts_buffers=ts_buffers, + ) + out_list.append(",".join([str(ind) for ind in sample_candidate_inds])) + out_list.append( + ",".join([str(rat) for rat in sample_candidate_ratings]) + ) + out_list.append(",".join([str(ind) for ind in sample_inds])) + out_list.append(",".join([str(rat) for rat in sample_ratings])) + else: + out_list += ["", "", "", ""] + # eval + ( + sample_inds, + sample_ratings, + sample_candidate_inds, + sample_candidate_ratings, + category_to_cnt, + ) = self.generate_one_timestamp( + category_to_cnt=category_to_cnt, + categories=categories, + t=int(self.num_timestamps * self.train_ratio), + id=id, + output_folder=output_folder, + uih_seq_len=self.num_eval_candidates, + eval=True, + inference=False, + file_idx=file_idx, + ts_buffers=ts_buffers, + ) + out_list.append(",".join([str(ind) for ind in sample_candidate_inds])) + out_list.append(",".join([str(rat) for rat in sample_candidate_ratings])) + out_list.append(",".join([str(ind) for ind in sample_inds])) + out_list.append(",".join([str(rat) for rat in sample_ratings])) + # inference + for t in range( + int(self.num_timestamps * self.train_ratio), self.num_timestamps + ): + if self.get_timestamp_sample(t): + ( + sample_inds, + sample_ratings, + sample_candidate_inds, + sample_candidate_ratings, + category_to_cnt, + ) = self.generate_one_timestamp( + category_to_cnt=category_to_cnt, + categories=categories, + t=t, + id=id, + output_folder=output_folder, + uih_seq_len=self.gen_rand_seq_len(), + eval=False, + inference=True, + file_idx=file_idx, + ts_buffers=ts_buffers, + ) + out_list.append(",".join([str(ind) for ind in sample_candidate_inds])) + out_list.append( + ",".join([str(rat) for rat in sample_candidate_ratings]) + ) + out_list.append(",".join([str(ind) for ind in sample_inds])) + out_list.append(",".join([str(rat) for rat in sample_ratings])) + else: + out_list += ["", "", "", ""] + return out_list + + def write_dataset( + self, output_folder: str, num_files: int, file_idx: int, seed: int + ) -> None: + """ + Write dataset for a single file partition. + + Args: + output_folder: Output directory path. + num_files: Total number of files in the dataset. + file_idx: Index of this file partition. + seed: Random seed for reproducibility. + """ + t0 = time.time() + num_users_per_file = self.num_users // num_files + user_id: int = num_users_per_file * file_idx + random.seed(seed + file_idx) + np.random.seed(seed + file_idx) + # Buffer timestamp data in memory to avoid excessive file I/O + ts_buffers: Dict[int, List[int]] = {} + output_file = output_folder + f"{file_idx}.csv" + with open(output_file, "w") as file: + writer = csv.writer(file) + for i in range(num_users_per_file): + out_list = self.generate_one_user( + id=user_id, + output_folder=output_folder, + file_idx=file_idx, + ts_buffers=ts_buffers, + ) + user_id += 1 + writer.writerow(out_list) + if i % 10000 == 0: + logger.warning( + f"rank {self.rank}: Done with users {i} for file {file_idx + 1} / {num_files}, total_cnt = {self.total_cnt}, spends {time.time() - t0} seconds." + ) + # Write buffered timestamp data after all users are processed + for ts, user_ids in ts_buffers.items(): + ts_file = output_folder + f"ts_{file_idx}_{ts}.csv" + with open(ts_file, "w") as f: + writer = csv.writer(f) + for uid in user_ids: + writer.writerow([uid]) + logger.warning( + f"rank {self.rank}: Wrote {len(ts_buffers)} timestamp files for file {file_idx}" + ) + + +def worker( + rank: int, + world_size: int, + num_files: int, + num_users: int, + num_items: int, + num_categories: int, + categories_per_user: int, + num_timestamps: int, + avg_samples_per_item: int, + num_eval_candidates: int, + num_inference_candidates: int, + train_ratio: float, + user_sampling_ratio: float, + output_folder: str, +) -> None: + """ + Worker function for parallel data generation. + + Each worker generates a subset of the dataset files. + + Args: + rank: Worker rank. + world_size: Total number of workers. + num_files: Total files to generate. + num_users: Total users in dataset. + num_items: Total items in catalog. + num_categories: Number of item categories. + categories_per_user: Categories per user. + num_timestamps: Number of time periods. + avg_samples_per_item: Average interactions per item. + num_eval_candidates: Eval candidates count. + num_inference_candidates: Inference candidates count. + train_ratio: Training data fraction. + user_sampling_ratio: User sampling probability. + output_folder: Output directory. + """ + generator = StreamingSyntheticDataGenerator( + num_categories=num_categories, + categories_per_user=categories_per_user, + num_users=num_users, + num_items=num_items, + num_timestamps=num_timestamps, + avg_samples_per_item=avg_samples_per_item, + train_ratio=train_ratio, + user_sampling_ratio=user_sampling_ratio, + num_eval_candidates=num_eval_candidates, + num_inference_candidates=num_inference_candidates, + debug=False, + rank=rank, + ) + num_files_per_rank = num_files // world_size + file_indices = [i + rank * num_files_per_rank for i in range(num_files_per_rank)] + for file_idx in file_indices: + logger.warning(f"rank {rank}: start generating file {file_idx}") + generator.write_dataset( + output_folder=output_folder, + num_files=num_files, + file_idx=file_idx, + seed=1001, + ) + logger.warning(f"rank {rank}: finish generating file {file_idx}") + + +def write_offset(output_folder: str, num_files: int, num_users: int) -> None: + """ + Write file byte offsets for random access to user data. + + Creates an offset.csv file containing byte positions for each user + within their respective data files. + + Args: + output_folder: Directory containing data files. + num_files: Number of data files. + num_users: Total number of users. + """ + with open(output_folder + "offset.csv", "a") as output_file: + writer = csv.writer(output_file) + for i in range(num_files): + input_file = output_folder + f"{i}.csv" + offsets = [] + with open(input_file, "r") as f: + while True: + offset = f.tell() + line = f.readline() + if not line: + break + offsets.append(offset) + assert len(offsets) == num_users // num_files, ( + f"num_users {num_users // num_files} != {len(offsets)}" + ) + logger.warning(f"offsets for file {i} finished") + writer.writerow([",".join([str(offset) for offset in offsets])]) + + +def write_ts_metadata(output_folder: str, total_ts: int, num_files: int) -> None: + """ + Write timestamp metadata for streaming simulation. + + Creates files tracking which users are active at each timestamp + and cumulative counts for efficient streaming access. + + Args: + output_folder: Output directory path. + total_ts: Total number of timestamps. + num_files: Number of data files. + """ + with open(output_folder + "requests_per_ts.csv", "w") as file_requests: + with open(output_folder + "users_cumsum_per_ts.csv", "w") as file_cumsum: + requests_writer = csv.writer(file_requests) + cumsum_writer = csv.writer(file_cumsum) + for ts in range(total_ts): + requests = [] + num_users_per_file = [] + for file in range(num_files): + with open(f"{output_folder}ts_{file}_{ts}.csv", "r") as file: + reader = csv.reader(file) + size = 0 + for row in reader: + requests.append(int(row[0])) + size += 1 + num_users_per_file.append(size) + cumsum = np.cumsum(num_users_per_file).tolist() + assert cumsum[-1] == len(requests) + requests_writer.writerow([",".join([str(r) for r in requests])]) + cumsum_writer.writerow([",".join([str(s) for s in cumsum])]) + logger.warning(f"ts {ts} finished") + with open( + output_folder + "requests_per_ts_offset.csv", "w" + ) as file_requests_offset: + writer = csv.writer(file_requests_offset) + input_file = output_folder + "requests_per_ts.csv" + offsets = [] + with open(input_file, "r") as f: + while True: + offset = f.tell() + line = f.readline() + if not line: + break + offsets.append(offset) + assert len(offsets) == total_ts, f"total_ts {total_ts} != {len(offsets)}" + logger.warning("offsets for file requests_per_ts.csv finished") + writer.writerow([",".join([str(offset) for offset in offsets])]) + + +def copy_sub_dataset(src_folder: str) -> None: + """ + Copy a subset of dataset files for quick testing. + + Creates a sampled_data subdirectory with essential files. + + Args: + src_folder: Source folder containing full dataset. + """ + dst_folder = src_folder + "sampled_data/" + files_to_copy = [ + "0.csv", + "offset.csv", + "requests_per_ts.csv", + "requests_per_ts_offset.csv", + "users_cumsum_per_ts.csv", + ] + os.makedirs(dst_folder, exist_ok=True) + for filename in files_to_copy: + src_path = os.path.join(src_folder, filename) + dst_path = os.path.join(dst_folder, filename) + shutil.copy2(src_path, dst_path) + logger.warning("Files copied successfully.") + + +def main() -> None: + """ + Main entry point for synthetic data generation. + + Configures and launches parallel workers to generate a complete + streaming recommendation dataset. + """ + processes = [] + num_files = 100 + num_users = 5_000_000 + num_items = 1_000_000_000 + num_categories = 128 + categories_per_user = 4 + num_timestamps = 100 + avg_samples_per_item = 50 + num_eval_candidates = 32 + num_inference_candidates = 2048 + train_ratio = 0.9 + user_sampling_ratio = 0.7 + world_size = 5 + username = os.getlogin() + output_folder = f"/home/{username}/data/streaming-100b/" + for i in range(world_size): + p = multiprocessing.Process( + target=worker, + args=( + i, + world_size, + num_files, + num_users, + num_items, + num_categories, + categories_per_user, + num_timestamps, + avg_samples_per_item, + num_eval_candidates, + num_inference_candidates, + train_ratio, + user_sampling_ratio, + output_folder, + ), + ) + processes.append(p) + p.start() + for p in processes: + p.join() + write_offset(output_folder, num_files, num_users) + write_ts_metadata(output_folder, num_timestamps, num_files) + copy_sub_dataset(src_folder=output_folder) + + +if __name__ == "__main__": + main() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/debug.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/debug.gin new file mode 100644 index 000000000..9261dc222 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/debug.gin @@ -0,0 +1,35 @@ +batch_size = 16 +dataset = "debug" + +# model parameters +make_model.dataset = %dataset + +# dense model optimizer +dense_optimizer_factory_and_class.learning_rate = 0.001 +dense_optimizer_factory_and_class.optimizer_name = "Adam" +dense_optimizer_factory_and_class.momentum = 0 +dense_optimizer_factory_and_class.weight_decay = 0 +dense_optimizer_factory_and_class.eps = 1e-8 +dense_optimizer_factory_and_class.betas = (0.95, 0.999) + +# sparse model optimizer +sparse_optimizer_factory_and_class.learning_rate = 0.001 +sparse_optimizer_factory_and_class.optimizer_name = "RowWiseAdagrad" +sparse_optimizer_factory_and_class.momentum = 0 +sparse_optimizer_factory_and_class.weight_decay = 0 +sparse_optimizer_factory_and_class.eps = 1e-8 +sparse_optimizer_factory_and_class.betas = (0.95, 0.999) + +# dataloader configs +make_train_test_dataloaders.batch_size = %batch_size +make_train_test_dataloaders.dataset_type = %dataset +make_train_test_dataloaders.train_split_percentage = 0.75 + +# train loop variables +train_loop.num_batches = 10 +train_loop.num_epochs = 1000 +train_loop.output_trace = True +train_loop.metric_log_frequency = 10 + +# logger variables +MetricsLogger.tensorboard_log_path = "/tmp/tensorboard_log_path.log" diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/kuairand_1k.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/kuairand_1k.gin new file mode 100644 index 000000000..46d8e1272 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/kuairand_1k.gin @@ -0,0 +1,41 @@ +batch_size = 16 +dataset = "kuairand-1k" + +# model parameters +make_model.dataset = %dataset + +# dense model optimizer +dense_optimizer_factory_and_class.learning_rate = 0.001 +dense_optimizer_factory_and_class.optimizer_name = "Adam" +dense_optimizer_factory_and_class.momentum = 0 +dense_optimizer_factory_and_class.weight_decay = 0 +dense_optimizer_factory_and_class.eps = 1e-8 +dense_optimizer_factory_and_class.betas = (0.95, 0.999) + +# sparse model optimizer +sparse_optimizer_factory_and_class.learning_rate = 0.001 +sparse_optimizer_factory_and_class.optimizer_name = "RowWiseAdagrad" +sparse_optimizer_factory_and_class.momentum = 0 +sparse_optimizer_factory_and_class.weight_decay = 0 +sparse_optimizer_factory_and_class.eps = 1e-8 +sparse_optimizer_factory_and_class.betas = (0.95, 0.999) + +# dataloader configs +make_train_test_dataloaders.batch_size = %batch_size +make_train_test_dataloaders.dataset_type = %dataset +make_train_test_dataloaders.train_split_percentage = 0.75 +make_train_test_dataloaders.new_path_prefix = "/home/linjianma" +make_train_test_dataloaders.num_workers = 2 +make_train_test_dataloaders.prefetch_factor = 4 + +# train loop variables +train_loop.num_epochs = 5 +train_loop.output_trace = True +train_loop.metric_log_frequency = 10 + +# logger variables +MetricsLogger.tensorboard_log_path = "/tmp/tensorboard_log_path.log" + +# checkpoint +# save_dmp_checkpoint.path = "/home/linjianma/ckpts/kuairand_1k" +# load_dmp_checkpoint.path = "/home/linjianma/ckpts/kuairand_1k/2025_01_12_17_56_43/" diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/movielens_13b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/movielens_13b.gin new file mode 100644 index 000000000..e2f371de4 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/movielens_13b.gin @@ -0,0 +1,41 @@ +batch_size = 128 +dataset = "movielens-13b" + +# model parameters +make_model.dataset = %dataset + +# dense model optimizer +dense_optimizer_factory_and_class.learning_rate = 0.001 +dense_optimizer_factory_and_class.optimizer_name = "Adam" +dense_optimizer_factory_and_class.momentum = 0 +dense_optimizer_factory_and_class.weight_decay = 0 +dense_optimizer_factory_and_class.eps = 1e-8 +dense_optimizer_factory_and_class.betas = (0.95, 0.999) + +# sparse model optimizer +sparse_optimizer_factory_and_class.learning_rate = 0.001 +sparse_optimizer_factory_and_class.optimizer_name = "RowWiseAdagrad" +sparse_optimizer_factory_and_class.momentum = 0 +sparse_optimizer_factory_and_class.weight_decay = 0 +sparse_optimizer_factory_and_class.eps = 1e-8 +sparse_optimizer_factory_and_class.betas = (0.95, 0.999) + +# dataloader configs +make_train_test_dataloaders.batch_size = %batch_size +make_train_test_dataloaders.dataset_type = %dataset +make_train_test_dataloaders.train_split_percentage = 0.75 +make_train_test_dataloaders.new_path_prefix = "/home/linjianma" +make_train_test_dataloaders.num_workers = 2 +make_train_test_dataloaders.prefetch_factor = 4 + +# train loop variables +train_loop.num_epochs = 1 +train_loop.output_trace = True +train_loop.metric_log_frequency = 10 +train_eval_loop.num_epochs = 1 +train_eval_loop.output_trace = True +train_eval_loop.metric_log_frequency = 10 + +# logger variables +MetricsLogger.tensorboard_log_path = "/tmp/tensorboard_log_path.log" +save_dmp_checkpoint.path = "/home/linjianma/ckpts/movielens_13b" diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/movielens_18b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/movielens_18b.gin new file mode 100644 index 000000000..094271b57 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/movielens_18b.gin @@ -0,0 +1,56 @@ +batch_size = 64 +dataset = "movielens-18b" + +# model parameters +make_model.dataset = %dataset + +# dense model optimizer +dense_optimizer_factory_and_class.learning_rate = 0.001 +dense_optimizer_factory_and_class.optimizer_name = "Adam" +dense_optimizer_factory_and_class.momentum = 0 +dense_optimizer_factory_and_class.weight_decay = 0 +dense_optimizer_factory_and_class.eps = 1e-8 +dense_optimizer_factory_and_class.betas = (0.95, 0.999) + +# sparse model optimizer +sparse_optimizer_factory_and_class.learning_rate = 0.001 +sparse_optimizer_factory_and_class.optimizer_name = "RowWiseAdagrad" +sparse_optimizer_factory_and_class.momentum = 0 +sparse_optimizer_factory_and_class.weight_decay = 0 +sparse_optimizer_factory_and_class.eps = 1e-8 +sparse_optimizer_factory_and_class.betas = (0.95, 0.999) + +# dataloader configs +make_train_test_dataloaders.batch_size = %batch_size +make_train_test_dataloaders.dataset_type = %dataset +make_train_test_dataloaders.train_split_percentage = 0.80 +make_train_test_dataloaders.new_path_prefix = "/home/linjianma" +make_train_test_dataloaders.num_workers = 2 +make_train_test_dataloaders.prefetch_factor = 4 +make_train_test_dataloaders.num_blocks = 20 + +# train loop variables +train_loop.num_epochs = 200 +train_loop.output_trace = False +train_loop.metric_log_frequency = 40 +train_loop.checkpoint_frequency = 4000 +train_loop.start_batch_idx = 0 + +# eval loop variables +eval_loop.metric_log_frequency = 40 + +# train eval loop variables +train_eval_loop.num_epochs = 20 +train_eval_loop.output_trace = False +train_eval_loop.start_train_batch_idx = 0 +train_eval_loop.start_eval_batch_idx = 0 +train_eval_loop.num_eval_batches = 200 +train_eval_loop.metric_log_frequency = 40 +train_eval_loop.checkpoint_frequency = 2000 +train_eval_loop.eval_frequency = 500 + + +# logger variables +MetricsLogger.tensorboard_log_path = "/home/linjianma/tensorboard/movielens_18b/" +save_dmp_checkpoint.path = "/home/linjianma/ckpts/movielens_18b/" +# load_dmp_checkpoint.path = "/home/linjianma/ckpts/movielens_18b/20000/" diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/movielens_1m.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/movielens_1m.gin new file mode 100644 index 000000000..2b6cd6b64 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/movielens_1m.gin @@ -0,0 +1,38 @@ +batch_size = 128 +dataset = "movielens-1m" + +# model parameters +make_model.dataset = %dataset + +# dense model optimizer +dense_optimizer_factory_and_class.learning_rate = 0.001 +dense_optimizer_factory_and_class.optimizer_name = "Adam" +dense_optimizer_factory_and_class.momentum = 0 +dense_optimizer_factory_and_class.weight_decay = 0 +dense_optimizer_factory_and_class.eps = 1e-8 +dense_optimizer_factory_and_class.betas = (0.9, 0.98) + +# sparse model optimizer +sparse_optimizer_factory_and_class.learning_rate = 0.001 +sparse_optimizer_factory_and_class.optimizer_name = "RowWiseAdagrad" +sparse_optimizer_factory_and_class.momentum = 0 +sparse_optimizer_factory_and_class.weight_decay = 0 +sparse_optimizer_factory_and_class.eps = 1e-8 +sparse_optimizer_factory_and_class.betas = (0.95, 0.999) + +# dataloader configs +make_train_test_dataloaders.batch_size = %batch_size +make_train_test_dataloaders.dataset_type = %dataset +make_train_test_dataloaders.train_split_percentage = 0.75 +make_train_test_dataloaders.new_path_prefix = "/home/linjianma" +make_train_test_dataloaders.num_workers = 2 +make_train_test_dataloaders.prefetch_factor = 4 + +# train-eval loop variables +train_eval_loop.num_epochs = 101 +train_eval_loop.output_trace = True +train_eval_loop.metric_log_frequency = 10 +train_eval_loop.eval_frequency = 1 + +# logger variables +MetricsLogger.tensorboard_log_path = "/tmp/tensorboard_log_path.log" diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/movielens_20m.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/movielens_20m.gin new file mode 100644 index 000000000..c01fab5af --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/movielens_20m.gin @@ -0,0 +1,56 @@ +batch_size = 64 +dataset = "movielens-20m" + +# model parameters +make_model.dataset = %dataset + +# dense model optimizer +dense_optimizer_factory_and_class.learning_rate = 0.001 +dense_optimizer_factory_and_class.optimizer_name = "Adam" +dense_optimizer_factory_and_class.momentum = 0 +dense_optimizer_factory_and_class.weight_decay = 0 +dense_optimizer_factory_and_class.eps = 1e-8 +dense_optimizer_factory_and_class.betas = (0.95, 0.999) + +# sparse model optimizer +sparse_optimizer_factory_and_class.learning_rate = 0.001 +sparse_optimizer_factory_and_class.optimizer_name = "RowWiseAdagrad" +sparse_optimizer_factory_and_class.momentum = 0 +sparse_optimizer_factory_and_class.weight_decay = 0 +sparse_optimizer_factory_and_class.eps = 1e-8 +sparse_optimizer_factory_and_class.betas = (0.95, 0.999) + +# dataloader configs +make_train_test_dataloaders.batch_size = %batch_size +make_train_test_dataloaders.dataset_type = %dataset +make_train_test_dataloaders.train_split_percentage = 0.80 +make_train_test_dataloaders.new_path_prefix = "/home/linjianma" +make_train_test_dataloaders.num_workers = 2 +make_train_test_dataloaders.prefetch_factor = 4 +make_train_test_dataloaders.num_blocks = 1 + +# train loop variables +train_loop.num_epochs = 200 +train_loop.output_trace = False +train_loop.metric_log_frequency = 40 +train_loop.checkpoint_frequency = 4000 +train_loop.start_batch_idx = 0 + +# eval loop variables +eval_loop.metric_log_frequency = 10 + +# train eval loop variables +train_eval_loop.num_epochs = 20 +train_eval_loop.output_trace = False +train_eval_loop.start_train_batch_idx = 0 +train_eval_loop.start_eval_batch_idx = 0 +train_eval_loop.num_eval_batches = 100 +train_eval_loop.metric_log_frequency = 40 +train_eval_loop.checkpoint_frequency = 2000 +train_eval_loop.eval_frequency = 200 + + +# logger variables +MetricsLogger.tensorboard_log_path = "/home/linjianma/tensorboard/movielens_20m/" +# save_dmp_checkpoint.path = "/home/linjianma/ckpts/movielens_18b/0.5T" +# load_dmp_checkpoint.path = "/home/linjianma/ckpts/movielens_18b/20000/" diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/streaming_100b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/streaming_100b.gin new file mode 100644 index 000000000..7d1df4bce --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/streaming_100b.gin @@ -0,0 +1,52 @@ +batch_size = 64 +num_workers = 2 +prefetch_factor = 4 +dataset = "streaming-100b" + +# model parameters +make_model.dataset = %dataset + +# dense model optimizer +dense_optimizer_factory_and_class.learning_rate = 0.001 +dense_optimizer_factory_and_class.optimizer_name = "Adam" +dense_optimizer_factory_and_class.momentum = 0 +dense_optimizer_factory_and_class.weight_decay = 0 +dense_optimizer_factory_and_class.eps = 1e-8 +dense_optimizer_factory_and_class.betas = (0.95, 0.999) + +# sparse model optimizer +sparse_optimizer_factory_and_class.learning_rate = 0.001 +sparse_optimizer_factory_and_class.optimizer_name = "RowWiseAdagrad" +sparse_optimizer_factory_and_class.momentum = 0 +sparse_optimizer_factory_and_class.weight_decay = 0 +sparse_optimizer_factory_and_class.eps = 1e-8 +sparse_optimizer_factory_and_class.betas = (0.95, 0.999) + +# dataloader configs +make_train_test_dataloaders.batch_size = %batch_size +make_train_test_dataloaders.dataset_type = %dataset +make_train_test_dataloaders.train_split_percentage = 0.80 +make_train_test_dataloaders.new_path_prefix = "/home/linjianma" +make_train_test_dataloaders.num_workers = %num_workers +make_train_test_dataloaders.prefetch_factor = %prefetch_factor +make_train_test_dataloaders.num_blocks = 20 + +get_dataset.name = %dataset +get_dataset.new_path_prefix = "/home/linjianma" + +make_streaming_dataloader.batch_size = %batch_size +make_streaming_dataloader.num_workers = %num_workers +make_streaming_dataloader.prefetch_factor = %prefetch_factor + +# train eval loop variables +streaming_train_eval_loop.num_train_ts = 90 +streaming_train_eval_loop.output_trace = False +streaming_train_eval_loop.num_eval_batches = 500 +streaming_train_eval_loop.metric_log_frequency = 40 +streaming_train_eval_loop.checkpoint_frequency = 3 + + +# logger variables +MetricsLogger.tensorboard_log_path = "/home/linjianma/tensorboard/streaming_100b/run4/" +save_dmp_checkpoint.path = "/home/linjianma/ckpts/streaming_100b/" +# load_dmp_checkpoint.path = "/home/linjianma/ckpts/streaming_100b/20000/" diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/streaming_200b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/streaming_200b.gin new file mode 100644 index 000000000..872019962 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/streaming_200b.gin @@ -0,0 +1,63 @@ +batch_size = 64 +num_workers = 2 +prefetch_factor = 4 +dataset = "streaming-200b" + +# model parameters +make_model.dataset = %dataset + +# dense model optimizer +dense_optimizer_factory_and_class.learning_rate = 0.001 +dense_optimizer_factory_and_class.optimizer_name = "Adam" +dense_optimizer_factory_and_class.momentum = 0 +dense_optimizer_factory_and_class.weight_decay = 0 +dense_optimizer_factory_and_class.eps = 1e-8 +dense_optimizer_factory_and_class.betas = (0.95, 0.999) + +# sparse model optimizer +sparse_optimizer_factory_and_class.learning_rate = 0.001 +sparse_optimizer_factory_and_class.optimizer_name = "RowWiseAdagrad" +sparse_optimizer_factory_and_class.momentum = 0 +sparse_optimizer_factory_and_class.weight_decay = 0 +sparse_optimizer_factory_and_class.eps = 1e-8 +sparse_optimizer_factory_and_class.betas = (0.95, 0.999) + +# dataloader configs +make_train_test_dataloaders.batch_size = %batch_size +make_train_test_dataloaders.dataset_type = %dataset +make_train_test_dataloaders.train_split_percentage = 0.80 +make_train_test_dataloaders.new_path_prefix = "/home/linjianma" +make_train_test_dataloaders.num_workers = %num_workers +make_train_test_dataloaders.prefetch_factor = %prefetch_factor +make_train_test_dataloaders.num_blocks = 20 + +get_dataset.name = %dataset +get_dataset.new_path_prefix = "/home/linjianma" + +make_streaming_dataloader.batch_size = %batch_size +make_streaming_dataloader.num_workers = %num_workers +make_streaming_dataloader.prefetch_factor = %prefetch_factor + +# train loop variables +train_loop.num_epochs = 200 +train_loop.output_trace = False +train_loop.metric_log_frequency = 40 +train_loop.checkpoint_frequency = 4000 +train_loop.start_batch_idx = 0 + +# eval loop variables +eval_loop.metric_log_frequency = 40 + +# train eval loop variables +streaming_train_eval_loop.num_train_ts = 90 +streaming_train_eval_loop.output_trace = False +streaming_train_eval_loop.num_train_batches = 5000 +streaming_train_eval_loop.num_eval_batches = 200 +streaming_train_eval_loop.metric_log_frequency = 40 +streaming_train_eval_loop.checkpoint_frequency = 2000 + + +# logger variables +MetricsLogger.tensorboard_log_path = "/home/linjianma/tensorboard/streaming_200b/" +# save_dmp_checkpoint.path = "/home/linjianma/ckpts/streaming_200b/" +# load_dmp_checkpoint.path = "/home/linjianma/ckpts/streaming_400m/20000/" diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/streaming_400m.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/streaming_400m.gin new file mode 100644 index 000000000..eba17bc23 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/streaming_400m.gin @@ -0,0 +1,61 @@ +batch_size = 64 +num_workers = 2 +prefetch_factor = 4 +dataset = "streaming-400m" + +# model parameters +make_model.dataset = %dataset + +# dense model optimizer +dense_optimizer_factory_and_class.learning_rate = 0.001 +dense_optimizer_factory_and_class.optimizer_name = "Adam" +dense_optimizer_factory_and_class.momentum = 0 +dense_optimizer_factory_and_class.weight_decay = 0 +dense_optimizer_factory_and_class.eps = 1e-8 +dense_optimizer_factory_and_class.betas = (0.95, 0.999) + +# sparse model optimizer +sparse_optimizer_factory_and_class.learning_rate = 0.001 +sparse_optimizer_factory_and_class.optimizer_name = "RowWiseAdagrad" +sparse_optimizer_factory_and_class.momentum = 0 +sparse_optimizer_factory_and_class.weight_decay = 0 +sparse_optimizer_factory_and_class.eps = 1e-8 +sparse_optimizer_factory_and_class.betas = (0.95, 0.999) + +# dataloader configs +make_train_test_dataloaders.batch_size = %batch_size +make_train_test_dataloaders.dataset_type = %dataset +make_train_test_dataloaders.train_split_percentage = 0.80 +make_train_test_dataloaders.new_path_prefix = "/home/linjianma" +make_train_test_dataloaders.num_workers = %num_workers +make_train_test_dataloaders.prefetch_factor = %prefetch_factor +make_train_test_dataloaders.num_blocks = 20 + +get_dataset.name = %dataset +get_dataset.new_path_prefix = "/home/linjianma" + +make_streaming_dataloader.batch_size = %batch_size +make_streaming_dataloader.num_workers = %num_workers +make_streaming_dataloader.prefetch_factor = %prefetch_factor + +# train loop variables +train_loop.num_epochs = 200 +train_loop.output_trace = False +train_loop.metric_log_frequency = 40 +train_loop.checkpoint_frequency = 4000 +train_loop.start_batch_idx = 0 + +# eval loop variables +eval_loop.metric_log_frequency = 40 + +# train eval loop variables +streaming_train_eval_loop.num_train_ts = 8 +streaming_train_eval_loop.output_trace = False +streaming_train_eval_loop.metric_log_frequency = 40 +streaming_train_eval_loop.checkpoint_frequency = 2000 + + +# logger variables +MetricsLogger.tensorboard_log_path = "/home/linjianma/tensorboard/streaming_400m/" +# save_dmp_checkpoint.path = "/home/linjianma/ckpts/streaming_400m/" +# load_dmp_checkpoint.path = "/home/linjianma/ckpts/streaming_400m/20000/" diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin new file mode 100644 index 000000000..a483f8766 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -0,0 +1,50 @@ +batch_size = 32 +num_workers = 1 +prefetch_factor = 2 +dataset = "yambda-5b" + +# model parameters +make_model.dataset = %dataset + +# dense model optimizer +dense_optimizer_factory_and_class.learning_rate = 0.001 +dense_optimizer_factory_and_class.optimizer_name = "Adam" +dense_optimizer_factory_and_class.momentum = 0 +dense_optimizer_factory_and_class.weight_decay = 0 +dense_optimizer_factory_and_class.eps = 1e-8 +dense_optimizer_factory_and_class.betas = (0.95, 0.999) + +# sparse model optimizer +sparse_optimizer_factory_and_class.learning_rate = 0.001 +sparse_optimizer_factory_and_class.optimizer_name = "RowWiseAdagrad" +sparse_optimizer_factory_and_class.momentum = 0 +sparse_optimizer_factory_and_class.weight_decay = 0 +sparse_optimizer_factory_and_class.eps = 1e-8 +sparse_optimizer_factory_and_class.betas = (0.95, 0.999) + +# dataloader configs +make_train_test_dataloaders.batch_size = %batch_size +make_train_test_dataloaders.eval_batch_size = 32 +make_train_test_dataloaders.dataset_type = %dataset +make_train_test_dataloaders.train_split_percentage = 0.90 +make_train_test_dataloaders.new_path_prefix = "/apps/chcai/dlrm_data" +make_train_test_dataloaders.num_workers = %num_workers +make_train_test_dataloaders.prefetch_factor = %prefetch_factor +make_train_test_dataloaders.num_blocks = 1 + +get_dataset.name = %dataset +get_dataset.new_path_prefix = "/apps/chcai/dlrm_data" + +# train-eval loop variables (yambda is non-streaming) +train_eval_loop.num_epochs = 1 +train_eval_loop.output_trace = False +train_eval_loop.metric_log_frequency = 50 +train_eval_loop.eval_frequency = 5000 +train_eval_loop.num_eval_batches = 500 +train_eval_loop.checkpoint_frequency = 1000000000 # disable mid-training checkpoints (disk-full guard) + +# logger variables +MetricsLogger.tensorboard_log_path = "/tmp/tb/yambda_5b/" +MetricsLogger.world_size = 8 +MetricsLogger.auc_threshold = 0.80275 +save_dmp_checkpoint.path = "/apps/chcai/ckpts/yambda_5b/" diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/train_test.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/train_test.py new file mode 100644 index 000000000..dfd58f6e5 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/train_test.py @@ -0,0 +1,29 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-strict +import unittest + +from generative_recommenders.common import gpu_unavailable +from generative_recommenders.dlrm_v3.train.train_ranker import main + + +class DLRMV3TrainTest(unittest.TestCase): + @unittest.skipIf(*gpu_unavailable) + def test_e2e(self) -> None: + main() + + +if __name__ == "__main__": + unittest.main() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py new file mode 100644 index 000000000..d17e2992d --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py @@ -0,0 +1,190 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-strict +import argparse +import logging + +logging.basicConfig(level=logging.INFO) +import os +import sys +import traceback + +import gin +import torch +from generative_recommenders.dlrm_v3.checkpoint import load_dmp_checkpoint +from generative_recommenders.dlrm_v3.train.utils import ( + cleanup, + eval_loop, + make_model, + make_optimizer_and_shard, + make_train_test_dataloaders, + setup, + streaming_train_eval_loop, + train_eval_loop, + train_loop, +) +from generative_recommenders.dlrm_v3.utils import MetricsLogger +from torch import multiprocessing as mp +from torchrec.test_utils import get_free_port + +logger: logging.Logger = logging.getLogger(__name__) + + +SUPPORTED_CONFIGS = { + "debug": "debug.gin", + "kuairand-1k": "kuairand_1k.gin", + "movielens-1m": "movielens_1m.gin", + "movielens-20m": "movielens_20m.gin", + "movielens-13b": "movielens_13b.gin", + "movielens-18b": "movielens_18b.gin", + "streaming-400m": "streaming_400m.gin", + "streaming-200b": "streaming_200b.gin", + "streaming-100b": "streaming_100b.gin", + "yambda-5b": "yambda_5b.gin", +} + + +def _main_func( + rank: int, + world_size: int, + master_port: int, + gin_file: str, + mode: str, +) -> None: + device = torch.device(f"cuda:{rank}") + logger.info(f"rank: {rank}, world_size: {world_size}, device: {device}") + setup( + rank=rank, + world_size=world_size, + master_port=master_port, + device=device, + ) + # parse all arguments + gin.parse_config_file(gin_file) + + model, model_configs, embedding_table_configs = make_model() + model, optimizer = make_optimizer_and_shard( + model=model, device=device, world_size=world_size + ) + train_dataloader, test_dataloader = make_train_test_dataloaders( + hstu_config=model_configs, + embedding_table_configs=embedding_table_configs, + ) + metrics = MetricsLogger( + multitask_configs=model_configs.multitask_configs, + batch_size=train_dataloader.batch_size, + window_size=2500, + device=device, + rank=rank, + ) + load_dmp_checkpoint( + model=model, optimizer=optimizer, metric_logger=metrics, device=device + ) + + # train loop + try: + if mode == "train": + train_loop( + rank=rank, + model=model, + dataloader=train_dataloader, + optimizer=optimizer, + metric_logger=metrics, + device=device, + ) + elif mode == "eval": + # reinit metrics logger for eval + metrics = MetricsLogger( + multitask_configs=model_configs.multitask_configs, + batch_size=train_dataloader.batch_size, + window_size=1000, + device=device, + rank=rank, + ) + eval_loop( + rank=rank, + model=model, + dataloader=test_dataloader, + metric_logger=metrics, + device=device, + ) + elif mode == "train-eval": + train_eval_loop( + rank=rank, + model=model, + train_dataloader=train_dataloader, + eval_dataloader=test_dataloader, + optimizer=optimizer, + metric_logger=metrics, + device=device, + ) + elif mode == "streaming-train-eval": + streaming_train_eval_loop( + rank=rank, + model=model, + optimizer=optimizer, + metric_logger=metrics, + device=device, + hstu_config=model_configs, + embedding_table_configs=embedding_table_configs, + ) + except Exception as e: + logger.info(traceback.format_exc()) + cleanup() + raise Exception(e) + + +def get_args(): # pyre-ignore [3] + """Parse commandline.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--dataset", default="debug", choices=SUPPORTED_CONFIGS.keys(), help="dataset" + ) + parser.add_argument( + "--mode", + default="train", + choices=["train", "eval", "train-eval", "streaming-train-eval"], + help="mode", + ) + args, unknown_args = parser.parse_known_args() + logger.warning(f"unknown_args: {unknown_args}") + return args + + +def main() -> None: + args = get_args() + logger.info(args) + assert args.dataset in SUPPORTED_CONFIGS, f"Unsupported dataset: {args.dataset}" + assert args.mode in [ + "train", + "eval", + "train-eval", + "streaming-train-eval", + ], f"Unsupported mode: {args.mode}" + WORLD_SIZE = int(os.environ.get("WORLD_SIZE", 1)) + MASTER_PORT = str(get_free_port()) + gin_path = f"{os.path.dirname(__file__)}/gin/{SUPPORTED_CONFIGS[args.dataset]}" + + mp.start_processes( + _main_func, + args=(WORLD_SIZE, MASTER_PORT, gin_path, args.mode), + nprocs=WORLD_SIZE, + join=True, + start_method="spawn", + ) + + +if __name__ == "__main__": + main() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py new file mode 100644 index 000000000..21d2baa6e --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -0,0 +1,902 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-strict +import logging +import os +from collections.abc import Iterator +from datetime import timedelta +from typing import ( + Any, + Callable, + Dict, + Iterable, + List, + Optional, + Set, + Tuple, + Type, + Union, +) + +import gin +import torch +import torchrec +from generative_recommenders.dlrm_v3.checkpoint import save_dmp_checkpoint +from generative_recommenders.dlrm_v3.configs import ( + get_embedding_table_config, + get_hstu_configs, +) +from generative_recommenders.dlrm_v3.datasets.dataset import collate_fn, Dataset +from generative_recommenders.dlrm_v3.utils import get_dataset, MetricsLogger, Profiler +from generative_recommenders.common import HammerKernel +from generative_recommenders.modules.dlrm_hstu import DlrmHSTU, DlrmHSTUConfig +from torch import distributed as dist +from torch.distributed.optim import ( + _apply_optimizer_in_backward as apply_optimizer_in_backward, +) +from torch.optim.optimizer import Optimizer +from torch.utils.data import DataLoader, Dataset as TorchDataset +from torch.utils.data.distributed import _T_co, DistributedSampler +from torchrec.distributed.model_parallel import DistributedModelParallel +from torchrec.distributed.planner import EmbeddingShardingPlanner, Topology +from torchrec.distributed.sharding_plan import get_default_sharders +from torchrec.distributed.types import ShardedTensor, ShardingEnv +from torchrec.modules.embedding_configs import EmbeddingConfig +from torchrec.modules.embedding_modules import ( + EmbeddingBagCollection, + EmbeddingCollection, +) +from torchrec.optim.keyed import CombinedOptimizer, KeyedOptimizerWrapper +from torchrec.optim.optimizers import in_backward_optimizer_filter +from torchrec.sparse.jagged_tensor import KeyedJaggedTensor + +logger: logging.Logger = logging.getLogger(__name__) + +TORCHREC_TYPES: Set[Type[Union[EmbeddingBagCollection, EmbeddingCollection]]] = { + EmbeddingBagCollection, + EmbeddingCollection, +} + + +def setup( + rank: int, world_size: int, master_port: int, device: torch.device +) -> dist.ProcessGroup: + os.environ["MASTER_ADDR"] = "localhost" + os.environ["MASTER_PORT"] = str(master_port) + + BACKEND = dist.Backend.NCCL + TIMEOUT = 1800 + + # initialize the process group + if not dist.is_initialized(): + dist.init_process_group("nccl", rank=rank, world_size=world_size) + + pg = dist.new_group( + backend=BACKEND, + timeout=timedelta(seconds=TIMEOUT), + ) + + # set device + torch.cuda.set_device(device) + + return pg + + +def cleanup() -> None: + dist.destroy_process_group() + + +class HammerToTorchDataset(TorchDataset): + def __init__( + self, + dataset: Dataset, + ) -> None: + self.dataset: Dataset = dataset + + def __getitem__(self, idx: int) -> Tuple[KeyedJaggedTensor, KeyedJaggedTensor]: + self.dataset.load_query_samples([idx]) + sample = self.dataset.get_sample(idx) + self.dataset.unload_query_samples([idx]) + return sample + + def __getitems__( + self, indices: List[int] + ) -> List[Tuple[KeyedJaggedTensor, KeyedJaggedTensor]]: + self.dataset.load_query_samples(indices) + samples = [self.dataset.get_sample(i) for i in indices] + self.dataset.unload_query_samples(indices) + return samples + + +class _ChainedRanges: + """O(1) __len__ + O(log K) __getitem__ over a sequence of `range`s. + + Lets `torch.utils.data.Subset(dataset, _ChainedRanges([r1, r2, ...]))` + avoid materializing a Python list of all per-block indices (which at + multi-billion totals is ~28 B/int and dominates host RAM). + """ + + def __init__(self, ranges: List[range]) -> None: + self._ranges: List[range] = list(ranges) + offsets = [0] + for r in self._ranges: + offsets.append(offsets[-1] + len(r)) + self._offsets: List[int] = offsets + + def __len__(self) -> int: + return self._offsets[-1] + + def __getitem__(self, idx: int) -> int: + import bisect + if idx < 0: + idx += self._offsets[-1] + if idx < 0 or idx >= self._offsets[-1]: + raise IndexError(idx) + bucket = bisect.bisect_right(self._offsets, idx) - 1 + return self._ranges[bucket][idx - self._offsets[bucket]] + + +class ChunkDistributedSampler(DistributedSampler[_T_co]): + """ + Each rank reads a contiguous chunk (trunk) of the input data + """ + + def __init__( + self, + dataset: TorchDataset, + num_replicas: Optional[int] = None, + rank: Optional[int] = None, + shuffle: bool = True, + seed: int = 1, + drop_last: bool = False, + ) -> None: + super().__init__( + dataset=dataset, + num_replicas=num_replicas, + rank=rank, + shuffle=shuffle, + seed=seed, + drop_last=drop_last, + ) + + def __iter__(self) -> Iterator[_T_co]: + if self.shuffle: + g = torch.Generator() + g.manual_seed(self.seed + self.epoch * 1001 + int(self.rank)) + indices_t = torch.randperm(self.num_samples, generator=g) + else: + indices_t = torch.arange(self.num_samples) + assert self.drop_last is True, ( + "drop_last must be True for ChunkDistributedSampler" + ) + indices_t = indices_t + (self.num_samples * int(self.rank)) + assert indices_t.numel() == self.num_samples + # Iterate via the numpy view, NOT directly over the tensor: iter(Tensor) + # calls Tensor.unbind(0) which eagerly materializes one zero-dim Tensor + # object per element (~600 B each). For 40 M eval / 525 M train samples + # that's 24 GB / 315 GB of [heap] growth per rank, blowing host RAM + # before the first batch. numpy's iter yields one Python int at a time + # with O(1) extra memory. + indices_np = indices_t.numpy() + return (int(x) for x in indices_np) + + def set_epoch(self, epoch: int) -> None: + logger.warning(f"Setting epoch to {epoch}") + self.epoch = epoch + + +@gin.configurable +def make_model( + dataset: str, +) -> Tuple[torch.nn.Module, DlrmHSTUConfig, Dict[str, EmbeddingConfig]]: + hstu_config = get_hstu_configs(dataset) + table_config = get_embedding_table_config(dataset) + + model = DlrmHSTU( + hstu_configs=hstu_config, + embedding_tables=table_config, + is_inference=False, + bf16_training=False, + ) + + # Triton on ROCm fails to compile some jagged kernels at our shapes + # (PassManager::run failed at make_ttgir). Allow the PyTorch backend as a + # global override so AMD smoke runs end-to-end. CUDA paths default to TRITON. + kernel_override = os.environ.get("HSTU_HAMMER_KERNEL", "").upper() + if kernel_override: + model.set_hammer_kernel(HammerKernel[kernel_override]) + logger.warning(f"HSTU_HAMMER_KERNEL override: {kernel_override}") + + return ( + model, + hstu_config, + table_config, + ) + + +@gin.configurable() +def dense_optimizer_factory_and_class( + optimizer_name: str, + betas: Tuple[float, float], + eps: float, + weight_decay: float, + momentum: float, + learning_rate: float, +) -> Tuple[ + Type[Optimizer], Dict[str, Any], Callable[[Iterable[torch.Tensor]], Optimizer] +]: + kwargs: Dict[str, Any] = {"lr": learning_rate} + if optimizer_name == "Adam": + optimizer_cls = torch.optim.Adam + kwargs.update({"betas": betas, "eps": eps, "weight_decay": weight_decay}) + elif optimizer_name == "SGD": + optimizer_cls = torch.optim.SGD + kwargs.update({"weight_decay": weight_decay, "momentum": momentum}) + elif optimizer_name == "AdamW": + optimizer_cls = torch.optim.AdamW + kwargs.update({"betas": betas, "eps": eps, "weight_decay": weight_decay}) + else: + raise Exception("Unsupported optimizer!") + + optimizer_factory = lambda params: optimizer_cls(params, **kwargs) + + return optimizer_cls, kwargs, optimizer_factory + + +@gin.configurable() +def sparse_optimizer_factory_and_class( + optimizer_name: str, + betas: Tuple[float, float], + eps: float, + weight_decay: float, + momentum: float, + learning_rate: float, +) -> Tuple[ + Type[Optimizer], Dict[str, Any], Callable[[Iterable[torch.Tensor]], Optimizer] +]: + kwargs: Dict[str, Any] = {"lr": learning_rate} + if optimizer_name == "Adam": + optimizer_cls = torch.optim.Adam + beta1, beta2 = betas + kwargs.update( + {"beta1": beta1, "beta2": beta2, "eps": eps, "weight_decay": weight_decay} + ) + elif optimizer_name == "SGD": + optimizer_cls = torchrec.optim.SGD + kwargs.update({"weight_decay": weight_decay, "momentum": momentum}) + elif optimizer_name == "RowWiseAdagrad": + optimizer_cls = torchrec.optim.RowWiseAdagrad + beta1, beta2 = betas + kwargs.update( + { + "eps": eps, + "beta1": beta1, + "beta2": beta2, + "weight_decay": weight_decay, + } + ) + else: + raise Exception("Unsupported optimizer!") + + optimizer_factory = lambda params: optimizer_cls(params, **kwargs) + + return optimizer_cls, kwargs, optimizer_factory + + +def make_optimizer_and_shard( + model: torch.nn.Module, + device: torch.device, + world_size: int, +) -> Tuple[DistributedModelParallel, torch.optim.Optimizer]: + dense_opt_cls, dense_opt_args, dense_opt_factory = ( + dense_optimizer_factory_and_class() + ) + + sparse_opt_cls, sparse_opt_args, sparse_opt_factory = ( + sparse_optimizer_factory_and_class() + ) + # Fuse sparse optimizer to backward step + for k, module in model.named_modules(): + if type(module) in TORCHREC_TYPES: + for _, param in module.named_parameters(prefix=k): + if param.requires_grad: + apply_optimizer_in_backward( + sparse_opt_cls, [param], sparse_opt_args + ) + sharders = get_default_sharders() + # MI350X has 288 GiB HBM3e per GPU; the 160 GiB cap was sized for older parts. + # Matches Primus-DLRM (hbm_cap_gb: 260) which runs the same 5b cross-feat + # table set on the same hardware without host materialization. + planner = EmbeddingShardingPlanner( + topology=Topology( + local_world_size=world_size, + world_size=world_size, + compute_device="cuda", + hbm_cap=260 * 1024 * 1024 * 1024, + ddr_cap=32 * 1024 * 1024 * 1024, + ) + ) + pg = dist.GroupMember.WORLD + env = ShardingEnv.from_process_group(pg) # pyre-ignore [6] + pg = env.process_group + + plan = planner.collective_plan(model, sharders, pg) + + # Shard model + model = DistributedModelParallel( + module=model, + device=device, + plan=plan, + sharders=sharders, + ) + # Create keyed optimizer + all_optimizers = [] + all_params = {} + non_fused_sparse_params = {} + for k, v in in_backward_optimizer_filter(model.named_parameters()): + if v.requires_grad: + if isinstance(v, ShardedTensor): + non_fused_sparse_params[k] = v + else: + all_params[k] = v + + if non_fused_sparse_params: + all_optimizers.append( + ( + "sparse_non_fused", + KeyedOptimizerWrapper( + params=non_fused_sparse_params, optim_factory=sparse_opt_factory + ), + ) + ) + + if all_params: + all_optimizers.append( + ( + "dense", + KeyedOptimizerWrapper( + params=all_params, + optim_factory=dense_opt_factory, + ), + ) + ) + output_optimizer = CombinedOptimizer(all_optimizers) + output_optimizer.init_state(set(model.sparse_grad_parameter_names())) + return model, output_optimizer + + +@gin.configurable +def make_streaming_dataloader( + dataset: HammerToTorchDataset, + ts: int, + batch_size: int, + num_workers: int, + prefetch_factor: int, +) -> DataLoader: + dataset.dataset.set_ts(ts) # pyre-ignore [16] + total_items = dataset.dataset.get_item_count() + subset = torch.utils.data.Subset(dataset, range(total_items)) + dataloader = DataLoader( + dataset=subset, + batch_size=batch_size, + shuffle=False, + collate_fn=collate_fn, + drop_last=True, + num_workers=num_workers, + prefetch_factor=prefetch_factor, + sampler=DistributedSampler(subset, drop_last=True), + ) + return dataloader + + +@gin.configurable +def make_train_test_dataloaders( + batch_size: int, + dataset_type: str, + hstu_config: DlrmHSTUConfig, + train_split_percentage: float, + embedding_table_configs: Dict[str, EmbeddingConfig], + new_path_prefix: str = "", + num_workers: int = 0, + num_blocks: int = 1, + prefetch_factor: Optional[int] = None, + eval_batch_size: Optional[int] = None, +) -> Tuple[DataLoader, DataLoader]: + dataset_class, kwargs = get_dataset( + name=dataset_type, new_path_prefix=new_path_prefix + ) + kwargs["embedding_config"] = embedding_table_configs + + # Create dataset + dataset = HammerToTorchDataset( + dataset=dataset_class(hstu_config=hstu_config, is_inference=False, **kwargs) + ) + total_items = dataset.dataset.get_item_count() + items_per_block = total_items // num_blocks + train_size_per_block = round(train_split_percentage * items_per_block) + # Avoid `extend(range(...))` which materializes a Python list of all sample + # indices — at 3.2B yambda samples × 28 bytes/int ≈ 90 GB/rank just for + # train_inds. Subset accepts any sequence with O(1) __len__ and __getitem__, + # so pass range objects (or a tiny chained view) directly. + if num_blocks == 1: + train_inds = range(0, train_size_per_block) + test_inds = range(train_size_per_block, items_per_block) + else: + train_inds = _ChainedRanges([ + range(i * items_per_block, i * items_per_block + train_size_per_block) + for i in range(num_blocks) + ]) + test_inds = _ChainedRanges([ + range(i * items_per_block + train_size_per_block, (i + 1) * items_per_block) + for i in range(num_blocks) + ]) + train_set = torch.utils.data.Subset(dataset, train_inds) + test_set = torch.utils.data.Subset(dataset, test_inds) + + # When the parent rank is started via mp.start_processes(start_method="spawn"), + # torch.multiprocessing's default Process context is also "spawn". DataLoader + # then pickles `self._dataset` to send to each worker — which for our mmap'd + # 211 GB yambda store materializes the entire dataset into the parent's anon + # memory (~230 GB/rank). Forcing "fork" lets workers inherit the parent's + # mmap'd pages via COW with zero extra anon. + mp_ctx = "fork" if num_workers and num_workers > 0 else None + train_dataloader = DataLoader( + dataset=train_set, + batch_size=batch_size, + shuffle=False, + collate_fn=collate_fn, + drop_last=True, + num_workers=num_workers, + prefetch_factor=prefetch_factor, + sampler=ChunkDistributedSampler(train_set, drop_last=True, shuffle=True), + multiprocessing_context=mp_ctx, + ) + test_dataloader = DataLoader( + dataset=test_set, + batch_size=eval_batch_size if eval_batch_size is not None else batch_size, + shuffle=False, + collate_fn=collate_fn, + drop_last=True, + num_workers=num_workers, + prefetch_factor=prefetch_factor, + sampler=ChunkDistributedSampler(test_set, drop_last=True, shuffle=True), + multiprocessing_context=mp_ctx, + ) + return train_dataloader, test_dataloader + + +@gin.configurable +def train_loop( + rank: int, + model: torch.nn.Module, + dataloader: torch.utils.data.DataLoader, + optimizer: Optimizer, + metric_logger: MetricsLogger, + device: torch.device, + num_epochs: int, + num_batches: Optional[int] = None, + output_trace: bool = False, + metric_log_frequency: int = 1, + checkpoint_frequency: int = 100, + start_batch_idx: int = 0, + # lr_scheduler: to-do: Add a scheduler +) -> None: + model.train() + batch_idx: int = start_batch_idx + profiler = Profiler(rank, active=10) if output_trace else None + + for epoch in range(num_epochs): + dataloader.sampler.set_epoch(epoch) # pyre-ignore [16] + for sample in dataloader: + optimizer.zero_grad() + sample.to(device) + ( + _, + _, + aux_losses, + mt_target_preds, + mt_target_labels, + mt_target_weights, + ) = model.forward( + sample.uih_features_kjt, + sample.candidates_features_kjt, + ) + # pyre-ignore + sum(aux_losses.values()).backward() + optimizer.step() + metric_logger.update( + mode="train", + predictions=mt_target_preds, + labels=mt_target_labels, + weights=mt_target_weights, + num_candidates=sample.candidates_features_kjt.lengths().view( + len(sample.candidates_features_kjt.keys()), -1 + )[0], + ) + if batch_idx % metric_log_frequency != 0: + metric_logger.compute_and_log( + mode="train", + additional_logs={ + "losses": aux_losses, + }, + ) + if batch_idx % checkpoint_frequency == 0 and batch_idx > 0: + save_dmp_checkpoint( + model=model, + optimizer=optimizer, + metric_logger=metric_logger, + rank=rank, + batch_idx=batch_idx, + ) + batch_idx += 1 + if output_trace: + assert profiler is not None + profiler.step() + if num_batches is not None and batch_idx >= num_batches: + break + if num_batches is not None and batch_idx >= num_batches: + break + + +@gin.configurable +def eval_loop( + rank: int, + model: torch.nn.Module, + dataloader: torch.utils.data.DataLoader, + metric_logger: MetricsLogger, + device: torch.device, + metric_log_frequency: int = 1, + num_batches: Optional[int] = None, + output_trace: bool = False, + # lr_scheduler: to-do: Add a scheduler +) -> None: + model.eval() + batch_idx: int = 0 + profiler = Profiler(rank, active=10) if output_trace else None + metric_logger.reset(mode="eval") + with torch.no_grad(): + for sample in dataloader: + sample.to(device) + ( + _, + _, + _, + mt_target_preds, + mt_target_labels, + mt_target_weights, + ) = model.forward( + sample.uih_features_kjt, + sample.candidates_features_kjt, + ) + metric_logger.update( + mode="eval", + predictions=mt_target_preds, + labels=mt_target_labels, + weights=mt_target_weights, + num_candidates=sample.candidates_features_kjt.lengths().view( + len(sample.candidates_features_kjt.keys()), -1 + )[0], + ) + if batch_idx % metric_log_frequency != 0: + metric_logger.compute_and_log(mode="eval") + batch_idx += 1 + if output_trace: + assert profiler is not None + profiler.step() + if num_batches is not None and batch_idx >= num_batches: + break + metric_logger.compute_and_log(mode="eval") + for k, v in metric_logger.compute(mode="eval").items(): + print(f"{k}: {v}") + + +@gin.configurable +def train_eval_loop( + rank: int, + model: torch.nn.Module, + optimizer: Optimizer, + metric_logger: MetricsLogger, + device: torch.device, + num_epochs: int, + num_train_batches: Optional[int] = None, + num_eval_batches: Optional[int] = None, + train_dataloader: Optional[torch.utils.data.DataLoader] = None, + eval_dataloader: Optional[torch.utils.data.DataLoader] = None, + output_trace: bool = False, + metric_log_frequency: int = 1, + checkpoint_frequency: int = 100, + eval_frequency: int = 1, + start_train_batch_idx: int = 0, + start_eval_batch_idx: int = 0, + # lr_scheduler: to-do: Add a scheduler +) -> None: + train_batch_idx: int = start_train_batch_idx + eval_batch_idx: int = start_eval_batch_idx + profiler = Profiler(rank, active=10) if output_trace else None + assert train_dataloader is not None and eval_dataloader is not None + + eval_data_iterator = iter(eval_dataloader) + train_data_iterator = iter(train_dataloader) + + for epoch in range(num_epochs): + train_dataloader.sampler.set_epoch(epoch) # pyre-ignore [16] + while True: + model.train() + try: + sample = next(train_data_iterator) + except StopIteration: + train_data_iterator = iter(train_dataloader) + break + optimizer.zero_grad() + sample.to(device) + ( + _, + _, + aux_losses, + mt_target_preds, + mt_target_labels, + mt_target_weights, + ) = model.forward( + sample.uih_features_kjt, + sample.candidates_features_kjt, + ) + # pyre-ignore + sum(aux_losses.values()).backward() + torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0) + optimizer.step() + metric_logger.update( + mode="train", + predictions=mt_target_preds, + labels=mt_target_labels, + weights=mt_target_weights, + num_candidates=sample.candidates_features_kjt.lengths().view( + len(sample.candidates_features_kjt.keys()), -1 + )[0], + ) + if train_batch_idx % metric_log_frequency == 0: + metric_logger.compute_and_log( + mode="train", + additional_logs={ + "losses": aux_losses, + }, + ) + if train_batch_idx % checkpoint_frequency == 0 and train_batch_idx > 0: + save_dmp_checkpoint( + model=model, + optimizer=optimizer, + metric_logger=metric_logger, + rank=rank, + batch_idx=train_batch_idx, + ) + train_batch_idx += 1 + if output_trace: + assert profiler is not None + profiler.step() + if train_batch_idx % eval_frequency == 0: + model.eval() + eval_batch_idx: int = 0 + with torch.no_grad(): + while True: + try: + sample = next(eval_data_iterator) + except StopIteration: + eval_data_iterator = iter(eval_dataloader) + sample = next(eval_data_iterator) + sample.to(device) + ( + _, + _, + _, + mt_target_preds, + mt_target_labels, + mt_target_weights, + ) = model.forward( + sample.uih_features_kjt, + sample.candidates_features_kjt, + ) + metric_logger.update( + mode="eval", + predictions=mt_target_preds, + labels=mt_target_labels, + weights=mt_target_weights, + num_candidates=sample.candidates_features_kjt.lengths().view( + len(sample.candidates_features_kjt.keys()), -1 + )[0], + ) + eval_batch_idx += 1 + if output_trace: + assert profiler is not None + profiler.step() + if eval_batch_idx % metric_log_frequency == 0: + metric_logger.compute_and_log(mode="eval") + if ( + num_eval_batches is not None + and eval_batch_idx >= num_eval_batches + ): + break + for k, v in metric_logger.compute(mode="eval").items(): + print(f"{k}: {v}") + model.train() + if num_train_batches is not None and train_batch_idx >= num_train_batches: + break + + +@gin.configurable +def streaming_train_eval_loop( + rank: int, + model: torch.nn.Module, + optimizer: Optimizer, + metric_logger: MetricsLogger, + device: torch.device, + num_train_ts: int, + hstu_config: DlrmHSTUConfig, + embedding_table_configs: Dict[str, EmbeddingConfig], + num_train_batches: Optional[int] = None, + num_eval_batches: Optional[int] = None, + output_trace: bool = False, + metric_log_frequency: int = 1, + checkpoint_frequency: int = 100, +) -> None: + profiler = Profiler(rank, active=10) if output_trace else None + dataset_class, kwargs = get_dataset() + kwargs["embedding_config"] = embedding_table_configs + dataset = HammerToTorchDataset( + dataset=dataset_class(hstu_config=hstu_config, is_inference=False, **kwargs) + ) + for train_ts in range(num_train_ts): + train_batch_idx: int = 0 + train_dataloader = make_streaming_dataloader(dataset=dataset, ts=train_ts) + train_data_iterator = iter(train_dataloader) + while True: + model.train() + try: + sample = next(train_data_iterator) + except StopIteration: + break + optimizer.zero_grad() + sample.to(device) + ( + _, + _, + aux_losses, + mt_target_preds, + mt_target_labels, + mt_target_weights, + ) = model.forward( + sample.uih_features_kjt, + sample.candidates_features_kjt, + ) + # pyre-ignore + sum(aux_losses.values()).backward() + optimizer.step() + metric_logger.update( + mode="train", + predictions=mt_target_preds, + labels=mt_target_labels, + weights=mt_target_weights, + num_candidates=sample.candidates_features_kjt.lengths().view( + len(sample.candidates_features_kjt.keys()), -1 + )[0], + ) + if train_batch_idx % metric_log_frequency == 0: + metric_logger.compute_and_log( + mode="train", + additional_logs={ + "losses": aux_losses, + }, + ) + train_batch_idx += 1 + if output_trace: + assert profiler is not None + profiler.step() + if num_train_batches is not None and train_batch_idx >= num_train_batches: + break + eval_ts = train_ts + 1 + dataset.dataset.is_eval = True # pyre-ignore [16] + model.eval() + eval_batch_idx: int = 0 + eval_dataloader = make_streaming_dataloader(dataset=dataset, ts=eval_ts) + eval_data_iterator = iter(eval_dataloader) + with torch.no_grad(): + while True: + try: + sample = next(eval_data_iterator) + except StopIteration: + break + sample.to(device) + ( + _, + _, + _, + mt_target_preds, + mt_target_labels, + mt_target_weights, + ) = model.forward( + sample.uih_features_kjt, + sample.candidates_features_kjt, + ) + metric_logger.update( + mode="eval", + predictions=mt_target_preds, + labels=mt_target_labels, + weights=mt_target_weights, + num_candidates=sample.candidates_features_kjt.lengths().view( + len(sample.candidates_features_kjt.keys()), -1 + )[0], + ) + eval_batch_idx += 1 + if output_trace: + assert profiler is not None + profiler.step() + if eval_batch_idx % metric_log_frequency == 0: + metric_logger.compute_and_log(mode="eval") + if num_eval_batches is not None and eval_batch_idx >= num_eval_batches: + break + for k, v in metric_logger.compute(mode="eval").items(): + print(f"{k}: {v}") + if ( + train_ts % checkpoint_frequency == 0 and train_ts > 0 + ) or train_ts == num_train_ts - 1: + save_dmp_checkpoint( + model=model, + optimizer=optimizer, + metric_logger=metric_logger, + rank=rank, + batch_idx=train_ts, + ) + + eval_ts = num_train_ts + dataset.dataset.is_eval = True + model.eval() + eval_batch_idx: int = 0 + eval_dataloader = make_streaming_dataloader(dataset=dataset, ts=eval_ts) + eval_data_iterator = iter(eval_dataloader) + with torch.no_grad(): + while True: + try: + sample = next(eval_data_iterator) + except StopIteration: + break + sample.to(device) + ( + _, + _, + _, + mt_target_preds, + mt_target_labels, + mt_target_weights, + ) = model.forward( + sample.uih_features_kjt, + sample.candidates_features_kjt, + ) + metric_logger.update( + mode="eval", + predictions=mt_target_preds, + labels=mt_target_labels, + weights=mt_target_weights, + num_candidates=sample.candidates_features_kjt.lengths().view( + len(sample.candidates_features_kjt.keys()), -1 + )[0], + ) + eval_batch_idx += 1 + if output_trace: + assert profiler is not None + profiler.step() + if eval_batch_idx % metric_log_frequency == 0: + metric_logger.compute_and_log(mode="eval") + if num_eval_batches is not None and eval_batch_idx >= num_eval_batches: + break + for k, v in metric_logger.compute(mode="eval").items(): + print(f"{k}: {v}") diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py new file mode 100644 index 000000000..52091d8dd --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py @@ -0,0 +1,652 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe +""" +mlperf dlrm_v3 inference benchmarking tool. +""" + +import contextlib +import logging +import os +import time +from typing import Callable, Dict, List, Optional + +import gin +import tensorboard # @manual=//tensorboard:lib # noqa: F401 - required implicit dep when using torch.utils.tensorboard +import torch +from generative_recommenders.dlrm_v3.datasets.dataset import DLRMv3RandomDataset +from generative_recommenders.dlrm_v3.datasets.kuairand import DLRMv3KuaiRandDataset +from generative_recommenders.dlrm_v3.datasets.movie_lens import DLRMv3MovieLensDataset +from generative_recommenders.dlrm_v3.datasets.synthetic_movie_lens import ( + DLRMv3SyntheticMovieLensDataset, +) +from generative_recommenders.dlrm_v3.datasets.synthetic_streaming import ( + DLRMv3SyntheticStreamingDataset, +) +from generative_recommenders.dlrm_v3.datasets.yambda import DLRMv3YambdaDataset +from generative_recommenders.modules.multitask_module import ( + MultitaskTaskType, + TaskConfig, +) +from torch.profiler import profile, profiler, ProfilerActivity # pyre-ignore [21] +from torch.utils.tensorboard import SummaryWriter +from torchrec.metrics.accuracy import AccuracyMetricComputation +from torchrec.metrics.auc import AUCMetricComputation, compute_auc +from torchrec.metrics.gauc import GAUCMetricComputation +from torchrec.metrics.mae import MAEMetricComputation +from torchrec.metrics.metrics_namespace import MetricName, MetricPrefix +from torchrec.metrics.mse import MSEMetricComputation +from torchrec.metrics.ne import NEMetricComputation +from torchrec.metrics.rec_metric import ( + MetricComputationReport, + RecMetricComputation, +) + + +class LifetimeAUCMetricComputation(AUCMetricComputation): + """AUC over all predictions seen so far (uncapped buffer); emits with the LIFETIME prefix.""" + + def _compute(self) -> List[MetricComputationReport]: + from typing import cast as _cast + from torchrec.metrics.auc import LABELS, PREDICTIONS, WEIGHTS + return [ + MetricComputationReport( + name=MetricName.AUC, + metric_prefix=MetricPrefix.LIFETIME, + value=compute_auc( + self._n_tasks, + _cast(List[torch.Tensor], getattr(self, PREDICTIONS)), + _cast(List[torch.Tensor], getattr(self, LABELS)), + _cast(List[torch.Tensor], getattr(self, WEIGHTS)), + self._apply_bin, + ), + ) + ] + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger("utils") + + +def _on_trace_ready_fn( + rank: Optional[int] = None, +) -> Callable[[torch.profiler.profile], None]: + """ + Create a callback function for handling profiler trace output. + + Args: + rank: Optional process rank for distributed training (included in filename). + + Returns: + A callback function that exports profiler traces to Manifold storage. + """ + + def handle_fn(p: torch.profiler.profile) -> None: + bucket_name = "hammer_gpu_traces" + pid = os.getpid() + rank_str = f"_rank_{rank}" if rank is not None else "" + file_name = f"libkineto_activities_{pid}_{rank_str}.json" + manifold_path = "tree/dlrm_v3_bench" + target_object_name = manifold_path + "/" + file_name + ".gz" + path = f"manifold://{bucket_name}/{manifold_path}/{file_name}" + logger.warning( + p.key_averages(group_by_input_shape=True).table( + sort_by="self_cuda_time_total" + ) + ) + logger.warning( + f"trace url: https://www.internalfb.com/intern/perfdoctor/trace_view?filepath={target_object_name}&bucket={bucket_name}" + ) + p.export_chrome_trace(path) + + return handle_fn + + +def profiler_or_nullcontext(enabled: bool, with_stack: bool): + """ + Create a profiler context manager or null context based on enabled flag. + + Args: + enabled: Whether to enable profiling. + with_stack: Whether to include stack traces in profile. + + Returns: + Either a torch.profiler.profile context manager or nullcontext. + """ + return ( + profile( + # pyre-fixme[16]: Module `profiler` has no attribute `ProfilerActivity`. + activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA], + on_trace_ready=_on_trace_ready_fn(), + with_stack=with_stack, + ) + if enabled + else contextlib.nullcontext() + ) + + +class Profiler: + """ + Wrapper around PyTorch profiler with scheduled profiling. + + Implements a wait-warmup-active schedule for controlled profiling that + avoids startup noise and captures representative performance data. + + Args: + rank: Process rank for trace file naming. + active: Number of active profiling steps (default: 50). + """ + + def __init__(self, rank, active: int = 50) -> None: + self.rank = rank + self._profiler: profiler.profile = torch.profiler.profile( + schedule=torch.profiler.schedule( + wait=10, + warmup=20, + active=active, + repeat=1, + ), + on_trace_ready=_on_trace_ready_fn(self.rank), + # pyre-fixme[16]: Module `profiler` has no attribute `ProfilerActivity`. + activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA], + record_shapes=True, + profile_memory=False, + with_stack=False, + with_flops=False, + with_modules=False, + ) + + def step(self) -> None: + """Advance the profiler to the next step.""" + self._profiler.step() + + +@gin.configurable +class MetricsLogger: + """ + Logger for tracking and computing recommendation metrics. + + Supports both classification metrics (NE, Accuracy, GAUC) and regression + metrics (MSE, MAE) based on multitask configuration. + + Args: + multitask_configs: List of task configurations defining metric types. + batch_size: Batch size for metric computation. + window_size: Window size for running metric aggregation. + device: Device to place metric tensors on. + rank: Process rank for distributed training. + tensorboard_log_path: Optional path for TensorBoard logging. + """ + + def __init__( + self, + multitask_configs: List[TaskConfig], + batch_size: int, + window_size: int, + device: torch.device, + rank: int, + tensorboard_log_path: str = "", + world_size: int = 1, + auc_threshold: Optional[float] = None, + ) -> None: + self.multitask_configs: List[TaskConfig] = multitask_configs + all_classification_tasks: List[str] = [ + task.task_name + for task in self.multitask_configs + if task.task_type != MultitaskTaskType.REGRESSION + ] + all_regression_tasks: List[str] = [ + task.task_name + for task in self.multitask_configs + if task.task_type == MultitaskTaskType.REGRESSION + ] + assert all_classification_tasks + all_regression_tasks == [ + task.task_name for task in multitask_configs + ] + self.task_names: List[str] = all_classification_tasks + all_regression_tasks + + self.class_metrics: Dict[str, List[RecMetricComputation]] = { + "train": [], + "eval": [], + } + if all_classification_tasks: + for mode in ["train", "eval"]: + self.class_metrics[mode].append( + NEMetricComputation( + my_rank=rank, + batch_size=batch_size, + n_tasks=len(all_classification_tasks), + window_size=window_size, + ).to(device) + ) + self.class_metrics[mode].append( + AccuracyMetricComputation( + my_rank=rank, + batch_size=batch_size, + n_tasks=len(all_classification_tasks), + window_size=window_size, + ).to(device) + ) + self.class_metrics[mode].append( + GAUCMetricComputation( + my_rank=rank, + batch_size=batch_size, + n_tasks=len(all_classification_tasks), + window_size=window_size, + ).to(device) + ) + self.class_metrics[mode].append( + AUCMetricComputation( + my_rank=rank, + batch_size=batch_size, + n_tasks=len(all_classification_tasks), + window_size=window_size, + ).to(device) + ) + self.class_metrics[mode].append( + LifetimeAUCMetricComputation( + my_rank=rank, + batch_size=batch_size, + n_tasks=len(all_classification_tasks), + window_size=10_000_000, + ).to(device) + ) + + self.regression_metrics: Dict[str, List[RecMetricComputation]] = { + "train": [], + "eval": [], + } + if all_regression_tasks: + for mode in ["train", "eval"]: + self.regression_metrics[mode].append( + MSEMetricComputation( + my_rank=rank, + batch_size=batch_size, + n_tasks=len(all_regression_tasks), + window_size=window_size, + ).to(device) + ) + self.regression_metrics[mode].append( + MAEMetricComputation( + my_rank=rank, + batch_size=batch_size, + n_tasks=len(all_regression_tasks), + window_size=window_size, + ).to(device) + ) + + self.global_step: Dict[str, int] = {"train": 0, "eval": 0} + self.tb_logger: Optional[SummaryWriter] = None + if tensorboard_log_path != "": + self.tb_logger = SummaryWriter(log_dir=tensorboard_log_path, purge_step=0) + self.tb_logger.flush() + + # Throughput / time-to-target tracking. Counters are train-only; eval + # samples are not relevant for headline samples/sec numbers. + self._world_size: int = max(1, int(world_size)) + self._auc_threshold: Optional[float] = auc_threshold + self._time_to_target_logged: bool = False + self._perf_t_start: float = time.perf_counter() + self._perf_t_window: float = self._perf_t_start + self._perf_steps_in_window: int = 0 + self._perf_total_samples: int = 0 + self._perf_samples_counter: torch.Tensor = torch.zeros( + 1, dtype=torch.long, device=device + ) + + @property + def all_metrics(self) -> Dict[str, List[RecMetricComputation]]: + """ + Get all metrics for train and eval modes. + + Returns: + Dictionary mapping mode ('train'/'eval') to list of metric computations. + """ + return { + "train": self.class_metrics["train"] + self.regression_metrics["train"], + "eval": self.class_metrics["eval"] + self.regression_metrics["eval"], + } + + def update( + self, + predictions: torch.Tensor, + weights: torch.Tensor, + labels: torch.Tensor, + num_candidates: torch.Tensor, + mode: str = "train", + ) -> None: + """ + Update metrics with new batch of predictions and labels. + + Args: + predictions: Model prediction tensor. + weights: Sample weight tensor. + labels: Ground truth label tensor. + num_candidates: Number of candidates per sample (for GAUC). + mode: Either 'train' or 'eval'. + """ + for metric in self.all_metrics[mode]: + if isinstance(metric, GAUCMetricComputation): + metric.update( + predictions=predictions, + labels=labels, + weights=weights, + num_candidates=num_candidates, + ) + else: + metric.update( + predictions=predictions, + labels=labels, + weights=weights, + ) + self.global_step[mode] += 1 + if mode == "train": + # Accumulate on-device to avoid a per-step GPU->CPU sync; we read + # the counter only at compute_and_log boundaries. + self._perf_samples_counter += num_candidates.sum().to( + self._perf_samples_counter.dtype + ) + self._perf_steps_in_window += 1 + + def compute(self, mode: str = "train") -> Dict[str, float]: + """ + Compute and return all metrics for the current window. + + Args: + mode: Either 'train' or 'eval'. + + Returns: + Dictionary mapping metric names to their computed values. + """ + all_computed_metrics = {} + + for metric in self.all_metrics[mode]: + computed_metrics = metric.compute() + for computed in computed_metrics: + all_values = computed.value.cpu() + for i, task_name in enumerate(self.task_names): + key = f"metric/{str(computed.metric_prefix) + str(computed.name)}/{task_name}" + all_computed_metrics[key] = all_values[i] + + logger.info( + f"{mode} - Step {self.global_step[mode]} metrics: {all_computed_metrics}" + ) + return all_computed_metrics + + def compute_and_log( + self, + mode: str = "train", + additional_logs: Optional[Dict[str, Dict[str, torch.Tensor]]] = None, + ) -> Dict[str, float]: + """ + Compute metrics and log to TensorBoard. + + Args: + mode: Either 'train' or 'eval'. + additional_logs: Optional additional data to log. + + Returns: + Dictionary mapping metric names to their computed values. + + Raises: + AssertionError: If TensorBoard logger is not configured. + """ + assert self.tb_logger is not None + all_computed_metrics = self.compute(mode) + for k, v in all_computed_metrics.items(): + self.tb_logger.add_scalar( # pyre-ignore [16] + f"{mode}_{k}", + v, + global_step=self.global_step[mode], + ) + + if additional_logs is not None: + for tag, data in additional_logs.items(): + for data_name, data_value in data.items(): + self.tb_logger.add_scalar( + f"{tag}/{mode}_{data_name}", + data_value.detach().clone().cpu(), + global_step=self.global_step[mode], + ) + + # Throughput metrics (train only). One GPU->CPU sync per call. + if mode == "train" and self._perf_steps_in_window > 0: + now = time.perf_counter() + dt = max(now - self._perf_t_window, 1e-6) + n_samples = int(self._perf_samples_counter.item()) + self._perf_total_samples += n_samples + local_sps = n_samples / dt + global_sps = local_sps * self._world_size + step_ms = dt * 1000.0 / self._perf_steps_in_window + elapsed = now - self._perf_t_start + step = self.global_step["train"] + self.tb_logger.add_scalar( + "perf/train_samples_per_sec_local", local_sps, global_step=step + ) + self.tb_logger.add_scalar( + "perf/train_samples_per_sec_global", global_sps, global_step=step + ) + self.tb_logger.add_scalar( + "perf/train_step_time_ms", step_ms, global_step=step + ) + self.tb_logger.add_scalar( + "perf/train_total_samples", self._perf_total_samples, global_step=step + ) + self.tb_logger.add_scalar( + "perf/train_elapsed_sec", elapsed, global_step=step + ) + logger.info( + f"train - Step {step} perf: local_sps={local_sps:.1f} " + f"global_sps={global_sps:.1f} step_ms={step_ms:.2f} " + f"elapsed_sec={elapsed:.1f} total_samples={self._perf_total_samples}" + ) + self._perf_t_window = now + self._perf_steps_in_window = 0 + self._perf_samples_counter.zero_() + + # Time-to-target: latch wall-clock once any task's AUC crosses threshold. + # Matches MLPerf DLRM-DCNv2 reporting style (default upstream target 0.80275). + if ( + self._auc_threshold is not None + and not self._time_to_target_logged + ): + for key, val in all_computed_metrics.items(): + metric_short = key.split("/")[-2] if "/" in key else key + if metric_short.endswith("auc") and not metric_short.endswith("gauc"): + if float(val) >= self._auc_threshold: + ttt = time.perf_counter() - self._perf_t_start + self.tb_logger.add_scalar( + f"perf/time_to_auc_{self._auc_threshold:.5f}_sec", + ttt, + global_step=self.global_step[mode], + ) + logger.info( + f"REACHED AUC>={self._auc_threshold} on {key}=" + f"{float(val):.6f} at elapsed_sec={ttt:.2f} " + f"step={self.global_step[mode]}" + ) + self._time_to_target_logged = True + break + + return all_computed_metrics + + def reset(self, mode: str = "train"): + """ + Reset all metrics for a given mode. + + Args: + mode: Either 'train' or 'eval'. + """ + for metric in self.all_metrics[mode]: + metric.reset() + + +# the datasets we support +SUPPORTED_DATASETS = [ + "debug", + "movielens-1m", + "movielens-20m", + "movielens-13b", + "movielens-18b", + "kuairand-1k", + "streaming-400m", + "streaming-200b", + "streaming-100b", + "sampled-streaming-100b", + "yambda-5b", +] + + +@gin.configurable +def get_dataset(name: str, new_path_prefix: str = ""): + """ + Get dataset class and configuration by name. + + Args: + name: Dataset identifier (must be in SUPPORTED_DATASETS). + new_path_prefix: Optional prefix to prepend to data paths. + + Returns: + Tuple of (dataset_class, kwargs_dict) for dataset instantiation. + + Raises: + AssertionError: If dataset name is not supported. + """ + assert name in SUPPORTED_DATASETS, f"dataset {name} not supported" + if name == "debug": + return DLRMv3RandomDataset, {} + if name == "movielens-1m": + return ( + DLRMv3MovieLensDataset, + { + "ratings_file": os.path.join( + new_path_prefix, "data/ml-1m/sasrec_format.csv" + ), + }, + ) + if name == "movielens-20m": + return ( + DLRMv3MovieLensDataset, + { + "ratings_file": os.path.join( + new_path_prefix, "data/ml-20m/sasrec_format.csv" + ), + }, + ) + if name == "movielens-13b": + return ( + DLRMv3SyntheticMovieLensDataset, + { + "ratings_file_prefix": os.path.join( + new_path_prefix, "data/ml-13b/16x16384" + ), + }, + ) + if name == "movielens-18b": + return ( + DLRMv3SyntheticMovieLensDataset, + { + "ratings_file_prefix": os.path.join( + new_path_prefix, "data/ml-18b/20x36864" + ), + }, + ) + if name == "kuairand-1k": + return ( + DLRMv3KuaiRandDataset, + { + "seq_logs_file": os.path.join( + new_path_prefix, "data/KuaiRand-1K/data/processed_seqs.csv" + ), + }, + ) + if name == "streaming-400m": + return ( + DLRMv3SyntheticStreamingDataset, + { + "ratings_file_prefix": os.path.join( + new_path_prefix, "data/streaming-400m/" + ), + "train_ts": 8, + "total_ts": 10, + "num_files": 3, + "num_users": 150_000, + "num_items": 1_500_000, + "num_categories": 128, + }, + ) + if name == "streaming-200b": + return ( + DLRMv3SyntheticStreamingDataset, + { + "ratings_file_prefix": os.path.join( + new_path_prefix, "data/streaming-200b/" + ), + "train_ts": 90, + "total_ts": 100, + "num_files": 100, + "num_users": 10_000_000, + "num_items": 1_000_000_000, + "num_categories": 128, + }, + ) + if name == "streaming-100b": + return ( + DLRMv3SyntheticStreamingDataset, + { + "ratings_file_prefix": os.path.join( + new_path_prefix, "data/streaming-100b/" + ), + "train_ts": 90, + "total_ts": 100, + "num_files": 100, + "num_users": 5_000_000, + "num_items": 1_000_000_000, + "num_categories": 128, + }, + ) + if name == "yambda-5b": + from generative_recommenders.dlrm_v3.configs import YAMBDA_5B_CROSS_SPECS + + return ( + DLRMv3YambdaDataset, + { + # Layout: /processed_5b/{train_sessions.parquet,...} + # and /shared_metadata/{artist,album}_item_mapping.parquet. + # The dataset auto-builds a MAP_SHARED-mmap'd cache of the + # flat columns + LISTEN-anchor positions under + # /hstu_cache_L/ on first use; + # all ranks on a node share the same physical pages. + "processed_dir": os.path.join(new_path_prefix, "processed_5b"), + "metadata_dir": os.path.join(new_path_prefix, "shared_metadata"), + "history_length": 4096, + "scan_window": 20000, + "cross_specs": YAMBDA_5B_CROSS_SPECS, + }, + ) + if name == "sampled-streaming-100b": + return ( + DLRMv3SyntheticStreamingDataset, + { + "ratings_file_prefix": os.path.join( + new_path_prefix, "data/streaming-100b/sampled_data/" + ), + "train_ts": 90, + "total_ts": 100, + "num_files": 1, + "num_users": 50_000, + "num_items": 1_000_000_000, + "num_categories": 128, + }, + ) diff --git a/recommendation_v4/generative_recommenders/modules/action_encoder.py b/recommendation_v4/generative_recommenders/modules/action_encoder.py new file mode 100644 index 000000000..13b65557e --- /dev/null +++ b/recommendation_v4/generative_recommenders/modules/action_encoder.py @@ -0,0 +1,112 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +from typing import Dict, List, Optional, Tuple + +import torch +from generative_recommenders.common import HammerModule +from generative_recommenders.ops.jagged_tensors import concat_2D_jagged + + +class ActionEncoder(HammerModule): + def __init__( + self, + action_embedding_dim: int, + action_feature_name: str, + action_weights: List[int], + watchtime_feature_name: str = "", + watchtime_to_action_thresholds_and_weights: Optional[ + List[Tuple[int, int]] + ] = None, + embedding_init_std: float = 0.1, + is_inference: bool = False, + ) -> None: + super().__init__(is_inference=is_inference) + self._watchtime_feature_name: str = watchtime_feature_name + self._action_feature_name: str = action_feature_name + self._watchtime_to_action_thresholds_and_weights: List[Tuple[int, int]] = ( + watchtime_to_action_thresholds_and_weights + if watchtime_to_action_thresholds_and_weights is not None + else [] + ) + self.register_buffer( + "_combined_action_weights", + torch.tensor( + action_weights + + [x[1] for x in self._watchtime_to_action_thresholds_and_weights] + ), + ) + self._num_action_types: int = len(action_weights) + len( + self._watchtime_to_action_thresholds_and_weights + ) + self._action_embedding_dim = action_embedding_dim + self._action_embedding_table: torch.nn.Parameter = torch.nn.Parameter( + torch.empty((self._num_action_types, action_embedding_dim)).normal_( + mean=0, std=embedding_init_std + ), + ) + self._target_action_embedding_table: torch.nn.Parameter = torch.nn.Parameter( + torch.empty((1, self._num_action_types * action_embedding_dim)).normal_( + mean=0, std=embedding_init_std + ), + ) + + @property + def output_embedding_dim(self) -> int: + return self._action_embedding_dim * self._num_action_types + + def forward( + self, + max_uih_len: int, + max_targets: int, + uih_offsets: torch.Tensor, + target_offsets: torch.Tensor, + seq_embeddings: torch.Tensor, + seq_payloads: Dict[str, torch.Tensor], + ) -> torch.Tensor: + seq_actions = seq_payloads[self._action_feature_name] + if len(self._watchtime_to_action_thresholds_and_weights) > 0: + watchtimes = seq_payloads[self._watchtime_feature_name] + for threshold, weight in self._watchtime_to_action_thresholds_and_weights: + seq_actions = torch.bitwise_or( + seq_actions, (watchtimes >= threshold).to(torch.int64) * weight + ) + exploded_actions = ( + torch.bitwise_and( + seq_actions.unsqueeze(-1), self._combined_action_weights.unsqueeze(0) + ) + > 0 + ) + action_embeddings = ( + exploded_actions.unsqueeze(-1) * self._action_embedding_table.unsqueeze(0) + ).view(-1, self._num_action_types * self._action_embedding_dim) + total_targets: int = seq_embeddings.size(0) - action_embeddings.size(0) + action_embeddings = concat_2D_jagged( + max_seq_len=max_uih_len + max_targets, + values_left=action_embeddings, + values_right=self._target_action_embedding_table.tile( + total_targets, + 1, + ), + max_len_left=max_uih_len, + max_len_right=max_targets, + offsets_left=uih_offsets, + offsets_right=target_offsets, + kernel=self.hammer_kernel(), + ) + return action_embeddings diff --git a/recommendation_v4/generative_recommenders/modules/content_encoder.py b/recommendation_v4/generative_recommenders/modules/content_encoder.py new file mode 100644 index 000000000..acca82dbf --- /dev/null +++ b/recommendation_v4/generative_recommenders/modules/content_encoder.py @@ -0,0 +1,109 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +from typing import Dict, List, Optional + +import torch +from generative_recommenders.common import HammerModule +from generative_recommenders.ops.jagged_tensors import concat_2D_jagged + + +class ContentEncoder(HammerModule): + def __init__( + self, + input_embedding_dim: int, + additional_content_features: Optional[Dict[str, int]] = None, + target_enrich_features: Optional[Dict[str, int]] = None, + is_inference: bool = False, + ) -> None: + super().__init__(is_inference=is_inference) + self._input_embedding_dim: int = input_embedding_dim + self._additional_content_features: Dict[str, int] = ( + additional_content_features + if additional_content_features is not None + else {} + ) + self._target_enrich_features: Dict[str, int] = ( + target_enrich_features if target_enrich_features is not None else {} + ) + self._target_enrich_dummy_embeddings: torch.nn.ParameterDict = ( + torch.nn.ParameterDict( + { + name: torch.nn.Parameter( + torch.empty((1, dim)).normal_(mean=0, std=0.1), + ) + for name, dim in self._target_enrich_features.items() + } + ) + ) + + @property + def output_embedding_dim(self) -> int: + return self._input_embedding_dim + sum( + list(self._additional_content_features.values()) + + list(self._target_enrich_features.values()) + ) + + def forward( + self, + max_uih_len: int, + max_targets: int, + uih_offsets: torch.Tensor, + target_offsets: torch.Tensor, + seq_embeddings: torch.Tensor, + seq_payloads: Dict[str, torch.Tensor], + ) -> torch.Tensor: + content_embeddings_list: List[torch.Tensor] = [seq_embeddings] + if len(self._additional_content_features) > 0: + content_embeddings_list = content_embeddings_list + [ + (seq_payloads[x].to(seq_embeddings.dtype)) + for x in self._additional_content_features.keys() + ] + + if self._target_enrich_dummy_embeddings: + total_seq_len: int = seq_embeddings.size(0) + for name, param in self._target_enrich_dummy_embeddings.items(): + enrich_embeddings_target = seq_payloads[name].to(seq_embeddings.dtype) + total_targets: int = enrich_embeddings_target.size(0) + total_uih_len: int = total_seq_len - total_targets + enrich_embeddings_uih = param.tile(total_uih_len, 1).to( + seq_embeddings.dtype + ) + enrich_embeddings = concat_2D_jagged( + max_seq_len=max_uih_len + max_targets, + values_left=enrich_embeddings_uih, + values_right=enrich_embeddings_target, + max_len_left=max_uih_len, + max_len_right=max_targets, + offsets_left=uih_offsets, + offsets_right=target_offsets, + kernel=self.hammer_kernel(), + ) + content_embeddings_list.append(enrich_embeddings) + + if ( + len(self._target_enrich_features) == 0 + and len(self._additional_content_features) == 0 + ): + return seq_embeddings + else: + content_embeddings = torch.cat( + content_embeddings_list, + dim=1, + ) + return content_embeddings diff --git a/recommendation_v4/generative_recommenders/modules/contextual_interleave_preprocessor.py b/recommendation_v4/generative_recommenders/modules/contextual_interleave_preprocessor.py new file mode 100644 index 000000000..fff0d72f0 --- /dev/null +++ b/recommendation_v4/generative_recommenders/modules/contextual_interleave_preprocessor.py @@ -0,0 +1,357 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +from math import sqrt +from typing import Callable, Dict, Optional, Tuple + +import torch +from generative_recommenders.common import fx_unwrap_optional_tensor +from generative_recommenders.modules.action_encoder import ActionEncoder +from generative_recommenders.modules.content_encoder import ContentEncoder +from generative_recommenders.modules.contextualize_mlps import ( + ContextualizedMLP, + ParameterizedContextualizedMLP, +) +from generative_recommenders.modules.preprocessors import ( + get_contextual_input_embeddings, + InputPreprocessor, +) +from generative_recommenders.ops.jagged_tensors import concat_2D_jagged + + +class ContextualInterleavePreprocessor(InputPreprocessor): + def __init__( + self, + input_embedding_dim: int, + output_embedding_dim: int, + contextual_feature_to_max_length: Dict[str, int], + contextual_feature_to_min_uih_length: Dict[str, int], + content_encoder: ContentEncoder, + content_contextualize_mlp_fn: Callable[ + [int, int, int, bool], ContextualizedMLP + ], + action_encoder: ActionEncoder, + action_contextualize_mlp_fn: Callable[[int, int, int, bool], ContextualizedMLP], + pmlp_contextual_dropout_ratio: float = 0.0, + enable_interleaving: bool = False, + is_inference: bool = False, + ) -> None: + super().__init__(is_inference=is_inference) + self._input_embedding_dim: int = input_embedding_dim + self._output_embedding_dim: int = output_embedding_dim + self._contextual_feature_to_max_length: Dict[str, int] = ( + contextual_feature_to_max_length + ) + self._max_contextual_seq_len: int = sum( + contextual_feature_to_max_length.values() + ) + self._contextual_feature_to_min_uih_length: Dict[str, int] = ( + contextual_feature_to_min_uih_length + ) + std = 1.0 * sqrt(2.0 / float(input_embedding_dim + output_embedding_dim)) + self._batched_contextual_linear_weights = torch.nn.Parameter( + torch.empty( + ( + self._max_contextual_seq_len, + input_embedding_dim, + output_embedding_dim, + ) + ).normal_(0.0, std) + ) + self._pmlp_contextual_dropout_ratio: float = pmlp_contextual_dropout_ratio + self._batched_contextual_linear_bias = torch.nn.Parameter( + torch.empty((self._max_contextual_seq_len, 1, output_embedding_dim)).fill_( + 0.0 + ) + ) + contextual_embedding_dim: int = ( + self._max_contextual_seq_len * input_embedding_dim + ) + self._content_encoder: ContentEncoder = content_encoder + self._content_embedding_mlp: ContextualizedMLP = content_contextualize_mlp_fn( + self._content_encoder.output_embedding_dim, + output_embedding_dim, + contextual_embedding_dim, + is_inference, + ) + self._action_encoder: ActionEncoder = action_encoder + self._action_embedding_mlp: ContextualizedMLP = action_contextualize_mlp_fn( + self._action_encoder.output_embedding_dim, + output_embedding_dim, + contextual_embedding_dim, + is_inference, + ) + self._enable_interleaving: bool = enable_interleaving + + def combine_embeddings( + self, + max_uih_len: int, + max_targets: int, + total_uih_len: int, + total_targets: int, + seq_lengths: torch.Tensor, + seq_timestamps: torch.Tensor, + content_embeddings: torch.Tensor, + action_embeddings: torch.Tensor, + contextual_embeddings: Optional[torch.Tensor], + num_targets: torch.Tensor, + ) -> Tuple[ + int, + int, + int, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + ]: + if self._enable_interleaving: + output_seq_timestamps = seq_timestamps.repeat_interleave(2) + output_seq_embeddings = torch.stack( + [content_embeddings, action_embeddings], dim=1 + ).reshape(-1, self._output_embedding_dim) + if self.interleave_targets(): + output_seq_lengths = seq_lengths * 2 + output_max_seq_len = (max_uih_len + max_targets) * 2 + output_num_targets = num_targets * 2 + output_total_uih_len = total_uih_len * 2 + output_total_targets = total_targets * 2 + else: + seq_lengths_by_2 = seq_lengths * 2 + output_seq_lengths = seq_lengths_by_2 - num_targets + output_max_seq_len = 2 * max_uih_len + max_targets + indices = torch.arange( + 2 * (max_uih_len + max_targets), device=seq_lengths.device + ).view(1, -1) + valid_mask = torch.logical_and( + indices < seq_lengths_by_2.view(-1, 1), + torch.logical_or( + indices < (output_seq_lengths - num_targets).view(-1, 1), + torch.remainder(indices, 2) == 0, + ), + ) + jagged_valid_mask = ( + torch.ops.fbgemm.dense_to_jagged( + valid_mask.int().unsqueeze(-1), + [ + torch.ops.fbgemm.asynchronous_complete_cumsum( + seq_lengths_by_2 + ) + ], + )[0] + .to(torch.bool) + .squeeze(1) + ) + output_seq_embeddings = output_seq_embeddings[jagged_valid_mask] + output_seq_timestamps = output_seq_timestamps[jagged_valid_mask] + output_num_targets = num_targets + output_total_uih_len = total_uih_len * 2 + output_total_targets = total_targets + else: + output_max_seq_len = max_uih_len + max_targets + output_seq_lengths = seq_lengths + output_num_targets = num_targets + output_seq_timestamps = seq_timestamps + output_seq_embeddings = content_embeddings + action_embeddings + output_total_uih_len = total_uih_len + output_total_targets = total_targets + + # concat contextual embeddings + output_seq_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum( + output_seq_lengths + ) + if self._max_contextual_seq_len > 0: + output_seq_embeddings = concat_2D_jagged( + max_seq_len=self._max_contextual_seq_len + output_max_seq_len, + values_left=fx_unwrap_optional_tensor(contextual_embeddings).reshape( + -1, self._output_embedding_dim + ), + values_right=output_seq_embeddings, + max_len_left=self._max_contextual_seq_len, + max_len_right=output_max_seq_len, + offsets_left=None, + offsets_right=output_seq_offsets, + kernel=self.hammer_kernel(), + ) + output_seq_timestamps = concat_2D_jagged( + max_seq_len=self._max_contextual_seq_len + output_max_seq_len, + values_left=torch.zeros( + (output_seq_lengths.size(0) * self._max_contextual_seq_len, 1), + dtype=output_seq_timestamps.dtype, + device=output_seq_timestamps.device, + ), + values_right=output_seq_timestamps.unsqueeze(-1), + max_len_left=self._max_contextual_seq_len, + max_len_right=output_max_seq_len, + offsets_left=None, + offsets_right=output_seq_offsets, + kernel=self.hammer_kernel(), + ).squeeze(-1) + output_max_seq_len = output_max_seq_len + self._max_contextual_seq_len + output_total_uih_len = ( + output_total_uih_len + + self._max_contextual_seq_len * output_seq_lengths.size(0) + ) + output_seq_lengths = output_seq_lengths + self._max_contextual_seq_len + output_seq_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum( + output_seq_lengths + ) + + return ( + output_max_seq_len, + output_total_uih_len, + output_total_targets, + output_seq_lengths, + output_seq_offsets, + output_seq_timestamps, + output_seq_embeddings, + output_num_targets, + ) + + def forward( # noqa C901 + self, + max_uih_len: int, + max_targets: int, + total_uih_len: int, + total_targets: int, + seq_lengths: torch.Tensor, + seq_timestamps: torch.Tensor, + seq_embeddings: torch.Tensor, + num_targets: torch.Tensor, + seq_payloads: Dict[str, torch.Tensor], + ) -> Tuple[ + int, + int, + int, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + Dict[str, torch.Tensor], + ]: + max_seq_len = max_uih_len + max_targets + with torch.autocast( + "cuda", + dtype=torch.bfloat16, + enabled=(not self.is_inference and self._training_dtype == torch.bfloat16), + ): + # get contextual_embeddings + contextual_embeddings: Optional[torch.Tensor] = None + pmlp_contextual_embeddings: Optional[torch.Tensor] = None + if self._max_contextual_seq_len > 0: + contextual_input_embeddings = get_contextual_input_embeddings( + seq_lengths=seq_lengths, + seq_payloads=seq_payloads, + contextual_feature_to_max_length=self._contextual_feature_to_max_length, + contextual_feature_to_min_uih_length=self._contextual_feature_to_min_uih_length, + dtype=seq_embeddings.dtype, + ) + if isinstance( + self._action_embedding_mlp, ParameterizedContextualizedMLP + ) or isinstance( + self._action_embedding_mlp, ParameterizedContextualizedMLP + ): + pmlp_contextual_embeddings = torch.nn.functional.dropout( + contextual_input_embeddings, + p=self._pmlp_contextual_dropout_ratio, + training=self.training, + ) + contextual_embeddings = torch.baddbmm( + self._batched_contextual_linear_bias.to( + contextual_input_embeddings.dtype + ), + contextual_input_embeddings.view( + -1, self._max_contextual_seq_len, self._input_embedding_dim + ).transpose(0, 1), + self._batched_contextual_linear_weights.to( + contextual_input_embeddings.dtype + ), + ).transpose(0, 1) + + # content embeddings + seq_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum(seq_lengths) + target_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum(num_targets) + uih_offsets = seq_offsets - target_offsets + content_embeddings = self._content_encoder( + max_uih_len=max_uih_len, + max_targets=max_targets, + uih_offsets=uih_offsets, + target_offsets=target_offsets, + seq_embeddings=seq_embeddings, + seq_payloads=seq_payloads, + ) + content_embeddings = self._content_embedding_mlp( + seq_embeddings=content_embeddings, + seq_offsets=seq_offsets, + max_seq_len=max_seq_len, + contextual_embeddings=pmlp_contextual_embeddings, + ) + + # action embeddings + action_embeddings = self._action_encoder( + max_uih_len=max_uih_len, + max_targets=max_targets, + uih_offsets=uih_offsets, + target_offsets=target_offsets, + seq_embeddings=seq_embeddings, + seq_payloads=seq_payloads, + ).to(seq_embeddings.dtype) + action_embeddings = self._action_embedding_mlp( + seq_embeddings=action_embeddings, + seq_offsets=seq_offsets, + max_seq_len=max_seq_len, + contextual_embeddings=pmlp_contextual_embeddings, + ) + + ( + output_max_seq_len, + output_total_uih_len, + output_total_targets, + output_seq_lengths, + output_seq_offsets, + output_seq_timestamps, + output_seq_embeddings, + output_num_targets, + ) = self.combine_embeddings( + max_uih_len=max_uih_len, + max_targets=max_targets, + total_uih_len=total_uih_len, + total_targets=total_targets, + seq_lengths=seq_lengths, + seq_timestamps=seq_timestamps, + content_embeddings=content_embeddings, + action_embeddings=action_embeddings, + contextual_embeddings=contextual_embeddings, + num_targets=num_targets, + ) + + return ( + output_max_seq_len, + output_total_uih_len, + output_total_targets, + output_seq_lengths, + output_seq_offsets, + output_seq_timestamps, + output_seq_embeddings, + output_num_targets, + seq_payloads, + ) + + def interleave_targets(self) -> bool: + return self.is_train and self._enable_interleaving diff --git a/recommendation_v4/generative_recommenders/modules/contextualize_mlps.py b/recommendation_v4/generative_recommenders/modules/contextualize_mlps.py new file mode 100644 index 000000000..dc49effeb --- /dev/null +++ b/recommendation_v4/generative_recommenders/modules/contextualize_mlps.py @@ -0,0 +1,141 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict +import abc +from typing import Optional + +import torch +from generative_recommenders.common import HammerModule, init_mlp_weights_optional_bias +from generative_recommenders.ops.jagged_tensors import jagged_dense_bmm_broadcast_add +from generative_recommenders.ops.layer_norm import LayerNorm, SwishLayerNorm +from libfb.py.pyre import none_throws + + +class ContextualizedMLP(HammerModule): + @abc.abstractmethod + def forward( + self, + max_seq_len: int, + seq_embeddings: torch.Tensor, + seq_offsets: torch.Tensor, + contextual_embeddings: Optional[torch.Tensor], + ) -> torch.Tensor: + """ + Args: + seq_embeddings: (L, D) + seq_offsets: (B + 1,) + max_seq_len: int + contextual_embeddings: (B, D') + """ + pass + + +class SimpleContextualizedMLP(ContextualizedMLP): + def __init__( + self, + sequential_input_dim: int, + sequential_output_dim: int, + hidden_dim: int, + is_inference: bool = False, + ) -> None: + super().__init__(is_inference=is_inference) + self._mlp: torch.nn.Module = torch.nn.Sequential( + torch.nn.Linear( + in_features=sequential_input_dim, + out_features=hidden_dim, + ), + SwishLayerNorm(hidden_dim, is_inference=is_inference), + torch.nn.Linear( + in_features=hidden_dim, + out_features=sequential_output_dim, + ), + LayerNorm(sequential_output_dim), + ).apply(init_mlp_weights_optional_bias) + + def forward( + self, + seq_embeddings: torch.Tensor, + seq_offsets: torch.Tensor, + max_seq_len: int, + contextual_embeddings: Optional[torch.Tensor], + ) -> torch.Tensor: + return self._mlp(seq_embeddings) + + +class ParameterizedContextualizedMLP(ContextualizedMLP): + def __init__( + self, + contextual_embedding_dim: int, + sequential_input_dim: int, + sequential_output_dim: int, + hidden_dim: int, + is_inference: bool = False, + ) -> None: + super().__init__(is_inference=is_inference) + + self._sequential_input_dim: int = sequential_input_dim + self._sequential_output_dim: int = sequential_output_dim + + self._dense_features_compress: torch.nn.Module = torch.nn.Linear( + in_features=contextual_embedding_dim, + out_features=hidden_dim, + ).apply(init_mlp_weights_optional_bias) + + self._attn_raw_weights: torch.nn.Module = torch.nn.Sequential( + torch.nn.Linear( + in_features=hidden_dim, + out_features=sequential_input_dim * sequential_output_dim, + ), + ).apply(init_mlp_weights_optional_bias) + + self._attn_weights_norm: torch.nn.Module = torch.nn.LayerNorm( + [sequential_input_dim, sequential_output_dim] + ) + + self._res_weights: torch.nn.Module = torch.nn.Sequential( + torch.nn.Linear( + in_features=hidden_dim, + out_features=hidden_dim, + ), + SwishLayerNorm(hidden_dim), + torch.nn.Linear( + in_features=hidden_dim, + out_features=sequential_output_dim, + ), + ).apply(init_mlp_weights_optional_bias) + + def forward( + self, + seq_embeddings: torch.Tensor, + seq_offsets: torch.Tensor, + max_seq_len: int, + contextual_embeddings: Optional[torch.Tensor], + ) -> torch.Tensor: + shared_input = self._dense_features_compress(none_throws(contextual_embeddings)) + attn_weights = self._attn_weights_norm( + self._attn_raw_weights(shared_input).reshape( + -1, self._sequential_input_dim, self._sequential_output_dim + ) + ) + return jagged_dense_bmm_broadcast_add( + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + jagged=seq_embeddings, + dense=attn_weights.to(seq_embeddings.dtype), + bias=self._res_weights(shared_input), + kernel=self.hammer_kernel(), + ) diff --git a/recommendation_v4/generative_recommenders/modules/dlrm_hstu.py b/recommendation_v4/generative_recommenders/modules/dlrm_hstu.py new file mode 100644 index 000000000..af2edc998 --- /dev/null +++ b/recommendation_v4/generative_recommenders/modules/dlrm_hstu.py @@ -0,0 +1,626 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + + +import logging +from dataclasses import dataclass, field +from typing import Dict, List, NamedTuple, Optional, Tuple + +import torch +from generative_recommenders.common import ( + fx_infer_max_len, + fx_mark_length_features, + HammerKernel, + HammerModule, + init_mlp_weights_optional_bias, + set_static_max_seq_lens, +) +from generative_recommenders.modules.hstu_transducer import HSTUTransducer +from generative_recommenders.modules.multitask_module import ( + DefaultMultitaskModule, + MultitaskTaskType, + TaskConfig, +) +from generative_recommenders.modules.positional_encoder import HSTUPositionalEncoder +from generative_recommenders.modules.postprocessors import ( + LayerNormPostprocessor, + TimestampLayerNormPostprocessor, +) +from generative_recommenders.modules.preprocessors import ContextualPreprocessor +from generative_recommenders.modules.stu import STU, STULayer, STULayerConfig, STUStack +from generative_recommenders.ops.jagged_tensors import concat_2D_jagged +from generative_recommenders.ops.layer_norm import LayerNorm, SwishLayerNorm +from torch.autograd.profiler import record_function +from torchrec import KeyedJaggedTensor +from torchrec.modules.embedding_configs import EmbeddingConfig +from torchrec.modules.embedding_modules import EmbeddingCollection + +logger: logging.Logger = logging.getLogger(__name__) + +torch.fx.wrap("fx_infer_max_len") +torch.fx.wrap("len") + + +class SequenceEmbedding(NamedTuple): + lengths: torch.Tensor + embedding: torch.Tensor + + +@dataclass +class DlrmHSTUConfig: + max_seq_len: int = 16384 + max_num_candidates: int = 10 + max_num_candidates_inference: int = 5 + hstu_num_heads: int = 1 + hstu_attn_linear_dim: int = 256 + hstu_attn_qk_dim: int = 128 + hstu_attn_num_layers: int = 12 + hstu_embedding_table_dim: int = 192 + hstu_preprocessor_hidden_dim: int = 256 + hstu_transducer_embedding_dim: int = 0 + hstu_group_norm: bool = False + hstu_input_dropout_ratio: float = 0.2 + hstu_linear_dropout_rate: float = 0.2 + contextual_feature_to_max_length: Dict[str, int] = field(default_factory=dict) + contextual_feature_to_min_uih_length: Dict[str, int] = field(default_factory=dict) + candidates_weight_feature_name: str = "" + candidates_watchtime_feature_name: str = "" + candidates_querytime_feature_name: str = "" + causal_multitask_weights: float = 0.2 + multitask_configs: List[TaskConfig] = field(default_factory=list) + user_embedding_feature_names: List[str] = field(default_factory=list) + item_embedding_feature_names: List[str] = field(default_factory=list) + uih_post_id_feature_name: str = "" + uih_action_time_feature_name: str = "" + uih_weight_feature_name: str = "" + hstu_uih_feature_names: List[str] = field(default_factory=list) + hstu_candidate_feature_names: List[str] = field(default_factory=list) + merge_uih_candidate_feature_mapping: List[Tuple[str, str]] = field( + default_factory=list + ) + action_weights: Optional[List[int]] = None + action_embedding_init_std: float = 0.1 + enable_postprocessor: bool = True + use_layer_norm_postprocessor: bool = False + + +def _get_supervision_labels_and_weights( + supervision_bitmasks: torch.Tensor, + watchtime_sequence: torch.Tensor, + task_configs: List[TaskConfig], +) -> Tuple[Dict[str, torch.Tensor], Dict[str, torch.Tensor]]: + supervision_labels: Dict[str, torch.Tensor] = {} + supervision_weights: Dict[str, torch.Tensor] = {} + for task in task_configs: + if task.task_type == MultitaskTaskType.REGRESSION: + supervision_labels[task.task_name] = watchtime_sequence.to(torch.float32) + elif task.task_type == MultitaskTaskType.BINARY_CLASSIFICATION: + supervision_labels[task.task_name] = ( + torch.bitwise_and(supervision_bitmasks, task.task_weight) > 0 + ).to(torch.float32) + else: + raise RuntimeError("Unsupported MultitaskTaskType") + return supervision_labels, supervision_weights + + +class DlrmHSTU(HammerModule): + def __init__( # noqa C901 + self, + hstu_configs: DlrmHSTUConfig, + embedding_tables: Dict[str, EmbeddingConfig], + is_inference: bool, + is_dense: bool = False, + bf16_training: bool = True, + ) -> None: + super().__init__(is_inference=is_inference) + logger.info(f"Initialize HSTU module with configs {hstu_configs}") + self._hstu_configs = hstu_configs + self._bf16_training: bool = bf16_training + set_static_max_seq_lens([self._hstu_configs.max_seq_len]) + + if not is_dense: + self._embedding_collection: EmbeddingCollection = EmbeddingCollection( + tables=list(embedding_tables.values()), + need_indices=False, + device=torch.device("meta"), + ) + + # multitask configs must be sorted by task types + self._multitask_configs: List[TaskConfig] = hstu_configs.multitask_configs + self._multitask_module = DefaultMultitaskModule( + task_configs=self._multitask_configs, + embedding_dim=hstu_configs.hstu_transducer_embedding_dim, + prediction_fn=lambda in_dim, num_tasks: torch.nn.Sequential( + torch.nn.Linear(in_features=in_dim, out_features=512), + SwishLayerNorm(512), + torch.nn.Linear(in_features=512, out_features=num_tasks), + ).apply(init_mlp_weights_optional_bias), + causal_multitask_weights=hstu_configs.causal_multitask_weights, + is_inference=self._is_inference, + ) + self._additional_embedding_features: List[str] = [ + uih_feature_name + for ( + uih_feature_name, + candidate_feature_name, + ) in self._hstu_configs.merge_uih_candidate_feature_mapping + if ( + candidate_feature_name + in self._hstu_configs.item_embedding_feature_names + ) + and (uih_feature_name in self._hstu_configs.user_embedding_feature_names) + and (uih_feature_name is not self._hstu_configs.uih_post_id_feature_name) + ] + + # preprocessor setup + preprocessor = ContextualPreprocessor( + input_embedding_dim=hstu_configs.hstu_embedding_table_dim, + hidden_dim=hstu_configs.hstu_preprocessor_hidden_dim, + output_embedding_dim=hstu_configs.hstu_transducer_embedding_dim, + contextual_feature_to_max_length=hstu_configs.contextual_feature_to_max_length, + contextual_feature_to_min_uih_length=hstu_configs.contextual_feature_to_min_uih_length, + action_embedding_dim=8, + action_feature_name=self._hstu_configs.uih_weight_feature_name, + action_weights=self._hstu_configs.action_weights, + action_embedding_init_std=self._hstu_configs.action_embedding_init_std, + additional_embedding_features=self._additional_embedding_features, + is_inference=is_inference, + ) + + # positional encoder + positional_encoder = HSTUPositionalEncoder( + num_position_buckets=8192, + num_time_buckets=2048, + embedding_dim=hstu_configs.hstu_transducer_embedding_dim, + contextual_seq_len=sum( + dict(hstu_configs.contextual_feature_to_max_length).values() + ), + is_inference=self._is_inference, + ) + + if hstu_configs.enable_postprocessor: + if hstu_configs.use_layer_norm_postprocessor: + postprocessor = LayerNormPostprocessor( + embedding_dim=hstu_configs.hstu_transducer_embedding_dim, + eps=1e-5, + is_inference=self._is_inference, + ) + else: + postprocessor = TimestampLayerNormPostprocessor( + embedding_dim=hstu_configs.hstu_transducer_embedding_dim, + time_duration_features=[ + (60 * 60, 24), # hour of day + (24 * 60 * 60, 7), # day of week + # (24 * 60 * 60, 365), # time of year (approximate) + ], + eps=1e-5, + is_inference=self._is_inference, + ) + else: + postprocessor = None + + # construct HSTU + stu_module: STU = STUStack( + stu_list=[ + STULayer( + config=STULayerConfig( + embedding_dim=hstu_configs.hstu_transducer_embedding_dim, + num_heads=hstu_configs.hstu_num_heads, + hidden_dim=hstu_configs.hstu_attn_linear_dim, + attention_dim=hstu_configs.hstu_attn_qk_dim, + output_dropout_ratio=hstu_configs.hstu_linear_dropout_rate, + use_group_norm=hstu_configs.hstu_group_norm, + causal=True, + target_aware=True, + max_attn_len=None, + attn_alpha=None, + recompute_normed_x=True, + recompute_uvqk=True, + recompute_y=True, + sort_by_length=True, + contextual_seq_len=0, + ), + is_inference=is_inference, + ) + for _ in range(hstu_configs.hstu_attn_num_layers) + ], + is_inference=is_inference, + ) + self._hstu_transducer: HSTUTransducer = HSTUTransducer( + stu_module=stu_module, + input_preprocessor=preprocessor, + output_postprocessor=postprocessor, + input_dropout_ratio=hstu_configs.hstu_input_dropout_ratio, + positional_encoder=positional_encoder, + is_inference=self._is_inference, + return_full_embeddings=False, + listwise=False, + ) + + # item embeddings + self._item_embedding_mlp: torch.nn.Module = torch.nn.Sequential( + torch.nn.Linear( + in_features=hstu_configs.hstu_embedding_table_dim + * len(self._hstu_configs.item_embedding_feature_names), + out_features=512, + ), + SwishLayerNorm(512), + torch.nn.Linear( + in_features=512, + out_features=hstu_configs.hstu_transducer_embedding_dim, + ), + LayerNorm(hstu_configs.hstu_transducer_embedding_dim), + ).apply(init_mlp_weights_optional_bias) + + def _construct_payload( + self, + payload_features: Dict[str, torch.Tensor], + seq_embeddings: Dict[str, SequenceEmbedding], + ) -> Dict[str, torch.Tensor]: + if len(self._hstu_configs.contextual_feature_to_max_length) > 0: + contextual_offsets: List[torch.Tensor] = [] + for x in self._hstu_configs.contextual_feature_to_max_length.keys(): + contextual_offsets.append( + torch.ops.fbgemm.asynchronous_complete_cumsum( + seq_embeddings[x].lengths + ) + ) + else: + # Dummy, offsets are unused + contextual_offsets = torch.empty((0, 0)) + if torch.jit.is_scripting(): + # Explicit loops are TS-clean (avoid the dict-merge / dict-comp + # idioms below, which TorchScript cannot script). + out: Dict[str, torch.Tensor] = {} + for k, v in payload_features.items(): + out[k] = v + for x in self._hstu_configs.contextual_feature_to_max_length.keys(): + out[x] = seq_embeddings[x].embedding + i = 0 + for x in self._hstu_configs.contextual_feature_to_max_length.keys(): + # pyre-ignore[6] + out[x + "_offsets"] = contextual_offsets[i] + i += 1 + for x in self._additional_embedding_features: + out[x] = seq_embeddings[x].embedding + return out + return { + **payload_features, + **{ + x: seq_embeddings[x].embedding + for x in self._hstu_configs.contextual_feature_to_max_length.keys() + }, + **{ + x + "_offsets": contextual_offsets[i] + for i, x in enumerate( + list(self._hstu_configs.contextual_feature_to_max_length.keys()) + ) + }, + **{ + x: seq_embeddings[x].embedding + for x in self._additional_embedding_features + }, + } + + def _user_forward( + self, + max_uih_len: int, + max_candidates: int, + seq_embeddings: Dict[str, SequenceEmbedding], + payload_features: Dict[str, torch.Tensor], + num_candidates: torch.Tensor, + total_uih_len: Optional[int] = None, + total_targets: Optional[int] = None, + ) -> torch.Tensor: + source_lengths = seq_embeddings[ + self._hstu_configs.uih_post_id_feature_name + ].lengths + source_timestamps = concat_2D_jagged( + max_seq_len=max_uih_len + max_candidates, + max_len_left=max_uih_len, + offsets_left=payload_features["uih_offsets"], + values_left=payload_features[ + self._hstu_configs.uih_action_time_feature_name + ].unsqueeze(-1), + max_len_right=max_candidates, + offsets_right=payload_features["candidate_offsets"], + values_right=payload_features[ + self._hstu_configs.candidates_querytime_feature_name + ].unsqueeze(-1), + kernel=self.hammer_kernel(), + ).squeeze(-1) + if total_targets is None: + total_targets = int(num_candidates.sum().item()) + if total_uih_len is None: + total_uih_len = source_timestamps.numel() - total_targets + embedding = seq_embeddings[ + self._hstu_configs.uih_post_id_feature_name + ].embedding + dtype = embedding.dtype + if (not self.is_inference) and self._bf16_training: + embedding = embedding.to(torch.bfloat16) + if torch.jit.is_scripting(): + # TorchScript does not support ``with torch.autocast(...)``. + # In script-mode inference the dense path is already in bf16 + # (move_sparse_output_to_device upcasts on the C++ side), so + # autocast is a no-op for the path the predictor exercises. + candidates_user_embeddings, _ = self._hstu_transducer( + max_uih_len=max_uih_len, + max_targets=max_candidates, + total_uih_len=total_uih_len, + total_targets=total_targets, + seq_embeddings=embedding, + seq_lengths=source_lengths, + seq_timestamps=source_timestamps, + seq_payloads=self._construct_payload( + payload_features=payload_features, + seq_embeddings=seq_embeddings, + ), + num_targets=num_candidates, + ) + else: + with torch.autocast( + "cuda", + dtype=torch.bfloat16, + enabled=(not self.is_inference) and self._bf16_training, + ): + candidates_user_embeddings, _ = self._hstu_transducer( + max_uih_len=max_uih_len, + max_targets=max_candidates, + total_uih_len=total_uih_len, + total_targets=total_targets, + seq_embeddings=embedding, + seq_lengths=source_lengths, + seq_timestamps=source_timestamps, + seq_payloads=self._construct_payload( + payload_features=payload_features, + seq_embeddings=seq_embeddings, + ), + num_targets=num_candidates, + ) + candidates_user_embeddings = candidates_user_embeddings.to(dtype) + + return candidates_user_embeddings + + def _item_forward( + self, + seq_embeddings: Dict[str, SequenceEmbedding], + ) -> torch.Tensor: # [L, D] + all_embeddings = torch.cat( + [ + seq_embeddings[name].embedding + for name in self._hstu_configs.item_embedding_feature_names + ], + dim=-1, + ) + item_embeddings = self._item_embedding_mlp(all_embeddings) + return item_embeddings + + def preprocess( + self, + uih_features: KeyedJaggedTensor, + candidates_features: KeyedJaggedTensor, + ) -> Tuple[ + Dict[str, SequenceEmbedding], + Dict[str, torch.Tensor], + int, + torch.Tensor, + int, + torch.Tensor, + ]: + # embedding lookup for uih and candidates + merged_sparse_features = KeyedJaggedTensor.from_lengths_sync( + keys=uih_features.keys() + candidates_features.keys(), + values=torch.cat( + [uih_features.values(), candidates_features.values()], + dim=0, + ), + lengths=torch.cat( + [uih_features.lengths(), candidates_features.lengths()], + dim=0, + ), + ) + seq_embeddings_dict = self._embedding_collection(merged_sparse_features) + num_candidates = fx_mark_length_features( + candidates_features.lengths().view(len(candidates_features.keys()), -1) + )[0] + max_num_candidates = fx_infer_max_len(num_candidates) + uih_seq_lengths = uih_features[ + self._hstu_configs.uih_post_id_feature_name + ].lengths() + max_uih_len = fx_infer_max_len(uih_seq_lengths) + + # prepare payload features + payload_features: Dict[str, torch.Tensor] = {} + for ( + uih_feature_name, + candidate_feature_name, + ) in self._hstu_configs.merge_uih_candidate_feature_mapping: + if ( + candidate_feature_name + not in self._hstu_configs.item_embedding_feature_names + and uih_feature_name + not in self._hstu_configs.user_embedding_feature_names + ): + values_left = uih_features[uih_feature_name].values() + if self._is_inference and ( + candidate_feature_name + == self._hstu_configs.candidates_weight_feature_name + or candidate_feature_name + == self._hstu_configs.candidates_watchtime_feature_name + ): + total_candidates = torch.sum(num_candidates).item() + values_right = torch.zeros( + total_candidates, # pyre-ignore + dtype=torch.int64, + device=values_left.device, + ) + else: + values_right = candidates_features[candidate_feature_name].values() + payload_features[uih_feature_name] = values_left + payload_features[candidate_feature_name] = values_right + payload_features["uih_offsets"] = torch.ops.fbgemm.asynchronous_complete_cumsum( + uih_seq_lengths + ) + payload_features["candidate_offsets"] = ( + torch.ops.fbgemm.asynchronous_complete_cumsum(num_candidates) + ) + + seq_embeddings = { + k: SequenceEmbedding( + lengths=seq_embeddings_dict[k].lengths(), + embedding=seq_embeddings_dict[k].values(), + ) + for k in self._hstu_configs.user_embedding_feature_names + + self._hstu_configs.item_embedding_feature_names + } + + return ( + seq_embeddings, + payload_features, + max_uih_len, + uih_seq_lengths, + max_num_candidates, + num_candidates, + ) + + def main_forward( + self, + seq_embeddings: Dict[str, SequenceEmbedding], + payload_features: Dict[str, torch.Tensor], + max_uih_len: int, + uih_seq_lengths: torch.Tensor, + max_num_candidates: int, + num_candidates: torch.Tensor, + total_uih_len: Optional[int] = None, + total_targets: Optional[int] = None, + ) -> Tuple[ + torch.Tensor, + torch.Tensor, + Dict[str, torch.Tensor], + Optional[torch.Tensor], + Optional[torch.Tensor], + Optional[torch.Tensor], + ]: + # merge uih and candidates embeddings + for ( + uih_feature_name, + candidate_feature_name, + ) in self._hstu_configs.merge_uih_candidate_feature_mapping: + if uih_feature_name in seq_embeddings: + seq_embeddings[uih_feature_name] = SequenceEmbedding( + lengths=uih_seq_lengths + num_candidates, + embedding=concat_2D_jagged( + max_seq_len=max_uih_len + max_num_candidates, + max_len_left=max_uih_len, + offsets_left=torch.ops.fbgemm.asynchronous_complete_cumsum( + uih_seq_lengths + ), + values_left=seq_embeddings[uih_feature_name].embedding, + max_len_right=max_num_candidates, + offsets_right=torch.ops.fbgemm.asynchronous_complete_cumsum( + num_candidates + ), + values_right=seq_embeddings[candidate_feature_name].embedding, + kernel=self.hammer_kernel(), + ), + ) + + with record_function("## item_forward ##"): + candidates_item_embeddings = self._item_forward( + seq_embeddings, + ) + with record_function("## user_forward ##"): + candidates_user_embeddings = self._user_forward( + max_uih_len=max_uih_len, + max_candidates=max_num_candidates, + seq_embeddings=seq_embeddings, + payload_features=payload_features, + num_candidates=num_candidates, + total_uih_len=total_uih_len, + total_targets=total_targets, + ) + with record_function("## multitask_module ##"): + supervision_labels, supervision_weights = ( + _get_supervision_labels_and_weights( + supervision_bitmasks=payload_features[ + self._hstu_configs.candidates_weight_feature_name + ], + watchtime_sequence=payload_features[ + self._hstu_configs.candidates_watchtime_feature_name + ], + task_configs=self._multitask_configs, + ) + ) + mt_target_preds, mt_target_labels, mt_target_weights, mt_losses = ( + self._multitask_module( + encoded_user_embeddings=candidates_user_embeddings, + item_embeddings=candidates_item_embeddings, + supervision_labels=supervision_labels, + supervision_weights=supervision_weights, + ) + ) + + aux_losses: Dict[str, torch.Tensor] = {} + if not self._is_inference and self.training: + for i, task in enumerate(self._multitask_configs): + aux_losses[task.task_name] = mt_losses[i] + + return ( + candidates_user_embeddings, + candidates_item_embeddings, + aux_losses, + mt_target_preds, + mt_target_labels, + mt_target_weights, + ) + + def forward( + self, + uih_features: KeyedJaggedTensor, + candidates_features: KeyedJaggedTensor, + ) -> Tuple[ + torch.Tensor, + torch.Tensor, + Dict[str, torch.Tensor], + Optional[torch.Tensor], + Optional[torch.Tensor], + Optional[torch.Tensor], + ]: + with record_function("## preprocess ##"): + ( + seq_embeddings, + payload_features, + max_uih_len, + uih_seq_lengths, + max_num_candidates, + num_candidates, + ) = self.preprocess( + uih_features=uih_features, + candidates_features=candidates_features, + ) + + with record_function("## main_forward ##"): + return self.main_forward( + seq_embeddings=seq_embeddings, + payload_features=payload_features, + max_uih_len=max_uih_len, + uih_seq_lengths=uih_seq_lengths, + max_num_candidates=max_num_candidates, + num_candidates=num_candidates, + ) diff --git a/recommendation_v4/generative_recommenders/modules/dynamic_stu.py b/recommendation_v4/generative_recommenders/modules/dynamic_stu.py new file mode 100644 index 000000000..e1fe8ad16 --- /dev/null +++ b/recommendation_v4/generative_recommenders/modules/dynamic_stu.py @@ -0,0 +1,304 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict +import abc +import contextlib +from typing import Any, Generator, Optional, Tuple + +import torch +from generative_recommenders.common import fx_infer_max_len +from generative_recommenders.modules.stu import STU +from generative_recommenders.ops.jagged_tensors import ( + hstu_concat_l2_embeddings, + hstu_split_l2_embeddings, +) + + +try: + torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops") + torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops_cpu") +except OSError: + pass + + +@contextlib.contextmanager +# pyre-ignore[3] +def _freeze_rng_state() -> Generator[Any, None, None]: + rng_state = torch.get_rng_state() + if torch.cuda.is_available(): + cuda_rng_state = torch.cuda.get_rng_state() + try: + yield + finally: + if torch.cuda.is_available(): + # pyre-ignore[61] + torch.cuda.set_rng_state(cuda_rng_state) + torch.set_rng_state(rng_state) + + +class DynamicSTU(STU): + def __init__(self, stu: STU, is_inference: bool) -> None: + super().__init__(is_inference) + self._stu = stu + + @abc.abstractmethod + def _preprocess( + self, + x: torch.Tensor, + x_lengths: torch.Tensor, + x_offsets: torch.Tensor, + max_seq_len: int, + num_targets: torch.Tensor, + max_kv_caching_len: int = 0, + kv_caching_lengths: Optional[torch.Tensor] = None, + ) -> Tuple[ + torch.Tensor, + torch.Tensor, + torch.Tensor, + int, + torch.Tensor, + int, + Optional[torch.Tensor], + ]: + pass + + @abc.abstractmethod + def _postprocess( + self, + stu_output: torch.Tensor, + ) -> torch.Tensor: + pass + + def forward( + self, + x: torch.Tensor, + x_lengths: torch.Tensor, + x_offsets: torch.Tensor, + max_seq_len: int, + num_targets: torch.Tensor, + max_kv_caching_len: int = 0, + kv_caching_lengths: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + ( + x, + x_lengths, + x_offsets, + max_seq_len, + num_targets, + max_kv_caching_len, + kv_caching_lengths, + ) = self._preprocess( + x=x, + x_lengths=x_lengths, + x_offsets=x_offsets, + max_seq_len=max_seq_len, + num_targets=num_targets, + max_kv_caching_len=max_kv_caching_len, + kv_caching_lengths=kv_caching_lengths, + ) + + stu_output = self._stu( + x=x, + x_lengths=x_lengths, + x_offsets=x_offsets, + max_seq_len=max_seq_len, + num_targets=num_targets, + max_kv_caching_len=max_kv_caching_len, + kv_caching_lengths=kv_caching_lengths, + ) + + return self._postprocess( + stu_output=stu_output, + ) + + +class SDSTU(DynamicSTU): + def __init__( + self, + stu: STU, + is_inference: bool, + dropout_ratio: float = 0.5, + seed: int = 0, + ) -> None: + """ + Stochastic Depth STU + """ + super().__init__(stu=stu, is_inference=is_inference) + self._dropout_ratio: float = dropout_ratio + self._iter: int = 0 + self._seed: int = seed + self._skip_x: Optional[torch.Tensor] = None + + def _preprocess( + self, + x: torch.Tensor, + x_lengths: torch.Tensor, + x_offsets: torch.Tensor, + max_seq_len: int, + num_targets: torch.Tensor, + max_kv_caching_len: int = 0, + kv_caching_lengths: Optional[torch.Tensor] = None, + ) -> Tuple[ + torch.Tensor, + torch.Tensor, + torch.Tensor, + int, + torch.Tensor, + int, + Optional[torch.Tensor], + ]: + if self.training: + with _freeze_rng_state(): + torch.manual_seed(self._iter + self._seed) + prob = torch.rand(1) + if prob.item() <= self._dropout_ratio: + new_x = torch.empty(size=(0, x.shape[1]), device=x.device) + self._skip_x = x + new_x_lengths = torch.zeros_like(x_lengths) + new_x_offsets = torch.zeros_like(x_offsets) + new_max_seq_len = 1 + else: + new_x = x + new_x_lengths = x_lengths + new_x_offsets = x_offsets + new_max_seq_len = max_seq_len + self._iter += 1 + else: + new_x = x + new_x_lengths = x_lengths + new_x_offsets = x_offsets + new_max_seq_len = max_seq_len + return ( + new_x, + new_x_lengths, + new_x_offsets, + new_max_seq_len, + num_targets, + max_kv_caching_len, + kv_caching_lengths, + ) + + def _postprocess( + self, + stu_output: torch.Tensor, + ) -> torch.Tensor: + if self.training and self._skip_x is not None: + ret = self._skip_x + self._skip_x = None + return ret + else: + return stu_output + + +@torch.fx.wrap +def _fx_unwrap_optional_tuple_tensor( + optional: Optional[Tuple[torch.Tensor, torch.Tensor, torch.Tensor]], +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + assert optional is not None, "Expected optional to be non-None" + return optional + + +class L2STU(DynamicSTU): + def __init__( + self, + stu: STU, + max_l2_len: int, + is_inference: bool, + contextual_seq_len: int = 0, + ) -> None: + """ + Stochastic Depth STU + """ + super().__init__(stu=stu, is_inference=is_inference) + self._max_l2_len: int = max_l2_len + self._contextual_seq_len: int = contextual_seq_len + self._saved_tensors: Optional[ + Tuple[torch.Tensor, torch.Tensor, torch.Tensor] + ] = None + self._runtime_max_l2_len: int = 0 + self._runtime_prefix_len: int = 0 + + def _preprocess( + self, + x: torch.Tensor, + x_lengths: torch.Tensor, + x_offsets: torch.Tensor, + max_seq_len: int, + num_targets: torch.Tensor, + max_kv_caching_len: int = 0, + kv_caching_lengths: Optional[torch.Tensor] = None, + ) -> Tuple[ + torch.Tensor, + torch.Tensor, + torch.Tensor, + int, + torch.Tensor, + int, + Optional[torch.Tensor], + ]: + prefix_lengths = ( + x_lengths - self._max_l2_len - num_targets - self._contextual_seq_len + ) + prefix_lengths = torch.clamp(prefix_lengths, min=0) + prefix_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum(prefix_lengths) + l2_lengths = x_lengths - prefix_lengths + l2_offsets = x_offsets - prefix_offsets + self._runtime_max_l2_len: int = fx_infer_max_len(l2_lengths) + self._runtime_prefix_len: int = fx_infer_max_len(prefix_lengths) + prefix_x, l2_x = hstu_split_l2_embeddings( + max_seq_len=max_seq_len, + x=x, + prefix_offsets=prefix_offsets, + l2_offsets=l2_offsets, + contextual_seq_len=self._contextual_seq_len, + kernel=self.hammer_kernel(), + ) + self._saved_tensors = ( + prefix_offsets, + prefix_x, + l2_offsets, + ) + return ( + l2_x, + l2_lengths, + l2_offsets, + self._runtime_max_l2_len, + num_targets, + max_kv_caching_len, + kv_caching_lengths, + ) + + def _postprocess( + self, + stu_output: torch.Tensor, + ) -> torch.Tensor: + ( + prefix_offsets, + prefix_x, + l2_offsets, + ) = _fx_unwrap_optional_tuple_tensor(self._saved_tensors) + self._saved_tensors = None + return hstu_concat_l2_embeddings( + max_prefix_len=self._runtime_prefix_len, + prefix_x=prefix_x, + prefix_offsets=prefix_offsets, + max_l2_len=self._runtime_max_l2_len, + l2_x=stu_output, + l2_offsets=l2_offsets, + contextual_seq_len=self._contextual_seq_len, + kernel=self.hammer_kernel(), + ) diff --git a/recommendation_v4/generative_recommenders/modules/hstu_transducer.py b/recommendation_v4/generative_recommenders/modules/hstu_transducer.py new file mode 100644 index 000000000..ce91a67c9 --- /dev/null +++ b/recommendation_v4/generative_recommenders/modules/hstu_transducer.py @@ -0,0 +1,330 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +import logging +from typing import Dict, Optional, Tuple + +import torch +from generative_recommenders.common import fx_unwrap_optional_tensor, HammerModule +from generative_recommenders.modules.positional_encoder import HSTUPositionalEncoder +from generative_recommenders.modules.postprocessors import ( + L2NormPostprocessor, + OutputPostprocessor, +) +from generative_recommenders.modules.preprocessors import InputPreprocessor +from generative_recommenders.modules.stu import STU +from generative_recommenders.ops.jagged_tensors import split_2D_jagged +from torch.profiler import record_function + +logger: logging.Logger = logging.getLogger(__name__) +torch.fx.wrap("len") + +try: + torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops") + torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops_cpu") +except OSError: + pass + + +@torch.fx.wrap +def default_seq_payload( + seq_payloads: Optional[Dict[str, torch.Tensor]], +) -> Dict[str, torch.Tensor]: + if seq_payloads is None: + return {} + else: + return torch.jit._unwrap_optional(seq_payloads) + + +class HSTUTransducer(HammerModule): + def __init__( + self, + stu_module: STU, + input_preprocessor: InputPreprocessor, + output_postprocessor: Optional[OutputPostprocessor] = None, + input_dropout_ratio: float = 0.0, + positional_encoder: Optional[HSTUPositionalEncoder] = None, + is_inference: bool = True, + return_full_embeddings: bool = False, + listwise: bool = False, + ) -> None: + super().__init__(is_inference=is_inference) + self._stu_module = stu_module + self._input_preprocessor: InputPreprocessor = input_preprocessor + self._output_postprocessor: OutputPostprocessor = ( + output_postprocessor + if output_postprocessor is not None + else L2NormPostprocessor(is_inference=is_inference) + ) + assert self._is_inference == self._input_preprocessor._is_inference, ( + f"input_preprocessor must have the same mode; self: {self._is_inference} vs input_preprocessor {self._input_preprocessor._is_inference}" + ) + self._positional_encoder: Optional[HSTUPositionalEncoder] = positional_encoder + self._input_dropout_ratio: float = input_dropout_ratio + self._return_full_embeddings: bool = return_full_embeddings + self._listwise_training: bool = listwise and self.is_train + + for name, m in self.named_modules(): + if "_stu_module" in name: + continue + elif isinstance(m, torch.nn.Linear): + torch.nn.init.xavier_normal_(m.weight) + elif isinstance(m, torch.nn.LayerNorm): + if m.weight.dim() >= 2: + torch.nn.init.xavier_normal_(m.weight) + if m.bias is not None and m.bias.dim() >= 2: + torch.nn.init.xavier_normal_(m.bias) + + def _preprocess( + self, + max_uih_len: int, + max_targets: int, + total_uih_len: int, + total_targets: int, + seq_lengths: torch.Tensor, + seq_timestamps: torch.Tensor, + seq_embeddings: torch.Tensor, + num_targets: torch.Tensor, + seq_payloads: Dict[str, torch.Tensor], + ) -> Tuple[ + int, + int, + int, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + Dict[str, torch.Tensor], + ]: + seq_payloads = default_seq_payload(seq_payloads) + + with record_function("hstu_input_preprocessor"): + ( + output_max_seq_len, + output_total_uih_len, + output_total_targets, + output_seq_lengths, + output_seq_offsets, + output_seq_timestamps, + output_seq_embeddings, + output_num_targets, + output_seq_payloads, + ) = self._input_preprocessor( + max_uih_len=max_uih_len, + max_targets=max_targets, + total_uih_len=total_uih_len, + total_targets=total_targets, + seq_lengths=seq_lengths, + seq_timestamps=seq_timestamps, + seq_embeddings=seq_embeddings, + num_targets=num_targets, + seq_payloads=seq_payloads, + ) + + with record_function("hstu_positional_encoder"): + if self._positional_encoder is not None: + output_seq_embeddings = self._positional_encoder( + max_seq_len=output_max_seq_len, + seq_lengths=output_seq_lengths, + seq_offsets=output_seq_offsets, + seq_timestamps=output_seq_timestamps, + seq_embeddings=output_seq_embeddings, + num_targets=( + None if self._listwise_training else output_num_targets + ), + ) + + output_seq_embeddings = torch.nn.functional.dropout( + output_seq_embeddings, + p=self._input_dropout_ratio, + training=self.training, + ) + + return ( + output_max_seq_len, + output_total_uih_len, + output_total_targets, + output_seq_lengths, + output_seq_offsets, + output_seq_timestamps, + output_seq_embeddings, + output_num_targets, + output_seq_payloads, + ) + + def _hstu_compute( + self, + max_seq_len: int, + seq_lengths: torch.Tensor, + seq_offsets: torch.Tensor, + seq_timestamps: torch.Tensor, + seq_embeddings: torch.Tensor, + num_targets: torch.Tensor, + ) -> torch.Tensor: + with record_function("hstu"): + seq_embeddings = self._stu_module( + max_seq_len=max_seq_len, + x=seq_embeddings, + x_lengths=seq_lengths, + x_offsets=seq_offsets, + num_targets=(None if self._listwise_training else num_targets), + ) + return seq_embeddings + + def _postprocess( + self, + max_seq_len: int, + max_uih_len: int, + max_targets: int, + total_uih_len: int, + total_targets: int, + seq_lengths: torch.Tensor, + seq_timestamps: torch.Tensor, + seq_embeddings: torch.Tensor, + num_targets: torch.Tensor, + seq_payloads: Dict[str, torch.Tensor], + ) -> Tuple[Optional[torch.Tensor], torch.Tensor]: + with record_function("hstu_output_postprocessor"): + if self._return_full_embeddings: + seq_embeddings = self._output_postprocessor( + seq_embeddings=seq_embeddings, + seq_timestamps=seq_timestamps, + seq_payloads=seq_payloads, + ) + uih_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum( + seq_lengths - num_targets + ) + candidates_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum( + num_targets + ) + _, candidate_embeddings = split_2D_jagged( + values=seq_embeddings, + max_seq_len=max_seq_len, + total_len_left=total_uih_len, + total_len_right=total_targets, + max_len_left=max_uih_len, + max_len_right=max_targets, + offsets_left=uih_offsets, + offsets_right=candidates_offsets, + kernel=self.hammer_kernel(), + ) + interleave_targets: bool = self._input_preprocessor.interleave_targets() + if interleave_targets: + candidate_embeddings = candidate_embeddings.view( + -1, 2, candidate_embeddings.size(-1) + )[:, 0, :] + if not self._return_full_embeddings: + _, candidate_timestamps = split_2D_jagged( + values=seq_timestamps.unsqueeze(-1), + max_seq_len=max_seq_len, + total_len_left=total_uih_len, + total_len_right=total_targets, + max_len_left=max_uih_len, + max_len_right=max_targets, + offsets_left=uih_offsets, + offsets_right=candidates_offsets, + kernel=self.hammer_kernel(), + ) + candidate_timestamps = candidate_timestamps.squeeze(-1) + if interleave_targets: + candidate_timestamps = candidate_timestamps.view(-1, 2)[:, 0] + candidate_embeddings = self._output_postprocessor( + seq_embeddings=candidate_embeddings, + seq_timestamps=candidate_timestamps, + seq_payloads=seq_payloads, + ) + + return ( + seq_embeddings if self._return_full_embeddings else None, + candidate_embeddings, + ) + + def forward( + self, + max_uih_len: int, + max_targets: int, + total_uih_len: int, + total_targets: int, + seq_lengths: torch.Tensor, + seq_embeddings: torch.Tensor, + seq_timestamps: torch.Tensor, + num_targets: torch.Tensor, + seq_payloads: Dict[str, torch.Tensor], + ) -> Tuple[ + torch.Tensor, + Optional[torch.Tensor], + ]: + orig_dtype = seq_embeddings.dtype + if not self._is_inference: + seq_embeddings = seq_embeddings.to(self._training_dtype) + + ( + max_seq_len, + total_uih_len, + total_targets, + seq_lengths, + seq_offsets, + seq_timestamps, + seq_embeddings, + num_targets, + seq_payloads, + ) = self._preprocess( + max_uih_len=max_uih_len, + max_targets=max_targets, + total_uih_len=total_uih_len, + total_targets=total_targets, + seq_lengths=seq_lengths, + seq_timestamps=seq_timestamps, + seq_embeddings=seq_embeddings, + num_targets=num_targets, + seq_payloads=seq_payloads, + ) + + encoded_embeddings = self._hstu_compute( + max_seq_len=max_seq_len, + seq_lengths=seq_lengths, + seq_offsets=seq_offsets, + seq_timestamps=seq_timestamps, + seq_embeddings=seq_embeddings, + num_targets=num_targets, + ) + + encoded_embeddings, encoded_candidate_embeddings = self._postprocess( + max_seq_len=max_seq_len, + max_uih_len=max_uih_len, + max_targets=max_targets, + total_uih_len=total_uih_len, + total_targets=total_targets, + seq_lengths=seq_lengths, + seq_embeddings=encoded_embeddings, + seq_timestamps=seq_timestamps, + num_targets=num_targets, + seq_payloads=seq_payloads, + ) + + if not self._is_inference: + encoded_candidate_embeddings = encoded_candidate_embeddings.to(orig_dtype) + if self._return_full_embeddings: + encoded_embeddings = fx_unwrap_optional_tensor(encoded_embeddings).to( + orig_dtype + ) + return ( + encoded_candidate_embeddings, + encoded_embeddings, + ) diff --git a/recommendation_v4/generative_recommenders/modules/multitask_module.py b/recommendation_v4/generative_recommenders/modules/multitask_module.py new file mode 100644 index 000000000..3cb11996f --- /dev/null +++ b/recommendation_v4/generative_recommenders/modules/multitask_module.py @@ -0,0 +1,288 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +import abc +import logging +from dataclasses import dataclass +from enum import IntEnum +from typing import Callable, Dict, List, Optional, Tuple + +import numpy as np +import torch +import torch.nn.functional as F +from generative_recommenders.common import HammerModule + +logger: logging.Logger = logging.getLogger(__name__) + + +class MultitaskTaskType(IntEnum): + BINARY_CLASSIFICATION = 0 + REGRESSION = 1 + + +@dataclass +class TaskConfig: + task_name: str + task_weight: int + task_type: MultitaskTaskType + + +class MultitaskModule(HammerModule): + @abc.abstractmethod + def forward( + self, + encoded_user_embeddings: torch.Tensor, + item_embeddings: torch.Tensor, + supervision_labels: Dict[str, torch.Tensor], + supervision_weights: Dict[str, torch.Tensor], + ) -> Tuple[ + torch.Tensor, + Optional[torch.Tensor], + Optional[torch.Tensor], + Optional[torch.Tensor], + ]: + """ + Computes multi-task predictions. + + Args: + encoded_user_embeddings: (L, D) x float. + item_embeddings: (L, D) x float. + supervision_labels: Dict[T, L] x float or int + supervision_weights: Dict[T', L] x float or int, T' <= T + Returns: + (T, L) x float, predictions, labels, weights, losses + """ + pass + + +def _compute_pred_and_logits( + prediction_module: torch.nn.Module, + encoded_user_embeddings: torch.Tensor, + item_embeddings: torch.Tensor, + task_offsets: List[int], + has_multiple_task_types: bool, +) -> Tuple[torch.Tensor, torch.Tensor]: + mt_logits = prediction_module(encoded_user_embeddings * item_embeddings).transpose( + 0, 1 + ) + mt_preds_list: List[torch.Tensor] = [] + for task_type in MultitaskTaskType: + logits = mt_logits[ + task_offsets[task_type] : task_offsets[task_type + 1], + :, + ] + if task_offsets[task_type + 1] - task_offsets[task_type] > 0: + if task_type == MultitaskTaskType.REGRESSION: + mt_preds_list.append(logits) + else: + mt_preds_list.append(F.sigmoid(logits)) + if has_multiple_task_types: + mt_preds: torch.Tensor = torch.concat(mt_preds_list, dim=0) + else: + mt_preds: torch.Tensor = mt_preds_list[0] + + return mt_preds, mt_logits + + +def _compute_labels_and_weights( + supervision_labels: Dict[str, torch.Tensor], + supervision_weights: Dict[str, torch.Tensor], + task_configs: List[TaskConfig], + device: torch.device, + dtype: torch.dtype = torch.float32, +) -> Tuple[torch.Tensor, torch.Tensor]: + first_label: torch.Tensor = list(supervision_labels.values())[0] + default_supervision_weight = torch.ones_like( + first_label, + dtype=dtype, + device=device, + ) + mt_lables_list: List[torch.Tensor] = [] + mt_weights_list: List[torch.Tensor] = [] + for task in task_configs: + mt_lables_list.append(supervision_labels[task.task_name]) + mt_weights_list.append( + supervision_weights.get(task.task_name, default_supervision_weight) + ) + if len(task_configs) > 1: + mt_labels = torch.stack(mt_lables_list, dim=0) + mt_weights = torch.stack(mt_weights_list, dim=0) + else: + mt_labels = mt_lables_list[0].unsqueeze(0) + mt_weights = mt_weights_list[0].unsqueeze(0) + return mt_labels, mt_weights + + +def _compute_loss( + task_offsets: List[int], + causal_multitask_weights: float, + mt_logits: torch.Tensor, + mt_labels: torch.Tensor, + mt_weights: torch.Tensor, + has_multiple_task_types: bool, +) -> torch.Tensor: + mt_losses_list: List[torch.Tensor] = [] + for task_type in MultitaskTaskType: + if task_offsets[task_type + 1] - task_offsets[task_type] > 0: + logits = mt_logits[ + task_offsets[task_type] : task_offsets[task_type + 1], + :, + ] + labels = mt_labels[ + task_offsets[task_type] : task_offsets[task_type + 1], + :, + ] + weights = mt_weights[ + task_offsets[task_type] : task_offsets[task_type + 1], + :, + ] + if task_type == MultitaskTaskType.REGRESSION: + mt_losses_list.append( + F.mse_loss(logits, labels, reduction="none") * weights + ) + else: + mt_losses_list.append( + F.binary_cross_entropy_with_logits( + input=logits, target=labels, reduction="none" + ) + * weights + ) + + if has_multiple_task_types: + mt_losses = torch.concat(mt_losses_list, dim=0) + else: + mt_losses = mt_losses_list[0] + mt_losses = ( + mt_losses.sum(-1) / mt_weights.sum(-1).clamp(min=1.0) * causal_multitask_weights + ) + return mt_losses + + +class DefaultMultitaskModule(MultitaskModule): + def __init__( + self, + task_configs: List[TaskConfig], + embedding_dim: int, + prediction_fn: Callable[[int, int], torch.nn.Module], + causal_multitask_weights: float, + is_inference: bool, + ) -> None: + super().__init__(is_inference) + assert sorted(task_configs, key=lambda x: x.task_type) == task_configs, ( + "task_configs must be sorted by task_type." + ) + assert len(task_configs) > 0, "task_configs must be non-empty." + self._task_configs: List[TaskConfig] = task_configs + self._task_offsets: List[int] = [0] * (len(MultitaskTaskType) + 1) + for task in self._task_configs: + self._task_offsets[task.task_type + 1] += 1 + self._has_multiple_task_types: bool = self._task_offsets.count(0) < len( + MultitaskTaskType + ) + self._task_offsets[1:] = np.cumsum(self._task_offsets[1:]).tolist() + self._causal_multitask_weights: float = causal_multitask_weights + self._prediction_module: torch.nn.Module = prediction_fn( + embedding_dim, len(task_configs) + ) + + def forward( + self, + encoded_user_embeddings: torch.Tensor, + item_embeddings: torch.Tensor, + supervision_labels: Dict[str, torch.Tensor], + supervision_weights: Dict[str, torch.Tensor], + ) -> Tuple[ + torch.Tensor, + Optional[torch.Tensor], + Optional[torch.Tensor], + Optional[torch.Tensor], + ]: + orig_dtype = encoded_user_embeddings.dtype + if not self._is_inference: + encoded_user_embeddings = encoded_user_embeddings.to(self._training_dtype) + item_embeddings = item_embeddings.to(self._training_dtype) + + if torch.jit.is_scripting(): + # Script-mode fast path: skip torch.autocast (unsupported in TS) + # and inline _compute_pred_and_logits to avoid its + # `torch.nn.Module` parameter annotation (TS only knows + # concrete module types). The dense module is already in bf16 + # at this point, so autocast is a no-op for the predictor path. + mt_logits = self._prediction_module( + encoded_user_embeddings * item_embeddings + ).transpose(0, 1) + mt_preds_list: List[torch.Tensor] = [] + # MultitaskTaskType is an IntEnum (BINARY_CLASSIFICATION=0, + # REGRESSION=1) but TorchScript treats it as an opaque Enum. + # Iterate by the integer task indices directly. + for task_type in range(len(self._task_offsets) - 1): + start = self._task_offsets[task_type] + end = self._task_offsets[task_type + 1] + logits = mt_logits[start:end, :] + if end - start > 0: + # 1 == MultitaskTaskType.REGRESSION + if task_type == 1: + mt_preds_list.append(logits) + else: + mt_preds_list.append(F.sigmoid(logits)) + if self._has_multiple_task_types: + mt_preds: torch.Tensor = torch.concat(mt_preds_list, dim=0) + else: + mt_preds: torch.Tensor = mt_preds_list[0] + return mt_preds, None, None, None + + with torch.autocast( + "cuda", + dtype=torch.bfloat16, + enabled=(not self.is_inference and self._training_dtype == torch.bfloat16), + ): + mt_preds, mt_logits = _compute_pred_and_logits( + prediction_module=self._prediction_module, + encoded_user_embeddings=encoded_user_embeddings, + item_embeddings=item_embeddings, + task_offsets=self._task_offsets, + has_multiple_task_types=self._has_multiple_task_types, + ) + + # losses are always computed in fp32 + mt_labels: Optional[torch.Tensor] = None + mt_weights: Optional[torch.Tensor] = None + mt_losses: Optional[torch.Tensor] = None + if not self._is_inference: + mt_labels, mt_weights = _compute_labels_and_weights( + supervision_labels=supervision_labels, + supervision_weights=supervision_weights, + task_configs=self._task_configs, + device=encoded_user_embeddings.device, + ) + mt_losses = _compute_loss( + task_offsets=self._task_offsets, + causal_multitask_weights=self._causal_multitask_weights, + mt_logits=mt_logits.to(mt_labels.dtype), + mt_labels=mt_labels, + mt_weights=mt_weights, + has_multiple_task_types=self._has_multiple_task_types, + ) + mt_preds = mt_preds.to(orig_dtype) + + return ( + mt_preds, + mt_labels, + mt_weights, + mt_losses, + ) diff --git a/recommendation_v4/generative_recommenders/modules/positional_encoder.py b/recommendation_v4/generative_recommenders/modules/positional_encoder.py new file mode 100644 index 000000000..99d904fd4 --- /dev/null +++ b/recommendation_v4/generative_recommenders/modules/positional_encoder.py @@ -0,0 +1,75 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +from math import sqrt +from typing import Optional + +import torch +from generative_recommenders.common import HammerModule +from generative_recommenders.ops.position import add_timestamp_positional_embeddings + + +class HSTUPositionalEncoder(HammerModule): + def __init__( + self, + num_position_buckets: int, + num_time_buckets: int, + embedding_dim: int, + contextual_seq_len: int, + is_inference: bool = True, + ) -> None: + super().__init__(is_inference=is_inference) + self._embedding_dim: int = embedding_dim + self._contextual_seq_len: int = contextual_seq_len + self._position_embeddings_weight: torch.nn.Parameter = torch.nn.Parameter( + torch.empty(num_position_buckets, embedding_dim).uniform_( + -sqrt(1.0 / num_position_buckets), + sqrt(1.0 / num_position_buckets), + ), + ) + self._timestamp_embeddings_weight: torch.nn.Parameter = torch.nn.Parameter( + torch.empty(num_time_buckets + 1, embedding_dim).uniform_( + -sqrt(1.0 / num_time_buckets), + sqrt(1.0 / num_time_buckets), + ), + ) + + def forward( + self, + max_seq_len: int, + seq_lengths: torch.Tensor, + seq_offsets: torch.Tensor, + seq_timestamps: torch.Tensor, + seq_embeddings: torch.Tensor, + num_targets: Optional[torch.Tensor], + ) -> torch.Tensor: + seq_embeddings = add_timestamp_positional_embeddings( + alpha=self._embedding_dim**0.5, + max_seq_len=max_seq_len, + max_contextual_seq_len=self._contextual_seq_len, + position_embeddings_weight=self._position_embeddings_weight, + timestamp_embeddings_weight=self._timestamp_embeddings_weight, + seq_offsets=seq_offsets, + seq_lengths=seq_lengths, + seq_embeddings=seq_embeddings, + timestamps=seq_timestamps, + num_targets=num_targets, + interleave_targets=False, + kernel=self.hammer_kernel(), + ) + return seq_embeddings diff --git a/recommendation_v4/generative_recommenders/modules/postprocessors.py b/recommendation_v4/generative_recommenders/modules/postprocessors.py new file mode 100644 index 000000000..7958e3fa9 --- /dev/null +++ b/recommendation_v4/generative_recommenders/modules/postprocessors.py @@ -0,0 +1,182 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +from abc import abstractmethod +from typing import Dict, List, Tuple + +import torch +from generative_recommenders.common import HammerModule, init_mlp_weights_optional_bias + + +@torch.fx.wrap +def _cast_dtype(t: torch.Tensor, dtype: torch.dtype) -> torch.Tensor: + if t.dtype != dtype: + return t.to(dtype) + return t + + +class OutputPostprocessor(HammerModule): + """An abstract class for post-processing user embeddings after HSTU layers.""" + + @abstractmethod + def forward( + self, + seq_embeddings: torch.Tensor, + seq_timestamps: torch.Tensor, + seq_payloads: Dict[str, torch.Tensor], + ) -> torch.Tensor: + """ + Args: + seq_embeddings: (L, D) + seq_timestamps: (L, ) + seq_payloads: str-keyed tensors. Implementation specific. + + Returns: + postprocessed seq_embeddings, (L, D) + """ + pass + + +class L2NormPostprocessor(OutputPostprocessor): + """Postprocesses user embeddings with l2 norm.""" + + def __init__(self, is_inference: bool = False) -> None: + super().__init__(is_inference=is_inference) + + def forward( + self, + seq_embeddings: torch.Tensor, + seq_timestamps: torch.Tensor, + seq_payloads: Dict[str, torch.Tensor], + ) -> torch.Tensor: + return seq_embeddings / torch.linalg.norm( + seq_embeddings, ord=2, dim=-1, keepdim=True + ).clamp(min=1e-6) + + +class LayerNormPostprocessor(OutputPostprocessor): + """Postprocesses user embeddings with layer norm.""" + + def __init__( + self, + embedding_dim: int, + eps: float = 1e-5, + is_inference: bool = False, + ) -> None: + super().__init__(is_inference=is_inference) + + self._layer_norm: torch.nn.Module = torch.nn.LayerNorm( + normalized_shape=[embedding_dim], eps=eps + ) + + def forward( + self, + seq_embeddings: torch.Tensor, + seq_timestamps: torch.Tensor, + seq_payloads: Dict[str, torch.Tensor], + ) -> torch.Tensor: + # pyre-fixme[6]: For 1st argument expected `dtype` but got `Union[dtype, + # Tensor, Module]`. + return self._layer_norm(seq_embeddings.to(self._layer_norm.weight.dtype)) + + +@torch.fx.wrap +def _unsqueeze_if_needed(t: torch.Tensor, embedding: torch.Tensor) -> torch.Tensor: + if embedding.dim() == 3: + return t.unsqueeze(0) + return t + + +class TimestampLayerNormPostprocessor(OutputPostprocessor): + """Postprocesses user embeddings with timestamp-based MLP -> layer norm.""" + + def __init__( + self, + embedding_dim: int, + time_duration_features: List[Tuple[int, int]], + eps: float = 1e-5, + is_inference: bool = False, + ) -> None: + super().__init__(is_inference=is_inference) + + self._layer_norm: torch.nn.Module = torch.nn.LayerNorm( + normalized_shape=[embedding_dim], eps=eps + ) + self.register_buffer( + "_period_units", + torch.Tensor([f[0] for f in time_duration_features]).view(1, -1), + ) + self.register_buffer( + "_units_per_period", + torch.Tensor([f[1] for f in time_duration_features]).view(1, -1), + ) + self._time_feature_combiner: torch.nn.Module = torch.nn.Linear( + embedding_dim + 2 * len(time_duration_features), + embedding_dim, + ).apply(init_mlp_weights_optional_bias) + + def _concat_time_features( + self, + combined_embeddings: torch.Tensor, + timestamps: torch.Tensor, # [B] or [B, D] + ) -> torch.Tensor: + # concat time representation to combined embeddings + period_units = self._period_units + units_per_period = self._units_per_period + + timestamps = timestamps.unsqueeze(-1) + period_units = _unsqueeze_if_needed(period_units, combined_embeddings) + units_per_period = _unsqueeze_if_needed( + units_per_period, combined_embeddings + ).float() + # Compute time features in float32 to avoid bf16 precision loss through + # discontinuous floor/remainder ops, matching Inductor fusion behavior. + _units_elapsed_type: torch.dtype = combined_embeddings.dtype + _units_since_epoch = torch.div( + timestamps.float(), period_units.float(), rounding_mode="floor" + ) # [sum(N_i), num_time_features] or [B, N, num_time_features] + _units_elapsed = ( + (torch.remainder(_units_since_epoch, units_per_period) / units_per_period) + * 2 + * 3.14 + ) + _units_elapsed = torch.view_as_real( + torch.polar( + _cast_dtype(torch.ones_like(_units_elapsed), torch.float32), + _cast_dtype(_units_elapsed, torch.float32), + ) + ).flatten( + -2, -1 + ) # [sum(N_i), num_time_features * 2] or [B, N, num_time_features * 2] + _units_elapsed = _cast_dtype(_units_elapsed, _units_elapsed_type) + combined_embeddings = torch.cat([combined_embeddings, _units_elapsed], dim=-1) + return combined_embeddings + + def forward( + self, + seq_embeddings: torch.Tensor, + seq_timestamps: torch.Tensor, + seq_payloads: Dict[str, torch.Tensor], + ) -> torch.Tensor: + user_embeddings = self._time_feature_combiner( + self._concat_time_features(seq_embeddings, timestamps=seq_timestamps).to( + self._time_feature_combiner.weight.dtype # pyre-fixme[6]: For 1st argument expected `dtype` but got `Union[dtype, + # Tensor, Module]`. + ) + ) + return self._layer_norm(user_embeddings) diff --git a/recommendation_v4/generative_recommenders/modules/preprocessors.py b/recommendation_v4/generative_recommenders/modules/preprocessors.py new file mode 100644 index 000000000..dc7806bb4 --- /dev/null +++ b/recommendation_v4/generative_recommenders/modules/preprocessors.py @@ -0,0 +1,334 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +import abc +from math import sqrt +from typing import Dict, List, Optional, Tuple + +import torch +from generative_recommenders.common import ( + fx_unwrap_optional_tensor, + HammerModule, + init_mlp_weights_optional_bias, + jagged_to_padded_dense, +) +from generative_recommenders.modules.action_encoder import ActionEncoder +from generative_recommenders.ops.jagged_tensors import concat_2D_jagged +from generative_recommenders.ops.layer_norm import LayerNorm, SwishLayerNorm + + +class InputPreprocessor(HammerModule): + """An abstract class for pre-processing sequence embeddings before HSTU layers.""" + + @abc.abstractmethod + def forward( + self, + max_uih_len: int, + max_targets: int, + total_uih_len: int, + total_targets: int, + seq_lengths: torch.Tensor, + seq_timestamps: torch.Tensor, + seq_embeddings: torch.Tensor, + num_targets: torch.Tensor, + seq_payloads: Dict[str, torch.Tensor], + ) -> Tuple[ + int, + int, + int, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + Dict[str, torch.Tensor], + ]: + """ + Args: + max_uih_len: int + max_targets: int + total_uih_len: int + total_targets: int + seq_lengths: (B,) + seq_embeddings: (L, D) + seq_timestamps: (B, N) + num_targets: (B,) Optional. + seq_payloads: str-keyed tensors. Implementation specific. + + Returns: + (max_seq_len, total_uih_len, total_targets, lengths, offsets, timestamps, embeddings, num_targets, payloads) updated based on input preprocessor. + """ + pass + + def interleave_targets(self) -> bool: + return False + + +def get_contextual_input_embeddings( + seq_lengths: torch.Tensor, + seq_payloads: Dict[str, torch.Tensor], + contextual_feature_to_max_length: Dict[str, int], + contextual_feature_to_min_uih_length: Dict[str, int], + dtype: torch.dtype, +) -> torch.Tensor: + padded_values: List[torch.Tensor] = [] + for key, max_len in contextual_feature_to_max_length.items(): + v = torch.flatten( + jagged_to_padded_dense( + values=seq_payloads[key].to(dtype), + offsets=[seq_payloads[key + "_offsets"]], + max_lengths=[max_len], + padding_value=0.0, + ), + 1, + 2, + ) + min_uih_length = contextual_feature_to_min_uih_length.get(key, 0) + if min_uih_length > 0: + v = v * (seq_lengths.view(-1, 1) >= min_uih_length) + padded_values.append(v) + return torch.cat(padded_values, dim=1) + + +class ContextualPreprocessor(InputPreprocessor): + def __init__( + self, + input_embedding_dim: int, + hidden_dim: int, + output_embedding_dim: int, + contextual_feature_to_max_length: Dict[str, int], + contextual_feature_to_min_uih_length: Dict[str, int], + action_embedding_dim: int = 8, + action_feature_name: str = "", + action_weights: Optional[List[int]] = None, + additional_embedding_features: List[str] = [], + action_embedding_init_std: float = 0.1, + is_inference: bool = True, + ) -> None: + super().__init__(is_inference=is_inference) + self._output_embedding_dim: int = output_embedding_dim + self._input_embedding_dim: int = input_embedding_dim + self._hidden_dim: int = hidden_dim + self._contextual_feature_to_max_length: Dict[str, int] = ( + contextual_feature_to_max_length + ) + self._max_contextual_seq_len: int = sum( + contextual_feature_to_max_length.values() + ) + self._contextual_feature_to_min_uih_length: Dict[str, int] = ( + contextual_feature_to_min_uih_length + ) + if self._max_contextual_seq_len > 0: + std = 1.0 * sqrt( + 2.0 / float(input_embedding_dim + self._output_embedding_dim) + ) + self._batched_contextual_linear_weights: torch.nn.Parameter = ( + torch.nn.Parameter( + torch.empty( + ( + self._max_contextual_seq_len, + input_embedding_dim, + self._output_embedding_dim, + ) + ).normal_(0.0, std) + ) + ) + self._batched_contextual_linear_bias: torch.nn.Parameter = ( + torch.nn.Parameter( + torch.empty( + (self._max_contextual_seq_len, self._output_embedding_dim) + ).fill_(0.0) + ) + ) + self._content_embedding_mlp: torch.nn.Module = torch.nn.Sequential( + torch.nn.Linear( + in_features=self._input_embedding_dim, + out_features=self._hidden_dim, + ), + SwishLayerNorm(self._hidden_dim), + torch.nn.Linear( + in_features=self._hidden_dim, + out_features=self._output_embedding_dim, + ), + LayerNorm(self._output_embedding_dim), + ).apply(init_mlp_weights_optional_bias) + self._additional_embedding_features: List[str] = additional_embedding_features + self._additional_embedding_mlp: torch.nn.Module = torch.nn.Sequential( + torch.nn.Linear( + in_features=self._input_embedding_dim + * len(additional_embedding_features), + out_features=self._hidden_dim, + ), + SwishLayerNorm(self._hidden_dim), + torch.nn.Linear( + in_features=self._hidden_dim, + out_features=self._output_embedding_dim, + ), + LayerNorm(self._output_embedding_dim), + ).apply(init_mlp_weights_optional_bias) + self._action_feature_name: str = action_feature_name + self._action_weights: Optional[List[int]] = action_weights + if self._action_weights is not None: + self._action_encoder: ActionEncoder = ActionEncoder( + action_feature_name=action_feature_name, + action_weights=self._action_weights, + action_embedding_dim=action_embedding_dim, + embedding_init_std=action_embedding_init_std, + is_inference=is_inference, + ) + self._action_embedding_mlp: torch.nn.Module = torch.nn.Sequential( + torch.nn.Linear( + in_features=self._action_encoder.output_embedding_dim, + out_features=self._hidden_dim, + ), + SwishLayerNorm(self._hidden_dim), + torch.nn.Linear( + in_features=self._hidden_dim, + out_features=self._output_embedding_dim, + ), + LayerNorm(self._output_embedding_dim), + ).apply(init_mlp_weights_optional_bias) + + def forward( # noqa C901 + self, + max_uih_len: int, + max_targets: int, + total_uih_len: int, + total_targets: int, + seq_lengths: torch.Tensor, + seq_timestamps: torch.Tensor, + seq_embeddings: torch.Tensor, + num_targets: torch.Tensor, + seq_payloads: Dict[str, torch.Tensor], + ) -> Tuple[ + int, + int, + int, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + Dict[str, torch.Tensor], + ]: + output_seq_embeddings = self._content_embedding_mlp(seq_embeddings) + if len(self._additional_embedding_features) > 0: + additional_embeddings = torch.cat( + [ + seq_payloads[feature] + for feature in self._additional_embedding_features + ], + dim=1, + ) + output_seq_embeddings = ( + output_seq_embeddings + + self._additional_embedding_mlp(additional_embeddings) + ) + max_seq_len = max_uih_len + max_targets + target_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum(num_targets) + seq_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum(seq_lengths) + uih_offsets = seq_offsets - target_offsets + if self._action_weights is not None: + action_embeddings = self._action_encoder( + max_uih_len=max_uih_len, + max_targets=max_targets, + uih_offsets=uih_offsets, + target_offsets=target_offsets, + seq_embeddings=seq_embeddings, + seq_payloads=seq_payloads, + ) + output_seq_embeddings = output_seq_embeddings + self._action_embedding_mlp( + action_embeddings + ) + + output_max_seq_len = max_seq_len + output_total_uih_len = total_uih_len + output_total_targets = total_targets + output_seq_lengths = seq_lengths + output_num_targets = num_targets + output_seq_timestamps = seq_timestamps + output_seq_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum( + output_seq_lengths + ) + # concat contextual embeddings + if self._max_contextual_seq_len > 0: + contextual_input_embeddings = get_contextual_input_embeddings( + seq_lengths=seq_lengths, + seq_payloads=seq_payloads, + contextual_feature_to_max_length=self._contextual_feature_to_max_length, + contextual_feature_to_min_uih_length=self._contextual_feature_to_min_uih_length, + dtype=seq_embeddings.dtype, + ) + contextual_embeddings = torch.baddbmm( + self._batched_contextual_linear_bias.view( + -1, 1, self._output_embedding_dim + ).to(contextual_input_embeddings.dtype), + contextual_input_embeddings.view( + -1, self._max_contextual_seq_len, self._input_embedding_dim + ).transpose(0, 1), + self._batched_contextual_linear_weights.to( + contextual_input_embeddings.dtype + ), + ).transpose(0, 1) + output_seq_embeddings = concat_2D_jagged( + max_seq_len=self._max_contextual_seq_len + output_max_seq_len, + values_left=fx_unwrap_optional_tensor(contextual_embeddings).reshape( + -1, self._output_embedding_dim + ), + values_right=output_seq_embeddings, + max_len_left=self._max_contextual_seq_len, + max_len_right=output_max_seq_len, + offsets_left=None, + offsets_right=output_seq_offsets, + kernel=self.hammer_kernel(), + ) + output_seq_timestamps = concat_2D_jagged( + max_seq_len=self._max_contextual_seq_len + output_max_seq_len, + values_left=torch.zeros( + (output_seq_lengths.size(0) * self._max_contextual_seq_len, 1), + dtype=output_seq_timestamps.dtype, + device=output_seq_timestamps.device, + ), + values_right=output_seq_timestamps.unsqueeze(-1), + max_len_left=self._max_contextual_seq_len, + max_len_right=output_max_seq_len, + offsets_left=None, + offsets_right=output_seq_offsets, + kernel=self.hammer_kernel(), + ).squeeze(-1) + output_max_seq_len = output_max_seq_len + self._max_contextual_seq_len + output_seq_lengths = output_seq_lengths + self._max_contextual_seq_len + output_seq_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum( + output_seq_lengths + ) + output_total_uih_len = ( + output_total_uih_len + + self._max_contextual_seq_len * output_seq_lengths.size(0) + ) + + return ( + output_max_seq_len, + output_total_uih_len, + output_total_targets, + output_seq_lengths, + output_seq_offsets, + output_seq_timestamps, + output_seq_embeddings, + output_num_targets, + seq_payloads, + ) diff --git a/recommendation_v4/generative_recommenders/modules/stu.py b/recommendation_v4/generative_recommenders/modules/stu.py new file mode 100644 index 000000000..45c6ea5f3 --- /dev/null +++ b/recommendation_v4/generative_recommenders/modules/stu.py @@ -0,0 +1,471 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict +import abc +from dataclasses import dataclass +from typing import List, Optional, Tuple + +import torch +from generative_recommenders.common import fx_unwrap_optional_tensor, HammerModule +from generative_recommenders.ops.hstu_attention import delta_hstu_mha +from generative_recommenders.ops.hstu_compute import ( + hstu_compute_output, + hstu_compute_uqvk, + hstu_preprocess_and_attention, +) +from generative_recommenders.ops.jagged_tensors import concat_2D_jagged, split_2D_jagged +from torch.autograd.profiler import record_function + + +try: + torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops") + torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops_cpu") +except OSError: + pass + + +class STU(HammerModule, abc.ABC): + def cached_forward( + self, + delta_x: torch.Tensor, + num_targets: torch.Tensor, + max_kv_caching_len: int = 0, + kv_caching_lengths: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + raise NotImplementedError + + @abc.abstractmethod + def forward( + self, + x: torch.Tensor, + x_lengths: torch.Tensor, + x_offsets: torch.Tensor, + max_seq_len: int, + num_targets: torch.Tensor, + max_kv_caching_len: int = 0, + kv_caching_lengths: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + pass + + +@dataclass +class STULayerConfig: + embedding_dim: int + num_heads: int + hidden_dim: int + attention_dim: int + output_dropout_ratio: float = 0.3 + causal: bool = True + target_aware: bool = True + max_attn_len: Optional[int] = None + attn_alpha: Optional[float] = None + use_group_norm: bool = False + recompute_normed_x: bool = True + recompute_uvqk: bool = True + recompute_y: bool = True + sort_by_length: bool = True + contextual_seq_len: int = 0 + + +@torch.fx.wrap +def _update_kv_cache( + max_seq_len: int, + seq_offsets: torch.Tensor, + k: Optional[torch.Tensor], + v: Optional[torch.Tensor], + max_kv_caching_len: int, + kv_caching_lengths: Optional[torch.Tensor], + orig_k_cache: Optional[torch.Tensor], + orig_v_cache: Optional[torch.Tensor], + orig_max_kv_caching_len: int, + orig_kv_caching_offsets: Optional[torch.Tensor], +) -> Tuple[Optional[torch.Tensor], Optional[torch.Tensor], int, Optional[torch.Tensor]]: + if kv_caching_lengths is not None: + kv_caching_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum( + kv_caching_lengths + ) + delta_offsets = seq_offsets - kv_caching_offsets + k_cache, _ = split_2D_jagged( + max_seq_len=max_seq_len, + values=fx_unwrap_optional_tensor(k).flatten(1, 2), + max_len_left=None, + max_len_right=None, + offsets_left=kv_caching_offsets, + offsets_right=delta_offsets, + ) + v_cache, _ = split_2D_jagged( + max_seq_len=max_seq_len, + values=fx_unwrap_optional_tensor(v).flatten(1, 2), + max_len_left=None, + max_len_right=None, + offsets_left=kv_caching_offsets, + offsets_right=delta_offsets, + ) + if max_kv_caching_len == 0: + max_kv_caching_len = int(kv_caching_lengths.max().item()) + return ( + k_cache, + v_cache, + max_kv_caching_len, + kv_caching_offsets, + ) + else: + return ( + orig_k_cache, + orig_v_cache, + orig_max_kv_caching_len, + orig_kv_caching_offsets, + ) + + +@torch.fx.wrap +def _construct_full_kv( + delta_k: torch.Tensor, + delta_v: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + max_kv_caching_len: int, + kv_caching_offsets: torch.Tensor, +) -> Tuple[torch.Tensor, torch.Tensor, int, torch.Tensor]: + L, _ = delta_k.shape + B = kv_caching_offsets.shape[0] - 1 + delta_size = L // B + full_k = concat_2D_jagged( + max_seq_len=max_kv_caching_len + delta_size, + values_left=k_cache, + values_right=delta_k, + max_len_left=max_kv_caching_len, + max_len_right=delta_size, + offsets_left=kv_caching_offsets, + offsets_right=None, + ) + full_v = concat_2D_jagged( + max_seq_len=max_kv_caching_len + delta_size, + values_left=v_cache, + values_right=delta_v, + max_len_left=max_kv_caching_len, + max_len_right=delta_size, + offsets_left=kv_caching_offsets, + offsets_right=None, + ) + full_kv_caching_offsets = kv_caching_offsets + delta_size * torch.arange( + B + 1, device=delta_k.device + ) + return ( + full_k, + full_v, + max_kv_caching_len + delta_size, + full_kv_caching_offsets, + ) + + +class STULayer(STU): + max_kv_caching_len: int + k_cache: Optional[torch.Tensor] + v_cache: Optional[torch.Tensor] + kv_caching_offsets: Optional[torch.Tensor] + + def __init__( + self, + config: STULayerConfig, + is_inference: bool = False, + ) -> None: + super().__init__( + is_inference=is_inference, + ) + self.reset_kv_cache() + self._num_heads: int = config.num_heads + self._embedding_dim: int = config.embedding_dim + self._hidden_dim: int = config.hidden_dim + self._attention_dim: int = config.attention_dim + self._output_dropout_ratio: float = config.output_dropout_ratio + self._target_aware: bool = config.target_aware + self._causal: bool = config.causal + self._max_attn_len: int = config.max_attn_len or 0 + self._attn_alpha: float = config.attn_alpha or 1.0 / (self._attention_dim**0.5) + self._use_group_norm: bool = config.use_group_norm + self._recompute_normed_x: bool = config.recompute_normed_x + self._recompute_uvqk: bool = config.recompute_uvqk + self._recompute_y: bool = config.recompute_y + self._sort_by_length: bool = config.sort_by_length + self._contextual_seq_len: int = config.contextual_seq_len + + self._uvqk_weight: torch.nn.Parameter = torch.nn.Parameter( + torch.empty( + ( + self._embedding_dim, + (self._hidden_dim * 2 + self._attention_dim * 2) * self._num_heads, + ) + ), + ) + torch.nn.init.xavier_uniform_(self._uvqk_weight) + self._uvqk_beta: torch.nn.Parameter = torch.nn.Parameter( + torch.zeros( + (self._hidden_dim * 2 + self._attention_dim * 2) * self._num_heads, + ), + ) + self._input_norm_weight: torch.nn.Parameter = torch.nn.Parameter( + torch.ones((self._embedding_dim,)), + ) + self._input_norm_bias: torch.nn.Parameter = torch.nn.Parameter( + torch.zeros((self._embedding_dim,)), + ) + self._output_weight = torch.nn.Parameter( + torch.empty( + ( + self._hidden_dim * self._num_heads * 3, + self._embedding_dim, + ) + ), + ) + torch.nn.init.xavier_uniform_(self._output_weight) + output_norm_shape: int = ( + self._hidden_dim * self._num_heads + if not self._use_group_norm + else self._num_heads + ) + self._output_norm_weight: torch.nn.Parameter = torch.nn.Parameter( + torch.ones((output_norm_shape,)), + ) + self._output_norm_bias: torch.nn.Parameter = torch.nn.Parameter( + torch.zeros((output_norm_shape,)), + ) + + def reset_kv_cache(self) -> None: + self.k_cache = None + self.v_cache = None + self.kv_caching_offsets = None + self.max_kv_caching_len = 0 + + def update_kv_cache( + self, + max_seq_len: int, + seq_offsets: torch.Tensor, + k: Optional[torch.Tensor], + v: Optional[torch.Tensor], + max_kv_caching_len: int, + kv_caching_lengths: Optional[torch.Tensor], + ) -> None: + self.k_cache, self.v_cache, self.max_kv_caching_len, self.kv_caching_offsets = ( + _update_kv_cache( + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + k=k, + v=v, + max_kv_caching_len=max_kv_caching_len, + kv_caching_lengths=kv_caching_lengths, + orig_k_cache=self.k_cache, + orig_v_cache=self.v_cache, + orig_max_kv_caching_len=self.max_kv_caching_len, + orig_kv_caching_offsets=self.kv_caching_offsets, + ) + ) + + def construct_full_kv( + self, + delta_k: torch.Tensor, + delta_v: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor, int, torch.Tensor]: + return _construct_full_kv( + delta_k=delta_k, + delta_v=delta_v, + k_cache=fx_unwrap_optional_tensor(self.k_cache), + v_cache=fx_unwrap_optional_tensor(self.v_cache), + max_kv_caching_len=self.max_kv_caching_len, + kv_caching_offsets=self.kv_caching_offsets, + ) + + def forward( + self, + x: torch.Tensor, + x_lengths: torch.Tensor, + x_offsets: torch.Tensor, + max_seq_len: int, + num_targets: torch.Tensor, + max_kv_caching_len: int = 0, + kv_caching_lengths: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + with record_function("## stu_preprocess_and_attention ##"): + u, attn_output, k, v = hstu_preprocess_and_attention( + x=x, + norm_weight=self._input_norm_weight.to(x.dtype), + norm_bias=self._input_norm_bias.to(x.dtype), + norm_eps=1e-6, + num_heads=self._num_heads, + attn_dim=self._attention_dim, + hidden_dim=self._hidden_dim, + uvqk_weight=self._uvqk_weight.to(x.dtype), + uvqk_bias=self._uvqk_beta.to(x.dtype), + max_seq_len=max_seq_len, + seq_offsets=x_offsets, + attn_alpha=self._attn_alpha, + causal=self._causal, + num_targets=num_targets if self._target_aware else None, + max_attn_len=self._max_attn_len, + contextual_seq_len=self._contextual_seq_len, + recompute_uvqk_in_backward=self._recompute_uvqk, + recompute_normed_x_in_backward=self._recompute_normed_x, + sort_by_length=self._sort_by_length, + prefill=kv_caching_lengths is not None, + kernel=self.hammer_kernel(), + ) + + self.update_kv_cache( + max_seq_len=max_seq_len, + seq_offsets=x_offsets, + k=k, + v=v, + max_kv_caching_len=max_kv_caching_len, + kv_caching_lengths=kv_caching_lengths, + ) + + with record_function("## stu_compute_output ##"): + return hstu_compute_output( + attn=attn_output, + u=u, + x=x, + norm_weight=self._output_norm_weight.to(x.dtype), + norm_bias=self._output_norm_bias.to(x.dtype), + norm_eps=1e-6, + dropout_ratio=self._output_dropout_ratio, + output_weight=self._output_weight.to(x.dtype), + group_norm=self._use_group_norm, + num_heads=self._num_heads, + linear_dim=self._hidden_dim, + concat_u=True, + concat_x=True, + mul_u_activation_type="none", + training=self.training, + kernel=self.hammer_kernel(), + recompute_y_in_backward=self._recompute_y, + ) + + def cached_forward( + self, + delta_x: torch.Tensor, + num_targets: torch.Tensor, + max_kv_caching_len: int = 0, + kv_caching_lengths: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + with record_function("## stu_compute_uqvk ##"): + delta_u, delta_q, delta_k, delta_v = hstu_compute_uqvk( + x=delta_x, + norm_weight=self._input_norm_weight.to(delta_x.dtype), + norm_bias=self._input_norm_bias.to(delta_x.dtype), + norm_eps=1e-6, + num_heads=self._num_heads, + attn_dim=self._attention_dim, + hidden_dim=self._hidden_dim, + uvqk_weight=self._uvqk_weight.to(delta_x.dtype), + uvqk_bias=self._uvqk_beta.to(delta_x.dtype), + kernel=self.hammer_kernel(), + ) + k, v, max_seq_len, seq_offsets = self.construct_full_kv( + delta_k=delta_k.flatten(1, 2), + delta_v=delta_v.flatten(1, 2), + ) + self.update_kv_cache( + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + k=k, + v=v, + max_kv_caching_len=max_kv_caching_len, + kv_caching_lengths=kv_caching_lengths, + ) + k = k.view(-1, self._num_heads, self._attention_dim) + v = v.view(-1, self._num_heads, self._hidden_dim) + with record_function("## delta_hstu_mha ##"): + delta_attn_output = delta_hstu_mha( + max_seq_len=max_seq_len, + alpha=self._attn_alpha, + delta_q=delta_q, + k=k, + v=v, + seq_offsets=seq_offsets, + num_targets=num_targets if self._target_aware else None, + max_attn_len=self._max_attn_len, + contextual_seq_len=self._contextual_seq_len, + kernel=self.hammer_kernel(), + ).view(-1, self._hidden_dim * self._num_heads) + with record_function("## stu_compute_output ##"): + return hstu_compute_output( + attn=delta_attn_output, + u=delta_u, + x=delta_x, + norm_weight=self._output_norm_weight.to(delta_x.dtype), + norm_bias=self._output_norm_bias.to(delta_x.dtype), + norm_eps=1e-6, + dropout_ratio=self._output_dropout_ratio, + output_weight=self._output_weight.to(delta_x.dtype), + group_norm=self._use_group_norm, + num_heads=self._num_heads, + linear_dim=self._hidden_dim, + concat_u=True, + concat_x=True, + mul_u_activation_type="none", + training=self.training, + kernel=self.hammer_kernel(), + recompute_y_in_backward=self._recompute_y, + ) + + +class STUStack(STU): + def __init__( + self, + stu_list: List[STU], + is_inference: bool = False, + ) -> None: + super().__init__(is_inference=is_inference) + self._stu_layers: torch.nn.ModuleList = torch.nn.ModuleList(modules=stu_list) + + def forward( + self, + x: torch.Tensor, + x_lengths: torch.Tensor, + x_offsets: torch.Tensor, + max_seq_len: int, + num_targets: torch.Tensor, + max_kv_caching_len: int = 0, + kv_caching_lengths: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + for layer in self._stu_layers: + x = layer( + x=x, + x_lengths=x_lengths, + x_offsets=x_offsets, + max_seq_len=max_seq_len, + num_targets=num_targets, + max_kv_caching_len=max_kv_caching_len, + kv_caching_lengths=kv_caching_lengths, + ) + return x + + def cached_forward( + self, + delta_x: torch.Tensor, + num_targets: torch.Tensor, + max_kv_caching_len: int = 0, + kv_caching_lengths: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + for layer in self._stu_layers: + delta_x = layer.cached_forward( # pyre-ignore [29] + delta_x=delta_x, + num_targets=num_targets, + max_kv_caching_len=max_kv_caching_len, + kv_caching_lengths=kv_caching_lengths, + ) + return delta_x diff --git a/recommendation_v4/generative_recommenders/modules/tests/action_encoder_test.py b/recommendation_v4/generative_recommenders/modules/tests/action_encoder_test.py new file mode 100644 index 000000000..184b314ea --- /dev/null +++ b/recommendation_v4/generative_recommenders/modules/tests/action_encoder_test.py @@ -0,0 +1,113 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +import unittest + +import torch +from generative_recommenders.common import gpu_unavailable +from generative_recommenders.modules.action_encoder import ActionEncoder + + +class ActionEncoderTest(unittest.TestCase): + @unittest.skipIf(*gpu_unavailable) + def test_forward(self) -> None: + device = torch.device("cuda") + action_embedding_dim = 32 + action_weights = [1, 2, 4, 8, 16] + watchtime_to_action_thresholds_and_weights = [ + (30, 32), + (60, 64), + (100, 128), + ] + num_action_types = len(action_weights) + len( + watchtime_to_action_thresholds_and_weights + ) + combined_action_weights = action_weights + [ + x[1] for x in watchtime_to_action_thresholds_and_weights + ] + enabled_actions = [ + [0], + [0, 1], + [1, 3, 4], + [1, 2, 3, 4], + [1, 2], + [2], + ] + watchtimes = [40, 20, 110, 31, 26, 55] + for i, wt in enumerate(watchtimes): + for j, w in enumerate(watchtime_to_action_thresholds_and_weights): + if wt > w[0]: + enabled_actions[i].append(j + len(action_weights)) + actions = [ + sum([combined_action_weights[t] for t in x]) for x in enabled_actions + ] + + encoder = ActionEncoder( + watchtime_feature_name="watchtimes", + action_feature_name="actions", + action_weights=action_weights, + watchtime_to_action_thresholds_and_weights=watchtime_to_action_thresholds_and_weights, + action_embedding_dim=action_embedding_dim, + is_inference=False, + ).to(device) + + seq_lengths = [6, 3] + seq_offsets = [0, 6, 9] + num_targets = [2, 1] + uih_offsets = [0, 4, 6] + target_offsets = [0, 2, 3] + seq_embeddings = torch.rand(9, 128, device=device) + action_embeddings = encoder( + max_uih_len=4, + max_targets=2, + uih_offsets=torch.tensor(uih_offsets, device=device), + target_offsets=torch.tensor(target_offsets, device=device), + seq_embeddings=seq_embeddings, + seq_payloads={ + "watchtimes": torch.tensor(watchtimes, device=device), + "actions": torch.tensor(actions, device=device), + }, + ) + self.assertEqual( + action_embeddings.shape, (9, action_embedding_dim * num_action_types) + ) + for b in range(len(seq_lengths)): + b_start = seq_offsets[b] + b_end = seq_offsets[b + 1] + u_start = uih_offsets[b] + for j in range(b_start, b_end): + embedding = action_embeddings[j].view(num_action_types, -1) + for atype in range(num_action_types): + if b_end - j <= num_targets[b]: + torch.testing.assert_allclose( + embedding[atype], + encoder._target_action_embedding_table.view( + num_action_types, -1 + )[atype], + ) + else: + if atype in enabled_actions[j - b_start + u_start]: + torch.testing.assert_allclose( + embedding[atype], + encoder._action_embedding_table[atype], + ) + else: + torch.testing.assert_allclose( + embedding[atype], torch.zeros_like(embedding[atype]) + ) + action_embeddings.sum().backward() diff --git a/recommendation_v4/generative_recommenders/modules/tests/content_encoder_test.py b/recommendation_v4/generative_recommenders/modules/tests/content_encoder_test.py new file mode 100644 index 000000000..e67656388 --- /dev/null +++ b/recommendation_v4/generative_recommenders/modules/tests/content_encoder_test.py @@ -0,0 +1,74 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +import unittest + +import torch +from generative_recommenders.common import gpu_unavailable +from generative_recommenders.modules.content_encoder import ContentEncoder + + +class ContentEncoderTest(unittest.TestCase): + @unittest.skipIf(*gpu_unavailable) + def test_forward(self) -> None: + device = torch.device("cuda") + input_embedding_dim = 32 + additional_embedding_dim = 64 + enrich_embedding_dim = 16 + encoder = ContentEncoder( + input_embedding_dim=input_embedding_dim, + additional_content_features={ + "a0": additional_embedding_dim, + "a1": additional_embedding_dim, + }, + target_enrich_features={ + "t0": enrich_embedding_dim, + "t1": enrich_embedding_dim, + }, + is_inference=False, + ).to(device) + seq_lengths = [6, 3] + num_targets = [2, 1] + uih_offsets = [0, 4, 6] + target_offsets = [0, 2, 3] + seq_embeddings = torch.rand( + sum(seq_lengths), input_embedding_dim, device=device + ).requires_grad_(True) + seq_payloads = { + "a0": torch.rand( + sum(seq_lengths), additional_embedding_dim, device=device + ).requires_grad_(True), + "a1": torch.rand( + sum(seq_lengths), additional_embedding_dim, device=device + ).requires_grad_(True), + "t0": torch.rand( + sum(num_targets), enrich_embedding_dim, device=device + ).requires_grad_(True), + "t1": torch.rand( + sum(num_targets), enrich_embedding_dim, device=device + ).requires_grad_(True), + } + content_embeddings = encoder( + max_uih_len=4, + max_targets=2, + uih_offsets=torch.tensor(uih_offsets, device=device), + target_offsets=torch.tensor(target_offsets, device=device), + seq_embeddings=seq_embeddings, + seq_payloads=seq_payloads, + ) + content_embeddings.sum().backward() diff --git a/recommendation_v4/generative_recommenders/modules/tests/contextual_interleave_preprocessor_test.py b/recommendation_v4/generative_recommenders/modules/tests/contextual_interleave_preprocessor_test.py new file mode 100644 index 000000000..c3202072c --- /dev/null +++ b/recommendation_v4/generative_recommenders/modules/tests/contextual_interleave_preprocessor_test.py @@ -0,0 +1,499 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +import unittest + +import torch +from generative_recommenders.common import gpu_unavailable +from generative_recommenders.modules.action_encoder import ActionEncoder +from generative_recommenders.modules.content_encoder import ContentEncoder +from generative_recommenders.modules.contextual_interleave_preprocessor import ( + ContextualInterleavePreprocessor, +) +from generative_recommenders.modules.contextualize_mlps import ( + ParameterizedContextualizedMLP, + SimpleContextualizedMLP, +) +from hypothesis import given, settings, strategies as st, Verbosity + + +class ContextualInterleavePreprocessorTest(unittest.TestCase): + # pyre-ignore + @given( + enable_interleaving=st.sampled_from([True, False]), + enable_pmlp=st.sampled_from([True, False]), + is_train=st.sampled_from([True, False]), + dtype=st.sampled_from( + [torch.float32, torch.bfloat16] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + ) + @unittest.skipIf(*gpu_unavailable) + @settings(verbosity=Verbosity.verbose, max_examples=50, deadline=None) + def test_forward( + self, + enable_interleaving: bool, + enable_pmlp: bool, + is_train: bool, + dtype: torch.dtype, + ) -> None: + device = torch.device("cuda") + + input_embedding_dim = 64 + output_embedding_dim = 32 + action_embedding_dim = 16 + action_encoder_hidden_dim = 256 + content_encoder_hidden_dim = 128 + contextual_len = 3 + + content_encoder = ContentEncoder( + input_embedding_dim=input_embedding_dim, + additional_content_features={ + "a0": input_embedding_dim, + "a1": input_embedding_dim, + }, + target_enrich_features={ + "t0": input_embedding_dim, + "t1": input_embedding_dim, + }, + is_inference=False, + ).to(device) + action_embedding_dim = 32 + action_weights = [1, 2, 4, 8, 16] + watchtime_to_action_thresholds_and_weights = [ + (30, 32), + (60, 64), + (100, 128), + ] + action_encoder = ActionEncoder( + watchtime_feature_name="watchtimes", + action_feature_name="actions", + action_weights=action_weights, + watchtime_to_action_thresholds_and_weights=watchtime_to_action_thresholds_and_weights, + action_embedding_dim=action_embedding_dim, + is_inference=False, + ).to(device) + + preprocessor = ContextualInterleavePreprocessor( + input_embedding_dim=input_embedding_dim, + output_embedding_dim=output_embedding_dim, + contextual_feature_to_max_length={"c_0": 1, "c_1": 2}, + contextual_feature_to_min_uih_length={"c_1": 4}, + pmlp_contextual_dropout_ratio=0.2, + content_encoder=content_encoder, + content_contextualize_mlp_fn=lambda in_dim, + out_dim, + contextual_dim, + is_inference: ParameterizedContextualizedMLP( + contextual_embedding_dim=contextual_dim, + sequential_input_dim=in_dim, + sequential_output_dim=out_dim, + hidden_dim=content_encoder_hidden_dim, + is_inference=is_inference, + ) + if enable_pmlp + else SimpleContextualizedMLP( + sequential_input_dim=in_dim, + sequential_output_dim=out_dim, + hidden_dim=content_encoder_hidden_dim, + is_inference=is_inference, + ), + action_encoder=action_encoder, + action_contextualize_mlp_fn=lambda in_dim, + out_dim, + contextual_dim, + is_inference: ParameterizedContextualizedMLP( + contextual_embedding_dim=contextual_dim, + sequential_input_dim=in_dim, + sequential_output_dim=out_dim, + hidden_dim=action_encoder_hidden_dim, + is_inference=is_inference, + ) + if enable_pmlp + else SimpleContextualizedMLP( + sequential_input_dim=in_dim, + sequential_output_dim=out_dim, + hidden_dim=action_encoder_hidden_dim, + is_inference=is_inference, + ), + enable_interleaving=enable_interleaving, + is_inference=False, + ).to(device) + preprocessor.set_training_dtype(dtype) + if not is_train: + preprocessor.eval() + + # inputs + seq_lengths = [6, 3] + num_targets = [2, 1] + seq_embeddings = torch.rand( + (sum(seq_lengths), input_embedding_dim), + device=device, + dtype=dtype, + ) + seq_timestamps = torch.tensor( + [1, 2, 3, 4, 5, 6, 10, 20, 30], + device=device, + ) + watchtimes = [40, 20, 110, 31, 26, 55] + actions = [1, 3, 26, 30, 6, 4] + ( + output_max_seq_len, + output_total_uih_len, + output_total_targets, + output_seq_lengths, + output_seq_offsets, + output_seq_timestamps, + output_seq_embeddings, + output_num_targets, + _, + ) = preprocessor( + max_uih_len=4, + max_targets=2, + total_uih_len=sum(seq_lengths) - sum(num_targets), + total_targets=sum(num_targets), + seq_lengths=torch.tensor(seq_lengths, device=device), + seq_timestamps=seq_timestamps, + seq_embeddings=seq_embeddings, + seq_payloads={ + # contextual + "c_0": torch.rand((2, input_embedding_dim), device=device, dtype=dtype), + "c_0_offsets": torch.tensor([0, 1, 1], device=device), + "c_1": torch.rand((4, input_embedding_dim), device=device, dtype=dtype), + "c_1_offsets": torch.tensor([0, 2, 3], device=device), + # action + "watchtimes": torch.tensor(watchtimes, device=device), + "actions": torch.tensor(actions, device=device), + # content + "a0": torch.rand_like(seq_embeddings).requires_grad_(True), + "a1": torch.rand_like(seq_embeddings).requires_grad_(True), + "t0": torch.rand( + sum(num_targets), input_embedding_dim, device=device, dtype=dtype + ).requires_grad_(True), + "t1": torch.rand( + sum(num_targets), input_embedding_dim, device=device, dtype=dtype + ).requires_grad_(True), + }, + num_targets=torch.tensor(num_targets, device=device), + ) + if enable_interleaving: + if is_train: + expected_output_seq_lengths = [ + 2 * s + contextual_len for s in seq_lengths + ] + expected_max_seq_len = max(expected_output_seq_lengths) + expected_output_num_targets = [2 * s for s in num_targets] + expected_seq_embedding_size = ( + sum(expected_output_seq_lengths), + output_embedding_dim, + ) + expected_seq_timestamps_size = (sum(expected_output_seq_lengths),) + expected_output_seq_timestamps = [ + 0, + 0, + 0, + 1, + 1, + 2, + 2, + 3, + 3, + 4, + 4, + 5, + 5, + 6, + 6, + 0, + 0, + 0, + 10, + 10, + 20, + 20, + 30, + 30, + ] + else: + expected_output_seq_lengths = [ + 2 * s - n + contextual_len for s, n in zip(seq_lengths, num_targets) + ] + expected_max_seq_len = max(expected_output_seq_lengths) + expected_output_num_targets = num_targets + expected_seq_embedding_size = ( + sum(expected_output_seq_lengths), + output_embedding_dim, + ) + expected_seq_timestamps_size = (sum(expected_output_seq_lengths),) + expected_output_seq_timestamps = [ + 0, + 0, + 0, + 1, + 1, + 2, + 2, + 3, + 3, + 4, + 4, + 5, + 6, + 0, + 0, + 0, + 10, + 10, + 20, + 20, + 30, + ] + else: + expected_output_seq_lengths = [s + contextual_len for s in seq_lengths] + expected_max_seq_len = max(expected_output_seq_lengths) + expected_output_num_targets = num_targets + expected_seq_embedding_size = ( + sum(expected_output_seq_lengths), + output_embedding_dim, + ) + expected_seq_timestamps_size = (sum(expected_output_seq_lengths),) + expected_output_seq_timestamps = [ + 0, + 0, + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 0, + 0, + 0, + 10, + 20, + 30, + ] + + self.assertEqual(output_max_seq_len, expected_max_seq_len) + self.assertEqual(output_seq_lengths.tolist(), expected_output_seq_lengths) + torch.testing.assert_allclose( + torch.ops.fbgemm.asynchronous_complete_cumsum(output_seq_lengths), + output_seq_offsets, + ) + self.assertEqual(output_num_targets.tolist(), expected_output_num_targets) + self.assertEqual( + output_seq_embeddings.size(), + expected_seq_embedding_size, + ) + self.assertEqual( + output_seq_timestamps.size(), + expected_seq_timestamps_size, + ) + self.assertEqual( + output_seq_timestamps.tolist(), + expected_output_seq_timestamps, + ) + + # test combine embeddings + batch_size = 10 + max_uih_len = 100 + max_targets = 20 + max_seq_len = max_uih_len + max_targets + seq_lengths = torch.randint(0, max_uih_len, (batch_size,), device=device) + total_uih_len = int(seq_lengths.sum().item()) + num_targets = torch.randint(1, max_targets, (batch_size,), device=device) + total_targets = int(num_targets.sum().item()) + seq_lengths = seq_lengths + num_targets + seq_offsets = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + seq_offsets[1:] = torch.cumsum(seq_lengths, dim=0) + total_seq_len = int(torch.sum(seq_lengths).item()) + seq_timestamps = torch.randint(0, 1000000, (total_seq_len,), device=device) + content_embeddings = torch.rand( + (total_seq_len, output_embedding_dim), + device=device, + ).requires_grad_(True) + action_embeddings = torch.rand( + (total_seq_len, output_embedding_dim), + device=device, + ).requires_grad_(True) + contextual_embeddings = torch.rand( + (total_seq_len, 3 * output_embedding_dim), device=device + ).requires_grad_(True) + ( + output_max_seq_len, + output_total_uih_len, + output_total_targets, + output_seq_lengths, + output_seq_offsets, + output_seq_timestamps, + output_seq_embeddings, + output_num_targets, + ) = preprocessor.combine_embeddings( + max_uih_len=max_uih_len, + max_targets=max_targets, + total_uih_len=total_uih_len, + total_targets=total_targets, + seq_lengths=seq_lengths, + seq_timestamps=seq_timestamps, + content_embeddings=content_embeddings, + action_embeddings=action_embeddings, + contextual_embeddings=contextual_embeddings, + num_targets=num_targets, + ) + seq_embeddings = action_embeddings + content_embeddings + if enable_interleaving: + if is_train: + self.assertEqual( + output_max_seq_len, + max_seq_len * 2 + contextual_len, + ) + self.assertEqual( + output_total_uih_len, + total_uih_len * 2 + contextual_len * batch_size, + ) + self.assertEqual( + output_total_targets, + total_targets * 2, + ) + torch.testing.assert_allclose( + output_seq_lengths, seq_lengths * 2 + contextual_len + ) + torch.testing.assert_allclose(output_num_targets, num_targets * 2) + else: + self.assertEqual( + output_max_seq_len, + max_uih_len * 2 + max_targets + contextual_len, + ) + self.assertEqual( + output_total_uih_len, + total_uih_len * 2 + contextual_len * batch_size, + ) + self.assertEqual( + output_total_targets, + total_targets, + ) + torch.testing.assert_allclose( + output_seq_lengths, seq_lengths * 2 - num_targets + contextual_len + ) + torch.testing.assert_allclose(output_num_targets, num_targets) + else: + self.assertEqual( + output_max_seq_len, + max_seq_len + contextual_len, + ) + self.assertEqual( + output_total_uih_len, + total_uih_len + contextual_len * batch_size, + ) + self.assertEqual( + output_total_targets, + total_targets, + ) + torch.testing.assert_allclose( + output_seq_lengths, seq_lengths + contextual_len + ) + torch.testing.assert_allclose(output_num_targets, num_targets) + for b in range(batch_size): + input_start = int(seq_offsets[b].item()) + input_end = int(seq_offsets[b + 1].item()) + output_start = int(output_seq_offsets[b].item()) + output_end = int(output_seq_offsets[b + 1].item()) + input_targets = int(num_targets[b].item()) + output_targets = int(output_num_targets[b].item()) + torch.testing.assert_allclose( + output_seq_timestamps[output_start : output_start + contextual_len], + torch.zeros(3, device=device), + ) + torch.testing.assert_allclose( + output_seq_embeddings[ + output_start : output_start + contextual_len + ].view(-1), + contextual_embeddings[b], + ) + if enable_interleaving: + torch.testing.assert_allclose( + output_seq_timestamps[ + output_start + contextual_len : output_end - output_targets + ].view(-1, 2)[:, 0], + seq_timestamps[input_start : input_end - input_targets], + ) + torch.testing.assert_allclose( + output_seq_timestamps[ + output_start + contextual_len : output_end - output_targets + ].view(-1, 2)[:, 1], + seq_timestamps[input_start : input_end - input_targets], + ) + torch.testing.assert_allclose( + output_seq_embeddings[ + output_start + contextual_len : output_end - output_targets + ].view(-1, 2, output_embedding_dim)[:, 0, :], + content_embeddings[input_start : input_end - input_targets], + ) + torch.testing.assert_allclose( + output_seq_embeddings[ + output_start + contextual_len : output_end - output_targets + ].view(-1, 2, output_embedding_dim)[:, 1, :], + action_embeddings[input_start : input_end - input_targets], + ) + if is_train: + torch.testing.assert_allclose( + output_seq_timestamps[ + output_end - output_targets : output_end + ].view(-1, 2)[:, 0], + seq_timestamps[input_end - input_targets : input_end], + ) + torch.testing.assert_allclose( + output_seq_timestamps[ + output_end - output_targets : output_end + ].view(-1, 2)[:, 1], + seq_timestamps[input_end - input_targets : input_end], + ) + torch.testing.assert_allclose( + output_seq_embeddings[ + output_end - output_targets : output_end + ].view(-1, 2, output_embedding_dim)[:, 0, :], + content_embeddings[input_end - input_targets : input_end], + ) + torch.testing.assert_allclose( + output_seq_embeddings[ + output_end - output_targets : output_end + ].view(-1, 2, output_embedding_dim)[:, 1, :], + action_embeddings[input_end - input_targets : input_end], + ) + else: + torch.testing.assert_allclose( + output_seq_timestamps[output_end - output_targets : output_end], + seq_timestamps[input_end - input_targets : input_end], + ) + torch.testing.assert_allclose( + output_seq_embeddings[output_end - output_targets : output_end], + content_embeddings[input_end - input_targets : input_end], + ) + else: + torch.testing.assert_allclose( + output_seq_timestamps[output_start + contextual_len : output_end], + seq_timestamps[input_start:input_end], + ) + torch.testing.assert_allclose( + output_seq_embeddings[output_start + contextual_len : output_end], + seq_embeddings[input_start:input_end], + ) diff --git a/recommendation_v4/generative_recommenders/modules/tests/dynamic_stu_test.py b/recommendation_v4/generative_recommenders/modules/tests/dynamic_stu_test.py new file mode 100644 index 000000000..c1c598f1f --- /dev/null +++ b/recommendation_v4/generative_recommenders/modules/tests/dynamic_stu_test.py @@ -0,0 +1,279 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +import copy +import unittest +from typing import List + +import torch +from generative_recommenders.common import gpu_unavailable, HammerKernel, set_dev_mode +from generative_recommenders.modules.dynamic_stu import L2STU, SDSTU +from generative_recommenders.modules.stu import STU, STULayer, STULayerConfig, STUStack +from hypothesis import given, settings, strategies as st, Verbosity + + +class DynamicStuTest(unittest.TestCase): + # pyre-ignore + @given( + causal=st.sampled_from([True]), + num_layers=st.sampled_from([2]), + num_heads=st.sampled_from([2]), + max_uih_len=st.sampled_from([300]), + batch_size=st.sampled_from([8]), + embedding_dim=st.sampled_from([16]), + attention_dim=st.sampled_from([32]), + linear_hidden_dim=st.sampled_from([64]), + has_multiple_targets=st.sampled_from([True, False]), + contextual_seq_len=st.sampled_from([0, 10]), + dtype=st.sampled_from( + [torch.float32, torch.bfloat16] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + ) + @unittest.skipIf(*gpu_unavailable) + @settings(verbosity=Verbosity.verbose, max_examples=20, deadline=None) + def test_l2_stu( + self, + causal: bool, + num_layers: int, + num_heads: int, + max_uih_len: int, + batch_size: int, + embedding_dim: int, + attention_dim: int, + linear_hidden_dim: int, + has_multiple_targets: bool, + contextual_seq_len: int, + dtype: torch.dtype, + ) -> None: + set_dev_mode(True) + device = torch.device("cuda") + l3_stu_layers: List[STU] = [ + STULayer( + config=STULayerConfig( + embedding_dim=embedding_dim, + num_heads=num_heads, + hidden_dim=linear_hidden_dim, + attention_dim=attention_dim, + output_dropout_ratio=0.0, + causal=causal, + target_aware=has_multiple_targets, + contextual_seq_len=contextual_seq_len, + ), + is_inference=False, + ) + for _ in range(num_layers) + ] + l3_stu: List[STU] = [ + L2STU( + stu=STUStack( + stu_list=l3_stu_layers, + is_inference=False, + ), + max_l2_len=100, + contextual_seq_len=contextual_seq_len, + is_inference=False, + ) + ] + l2_stu_layers: List[STU] = [ + STULayer( + config=STULayerConfig( + embedding_dim=embedding_dim, + num_heads=num_heads, + hidden_dim=linear_hidden_dim, + attention_dim=attention_dim, + output_dropout_ratio=0.0, + causal=causal, + target_aware=has_multiple_targets, + contextual_seq_len=contextual_seq_len, + ), + is_inference=False, + ) + for _ in range(num_layers) + ] + l3_stu + l2_stu: List[STU] = [ + L2STU( + stu=STUStack( + stu_list=l2_stu_layers, + is_inference=False, + ), + max_l2_len=200, + contextual_seq_len=contextual_seq_len, + is_inference=False, + ) + ] + stu_layers: List[STU] = [ + STULayer( + config=STULayerConfig( + embedding_dim=embedding_dim, + num_heads=num_heads, + hidden_dim=linear_hidden_dim, + attention_dim=attention_dim, + output_dropout_ratio=0.0, + causal=causal, + target_aware=has_multiple_targets, + contextual_seq_len=contextual_seq_len, + ), + is_inference=False, + ) + for _ in range(num_layers) + ] + l2_stu + stu = STUStack( + stu_list=stu_layers, + is_inference=False, + ).to(device) + stu.recursive_setattr("_hammer_kernel", HammerKernel.TRITON) + + x_lengths = torch.randint(max_uih_len + 1, (batch_size,), device=device) + x_lengths = x_lengths + contextual_seq_len + max_seq_len = max_uih_len + contextual_seq_len + max_targets = 20 + num_targets = torch.randint(1, max_targets, size=(batch_size,), device=device) + if has_multiple_targets: + x_lengths = x_lengths + num_targets + max_seq_len = max_seq_len + max_targets + x_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum(x_lengths) + total_seq_len = int(x_offsets[-1].cpu().item()) + x = torch.randn( + int(total_seq_len), + embedding_dim, + device=device, + dtype=dtype, + ).requires_grad_(True) + stu_output = stu( + x=x, + x_lengths=x_lengths, + x_offsets=x_offsets, + max_seq_len=max_seq_len, + num_targets=num_targets, + ) + dout = torch.randn_like(stu_output) + stu_output.backward(dout) + self.assertTrue(stu_output.shape == x.shape) + + # pyre-ignore + @given( + causal=st.sampled_from([True]), + num_layers=st.sampled_from([2]), + num_heads=st.sampled_from([2]), + max_uih_len=st.sampled_from([300]), + batch_size=st.sampled_from([8]), + embedding_dim=st.sampled_from([16]), + attention_dim=st.sampled_from([32]), + linear_hidden_dim=st.sampled_from([64]), + has_multiple_targets=st.sampled_from([True, False]), + contextual_seq_len=st.sampled_from([0, 10]), + dropout_ratio=st.sampled_from([0.0, 0.3, 1.0]), + dtype=st.sampled_from( + [torch.float32, torch.bfloat16] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + ) + @unittest.skipIf(*gpu_unavailable) + @settings(verbosity=Verbosity.verbose, max_examples=20, deadline=None) + def test_sd_stu( + self, + causal: bool, + num_layers: int, + num_heads: int, + max_uih_len: int, + batch_size: int, + embedding_dim: int, + attention_dim: int, + linear_hidden_dim: int, + has_multiple_targets: bool, + contextual_seq_len: int, + dropout_ratio: float, + dtype: torch.dtype, + ) -> None: + set_dev_mode(True) + device = torch.device("cuda") + stu_layers: List[STU] = [ + STULayer( + config=STULayerConfig( + embedding_dim=embedding_dim, + num_heads=num_heads, + hidden_dim=linear_hidden_dim, + attention_dim=attention_dim, + output_dropout_ratio=0.0, + causal=causal, + target_aware=has_multiple_targets, + contextual_seq_len=contextual_seq_len, + ), + is_inference=False, + ) + for _ in range(num_layers) + ] + stu = STUStack( + stu_list=stu_layers, + is_inference=False, + ).to(device) + sd_stu = SDSTU( + stu=copy.deepcopy(stu), + dropout_ratio=dropout_ratio, + is_inference=False, + ).to(device) + stu.recursive_setattr("_hammer_kernel", HammerKernel.PYTORCH) + sd_stu.recursive_setattr("_hammer_kernel", HammerKernel.PYTORCH) + x_lengths = torch.randint(max_uih_len + 1, (batch_size,), device=device) + x_lengths = x_lengths + contextual_seq_len + max_seq_len = max_uih_len + contextual_seq_len + max_targets = 20 + num_targets = torch.randint(1, max_targets, size=(batch_size,), device=device) + if has_multiple_targets: + x_lengths = x_lengths + num_targets + max_seq_len = max_seq_len + max_targets + x_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum(x_lengths) + total_seq_len = int(x_offsets[-1].cpu().item()) + x = torch.randn( + int(total_seq_len), + embedding_dim, + device=device, + dtype=dtype, + ).requires_grad_(True) + stu_output = stu( + x=x, + x_lengths=x_lengths, + x_offsets=x_offsets, + max_seq_len=max_seq_len, + num_targets=num_targets, + ) + dout = torch.randn_like(stu_output) + stu_output.backward(dout) + assert x.grad is not None + d_x, x.grad = x.grad.detach().clone(), None + x = x.detach().clone().requires_grad_(True) + sd_stu_output = sd_stu( + x=x, + x_lengths=x_lengths, + x_offsets=x_offsets, + max_seq_len=max_seq_len, + num_targets=num_targets, + ) + dout = dout.detach().clone() + sd_stu_output.backward(dout) + d_sd_x = x.grad.detach().clone() + + self.assertTrue(sd_stu_output.shape == x.shape) + if dropout_ratio == 0.0: + torch.testing.assert_close(stu_output, sd_stu_output) + torch.testing.assert_close(d_x, d_sd_x) + if dropout_ratio == 1.0: + torch.testing.assert_close(x, sd_stu_output) diff --git a/recommendation_v4/generative_recommenders/modules/tests/multitask_module_test.py b/recommendation_v4/generative_recommenders/modules/tests/multitask_module_test.py new file mode 100644 index 000000000..66f2db185 --- /dev/null +++ b/recommendation_v4/generative_recommenders/modules/tests/multitask_module_test.py @@ -0,0 +1,233 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +import unittest +from typing import Dict, List, Tuple + +import torch +from generative_recommenders.common import gpu_unavailable, set_dev_mode +from generative_recommenders.modules.multitask_module import ( + DefaultMultitaskModule, + MultitaskTaskType, + TaskConfig, +) +from generative_recommenders.ops.layer_norm import SwishLayerNorm +from hypothesis import given, settings, strategies as st, Verbosity + + +_task_configs: List[List[TaskConfig]] = [ + [ + TaskConfig( + task_name="is_click", + task_weight=1, + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ), + ], + [ + TaskConfig( + task_name="vvp", + task_weight=2, + task_type=MultitaskTaskType.REGRESSION, + ), + ], + [ + TaskConfig( + task_name="is_click", + task_weight=1, + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ), + TaskConfig( + task_name="is_like", + task_weight=2, + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ), + TaskConfig( + task_name="is_follow", + task_weight=4, + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ), + ], + [ + TaskConfig( + task_name="rating", + task_weight=1, + task_type=MultitaskTaskType.REGRESSION, + ), + TaskConfig( + task_name="vvp", + task_weight=2, + task_type=MultitaskTaskType.REGRESSION, + ), + ], + [ + TaskConfig( + task_name="type_1", + task_weight=2, + task_type=MultitaskTaskType.REGRESSION, + ), + ], + [ + TaskConfig( + task_name="is_click", + task_weight=1, + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ), + TaskConfig( + task_name="is_like", + task_weight=2, + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ), + TaskConfig( + task_name="is_follow", + task_weight=4, + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ), + TaskConfig( + task_name="rating", + task_weight=1, + task_type=MultitaskTaskType.REGRESSION, + ), + TaskConfig( + task_name="vvp", + task_weight=2, + task_type=MultitaskTaskType.REGRESSION, + ), + ], + [ + TaskConfig( + task_name="is_click", + task_weight=1, + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ), + TaskConfig( + task_name="is_like", + task_weight=2, + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ), + TaskConfig( + task_name="is_follow", + task_weight=4, + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ), + TaskConfig( + task_name="rating", + task_weight=1, + task_type=MultitaskTaskType.REGRESSION, + ), + TaskConfig( + task_name="vvp", + task_weight=2, + task_type=MultitaskTaskType.REGRESSION, + ), + ], +] + + +def _get_random_supervision_labels_and_weights( + num_examples: int, + task_configs: List[TaskConfig], + device: torch.device, +) -> Tuple[Dict[str, torch.Tensor], Dict[str, torch.Tensor]]: + supervision_labels: Dict[str, torch.Tensor] = {} + supervision_weights: Dict[str, torch.Tensor] = {} + for task in task_configs: + if task.task_type == MultitaskTaskType.REGRESSION: + supervision_labels[task.task_name] = torch.randn( + num_examples, device=device + ).to(torch.float32) + else: + supervision_labels[task.task_name] = torch.randint( + 0, + 11, + (num_examples,), + device=device, + ).to(torch.float32) + + return supervision_labels, supervision_weights + + +class MultiTaskModuleTest(unittest.TestCase): + # pyre-ignore + @given( + task_config_idx=st.sampled_from(range(len(_task_configs))), + training=st.booleans(), + is_inference=st.booleans(), + dtype=st.sampled_from( + [torch.float32, torch.bfloat16] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + ) + @unittest.skipIf(*gpu_unavailable) + @settings(verbosity=Verbosity.verbose, max_examples=50, deadline=None) + def test_default_multitask_module( + self, + task_config_idx: int, + training: bool, + is_inference: bool, + dtype: torch.dtype, + ) -> None: + set_dev_mode(True) + device = torch.device("cuda") + + L = 200 + embedding_dim = 64 + causal_multitask_weights = 0.3 + + task_configs: List[TaskConfig] = _task_configs[task_config_idx] + task_configs.sort(key=lambda x: x.task_type) + multitask_module = DefaultMultitaskModule( + task_configs=task_configs, + embedding_dim=embedding_dim, + prediction_fn=lambda in_dim, num_tasks: torch.nn.Sequential( + torch.nn.Linear(in_features=in_dim, out_features=512), + SwishLayerNorm(512), + torch.nn.Linear(in_features=512, out_features=num_tasks), + ), + causal_multitask_weights=causal_multitask_weights, + is_inference=is_inference, + ).to(device) + multitask_module.set_training_dtype(dtype) + supervision_labels, supervision_weights = ( + _get_random_supervision_labels_and_weights( + num_examples=L, + task_configs=task_configs, + device=device, + ) + ) + encoded_user_embeddings = torch.rand(L, embedding_dim, device=device) + item_embeddings = torch.rand(L, embedding_dim, device=device) + + ( + mt_preds, + mt_labels, + mt_weights, + mt_losses, + ) = multitask_module( + encoded_user_embeddings=encoded_user_embeddings, + item_embeddings=item_embeddings, + supervision_labels=supervision_labels, + supervision_weights=supervision_weights, + ) + + self.assertEqual(mt_preds.size(), (len(task_configs), L)) + if not is_inference: + self.assertEqual(mt_labels.size(), (len(task_configs), L)) + self.assertEqual(mt_weights.size(), (len(task_configs), L)) + if training: + self.assertEqual(mt_losses.size(), (len(task_configs),)) diff --git a/recommendation_v4/generative_recommenders/modules/tests/stu_test.py b/recommendation_v4/generative_recommenders/modules/tests/stu_test.py new file mode 100644 index 000000000..f6440e55e --- /dev/null +++ b/recommendation_v4/generative_recommenders/modules/tests/stu_test.py @@ -0,0 +1,453 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +import copy +import unittest +from typing import List + +import torch +from generative_recommenders.common import gpu_unavailable, HammerKernel, set_dev_mode +from generative_recommenders.modules.stu import STU, STULayer, STULayerConfig, STUStack +from generative_recommenders.ops.jagged_tensors import split_2D_jagged +from hypothesis import given, settings, strategies as st, Verbosity + + +def _inplace_swap( + batch_size: int, + x: torch.Tensor, + swap_from: torch.Tensor, + swap_to: torch.Tensor, +) -> torch.Tensor: + for i in range(batch_size): + tmp = x[i, swap_from[i], :].detach().clone() + x[i, swap_from[i], :] = x[i, swap_to[i], :] + x[i, swap_to[i], :] = tmp + return x + + +class StuTest(unittest.TestCase): + # pyre-ignore + @given( + causal=st.sampled_from([True]), + num_layers=st.sampled_from([2]), + num_heads=st.sampled_from([1, 2]), + max_uih_len=st.sampled_from([20, 64]), + batch_size=st.sampled_from([8]), + embedding_dim=st.sampled_from([16]), + attention_dim=st.sampled_from([32]), + linear_hidden_dim=st.sampled_from([64]), + has_multiple_targets=st.sampled_from([True, False]), + contextual_seq_len=st.sampled_from([0, 10]), + use_group_norm=st.sampled_from([True, False]), + recompute_uvqk_in_backward=st.sampled_from([True, False]), + recompute_normed_x_in_backward=st.sampled_from([True, False]), + recompute_y_in_backward=st.sampled_from([True, False]), + empty_inputs=st.sampled_from([False]), + dtype=st.sampled_from( + [torch.float32] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + ) + @unittest.skipIf(*gpu_unavailable) + @settings(verbosity=Verbosity.verbose, max_examples=100, deadline=None) + def test_triton( + self, + causal: bool, + num_layers: int, + num_heads: int, + max_uih_len: int, + batch_size: int, + embedding_dim: int, + attention_dim: int, + linear_hidden_dim: int, + has_multiple_targets: bool, + contextual_seq_len: int, + use_group_norm: bool, + recompute_uvqk_in_backward: bool, + recompute_normed_x_in_backward: bool, + recompute_y_in_backward: bool, + empty_inputs: bool, # test the case where all the seqlen in the batch are 0 + dtype: torch.dtype, + ) -> None: + set_dev_mode(True) + device = torch.device("cuda") + + stu_layers: List[STU] = [ + STULayer( + config=STULayerConfig( + embedding_dim=embedding_dim, + num_heads=num_heads, + hidden_dim=linear_hidden_dim, + attention_dim=attention_dim, + output_dropout_ratio=0.0, + causal=causal, + target_aware=has_multiple_targets, + max_attn_len=None, + attn_alpha=None, + use_group_norm=use_group_norm, + recompute_normed_x=recompute_normed_x_in_backward, + recompute_uvqk=recompute_uvqk_in_backward, + recompute_y=recompute_y_in_backward, + sort_by_length=True, + contextual_seq_len=contextual_seq_len, + ), + is_inference=False, + ) + for _ in range(num_layers) + ] + stu = STUStack( + stu_list=stu_layers, + is_inference=False, + ).to(device) + stu.recursive_setattr("_hammer_kernel", HammerKernel.PYTORCH) + stu_triton = copy.deepcopy(stu) + stu_triton.recursive_setattr("_hammer_kernel", HammerKernel.TRITON) + + if empty_inputs: + x_lengths = torch.zeros(batch_size, dtype=torch.int32, device=device) + num_targets = torch.zeros(batch_size, dtype=torch.int32, device=device) + contextual_seq_len = 0 + max_seq_len = 16 + else: + x_lengths = torch.randint(max_uih_len + 1, (batch_size,), device=device) + x_lengths = x_lengths + contextual_seq_len + max_seq_len = max_uih_len + contextual_seq_len + max_targets = 20 + num_targets = torch.randint( + 1, max_targets, size=(batch_size,), device=device + ) + if has_multiple_targets: + x_lengths = x_lengths + num_targets + max_seq_len = max_seq_len + max_targets + x_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum(x_lengths) + total_seq_len = int(x_offsets[-1].cpu().item()) + x = torch.randn( + int(total_seq_len), + embedding_dim, + device=device, + dtype=dtype, + ).requires_grad_(True) + x_triton = x.clone().detach().requires_grad_() + stu_output = stu( + x=x, + x_lengths=x_lengths, + x_offsets=x_offsets, + max_seq_len=max_seq_len, + num_targets=num_targets, + ) + stu_triton_output = stu_triton( + x=x_triton, + x_lengths=x_lengths, + x_offsets=x_offsets, + max_seq_len=max_seq_len, + num_targets=num_targets, + ) + atol = 5e-3 if dtype == torch.bfloat16 else None + rtol = 1e-2 if dtype == torch.bfloat16 else None + torch.testing.assert_close(stu_triton_output, stu_output, atol=atol, rtol=rtol) + dout = torch.randn_like(stu_output) + stu_output.backward(dout) + dout = dout.detach().clone() + stu_triton_output.backward(dout) + torch.testing.assert_close(x.grad, x_triton.grad, atol=atol, rtol=rtol) + + # pyre-ignore + @given( + dtype=st.sampled_from( + [torch.float32] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + ) + @unittest.skipIf(*gpu_unavailable) + @settings(verbosity=Verbosity.verbose, max_examples=8, deadline=None) + def test_target_invariance( + self, + dtype: torch.dtype, + ) -> None: + set_dev_mode(True) + device = torch.device("cuda") + num_layers = 2 + num_heads = 2 + max_seq_len = 32 + batch_size = 8 + embedding_dim = 16 + attention_dim = 32 + linear_hidden_dim = 32 + causal = True + use_group_norm = False + recompute_normed_x_in_backward = False + recompute_uvqk_in_backward = False + recompute_y_in_backward = False + max_attn_len = None + stu_layers: List[STU] = [ + STULayer( + config=STULayerConfig( + embedding_dim=embedding_dim, + num_heads=num_heads, + hidden_dim=linear_hidden_dim, + attention_dim=attention_dim, + output_dropout_ratio=0.0, + causal=causal, + target_aware=True, + max_attn_len=max_attn_len, + attn_alpha=None, + use_group_norm=use_group_norm, + recompute_normed_x=recompute_normed_x_in_backward, + recompute_uvqk=recompute_uvqk_in_backward, + recompute_y=recompute_y_in_backward, + sort_by_length=True, + contextual_seq_len=0, + ), + is_inference=False, + ) + for _ in range(num_layers) + ] + stu = STUStack( + stu_list=stu_layers, + is_inference=False, + ).to(device) + + x_lengths = torch.randint( + low=2, high=max_seq_len + 1, size=(batch_size,), device=device + ) + num_targets = torch.randint(low=2, high=10, size=(batch_size,), device=device) + x_lengths = x_lengths + num_targets + x_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum(x_lengths) + total_seq_len = int(x_offsets[-1].cpu()) + + swap_from = torch.remainder( + torch.randint(20, (batch_size,), device=device), num_targets + ) + swap_to = torch.remainder( + torch.randint(20, (batch_size,), device=device), num_targets + ) + swap_from = x_lengths - 1 - swap_from + swap_to = x_lengths - 1 - swap_to + max_seq_len = int(x_lengths.max().item()) + + # forward() + x = torch.randn( + int(total_seq_len), + embedding_dim, + device=device, + dtype=dtype, + ).requires_grad_(True) + stu_output = stu( + x=x, + x_lengths=x_lengths, + x_offsets=x_offsets, + max_seq_len=max_seq_len, + num_targets=num_targets, + ) + stu_output_dense = torch.ops.fbgemm.jagged_to_padded_dense( + values=stu_output, + offsets=[x_offsets], + max_lengths=[max_seq_len], + padding_value=0.0, + ) + + # swapped forward(). + dense_x = torch.ops.fbgemm.jagged_to_padded_dense( + x.detach(), + [x_offsets], + [max_seq_len], + ) + swapped_dense_x = _inplace_swap(batch_size, dense_x, swap_from, swap_to) + swapped_x = torch.ops.fbgemm.dense_to_jagged( + swapped_dense_x, + [x_offsets], + )[0].requires_grad_(True) + swapped_stu_output = stu( + x=swapped_x, + x_lengths=x_lengths, + x_offsets=x_offsets, + max_seq_len=max_seq_len, + num_targets=num_targets, + ) + swapped_stu_output_dense = torch.ops.fbgemm.jagged_to_padded_dense( + values=swapped_stu_output, + offsets=[x_offsets], + max_lengths=[max_seq_len], + padding_value=0.0, + ) + + # backward + dout = torch.randn_like(stu_output_dense) + stu_output_dense.backward(dout) + dout = dout.detach().clone() + swapped_stu_output_dense.backward( + _inplace_swap(batch_size, dout, swap_from, swap_to) + ) + + swapped_swapped_stu_output_dense = _inplace_swap( + batch_size, swapped_stu_output_dense, swap_from, swap_to + ) + torch.testing.assert_close(stu_output_dense, swapped_swapped_stu_output_dense) + + # backward + torch.testing.assert_close( + torch.ops.fbgemm.jagged_to_padded_dense( + swapped_x.grad, + [x_offsets], + [max_seq_len], + ), + _inplace_swap( + batch_size, + torch.ops.fbgemm.jagged_to_padded_dense( + x.grad, + [x_offsets], + [max_seq_len], + ), + swap_from, + swap_to, + ), + ) + + # pyre-ignore[56] + @given( + num_layers=st.sampled_from([1, 2, 4]), + num_heads=st.sampled_from([1, 4]), + max_uih_len=st.sampled_from([20, 128]), + batch_size=st.sampled_from([4, 8]), + embedding_dim=st.sampled_from([32]), + attention_dim=st.sampled_from([16]), + linear_hidden_dim=st.sampled_from([64]), + contextual_seq_len=st.sampled_from([0, 10]), + ) + @settings(verbosity=Verbosity.verbose, max_examples=20, deadline=None) + @unittest.skipIf(*gpu_unavailable) + @torch.inference_mode() + def test_cached_forward( + self, + num_layers: int, + num_heads: int, + max_uih_len: int, + batch_size: int, + embedding_dim: int, + attention_dim: int, + linear_hidden_dim: int, + contextual_seq_len: int, + ) -> None: + set_dev_mode(True) + device = torch.device("cuda") + + torch.backends.cudnn.allow_tf32 = True + torch.backends.cuda.matmul.allow_tf32 = True + + use_group_norm = False + recompute_normed_x_in_backward = False + recompute_uvqk_in_backward = False + recompute_y_in_backward = False + max_attn_len = None + stu_layers: List[STU] = [ + STULayer( + config=STULayerConfig( + embedding_dim=embedding_dim, + num_heads=num_heads, + hidden_dim=linear_hidden_dim, + attention_dim=attention_dim, + output_dropout_ratio=0.0, + causal=True, + target_aware=True, + max_attn_len=max_attn_len, + attn_alpha=None, + use_group_norm=use_group_norm, + recompute_normed_x=recompute_normed_x_in_backward, + recompute_uvqk=recompute_uvqk_in_backward, + recompute_y=recompute_y_in_backward, + sort_by_length=True, + contextual_seq_len=contextual_seq_len, + ), + is_inference=True, + ) + for _ in range(num_layers) + ] + stu = STUStack( + stu_list=stu_layers, + is_inference=True, + ).to(device) + stu.recursive_setattr("_hammer_kernel", HammerKernel.TRITON) + stu.eval() + + x_lengths = torch.randint( + max_uih_len, max_uih_len + 1, (batch_size,), device=device + ) + x_lengths = x_lengths + contextual_seq_len + max_seq_len = max_uih_len + contextual_seq_len + delta_size = 20 + max_targets = delta_size * 2 + num_targets = torch.randint( + delta_size, max_targets + 1, size=(batch_size,), device=device + ) + x_lengths = x_lengths + num_targets + contextual_seq_len + max_seq_len = max_seq_len + max_targets + contextual_seq_len + x_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum(x_lengths) + total_seq_len = int(x_offsets[-1].cpu().item()) + x = torch.randn( + int(total_seq_len), + embedding_dim, + device=device, + ).requires_grad_(True) + + # default forward(). + ref_y = stu( + x=x, + x_lengths=x_lengths, + x_offsets=x_offsets, + max_seq_len=max_seq_len, + num_targets=num_targets, + ) + prime_lengths = x_lengths - delta_size + prime_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum(prime_lengths) + _, ref_delta_y = split_2D_jagged( + max_seq_len=max_seq_len, + values=ref_y, + max_len_left=None, + max_len_right=delta_size, + offsets_left=prime_offsets, + offsets_right=None, + kernel=HammerKernel.TRITON, + ) + + # cached forward(). + prime_x, delta_x = split_2D_jagged( + max_seq_len=max_seq_len, + values=x, + max_len_left=None, + max_len_right=delta_size, + offsets_left=prime_offsets, + offsets_right=None, + kernel=HammerKernel.TRITON, + ) + _ = stu( + x=prime_x, + x_lengths=prime_lengths, + x_offsets=prime_offsets, + max_seq_len=max_seq_len, + num_targets=num_targets - delta_size, + max_kv_caching_len=max_seq_len - delta_size, + kv_caching_lengths=x_lengths - delta_size, + ) + delta_y = stu.cached_forward( + delta_x=delta_x, + num_targets=num_targets, + ) + + torch.testing.assert_close(ref_delta_y, delta_y) diff --git a/recommendation_v4/generative_recommenders/ops/benchmarks/addmm_bench.py b/recommendation_v4/generative_recommenders/ops/benchmarks/addmm_bench.py new file mode 100644 index 000000000..b1be3a803 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/benchmarks/addmm_bench.py @@ -0,0 +1,174 @@ +# pyre-unsafe +import time +from typing import List, Optional, Tuple + +import click +import pandas as pd +import torch + +# @manual=//triton:triton +import triton +from generative_recommenders.common import HammerKernel +from generative_recommenders.ops.triton.triton_addmm import ( + triton_addmm_fwd, + triton_addmm_fwd_tma_persistent, + triton_addmm_fwd_tma_ws_persistent_tlx, + triton_addmm_fwd_tma_ws_tlx, +) +from generative_recommenders.ops.utils import is_sm100 + +try: + # @manual=//triton:triton + import triton.language.extra.tlx as tlx # type: ignore + + HAS_TLX = True +except ImportError: + tlx = None + HAS_TLX = False + + +def get_kernel(provider: str) -> HammerKernel: + if provider == "triton": + return HammerKernel.TRITON + elif provider == "pytorch": + return HammerKernel.PYTORCH + else: + raise ValueError(f"Unknown provider {provider}") + + +def get_dtype(dtype: str) -> torch.dtype: + if dtype == "bfloat16": + return torch.bfloat16 + elif dtype == "float32": + return torch.float32 + elif dtype == "float16": + return torch.float16 + else: + raise ValueError(f"Not supported dtype {dtype}") + + +@click.command() +@click.option("--m", type=int, default=0) +@click.option("--k", type=int, default=4096) +@click.option("--n", type=int, default=4096) +@click.option("--dtype", type=str, default="bfloat16") +@click.option("--return-result", type=bool, default=False) +@click.option("--broadcast-y", type=bool, is_flag=True, default=False) +def main( + m: int, + k: int, + n: int, + dtype: str, + return_result: bool, + broadcast_y: bool, +) -> Optional[Tuple[List[triton.testing.Benchmark], List[pd.DataFrame]]]: + if m == 0: + batch_sizes = [64, 128, 256, 512] + else: + batch_sizes = [m] + line_vals = [ + "pytorch", + "triton", + "triton_tma_persistent", + "triton_tma_persistent_ws", + ] + line_names = [ + "PyTorch", + "Triton", + "Triton TMA Persistent", + "Triton TMA Persistent WS", + ] + styles = [ + ("red", "-"), + ("green", "-"), + ("orange", "-"), + ("purple", "-"), + ] + if is_sm100() and HAS_TLX: # tmem is only supported on Blackwell + line_vals.append("triton_tma_ws_tlx") + line_names.append("Triton TMA WS TLX") + styles.append(("cyan", "-")) + line_vals.append("triton_tma_persistent_ws_tlx") + line_names.append("Triton TMA Persistent WS TLX") + styles.append(("magenta", "-")) + configs: List[triton.testing.Benchmark] = [ + triton.testing.Benchmark( + x_names=["batch_size"], + x_vals=batch_sizes, + line_arg="provider", + line_vals=line_vals, + line_names=line_names, + styles=styles, + ylabel="ms", + plot_name=f"addmm-K-{k}-N-{n}-mode-{mode}-dtype-{dtype}-broadcast_y-{broadcast_y}", + args={ + "K": k, + "N": n, + "dtype": dtype, + "broadcast_y": broadcast_y, + }, + ) + for mode in ["fwd"] + ] + + @triton.testing.perf_report(configs) + def bench_addmm( + batch_size: int, + K: int, + N: int, + dtype: str, + provider: str, + broadcast_y: bool, + ) -> float: + warmup = 20 + rep = 2000 + x = torch.randn( + (batch_size, K), dtype=get_dtype(dtype), device=torch.device("cuda") + ).requires_grad_(True) + weight = torch.randn( + (N, K), dtype=get_dtype(dtype), device=torch.device("cuda") + ).requires_grad_(True) + if broadcast_y: + y = torch.randn( + (N), dtype=get_dtype(dtype), device=torch.device("cuda") + ).requires_grad_(True) + else: + y = torch.randn( + (batch_size, N), dtype=get_dtype(dtype), device=torch.device("cuda") + ).requires_grad_(True) + + # Make sure tensors are contiguous for TMA kernels + weight_t_contiguous = weight.T.contiguous() + + if provider == "pytorch": + fn = lambda: torch.addmm(y, x, weight.T) # noqa E731 + elif provider == "triton_tma_persistent": + fn = lambda: triton_addmm_fwd_tma_persistent( + x, weight_t_contiguous, y, warp_specialize=False + ) # noqa E731 + elif provider == "triton_tma_persistent_ws": + fn = lambda: triton_addmm_fwd_tma_persistent( + x, weight_t_contiguous, y, warp_specialize=True + ) # noqa E731 + elif provider == "triton_tma_persistent_ws_tlx": + fn = lambda: triton_addmm_fwd_tma_ws_persistent_tlx( + x, weight_t_contiguous, y + ) # noqa E731 + elif provider == "triton_tma_ws_tlx": + fn = lambda: triton_addmm_fwd_tma_ws_tlx(x, weight_t_contiguous, y) # noqa E731 + elif provider == "triton": + fn = lambda: triton_addmm_fwd(x, weight_t_contiguous, y) # noqa E731 + else: + raise ValueError(f"Unknown provider: {provider}") + time.sleep(2) + ms = triton.testing.do_bench(fn, warmup=warmup, rep=rep) + return ms + + df = bench_addmm.run(print_data=True, return_df=return_result) + + if return_result: + return configs, df + + +if __name__ == "__main__": + main() diff --git a/recommendation_v4/generative_recommenders/ops/benchmarks/hstu_attention_bench.py b/recommendation_v4/generative_recommenders/ops/benchmarks/hstu_attention_bench.py new file mode 100644 index 000000000..cc7fbede7 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/benchmarks/hstu_attention_bench.py @@ -0,0 +1,406 @@ +# pyre-strict +import os +from typing import List, Optional, Tuple + +import click +import pandas as pd +import torch + +# @manual=//triton:triton +import triton +from generative_recommenders.common import ( + apply_sampling, + blackwell_tlx_unavailable, + generate_sparse_seq_len, + HammerKernel, +) +from generative_recommenders.ops.cpp.cuda_hstu_attention import cuda_hstu_mha +from generative_recommenders.ops.hstu_attention import delta_hstu_mha, hstu_mha + +try: + from hammer.ops.ragged_hstu_attention import ragged_hstu_mha + from hammer.utils import HammerKernel as HammerKernel2 +except ImportError: + pass + + +def _get_kernel(provider: str) -> HammerKernel: + if provider == "triton": + return HammerKernel.TRITON + elif provider == "tlx": + return HammerKernel.TLX + elif provider == "pytorch": + return HammerKernel.PYTORCH + else: + raise ValueError(f"Unknown provider {provider}") + + +def _flops( + batch_size: int, + max_seqlen: int, + attn_dim: int, + hidden_dim: int, + nheads: int, + seq_offsets: torch.Tensor, + mode: str = "fwd", +) -> float: + assert mode in ["fwd", "bwd", "fwd_bwd"] + ratio = 2.0 # triangular masking + f1 = 0.0 + f2 = 0.0 + for i in range(batch_size): + seq_len = int((seq_offsets[i + 1] - seq_offsets[i]).item()) + # (QK^T), dQ = d(QK^T)K, dK^T = Q^Td(QK^T) + f1 += 2 * nheads * attn_dim * seq_len**2 // ratio + # (QK^T)V, d(QK^T) = dOV^T, dV = (QK^T)^TdO, + f2 += 2 * nheads * hidden_dim * seq_len**2 // ratio + if mode == "fwd": + return f1 + f2 # computes (QK^T) and (QK^T)V + elif mode == "bwd": + return 3 * f1 + 2 * f2 # computes (QK^T), dQ, dK, dV, d(QK^T) + else: + return 4 * f1 + 3 * f2 + + +@click.command() +@click.option( + "--batch-size", + type=int, + default=512, +) +@click.option("--heads", type=int, default=4) +@click.option("--attn-dim", type=int, default=128) +@click.option("--hidden-dim", type=int, default=128) +@click.option("--max-seq-len-log2", type=int, default=13) +@click.option("--data-type", type=str, default="bf16") +@click.option("--seq-sparsity", type=float, default=0.95) +@click.option("--has-delta-q", type=bool, default=False) +@click.option("--delta-size", type=int, default=256) +@click.option("--target-size", type=int, default=20) +@click.option("--bench-backward", type=bool, default=True) +@click.option("--bench-forward", type=bool, default=True) +@click.option("--bench-tlx", type=bool, default=False) +@click.option("--bench-pytorch", type=bool, default=False) +@click.option("--bench-ragged", type=bool, default=True) +@click.option("--report-flops", type=bool, default=False) +@click.option("--return-result", type=bool, default=False) +@click.option("--max-attn-len", type=int, default=0) +@click.option("--min-full-attn-seq-len", type=int, default=0) +@click.option("--contextual-seq-len", type=int, default=0) +@click.option("--sampling-alpha", type=float, default=2.0) +@click.option("--triton-enable-tma", type=bool, default=False) +@click.option("--dynamic-attn-scale", type=bool, default=False) +@click.option("--num-softmax-heads", type=int, default=0) +def main( # noqa: C901 + batch_size: int, + heads: int, + attn_dim: int, + hidden_dim: int, + max_seq_len_log2: int, + data_type: str, + seq_sparsity: float, + has_delta_q: bool, + delta_size: int, + target_size: int, + bench_backward: bool, + bench_forward: bool, + bench_tlx: bool, + bench_pytorch: bool, + bench_ragged: bool, + report_flops: bool, + return_result: bool, + max_attn_len: int, + min_full_attn_seq_len: int, + contextual_seq_len: int, + sampling_alpha: float, + triton_enable_tma: bool, + dynamic_attn_scale: bool, + num_softmax_heads: int, +) -> Optional[Tuple[List[triton.testing.Benchmark], List[pd.DataFrame]]]: + torch.backends.cudnn.allow_tf32 = True + torch.backends.cuda.matmul.allow_tf32 = True + if data_type == "fp32": + dtype = torch.float32 + elif data_type == "fp16": + dtype = torch.float16 + elif data_type == "bf16": + dtype = torch.bfloat16 + else: + raise ValueError(f"Unsupported data type: {data_type}.") + + line_vals = ["triton", "flash_cuda_jagged"] + line_names = ["triton", "flash_cuda_jagged"] + styles = [("blue", "-"), ("green", "-")] + if bench_pytorch: + line_vals.append("pytorch") + line_names.append("PyTorch") + styles.append(("green", "-")) + if bench_ragged: + line_vals.append("ragged") + line_names.append("ragged") + styles.append(("red", "-")) + if bench_tlx and not blackwell_tlx_unavailable[0]: + line_vals.append("tlx") + line_names.append("tlx") + styles.append(("cyan", "-")) + + bench_backward = False if has_delta_q else bench_backward + modes = [] + if bench_forward: + modes.append("fwd") + if bench_backward: + modes.append("bwd") + assert len(modes) > 0 + + configs: List[triton.testing.Benchmark] = [ + triton.testing.Benchmark( + x_names=["seq_len"], + x_vals=[2**i for i in range(8, max_seq_len_log2)], + line_arg="provider", + line_vals=line_vals, + line_names=line_names, + styles=styles, + ylabel="ms", + plot_name=f"hstu-attn-b{batch_size}-h{heads}-d{attn_dim}-v{hidden_dim}--sparsity{seq_sparsity}-{mode}-{dtype}-target{target_size}-mattn{max_attn_len}-full{min_full_attn_seq_len}-c{contextual_seq_len}-sl_alpha{sampling_alpha}-triton_tma{triton_enable_tma}-dynamic_scale{dynamic_attn_scale}-num_softmax_heads{num_softmax_heads}", + args={ + "batch_size": batch_size, + "heads": heads, + "attn_dim": attn_dim, + "hidden_dim": hidden_dim, + "dtype": dtype, + "mode": mode, + "seq_sparsity": seq_sparsity, + "has_delta_q": has_delta_q, + "delta_size": delta_size, + "target_size": target_size, + "bench_backward": bench_backward, + "report_flops": report_flops, + "max_attn_len": max_attn_len, + "min_full_attn_seq_len": min_full_attn_seq_len, + "contextual_seq_len": contextual_seq_len, + "sampling_alpha": sampling_alpha, + "triton_enable_tma": triton_enable_tma, + "dynamic_attn_scale": dynamic_attn_scale, + "num_softmax_heads": num_softmax_heads, + }, + ) + for mode in modes + ] + + @triton.testing.perf_report(configs) + def _bench_hstu_attention( + batch_size: int, + heads: int, + seq_len: int, + attn_dim: int, + hidden_dim: int, + mode: str, + provider: str, + dtype: torch.dtype, + seq_sparsity: float, + has_delta_q: bool, + delta_size: int, + target_size: int, + bench_backward: bool, + report_flops: bool, + max_attn_len: int, + min_full_attn_seq_len: int, + contextual_seq_len: int, + sampling_alpha: float, + triton_enable_tma: bool, + dynamic_attn_scale: bool, + num_softmax_heads: int, + ) -> float: + assert mode in ["fwd", "bwd"] + warmup = 25 + rep = 1000 + torch.manual_seed(1001) # for reproducibility + alpha = 1.0 / attn_dim + causal = True + lengths = generate_sparse_seq_len( + size=batch_size, + max_seq_len=seq_len, + sparsity=seq_sparsity, + device=torch.device("cuda"), + ) + lengths = apply_sampling(lengths, sampling_alpha, max_seq_len=seq_len) + if has_delta_q: + lengths = lengths + delta_size + num_targets = torch.ones_like(lengths) * delta_size + seq_len = seq_len + delta_size + else: + delta_size = 0 + num_targets = None + if target_size != 0: + num_targets = torch.randint( + 1, + target_size + 1, + (batch_size,), + device=lengths.device, + dtype=lengths.dtype, + ) + num_targets = torch.where( + num_targets > lengths, lengths, num_targets + ).to(torch.int32) + max_attn_len = max_attn_len if max_attn_len < seq_len else seq_len + seq_offsets = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + seq_offsets[1:] = torch.cumsum(lengths, dim=0) + L = int(seq_offsets[-1].item()) + x = torch.empty( + (L, heads, attn_dim * 2 + hidden_dim), + dtype=dtype, + device=torch.device("cuda"), + ).uniform_(-0.01, 0.01) + q, k, v = torch.split(x, [attn_dim, attn_dim, hidden_dim], dim=-1) + delta_q = torch.empty( + (batch_size * delta_size, heads, attn_dim), + dtype=dtype, + device=torch.device("cuda"), + ).uniform_(-0.1, 0.1) + delta_x_offsets = torch.arange(0, delta_size, device=torch.device("cuda")) + delta_x_offsets = (seq_offsets[1:] - delta_size).view( + batch_size, 1 + ) + delta_x_offsets.view(1, delta_size) + delta_x_offsets = delta_x_offsets.view(-1) + attn_scale = torch.empty( + (L,), + dtype=torch.float32, + device=torch.device("cuda"), + ).uniform_(0.5, 1.0) + + if bench_backward: + q = q.requires_grad_(True) + k = k.requires_grad_(True) + v = v.requires_grad_(True) + assert provider in [ + "triton", + "pytorch", + "flash_cuda_jagged", + "flash_cuda", + "tlx", + "ragged", + ] + if has_delta_q: + fn = lambda: delta_hstu_mha( # noqa E731 + max_seq_len=seq_len, + alpha=alpha, + delta_q=delta_q, + k=k, + v=v, + seq_offsets=seq_offsets, + num_targets=num_targets, + kernel=_get_kernel(provider), + ) + else: + if provider == "flash_cuda_jagged": + fn = lambda: cuda_hstu_mha( # noqa E731 + q=q, + k=k, + v=v, + alpha=alpha, + causal=True, + seq_offsets=seq_offsets.to(torch.int32), + attn_scale=attn_scale if dynamic_attn_scale else None, + max_seq_len=seq_len, + max_attn_len=max_attn_len, + min_full_attn_seq_len=min_full_attn_seq_len, + contextual_seq_len=contextual_seq_len, + num_targets=num_targets, + sort_by_length=False, + num_softmax_heads=num_softmax_heads, + ) + elif provider == "flash_cuda": + q, k, v = [ + torch.randn( + batch_size, + seq_len, + heads, + attn_dim, + device="cuda", + dtype=dtype, + requires_grad=True, + ) + for _ in range(3) + ] + fn = lambda: cuda_hstu_mha( # noqa E731 + q=q, + k=k, + v=v, + alpha=alpha, + causal=True, + max_seq_len=seq_len, + max_attn_len=max_attn_len, + min_full_attn_seq_len=min_full_attn_seq_len, + contextual_seq_len=contextual_seq_len, + num_targets=num_targets, + sort_by_length=False, + num_softmax_heads=num_softmax_heads, + ) + elif provider == "ragged": + fn = lambda: ragged_hstu_mha( # noqa E731 + max_seq_len=seq_len, + alpha=alpha, + q=q, + k=k, + v=v, + seq_offsets=seq_offsets, + dropout_pr=0.0, + training=True, + invalid_attn_mask_type="lower_triangular", + num_targets=num_targets, + attn_scale=attn_scale if dynamic_attn_scale else None, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + full_attn_size=min_full_attn_seq_len, + sort_by_length=True, + kernel=HammerKernel2.TRITON, + num_softmax_heads=num_softmax_heads, + ) + else: + fn = lambda: hstu_mha( # noqa E731 + max_seq_len=seq_len, + alpha=alpha, + q=q, + k=k, + v=v, + seq_offsets=seq_offsets, + causal=causal, + dropout_pr=0.0, + training=True, + num_targets=num_targets, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + sort_by_length=True, + kernel=_get_kernel(provider), + enable_tma=triton_enable_tma, + ) + if mode == "bwd": + o = fn() + do = torch.randn_like(o) + fn = lambda: o.backward(do, retain_graph=True) # noqa E731 + ms = triton.testing.do_bench(fn, warmup=warmup, rep=rep) + all_flops = _flops( + batch_size, seq_len, attn_dim, hidden_dim, heads, seq_offsets, mode + ) + if has_delta_q: + all_flops = all_flops / seq_len * delta_size + if report_flops: + return all_flops / ms / 1e9 + else: + return ms + + df = _bench_hstu_attention.run( + print_data=True, + show_plots=False, + save_path="/tmp/" + os.environ["USER"], + return_df=return_result, + ) + + if return_result: + return configs, df + + +if __name__ == "__main__": + main() diff --git a/recommendation_v4/generative_recommenders/ops/benchmarks/jagged_dense_bmm_bench.py b/recommendation_v4/generative_recommenders/ops/benchmarks/jagged_dense_bmm_bench.py new file mode 100644 index 000000000..dcfb9819e --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/benchmarks/jagged_dense_bmm_bench.py @@ -0,0 +1,199 @@ +# pyre-strict +import math +from typing import List, Optional, Tuple + +import click +import pandas as pd +import torch + +# @manual=//triton:triton +import triton +from generative_recommenders.common import HammerKernel +from generative_recommenders.ops.triton.triton_jagged import triton_jagged_dense_bmm + +# buck2 run @mode/{opt,inplace} //generative_recommenders/ops/benchmarks:jagged_dense_bmm_bench -- --fwd-only + + +def jagged_dense_bmm( + max_seq_len: int, + seq_offsets: torch.Tensor, + jagged: torch.Tensor, + dense: torch.Tensor, + kernel: HammerKernel = HammerKernel.PYTORCH, +) -> torch.Tensor: + """ + Wrapper function for jagged_dense_bmm with kernel selection. + Computing out = jagged x dense + jagged has shape (sum_B(M_i), K), dense has shape (B, K, N) + out has shape (sum_B(M_i), N) + """ + if kernel == HammerKernel.TRITON: + return triton_jagged_dense_bmm( + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + jagged=jagged, + dense=dense, + ) + elif kernel == HammerKernel.PYTORCH: + # PyTorch implementation - manual implementation using standard operations + B, K, N = dense.shape + outputs = [] + for i in range(B): + start_idx = seq_offsets[i] + end_idx = seq_offsets[i + 1] + if start_idx < end_idx: + jagged_seq = jagged[start_idx:end_idx] # (seq_len, K) + dense_seq = dense[i] # (K, N) + output_seq = torch.mm(jagged_seq, dense_seq) # (seq_len, N) + outputs.append(output_seq) + return ( + torch.cat(outputs, dim=0) + if outputs + else torch.empty(0, N, device=jagged.device, dtype=jagged.dtype) + ) + else: + raise ValueError(f"Unsupported kernel: {kernel}") + + +@click.command() +@click.option( + "--batch-size", + type=int, + default=512, +) +@click.option( + "--max-seq-len", + type=int, + default=8192, + show_default=True, +) +@click.option( + "-d", + type=int, + default=64, + show_default=True, +) +@click.option( + "-k", + type=int, + default=64, + show_default=True, +) +@click.option("--dtype", type=str, default="bf16") +@click.option("--fwd-only", is_flag=True) +@click.option("--return-result", type=bool, default=False) +def main( + batch_size: int, + max_seq_len: int, + d: int, + k: int, + dtype: str, + fwd_only: bool, + return_result: bool, +) -> Optional[Tuple[List[triton.testing.Benchmark], List[pd.DataFrame]]]: + torch.backends.cudnn.allow_tf32 = True + torch.backends.cuda.matmul.allow_tf32 = True + max_seq_len_log2 = int(round(math.log2(max_seq_len))) + if dtype == "fp32": + pt_dtype = torch.float32 + elif dtype == "fp16": + pt_dtype = torch.float16 + elif dtype == "bf16": + pt_dtype = torch.bfloat16 + else: + raise ValueError(f"Unsupported data type: {dtype}.") + + configs: List[triton.testing.Benchmark] = [ + triton.testing.Benchmark( + x_names=["seq_len"], + x_vals=[2**i for i in range(5, max_seq_len_log2 + 1)], + line_arg="provider", + line_vals=["triton", "pytorch"], + line_names=["Triton", "Pytorch"], + styles=[("red", "-"), ("blue", "-")], + ylabel="ms", + plot_name=f"jagged_dense_bmm-b{batch_size}-D{d}-K{k}-{dtype}", + args={ + "batch_size": batch_size, + "D": d, + "K": k, + "dtype": pt_dtype, + "mode": mode, + }, + ) + for mode in (["fwd"] if fwd_only else ["fwd", "bwd"]) + ] + + @triton.testing.perf_report(configs) + def bench_jagged_dense_bmm( + batch_size: int, + seq_len: int, + D: int, + K: int, + mode: str, + provider: str, + dtype: torch.dtype, + ) -> float: + assert mode in ["fwd", "bwd"] + warmup = 25 + rep = 100 + + max_seq_len = seq_len + lengths = torch.randint( + max_seq_len + 1, size=(batch_size,), device=torch.device("cuda") + ) + seq_offsets = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + seq_offsets[1:] = torch.cumsum(lengths, dim=0) + jagged_size = int(seq_offsets[-1].item()) + jagged = ( + torch.empty((jagged_size, D), dtype=dtype, device=torch.device("cuda")) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + dense = ( + torch.empty((batch_size, D, K), dtype=dtype, device=torch.device("cuda")) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + + if provider == "triton": + fn = lambda: jagged_dense_bmm( # noqa E731 + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + jagged=jagged, + dense=dense, + kernel=HammerKernel.TRITON, + ) + if mode == "bwd": + o = fn() + do = torch.randn_like(o) + fn = lambda: o.backward(do, retain_graph=True) # noqa E731 + ms = triton.testing.do_bench(fn, warmup=warmup, rep=rep) + return ms + elif provider == "pytorch": + fn = lambda: jagged_dense_bmm( # noqa E731 + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + jagged=jagged, + dense=dense, + kernel=HammerKernel.PYTORCH, + ) + if mode == "bwd": + o = fn() + do = torch.randn_like(o) + fn = lambda: o.backward(do, retain_graph=True) # noqa E731 + ms = triton.testing.do_bench(fn, warmup=warmup, rep=rep) + return ms + else: + raise ValueError(f"unsupported provider: {provider}") + + df = bench_jagged_dense_bmm.run(print_data=True, return_df=return_result) + + if return_result: + return configs, df + + +if __name__ == "__main__": + main() diff --git a/recommendation_v4/generative_recommenders/ops/benchmarks/jagged_dense_bmm_broadcast_add_bench.py b/recommendation_v4/generative_recommenders/ops/benchmarks/jagged_dense_bmm_broadcast_add_bench.py new file mode 100644 index 000000000..193704bf9 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/benchmarks/jagged_dense_bmm_broadcast_add_bench.py @@ -0,0 +1,270 @@ +# pyre-strict +import math +import pickle +from typing import List + +import click +import torch + +# @manual=//triton:triton +import triton +from generative_recommenders.common import HammerKernel +from generative_recommenders.ops.jagged_tensors import jagged_dense_bmm_broadcast_add +from generative_recommenders.ops.triton.triton_jagged import ( + jagged_dense_bmm_broadcast_add_kernel, + triton_jagged_dense_bmm, + triton_jagged_dense_broadcast_add, +) + +# buck2 run @mode/{opt,inplace} //generative_recommenders/ops/benchmarks:jagged_dense_bmm_broadcast_add_bench -- --fwd-only + +# To dump the jagged_dense_bmm_broadcast_add_kernel cache +# buck2 run @mode/opt //generative_recommenders/ops/benchmarks:jagged_dense_bmm_broadcast_add_bench -- --fwd-only --dump-cache-dir=/home/${USER}/fbsource/fbcode/generative_recommenders/ops/triton/jagged_dense_bmm_broadcast_add_kernel_cache.pkl + + +def get_kernel(provider: str) -> HammerKernel: + if provider == "triton": + return HammerKernel.TRITON + elif provider == "pytorch": + return HammerKernel.PYTORCH + else: + raise ValueError(f"Unknown provider {provider}") + + +def jagged_dense_broadcast_add( + max_seq_len: int, + seq_offsets: torch.Tensor, + jagged: torch.Tensor, + dense: torch.Tensor, + kernel: HammerKernel = HammerKernel.PYTORCH, +) -> torch.Tensor: + """ + Wrapper function for jagged_dense_broadcast_add with kernel selection. + Computing out = jagged + dense (broadcasted) + jagged has shape (sum_B(M_i), N), dense has shape (B, N) + out has shape (sum_B(M_i), N) + """ + if kernel == HammerKernel.TRITON: + return triton_jagged_dense_broadcast_add( + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + jagged=jagged, + dense=dense, + ) + elif kernel == HammerKernel.PYTORCH: + # PyTorch implementation - manual implementation using standard operations + B, N = dense.shape + outputs = [] + for i in range(B): + start_idx = seq_offsets[i] + end_idx = seq_offsets[i + 1] + if start_idx < end_idx: + jagged_seq = jagged[start_idx:end_idx] # (seq_len, N) + dense_seq = dense[i] # (N,) + output_seq = jagged_seq + dense_seq # (seq_len, N) + outputs.append(output_seq) + return ( + torch.cat(outputs, dim=0) + if outputs + else torch.empty(0, N, device=jagged.device, dtype=jagged.dtype) + ) + else: + raise ValueError(f"Unsupported kernel: {kernel}") + + +def jagged_dense_bmm( + max_seq_len: int, + seq_offsets: torch.Tensor, + jagged: torch.Tensor, + dense: torch.Tensor, + kernel: HammerKernel = HammerKernel.PYTORCH, +) -> torch.Tensor: + """ + Wrapper function for jagged_dense_bmm with kernel selection. + Computing out = jagged x dense + jagged has shape (sum_B(M_i), K), dense has shape (B, K, N) + out has shape (sum_B(M_i), N) + """ + if kernel == HammerKernel.TRITON: + return triton_jagged_dense_bmm( + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + jagged=jagged, + dense=dense, + ) + elif kernel == HammerKernel.PYTORCH: + # PyTorch implementation - manual implementation using standard operations + B, K, N = dense.shape + outputs = [] + for i in range(B): + start_idx = seq_offsets[i] + end_idx = seq_offsets[i + 1] + if start_idx < end_idx: + jagged_seq = jagged[start_idx:end_idx] # (seq_len, K) + dense_seq = dense[i] # (K, N) + output_seq = torch.mm(jagged_seq, dense_seq) # (seq_len, N) + outputs.append(output_seq) + return ( + torch.cat(outputs, dim=0) + if outputs + else torch.empty(0, N, device=jagged.device, dtype=jagged.dtype) + ) + else: + raise ValueError(f"Unsupported kernel: {kernel}") + + +@click.command() +@click.option( + "--batch-size", + type=int, + default=384, +) +@click.option( + "--max-seq-len", + type=int, + default=4096, + show_default=True, +) +@click.option( + "-d", + type=int, + default=512, + show_default=True, +) +@click.option( + "-k", + type=int, + default=512, + show_default=True, +) +@click.option("--dtype", type=str, default="bf16") +@click.option("--fwd-only", is_flag=True) +@click.option("--dump-cache-dir", type=str, default="") +def main( + batch_size: int, + max_seq_len: int, + d: int, + k: int, + dtype: str, + fwd_only: bool, + dump_cache_dir: str, +) -> None: + torch.backends.cudnn.allow_tf32 = True + torch.backends.cuda.matmul.allow_tf32 = True + max_seq_len_log2 = int(round(math.log2(max_seq_len))) + if dtype == "fp32": + pt_dtype = torch.float32 + elif dtype == "fp16": + pt_dtype = torch.float16 + elif dtype == "bf16": + pt_dtype = torch.bfloat16 + else: + raise ValueError(f"Unsupported data type: {dtype}.") + + configs: List[triton.testing.Benchmark] = [ + triton.testing.Benchmark( + x_names=["seq_len"], + x_vals=[2**i for i in range(8, max_seq_len_log2 + 1)], + line_arg="provider", + line_vals=["triton", "pytorch", "triton_nonfused"], + line_names=["Triton", "Pytorch", "Triton_Nonfused"], + styles=[("red", "-"), ("blue", "-"), ("green", "-")], + ylabel="ms", + plot_name=f"jagged_dense_bmm_broadcast_add-{mode}-b{batch_size}-D{d}-K{k}-{dtype}", + args={ + "batch_size": batch_size, + "D": d, + "K": k, + "dtype": pt_dtype, + "mode": mode, + }, + ) + for mode in (["fwd"] if fwd_only else ["fwd", "bwd"]) + ] + + @triton.testing.perf_report(configs) + def bench_jagged_dense_bmm_broadcast_add( + batch_size: int, + seq_len: int, + D: int, + K: int, + mode: str, + provider: str, + dtype: torch.dtype, + ) -> float: + assert mode in ["fwd", "bwd"] + warmup = 25 + rep = 100 + + max_seq_len = seq_len + lengths = torch.randint( + max_seq_len + 1, size=(batch_size,), device=torch.device("cuda") + ) + seq_offsets = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + seq_offsets[1:] = torch.cumsum(lengths, dim=0) + jagged_size = int(seq_offsets[-1].item()) + jagged = ( + torch.empty((jagged_size, D), dtype=dtype, device=torch.device("cuda")) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + dense = ( + torch.empty((batch_size, D, K), dtype=dtype, device=torch.device("cuda")) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + bias = ( + torch.empty((batch_size, K), dtype=dtype, device=torch.device("cuda")) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + + if provider in ["triton", "pytorch"]: + fn = lambda: jagged_dense_bmm_broadcast_add( # noqa E731 + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + jagged=jagged, + dense=dense, + bias=bias, + kernel=get_kernel(provider), + ) + if mode == "bwd": + o = fn() + do = torch.randn_like(o) + fn = lambda: o.backward(do, retain_graph=True) # noqa E731 + ms = triton.testing.do_bench(fn, warmup=warmup, rep=rep) + return ms + elif provider == "triton_nonfused": + fn = lambda: jagged_dense_broadcast_add( # noqa E731 + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + jagged=jagged_dense_bmm( + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + jagged=jagged, + dense=dense, + kernel=HammerKernel.TRITON, + ), + dense=bias, + kernel=HammerKernel.TRITON, + ) + if mode == "bwd": + o = fn() + do = torch.randn_like(o) + fn = lambda: o.backward(do, retain_graph=True) # noqa E731 + ms = triton.testing.do_bench(fn, warmup=warmup, rep=rep) + return ms + else: + raise ValueError(f"unsupported provider: {provider}") + + bench_jagged_dense_bmm_broadcast_add.run(print_data=True) + if dump_cache_dir: + with open(dump_cache_dir, "wb") as data: + # @lint-ignore PYTHONPICKLEISBAD + pickle.dump(jagged_dense_bmm_broadcast_add_kernel.cache, data) + + +if __name__ == "__main__": + main() diff --git a/recommendation_v4/generative_recommenders/ops/benchmarks/jagged_dense_broadcast_add_bench.py b/recommendation_v4/generative_recommenders/ops/benchmarks/jagged_dense_broadcast_add_bench.py new file mode 100644 index 000000000..049258e3d --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/benchmarks/jagged_dense_broadcast_add_bench.py @@ -0,0 +1,205 @@ +# pyre-strict +import math +import pickle +from typing import List, Optional, Tuple + +import click +import pandas as pd +import torch + +# @manual=//triton:triton +import triton +from generative_recommenders.common import HammerKernel +from generative_recommenders.ops.triton.triton_jagged import ( + jagged_dense_broadcast_add_kernel, + triton_jagged_dense_broadcast_add, +) + +# buck2 run @mode/{opt,inplace} //generative_recommenders/ops/benchmarks:jagged_dense_broadcast_add_bench + + +# To dump the jagged_dense_broadcast_add_kernel cache, run: +# buck2 run @mode/{opt,inplace} //generative_recommenders/ops/benchmarks:jagged_dense_broadcast_add_bench -- --dump-ragged-tuner-cache-dir=/home/${USER}/fbsource/fbcode/generative_recommenders/ops/triton/jagged_dense_broadcast_add_kernel_cache.pkl + + +def jagged_dense_broadcast_add( + max_seq_len: int, + seq_offsets: torch.Tensor, + jagged: torch.Tensor, + dense: torch.Tensor, + kernel: HammerKernel = HammerKernel.PYTORCH, +) -> torch.Tensor: + """ + Wrapper function for jagged_dense_broadcast_add with kernel selection. + Computing out = jagged + dense (broadcasted) + jagged has shape (sum_B(M_i), N), dense has shape (B, N) + out has shape (sum_B(M_i), N) + """ + if kernel == HammerKernel.TRITON: + return triton_jagged_dense_broadcast_add( + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + jagged=jagged, + dense=dense, + ) + elif kernel == HammerKernel.PYTORCH: + # PyTorch implementation - manual implementation using standard operations + B, N = dense.shape + outputs = [] + for i in range(B): + start_idx = seq_offsets[i] + end_idx = seq_offsets[i + 1] + if start_idx < end_idx: + jagged_seq = jagged[start_idx:end_idx] # (seq_len, N) + dense_seq = dense[i] # (N,) + output_seq = jagged_seq + dense_seq # (seq_len, N) + outputs.append(output_seq) + return ( + torch.cat(outputs, dim=0) + if outputs + else torch.empty(0, N, device=jagged.device, dtype=jagged.dtype) + ) + else: + raise ValueError(f"Unsupported kernel: {kernel}") + + +@click.command() +@click.option( + "--batch-size", + type=int, + default=512, +) +@click.option( + "--max-seq-len", + type=int, + default=8192, + show_default=True, +) +@click.option( + "-d", + type=int, + default=64, + show_default=True, +) +@click.option("--dtype", type=str, default="fp32") +@click.option("--fwd-only", is_flag=True) +@click.option("--dump-ragged-tuner-cache-dir", type=str, default="") +@click.option("--return-result", type=bool, default=False) +def main( + batch_size: int, + max_seq_len: int, + d: int, + dtype: str, + fwd_only: bool, + dump_ragged_tuner_cache_dir: str, + return_result: bool, +) -> Optional[Tuple[List[triton.testing.Benchmark], List[pd.DataFrame]]]: + torch.backends.cudnn.allow_tf32 = True + torch.backends.cuda.matmul.allow_tf32 = True + max_seq_len_log2 = int(round(math.log2(max_seq_len))) + if dtype == "fp32": + pt_dtype = torch.float32 + elif dtype == "fp16": + pt_dtype = torch.float16 + elif dtype == "bf16": + pt_dtype = torch.bfloat16 + else: + raise ValueError(f"Unsupported data type: {dtype}.") + + configs: List[triton.testing.Benchmark] = [ + triton.testing.Benchmark( + x_names=["seq_len"], + x_vals=[2**i for i in range(5, max_seq_len_log2 + 1)], + line_arg="provider", + line_vals=["triton", "pytorch"], + line_names=["Triton", "Pytorch"], + styles=[("red", "-"), ("blue", "-")], + ylabel="ms", + plot_name=f"jagged_dense_broadcast_add-b{batch_size}-D{d}-{dtype}-{mode}", + args={ + "batch_size": batch_size, + "D": d, + "dtype": pt_dtype, + "mode": mode, + }, + ) + for mode in (["fwd"] if fwd_only else ["fwd", "fwd+bwd"]) + ] + + @triton.testing.perf_report(configs) + def bench_jagged_dense_broadcast_add( + batch_size: int, + seq_len: int, + D: int, + mode: str, + provider: str, + dtype: torch.dtype, + ) -> float: + assert mode in ["fwd", "bwd", "fwd+bwd"] + warmup = 25 + rep = 100 + + max_seq_len = seq_len + lengths = torch.randint( + max_seq_len + 1, size=(batch_size,), device=torch.device("cuda") + ) + seq_offsets = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + seq_offsets[1:] = torch.cumsum(lengths, dim=0) + jagged_size = int(seq_offsets[-1].item()) + jagged = ( + torch.empty((jagged_size, D), dtype=dtype, device=torch.device("cuda")) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + dense = ( + torch.empty((batch_size, D), dtype=dtype, device=torch.device("cuda")) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + + if provider == "triton": + fn = lambda: jagged_dense_broadcast_add( # noqa E731 + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + jagged=jagged, + dense=dense, + kernel=HammerKernel.TRITON, + ) + if mode == "bwd": + o = fn() + do = torch.randn_like(o) + fn = lambda: o.backward(do, retain_graph=True) # noqa E731 + ms = triton.testing.do_bench(fn, warmup=warmup, rep=rep) + return ms + elif provider == "pytorch": + fn = lambda: jagged_dense_broadcast_add( # noqa E731 + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + jagged=jagged, + dense=dense, + kernel=HammerKernel.PYTORCH, + ) + if mode == "bwd": + o = fn() + do = torch.randn_like(o) + fn = lambda: o.backward(do, retain_graph=True) # noqa E731 + ms = triton.testing.do_bench(fn, warmup=warmup, rep=rep) + return ms + else: + raise ValueError(f"unsupported provider: {provider}") + + df = bench_jagged_dense_broadcast_add.run(print_data=True, return_df=return_result) + + if dump_ragged_tuner_cache_dir: + with open(dump_ragged_tuner_cache_dir, "wb") as data: + # @lint-ignore PYTHONPICKLEISBAD + pickle.dump(jagged_dense_broadcast_add_kernel.cache, data) + + if return_result: + return configs, df + + +if __name__ == "__main__": + main() diff --git a/recommendation_v4/generative_recommenders/ops/cpp/benchmarks/concat_1d_jagged_jagged_bench.py b/recommendation_v4/generative_recommenders/ops/cpp/benchmarks/concat_1d_jagged_jagged_bench.py new file mode 100644 index 000000000..95c43853f --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/benchmarks/concat_1d_jagged_jagged_bench.py @@ -0,0 +1,125 @@ +# pyre-strict +from typing import List + +import click +import torch + +# @manual=//triton:triton +import triton +from hammer.ops.jagged import concat_1D_jagged_jagged + +# buck2 run @//mode/opt -c fbcode.nvcc_arch=h100 //generative_recommenders/ops/cpp/benchmarks:concat_1d_jagged_jagged_bench + +torch.ops.load_library("//generative_recommenders/ops/cpp:cpp_ops") +torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops") +torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops_cpu") + + +@click.command() +@click.option("--data-type", type=str, default="float32") +@click.option("--batch-size", type=int, default=512) +@click.option("--max-seq-len-log2", type=int, default=20) +@click.option("--seq-sparsity", type=float, default=0.8) +def main( + data_type: str, + batch_size: int, + max_seq_len_log2: int, + seq_sparsity: float, +) -> None: + if data_type == "float32": + dtype = torch.float32 + elif data_type == "float16": + dtype = torch.float16 + elif data_type == "bfloat16": + dtype = torch.bfloat16 + else: + raise ValueError(f"Unsupported data type: {data_type}.") + + configs: List[triton.testing.Benchmark] = [ + triton.testing.Benchmark( + x_names=["max_seq_len"], + x_vals=[2**i for i in range(6, max_seq_len_log2)], + line_arg="method", + line_vals=[ + "custom_cuda", + "hammer_pytorch", + ], + line_names=["Custom CUDA", "Hammer PyTorch"], + styles=[("green", "-"), ("orange", "--")], + ylabel="ms", + plot_name=f"concat_1d_jagged_jagged_batch{batch_size}_sparsity{seq_sparsity}_{data_type}", + args={ + "dtype": dtype, + "batch_size": batch_size, + "seq_sparsity": seq_sparsity, + }, + ) + ] + + @triton.testing.perf_report(configs) + def bench_concat_1d_jagged_jagged( + max_seq_len: int, + batch_size: int, + method: str, + dtype: torch.dtype, + seq_sparsity: float, + ) -> float: + warmup = 50 + rep = 500 + torch.manual_seed(1001) + + lengths_left = torch.randint( + 1, int(max_seq_len * seq_sparsity) + 1, (batch_size,), dtype=torch.int32 + ) + lengths_right = torch.randint( + 1, int(max_seq_len * seq_sparsity) + 1, (batch_size,), dtype=torch.int32 + ) + + total_left = int(lengths_left.sum().item()) + total_right = int(lengths_right.sum().item()) + + values_left = torch.randn(total_left, dtype=dtype) + values_right = torch.randn(total_right, dtype=dtype) + + offsets_left = torch.zeros( + (batch_size + 1,), dtype=lengths_left.dtype, device=lengths_left.device + ) + offsets_left[1:] = torch.cumsum(lengths_left.view(-1), dim=0) + offsets_right = torch.zeros( + (batch_size + 1,), dtype=lengths_right.dtype, device=lengths_right.device + ) + offsets_right[1:] = torch.cumsum(lengths_right.view(-1), dim=0) + max_seq_len_left = int(lengths_left.max().item()) + max_seq_len_right = int(lengths_right.max().item()) + + lengths_left = lengths_left.cuda() + lengths_right = lengths_right.cuda() + values_left = values_left.cuda() + values_right = values_right.cuda() + offsets_left = offsets_left.cuda() + offsets_right = offsets_right.cuda() + + if method == "custom_cuda": + fn = lambda: torch.ops.hstu.concat_1d_jagged_jagged( # noqa E731 + lengths_left, values_left, lengths_right, values_right + ) + elif method == "hammer_pytorch": + fn = lambda: concat_1D_jagged_jagged( # noqa E731 + max_seq_len_left, + offsets_left, + values_left, + max_seq_len_right, + offsets_right, + values_right, + ) + else: + raise ValueError(f"unknown method: {method}") + + ms = triton.testing.do_bench(fn, warmup=warmup, rep=rep) + return ms + + bench_concat_1d_jagged_jagged.run(print_data=True) + + +if __name__ == "__main__": + main() diff --git a/recommendation_v4/generative_recommenders/ops/cpp/benchmarks/jagged_transpose_1d_bench.py b/recommendation_v4/generative_recommenders/ops/cpp/benchmarks/jagged_transpose_1d_bench.py new file mode 100644 index 000000000..7806d6970 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/benchmarks/jagged_transpose_1d_bench.py @@ -0,0 +1,117 @@ +# pyre-strict +from typing import List + +import click +import torch + +# @manual=//triton:triton +import triton +from hammer.ops.jagged import jagged_transpose_1D + +# buck2 run @//mode/opt -c fbcode.nvcc_arch=h100 //generative_recommenders/ops/cpp/benchmarks:jagged_transpose_1d_bench + +torch.ops.load_library("//generative_recommenders/ops/cpp:cpp_ops") +torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops") +torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops_cpu") + + +@click.command() +@click.option("--data-type", type=str, default="float32") +@click.option("--size1", type=int, default=32) +@click.option("--size2", type=int, default=16) +@click.option("--max-len-log2", type=int, default=19) +@click.option("--seq-sparsity", type=float, default=0.8) +def main( + data_type: str, + size1: int, + size2: int, + max_len_log2: int, + seq_sparsity: float, +) -> None: + if data_type == "float32": + dtype = torch.float32 + elif data_type == "float16": + dtype = torch.float16 + elif data_type == "bfloat16": + dtype = torch.bfloat16 + else: + raise ValueError(f"Unsupported data type: {data_type}.") + + configs: List[triton.testing.Benchmark] = [ + triton.testing.Benchmark( + x_names=["max_len"], + x_vals=[2**i for i in range(4, max_len_log2)], + line_arg="method", + line_vals=[ + "custom_cuda", + "hammer_pytorch", + ], + line_names=["Custom CUDA", "Hammer PyTorch"], + styles=[("green", "-"), ("orange", "--")], + ylabel="ms", + plot_name=f"jagged_transpose_1d_size1_{size1}_size2_{size2}_sparsity{seq_sparsity}_{data_type}", + args={ + "dtype": dtype, + "size1": size1, + "size2": size2, + "seq_sparsity": seq_sparsity, + }, + ) + ] + + @triton.testing.perf_report(configs) + def bench_jagged_transpose_1d( + max_len: int, + size1: int, + size2: int, + method: str, + dtype: torch.dtype, + seq_sparsity: float, + ) -> float: + warmup = 50 + rep = 500 + torch.manual_seed(1001) + + lengths = torch.randint( + 1, int(max_len * seq_sparsity) + 1, (size1 * size2,), dtype=torch.int32 + ) + offsets = torch.zeros( + (size1 * size2 + 1,), dtype=lengths.dtype, device=lengths.device + ) + offsets[1:] = torch.cumsum(lengths.view(-1), dim=0) + + values = torch.randn(int(offsets[-1].item()), dtype=dtype) + + lengths = lengths.cuda() + offsets = offsets.cuda() + values = values.cuda() + + if method == "custom_cuda": + fn = lambda: torch.ops.hstu.jagged_transpose_1d( # noqa E731 + values=values, + offsets=offsets, + lengths=lengths, + max_len=max_len, + size1=size1, + size2=size2, + ) + elif method == "hammer_pytorch": + fn = lambda: jagged_transpose_1D( # noqa E731 + values=values, + offsets=offsets, + lengths=lengths, + max_len=max_len, + size1=size1, + size2=size2, + ) + else: + raise ValueError(f"unknown method: {method}") + + ms = triton.testing.do_bench(fn, warmup=warmup, rep=rep) + return ms + + bench_jagged_transpose_1d.run(print_data=True) + + +if __name__ == "__main__": + main() diff --git a/recommendation_v4/generative_recommenders/ops/cpp/benchmarks/replace_last_n_with_jagged_bench.py b/recommendation_v4/generative_recommenders/ops/cpp/benchmarks/replace_last_n_with_jagged_bench.py new file mode 100644 index 000000000..a3f2483fa --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/benchmarks/replace_last_n_with_jagged_bench.py @@ -0,0 +1,150 @@ +# pyre-strict +from typing import List + +import click +import torch + +# @manual=//triton:triton +import triton +from hammer.ops.jagged import replace_last_n_with_jagged +from hammer.utils import HammerKernel + +# buck2 run @//mode/opt -c fbcode.nvcc_arch=h100 //generative_recommenders/ops/cpp/benchmarks:replace_last_n_with_jagged_bench + +torch.ops.load_library("//generative_recommenders/ops/cpp:cpp_ops") +torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops") +torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops_cpu") + + +@click.command() +@click.option("--data-type", type=str, default="float32") +@click.option("--batch-size", type=int, default=512) +@click.option("--embedding-dim", type=int, default=64) +@click.option("--max-seq-len-log2", type=int, default=16) +@click.option("--seq-sparsity", type=float, default=0.8) +def main( + data_type: str, + batch_size: int, + embedding_dim: int, + max_seq_len_log2: int, + seq_sparsity: float, +) -> None: + if data_type == "float32": + dtype = torch.float32 + elif data_type == "float16": + dtype = torch.float16 + elif data_type == "bfloat16": + dtype = torch.bfloat16 + else: + raise ValueError(f"Unsupported data type: {data_type}.") + + configs: List[triton.testing.Benchmark] = [ + triton.testing.Benchmark( + x_names=["max_seq_len"], + x_vals=[2**i for i in range(6, max_seq_len_log2)], + line_arg="method", + line_vals=[ + "custom_cuda", + "hammer_pytorch", + "hammer_triton", + ], + line_names=[ + "Custom CUDA", + "Hammer PyTorch", + "Hammer Triton", + ], + styles=[ + ("green", "-"), + ("orange", "--"), + ("purple", "-."), + ], + ylabel="ms", + plot_name=f"replace_last_n_with_jagged_batch{batch_size}_dim{embedding_dim}_sparsity{seq_sparsity}_{data_type}", + args={ + "dtype": dtype, + "batch_size": batch_size, + "embedding_dim": embedding_dim, + "seq_sparsity": seq_sparsity, + }, + ) + ] + + @triton.testing.perf_report(configs) + def bench_replace_last_n_with_jagged( + max_seq_len: int, + batch_size: int, + method: str, + dtype: torch.dtype, + embedding_dim: int, + seq_sparsity: float, + ) -> float: + warmup = 50 + rep = 500 + torch.manual_seed(1001) + + min_left_len = max(1, int(max_seq_len * seq_sparsity * 0.3)) + max_left_len = int(max_seq_len * seq_sparsity) + + lengths_left = torch.randint( + min_left_len, max_left_len + 1, (batch_size,), dtype=torch.int32 + ) + lengths_right = torch.randint( + 1, min_left_len + 1, (batch_size,), dtype=torch.int32 + ) + + lengths_right = torch.min(lengths_right, lengths_left) + + total_left = int(lengths_left.sum().item()) + total_right = int(lengths_right.sum().item()) + + values_left = torch.randn(total_left, embedding_dim, dtype=dtype) + values_right = torch.randn(total_right, embedding_dim, dtype=dtype) + + offsets_left = torch.zeros( + (batch_size + 1,), dtype=lengths_left.dtype, device=lengths_left.device + ) + offsets_left[1:] = torch.cumsum(lengths_left.view(-1), dim=0) + offsets_right = torch.zeros( + (batch_size + 1,), dtype=lengths_right.dtype, device=lengths_right.device + ) + offsets_right[1:] = torch.cumsum(lengths_right.view(-1), dim=0) + + lengths_left = lengths_left.cuda() + lengths_right = lengths_right.cuda() + values_left = values_left.cuda() + values_right = values_right.cuda() + offsets_left = offsets_left.cuda() + offsets_right = offsets_right.cuda() + + if method == "custom_cuda": + fn = lambda: torch.ops.hstu.replace_last_n_with_jagged( # noqa E731 + lengths_left, values_left, lengths_right, values_right + ) + elif method == "hammer_pytorch": + fn = lambda: replace_last_n_with_jagged( # noqa E731 + max_seq_len_left=max_seq_len, + offsets_left=offsets_left, + values_left=values_left, + offsets_right=offsets_right, + values_right=values_right, + ) + elif method == "hammer_triton": + fn = lambda: replace_last_n_with_jagged( # noqa E731 + max_seq_len_left=max_seq_len, + offsets_left=offsets_left, + values_left=values_left, + offsets_right=offsets_right, + values_right=values_right, + kernel=HammerKernel.TRITON, + ) + else: + raise ValueError(f"unknown method: {method}") + + ms = triton.testing.do_bench(fn, warmup=warmup, rep=rep) + return ms + + bench_replace_last_n_with_jagged.run(print_data=True) + + +if __name__ == "__main__": + main() diff --git a/recommendation_v4/generative_recommenders/ops/cpp/benchmarks/split_1d_jagged_jagged_bench.py b/recommendation_v4/generative_recommenders/ops/cpp/benchmarks/split_1d_jagged_jagged_bench.py new file mode 100644 index 000000000..4aaa9d77c --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/benchmarks/split_1d_jagged_jagged_bench.py @@ -0,0 +1,116 @@ +# pyre-strict +from typing import List + +import click +import torch + +# @manual=//triton:triton +import triton +from hammer.ops.jagged import split_1D_jagged_jagged + +# buck2 run @//mode/opt -c fbcode.nvcc_arch=h100 //generative_recommenders/ops/cpp/benchmarks:split_1d_jagged_jagged_bench + +torch.ops.load_library("//generative_recommenders/ops/cpp:cpp_ops") +torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops") +torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops_cpu") + + +@click.command() +@click.option("--data-type", type=str, default="float32") +@click.option("--batch-size", type=int, default=512) +@click.option("--max-seq-len-log2", type=int, default=20) +@click.option("--seq-sparsity", type=float, default=0.8) +def main( + data_type: str, + batch_size: int, + max_seq_len_log2: int, + seq_sparsity: float, +) -> None: + if data_type == "float32": + dtype = torch.float32 + elif data_type == "float16": + dtype = torch.float16 + elif data_type == "bfloat16": + dtype = torch.bfloat16 + else: + raise ValueError(f"Unsupported data type: {data_type}.") + + configs: List[triton.testing.Benchmark] = [ + triton.testing.Benchmark( + x_names=["max_seq_len"], + x_vals=[2**i for i in range(6, max_seq_len_log2)], + line_arg="method", + line_vals=[ + "custom_cuda", + "hammer_pytorch", + ], + line_names=["Custom CUDA", "Hammer PyTorch"], + styles=[("green", "-"), ("orange", "--")], + ylabel="ms", + plot_name=f"split_1d_jagged_jagged_batch{batch_size}_sparsity{seq_sparsity}_{data_type}", + args={ + "dtype": dtype, + "batch_size": batch_size, + "seq_sparsity": seq_sparsity, + }, + ) + ] + + @triton.testing.perf_report(configs) + def bench_split_1d_jagged_jagged( + max_seq_len: int, + batch_size: int, + method: str, + dtype: torch.dtype, + seq_sparsity: float, + ) -> float: + warmup = 50 + rep = 500 + torch.manual_seed(1001) + + lengths_left = torch.randint( + 1, int(max_seq_len * seq_sparsity) + 1, (batch_size,), dtype=torch.int32 + ) + lengths_right = torch.randint( + 1, int(max_seq_len * seq_sparsity) + 1, (batch_size,), dtype=torch.int32 + ) + + offsets_left = torch.zeros( + (batch_size + 1,), dtype=lengths_left.dtype, device=lengths_left.device + ) + offsets_left[1:] = torch.cumsum(lengths_left.view(-1), dim=0) + offsets_right = torch.zeros( + (batch_size + 1,), dtype=lengths_right.dtype, device=lengths_right.device + ) + offsets_right[1:] = torch.cumsum(lengths_right.view(-1), dim=0) + + combined_offsets = offsets_left + offsets_right + combined_values = torch.randn(int(combined_offsets[-1].item()), dtype=dtype) + + max_seq_len_combined = int((lengths_left + lengths_right).max().item()) + + lengths_left = lengths_left.cuda() + lengths_right = lengths_right.cuda() + combined_values = combined_values.cuda() + offsets_left = offsets_left.cuda() + offsets_right = offsets_right.cuda() + + if method == "custom_cuda": + fn = lambda: torch.ops.hstu.split_1d_jagged_jagged( # noqa E731 + lengths_left, lengths_right, combined_values + ) + elif method == "hammer_pytorch": + fn = lambda: split_1D_jagged_jagged( # noqa E731 + max_seq_len_combined, combined_values, offsets_left, offsets_right + ) + else: + raise ValueError(f"unknown method: {method}") + + ms = triton.testing.do_bench(fn, warmup=warmup, rep=rep) + return ms + + bench_split_1d_jagged_jagged.run(print_data=True) + + +if __name__ == "__main__": + main() diff --git a/recommendation_v4/generative_recommenders/ops/cpp/common.h b/recommendation_v4/generative_recommenders/ops/cpp/common.h new file mode 100644 index 000000000..1c4b43768 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/common.h @@ -0,0 +1,60 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +#define AT_DISPATCH_CASE_FLOATING_TYPES_AND4( \ + SCALARTYPE1, SCALARTYPE2, SCALARTYPE3, SCALARTYPE4, ...) \ + AT_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__) \ + AT_DISPATCH_CASE(SCALARTYPE1, __VA_ARGS__) \ + AT_DISPATCH_CASE(SCALARTYPE2, __VA_ARGS__) \ + AT_DISPATCH_CASE(SCALARTYPE3, __VA_ARGS__) \ + AT_DISPATCH_CASE(SCALARTYPE4, __VA_ARGS__) + +#define AT_DISPATCH_FLOATING_TYPES_AND4( \ + SCALARTYPE1, SCALARTYPE2, SCALARTYPE3, SCALARTYPE4, TYPE, NAME, ...) \ + AT_DISPATCH_SWITCH( \ + TYPE, \ + NAME, \ + AT_DISPATCH_CASE_FLOATING_TYPES_AND4( \ + SCALARTYPE1, SCALARTYPE2, SCALARTYPE3, SCALARTYPE4, __VA_ARGS__)) + +inline __attribute__((always_inline)) uint32_t +div_round_up(uint32_t a, uint32_t b) { + return (a + b - 1) / b; +}; + +inline __attribute__((always_inline)) uint32_t next_power_of_2(uint32_t n) { + n--; + n |= n >> 1; + n |= n >> 2; + n |= n >> 4; + n |= n >> 8; + n |= n >> 16; + n++; + return n; +} + +/* + * Because different .SO may include the same CUDA CUB kernels, this results in + * confusion, where libA may end up calling libB's cub kernel and causing + * failures when we static link libcudart_static.a. To avoid this, we annotate + * only the public functions and hide the rest. + */ +#define DLL_PUBLIC __attribute__((visibility("default"))) diff --git a/recommendation_v4/generative_recommenders/ops/cpp/complete_cumsum.cpp b/recommendation_v4/generative_recommenders/ops/cpp/complete_cumsum.cpp new file mode 100644 index 000000000..4ebd426d7 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/complete_cumsum.cpp @@ -0,0 +1,44 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include "fbgemm_gpu/sparse_ops.h" // @manual + +namespace hstu { + +at::Tensor complete_cumsum_cpu(const at::Tensor& values) { + TORCH_CHECK(values.dim() == 1); + auto len = values.size(0); + const torch::Tensor index = at::range(0, len, at::kLong).cpu(); + auto output = fbgemm_gpu::asynchronous_complete_cumsum_cpu(values); + return output; +} + +at::Tensor complete_cumsum_meta(const at::Tensor& values) { + auto len = values.sym_size(0); + auto output = at::native::empty_meta_symint( + {len + 1}, + /*dtype=*/::std::make_optional(values.scalar_type()), + /*layout=*/::std::make_optional(values.layout()), + /*device=*/::std::make_optional(c10::Device(c10::kMeta)), + /*pin_memory=*/::std::nullopt); + return output; +} + +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/complete_cumsum.cu b/recommendation_v4/generative_recommenders/ops/cpp/complete_cumsum.cu new file mode 100644 index 000000000..06d1abdd6 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/complete_cumsum.cu @@ -0,0 +1,51 @@ +#include "common.h" + +#include + +namespace hstu { + +DLL_PUBLIC at::Tensor complete_cumsum_cuda(const at::Tensor& values) { + at::cuda::OptionalCUDAGuard device_guard; + device_guard.set_index(values.get_device()); + + TORCH_CHECK(values.numel() < std::numeric_limits::max()); + TORCH_CHECK(values.dim() == 1); + const auto values_contig = values.contiguous(); + + auto cumsum = at::empty({values_contig.numel() + 1}, values_contig.options()); + cumsum[0].zero_(); + + AT_DISPATCH_FLOATING_TYPES_AND4( + at::ScalarType::Int, + at::ScalarType::Long, + at::ScalarType::Half, + at::ScalarType::BFloat16, + values_contig.scalar_type(), + "complete_cumsum_cuda", + [&] { + size_t temp_storage_bytes = 0; + AT_CUDA_CHECK( + cub::DeviceScan::InclusiveSum( + nullptr, + temp_storage_bytes, + values_contig.data_ptr(), + cumsum.data_ptr() + 1, + values_contig.numel(), + at::cuda::getCurrentCUDAStream())); + auto temp_storage = at::empty( + {static_cast(temp_storage_bytes)}, + values_contig.options().dtype(at::kByte)); + AT_CUDA_CHECK( + cub::DeviceScan::InclusiveSum( + temp_storage.data_ptr(), + temp_storage_bytes, + values_contig.data_ptr(), + cumsum.data_ptr() + 1, + values_contig.numel(), + at::cuda::getCurrentCUDAStream())); + }); + + return cumsum; +} + +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/concat_1d_jagged_jagged.cpp b/recommendation_v4/generative_recommenders/ops/cpp/concat_1d_jagged_jagged.cpp new file mode 100644 index 000000000..51b313443 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/concat_1d_jagged_jagged.cpp @@ -0,0 +1,111 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "fbgemm_gpu/sparse_ops.h" // @manual + +namespace hstu { + +template +void _concat_1d_jagged_jagged_cpu_kernel( + int32_t B, + const at::TensorAccessor& offsets_left, + const at::TensorAccessor& values_left, + const at::TensorAccessor& offsets_right, + const at::TensorAccessor& values_right, + at::TensorAccessor combined_values) { + for (auto b : c10::irange(B)) { + auto left_start = offsets_left[b]; + auto left_len = offsets_left[b + 1] - left_start; + auto right_start = offsets_right[b]; + auto right_len = offsets_right[b + 1] - right_start; + auto combined_start = left_start + right_start; + for (auto i = 0; i < left_len; ++i) { + combined_values[combined_start + i] = values_left[left_start + i]; + } + for (auto i = 0; i < right_len; ++i) { + combined_values[left_len + combined_start + i] = + values_right[right_start + i]; + } + } +} + +at::Tensor concat_1d_jagged_jagged_cpu( + const at::Tensor& lengths_left, + const at::Tensor& values_left, + const at::Tensor& lengths_right, + const at::Tensor& values_right) { + TORCH_INTERNAL_ASSERT(lengths_left.device().type() == at::DeviceType::CPU); + TORCH_INTERNAL_ASSERT(values_left.device().type() == at::DeviceType::CPU); + TORCH_INTERNAL_ASSERT(lengths_right.device().type() == at::DeviceType::CPU); + TORCH_INTERNAL_ASSERT(values_right.device().type() == at::DeviceType::CPU); + auto L = values_left.numel() + values_right.numel(); + TORCH_CHECK(L < std::numeric_limits::max()); + TORCH_CHECK(lengths_left.size(0) == lengths_right.size(0)); + auto B = lengths_left.size(0); + auto combined_values = at::empty({L}, values_left.options()); + if (L == 0) { + return combined_values; + } + const auto offsets_left = + fbgemm_gpu::asynchronous_complete_cumsum_cpu(lengths_left.view({-1})); + const auto offsets_right = + fbgemm_gpu::asynchronous_complete_cumsum_cpu(lengths_right.view({-1})); + AT_DISPATCH_INTEGRAL_TYPES( + lengths_left.scalar_type(), + "concat_1d_jagged_jagged_values_cpu_kernel_input1", + [&] { + using index_t = scalar_t; + AT_DISPATCH_ALL_TYPES_AND2( + at::ScalarType::BFloat16, + at::ScalarType::Half, + values_left.scalar_type(), + "concat_1d_jagged_jagged_values_cpu_kernel_input2", + [&] { + using val_t = scalar_t; + _concat_1d_jagged_jagged_cpu_kernel( + B, + offsets_left.accessor(), + values_left.accessor(), + offsets_right.accessor(), + values_right.accessor(), + combined_values.accessor()); + }); + }); + return combined_values; +} + +at::Tensor concat_1d_jagged_jagged_meta( + const at::Tensor& lengths_left, + const at::Tensor& values_left, + const at::Tensor& lengths_right, + const at::Tensor& values_right) { + auto L = values_left.numel() + values_right.numel(); + return at::native::empty_meta_symint( + {L}, + /*dtype=*/::std::make_optional(values_left.scalar_type()), + /*layout=*/::std::make_optional(values_left.layout()), + /*device=*/::std::make_optional(c10::Device(c10::kMeta)), + /*pin_memory=*/::std::nullopt); +} +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/concat_1d_jagged_jagged.cu b/recommendation_v4/generative_recommenders/ops/cpp/concat_1d_jagged_jagged.cu new file mode 100644 index 000000000..8eeae9d59 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/concat_1d_jagged_jagged.cu @@ -0,0 +1,130 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "common.h" +#include "fbgemm_gpu/sparse_ops.h" // @manual +#include "fbgemm_gpu/utils/fixed_divisor.cuh" // @manual + +namespace hstu { + +static constexpr int32_t kMaxThreads = 1024; + +template +__global__ +__launch_bounds__(kMaxThreads) void _concat_1d_jagged_jagged_cuda_kernel( + int32_t B, + const at::PackedTensorAccessor32 + offsets_left, + const at::PackedTensorAccessor32 + values_left, + const at::PackedTensorAccessor32 + offsets_right, + const at::PackedTensorAccessor32 + values_right, + at::PackedTensorAccessor32 + combined_values) { + for (auto b = blockIdx.x * blockDim.y + threadIdx.y; + b < static_cast(B); + b += gridDim.x * blockDim.y) { + auto left_start = offsets_left[b]; + auto left_len = offsets_left[b + 1] - left_start; + auto right_start = offsets_right[b]; + auto right_len = offsets_right[b + 1] - right_start; + auto combined_start = left_start + right_start; + for (auto i = threadIdx.x; i < static_cast(left_len + right_len); + i += blockDim.x) { + if (i < static_cast(left_len)) { + combined_values[combined_start + i] = values_left[left_start + i]; + } else { + combined_values[combined_start + i] = + values_right[right_start + i - left_len]; + } + } + } +} + +at::Tensor concat_1d_jagged_jagged_cuda( + const at::Tensor& lengths_left, + const at::Tensor& values_left, + const at::Tensor& lengths_right, + const at::Tensor& values_right) { + at::cuda::OptionalCUDAGuard device_guard; + device_guard.set_index(values_left.get_device()); + TORCH_INTERNAL_ASSERT(lengths_left.device().type() == at::DeviceType::CUDA); + TORCH_INTERNAL_ASSERT(values_left.device().type() == at::DeviceType::CUDA); + TORCH_INTERNAL_ASSERT(lengths_right.device().type() == at::DeviceType::CUDA); + TORCH_INTERNAL_ASSERT(values_right.device().type() == at::DeviceType::CUDA); + auto L = values_left.numel() + values_right.numel(); + TORCH_CHECK(L < std::numeric_limits::max()); + TORCH_CHECK(values_left.get_device() == lengths_left.get_device()); + TORCH_CHECK(values_left.get_device() == lengths_right.get_device()); + TORCH_CHECK(values_left.get_device() == values_right.get_device()); + auto B = lengths_left.size(0); + auto combined_values = at::empty({L}, values_left.options()); + if (L == 0) { + return combined_values; + } + const auto offsets_left = + fbgemm_gpu::asynchronous_complete_cumsum_gpu(lengths_left.view({-1})); + const auto offsets_right = + fbgemm_gpu::asynchronous_complete_cumsum_gpu(lengths_right.view({-1})); + // Optimized thread block configuration based on benchmark results + uint32_t B_blocks = 4; + dim3 threads(256, B_blocks); + auto blocks = div_round_up(B, B_blocks); + AT_DISPATCH_INTEGRAL_TYPES( + lengths_left.scalar_type(), + "concat_1d_jagged_jagged_values_cuda_kernel_input1", + [&] { + using index_t = scalar_t; + AT_DISPATCH_ALL_TYPES_AND2( + at::ScalarType::BFloat16, + at::ScalarType::Half, + values_left.scalar_type(), + "concat_1d_jagged_jagged_values_cuda_kernel_input2", + [&] { + using val_t = scalar_t; + _concat_1d_jagged_jagged_cuda_kernel + <<>>( + B, + offsets_left.packed_accessor32< + index_t, + 1, + at::RestrictPtrTraits>(), + values_left + .packed_accessor32(), + offsets_right.packed_accessor32< + index_t, + 1, + at::RestrictPtrTraits>(), + values_right + .packed_accessor32(), + combined_values.packed_accessor32< + val_t, + 1, + at::RestrictPtrTraits>()); + }); + }); + return combined_values; +} +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/cpp_ops.cpp b/recommendation_v4/generative_recommenders/ops/cpp/cpp_ops.cpp new file mode 100644 index 000000000..155cc7572 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/cpp_ops.cpp @@ -0,0 +1,207 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * Because different .SO may include the same CUDA CUB kernels, this results in + * confusion, where libA may end up calling libB's cub kernel and causing + * failures when we static link libcudart_static.a. To avoid this, we annotate + * only the public functions and hide the rest. + */ +#define DLL_PUBLIC __attribute__((visibility("default"))) + +namespace hstu { +at::Tensor expand_1d_jagged_to_dense_cpu( + const at::Tensor& values, + const at::Tensor& offsets, + const int64_t max_len); + +at::Tensor expand_1d_jagged_to_dense_meta( + const at::Tensor& values, + const at::Tensor& offsets, + const c10::SymInt max_len); + +at::Tensor expand_1d_jagged_to_dense_cuda( + const at::Tensor& values, + const at::Tensor& offsets, + const int64_t max_len); + +at::Tensor complete_cumsum_cpu(const at::Tensor& values); + +at::Tensor complete_cumsum_cuda(const at::Tensor& values); + +at::Tensor complete_cumsum_meta(const at::Tensor& values); + +at::Tensor concat_1d_jagged_jagged_cpu( + const at::Tensor& lengths_left, + const at::Tensor& values_left, + const at::Tensor& lengths_right, + const at::Tensor& values_right); + +at::Tensor concat_1d_jagged_jagged_cuda( + const at::Tensor& lengths_left, + const at::Tensor& values_left, + const at::Tensor& lengths_right, + const at::Tensor& values_right); + +at::Tensor concat_1d_jagged_jagged_meta( + const at::Tensor& lengths_left, + const at::Tensor& values_left, + const at::Tensor& lengths_right, + const at::Tensor& values_right); + +std::tuple split_1d_jagged_jagged_cpu( + const at::Tensor& lengths_left, + const at::Tensor& lengths_right, + const at::Tensor& combined_values); + +std::tuple split_1d_jagged_jagged_cuda( + const at::Tensor& lengths_left, + const at::Tensor& lengths_right, + const at::Tensor& combined_values); + +std::tuple split_1d_jagged_jagged_meta( + const at::Tensor& lengths_left, + const at::Tensor& lengths_right, + const at::Tensor& combined_values); + +at::Tensor replace_last_n_with_jagged_cpu( + const at::Tensor& lengths_left, + const at::Tensor& values_left, + const at::Tensor& lengths_right, + const at::Tensor& values_right); + +at::Tensor replace_last_n_with_jagged_cuda( + const at::Tensor& lengths_left, + const at::Tensor& values_left, + const at::Tensor& lengths_right, + const at::Tensor& values_right); + +at::Tensor replace_last_n_with_jagged_meta( + const at::Tensor& lengths_left, + const at::Tensor& values_left, + const at::Tensor& lengths_right, + const at::Tensor& values_right); + +std::tuple jagged_transpose_1d_cpu( + const at::Tensor& values, + const at::Tensor& offsets, + const at::Tensor& lengths, + const int64_t max_len, + const int64_t size1, + const int64_t size2); + +std::tuple jagged_transpose_1d_cuda( + const at::Tensor& values, + const at::Tensor& offsets, + const at::Tensor& lengths, + const int64_t max_len, + const int64_t size1, + const int64_t size2); + +std::tuple jagged_transpose_1d_meta( + const at::Tensor& values, + const at::Tensor& offsets, + const at::Tensor& lengths, + const int64_t max_len, + const int64_t size1, + const int64_t size2); + +DLL_PUBLIC std::tuple sort_kv_pairs_meta( + const at::Tensor& keys, + const at::Tensor& values, + const std::optional& end_bit, + const bool descending = false) { + TORCH_CHECK( + keys.dtype() == at::kInt || keys.dtype() == at::kLong || + keys.dtype() == at::kByte || keys.dtype() == at::kShort); + TORCH_CHECK(keys.dim() == 1); + TORCH_CHECK(values.dim() == 1); + return {at::empty_like(keys), at::empty_like(values)}; +} + +std::tuple sort_kv_pairs_cuda( + const at::Tensor& keys, + const at::Tensor& values, + const std::optional& end_bit, + const bool descending = false); + +} // namespace hstu + +TORCH_LIBRARY_FRAGMENT(hstu, m) { + m.def( + "expand_1d_jagged_to_dense(Tensor values, Tensor offsets, SymInt max_len) -> Tensor"); + m.def( + "concat_1d_jagged_jagged(Tensor lengths_left, Tensor values_left, Tensor lengths_right, Tensor values_right) -> Tensor"); + m.def( + "split_1d_jagged_jagged(Tensor lengths_left, Tensor lengths_right, Tensor combined_values) -> (Tensor, Tensor)"); + m.def( + "replace_last_n_with_jagged(Tensor lengths_left, Tensor values_left, Tensor lengths_right, Tensor values_right) -> Tensor"); + m.def( + "jagged_transpose_1d(Tensor values, Tensor offsets, Tensor lengths, int max_len, int size1, int size2) -> (Tensor, Tensor, Tensor)"); + m.def("complete_cumsum(Tensor values) -> Tensor"); + m.def( + "sort_kv_pairs(Tensor keys, Tensor values, int? end_bit=None, bool descending=False) -> (Tensor, Tensor)"); +} + +TORCH_LIBRARY_IMPL(hstu, CPU, m) { + m.impl("expand_1d_jagged_to_dense", hstu::expand_1d_jagged_to_dense_cpu); + m.impl("concat_1d_jagged_jagged", hstu::concat_1d_jagged_jagged_cpu); + m.impl("split_1d_jagged_jagged", hstu::split_1d_jagged_jagged_cpu); + m.impl("replace_last_n_with_jagged", hstu::replace_last_n_with_jagged_cpu); + m.impl("jagged_transpose_1d", hstu::jagged_transpose_1d_cpu); + m.impl("complete_cumsum", hstu::complete_cumsum_cpu); +} + +TORCH_LIBRARY_IMPL(hstu, CUDA, m) { + m.impl("expand_1d_jagged_to_dense", hstu::expand_1d_jagged_to_dense_cuda); + m.impl("concat_1d_jagged_jagged", hstu::concat_1d_jagged_jagged_cuda); + m.impl("split_1d_jagged_jagged", hstu::split_1d_jagged_jagged_cuda); + m.impl("replace_last_n_with_jagged", hstu::replace_last_n_with_jagged_cuda); + m.impl("jagged_transpose_1d", hstu::jagged_transpose_1d_cuda); + m.impl("complete_cumsum", hstu::complete_cumsum_cuda); + m.impl( + "sort_kv_pairs", + torch::dispatch( + c10::DispatchKey::CUDA, TORCH_FN(hstu::sort_kv_pairs_cuda))); +} + +TORCH_LIBRARY_IMPL(hstu, Meta, m) { + m.impl("expand_1d_jagged_to_dense", hstu::expand_1d_jagged_to_dense_meta); + m.impl("concat_1d_jagged_jagged", hstu::concat_1d_jagged_jagged_meta); + m.impl("split_1d_jagged_jagged", hstu::split_1d_jagged_jagged_meta); + m.impl("replace_last_n_with_jagged", hstu::replace_last_n_with_jagged_meta); + m.impl("jagged_transpose_1d", hstu::jagged_transpose_1d_meta); + m.impl("complete_cumsum", hstu::complete_cumsum_meta); + m.impl( + "sort_kv_pairs", + torch::dispatch( + c10::DispatchKey::Meta, TORCH_FN(hstu::sort_kv_pairs_meta))); +} + +TORCH_LIBRARY_IMPL(hstu, Autograd, m) { + m.impl( + "expand_1d_jagged_to_dense", + torch::autograd::autogradNotImplementedFallback()); + m.impl("complete_cumsum", torch::autograd::autogradNotImplementedFallback()); +} diff --git a/recommendation_v4/generative_recommenders/ops/cpp/cuda_hstu_attention.py b/recommendation_v4/generative_recommenders/ops/cpp/cuda_hstu_attention.py new file mode 100644 index 000000000..0f9458c8b --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/cuda_hstu_attention.py @@ -0,0 +1,193 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-strict + +from typing import Optional + +import torch +from generative_recommenders.ops.utils import is_sm100_plus + +try: + # We need to import the CUDA kernels after importing torch + import hstu._C # pyre-ignore [21] +except: + pass +try: + torch.ops.load_library( + "//generative_recommenders/fb/ultra/ops/blackwell/hstu_attention:hstu_flash_attention" + ) + torch.ops.load_library( + "//generative_recommenders/ops/cpp/hstu_attention:hstu_flash_attention" + ) +except: + pass + + +def cuda_hstu_mha( + max_seq_len: int, + alpha: float, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + seq_offsets: Optional[torch.Tensor] = None, + causal: bool = False, + num_targets: Optional[torch.Tensor] = None, + attn_scale: Optional[torch.Tensor] = None, + max_attn_len: int = 0, + min_full_attn_seq_len: int = 0, + contextual_seq_len: int = 0, + q_descale: Optional[torch.Tensor] = None, + k_descale: Optional[torch.Tensor] = None, + v_descale: Optional[torch.Tensor] = None, + sort_by_length: bool = False, + deterministic: bool = False, + sm_margin: int = 0, + max_q_len: int = 0, + seq_offsets_q: Optional[torch.Tensor] = None, + num_softmax_heads: int = 0, + training: bool = True, + max_seq_len_tensor: Optional[torch.Tensor] = None, + contextual_seq_len_tensor: Optional[torch.Tensor] = None, + max_attn_len_tensor: Optional[torch.Tensor] = None, + min_full_attn_seq_len_tensor: Optional[torch.Tensor] = None, + num_groups: int = 1, + is_inference: bool = False, +) -> torch.Tensor: + """ + Arguments: + q, k, v: (batch_size, seqlen, nheads, headdim) or (total_seqlen, nheads, headdim) + """ + if is_sm100_plus() and not is_inference: + return torch.ops.bw_hstu.bw_hstu_mha( + max_seq_len, + alpha, + q, + k, + v, + seq_offsets, + causal, + num_targets, + attn_scale, + max_attn_len, + min_full_attn_seq_len, + contextual_seq_len, + q_descale, + k_descale, + v_descale, + sort_by_length, + deterministic, + sm_margin, + max_q_len, + seq_offsets_q, + max_seq_len_tensor, + contextual_seq_len_tensor, + max_attn_len_tensor, + min_full_attn_seq_len_tensor, + num_groups, + num_softmax_heads, + ) + else: + return cuda_hstu_mha_inference_wrapper( + max_seq_len, + alpha, + q, + k, + v, + seq_offsets, + causal, + num_targets, + attn_scale, + max_attn_len, + min_full_attn_seq_len, + contextual_seq_len, + q_descale, + k_descale, + v_descale, + sort_by_length, + deterministic, + sm_margin, + max_q_len, + seq_offsets_q, + num_softmax_heads, + training, + max_seq_len_tensor, + contextual_seq_len_tensor, + max_attn_len_tensor, + min_full_attn_seq_len_tensor, + num_groups, + ) + + +@torch.fx.wrap +def cuda_hstu_mha_inference_wrapper( + max_seq_len: int, + alpha: float, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + seq_offsets: Optional[torch.Tensor] = None, + causal: bool = False, + num_targets: Optional[torch.Tensor] = None, + attn_scale: Optional[torch.Tensor] = None, + max_attn_len: int = 0, + min_full_attn_seq_len: int = 0, + contextual_seq_len: int = 0, + q_descale: Optional[torch.Tensor] = None, + k_descale: Optional[torch.Tensor] = None, + v_descale: Optional[torch.Tensor] = None, + sort_by_length: bool = False, + deterministic: bool = False, + sm_margin: int = 0, + max_q_len: int = 0, + seq_offsets_q: Optional[torch.Tensor] = None, + num_softmax_heads: int = 0, + training: bool = True, + max_seq_len_tensor: Optional[torch.Tensor] = None, + contextual_seq_len_tensor: Optional[torch.Tensor] = None, + max_attn_len_tensor: Optional[torch.Tensor] = None, + min_full_attn_seq_len_tensor: Optional[torch.Tensor] = None, + num_groups: int = 1, +) -> torch.Tensor: + attn_scale = attn_scale.to(torch.float32) if attn_scale is not None else attn_scale + + return torch.ops.hstu.hstu_mha( + max_seq_len, + alpha, + q, + k, + v, + seq_offsets, + causal, + num_targets, + attn_scale, + max_attn_len, + min_full_attn_seq_len, + contextual_seq_len, + q_descale, + k_descale, + v_descale, + sort_by_length, + deterministic, + sm_margin, + max_q_len, + seq_offsets_q, + num_softmax_heads, + training, + max_seq_len_tensor, + contextual_seq_len_tensor, + max_attn_len_tensor, + min_full_attn_seq_len_tensor, + num_groups, + ) diff --git a/recommendation_v4/generative_recommenders/ops/cpp/cuda_hstu_preprocess_and_attention.py b/recommendation_v4/generative_recommenders/ops/cpp/cuda_hstu_preprocess_and_attention.py new file mode 100644 index 000000000..2184ef2a5 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/cuda_hstu_preprocess_and_attention.py @@ -0,0 +1,668 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +from typing import Optional, Tuple + +import torch +from generative_recommenders.ops.triton.triton_addmm import ( + maybe_triton_addmm_fwd, + triton_addmm_bwd, +) +from generative_recommenders.ops.triton.triton_layer_norm import ( + triton_weighted_layer_norm_bwd, +) +from generative_recommenders.ops.utils import copy_if_different_ptr, is_sm100_plus +from torch.nn import functional as F + +try: + from generative_recommenders.fb.ultra.ops.fp8.fp8_addmm import ( + fp8_rowwise_quantize_addmm, + ) + from generative_recommenders.fb.ultra.ops.fp8.layer_norm_quantization import ( + triton_weighted_layer_norm_quantization_fwd, + ) + from hammer.ops.triton.triton_apply_rope import ( + triton_apply_rope_bwd, + triton_apply_rope_fwd, + ) + + if is_sm100_plus(): + print("is sm100_plus architecture, loading hstu flash attention for blackwell") + torch.ops.load_library( + "//generative_recommenders/fb/ultra/ops/blackwell/hstu_attention:hstu_flash_attention" + ) + print("loading hstu flash attention for general architecture") + torch.ops.load_library( + "//generative_recommenders/ops/cpp/hstu_attention:hstu_flash_attention" + ) +except Exception as ex: + print(f"Library importing error when importing library: {ex}") + + +class _HSTUPreprocessAndAttentionFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore [14] + def forward( + ctx, # pyre-ignore [2] + x: torch.Tensor, + norm_weight: torch.Tensor, + norm_bias: torch.Tensor, + norm_eps: float, + num_heads: int, + attn_dim: int, + hidden_dim: int, + uvqk_weight: torch.Tensor, + uvqk_bias: Optional[torch.Tensor], + max_seq_len: int, + seq_offsets: torch.Tensor, + alpha: float, + invalid_attn_mask_type: str, + num_targets: Optional[torch.Tensor], + rotary_weights: Optional[ + Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor] + ] = None, + attn_scale: Optional[torch.Tensor] = None, + recompute_uvqk_in_backward: bool = False, + recompute_normed_x_in_backward: bool = False, + contextual_seq_len: int = 0, + sort_by_length: bool = False, + max_attn_len: Optional[int] = None, + full_attn_size: Optional[int] = None, + silu_u: bool = True, + fp8_in_addmm_fwd: bool = False, + num_softmax_heads: int = 0, + ) -> Tuple[torch.Tensor, torch.Tensor]: + max_attn_len = max_attn_len or 0 + full_attn_size = full_attn_size or 0 + normed_x, x_mean, x_rstd, BLOCK_D, x_scale, normed_x_fp8 = ( + triton_weighted_layer_norm_quantization_fwd( + x=x, + weight=norm_weight, + bias=norm_bias, + eps=norm_eps, + quantize_output=fp8_in_addmm_fwd, + ) + ) + # When silu_u is False and we want to recompute in backward, we split the weight + # for u and vqk separately during training to compute them independently. + # This avoids needing to clone u (which would otherwise keep the whole uvqk alive). + if not silu_u and recompute_uvqk_in_backward: + # Split the weights/biases to compute u and vqk separately + u_weight, vqk_weight = uvqk_weight.split( + [ + hidden_dim * num_heads, + hidden_dim * num_heads + + attn_dim * num_heads + + attn_dim * num_heads, + ], + dim=1, + ) + if uvqk_bias is not None: + u_bias, vqk_bias = uvqk_bias.split( + [ + hidden_dim * num_heads, + hidden_dim * num_heads + + attn_dim * num_heads + + attn_dim * num_heads, + ], + dim=0, + ) + else: + u_bias, vqk_bias = None, None + if fp8_in_addmm_fwd: + assert x_scale is not None and normed_x_fp8 is not None + u = fp8_rowwise_quantize_addmm( + x=normed_x, + x_fp8=normed_x_fp8, + w=u_weight, + y=u_bias, + x_scale=x_scale, + custom_kernel=False, + is_inference=False, + ).contiguous() + vqk = fp8_rowwise_quantize_addmm( + x=normed_x, + x_fp8=normed_x_fp8, + w=vqk_weight, + y=vqk_bias, + x_scale=x_scale, + custom_kernel=False, + is_inference=False, + ).contiguous() + else: + u = maybe_triton_addmm_fwd(normed_x, u_weight, u_bias).contiguous() + vqk = maybe_triton_addmm_fwd( + normed_x, vqk_weight, vqk_bias + ).contiguous() + v, q, k = vqk.split( + [ + hidden_dim * num_heads, + attn_dim * num_heads, + attn_dim * num_heads, + ], + dim=1, + ) + # uvqk is not used since we split the computation, but we need it + # for saving in case recompute_uvqk_in_backward is False in a + # different code path. Set to None to satisfy type checker. + uvqk = None + else: + if fp8_in_addmm_fwd: + assert ( + x_scale is not None + and normed_x_fp8 is not None + and uvqk_bias is not None + ) + uvqk = fp8_rowwise_quantize_addmm( + x=normed_x, + x_fp8=normed_x_fp8, + w=uvqk_weight, + y=uvqk_bias, + x_scale=x_scale, + custom_kernel=False, + is_inference=False, + ).contiguous() + else: + uvqk = maybe_triton_addmm_fwd( + normed_x, uvqk_weight, uvqk_bias + ).contiguous() + u, v, q, k = uvqk.split( + [ + hidden_dim * num_heads, + hidden_dim * num_heads, + attn_dim * num_heads, + attn_dim * num_heads, + ], + dim=1, + ) + if silu_u: + u = F.silu(u) + if rotary_weights is not None: + q_cos_weights = rotary_weights[0] + q_sin_weights = rotary_weights[1] + k_cos_weights = rotary_weights[2] + k_sin_weights = rotary_weights[3] + _q = triton_apply_rope_fwd( + x=q.view(-1, num_heads, attn_dim), + N=max_seq_len, + seq_offsets=seq_offsets, + cos_rope=q_cos_weights, + sin_rope=q_sin_weights, + ).view(-1, num_heads * attn_dim) + _k = triton_apply_rope_fwd( + x=k.view(-1, num_heads, attn_dim), + N=max_seq_len, + seq_offsets=seq_offsets, + cos_rope=k_cos_weights, + sin_rope=k_sin_weights, + ).view(-1, num_heads * attn_dim) + copy_if_different_ptr(q, _q) + copy_if_different_ptr(k, _k) + q = q.view(-1, num_heads, attn_dim) + k = k.view(-1, num_heads, attn_dim) + v = v.view(-1, num_heads, hidden_dim) + if is_sm100_plus(): + out, softmax_lse = torch.ops.bw_hstu.bw_hstu_mha_fwd( + max_seq_len, + alpha, + q, + k, + v, + seq_offsets, + True, # causal + num_targets, + attn_scale, + max_attn_len, + full_attn_size, + contextual_seq_len, + None, # q_descale + None, # k_descale + None, # v_descale + 0, # sm_margin + max_seq_len, # max_q_len, + None, # seq_offsets_q, + None, # max_seq_len_tensor, + None, # contextual_seq_len_tensor, + None, # max_attn_len_tensor, + None, # min_full_attn_seq_len_tensor, + 1, # num_groups + num_softmax_heads, # num_softmax_heads + ) + else: + out, softmax_lse = torch.ops.hstu.hstu_mha_fwd( + max_seq_len, + alpha, + q, + k, + v, + seq_offsets, + True, # causal + num_targets, + attn_scale, + max_attn_len, + full_attn_size, + contextual_seq_len, + None, # q_descale + None, # k_descale + None, # v_descale + 0, # sm_margin + 0, # max_q_len, + None, # seq_offsets_q, + num_softmax_heads, # num_softmax_heads, + ) + # update ctx + saved_tensors = [ + x, + norm_weight, + norm_bias, + x_mean, + x_rstd, + uvqk_weight, + seq_offsets, + out, + ] + if num_softmax_heads > 0: + saved_tensors.append(softmax_lse) + if num_targets is not None: + saved_tensors.append(num_targets) + if attn_scale is not None: + saved_tensors.append(attn_scale) + if not recompute_normed_x_in_backward: + saved_tensors.append(normed_x) + if recompute_uvqk_in_backward: + if uvqk_bias is not None: + saved_tensors.append(uvqk_bias) + if fp8_in_addmm_fwd: + saved_tensors.append(x_scale) # pyre-ignore + saved_tensors.append(normed_x_fp8) # pyre-ignore + else: + saved_tensors.append(uvqk) + if rotary_weights is not None: + saved_tensors.append(rotary_weights[0]) + saved_tensors.append(rotary_weights[1]) + saved_tensors.append(rotary_weights[2]) + saved_tensors.append(rotary_weights[3]) + ctx.save_for_backward(*saved_tensors) + ctx.alpha = alpha + ctx.invalid_attn_mask_type = invalid_attn_mask_type + ctx.has_multiple_targets = num_targets is not None + ctx.has_rotary_weights = rotary_weights is not None + ctx.has_attn_scale = attn_scale is not None + ctx.max_seq_len = max_seq_len + ctx.max_attn_len = max_attn_len + ctx.full_attn_size = full_attn_size + ctx.recompute_normed_x_in_backward = recompute_normed_x_in_backward + ctx.recompute_uvqk_in_backward = recompute_uvqk_in_backward + ctx.hidden_dim = hidden_dim + ctx.attn_dim = attn_dim + ctx.num_heads = num_heads + ctx.has_uvqk_bias = uvqk_bias is not None + ctx.uvqk_bias_1d = uvqk_bias.dim() == 1 if uvqk_bias is not None else False + ctx.norm_eps = norm_eps + ctx.norm_BLOCK_D = BLOCK_D + ctx.contextual_seq_len = contextual_seq_len + ctx.sort_by_length = sort_by_length + ctx.silu_u = silu_u + ctx.fp8_in_addmm_fwd = fp8_in_addmm_fwd + ctx.num_softmax_heads = num_softmax_heads + return u, out + + @staticmethod + # pyre-ignore[14] + def backward( + ctx, # pyre-ignore[2] + _du: torch.Tensor, + dout: torch.Tensor, + ) -> Tuple[ + torch.Tensor, # d_x + torch.Tensor, # d_norm_weight + torch.Tensor, # d_norm_bias + None, + None, + None, + None, + torch.Tensor, # d_uvqk_weight + torch.Tensor, # d_uvqk_bias + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + ]: + x, norm_weight, norm_bias, x_mean, x_rstd, uvqk_weight, seq_offsets, out = ( + ctx.saved_tensors[:8] + ) + idx = 8 + if ctx.num_softmax_heads > 0: + softmax_lse = ctx.saved_tensors[idx] + idx += 1 + else: + softmax_lse = None + if ctx.has_multiple_targets: + num_targets = ctx.saved_tensors[idx] + idx += 1 + else: + num_targets = None + if ctx.has_attn_scale: + attn_scale = ctx.saved_tensors[idx] + idx += 1 + else: + attn_scale = None + if ctx.recompute_normed_x_in_backward: + normed_x, _, _, _, _, _ = triton_weighted_layer_norm_quantization_fwd( + x=x, + weight=norm_weight, + bias=norm_bias, + eps=ctx.norm_eps, + mean=x_mean, + rstd=x_rstd, + quantize_output=ctx.fp8_in_addmm_fwd, + ) + else: + normed_x = ctx.saved_tensors[idx] + idx += 1 + if ctx.recompute_uvqk_in_backward: + if ctx.has_uvqk_bias: + uvqk_bias = ctx.saved_tensors[idx] + idx += 1 + else: + uvqk_bias = None + if not ctx.silu_u: + # When silu_u is False, we only recompute vqk (not u) + # Split the weights/biases to extract vqk portion + _, vqk_weight = uvqk_weight.split( + [ + ctx.hidden_dim * ctx.num_heads, + ctx.hidden_dim * ctx.num_heads + + ctx.attn_dim * ctx.num_heads + + ctx.attn_dim * ctx.num_heads, + ], + dim=1, + ) + vqk_bias = None + if ctx.has_uvqk_bias: + _, vqk_bias = uvqk_bias.split( + [ + ctx.hidden_dim * ctx.num_heads, + ctx.hidden_dim * ctx.num_heads + + ctx.attn_dim * ctx.num_heads + + ctx.attn_dim * ctx.num_heads, + ], + dim=0, + ) + if ctx.fp8_in_addmm_fwd: + x_scale, normed_x_fp8 = ctx.saved_tensors[idx : idx + 2] + vqk = fp8_rowwise_quantize_addmm( + x=normed_x, + x_fp8=normed_x_fp8, + w=vqk_weight, + y=vqk_bias, + x_scale=x_scale, + custom_kernel=False, + is_inference=False, + ) + idx += 2 + else: + vqk = maybe_triton_addmm_fwd( + normed_x, vqk_weight, vqk_bias + ).contiguous() + # Split vqk into v, q, k components + v, q, k = vqk.split( + [ + ctx.hidden_dim * ctx.num_heads, + ctx.attn_dim * ctx.num_heads, + ctx.attn_dim * ctx.num_heads, + ], + dim=1, + ) + u = None + else: + # When silu_u is True, we recompute uvqk (all components) + if ctx.fp8_in_addmm_fwd: + x_scale, normed_x_fp8 = ctx.saved_tensors[idx : idx + 2] + uvqk = fp8_rowwise_quantize_addmm( + x=normed_x, + x_fp8=normed_x_fp8, + w=uvqk_weight, + y=uvqk_bias, + x_scale=x_scale, + custom_kernel=False, + is_inference=False, + ) + idx += 2 + else: + uvqk = maybe_triton_addmm_fwd( + normed_x, uvqk_weight, uvqk_bias + ).contiguous() + # Split uvqk into u, v, q, k components + u, v, q, k = uvqk.split( + [ + ctx.hidden_dim * ctx.num_heads, + ctx.hidden_dim * ctx.num_heads, + ctx.attn_dim * ctx.num_heads, + ctx.attn_dim * ctx.num_heads, + ], + dim=1, + ) + else: + uvqk = ctx.saved_tensors[idx] + idx += 1 + # Split saved uvqk into u, v, q, k components + u, v, q, k = uvqk.split( + [ + ctx.hidden_dim * ctx.num_heads, + ctx.hidden_dim * ctx.num_heads, + ctx.attn_dim * ctx.num_heads, + ctx.attn_dim * ctx.num_heads, + ], + dim=1, + ) + if ctx.has_rotary_weights: + q_cos_weights, q_sin_weights, k_cos_weights, k_sin_weights = ( + ctx.saved_tensors[idx : idx + 4] + ) + idx += 4 + else: + q_cos_weights, q_sin_weights, k_cos_weights, k_sin_weights = ( + None, + None, + None, + None, + ) + + duvqk = torch.empty( + [ + x.size(0), + ctx.hidden_dim * ctx.num_heads * 2 + ctx.attn_dim * ctx.num_heads * 2, + ], + device=x.device, + dtype=x.dtype, + ) + du, dv, dq, dk = duvqk.split( + [ + ctx.hidden_dim * ctx.num_heads, + ctx.hidden_dim * ctx.num_heads, + ctx.attn_dim * ctx.num_heads, + ctx.attn_dim * ctx.num_heads, + ], + dim=1, + ) + q = q.view(-1, ctx.num_heads, ctx.attn_dim) + k = k.view(-1, ctx.num_heads, ctx.attn_dim) + v = v.view(-1, ctx.num_heads, ctx.hidden_dim) + dq = dq.view(-1, ctx.num_heads, ctx.attn_dim) + dk = dk.view(-1, ctx.num_heads, ctx.attn_dim) + dv = dv.view(-1, ctx.num_heads, ctx.hidden_dim) + if ( + ctx.recompute_uvqk_in_backward and ctx.has_rotary_weights + ): # recompute ROPE on qk + q = triton_apply_rope_fwd( + x=q, + N=ctx.max_seq_len, + seq_offsets=seq_offsets, + cos_rope=q_cos_weights, + sin_rope=q_sin_weights, + ) + k = triton_apply_rope_fwd( + x=k, + N=ctx.max_seq_len, + seq_offsets=seq_offsets, + cos_rope=k_cos_weights, + sin_rope=k_sin_weights, + ) + dq = dq.view(-1, ctx.num_heads, ctx.attn_dim) + dk = dk.view(-1, ctx.num_heads, ctx.attn_dim) + dv = dv.view(-1, ctx.num_heads, ctx.hidden_dim) + # Note: the two operations below update duvqk in place + if is_sm100_plus(): + _dq, _dk, _dv = torch.ops.bw_hstu.bw_hstu_mha_bwd( + ctx.max_seq_len, + ctx.alpha, + dout, + q, + k, + v, + dq, + dk, + dv, + seq_offsets, + True, # causal + num_targets, + attn_scale, + ctx.max_attn_len, + ctx.full_attn_size, + ctx.contextual_seq_len, + ctx.sort_by_length, + False, # deterministic + 0, # sm_margin + ctx.max_seq_len, # max_q_len, + None, # seq_offsets_q, + None, # max_seq_len_tensor, + None, # contextual_seq_len_tensor, + None, # max_attn_len_tensor, + None, # min_full_attn_seq_len_tensor, + 1, # num_groups + ctx.num_softmax_heads, # num_softmax_heads + out, # out + softmax_lse, # lse + ) + else: + _dq, _dk, _dv = torch.ops.hstu.hstu_mha_bwd( + ctx.max_seq_len, + ctx.alpha, + dout, + q, + k, + v, + dq, + dk, + dv, + out, + seq_offsets, + True, # causal + num_targets, + attn_scale, + ctx.max_attn_len, + ctx.full_attn_size, + ctx.contextual_seq_len, + ctx.sort_by_length, + False, # deterministic + 0, # sm_margin + 0, # max_q_len, + None, # seq_offsets_q, + ctx.num_softmax_heads, # num_softmax_heads, + softmax_lse, + ) + if ctx.has_rotary_weights: + _dq = triton_apply_rope_bwd( + grad=_dq, + N=ctx.max_seq_len, + seq_offsets=seq_offsets, + cos_rope=q_cos_weights, + sin_rope=q_sin_weights, + ) + _dk = triton_apply_rope_bwd( + grad=_dk, + N=ctx.max_seq_len, + seq_offsets=seq_offsets, + cos_rope=k_cos_weights, + sin_rope=k_sin_weights, + ) + copy_if_different_ptr(dq, _dq) + copy_if_different_ptr(dk, _dk) + copy_if_different_ptr(dv, _dv) + if ctx.silu_u: + torch.ops.aten.silu_backward(_du, u, grad_input=du) + else: + copy_if_different_ptr(du, _du) + d_normed_x, d_uvqk_weight, d_uvqk_bias = triton_addmm_bwd( + x=normed_x, + w=uvqk_weight, + dz=duvqk, + is_y_1d=ctx.uvqk_bias_1d and ctx.has_uvqk_bias, + ) + d_x, d_norm_weight, d_norm_bias = triton_weighted_layer_norm_bwd( + dy=d_normed_x, + x=x, + weight=norm_weight, + bias=norm_bias, + mean=x_mean, + rstd=x_rstd, + learnable=True, + eps=ctx.norm_eps, + BLOCK_D=ctx.norm_BLOCK_D, + ) + # pyre-ignore[7] + return ( + d_x, + d_norm_weight, + d_norm_bias, + None, + None, + None, + None, + d_uvqk_weight, + d_uvqk_bias if ctx.has_uvqk_bias else None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + ) diff --git a/recommendation_v4/generative_recommenders/ops/cpp/expand_1d_jagged_to_dense.cpp b/recommendation_v4/generative_recommenders/ops/cpp/expand_1d_jagged_to_dense.cpp new file mode 100644 index 000000000..4730078e6 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/expand_1d_jagged_to_dense.cpp @@ -0,0 +1,97 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace hstu { + +template +void expand_1d_jagged_to_dense_cpu_kernel_( + int64_t B, + int64_t max_len, + const at::TensorAccessor& values, + const at::TensorAccessor& offsets, + at::TensorAccessor output) { + for (auto i : c10::irange(B)) { + int64_t begin = offsets[i]; + int64_t end = offsets[i + 1]; + if (end - begin == 0) { + for (int64_t j : c10::irange(max_len)) { + output[i][j] = 0; + continue; + } + } else { + int64_t j = 0; + for (; j < std::min(end - begin, max_len); ++j) { + output[i][j] = values[begin + j]; + } + for (; j < max_len; ++j) { + output[i][j] = values[end - 1]; + } + } + } // for each i +} + +at::Tensor expand_1d_jagged_to_dense_cpu( + const at::Tensor& values, + const at::Tensor& offsets, + const int64_t max_len) { + TORCH_INTERNAL_ASSERT(values.device().type() == at::DeviceType::CPU); + TORCH_INTERNAL_ASSERT(offsets.device().type() == at::DeviceType::CPU); + TORCH_CHECK(values.numel() < std::numeric_limits::max()); + TORCH_CHECK(max_len >= 0); + auto B = offsets.size(0) - 1; + auto output = at::empty({B, max_len}, values.options()); + if (values.numel() == 0 || max_len == 0) { + return output; + } + AT_DISPATCH_ALL_TYPES_AND2( + at::ScalarType::BFloat16, + at::ScalarType::Half, + values.scalar_type(), + "expand_1d_jagged_to_dense_cpu_input1", + [&] { + using val_t = scalar_t; + AT_DISPATCH_INTEGRAL_TYPES( + offsets.scalar_type(), "expand_1d_jagged_to_dense_cpu_input2", [&] { + using index_t = scalar_t; + expand_1d_jagged_to_dense_cpu_kernel_( + B, + max_len, + values.accessor(), + offsets.accessor(), + output.accessor()); + }); + }); + return output; +} + +at::Tensor expand_1d_jagged_to_dense_meta( + const at::Tensor& values, + const at::Tensor& offsets, + const c10::SymInt max_len) { + auto B = offsets.sym_size(0) - 1; + auto output = at::empty_symint({B, max_len}, values.options()); + return output; +} + +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/expand_1d_jagged_to_dense.cu b/recommendation_v4/generative_recommenders/ops/cpp/expand_1d_jagged_to_dense.cu new file mode 100644 index 000000000..aa3678d2b --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/expand_1d_jagged_to_dense.cu @@ -0,0 +1,103 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "common.h" + +static constexpr int32_t kMaxThreads = 1024; + +namespace hstu { + +template +__global__ +__launch_bounds__(kMaxThreads) void expand_1d_jagged_to_dense_cuda_kernel_( + int64_t B, + int64_t max_len, + const at::PackedTensorAccessor32 values, + const at::PackedTensorAccessor32 offsets, + at::PackedTensorAccessor32 output) { + int64_t b = blockIdx.y; + int64_t begin = offsets[b]; + int64_t i = blockIdx.x * blockDim.x + threadIdx.x; + int64_t end = offsets[b + 1]; + if (end - begin == 0) { + if (i < max_len) { + output[b][i] = 0; + } + } else { + if (i < std::min(end - begin, max_len)) { + output[b][i] = values[i + begin]; + } else if (i < max_len) { + output[b][i] = values[end - 1]; + } + } +} + +at::Tensor expand_1d_jagged_to_dense_cuda( + const at::Tensor& values, + const at::Tensor& offsets, + const int64_t max_len) { + at::cuda::OptionalCUDAGuard device_guard; + device_guard.set_index(values.get_device()); + TORCH_INTERNAL_ASSERT(values.device().type() == at::DeviceType::CUDA); + TORCH_INTERNAL_ASSERT(offsets.device().type() == at::DeviceType::CUDA); + TORCH_CHECK(values.numel() < std::numeric_limits::max()); + TORCH_CHECK(values.get_device() == offsets.get_device()); + TORCH_CHECK(max_len >= 0); + auto B = offsets.size(0) - 1; + auto output = at::empty({B, max_len}, values.options()); + if (values.numel() == 0 || max_len == 0) { + return output; + } + uint32_t nthreads_per_block = max_len > 64 ? 64 : max_len; + dim3 grid_size = dim3(div_round_up(max_len, nthreads_per_block), B); + AT_DISPATCH_ALL_TYPES_AND2( + at::ScalarType::BFloat16, + at::ScalarType::Half, + values.scalar_type(), + "expand_1d_jagged_to_dense_cuda_input1", + [&] { + using val_t = scalar_t; + AT_DISPATCH_INTEGRAL_TYPES( + offsets.scalar_type(), + "expand_1d_jagged_to_dense_cuda_input2", + [&] { + using index_t = scalar_t; + expand_1d_jagged_to_dense_cuda_kernel_<<< + grid_size, + nthreads_per_block, + 0, + at::cuda::getCurrentCUDAStream()>>>( + B, + max_len, + values.packed_accessor32(), + offsets + .packed_accessor32(), + output.packed_accessor32()); + C10_CUDA_KERNEL_LAUNCH_CHECK(); + }); + }); + + return output; +} + +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/copy_sm90_bulk_reduce.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/copy_sm90_bulk_reduce.h new file mode 100644 index 000000000..a22ae7745 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/copy_sm90_bulk_reduce.h @@ -0,0 +1,66 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/****************************************************************************** + * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, + *Pradeep Ramani, Tri Dao. + ******************************************************************************/ + +#pragma once + +#include + +namespace cute { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +struct SM90_BULK_REDUCE_ADD { + CUTE_HOST_DEVICE static void + copy(float const* smem_ptr, float* gmem_ptr, int32_t store_bytes) { +#if defined(CUTE_ARCH_TMA_SM90_ENABLED) + uint32_t smem_int_ptr = cast_smem_ptr_to_uint(smem_ptr); + asm volatile( + "cp.reduce.async.bulk.global.shared::cta.bulk_group.add.f32 [%0], [%1], %2;\n" + : + : "l"(gmem_ptr), "r"(smem_int_ptr), "r"(store_bytes) + : "memory"); +#else + CUTE_INVALID_CONTROL_PATH( + "Trying to use BULK_REDUCE_ADD without CUTE_ARCH_TMA_SM90_ENABLED."); +#endif + } + + CUTE_HOST_DEVICE static void copy( + float const* smem_ptr, + float* gmem_ptr, + int32_t store_bytes, + uint64_t cache_hint) { +#if defined(CUTE_ARCH_TMA_SM90_ENABLED) + uint32_t smem_int_ptr = cast_smem_ptr_to_uint(smem_ptr); + asm volatile( + "cp.reduce.async.bulk.global.shared::cta.bulk_group.L2::cache_hint.add.f32 [%0], [%1], %2, %3;\n" + : + : "l"(gmem_ptr), "r"(smem_int_ptr), "r"(store_bytes), "l"(cache_hint) + : "memory"); +#else + CUTE_INVALID_CONTROL_PATH( + "Trying to use BULK_REDUCE_ADD without CUTE_ARCH_TMA_SM90_ENABLED."); +#endif + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // end namespace cute diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/epilogue_bwd.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/epilogue_bwd.h new file mode 100644 index 000000000..833f3ae28 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/epilogue_bwd.h @@ -0,0 +1,481 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/****************************************************************************** + * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, + *Pradeep Ramani, Tri Dao. + ******************************************************************************/ + +#pragma once + +#include +#include + +#include "cutlass/gemm/collective/builders/sm90_common.inl" + +#include "copy_sm90_bulk_reduce.h" +#include "named_barrier.h" +#include "seqlen.h" +#include "utils.h" + +namespace hstu { + +using namespace cute; + +template < + class TileShape_MNK_, + class Element_, + class ArchTag_, + int NumEpilogueThreads_, + bool Jagged, + bool dKV_swapAB_, + int AtomLayoutKdKV = 1> +struct CollectiveEpilogueBwd { + using TileShape_MNK = TileShape_MNK_; + using Element = Element_; + using ArchTag = ArchTag_; + static constexpr int NumEpilogueThreads = NumEpilogueThreads_; + static constexpr bool dKV_swapAB = dKV_swapAB_; + static constexpr bool Use_TMA = + !Jagged && ArchTag::kMinComputeCapability >= 90; + + static_assert(ArchTag::kMinComputeCapability >= 80); + + using GmemTiledCopydKVTMA = cute::SM90_TMA_STORE; + + // These are for storing the output tensor without TMA (e.g., for setting + // output to zero) + static constexpr int kGmemElemsPerLoad = + sizeof(cute::uint128_t) / sizeof(Element); + static_assert( + get<2>(TileShape_MNK{}) % kGmemElemsPerLoad == 0, + "Headdim must be a multiple of kGmemElemsPerLoad"); + static constexpr int kHeadDim = get<2>(TileShape_MNK{}); + static constexpr int kGmemThreadsPerRow = + cutlass::gcd(kHeadDim / kGmemElemsPerLoad, NumEpilogueThreads); + static_assert( + NumEpilogueThreads % kGmemThreadsPerRow == 0, + "NumEpilogueThreads must be a multiple of kGmemThreadsPerRow"); + using GmemLayoutAtom = Layout< + Shape< + Int, + Int>, + Stride, _1>>; + using GmemTiledCopydKV = decltype(make_tiled_copy( + Copy_Atom, Element>{}, + GmemLayoutAtom{}, + Layout>>{})); // Val layout, 8 or 16 vals + // per store + + using SmemLayoutAtomdKVTMA = + decltype(cutlass::gemm::collective::detail::ss_smem_selector< + GMMA::Major::K, + Element, + // TODO: do we have to change this if dKV_swapAB is true? + decltype(cute::get<1>(TileShape_MNK{})), + Int(TileShape_MNK{})) / + AtomLayoutKdKV>>()); + using SmemLayoutdKVTMA = decltype(tile_to_shape( + SmemLayoutAtomdKVTMA{}, + select<1, 2>(TileShape_MNK{}))); + using SmemLayoutdKVtTMA = decltype(cute::composition( + SmemLayoutdKVTMA{}, + make_layout( + make_shape(get<2>(TileShape_MNK{}), get<1>(TileShape_MNK{})), + make_stride(decltype(get<1>(TileShape_MNK{})){}, _1{})))); + + // If we don't use TMA + static constexpr int kBlockKSmem = + kHeadDim % 64 == 0 ? 64 : (kHeadDim % 32 == 0 ? 32 : 16); + static constexpr int kSwizzle = + kBlockKSmem == 64 ? 3 : (kBlockKSmem == 32 ? 2 : 1); + using SmemLayoutAtomdKVSTG = decltype(composition( + Swizzle{}, + Layout, Int>, Stride, _1>>{})); + + using SmemLayoutAtomdKV = + std::conditional_t; + using SmemLayoutdKV = decltype(tile_to_shape( + SmemLayoutAtomdKV{}, + select<1, 2>(TileShape_MNK{}))); + using SmemLayoutdKVt = decltype(cute::composition( + SmemLayoutdKV{}, + make_layout( + make_shape(get<2>(TileShape_MNK{}), get<1>(TileShape_MNK{})), + make_stride(decltype(get<1>(TileShape_MNK{})){}, _1{})))); + + using SmemCopyAtomdKV = Copy_Atom< + std::conditional_t< + ArchTag::kMinComputeCapability >= 90, + std::conditional_t< + !dKV_swapAB, + cute::SM90_U32x4_STSM_N, + cute::SM90_U16x8_STSM_T>, + AutoVectorizingCopyWithAssumedAlignment<128>>, + Element>; + + static constexpr size_t SmemAlignmentdKV = + ArchTag::kMinComputeCapability >= 90 + ? cutlass::detail::alignment_for_swizzle(SmemLayoutdKV{}) + : 128; + static_assert(SmemAlignmentdKV >= 128, "Require at least 128B alignment"); + + struct TensorStorage : cute::aligned_struct { + cute:: + array_aligned, SmemAlignmentdKV> + smem_dk; + cute:: + array_aligned, SmemAlignmentdKV> + smem_dv; + }; + + using ShapedKV = + cute::Shape; // (seqlen_k, d, head, + // batch) + using StridedKV = cute::Stride; + + using TMA_dKV = std::conditional_t< + Use_TMA, + decltype(make_tma_copy( + GmemTiledCopydKVTMA{}, + make_tensor( + make_gmem_ptr(static_cast(nullptr)), + ShapedKV{}, + StridedKV{}), + SmemLayoutdKVTMA{}, + select<1, 2>(TileShape_MNK{}), + _1{})), // no mcast for dKV + std::nullptr_t>; + + // Host side kernel arguments + struct Arguments { + Element* ptr_dK; + ShapedKV const shape_dK; + StridedKV const stride_dK; + Element* ptr_dV; + StridedKV const stride_dV; + int const num_heads_q; + int const* seq_offsets; + }; + + // Device side kernel params + struct Params { + Element* ptr_dK; + ShapedKV const shape_dK; + StridedKV const stride_dK; + Element* ptr_dV; + StridedKV const stride_dV; + TMA_dKV tma_store_dK, tma_store_dV; + int const* seq_offsets = nullptr; + }; + + static Params to_underlying_arguments(Arguments const& args) { + Tensor mdK = + make_tensor(make_gmem_ptr(args.ptr_dK), args.shape_dK, args.stride_dK); + Tensor mdV = + make_tensor(make_gmem_ptr(args.ptr_dV), args.shape_dK, args.stride_dV); + TMA_dKV tma_store_dK = [&] { + if constexpr (Use_TMA) { + return make_tma_copy( + GmemTiledCopydKVTMA{}, + mdK, + SmemLayoutdKVTMA{}, + select<1, 2>(TileShape_MNK{}), + _1{}); // no mcast for dKV + } else { + return nullptr; + } + }(); + TMA_dKV tma_store_dV = [&] { + if constexpr (Use_TMA) { + return make_tma_copy( + GmemTiledCopydKVTMA{}, + mdV, + SmemLayoutdKVTMA{}, + select<1, 2>(TileShape_MNK{}), + _1{}); // no mcast for dKV + } else { + return nullptr; + } + }(); + return { + args.ptr_dK, + args.shape_dK, + args.stride_dK, + args.ptr_dV, + args.stride_dV, + tma_store_dK, + tma_store_dV, + args.seq_offsets}; + } + + /// Issue Tma Descriptor Prefetch -- ideally from a single thread for best + /// performance + CUTLASS_DEVICE + static void prefetch_tma_descriptors(Params const& params) { + if constexpr (Use_TMA) { + cute::prefetch_tma_descriptor(params.tma_store_dK.get_tma_descriptor()); + cute::prefetch_tma_descriptor(params.tma_store_dV.get_tma_descriptor()); + } + } + + template + CUTLASS_DEVICE void store( + Params const& params, + FrgTensorO const& tdKrdK, + FrgTensorO const& tdVrdV, + SharedStorage& shared_storage, + TiledMma tiled_mma, + int thread_idx, + cute::tuple const& block_coord) { + auto [n_block, bidh, bidb] = block_coord; + Tensor sdK = cute::as_position_independent_swizzle_tensor(make_tensor( + make_smem_ptr(shared_storage.tensors.epilogue.smem_dk.data()), + SmemLayoutdKV{})); + Tensor sdV = cute::as_position_independent_swizzle_tensor(make_tensor( + make_smem_ptr(shared_storage.tensors.epilogue.smem_dv.data()), + SmemLayoutdKV{})); + Tensor sdKt = cute::as_position_independent_swizzle_tensor(make_tensor( + make_smem_ptr(shared_storage.tensors.epilogue.smem_dk.data()), + SmemLayoutdKVt{})); + Tensor sdVt = cute::as_position_independent_swizzle_tensor(make_tensor( + make_smem_ptr(shared_storage.tensors.epilogue.smem_dv.data()), + SmemLayoutdKVt{})); + auto smem_tiled_copy_dKV = make_tiled_copy_C(SmemCopyAtomdKV{}, tiled_mma); + auto smem_thr_copy_dKV = smem_tiled_copy_dKV.get_thread_slice(thread_idx); + + Tensor tdVrdV_out = make_tensor_like(tdVrdV); + hstu::convert_type_out(tdVrdV, tdVrdV_out); + Tensor tdKrdK_out = make_tensor_like(tdKrdK); + hstu::convert_type_out(tdKrdK, tdKrdK_out); + Tensor taccdKrdK = smem_thr_copy_dKV.retile_S( + tdKrdK_out); // ((Atom,AtomNum), MMA_M, MMA_N) + Tensor taccdVrdV = smem_thr_copy_dKV.retile_S( + tdVrdV_out); // ((Atom,AtomNum), MMA_M, MMA_N) + // if (blockIdx.x == 0 && threadIdx.x == 128) { print(smem_thr_copy_dKV); + // print(sdK); printf("\n"); print(sdKt); printf("\n"); } + Tensor taccdKsdK = smem_thr_copy_dKV.partition_D( + cute::conditional_return( + sdK, sdKt)); // ((Atom,AtomNum),PIPE_M,PIPE_N) + Tensor taccdVsdV = smem_thr_copy_dKV.partition_D( + cute::conditional_return( + sdV, sdVt)); // ((Atom,AtomNum),PIPE_M,PIPE_N) + + // Make sure all WGs have finished reading K and V + hstu::named_barrier_sync( + NumEpilogueThreads, + cutlass::arch::ReservedNamedBarriers::EpilogueBarrier); + cute::copy(smem_tiled_copy_dKV, taccdVrdV, taccdVsdV); + cute::copy(smem_tiled_copy_dKV, taccdKrdK, taccdKsdK); + if constexpr (Use_TMA) { + cutlass::arch::fence_view_async_shared(); // ensure smem writes are + // visible to TMA + cutlass::arch::NamedBarrier::arrive( + NumEpilogueThreads + cutlass::NumThreadsPerWarp, + cutlass::arch::ReservedNamedBarriers::EpilogueBarrier); + + Tensor mdK = params.tma_store_dK.get_tma_tensor(params.shape_dK); + Tensor mdV = params.tma_store_dV.get_tma_tensor(params.shape_dK); + Tensor gdK = local_tile( + mdK(_, _, bidh, bidb), + select<1, 2>(TileShape_MNK{}), + make_coord(n_block, _0{})); // (M, K) + Tensor gdV = local_tile( + mdV(_, _, bidh, bidb), + select<1, 2>(TileShape_MNK{}), + make_coord(n_block, _0{})); // (M, K) + auto block_tma_dK = params.tma_store_dK.get_slice(_0{}); + auto block_tma_dV = params.tma_store_dV.get_slice(_0{}); + Tensor tdKgdK = block_tma_dK.partition_D(gdK); // (TMA, TMA_M, TMA_K) + Tensor tdKsdK = block_tma_dK.partition_S(sdK); // (TMA, TMA_M, TMA_K) + Tensor tdVgdV = block_tma_dV.partition_D(gdV); // (TMA, TMA_M, TMA_K) + Tensor tdVsdV = block_tma_dV.partition_S(sdV); // (TMA, TMA_M, TMA_K) + int warp_idx_sync = + __shfl_sync(0xffffffff, thread_idx / cutlass::NumThreadsPerWarp, 0); + if (warp_idx_sync == + NumEpilogueThreads / cutlass::NumThreadsPerWarp - 1) { + cutlass::arch::NamedBarrier::sync( + NumEpilogueThreads + cutlass::NumThreadsPerWarp, + cutlass::arch::ReservedNamedBarriers::EpilogueBarrier); + if (cute::elect_one_sync()) { + cute::copy(params.tma_store_dV, tdVsdV, tdVgdV); + cute::copy(params.tma_store_dK, tdKsdK, tdKgdK); + tma_store_arrive(); + } + } + tma_store_wait<0>(); + // // Tell warp 0 that smem_k and smem_v are ready + // cutlass::arch::NamedBarrier::arrive(NumEpilogueThreads + + // cutlass::NumThreadsPerWarp, + // static_cast(BwdNamedBarriers::KVEmpty) /*id*/); + + } else { + hstu::named_barrier_sync( + NumEpilogueThreads, + cutlass::arch::ReservedNamedBarriers::EpilogueBarrier); + static constexpr int kBlockN = get<1>(TileShape_MNK{}); + hstu::SeqlenInfo seqlen_info{ + bidb, size<0>(params.shape_dK), params.seq_offsets}; + Tensor mdK = make_tensor( + make_gmem_ptr(params.ptr_dK), params.shape_dK, params.stride_dK)( + _, _, bidh, !Jagged ? bidb : 0); + Tensor gdK = local_tile( + cute::domain_offset(make_coord(seqlen_info.offset, _0{}), mdK), + select<1, 2>(TileShape_MNK{}), + make_coord(n_block, _0{})); // (M, K) + Tensor mdV = make_tensor( + make_gmem_ptr(params.ptr_dV), params.shape_dK, params.stride_dV)( + _, _, bidh, !Jagged ? bidb : 0); + Tensor gdV = local_tile( + cute::domain_offset(make_coord(seqlen_info.offset, _0{}), mdV), + select<1, 2>(TileShape_MNK{}), + make_coord(n_block, _0{})); // (M, K) + + GmemTiledCopydKV gmem_tiled_copy_dKV; + auto gmem_thr_copy_dKV = gmem_tiled_copy_dKV.get_thread_slice(thread_idx); + Tensor tdKVgdV = gmem_thr_copy_dKV.partition_D(gdV); + Tensor tdKVsdV = + gmem_thr_copy_dKV.partition_S(sdV); // (TMA, TMA_M, TMA_K) + Tensor tdKVgdK = gmem_thr_copy_dKV.partition_D(gdK); + Tensor tdKVsdK = + gmem_thr_copy_dKV.partition_S(sdK); // (TMA, TMA_M, TMA_K) + Tensor tdKVrdV = make_fragment_like(tdKVgdV); + Tensor tdKVrdK = make_fragment_like(tdKVgdK); + Tensor cdKV = cute::make_identity_tensor( + select<1, 2>(TileShape_MNK{})); // (BLK_M,BLK_K) -> (blk_m,blk_k) + // Repeat the partitioning with identity layouts + Tensor tdKVcdKV = gmem_thr_copy_dKV.partition_D(cdKV); + Tensor tdKVpdKV = make_tensor(make_shape(size<2>(tdKVgdV))); +#pragma unroll + for (int k = 0; k < size(tdKVpdKV); ++k) { + tdKVpdKV(k) = get<1>(tdKVcdKV(_0{}, _0{}, k)) < get<1>(params.shape_dK); + } + // Need to check OOB when reading from smem if kBlockN isn't evenly tiled + static constexpr bool EvenN = + kBlockN % CUTE_STATIC_V(size<0>(GmemLayoutAtom{})) == 0; + hstu::copy< + /*Is_even_MN=*/EvenN, + /*Is_even_K=*/true, + /*Clear_OOB_MN=*/false>( + gmem_tiled_copy_dKV, tdKVsdV, tdKVrdV, tdKVcdKV, tdKVpdKV, kBlockN); + hstu::copy< + /*Is_even_MN=*/EvenN, + /*Is_even_K=*/true, + /*Clear_OOB_MN=*/false>( + gmem_tiled_copy_dKV, tdKVsdK, tdKVrdK, tdKVcdKV, tdKVpdKV, kBlockN); + // // Tell warp 0 that smem_k and smem_v are ready + // cutlass::arch::fence_view_async_shared(); // ensure smem reads are done + // before next TMA to smem_k/v + // hstu::named_barrier_arrive(NumEpilogueThreads + + // cutlass::NumThreadsPerWarp, + // static_cast(BwdNamedBarriers::KVEmpty) /*id*/); Construct + // identity layout for gdKV Clear_OOB_K must be false since we don't want + // to write zeros to gmem + hstu::copy< + /*Is_even_MN=*/false, + /*Is_even_K=*/false, + /*Clear_OOB_MN=*/false, + /*Clear_OOB_K=*/false>( + gmem_tiled_copy_dKV, + tdKVrdV, + tdKVgdV, + tdKVcdKV, + tdKVpdKV, + std::min(seqlen_info.seqlen - n_block * kBlockN, kBlockN)); + hstu::copy< + /*Is_even_MN=*/false, + /*Is_even_K=*/false, + /*Clear_OOB_MN=*/false, + /*Clear_OOB_K=*/false>( + gmem_tiled_copy_dKV, + tdKVrdK, + tdKVgdK, + tdKVcdKV, + tdKVpdKV, + std::min(seqlen_info.seqlen - n_block * kBlockN, kBlockN)); + } + } + + CUTLASS_DEVICE void store_tail() { + // if constexpr (Use_TMA) { tma_store_wait<0>(); } + } + + // Write 0 to dK and dV + CUTLASS_DEVICE void store_zero( + Params const& params, + int thread_idx, + cute::tuple const& block_coord) { + static constexpr int kBlockN = get<1>(TileShape_MNK{}); + auto [n_block, bidh, bidb] = block_coord; + hstu::SeqlenInfo seqlen_info{ + bidb, size<0>(params.shape_dK), params.seq_offsets}; + Tensor mdK = make_tensor( + make_gmem_ptr(params.ptr_dK), params.shape_dK, params.stride_dK)( + _, _, bidh, !Jagged ? bidb : 0); + Tensor gdK = local_tile( + cute::domain_offset(make_coord(seqlen_info.offset, _0{}), mdK), + select<1, 2>(TileShape_MNK{}), + make_coord(n_block, _0{})); // (M, K) + Tensor mdV = make_tensor( + make_gmem_ptr(params.ptr_dV), params.shape_dK, params.stride_dV)( + _, _, bidh, !Jagged ? bidb : 0); + Tensor gdV = local_tile( + cute::domain_offset(make_coord(seqlen_info.offset, _0{}), mdV), + select<1, 2>(TileShape_MNK{}), + make_coord(n_block, _0{})); // (M, K) + + GmemTiledCopydKV gmem_tiled_copy_dKV; + auto gmem_thr_copy_dKV = gmem_tiled_copy_dKV.get_thread_slice(thread_idx); + Tensor tdKVgdK = gmem_thr_copy_dKV.partition_D(gdK); + Tensor tdKVgdV = gmem_thr_copy_dKV.partition_D(gdV); + Tensor tdKVrdKV = make_fragment_like(tdKVgdK); + clear(tdKVrdKV); + // Construct identity layout for gdKV + Tensor cdKV = cute::make_identity_tensor( + select<1, 2>(TileShape_MNK{})); // (BLK_M,BLK_K) -> (blk_m,blk_k) + // Repeat the partitioning with identity layouts + Tensor tdKVcdKV = gmem_thr_copy_dKV.partition_D(cdKV); + Tensor tdKVpdKV = make_tensor(make_shape(size<2>(tdKVgdK))); +#pragma unroll + for (int k = 0; k < size(tdKVpdKV); ++k) { + tdKVpdKV(k) = get<1>(tdKVcdKV(_0{}, _0{}, k)) < get<1>(params.shape_dK); + } + // Clear_OOB_K must be false since we don't want to write zeros to gmem + hstu::copy< + /*Is_even_MN=*/false, + /*Is_even_K=*/false, + /*Clear_OOB_MN=*/false, + /*Clear_OOB_K=*/false>( + gmem_tiled_copy_dKV, + tdKVrdKV, + tdKVgdK, + tdKVcdKV, + tdKVpdKV, + seqlen_info.seqlen - n_block * kBlockN); + hstu::copy< + /*Is_even_MN=*/false, + /*Is_even_K=*/false, + /*Clear_OOB_MN=*/false, + /*Clear_OOB_K=*/false>( + gmem_tiled_copy_dKV, + tdKVrdKV, + tdKVgdV, + tdKVcdKV, + tdKVpdKV, + seqlen_info.seqlen - n_block * kBlockN); + } +}; + +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/epilogue_fwd.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/epilogue_fwd.h new file mode 100644 index 000000000..c794a114c --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/epilogue_fwd.h @@ -0,0 +1,550 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/****************************************************************************** + * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, + *Pradeep Ramani, Tri Dao. + ******************************************************************************/ + +#pragma once + +#include +#include // For FastDivMod +#include "cute/tensor.hpp" + +#include "cutlass/epilogue/collective/builders/sm90_common.inl" +#include "cutlass/gemm/collective/builders/sm90_common.inl" + +#include "named_barrier.h" +#include "seqlen.h" +#include "utils.h" + +namespace hstu { + +using namespace cute; + +template < + class TileShape_MNK_, + class ClusterShape_, + class Element_, + class ArchTag_, + int NumEpilogueThreads_, + bool Jagged, + bool FP8PermuteCol = false> +struct CollectiveEpilogueFwd { + using TileShape_MNK = TileShape_MNK_; + using ClusterShape = ClusterShape_; + using Element = Element_; + using ArchTag = ArchTag_; + static constexpr int NumEpilogueThreads = NumEpilogueThreads_; + static constexpr bool Use_smem = sizeof(Element) <= 2; + static constexpr bool Use_TMA_O = + ArchTag::kMinComputeCapability >= 90 && !Jagged && Use_smem; + + static_assert(ArchTag::kMinComputeCapability >= 80); + static_assert( + ArchTag::kMinComputeCapability >= 90 || + CUTE_STATIC_V(size(ClusterShape{})) == 1); + + static constexpr int kBlockM = get<0>(TileShape_MNK{}); + static constexpr int kHeadDim = get<2>(TileShape_MNK{}); + + using GmemTiledCopyOTMA = cute::SM90_TMA_STORE; + + // These are for storing the output tensor without TMA (e.g., for setting + // output to zero) + static constexpr int kGmemElemsPerStore = + sizeof(cute::uint128_t) / sizeof(Element); + static_assert( + kHeadDim % kGmemElemsPerStore == 0, + "Headdim must be a multiple of kGmemElemsPerStore"); + // We want each "row" to have 64 elements (128 bytes, i.e. 1 cache line). We + // want each thread to have 4 elements in the M direction and 2 elements in + // the K direction. In the case of PackGQA, this reduces the number of times + // we need to call divmod. + static constexpr int kBytePerRow = kHeadDim * sizeof(Element); + static constexpr int kBlockKGmem = + (kBytePerRow % 128 == 0 ? 128 : (kBytePerRow % 64 == 0 ? 64 : 32)) / + sizeof(Element); + // static constexpr int kBlockKGmem = kHeadDim % 128 == 0 ? 128 : (kHeadDim % + // 64 == 0 ? 64 : 32); static constexpr int kGmemThreadsPerRow = + // cutlass::gcd(kHeadDim / kGmemElemsPerStore, NumEpilogueThreads); + static constexpr int kGmemThreadsPerRow = kBlockKGmem / kGmemElemsPerStore; + // If PackGQA, we split the work of compute O_ptr among threads in the same + // row, so we need this to within a warp + static_assert(cutlass::NumThreadsPerWarp % kGmemThreadsPerRow == 0); + static_assert( + NumEpilogueThreads % kGmemThreadsPerRow == 0, + "NumEpilogueThreads must be a multiple of kGmemThreadsPerRow"); + using GmemLayoutAtom = Layout< + Shape< + Int, + Int>, + Stride, _1>>; + static_assert( + kBlockM % CUTE_STATIC_V(shape<0>(GmemLayoutAtom{})) == 0, + "kBlockM must be a multiple of NumEpilogueThreads / kGmemThreadsPerRow"); + using GmemTiledCopyO = decltype(make_tiled_copy( + Copy_Atom, Element>{}, + GmemLayoutAtom{}, + Layout>>{})); // Val layout, 8 or 16 + // vals per store + + using SmemLayoutAtomOTMA = + decltype(cutlass::gemm::collective::detail::ss_smem_selector< + GMMA::Major::K, + Element, + decltype(cute::get<0>(TileShape_MNK{})), + decltype(cute::get<2>(TileShape_MNK{}))>()); + using SmemLayoutOTMA = decltype(tile_to_shape( + SmemLayoutAtomOTMA{}, + select<0, 2>(TileShape_MNK{}))); + static constexpr int kSwizzle = kBlockKGmem == 128 + ? 4 + : (kBlockKGmem == 64 ? 3 : (kBlockKGmem == 32 ? 2 : 1)); + static constexpr int kSwizzleBase = + sizeof(Element) == 4 ? 2 : (sizeof(Element) == 2 ? 3 : 4); + using SmemLayoutAtomO = decltype(composition( + Swizzle{}, + Layout>, Stride, _1>>{})); + using SmemLayoutOSTS = + decltype(tile_to_shape(SmemLayoutAtomO{}, select<0, 2>(TileShape_MNK{}))); + using SmemLayoutO = std::conditional_t< + ArchTag::kMinComputeCapability >= 90, + SmemLayoutOTMA, + SmemLayoutOSTS>; + + using ShapeO = + cute::Shape; // (seqlen_q, d, + // head, batch, + // num_splits) + using StrideO = cute::Stride; + // ((qhead_per_khead, seqlen_q), d, nheads, batch, num_splits) + using ShapeOPacked = ShapeO; + using StrideOPacked = StrideO; + // ((qhead_per_khead, seqlen_q), nheads, batch, num_splits) + using StrideLSE = + cute::Stride<_1, int64_t, int64_t, int64_t>; // (seqlen_q, head, batch, + // num_splits) + using ShapeLSEPacked = cute::Shape; + using StrideLSEPacked = StrideLSE; + using EpilogueTile_MN = decltype(select<0, 1>(TileShape_MNK{})); + using CopyOpR2S = std::conditional_t< + ArchTag::kMinComputeCapability >= 90, + // cute::SM90_U32x4_STSM_N if Element size is 2 bytes (fp16, bf16) + decltype(cutlass::epilogue::collective::detail:: + sm90_get_smem_store_op_for_accumulator< + StrideO, + Element, + EpilogueTile_MN>()), + AutoVectorizingCopyWithAssumedAlignment<128>>; + using SmemCopyAtomO = Copy_Atom; + + // static constexpr size_t SmemAlignmentO = + // cutlass::detail::alignment_for_swizzle(SmemLayoutO{}); + // static_assert(SmemAlignmentO >= 128, "Require at least 128B alignment"); + // struct TensorStorage : cute::aligned_struct { + // cute::array_aligned : + // 0, SmemAlignmentO> smem_o; + // }; + struct TensorStorage : cute::aligned_struct<128> { + cute::array_aligned : 0> + smem_o; + }; + + using TMA_O = std::conditional_t< + Use_TMA_O, + decltype(make_tma_copy( + GmemTiledCopyOTMA{}, + make_tensor( + make_gmem_ptr(static_cast(nullptr)), + ShapeO{}, + StrideO{}), + SmemLayoutOTMA{}, + select<0, 2>(TileShape_MNK{}), + _1{})), // no mcast for O + std::nullptr_t>; + + // Host side kernel arguments + struct Arguments { + Element* ptr_O; + ShapeO const shape_O; + StrideO const stride_O; + int32_t const nheads; + int32_t const num_softmax_heads; + StrideLSE const stride_lse; + float* ptr_lse = nullptr; + int const* seq_offsets = nullptr; + }; + + // Device side kernel params + struct Params { + Element* ptr_O; + ShapeO const shape_O; + StrideO const stride_O; + ShapeOPacked const shape_O_packed; + StrideOPacked const stride_O_packed; + float* ptr_lse; + StrideLSE const stride_lse; + ShapeLSEPacked const shape_lse_packed; + StrideLSEPacked const stride_lse_packed; + TMA_O tma_store_O; + int const* seq_offsets = nullptr; + }; + + static Params to_underlying_arguments(Arguments const& args) { + Tensor mO = + make_tensor(make_gmem_ptr(args.ptr_O), args.shape_O, args.stride_O); + TMA_O tma_store_O = [&] { + if constexpr (Use_TMA_O) { + return make_tma_copy( + GmemTiledCopyOTMA{}, + mO, + SmemLayoutO{}, + select<0, 2>(TileShape_MNK{}), + _1{}); // no mcast + } else { + return nullptr; + } + }(); + // If PackGQA, reshape O to be ((qhead_per_khead, seqlen_q), head_size, + // nhead_k, batch_size, num_splits) + int const qhead_per_khead = 1; + auto const shape_O_packed = cute::conditional_return( + args.shape_O, + make_shape( + make_shape(qhead_per_khead, get<0>(args.shape_O)), + get<1>(args.shape_O), + args.nheads, + get<3>(args.shape_O), + get<4>(args.shape_O))); + auto const stride_O_packed = cute::conditional_return( + args.stride_O, + make_stride( + make_stride(get<2>(args.stride_O), get<0>(args.stride_O)), + get<1>(args.stride_O), + get<2>(args.stride_O) * qhead_per_khead, + get<3>(args.stride_O), + get<4>(args.stride_O))); + auto const shape_lse_packed = select<0, 2, 3, 4>(args.shape_O); + auto const stride_lse_packed = args.stride_lse; + return { + args.ptr_O, + args.shape_O, + args.stride_O, + shape_O_packed, + stride_O_packed, + args.ptr_lse, + args.stride_lse, + shape_lse_packed, + stride_lse_packed, + tma_store_O, + args.seq_offsets}; + } + + /// Issue Tma Descriptor Prefetch -- ideally from a single thread for best + /// performance + CUTLASS_DEVICE + static void prefetch_tma_descriptors(Params const& params) { + if constexpr (Use_TMA_O) { + cute::prefetch_tma_descriptor(params.tma_store_O.get_tma_descriptor()); + } + } + + template + CUTLASS_DEVICE void store( + Params const& params, + FrgTensorO const& tOrO, + SharedStorage& shared_storage, + TiledMma tiled_mma, + int thread_idx, + cute::tuple const& block_coord) { + auto [m_block, bidh, bidb, split_idx] = block_coord; + Tensor sO = make_tensor( + make_smem_ptr(shared_storage.tensors.epilogue.smem_o.data()), + SmemLayoutO{}); + // Tensor sO_pi = cute::as_position_independent_swizzle_tensor(sO); + + Tensor tOrO_out = make_tensor_like(tOrO); + hstu::convert_type_out(tOrO, tOrO_out); + if constexpr ( + FP8PermuteCol && (sizeof(Element) == 2 || sizeof(Element) == 4)) { + hstu::permute_output_fp8_Vcolmajor(tOrO_out); + } + + // Make sure all WGs have finished reading V + // Technically we don't need this if we're not using smem, but the mainloop + // makes the assumption that all epilogue threads sync at least once during + // the epilogue (so that we can start loading Q with cp.async if we need). + hstu::named_barrier_sync( + NumEpilogueThreads, + cutlass::arch::ReservedNamedBarriers::EpilogueBarrier); + + // Step 1: Write O from rmem -> smem + if constexpr (Use_smem) { + auto smem_tiled_copy_O = make_tiled_copy_C(SmemCopyAtomO{}, tiled_mma); + auto smem_thr_copy_O = smem_tiled_copy_O.get_thread_slice(thread_idx); + Tensor taccOrO = + smem_thr_copy_O.retile_S(tOrO_out); // ((Atom,AtomNum), MMA_M, MMA_N) + Tensor taccOsO = + smem_thr_copy_O.partition_D(sO); // ((Atom,AtomNum),PIPE_M,PIPE_N) + // Tensor taccOsO = smem_thr_copy_O.partition_D(sO_pi); // + // ((Atom,AtomNum),PIPE_M,PIPE_N) + cute::copy(smem_tiled_copy_O, taccOrO, taccOsO); + if constexpr (Use_TMA_O) { + cutlass::arch::fence_view_async_shared(); // ensure smem writes are + // visible to TMA + cutlass::arch::NamedBarrier::arrive( + NumEpilogueThreads + cutlass::NumThreadsPerWarp, + cutlass::arch::ReservedNamedBarriers::EpilogueBarrier); + } else { + hstu::named_barrier_sync( + NumEpilogueThreads, + cutlass::arch::ReservedNamedBarriers::EpilogueBarrier); + } + } else { + if constexpr (ArchTag::kMinComputeCapability >= 90) { +#pragma unroll + for (uint32_t cta_id = 0; cta_id < size(ClusterShape{}); ++cta_id) { + shared_storage.pipelines.barrier_O.arrive(cta_id); + } + } + } + + hstu::SeqlenInfo seqlen_info{ + bidb, size<0>(params.shape_O), params.seq_offsets}; + int offset_o = seqlen_info.offset; + int seqlen_o = seqlen_info.seqlen; + + // Step 2: Write LSE from rmem -> gmem + auto thread_mma = tiled_mma.get_thread_slice(thread_idx); + // (MMA,MMA_M,MMA_K) + Tensor taccOcO = thread_mma.partition_C( + cute::make_identity_tensor(select<0, 2>(TileShape_MNK{}))); + static_assert(decltype(size<0, 0>(taccOcO))::value == 2); + static_assert(decltype(size<0, 1>(taccOcO))::value == 2); + Tensor taccOcO_rowcol = make_tensor( + taccOcO.data(), hstu::convert_layout_acc_rowcol(taccOcO.layout())); + Tensor taccOcO_row = taccOcO_rowcol(_, _0{}); + // Step 3: Write O from smem -> gmem + if constexpr (Use_TMA_O) { + Tensor mO = params.tma_store_O.get_tma_tensor(params.shape_O)( + _, _, bidh, bidb, split_idx); + Tensor gO = local_tile( + mO, + select<0, 2>(TileShape_MNK{}), + make_coord(m_block, _0{})); // (M, K) + auto block_tma_O = params.tma_store_O.get_slice(_0{}); + Tensor tOgO = block_tma_O.partition_D(gO); // (TMA, TMA_M, TMA_K) + Tensor tOsO = block_tma_O.partition_S(sO); // (TMA, TMA_M, TMA_K) + int warp_idx_sync = + __shfl_sync(0xffffffff, thread_idx / cutlass::NumThreadsPerWarp, 0); + if (warp_idx_sync == + NumEpilogueThreads / cutlass::NumThreadsPerWarp - 1) { + cutlass::arch::NamedBarrier::sync( + NumEpilogueThreads + cutlass::NumThreadsPerWarp, + cutlass::arch::ReservedNamedBarriers::EpilogueBarrier); + if (cute::elect_one_sync()) { + cute::copy(params.tma_store_O, tOsO, tOgO); + tma_store_arrive(); + tma_store_wait<0>(); +#pragma unroll + for (uint32_t cta_id = 0; cta_id < size(ClusterShape{}); ++cta_id) { + shared_storage.pipelines.barrier_O.arrive(cta_id); + } + } + } + } else { // Don't use TMA in Jagged case since we don't want to overwrite + // the output of another sequence + Tensor mO = make_tensor( + make_gmem_ptr(params.ptr_O + offset_o * get<0>(params.stride_O)), + params.shape_O_packed, + params.stride_O_packed)(_, _, bidh, !Jagged ? bidb : 0, split_idx); + Tensor gO = local_tile( + mO, + select<0, 2>(TileShape_MNK{}), + make_coord(m_block, _0{})); // (M, K) + // if (thread_idx == 0) { printf("Before O write, m_block: %d, bidh: %d, + // bidb: %d, split_idx: %d, offset_o: %d, seqlen_o: %d, mO_addr = %p, addr + // diff = %d\n", m_block, bidh, bidb, split_idx, offset_o, seqlen_o, + // mO.data(), reinterpret_cast(&mO(0)) - + // reinterpret_cast(params.ptr_O)); } + if constexpr (Use_smem) { + GmemTiledCopyO gmem_tiled_copy_O; + auto gmem_thr_copy_O = gmem_tiled_copy_O.get_thread_slice(thread_idx); + Tensor tOsO = + gmem_thr_copy_O.partition_S(sO); // ((Atom,AtomNum),ATOM_M,ATOM_N) + // Tensor tOsO = gmem_thr_copy_O.partition_S(sO_pi); // + // ((Atom,AtomNum),ATOM_M,ATOM_N) + Tensor tOrO = make_fragment_like(tOsO); + cute::copy(gmem_tiled_copy_O, tOsO, tOrO); + if constexpr (ArchTag::kMinComputeCapability >= 90) { + cutlass::arch::fence_view_async_shared(); // ensure smem reads are + // done before next TMA to + // smem_v +#pragma unroll + for (uint32_t cta_id = 0; cta_id < size(ClusterShape{}); ++cta_id) { + shared_storage.pipelines.barrier_O.arrive(cta_id); + } + } + // (BLK_M,BLK_K) -> (blk_m,blk_k) + Tensor tOcO = gmem_thr_copy_O.partition_D( + cute::make_identity_tensor(select<0, 2>(TileShape_MNK{}))); + Tensor tOpO = make_tensor(make_shape(size<2>(tOsO))); +#pragma unroll + for (int k = 0; k < size(tOpO); ++k) { + tOpO(k) = get<1>(tOcO(_0{}, _0{}, k)) < get<1>(params.shape_O); + } + Tensor tOgO = gmem_thr_copy_O.partition_D(gO); + // Clear_OOB_K must be false since we don't want to write zeros to + // gmem + hstu::copy< + /*Is_even_MN=*/false, + /*Is_even_K=*/false, + /*Clear_OOB_MN=*/false, + /*Clear_OOB_K=*/false>( + gmem_tiled_copy_O, + tOrO, + tOgO, + tOcO, + tOpO, + seqlen_o - m_block * kBlockM); + } else { + // We already arrived on barrier_O earlier + static constexpr int kGmemElemsPerStoreDirect = 2; + cute::Copy_Atom, Element> + gmem_copy_direct; + // Reshape acc from ((2, 2, V), MMA_M, MMA_N) to (nrow=(2, MMA_M), + // ncol=(2, V, MMA_N)) + Tensor tOrO_rowcol = make_tensor( + tOrO_out.data(), hstu::convert_layout_acc_rowcol(tOrO.layout())); + Tensor tOrO_copy = cute::tiled_divide( + tOrO_rowcol, Shape<_1, Int>{}); + Tensor tOgO = thread_mma.partition_C(gO); + Tensor tOgO_rowcol = make_tensor( + tOgO.data(), hstu::convert_layout_acc_rowcol(tOgO.layout())); + Tensor tOgO_copy = cute::tiled_divide( + tOgO_rowcol, Shape<_1, Int>{}); + Tensor taccOcO_col = taccOcO_rowcol(_0{}, _); +#pragma unroll + for (int m = 0; m < size(taccOcO_row); ++m) { + if (get<0>(taccOcO_row(m)) < seqlen_o - m_block * kBlockM) { +#pragma unroll + for (int k = 0; k < size(taccOcO_col) / kGmemElemsPerStoreDirect; + ++k) { + if (get<1>(taccOcO_col(k * kGmemElemsPerStoreDirect)) < + get<1>(params.shape_O)) { + cute::copy( + gmem_copy_direct, tOrO_copy(_, m, k), tOgO_copy(_, m, k)); + } + } + } + } + } + } + } + + template + CUTLASS_DEVICE void store_softmax( + Params const& params, + FrgTensorLSE const& lse, + TiledMma tiled_mma, + int thread_idx, + cute::tuple const& block_coord) { + auto [m_block, bidh, bidb, split_idx] = block_coord; + hstu::SeqlenInfo seqlen_info{ + bidb, size<0>(params.shape_O), params.seq_offsets}; + int offset_o = seqlen_info.offset; + int seqlen_o = seqlen_info.seqlen; + // Step 2: Write LSE from rmem -> gmem + auto thread_mma = tiled_mma.get_thread_slice(thread_idx); + // (MMA,MMA_M,MMA_K) + Tensor taccOcO = thread_mma.partition_C( + cute::make_identity_tensor(select<0, 2>(TileShape_MNK{}))); + static_assert(decltype(size<0, 0>(taccOcO))::value == 2); + static_assert(decltype(size<0, 1>(taccOcO))::value == 2); + Tensor taccOcO_rowcol = make_tensor( + taccOcO.data(), hstu::convert_layout_acc_rowcol(taccOcO.layout())); + Tensor taccOcO_row = taccOcO_rowcol(_, _0{}); + CUTE_STATIC_ASSERT_V(size(lse) == size(taccOcO_row)); // MMA_M + Tensor mLSE = make_tensor( + make_gmem_ptr(params.ptr_lse + offset_o * get<0>(params.stride_lse)), + params.shape_lse_packed, + params.stride_lse_packed)(_, bidh, !Jagged ? bidb : 0, 0); +#pragma unroll + for (int mi = 0; mi < size(lse); ++mi) { + int const row = m_block * kBlockM + get<0>(taccOcO_row(mi)); + if (get<1>(taccOcO_row(_0{})) == 0 && row < seqlen_o) { + mLSE(row) = lse(mi); + } + } + } + + CUTLASS_DEVICE void store_tail() { + // Don't need to do tma_store_wait<0>() here since we already did in @store + } + + // Write 0 to output and -inf to LSE + template + CUTLASS_DEVICE void store_zero( + Params const& params, + int thread_idx, + cute::tuple const& block_coord) { + static constexpr int kBlockM = get<0>(TileShape_MNK{}); + auto [m_block, bidh, bidb, split_idx] = block_coord; + hstu::SeqlenInfo seqlen_info{ + bidb, size<0>(params.shape_O), params.seq_offsets}; + int offset_o = seqlen_info.offset; + int seqlen_o = seqlen_info.seqlen; + Tensor mO = make_tensor( + make_gmem_ptr(params.ptr_O + offset_o * get<0>(params.stride_O)), + params.shape_O_packed, + params.stride_O_packed)(_, _, bidh, !Jagged ? bidb : 0, split_idx); + + static_assert(kBlockM <= NumEpilogueThreads); + if constexpr (!Clear_O) { + return; + } + + GmemTiledCopyO gmem_tiled_copy_O; + auto gmem_thr_copy_O = gmem_tiled_copy_O.get_thread_slice(thread_idx); + Tensor tOcO = gmem_thr_copy_O.partition_D( + cute::make_identity_tensor(select<0, 2>(TileShape_MNK{}))); + Tensor tOpO = make_tensor(make_shape(size<2>(tOcO))); +#pragma unroll + for (int k = 0; k < size(tOpO); ++k) { + tOpO(k) = get<1>(tOcO(_0{}, _0{}, k)) < get<1>(params.shape_O); + } + Tensor gO = local_tile( + mO, select<0, 2>(TileShape_MNK{}), make_coord(m_block, _0{})); // (M, K) + Tensor tOgO = gmem_thr_copy_O.partition_D(gO); + Tensor tOrO = make_fragment_like(tOgO); + cute::clear(tOrO); + // Clear_OOB_K must be false since we don't want to write zeros to gmem + hstu::copy< + /*Is_even_MN=*/false, + /*Is_even_K=*/false, + /*Clear_OOB_MN=*/false, + /*Clear_OOB_K=*/false>( + gmem_tiled_copy_O, + tOrO, + tOgO, + tOcO, + tOpO, + seqlen_o - m_block * kBlockM); + } +}; + +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash.h new file mode 100644 index 000000000..ef37e3408 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash.h @@ -0,0 +1,157 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +//////////////////////////////////////////////////////////////////////////////////////////////////// +namespace hstu { + +struct Qkv_params { + using index_t = int64_t; + // The QKV matrices. + void* __restrict__ q_ptr; + void* __restrict__ k_ptr; + void* __restrict__ v_ptr; + + // The stride between rows of the Q, K and V matrices. + index_t q_batch_stride; + index_t k_batch_stride; + index_t v_batch_stride; + index_t q_row_stride; + index_t k_row_stride; + index_t v_row_stride; + index_t q_head_stride; + index_t k_head_stride; + index_t v_head_stride; + index_t v_dim_stride; + + // The number of heads. + int h; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +struct Flash_fwd_params : public Qkv_params { + using index_t = int64_t; + + // The O matrix (output). + void* __restrict__ o_ptr; + + // The stride between rows of O. + index_t o_batch_stride; + index_t o_row_stride; + index_t o_head_stride; + + // For FP8 scaling + float* __restrict__ q_descale_ptr; + float* __restrict__ k_descale_ptr; + float* __restrict__ v_descale_ptr; + index_t q_descale_batch_stride; + index_t q_descale_head_stride; + index_t k_descale_batch_stride; + index_t k_descale_head_stride; + index_t v_descale_batch_stride; + index_t v_descale_head_stride; + + // The dimensions. + int b, max_kv_len, max_q_len, qk_d, v_d, total_seq_len_q, total_seq_len_kv; + + // groups + int num_groups, batch_size_per_group; + int* __restrict__ max_seq_len_tensor; + int* __restrict__ contextual_seq_len_tensor; + int* __restrict__ max_attn_len_tensor; + int* __restrict__ min_full_attn_seq_len_tensor; + + // The scaling factors for the kernel. + float alpha; + + int* __restrict__ seq_offsets; + int* __restrict__ seq_offsets_q; + float* __restrict__ softmax_lse; + int* __restrict__ num_targets; + float* __restrict__ attn_scale; + + // Local window size + int max_attn_len, contextual_seq_len, min_full_attn_seq_len, + num_softmax_heads; + + // Pointer to the RNG seed (idx 0) and offset (idx 1). + uint64_t* rng_state; + + bool is_bf16; + bool is_fp32; + bool is_e4m3; + bool is_causal; + bool is_local; + bool has_contexual_mask; + bool scalar_scale; + bool training; + + int* __restrict__ tile_count_semaphore; + + int arch; + int num_sm; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +struct Flash_bwd_params : public Flash_fwd_params { + using index_t = int64_t; + + // The dO and dQKV matrices. + void* __restrict__ do_ptr; + void* __restrict__ dq_ptr; + void* __restrict__ dk_ptr; + void* __restrict__ dv_ptr; + float* __restrict__ softmax_lse_log2; + float* __restrict__ softmax_d; + + // To accumulate dQ + void* __restrict__ dq_accum_ptr; + int* __restrict__ dq_semaphore; + + // The stride between rows of the dO, dQ, dK and dV matrices. + index_t do_batch_stride; + index_t do_row_stride; + index_t do_head_stride; + index_t dq_batch_stride; + index_t dk_batch_stride; + index_t dv_batch_stride; + index_t dq_row_stride; + index_t dk_row_stride; + index_t dv_row_stride; + index_t dq_head_stride; + index_t dk_head_stride; + index_t dv_head_stride; + + int* __restrict__ sort_by_length_indices; + + int max_q_len_rounded, qk_d_rounded, v_d_rounded; + + bool deterministic; + index_t dq_accum_split_stride; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +void run_mha_fwd_(Flash_fwd_params& params, cudaStream_t stream); +template +void run_mha_bwd_(Flash_bwd_params& params, cudaStream_t stream); +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_api.cpp b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_api.cpp new file mode 100644 index 000000000..389e6620f --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_api.cpp @@ -0,0 +1,322 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/****************************************************************************** + * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, + *Pradeep Ramani, Tri Dao. + ******************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include // @manual +#include +#include "flash_common.h" + +extern "C" { +/* Creates a dummy empty _C module that can be imported from Python. + The import from Python will load the .so consisting of this file + in this extension, so that the TORCH_LIBRARY static initializers + below are run. */ +PyObject* PyInit__C(void) { + static struct PyModuleDef module_def = { + PyModuleDef_HEAD_INIT, + "_C", /* name of module */ + NULL, /* module documentation, may be NULL */ + -1, /* size of per-interpreter state of the module, + or -1 if the module keeps state in global variables. */ + NULL, /* methods */ + }; + return PyModule_Create(&module_def); +} +} + +namespace hstu { + +class HSTUFlashAttentionFunctionGPU + : public torch::autograd::Function { + public: + static at::Tensor forward( + torch::autograd::AutogradContext* ctx, + int64_t max_seq_len, + double alpha, + at::Tensor& q, // (b, s, h, d) or (total_s, h, d) + at::Tensor& k, // (b, s, h, d) or (total_s, h, d) + at::Tensor& v, // (b, s, h, d) or (total_s, h, d) + const std::optional& seq_offsets, + bool causal, + const std::optional& num_targets, + const std::optional& attn_scale, + int64_t max_attn_len, + int64_t min_full_attn_seq_len, + int64_t contextual_seq_len, + const std::optional& q_descale, // (b, h_k), not (b, h) + const std::optional& k_descale, // (b, h_k) + const std::optional& v_descale, // (b, h_k) + bool sort_by_length, + bool deterministic, + const int64_t sm_margin, + int64_t max_q_len, + const std::optional& seq_offsets_q, + int64_t num_softmax_heads, + bool training, + const std::optional& max_seq_len_tensor = std::nullopt, + const std::optional& contextual_seq_len_tensor = std::nullopt, + const std::optional& max_attn_len_tensor = std::nullopt, + const std::optional& min_full_attn_seq_len_tensor = + std::nullopt, + int64_t num_groups = 1) { + ctx->saved_data["max_seq_len"] = max_seq_len; + ctx->saved_data["alpha"] = alpha; + ctx->saved_data["causal"] = causal; + ctx->saved_data["max_attn_len"] = max_attn_len; + ctx->saved_data["min_full_attn_seq_len"] = min_full_attn_seq_len; + ctx->saved_data["contextual_seq_len"] = contextual_seq_len; + ctx->saved_data["deterministic"] = deterministic; + ctx->saved_data["sort_by_length"] = sort_by_length; + ctx->saved_data["sm_margin"] = sm_margin; + ctx->saved_data["max_q_len"] = max_q_len; + ctx->saved_data["num_softmax_heads"] = num_softmax_heads; + ctx->saved_data["num_groups"] = num_groups; + auto fwd_out = hstu::hstu_mha_fwd( + max_seq_len, // max_seq_len + alpha, // alpha + q, // q + k, // k + v, // v + seq_offsets, // seq_offsets + causal, // causal + num_targets, // num_targets + attn_scale, // attn_scale + max_attn_len, // max_attn_len + min_full_attn_seq_len, // min_full_attn_seq_len + contextual_seq_len, // contextual_seq_len + q_descale, // q_descale + k_descale, // k_descale + v_descale, // v_descale + sm_margin, // sm_margin + max_q_len, // max_q_len + seq_offsets_q, // seq_offsets_q + num_softmax_heads, // num_softmax_heads + training, + max_seq_len_tensor, + contextual_seq_len_tensor, + max_attn_len_tensor, + min_full_attn_seq_len_tensor, + num_groups); + auto out = get<0>(fwd_out); + auto softmax_lse = get<1>(fwd_out); + ctx->save_for_backward( + {q, + k, + v, + out, + seq_offsets.value_or(at::Tensor()), + num_targets.value_or(at::Tensor()), + attn_scale.value_or(at::Tensor()), + seq_offsets_q.value_or(at::Tensor()), + softmax_lse.value_or(at::Tensor()), + max_seq_len_tensor.value_or(at::Tensor()), + contextual_seq_len_tensor.value_or(at::Tensor()), + max_attn_len_tensor.value_or(at::Tensor()), + min_full_attn_seq_len_tensor.value_or(at::Tensor())}); + return out; + } + + static torch::autograd::variable_list backward( + torch::autograd::AutogradContext* ctx, + torch::autograd::variable_list grad_outputs) { + auto saved_tensors = ctx->get_saved_variables(); + auto saved_data = ctx->saved_data; + auto q = saved_tensors[0]; + auto k = saved_tensors[1]; + auto v = saved_tensors[2]; + auto out = saved_tensors[3]; + auto seq_offsets = saved_tensors[4]; + auto num_targets = saved_tensors[5]; + auto attn_scale = saved_tensors[6]; + auto seq_offsets_q = saved_tensors[7]; + auto softmax_lse = saved_tensors[8]; + auto max_seq_len_tensor = saved_tensors[9]; + auto contextual_seq_len_tensor = saved_tensors[10]; + auto max_attn_len_tensor = saved_tensors[11]; + auto min_full_attn_seq_len_tensor = saved_tensors[12]; + auto seq_offsets_opt = + seq_offsets.defined() ? std::optional(seq_offsets) : std::nullopt; + auto num_targets_opt = + num_targets.defined() ? std::optional(num_targets) : std::nullopt; + auto attn_scale_opt = + attn_scale.defined() ? std::optional(attn_scale) : std::nullopt; + auto seq_offsets_q_opt = + seq_offsets_q.defined() ? std::optional(seq_offsets_q) : std::nullopt; + auto softmax_lse_opt = + softmax_lse.defined() ? std::optional(softmax_lse) : std::nullopt; + auto max_seq_len_tensor_opt = max_seq_len_tensor.defined() + ? std::optional(max_seq_len_tensor) + : std::nullopt; + auto contextual_seq_len_tensor_opt = contextual_seq_len_tensor.defined() + ? std::optional(contextual_seq_len_tensor) + : std::nullopt; + auto max_attn_len_tensor_opt = max_attn_len_tensor.defined() + ? std::optional(max_attn_len_tensor) + : std::nullopt; + auto min_full_attn_seq_len_tensor_opt = + min_full_attn_seq_len_tensor.defined() + ? std::optional(min_full_attn_seq_len_tensor) + : std::nullopt; + + auto dq = at::empty_like(q); + auto dk = at::empty_like(k); + auto dv = at::empty_like(v); + + auto bwd_res = hstu::hstu_mha_bwd( + saved_data["max_seq_len"].toInt(), // max_seq_len + saved_data["alpha"].toDouble(), // alpha + grad_outputs[0], // dout + q, // q + k, // k + v, // v + dq, // dq + dk, // dk + dv, // dv + out, // out + seq_offsets_opt, // seq_offsets + saved_data["causal"].toBool(), // causal + num_targets_opt, // num_targets + attn_scale_opt, // attn_scale + saved_data["max_attn_len"].toInt(), // max_attn_len + saved_data["min_full_attn_seq_len"].toInt(), // min_full_attn_seq_len + saved_data["contextual_seq_len"].toInt(), // contextual_seq_len + saved_data["sort_by_length"].toBool(), // sort_by_length + saved_data["deterministic"].toBool(), // deterministic + saved_data["sm_margin"].toInt(), // sm_margin + saved_data["max_q_len"].toInt(), // max_q_len + seq_offsets_q_opt, // seq_offsets_q + saved_data["num_softmax_heads"].toInt(), // num_softmax_heads + softmax_lse_opt, + max_seq_len_tensor_opt, + contextual_seq_len_tensor_opt, + max_attn_len_tensor_opt, + min_full_attn_seq_len_tensor_opt, + saved_data["num_groups"].toInt()); + + return { + torch::autograd::Variable(), // max_seq_len + torch::autograd::Variable(), // alpha + bwd_res[0], // dq + bwd_res[1], // dk + bwd_res[2], // dv + torch::autograd::Variable(), // seq_offsets + torch::autograd::Variable(), // causal + torch::autograd::Variable(), // num_targets + torch::autograd::Variable(), // attn_scale + torch::autograd::Variable(), // max_attn_len + torch::autograd::Variable(), // min_full_attn_seq_len + torch::autograd::Variable(), // contextual_seq_len + torch::autograd::Variable(), // q_descale + torch::autograd::Variable(), // k_descale + torch::autograd::Variable(), // v_descale + torch::autograd::Variable(), // sort_by_length + torch::autograd::Variable(), // deterministic + torch::autograd::Variable(), // sm_margin + torch::autograd::Variable(), // max_q_len + torch::autograd::Variable(), // seq_offsets_q + torch::autograd::Variable(), // num_softmax_heads + torch::autograd::Variable(), // training + torch::autograd::Variable(), // max_seq_len_tensor + torch::autograd::Variable(), // contextual_seq_len_tensor + torch::autograd::Variable(), // max_attn_len_tensor + torch::autograd::Variable(), // min_full_attn_seq_len_tensor + torch::autograd::Variable(), // num_groups + }; + } +}; + +at::Tensor cuda_hstu_mha( + int64_t max_seq_len, + double alpha, + at::Tensor& q, // (b, s, h, d) or (total_s, h, d) + at::Tensor& k, // (b, s, h, d) or (total_s, h, d) + at::Tensor& v, // (b, s, h, d) or (total_s, h, d) + const std::optional& seq_offsets, + bool causal, + const std::optional& num_targets, + const std::optional& attn_scale, + int64_t max_attn_len, + int64_t min_full_attn_seq_len, + int64_t contextual_seq_len, + const std::optional& q_descale, // (b, h_k), not (b, h) + const std::optional& k_descale, // (b, h_k) + const std::optional& v_descale, // (b, h_k) + bool sort_by_length, + bool deterministic, + const int64_t sm_margin = 0, + int64_t max_q_len = 0, + const std::optional& seq_offsets_q = std::nullopt, + int64_t num_softmax_heads = 0, + bool training = true, + const std::optional& max_seq_len_tensor = std::nullopt, + const std::optional& contextual_seq_len_tensor = std::nullopt, + const std::optional& max_attn_len_tensor = std::nullopt, + const std::optional& min_full_attn_seq_len_tensor = + std::nullopt, + int64_t num_groups = 1) { + return hstu::HSTUFlashAttentionFunctionGPU::apply( + max_seq_len, + alpha, + q, + k, + v, + seq_offsets, + causal, + num_targets, + attn_scale, + max_attn_len, + min_full_attn_seq_len, + contextual_seq_len, + q_descale, + k_descale, + v_descale, + sort_by_length, + deterministic, + sm_margin, + max_q_len, + seq_offsets_q, + num_softmax_heads, + training, + max_seq_len_tensor, + contextual_seq_len_tensor, + max_attn_len_tensor, + min_full_attn_seq_len_tensor, + num_groups); +} + +TORCH_LIBRARY_FRAGMENT(hstu, m) { + m.impl( + "hstu_mha", + torch::dispatch(c10::DispatchKey::CUDA, TORCH_FN(cuda_hstu_mha))); + + m.impl( + "hstu_mha_fwd", + torch::dispatch(c10::DispatchKey::CUDA, TORCH_FN(hstu::hstu_mha_fwd))); + + m.impl( + "hstu_mha_bwd", + torch::dispatch(c10::DispatchKey::CUDA, TORCH_FN(hstu::hstu_mha_bwd))); +} +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_api_cpu.cpp b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_api_cpu.cpp new file mode 100644 index 000000000..c02424efe --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_api_cpu.cpp @@ -0,0 +1,256 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/****************************************************************************** + * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, + *Pradeep Ramani, Tri Dao. + ******************************************************************************/ + +#include // @manual +#include +#include "flash_common_cpu.h" + +namespace hstu { + +at::Tensor hstu_mha_cpu( + int64_t max_seq_len, + double alpha, + at::Tensor& q, // (b, s, h, d) or (total_s, h, d) + at::Tensor& k, // (b, s, h, d) or (total_s, h, d) + at::Tensor& v, // (b, s, h, d) or (total_s, h, d) + const std::optional& seq_offsets, + bool causal, + const std::optional& num_targets, + const std::optional& attn_scale, + int64_t max_attn_len, + int64_t min_full_attn_seq_len, + int64_t contextual_seq_len, + const std::optional& q_descale, // (b, h_k), not (b, h) + const std::optional& k_descale, // (b, h_k) + const std::optional& v_descale, // (b, h_k) + bool sort_by_length, + bool deterministic, + const int64_t sm_margin = 0, + int64_t max_q_len = 0, + const std::optional& seq_offsets_q = std::nullopt, + int64_t num_softmax_heads = 0, + bool training = true, + const std::optional& max_seq_len_tensor = std::nullopt, + const std::optional& contextual_seq_len_tensor = std::nullopt, + const std::optional& max_attn_len_tensor = std::nullopt, + const std::optional& min_full_attn_seq_len_tensor = + std::nullopt, + int64_t num_groups = 1) { + auto fwd_out = hstu::hstu_mha_fwd_dummy( + max_seq_len, + alpha, + q, + k, + v, + seq_offsets, + causal, + num_targets, + attn_scale, + max_attn_len, + min_full_attn_seq_len, + contextual_seq_len, + q_descale, + k_descale, + v_descale, + sm_margin, + max_q_len, + seq_offsets_q, + num_softmax_heads, + training); + return get<0>(fwd_out); +} + +at::Tensor hstu_mha_meta( + const at::SymInt max_seq_len, + double alpha, + at::Tensor& q, // (b, s, h, d) or (total_s, h, d) + at::Tensor& k, // (b, s, h, d) or (total_s, h, d) + at::Tensor& v, // (b, s, h, d) or (total_s, h, d) + const std::optional& seq_offsets, + bool causal, + const std::optional& num_targets, + const std::optional& attn_scale, + int64_t max_attn_len, + int64_t min_full_attn_seq_len, + int64_t contextual_seq_len, + const std::optional& q_descale, // (b, h_k), not (b, h) + const std::optional& k_descale, // (b, h_k) + const std::optional& v_descale, // (b, h_k) + bool sort_by_length, + bool deterministic, + const int64_t sm_margin = 0, + int64_t max_q_len = 0, + const std::optional& seq_offsets_q = std::nullopt, + int64_t num_softmax_heads = 0, + bool training = true, + const std::optional& max_seq_len_tensor = std::nullopt, + const std::optional& contextual_seq_len_tensor = std::nullopt, + const std::optional& max_attn_len_tensor = std::nullopt, + const std::optional& min_full_attn_seq_len_tensor = + std::nullopt, + int64_t num_groups = 1) { + auto fwd_out = hstu::hstu_mha_fwd_meta( + max_seq_len, + alpha, + q, + k, + v, + seq_offsets, + causal, + num_targets, + attn_scale, + max_attn_len, + min_full_attn_seq_len, + contextual_seq_len, + q_descale, + k_descale, + v_descale, + sm_margin, + max_q_len, + seq_offsets_q, + num_softmax_heads, + training); + return get<0>(fwd_out); +} + +// CPU-only implementation that registers under main hstu namespace +// This provides fallback implementations when GPU code is not compiled +TORCH_LIBRARY_FRAGMENT(hstu, m) { + // Only register operators if they haven't been registered by GPU code + // This allows CPU-only builds to work while GPU builds use GPU + // implementations + + m.def( + "hstu_mha_fwd(" + "SymInt max_seq_len, " + "float alpha, " + "Tensor q, " + "Tensor k, " + "Tensor v, " + "Tensor? seq_offsets, " + "bool causal, " + "Tensor? num_targets, " + "Tensor? attn_scale, " + "int max_attn_len, " + "int min_full_attn_seq_len, " + "int contextual_seq_len, " + "Tensor? q_descale, " + "Tensor? k_descale, " + "Tensor? v_descale, " + "int sm_margin = 0," + "int max_q_len = 0," + "Tensor? seq_offsets_q = None," + "int num_softmax_heads = 0," + "bool training = True," + "Tensor? max_seq_len_tensor = None," + "Tensor? contextual_seq_len_tensor = None," + "Tensor? max_attn_len_tensor = None," + "Tensor? min_full_attn_seq_len_tensor = None," + "int num_groups = 1" + ") -> (Tensor, Tensor?)"); + + m.def( + "hstu_mha_bwd(" + "int max_seq_len, " + "float alpha, " + "Tensor dout, " + "Tensor q, " + "Tensor k, " + "Tensor v, " + "Tensor dq, " + "Tensor dk, " + "Tensor dv, " + "Tensor out, " + "Tensor? seq_offsets, " + "bool causal, " + "Tensor? num_targets, " + "Tensor? attn_scale, " + "int max_attn_len, " + "int min_full_attn_seq_len, " + "int contextual_seq_len, " + "bool sort_by_length," + "bool deterministic," + "int sm_margin = 0," + "int max_q_len = 0," + "Tensor? seq_offsets_q = None," + "int num_softmax_heads = 0," + "Tensor? softmax_lse = None," + "Tensor? max_seq_len_tensor = None," + "Tensor? contextual_seq_len_tensor = None," + "Tensor? max_attn_len_tensor = None," + "Tensor? min_full_attn_seq_len_tensor = None," + "int num_groups = 1" + ") -> Tensor[]"); + + m.def( + "hstu_mha(" + "SymInt max_seq_len, " + "float alpha, " + "Tensor q, " + "Tensor k, " + "Tensor v, " + "Tensor? seq_offsets, " + "bool causal, " + "Tensor? num_targets, " + "Tensor? attn_scale, " + "int max_attn_len, " + "int min_full_attn_seq_len, " + "int contextual_seq_len, " + "Tensor? q_descale, " + "Tensor? k_descale, " + "Tensor? v_descale, " + "bool sort_by_length, " + "bool deterministic, " + "int sm_margin = 0," + "int max_q_len = 0," + "Tensor? seq_offsets_q = None," + "int num_softmax_heads = 0," + "bool training = True," + "Tensor? max_seq_len_tensor = None," + "Tensor? contextual_seq_len_tensor = None," + "Tensor? max_attn_len_tensor = None," + "Tensor? min_full_attn_seq_len_tensor = None," + "int num_groups = 1" + ") -> Tensor"); + + // Register CPU implementations + m.impl( + "hstu_mha", + torch::dispatch(c10::DispatchKey::CPU, TORCH_FN(hstu_mha_cpu))); + m.impl( + "hstu_mha", + torch::dispatch(c10::DispatchKey::Meta, TORCH_FN(hstu_mha_meta))); + + m.impl( + "hstu_mha_fwd", + torch::dispatch( + c10::DispatchKey::CPU, TORCH_FN(hstu::hstu_mha_fwd_dummy))); + m.impl( + "hstu_mha_fwd", + torch::dispatch( + c10::DispatchKey::Meta, TORCH_FN(hstu::hstu_mha_fwd_meta))); + + m.impl( + "hstu_mha_bwd", + torch::dispatch( + c10::DispatchKey::CPU, TORCH_FN(hstu::hstu_mha_bwd_dummy))); +} + +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_bwd_kernel_sm90.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_bwd_kernel_sm90.h new file mode 100644 index 000000000..051d5141b --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_bwd_kernel_sm90.h @@ -0,0 +1,402 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/****************************************************************************** + * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, + *Pradeep Ramani, Tri Dao. + ******************************************************************************/ + +#pragma once + +#include "cute/tensor.hpp" + +#include +#include +#include +#include +#include +#include +#include "cutlass/pipeline/pipeline.hpp" + +#include "tile_scheduler.h" +#include "utils.h" + +namespace hstu { + +using namespace cute; + +template < + bool Softmax, + class CollectiveMainloop_, + class CollectiveEpilogue_, + class TileScheduler_> +class FlashAttnBwdSm90 { + public: + // Mainloop derived types + using CollectiveMainloop = CollectiveMainloop_; + using TileShape_MNK = typename CollectiveMainloop::TileShape_MNK; + using TiledMmaSdP = typename CollectiveMainloop::TiledMmaSdP; + using TiledMmadKV = typename CollectiveMainloop::TiledMmadKV; + using ArchTag = typename CollectiveMainloop::ArchTag; + using ClusterShape = typename CollectiveMainloop::ClusterShape; + using MainloopArguments = typename CollectiveMainloop::Arguments; + using MainloopParams = typename CollectiveMainloop::Params; + static constexpr bool dKV_swapAB = CollectiveMainloop::dKV_swapAB; + + // Epilogue derived types + using CollectiveEpilogue = CollectiveEpilogue_; + using EpilogueArguments = typename CollectiveEpilogue::Arguments; + using EpilogueParams = typename CollectiveEpilogue::Params; + + static_assert(ArchTag::kMinComputeCapability >= 90); + + using TileScheduler = TileScheduler_; + using TileSchedulerArguments = typename hstu::TileSchedulerArguments; + using TileSchedulerParams = typename TileScheduler::Params; + + static constexpr uint32_t NumLoadWarpGroups = 1; + static constexpr uint32_t NumMmaWarpGroups = + CUTE_STATIC_V(size(TiledMmaSdP{})) / cutlass::NumThreadsPerWarpGroup; + static constexpr uint32_t MaxThreadsPerBlock = + CUTE_STATIC_V(size(TiledMmaSdP{})) + + (NumLoadWarpGroups * cutlass::NumThreadsPerWarpGroup); + static constexpr uint32_t MinBlocksPerMultiprocessor = 1; + static_assert(NumMmaWarpGroups == 2 || NumMmaWarpGroups == 3); + + /// Register requirement for Load and Math WGs + static constexpr uint32_t LoadRegisterRequirement = + NumMmaWarpGroups == 2 ? 24 : 32; + static constexpr uint32_t MmaRegisterRequirement = + NumMmaWarpGroups == 2 ? 240 : 160; + // If you want to print from the producer warp, you'd need to increase the + // number of registers Otherwise you'll get CUDA error. static constexpr + // uint32_t LoadRegisterRequirement = 40; static constexpr uint32_t + // MmaRegisterRequirement = NumMmaWarpGroups == 2 ? 232 : 152; + + // Kernel level shared memory storage + struct SharedStorage { + struct TensorStorage : cute::aligned_struct<128> { + union { + typename CollectiveMainloop::TensorStorage mainloop; + typename CollectiveEpilogue::TensorStorage epilogue; + }; + } tensors; + + struct PipelineStorage : cute::aligned_struct<16> { + alignas(16) cutlass::arch::ClusterTransactionBarrier barrier_KV; + alignas(16) typename CollectiveMainloop::MainloopPipeline::SharedStorage + pipeline_q; + alignas(16) + typename CollectiveMainloop::MainloopPipeline_dO::SharedStorage + pipeline_do; + alignas(16) typename TileScheduler::SharedStorage smem_scheduler; + } pipelines; + }; + + static constexpr int SharedStorageSize = sizeof(SharedStorage); + + // Device side arguments + struct Arguments { + MainloopArguments mainloop{}; + EpilogueArguments epilogue{}; + cutlass::KernelHardwareInfo hw_info{}; + TileSchedulerArguments scheduler{}; + }; + + // Kernel entry point API + struct Params { + MainloopParams mainloop{}; + EpilogueParams epilogue{}; + cutlass::KernelHardwareInfo hw_info{}; + TileSchedulerParams scheduler{}; + }; + + // + // Methods + // + + // Convert to underlying arguments. In this case, a simple copy for the + // aliased type. + static Params to_underlying_arguments(Arguments const& args) { + CUTLASS_TRACE_HOST("to_underlying_arguments():"); + + // Get SM count if needed, otherwise use user supplied SM count + int sm_count = args.hw_info.sm_count; + if (sm_count <= 0) { + CUTLASS_TRACE_HOST( + " WARNING: Arguments do not include a valid SM count.\n" + " For optimal performance, populate the arguments KernelHardwareInfo struct with the SM count."); + sm_count = cutlass::KernelHardwareInfo::query_device_multiprocessor_count( + args.hw_info.device_id); + } + + CUTLASS_TRACE_HOST( + "to_underlying_arguments(): Setting persistent grid SM count to " + << sm_count); + + cutlass::KernelHardwareInfo hw_info{args.hw_info.device_id, sm_count}; + return { + CollectiveMainloop::to_underlying_arguments(args.mainloop), + CollectiveEpilogue::to_underlying_arguments(args.epilogue), + hw_info, + TileScheduler::to_underlying_arguments(args.scheduler)}; + } + + // Computes the kernel launch grid shape based on runtime parameters + static dim3 get_grid_shape(Params const& params) { + return TileScheduler::get_grid_shape( + params.scheduler, params.hw_info.sm_count); + } + + static dim3 get_block_shape() { + return dim3(MaxThreadsPerBlock, 1, 1); + } + + CUTLASS_DEVICE + void operator()(Params const& params, char* smem_buf) { + static constexpr int NumMmaThreads = + NumMmaWarpGroups * cutlass::NumThreadsPerWarpGroup; + static constexpr int NumCopyThreads = + NumLoadWarpGroups * cutlass::NumThreadsPerWarpGroup; + + using MainloopPipeline = typename CollectiveMainloop::MainloopPipeline; + using PipelineParams = typename MainloopPipeline::Params; + using PipelineState = typename MainloopPipeline::PipelineState; + using MainloopPipeline_dO = + typename CollectiveMainloop::MainloopPipeline_dO; + using PipelineParams_dO = typename MainloopPipeline_dO::Params; + using PipelineState_dO = typename MainloopPipeline_dO::PipelineState; + static constexpr bool Q_dO_same_stages = + std::is_same_v; + + SharedStorage& shared_storage = *reinterpret_cast(smem_buf); + + int const lane_predicate = cute::elect_one_sync(); + int const warp_idx = cutlass::canonical_warp_idx_sync(); + + // Issue Tma Descriptor Prefetch from a single thread + if (warp_idx == 0 && lane_predicate) { + CollectiveMainloop::prefetch_tma_descriptors(params.mainloop); + CollectiveEpilogue::prefetch_tma_descriptors(params.epilogue); + } + + // Obtain warp index + int const warp_group_thread_idx = + threadIdx.x % cutlass::NumThreadsPerWarpGroup; + + PipelineParams pipeline_params; + if constexpr (Softmax) { + pipeline_params.transaction_bytes = + CollectiveMainloop::TmaTransactionBytesQ + + CollectiveMainloop::TmaTransactionBytesLSE; + } else { + pipeline_params.transaction_bytes = + CollectiveMainloop::TmaTransactionBytesQ; + } + int warp_group_idx = cutlass::canonical_warp_group_idx(); + pipeline_params.role = warp_group_idx == 0 + ? MainloopPipeline::ThreadCategory::Producer + : MainloopPipeline::ThreadCategory::Consumer; + pipeline_params.is_leader = warp_group_thread_idx == 0; + pipeline_params.num_consumers = NumMmaThreads; + + if (warp_idx == 0 && lane_predicate) { + shared_storage.pipelines.barrier_KV.init(1 /*numThreads*/); + } + // We're counting on pipeline_q to call cutlass::arch::fence_barrier_init(); + MainloopPipeline pipeline_q( + shared_storage.pipelines.pipeline_q, pipeline_params, ClusterShape{}); + auto role_dO = warp_group_idx == 0 + ? MainloopPipeline_dO::ThreadCategory::Producer + : MainloopPipeline_dO::ThreadCategory::Consumer; + PipelineParams_dO pipeline_params_dO{ + pipeline_params.transaction_bytes, + role_dO, + pipeline_params.is_leader, + pipeline_params.num_consumers}; + MainloopPipeline_dO pipeline_do( + shared_storage.pipelines.pipeline_do, + cute::conditional_return( + pipeline_params, pipeline_params_dO), + ClusterShape{}); + + CollectiveMainloop collective_mainloop; + CollectiveEpilogue collective_epilogue; + + // We need this to guarantee that the Pipeline init is visible to all + // producers and consumer blocks in the Cluster + if constexpr (size(ClusterShape{}) > 1) { + cute::cluster_arrive_relaxed(); + cute::cluster_wait(); + } else { + __syncthreads(); + } + + if (warp_group_idx == 0) { // Producer + cutlass::arch::warpgroup_reg_dealloc(); + + int warp_idx_in_warpgroup = + __shfl_sync(0xffffffff, (threadIdx.x / 32) % 4, 0); + if (warp_idx_in_warpgroup == 0) { // Load K, V, and do TMA on Q and dO + PipelineState smem_pipe_write = + cutlass::make_producer_start_state(); + PipelineState_dO smem_pipe_write_do = + cutlass::make_producer_start_state(); + + TileScheduler scheduler( + reinterpret_cast( + &shared_storage.pipelines.smem_scheduler)); + for (auto work_tile_info = + scheduler.template get_initial_work( + params.scheduler); + work_tile_info.is_valid(params.scheduler); + work_tile_info = + scheduler.template get_next_work( + params.scheduler, work_tile_info)) { + auto block_coord_ = work_tile_info.get_block_coord(params.scheduler); + auto [n_block, bidh, bidb, _ /*split_idx*/] = block_coord_; + cute::tuple block_coord = { + n_block, bidh, bidb}; + auto scheduler_prefetch = [&scheduler, ¶ms, &work_tile_info]() { + scheduler.prefetch_next_work(params.scheduler, work_tile_info); + }; + collective_mainloop.load( + params.mainloop, + pipeline_q, + pipeline_do, + smem_pipe_write, + smem_pipe_write_do, + shared_storage, + scheduler_prefetch, + block_coord); + } + collective_mainloop.load_tail( + pipeline_q, pipeline_do, smem_pipe_write, smem_pipe_write_do); + } else if (warp_idx_in_warpgroup == 1) { + TileScheduler scheduler( + reinterpret_cast( + &shared_storage.pipelines.smem_scheduler)); + for (auto work_tile_info = + scheduler.template get_initial_work( + params.scheduler); + work_tile_info.is_valid(params.scheduler); + work_tile_info = + scheduler.template get_next_work( + params.scheduler, work_tile_info)) { + auto block_coord_ = work_tile_info.get_block_coord(params.scheduler); + auto [n_block, bidh, bidb, _ /*split_idx*/] = block_coord_; + cute::tuple block_coord = { + n_block, bidh, bidb}; + collective_mainloop.store_dq( + params.mainloop, shared_storage, block_coord); + } + } + } else { // Consumer + cutlass::arch::warpgroup_reg_alloc(); + + TileScheduler scheduler( + reinterpret_cast( + &shared_storage.pipelines.smem_scheduler)); + // Initialize matmul objects. + TiledMmadKV tiled_mma_dKV; + + PipelineState smem_pipe_read; + PipelineState_dO smem_pipe_read_do; + + collective_mainloop.mma_init(); + scheduler.init_consumer(); + + int work_idx = 0; + CUTLASS_PRAGMA_NO_UNROLL + for (auto work_tile_info = + scheduler.template get_initial_work( + params.scheduler); + work_tile_info.is_valid(params.scheduler); + work_tile_info = + scheduler.template get_next_work( + params.scheduler, work_tile_info)) { + auto block_coord_ = work_tile_info.get_block_coord(params.scheduler); + auto [n_block, bidh, bidb, _ /*split_idx*/] = block_coord_; +#ifdef HSTU_FLASH_ATTN_DEBUG_INFO + if (threadIdx.x == 0 || threadIdx.x == 128) { + std::printf( + "n_block: (%d), bidh: (%d), bidb: (%d), blockIdx.x: (%d), blockIdx.y: (%d), blockIdx.z: (%d)\n", + n_block, + bidh, + bidb, + blockIdx.x, + blockIdx.y, + blockIdx.z); + } +#endif + cute::tuple block_coord = { + n_block, bidh, bidb}; + + // dK and dV output accumulator. + Tensor tdKrdK = partition_fragment_C( + tiled_mma_dKV, + select(TileShape_MNK{})); + Tensor tdVrdV = partition_fragment_C( + tiled_mma_dKV, + select(TileShape_MNK{})); + + bool tile_valid; + if constexpr (Softmax) { + tile_valid = collective_mainloop.mma_softmax( + params.mainloop, + pipeline_q, + pipeline_do, + smem_pipe_read, + smem_pipe_read_do, + tdKrdK, + tdVrdV, + threadIdx.x - NumCopyThreads, + work_idx, + block_coord, + shared_storage); + } else { + tile_valid = collective_mainloop.mma( + params.mainloop, + pipeline_q, + pipeline_do, + smem_pipe_read, + smem_pipe_read_do, + tdKrdK, + tdVrdV, + threadIdx.x - NumCopyThreads, + work_idx, + block_coord, + shared_storage); + } + if (tile_valid) { + collective_epilogue.store( + params.epilogue, + tdKrdK, + tdVrdV, + shared_storage, + tiled_mma_dKV, + threadIdx.x - NumCopyThreads, + block_coord); + } else { + collective_epilogue.store_zero( + params.epilogue, threadIdx.x - NumCopyThreads, block_coord); + } + } + collective_epilogue.store_tail(); + } + } +}; + +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_bwd_launch_template.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_bwd_launch_template.h new file mode 100644 index 000000000..6900852df --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_bwd_launch_template.h @@ -0,0 +1,492 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/****************************************************************************** + * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, + *Pradeep Ramani, Tri Dao. + ******************************************************************************/ + +#pragma once + +#include "cute/tensor.hpp" + +#include "cutlass/cluster_launch.hpp" // For ClusterLauncher +#include "cutlass/device_kernel.h" // For device_kernel +#include "cutlass/kernel_launch.h" // For kernel_launch + +#include "epilogue_bwd.h" +#include "flash.h" +#include "flash_bwd_kernel_sm90.h" +#include "flash_bwd_postprocess_kernel.h" +#include "flash_bwd_preprocess_kernel.h" +#include "mainloop_bwd_sm90_tma_gmma_ws.h" +#include "static_switch.h" +#include "tile_scheduler.h" +#include "tile_size.h" + +namespace hstu { + +using namespace cute; + +template < + int Arch, + int kHeadDim, + int kBlockM, + int kBlockN, + typename Element, + bool Causal, + bool Local, + bool Contexual_mask, + bool Jagged, + bool Has_targets, + bool Deterministic, + int Stages_dO = 2, + int Stages_dS_or_QSm80 = 2, + bool SdP_swapAB = true, + bool dKV_swapAB = false, + bool dQ_swapAB = false, + int NumMmaWarpGroups = 2, + int AtomLayoutMSdP = 1, + int AtomLayoutNdKV = 2, + int AtomLayoutMdQ = 1, + bool V_in_regs = false, + bool Cross = false, + bool Softmax = false> +void run_flash_bwd(hstu::Flash_bwd_params& params, cudaStream_t stream) { +#ifdef HSTU_FLASH_ATTN_DEBUG_INFO + std::printf( + "[flash_bwd_launch_template] Local: (%d), Jagged: (%d), Has_targets: (%d), Causal: (%d), max_kv_len: (%d), kHeadDim: (%d), kBlockM: (%d), kBlockN: (%d)\n", + Local, + Jagged, + Has_targets, + Causal, + params.max_kv_len, + kHeadDim, + kBlockM, + kBlockN); +#endif + static_assert( + !(Causal && Local), "Causal and Local cannot be true at the same time."); + using ElementAccum = float; + using ArchTag = + std::conditional_t= 90, cutlass::arch::Sm90, cutlass::arch::Sm80>; + + int const total_q_padded_rounded = + cute::round_up(params.total_seq_len_q + params.b * kBlockM, kBlockM); + int seqlen_q = !Jagged ? params.max_q_len : params.total_seq_len_q; + int seqlen_kv = !Jagged ? params.max_kv_len : params.total_seq_len_kv; + int seqlen_q_rounded = + !Jagged ? params.max_q_len_rounded : total_q_padded_rounded; + int batch = !Jagged ? params.b : 1; + + using TileShape_MK = cute::Shape, Int>; + using PreprocessKernel = hstu::FlashAttnBwdPreprocess< + TileShape_MK, + Element, + ElementAccum, + ArchTag, + /*Clear_dQaccum=*/true, + Jagged, + Softmax>; + typename PreprocessKernel::Arguments preprocess_args{ + static_cast(params.o_ptr), + {seqlen_q, params.v_d, params.h, batch}, // shape_O + {params.o_row_stride, + _1{}, + params.o_head_stride, + !Jagged ? params.o_batch_stride : 0}, // stride_O + static_cast(params.do_ptr), + {params.do_row_stride, + _1{}, + params.do_head_stride, + !Jagged ? params.do_batch_stride : 0}, // stride_dO + static_cast(params.softmax_d), + {seqlen_q_rounded, params.num_softmax_heads, batch}, // shape_dPsum + {_1{}, + seqlen_q_rounded, + !Jagged ? params.num_softmax_heads * params.max_q_len_rounded + : 0}, // stride_dPsum + static_cast(params.softmax_lse), + {_1{}, + seqlen_q, + !Jagged ? params.num_softmax_heads * params.max_q_len_rounded + : 0}, // stride_LSE + static_cast(params.softmax_lse_log2), + {_1{}, + seqlen_q_rounded, + !Jagged ? params.num_softmax_heads * params.max_q_len_rounded + : 0}, // stride_LSE_log2 + static_cast(params.dq_accum_ptr), + {seqlen_q_rounded * params.qk_d_rounded, + params.h, + batch}, // shape_dQaccum + {_1{}, + seqlen_q_rounded * params.qk_d_rounded, + !Jagged ? params.qk_d_rounded * params.max_q_len_rounded * params.h + : 0}, // stride_dQaccum + params.b, + params.h, + params.num_softmax_heads, + params.max_q_len, + params.dq_semaphore, + Cross ? params.seq_offsets_q : params.seq_offsets}; + typename PreprocessKernel::Params preprocess_params = + PreprocessKernel::to_underlying_arguments(preprocess_args); + int num_m_block = cute::ceil_div(params.max_q_len, kBlockM); + dim3 grid_m(num_m_block, params.h, params.b); + cutlass::kernel_launch( + grid_m, + PreprocessKernel::MaxThreadsPerBlock, + PreprocessKernel::SharedStorageSize, + stream, + preprocess_params, + false /*launch_with_pdl*/); + CHECK_CUDA_KERNEL_LAUNCH(); + + using TileShape_MNK = cute::Shape, Int, Int>; + using ClusterShape = + cute::Shape<_1, Int<1>, _1>; // Currently doesn't not support cluster + // Stages_dS_or_QSm80 is Stages_dS if Sm90 and Stages if Sm80 + static constexpr int Stages = Arch >= 90 ? 2 : Stages_dS_or_QSm80; + static constexpr int Stages_dS = Arch >= 90 ? Stages_dS_or_QSm80 : 1; + using CollectiveMainloop = hstu::CollectiveMainloopBwdSm90< + Stages, + Stages_dO, + Stages_dS, + ClusterShape, + TileShape_MNK, + Element, + ElementAccum, + cutlass::arch::Sm90, + Causal, + Local, + Contexual_mask, + Jagged, + Has_targets, + Deterministic, + SdP_swapAB, + dKV_swapAB, + dQ_swapAB, + NumMmaWarpGroups, + AtomLayoutMSdP, + AtomLayoutNdKV, + AtomLayoutMdQ, + V_in_regs, + Cross, + Softmax>; + using CollectiveEpilogue = hstu::CollectiveEpilogueBwd< + TileShape_MNK, + Element, + ArchTag, + CollectiveMainloop::NumMmaThreads, + Jagged, + dKV_swapAB, + NumMmaWarpGroups*(Arch >= 90 ? 1 : cutlass::NumWarpsPerWarpGroup) / + AtomLayoutNdKV>; + using Scheduler = + hstu::SingleTileScheduler; + using AttnKernel = hstu::enable_sm90_or_later>; + + typename CollectiveMainloop::Arguments mainloop_args{ + static_cast(params.q_ptr), + {seqlen_q, params.qk_d, params.h, batch}, // shape_Q + {params.q_row_stride, + _1{}, + params.q_head_stride, + !Jagged ? params.q_batch_stride : 0}, // stride_Q + static_cast(params.k_ptr), + {seqlen_kv, params.qk_d, params.h, batch}, // shape_K + {params.k_row_stride, + _1{}, + params.k_head_stride, + !Jagged ? params.k_batch_stride : 0}, // stride_K + static_cast(params.v_ptr), + {seqlen_kv, params.v_d, params.h, batch}, // shape_V + {params.v_row_stride, + _1{}, + params.v_head_stride, + !Jagged ? params.v_batch_stride : 0}, // stride_V + static_cast(params.do_ptr), + {seqlen_q, params.v_d, params.h, batch}, // shape_dO + {params.do_row_stride, + _1{}, + params.do_head_stride, + !Jagged ? params.do_batch_stride : 0}, // stride_dO + static_cast(params.dq_accum_ptr), + {seqlen_q_rounded * params.qk_d_rounded, + params.h, + batch}, // shape_dQaccum + {_1{}, + seqlen_q_rounded * params.qk_d_rounded, + !Jagged ? params.qk_d_rounded * params.max_q_len_rounded * params.h + : 0}, // stride_dQaccum + static_cast(params.softmax_lse_log2), + {seqlen_q_rounded, params.num_softmax_heads, batch}, // shape_LSE + {_1{}, + seqlen_q_rounded, + !Jagged ? params.num_softmax_heads * params.max_q_len_rounded + : 0}, // stride_LSE_log2 + static_cast(params.softmax_d), + {_1{}, + seqlen_q_rounded, + !Jagged ? params.num_softmax_heads * params.max_q_len_rounded + : 0}, // stride_dPsum + params.max_attn_len, + params.min_full_attn_seq_len, + params.contextual_seq_len, + 1.0f / params.max_kv_len, + params.alpha, + params.b, + params.num_softmax_heads, + params.num_groups, + params.batch_size_per_group, + params.dq_semaphore, + params.seq_offsets, + params.seq_offsets_q, + params.num_targets, + params.max_seq_len_tensor, + params.contextual_seq_len_tensor, + params.max_attn_len_tensor, + params.min_full_attn_seq_len_tensor, + params.attn_scale, + params.scalar_scale}; + typename CollectiveEpilogue::Arguments epilogue_args{ + static_cast(params.dk_ptr), + [&] { + return typename CollectiveEpilogue::ShapedKV{ + seqlen_kv, params.qk_d, params.h, batch}; // shape_dK + }(), + [&] { + return typename CollectiveEpilogue::StridedKV{ + params.dk_row_stride, + _1{}, + params.dk_head_stride, + !Jagged ? params.dk_batch_stride : 0}; // stride_dK + }(), + static_cast(params.dv_ptr), + [&] { + return typename CollectiveEpilogue::StridedKV{ + params.dv_row_stride, + _1{}, + params.dv_head_stride, + !Jagged ? params.dv_batch_stride : 0}; // stride_dV + }(), + params.h, + params.seq_offsets}; + + int num_blocks_n = + cutlass::ceil_div(params.max_kv_len, get<1>(TileShape_MNK{})); + num_blocks_n = cutlass::round_up(num_blocks_n, size<1>(ClusterShape{})); + typename hstu::TileSchedulerArguments scheduler_args{ + num_blocks_n, + params.h, + params.b, + params.max_kv_len, + params.qk_d, + sizeof(Element), + params.tile_count_semaphore, + params.seq_offsets, + params.sort_by_length_indices}; + + int device; + cudaGetDevice(&device); + typename AttnKernel::Params kernel_params = + AttnKernel::to_underlying_arguments( + {mainloop_args, + epilogue_args, + {device, params.num_sm}, + scheduler_args}); + + dim3 grid_dims = AttnKernel::get_grid_shape(kernel_params); + dim3 block_dims = AttnKernel::get_block_shape(); + int smem_size = AttnKernel::SharedStorageSize; + if constexpr (size(ClusterShape{}) > 1) { + void const* kernel = (void const*)cutlass::device_kernel; + if (smem_size >= 48 * 1024) { + CHECK_CUDA(cudaFuncSetAttribute( + kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size)); + } + dim3 cluster_dims( + size<0>(ClusterShape{}), + size<1>(ClusterShape{}), + size<2>(ClusterShape{})); + cutlass::ClusterLauncher::launch( + grid_dims, + cluster_dims, + block_dims, + smem_size, + stream, + kernel, + kernel_params, + false /*launch_with_pdl*/); + } else { + if (smem_size >= 48 * 1024) { + CHECK_CUDA(cudaFuncSetAttribute( + cutlass::device_kernel, + cudaFuncAttributeMaxDynamicSharedMemorySize, + smem_size)); + } + cutlass::kernel_launch( + grid_dims, + block_dims, + smem_size, + stream, + kernel_params, + false /*launch_with_pdl*/); + } + CHECK_CUDA_KERNEL_LAUNCH(); + + using PostprocessKernel = hstu::FlashAttnBwdPostprocessConvertdQ< + TileShape_MK, + Element, + ElementAccum, + ArchTag, + AttnKernel::CollectiveMainloop::NumMmaThreads, + typename AttnKernel::CollectiveMainloop::TiledMmadQ, + AttnKernel::CollectiveMainloop::dQ_swapAB, + Jagged, + Softmax>; + typename PostprocessKernel::Arguments postprocess_args{ + static_cast(params.dq_accum_ptr), + {seqlen_q_rounded * params.qk_d_rounded, + params.h, + batch}, // shape_dQaccum + {_1{}, + seqlen_q_rounded * params.qk_d_rounded, + !Jagged ? params.qk_d_rounded * params.max_q_len_rounded * params.h + : 0}, // stride_dQaccum + static_cast(params.dq_ptr), + {seqlen_q, params.qk_d, params.h, batch}, // shape_dQ + {params.dq_row_stride, + _1{}, + params.dq_head_stride, + params.dq_batch_stride}, // stride_dQ + Cross ? params.seq_offsets_q : params.seq_offsets}; + typename PostprocessKernel::Params postprocess_params = + PostprocessKernel::to_underlying_arguments(postprocess_args); + int num_m_block_postprocess = + cute::ceil_div(params.max_q_len, get<0>(TileShape_MK{})); + dim3 grid_m_postprocess(num_m_block_postprocess, params.h, params.b); + int smem_size_postprocess = PostprocessKernel::SharedStorageSize; + if (smem_size_postprocess >= 48 * 1024) { + CHECK_CUDA(cudaFuncSetAttribute( + cutlass::device_kernel, + cudaFuncAttributeMaxDynamicSharedMemorySize, + smem_size_postprocess)); + } + cutlass::kernel_launch( + grid_m_postprocess, + PostprocessKernel::MaxThreadsPerBlock, + smem_size_postprocess, + stream, + postprocess_params, + false /*launch_with_pdl*/); + CHECK_CUDA_KERNEL_LAUNCH(); +} + +template < + int Arch, + typename T, + int kBlockM, + int kBlockN, + int kHeadDim, + bool Causal, + bool Local, + int Stages_dO = 2, + int Stages_dS_or_QSm80 = 2, + bool SdP_swapAB = true, + bool dKV_swapAB = false, + bool dQ_swapAB = false, + int NumMmaWarpGroups = 2, + int AtomLayoutMSdP = 1, + int AtomLayoutNdKV = 2, + int AtomLayoutMdQ = 1, + bool V_in_regs = false, + bool Softmax = false> +void run_mha_bwd_dispatch(hstu::Flash_bwd_params& params, cudaStream_t stream) { + BOOL_SWITCH(params.seq_offsets != nullptr, Jagged, [&] { + BOOL_SWITCH(params.num_targets != nullptr, Has_targets, [&] { + BOOL_SWITCH(params.has_contexual_mask, Contexual_mask, [&] { + BOOL_SWITCH(params.seq_offsets_q, Cross, [&] { + run_flash_bwd< + Arch, + kHeadDim, + kBlockM, + kBlockN, + T, + Causal, + Local, + Contexual_mask, + Jagged, + Has_targets, + false /*Deterministic*/, + Stages_dO, + Stages_dS_or_QSm80, + SdP_swapAB, + dKV_swapAB, + dQ_swapAB, + NumMmaWarpGroups, + AtomLayoutMSdP, + AtomLayoutNdKV, + AtomLayoutMdQ, + V_in_regs, + Cross, + Softmax>(params, stream); + }); + }); + }); + }); +} + +template +void run_mha_bwd_(hstu::Flash_bwd_params& params, cudaStream_t stream) { + CAUSAL_LOCAL_SWITCH(params.is_causal, params.is_local, Causal, Local, [&] { + int const kBlockM = hstu::kBlockM_bwd(Arch, kHeadDim, Causal, Local); + int const kBlockN = hstu::kBlockN_bwd(Arch, kHeadDim); + bool const V_in_regs = hstu::V_in_regs_bwd(Arch, kHeadDim); + static constexpr std::tuple Stages = + hstu::Stages_bwd(Arch, kHeadDim); + static constexpr std::tuple swapAB = + hstu::swapAB_bwd(Arch, kHeadDim, Causal, Local); + int const NumMmaWarpGroups = hstu::NumMmaWarpGroups_bwd(Arch, kHeadDim); + static constexpr std::tuple AtomLayout = + hstu::AtomLayout_bwd(Arch, kHeadDim); + run_mha_bwd_dispatch< + Arch, + T, + kBlockM, + kBlockN, + kHeadDim, + Causal, + Local, + std::get<0>(Stages), /*Stages_dO*/ + std::get<1>(Stages), /*Stages_dS_or_QSm80*/ + std::get<0>(swapAB), /*SdP_swapAB*/ + std::get<1>(swapAB), /*dKV_swapAB*/ + std::get<2>(swapAB), /*dQ_swapAB*/ + NumMmaWarpGroups, + std::get<0>(AtomLayout), /*AtomLayoutMSdP*/ + std::get<1>(AtomLayout), /*AtomLayoutNdKV*/ + std::get<2>(AtomLayout), /*AtomLayoutMdQ*/ + V_in_regs, + Softmax>(params, stream); + }); +} + +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_bwd_postprocess_kernel.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_bwd_postprocess_kernel.h new file mode 100644 index 000000000..ca04a1456 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_bwd_postprocess_kernel.h @@ -0,0 +1,348 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/****************************************************************************** + * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, + *Pradeep Ramani, Tri Dao. + ******************************************************************************/ + +#pragma once + +#include "cute/tensor.hpp" + +#include +#include +#include +#include +#include "cutlass/arch/barrier.h" + +#include "seqlen.h" +#include "utils.h" + +namespace hstu { + +using namespace cute; + +template < + class TileShape_MK_, + class Element, + class ElementAccum, + class ArchTag_, + int kNThreads, + class TiledMma, + bool dQ_swapAB, + bool Jagged, + bool Softmax> +class FlashAttnBwdPostprocessConvertdQ { + public: + // Type Aliases + using TileShape_MK = TileShape_MK_; + using ArchTag = ArchTag_; + + static_assert(ArchTag::kMinComputeCapability >= 75); + static constexpr bool IsSm90 = ArchTag::kMinComputeCapability >= 90; + + static constexpr uint32_t MaxThreadsPerBlock = kNThreads; + static constexpr uint32_t MinBlocksPerMultiprocessor = 2; + + static constexpr int kBlockM = get<0>(TileShape_MK{}); + static constexpr int kHeadDim = get<1>(TileShape_MK{}); + static_assert( + !IsSm90 || kNThreads % cutlass::NumThreadsPerWarpGroup == 0, + "kNThreads must be a multiple of NumThreadsPerWarpGroup"); + static constexpr int NumdQWarpGgroups = + kNThreads / cutlass::NumThreadsPerWarpGroup; + using R2SLayoutAtomdQaccum = std::conditional_t< + IsSm90, + Layout< + Shape, Int>>, + Layout>>>; + using R2STiledCopydQaccum = decltype(make_tiled_copy( + Copy_Atom, ElementAccum>{}, + R2SLayoutAtomdQaccum{}, + Layout>>{})); // Val layout, 1 or 4 vals per + // read + using G2SLayoutAtomdQaccum = Layout>>; + // UniversalCopy instead of AutoVectorizingCopyWithAssumedAlignment as the + // latter generates cp.async instructions + using G2STiledCopydQaccum = decltype(make_tiled_copy( + Copy_Atom, ElementAccum>{}, + G2SLayoutAtomdQaccum{}, + Layout>{})); // Val layout, 4 vals per read + // We don't do bound checking for the gmem -> smem load so we just assert + // here. + static_assert(IsSm90 || (kBlockM * kHeadDim) % (kNThreads * 4) == 0); + static constexpr int SmemdQaccumSize = size(TileShape_MK{}); + using SmemLayoutdQaccumFlat = Layout>>; + using SmemLayoutdQaccum = std::conditional_t< + IsSm90, + Layout, + Int>>, + Layout>>>; + + // We can't just use kHeadDim here. E.g. if MMA shape is 64 x 96 but split + // across 2 WGs, then setting kBlockKSmem to 32 will cause "Static shape_div + // failure". We want to treat it as 64 x 48, so kBlockKSmem should be 16. + static constexpr int MmaShapeN = get<1>(typename TiledMma::AtomShape_MNK{}); + static constexpr int kBlockKSmem = + MmaShapeN % 64 == 0 ? 64 : (MmaShapeN % 32 == 0 ? 32 : 16); + static constexpr int kSwizzle = + kBlockKSmem == 64 ? 3 : (kBlockKSmem == 32 ? 2 : 1); + using SmemLayoutAtomdQ = decltype(composition( + Swizzle{}, + Layout, Int>, Stride, _1>>{})); + using SmemLayoutdQ = + decltype(tile_to_shape(SmemLayoutAtomdQ{}, TileShape_MK{})); + using SmemLayoutdQt = decltype(cute::composition( + SmemLayoutdQ{}, + make_layout( + make_shape(get<1>(TileShape_MK{}), get<0>(TileShape_MK{})), + make_stride(Int(TileShape_MK{})>{}, _1{})))); + + using SmemCopyAtomdQ = Copy_Atom< + std::conditional_t< + IsSm90, + std::conditional_t< + !dQ_swapAB, + cute::SM90_U32x4_STSM_N, + cute::SM90_U16x8_STSM_T>, + AutoVectorizingCopyWithAssumedAlignment<128>>, + Element>; + + static constexpr int kGmemElemsPerLoad = + sizeof(cute::uint128_t) / sizeof(Element); + static_assert( + kHeadDim % kGmemElemsPerLoad == 0, + "Headdim must be a multiple of kGmemElemsPerLoad"); + static constexpr int kGmemThreadsPerRow = + cutlass::gcd(kHeadDim / kGmemElemsPerLoad, int(MaxThreadsPerBlock)); + static_assert( + MaxThreadsPerBlock % kGmemThreadsPerRow == 0, + "MaxThreadsPerBlock must be a multiple of kGmemThreadsPerRow"); + using GmemLayoutAtom = Layout< + Shape< + Int, + Int>, + Stride, _1>>; + using GmemTiledCopy = decltype(make_tiled_copy( + Copy_Atom, Element>{}, + GmemLayoutAtom{}, + Layout>>{})); // Val layout, 8 or 16 vals + // per load + + struct SharedStorage : cute::aligned_struct<128> { + cute::array_aligned> + smem_dqacc; + cute::array_aligned> smem_dq; + alignas(16) cutlass::arch::ClusterTransactionBarrier barrier_dQaccum; + }; + + static constexpr int SharedStorageSize = sizeof(SharedStorage); + + using ShapedQ = + cute::Shape; // (seqlen_q, d, head, + // batch) + using StridedQ = cute::Stride; + using ShapedQaccum = + cute::Shape; // (seqlen_q * d, head, batch) + using StridedQaccum = cute::Stride<_1, int64_t, int64_t>; + + // Device side arguments + struct Arguments { + ElementAccum const* ptr_dQaccum; + ShapedQaccum const shape_dQaccum; + StridedQaccum const stride_dQaccum; + Element* ptr_dQ; + ShapedQ const shape_dQ; + StridedQ const stride_dQ; + int const* seq_offsets = nullptr; + }; + + // Kernel entry point API + struct Params { + ElementAccum const* ptr_dQaccum; + ShapedQaccum const shape_dQaccum; + StridedQaccum const stride_dQaccum; + Element* ptr_dQ; + ShapedQ const shape_dQ; + StridedQ const stride_dQ; + int const* seq_offsets = nullptr; + }; + + // Convert to underlying arguments. In this case, a simple copy for the + // aliased type. + static Params to_underlying_arguments(Arguments const& args) { + return { + args.ptr_dQaccum, + args.shape_dQaccum, + args.stride_dQaccum, + args.ptr_dQ, + args.shape_dQ, + args.stride_dQ, + args.seq_offsets}; + } + + CUTLASS_DEVICE + void operator()(Params const& params, char* smem_buf) { + static constexpr int kBlockM = get<0>(TileShape_MK{}); + SharedStorage& shared_storage = *reinterpret_cast(smem_buf); + + Tensor sdQaccum = make_tensor( + make_smem_ptr(shared_storage.smem_dqacc.data()), SmemLayoutdQaccum{}); + Tensor sdQaccum_flat = make_tensor( + make_smem_ptr(shared_storage.smem_dqacc.data()), + SmemLayoutdQaccumFlat{}); + Tensor sdQ = make_tensor( + make_smem_ptr(shared_storage.smem_dq.data()), SmemLayoutdQ{}); + Tensor sdQt = make_tensor( + make_smem_ptr(shared_storage.smem_dq.data()), SmemLayoutdQt{}); + + int const thread_idx = threadIdx.x; + int const m_block = blockIdx.x; + int const bidh = blockIdx.y; + int const bidb = blockIdx.z; + + hstu::SeqlenInfo seqlen_info( + bidb, size<0>(params.shape_dQ), params.seq_offsets); + if (Jagged && m_block * kBlockM >= seqlen_info.seqlen) { + return; + } + + // Step 1: load dQaccum from gmem to smem + Tensor mdQaccum = make_tensor( + make_gmem_ptr( + reinterpret_cast(params.ptr_dQaccum)), + params.shape_dQaccum, + params.stride_dQaccum)(_, bidh, !Jagged ? bidb : 0); + Tensor gdQaccum = local_tile( + domain_offset( + make_coord(seqlen_info.offset_padded * kHeadDim), mdQaccum), + Shape>{}, + make_coord(m_block)); // (M * K) + if constexpr (IsSm90) { // Use BulkCopy + static constexpr uint32_t TmaTransactionBytesdQaccum = + static_cast( + size(SmemLayoutdQaccumFlat{}) * + cute::sizeof_bits_v / 8); + auto bulk_copy = Copy_Traits{}; + // if (thread0()) { print(gdQaccum); printf("\n"); print(sdQaccum_flat); + // printf("\n"); } + if (thread_idx == 0) { + shared_storage.barrier_dQaccum.init(1 /*numThreads*/); + shared_storage.barrier_dQaccum.arrive_and_expect_tx( + TmaTransactionBytesdQaccum); + copy( + bulk_copy.with( + *reinterpret_cast(&shared_storage.barrier_dQaccum)), + gdQaccum, + sdQaccum_flat); + } + __syncthreads(); + shared_storage.barrier_dQaccum.wait(0); + } else { + G2STiledCopydQaccum g2s_tiled_copy_dQaccum; + auto g2s_thr_copy_dQaccum = + g2s_tiled_copy_dQaccum.get_thread_slice(thread_idx); + Tensor tdQgdQaccumg2s = g2s_thr_copy_dQaccum.partition_S(gdQaccum); + Tensor tdQsdQaccumg2s = g2s_thr_copy_dQaccum.partition_D(sdQaccum); + cute::copy(g2s_tiled_copy_dQaccum, tdQgdQaccumg2s, tdQsdQaccumg2s); + __syncthreads(); + } + + // __syncthreads(); if (cute::thread0()) { print_tensor(sdQaccum); } + + // Step 2: Load dQaccum from smem to register, then convert fp32 -> + // fp16/bf16 + R2STiledCopydQaccum s2r_tiled_copy_dQaccum; + auto s2r_thr_copy_dQaccum = + s2r_tiled_copy_dQaccum.get_thread_slice(thread_idx); + Tensor tdQsdQaccum = s2r_thr_copy_dQaccum.partition_S(sdQaccum); + TiledMma tiled_mma_dQ; + Tensor taccdQrdQaccum = partition_fragment_C( + tiled_mma_dQ, + select(TileShape_MK{})); + // if (blockIdx.x == 0 && blockIdx.y == 0 && threadIdx.x == 1) { + // print(tiled_mma_dQ); printf("\n"); } if (blockIdx.x == 0 && blockIdx.y == + // 0 && threadIdx.x == 1) { print(tdQsdQaccum); } if (blockIdx.x == 0 && + // blockIdx.y == 0 && threadIdx.x == 1) { print(taccdQrdQaccum); } + CUTE_STATIC_ASSERT_V(size(taccdQrdQaccum) == size(tdQsdQaccum)); + Tensor tdQrdQaccum = s2r_thr_copy_dQaccum.retile_D(taccdQrdQaccum); + cute::copy(s2r_tiled_copy_dQaccum, tdQsdQaccum, tdQrdQaccum); + // Convert tdQrdQ from fp32 to fp16 + Tensor rdQ = make_tensor_like(taccdQrdQaccum); + hstu::convert_type_out(taccdQrdQaccum, rdQ); + + // Step 3: Copy dQ from register to smem + auto smem_tiled_copy_dQ = make_tiled_copy_C(SmemCopyAtomdQ{}, tiled_mma_dQ); + auto smem_thr_copy_dQ = smem_tiled_copy_dQ.get_thread_slice(thread_idx); + Tensor taccdQrdQ = + smem_thr_copy_dQ.retile_S(rdQ); // ((Atom,AtomNum), MMA_N, MMA_N) + // if (cute::thread0()) { print(smem_tiled_copy_dQ); } + // if (cute::thread0()) { print(smem_thr_copy_dQ); } + // if (cute::thread0()) { print(sdQ); } + Tensor taccdQsdQ = smem_thr_copy_dQ.partition_D( + cute::conditional_return( + sdQ, sdQt)); // ((Atom,AtomNum),PIPE_M,PIPE_N) + cute::copy(smem_tiled_copy_dQ, taccdQrdQ, taccdQsdQ); + __syncthreads(); + + // Step 4: Copy dQ from smem to register to prepare for coalesced write to + // gmem + Tensor mdQ = make_tensor( + make_gmem_ptr(params.ptr_dQ), params.shape_dQ, params.stride_dQ)( + _, _, bidh, !Jagged ? bidb : 0); + Tensor gdQ = local_tile( + domain_offset(make_coord(seqlen_info.offset, _0{}), mdQ), + TileShape_MK{}, + make_coord(m_block, _0{})); // (M, K) + GmemTiledCopy gmem_tiled_copy_dQ; + auto gmem_thr_copy_dQ = gmem_tiled_copy_dQ.get_thread_slice(thread_idx); + Tensor tdQsdQ = + gmem_thr_copy_dQ.partition_S(sdQ); // ((Atom,AtomNum),ATOM_M,ATOM_N) + Tensor tdQgdQ = gmem_thr_copy_dQ.partition_D(gdQ); + + Tensor tdQrdQ = make_fragment_like(tdQsdQ); + Tensor tdQcdQ = gmem_thr_copy_dQ.partition_D( + cute::make_identity_tensor(TileShape_MK{})); + Tensor tdQpdQ = make_tensor(make_shape(size<2>(tdQgdQ))); +#pragma unroll + for (int k = 0; k < size(tdQpdQ); ++k) { + tdQpdQ(k) = get<1>(tdQcdQ(_0{}, _0{}, k)) < get<1>(params.shape_dQ); + } + // Need to check OOB when reading from smem if kBlockM isn't evenly tiled + static constexpr bool EvenM = + kBlockM % CUTE_STATIC_V(size<0>(GmemLayoutAtom{})) == 0; + hstu:: + copy( + gmem_tiled_copy_dQ, tdQsdQ, tdQrdQ, tdQcdQ, tdQpdQ, kBlockM); + + // Step 5: Copy dQ from register to gmem + // Clear_OOB_K must be false since we don't want to write zeros to gmem + hstu::copy< + /*Is_even_MN=*/false, + /*Is_even_K=*/false, + /*Clear_OOB_MN=*/false, + /*Clear_OOB_K=*/false>( + gmem_tiled_copy_dQ, + tdQrdQ, + tdQgdQ, + tdQcdQ, + tdQpdQ, + std::min(seqlen_info.seqlen - m_block * kBlockM, kBlockM)); + } +}; + +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_bwd_preprocess_kernel.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_bwd_preprocess_kernel.h new file mode 100644 index 000000000..8d29778af --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_bwd_preprocess_kernel.h @@ -0,0 +1,349 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/****************************************************************************** + * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, + *Pradeep Ramani, Tri Dao. + ******************************************************************************/ + +#pragma once + +#include "cute/tensor.hpp" + +#include +#include +#include +#include + +#include "seqlen.h" + +namespace hstu { + +using namespace cute; + +template < + class TileShape_MK_, + class Element, + class ElementAccum, + class ArchTag_, + bool Clear_dQaccum, + bool Jagged, + bool Softmax> +class FlashAttnBwdPreprocess { + public: + // Type Aliases + using TileShape_MK = TileShape_MK_; + using ArchTag = ArchTag_; + + static_assert( + std::is_same_v && + ArchTag::kMinComputeCapability >= 75 || + std::is_same_v && + ArchTag::kMinComputeCapability >= 80 || + std::is_same_v && + ArchTag::kMinComputeCapability >= 89); + + static constexpr uint32_t MaxThreadsPerBlock = 256; + static constexpr uint32_t MinBlocksPerMultiprocessor = 2; + static constexpr int SharedStorageSize = 0; + + static constexpr int kGmemElemsPerLoad = + sizeof(cute::uint128_t) / sizeof(Element); + static_assert( + get<1>(TileShape_MK{}) % kGmemElemsPerLoad == 0, + "Headdim must be a multiple of kGmemElemsPerLoad"); + static constexpr int kBlockM = get<0>(TileShape_MK{}); + static constexpr int kHeadDim = get<1>(TileShape_MK{}); + // We want kBlockKGmem to be a power of 2 so that when we do the summing, + // it's just between threads in the same warp + static constexpr int kBlockKGmem = + kHeadDim % 128 == 0 ? 128 : (kHeadDim % 64 == 0 ? 64 : 32); + static constexpr int kGmemThreadsPerRow = kBlockKGmem / kGmemElemsPerLoad; + static_assert( + MaxThreadsPerBlock % kGmemThreadsPerRow == 0, + "MaxThreadsPerBlock must be a multiple of kGmemThreadsPerRow"); + using GmemLayoutAtom = Layout< + Shape< + Int, + Int>, + Stride, _1>>; + using GmemTiledCopy = decltype(make_tiled_copy( + Copy_Atom, Element>{}, + GmemLayoutAtom{}, + Layout>>{})); // Val layout, 8 or 16 vals + // per load + + static constexpr int kGmemElemsPerLoadAccum = + sizeof(cute::uint128_t) / sizeof(ElementAccum); + static_assert( + (kBlockM * kHeadDim / kGmemElemsPerLoadAccum) % MaxThreadsPerBlock == 0, + "MaxThreadsPerBlock must divide kBlockM * kHeadDim / kGmemElemsPerLoadAccum"); + using GmemLayoutAtomAccum = Layout>>; + using GmemTiledCopyAccum = decltype(make_tiled_copy( + Copy_Atom, ElementAccum>{}, + GmemLayoutAtomAccum{}, + Layout>>{})); // Val layout, 4 vals per + // store + + using ShapeO = + cute::Shape; // (seqlen_q, d, head, + // batch) + using StrideO = cute::Stride; + using ShapedPsum = + cute::Shape; // (seqlen_q, head, batch) + using StridedPsum = cute::Stride<_1, int64_t, int64_t>; + using ShapedQaccum = + cute::Shape; // (seqlen_q * d, head, batch) + using StridedQaccum = cute::Stride<_1, int64_t, int64_t>; + + // Device side arguments + struct Arguments { + Element const* ptr_O; + ShapeO const shape_O; + StrideO const stride_O; + Element const* ptr_dO; + StrideO const stride_dO; + float* ptr_dPsum; + ShapedPsum const shape_dPsum; + StridedPsum const stride_dPsum; + float const* ptr_LSE; + StridedPsum const stride_LSE; + float* ptr_LSE_log2; + StridedPsum const stride_LSE_log2; + ElementAccum* ptr_dQaccum; + ShapedQaccum const shape_dQaccum; + StridedQaccum const stride_dQaccum; + int num_batch; // We need this to know the size of dq_semaphore in case of + // jagged + int num_heads; + int num_softmax_heads; + int max_seq_len; + int* dq_semaphore; + int const* seq_offsets = nullptr; + }; + + // Kernel entry point API + struct Params { + Element const* ptr_O; + ShapeO const shape_O; + StrideO const stride_O; + Element const* ptr_dO; + StrideO const stride_dO; + float* ptr_dPsum; + ShapedPsum const shape_dPsum; + StridedPsum const stride_dPsum; + float const* ptr_LSE; + StridedPsum const stride_LSE; + float* ptr_LSE_log2; + StridedPsum const stride_LSE_log2; + ElementAccum* ptr_dQaccum; + ShapedQaccum const shape_dQaccum; + StridedQaccum const stride_dQaccum; + int num_batch; + int num_heads; + int num_softmax_heads; + int max_seq_len; + int* dq_semaphore; + int const* seq_offsets = nullptr; + }; + + // Convert to underlying arguments. In this case, a simple copy for the + // aliased type. + static Params to_underlying_arguments(Arguments const& args) { + return {args.ptr_O, args.shape_O, args.stride_O, + args.ptr_dO, args.stride_dO, args.ptr_dPsum, + args.shape_dPsum, args.stride_dPsum, args.ptr_LSE, + args.stride_LSE, args.ptr_LSE_log2, args.stride_LSE_log2, + args.ptr_dQaccum, args.shape_dQaccum, args.stride_dQaccum, + args.num_batch, args.num_heads, args.num_softmax_heads, + args.max_seq_len, args.dq_semaphore, args.seq_offsets}; + } + + CUTLASS_DEVICE + void operator()(Params const& params, [[maybe_unused]] char* smem_buf) { + static constexpr int kBlockM = get<0>(TileShape_MK{}); + + int const thread_idx = threadIdx.x; + int const m_block = blockIdx.x; + int const bidh = blockIdx.y; + int const bidb = blockIdx.z; + + hstu::SeqlenInfo seqlen_info( + bidb, params.max_seq_len, params.seq_offsets); + int const seqlen_o = seqlen_info.seqlen; + if (Jagged && m_block * kBlockM >= seqlen_o) { + return; + } + + if constexpr (Softmax) { + Tensor mO = make_tensor( + make_gmem_ptr(params.ptr_O), params.shape_O, params.stride_O)( + _, _, bidh, !Jagged ? bidb : 0); + Tensor gO = local_tile( + cute::domain_offset(make_coord(seqlen_info.offset, _0{}), mO), + TileShape_MK{}, + make_coord(m_block, _0{})); // (M, K) + Tensor mdO = make_tensor( + make_gmem_ptr(params.ptr_dO), params.shape_O, params.stride_dO)( + _, _, bidh, !Jagged ? bidb : 0); + Tensor gdO = local_tile( + cute::domain_offset(make_coord(seqlen_info.offset, _0{}), mdO), + TileShape_MK{}, + make_coord(m_block, _0{})); // (M, K) + + auto shape_LSE = select<0, 2, 3>(params.shape_O); + Tensor mLSE = make_tensor( + make_gmem_ptr(params.ptr_LSE), shape_LSE, params.stride_LSE)( + _, bidh, !Jagged ? bidb : 0); + Tensor gLSE = local_tile( + cute::domain_offset(make_coord(seqlen_info.offset), mLSE), + Shape>{}, + make_coord(m_block)); + static_assert(kBlockM <= MaxThreadsPerBlock); + float lse = + thread_idx < seqlen_o - m_block * kBlockM && thread_idx < kBlockM + ? gLSE(thread_idx) + : 0.0f; + + GmemTiledCopy gmem_tiled_copy_O; + auto gmem_thr_copy_O = gmem_tiled_copy_O.get_thread_slice(thread_idx); + + Tensor tOgO = gmem_thr_copy_O.partition_S(gO); + Tensor tOgdO = gmem_thr_copy_O.partition_S(gdO); + // Construct identity layout for gO + Tensor cO = cute::make_identity_tensor( + TileShape_MK{}); // (BLK_M,BLK_K) -> (blk_m,blk_k) + // Repeat the partitioning with identity layouts + Tensor tOcO = gmem_thr_copy_O.partition_D(cO); + Tensor tOpO = make_tensor(make_shape(size<2>(tOgO))); +#pragma unroll + for (int k = 0; k < size(tOpO); ++k) { + tOpO(k) = get<1>(tOcO(_0{}, _0{}, k)) < get<1>(params.shape_O); + } + + // (8, kBlockM / 32, kHeadDim / 64) or (8, kBlockM / 16, kHeadDim / 128) + Tensor tOrO = make_fragment_like(tOgO); + Tensor tOrdO = make_fragment_like(tOgdO); + hstu::copy< + /*Is_even_MN=*/false, + /*Is_even_K=*/false, + /*Clear_OOB_MN=*/true, + /*Clearn_OOB_K=*/true>( + gmem_tiled_copy_O, + tOgO, + tOrO, + tOcO, + tOpO, + seqlen_o - m_block * kBlockM); + hstu::copy< + /*Is_even_MN=*/false, + /*Is_even_K=*/false, + /*Clear_OOB_MN=*/true, + /*Clearn_OOB_K=*/true>( + gmem_tiled_copy_O, + tOgdO, + tOrdO, + tOcO, + tOpO, + seqlen_o - m_block * kBlockM); + // Reshape from e.g. (8, kBlockM / 32, kHeadDim / 64) to (kBlockM / 32, + // (8, kHeadDim / 64)) + Layout l = make_layout( + get<1>(tOrO.layout()), + make_layout(get<0>(tOrO.layout()), get<2>(tOrO.layout()))); + Tensor tOrO_l = make_tensor(tOrO.data(), l); + Tensor o_fp32 = make_tensor_like(tOrO_l); + hstu::convert_type_out(tOrO_l, o_fp32); + Tensor tOrdO_l = make_tensor(tOrdO.data(), l); + Tensor do_fp32 = make_tensor_like(tOrdO_l); + hstu::convert_type_out(tOrdO_l, do_fp32); + // Sum across the last dimension + Tensor dP_sum = make_tensor(make_shape(size<0>(o_fp32))); +#pragma unroll + for (int mi = 0; mi < size<0>(o_fp32); ++mi) { + float dP_sum_cur = do_fp32(mi, 0) * o_fp32(mi, 0); +#pragma unroll + for (int ni = 1; ni < size<1>(o_fp32); ni++) { + dP_sum_cur += do_fp32(mi, ni) * o_fp32(mi, ni); + } + hstu::SumOp sum_op; + dP_sum(mi) = + hstu::Allreduce::run(dP_sum_cur, sum_op); + } + + Tensor mdPsum = make_tensor( + make_gmem_ptr(params.ptr_dPsum), + params.shape_dPsum, + params.stride_dPsum)(_, bidh, !Jagged ? bidb : 0); + Tensor gdPsum = local_tile( + cute::domain_offset(make_coord(seqlen_info.offset_padded), mdPsum), + Shape>{}, + make_coord(m_block)); + if (get<1>(tOcO(_0{}, _0{}, _0{})) == 0) { +#pragma unroll + for (int mi = 0; mi < size(dP_sum); ++mi) { + int const row = get<0>(tOcO(_0{}, mi, _0{})); + gdPsum(row) = row < seqlen_o - m_block * kBlockM ? dP_sum(mi) : 0; + } + } + + int const seqlen_rounded = cute::round_up(seqlen_o, kBlockM); + Tensor mLSElog2 = make_tensor( + make_gmem_ptr(params.ptr_LSE_log2), + params.shape_dPsum, + params.stride_LSE_log2)(_, bidh, !Jagged ? bidb : 0); + Tensor gLSElog2 = local_tile( + cute::domain_offset(make_coord(seqlen_info.offset_padded), mLSElog2), + Shape>{}, + make_coord(m_block)); + if (thread_idx < seqlen_rounded - m_block * kBlockM && + thread_idx < kBlockM) { + gLSElog2(thread_idx) = lse * float(M_LOG2E); + } + } + if constexpr (Clear_dQaccum) { + Tensor mdQaccum = make_tensor( + make_gmem_ptr(params.ptr_dQaccum), + params.shape_dQaccum, + params.stride_dQaccum)(_, bidh, !Jagged ? bidb : 0); + Tensor gdQaccum = local_tile( + cute::domain_offset( + make_coord(seqlen_info.offset_padded * kHeadDim), mdQaccum), + Shape>{}, + make_coord(m_block)); + GmemTiledCopyAccum gmem_tiled_copy_dQaccum; + auto gmem_thr_copy_dQaccum = + gmem_tiled_copy_dQaccum.get_thread_slice(thread_idx); + Tensor tdQgdQaccum = gmem_thr_copy_dQaccum.partition_D(gdQaccum); + Tensor zero = make_fragment_like(tdQgdQaccum); + clear(zero); + cute::copy( + Copy_Atom< + AutoVectorizingCopyWithAssumedAlignment<128>, + ElementAccum>{}, + zero, + tdQgdQaccum); + } + + if (params.dq_semaphore != nullptr && thread_idx == 0) { + int const num_batch = params.num_batch; + int const num_head = params.num_heads; + params.dq_semaphore + [bidh + bidb * num_head + m_block * num_head * num_batch] = 0; + } + } +}; + +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_common.cpp b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_common.cpp new file mode 100644 index 000000000..66ec445be --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_common.cpp @@ -0,0 +1,1165 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/****************************************************************************** + * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, + *Pradeep Ramani, Tri Dao. + ******************************************************************************/ + +// Include these 2 headers instead of torch/extension.h since we don't need all +// of the torch headers. +#include +#include +#include +#include +#include // For TORCH_VERSION* macros + +#include + +#include "flash.h" +#include "flash_common.h" +#include "static_switch.h" +#include "tile_size.h" + +#define CHECK_DEVICE(x) TORCH_CHECK(x.is_cuda(), #x " must be on CUDA") +#define CHECK_SHAPE(x, ...) \ + TORCH_CHECK( \ + x.sizes() == torch::IntArrayRef({__VA_ARGS__}), \ + #x " must have shape (" #__VA_ARGS__ ")") +#define CHECK_CONTIGUOUS(x) \ + TORCH_CHECK(x.is_contiguous(), #x " must be contiguous") + +at::Tensor switch_to_contiguous_if_needed(const at::Tensor& x) { + if (x.stride(x.dim() - 1) == 1) { + return x; + } + return x.contiguous(); +} + +namespace hstu { + +void set_params_fprop( + hstu::Flash_fwd_params& params, + // sizes + const size_t b, + const size_t total_seq_len_kv, + const size_t total_seq_len_q, + const size_t max_seq_len, + const size_t max_q_len, + const size_t h, + const size_t qk_d, + const size_t v_d, + // device pointers + const at::Tensor& q, + const at::Tensor& k, + const at::Tensor& v, + const at::Tensor& out, + void* seq_offsets, + void* num_targets, + void* attn_scale, + void* seq_offsets_q, + void* softmax_lse, + void* max_seq_len_tensor, + void* contextual_seq_len_tensor, + void* max_attn_len_tensor, + void* min_full_attn_seq_len_tensor, + const int num_groups, + bool causal, + float alpha, + const bool scalar_scale, + const int max_attn_len, + const int min_full_attn_seq_len, + const int contextual_seq_len, + const int num_softmax_heads, + const bool training, + const int sm_margin = 0) { + // Reset the parameters + params = {}; + + params.is_bf16 = q.dtype() == torch::kBFloat16; + params.is_e4m3 = q.dtype() == torch::kFloat8_e4m3fn; + + // Set the pointers and strides. + params.q_ptr = q.data_ptr(); + params.k_ptr = k.data_ptr(); + params.v_ptr = v.data_ptr(); + params.o_ptr = out.data_ptr(); + // All stride are in elements, not bytes. + params.q_row_stride = q.stride(-3); + params.k_row_stride = k.stride(-3); + params.v_row_stride = v.stride(-3); + params.o_row_stride = out.stride(-3); + params.q_head_stride = q.stride(-2); + params.k_head_stride = k.stride(-2); + params.v_head_stride = v.stride(-2); + params.o_head_stride = out.stride(-2); + params.v_dim_stride = v.stride(-1); + + if (seq_offsets == nullptr) { + params.q_batch_stride = q.stride(0); + params.k_batch_stride = k.stride(0); + params.v_batch_stride = v.stride(0); + params.o_batch_stride = out.stride(0); + } + + params.seq_offsets = static_cast(seq_offsets); + params.seq_offsets_q = static_cast(seq_offsets_q); + params.num_targets = static_cast(num_targets); + params.attn_scale = static_cast(attn_scale); + params.softmax_lse = static_cast(softmax_lse); + params.max_seq_len_tensor = static_cast(max_seq_len_tensor); + params.contextual_seq_len_tensor = + static_cast(contextual_seq_len_tensor); + params.max_attn_len_tensor = static_cast(max_attn_len_tensor); + params.min_full_attn_seq_len_tensor = + static_cast(min_full_attn_seq_len_tensor); + params.num_groups = num_groups; + params.batch_size_per_group = b / num_groups; + + // Set the dimensions. + params.b = b; + params.h = h; + params.total_seq_len_q = total_seq_len_q; + params.total_seq_len_kv = total_seq_len_kv; + params.max_kv_len = max_seq_len; + params.max_q_len = max_q_len; + params.qk_d = qk_d; + params.v_d = v_d; + + params.alpha = alpha; + + // Note: when num_groups > 1, max_attn_len, contextual_seq_len, + // min_full_attn_seq_len represent the max value in the tensor. + params.is_local = max_attn_len > 0; + params.is_causal = causal && (!params.is_local); + params.has_contexual_mask = contextual_seq_len > 0; + params.scalar_scale = scalar_scale; + params.num_softmax_heads = num_softmax_heads; + params.training = training; + + params.max_attn_len = max_attn_len; + params.min_full_attn_seq_len = min_full_attn_seq_len; + params.contextual_seq_len = contextual_seq_len; + + params.arch = at::cuda::getCurrentDeviceProperties()->major * 10 + + at::cuda::getCurrentDeviceProperties()->minor; + params.num_sm = + at::cuda::getCurrentDeviceProperties()->multiProcessorCount - sm_margin; + +#ifdef FLASHATTENTION_DISABLE_LOCAL + TORCH_CHECK( + !params.is_local, + "This flash attention build does not support local attention."); +#endif +} + +void set_params_dgrad( + hstu::Flash_bwd_params& params, + // sizes + const size_t b, + const size_t total_seq_len_kv, + const size_t total_seq_len_q, + const size_t max_seq_len, + const size_t max_q_len, + const size_t max_q_len_rounded, + const size_t h, + const size_t qk_d, + const size_t v_d, + const size_t qk_d_rounded, + const size_t v_d_rounded, + // device pointers + const at::Tensor& q, + const at::Tensor& k, + const at::Tensor& v, + const at::Tensor& out, + const at::Tensor& dout, + const at::Tensor& dq, + const at::Tensor& dk, + const at::Tensor& dv, + void* dq_accum_d, + void* seq_offsets, + void* num_targets, + void* attn_scale, + void* sort_by_length_indices, + void* seq_offsets_q, + void* softmax_lse, + void* softmax_d, + void* softmax_lse_log2, + void* max_seq_len_tensor, + void* contextual_seq_len_tensor, + void* max_attn_len_tensor, + void* min_full_attn_seq_len_tensor, + const int num_groups, + const bool scalar_scale, + const bool causal, + const float alpha, + const int max_attn_len, + const int min_full_attn_seq_len, + const int contextual_seq_len, + const int num_softmax_heads, + bool deterministic = false, + int const sm_margin = 0) { + hstu::set_params_fprop( + params, + b, + total_seq_len_kv, + total_seq_len_q, + max_seq_len, + max_q_len, + h, + qk_d, + v_d, + q, + k, + v, + out, + seq_offsets, + num_targets, + attn_scale, + seq_offsets_q, + softmax_lse, + max_seq_len_tensor, + contextual_seq_len_tensor, + max_attn_len_tensor, + min_full_attn_seq_len_tensor, + num_groups, + causal, + alpha, + scalar_scale, + max_attn_len, + min_full_attn_seq_len, + contextual_seq_len, + num_softmax_heads, + false /* training */, + sm_margin); + + // Set the pointers and strides. + params.do_ptr = dout.data_ptr(); + params.do_row_stride = dout.stride(-3); + params.do_head_stride = dout.stride(-2); + params.dq_ptr = dq.data_ptr(); + params.dk_ptr = dk.data_ptr(); + params.dv_ptr = dv.data_ptr(); + params.dq_row_stride = dq.stride(-3); + params.dk_row_stride = dk.stride(-3); + params.dv_row_stride = dv.stride(-3); + params.dq_head_stride = dq.stride(-2); + params.dk_head_stride = dk.stride(-2); + params.dv_head_stride = dv.stride(-2); + + params.qk_d_rounded = qk_d_rounded; + params.v_d_rounded = v_d_rounded; + params.max_q_len_rounded = max_q_len_rounded; + + params.sort_by_length_indices = static_cast(sort_by_length_indices); + + if (seq_offsets == nullptr) { + params.do_batch_stride = dout.stride(0); + params.dq_batch_stride = dq.stride(0); + params.dk_batch_stride = dk.stride(0); + params.dv_batch_stride = dv.stride(0); + } + params.dq_accum_ptr = dq_accum_d; + params.softmax_lse_log2 = static_cast(softmax_lse_log2); + params.softmax_d = static_cast(softmax_d); + params.deterministic = deterministic; +} + +void run_mha_fwd(hstu::Flash_fwd_params& params, cudaStream_t stream) { + // HEADDIM_SWITCH(params.d, [&] { + // hstu::run_mha_fwd_(params, stream); + // }); + ARCH_SWITCH(params.arch, Arch, [&] { + BOOL_SWITCH(params.num_softmax_heads == params.h, Softmax, [&] { + if (!params.is_e4m3) { + if (params.is_bf16) { +#ifndef FLASHATTENTION_DISABLE_HDIM64 + if (params.qk_d <= 64) { + return hstu::run_mha_fwd_( + params, stream); + } +#endif +#ifndef FLASHATTENTION_DISABLE_HDIM96 + if (params.qk_d <= 96) { + return hstu::run_mha_fwd_( + params, stream); + } +#endif +#ifndef FLASHATTENTION_DISABLE_HDIM128 + if (params.qk_d <= 128) { + return hstu::run_mha_fwd_( + params, stream); + } +#endif +#ifndef FLASHATTENTION_DISABLE_HDIM192 + if (params.qk_d <= 192) { + return hstu::run_mha_fwd_( + params, stream); + } +#endif +#ifndef FLASHATTENTION_DISABLE_HDIM256 + if (params.qk_d <= 256) { + return hstu::run_mha_fwd_( + params, stream); + } +#endif + } else { +#ifndef FLASHATTENTION_DISABLE_FP16 +#ifndef FLASHATTENTION_DISABLE_HDIM64 + if (params.qk_d <= 64) { + return hstu::run_mha_fwd_( + params, stream); + } +#endif +#ifndef FLASHATTENTION_DISABLE_HDIM96 + if (params.qk_d <= 96) { + return hstu::run_mha_fwd_( + params, stream); + } +#endif +#ifndef FLASHATTENTION_DISABLE_HDIM128 + if (params.qk_d <= 128) { + return hstu::run_mha_fwd_( + params, stream); + } +#endif +#ifndef FLASHATTENTION_DISABLE_HDIM192 + if (params.qk_d <= 192) { + return hstu::run_mha_fwd_( + params, stream); + } +#endif +#ifndef FLASHATTENTION_DISABLE_HDIM256 + if (params.qk_d <= 256) { + return hstu::run_mha_fwd_( + params, stream); + } +#endif +#else + TORCH_CHECK(false, "This flash attention build does not support FP16."); +#endif + } + } else { +#ifndef FLASHATTENTION_DISABLE_FP8 +#ifndef FLASHATTENTION_DISABLE_HDIM64 + if (params.qk_d <= 64) { + return hstu::run_mha_fwd_<90, cutlass::float_e4m3_t, 64, Softmax>( + params, stream); + } +#endif +#ifndef FLASHATTENTION_DISABLE_HDIM96 + if (params.qk_d <= 96) { + return hstu::run_mha_fwd_<90, cutlass::float_e4m3_t, 96, Softmax>( + params, stream); + } +#endif +#ifndef FLASHATTENTION_DISABLE_HDIM128 + if (params.qk_d <= 128) { + return hstu::run_mha_fwd_<90, cutlass::float_e4m3_t, 128, Softmax>( + params, stream); + } +#endif +#ifndef FLASHATTENTION_DISABLE_HDIM192 + if (params.qk_d <= 192) { + return hstu::run_mha_fwd_<90, cutlass::float_e4m3_t, 192, Softmax>( + params, stream); + } +#endif +#ifndef FLASHATTENTION_DISABLE_HDIM256 + if (params.qk_d <= 256) { + return hstu::run_mha_fwd_<90, cutlass::float_e4m3_t, 256, Softmax>( + params, stream); + } +#endif +#else + TORCH_CHECK(false, "This flash attention build does not support FP8."); +#endif + } + }); + }); +} + +std::tuple> hstu_mha_fwd( + int64_t max_seq_len, + double alpha, + at::Tensor& q, // (b, s, h, d) or (total_s, h, d) + at::Tensor& k, // (b, s, h, d) or (total_s, h, d) + at::Tensor& v, // (b, s, h, d) or (total_s, h, d) + const std::optional& seq_offsets, + bool causal, + const std::optional& num_targets, + const std::optional& attn_scale, + int64_t max_attn_len, + int64_t min_full_attn_seq_len, + int64_t contextual_seq_len, + const std::optional& q_descale, // (b, h_k), not (b, h) + const std::optional& k_descale, // (b, h_k) + const std::optional& v_descale, // (b, h_k) + const int64_t sm_margin, + int64_t max_q_len, + const std::optional& seq_offsets_q, + int64_t num_softmax_heads, + bool training, + const std::optional& max_seq_len_tensor, + const std::optional& contextual_seq_len_tensor, + const std::optional& max_attn_len_tensor, + const std::optional& min_full_attn_seq_len_tensor, + int64_t num_groups) { + auto dprops = at::cuda::getCurrentDeviceProperties(); + bool is_sm9x = dprops->major >= 9; + TORCH_CHECK(is_sm9x, "HSTU Attention only supports Hopper GPUs or newer."); + + q = switch_to_contiguous_if_needed(q); + k = switch_to_contiguous_if_needed(k); + v = switch_to_contiguous_if_needed(v); + + auto q_type = q.scalar_type(); + TORCH_CHECK( + q_type == at::ScalarType::Half || q_type == at::ScalarType::BFloat16 || + q_type == at::ScalarType::Float8_e4m3fn, + "FlashAttention only supports fp16, bf16, and fp8_e4m3 data type"); + if (dprops->major < 9) { + TORCH_CHECK( + q_type == at::ScalarType::Half || q_type == at::ScalarType::BFloat16, + "FlashAttention on Ampere/Ada cards only supports fp16 and bf16 data type"); + } + TORCH_CHECK( + k.scalar_type() == q_type, "query and key must have the same dtype"); + TORCH_CHECK( + v.scalar_type() == q_type, "query and value must have the same dtype"); + + CHECK_DEVICE(q); + CHECK_DEVICE(k); + CHECK_DEVICE(v); + + TORCH_CHECK( + q.stride(-1) == 1, "Input tensor must have contiguous last dimension"); + TORCH_CHECK( + k.stride(-1) == 1, "Input tensor must have contiguous last dimension"); + TORCH_CHECK( + v.stride(-1) == 1, "Input tensor must have contiguous last dimension"); + + at::Tensor seq_offsets_; + bool const is_jagged = seq_offsets.has_value(); + if (is_jagged) { + seq_offsets_ = seq_offsets.value(); + CHECK_DEVICE(seq_offsets_); + CHECK_CONTIGUOUS(seq_offsets_); + TORCH_CHECK( + seq_offsets_.dtype() == torch::kInt32, + "seq_offsets_ must have dtype torch.int32"); + } + at::Tensor num_targets_; + bool const has_multiple_targets = num_targets.has_value(); + if (has_multiple_targets) { + num_targets_ = num_targets.value(); + CHECK_DEVICE(num_targets_); + CHECK_CONTIGUOUS(num_targets_); + TORCH_CHECK( + num_targets_.dtype() == torch::kInt32, + "num_targets_ must have dtype torch.int32"); + } + at::Tensor seq_offsets_q_; + bool const is_cross_attn = seq_offsets_q.has_value(); + if (is_cross_attn) { + seq_offsets_q_ = seq_offsets_q.value(); + CHECK_DEVICE(seq_offsets_q_); + CHECK_CONTIGUOUS(seq_offsets_q_); + TORCH_CHECK( + seq_offsets_q_.dtype() == torch::kInt32, + "seq_offsets_q_ must have dtype torch.int32"); + } else { + max_q_len = max_seq_len; + } + at::Tensor attn_scale_; + bool scalar_scale = true; + bool const has_attn_scale = attn_scale.has_value(); + if (has_attn_scale) { + attn_scale_ = attn_scale.value(); + scalar_scale = attn_scale_.numel() == num_groups; + CHECK_DEVICE(attn_scale_); + TORCH_CHECK( + attn_scale_.dtype() == torch::kFloat32, + "attn_scale_ must have dtype torch.float32"); + } + at::Tensor max_seq_len_tensor_; + at::Tensor contextual_seq_len_tensor_; + at::Tensor max_attn_len_tensor_; + at::Tensor min_full_attn_seq_len_tensor_; + if (num_groups > 1) { + TORCH_CHECK( + max_seq_len_tensor.has_value(), + "max_seq_len_tensor cannot be empty for num_groups > 1."); + max_seq_len_tensor_ = max_seq_len_tensor.value(); + CHECK_DEVICE(max_seq_len_tensor_); + TORCH_CHECK(max_seq_len_tensor_.dtype() == torch::kInt32); + if (!is_cross_attn) { + TORCH_CHECK( + contextual_seq_len_tensor.has_value(), + "contextual_seq_len_tensor cannot be empty for num_groups > 1 and not cross_attn."); + TORCH_CHECK( + max_attn_len_tensor.has_value(), + "max_attn_len_tensor cannot be empty for num_groups > 1 and not cross_attn."); + TORCH_CHECK( + min_full_attn_seq_len_tensor.has_value(), + "min_full_attn_seq_len_tensor cannot be empty for num_groups > 1 and not cross_attn."); + contextual_seq_len_tensor_ = contextual_seq_len_tensor.value(); + max_attn_len_tensor_ = max_attn_len_tensor.value(); + min_full_attn_seq_len_tensor_ = min_full_attn_seq_len_tensor.value(); + CHECK_DEVICE(contextual_seq_len_tensor_); + CHECK_DEVICE(max_attn_len_tensor_); + CHECK_DEVICE(min_full_attn_seq_len_tensor_); + TORCH_CHECK(contextual_seq_len_tensor_.dtype() == torch::kInt32); + TORCH_CHECK(max_attn_len_tensor_.dtype() == torch::kInt32); + TORCH_CHECK(min_full_attn_seq_len_tensor_.dtype() == torch::kInt32); + } + } +#ifdef HSTU_FLASH_ATTN_DEBUG_INFO + if (is_jagged && has_multiple_targets) { + auto uih_lengths = seq_offsets_.slice(0, 1) + .sub(seq_offsets_.slice(0, 0, -1)) + .sub(num_targets_); + TORCH_CHECK( + (uih_lengths.gt(0)).sum().item() == num_targets_.size(0), + "some uih seqlen is 0"); + TORCH_CHECK( + (uih_lengths.greater_equal(contextual_seq_len)).sum().item() == + num_targets_.size(0), + "some uih seqlen is less than contextual_seq_len"); + } +#endif + TORCH_CHECK( + q.size(-1) == k.size(-1) && k.size(-1) == v.size(-1), + "only attndim == hidden_dim is supported"); + + auto const sizes_q = q.sizes(); + auto const sizes_k = k.sizes(); + const int batch_size = !is_jagged ? sizes_q[0] : seq_offsets_.size(0) - 1; + TORCH_CHECK( + batch_size % num_groups == 0, "batch_size not divisible by num_groups"); + int total_seq_len_q = !is_jagged ? batch_size * max_q_len : sizes_q[0]; + int total_seq_len_kv = !is_jagged ? batch_size * max_seq_len : sizes_k[0]; + int num_heads = q.size(-2); + int const qk_head_size = q.size(-1); + int const v_head_size = v.size(-1); + int const max_headdim = get_max_headdim(); + TORCH_CHECK( + qk_head_size <= max_headdim && v_head_size <= max_headdim, + "FlashAttention forward only supports head dimension at most " + + std::to_string(max_headdim)); + TORCH_CHECK(max_attn_len >= 0, "max_attn_len must be at least 0"); + TORCH_CHECK( + min_full_attn_seq_len >= 0, "min_full_attn_seq_len must be at least 0"); + TORCH_CHECK(contextual_seq_len >= 0, "contextual_seq_len must be at least 0"); + if (max_attn_len > 0) { + TORCH_CHECK( + min_full_attn_seq_len > 0, + "min_full_attn_seq_len=0 not supported when max_attn_len > 0"); + } + TORCH_CHECK( + 0 == num_softmax_heads || num_softmax_heads == num_heads, + "num_softmax_heads must be either 0 or num_heads"); + if (!is_jagged) { + CHECK_SHAPE(q, batch_size, max_q_len, num_heads, qk_head_size); + CHECK_SHAPE(k, batch_size, max_seq_len, num_heads, qk_head_size); + CHECK_SHAPE(v, batch_size, max_seq_len, num_heads, v_head_size); + } else { + CHECK_SHAPE(q, total_seq_len_q, num_heads, qk_head_size); + CHECK_SHAPE(k, total_seq_len_kv, num_heads, qk_head_size); + CHECK_SHAPE(v, total_seq_len_kv, num_heads, v_head_size); + CHECK_SHAPE(seq_offsets_, batch_size + 1); + } + if (has_multiple_targets) { + CHECK_SHAPE(num_targets_, batch_size); + } + if (is_cross_attn) { + CHECK_SHAPE(seq_offsets_q_, batch_size + 1); + } + + int const alignment = q_type == torch::kFloat8_e4m3fn ? 16 : 8; + TORCH_CHECK( + qk_head_size % alignment == 0 && v_head_size % alignment == 0, + "head_size should be a multiple of " + std::to_string(alignment)); + + auto opts = q.options(); + auto out_type = q_type == at::ScalarType::Float8_e4m3fn + ? at::ScalarType::BFloat16 + : q_type; + at::Tensor out; + if (!is_jagged) { + out = torch::empty( + {batch_size, max_q_len, num_heads, v_head_size}, opts.dtype(out_type)); + } else { + out = torch::empty( + {total_seq_len_q, num_heads, v_head_size}, opts.dtype(out_type)); + } + std::optional softmax_lse = std::nullopt; + + // Early return for empty sequences to avoid TMA descriptor + // initialization failure + if (total_seq_len_kv == 0 || total_seq_len_q == 0) { + return {out, std::nullopt}; + } + + if (num_softmax_heads > 0) { + if (!is_jagged) { + softmax_lse = torch::empty( + {batch_size, num_softmax_heads, max_q_len}, opts.dtype(at::kFloat)); + } else { + softmax_lse = torch::empty( + {num_softmax_heads, total_seq_len_q}, opts.dtype(at::kFloat)); + } + } + + // Otherwise the kernel will be launched from cuda:0 device + // Cast to char to avoid compiler warning about narrowing + at::cuda::CUDAGuard device_guard{(char)q.get_device()}; + hstu::Flash_fwd_params params; + hstu::set_params_fprop( + params, + batch_size, + total_seq_len_kv, + total_seq_len_q, + max_seq_len, + max_q_len, + num_heads, + qk_head_size, + v_head_size, + q, + k, + v, + out, + !is_jagged ? nullptr : seq_offsets_.data_ptr(), + !has_multiple_targets ? nullptr : num_targets_.data_ptr(), + !has_attn_scale ? nullptr : attn_scale_.data_ptr(), + !is_cross_attn ? nullptr : seq_offsets_q_.data_ptr(), + (num_softmax_heads == 0) ? nullptr : softmax_lse.value().data_ptr(), + num_groups > 1 ? max_seq_len_tensor_.data_ptr() : nullptr, + ((num_groups > 1) && (!is_cross_attn)) + ? contextual_seq_len_tensor_.data_ptr() + : nullptr, + ((num_groups > 1) && (!is_cross_attn)) ? max_attn_len_tensor_.data_ptr() + : nullptr, + ((num_groups > 1) && (!is_cross_attn)) + ? min_full_attn_seq_len_tensor_.data_ptr() + : nullptr, + num_groups, + causal, + alpha, + scalar_scale, + max_attn_len, + min_full_attn_seq_len, + contextual_seq_len, + num_softmax_heads, + training, + sm_margin); + at::Tensor tile_count_semaphore; + // We don't use the persistent scheduler if not jagged + bool const persistent_scheduler = params.arch >= 90 + ? (params.is_causal || params.is_local || is_jagged) + : (params.is_causal || is_jagged); + if (persistent_scheduler) { + tile_count_semaphore = torch::zeros({1}, opts.dtype(torch::kInt32)); + params.tile_count_semaphore = tile_count_semaphore.data_ptr(); + } else { + params.tile_count_semaphore = nullptr; + } + + if (q_type == at::ScalarType::Float8_e4m3fn) { + if (q_descale.has_value()) { + auto q_descale_ = q_descale.value(); + CHECK_DEVICE(q_descale_); + CHECK_SHAPE(q_descale_, batch_size, num_heads); + params.q_descale_ptr = q_descale_.data_ptr(); + params.q_descale_batch_stride = q_descale_.stride(0); + params.q_descale_head_stride = q_descale_.stride(1); + } else { + params.q_descale_ptr = nullptr; + } + if (k_descale.has_value()) { + auto k_descale_ = k_descale.value(); + CHECK_DEVICE(k_descale_); + CHECK_SHAPE(k_descale_, batch_size, num_heads); + params.k_descale_ptr = k_descale_.data_ptr(); + params.k_descale_batch_stride = k_descale_.stride(0); + params.k_descale_head_stride = k_descale_.stride(1); + } else { + params.k_descale_ptr = nullptr; + } + if (v_descale.has_value()) { + auto v_descale_ = v_descale.value(); + CHECK_DEVICE(v_descale_); + CHECK_SHAPE(v_descale_, batch_size, num_heads); + params.v_descale_ptr = v_descale_.data_ptr(); + params.v_descale_batch_stride = v_descale_.stride(0); + params.v_descale_head_stride = v_descale_.stride(1); + } else { + params.v_descale_ptr = nullptr; + } + } + +#ifdef FLASHATTENTION_DISABLE_LOCAL + TORCH_CHECK( + !params.is_local, + "This flash attention build does not support local attention."); +#endif + + if (total_seq_len_q > 0 && num_heads > 0) { + auto stream = at::cuda::getCurrentCUDAStream().stream(); + run_mha_fwd(params, stream); + } + return {out, softmax_lse}; +} + +void run_mha_bwd(hstu::Flash_bwd_params& params, cudaStream_t stream) { +#ifndef FLASHATTENTION_DISABLE_BACKWARD + // FP16_SWITCH(!params.is_bf16, [&] { + // HEADDIM_SWITCH(params.d, [&] { + // hstu::run_mha_bwd_(params, stream); + // }); + // }); + ARCH_SWITCH(params.arch, Arch, [&] { + BOOL_SWITCH(params.num_softmax_heads == params.h, Softmax, [&] { + if (!params.is_bf16) { +#ifndef FLASHATTENTION_DISABLE_FP16 +#ifndef FLASHATTENTION_DISABLE_HDIM64 + if (params.qk_d <= 64) { + return hstu::run_mha_bwd_( + params, stream); + } +#endif +#ifndef FLASHATTENTION_DISABLE_HDIM96 + if (params.qk_d <= 96) { + return hstu::run_mha_bwd_( + params, stream); + } +#endif +#ifndef FLASHATTENTION_DISABLE_HDIM128 + if (params.qk_d <= 128) { + return hstu::run_mha_bwd_( + params, stream); + } +#endif +#ifndef FLASHATTENTION_DISABLE_HDIM192 + if (params.qk_d <= 192) { + return hstu::run_mha_bwd_( + params, stream); + } +#endif +#ifndef FLASHATTENTION_DISABLE_HDIM256 + if (params.qk_d <= 256) { + return hstu::run_mha_bwd_( + params, stream); + } +#endif +#else + TORCH_CHECK(false, "This flash attention build does not support FP16."); +#endif + } else { +#ifndef FLASHATTENTION_DISABLE_HDIM64 + if (params.qk_d <= 64) { + return hstu::run_mha_bwd_( + params, stream); + } +#endif +#ifndef FLASHATTENTION_DISABLE_HDIM96 + if (params.qk_d <= 96) { + return hstu::run_mha_bwd_( + params, stream); + } +#endif +#ifndef FLASHATTENTION_DISABLE_HDIM128 + if (params.qk_d <= 128) { + return hstu::run_mha_bwd_( + params, stream); + } +#endif +#ifndef FLASHATTENTION_DISABLE_HDIM192 + if (params.qk_d <= 192) { + return hstu::run_mha_bwd_( + params, stream); + } +#endif +#ifndef FLASHATTENTION_DISABLE_HDIM256 + if (params.qk_d <= 256) { + return hstu::run_mha_bwd_( + params, stream); + } +#endif + } + }); + }); +#endif +} + +std::vector hstu_mha_bwd( + int64_t max_seq_len, + double alpha, + at::Tensor& dout, + at::Tensor& q, + at::Tensor& k, + at::Tensor& v, + at::Tensor& dq, + at::Tensor& dk, + at::Tensor& dv, + at::Tensor& out, + const std::optional& seq_offsets, + bool causal, + const std::optional& num_targets, + const std::optional& attn_scale, + int64_t max_attn_len, + int64_t min_full_attn_seq_len, + int64_t contextual_seq_len, + bool sort_by_length, + bool const deterministic, + const int64_t sm_margin, + int64_t max_q_len, + const std::optional& seq_offsets_q, + int64_t num_softmax_heads, + const std::optional& softmax_lse, + const std::optional& max_seq_len_tensor, + const std::optional& contextual_seq_len_tensor, + const std::optional& max_attn_len_tensor, + const std::optional& min_full_attn_seq_len_tensor, + int64_t num_groups) { +#ifdef FLASHATTENTION_DISABLE_BACKWARD + TORCH_CHECK(false, "This flash attention build does not support backward."); +#endif + + auto dprops = at::cuda::getCurrentDeviceProperties(); + bool is_sm9x = dprops->major >= 9; + TORCH_CHECK(is_sm9x, "HSTU Attention only supports Hopper GPUs or newer."); + + q = switch_to_contiguous_if_needed(q); + k = switch_to_contiguous_if_needed(k); + v = switch_to_contiguous_if_needed(v); + out = switch_to_contiguous_if_needed(out); + dout = switch_to_contiguous_if_needed(dout); + + auto q_type = q.dtype(); + TORCH_CHECK( + q_type == torch::kFloat16 || q_type == torch::kBFloat16, + "FlashAttention only support fp16 and bf16 data type"); + TORCH_CHECK(k.dtype() == q_type, "query and key must have the same dtype"); + TORCH_CHECK(v.dtype() == q_type, "query and value must have the same dtype"); + TORCH_CHECK( + dout.dtype() == q_type, "query and dout must have the same dtype"); + + CHECK_DEVICE(q); + CHECK_DEVICE(k); + CHECK_DEVICE(v); + CHECK_DEVICE(dout); + + TORCH_CHECK( + q.stride(-1) == 1, "Input tensor must have contiguous last dimension"); + TORCH_CHECK( + k.stride(-1) == 1, "Input tensor must have contiguous last dimension"); + TORCH_CHECK( + v.stride(-1) == 1, "Input tensor must have contiguous last dimension"); + TORCH_CHECK( + dout.stride(-1) == 1, "dout tensor must have contiguous last dimension"); + + at::Tensor seq_offsets_; + bool const is_jagged = seq_offsets.has_value(); + if (is_jagged) { + seq_offsets_ = seq_offsets.value(); + CHECK_DEVICE(seq_offsets_); + CHECK_CONTIGUOUS(seq_offsets_); + TORCH_CHECK( + seq_offsets_.dtype() == torch::kInt32, + "seq_offsets_ must have dtype torch.int32"); + } + at::Tensor sort_by_length_indices_; + if (sort_by_length && is_jagged) { + auto seq_lengths = + seq_offsets_.slice(0, 1).sub(seq_offsets_.slice(0, 0, -1)); + std::tuple sort_result = torch::sort( + seq_lengths, false /*stable*/, 0 /*dim*/, true /*descending*/); + sort_by_length_indices_ = std::get<1>(sort_result).to(torch::kInt32); + CHECK_DEVICE(sort_by_length_indices_); + CHECK_CONTIGUOUS(sort_by_length_indices_); + TORCH_CHECK( + sort_by_length_indices_.dtype() == torch::kInt32, + "sort_by_length_indices_ must have dtype torch.int32"); + } + at::Tensor num_targets_; + bool const has_multiple_targets = num_targets.has_value(); + if (has_multiple_targets) { + num_targets_ = num_targets.value(); + CHECK_DEVICE(num_targets_); + CHECK_CONTIGUOUS(num_targets_); + TORCH_CHECK( + num_targets_.dtype() == torch::kInt32, + "num_targets_ must have dtype torch.int32"); + } + at::Tensor attn_scale_; + bool scalar_scale = true; + bool const has_attn_scale = attn_scale.has_value(); + if (has_attn_scale) { + attn_scale_ = attn_scale.value(); + scalar_scale = attn_scale_.numel() == num_groups; + CHECK_DEVICE(attn_scale_); + TORCH_CHECK( + attn_scale_.dtype() == torch::kFloat32, + "attn_scale_ must have dtype torch.float32"); + } + at::Tensor seq_offsets_q_; + bool const is_cross_attn = seq_offsets_q.has_value(); + if (is_cross_attn) { + seq_offsets_q_ = seq_offsets_q.value(); + CHECK_DEVICE(seq_offsets_q_); + CHECK_CONTIGUOUS(seq_offsets_q_); + TORCH_CHECK( + seq_offsets_q_.dtype() == torch::kInt32, + "seq_offsets_q_ must have dtype torch.int32"); + } else { + max_q_len = max_seq_len; + } + at::Tensor max_seq_len_tensor_; + at::Tensor contextual_seq_len_tensor_; + at::Tensor max_attn_len_tensor_; + at::Tensor min_full_attn_seq_len_tensor_; + if (num_groups > 1) { + TORCH_CHECK( + max_seq_len_tensor.has_value(), + "max_seq_len_tensor cannot be empty for num_groups > 1."); + max_seq_len_tensor_ = max_seq_len_tensor.value(); + CHECK_DEVICE(max_seq_len_tensor_); + TORCH_CHECK(max_seq_len_tensor_.dtype() == torch::kInt32); + if (!is_cross_attn) { + TORCH_CHECK( + contextual_seq_len_tensor.has_value(), + "contextual_seq_len_tensor cannot be empty for num_groups > 1 and not cross_attn."); + TORCH_CHECK( + max_attn_len_tensor.has_value(), + "max_attn_len_tensor cannot be empty for num_groups > 1 and not cross_attn."); + TORCH_CHECK( + min_full_attn_seq_len_tensor.has_value(), + "min_full_attn_seq_len_tensor cannot be empty for num_groups > 1 and not cross_attn."); + contextual_seq_len_tensor_ = contextual_seq_len_tensor.value(); + max_attn_len_tensor_ = max_attn_len_tensor.value(); + min_full_attn_seq_len_tensor_ = min_full_attn_seq_len_tensor.value(); + CHECK_DEVICE(contextual_seq_len_tensor_); + CHECK_DEVICE(max_attn_len_tensor_); + CHECK_DEVICE(min_full_attn_seq_len_tensor_); + TORCH_CHECK(contextual_seq_len_tensor_.dtype() == torch::kInt32); + TORCH_CHECK(max_attn_len_tensor_.dtype() == torch::kInt32); + TORCH_CHECK(min_full_attn_seq_len_tensor_.dtype() == torch::kInt32); + } + } + auto const sizes_q = q.sizes(); + auto const sizes_kv = k.sizes(); + int const batch_size = !is_jagged ? sizes_q[0] : seq_offsets_.size(0) - 1; + TORCH_CHECK( + batch_size % num_groups == 0, "batch_size not divisible by num_groups"); + if (!is_jagged) { + max_seq_len = sizes_kv[1]; + } + int const total_seq_len_q = !is_jagged ? batch_size * sizes_q[1] : sizes_q[0]; + int const total_seq_len_kv = + !is_jagged ? batch_size * sizes_kv[1] : sizes_kv[0]; + int const num_heads = q.size(-2); + int const qk_head_size = q.size(-1); + int const v_head_size = v.size(-1); + TORCH_CHECK( + qk_head_size % 8 == 0 && v_head_size % 8 == 0, + "head_size should be a multiple of 8"); + int const max_headdim = get_max_headdim(); + TORCH_CHECK( + qk_head_size <= max_headdim && v_head_size <= max_headdim, + "FlashAttention backward only supports head dimension at most " + + std::to_string(max_headdim)); + TORCH_CHECK(max_attn_len >= 0, "max_attn_len must be at least 0"); + TORCH_CHECK( + min_full_attn_seq_len >= 0, "min_full_attn_seq_len must be at least 0"); + TORCH_CHECK(contextual_seq_len >= 0, "contextual_seq_len must be at least 0"); + if (!is_jagged) { + CHECK_SHAPE(q, batch_size, max_q_len, num_heads, qk_head_size); + CHECK_SHAPE(k, batch_size, max_seq_len, num_heads, qk_head_size); + CHECK_SHAPE(v, batch_size, max_seq_len, num_heads, v_head_size); + CHECK_SHAPE(dout, batch_size, max_q_len, num_heads, v_head_size); + CHECK_SHAPE(dq, batch_size, max_q_len, num_heads, qk_head_size); + CHECK_SHAPE(dk, batch_size, max_seq_len, num_heads, qk_head_size); + CHECK_SHAPE(dv, batch_size, max_seq_len, num_heads, v_head_size); + } else { + CHECK_SHAPE(q, total_seq_len_q, num_heads, qk_head_size); + CHECK_SHAPE(k, total_seq_len_kv, num_heads, qk_head_size); + CHECK_SHAPE(v, total_seq_len_kv, num_heads, v_head_size); + CHECK_SHAPE(dout, total_seq_len_q, num_heads, v_head_size); + CHECK_SHAPE(dq, total_seq_len_q, num_heads, qk_head_size); + CHECK_SHAPE(dk, total_seq_len_kv, num_heads, qk_head_size); + CHECK_SHAPE(dv, total_seq_len_kv, num_heads, v_head_size); + CHECK_SHAPE(seq_offsets_, batch_size + 1); + } + if (has_multiple_targets) { + CHECK_SHAPE(num_targets_, batch_size); + } + if (is_cross_attn) { + CHECK_SHAPE(seq_offsets_q_, batch_size + 1); + } + int const arch = at::cuda::getCurrentDeviceProperties()->major * 10 + + at::cuda::getCurrentDeviceProperties()->minor; + int const qk_head_size_rounded = round_up_headdim(qk_head_size); + int const v_head_size_rounded = round_up_headdim(v_head_size); + // Very important that these match the kernel configs + bool const is_local = max_attn_len > 0; + int const kBlockM = + hstu::kBlockM_bwd(arch, qk_head_size_rounded, causal, is_local); + auto round_multiple = [](int x, int m) { return (x + m - 1) / m * m; }; + int const max_q_len_rounded = round_multiple(max_q_len, kBlockM); + int const total_seq_len_q_padded_rounded = + round_multiple(total_seq_len_q + batch_size * kBlockM, kBlockM); + + TORCH_CHECK(dq.dtype() == q_type, "dq must have the same dtype as q"); + CHECK_DEVICE(dq); + TORCH_CHECK(dq.stride(-1) == 1, "dq must have contiguous last dimension"); + if (!is_jagged) { + CHECK_SHAPE(dq, batch_size, max_q_len, num_heads, qk_head_size); + } else { + CHECK_SHAPE(dq, total_seq_len_q, num_heads, qk_head_size); + } + TORCH_CHECK(dk.dtype() == q_type, "dk must have the same dtype as q"); + CHECK_DEVICE(dk); + TORCH_CHECK(dk.stride(-1) == 1, "dk must have contiguous last dimension"); + if (!is_jagged) { + CHECK_SHAPE(dk, batch_size, max_seq_len, num_heads, qk_head_size); + } else { + CHECK_SHAPE(dk, total_seq_len_kv, num_heads, qk_head_size); + } + TORCH_CHECK(dv.dtype() == q_type, "dv must have the same dtype as q"); + CHECK_DEVICE(dv); + TORCH_CHECK(dv.stride(-1) == 1, "dv must have contiguous last dimension"); + if (!is_jagged) { + CHECK_SHAPE(dv, batch_size, max_seq_len, num_heads, v_head_size); + } else { + CHECK_SHAPE(dv, total_seq_len_kv, num_heads, v_head_size); + } + + // Otherwise the kernel will be launched from cuda:0 device + // Cast to char to avoid compiler warning about narrowing + at::cuda::CUDAGuard device_guard{(char)q.get_device()}; + auto opts = q.options(); + + at::Tensor dq_accum; + if (!is_jagged) { + dq_accum = torch::empty( + {batch_size, num_heads, max_q_len_rounded * qk_head_size_rounded}, + opts.dtype(at::kFloat)); + } else { + dq_accum = torch::empty( + {num_heads, total_seq_len_q_padded_rounded * qk_head_size_rounded}, + opts.dtype(at::kFloat)); + } + at::Tensor softmax_d, softmax_lse_log2; + if (!is_jagged) { + // Need softmax_d to have seqlen_q_rounded since we want its address to be + // aligned by 16/8 bytes for TMA / LDG.64 + softmax_d = torch::empty( + {batch_size, num_softmax_heads, max_q_len_rounded}, + opts.dtype(at::kFloat)); + softmax_lse_log2 = torch::empty( + {batch_size, num_softmax_heads, max_q_len_rounded}, + opts.dtype(at::kFloat)); + } else { + softmax_d = torch::empty( + {num_softmax_heads, total_seq_len_q_padded_rounded}, + opts.dtype(at::kFloat)); + softmax_lse_log2 = torch::empty( + {num_softmax_heads, total_seq_len_q_padded_rounded}, + opts.dtype(at::kFloat)); + } + + // Early return for empty sequences; analog to TMA prevention guard + // in hstu_mha_fwd + if (total_seq_len_kv == 0 || total_seq_len_q == 0) { + return {dq, dk, dv}; + } + + hstu::Flash_bwd_params params; + hstu::set_params_dgrad( + params, + batch_size, + total_seq_len_kv, + total_seq_len_q, + max_seq_len, + max_q_len, + max_q_len_rounded, + num_heads, + qk_head_size, + v_head_size, + qk_head_size_rounded, + v_head_size_rounded, + q, + k, + v, + out, + dout, + dq, + dk, + dv, + dq_accum.data_ptr(), + !is_jagged ? nullptr : seq_offsets_.data_ptr(), + !has_multiple_targets ? nullptr : num_targets_.data_ptr(), + !has_attn_scale ? nullptr : attn_scale_.data_ptr(), + !(sort_by_length && is_jagged) ? nullptr + : sort_by_length_indices_.data_ptr(), + !is_cross_attn ? nullptr : seq_offsets_q_.data_ptr(), + num_softmax_heads == 0 ? nullptr : softmax_lse.value().data_ptr(), + num_softmax_heads == 0 ? nullptr : softmax_d.data_ptr(), + num_softmax_heads == 0 ? nullptr : softmax_lse_log2.data_ptr(), + num_groups > 1 ? max_seq_len_tensor_.data_ptr() : nullptr, + ((num_groups > 1) && (!is_cross_attn)) + ? contextual_seq_len_tensor_.data_ptr() + : nullptr, + ((num_groups > 1) && (!is_cross_attn)) ? max_attn_len_tensor_.data_ptr() + : nullptr, + ((num_groups > 1) && (!is_cross_attn)) + ? min_full_attn_seq_len_tensor_.data_ptr() + : nullptr, + num_groups, + scalar_scale, + causal, + alpha, + max_attn_len, + min_full_attn_seq_len, + contextual_seq_len, + num_softmax_heads, + deterministic, + sm_margin); + + // auto tile_count_semaphore = (params.is_causal || params.is_local) ? + // torch::zeros({1}, opts.dtype(torch::kInt32)) : torch::empty({1}, + // opts.dtype(torch::kInt32)); params.tile_count_semaphore = + // tile_count_semaphore.data_ptr(); Will be zero'ed out in the + // backward preprocess kernel + at::Tensor dq_semaphore = torch::empty( + {(max_seq_len + kBlockM - 1) / kBlockM, batch_size, num_heads}, + opts.dtype(torch::kInt32)); + params.dq_semaphore = dq_semaphore.data_ptr(); + +#ifdef FLASHATTENTION_DISABLE_LOCAL + TORCH_CHECK( + !params.is_local, + "This flash attention build does not support local attention."); +#endif + + if (total_seq_len_q > 0 && num_heads > 0) { + auto stream = at::cuda::getCurrentCUDAStream().stream(); + run_mha_bwd(params, stream); + } + return {dq, dk, dv}; +} + +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_common.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_common.h new file mode 100644 index 000000000..98ca009f5 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_common.h @@ -0,0 +1,149 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/****************************************************************************** + * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, + *Pradeep Ramani, Tri Dao. + ******************************************************************************/ + +// Include these 2 headers instead of torch/extension.h since we don't need all +// of the torch headers. +#include +#include +#include + +#include + +#include // @manual + +#define CHECK_DEVICE(x) TORCH_CHECK(x.is_cuda(), #x " must be on CUDA") +#define CHECK_SHAPE(x, ...) \ + TORCH_CHECK( \ + x.sizes() == torch::IntArrayRef({__VA_ARGS__}), \ + #x " must have shape (" #__VA_ARGS__ ")") +#define CHECK_CONTIGUOUS(x) \ + TORCH_CHECK(x.is_contiguous(), #x " must be contiguous") + +inline int round_up_headdim(int head_size) { +#ifndef FLASHATTENTION_DISABLE_HDIM64 + if (head_size <= 64) { + return 64; + } +#endif +#ifndef FLASHATTENTION_DISABLE_HDIM96 + if (head_size <= 96) { + return 96; + } +#endif +#ifndef FLASHATTENTION_DISABLE_HDIM128 + if (head_size <= 128) { + return 128; + } +#endif +#ifndef FLASHATTENTION_DISABLE_HDIM192 + if (head_size <= 192) { + return 192; + } +#endif +#ifndef FLASHATTENTION_DISABLE_HDIM256 + if (head_size <= 256) { + return 256; + } +#endif + return 256; +} + +inline int get_max_headdim() { +#ifndef FLASHATTENTION_DISABLE_HDIM256 + return 256; +#endif +#ifndef FLASHATTENTION_DISABLE_HDIM192 + return 192; +#endif +#ifndef FLASHATTENTION_DISABLE_HDIM128 + return 128; +#endif +#ifndef FLASHATTENTION_DISABLE_HDIM96 + return 96; +#endif +#ifndef FLASHATTENTION_DISABLE_HDIM64 + return 64; +#endif + return 0; +} + +namespace hstu { + +std::tuple> hstu_mha_fwd( + int64_t max_seq_len, + double alpha, + at::Tensor& q, // (b, s, h, d) or (total_s, h, d) + at::Tensor& k, // (b, s, h, d) or (total_s, h, d) + at::Tensor& v, // (b, s, h, d) or (total_s, h, d) + const std::optional& seq_offsets, + bool causal, + const std::optional& num_targets, + const std::optional& attn_scale, + int64_t max_attn_len, + int64_t min_full_attn_seq_len, + int64_t contextual_seq_len, + const std::optional& q_descale, // (b, h_k), not (b, h) + const std::optional& k_descale, // (b, h_k) + const std::optional& v_descale, // (b, h_k) + const int64_t sm_margin = 0, + int64_t max_q_len = 0, + const std::optional& seq_offsets_q = std::nullopt, + int64_t num_softmax_heads = 0, + bool training = true, + const std::optional& max_seq_len_tensor = std::nullopt, + const std::optional& contextual_seq_len_tensor = std::nullopt, + const std::optional& max_attn_len_tensor = std::nullopt, + const std::optional& min_full_attn_seq_len_tensor = + std::nullopt, + int64_t num_groups = 1); + +std::vector hstu_mha_bwd( + int64_t max_seq_len, + double alpha, + at::Tensor& dout, + at::Tensor& q, + at::Tensor& k, + at::Tensor& v, + at::Tensor& dq, + at::Tensor& dk, + at::Tensor& dv, + at::Tensor& out, + const std::optional& seq_offsets, + bool causal, + const std::optional& num_targets, + const std::optional& attn_scale, + int64_t max_attn_len, + int64_t min_full_attn_seq_len, + int64_t contextual_seq_len, + bool sort_by_length, + bool const deterministic, + const int64_t sm_margin = 0, + int64_t max_q_len = 0, + const std::optional& seq_offsets_q = std::nullopt, + int64_t num_softmax_heads = 0, + const std::optional& softmax_lse = std::nullopt, + const std::optional& max_seq_len_tensor = std::nullopt, + const std::optional& contextual_seq_len_tensor = std::nullopt, + const std::optional& max_attn_len_tensor = std::nullopt, + const std::optional& min_full_attn_seq_len_tensor = + std::nullopt, + int64_t num_groups = 1); + +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_common_cpu.cpp b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_common_cpu.cpp new file mode 100644 index 000000000..30d4f792c --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_common_cpu.cpp @@ -0,0 +1,172 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/****************************************************************************** + * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, + *Pradeep Ramani, Tri Dao. + ******************************************************************************/ + +#include +#include + +#include "flash_common_cpu.h" + +namespace hstu { + +std::tuple> hstu_mha_fwd_meta( + const at::SymInt max_seq_len, + double alpha, + at::Tensor& q, // (b, s, h, d) or (total_s, h, d) + at::Tensor& k, // (b, s, h, d) or (total_s, h, d) + at::Tensor& v, // (b, s, h, d) or (total_s, h, d) + const std::optional& seq_offsets, + bool causal, + const std::optional& num_targets, + const std::optional& attn_scale, + int64_t max_attn_len, + int64_t min_full_attn_seq_len, + int64_t contextual_seq_len, + const std::optional& q_descale, // (b, h_k), not (b, h) + const std::optional& k_descale, // (b, h_k) + const std::optional& v_descale, // (b, h_k) + const int64_t sm_margin, + int64_t max_q_len, + const std::optional& seq_offsets_q, + int64_t num_softmax_heads, + bool training, + const std::optional& max_seq_len_tensor, + const std::optional& contextual_seq_len_tensor, + const std::optional& max_attn_len_tensor, + const std::optional& min_full_attn_seq_len_tensor, + int64_t num_groups) { + auto q_type = q.scalar_type(); + auto const sizes = q.sym_sizes(); + at::Tensor seq_offsets_; + bool const is_jagged = seq_offsets.has_value(); + if (is_jagged) { + seq_offsets_ = seq_offsets.value(); + } + const c10::SymInt batch_size = + !is_jagged ? sizes[0] : seq_offsets_.sym_sizes()[0] - 1; + auto total_seq_len = !is_jagged ? batch_size * max_seq_len : sizes[0]; + const auto& num_heads = sizes[sizes.size() - 2]; + auto v_head_size = v.sym_sizes()[v.sym_sizes().size() - 1]; + auto out_type = q_type == at::ScalarType::Float8_e4m3fn + ? at::ScalarType::BFloat16 + : q_type; + auto opts = q.options(); + + at::Tensor out; + if (!is_jagged) { + out = at::empty_symint( + {batch_size, max_seq_len, num_heads, v_head_size}, + opts.dtype(out_type)); + } else { + out = at::empty_symint( + {total_seq_len, num_heads, v_head_size}, opts.dtype(out_type)); + } + return {out, std::nullopt}; +}; + +std::tuple> hstu_mha_fwd_dummy( + int64_t max_seq_len, + double alpha, + at::Tensor& q, // (b, s, h, d) or (total_s, h, d) + at::Tensor& k, // (b, s, h, d) or (total_s, h, d) + at::Tensor& v, // (b, s, h, d) or (total_s, h, d) + const std::optional& seq_offsets, + bool causal, + const std::optional& num_targets, + const std::optional& attn_scale, + int64_t max_attn_len, + int64_t min_full_attn_seq_len, + int64_t contextual_seq_len, + const std::optional& q_descale, // (b, h_k), not (b, h) + const std::optional& k_descale, // (b, h_k) + const std::optional& v_descale, // (b, h_k) + const int64_t sm_margin, + const int64_t max_q_len, + const std::optional& seq_offsets_q, + int64_t num_softmax_heads, + bool training, + const std::optional& max_seq_len_tensor, + const std::optional& contextual_seq_len_tensor, + const std::optional& max_attn_len_tensor, + const std::optional& min_full_attn_seq_len_tensor, + int64_t num_groups) { + auto q_type = q.scalar_type(); + auto const sizes = q.sizes(); + at::Tensor seq_offsets_; + bool const is_jagged = seq_offsets.has_value(); + if (is_jagged) { + seq_offsets_ = seq_offsets.value(); + } + const int batch_size = !is_jagged ? sizes[0] : seq_offsets_.size(0) - 1; + int total_seq_len = !is_jagged ? batch_size * max_seq_len : sizes[0]; + int num_heads = q.size(-2); + // int const qk_head_size = q.size(-1); + int const v_head_size = v.size(-1); + // int const max_headdim = get_max_headdim(); + auto out_type = q_type == at::ScalarType::Float8_e4m3fn + ? at::ScalarType::BFloat16 + : q_type; + auto opts = q.options(); + + at::Tensor out; + if (!is_jagged) { + out = torch::empty( + {batch_size, max_seq_len, num_heads, v_head_size}, + opts.dtype(out_type)); + } else { + out = torch::empty( + {total_seq_len, num_heads, v_head_size}, opts.dtype(out_type)); + } + return {out, std::nullopt}; +}; + +std::vector hstu_mha_bwd_dummy( + int64_t max_seq_len, + double alpha, + at::Tensor& dout, + at::Tensor& q, + at::Tensor& k, + at::Tensor& v, + at::Tensor& dq, + at::Tensor& dk, + at::Tensor& dv, + at::Tensor& out, + const std::optional& seq_offsets, + bool causal, + const std::optional& num_targets, + const std::optional& attn_scale, + int64_t max_attn_len, + int64_t min_full_attn_seq_len, + int64_t contextual_seq_len, + bool sort_by_length, + bool const deterministic, + const int64_t sm_margin, + const int64_t max_q_len, + const std::optional& seq_offsets_q, + int64_t num_softmax_heads, + const std::optional& softmax_lse, + const std::optional& max_seq_len_tensor, + const std::optional& contextual_seq_len_tensor, + const std::optional& max_attn_len_tensor, + const std::optional& min_full_attn_seq_len_tensor, + int64_t num_groups) { + return {dq, dk, dv}; +}; + +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_common_cpu.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_common_cpu.h new file mode 100644 index 000000000..9d0e18a71 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_common_cpu.h @@ -0,0 +1,114 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/****************************************************************************** + * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, + *Pradeep Ramani, Tri Dao. + ******************************************************************************/ + +#include +#include // @manual +#include + +namespace hstu { + +std::tuple> hstu_mha_fwd_dummy( + int64_t max_seq_len, + double alpha, + at::Tensor& q, // (b, s, h, d) or (total_s, h, d) + at::Tensor& k, // (b, s, h, d) or (total_s, h, d) + at::Tensor& v, // (b, s, h, d) or (total_s, h, d) + const std::optional& seq_offsets, + bool causal, + const std::optional& num_targets, + const std::optional& attn_scale, + int64_t max_attn_len, + int64_t min_full_attn_seq_len, + int64_t contextual_seq_len, + const std::optional& q_descale, // (b, h_k), not (b, h) + const std::optional& k_descale, // (b, h_k) + const std::optional& v_descale, // (b, h_k) + const int64_t sm_margin = 0, + int64_t max_q_len = 0, + const std::optional& seq_offsets_q = std::nullopt, + int64_t num_softmax_heads = 0, + bool training = true, + const std::optional& max_seq_len_tensor = std::nullopt, + const std::optional& contextual_seq_len_tensor = std::nullopt, + const std::optional& max_attn_len_tensor = std::nullopt, + const std::optional& min_full_attn_seq_len_tensor = + std::nullopt, + int64_t num_groups = 1); + +std::vector hstu_mha_bwd_dummy( + int64_t max_seq_len, + double alpha, + at::Tensor& dout, + at::Tensor& q, + at::Tensor& k, + at::Tensor& v, + at::Tensor& dq, + at::Tensor& dk, + at::Tensor& dv, + at::Tensor& out, + const std::optional& seq_offsets, + bool causal, + const std::optional& num_targets, + const std::optional& attn_scale, + int64_t max_attn_len, + int64_t min_full_attn_seq_len, + int64_t contextual_seq_len, + bool sort_by_length, + bool const deterministic, + const int64_t sm_margin = 0, + int64_t max_q_len = 0, + const std::optional& seq_offsets_q = std::nullopt, + int64_t num_softmax_heads = 0, + const std::optional& softmax_lse = std::nullopt, + const std::optional& max_seq_len_tensor = std::nullopt, + const std::optional& contextual_seq_len_tensor = std::nullopt, + const std::optional& max_attn_len_tensor = std::nullopt, + const std::optional& min_full_attn_seq_len_tensor = + std::nullopt, + int64_t num_groups = 1); + +std::tuple> hstu_mha_fwd_meta( + const at::SymInt max_seq_len, + double alpha, + at::Tensor& q, // (b, s, h, d) or (total_s, h, d) + at::Tensor& k, // (b, s, h, d) or (total_s, h, d) + at::Tensor& v, // (b, s, h, d) or (total_s, h, d) + const std::optional& seq_offsets, + bool causal, + const std::optional& num_targets, + const std::optional& attn_scale, + int64_t max_attn_len, + int64_t min_full_attn_seq_len, + int64_t contextual_seq_len, + const std::optional& q_descale, // (b, h_k), not (b, h) + const std::optional& k_descale, // (b, h_k) + const std::optional& v_descale, // (b, h_k) + const int64_t sm_margin = 0, + int64_t max_q_len = 0, + const std::optional& seq_offsets_q = std::nullopt, + int64_t num_softmax_heads = 0, + bool training = true, + const std::optional& max_seq_len_tensor = std::nullopt, + const std::optional& contextual_seq_len_tensor = std::nullopt, + const std::optional& max_attn_len_tensor = std::nullopt, + const std::optional& min_full_attn_seq_len_tensor = + std::nullopt, + int64_t num_groups = 1); +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_fwd_kernel_sm90.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_fwd_kernel_sm90.h new file mode 100644 index 000000000..2e3d0916b --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_fwd_kernel_sm90.h @@ -0,0 +1,511 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/****************************************************************************** + * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, + *Pradeep Ramani, Tri Dao. + ******************************************************************************/ + +#pragma once + +#include "cute/tensor.hpp" + +#include +#include +#include +#include +#include +#include +#include + +#include "seqlen.h" +#include "softmax.h" +#include "tile_scheduler.h" +#include "utils.h" + +namespace hstu { + +using namespace cute; + +template < + bool Softmax, + class CollectiveMainloop_, + class CollectiveEpilogue_, + class TileScheduler_> +class FlashAttnFwdSm90 { + public: + // Type Aliases + using CollectiveMainloop = CollectiveMainloop_; + using CollectiveEpilogue = CollectiveEpilogue_; + static constexpr bool Is_FP8 = CollectiveMainloop::Is_FP8; + static constexpr bool Transpose_V = CollectiveMainloop::Transpose_V; + static constexpr bool Use_TMA_O = CollectiveEpilogue::Use_TMA_O; + static constexpr int NumProducerThreads = + CollectiveMainloop::NumProducerThreads; + using SeqlenInfo_t = typename CollectiveMainloop::SeqlenInfo_t; + + // Mainloop derived types + using TileShape_MNK = typename CollectiveMainloop::TileShape_MNK; + using TiledMma0 = typename CollectiveMainloop::TiledMma0; + using TiledMma1 = typename CollectiveMainloop::TiledMma1; + using ArchTag = typename CollectiveMainloop::ArchTag; + using ClusterShape = typename CollectiveMainloop::ClusterShape; + using MainloopArguments = typename CollectiveMainloop::Arguments; + using MainloopParams = typename CollectiveMainloop::Params; + using BarrierQ = cutlass::arch::ClusterTransactionBarrier; + + // Epilogue derived types + using EpilogueArguments = typename CollectiveEpilogue::Arguments; + using EpilogueParams = typename CollectiveEpilogue::Params; + + static_assert(ArchTag::kMinComputeCapability >= 90); + + using TileScheduler = TileScheduler_; + using TileSchedulerArguments = typename hstu::TileSchedulerArguments; + using TileSchedulerParams = typename TileScheduler::Params; + + static constexpr uint32_t NumLoadWarpGroups = 1; + static constexpr uint32_t NumMmaWarpGroups = + CUTE_STATIC_V(size(TiledMma0{})) / cutlass::NumThreadsPerWarpGroup; + static constexpr uint32_t MaxThreadsPerBlock = + CUTE_STATIC_V(size(TiledMma0{})) + + (NumLoadWarpGroups * cutlass::NumThreadsPerWarpGroup); + static constexpr uint32_t MinBlocksPerMultiprocessor = 1; + static_assert( + NumMmaWarpGroups == 1 || NumMmaWarpGroups == 2 || NumMmaWarpGroups == 3); + + /// Register requirement for Load and Math WGs + // If we use cp.async to load K and V, we need more registers for the producer + // WG. + static constexpr uint32_t LoadRegisterRequirement = + NumMmaWarpGroups == 1 ? 56 : (NumMmaWarpGroups == 2 ? 24 : 32); + static constexpr uint32_t MmaRegisterRequirement = + NumMmaWarpGroups == 1 ? 256 : (NumMmaWarpGroups == 2 ? 240 : 160); + // If you want to print from the producer warp, you'd need to increase the + // number of registers Otherwise you'll get CUDA error. static constexpr + // uint32_t LoadRegisterRequirement = 40; static constexpr uint32_t + // MmaRegisterRequirement = NumMmaWarpGroups == 2 ? 232 : 152; + + // Kernel level shared memory storage + // We overlap the shared memory for the mainloop and epilogue. However, we + // only want smem_o to overlap with smem_v and nothing else, so we'll pad in + // case sizeof(smem_o) > sizeof(smem_v). + static constexpr int mainloop_smem_padding_ = + int(sizeof(typename CollectiveEpilogue::TensorStorage)) - + int(sizeof( + decltype((typename CollectiveMainloop::TensorStorage{}).smem_v))); + static constexpr int mainloop_smem_padding = + mainloop_smem_padding_ < 0 ? 0 : mainloop_smem_padding_; + struct SharedStorage { + struct TensorStorage : cute::aligned_struct<128> { + union { + struct { + cute::array + padding_; + typename CollectiveMainloop::TensorStorage mainloop; + }; + // We want smem_o to line up with the start of smem_v + typename CollectiveEpilogue::TensorStorage epilogue; + }; + } tensors; + + struct PipelineStorage : cute::aligned_struct<16> { + alignas(16) BarrierQ barrier_Q; + alignas(16) cutlass::arch::ClusterBarrier barrier_O; + alignas(16) typename CollectiveMainloop::MainloopPipelineK::SharedStorage + pipeline_k; + alignas(16) typename CollectiveMainloop::MainloopPipelineV::SharedStorage + pipeline_v; + alignas(16) typename CollectiveMainloop::MainloopPipelineVt::SharedStorage + pipeline_vt; + alignas(16) typename TileScheduler::SharedStorage smem_scheduler; + } pipelines; + }; + + static constexpr int SharedStorageSize = sizeof(SharedStorage); + + // Device side arguments + struct Arguments { + MainloopArguments mainloop{}; + EpilogueArguments epilogue{}; + cutlass::KernelHardwareInfo hw_info{}; + TileSchedulerArguments scheduler{}; + }; + + // Kernel entry point API + struct Params { + MainloopParams mainloop{}; + EpilogueParams epilogue{}; + cutlass::KernelHardwareInfo hw_info{}; + TileSchedulerParams scheduler{}; + }; + + // + // Methods + // + + // Convert to underlying arguments. In this case, a simple copy for the + // aliased type. + static Params to_underlying_arguments(Arguments const& args) { + CUTLASS_TRACE_HOST("to_underlying_arguments():"); + + // Get SM count if needed, otherwise use user supplied SM count + int sm_count = args.hw_info.sm_count; + if (sm_count <= 0) { + CUTLASS_TRACE_HOST( + " WARNING: Arguments do not include a valid SM count.\n" + " For optimal performance, populate the arguments KernelHardwareInfo struct with the SM count."); + sm_count = cutlass::KernelHardwareInfo::query_device_multiprocessor_count( + args.hw_info.device_id); + } + + CUTLASS_TRACE_HOST( + "to_underlying_arguments(): Setting persistent grid SM count to " + << sm_count); + + cutlass::KernelHardwareInfo hw_info{args.hw_info.device_id, sm_count}; + return { + CollectiveMainloop::to_underlying_arguments(args.mainloop), + CollectiveEpilogue::to_underlying_arguments(args.epilogue), + hw_info, + TileScheduler::to_underlying_arguments(args.scheduler)}; + } + + // Computes the kernel launch grid shape based on runtime parameters + static dim3 get_grid_shape(Params const& params) { + return TileScheduler::get_grid_shape( + params.scheduler, params.hw_info.sm_count); + } + + static dim3 get_block_shape() { + return dim3(MaxThreadsPerBlock, 1, 1); + } + + CUTLASS_DEVICE + void operator()(Params const& params, char* smem_buf) { + static constexpr int NumMmaThreads = + NumMmaWarpGroups * cutlass::NumThreadsPerWarpGroup; + static constexpr int MmaThreadOffset = + NumLoadWarpGroups * cutlass::NumThreadsPerWarpGroup; + static constexpr int kBlockM = get<0>(TileShape_MNK{}); + + using MainloopPipelineK = typename CollectiveMainloop::MainloopPipelineK; + using MainloopPipelineV = typename CollectiveMainloop::MainloopPipelineV; + using MainloopPipelineVt = typename CollectiveMainloop::MainloopPipelineVt; + using MainloopPipelineKVNew = + typename CollectiveMainloop::MainloopPipelineKVNew; + using PipelineState = typename CollectiveMainloop::PipelineState; + using PipelineParamsK = typename MainloopPipelineK::Params; + using PipelineParamsV = typename MainloopPipelineV::Params; + using PipelineParamsVt = typename MainloopPipelineVt::Params; + using PipelineParamsKVNew = typename MainloopPipelineKVNew::Params; + + SharedStorage& shared_storage = *reinterpret_cast(smem_buf); + + int const lane_predicate = cute::elect_one_sync(); + int const warp_idx = cutlass::canonical_warp_idx_sync(); + + // Issue Tma Descriptor Prefetch from a single thread + if (warp_idx == 0 && lane_predicate) { + CollectiveMainloop::prefetch_tma_descriptors(params.mainloop); + CollectiveEpilogue::prefetch_tma_descriptors(params.epilogue); + } + + // Obtain warp index + int const warp_group_thread_idx = + threadIdx.x % cutlass::NumThreadsPerWarpGroup; + int warp_group_idx = cutlass::canonical_warp_group_idx(); + + if (warp_idx == 0 && lane_predicate) { + shared_storage.pipelines.barrier_Q.init(1 /*numThreads*/); + shared_storage.pipelines.barrier_O.init( + size(ClusterShape{}) * + (Use_TMA_O ? 1 : NumMmaThreads) /*numThreads*/); + } + + // We're counting on pipeline_k to call cutlass::arch::fence_barrier_init(); + PipelineParamsK pipeline_params_k; + pipeline_params_k.role = warp_group_idx == 0 + ? MainloopPipelineK::ThreadCategory::Producer + : MainloopPipelineK::ThreadCategory::Consumer; + pipeline_params_k.transaction_bytes = + CollectiveMainloop::TmaTransactionBytesK; + pipeline_params_k.is_leader = warp_group_thread_idx == 0; + pipeline_params_k.num_consumers = NumMmaThreads; + + MainloopPipelineK pipeline_k = [&] { + return MainloopPipelineK( + shared_storage.pipelines.pipeline_k, + pipeline_params_k, + ClusterShape{}); + }(); + // MainloopPipelineV pipeline_v(shared_storage.pipelines.pipeline_v, + // pipeline_params_v, ClusterShape{}); + MainloopPipelineV pipeline_v = [&] { + if constexpr (!Transpose_V) { + static_assert(is_same_v); + return MainloopPipelineV( + shared_storage.pipelines.pipeline_v, + pipeline_params_k, + ClusterShape{}); + } else { + PipelineParamsV pipeline_params_v; + pipeline_params_v.role = warp_group_idx == 0 + ? MainloopPipelineV::ThreadCategory::Producer + : MainloopPipelineV::ThreadCategory::Consumer; + pipeline_params_v.producer_arv_count = NumProducerThreads; + pipeline_params_v.consumer_arv_count = NumMmaThreads; + return MainloopPipelineV( + shared_storage.pipelines.pipeline_v, pipeline_params_v); + } + }(); + static_assert(is_same_v); + // If we need to transpose V (e.g. FP8 and V is row-major), we use + // pipeline_vt for the TMA, then the producer WG will read from pipeline_vt + // and write to pipeline_v. If we don't need to transpose V, we use + // pipeline_v for the TMA, and pipeline_vt won't be used. Technically for + // pipeline_params_vt, warp0 of WG0 is the producer and all of WG0 are + // consumers. However, the thread role isn't used in the pipeline + // implementation. + MainloopPipelineVt pipeline_vt = [&] { + pipeline_params_k.num_consumers = + NumProducerThreads; // TMA_V is only consumed by the producer WG + return MainloopPipelineVt( + shared_storage.pipelines.pipeline_vt, + pipeline_params_k, + ClusterShape{}); + }(); + + CollectiveMainloop collective_mainloop; + CollectiveEpilogue collective_epilogue; + + // We need this to guarantee that the Pipeline init is visible to all + // producers and consumer blocks in the Cluster + if constexpr (size(ClusterShape{}) > 1) { + cute::cluster_arrive_relaxed(); + cute::cluster_wait(); + } else { + __syncthreads(); + } + + if (warp_group_idx == 0) { // Producer + cutlass::arch::warpgroup_reg_dealloc(); + + // The pipelines for AppendKV and main attention are different, since e.g. + // main attention might use cp.async to load KV (if PagedKV) while + // AppendKV always uses TMA to load KV_new. Since the pipeline states are + // different, we have to manually sync to make sure the two pipelines + // don't race when accessing smem_k and smem_v. + PipelineState smem_pipe_write = + cutlass::make_producer_start_state(); + PipelineState smem_pipe_write_new = + cutlass::make_producer_start_state(); + int work_idx = 0; + + TileScheduler scheduler( + reinterpret_cast( + &shared_storage.pipelines.smem_scheduler)); + int warp_idx_in_warpgroup = + __shfl_sync(0xffffffff, (threadIdx.x / 32) % 4, 0); + static constexpr bool SingleProducerWarp = + NumProducerThreads == cutlass::NumThreadsPerWarp; + if constexpr (SingleProducerWarp) { + if (warp_idx_in_warpgroup != 0) { + return; + } + } + if (!SingleProducerWarp && warp_idx_in_warpgroup != 0) { + scheduler.init_consumer(); + } + + // Load Q, K, V + for (auto work_tile_info = SingleProducerWarp || + warp_idx_in_warpgroup == 0 + ? scheduler.template get_initial_work( + params.scheduler) + : scheduler.template get_initial_work( + params.scheduler); + work_tile_info.is_valid(params.scheduler); + work_tile_info = SingleProducerWarp || warp_idx_in_warpgroup == 0 + ? scheduler.template get_next_work( + params.scheduler, work_tile_info) + : scheduler.template get_next_work( + params.scheduler, work_tile_info)) { + auto block_coord = work_tile_info.get_block_coord(params.scheduler); + SeqlenInfo_t seqlen_info{ + get<2>(block_coord) /*bidb*/, + get<0>(params.mainloop.shape_Q), + get<0>(params.mainloop.shape_K), + params.mainloop.seq_offsets, + params.mainloop.seq_offsets_q, + params.mainloop.num_targets, + }; + auto scheduler_prefetch = [&scheduler, ¶ms, &work_tile_info]() { + scheduler.prefetch_next_work(params.scheduler, work_tile_info); + }; + // pipeline_vt won't be used if we don't need to transpose V. + collective_mainloop.load( + params.mainloop, + pipeline_k, + pipeline_v, + pipeline_vt, + smem_pipe_write, + shared_storage, + scheduler_prefetch, + seqlen_info, + block_coord, + work_idx); + } + collective_mainloop.load_tail( + pipeline_k, + pipeline_v, + pipeline_vt, + smem_pipe_write, + shared_storage, + work_idx); + } else { // Consumer + cutlass::arch::warpgroup_reg_alloc(); + + TileScheduler scheduler( + reinterpret_cast( + &shared_storage.pipelines.smem_scheduler)); + // Initialize matmul objects. + TiledMma1 tiled_mma1; + + PipelineState smem_pipe_read; + // We don't need separate variables smem_pipe_release_k and + // smem_pipe_release_v (like in Cutlass's gemm) because the read and + // release pipeline states are always the same. + + scheduler.init_consumer(); + collective_mainloop.mma_init(); + + int work_idx = 0; + CUTLASS_PRAGMA_NO_UNROLL + for (auto work_tile_info = + scheduler.template get_initial_work( + params.scheduler); + work_tile_info.is_valid(params.scheduler); + work_tile_info = + scheduler.template get_next_work( + params.scheduler, work_tile_info)) { + // Attention output (GEMM-II) accumulator. + Tensor tOrO = + partition_fragment_C(tiled_mma1, select<0, 2>(TileShape_MNK{})); + // If there's tanh softcap, the scaling will be done before tanh. + auto block_coord = work_tile_info.get_block_coord(params.scheduler); + int const bidb = get<2>(block_coord); + int const bidh = get<1>(block_coord); + if constexpr (Is_FP8) { + int const bidh_kv = bidh; + float const q_descale = params.mainloop.ptr_q_descale == nullptr + ? 1.0f + : params.mainloop.ptr_q_descale + [bidb * get<0>(params.mainloop.stride_q_descale) + + bidh_kv * get<1>(params.mainloop.stride_q_descale)]; + float const k_descale = params.mainloop.ptr_k_descale == nullptr + ? 1.0f + : params.mainloop.ptr_k_descale + [bidb * get<0>(params.mainloop.stride_k_descale) + + bidh_kv * get<1>(params.mainloop.stride_k_descale)]; + } + + SeqlenInfo_t seqlen_info{ + bidb, + get<0>(params.mainloop.shape_Q), + get<0>(params.mainloop.shape_K), + params.mainloop.seq_offsets, + params.mainloop.seq_offsets_q, + params.mainloop.num_targets, + }; + float alpha_log2 = params.mainloop.alpha_log2; + bool tile_valid; + if constexpr (Softmax) { + hstu::Softmax< + 2 * (2 * kBlockM / NumMmaThreads), + /*Max_offset=*/!Is_FP8 ? 0 : 8> + softmax(alpha_log2); + tile_valid = collective_mainloop.mma_softmax( + params.mainloop, + pipeline_k, + pipeline_v, + smem_pipe_read, + tOrO, + softmax, + threadIdx.x - MmaThreadOffset, + work_idx, + seqlen_info, + block_coord, + shared_storage); + if (tile_valid) { + collective_epilogue.store( + params.epilogue, + tOrO, + shared_storage, + tiled_mma1, + threadIdx.x - MmaThreadOffset, + block_coord); + collective_epilogue.store_softmax( + params.epilogue, + softmax.row_sum, + tiled_mma1, + threadIdx.x - MmaThreadOffset, + block_coord); + } else { + // Write 0 to gO and -inf to gLSE. + // If Split, we don't have to write 0 to O if the mha_combine kernel + // is used, since it will not use the value of O if LSE is -inf. + collective_epilogue.template store_zero( + params.epilogue, threadIdx.x - MmaThreadOffset, block_coord); + // collective_epilogue.store_zero(params.epilogue, threadIdx.x - + // MmaThreadOffset, block_coord); + } + } else { + tile_valid = collective_mainloop.mma( + params.mainloop, + pipeline_k, + pipeline_v, + smem_pipe_read, + tOrO, + threadIdx.x - MmaThreadOffset, + work_idx, + seqlen_info, + block_coord, + shared_storage); + if (tile_valid) { + collective_epilogue.store( + params.epilogue, + tOrO, + shared_storage, + tiled_mma1, + threadIdx.x - MmaThreadOffset, + block_coord); + } else { + // Write 0 to gO and -inf to gLSE. + // If Split, we don't have to write 0 to O if the mha_combine kernel + // is used, since it will not use the value of O if LSE is -inf. + collective_epilogue.template store_zero( + params.epilogue, threadIdx.x - MmaThreadOffset, block_coord); + // collective_epilogue.store_zero(params.epilogue, threadIdx.x - + // MmaThreadOffset, block_coord); + } + } + } + collective_epilogue.store_tail(); + } + } +}; + +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_fwd_launch_template.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_fwd_launch_template.h new file mode 100644 index 000000000..c79ea3a3f --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_fwd_launch_template.h @@ -0,0 +1,376 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/****************************************************************************** + * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, + *Pradeep Ramani, Tri Dao. + ******************************************************************************/ + +#pragma once + +// clang-format off +#include "cute/tensor.hpp" + +#include "cutlass/cutlass.h" +#include "cutlass/device_kernel.h" // For device_kernel +#include +#include "cutlass/cluster_launch.hpp" +#include "cutlass/kernel_launch.h" + +#include "static_switch.h" +#include "flash.h" +#include "tile_size.h" +#include "tile_scheduler.h" +#include "flash_fwd_kernel_sm90.h" +#include "mainloop_fwd_sm90_tma_gmma_ws.h" +#include "epilogue_fwd.h" +// clang-format on + +namespace hstu { + +using namespace cute; + +template < + int Arch, + int kHeadDim, + int ClusterM, + typename Element, + typename ElementOut, + bool Causal, + bool Local, + bool Contexual_mask, + bool Jagged, + bool Has_targets, + bool V_colmajor, + bool Cross, + bool Softmax, + bool Training> +void run_flash_fwd(hstu::Flash_fwd_params& params, cudaStream_t stream) { + static_assert( + !(Causal && Local), + "Causal and Local cannot be enabled at the same time"); + static constexpr bool Is_FP8 = + cute::is_same_v || + cute::is_same_v; + static constexpr bool FP8_TransposeV = Is_FP8 && !V_colmajor; + using ArchTag = + std::conditional_t= 90, cutlass::arch::Sm90, cutlass::arch::Sm80>; + + // Can't use structured binding since it's not compatible with constexpr + static constexpr std::tuple kBlockMN_RS = + hstu::tile_size_fwd_sm90( + kHeadDim, + Causal, + Local, + sizeof(Element) /*element_size*/, + V_colmajor, + Cross, + Training); + static constexpr std::tuple + kBlockMN_kNWarps_Stages_RS = hstu::tile_size_fwd_sm8x( + Arch == 86 || Arch == 89, + kHeadDim, + Causal, + Local, + sizeof(Element) /*element_size*/); + static constexpr int kBlockM = Arch >= 90 + ? std::get<0>(kBlockMN_RS) + : std::get<0>(kBlockMN_kNWarps_Stages_RS); + static constexpr int kBlockN = Arch >= 90 + ? std::get<1>(kBlockMN_RS) + : std::get<1>(kBlockMN_kNWarps_Stages_RS); + static constexpr bool Mma1_is_RS = std::get<2>(kBlockMN_RS); + static constexpr int kNWarps = std::get<2>(kBlockMN_kNWarps_Stages_RS); + static constexpr int kStages = + Arch >= 90 ? 2 : std::get<3>(kBlockMN_kNWarps_Stages_RS); + static constexpr bool Q_in_regs = + Arch >= 90 ? false : std::get<4>(kBlockMN_kNWarps_Stages_RS); + +#ifdef HSTU_FLASH_ATTN_DEBUG_INFO + std::printf( + "kBlockM: (%d), kBlockN: (%d), Mma1_is_RS: (%d), kNWarps: (%d), kStages: (%d), Q_in_regs: (%d)\n", + kBlockM, + kBlockN, + Mma1_is_RS, + kNWarps, + kStages, + Q_in_regs); +#endif + + using TileShape_MNK = cute::Shape, Int, Int>; + using ClusterShape = cute::Shape, _1, _1>; + using CollectiveMainloop = hstu::CollectiveMainloopFwdSm90< + kStages, + ClusterShape, + TileShape_MNK, + Element, + float, + cutlass::arch::Sm90, + Causal, + Local, + Contexual_mask, + Jagged, + Has_targets, + Mma1_is_RS, + V_colmajor, + Cross>; + using CollectiveEpilogue = hstu::CollectiveEpilogueFwd< + TileShape_MNK, + ClusterShape, + ElementOut, + ArchTag, + CollectiveMainloop::NumMmaThreads, + Jagged, + FP8_TransposeV>; + + static constexpr int NumProducerThreads = Arch >= 90 + ? CollectiveMainloop::NumProducerThreads + : CollectiveMainloop::NumMmaThreads; + using SchedulerPersistent = std::conditional_t< + Jagged, + hstu::VarlenDynamicPersistentTileScheduler< + kBlockM, + CollectiveMainloop::NumMmaThreads, + NumProducerThreads, + Arch >= 90 /*WarpSpecialized*/>, + std::conditional_t< + !Causal && !Local, + hstu::StaticPersistentTileScheduler, + hstu::DynamicPersistentTileScheduler< + CollectiveMainloop::NumMmaThreads, + NumProducerThreads, + Arch >= 90 /*WarpSpecialized*/>>>; + using SchedulerSingleTile = hstu:: + SingleTileScheduler; + // If Split then we probably don't have enough work for PersistentScheduler to + // be useful. However, if Jagged (e.g., during decode where we have + // max_seqlens), using PersistentScheduler is better since we'll avoid + // launching a bunch of thread blocks that immediately exit. On Sm80, + // noncausal persistent seems a bit slower. + using Scheduler = std::conditional_t< + Arch >= 90 ? false : !(Causal && !Jagged), + SchedulerSingleTile, + SchedulerPersistent>; + using AttnKernel = hstu::enable_sm90_or_later>; + + int seqlen_q = !Jagged ? params.max_q_len : params.total_seq_len_q; + int seqlen_kv = !Jagged ? params.max_kv_len : params.total_seq_len_kv; + int batch = !Jagged ? params.b : 1; +#ifdef HSTU_FLASH_ATTN_DEBUG_INFO + std::printf("max/total seqlen: (%d), batch: (%d)\n", seqlen, batch); +#endif + typename CollectiveMainloop::StrideV v_strides = + cute::conditional_return( + make_stride( + params.v_row_stride, + _1{}, + params.v_head_stride, + !Jagged ? params.v_batch_stride : 0), + make_stride( + _1{}, + params.v_dim_stride, + params.v_head_stride, + !Jagged ? params.v_batch_stride : 0)); + typename CollectiveMainloop::Arguments mainloop_args{ + static_cast(params.q_ptr), + {seqlen_q, params.qk_d, params.h, batch}, // shape_Q + {params.q_row_stride, + _1{}, + params.q_head_stride, + !Jagged ? params.q_batch_stride : 0}, // stride_Q + static_cast(params.k_ptr), + {seqlen_kv, params.qk_d, params.h, batch}, // shape_K + {params.k_row_stride, + _1{}, + params.k_head_stride, + !Jagged ? params.k_batch_stride : 0}, // stride_K + static_cast(params.v_ptr), + v_strides, // stride_V + params.q_descale_ptr, + params.k_descale_ptr, + params.v_descale_ptr, + {params.q_descale_batch_stride, params.q_descale_head_stride}, + {params.k_descale_batch_stride, params.k_descale_head_stride}, + {params.v_descale_batch_stride, params.v_descale_head_stride}, + 1.0f / params.max_kv_len, + params.alpha, + params.max_attn_len, + params.min_full_attn_seq_len, + params.contextual_seq_len, + params.num_softmax_heads, + params.num_groups, + params.batch_size_per_group, + params.seq_offsets, + params.seq_offsets_q, + params.num_targets, + params.max_seq_len_tensor, + params.contextual_seq_len_tensor, + params.max_attn_len_tensor, + params.min_full_attn_seq_len_tensor, + params.attn_scale, + params.scalar_scale, + }; + typename CollectiveEpilogue::Arguments epilogue_args{ + static_cast(params.o_ptr), + {seqlen_q, params.v_d, params.h, batch, 1}, // shape_O + {params.o_row_stride, + _1{}, + params.o_head_stride, + !Jagged ? params.o_batch_stride : 0, + 0}, // stride_O + params.h, + params.num_softmax_heads, + {_1{}, seqlen_q, !Jagged ? params.h * seqlen_q : 0, 0}, // stride_LSE} + static_cast(params.softmax_lse), + Cross ? params.seq_offsets_q : params.seq_offsets}; + + int num_blocks_m = + cutlass::ceil_div(params.max_q_len, get<0>(TileShape_MNK{})); + num_blocks_m = cutlass::round_up(num_blocks_m, size<0>(ClusterShape{})); + typename hstu::TileSchedulerArguments scheduler_args{ + num_blocks_m, + params.h, + params.b, + params.max_q_len, + params.qk_d, + sizeof(Element), + params.tile_count_semaphore, + Cross ? params.seq_offsets_q : params.seq_offsets, + nullptr /*sort_by_length_indices*/}; + + int device; + CHECK_CUDA(cudaGetDevice(&device)); + typename AttnKernel::Params kernel_params = + AttnKernel::to_underlying_arguments( + {mainloop_args, + epilogue_args, + {device, params.num_sm}, + scheduler_args}); + + dim3 grid_dims = AttnKernel::get_grid_shape(kernel_params); + dim3 block_dims = AttnKernel::get_block_shape(); + int smem_size = AttnKernel::SharedStorageSize; + // int smem_size_q = sizeof(decltype((typename + // CollectiveMainloop::TensorStorage{}).smem_q)); int smem_size_k = + // sizeof(decltype((typename CollectiveMainloop::TensorStorage{}).smem_k)); + // int smem_size_v = sizeof(decltype((typename + // CollectiveMainloop::TensorStorage{}).smem_v)); printf("smem_size = %d, q = + // %d, k = %d, v = %d\n", smem_size, smem_size_q, smem_size_k, smem_size_v); + // Get the ptr to kernel function. + if constexpr (size(ClusterShape{}) > 1) { + void const* kernel = (void const*)cutlass::device_kernel; + if (smem_size >= 48 * 1024) { + CHECK_CUDA(cudaFuncSetAttribute( + kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size)); + } + dim3 cluster_dims( + size<0>(ClusterShape{}), + size<1>(ClusterShape{}), + size<2>(ClusterShape{})); + cutlass::ClusterLaunchParams launch_params{ + grid_dims, block_dims, cluster_dims, smem_size, stream}; + cutlass::launch_kernel_on_cluster(launch_params, kernel, kernel_params); + } else { +#ifdef HSTU_FLASH_ATTN_DEBUG_INFO + std::cout << "ClusterShape = 1" << std::endl; + std::cout << "grid_dims = " << grid_dims << std::endl; + std::cout << "block_dims = " << block_dims << std::endl; + std::cout << "smem_size = " << smem_size << std::endl; +#endif + auto kernel = cutlass::device_kernel; + if (smem_size >= 48 * 1024) { + CHECK_CUDA(cudaFuncSetAttribute( + kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size)); + } + kernel<<>>(kernel_params); + } + CHECK_CUDA_KERNEL_LAUNCH(); +} + +template < + int Arch, + int kHeadDim, + bool Causal, + bool Local, + bool Softmax, + typename T, + typename T_out> +void run_mha_fwd_dispatch(hstu::Flash_fwd_params& params, cudaStream_t stream) { + static constexpr bool V_colmajor = false; // V_colmajor_ && sizeof(T) == 1; + BOOL_SWITCH(params.num_targets, Has_targets, [&] { + BOOL_SWITCH(params.seq_offsets, Jagged, [&] { + BOOL_SWITCH(params.seq_offsets_q, Cross, [&] { + BOOL_SWITCH(params.has_contexual_mask, Contexual_mask, [&] { + BOOL_SWITCH(params.training, Training, [&] { +#ifdef HSTU_FLASH_ATTN_DEBUG_INFO + std::printf( + "[flash_fwd_launch_template] Local: (%d), Jagged: (%d), Has_targets: (%d), Causal: (%d), max_kv_len: (%d), kHeadDim: (%d)\n", + Local, + Jagged, + Has_targets, + Causal, + params.max_kv_len, + kHeadDim); +#endif + // static constexpr bool Enable_cluster = Arch >= 90 && + // (sizeof(T) == 2 ? (kHeadDim >= 128) : (kHeadDim == 192)) && + // !Causal && !Local && !Jagged; + // static constexpr bool Enable_cluster = false; + // CLUSTER_SWITCH( + // cutlass::ceil_div(params.max_q_len, kBlockM) % 2 == 0, + // Use_cluster, + // [&] { + // static constexpr int ClusterM = + // Enable_cluster && Use_cluster ? 2 : 1; + run_flash_fwd< + Arch, + kHeadDim, + 1, // ClusterM, + T, + T_out, + Causal, + Local, + Contexual_mask, + Jagged, + Has_targets, + V_colmajor, + Cross, + Softmax, + Training>(params, stream); + }); + }); + }); + }); + }); +} + +template +void run_mha_fwd_(hstu::Flash_fwd_params& params, cudaStream_t stream) { + static_assert( + sizeof(T) == 2 || sizeof(T) == 1, "Only 16bit and 8bit are supported"); + static constexpr bool Is_FP8 = cute::is_same_v || + cute::is_same_v; + using T_out = std::conditional_t; + CAUSAL_LOCAL_SWITCH(params.is_causal, params.is_local, Causal, Local, [&] { + // VCOLMAJOR_SWITCH(params.v_dim_stride != 1, V_colmajor_, [&] { + run_mha_fwd_dispatch( + params, stream); + }); +} +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/generate_kernels.py b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/generate_kernels.py new file mode 100644 index 000000000..6c3a03188 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/generate_kernels.py @@ -0,0 +1,236 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + + +# Copied from Driss Guessous's PR in PyTorch: https://github.com/pytorch/pytorch/pull/105602 + +# This file is run to generate the kernel instantiations for the flash_attn kernels +# They are written to several files in order to speed up compilation + +import argparse +import itertools +from dataclasses import dataclass +from pathlib import Path +from typing import List, Optional, Union + + +DTYPE_MAP = { + "fp16": "cutlass::half_t", + "bf16": "cutlass::bfloat16_t", + "e4m3": "cutlass::float_e4m3_t", +} + +DTYPE_MAP_FWD_SM8x = { + "fp16": "cutlass::half_t", + "bf16": "cutlass::bfloat16_t", +} + +DTYPE_MAP_BWD = { + "fp16": "cutlass::half_t", + "bf16": "cutlass::bfloat16_t", +} + +SM = [90] # Sm kernels support up to +SOFTMAX = ["true", "false"] +HEAD_DIMENSIONS = [64, 96, 128, 192, 256] + +KERNEL_IMPL_TEMPLATE_FWD_SM90 = """ +#ifdef OSS_ENV +#include "hstu_attention/flash_fwd_launch_template.h" +#else +#include "flash_fwd_launch_template.h" +#endif + +namespace hstu {{ +#ifndef FLASHATTENTION_DISABLE_HDIM{HEAD_DIM} +template void run_mha_fwd_<{ARCH}, {DTYPE}, {HEAD_DIM}, {SOFTMAX}>(Flash_fwd_params ¶ms, cudaStream_t stream); +#endif +}} // namespace hstu +""" + +KERNEL_IMPL_TEMPLATE_FWD_SM8x = """ +#ifdef OSS_ENV +#include "hstu_attention/flash_fwd_launch_template.h" +#else +#include "flash_fwd_launch_template.h" +#endif + +namespace hstu {{ +#ifndef FLASHATTENTION_DISABLE_SM8x +#ifndef FLASHATTENTION_DISABLE_HDIM{HEAD_DIM} +template void run_mha_fwd_<80, {DTYPE}, {HEAD_DIM}, {SOFTMAX}>(Flash_fwd_params ¶ms, cudaStream_t stream); +#endif +#endif +}} // namespace hstu +""" + +KERNEL_IMPL_TEMPLATE_BWD_SM90 = """ +#ifdef OSS_ENV +#include "hstu_attention/flash_bwd_launch_template.h" +#else +#include "flash_bwd_launch_template.h" +#endif + +namespace hstu {{ +#ifndef FLASHATTENTION_DISABLE_HDIM{HEAD_DIM} +template void run_mha_bwd_<{ARCH}, {DTYPE}, {HEAD_DIM}, {SOFTMAX}>(Flash_bwd_params ¶ms, cudaStream_t stream); +#endif +}} // namespace hstu +""" + +KERNEL_IMPL_TEMPLATE_BWD_SM8x = """ +#ifdef OSS_ENV +#include "hstu_attention/flash_bwd_launch_template.h" +#else +#include "flash_bwd_launch_template.h" +#endif + +namespace hstu {{ +#ifndef FLASHATTENTION_DISABLE_SM8x +#ifndef FLASHATTENTION_DISABLE_HDIM{HEAD_DIM} +template void run_mha_bwd_<80, {DTYPE}, {HEAD_DIM}, {SOFTMAX}>(Flash_bwd_params ¶ms, cudaStream_t stream); +#endif +#endif +}} // namespace hstu +""" + + +@dataclass +class Kernel: + sm: int + dtype: str + head_dim: int + softmax: str + direction: str + + @property + def template(self) -> str: + if self.direction == "fwd": + if self.sm == 90: + return KERNEL_IMPL_TEMPLATE_FWD_SM90.format( + ARCH=str(self.sm), + DTYPE=DTYPE_MAP[self.dtype], + HEAD_DIM=self.head_dim, + SOFTMAX=self.softmax, + ) + else: + # Always enable PackGQA for Sm8x to reduce compilation + return KERNEL_IMPL_TEMPLATE_FWD_SM8x.format( + DTYPE=DTYPE_MAP[self.dtype], + HEAD_DIM=self.head_dim, + SOFTMAX=self.softmax, + ) + else: + assert self.direction == "bwd" + if self.sm == 90: + return KERNEL_IMPL_TEMPLATE_BWD_SM90.format( + ARCH=str(self.sm), + DTYPE=DTYPE_MAP[self.dtype], + HEAD_DIM=self.head_dim, + SOFTMAX=self.softmax, + ) + else: + return KERNEL_IMPL_TEMPLATE_BWD_SM8x.format( + DTYPE=DTYPE_MAP[self.dtype], + HEAD_DIM=self.head_dim, + SOFTMAX=self.softmax, + ) + + @property + def filename(self) -> str: + return f"flash_{self.direction}_hdim{self.head_dim}_{self.dtype}_softmax{self.softmax}_sm{self.sm}.cu" + + +def get_all_kernels() -> List[Kernel]: + kernels: List[Kernel] = [] + for dtype, head_dim, sm, softmax in itertools.product( + DTYPE_MAP.keys(), HEAD_DIMENSIONS, SM, SOFTMAX + ): + # We always enable PackGQA for Sm8x or Split + # so we should just pass in packgqa=False to avoid the `_packgqa` in the filename. + if sm >= 90 or dtype in DTYPE_MAP_FWD_SM8x: + kernels.append( + Kernel( + sm=sm, + dtype=dtype, + head_dim=head_dim, + direction="fwd", + softmax=softmax, + ) + ) + for dtype, head_dim, sm, softmax in itertools.product( + DTYPE_MAP_BWD.keys(), HEAD_DIMENSIONS, SM, SOFTMAX + ): + kernels.append( + Kernel( + sm=sm, + dtype=dtype, + head_dim=head_dim, + direction="bwd", + softmax=softmax, + ) + ) + return kernels + + +def write_kernel(kernel: Union[Kernel], autogen_dir: Path) -> None: + prelude = """ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ \n +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, Pradeep Ramani, Tri Dao. +// Splitting the different template instantiations to different files to speed up compilation. +// This file is auto-generated. See "generate_kernels.py"\n +""" + (autogen_dir / kernel.filename).write_text(prelude + kernel.template) + + +def main(output_dir_name: Optional[str]) -> None: + output_dir = ( + Path(output_dir_name) if output_dir_name is not None else Path(__file__).parent + ) + output_dir.mkdir(parents=True, exist_ok=True) + kernels_all = list(get_all_kernels()) + for kernel in kernels_all: + write_kernel(kernel, output_dir) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + prog="generate_kernels", + description="Generate the flash_attention kernels template instantiations", + ) + # Set an optional output directory + parser.add_argument( + "-o", + "--output_dir", + default="instantiations", + required=False, + help="Where to generate the kernels will default to the current directory ", + ) + args = parser.parse_args() + main(args.output_dir) diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim128_bf16_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim128_bf16_softmaxfalse_sm90.cu new file mode 100644 index 000000000..da0eeb2df --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim128_bf16_softmaxfalse_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_bwd_launch_template.h" +#else +#include "flash_bwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM128 +template void run_mha_bwd_<90, cutlass::bfloat16_t, 128, false>( + Flash_bwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim128_bf16_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim128_bf16_softmaxtrue_sm90.cu new file mode 100644 index 000000000..8d85c2235 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim128_bf16_softmaxtrue_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_bwd_launch_template.h" +#else +#include "flash_bwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM128 +template void run_mha_bwd_<90, cutlass::bfloat16_t, 128, true>( + Flash_bwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim128_fp16_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim128_fp16_softmaxfalse_sm90.cu new file mode 100644 index 000000000..09226cd80 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim128_fp16_softmaxfalse_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_bwd_launch_template.h" +#else +#include "flash_bwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM128 +template void run_mha_bwd_<90, cutlass::half_t, 128, false>( + Flash_bwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim128_fp16_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim128_fp16_softmaxtrue_sm90.cu new file mode 100644 index 000000000..63e451d14 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim128_fp16_softmaxtrue_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_bwd_launch_template.h" +#else +#include "flash_bwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM128 +template void run_mha_bwd_<90, cutlass::half_t, 128, true>( + Flash_bwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim192_bf16_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim192_bf16_softmaxfalse_sm90.cu new file mode 100644 index 000000000..e379d9918 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim192_bf16_softmaxfalse_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_bwd_launch_template.h" +#else +#include "flash_bwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM192 +template void run_mha_bwd_<90, cutlass::bfloat16_t, 192, false>( + Flash_bwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim192_bf16_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim192_bf16_softmaxtrue_sm90.cu new file mode 100644 index 000000000..7faa31376 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim192_bf16_softmaxtrue_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_bwd_launch_template.h" +#else +#include "flash_bwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM192 +template void run_mha_bwd_<90, cutlass::bfloat16_t, 192, true>( + Flash_bwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim192_fp16_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim192_fp16_softmaxfalse_sm90.cu new file mode 100644 index 000000000..5ddc7d7fc --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim192_fp16_softmaxfalse_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_bwd_launch_template.h" +#else +#include "flash_bwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM192 +template void run_mha_bwd_<90, cutlass::half_t, 192, false>( + Flash_bwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim192_fp16_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim192_fp16_softmaxtrue_sm90.cu new file mode 100644 index 000000000..530deae2b --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim192_fp16_softmaxtrue_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_bwd_launch_template.h" +#else +#include "flash_bwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM192 +template void run_mha_bwd_<90, cutlass::half_t, 192, true>( + Flash_bwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim256_bf16_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim256_bf16_softmaxfalse_sm90.cu new file mode 100644 index 000000000..185907c5e --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim256_bf16_softmaxfalse_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_bwd_launch_template.h" +#else +#include "flash_bwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM256 +template void run_mha_bwd_<90, cutlass::bfloat16_t, 256, false>( + Flash_bwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim256_bf16_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim256_bf16_softmaxtrue_sm90.cu new file mode 100644 index 000000000..39df173bb --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim256_bf16_softmaxtrue_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_bwd_launch_template.h" +#else +#include "flash_bwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM256 +template void run_mha_bwd_<90, cutlass::bfloat16_t, 256, true>( + Flash_bwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim256_fp16_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim256_fp16_softmaxfalse_sm90.cu new file mode 100644 index 000000000..cdc0a9f7e --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim256_fp16_softmaxfalse_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_bwd_launch_template.h" +#else +#include "flash_bwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM256 +template void run_mha_bwd_<90, cutlass::half_t, 256, false>( + Flash_bwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim256_fp16_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim256_fp16_softmaxtrue_sm90.cu new file mode 100644 index 000000000..6f3182d34 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim256_fp16_softmaxtrue_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_bwd_launch_template.h" +#else +#include "flash_bwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM256 +template void run_mha_bwd_<90, cutlass::half_t, 256, true>( + Flash_bwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim64_bf16_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim64_bf16_softmaxfalse_sm90.cu new file mode 100644 index 000000000..89285d0be --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim64_bf16_softmaxfalse_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_bwd_launch_template.h" +#else +#include "flash_bwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM64 +template void run_mha_bwd_<90, cutlass::bfloat16_t, 64, false>( + Flash_bwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim64_bf16_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim64_bf16_softmaxtrue_sm90.cu new file mode 100644 index 000000000..ab39c7e06 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim64_bf16_softmaxtrue_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_bwd_launch_template.h" +#else +#include "flash_bwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM64 +template void run_mha_bwd_<90, cutlass::bfloat16_t, 64, true>( + Flash_bwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim64_fp16_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim64_fp16_softmaxfalse_sm90.cu new file mode 100644 index 000000000..8d62b8827 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim64_fp16_softmaxfalse_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_bwd_launch_template.h" +#else +#include "flash_bwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM64 +template void run_mha_bwd_<90, cutlass::half_t, 64, false>( + Flash_bwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim64_fp16_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim64_fp16_softmaxtrue_sm90.cu new file mode 100644 index 000000000..5192d945f --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim64_fp16_softmaxtrue_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_bwd_launch_template.h" +#else +#include "flash_bwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM64 +template void run_mha_bwd_<90, cutlass::half_t, 64, true>( + Flash_bwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim96_bf16_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim96_bf16_softmaxfalse_sm90.cu new file mode 100644 index 000000000..cbeeac64a --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim96_bf16_softmaxfalse_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_bwd_launch_template.h" +#else +#include "flash_bwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM96 +template void run_mha_bwd_<90, cutlass::bfloat16_t, 96, false>( + Flash_bwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim96_bf16_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim96_bf16_softmaxtrue_sm90.cu new file mode 100644 index 000000000..b654969e4 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim96_bf16_softmaxtrue_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_bwd_launch_template.h" +#else +#include "flash_bwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM96 +template void run_mha_bwd_<90, cutlass::bfloat16_t, 96, true>( + Flash_bwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim96_fp16_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim96_fp16_softmaxfalse_sm90.cu new file mode 100644 index 000000000..ea81f7ee4 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim96_fp16_softmaxfalse_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_bwd_launch_template.h" +#else +#include "flash_bwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM96 +template void run_mha_bwd_<90, cutlass::half_t, 96, false>( + Flash_bwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim96_fp16_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim96_fp16_softmaxtrue_sm90.cu new file mode 100644 index 000000000..7439f322e --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim96_fp16_softmaxtrue_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_bwd_launch_template.h" +#else +#include "flash_bwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM96 +template void run_mha_bwd_<90, cutlass::half_t, 96, true>( + Flash_bwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_bf16_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_bf16_softmaxfalse_sm90.cu new file mode 100644 index 000000000..a39bcd505 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_bf16_softmaxfalse_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_fwd_launch_template.h" +#else +#include "flash_fwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM128 +template void run_mha_fwd_<90, cutlass::bfloat16_t, 128, false>( + Flash_fwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_bf16_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_bf16_softmaxtrue_sm90.cu new file mode 100644 index 000000000..464a0f443 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_bf16_softmaxtrue_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_fwd_launch_template.h" +#else +#include "flash_fwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM128 +template void run_mha_fwd_<90, cutlass::bfloat16_t, 128, true>( + Flash_fwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_e4m3_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_e4m3_softmaxfalse_sm90.cu new file mode 100644 index 000000000..3075657bb --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_e4m3_softmaxfalse_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_fwd_launch_template.h" +#else +#include "flash_fwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM128 +template void run_mha_fwd_<90, cutlass::float_e4m3_t, 128, false>( + Flash_fwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_e4m3_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_e4m3_softmaxtrue_sm90.cu new file mode 100644 index 000000000..1ab6e4394 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_e4m3_softmaxtrue_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_fwd_launch_template.h" +#else +#include "flash_fwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM128 +template void run_mha_fwd_<90, cutlass::float_e4m3_t, 128, true>( + Flash_fwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_fp16_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_fp16_softmaxfalse_sm90.cu new file mode 100644 index 000000000..be5a6cb0d --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_fp16_softmaxfalse_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_fwd_launch_template.h" +#else +#include "flash_fwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM128 +template void run_mha_fwd_<90, cutlass::half_t, 128, false>( + Flash_fwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_fp16_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_fp16_softmaxtrue_sm90.cu new file mode 100644 index 000000000..7c303e7ef --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_fp16_softmaxtrue_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_fwd_launch_template.h" +#else +#include "flash_fwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM128 +template void run_mha_fwd_<90, cutlass::half_t, 128, true>( + Flash_fwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_bf16_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_bf16_softmaxfalse_sm90.cu new file mode 100644 index 000000000..6e8d906d5 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_bf16_softmaxfalse_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_fwd_launch_template.h" +#else +#include "flash_fwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM192 +template void run_mha_fwd_<90, cutlass::bfloat16_t, 192, false>( + Flash_fwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_bf16_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_bf16_softmaxtrue_sm90.cu new file mode 100644 index 000000000..80367708f --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_bf16_softmaxtrue_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_fwd_launch_template.h" +#else +#include "flash_fwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM192 +template void run_mha_fwd_<90, cutlass::bfloat16_t, 192, true>( + Flash_fwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_e4m3_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_e4m3_softmaxfalse_sm90.cu new file mode 100644 index 000000000..67ade004b --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_e4m3_softmaxfalse_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_fwd_launch_template.h" +#else +#include "flash_fwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM192 +template void run_mha_fwd_<90, cutlass::float_e4m3_t, 192, false>( + Flash_fwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_e4m3_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_e4m3_softmaxtrue_sm90.cu new file mode 100644 index 000000000..9f40d2726 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_e4m3_softmaxtrue_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_fwd_launch_template.h" +#else +#include "flash_fwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM192 +template void run_mha_fwd_<90, cutlass::float_e4m3_t, 192, true>( + Flash_fwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_fp16_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_fp16_softmaxfalse_sm90.cu new file mode 100644 index 000000000..1779657c0 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_fp16_softmaxfalse_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_fwd_launch_template.h" +#else +#include "flash_fwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM192 +template void run_mha_fwd_<90, cutlass::half_t, 192, false>( + Flash_fwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_fp16_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_fp16_softmaxtrue_sm90.cu new file mode 100644 index 000000000..0037dbc17 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_fp16_softmaxtrue_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_fwd_launch_template.h" +#else +#include "flash_fwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM192 +template void run_mha_fwd_<90, cutlass::half_t, 192, true>( + Flash_fwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_bf16_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_bf16_softmaxfalse_sm90.cu new file mode 100644 index 000000000..93440571c --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_bf16_softmaxfalse_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_fwd_launch_template.h" +#else +#include "flash_fwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM256 +template void run_mha_fwd_<90, cutlass::bfloat16_t, 256, false>( + Flash_fwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_bf16_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_bf16_softmaxtrue_sm90.cu new file mode 100644 index 000000000..c0634db8f --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_bf16_softmaxtrue_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_fwd_launch_template.h" +#else +#include "flash_fwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM256 +template void run_mha_fwd_<90, cutlass::bfloat16_t, 256, true>( + Flash_fwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_e4m3_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_e4m3_softmaxfalse_sm90.cu new file mode 100644 index 000000000..a0eb625f5 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_e4m3_softmaxfalse_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_fwd_launch_template.h" +#else +#include "flash_fwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM256 +template void run_mha_fwd_<90, cutlass::float_e4m3_t, 256, false>( + Flash_fwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_e4m3_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_e4m3_softmaxtrue_sm90.cu new file mode 100644 index 000000000..8b7216302 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_e4m3_softmaxtrue_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_fwd_launch_template.h" +#else +#include "flash_fwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM256 +template void run_mha_fwd_<90, cutlass::float_e4m3_t, 256, true>( + Flash_fwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_fp16_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_fp16_softmaxfalse_sm90.cu new file mode 100644 index 000000000..fe89b532f --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_fp16_softmaxfalse_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_fwd_launch_template.h" +#else +#include "flash_fwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM256 +template void run_mha_fwd_<90, cutlass::half_t, 256, false>( + Flash_fwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_fp16_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_fp16_softmaxtrue_sm90.cu new file mode 100644 index 000000000..c0857f941 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_fp16_softmaxtrue_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_fwd_launch_template.h" +#else +#include "flash_fwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM256 +template void run_mha_fwd_<90, cutlass::half_t, 256, true>( + Flash_fwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_bf16_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_bf16_softmaxfalse_sm90.cu new file mode 100644 index 000000000..841e9359e --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_bf16_softmaxfalse_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_fwd_launch_template.h" +#else +#include "flash_fwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM64 +template void run_mha_fwd_<90, cutlass::bfloat16_t, 64, false>( + Flash_fwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_bf16_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_bf16_softmaxtrue_sm90.cu new file mode 100644 index 000000000..3da54d69f --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_bf16_softmaxtrue_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_fwd_launch_template.h" +#else +#include "flash_fwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM64 +template void run_mha_fwd_<90, cutlass::bfloat16_t, 64, true>( + Flash_fwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_e4m3_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_e4m3_softmaxfalse_sm90.cu new file mode 100644 index 000000000..4761ca635 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_e4m3_softmaxfalse_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_fwd_launch_template.h" +#else +#include "flash_fwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM64 +template void run_mha_fwd_<90, cutlass::float_e4m3_t, 64, false>( + Flash_fwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_e4m3_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_e4m3_softmaxtrue_sm90.cu new file mode 100644 index 000000000..33e66d0a7 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_e4m3_softmaxtrue_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_fwd_launch_template.h" +#else +#include "flash_fwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM64 +template void run_mha_fwd_<90, cutlass::float_e4m3_t, 64, true>( + Flash_fwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_fp16_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_fp16_softmaxfalse_sm90.cu new file mode 100644 index 000000000..fab2951ee --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_fp16_softmaxfalse_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_fwd_launch_template.h" +#else +#include "flash_fwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM64 +template void run_mha_fwd_<90, cutlass::half_t, 64, false>( + Flash_fwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_fp16_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_fp16_softmaxtrue_sm90.cu new file mode 100644 index 000000000..2ef1f29c9 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_fp16_softmaxtrue_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_fwd_launch_template.h" +#else +#include "flash_fwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM64 +template void run_mha_fwd_<90, cutlass::half_t, 64, true>( + Flash_fwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_bf16_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_bf16_softmaxfalse_sm90.cu new file mode 100644 index 000000000..bc52514e9 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_bf16_softmaxfalse_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_fwd_launch_template.h" +#else +#include "flash_fwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM96 +template void run_mha_fwd_<90, cutlass::bfloat16_t, 96, false>( + Flash_fwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_bf16_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_bf16_softmaxtrue_sm90.cu new file mode 100644 index 000000000..11ea3bb20 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_bf16_softmaxtrue_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_fwd_launch_template.h" +#else +#include "flash_fwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM96 +template void run_mha_fwd_<90, cutlass::bfloat16_t, 96, true>( + Flash_fwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_e4m3_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_e4m3_softmaxfalse_sm90.cu new file mode 100644 index 000000000..9e0b05a31 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_e4m3_softmaxfalse_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_fwd_launch_template.h" +#else +#include "flash_fwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM96 +template void run_mha_fwd_<90, cutlass::float_e4m3_t, 96, false>( + Flash_fwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_e4m3_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_e4m3_softmaxtrue_sm90.cu new file mode 100644 index 000000000..7fa79aa76 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_e4m3_softmaxtrue_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_fwd_launch_template.h" +#else +#include "flash_fwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM96 +template void run_mha_fwd_<90, cutlass::float_e4m3_t, 96, true>( + Flash_fwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_fp16_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_fp16_softmaxfalse_sm90.cu new file mode 100644 index 000000000..83a25a649 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_fp16_softmaxfalse_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_fwd_launch_template.h" +#else +#include "flash_fwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM96 +template void run_mha_fwd_<90, cutlass::half_t, 96, false>( + Flash_fwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_fp16_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_fp16_softmaxtrue_sm90.cu new file mode 100644 index 000000000..e0526dec8 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_fp16_softmaxtrue_sm90.cu @@ -0,0 +1,33 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, +// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to +// different files to speed up compilation. This file is auto-generated. See +// "generate_kernels.py" + +#ifdef OSS_ENV +#include "hstu_attention/flash_fwd_launch_template.h" +#else +#include "flash_fwd_launch_template.h" +#endif + +namespace hstu { +#ifndef FLASHATTENTION_DISABLE_HDIM96 +template void run_mha_fwd_<90, cutlass::half_t, 96, true>( + Flash_fwd_params& params, + cudaStream_t stream); +#endif +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/mainloop_bwd_sm90_tma_gmma_ws.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/mainloop_bwd_sm90_tma_gmma_ws.h new file mode 100644 index 000000000..e702faf0b --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/mainloop_bwd_sm90_tma_gmma_ws.h @@ -0,0 +1,3166 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/****************************************************************************** + * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, + *Pradeep Ramani, Tri Dao. + ******************************************************************************/ + +#pragma once + +#include +#include +#include +#include +#include +#include "cutlass/pipeline/pipeline.hpp" + +#include "cute/tensor.hpp" + +#include "cutlass/gemm/collective/builders/sm90_common.inl" + +#include "copy_sm90_bulk_reduce.h" +#include "mask.h" +#include "named_barrier.h" +#include "seqlen.h" +#include "utils.h" + +namespace hstu { + +using namespace cute; + +template < + int Stages, + int Stages_dO, + int Stages_dS, + class ClusterShape_, + class TileShape_MNK_, + class Element_, + class ElementAccum_, + class ArchTag_, + bool Causal, + bool Local, + bool Contexual_mask, + bool Jagged, + bool Has_targets, + bool Deterministic, + bool SdP_swapAB_, + bool dKV_swapAB_, + bool dQ_swapAB_, + int NumMmaWarpGroups = 2, + int AtomLayoutMSdP = 1, + int AtomLayoutNdKV = 2, + int AtomLayoutMdQ = 1, + bool Mma_dP_is_RS = false, + bool Cross = false, + bool Softmax = false> +struct CollectiveMainloopBwdSm90 { + static constexpr int kStages = Stages; + static constexpr int kStages_dO = Stages_dO; + static constexpr int kStages_dS = Stages_dS; + static_assert(kStages >= kStages_dO); + static_assert(Stages_dS == 1 || Stages_dS == kStages); + static_assert( + !Mma_dP_is_RS || SdP_swapAB_); // If Mma_dP_is_RS, we need SdP_SwapAB + using ClusterShape = ClusterShape_; + using TileShape_MNK = TileShape_MNK_; + using Element = Element_; + using ElementAccum = ElementAccum_; + using ArchTag = ArchTag_; + using SeqlenInfo_t = hstu::SeqlenInfoQKBwd< + Jagged, + Cross, + Has_targets, + CUTE_STATIC_V(get<0>(TileShape_MNK{}))>; + + static constexpr bool SdP_swapAB = SdP_swapAB_; + static constexpr bool dKV_swapAB = dKV_swapAB_; + static constexpr bool dQ_swapAB = dQ_swapAB_; + + static constexpr bool Q_dO_same_stages = kStages == kStages_dO; + + static constexpr int kBlockM = get<0>(TileShape_MNK{}); + static constexpr int kBlockN = get<1>(TileShape_MNK{}); + static constexpr int kHeadDim = get<2>(TileShape_MNK{}); + + static_assert(ArchTag::kMinComputeCapability >= 90); + static_assert(get<0>(ClusterShape{}) == 1 && get<2>(ClusterShape{}) == 1); + + static constexpr int NumMmaThreads = + NumMmaWarpGroups * cutlass::NumThreadsPerWarpGroup; + static constexpr int NumProducerThreads = cutlass::NumThreadsPerWarp * 2; + + static_assert(NumMmaWarpGroups % AtomLayoutMSdP == 0); + static_assert(NumMmaWarpGroups % AtomLayoutNdKV == 0); + static_assert(NumMmaWarpGroups % AtomLayoutMdQ == 0); + static constexpr bool Mma_dKV_is_RS = AtomLayoutMSdP == 1 && + AtomLayoutNdKV == NumMmaWarpGroups && SdP_swapAB && !dKV_swapAB; + static constexpr bool Mma_dQ_is_RS = AtomLayoutMSdP == NumMmaWarpGroups && + AtomLayoutMdQ == NumMmaWarpGroups && !SdP_swapAB && + !dQ_swapAB; // If dQ_swapAB we can't use RS + + static constexpr GMMA::Major PdS_Major = GMMA::Major::K; + // static constexpr GMMA::Major PdS_Major = GMMA::Major::MN; + static constexpr GMMA::Major PdSt_Major = + PdS_Major == GMMA::Major::K ? GMMA::Major::MN : GMMA::Major::K; + + using TileShapeAtomSdP = std::conditional_t< + !SdP_swapAB, + Shape< + Int, + Int, + Int>, + Shape, Int, Int>>; + using AtomLayoutSdP = std::conditional_t< + !SdP_swapAB, + Layout, + Int, + _1>>, + Layout, + Int, + _1>>>; + using TiledMmaSdP = decltype(cute::make_tiled_mma( + cute::GMMA:: + ss_op_selector(), + AtomLayoutSdP{})); + + using TiledMmadPRS = decltype(cute::make_tiled_mma( + cute::GMMA:: + rs_op_selector(), + AtomLayoutSdP{})); + + using TileShapeAtomdKV = std::conditional_t< + !dKV_swapAB, + Shape< + Int, + Int, + Int>, + Shape, Int, Int>>; + using AtomLayoutdKV = std::conditional_t< + !dKV_swapAB, + Layout, + Int, + _1>>, + Layout, + Int, + _1>>>; + using TiledMmadKV = decltype(cute::make_tiled_mma( + std::conditional_t< + Mma_dKV_is_RS, + decltype(cute::GMMA::rs_op_selector< + Element, + Element, + ElementAccum, + TileShapeAtomdKV, + GMMA::Major::K, + GMMA::Major::MN>()), + decltype(cute::GMMA::ss_op_selector< + Element, + Element, + ElementAccum, + TileShapeAtomdKV, + !dKV_swapAB ? PdSt_Major : GMMA::Major::MN, + !dKV_swapAB ? GMMA::Major::MN : PdSt_Major>())>{}, + AtomLayoutdKV{})); + + static constexpr bool dQacc_use_TMA = kHeadDim < 256; + // For hdim256, we want to slice the dQ MMA (64 x 256 on 2 WGs) into two (64 x + // 128 on 2 WGs) so that we can do atomic add on one half before doing the + // other half of the MMA, to reduce register pressure. + static constexpr bool Slice_dQKV_Mma = kHeadDim == 256 && !dQacc_use_TMA && + dQ_swapAB && AtomLayoutMdQ == 1 && NumMmaWarpGroups == 2; + static_assert( + !(Deterministic && Slice_dQKV_Mma), + "Deterministic mode not supported with Slice_dQKV_Mma"); + + static constexpr int TileShapeAtomdQ_BlockM = kBlockM / AtomLayoutMdQ; + static constexpr int TileShapeAtomdQ_HeadDim = + (Slice_dQKV_Mma ? kHeadDim / 2 : kHeadDim) / + (NumMmaWarpGroups / AtomLayoutMdQ); + static_assert( + !dQ_swapAB ? TileShapeAtomdQ_BlockM == 64 : TileShapeAtomdQ_HeadDim == 64, + "Tile_M must be 64."); + using TileShapeAtomdQ = std::conditional_t< + !dQ_swapAB, + Shape< + Int, + Int, + Int>, + Shape< + Int, + Int, + Int>>; + using AtomLayoutdQ = std::conditional_t< + !dQ_swapAB, + Layout< + Shape, Int, _1>>, + Layout, + Int, + _1>>>; + using TiledMmadQ = decltype(cute::make_tiled_mma( + std::conditional_t< + Mma_dQ_is_RS, + decltype(cute::GMMA::rs_op_selector< + Element, + Element, + ElementAccum, + TileShapeAtomdQ, + GMMA::Major::K, + GMMA::Major::MN>()), + decltype(cute::GMMA::ss_op_selector< + Element, + Element, + ElementAccum, + TileShapeAtomdQ, + !dQ_swapAB ? PdS_Major : GMMA::Major::MN, + !dQ_swapAB ? GMMA::Major::MN : PdS_Major>())>{}, + AtomLayoutdQ{})); + + // We need to accommodate both Q and Q^T (and dO and dO^T) in shared memory. + // Q & dO are used in the SdP Mma and Q^T and dO^T are used in the dKV Mma. + // Since this is GMMA::Major::K, the M dimension (kBlockM) doesn't matter for + // the layout, only the K dimension changes the layout. + using SmemLayoutAtomQdO = + decltype(cutlass::gemm::collective::detail::ss_smem_selector< + GMMA::Major::K, + Element, + Int, + Int>()); // for dKV_Mma + using SmemLayoutQ = decltype(tile_to_shape( + SmemLayoutAtomQdO{}, + make_shape( + shape<0>(TileShape_MNK{}), + shape<2>(TileShape_MNK{}), + Int{}))); + using SmemLayoutdO = decltype(tile_to_shape( + SmemLayoutAtomQdO{}, + make_shape( + shape<0>(TileShape_MNK{}), + shape<2>(TileShape_MNK{}), + Int{}))); + + using SmemLayoutAtomK = + decltype(cutlass::gemm::collective::detail::ss_smem_selector< + GMMA::Major::K, + Element, + Int, + Int>()); + using SmemLayoutK = + decltype(tile_to_shape(SmemLayoutAtomK{}, select<1, 2>(TileShape_MNK{}))); + + using SmemLayoutAtomV = + decltype(cutlass::gemm::collective::detail::ss_smem_selector< + GMMA::Major::K, + Element, + decltype(cute::get<1>(TileShape_MNK{})), + decltype(cute::get<2>(TileShape_MNK{}))>()); + using SmemLayoutV = + decltype(tile_to_shape(SmemLayoutAtomV{}, select<1, 2>(TileShape_MNK{}))); + + using SmemLayoutAtomPdS = + decltype(cutlass::gemm::collective::detail::ss_smem_selector< + PdS_Major, + Element, + Int, + Int>()); + using SmemLayoutPdS = decltype(tile_to_shape( + SmemLayoutAtomPdS{}, + make_shape(Int{}, Int{}, Int{}), + std::conditional_t< + PdS_Major == GMMA::Major::K, + cute::Step<_1, _2, _3>, + cute::Step<_2, _1, _3>>{})); + // Need stride to be multiple of 32, otherwise we get error (misaligned + // address) when doing TMA if e.g. kBlockM=80 We set stride to be multiple of + // 64 so that if ShuffleLSE, even if threads read from sLSE but out of bounds, + // it's still a valid smem address. + using SmemLayoutLSE = cute::Layout< + cute::Shape, Int>, + cute::Stride<_1, Int>>; + using SmemLayoutLSEMma = std::conditional_t< + SdP_swapAB, + cute::Layout< + cute::Shape, Int, Int>, + cute::Stride<_0, _1, Int>>, + cute::Layout< + cute::Shape, Int, Int>, + cute::Stride<_1, _0, Int>>>; + + // Note this is the transpose in terms of the view, not in terms of memory. + using SmemLayoutQt = decltype(cute::composition( + SmemLayoutQ{}, + make_layout( + make_shape( + get<2>(TileShape_MNK{}), + get<0>(TileShape_MNK{}), + Int{}), + make_stride(Int{}, _1{}, Int{})))); + using SmemLayoutdOt = decltype(cute::composition( + SmemLayoutdO{}, + make_layout( + make_shape( + get<2>(TileShape_MNK{}), + get<0>(TileShape_MNK{}), + Int{}), + make_stride(Int{}, _1{}, Int{})))); + using SmemLayoutKt = decltype(cute::composition( + SmemLayoutK{}, + make_layout( + make_shape(get<2>(TileShape_MNK{}), get<1>(TileShape_MNK{})), + make_stride(Int{}, _1{})))); + using SmemLayoutPdSt = decltype(cute::composition( + SmemLayoutPdS{}, + make_layout( + make_shape(Int{}, Int{}, Int{}), + make_stride(Int{}, _1{}, Int{})))); + + // Thread layout, 256 or 384 threads per row + // We split into NumMmaWarpGroups so that we can do Bulk reduce add for each + // WG separately. + using R2SLayoutAtomdQaccum = Layout< + Shape, Int>>; + using R2STiledCopydQaccum = decltype(make_tiled_copy( + Copy_Atom, ElementAccum>{}, + R2SLayoutAtomdQaccum{}, + Layout>{})); // Val layout, 4 vals per store + using SmemLayoutdQaccum = Layout< + Shape, Int>>; + + static constexpr int kNumPdSStore = kBlockM * kBlockN / NumMmaThreads; + // If !SdP_swapAB, the accum registers hold P / dS, otherwise they hold Pt / + // dSt. If PdS_major is MN, then we need to "transpose" the write. + using SmemCopyAtomPdS = Copy_Atom< + std::conditional_t< + (!SdP_swapAB) ^ (PdS_Major == GMMA::Major::MN), + std::conditional_t< + kNumPdSStore % 8 == 0, + cute::SM90_U32x4_STSM_N, + cute::SM90_U32x2_STSM_N>, + std::conditional_t< + kNumPdSStore % 8 == 0, + cute::SM90_U16x8_STSM_T, + cute::SM90_U16x4_STSM_T>>, + Element>; + + using GmemTiledCopyQdO = + decltype(cutlass::gemm::collective::detail:: + sm90_cluster_shape_to_tma_atom(shape<1>(ClusterShape{}))); + using GmemTiledCopyKV = cute::SM90_TMA_LOAD; + + using ShapeQKV = + cute::Shape; // (seqlen, d, head, + // batch) + using StrideQKV = cute::Stride; + using ShapeLSE = + cute::Shape; // (seqlen, head, batch) + using StrideLSE = cute::Stride<_1, int64_t, int64_t>; // (seqlen, head, batch) + using ShapedQaccum = + cute::Shape; // (seqlen * d, head, batch) + using StridedQaccum = cute::Stride<_1, int64_t, int64_t>; + + using TMA_QdO = decltype(make_tma_copy_A_sm90( + GmemTiledCopyQdO{}, + make_tensor( + make_gmem_ptr(static_cast(nullptr)), + ShapeQKV{}, + StrideQKV{}), + take<0, 2>(SmemLayoutQ{}), + TileShape_MNK{}, + ClusterShape{})); // mcast along N mode for this M load, if any + + using TMA_K = decltype(make_tma_copy_B_sm90( + GmemTiledCopyKV{}, + make_tensor( + make_gmem_ptr(static_cast(nullptr)), + ShapeQKV{}, + StrideQKV{}), + SmemLayoutK{}, + TileShape_MNK{}, + ClusterShape{})); // no mcast for KV + + using TMA_V = decltype(make_tma_copy_B_sm90( + GmemTiledCopyKV{}, + make_tensor( + make_gmem_ptr(static_cast(nullptr)), + ShapeQKV{}, + StrideQKV{}), + SmemLayoutV{}, + TileShape_MNK{}, + ClusterShape{})); // no mcast for KV + + using MainloopPipeline = typename cutlass::PipelineTmaAsync; + using PipelineState = typename MainloopPipeline::PipelineState; + using MainloopPipeline_dO = typename cutlass::PipelineTmaAsync; + using PipelineState_dO = typename MainloopPipeline_dO::PipelineState; + + // Set the bytes transferred in this TMA transaction (may involve multiple + // issues) + static constexpr uint32_t TmaTransactionBytesQ = static_cast( + size(take<0, 2>(SmemLayoutQ{})) * cutlass::sizeof_bits_v / 8); + static constexpr uint32_t TmaTransactionBytesK = static_cast( + size(SmemLayoutK{}) * cutlass::sizeof_bits_v / 8); + static constexpr uint32_t TmaTransactionBytesV = static_cast( + size(SmemLayoutV{}) * cutlass::sizeof_bits_v / 8); + static constexpr uint32_t TmaTransactionBytesLSE = static_cast( + size(select<0>(SmemLayoutLSE{})) * cutlass::sizeof_bits_v / + 8); + + // These are tuned for speed. They don't affect correctness. + // We have separate iterations with causal masking. Not necessary for hdim 128 + // but for hdim 64 this helps quite a bit to not have to do causal masking for + // most of the iterations. For hdim 192, separating masking iterations results + // in register spills. + static constexpr bool SeparateMaskingIterations = false; + // Do we keep the LSE and dPsum in each thread, or split them across 8 threads + // that share them and then shuffle to get the value whenever we need? This + // can reduce register pressure when SdP_swapAB, where each thread needs to + // keep statistics for (kBlockM / 4) rows. If !SdP_swapAB, each thread only + // needs to keep statistic for 2 rows. + static constexpr bool ShuffleLSE = SdP_swapAB && kHeadDim <= 64; + static constexpr bool ShuffledPsum = SdP_swapAB && kHeadDim <= 64; + static constexpr size_t SmemAlignmentP = + cutlass::detail::alignment_for_swizzle(SmemLayoutPdS{}); + static constexpr size_t SmemAlignmentdS = + cutlass::detail::alignment_for_swizzle(SmemLayoutPdS{}); + // Without this SmemAlignment, with hdim 256 we get "misaligned address" error + // in TMA + static constexpr size_t SmemAlignmentQKVdO = kHeadDim % 256 == 0 ? 256 : 128; + static constexpr size_t SmemAlignmentV = !Mma_dP_is_RS + ? SmemAlignmentQKVdO + : cutlass::detail::alignment_for_swizzle(SmemLayoutV{}); + static_assert( + SmemAlignmentP >= 128 && SmemAlignmentdS >= 128, + "Require at least 128B alignment"); + + // TODO: do we have to worry that smem_dk and smem_dv in the epilogue don't + // line up w smem_k and smem_v due to alignment? + using SmemdQacc_t = std::conditional_t< + !dQacc_use_TMA, + cute::array, + cute::array_aligned>>; + using SmemP_t = std::conditional_t< + Mma_dKV_is_RS, + cute::array, + cute::array_aligned< + Element, + cute::cosize_v, + SmemAlignmentP>>; + struct TensorStorage + : cute::aligned_struct< + cute::max(SmemAlignmentP, SmemAlignmentdS, SmemAlignmentQKVdO)> { + cute:: + array_aligned, SmemAlignmentQKVdO> + smem_k; + cute::array_aligned, SmemAlignmentV> + smem_v; + SmemdQacc_t smem_dqacc; + cute:: + array_aligned, SmemAlignmentQKVdO> + smem_q; + cute:: + array_aligned, SmemAlignmentQKVdO> + smem_do; + cute::array_aligned, 128> + smem_lse; + cute::array_aligned, 128> + smem_dpsum; + SmemP_t smem_p; + cute::array_aligned, SmemAlignmentdS> + smem_ds; + }; + + // Host side kernel arguments + struct Arguments { + Element const* const ptr_Q; + ShapeQKV const shape_Q; + StrideQKV const stride_Q; + Element const* const ptr_K; + ShapeQKV const shape_K; + StrideQKV const stride_K; + Element const* const ptr_V; + ShapeQKV const shape_V; + StrideQKV const stride_V; + Element const* const ptr_dO; + ShapeQKV const shape_dO; + StrideQKV const stride_dO; + ElementAccum* const ptr_dQaccum; + ShapedQaccum const shape_dQaccum; + StridedQaccum const stride_dQaccum; + float const* const ptr_LSE_log2; + ShapeLSE const shape_LSE; + StrideLSE const stride_LSE_log2; + float const* const ptr_dPsum; + StrideLSE const stride_dPsum; + int const max_attn_len; + int const min_full_attn_seq_len; + int const contextual_seq_len; + float const max_seq_len_inv; + float const alpha; + int const num_batch; + int const num_softmax_heads; + int const num_groups; + int const batch_size_per_group; + int* const dq_semaphore; + int const* const seq_offsets = nullptr; + int const* const seq_offsets_q = nullptr; + int const* const num_targets = nullptr; + int const* const max_seq_len_tensor = nullptr; + int const* const contextual_seq_len_tensor = nullptr; + int const* const max_attn_len_tensor = nullptr; + int const* const min_full_attn_seq_len_tensor = nullptr; + float const* const attn_scale = nullptr; + bool const scalar_scale = true; + }; + + // Device side kernel params + struct Params { + ShapeQKV const shape_Q; + ShapeQKV const shape_K; + ShapeQKV const shape_V; + ShapeQKV const shape_dO; + ElementAccum* const ptr_dQaccum; + ShapedQaccum const shape_dQaccum; + StridedQaccum stride_dQaccum; + TMA_QdO tma_load_Q, tma_load_dO; + TMA_K tma_load_K; + TMA_V tma_load_V; + float const* const ptr_LSE_log2; + ShapeLSE const shape_LSE; + StrideLSE const stride_LSE_log2; + float const* const ptr_dPsum; + StrideLSE const stride_dPsum; + int const max_attn_len; + int const min_full_attn_seq_len; + int const contextual_seq_len; + float const max_seq_len_inv; + float const alpha; + float const alpha_log2; + int const num_batch; + int const num_softmax_heads; + int const num_groups; + int const batch_size_per_group; + int* const dq_semaphore; + int const* const seq_offsets = nullptr; + int const* const seq_offsets_q = nullptr; + int const* const num_targets; + int const* const max_seq_len_tensor = nullptr; + int const* const contextual_seq_len_tensor = nullptr; + int const* const max_attn_len_tensor = nullptr; + int const* const min_full_attn_seq_len_tensor = nullptr; + float const* const attn_scale; + bool const scalar_scale = true; + }; + + static Params to_underlying_arguments(Arguments const& args) { + Tensor mQ = + make_tensor(make_gmem_ptr(args.ptr_Q), args.shape_Q, args.stride_Q); + TMA_QdO tma_load_Q = make_tma_copy_A_sm90( + GmemTiledCopyQdO{}, + mQ, + SmemLayoutQ{}(_, _, _0{}), + TileShape_MNK{}, + ClusterShape{}); // mcast along N mode for this M load, if any + Tensor mdO = + make_tensor(make_gmem_ptr(args.ptr_dO), args.shape_Q, args.stride_dO); + TMA_QdO tma_load_dO = make_tma_copy_A_sm90( + GmemTiledCopyQdO{}, + mdO, + SmemLayoutdO{}(_, _, _0{}), + TileShape_MNK{}, + ClusterShape{}); // mcast along N mode for this M load, if any + Tensor mK = + make_tensor(make_gmem_ptr(args.ptr_K), args.shape_K, args.stride_K); + TMA_K tma_load_K = make_tma_copy_B_sm90( + GmemTiledCopyKV{}, + mK, + SmemLayoutK{}, + TileShape_MNK{}, + ClusterShape{}); // no mcast for KV + Tensor mV = + make_tensor(make_gmem_ptr(args.ptr_V), args.shape_K, args.stride_V); + TMA_V tma_load_V = make_tma_copy_B_sm90( + GmemTiledCopyKV{}, + mV, + SmemLayoutV{}, + TileShape_MNK{}, + ClusterShape{}); // no mcast for KV + if constexpr (Deterministic) { + assert(args.dq_semaphore != nullptr); + } + return { + args.shape_Q, + args.shape_K, + args.shape_V, + args.shape_dO, + args.ptr_dQaccum, + args.shape_dQaccum, + args.stride_dQaccum, + tma_load_Q, + tma_load_dO, + tma_load_K, + tma_load_V, + args.ptr_LSE_log2, + args.shape_LSE, + args.stride_LSE_log2, + args.ptr_dPsum, + args.stride_dPsum, + args.max_attn_len, + args.min_full_attn_seq_len, + args.contextual_seq_len, + args.max_seq_len_inv, + args.alpha, + float(args.alpha * M_LOG2E), + args.num_batch, + args.num_softmax_heads, + args.num_groups, + args.batch_size_per_group, + args.dq_semaphore, + args.seq_offsets, + args.seq_offsets_q, + args.num_targets, + args.max_seq_len_tensor, + args.contextual_seq_len_tensor, + args.max_attn_len_tensor, + args.min_full_attn_seq_len_tensor, + args.attn_scale, + args.scalar_scale}; + } + + /// Issue Tma Descriptor Prefetch -- ideally from a single thread for best + /// performance + CUTLASS_DEVICE + static void prefetch_tma_descriptors(Params const& params) { + cute::prefetch_tma_descriptor(params.tma_load_Q.get_tma_descriptor()); + cute::prefetch_tma_descriptor(params.tma_load_dO.get_tma_descriptor()); + cute::prefetch_tma_descriptor(params.tma_load_K.get_tma_descriptor()); + cute::prefetch_tma_descriptor(params.tma_load_V.get_tma_descriptor()); + } + + CUTLASS_DEVICE + cute::tuple get_m_block_min_max( + int const max_attn_len, + int const contextual_seq_len, + int const uihlen, + int const seqlen, + int const n_block) { + static constexpr int kBlockM = get<0>(TileShape_MNK{}); + static constexpr int kBlockN = get<1>(TileShape_MNK{}); + if constexpr (Has_targets) { + int n_idx_min = n_block * kBlockN; + if (n_idx_min >= uihlen) { + int n_idx_max = (n_block + 1) * kBlockN; + return { + std::max(0, n_idx_min / kBlockM), + cute::ceil_div(std::min(n_idx_max, seqlen), kBlockM)}; + } + } + // uih part + int m_block_max = cute::ceil_div(seqlen, kBlockM); + if constexpr (Local) { + int local_m_block_max = + cute::ceil_div((n_block + 1) * kBlockN + max_attn_len, kBlockM); + if constexpr (Contexual_mask) { + // row contexual without sink + if (n_block * kBlockN < contextual_seq_len) { + local_m_block_max = std::max( + local_m_block_max, + cute::ceil_div(contextual_seq_len + max_attn_len, kBlockM)); + } + } + m_block_max = std::min(m_block_max, local_m_block_max); + } + int m_block_min = 0; + if constexpr (Causal || Local) { + m_block_min = std::max(m_block_min, (n_block * kBlockN) / kBlockM); + } + return {m_block_min, m_block_max}; + } + + CUTLASS_DEVICE + cute::tuple get_full_m_block_min_max( + int const uihlen, + int const seqlen, + int const min_full_attn_seq_len, + int const m_block_max, + int const n_block) { + static constexpr int kBlockM = get<0>(TileShape_MNK{}); + static constexpr int kBlockN = get<1>(TileShape_MNK{}); + if constexpr (Cross) { + return {0, 0}; + } + if constexpr (!Local) { + return {0, 0}; + } + if constexpr (Has_targets) { + int n_idx_min = n_block * kBlockN; + if (n_idx_min >= uihlen) { + return {0, 0}; + } + } + if constexpr (Local) { + int full_m_block_max = cute::ceil_div(seqlen, kBlockM); + int full_m_block_min = + std::max(m_block_max, (uihlen - min_full_attn_seq_len) / kBlockM); + return {full_m_block_min, full_m_block_max}; + } + return {0, 0}; + } + + CUTLASS_DEVICE + int get_contexual_m_block_max( + int const uihlen, + int const contextual_seq_len, + int const m_block_min, + int const n_block) { + static constexpr int kBlockM = get<0>(TileShape_MNK{}); + static constexpr int kBlockN = get<1>(TileShape_MNK{}); + if constexpr (Cross) { + return 0; + } + if constexpr (!Contexual_mask) { + return 0; + } + if constexpr (Has_targets) { + int n_idx_min = n_block * kBlockN; + if (n_idx_min >= uihlen) { + return 0; + } + } + if constexpr (Causal || Local) { + int contexual_m_block_max = + std::min(m_block_min, cute::ceil_div(contextual_seq_len, kBlockM)); + return contexual_m_block_max; + } + return 0; + } + + CUTLASS_DEVICE + int get_next_m_block( + int const m_block, + int const m_block_min, + int const m_block_max, + int const contexual_m_block_max, + int const full_m_block_min, + int const full_m_block_max) { + int const out_m_block = m_block + 1; + if constexpr (Contexual_mask || Local) { + if (out_m_block == m_block_max) { + if (contexual_m_block_max > 0) { + return 0; + } + if (full_m_block_max > full_m_block_min) { + return full_m_block_min; + } + return -1; + } + if (out_m_block == contexual_m_block_max) { + if (full_m_block_max > full_m_block_min) { + return full_m_block_min; + } + return -1; + } + if (out_m_block == full_m_block_max) { + return -1; + } + return out_m_block; + } + if (out_m_block == m_block_max) { + return -1; + } + return out_m_block; + } + + CUTLASS_DEVICE + cute::tuple get_cross_m_block_min_max( + int const uihlen_q, + int const seqlen_q, + int const seqlen_kv, + int const n_block) { + static constexpr int kBlockM = get<0>(TileShape_MNK{}); + static constexpr int kBlockN = get<1>(TileShape_MNK{}); + int m_block_max = cute::ceil_div(seqlen_q, kBlockM); + if constexpr (!Causal) { + return {0, m_block_max}; + } + int m_block_min = + std::max(0, (n_block * kBlockN + uihlen_q - seqlen_kv) / kBlockM); + return {m_block_min, m_block_max}; + } + + template + CUTLASS_DEVICE void load( + Params const& params, + MainloopPipeline pipeline_q, + MainloopPipeline_dO pipeline_do, + PipelineState& smem_pipe_write, + PipelineState_dO& smem_pipe_write_do, + SharedStorage& shared_storage, + SchedulerPrefetch const& scheduler_prefetch, + cute::tuple block_coord) { + auto [n_block, bidh, bidb] = block_coord; + SeqlenInfo_t seqlen_info{ + bidb, + get<0>(params.shape_Q), + get<0>(params.shape_K), + params.seq_offsets, + params.seq_offsets_q, + params.num_targets}; + if constexpr (Jagged) { + static constexpr int kBlockN = get<1>(TileShape_MNK{}); + if (n_block * kBlockN >= seqlen_info.seqlen_kv) { + scheduler_prefetch(); + return; + } + } + int min_full_attn_seq_len_, max_attn_len_, contextual_seq_len_; + if constexpr (!Cross) { + if (params.num_groups > 1) { + int group_id = bidb / params.batch_size_per_group; + min_full_attn_seq_len_ = params.min_full_attn_seq_len_tensor[group_id]; + max_attn_len_ = params.max_attn_len_tensor[group_id]; + contextual_seq_len_ = params.contextual_seq_len_tensor[group_id]; + } else { + min_full_attn_seq_len_ = params.min_full_attn_seq_len; + max_attn_len_ = params.max_attn_len; + contextual_seq_len_ = params.contextual_seq_len; + } + } + int m_block_min, m_block_max; + if constexpr (Cross) { + auto m_block_min_max = get_cross_m_block_min_max( + seqlen_info.uihlen_q, + seqlen_info.seqlen_q, + seqlen_info.seqlen_kv, + n_block); + m_block_min = get<0>(m_block_min_max); + m_block_max = get<1>(m_block_min_max); + } else { + auto m_block_min_max = get_m_block_min_max( + max_attn_len_, + contextual_seq_len_, + seqlen_info.uihlen_q, + seqlen_info.seqlen_q, + n_block); + m_block_min = get<0>(m_block_min_max); + m_block_max = get<1>(m_block_min_max); + } + auto full_m_block_min_max = get_full_m_block_min_max( + seqlen_info.uihlen_q, + seqlen_info.seqlen_q, + min_full_attn_seq_len_, + m_block_max, + n_block); + int const full_m_block_min = get<0>(full_m_block_min_max); + int const full_m_block_max = get<1>(full_m_block_min_max); + int contexual_m_block_max = get_contexual_m_block_max( + seqlen_info.uihlen_q, contextual_seq_len_, m_block_min, n_block); + + Tensor sQ = make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_q.data()), + SmemLayoutQ{}); + Tensor sdO = make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_do.data()), + SmemLayoutdO{}); + Tensor sK = make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_k.data()), + SmemLayoutK{}); + Tensor sV = make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_v.data()), + SmemLayoutV{}); + Tensor sLSE = make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_lse.data()), + SmemLayoutLSE{}); + Tensor sdPsum = make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_dpsum.data()), + SmemLayoutLSE{}); + + // Prepare the TMA loads + uint32_t block_rank_in_cluster = cute::block_rank_in_cluster(); + constexpr uint32_t cluster_shape_x = get<0>(ClusterShape()); + uint2 cluster_local_block_id = { + block_rank_in_cluster % cluster_shape_x, + block_rank_in_cluster / cluster_shape_x}; + Tensor mQ = params.tma_load_Q.get_tma_tensor(params.shape_Q)( + _, _, bidh, !Jagged ? bidb : 0); + Tensor mdO = params.tma_load_dO.get_tma_tensor(params.shape_Q)( + _, _, bidh, !Jagged ? bidb : 0); + Tensor mK = params.tma_load_K.get_tma_tensor(params.shape_K)( + _, _, bidh, !Jagged ? bidb : 0); + Tensor mV = params.tma_load_V.get_tma_tensor(params.shape_K)( + _, _, bidh, !Jagged ? bidb : 0); + Tensor mLSE = make_tensor( + make_gmem_ptr(params.ptr_LSE_log2), + params.shape_LSE, + params.stride_LSE_log2)(_, bidh, !Jagged ? bidb : 0); + Tensor mdPsum = make_tensor( + make_gmem_ptr(params.ptr_dPsum), params.shape_LSE, params.stride_dPsum)( + _, bidh, !Jagged ? bidb : 0); + + Tensor gQ = local_tile( + domain_offset(make_coord(seqlen_info.offset_q, _0{}), mQ), + select<0, 2>(TileShape_MNK{}), + make_coord(_, _0{})); // (M, K, _) + Tensor gdO = local_tile( + domain_offset(make_coord(seqlen_info.offset_q, _0{}), mdO), + select<0, 2>(TileShape_MNK{}), + make_coord(_, _0{})); // (M, K, _) + Tensor gK = local_tile( + domain_offset(make_coord(seqlen_info.offset_k, _0{}), mK), + select<1, 2>(TileShape_MNK{}), + make_coord(n_block, _0{})); // (N, K) + Tensor gV = local_tile( + domain_offset(make_coord(seqlen_info.offset_k, _0{}), mV), + select<1, 2>(TileShape_MNK{}), + make_coord(n_block, _0{})); // (N, K) + Tensor gLSE = local_tile( + domain_offset(make_coord(seqlen_info.offset_q_padded), mLSE), + select<0>(TileShape_MNK{}), + make_coord(_)); // (M, _) + Tensor gdPsum = local_tile( + domain_offset(make_coord(seqlen_info.offset_q_padded), mdPsum), + select<0>(TileShape_MNK{}), + make_coord(_)); // (M, _) + + Tensor sK_x = + make_tensor(sK.data(), make_layout(sK.layout(), Layout<_1>{})); + Tensor gK_x = + make_tensor(gK.data(), make_layout(gK.layout(), Layout<_1>{})); + Tensor sV_x = + make_tensor(sV.data(), make_layout(sV.layout(), Layout<_1>{})); + Tensor gV_x = + make_tensor(gV.data(), make_layout(gV.layout(), Layout<_1>{})); + // auto [tQgQ, tQsQ] = tma_partition(params.tma_load_Q, + // block_rank_in_cluster, Layout{}, + // group_modes<0, 2>(sQ), group_modes<0, + // 2>(gQ)); // (TMA, k), (TMA, PIPE) + // auto [tdOgdO, tdOsdO] = tma_partition(params.tma_load_dO, + // block_rank_in_cluster, Layout{}, + // group_modes<0, 2>(sdO), group_modes<0, + // 2>(gdO)); // (TMA, k), (TMA, PIPE) + auto block_tma_Q = params.tma_load_Q.get_slice(cluster_local_block_id.y); + auto block_tma_dO = params.tma_load_dO.get_slice(cluster_local_block_id.y); + Tensor tQgQ = group_modes<0, 3>(block_tma_Q.partition_S(gQ)); + Tensor tQsQ = group_modes<0, 3>(block_tma_Q.partition_D(sQ)); + Tensor tdOgdO = group_modes<0, 3>(block_tma_dO.partition_S(gdO)); + Tensor tdOsdO = group_modes<0, 3>(block_tma_dO.partition_D(sdO)); + auto [tKgK, tKsK] = tma_partition( + params.tma_load_K, + _0{}, + Layout<_1>{}, + group_modes<0, 2>(sK_x), + group_modes<0, 2>(gK_x)); // (TMA), (TMA) + auto [tVgV, tVsV] = tma_partition( + params.tma_load_V, + _0{}, + Layout<_1>{}, + group_modes<0, 2>(sV_x), + group_modes<0, 2>(gV_x)); // (TMA), (TMA) + auto bulk_copy = Copy_Traits{}; + + uint16_t mcast_mask_qdo = 0; + if constexpr (cute::is_same_v) { + auto block_layout = Layout{}; // (m,n) -> block_id + for (int n = 0; n < size<1>(block_layout); ++n) { + mcast_mask_qdo |= + (uint16_t(1) << block_layout(cluster_local_block_id.x, n, _0{})); + } + } + + int m_block = m_block_min; + int next_m_block = -1; + int lane_predicate = cute::elect_one_sync(); + + if (lane_predicate) { + pipeline_q.producer_acquire(smem_pipe_write); + copy( + params.tma_load_Q.with( + *pipeline_q.producer_get_barrier(smem_pipe_write), + mcast_mask_qdo, + TMA::CacheHintSm90::EVICT_LAST), + tQgQ(_, m_block), + tQsQ(_, smem_pipe_write.index())); + if constexpr (Softmax) { + copy( + bulk_copy.with(*pipeline_q.producer_get_barrier(smem_pipe_write)), + gLSE(_, m_block), + sLSE(_, smem_pipe_write.index())); + } + } + + // // Wait for the MMA warpgroups to say that smem_k and smem_v are ready + // cutlass::arch::NamedBarrier::sync(NumMmaThreads + + // cutlass::NumThreadsPerWarp, + // static_cast(BwdNamedBarriers::KVEmpty) /*id*/); + + auto load_step = [&](int m_block) { + // If Q and dO have the same number of stages, we can use the same + // pipeline state variable to reduce registers + PipelineState_dO smem_pipe_write_do_cur = + cute::conditional_return( + smem_pipe_write, smem_pipe_write_do); + pipeline_do.producer_acquire(smem_pipe_write_do_cur); + copy( + params.tma_load_dO.with( + *pipeline_do.producer_get_barrier(smem_pipe_write_do_cur), + mcast_mask_qdo, + TMA::CacheHintSm90::EVICT_LAST), + tdOgdO(_, m_block), + tdOsdO(_, smem_pipe_write_do_cur.index())); + if constexpr (Softmax) { + copy( + bulk_copy.with( + *pipeline_do.producer_get_barrier(smem_pipe_write_do_cur)), + gdPsum(_, m_block), + sdPsum(_, smem_pipe_write_do_cur.index())); + } + if constexpr (!Q_dO_same_stages) { + ++smem_pipe_write_do; + } + ++smem_pipe_write; + next_m_block = get_next_m_block( + m_block, + m_block_min, + m_block_max, + contexual_m_block_max, + full_m_block_min, + full_m_block_max); + if (next_m_block != -1) { + pipeline_q.producer_acquire(smem_pipe_write); + copy( + params.tma_load_Q.with( + *pipeline_q.producer_get_barrier(smem_pipe_write), + mcast_mask_qdo, + TMA::CacheHintSm90::EVICT_LAST), + tQgQ(_, next_m_block), + tQsQ(_, smem_pipe_write.index())); + if constexpr (Softmax) { + copy( + bulk_copy.with(*pipeline_q.producer_get_barrier(smem_pipe_write)), + gLSE(_, next_m_block), + sLSE(_, smem_pipe_write.index())); + } + } + }; + + if (lane_predicate) { + // Copy K tile and V tile from GMEM to SMEM. + shared_storage.pipelines.barrier_KV.arrive_and_expect_tx( + TmaTransactionBytesK + TmaTransactionBytesV); + copy( + params.tma_load_K.with( + reinterpret_cast< + cutlass::arch::ClusterTransactionBarrier::ValueType&>( + shared_storage.pipelines.barrier_KV), + 0 /*mcast_mask*/), + tKgK, + tKsK); + copy( + params.tma_load_V.with( + reinterpret_cast< + cutlass::arch::ClusterTransactionBarrier::ValueType&>( + shared_storage.pipelines.barrier_KV), + 0 /*mcast_mask*/), + tVgV, + tVsV); + +#pragma unroll(kHeadDim < 256 ? 2 : 1) + for (; m_block < m_block_max; ++m_block) { + load_step(m_block); + } + } + scheduler_prefetch(); + m_block = next_m_block; + if constexpr (Contexual_mask) { + if (lane_predicate) { + if (m_block >= 0) { +#pragma unroll(kHeadDim < 256 ? 2 : 1) + for (; m_block < contexual_m_block_max; ++m_block) { + load_step(m_block); + } + } + } + } + m_block = next_m_block; + if constexpr (Local) { + if (lane_predicate) { + if (m_block >= 0) { +#pragma unroll(kHeadDim < 256 ? 2 : 1) + for (; m_block < full_m_block_max; ++m_block) { + load_step(m_block); + } + } + } + } + if constexpr (Q_dO_same_stages) { + smem_pipe_write_do = smem_pipe_write; + } + } + + /// Perform a Producer Epilogue to prevent early exit of blocks in a Cluster + CUTLASS_DEVICE void load_tail( + MainloopPipeline pipeline_q, + MainloopPipeline_dO pipeline_do, + PipelineState& smem_pipe_write) { + static_assert( + Q_dO_same_stages, "Q and dO must have the same number of stages"); + // Need to copy since pipeline_q.producer_tail(smem_pipe_write) will + // increment smem_pipe_write + PipelineState smem_pipe_write_do = smem_pipe_write; + // Issue the epilogue waits + if (cute::elect_one_sync()) { + /* This helps avoid early exit of blocks in Cluster + * Waits for all stages to either be released (all Consumer UNLOCKs), or + * if the stage was never used then would just be acquired since the phase + * was still inverted from make_producer_start_state + */ + pipeline_q.producer_tail(smem_pipe_write); + pipeline_do.producer_tail(smem_pipe_write_do); + } + } + + /// Perform a Producer Epilogue to prevent early exit of blocks in a Cluster + CUTLASS_DEVICE void load_tail( + MainloopPipeline pipeline_q, + MainloopPipeline_dO pipeline_do, + PipelineState& smem_pipe_write, + PipelineState_dO& smem_pipe_write_do) { + // Issue the epilogue waits + if (cute::elect_one_sync()) { + /* This helps avoid early exit of blocks in Cluster + * Waits for all stages to either be released (all Consumer UNLOCKs), or + * if the stage was never used then would just be acquired since the phase + * was still inverted from make_producer_start_state + */ + pipeline_q.producer_tail(smem_pipe_write); + pipeline_do.producer_tail(smem_pipe_write_do); + } + } + + template + CUTLASS_DEVICE void store_dq( + Params const& params, + SharedStorage& shared_storage, + cute::tuple block_coord) { + if constexpr (!dQacc_use_TMA) { + return; + } + + auto [n_block, bidh, bidb] = block_coord; + SeqlenInfo_t seqlen_info{ + bidb, + get<0>(params.shape_Q), + get<0>(params.shape_K), + params.seq_offsets, + params.seq_offsets_q, + params.num_targets}; + if constexpr (Jagged) { + static constexpr int kBlockN = get<1>(TileShape_MNK{}); + if (n_block * kBlockN >= seqlen_info.seqlen_kv) { + return; + } + } + int min_full_attn_seq_len_, max_attn_len_, contextual_seq_len_; + if constexpr (!Cross) { + if (params.num_groups > 1) { + int group_id = bidb / params.batch_size_per_group; + min_full_attn_seq_len_ = params.min_full_attn_seq_len_tensor[group_id]; + max_attn_len_ = params.max_attn_len_tensor[group_id]; + contextual_seq_len_ = params.contextual_seq_len_tensor[group_id]; + } else { + min_full_attn_seq_len_ = params.min_full_attn_seq_len; + max_attn_len_ = params.max_attn_len; + contextual_seq_len_ = params.contextual_seq_len; + } + } + int m_block_min, m_block_max; + if constexpr (Cross) { + auto m_block_min_max = get_cross_m_block_min_max( + seqlen_info.uihlen_q, + seqlen_info.seqlen_q, + seqlen_info.seqlen_kv, + n_block); + m_block_min = get<0>(m_block_min_max); + m_block_max = get<1>(m_block_min_max); + } else { + auto m_block_min_max = get_m_block_min_max( + max_attn_len_, + contextual_seq_len_, + seqlen_info.uihlen_q, + seqlen_info.seqlen_q, + n_block); + m_block_min = get<0>(m_block_min_max); + m_block_max = get<1>(m_block_min_max); + } + auto full_m_block_min_max = get_full_m_block_min_max( + seqlen_info.uihlen_q, + seqlen_info.seqlen_q, + min_full_attn_seq_len_, + m_block_max, + n_block); + int const full_m_block_min = get<0>(full_m_block_min_max); + int const full_m_block_max = get<1>(full_m_block_min_max); + int contexual_m_block_max = get_contexual_m_block_max( + seqlen_info.uihlen_q, contextual_seq_len_, m_block_min, n_block); + + Tensor sdQ = make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_dqacc.data()), + SmemLayoutdQaccum{}); + static constexpr int dQ_TMA_num_bytes = + CUTE_STATIC_V(size<0>(sdQ)) * sizeof(ElementAccum); + + Tensor mdQaccum = make_tensor( + make_gmem_ptr(reinterpret_cast(params.ptr_dQaccum)), + params.shape_dQaccum, + params.stride_dQaccum)(_, bidh, !Jagged ? bidb : 0); + Tensor gdQaccum_ = local_tile( + domain_offset( + make_coord(seqlen_info.offset_q_padded * kHeadDim), mdQaccum), + Shape>{}, + make_coord(_)); // (M * K, _) + Tensor gdQaccum = cute::flat_divide( + gdQaccum_, + Int{}); // (M * K / WG, WG, _) + + int const num_batch = params.num_batch; + int const num_head = get<2>(params.shape_Q); + int* lock_ptr = + !Deterministic ? nullptr : params.dq_semaphore + bidb * num_head + bidh; + using Barrier = cutlass::GenericBarrier; + bool const lane_predicate = cute::elect_one_sync(); + + auto store_dq_step = [&](int m_block) { + if constexpr (Deterministic) { + Barrier::wait_eq( + lock_ptr, + threadIdx.x % cutlass::NumThreadsPerWarp, + m_block * num_batch * num_head, + n_block); + } +#pragma unroll + for (int warpgroup_idx = 0; warpgroup_idx < NumMmaWarpGroups; + ++warpgroup_idx) { + cutlass::arch::NamedBarrier::sync( + cutlass::NumThreadsPerWarpGroup + cutlass::NumThreadsPerWarp, + static_cast(BwdNamedBarriers::dQFullWG1) + + warpgroup_idx /*id*/); // sdQ full, to be written to gmem + if (lane_predicate) { + SM90_BULK_REDUCE_ADD::copy( + raw_pointer_cast(sdQ(_, warpgroup_idx).data()), + raw_pointer_cast(gdQaccum(_, warpgroup_idx, m_block).data()), + dQ_TMA_num_bytes, + static_cast(TMA::CacheHintSm90::EVICT_LAST)); + tma_store_arrive(); + } + } + // Note, the for_each() function is required here to ensure + // `warpgroup_idx` is of type Int. + for_each(make_int_sequence{}, [&](auto warpgroup_idx) { + if (lane_predicate) { + tma_store_wait(); + } + cutlass::arch::NamedBarrier::arrive( + cutlass::NumThreadsPerWarpGroup + cutlass::NumThreadsPerWarp, + static_cast(BwdNamedBarriers::dQEmptyWG1) + + warpgroup_idx /*id*/); // sdQ empty, ready to be written to + }); + if constexpr (Deterministic) { + Barrier::arrive_inc( + lock_ptr, + threadIdx.x % cutlass::NumThreadsPerWarp, + m_block * num_batch * num_head); + } + }; + +#pragma unroll 2 + for (int m_block = m_block_min; m_block < m_block_max; ++m_block) { + store_dq_step(m_block); + } + if constexpr (Contexual_mask) { +#pragma unroll 2 + for (int m_block = 0; m_block < contexual_m_block_max; ++m_block) { + store_dq_step(m_block); + } + } + if constexpr (Local) { +#pragma unroll 2 + for (int m_block = full_m_block_min; m_block < full_m_block_max; + ++m_block) { + store_dq_step(m_block); + } + } + if constexpr (Local && Deterministic) { + constexpr int kBlockM = get<0>(TileShape_MNK{}); + int const m_block_global_max = + cute::ceil_div(seqlen_info.seqlen_q, kBlockM); +#pragma unroll 2 + for (int m_block = m_block_max; m_block < m_block_global_max; ++m_block) { + Barrier::arrive_inc( + lock_ptr, + threadIdx.x % cutlass::NumThreadsPerWarp, + m_block * num_batch * num_head); + } + } + } + + CUTLASS_DEVICE void mma_init() { + // We're not currently using this bc we're not using persistent scheduler + // // Tell producer (warp 0) that smem_k and smem_v are ready + // cutlass::arch::NamedBarrier::arrive(NumMmaThreads + + // cutlass::NumThreadsPerWarp, + // static_cast(BwdNamedBarriers::KVEmpty) /*id*/); + int warp_idx_in_warpgroup = + __shfl_sync(0xffffffff, (threadIdx.x / 32) % 4, 0); + if constexpr (dQacc_use_TMA) { + if (warp_idx_in_warpgroup == 0) { + cutlass::arch::NamedBarrier::arrive( + cutlass::NumThreadsPerWarpGroup + cutlass::NumThreadsPerWarp, + static_cast(BwdNamedBarriers::dQEmptyWG1) - 1 + + hstu::canonical_warp_group_idx_nosync() /*id*/); // sdQ empty, + // ready to be + // written to + } + } + } + + template + CUTLASS_DEVICE bool mma( + Params const& params, + MainloopPipeline pipeline_q, + MainloopPipeline_dO pipeline_do, + PipelineState& smem_pipe_read, + PipelineState_dO& smem_pipe_read_do, + FrgTensordKV& tdKrdK, + FrgTensordKV& tdVrdV, + int thread_idx, + int& work_idx, + cute::tuple block_coord, + SharedStorage& shared_storage) { + static_assert( + is_rmem::value, + "dK and dV tensor must be rmem resident."); + + int n_block = get<0>(block_coord); + int bidb = get<2>(block_coord); + SeqlenInfo_t seqlen_info{ + bidb, + get<0>(params.shape_Q), + get<0>(params.shape_K), + params.seq_offsets, + params.seq_offsets_q, + params.num_targets}; + if constexpr (Jagged) { + static constexpr int kBlockN = get<1>(TileShape_MNK{}); + if (n_block * kBlockN >= seqlen_info.seqlen_kv) { + return false; + } + } + int min_full_attn_seq_len_, max_attn_len_, contextual_seq_len_; + float scalar_scale_val_; + if constexpr (!Cross) { + if (params.num_groups > 1) { + int group_id = bidb / params.batch_size_per_group; + min_full_attn_seq_len_ = params.min_full_attn_seq_len_tensor[group_id]; + max_attn_len_ = params.max_attn_len_tensor[group_id]; + contextual_seq_len_ = params.contextual_seq_len_tensor[group_id]; + } else { + min_full_attn_seq_len_ = params.min_full_attn_seq_len; + max_attn_len_ = params.max_attn_len; + contextual_seq_len_ = params.contextual_seq_len; + } + } + if (params.num_groups > 1) { + int group_id = bidb / params.batch_size_per_group; + int max_seq_len_per_group = params.max_seq_len_tensor[group_id]; + // attention scale + scalar_scale_val_ = params.scalar_scale + ? (params.attn_scale == nullptr ? 1.0f / max_seq_len_per_group + : params.attn_scale[group_id]) + : 0; + } else { + // attention scale + scalar_scale_val_ = params.scalar_scale + ? (params.attn_scale == nullptr ? params.max_seq_len_inv + : params.attn_scale[0]) + : 0; + } + int m_block_min, m_block_max; + if constexpr (Cross) { + auto m_block_min_max = get_cross_m_block_min_max( + seqlen_info.uihlen_q, + seqlen_info.seqlen_q, + seqlen_info.seqlen_kv, + n_block); + m_block_min = get<0>(m_block_min_max); + m_block_max = get<1>(m_block_min_max); + } else { + auto m_block_min_max = get_m_block_min_max( + max_attn_len_, + contextual_seq_len_, + seqlen_info.uihlen_q, + seqlen_info.seqlen_q, + n_block); + m_block_min = get<0>(m_block_min_max); + m_block_max = get<1>(m_block_min_max); + } + auto full_m_block_min_max = get_full_m_block_min_max( + seqlen_info.uihlen_q, + seqlen_info.seqlen_q, + min_full_attn_seq_len_, + m_block_max, + n_block); + int const full_m_block_min = get<0>(full_m_block_min_max); + int const full_m_block_max = get<1>(full_m_block_min_max); + int contexual_m_block_max = get_contexual_m_block_max( + seqlen_info.uihlen_q, contextual_seq_len_, m_block_min, n_block); + + Tensor sQ = make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_q.data()), + SmemLayoutQ{}); + Tensor sdO = make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_do.data()), + SmemLayoutdO{}); + Tensor sK = make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_k.data()), + SmemLayoutK{}); + Tensor sV = make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_v.data()), + SmemLayoutV{}); + Tensor sQt = make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_q.data()), + SmemLayoutQt{}); + Tensor sdOt = make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_do.data()), + SmemLayoutdOt{}); + Tensor sKt = make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_k.data()), + SmemLayoutKt{}); + Tensor sP = make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_p.data()), + SmemLayoutPdS{}); + Tensor sP_pi = cute::as_position_independent_swizzle_tensor(sP); + Tensor sPt = make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_p.data()), + SmemLayoutPdSt{}); + Tensor sPt_pi = cute::as_position_independent_swizzle_tensor(sPt); + Tensor sdS = make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_ds.data()), + SmemLayoutPdS{}); + Tensor sdS_pi = cute::as_position_independent_swizzle_tensor(sdS); + Tensor sdSt = make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_ds.data()), + SmemLayoutPdSt{}); + Tensor sdSt_pi = cute::as_position_independent_swizzle_tensor(sdSt); + Tensor sdQ = make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_dqacc.data()), + SmemLayoutdQaccum{}); + + static_assert( + stride<0>(typename TiledMmaSdP::ALayout{}) == 0 and + stride<0>(typename TiledMmaSdP::BLayout{}) == 0 and + size<0>(typename TiledMmaSdP::ALayout{}) == + cutlass::NumThreadsPerWarpGroup and + size<0>(typename TiledMmaSdP::BLayout{}) == + cutlass::NumThreadsPerWarpGroup, + "Stride of the first mode must be 0 and the size of the mode must be NumThreadsPerWarpGroup"); + constexpr int MmaWarpGroups = + NumMmaThreads / cutlass::NumThreadsPerWarpGroup; + Layout warp_group_thread_layout = make_layout( + make_shape(Int{}), + make_stride(Int{})); + Layout warp_group_thread_layout_dq = make_layout( + make_shape(Int{}), + make_stride(Int{})); + + int warp_group_idx = __shfl_sync( + 0xFFFFFFFF, thread_idx / cutlass::NumThreadsPerWarpGroup, 0); + TiledMmaSdP tiled_mma_SdP; + using TiledMmadP = + std::conditional_t; + TiledMmadP tiled_mma_dP; + TiledMmadKV tiled_mma_dKV; + TiledMmadQ tiled_mma_dQ; + + auto wg_mma_SdP = + tiled_mma_SdP.get_slice(warp_group_thread_layout(warp_group_idx)); + auto wg_mma_dP = + tiled_mma_dP.get_slice(warp_group_thread_layout(warp_group_idx)); + auto thread_mma_SdP = tiled_mma_SdP.get_thread_slice(thread_idx); + auto wg_mma_dKV = + tiled_mma_dKV.get_slice(warp_group_thread_layout(warp_group_idx)); + auto wg_mma_dQ = + tiled_mma_dQ.get_slice(warp_group_thread_layout_dq(warp_group_idx)); + + auto smem_tiled_copy_PdS = + make_tiled_copy_C(SmemCopyAtomPdS{}, tiled_mma_SdP); + auto smem_thr_copy_PdS = smem_tiled_copy_PdS.get_thread_slice(thread_idx); + + R2STiledCopydQaccum r2s_tiled_copy_dQaccum; + auto r2s_thr_copy_dQaccum = + r2s_tiled_copy_dQaccum.get_thread_slice(thread_idx); + Tensor tdQsdQaccum = r2s_thr_copy_dQaccum.partition_D(sdQ); + // if (thread_idx == 0) { print(sdQ); printf("\n"); print(tdQsdQaccum); + // printf("\n"); } + + // Allocate "fragments/descriptors" + // We have to use the templated mma_partition_fragment_AB instead of + // cute::conditional_return or lambda, because some partition_fragment_A/B + // don't compile. + // https://stackoverflow.com/questions/50051473/if-constexpr-in-c17-does-not-work-in-a-non-templated-function + Tensor tSrQ = mma_partition_fragment_AB(wg_mma_SdP, sQ); + Tensor tSrK = mma_partition_fragment_AB(wg_mma_SdP, sK); + Tensor tdPrdO = + mma_partition_fragment_AB(wg_mma_SdP, sdO); + Tensor tdPrV = mma_partition_fragment_AB(wg_mma_dP, sV); + Tensor tdVrdO = + mma_partition_fragment_AB(wg_mma_dKV, sdOt); + Tensor tdKrQ = mma_partition_fragment_AB(wg_mma_dKV, sQt); + Tensor tdQrdS = mma_partition_fragment_AB(wg_mma_dQ, sdS); + Tensor tdQrK = mma_partition_fragment_AB(wg_mma_dQ, sKt); + + Tensor tPsP = smem_thr_copy_PdS.partition_D( + cute::conditional_return( + sP_pi, sPt_pi)); // ((Atom,AtomNum),PIPE_M,PIPE_N) + Tensor tdSsdS = smem_thr_copy_PdS.partition_D( + cute::conditional_return( + sdS_pi, sdSt_pi)); // ((Atom,AtomNum),PIPE_M,PIPE_N) + // if (blockIdx.x == 0 && threadIdx.x == 128) { print(smem_thr_copy_PdS); + // print(sP_pi); printf("\n"); print(sPt_pi); printf("\n"); print(tPsP); + // printf("\n"); print(tdSsdS); printf("\n"); } + + auto consumer_wait = [](auto& pipeline, auto& smem_pipe_read) { + auto barrier_token = pipeline.consumer_try_wait(smem_pipe_read); + pipeline.consumer_wait(smem_pipe_read, barrier_token); + }; + + int bidh = get<1>(block_coord); + // For the case where we do atomicAdd directly to gdQaccum instead of using + // TMA + Tensor mdQaccum = make_tensor( + make_gmem_ptr(reinterpret_cast(params.ptr_dQaccum)), + params.shape_dQaccum, + params.stride_dQaccum)(_, bidh, !Jagged ? bidb : 0); + Tensor gdQaccum_ = local_tile( + domain_offset( + make_coord(seqlen_info.offset_q_padded * kHeadDim), mdQaccum), + Shape>{}, + make_coord(_)); // (M * K, _) + Tensor gdQaccum = cute::flat_divide( + gdQaccum_, + Int{}); // (M * K / WG, WG, _) + // We can reuse r2s_thr_copy_dQaccum for this partitioning + Tensor tdQgdQaccum = r2s_thr_copy_dQaccum.partition_D(gdQaccum); + // if (blockIdx.x == 0 && threadIdx.x == 128) { print(mdQaccum); + // printf("\n"); print(gdQaccum_); printf("\n"); print(gdQaccum); + // printf("\n"); print(tdQgdQaccum); printf("\n"); } + + hstu::Mask mask( + thread_idx, + seqlen_info.seqlen_q, + seqlen_info.seqlen_kv, + max_attn_len_, + min_full_attn_seq_len_, + contextual_seq_len_, + seqlen_info.uihlen_q); + + int m_block = m_block_min; + + clear(tdKrdK); + clear(tdVrdV); + // tiled_mma_dKV.accumulate_ = GMMA::ScaleOut::Zero; + + cutlass::ConsumerToken barrier_token = static_cast( + shared_storage.pipelines.barrier_KV.try_wait(work_idx % 2)); + if (barrier_token == cutlass::BarrierStatus::WaitAgain) { + shared_storage.pipelines.barrier_KV.wait(work_idx % 2); + } + + if constexpr (Mma_dP_is_RS) { + using SmemCopyAtomV = Copy_Atom; + auto smem_tiled_copy_V = make_tiled_copy_A(SmemCopyAtomV{}, tiled_mma_dP); + auto smem_thr_copy_V = smem_tiled_copy_V.get_thread_slice(thread_idx); + Tensor tdPrV_copy_view = smem_thr_copy_V.retile_D(tdPrV); + Tensor tdPsV_copy_view = smem_thr_copy_V.partition_S( + cute::as_position_independent_swizzle_tensor(sV)); + cute::copy(smem_tiled_copy_V, tdPsV_copy_view, tdPrV_copy_view); + } + static constexpr int Qdim = !SdP_swapAB ? 0 : 1; + auto thread0_mma_SdP = tiled_mma_SdP.get_thread_slice(_0{}); + Tensor cS = cute::make_identity_tensor( + Shape< + Int, + Int>{}); + Tensor tScS = thread_mma_SdP.partition_C(cS); + Tensor tScS_rowcol = make_tensor( + tScS.data(), + hstu::convert_layout_acc_rowcol( + tScS.layout())); + Tensor t0ScS = thread0_mma_SdP.partition_C(cS); + Tensor t0ScS_rowcol = make_tensor( + t0ScS.data(), + hstu::convert_layout_acc_rowcol( + t0ScS.layout())); + int const thread_qdim_offset = get(tScS_rowcol(_0{}, _0{})); + + auto bwd_step = [&](int m_block, auto mask_fn) { + Tensor tSrS = partition_fragment_C( + tiled_mma_SdP, + select(TileShape_MNK{})); + consumer_wait(pipeline_q, smem_pipe_read); + hstu::gemm( + tiled_mma_SdP, tSrQ(_, _, _, smem_pipe_read.index()), tSrK, tSrS); + Tensor tdPrdP = partition_fragment_C( + tiled_mma_SdP, + select(TileShape_MNK{})); + PipelineState_dO smem_pipe_read_do_cur = + cute::conditional_return( + smem_pipe_read, smem_pipe_read_do); + consumer_wait(pipeline_do, smem_pipe_read_do_cur); + hstu::gemm( + tiled_mma_dP, + tdPrdO(_, _, _, smem_pipe_read_do_cur.index()), + tdPrV, + tdPrdP); + warpgroup_wait<1>(); + // Reshape tSrS from ((2, 2, V), MMA_N, MMA_M) to (nrow=(2, V, MMA_M), + // ncol=(2, MMA_N)) + Tensor scores = make_tensor( + tSrS.data(), + hstu::convert_layout_acc_rowcol( + tSrS.layout())); + Tensor tSrS_sigmoid = make_tensor_like(tSrS); + Tensor sigmoid = make_tensor( + tSrS_sigmoid.data(), + hstu::convert_layout_acc_rowcol( + tSrS_sigmoid.layout())); + int qdim_offset = params.scalar_scale + ? 0 + : m_block * kBlockM + thread_qdim_offset + seqlen_info.offset_q; + mask_fn(tSrS, m_block); +#pragma unroll + for (int mi = 0; mi < size<0>(scores); ++mi) { + float scale = scalar_scale_val_; + if (!params.scalar_scale) { + int q_index = qdim_offset + int(get(t0ScS_rowcol(mi, _0{}))); + int q_local = q_index - seqlen_info.offset_q; + if (q_local < seqlen_info.seqlen_q) { + scale = params.attn_scale[q_index]; + } + } +#pragma unroll + for (int ni = 0; ni < size<1>(scores); ++ni) { + scores(mi, ni) = scores(mi, ni) * params.alpha; + sigmoid(mi, ni) = + __fdividef(1., 1.0f + cutlass::fast_exp(-scores(mi, ni))); + scores(mi, ni) = sigmoid(mi, ni) * scores(mi, ni) * scale; + } + } + mask_fn(tSrS_sigmoid, m_block); + + warpgroup_wait<0>(); + // Reshape tdPrdP from ((2, 2, V), MMA_N, MMA_M) to (nrow=(2, V, MMA_M), + // ncol=(2, MMA_N)) + Tensor dS = make_tensor(tdPrdP.data(), scores.layout()); +#pragma unroll + for (int mi = 0; mi < size<0>(dS); ++mi) { + float scale = scalar_scale_val_; + if (!params.scalar_scale) { + int q_index = qdim_offset + int(get(t0ScS_rowcol(mi, _0{}))); + int q_local = q_index - seqlen_info.offset_q; + if (q_local < seqlen_info.seqlen_q) { + scale = params.attn_scale[q_index]; + } + } +#pragma unroll + for (int ni = 0; ni < size<1>(dS); ++ni) { + dS(mi, ni) = dS(mi, ni) * sigmoid(mi, ni) * scale + + dS(mi, ni) * scores(mi, ni) * (1.f - sigmoid(mi, ni)); + dS(mi, ni) = dS(mi, ni) * params.alpha; + // if (dS(mi, ni) > 0.0001) { + // std::printf( + // "dS(mi, ni) is (%f), (m, n) is (%d, %d), thread_idx is + // (%d), blockIdx.z is (%d)\n", dS(mi, ni), mi, ni, + // threadIdx.x, + // blockIdx.z); + // } + } + } + // Convert scores from fp32 to fp16/bf16 + Tensor rP = make_tensor_like(tSrS); + hstu::convert_type_out(tSrS, rP); + if constexpr (!Mma_dKV_is_RS) { + // Need to sync to make sure P has already been used in the previous + // iteration before writing new values + if constexpr (kStages_dS == 1) { + cutlass::arch::NamedBarrier::sync( + NumMmaThreads, + static_cast(BwdNamedBarriers::PdS) /*id*/); + } + Tensor tPaP = + smem_thr_copy_PdS.retile_S(rP); // ((Atom,AtomNum), MMA_N, MMA_N) + cute::copy( + smem_tiled_copy_PdS, + tPaP, + tPsP( + _, + _, + _, + cute::conditional_return( + _0{}, smem_pipe_read.index()))); + } + Tensor rdS = make_tensor_like(tdPrdP); + hstu::convert_type_out(tdPrdP, rdS); + // If there's double buffering on dS, we don't need to sync here. + // Otherwise we might have WG1 writing to dS before WG2 is done reading + // from it during MmadQ. But because both WGs have to sync at the end of + // the loop and double buffering, this race condition is not possible. + // This sync is to ensure (1) P is written in case of !Mma_dKV_is_RS and + // (2) dS is already read by the Mma in the previous iteration in case of + // Mma_dKV_is_RS. + if constexpr (!Mma_dKV_is_RS || (kStages_dS == 1 && Mma_dKV_is_RS)) { + cutlass::arch::fence_view_async_shared(); + cutlass::arch::NamedBarrier::sync( + NumMmaThreads, static_cast(BwdNamedBarriers::PdS) /*id*/); + } + // For hdim 64, It's faster to write to smem_dS first before the dV gemm + Tensor tdSadS = + smem_thr_copy_PdS.retile_S(rdS); // ((Atom,AtomNum), MMA_N, MMA_N) + cute::copy( + smem_tiled_copy_PdS, + tdSadS, + tdSsdS( + _, + _, + _, + cute::conditional_return( + _0{}, smem_pipe_read.index()))); + + if constexpr (!Slice_dQKV_Mma) { + // Most cases take this path, except for hdim256 where we want to slice + // to reduce register pressure + if constexpr (Mma_dKV_is_RS) { + Tensor tdVrP = make_tensor( + rP.data(), convert_layout_acc_Aregs(tSrS.layout())); + hstu::gemm( + tiled_mma_dKV, + tdVrP, + tdVrdO(_, _, _, smem_pipe_read_do_cur.index()), + tdVrdV); + } else { + Tensor tdVrP = + mma_partition_fragment_AB(wg_mma_dKV, sPt); + Tensor tdVrP_cur = tdVrP( + _, + _, + _, + cute::conditional_return( + _0{}, smem_pipe_read.index())); + hstu:: + gemm( + tiled_mma_dKV, + tdVrP_cur, + tdVrdO(_, _, _, smem_pipe_read_do_cur.index()), + tdVrdV); + } + // SMEM fence to make sure sdS is written before it's read by WGMMA + cutlass::arch::fence_view_async_shared(); + cutlass::arch::NamedBarrier::sync( + NumMmaThreads, static_cast(BwdNamedBarriers::PdS) /*id*/); + Tensor tdQrdQ = partition_fragment_C( + tiled_mma_dQ, + select(TileShape_MNK{})); + Tensor tdQrdS_cur = tdQrdS( + _, + _, + _, + cute::conditional_return( + _0{}, smem_pipe_read.index())); + hstu::gemm( + tiled_mma_dQ, tdQrdS_cur, tdQrK, tdQrdQ); + pipeline_do.consumer_release(smem_pipe_read_do_cur); // release dQ + + if constexpr (Mma_dKV_is_RS) { + Tensor tdKrdS = make_tensor( + rdS.data(), + convert_layout_acc_Aregs(tdPrdP.layout())); + hstu::gemm( + tiled_mma_dKV, + tdKrdS, + tdKrQ(_, _, _, smem_pipe_read.index()), + tdKrdK); + } else { + Tensor tdKrdS = + mma_partition_fragment_AB(wg_mma_dKV, sdSt); + Tensor tdKrdS_cur = tdKrdS( + _, + _, + _, + cute::conditional_return( + _0{}, smem_pipe_read.index())); + hstu::gemm( + tiled_mma_dKV, + tdKrdS_cur, + tdKrQ(_, _, _, smem_pipe_read.index()), + tdKrdK); + } + if constexpr (dQacc_use_TMA) { + int const warp_group_idx = + hstu::canonical_warp_group_idx_nosync() - 1; + cutlass::arch::NamedBarrier::sync( + cutlass::NumThreadsPerWarpGroup + cutlass::NumThreadsPerWarp, + static_cast(BwdNamedBarriers::dQEmptyWG1) + + warp_group_idx /*id*/); // sdQ full, to be written to gmem + Tensor taccdQrdQ = r2s_thr_copy_dQaccum.retile_S(tdQrdQ); + cute::copy(r2s_tiled_copy_dQaccum, taccdQrdQ, tdQsdQaccum); + cutlass::arch::fence_view_async_shared(); + cutlass::arch::NamedBarrier::arrive( + cutlass::NumThreadsPerWarpGroup + cutlass::NumThreadsPerWarp, + static_cast(BwdNamedBarriers::dQFullWG1) + + warp_group_idx /*id*/); // sdQ full, to be written to gmem + } else { + // We can reuse r2s_thr_copy_dQaccum for this partitioning + Tensor tdQrdQ_atomic = + recast(r2s_thr_copy_dQaccum.retile_S(tdQrdQ)); + Tensor tdQgdQaccum_atomic = + recast(tdQgdQaccum(_, _, _, m_block)); + static_assert( + CUTE_STATIC_V(size(tdQrdQ_atomic)) == + CUTE_STATIC_V(size(tdQgdQaccum_atomic))); +#pragma unroll + for (int i = 0; i < size(tdQrdQ_atomic); ++i) { + atomicAdd(&tdQgdQaccum_atomic(i), tdQrdQ_atomic(i)); + } + } + + } else { // Slice_dQKV_Mma + + static_assert(!(Slice_dQKV_Mma && Mma_dKV_is_RS)); + Tensor tdVrP = + mma_partition_fragment_AB(wg_mma_dKV, sPt); + Tensor tdVrP_cur = tdVrP( + _, + _, + _, + cute::conditional_return( + _0{}, smem_pipe_read.index())); + hstu::gemm< + /*zero_init=*/false, + /*wg_wait=*/-1, + /*SwapAB=*/dKV_swapAB, + /*M_slice=*/0>( + tiled_mma_dKV, + tdVrP_cur, + tdVrdO(_, _, _, smem_pipe_read_do_cur.index()), + tdVrdV); + + cutlass::arch::fence_view_async_shared(); + cutlass::arch::NamedBarrier::sync( + NumMmaThreads, static_cast(BwdNamedBarriers::PdS) /*id*/); + Tensor tdQrdQ = partition_fragment_C( + tiled_mma_dQ, + select(TileShape_MNK{})); + Tensor tdQrdS_cur = tdQrdS( + _, + _, + _, + cute::conditional_return( + _0{}, smem_pipe_read.index())); + hstu::gemm< + /*zero_init=*/true, + /*wg_wait=*/-1, + /*SwapAB=*/dQ_swapAB, + /*M_slice=*/0>(tiled_mma_dQ, tdQrdS_cur, tdQrK, tdQrdQ); + hstu::gemm< + /*zero_init=*/false, + /*wg_wait=*/1, + /*SwapAB=*/dKV_swapAB, + /*M_slice=*/1>( + tiled_mma_dKV, + tdVrP_cur, + tdVrdO(_, _, _, smem_pipe_read_do_cur.index()), + tdVrdV); + Tensor tdQrdQ_atomic = + recast(r2s_thr_copy_dQaccum.retile_S(tdQrdQ)); + Tensor tdQgdQaccum_atomic = + recast(tdQgdQaccum(_, _, _, m_block)); +#pragma unroll + for (int i = 0; i < size(tdQrdQ_atomic) / 2; ++i) { + atomicAdd(&tdQgdQaccum_atomic(i), tdQrdQ_atomic(i)); + } + + Tensor tdKrdS = + mma_partition_fragment_AB(wg_mma_dKV, sdSt); + Tensor tdKrdS_cur = tdKrdS( + _, + _, + _, + cute::conditional_return( + _0{}, smem_pipe_read.index())); + hstu::gemm< + /*zero_init=*/false, + /*wg_wait=*/1, + /*SwapAB=*/dKV_swapAB, + /*M_slice=*/0>( + tiled_mma_dKV, + tdKrdS_cur, + tdKrQ(_, _, _, smem_pipe_read.index()), + tdKrdK); + pipeline_do.consumer_release(smem_pipe_read_do_cur); // release dO + + hstu::gemm< + /*zero_init=*/true, + /*wg_wait=*/0, + /*SwapAB=*/dQ_swapAB, + /*M_slice=*/1>(tiled_mma_dQ, tdQrdS_cur, tdQrK, tdQrdQ); +#pragma unroll + for (int i = size(tdQrdQ_atomic) / 2; i < size(tdQrdQ_atomic); ++i) { + atomicAdd(&tdQgdQaccum_atomic(i), tdQrdQ_atomic(i)); + } + + hstu::gemm< + /*zero_init=*/false, + /*wg_wait=*/-1, + /*SwapAB=*/dKV_swapAB, + /*M_slice=*/1>( + tiled_mma_dKV, + tdKrdS_cur, + tdKrQ(_, _, _, smem_pipe_read.index()), + tdKrdK); + } + + warpgroup_wait<0>(); + pipeline_q.consumer_release(smem_pipe_read); // release Q + ++smem_pipe_read; + if constexpr (!Q_dO_same_stages) { + ++smem_pipe_read_do; + } + }; + static constexpr int kBlockM = get<0>(TileShape_MNK{}); + static constexpr int kBlockN = get<1>(TileShape_MNK{}); + + if constexpr (Cross) { + auto mask_fn = [&](auto& tSrS, int m_block) { + mask.template apply< + true /*Seqlenq_mask*/, + true /*Seqlenk_mask*/, + Causal, + false /*Local*/, + false /*Contexual_mask*/, + false /*Target_mask*/, + Cross, + false /*Softmax*/>(tSrS, m_block, n_block); + }; + CUTLASS_PRAGMA_NO_UNROLL + for (; m_block < m_block_max; ++m_block) { + bwd_step(m_block, mask_fn); + } + if constexpr (Q_dO_same_stages) { + smem_pipe_read_do = smem_pipe_read; + } + ++work_idx; + return true; + } + if constexpr (Has_targets) { + if (n_block * kBlockN >= seqlen_info.uihlen_q) { + auto mask_fn = [&](auto& tSrS, int m_block) { + mask.template apply< + true /*Seqlenq_mask*/, + true /*Seqlenk_mask*/, + false /*Causal*/, + false /*Local*/, + false /*Contexual_mask*/, + Has_targets /*Target_mask*/, + false /*Cross*/, + false /*Softmax*/>(tSrS, m_block, n_block); + }; + CUTLASS_PRAGMA_NO_UNROLL + for (; m_block < m_block_max; ++m_block) { + bwd_step(m_block, mask_fn); + } + if constexpr (Q_dO_same_stages) { + smem_pipe_read_do = smem_pipe_read; + } + ++work_idx; + return true; + } else if ((n_block + 1) * kBlockN >= seqlen_info.uihlen_q) { + if constexpr ((Causal || Local) && SeparateMaskingIterations) { + auto mask_fn = [&](auto& tSrS, int m_block) { + mask.template apply< + true /*Seqlenq_mask*/, + true /*Seqlenk_mask*/, + Causal, + Local, + Contexual_mask, + Has_targets /*Target_mask*/, + false /*Cross*/, + false /*Softmax*/>(tSrS, m_block, n_block); + }; + int const m_block_masking_max = + ((n_block + 1) * kBlockN - 1) / kBlockM + 1; + CUTLASS_PRAGMA_NO_UNROLL + for (; m_block < std::min(m_block_max, m_block_masking_max); + ++m_block) { + bwd_step(m_block, mask_fn); + } + } + + auto mask_fn = [&](auto& tSrS, int m_block) { + mask.template apply< + true /*Seqlenq_mask*/, + true /*Seqlenk_mask*/, + Causal && !SeparateMaskingIterations, + Local && !SeparateMaskingIterations, + Contexual_mask, + Has_targets /*Target_mask*/, + false /*Cross*/, + false /*Softmax*/>(tSrS, m_block, n_block); + }; + if constexpr (SeparateMaskingIterations) { + int const m_block_max_before_local_mask = + !Local || !SeparateMaskingIterations + ? m_block_max + : std::min( + m_block_max, (n_block * kBlockN + max_attn_len_) / kBlockM); + CUTLASS_PRAGMA_NO_UNROLL + for (; m_block < m_block_max_before_local_mask; ++m_block) { + bwd_step(m_block, mask_fn); + } + } else { + int num_m_block = m_block_max - m_block_min; + CUTLASS_PRAGMA_NO_UNROLL + for (int i = 0; i < num_m_block + full_m_block_max - + full_m_block_min + contexual_m_block_max; + ++i) { + if (i < num_m_block) { + m_block = m_block_min + i; + } else if (i < num_m_block + contexual_m_block_max) { + m_block = i - num_m_block; + } else { + m_block = + i - num_m_block - contexual_m_block_max + full_m_block_min; + } + bwd_step(m_block, mask_fn); + } + } + + if constexpr (Local && SeparateMaskingIterations) { + auto mask_fn = [&](auto& tSrS, int m_block) { + mask.template apply< + true /*Seqlenq_mask*/, + true /*Seqlenk_mask*/, + false /*Causal_mask*/, + Local, + Contexual_mask, + Has_targets /*Target_mask*/, + false /*Cross*/, + false /*Softmax*/>(tSrS, m_block, n_block); + }; + CUTLASS_PRAGMA_NO_UNROLL + for (; m_block < m_block_max; ++m_block) { + bwd_step(m_block, mask_fn); + } + } + if constexpr (Contexual_mask && SeparateMaskingIterations) { + auto mask_fn = [&](auto& tSrS, int m_block) { + mask.template apply< + true /*Seqlenq_mask*/, + true /*Seqlenk_mask*/, + Causal /*Causal_mask*/, + Local /*Local_mask*/, + Contexual_mask, + Has_targets, + false /*Cross*/, + false /*Softmax*/>(tSrS, m_block, n_block); + }; + CUTLASS_PRAGMA_NO_UNROLL + for (m_block = 0; m_block < contexual_m_block_max; ++m_block) { + bwd_step(m_block, mask_fn); + } + } + + if constexpr (Local && SeparateMaskingIterations) { + auto mask_fn = [&](auto& tSrS, int m_block) { + mask.template apply< + true /*Seqlenq_mask*/, + true /*Seqlenk_mask*/, + false /*Causal_mask*/, + Local, + Contexual_mask, + Has_targets, + false /*Cross*/, + false /*Softmax*/>(tSrS, m_block, n_block); + }; + CUTLASS_PRAGMA_NO_UNROLL + for (m_block = full_m_block_min; m_block < full_m_block_max; + ++m_block) { + bwd_step(m_block, mask_fn); + } + } + if constexpr (Q_dO_same_stages) { + smem_pipe_read_do = smem_pipe_read; + } + ++work_idx; + return true; + } + } + // We have separate iterations with causal masking. Not necessary for hdim + // 128 but for hdim 64 this helps quite a bit to not have to do causal + // masking for most of the iterations. + if constexpr ((Causal || Local) && SeparateMaskingIterations) { + auto mask_fn = [&](auto& tSrS, int m_block) { + mask.template apply< + true /*Seqlenq_mask*/, + true /*Seqlenk_mask*/, + Causal, + Local, + Contexual_mask, + false /*Target_mask*/, + false /*Cross*/, + false /*Softmax*/>(tSrS, m_block, n_block); + }; + static constexpr int kBlockM = get<0>(TileShape_MNK{}); + int const m_block_masking_max = + ((n_block + 1) * kBlockN - 1) / kBlockM + 1; + CUTLASS_PRAGMA_NO_UNROLL + for (; m_block < std::min(m_block_max, m_block_masking_max); ++m_block) { + bwd_step(m_block, mask_fn); + } + } + + auto mask_fn = [&](auto& tSrS, int m_block) { + mask.template apply< + true /*Seqlenq_mask*/, + true /*Seqlenk_mask*/, + Causal && !SeparateMaskingIterations, + Local && !SeparateMaskingIterations, + Contexual_mask, + false /*Target_mask*/, + false /*Cross*/, + false /*Softmax*/>(tSrS, m_block, n_block); + }; + if constexpr (SeparateMaskingIterations) { + int const m_block_max_before_local_mask = + !Local || !SeparateMaskingIterations + ? m_block_max + : std::min( + m_block_max, (n_block * kBlockN + max_attn_len_) / kBlockM); + CUTLASS_PRAGMA_NO_UNROLL + for (; m_block < m_block_max_before_local_mask; ++m_block) { + bwd_step(m_block, mask_fn); + } + } else { + int num_m_block = m_block_max - m_block_min; + CUTLASS_PRAGMA_NO_UNROLL + for (int i = 0; i < num_m_block + full_m_block_max - full_m_block_min + + contexual_m_block_max; + ++i) { + if (i < num_m_block) { + m_block = m_block_min + i; + } else if (i < num_m_block + contexual_m_block_max) { + m_block = i - num_m_block; + } else { + m_block = i - num_m_block - contexual_m_block_max + full_m_block_min; + } + bwd_step(m_block, mask_fn); + } + } + + if constexpr (Local && SeparateMaskingIterations) { + auto mask_fn = [&](auto& tSrS, int m_block) { + mask.template apply< + true /*Seqlenq_mask*/, + true /*Seqlenk_mask*/, + false /*Causal_mask*/, + Local, + Contexual_mask, + false /*Target_mask*/, + false /*Cross*/, + false /*Softmax*/>(tSrS, m_block, n_block); + }; + CUTLASS_PRAGMA_NO_UNROLL + for (; m_block < m_block_max; ++m_block) { + bwd_step(m_block, mask_fn); + } + } + if constexpr (Contexual_mask && SeparateMaskingIterations) { + auto mask_fn = [&](auto& tSrS, int m_block) { + mask.template apply< + true /*Seqlenq_mask*/, + true /*Seqlenk_mask*/, + Causal /*Causal_mask*/, + Local /*Local_mask*/, + Contexual_mask, + false /*Target_mask*/, + false /*Cross*/, + false /*Softmax*/>(tSrS, m_block, n_block); + }; + CUTLASS_PRAGMA_NO_UNROLL + for (m_block = 0; m_block < contexual_m_block_max; ++m_block) { + bwd_step(m_block, mask_fn); + } + } + + if constexpr (Local && SeparateMaskingIterations) { + auto mask_fn = [&](auto& tSrS, int m_block) { + mask.template apply< + true /*Seqlenq_mask*/, + true /*Seqlenk_mask*/, + false /*Causal_mask*/, + Local, + Contexual_mask, + false /*Target_mask*/, + false /*Cross*/, + false /*Softmax*/>(tSrS, m_block, n_block); + }; + CUTLASS_PRAGMA_NO_UNROLL + for (m_block = full_m_block_min; m_block < full_m_block_max; ++m_block) { + bwd_step(m_block, mask_fn); + } + } + + // if (blockIdx.x == 0 && threadIdx.x == 128) { print_tensor(tdVrdV); } + if constexpr (Q_dO_same_stages) { + smem_pipe_read_do = smem_pipe_read; + } + ++work_idx; + return true; + } + + template + CUTLASS_DEVICE bool mma_softmax( + Params const& params, + MainloopPipeline pipeline_q, + MainloopPipeline_dO pipeline_do, + PipelineState& smem_pipe_read, + PipelineState_dO& smem_pipe_read_do, + FrgTensordKV& tdKrdK, + FrgTensordKV& tdVrdV, + int thread_idx, + int& work_idx, + cute::tuple block_coord, + SharedStorage& shared_storage) { + static_assert( + is_rmem::value, + "dK and dV tensor must be rmem resident."); + + int n_block = get<0>(block_coord); + int bidb = get<2>(block_coord); + SeqlenInfo_t seqlen_info{ + bidb, + get<0>(params.shape_Q), + get<0>(params.shape_K), + params.seq_offsets, + params.seq_offsets_q, + params.num_targets}; + if constexpr (Jagged) { + static constexpr int kBlockN = get<1>(TileShape_MNK{}); + if (n_block * kBlockN >= seqlen_info.seqlen_kv) { + return false; + } + } + int min_full_attn_seq_len_, max_attn_len_, contextual_seq_len_; + if constexpr (!Cross) { + if (params.num_groups > 1) { + int group_id = bidb / params.num_groups; + min_full_attn_seq_len_ = params.min_full_attn_seq_len_tensor[group_id]; + max_attn_len_ = params.max_attn_len_tensor[group_id]; + contextual_seq_len_ = params.contextual_seq_len_tensor[group_id]; + } else { + min_full_attn_seq_len_ = params.min_full_attn_seq_len; + max_attn_len_ = params.max_attn_len; + contextual_seq_len_ = params.contextual_seq_len; + } + } + int m_block_min, m_block_max; + if constexpr (Cross) { + auto m_block_min_max = get_cross_m_block_min_max( + seqlen_info.uihlen_q, + seqlen_info.seqlen_q, + seqlen_info.seqlen_kv, + n_block); + m_block_min = get<0>(m_block_min_max); + m_block_max = get<1>(m_block_min_max); + } else { + auto m_block_min_max = get_m_block_min_max( + max_attn_len_, + contextual_seq_len_, + seqlen_info.uihlen_q, + seqlen_info.seqlen_q, + n_block); + m_block_min = get<0>(m_block_min_max); + m_block_max = get<1>(m_block_min_max); + } + auto full_m_block_min_max = get_full_m_block_min_max( + seqlen_info.uihlen_q, + seqlen_info.seqlen_q, + min_full_attn_seq_len_, + m_block_max, + n_block); + int const full_m_block_min = get<0>(full_m_block_min_max); + int const full_m_block_max = get<1>(full_m_block_min_max); + int contexual_m_block_max = get_contexual_m_block_max( + seqlen_info.uihlen_q, contextual_seq_len_, m_block_min, n_block); + + Tensor sQ = make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_q.data()), + SmemLayoutQ{}); + Tensor sdO = make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_do.data()), + SmemLayoutdO{}); + Tensor sK = make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_k.data()), + SmemLayoutK{}); + Tensor sV = make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_v.data()), + SmemLayoutV{}); + Tensor sQt = make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_q.data()), + SmemLayoutQt{}); + Tensor sdOt = make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_do.data()), + SmemLayoutdOt{}); + Tensor sKt = make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_k.data()), + SmemLayoutKt{}); + Tensor sP = make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_p.data()), + SmemLayoutPdS{}); + Tensor sP_pi = cute::as_position_independent_swizzle_tensor(sP); + Tensor sPt = make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_p.data()), + SmemLayoutPdSt{}); + Tensor sPt_pi = cute::as_position_independent_swizzle_tensor(sPt); + Tensor sdS = make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_ds.data()), + SmemLayoutPdS{}); + Tensor sdS_pi = cute::as_position_independent_swizzle_tensor(sdS); + Tensor sdSt = make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_ds.data()), + SmemLayoutPdSt{}); + Tensor sdSt_pi = cute::as_position_independent_swizzle_tensor(sdSt); + Tensor sdQ = make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_dqacc.data()), + SmemLayoutdQaccum{}); + Tensor sLSEMma = make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_lse.data()), + SmemLayoutLSEMma{}); + Tensor sdPsumMma = make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_dpsum.data()), + SmemLayoutLSEMma{}); + + static_assert( + stride<0>(typename TiledMmaSdP::ALayout{}) == 0 and + stride<0>(typename TiledMmaSdP::BLayout{}) == 0 and + size<0>(typename TiledMmaSdP::ALayout{}) == + cutlass::NumThreadsPerWarpGroup and + size<0>(typename TiledMmaSdP::BLayout{}) == + cutlass::NumThreadsPerWarpGroup, + "Stride of the first mode must be 0 and the size of the mode must be NumThreadsPerWarpGroup"); + constexpr int MmaWarpGroups = + NumMmaThreads / cutlass::NumThreadsPerWarpGroup; + Layout warp_group_thread_layout = make_layout( + make_shape(Int{}), + make_stride(Int{})); + Layout warp_group_thread_layout_dq = make_layout( + make_shape(Int{}), + make_stride(Int{})); + + int warp_group_idx = __shfl_sync( + 0xFFFFFFFF, thread_idx / cutlass::NumThreadsPerWarpGroup, 0); + TiledMmaSdP tiled_mma_SdP; + using TiledMmadP = + std::conditional_t; + TiledMmadP tiled_mma_dP; + TiledMmadKV tiled_mma_dKV; + TiledMmadQ tiled_mma_dQ; + + auto wg_mma_SdP = + tiled_mma_SdP.get_slice(warp_group_thread_layout(warp_group_idx)); + auto wg_mma_dP = + tiled_mma_dP.get_slice(warp_group_thread_layout(warp_group_idx)); + auto thread_mma_SdP = tiled_mma_SdP.get_thread_slice(thread_idx); + auto wg_mma_dKV = + tiled_mma_dKV.get_slice(warp_group_thread_layout(warp_group_idx)); + auto wg_mma_dQ = + tiled_mma_dQ.get_slice(warp_group_thread_layout_dq(warp_group_idx)); + + auto smem_tiled_copy_PdS = + make_tiled_copy_C(SmemCopyAtomPdS{}, tiled_mma_SdP); + auto smem_thr_copy_PdS = smem_tiled_copy_PdS.get_thread_slice(thread_idx); + + R2STiledCopydQaccum r2s_tiled_copy_dQaccum; + auto r2s_thr_copy_dQaccum = + r2s_tiled_copy_dQaccum.get_thread_slice(thread_idx); + Tensor tdQsdQaccum = r2s_thr_copy_dQaccum.partition_D(sdQ); + // if (thread_idx == 0) { print(sdQ); printf("\n"); print(tdQsdQaccum); + // printf("\n"); } + + // Allocate "fragments/descriptors" + // We have to use the templated mma_partition_fragment_AB instead of + // cute::conditional_return or lambda, because some partition_fragment_A/B + // don't compile. + // https://stackoverflow.com/questions/50051473/if-constexpr-in-c17-does-not-work-in-a-non-templated-function + Tensor tSrQ = mma_partition_fragment_AB(wg_mma_SdP, sQ); + Tensor tSrK = mma_partition_fragment_AB(wg_mma_SdP, sK); + Tensor tdPrdO = + mma_partition_fragment_AB(wg_mma_SdP, sdO); + Tensor tdPrV = mma_partition_fragment_AB(wg_mma_dP, sV); + Tensor tdVrdO = + mma_partition_fragment_AB(wg_mma_dKV, sdOt); + Tensor tdKrQ = mma_partition_fragment_AB(wg_mma_dKV, sQt); + Tensor tdQrdS = mma_partition_fragment_AB(wg_mma_dQ, sdS); + Tensor tdQrK = mma_partition_fragment_AB(wg_mma_dQ, sKt); + + Tensor tPsP = smem_thr_copy_PdS.partition_D( + cute::conditional_return( + sP_pi, sPt_pi)); // ((Atom,AtomNum),PIPE_M,PIPE_N) + Tensor tdSsdS = smem_thr_copy_PdS.partition_D( + cute::conditional_return( + sdS_pi, sdSt_pi)); // ((Atom,AtomNum),PIPE_M,PIPE_N) + // if (blockIdx.x == 0 && threadIdx.x == 128) { print(smem_thr_copy_PdS); + // print(sP_pi); printf("\n"); print(sPt_pi); printf("\n"); print(tPsP); + // printf("\n"); print(tdSsdS); printf("\n"); } + + // thread_mma_SdP.partition_C(sLSEMma) has shape ((2, 2, V), MMA_M, MMA_N, + // PIPE), we only take the col indices or row indices, depending on whether + // SdP_swapAB. + Tensor tLSEsLSE = cute::conditional_return( + group_modes<0, 2>(thread_mma_SdP.partition_C(sLSEMma)( + make_coord(_0{}, _, _0{}), _, _0{}, _)), // (2, MMA_M, PIPE) + group_modes<0, 3>(thread_mma_SdP.partition_C(sLSEMma)( + make_coord(_, _0{}, _), _0{}, _, _))); // (2, V, MMA_N, PIPE) + Tensor tLSEsdPsum = cute::conditional_return( + group_modes<0, 2>(thread_mma_SdP.partition_C(sdPsumMma)( + make_coord(_0{}, _, _0{}), _, _0{}, _)), + group_modes<0, 3>(thread_mma_SdP.partition_C(sdPsumMma)( + make_coord(_, _0{}, _), _0{}, _, _))); + // if (blockIdx.x == 0 && threadIdx.x == 128) { print(sLSEMma); + // printf("\n"); print(tLSEsLSE); printf("\n"); } If we want to split the + // stats among the 8 threads that share the same rows. + static constexpr int kStatsPerThread = + cute::ceil_div(decltype(size(tLSEsLSE))::value, 8); + + auto consumer_wait = [](auto& pipeline, auto& smem_pipe_read) { + auto barrier_token = pipeline.consumer_try_wait(smem_pipe_read); + pipeline.consumer_wait(smem_pipe_read, barrier_token); + }; + + int bidh = get<1>(block_coord); + // For the case where we do atomicAdd directly to gdQaccum instead of using + // TMA + Tensor mdQaccum = make_tensor( + make_gmem_ptr(reinterpret_cast(params.ptr_dQaccum)), + params.shape_dQaccum, + params.stride_dQaccum)(_, bidh, !Jagged ? bidb : 0); + Tensor gdQaccum_ = local_tile( + domain_offset( + make_coord(seqlen_info.offset_q_padded * kHeadDim), mdQaccum), + Shape>{}, + make_coord(_)); // (M * K, _) + Tensor gdQaccum = cute::flat_divide( + gdQaccum_, + Int{}); // (M * K / WG, WG, _) + // We can reuse r2s_thr_copy_dQaccum for this partitioning + Tensor tdQgdQaccum = r2s_thr_copy_dQaccum.partition_D(gdQaccum); + // if (blockIdx.x == 0 && threadIdx.x == 128) { print(mdQaccum); + // printf("\n"); print(gdQaccum_); printf("\n"); print(gdQaccum); + // printf("\n"); print(tdQgdQaccum); printf("\n"); } + + hstu::Mask mask( + thread_idx, + seqlen_info.seqlen_q, + seqlen_info.seqlen_kv, + max_attn_len_, + min_full_attn_seq_len_, + contextual_seq_len_, + seqlen_info.uihlen_q); + + int m_block = m_block_min; + + clear(tdKrdK); + clear(tdVrdV); + // tiled_mma_dKV.accumulate_ = GMMA::ScaleOut::Zero; + + cutlass::ConsumerToken barrier_token = static_cast( + shared_storage.pipelines.barrier_KV.try_wait(work_idx % 2)); + if (barrier_token == cutlass::BarrierStatus::WaitAgain) { + shared_storage.pipelines.barrier_KV.wait(work_idx % 2); + } + + if constexpr (Mma_dP_is_RS) { + using SmemCopyAtomV = Copy_Atom; + auto smem_tiled_copy_V = make_tiled_copy_A(SmemCopyAtomV{}, tiled_mma_dP); + auto smem_thr_copy_V = smem_tiled_copy_V.get_thread_slice(thread_idx); + Tensor tdPrV_copy_view = smem_thr_copy_V.retile_D(tdPrV); + Tensor tdPsV_copy_view = smem_thr_copy_V.partition_S( + cute::as_position_independent_swizzle_tensor(sV)); + cute::copy(smem_tiled_copy_V, tdPsV_copy_view, tdPrV_copy_view); + } + static constexpr int Qdim = !SdP_swapAB ? 0 : 1; + auto thread0_mma_SdP = tiled_mma_SdP.get_thread_slice(_0{}); + Tensor cS = cute::make_identity_tensor( + Shape< + Int, + Int>{}); + Tensor tScS = thread_mma_SdP.partition_C(cS); + Tensor tScS_rowcol = make_tensor( + tScS.data(), + hstu::convert_layout_acc_rowcol( + tScS.layout())); + Tensor t0ScS = thread0_mma_SdP.partition_C(cS); + Tensor t0ScS_rowcol = make_tensor( + t0ScS.data(), + hstu::convert_layout_acc_rowcol( + t0ScS.layout())); + int const thread_qdim_offset = get(tScS_rowcol(_0{}, _0{})); + + auto bwd_step = [&](int m_block, auto mask_fn) { + Tensor tSrS = partition_fragment_C( + tiled_mma_SdP, + select(TileShape_MNK{})); + consumer_wait(pipeline_q, smem_pipe_read); + hstu::gemm( + tiled_mma_SdP, tSrQ(_, _, _, smem_pipe_read.index()), tSrK, tSrS); + Tensor tLSErLSE = cute::conditional_return( + make_fragment_like(tLSEsLSE(_, _0{})), + make_tensor(Int{})); + if constexpr (!ShuffleLSE) { + cute::copy(tLSEsLSE(_, smem_pipe_read.index()), tLSErLSE); + } else { +#pragma unroll + for (int i = 0; i < kStatsPerThread; ++i) { + // It's ok to read OOB, since we made sure sLSE is large enough and we + // won't use the OOB values + tLSErLSE(i) = + tLSEsLSE((thread_idx % 32) / 4 + i * 8, smem_pipe_read.index()); + } + } + Tensor tdPrdP = partition_fragment_C( + tiled_mma_SdP, + select(TileShape_MNK{})); + PipelineState_dO smem_pipe_read_do_cur = + cute::conditional_return( + smem_pipe_read, smem_pipe_read_do); + consumer_wait(pipeline_do, smem_pipe_read_do_cur); + hstu::gemm( + tiled_mma_dP, + tdPrdO(_, _, _, smem_pipe_read_do_cur.index()), + tdPrV, + tdPrdP); + warpgroup_wait<1>(); + // Reshape tSrS from ((2, 2, V), MMA_N, MMA_M) to (nrow=(2, V, MMA_M), + // ncol=(2, MMA_N)) + Tensor scores = make_tensor( + tSrS.data(), + hstu::convert_layout_acc_rowcol( + tSrS.layout())); + mask_fn(tSrS, m_block); +#pragma unroll + for (int mi = 0; mi < size<0>(scores); ++mi) { + float const lse_scaled = [&] { + if constexpr (!ShuffleLSE) + return tLSErLSE(mi); + else + return __shfl_sync( + 0xffffffff, tLSErLSE(mi / 8), (mi % 8) * 4 + (thread_idx % 4)); + }(); +#pragma unroll + for (int ni = 0; ni < size<1>(scores); ++ni) { + scores(mi, ni) = + exp2f(scores(mi, ni) * params.alpha_log2 - lse_scaled); + } + } + Tensor tLSErdPsum = cute::conditional_return( + make_fragment_like(tLSEsdPsum(_, _0{})), + make_tensor(Int{})); + if constexpr (!ShuffledPsum) { + cute::copy(tLSEsdPsum(_, smem_pipe_read_do_cur.index()), tLSErdPsum); + } else { +#pragma unroll + for (int i = 0; i < kStatsPerThread; ++i) { + tLSErdPsum(i) = tLSEsdPsum( + (thread_idx % 32) / 4 + i * 8, smem_pipe_read_do_cur.index()); + } + } + + warpgroup_wait<0>(); + // Reshape tdPrdP from ((2, 2, V), MMA_N, MMA_M) to (nrow=(2, V, MMA_M), + // ncol=(2, MMA_N)) + Tensor dS = make_tensor(tdPrdP.data(), scores.layout()); +#pragma unroll + for (int mi = 0; mi < size<0>(dS); ++mi) { + float const dP_sum_cur = [&] { + if constexpr (!ShuffledPsum) + return tLSErdPsum(mi); + else + return __shfl_sync( + 0xffffffff, + tLSErdPsum(mi / 8), + (mi % 8) * 4 + (thread_idx % 4)); + }(); +#pragma unroll + for (int ni = 0; ni < size<1>(dS); ++ni) { + dS(mi, ni) = + scores(mi, ni) * (dS(mi, ni) - dP_sum_cur) * params.alpha; + } + } + // Convert scores from fp32 to fp16/bf16 + Tensor rP = make_tensor_like(tSrS); + hstu::convert_type_out(tSrS, rP); + if constexpr (!Mma_dKV_is_RS) { + // Need to sync to make sure P has already been used in the previous + // iteration before writing new values + if constexpr (kStages_dS == 1) { + cutlass::arch::NamedBarrier::sync( + NumMmaThreads, + static_cast(BwdNamedBarriers::PdS) /*id*/); + } + Tensor tPaP = + smem_thr_copy_PdS.retile_S(rP); // ((Atom,AtomNum), MMA_N, MMA_N) + cute::copy( + smem_tiled_copy_PdS, + tPaP, + tPsP( + _, + _, + _, + cute::conditional_return( + _0{}, smem_pipe_read.index()))); + } + Tensor rdS = make_tensor_like(tdPrdP); + hstu::convert_type_out(tdPrdP, rdS); + // If there's double buffering on dS, we don't need to sync here. + // Otherwise we might have WG1 writing to dS before WG2 is done reading + // from it during MmadQ. But because both WGs have to sync at the end of + // the loop and double buffering, this race condition is not possible. + // This sync is to ensure (1) P is written in case of !Mma_dKV_is_RS and + // (2) dS is already read by the Mma in the previous iteration in case of + // Mma_dKV_is_RS. + if constexpr (!Mma_dKV_is_RS || (kStages_dS == 1 && Mma_dKV_is_RS)) { + cutlass::arch::fence_view_async_shared(); + cutlass::arch::NamedBarrier::sync( + NumMmaThreads, static_cast(BwdNamedBarriers::PdS) /*id*/); + } + // For hdim 64, It's faster to write to smem_dS first before the dV gemm + Tensor tdSadS = + smem_thr_copy_PdS.retile_S(rdS); // ((Atom,AtomNum), MMA_N, MMA_N) + cute::copy( + smem_tiled_copy_PdS, + tdSadS, + tdSsdS( + _, + _, + _, + cute::conditional_return( + _0{}, smem_pipe_read.index()))); + + if constexpr (!Slice_dQKV_Mma) { + // Most cases take this path, except for hdim256 where we want to slice + // to reduce register pressure + if constexpr (Mma_dKV_is_RS) { + Tensor tdVrP = make_tensor( + rP.data(), convert_layout_acc_Aregs(tSrS.layout())); + hstu::gemm( + tiled_mma_dKV, + tdVrP, + tdVrdO(_, _, _, smem_pipe_read_do_cur.index()), + tdVrdV); + } else { + Tensor tdVrP = + mma_partition_fragment_AB(wg_mma_dKV, sPt); + Tensor tdVrP_cur = tdVrP( + _, + _, + _, + cute::conditional_return( + _0{}, smem_pipe_read.index())); + hstu:: + gemm( + tiled_mma_dKV, + tdVrP_cur, + tdVrdO(_, _, _, smem_pipe_read_do_cur.index()), + tdVrdV); + } + // SMEM fence to make sure sdS is written before it's read by WGMMA + cutlass::arch::fence_view_async_shared(); + cutlass::arch::NamedBarrier::sync( + NumMmaThreads, static_cast(BwdNamedBarriers::PdS) /*id*/); + Tensor tdQrdQ = partition_fragment_C( + tiled_mma_dQ, + select(TileShape_MNK{})); + Tensor tdQrdS_cur = tdQrdS( + _, + _, + _, + cute::conditional_return( + _0{}, smem_pipe_read.index())); + hstu::gemm( + tiled_mma_dQ, tdQrdS_cur, tdQrK, tdQrdQ); + pipeline_do.consumer_release(smem_pipe_read_do_cur); // release dQ + + if constexpr (Mma_dKV_is_RS) { + Tensor tdKrdS = make_tensor( + rdS.data(), + convert_layout_acc_Aregs(tdPrdP.layout())); + hstu::gemm( + tiled_mma_dKV, + tdKrdS, + tdKrQ(_, _, _, smem_pipe_read.index()), + tdKrdK); + } else { + Tensor tdKrdS = + mma_partition_fragment_AB(wg_mma_dKV, sdSt); + Tensor tdKrdS_cur = tdKrdS( + _, + _, + _, + cute::conditional_return( + _0{}, smem_pipe_read.index())); + hstu::gemm( + tiled_mma_dKV, + tdKrdS_cur, + tdKrQ(_, _, _, smem_pipe_read.index()), + tdKrdK); + } + if constexpr (dQacc_use_TMA) { + int const warp_group_idx = + hstu::canonical_warp_group_idx_nosync() - 1; + cutlass::arch::NamedBarrier::sync( + cutlass::NumThreadsPerWarpGroup + cutlass::NumThreadsPerWarp, + static_cast(BwdNamedBarriers::dQEmptyWG1) + + warp_group_idx /*id*/); // sdQ full, to be written to gmem + Tensor taccdQrdQ = r2s_thr_copy_dQaccum.retile_S(tdQrdQ); + cute::copy(r2s_tiled_copy_dQaccum, taccdQrdQ, tdQsdQaccum); + cutlass::arch::fence_view_async_shared(); + cutlass::arch::NamedBarrier::arrive( + cutlass::NumThreadsPerWarpGroup + cutlass::NumThreadsPerWarp, + static_cast(BwdNamedBarriers::dQFullWG1) + + warp_group_idx /*id*/); // sdQ full, to be written to gmem + } else { + // We can reuse r2s_thr_copy_dQaccum for this partitioning + Tensor tdQrdQ_atomic = + recast(r2s_thr_copy_dQaccum.retile_S(tdQrdQ)); + Tensor tdQgdQaccum_atomic = + recast(tdQgdQaccum(_, _, _, m_block)); + static_assert( + CUTE_STATIC_V(size(tdQrdQ_atomic)) == + CUTE_STATIC_V(size(tdQgdQaccum_atomic))); +#pragma unroll + for (int i = 0; i < size(tdQrdQ_atomic); ++i) { + atomicAdd(&tdQgdQaccum_atomic(i), tdQrdQ_atomic(i)); + } + } + + } else { // Slice_dQKV_Mma + + static_assert(!(Slice_dQKV_Mma && Mma_dKV_is_RS)); + Tensor tdVrP = + mma_partition_fragment_AB(wg_mma_dKV, sPt); + Tensor tdVrP_cur = tdVrP( + _, + _, + _, + cute::conditional_return( + _0{}, smem_pipe_read.index())); + hstu::gemm< + /*zero_init=*/false, + /*wg_wait=*/-1, + /*SwapAB=*/dKV_swapAB, + /*M_slice=*/0>( + tiled_mma_dKV, + tdVrP_cur, + tdVrdO(_, _, _, smem_pipe_read_do_cur.index()), + tdVrdV); + + cutlass::arch::fence_view_async_shared(); + cutlass::arch::NamedBarrier::sync( + NumMmaThreads, static_cast(BwdNamedBarriers::PdS) /*id*/); + Tensor tdQrdQ = partition_fragment_C( + tiled_mma_dQ, + select(TileShape_MNK{})); + Tensor tdQrdS_cur = tdQrdS( + _, + _, + _, + cute::conditional_return( + _0{}, smem_pipe_read.index())); + hstu::gemm< + /*zero_init=*/true, + /*wg_wait=*/-1, + /*SwapAB=*/dQ_swapAB, + /*M_slice=*/0>(tiled_mma_dQ, tdQrdS_cur, tdQrK, tdQrdQ); + hstu::gemm< + /*zero_init=*/false, + /*wg_wait=*/1, + /*SwapAB=*/dKV_swapAB, + /*M_slice=*/1>( + tiled_mma_dKV, + tdVrP_cur, + tdVrdO(_, _, _, smem_pipe_read_do_cur.index()), + tdVrdV); + Tensor tdQrdQ_atomic = + recast(r2s_thr_copy_dQaccum.retile_S(tdQrdQ)); + Tensor tdQgdQaccum_atomic = + recast(tdQgdQaccum(_, _, _, m_block)); +#pragma unroll + for (int i = 0; i < size(tdQrdQ_atomic) / 2; ++i) { + atomicAdd(&tdQgdQaccum_atomic(i), tdQrdQ_atomic(i)); + } + + Tensor tdKrdS = + mma_partition_fragment_AB(wg_mma_dKV, sdSt); + Tensor tdKrdS_cur = tdKrdS( + _, + _, + _, + cute::conditional_return( + _0{}, smem_pipe_read.index())); + hstu::gemm< + /*zero_init=*/false, + /*wg_wait=*/1, + /*SwapAB=*/dKV_swapAB, + /*M_slice=*/0>( + tiled_mma_dKV, + tdKrdS_cur, + tdKrQ(_, _, _, smem_pipe_read.index()), + tdKrdK); + pipeline_do.consumer_release(smem_pipe_read_do_cur); // release dO + + hstu::gemm< + /*zero_init=*/true, + /*wg_wait=*/0, + /*SwapAB=*/dQ_swapAB, + /*M_slice=*/1>(tiled_mma_dQ, tdQrdS_cur, tdQrK, tdQrdQ); +#pragma unroll + for (int i = size(tdQrdQ_atomic) / 2; i < size(tdQrdQ_atomic); ++i) { + atomicAdd(&tdQgdQaccum_atomic(i), tdQrdQ_atomic(i)); + } + + hstu::gemm< + /*zero_init=*/false, + /*wg_wait=*/-1, + /*SwapAB=*/dKV_swapAB, + /*M_slice=*/1>( + tiled_mma_dKV, + tdKrdS_cur, + tdKrQ(_, _, _, smem_pipe_read.index()), + tdKrdK); + } + + warpgroup_wait<0>(); + pipeline_q.consumer_release(smem_pipe_read); // release Q + ++smem_pipe_read; + if constexpr (!Q_dO_same_stages) { + ++smem_pipe_read_do; + } + }; + static constexpr int kBlockM = get<0>(TileShape_MNK{}); + static constexpr int kBlockN = get<1>(TileShape_MNK{}); + + if constexpr (Cross) { + auto mask_fn = [&](auto& tSrS, int m_block) { + mask.template apply< + true /*Seqlenq_mask*/, + true /*Seqlenk_mask*/, + Causal, + false /*Local*/, + false /*Contexual_mask*/, + false /*Target_mask*/, + Cross, + true /*Softmax*/>(tSrS, m_block, n_block); + }; + CUTLASS_PRAGMA_NO_UNROLL + for (; m_block < m_block_max; ++m_block) { + bwd_step(m_block, mask_fn); + } + if constexpr (Q_dO_same_stages) { + smem_pipe_read_do = smem_pipe_read; + } + ++work_idx; + return true; + } + if constexpr (Has_targets) { + if (n_block * kBlockN >= seqlen_info.uihlen_q) { + auto mask_fn = [&](auto& tSrS, int m_block) { + mask.template apply< + true /*Seqlenq_mask*/, + true /*Seqlenk_mask*/, + false /*Causal*/, + false /*Local*/, + false /*Contexual_mask*/, + Has_targets /*Target_mask*/, + false /*Cross*/, + true /*Softmax*/>(tSrS, m_block, n_block); + }; + CUTLASS_PRAGMA_NO_UNROLL + for (; m_block < m_block_max; ++m_block) { + bwd_step(m_block, mask_fn); + } + if constexpr (Q_dO_same_stages) { + smem_pipe_read_do = smem_pipe_read; + } + ++work_idx; + return true; + } else if ((n_block + 1) * kBlockN >= seqlen_info.uihlen_q) { + if constexpr ((Causal || Local) && SeparateMaskingIterations) { + auto mask_fn = [&](auto& tSrS, int m_block) { + mask.template apply< + true /*Seqlenq_mask*/, + true /*Seqlenk_mask*/, + Causal, + Local, + Contexual_mask, + Has_targets /*Target_mask*/, + false /*Cross*/, + true /*Softmax*/>(tSrS, m_block, n_block); + }; + int const m_block_masking_max = + ((n_block + 1) * kBlockN - 1) / kBlockM + 1; + CUTLASS_PRAGMA_NO_UNROLL + for (; m_block < std::min(m_block_max, m_block_masking_max); + ++m_block) { + bwd_step(m_block, mask_fn); + } + } + + auto mask_fn = [&](auto& tSrS, int m_block) { + mask.template apply< + true /*Seqlenq_mask*/, + true /*Seqlenk_mask*/, + Causal && !SeparateMaskingIterations, + Local && !SeparateMaskingIterations, + Contexual_mask, + Has_targets /*Target_mask*/, + false /*Cross*/, + true /*Softmax*/>(tSrS, m_block, n_block); + }; + if constexpr (SeparateMaskingIterations) { + int const m_block_max_before_local_mask = + !Local || !SeparateMaskingIterations + ? m_block_max + : std::min( + m_block_max, (n_block * kBlockN + max_attn_len_) / kBlockM); + CUTLASS_PRAGMA_NO_UNROLL + for (; m_block < m_block_max_before_local_mask; ++m_block) { + bwd_step(m_block, mask_fn); + } + } else { + int num_m_block = m_block_max - m_block_min; + CUTLASS_PRAGMA_NO_UNROLL + for (int i = 0; i < num_m_block + full_m_block_max - + full_m_block_min + contexual_m_block_max; + ++i) { + if (i < num_m_block) { + m_block = m_block_min + i; + } else if (i < num_m_block + contexual_m_block_max) { + m_block = i - num_m_block; + } else { + m_block = + i - num_m_block - contexual_m_block_max + full_m_block_min; + } + bwd_step(m_block, mask_fn); + } + } + + if constexpr (Local && SeparateMaskingIterations) { + auto mask_fn = [&](auto& tSrS, int m_block) { + mask.template apply< + true /*Seqlenq_mask*/, + true /*Seqlenk_mask*/, + false /*Causal_mask*/, + Local, + Contexual_mask, + Has_targets /*Target_mask*/, + false /*Cross*/, + true /*Softmax*/>(tSrS, m_block, n_block); + }; + CUTLASS_PRAGMA_NO_UNROLL + for (; m_block < m_block_max; ++m_block) { + bwd_step(m_block, mask_fn); + } + } + if constexpr (Contexual_mask && SeparateMaskingIterations) { + auto mask_fn = [&](auto& tSrS, int m_block) { + mask.template apply< + true /*Seqlenq_mask*/, + true /*Seqlenk_mask*/, + Causal /*Causal_mask*/, + Local /*Local_mask*/, + Contexual_mask, + Has_targets, + false /*Cross*/, + true /*Softmax*/>(tSrS, m_block, n_block); + }; + CUTLASS_PRAGMA_NO_UNROLL + for (m_block = 0; m_block < contexual_m_block_max; ++m_block) { + bwd_step(m_block, mask_fn); + } + } + + if constexpr (Local && SeparateMaskingIterations) { + auto mask_fn = [&](auto& tSrS, int m_block) { + mask.template apply< + true /*Seqlenq_mask*/, + true /*Seqlenk_mask*/, + false /*Causal_mask*/, + Local, + Contexual_mask, + Has_targets, + false /*Cross*/, + true /*Softmax*/>(tSrS, m_block, n_block); + }; + CUTLASS_PRAGMA_NO_UNROLL + for (m_block = full_m_block_min; m_block < full_m_block_max; + ++m_block) { + bwd_step(m_block, mask_fn); + } + } + if constexpr (Q_dO_same_stages) { + smem_pipe_read_do = smem_pipe_read; + } + ++work_idx; + return true; + } + } + // We have separate iterations with causal masking. Not necessary for hdim + // 128 but for hdim 64 this helps quite a bit to not have to do causal + // masking for most of the iterations. + if constexpr ((Causal || Local) && SeparateMaskingIterations) { + auto mask_fn = [&](auto& tSrS, int m_block) { + mask.template apply< + true /*Seqlenq_mask*/, + true /*Seqlenk_mask*/, + Causal, + Local, + Contexual_mask, + false /*Target_mask*/, + false /*Cross*/, + true /*Softmax*/>(tSrS, m_block, n_block); + }; + static constexpr int kBlockM = get<0>(TileShape_MNK{}); + int const m_block_masking_max = + ((n_block + 1) * kBlockN - 1) / kBlockM + 1; + CUTLASS_PRAGMA_NO_UNROLL + for (; m_block < std::min(m_block_max, m_block_masking_max); ++m_block) { + bwd_step(m_block, mask_fn); + } + } + + auto mask_fn = [&](auto& tSrS, int m_block) { + mask.template apply< + true /*Seqlenq_mask*/, + true /*Seqlenk_mask*/, + Causal && !SeparateMaskingIterations, + Local && !SeparateMaskingIterations, + Contexual_mask, + false /*Target_mask*/, + false /*Cross*/, + true /*Softmax*/>(tSrS, m_block, n_block); + }; + if constexpr (SeparateMaskingIterations) { + int const m_block_max_before_local_mask = + !Local || !SeparateMaskingIterations + ? m_block_max + : std::min( + m_block_max, (n_block * kBlockN + max_attn_len_) / kBlockM); + CUTLASS_PRAGMA_NO_UNROLL + for (; m_block < m_block_max_before_local_mask; ++m_block) { + bwd_step(m_block, mask_fn); + } + } else { + int num_m_block = m_block_max - m_block_min; + CUTLASS_PRAGMA_NO_UNROLL + for (int i = 0; i < num_m_block + full_m_block_max - full_m_block_min + + contexual_m_block_max; + ++i) { + if (i < num_m_block) { + m_block = m_block_min + i; + } else if (i < num_m_block + contexual_m_block_max) { + m_block = i - num_m_block; + } else { + m_block = i - num_m_block - contexual_m_block_max + full_m_block_min; + } + bwd_step(m_block, mask_fn); + } + } + + if constexpr (Local && SeparateMaskingIterations) { + auto mask_fn = [&](auto& tSrS, int m_block) { + mask.template apply< + true /*Seqlenq_mask*/, + true /*Seqlenk_mask*/, + false /*Causal_mask*/, + Local, + Contexual_mask, + false /*Target_mask*/, + false /*Cross*/, + true /*Softmax*/>(tSrS, m_block, n_block); + }; + CUTLASS_PRAGMA_NO_UNROLL + for (; m_block < m_block_max; ++m_block) { + bwd_step(m_block, mask_fn); + } + } + if constexpr (Contexual_mask && SeparateMaskingIterations) { + auto mask_fn = [&](auto& tSrS, int m_block) { + mask.template apply< + true /*Seqlenq_mask*/, + true /*Seqlenk_mask*/, + Causal /*Causal_mask*/, + Local /*Local_mask*/, + Contexual_mask, + false /*Target_mask*/, + false /*Cross*/, + true /*Softmax*/>(tSrS, m_block, n_block); + }; + CUTLASS_PRAGMA_NO_UNROLL + for (m_block = 0; m_block < contexual_m_block_max; ++m_block) { + bwd_step(m_block, mask_fn); + } + } + + if constexpr (Local && SeparateMaskingIterations) { + auto mask_fn = [&](auto& tSrS, int m_block) { + mask.template apply< + true /*Seqlenq_mask*/, + true /*Seqlenk_mask*/, + false /*Causal_mask*/, + Local, + Contexual_mask, + false /*Target_mask*/, + false /*Cross*/, + true /*Softmax*/>(tSrS, m_block, n_block); + }; + CUTLASS_PRAGMA_NO_UNROLL + for (m_block = full_m_block_min; m_block < full_m_block_max; ++m_block) { + bwd_step(m_block, mask_fn); + } + } + + // if (blockIdx.x == 0 && threadIdx.x == 128) { print_tensor(tdVrdV); } + if constexpr (Q_dO_same_stages) { + smem_pipe_read_do = smem_pipe_read; + } + ++work_idx; + return true; + } +}; + +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/mainloop_fwd_sm90_tma_gmma_ws.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/mainloop_fwd_sm90_tma_gmma_ws.h new file mode 100644 index 000000000..7c8a447af --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/mainloop_fwd_sm90_tma_gmma_ws.h @@ -0,0 +1,2180 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/****************************************************************************** + * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, + *Pradeep Ramani, Tri Dao. + ******************************************************************************/ + +#pragma once + +#include +#include +#include +#include +#include "cutlass/pipeline/pipeline.hpp" + +#include "cute/tensor.hpp" + +#include "cutlass/gemm/collective/builders/sm90_common.inl" + +#include "mask.h" +#include "named_barrier.h" +#include "seqlen.h" +#include "sm90_pipeline_no_cluster.h" +#include "utils.h" + +namespace hstu { + +using namespace cute; + +template < + int Stages, + class ClusterShape_, + class TileShape_MNK_, + class Element_, + class ElementAccum_, + class ArchTag_, + bool Causal, + bool Local, + bool Contexual_mask, + bool Jagged, + bool Has_targets, + bool Mma1_is_RS, + bool V_colmajor_, + bool Cross> +struct CollectiveMainloopFwdSm90 { + static constexpr int kStages = Stages; + using ClusterShape = ClusterShape_; + using TileShape_MNK = TileShape_MNK_; + using Element = Element_; + using ElementAccum = ElementAccum_; + using ArchTag = ArchTag_; + static constexpr bool Is_FP8 = + cute::is_same_v || + cute::is_same_v; + ; + static constexpr bool V_colmajor = V_colmajor_; + static constexpr bool Transpose_V = Is_FP8 && !V_colmajor; + using SeqlenInfo_t = hstu::SeqlenInfoQKFwd; + + static_assert(ArchTag::kMinComputeCapability >= 90); + + static constexpr cute::GMMA::Major MmaMajorV = + !Is_FP8 && !V_colmajor ? GMMA::Major::MN : GMMA::Major::K; + static constexpr cute::GMMA::Major TmaMajorV = + !V_colmajor ? GMMA::Major::MN : GMMA::Major::K; + + static constexpr int kBlockM = get<0>(TileShape_MNK{}); + static constexpr int kBlockN = get<1>(TileShape_MNK{}); + static constexpr int kHeadDim = get<2>(TileShape_MNK{}); + + // Register bandwidth is actually a bottleneck so we don't want Q to be in + // registers. Leaving this option here for reference. + static constexpr bool Mma0_is_RS = false; + // We can have Mma1 (P @ V) with P in smem in rmem to reduce register pressure + // at the cost of more smem. + static_assert(!(!Mma1_is_RS && Is_FP8), "Mma1 must be RS if FP8"); + static_assert( + !(!Mma1_is_RS && Transpose_V), + "Mma1 must be RS if Transpose_V"); + + using AtomLayoutMNK = Layout, _1, _1>>; + using TiledMma0 = decltype(cute::make_tiled_mma( + std::conditional_t< + !Mma0_is_RS, + decltype(cute::GMMA::ss_op_selector< + Element, + Element, + ElementAccum, + TileShape_MNK>()), + decltype(cute::GMMA::rs_op_selector< + Element, + Element, + ElementAccum, + TileShape_MNK>())>{}, + AtomLayoutMNK{})); + using TiledMma1 = decltype(cute::make_tiled_mma( + std::conditional_t< + !Mma1_is_RS, + decltype(cute::GMMA::ss_op_selector< + Element, + Element, + ElementAccum, + decltype(select<0, 2, 1>(TileShape_MNK{})), + GMMA::Major::K, + MmaMajorV>()), + decltype(cute::GMMA::rs_op_selector< + Element, + Element, + ElementAccum, + decltype(select<0, 2, 1>(TileShape_MNK{})), + GMMA::Major::K, + MmaMajorV>())>{}, + AtomLayoutMNK{})); + + static constexpr int NumMmaThreads = size(TiledMma0{}); + static constexpr int NumProducerThreads = !Transpose_V + ? cutlass::NumThreadsPerWarp + : cutlass::NumThreadsPerWarpGroup; + static_assert(NumMmaThreads % cutlass::NumThreadsPerWarpGroup == 0); + static constexpr int NumMmaWarpGroups = + NumMmaThreads / cutlass::NumThreadsPerWarpGroup; + static_assert( + NumMmaWarpGroups == 1 || NumMmaWarpGroups == 2 || NumMmaWarpGroups == 3); + + using SmemLayoutAtomQ = + decltype(cutlass::gemm::collective::detail::ss_smem_selector< + GMMA::Major::K, + Element, + decltype(cute::get<0>(TileShape_MNK{})), + decltype(cute::get<2>(TileShape_MNK{}))>()); + using SmemLayoutQ = + decltype(tile_to_shape(SmemLayoutAtomQ{}, select<0, 2>(TileShape_MNK{}))); + + using SmemLayoutAtomK = + decltype(cutlass::gemm::collective::detail::ss_smem_selector< + GMMA::Major::K, + Element, + decltype(cute::get<1>(TileShape_MNK{})), + decltype(cute::get<2>(TileShape_MNK{}))>()); + using SmemLayoutK = decltype(tile_to_shape( + SmemLayoutAtomK{}, + make_shape( + shape<1>(TileShape_MNK{}), + shape<2>(TileShape_MNK{}), + Int{}))); + + using SmemLayoutAtomVt = + decltype(cutlass::gemm::collective::detail::ss_smem_selector< + TmaMajorV, + Element, + decltype(cute::get<2>(TileShape_MNK{})), + decltype(cute::get<1>(TileShape_MNK{}))>()); + using SmemLayoutVt = decltype(tile_to_shape( + SmemLayoutAtomVt{}, + make_shape( + shape<2>(TileShape_MNK{}), + shape<1>(TileShape_MNK{}), + Int{}), + std::conditional_t< + TmaMajorV == GMMA::Major::K, + cute::Step<_1, _2, _3>, + cute::Step<_2, _1, _3>>{})); + + using SmemLayoutAtomVtMma = + decltype(cutlass::gemm::collective::detail::ss_smem_selector< + MmaMajorV, + Element, + decltype(cute::get<2>(TileShape_MNK{})), + decltype(cute::get<1>(TileShape_MNK{}))>()); + using SmemLayoutVtMma = decltype(tile_to_shape( + SmemLayoutAtomVtMma{}, + make_shape( + shape<2>(TileShape_MNK{}), + shape<1>(TileShape_MNK{}), + Int{}), + std::conditional_t< + MmaMajorV == GMMA::Major::K, + cute::Step<_1, _2, _3>, + cute::Step<_2, _1, _3>>{})); + + // Only used if we're using cp.async to load V + using SmemLayoutAtomVCpAsync = + decltype(cutlass::gemm::collective::detail::ss_smem_selector< + GMMA::Major::K, + Element, + decltype(cute::get<1>(TileShape_MNK{})), + decltype(cute::get<2>(TileShape_MNK{}))>()); + using SmemLayoutVCpAsync = decltype(tile_to_shape( + SmemLayoutAtomVCpAsync{}, + make_shape( + shape<1>(TileShape_MNK{}), + shape<2>(TileShape_MNK{}), + Int{}))); + + using SmemLayoutAtomP = + decltype(cutlass::gemm::collective::detail::ss_smem_selector< + GMMA::Major::K, + Element, + decltype(cute::get<0>(TileShape_MNK{})), + decltype(cute::get<1>(TileShape_MNK{}))>()); + using SmemLayoutP = + decltype(tile_to_shape(SmemLayoutAtomP{}, select<0, 1>(TileShape_MNK{}))); + + using SmemCopyAtomP = Copy_Atom; + + // Use LDSM.T and STSM to transpose V in the case of FP8 and V being + // row-major. For FP16/BF16 we don't do any transposing. + static_assert(!Transpose_V || (kHeadDim % 32 == 0 && kBlockN % 32 == 0)); + static constexpr bool kHeadDim_multiple_64 = kHeadDim % 64 == 0; + // Either kHeadDim is a multiple of 64 (in which case we use a block size of + // 64 x 32 for the transpose), or we need kBlockN to be a multiple of 64 (in + // which case we use a block size of 32 x 64 for the transpose). + static_assert(!Transpose_V || (kHeadDim_multiple_64 || kBlockN % 64 == 0)); + using LDSM_thread_shape = std::conditional_t< + kHeadDim_multiple_64, + Shape<_32, _4, _1, _1>, + Shape<_16, _4, _1, _2>>; + using LDSM_thread_stride = std::conditional_t< + kHeadDim_multiple_64, + Stride<_4, _1, _0, _0>, + Stride<_4, _1, _0, _64>>; + using LDSM_value_shape = Shape<_2, _2, _1, _4>; + using LDSM_value_stride = Stride<_1, _2, _16, _4>; + using LDSM_divide_shape = + std::conditional_t, Shape<_32, _8>>; + using S2RTiledCopyVt = decltype(make_tiled_copy( + Copy_Atom{}, + Layout{}, + Layout{})); + + using STSM_thread_shape = std::conditional_t< + kHeadDim_multiple_64, + Shape<_8, _4, _4, _1>, + Shape<_8, _4, _2, _2>>; + using STSM_thread_stride = std::conditional_t< + kHeadDim_multiple_64, + Stride<_4, _1, _32, _0>, + Stride<_4, _1, _32, _64>>; + using STSM_value_shape = Shape<_1, _4, _2, _2>; + using STSM_value_stride = Stride<_0, _1, _4, _8>; + using STSM_divide_shape = Shape<_8, _16>; + // These will not permute the columns of V (the kHeadDim dimension) but incur + // bank conflicts so a little slower (e.g. 1150 TFLOPS for hdim 256 instead of + // 1200 TFLOPS). Instead we will permute the cols of V, and un-permute the + // cols of O in the epilogue. using STSM_value_shape = Shape<_2, _4, _1, _2>; + // using STSM_value_stride = Stride<_4, _1, _0, _8>; + // using STSM_divide_shape = Shape<_16, _16>; + using R2STiledCopyV = decltype(make_tiled_copy( + Copy_Atom{}, + Layout{}, + Layout{})); + + using GmemTiledCopyQ = cute::SM90_TMA_LOAD; + using GmemTiledCopyKV = + decltype(cutlass::gemm::collective::detail:: + sm90_cluster_shape_to_tma_atom(shape<0>(ClusterShape{}))); + + // We use CpAsync for K and V if PagedKV and AppendKV, since TMA doesn't work + // there + static constexpr int kGmemElemsPerLoad = + sizeof(cute::uint128_t) / sizeof(Element); + static_assert( + kHeadDim % kGmemElemsPerLoad == 0, + "Headdim must be a multiple of kGmemElemsPerLoad"); + // We want each "row" to have 64 elements (128 bytes, i.e. 1 cache line). E.g. + // if hdim=128, we want each thread to have 4 loads in the M direction and 2 + // vectorized load in the K direction. We want each thread to have at least 2 + // loads in the K direction since in the case of non-interleaved rotary + // (combining elements at indices 0 and rotary_dim/2, 1 and rotary_dim/2+1, + // etc), each thread will load twice from the same row. + static constexpr int kBytePerHalfRow = kHeadDim / 2 * sizeof(Element); + static constexpr int kBlockKGmem = + (kBytePerHalfRow % 128 == 0 ? 128 + : (kBytePerHalfRow % 64 == 0 ? 64 : 32)) / + sizeof(Element); + static constexpr int kGmemThreadsPerRow = kBlockKGmem / kGmemElemsPerLoad; + static_assert( + NumMmaThreads % kGmemThreadsPerRow == 0, + "NumMmaThreads must be a multiple of kGmemThreadsPerRow"); + // We assume threads loading the same row are in the same warp. This is for an + // optimization in PagedKV where these threads share the same page table entry + // and share the work of computing pointers to paged K and paged V. + static_assert( + cutlass::NumThreadsPerWarp % kGmemThreadsPerRow == 0, + "kGmemThreadsPerRow must divide NumThreadsPerWarp"); + using GmemLayoutAtom = Layout< + Shape, Int>, + Stride, _1>>; + // If AppendKV, we'll be loading Q for rotary, and we assume divisibility to + // avoid predication + static_assert( + kBlockM % CUTE_STATIC_V(shape<0>(GmemLayoutAtom{})) == 0, + "kBlockM must be a multiple of NumMmaThreads / kGmemThreadsPerRow"); + + using ShapeQKV = + cute::Shape; // (seqlen, d, head, + // batch) + using StrideQK = cute::Stride; + using StrideV = std::conditional_t< + !V_colmajor, + StrideQK, + cute::Stride<_1, int64_t, int64_t, int64_t>>; + // ((qhead_per_khead, seqlen), d, nheads_kv, batch, num_splits) + using ShapeQPacked = ShapeQKV; + using StrideQPacked = StrideQK; + using StrideDescale = cute::Stride; + + using TMA_Q = decltype(make_tma_copy_A_sm90( + GmemTiledCopyQ{}, + make_tensor( + make_gmem_ptr(static_cast(nullptr)), + ShapeQKV{}, + StrideQK{}), + SmemLayoutQ{}, + TileShape_MNK{}, + ClusterShape{})); + + using TMA_K = decltype(make_tma_copy_B_sm90( + GmemTiledCopyKV{}, + make_tensor( + make_gmem_ptr(static_cast(nullptr)), + ShapeQKV{}, + StrideQK{}), + take<0, 2>(SmemLayoutK{}), + TileShape_MNK{}, + ClusterShape{})); // mcast along M mode for this N load, if any + + using TMA_V = decltype(make_tma_copy( + GmemTiledCopyKV{}, + make_tensor( + make_gmem_ptr(static_cast(nullptr)), + ShapeQKV{}, + select<1, 0, 2, 3>(StrideV{})), + take<0, 2>(SmemLayoutVt{}), + select<2, 1>(TileShape_MNK{}), + size<0>(ClusterShape{}))); // mcast along M mode for this N load, if any + + // Set the bytes transferred in this TMA transaction (may involve multiple + // issues) + static constexpr uint32_t TmaTransactionBytesQ = static_cast( + size(SmemLayoutQ{}) * cutlass::sizeof_bits_v / 8); + static constexpr uint32_t TmaTransactionBytesK = static_cast( + size(take<0, 2>(SmemLayoutK{})) * cutlass::sizeof_bits_v / 8); + static constexpr uint32_t TmaTransactionBytesV = static_cast( + size(take<0, 2>(SmemLayoutVt{})) * cutlass::sizeof_bits_v / 8); + static_assert(TmaTransactionBytesK == TmaTransactionBytesV); + + using PipelineTmaAsync = std::conditional_t< + CUTE_STATIC_V(size(ClusterShape{})) == 1, + typename cutlass::PipelineTmaAsyncNoCluster, + typename cutlass::PipelineTmaAsync>; + using MainloopPipelineK = PipelineTmaAsync; + using MainloopPipelineV = std::conditional_t< + !Transpose_V, + PipelineTmaAsync, + typename cutlass::PipelineAsync>; + using MainloopPipelineVt = PipelineTmaAsync; + // We always use TMA for K_new and V_new + using MainloopPipelineKVNew = PipelineTmaAsync; + using PipelineState = cutlass::PipelineState; + + // If PackGQA, we use cp.async (instead of TMA) to load Q, so we want smem_q + // to be aligned and have sQ being position_independent_swizzle_tensor. If + // !Use_TMA_KV, we use cp.async (instead of TMA) to load K & V, so we want + // smem_k and smem_v to be aligned. + static constexpr size_t SmemAlignmentQ = + !Mma0_is_RS ? 128 : cutlass::detail::alignment_for_swizzle(SmemLayoutQ{}); + static constexpr size_t SmemAlignmentK = 128; + static constexpr size_t SmemAlignmentVtNoTranspose = + cutlass::detail::alignment_for_swizzle(SmemLayoutVt{}); + static_assert( + SmemAlignmentQ >= 128 and SmemAlignmentK >= 128 && + SmemAlignmentVtNoTranspose >= 128, + "Require at least 128B alignment"); + static constexpr size_t SmemAlignmentP = + cutlass::detail::alignment_for_swizzle(SmemLayoutP{}); + static_assert(SmemAlignmentP >= 128, "Require at least 128B alignment"); + + using SmemP_t = std::conditional_t< + Mma1_is_RS, + cute::array, + cute:: + array_aligned, SmemAlignmentP>>; + // Sometimes even with SmemP_t = cute::array, putting it in the + // TensorStorage struct causes smem size to go from 227KB to 228KB and we get + // "invalid argument". + + struct TensorStorageWithoutPNoTranspose : cute::aligned_struct { + cute::array_aligned< + Element, + cute::cosize_v, + SmemAlignmentVtNoTranspose> + smem_v; + cute::array_aligned, SmemAlignmentQ> + smem_q; + cute::array_aligned, SmemAlignmentK> + smem_k; + }; + + struct TensorStorageWithPNoTranspose : cute::aligned_struct { + cute::array_aligned< + Element, + cute::cosize_v, + SmemAlignmentVtNoTranspose> + smem_v; + cute::array_aligned, SmemAlignmentQ> + smem_q; + cute::array_aligned, SmemAlignmentK> + smem_k; + SmemP_t smem_p; + }; + + using TensorStorageNoTranspose = std::conditional_t< + Mma1_is_RS, + TensorStorageWithoutPNoTranspose, + TensorStorageWithPNoTranspose>; + + static constexpr size_t SmemAlignmentVt = + cutlass::detail::alignment_for_swizzle(SmemLayoutVt{}); + static constexpr size_t SmemAlignmentV = + cutlass::detail::alignment_for_swizzle(SmemLayoutVtMma{}); + static_assert( + SmemAlignmentVt >= 128 and SmemAlignmentV >= 128, + "Require at least 128B alignment"); + struct TensorStorageTransposeV + : cute::aligned_struct< + cute::max(SmemAlignmentQ, SmemAlignmentK, SmemAlignmentV)> { + cute:: + array_aligned, SmemAlignmentV> + smem_v; + cute::array_aligned, SmemAlignmentVt> + smem_vt; + cute::array_aligned, SmemAlignmentQ> + smem_q; + cute::array_aligned, SmemAlignmentK> + smem_k; + }; + + using TensorStorage = std::conditional_t< + !Transpose_V, + TensorStorageNoTranspose, + TensorStorageTransposeV>; + + // These are tuned for speed. They don't affect correctness. + static constexpr bool UseSchedulerBarrier = + (NumMmaWarpGroups >= 2) && (!Is_FP8 ? kHeadDim <= 128 : kHeadDim >= 128); + static constexpr bool RescaleOBeforeGemm = + kHeadDim > 128 && (!Is_FP8 || V_colmajor); + + // Host side kernel arguments + struct Arguments { + Element const* const ptr_Q; + ShapeQKV const shape_Q; + StrideQK const stride_Q; + Element* const + ptr_K; // Not Element const* since we might append to KV cache in-place + ShapeQKV const shape_K; + StrideQK const stride_K; + Element* const ptr_V; + StrideV const stride_V; + float const *ptr_q_descale, *ptr_k_descale, *ptr_v_descale; + StrideDescale const stride_q_descale, stride_k_descale, stride_v_descale; + float const max_seq_len_inv; + float const alpha; + int const max_attn_len; + int const min_full_attn_seq_len; + int const contextual_seq_len; + int const num_softmax_heads; + int const num_groups; + int const batch_size_per_group; + int const* const seq_offsets = nullptr; + int const* const seq_offsets_q = nullptr; + int const* const num_targets = nullptr; + int const* const max_seq_len_tensor = nullptr; + int const* const contextual_seq_len_tensor = nullptr; + int const* const max_attn_len_tensor = nullptr; + int const* const min_full_attn_seq_len_tensor = nullptr; + float const* const attn_scale = nullptr; + bool const scalar_scale = true; + }; + + // Device side kernel params + struct Params { + Element const* const ptr_Q; + ShapeQKV const shape_Q; + StrideQK const stride_Q; + ShapeQPacked const shape_Q_packed; + StrideQPacked const stride_Q_packed; + Element* const ptr_K; + ShapeQKV const shape_K; + StrideQK const stride_K; + Element* const ptr_V; + StrideV const stride_V; + TMA_Q tma_load_Q; + TMA_K tma_load_K; + TMA_V tma_load_V; + float const *ptr_q_descale, *ptr_k_descale, *ptr_v_descale; + StrideDescale const stride_q_descale, stride_k_descale, stride_v_descale; + float const max_seq_len_inv; + float const alpha; + float const alpha_log2; + int const max_attn_len; + int const min_full_attn_seq_len; + int const contextual_seq_len; + int const num_softmax_heads; + int const num_groups; + int const batch_size_per_group; + int const* const seq_offsets = nullptr; + int const* const seq_offsets_q = nullptr; + int const* const num_targets = nullptr; + int const* const max_seq_len_tensor = nullptr; + int const* const contextual_seq_len_tensor = nullptr; + int const* const max_attn_len_tensor = nullptr; + int const* const min_full_attn_seq_len_tensor = nullptr; + float const* const attn_scale = nullptr; + bool const scalar_scale = true; + }; + + static Params to_underlying_arguments(Arguments const& args) { + Tensor mQ = + make_tensor(make_gmem_ptr(args.ptr_Q), args.shape_Q, args.stride_Q); + TMA_Q tma_load_Q = make_tma_copy_A_sm90( + GmemTiledCopyQ{}, + mQ, + SmemLayoutQ{}, + TileShape_MNK{}, + ClusterShape{}); // no mcast for Q + Tensor mK = + make_tensor(make_gmem_ptr(args.ptr_K), args.shape_K, args.stride_K); + TMA_K tma_load_K = make_tma_copy_B_sm90( + GmemTiledCopyKV{}, + mK, + take<0, 2>(SmemLayoutK{}), + TileShape_MNK{}, + ClusterShape{}); // mcast along M mode for this N load, if any + Tensor mV = make_tensor( + make_gmem_ptr(args.ptr_V), + select<1, 0, 2, 3>(args.shape_K), + select<1, 0, 2, 3>(args.stride_V)); + TMA_V tma_load_V = make_tma_copy( + GmemTiledCopyKV{}, + mV, + take<0, 2>(SmemLayoutVt{}), + select<2, 1>(TileShape_MNK{}), + size<0>(ClusterShape{})); // mcast along M mode for this N load, if any + auto const shape_Q_packed = cute::conditional_return( + args.shape_Q, + make_shape( + make_shape(1, get<0>(args.shape_Q)), + get<1>(args.shape_Q), + get<2>(args.shape_K), + get<3>(args.shape_Q))); + auto const stride_Q_packed = cute::conditional_return( + args.stride_Q, + make_stride( + make_stride(get<2>(args.stride_Q), get<0>(args.stride_Q)), + get<1>(args.stride_Q), + get<2>(args.stride_Q), + get<3>(args.stride_Q))); + return { + args.ptr_Q, + args.shape_Q, + args.stride_Q, + shape_Q_packed, + stride_Q_packed, + args.ptr_K, + args.shape_K, + args.stride_K, + args.ptr_V, + args.stride_V, + tma_load_Q, + tma_load_K, + tma_load_V, + args.ptr_q_descale, + args.ptr_k_descale, + args.ptr_v_descale, + args.stride_q_descale, + args.stride_k_descale, + args.stride_v_descale, + args.max_seq_len_inv, + args.alpha, + float(args.alpha * M_LOG2E), + args.max_attn_len, + args.min_full_attn_seq_len, + args.contextual_seq_len, + args.num_softmax_heads, + args.num_groups, + args.batch_size_per_group, + args.seq_offsets, + args.seq_offsets_q, + args.num_targets, + args.max_seq_len_tensor, + args.contextual_seq_len_tensor, + args.max_attn_len_tensor, + args.min_full_attn_seq_len_tensor, + args.attn_scale, + args.scalar_scale}; + } + + /// Issue Tma Descriptor Prefetch -- ideally from a single thread for best + /// performance + CUTLASS_DEVICE + static void prefetch_tma_descriptors(Params const& params) { + cute::prefetch_tma_descriptor(params.tma_load_Q.get_tma_descriptor()); + cute::prefetch_tma_descriptor(params.tma_load_K.get_tma_descriptor()); + cute::prefetch_tma_descriptor(params.tma_load_V.get_tma_descriptor()); + } + + CUTLASS_DEVICE + cute::tuple get_n_block_min_max( + int max_attn_len, + int min_full_attn_seq_len, + int contextual_seq_len, + int uihlen, + int m_block) { + static constexpr int kBlockM = get<0>(TileShape_MNK{}); + static constexpr int kBlockN = get<1>(TileShape_MNK{}); + if constexpr (Contexual_mask) { + if (m_block * kBlockM < contextual_seq_len) { + return {0, cute::ceil_div(uihlen, kBlockN)}; + } + } + if constexpr (Has_targets) { + int m_idx_max = (m_block + 1) * kBlockM; + if (m_idx_max > uihlen) { + return {0, cute::ceil_div(uihlen, kBlockN)}; + } + } + int n_block_max; + int n_block_min; + // Non-target part, n_block_max + if constexpr (Causal || Local) { + int m_idx_max = (m_block + 1) * kBlockM; + n_block_max = cute::ceil_div(std::min(m_idx_max, uihlen), kBlockN); + } else { + n_block_max = cute::ceil_div(uihlen, kBlockN); + } + // Non-target part, n_block_min + if constexpr (Local) { + int m_idx_min = m_block * kBlockM; + int m_idx_max = (m_block + 1) * kBlockM; + if (min_full_attn_seq_len == 0 || + m_idx_max <= uihlen - min_full_attn_seq_len) { + n_block_min = std::max(int(0), (m_idx_min - max_attn_len) / kBlockN); + if constexpr (Contexual_mask) { + // row contexual without sink + if (n_block_min * kBlockN < contextual_seq_len) { + n_block_min = 0; + } + } + } else { + n_block_min = 0; + } + } else { + n_block_min = 0; + } + return {n_block_min, n_block_max}; + } + + CUTLASS_DEVICE + cute::tuple get_target_n_block_min_max( + int n_block_max, + int uihlen, + int seqlen, + int m_block) { + static constexpr int kBlockM = get<0>(TileShape_MNK{}); + static constexpr int kBlockN = get<1>(TileShape_MNK{}); + int m_idx_max = (m_block + 1) * kBlockM; + if (m_idx_max <= uihlen) { // Non-target part + return {n_block_max, n_block_max}; + } else { // Target part + int m_idx_min = m_block * kBlockM; + return { + std::max(n_block_max, m_idx_min / kBlockN), + cute::ceil_div(std::min(m_idx_max, seqlen), kBlockN)}; + } + } + + CUTLASS_DEVICE + int get_contexual_n_block_max( + int n_block_min, + int min_full_attn_seq_len, + int contextual_seq_len, + int uihlen, + int m_block) { + return 0; + // TODO: reenable below once contexual + semi local implementation is + // finalized + static constexpr int kBlockM = get<0>(TileShape_MNK{}); + static constexpr int kBlockN = get<1>(TileShape_MNK{}); + if constexpr (!Local) { + return 0; + } + if (m_block * kBlockM < contextual_seq_len) { + return 0; + } + int m_idx_max = (m_block + 1) * kBlockM; + if constexpr (Has_targets) { + if (m_idx_max > uihlen) { + return 0; + } + } + if (min_full_attn_seq_len == 0 || + m_idx_max <= uihlen - min_full_attn_seq_len) { + return std::min(n_block_min, cute::ceil_div(contextual_seq_len, kBlockN)); + } + return 0; + } + + CUTLASS_DEVICE + cute::tuple get_cross_n_block_min_max( + int const uihlen_q, + int const seqlen_q, + int const seqlen_kv, + int const m_block) { + static constexpr int kBlockM = get<0>(TileShape_MNK{}); + static constexpr int kBlockN = get<1>(TileShape_MNK{}); + if constexpr (!Causal) { + return {0, cute::ceil_div(seqlen_kv, kBlockN)}; + } + int n_block_max = + std::min(seqlen_kv, (m_block + 1) * kBlockM + seqlen_kv - uihlen_q); + return {0, cute::ceil_div(n_block_max, kBlockN)}; + } + + template + CUTLASS_DEVICE void load( + Params const& params, + MainloopPipelineK pipeline_k, + MainloopPipelineV pipeline_v, + MainloopPipelineVt pipeline_vt, + PipelineState& smem_pipe_write, + SharedStorage& shared_storage, + SchedulerPrefetch const& scheduler_prefetch, + SeqlenInfo_t const& seqlen_info, + cute::tuple block_coord, + int& work_idx) { + auto [m_block, bidh, bidb, split_idx] = block_coord; + if constexpr (Jagged) { + static constexpr int kBlockM = get<0>(TileShape_MNK{}); + if (m_block * kBlockM >= seqlen_info.seqlen_q) { + scheduler_prefetch(); + return; + } + } + int min_full_attn_seq_len_, max_attn_len_, contextual_seq_len_; + if constexpr (!Cross) { + if (params.num_groups > 1) { + int group_id = bidb / params.batch_size_per_group; + min_full_attn_seq_len_ = params.min_full_attn_seq_len_tensor[group_id]; + max_attn_len_ = params.max_attn_len_tensor[group_id]; + contextual_seq_len_ = params.contextual_seq_len_tensor[group_id]; + } else { + min_full_attn_seq_len_ = params.min_full_attn_seq_len; + max_attn_len_ = params.max_attn_len; + contextual_seq_len_ = params.contextual_seq_len; + } + } + int n_block_min, n_block_max; + if constexpr (Cross) { + auto n_block_min_max = get_cross_n_block_min_max( + seqlen_info.uihlen_q, + seqlen_info.seqlen_q, + seqlen_info.seqlen_kv, + m_block); + n_block_min = get<0>(n_block_min_max); + n_block_max = get<1>(n_block_min_max); + } else { + auto n_block_min_max = get_n_block_min_max( + max_attn_len_, + min_full_attn_seq_len_, + contextual_seq_len_, + seqlen_info.uihlen_q, + m_block); + n_block_min = get<0>(n_block_min_max); + n_block_max = get<1>(n_block_min_max); + } +#ifdef HSTU_FLASH_ATTN_DEBUG_INFO + if (n_block_max <= n_block_min) { + std::printf( + "mainloop_fwd_sm90: n_block_max <= n_block_min not expected."); + scheduler_prefetch(); + return; + } +#endif + + Tensor sQ = make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_q.data()), + SmemLayoutQ{}); + Tensor sK = make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_k.data()), + SmemLayoutK{}); + Tensor sK_pi = as_position_independent_swizzle_tensor(sK); + // as_position_independent_swizzle_tensor makes address calculation easier + // when we do LDSM & STSM to transpose. But it requires smem_vt and smem_v + // to be aligned to e.g 512 bytes. + Tensor sVt = [&] { + if constexpr (!Transpose_V) { + return make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_v.data()), + SmemLayoutVt{}); + } else { + return cute::as_position_independent_swizzle_tensor(make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_vt.data()), + SmemLayoutVt{})); + } + }(); + // Only used if Transpose_V + Tensor sV = cute::as_position_independent_swizzle_tensor(make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_v.data()), + SmemLayoutVtMma{})); + + int const thread_idx = threadIdx.x % NumProducerThreads; + + // Prepare the TMA loads + uint32_t block_rank_in_cluster = cute::block_rank_in_cluster(); + constexpr uint32_t cluster_shape_x = get<0>(ClusterShape()); + uint2 cluster_local_block_id = { + block_rank_in_cluster % cluster_shape_x, + block_rank_in_cluster / cluster_shape_x}; + + Tensor mQ = params.tma_load_Q.get_tma_tensor(params.shape_Q)( + _, _, bidh, !Jagged ? bidb : 0); + Tensor mK_TMA = params.tma_load_K.get_tma_tensor(params.shape_K)( + _, _, bidh, !Jagged ? bidb : 0); + Tensor mVt_TMA = params.tma_load_V.get_tma_tensor( + select<1, 0, 2, 3>(params.shape_K))(_, _, bidh, !Jagged ? bidb : 0); + + Tensor gQ = local_tile( + domain_offset(make_coord(seqlen_info.offset_q, _0{}), mQ), + select<0, 2>(TileShape_MNK{}), + make_coord(m_block, _0{})); // (M, K) + // if (cute::thread0()) { printf("Jagged = %d, params.leftpad_k = %p, + // leftpad_k = %d\n", Jagged, params.leftpad_k, leftpad_k); } + Tensor gK_TMA = local_tile( + domain_offset(make_coord(seqlen_info.offset_k, _0{}), mK_TMA), + select<1, 2>(TileShape_MNK{}), + make_coord(_, _0{})); // (N, K, _) + Tensor gVt_TMA = local_tile( + domain_offset(make_coord(_0{}, seqlen_info.offset_k), mVt_TMA), + select<2, 1>(TileShape_MNK{}), + make_coord(_0{}, _)); // (K, N, _) + + auto block_tma_Q = params.tma_load_Q.get_slice(_0{}); + Tensor tQgQ = group_modes<0, 3>(block_tma_Q.partition_S(gQ)); // (TMA) + Tensor tQsQ = group_modes<0, 3>(block_tma_Q.partition_D(sQ)); // (TMA) + // tma_partition doesn't handle position_independent_swizzle_tensor + // correctly, so we need to do it manually + auto block_tma_K = params.tma_load_K.get_slice(cluster_local_block_id.x); + Tensor tKgK_TMA = + group_modes<0, 3>(block_tma_K.partition_S(gK_TMA)); // (TMA, k) + Tensor tKsK_TMA = + group_modes<0, 3>(block_tma_K.partition_D(sK)); // (TMA, PIPE) + auto block_tma_V = params.tma_load_V.get_slice(cluster_local_block_id.x); + Tensor tVgVt_TMA = + group_modes<0, 3>(block_tma_V.partition_S(gVt_TMA)); // (TMA, k) + Tensor tVsVt_TMA = + group_modes<0, 3>(block_tma_V.partition_D(sVt)); // (TMA, PIPE) + + // Set up for transposing V, only used if Transpose_V + S2RTiledCopyVt s2r_tiled_copy_vt; + R2STiledCopyV r2s_tiled_copy_v; + auto s2r_thr_copy_vt = s2r_tiled_copy_vt.get_thread_slice(thread_idx); + auto r2s_thr_copy_v = r2s_tiled_copy_v.get_thread_slice(thread_idx); + // flat_divide(sVt, LDSM_divide_shape{}): (64, 8, kHeadDim / 64, kBlockN / + // 8, kStages) + Tensor tTranssVt_ = s2r_thr_copy_vt.partition_S( + flat_divide(sVt, LDSM_divide_shape{})); // ((16, 1), 1, 1, kHeadDim / + // 64, kBlockN / 32, kStages) + // flat_divide(sV, STSM_divide_shape{}): (8, 16, kHeadDim / 8, (4, kBlockN + // / 64), kStages) + Tensor tTranssV_ = r2s_thr_copy_v.partition_D( + flat_divide(sV, STSM_divide_shape{})); // ((16, 1), 1, 1, kHeadDim / 64, + // (2, kBlockN / 64), kStages) + CUTE_STATIC_ASSERT_V(rank(tTranssVt_) == rank(tTranssV_)); + CUTE_STATIC_ASSERT_V(size<0>(tTranssVt_) == size<0>(tTranssV_)); + CUTE_STATIC_ASSERT_V(size<1>(tTranssVt_) == size<1>(tTranssV_)); + CUTE_STATIC_ASSERT_V(size<2>(tTranssVt_) == size<2>(tTranssV_)); + CUTE_STATIC_ASSERT_V(size<3>(tTranssVt_) == size<3>(tTranssV_)); + CUTE_STATIC_ASSERT_V(size<4>(tTranssVt_) == size<4>(tTranssV_)); + // Faster to have 2 LDSM.T, byte permute, STSM for better ILP + static constexpr int Transpose_ILP = + (size<2>(tTranssVt_) * size<3>(tTranssVt_)) % 2 == 0 ? 2 : 1; + Tensor tTranssVt = logical_divide( + group_modes<1, rank(tTranssVt_) - 1>(tTranssVt_), + Shape>{}); // ((16, 1), (2, kHeadDim / 64 + // * kBlockN / 32 / 2), + // kStages) + Tensor tTranssV = logical_divide( + group_modes<1, rank(tTranssV_) - 1>(tTranssV_), + Shape>{}); // ((16, 1), (2, kHeadDim / 64 + // * kBlockN / 32 / 2), + // kStages) + auto transpose_V = [&](int stage) { + if constexpr (Transpose_V) { +#pragma unroll + for (int i = 0; i < size<1, 1>(tTranssVt); ++i) { + Tensor tTransrV = + make_fragment_like(tTranssV(_, make_coord(_, _0{}), _0{})); + static_assert(size<0>(tTransrV) == 16); + Tensor tTransrV_64 = recast(tTransrV); + cute::copy( + s2r_tiled_copy_vt, + tTranssVt(_, make_coord(_, i), stage), + tTransrV); +#pragma unroll + for (int j = 0; j < size(tTransrV_64); ++j) { + uint32_t upper = tTransrV_64[j].x; + uint32_t lower = tTransrV_64[j].y; + tTransrV_64[j].x = __byte_perm(upper, lower, 0x6420); + tTransrV_64[j].y = __byte_perm(upper, lower, 0x7531); + } + cute::copy( + r2s_tiled_copy_v, tTransrV, tTranssV(_, make_coord(_, i), stage)); + } + } + }; + + uint16_t mcast_mask_kv = 0; + if constexpr (cute::is_same_v) { + auto block_layout = Layout{}; // (m,n) -> block_id + for (int m = 0; m < size<0>(block_layout); ++m) { + mcast_mask_kv |= + (uint16_t(1) << block_layout(m, cluster_local_block_id.y, _0{})); + } + } + + auto load_K = [&](int const n_block, auto const& smem_pipe_write) { + pipeline_k.producer_acquire(smem_pipe_write); + copy( + params.tma_load_K.with( + *pipeline_k.producer_get_barrier(smem_pipe_write), + mcast_mask_kv, + TMA::CacheHintSm90::EVICT_LAST), + tKgK_TMA(_, n_block), + tKsK_TMA(_, smem_pipe_write.index())); + }; + + auto load_V = [&](int const n_block, auto const& smem_pipe_write) { + auto pipeline_v_load = + cute::conditional_return(pipeline_v, pipeline_vt); + pipeline_v_load.producer_acquire(smem_pipe_write); + copy( + params.tma_load_V.with( + *pipeline_v_load.producer_get_barrier(smem_pipe_write), + mcast_mask_kv, + TMA::CacheHintSm90::EVICT_LAST), + tVgVt_TMA(_, n_block), + tVsVt_TMA(_, smem_pipe_write.index())); + }; + + auto copy_Vt_to_V = [&](auto const& smem_pipe_write) { + // Instead of maintaining smem_pipe_read as a separate variable, we can + // just use smem_pipe_write, and exploit the invariance that + // smem_pipe_write.phase() == smem_pipe_read.phase() ^ 1. This saves 1 or + // 2 registers. + PipelineState smem_pipe_read{ + smem_pipe_write.index(), + smem_pipe_write.phase() ^ 1, + smem_pipe_write.count()}; + pipeline_vt.consumer_wait(smem_pipe_read); + pipeline_v.producer_acquire(smem_pipe_write); + transpose_V(smem_pipe_write.index()); + // SMEM fence to make sure V is transposed before math + cutlass::arch::fence_view_async_shared(); + pipeline_v.producer_commit(smem_pipe_write); + // Very important: PipelineTmaAsync::consumer_release assumes that the + // warpgroup is synchronized before calling. Without this we get race + // conditions. + cutlass::arch::NamedBarrier::sync( + cutlass::NumThreadsPerWarpGroup, + static_cast(FwdNamedBarriers::ProducerWG) /*id*/); + pipeline_vt.consumer_release(smem_pipe_read); + }; + + int n_block = n_block_max - 1; + + int warp_idx_in_warpgroup = + __shfl_sync(0xffffffff, (threadIdx.x / 32) % 4, 0); + // If this is true, we're guaranteed that only the first warp will execute + // this function + static constexpr bool SingleProducerWarp = + NumProducerThreads == cutlass::NumThreadsPerWarp; + bool should_load_KV = + ((SingleProducerWarp || warp_idx_in_warpgroup == 0) && + cute::elect_one_sync()); + + if (should_load_KV) { + if constexpr (Transpose_V) { + load_V(n_block, smem_pipe_write); + } + // if (thread_idx == 0) { printf("Producer: main load, before load_K, + // index = %d\n", smem_pipe_write.index());} + load_K(n_block, smem_pipe_write); + // if (thread_idx == 0) { printf("Producer: main load, after load K, index + // = %d\n", smem_pipe_write.index());} + } + + // TMA_Q, Wait for the MMA warpgroups to signal that smem_q is ready + if (SingleProducerWarp || warp_idx_in_warpgroup == 0) { + cutlass::arch::NamedBarrier::sync( + NumMmaThreads + cutlass::NumThreadsPerWarp, + static_cast(FwdNamedBarriers::QueryEmpty) /*id*/); + } + if ((SingleProducerWarp || warp_idx_in_warpgroup == 0) && + cute::elect_one_sync()) { + shared_storage.pipelines.barrier_Q.arrive_and_expect_tx( + TmaTransactionBytesQ); + copy( + params.tma_load_Q.with( + reinterpret_cast( + shared_storage.pipelines.barrier_Q), + 0 /*mcast_mask*/, + TMA::CacheHintSm90::EVICT_FIRST), + tQgQ, + tQsQ); + } + + // Wait for the MMA WGs to signal that smem_v are ready and V can be copied + // from gmem Need ClusterBarrier, not just NamedBarrier. Otherwise we might + // have CTA 0 finishing the TMA store on O first, call TMA multicast load on + // V, before CTA 1 can finishing TMA store on O. if (thread_idx == 0) { + // printf("Producer: main load, before barrier_O, work_idx = %d\n", + // work_idx);} + shared_storage.pipelines.barrier_O.wait((work_idx + 1) % 2); + // if (thread_idx == 0) { printf("Producer: main load, after barrier_O\n");} + + int n_block_prev = n_block; + --n_block; +#pragma unroll(!Transpose_V ? 2 : 1) + for (; n_block >= n_block_min; --n_block) { + PipelineState smem_pipe_write_v = + smem_pipe_write; // copy the state, write_v is always 1 step behind + ++smem_pipe_write; + if (should_load_KV) { + if constexpr (Transpose_V) { + load_V(n_block, smem_pipe_write); + } else { + load_V(n_block_prev, smem_pipe_write_v); + } + load_K(n_block, smem_pipe_write); + } + n_block_prev = n_block; + if constexpr (Transpose_V) { + copy_Vt_to_V(smem_pipe_write_v); + } + } + scheduler_prefetch(); + if constexpr (!Transpose_V) { + if (should_load_KV) { + load_V(n_block_prev, smem_pipe_write); + } + } + if constexpr (Transpose_V) { + copy_Vt_to_V(smem_pipe_write); + } + ++smem_pipe_write; + if constexpr (!Cross) { + if constexpr (Has_targets) { + auto [target_n_block_min, target_n_block_max] = + get_target_n_block_min_max( + n_block_max, + seqlen_info.uihlen_q, + seqlen_info.seqlen_kv, + m_block); +#pragma unroll 1 + for (n_block = target_n_block_max - 1; n_block >= target_n_block_min; + --n_block) { + if (should_load_KV) { + load_V(n_block, smem_pipe_write); + load_K(n_block, smem_pipe_write); + } + if constexpr (Transpose_V) { + copy_Vt_to_V(smem_pipe_write); + } + ++smem_pipe_write; + } + } + if constexpr (Contexual_mask) { + int contexual_n_block_max = get_contexual_n_block_max( + n_block_min, + min_full_attn_seq_len_, + contextual_seq_len_, + seqlen_info.uihlen_q, + m_block); +#pragma unroll 1 + for (n_block = contexual_n_block_max - 1; n_block >= 0; --n_block) { + if (should_load_KV) { + load_V(n_block, smem_pipe_write); + load_K(n_block, smem_pipe_write); + } + if constexpr (Transpose_V) { + copy_Vt_to_V(smem_pipe_write); + } + ++smem_pipe_write; + } + } + } + // At the end, all threads have the correct smem_pipe_write. + ++work_idx; + } + + template + CUTLASS_DEVICE void load_tail( + MainloopPipelineK pipeline_k, + MainloopPipelineV pipeline_v, + MainloopPipelineVt pipeline_vt, + PipelineState& smem_pipe_write, + SharedStorage& shared_storage, + int const work_idx) { + // If we don't wait for barrier_O here, when using Cluster, CTA0 might exit + // early and CTA1 will try to arrive on barrier_O of CTA0, causing + // "unspecified launch failure". + shared_storage.pipelines.barrier_O.wait((work_idx + 1) % 2); + int warp_idx_in_warpgroup = + __shfl_sync(0xffffffff, (threadIdx.x / 32) % 4, 0); + // Issue the epilogue waits + // TODO: check if this should be called by 1 thread or more + if (warp_idx_in_warpgroup == 0 && cute::elect_one_sync()) { + /* This helps avoid early exit of blocks in Cluster + * Waits for all stages to either be released (all Consumer UNLOCKs), or + * if the stage was never used then would just be acquired since the phase + * was still inverted from make_producer_start_state + */ + pipeline_k.producer_tail(smem_pipe_write); + pipeline_v.producer_tail(smem_pipe_write); + if constexpr (Transpose_V) { + pipeline_vt.producer_tail(smem_pipe_write); + } + } + } + + CUTLASS_DEVICE void warp_scheduler_barrier_sync() { + if constexpr (UseSchedulerBarrier) { + cutlass::arch::NamedBarrier::sync( + 2 * cutlass::NumThreadsPerWarpGroup, + static_cast(FwdNamedBarriers::WarpSchedulerWG1) - 1 + + hstu::canonical_warp_group_idx_nosync() /*id*/); + } + } + + CUTLASS_DEVICE void warp_scheduler_barrier_arrive() { + if constexpr (UseSchedulerBarrier) { + static_assert(NumMmaWarpGroups == 2 || NumMmaWarpGroups == 3); + int const cur_WG = hstu::canonical_warp_group_idx_nosync() - 1; + int const next_WG = NumMmaWarpGroups == 2 + ? 1 - cur_WG + : (cur_WG < NumMmaWarpGroups - 1 ? cur_WG + 1 : 0); + cutlass::arch::NamedBarrier::arrive( + 2 * cutlass::NumThreadsPerWarpGroup, + static_cast(FwdNamedBarriers::WarpSchedulerWG1) + + next_WG /*id*/); + } + } + + CUTLASS_DEVICE void mma_init() { + // Tell producers that smem_q is ready + cutlass::arch::NamedBarrier::arrive( + NumMmaThreads + cutlass::NumThreadsPerWarp, + static_cast(FwdNamedBarriers::QueryEmpty) /*id*/); + if constexpr (UseSchedulerBarrier) { + // We have NamedBarrier for up to 3 WGs + static_assert(NumMmaWarpGroups == 2 || NumMmaWarpGroups == 3); + // WG1 needs the very first signal to start + if (hstu::canonical_warp_group_idx_nosync() == 1) { + cutlass::arch::NamedBarrier::arrive( + 2 * cutlass::NumThreadsPerWarpGroup, + static_cast(FwdNamedBarriers::WarpSchedulerWG1) /*id*/); + } + } + } + + template + CUTLASS_DEVICE bool mma( + Params const& params, + MainloopPipelineK pipeline_k, + MainloopPipelineV pipeline_v, + PipelineState& smem_pipe_read, + FrgTensorO& tOrO, + int const thread_idx, + int& work_idx, + SeqlenInfo_t const& seqlen_info, + cute::tuple block_coord, + SharedStorage& shared_storage) { + static_assert( + is_rmem::value, "O tensor must be rmem resident."); + static constexpr int kBlockM = get<0>(TileShape_MNK{}); + static constexpr int kBlockN = get<1>(TileShape_MNK{}); + + // can't use auto [m_block, ...] = block_coord since structured binding + // cannot be captured in lambda + int const m_block = get<0>(block_coord); + int const bidh = get<1>(block_coord); + int const bidb = get<2>(block_coord); + int const split_idx = get<3>(block_coord); + if constexpr (Jagged) { + static constexpr int kBlockM = get<0>(TileShape_MNK{}); + if (m_block * kBlockM >= seqlen_info.seqlen_q) { + return false; + } + } + int min_full_attn_seq_len_, max_attn_len_, contextual_seq_len_; + float scalar_scale_val_; + if constexpr (!Cross) { + if (params.num_groups > 1) { + int group_id = bidb / params.batch_size_per_group; + min_full_attn_seq_len_ = params.min_full_attn_seq_len_tensor[group_id]; + max_attn_len_ = params.max_attn_len_tensor[group_id]; + contextual_seq_len_ = params.contextual_seq_len_tensor[group_id]; + } else { + min_full_attn_seq_len_ = params.min_full_attn_seq_len; + max_attn_len_ = params.max_attn_len; + contextual_seq_len_ = params.contextual_seq_len; + } + } + if (params.num_groups > 1) { + int group_id = bidb / params.batch_size_per_group; + int max_seq_len_per_group = params.max_seq_len_tensor[group_id]; + // attention scale + scalar_scale_val_ = params.scalar_scale + ? (params.attn_scale == nullptr ? 1.0f / max_seq_len_per_group + : params.attn_scale[group_id]) + : 0; + } else { + // attention scale + scalar_scale_val_ = params.scalar_scale + ? (params.attn_scale == nullptr ? params.max_seq_len_inv + : params.attn_scale[0]) + : 0; + } + int n_block_min, n_block_max; + if constexpr (Cross) { + auto n_block_min_max = get_cross_n_block_min_max( + seqlen_info.uihlen_q, + seqlen_info.seqlen_q, + seqlen_info.seqlen_kv, + m_block); + n_block_min = get<0>(n_block_min_max); + n_block_max = get<1>(n_block_min_max); + } else { + auto n_block_min_max = get_n_block_min_max( + max_attn_len_, + min_full_attn_seq_len_, + contextual_seq_len_, + seqlen_info.uihlen_q, + m_block); + n_block_min = get<0>(n_block_min_max); + n_block_max = get<1>(n_block_min_max); + } + +#ifdef HSTU_FLASH_ATTN_DEBUG_INFO + if (n_block_max <= n_block_min) { + std::printf( + "mainloop_fwd_sm90: n_block_max <= n_block_min not expected."); + return false; + } +#endif + + Tensor sQ = make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_q.data()), + SmemLayoutQ{}); + Tensor sK = make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_k.data()), + SmemLayoutK{}); + Tensor sV = make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_v.data()), + SmemLayoutVtMma{}); + Tensor sP = [&] { + if constexpr (Mma1_is_RS) { + // We might not have smem_p if !Mma1_is_RS1, just use smem_q as a + // placeholder since we don't use it + return make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_q.data()), + SmemLayoutP{}); + } else { + return make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_p.data()), + SmemLayoutP{}); + } + }(); + + if constexpr (!Mma0_is_RS) { + static_assert( + stride<0>(typename TiledMma0::ALayout{}) == 0 and + stride<0>(typename TiledMma0::BLayout{}) == 0 and + size<0>(typename TiledMma0::ALayout{}) == + cutlass::NumThreadsPerWarpGroup and + size<0>(typename TiledMma0::BLayout{}) == + cutlass::NumThreadsPerWarpGroup, + "Stride of the first mode must be 0 and the size of the mode must be NumThreadsPerWarpGroup"); + } + constexpr int MmaWarpGroups = + size(TiledMma0{}) / cutlass::NumThreadsPerWarpGroup; + Layout warp_group_thread_layout = make_layout( + make_shape(Int{}), + make_stride(Int{})); + + int warp_group_idx = __shfl_sync( + 0xFFFFFFFF, thread_idx / cutlass::NumThreadsPerWarpGroup, 0); + TiledMma0 tiled_mma0; + TiledMma1 tiled_mma1; + auto wg_mma0 = + tiled_mma0.get_slice(warp_group_thread_layout(warp_group_idx)); + auto wg_mma1 = + tiled_mma1.get_slice(warp_group_thread_layout(warp_group_idx)); + + auto smem_tiled_copy_P = make_tiled_copy_C(SmemCopyAtomP{}, tiled_mma0); + auto smem_thr_copy_P = smem_tiled_copy_P.get_thread_slice(thread_idx); + + // Allocate "fragments/descriptors" + Tensor tSrQ = wg_mma0.partition_fragment_A(sQ); + Tensor tSrK = wg_mma0.partition_fragment_B(sK); + Tensor tOrV = wg_mma1.partition_fragment_B(sV); + Tensor tOsP = wg_mma1.partition_fragment_A(sP); + Tensor tPsP = smem_thr_copy_P.partition_D( + cute::as_position_independent_swizzle_tensor(sP)); + + auto consumer_wait = [](auto& pipeline, auto& smem_pipe_read) { + auto barrier_token = pipeline.consumer_try_wait(smem_pipe_read); + pipeline.consumer_wait(smem_pipe_read, barrier_token); + }; + + clear(tOrO); + + int n_block = n_block_max - 1; + + hstu::Mask mask( + thread_idx, + seqlen_info.seqlen_q, + seqlen_info.seqlen_kv, + max_attn_len_, + min_full_attn_seq_len_, + contextual_seq_len_, + seqlen_info.uihlen_q); + + auto& barrier_Q = shared_storage.pipelines.barrier_Q; + barrier_Q.wait(work_idx % 2); + + static constexpr int Qdim = 0; + auto thread_mma = tiled_mma0.get_thread_slice(thread_idx); + auto thread0_mma = tiled_mma0.get_thread_slice(_0{}); + Tensor cS = cute::make_identity_tensor(Shape, Int>{}); + Tensor tScS = thread_mma.partition_C(cS); + Tensor tScS_rowcol = make_tensor( + tScS.data(), + hstu::convert_layout_acc_rowcol(tScS.layout())); + Tensor t0ScS = thread0_mma.partition_C(cS); + Tensor t0ScS_rowcol = make_tensor( + t0ScS.data(), + hstu::convert_layout_acc_rowcol(t0ScS.layout())); + int const thread_qdim_offset = get(tScS_rowcol(_0{}, _0{})); + SiluScaleOp silu_scale_op; + int qdim_offset = params.scalar_scale + ? 0 + : m_block * kBlockM + thread_qdim_offset + seqlen_info.offset_q; + + if constexpr (Mma0_is_RS) { + using SmemCopyAtomQ = Copy_Atom; + auto smem_tiled_copy_Q = make_tiled_copy_A(SmemCopyAtomQ{}, tiled_mma0); + auto smem_thr_copy_Q = smem_tiled_copy_Q.get_thread_slice(thread_idx); + Tensor tSrQ_copy_view = smem_thr_copy_Q.retile_D(tSrQ); + Tensor tSsQ_copy_view = smem_thr_copy_Q.partition_S( + cute::as_position_independent_swizzle_tensor(sQ)); + cute::copy(smem_tiled_copy_Q, tSsQ_copy_view, tSrQ_copy_view); + } + + Tensor tSrS = + partition_fragment_C(tiled_mma0, select<0, 1>(TileShape_MNK{})); + Tensor tSrS_rowcol = make_tensor( + tSrS.data(), + hstu::convert_layout_acc_rowcol(tSrS.layout())); + consumer_wait(pipeline_k, smem_pipe_read); + hstu::gemm( + tiled_mma0, tSrQ, tSrK(_, _, _, smem_pipe_read.index()), tSrS); + warpgroup_wait<0>(); + pipeline_k.consumer_release(smem_pipe_read); +#pragma unroll + for (int mi = 0; mi < size<0>(tSrS_rowcol); ++mi) { + float scale = scalar_scale_val_; + if (!params.scalar_scale) { + int q_index = qdim_offset + int(get(t0ScS_rowcol(mi, _0{}))); + // Convert global index to local sequence position for bounds checking + int q_local = q_index - seqlen_info.offset_q; + if (q_local < seqlen_info.seqlen_q) { + scale = params.attn_scale[q_index]; + } + } +#pragma unroll + for (int ni = 0; ni < size<1>(tSrS_rowcol); ++ni) { + tSrS_rowcol(mi, ni) = + silu_scale_op(tSrS_rowcol(mi, ni) * params.alpha, scale); + } + } + int const m_idx_max = (m_block + 1) * kBlockM; + if constexpr (Cross) { + mask.template apply< + false /*Seqlenq_mask*/, + true /*Seqlenk_mask*/, + Causal, + false /*Local*/, + false /*Contexual_mask*/, + false /*Target_mask*/, + Cross, + false /*Softmax*/>(tSrS, m_block, n_block); + } else { + if (m_idx_max <= seqlen_info.uihlen_q) { + mask.template apply< + false /*Seqlenq_mask*/, + false /*Seqlenk_mask*/, + Causal, + Local, + Contexual_mask, + false /*Target_mask*/, + Cross, + false /*Softmax*/>(tSrS, m_block, n_block); + } else if ( + m_idx_max <= + cute::ceil_div(seqlen_info.uihlen_q, kBlockM) * kBlockM) { + mask.template apply< + false /*Seqlenq_mask*/, + true /*Seqlenk_mask*/, + Causal, + Local, + Contexual_mask, + Has_targets, + Cross, + false /*Softmax*/>(tSrS, m_block, n_block); + } else { + mask.template apply< + false /*Seqlenq_mask*/, + true /*Seqlenk_mask*/, + false /*Causal*/, + false, + Contexual_mask, + Has_targets, + Cross, + false /*Softmax*/>(tSrS, m_block, n_block); + } + } + if constexpr (Is_FP8 && !V_colmajor) { + hstu::permute_Cregs_fp8(tSrS); + } + Tensor tOrP_acc = make_tensor( + tSrS.data(), hstu::convert_layout_acc_Aregs(tSrS.layout())); + Tensor tOrP = make_tensor_like(tOrP_acc); + convert_type_out(tOrP_acc, tOrP); + if constexpr (Is_FP8 && V_colmajor) { + hstu::permute_Aregs_fp8(tOrP); + } + if constexpr (!Mma1_is_RS) { + cute::copy(smem_tiled_copy_P, smem_thr_copy_P.retile_S(tOrP), tPsP); + cutlass::arch::fence_view_async_shared(); + __syncwarp(); // Only need syncwarp since each warp is using its own P + // values for Mma1 + } + --n_block; + + // Each step does gemm0 and silu for iter n_block and gemm1 for prev iter. + auto fwd_step_intra_warp_pipeline = [&](int const n_block, auto mask_fn) { + PipelineState smem_pipe_read_v( + smem_pipe_read.index(), + smem_pipe_read.phase(), + smem_pipe_read.count()); + ++smem_pipe_read; + Tensor tSrS = + partition_fragment_C(tiled_mma0, select<0, 1>(TileShape_MNK{})); + Tensor tSrS_rowcol = make_tensor( + tSrS.data(), + hstu::convert_layout_acc_rowcol(tSrS.layout())); + if (!UseSchedulerBarrier || warp_group_idx == 0) { + consumer_wait(pipeline_k, smem_pipe_read); + } + warp_scheduler_barrier_sync(); + hstu::gemm( + tiled_mma0, tSrQ, tSrK(_, _, _, smem_pipe_read.index()), tSrS); + if (!UseSchedulerBarrier || warp_group_idx == 0) { + consumer_wait(pipeline_v, smem_pipe_read_v); + } + hstu::gemm( + tiled_mma1, + cute::conditional_return(tOrP, tOsP), + tOrV(_, _, _, smem_pipe_read_v.index()), + tOrO); + warp_scheduler_barrier_arrive(); + warpgroup_wait<1>(); + pipeline_k.consumer_release(smem_pipe_read); // release K +#pragma unroll + for (int mi = 0; mi < size<0>(tSrS_rowcol); ++mi) { + float scale = scalar_scale_val_; + if (!params.scalar_scale) { + int q_index = qdim_offset + int(get(t0ScS_rowcol(mi, _0{}))); + // Convert global index to local sequence position for bounds checking + int q_local = q_index - seqlen_info.offset_q; + if (q_local < seqlen_info.seqlen_q) { + scale = params.attn_scale[q_index]; + } + } +#pragma unroll + for (int ni = 0; ni < size<1>(tSrS_rowcol); ++ni) { + tSrS_rowcol(mi, ni) = + silu_scale_op(tSrS_rowcol(mi, ni) * params.alpha, scale); + } + } + mask_fn(tSrS, n_block); + warpgroup_wait<0>(); + pipeline_v.consumer_release(smem_pipe_read_v); // release V + if constexpr (Is_FP8 && !V_colmajor) { + hstu::permute_Cregs_fp8(tSrS); + } + convert_type_out(make_tensor(tSrS.data(), tOrP.layout()), tOrP); + if constexpr (Is_FP8 && V_colmajor) { + hstu::permute_Aregs_fp8(tOrP); + } + if constexpr (!Mma1_is_RS) { + cute::copy(smem_tiled_copy_P, smem_thr_copy_P.retile_S(tOrP), tPsP); + } + if constexpr (!Mma1_is_RS) { + cutlass::arch::fence_view_async_shared(); + __syncwarp(); + } + }; + + if constexpr (Cross) { + if constexpr (Causal) { + if (m_idx_max <= + cute::ceil_div(seqlen_info.uihlen_q, kBlockM) * kBlockM) { + auto mask_fn = [&](auto& tSrS, int n_block) { + mask.template apply< + false /*Seqlenq_mask*/, + false /*Seqlenk_mask*/, + Causal, + false /*Local*/, + false /*Contexual_mask*/, + false /*Target_mask*/, + Cross, + false /*Softmax*/>(tSrS, m_block, n_block); + }; + int const m_idx_min = m_block * kBlockM; + int const n_block_min_causal_mask = std::max( + n_block_min, + (m_idx_min + seqlen_info.seqlen_kv - seqlen_info.uihlen_q) / + kBlockN); +#pragma unroll 1 + for (; n_block >= n_block_min_causal_mask; --n_block) { + fwd_step_intra_warp_pipeline(n_block, mask_fn); + } + } + } + auto no_mask_fn = [](auto& tSrS, int n_block) {}; +#pragma unroll 1 + for (; n_block >= n_block_min; --n_block) { + fwd_step_intra_warp_pipeline(n_block, no_mask_fn); + } + } else { + if constexpr (Causal || Local) { // Separate iterations with causal + // or local masking + if (m_idx_max <= + cute::ceil_div(seqlen_info.uihlen_q, kBlockM) * kBlockM) { + auto mask_fn = [&](auto& tSrS, int n_block) { + mask.template apply< + false /*Seqlenq_mask*/, + false /*Seqlenk_mask*/, + Causal, + Local, + Contexual_mask, + false /*Has_targets*/, + Cross, + false /*Softmax*/>(tSrS, m_block, n_block); + }; + int const m_idx_min = m_block * kBlockM; + int const n_block_min_causal_local_mask = + std::max(n_block_min, m_idx_min / kBlockN); +#pragma unroll 1 + for (; n_block >= n_block_min_causal_local_mask; --n_block) { + fwd_step_intra_warp_pipeline(n_block, mask_fn); + } + } + } + int n_block_min_before_local_mask = n_block_min; + if constexpr (Local) { + if (m_idx_max <= + cute::ceil_div( + seqlen_info.uihlen_q - min_full_attn_seq_len_, kBlockM) * + kBlockM) { + n_block_min_before_local_mask = std::max( + n_block_min, cute::ceil_div(m_idx_max - max_attn_len_, kBlockN)); + } + } + auto no_mask_fn = [](auto& tSrS, int n_block) {}; +#pragma unroll 1 + for (; n_block >= n_block_min_before_local_mask; --n_block) { + fwd_step_intra_warp_pipeline(n_block, no_mask_fn); + } + // Separate masking iterations on the left for local attention + if constexpr (Local) { + auto local_mask_fn = [&](auto& tSrS, int n_block) { + mask.template apply< + false /*Seqlenq_mask*/, + false /*Seqlenk_mask*/, + false /*Causal_mask*/, + Local, + Contexual_mask, + false /*Has_targets*/, + Cross, + false /*Softmax*/>(tSrS, m_block, n_block); + }; +#pragma unroll 1 + for (; n_block >= n_block_min; --n_block) { + fwd_step_intra_warp_pipeline(n_block, local_mask_fn); + } + } + // Target part GEMM + if constexpr (Has_targets) { + auto [target_n_block_min, target_n_block_max] = + get_target_n_block_min_max( + n_block_max, + seqlen_info.uihlen_q, + seqlen_info.seqlen_kv, + m_block); + auto target_mask_fn = [&](auto& tSrS, int n_block) { + mask.template apply< + false /*Seqlenq_mask*/, + true /*Seqlenk_mask*/, + false /*Causal_mask*/, + false /*Local*/, + Contexual_mask, + Has_targets, + Cross, + false /*Softmax*/>(tSrS, m_block, n_block); + }; +#pragma unroll 1 + for (n_block = target_n_block_max - 1; n_block >= target_n_block_min; + --n_block) { + fwd_step_intra_warp_pipeline(n_block, target_mask_fn); + } + } + if constexpr (Contexual_mask) { + int contexual_n_block_max = get_contexual_n_block_max( + n_block_min, + min_full_attn_seq_len_, + contextual_seq_len_, + seqlen_info.uihlen_q, + m_block); + auto contexual_mask_fn = [&](auto& tSrS, int n_block) { + mask.template apply< + false /*Seqlenq_mask*/, + true /*Seqlenk_mask*/, + false /*Causal_mask*/, + Local, + Contexual_mask, + Has_targets, + Cross, + false /*Softmax*/>(tSrS, m_block, n_block); + }; +#pragma unroll 1 + for (n_block = contexual_n_block_max - 1; n_block >= 0; --n_block) { + fwd_step_intra_warp_pipeline(n_block, contexual_mask_fn); + } + } + } + // Tell producers that smem_q is ready + cutlass::arch::NamedBarrier::arrive( + NumMmaThreads + cutlass::NumThreadsPerWarp, + static_cast(FwdNamedBarriers::QueryEmpty) /*id*/); + consumer_wait(pipeline_v, smem_pipe_read); + hstu::gemm( + tiled_mma1, + cute::conditional_return(tOrP, tOsP), + tOrV(_, _, _, smem_pipe_read.index()), + tOrO); + warpgroup_wait<0>(); + pipeline_v.consumer_release( + smem_pipe_read); // release V, otherwise producers will hang + if constexpr (Is_FP8 && !V_colmajor) { + hstu::permute_output_fp8(tOrO); + } + ++smem_pipe_read; + ++work_idx; + return true; + } + + template + CUTLASS_DEVICE bool mma_softmax( + Params const& params, + MainloopPipelineK pipeline_k, + MainloopPipelineV pipeline_v, + PipelineState& smem_pipe_read, + FrgTensorO& tOrO, + Softmax& softmax, + int const thread_idx, + int& work_idx, + SeqlenInfo_t const& seqlen_info, + cute::tuple block_coord, + SharedStorage& shared_storage) { + static_assert( + is_rmem::value, "O tensor must be rmem resident."); + static constexpr int kBlockM = get<0>(TileShape_MNK{}); + static constexpr int kBlockN = get<1>(TileShape_MNK{}); + + // can't use auto [m_block, ...] = block_coord since structured binding + // cannot be captured in lambda + int const m_block = get<0>(block_coord); + int const bidh = get<1>(block_coord); + int const bidb = get<2>(block_coord); + int const split_idx = get<3>(block_coord); + if constexpr (Jagged) { + static constexpr int kBlockM = get<0>(TileShape_MNK{}); + if (m_block * kBlockM >= seqlen_info.seqlen_q) { + return false; + } + } + int min_full_attn_seq_len_, max_attn_len_, contextual_seq_len_; + if constexpr (!Cross) { + if (params.num_groups > 1) { + int group_id = bidb / params.batch_size_per_group; + min_full_attn_seq_len_ = params.min_full_attn_seq_len_tensor[group_id]; + max_attn_len_ = params.max_attn_len_tensor[group_id]; + contextual_seq_len_ = params.contextual_seq_len_tensor[group_id]; + } else { + min_full_attn_seq_len_ = params.min_full_attn_seq_len; + max_attn_len_ = params.max_attn_len; + contextual_seq_len_ = params.contextual_seq_len; + } + } + int n_block_min, n_block_max; + if constexpr (Cross) { + auto n_block_min_max = get_cross_n_block_min_max( + seqlen_info.uihlen_q, + seqlen_info.seqlen_q, + seqlen_info.seqlen_kv, + m_block); + n_block_min = get<0>(n_block_min_max); + n_block_max = get<1>(n_block_min_max); + } else { + auto n_block_min_max = get_n_block_min_max( + max_attn_len_, + min_full_attn_seq_len_, + contextual_seq_len_, + seqlen_info.uihlen_q, + m_block); + n_block_min = get<0>(n_block_min_max); + n_block_max = get<1>(n_block_min_max); + } + +#ifdef HSTU_FLASH_ATTN_DEBUG_INFO + if (n_block_max <= n_block_min) { + std::printf( + "mainloop_fwd_sm90: n_block_max <= n_block_min not expected."); + return false; + } +#endif + + Tensor sQ = make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_q.data()), + SmemLayoutQ{}); + Tensor sK = make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_k.data()), + SmemLayoutK{}); + Tensor sV = make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_v.data()), + SmemLayoutVtMma{}); + Tensor sP = [&] { + if constexpr (Mma1_is_RS) { + // We might not have smem_p if !Mma1_is_RS1, just use smem_q as a + // placeholder since we don't use it + return make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_q.data()), + SmemLayoutP{}); + } else { + return make_tensor( + make_smem_ptr(shared_storage.tensors.mainloop.smem_p.data()), + SmemLayoutP{}); + } + }(); + + if constexpr (!Mma0_is_RS) { + static_assert( + stride<0>(typename TiledMma0::ALayout{}) == 0 and + stride<0>(typename TiledMma0::BLayout{}) == 0 and + size<0>(typename TiledMma0::ALayout{}) == + cutlass::NumThreadsPerWarpGroup and + size<0>(typename TiledMma0::BLayout{}) == + cutlass::NumThreadsPerWarpGroup, + "Stride of the first mode must be 0 and the size of the mode must be NumThreadsPerWarpGroup"); + } + constexpr int MmaWarpGroups = + size(TiledMma0{}) / cutlass::NumThreadsPerWarpGroup; + Layout warp_group_thread_layout = make_layout( + make_shape(Int{}), + make_stride(Int{})); + + int warp_group_idx = __shfl_sync( + 0xFFFFFFFF, thread_idx / cutlass::NumThreadsPerWarpGroup, 0); + TiledMma0 tiled_mma0; + TiledMma1 tiled_mma1; + auto wg_mma0 = + tiled_mma0.get_slice(warp_group_thread_layout(warp_group_idx)); + auto wg_mma1 = + tiled_mma1.get_slice(warp_group_thread_layout(warp_group_idx)); + + auto smem_tiled_copy_P = make_tiled_copy_C(SmemCopyAtomP{}, tiled_mma0); + auto smem_thr_copy_P = smem_tiled_copy_P.get_thread_slice(thread_idx); + + // Allocate "fragments/descriptors" + Tensor tSrQ = wg_mma0.partition_fragment_A(sQ); + Tensor tSrK = wg_mma0.partition_fragment_B(sK); + Tensor tOrV = wg_mma1.partition_fragment_B(sV); + Tensor tOsP = wg_mma1.partition_fragment_A(sP); + Tensor tPsP = smem_thr_copy_P.partition_D( + cute::as_position_independent_swizzle_tensor(sP)); + + auto consumer_wait = [](auto& pipeline, auto& smem_pipe_read) { + auto barrier_token = pipeline.consumer_try_wait(smem_pipe_read); + pipeline.consumer_wait(smem_pipe_read, barrier_token); + }; + + clear(tOrO); + + int n_block = n_block_max - 1; + + hstu::Mask mask( + thread_idx, + seqlen_info.seqlen_q, + seqlen_info.seqlen_kv, + max_attn_len_, + min_full_attn_seq_len_, + contextual_seq_len_, + seqlen_info.uihlen_q); + + auto& barrier_Q = shared_storage.pipelines.barrier_Q; + barrier_Q.wait(work_idx % 2); + static constexpr int Qdim = 0; + auto thread_mma = tiled_mma0.get_thread_slice(thread_idx); + auto thread0_mma = tiled_mma0.get_thread_slice(_0{}); + Tensor cS = cute::make_identity_tensor(Shape, Int>{}); + Tensor tScS = thread_mma.partition_C(cS); + Tensor tScS_rowcol = make_tensor( + tScS.data(), + hstu::convert_layout_acc_rowcol(tScS.layout())); + Tensor t0ScS = thread0_mma.partition_C(cS); + Tensor t0ScS_rowcol = make_tensor( + t0ScS.data(), + hstu::convert_layout_acc_rowcol(t0ScS.layout())); + int const thread_qdim_offset = get(tScS_rowcol(_0{}, _0{})); + int qdim_offset = params.scalar_scale + ? 0 + : m_block * kBlockM + thread_qdim_offset + seqlen_info.offset_q; + + if constexpr (Mma0_is_RS) { + using SmemCopyAtomQ = Copy_Atom; + auto smem_tiled_copy_Q = make_tiled_copy_A(SmemCopyAtomQ{}, tiled_mma0); + auto smem_thr_copy_Q = smem_tiled_copy_Q.get_thread_slice(thread_idx); + Tensor tSrQ_copy_view = smem_thr_copy_Q.retile_D(tSrQ); + Tensor tSsQ_copy_view = smem_thr_copy_Q.partition_S( + cute::as_position_independent_swizzle_tensor(sQ)); + cute::copy(smem_tiled_copy_Q, tSsQ_copy_view, tSrQ_copy_view); + } + + Tensor tSrS = + partition_fragment_C(tiled_mma0, select<0, 1>(TileShape_MNK{})); + Tensor tSrS_rowcol = make_tensor( + tSrS.data(), + hstu::convert_layout_acc_rowcol(tSrS.layout())); + consumer_wait(pipeline_k, smem_pipe_read); + hstu::gemm( + tiled_mma0, tSrQ, tSrK(_, _, _, smem_pipe_read.index()), tSrS); + warpgroup_wait<0>(); + pipeline_k.consumer_release(smem_pipe_read); + int const m_idx_max = (m_block + 1) * kBlockM; + if constexpr (Cross) { + mask.template apply< + false /*Seqlenq_mask*/, + true /*Seqlenk_mask*/, + Causal, + false /*Local*/, + false /*Contexual_mask*/, + false /*Target_mask*/, + Cross, + true /*Softmax*/>(tSrS, m_block, n_block); + } else { + if (m_idx_max <= seqlen_info.uihlen_q) { + mask.template apply< + false /*Seqlenq_mask*/, + false /*Seqlenk_mask*/, + Causal, + Local, + Contexual_mask, + false /*Target_mask*/, + Cross, + true /*Softmax*/>(tSrS, m_block, n_block); + } else if ( + m_idx_max <= + cute::ceil_div(seqlen_info.uihlen_q, kBlockM) * kBlockM) { + mask.template apply< + false /*Seqlenq_mask*/, + true /*Seqlenk_mask*/, + Causal, + Local, + Contexual_mask, + Has_targets, + Cross, + true /*Softmax*/>(tSrS, m_block, n_block); + } else { + mask.template apply< + false /*Seqlenq_mask*/, + true /*Seqlenk_mask*/, + false /*Causal*/, + false, + Contexual_mask, + Has_targets, + Cross, + true /*Softmax*/>(tSrS, m_block, n_block); + } + } + Tensor scores_scale = softmax.template max_get_scale< + /*Is_first=*/true, + /*Check_inf=*/true>(tSrS); + softmax.template online_softmax( + tSrS); + if constexpr (Is_FP8 && !V_colmajor) { + hstu::permute_Cregs_fp8(tSrS); + } + Tensor tOrP_acc = make_tensor( + tSrS.data(), hstu::convert_layout_acc_Aregs(tSrS.layout())); + Tensor tOrP = make_tensor_like(tOrP_acc); + convert_type_out(tOrP_acc, tOrP); + if constexpr (Is_FP8 && V_colmajor) { + hstu::permute_Aregs_fp8(tOrP); + } + if constexpr (!Mma1_is_RS) { + cute::copy(smem_tiled_copy_P, smem_thr_copy_P.retile_S(tOrP), tPsP); + cutlass::arch::fence_view_async_shared(); + __syncwarp(); // Only need syncwarp since each warp is using its own P + // values for Mma1 + } + --n_block; + + // Each step does gemm0 and softmax for iter n_block and gemm1 for prev + auto fwd_step_intra_warp_pipeline = [&](int const n_block, + auto mask_fn, + auto check_inf_type) { + static constexpr bool Check_inf = decltype(check_inf_type)::value; + PipelineState smem_pipe_read_v( + smem_pipe_read.index(), + smem_pipe_read.phase(), + smem_pipe_read.count()); + ++smem_pipe_read; + Tensor tSrS = + partition_fragment_C(tiled_mma0, select<0, 1>(TileShape_MNK{})); + Tensor tSrS_rowcol = make_tensor( + tSrS.data(), + hstu::convert_layout_acc_rowcol(tSrS.layout())); + if (!UseSchedulerBarrier || warp_group_idx == 0) { + consumer_wait(pipeline_k, smem_pipe_read); + } + warp_scheduler_barrier_sync(); + hstu::gemm( + tiled_mma0, tSrQ, tSrK(_, _, _, smem_pipe_read.index()), tSrS); + if (!UseSchedulerBarrier || warp_group_idx == 0) { + consumer_wait(pipeline_v, smem_pipe_read_v); + } + hstu::gemm( + tiled_mma1, + cute::conditional_return(tOrP, tOsP), + tOrV(_, _, _, smem_pipe_read_v.index()), + tOrO); + warp_scheduler_barrier_arrive(); + warpgroup_wait<1>(); + pipeline_k.consumer_release(smem_pipe_read); // release K + mask_fn(tSrS, n_block); + cute::copy( + softmax.template max_get_scale(tSrS), + scores_scale); + softmax.template online_softmax(tSrS); + warpgroup_wait<0>(); + pipeline_v.consumer_release(smem_pipe_read_v); // release V + if constexpr (Is_FP8 && !V_colmajor) { + hstu::permute_Cregs_fp8(tSrS); + } + convert_type_out(make_tensor(tSrS.data(), tOrP.layout()), tOrP); + if constexpr (Is_FP8 && V_colmajor) { + hstu::permute_Aregs_fp8(tOrP); + } + softmax.rescale_o(tOrO, scores_scale); + if constexpr (!Mma1_is_RS) { + cute::copy(smem_tiled_copy_P, smem_thr_copy_P.retile_S(tOrP), tPsP); + } + if constexpr (!Mma1_is_RS) { + cutlass::arch::fence_view_async_shared(); + __syncwarp(); + } + }; + + if constexpr (Cross) { + if constexpr (Causal) { + if (m_idx_max <= + cute::ceil_div(seqlen_info.uihlen_q, kBlockM) * kBlockM) { + auto mask_fn = [&](auto& tSrS, int n_block) { + mask.template apply< + false /*Seqlenq_mask*/, + false /*Seqlenk_mask*/, + Causal, + false /*Local*/, + false /*Contexual_mask*/, + false /*Target_mask*/, + Cross, + true /*Softmax*/>(tSrS, m_block, n_block); + }; + int const m_idx_min = m_block * kBlockM; + int const n_block_min_causal_mask = std::max( + n_block_min, + (m_idx_min + seqlen_info.seqlen_kv - seqlen_info.uihlen_q) / + kBlockN); +#pragma unroll 1 + for (; n_block >= n_block_min_causal_mask; --n_block) { + fwd_step_intra_warp_pipeline(n_block, mask_fn, cute::true_type{}); + } + } + } + auto no_mask_fn = [](auto& tSrS, int n_block) {}; +#pragma unroll 1 + for (; n_block >= n_block_min; --n_block) { + fwd_step_intra_warp_pipeline(n_block, no_mask_fn, cute::false_type{}); + } + } else { + if constexpr (Causal || Local) { // Separate iterations with causal + // or local masking + if (m_idx_max <= + cute::ceil_div(seqlen_info.uihlen_q, kBlockM) * kBlockM) { + auto mask_fn = [&](auto& tSrS, int n_block) { + mask.template apply< + false /*Seqlenq_mask*/, + false /*Seqlenk_mask*/, + Causal, + Local, + Contexual_mask, + false /*Has_targets*/, + Cross, + true /*Softmax*/>(tSrS, m_block, n_block); + }; + int const m_idx_min = m_block * kBlockM; + int const n_block_min_causal_local_mask = + std::max(n_block_min, m_idx_min / kBlockN); +#pragma unroll 1 + for (; n_block >= n_block_min_causal_local_mask; --n_block) { + fwd_step_intra_warp_pipeline(n_block, mask_fn, cute::true_type{}); + } + } + } + int n_block_min_before_local_mask = n_block_min; + if constexpr (Local) { + if (m_idx_max <= + cute::ceil_div( + seqlen_info.uihlen_q - min_full_attn_seq_len_, kBlockM) * + kBlockM) { + n_block_min_before_local_mask = std::max( + n_block_min, cute::ceil_div(m_idx_max - max_attn_len_, kBlockN)); + } + } + auto no_mask_fn = [](auto& tSrS, int n_block) {}; +#pragma unroll 1 + for (; n_block >= n_block_min_before_local_mask; --n_block) { + fwd_step_intra_warp_pipeline(n_block, no_mask_fn, cute::false_type{}); + } + // Separate masking iterations on the left for local attention + if constexpr (Local) { + auto local_mask_fn = [&](auto& tSrS, int n_block) { + mask.template apply< + false /*Seqlenq_mask*/, + false /*Seqlenk_mask*/, + false /*Causal_mask*/, + Local, + Contexual_mask, + false /*Has_targets*/, + Cross, + true /*Softmax*/>(tSrS, m_block, n_block); + }; +#pragma unroll 1 + for (; n_block >= n_block_min; --n_block) { + fwd_step_intra_warp_pipeline( + n_block, local_mask_fn, cute::true_type{}); + } + } + // Target part GEMM + if constexpr (Has_targets) { + auto [target_n_block_min, target_n_block_max] = + get_target_n_block_min_max( + n_block_max, + seqlen_info.uihlen_q, + seqlen_info.seqlen_kv, + m_block); + auto target_mask_fn = [&](auto& tSrS, int n_block) { + mask.template apply< + false /*Seqlenq_mask*/, + true /*Seqlenk_mask*/, + false /*Causal_mask*/, + false /*Local*/, + Contexual_mask, + Has_targets, + Cross, + true /*Softmax*/>(tSrS, m_block, n_block); + }; +#pragma unroll 1 + for (n_block = target_n_block_max - 1; n_block >= target_n_block_min; + --n_block) { + fwd_step_intra_warp_pipeline( + n_block, target_mask_fn, cute::true_type{}); + } + } + if constexpr (Contexual_mask) { + int contexual_n_block_max = get_contexual_n_block_max( + n_block_min, + min_full_attn_seq_len_, + contextual_seq_len_, + seqlen_info.uihlen_q, + m_block); + auto contexual_mask_fn = [&](auto& tSrS, int n_block) { + mask.template apply< + false /*Seqlenq_mask*/, + true /*Seqlenk_mask*/, + false /*Causal_mask*/, + Local, + Contexual_mask, + Has_targets, + Cross, + true /*Softmax*/>(tSrS, m_block, n_block); + }; +#pragma unroll 1 + for (n_block = contexual_n_block_max - 1; n_block >= 0; --n_block) { + fwd_step_intra_warp_pipeline( + n_block, contexual_mask_fn, cute::true_type{}); + } + } + } + // Tell producers that smem_q is ready + cutlass::arch::NamedBarrier::arrive( + NumMmaThreads + cutlass::NumThreadsPerWarp, + static_cast(FwdNamedBarriers::QueryEmpty) /*id*/); + consumer_wait(pipeline_v, smem_pipe_read); + hstu::gemm( + tiled_mma1, + cute::conditional_return(tOrP, tOsP), + tOrV(_, _, _, smem_pipe_read.index()), + tOrO); + cute::copy(softmax.finalize(1.0f), scores_scale); + warpgroup_wait<0>(); + pipeline_v.consumer_release( + smem_pipe_read); // release V, otherwise producers will hang + softmax.rescale_o(tOrO, scores_scale); + if constexpr (Is_FP8 && !V_colmajor) { + hstu::permute_output_fp8(tOrO); + } + ++smem_pipe_read; + ++work_idx; + return true; + } +}; + +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/mask.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/mask.h new file mode 100644 index 000000000..e35af5193 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/mask.h @@ -0,0 +1,396 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/****************************************************************************** + * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, + *Pradeep Ramani, Tri Dao. + ******************************************************************************/ + +#pragma once + +#include + +#include "utils.h" + +namespace hstu { + +using namespace cute; + +template +struct Mask { + int const thread_idx; + int const max_q_len; + int const max_kv_len; + int const max_attn_len; + int const min_full_attn_seq_len; + int const contextual_seq_len; + int const max_uih_len; + + CUTLASS_DEVICE + Mask( + const int thread_idx, + const int max_q_len, + const int max_kv_len, + const int max_attn_len, + const int min_full_attn_seq_len, + const int contextual_seq_len, + const int max_uih_len) + : thread_idx(thread_idx), + max_q_len(max_q_len), + max_kv_len(max_kv_len), + max_attn_len(max_attn_len), + min_full_attn_seq_len(min_full_attn_seq_len), + contextual_seq_len(contextual_seq_len), + max_uih_len(max_uih_len) {}; + + template < + bool Seqlenq_mask = false, + bool Seqlenk_mask = false, + bool Causal_mask = false, + bool Local_mask = false, + bool Contexual_mask = false, + bool Target_mask = false, // If Target_mask, Seqlenk_mask will be disabled + bool Cross = false, + bool Softmax = false, + typename Engine, + typename Layout> + CUTLASS_DEVICE void apply( + Tensor& tSrS, + const int m_block, + const int n_block) const { + static_assert( + !(Causal_mask && Local_mask), "Cannot be both causal and local"); + static_assert(Layout::rank == 3, "Only support 3D Tensor"); + if constexpr (Cross) { + static_assert( + (!Local_mask) && (!Contexual_mask) && (!Target_mask), + "Local, contexual, and target masks not supported under cross attention"); + } + if (!Seqlenq_mask && !Seqlenk_mask && !Causal_mask && !Local_mask && + !Target_mask) { + return; + } + + auto thread_mma = TiledMma{}.get_thread_slice(thread_idx); + auto thread0_mma = TiledMma{}.get_thread_slice(_0{}); + + static constexpr int Qdim = !SwapAB ? 0 : 1, Kdim = !SwapAB ? 1 : 0; + + Tensor cS = cute::make_identity_tensor( + Shape< + Int, + Int>{}); + Tensor tScS = thread_mma.partition_C(cS); + Tensor tSrS_rowcol = make_tensor( + tSrS.data(), + hstu::convert_layout_acc_rowcol(tSrS.layout())); + Tensor tScS_rowcol = make_tensor( + tScS.data(), + hstu::convert_layout_acc_rowcol(tScS.layout())); + Tensor t0ScS = thread0_mma.partition_C(cS); + Tensor t0ScS_rowcol = make_tensor( + t0ScS.data(), + hstu::convert_layout_acc_rowcol(t0ScS.layout())); + // We want to use the col indices of thread0 to compare, since that is known + // at compile time. So we subtract the limit by the first col index of this + // thread + int const thread_kdim_offset = get(tScS_rowcol(_0{}, _0{})); + int const thread_qdim_offset = get(tScS_rowcol(_0{}, _0{})); + int const seqlen_k_limit = + max_kv_len - n_block * BlockN - thread_kdim_offset; + int const uihlen_k_limit = + max_uih_len - n_block * BlockN - thread_kdim_offset; + int const seqlen_q_limit = + max_q_len - m_block * BlockM - thread_qdim_offset; + if constexpr (Seqlenq_mask) { +#pragma unroll + for (int m = 0; m < size<0>(tSrS_rowcol); ++m) { + if (int(get(t0ScS_rowcol(m, _0{}))) >= seqlen_q_limit) { +#pragma unroll + for (int n = 0; n < size<1>(tSrS_rowcol); ++n) { + if constexpr (Softmax) { + tSrS_rowcol(m, n) = -INFINITY; + } else { + tSrS_rowcol(m, n) = 0.0f; + } + } + } + } + } + if constexpr (Cross) { + if constexpr (!Causal_mask) { + if constexpr (Seqlenk_mask) { +#pragma unroll + for (int n = 0; n < size<1>(tSrS_rowcol); ++n) { + int const t0_col_idx = int(get(t0ScS_rowcol(_0{}, n))); + if (t0_col_idx >= seqlen_k_limit) { +#pragma unroll + for (int m = 0; m < size<0>(tSrS_rowcol); ++m) { + if constexpr (Softmax) { + tSrS_rowcol(m, n) = -INFINITY; + } else { + tSrS_rowcol(m, n) = 0.0f; + } + } + } + } + } + } else { + int const causal_row_offset = max_kv_len - max_uih_len + 1 - + n_block * BlockN - thread_kdim_offset; +#pragma unroll + for (int m = 0; m < size<0>(tSrS_rowcol); ++m) { + if constexpr (Seqlenq_mask) { + if (int(get(t0ScS_rowcol(m, _0{}))) >= seqlen_q_limit) { + continue; + } + } + int const row_idx = get(t0ScS_rowcol(m, _0{})) + + m_block * BlockM + thread_qdim_offset; + int const col_limit_right = !Seqlenk_mask + ? row_idx + causal_row_offset + : __viaddmin_s32(row_idx, causal_row_offset, seqlen_k_limit); +#pragma unroll + for (int n = 0; n < size<1>(tSrS_rowcol); ++n) { + int const t0_col_idx = int(get(t0ScS_rowcol(_0{}, n))); + if (t0_col_idx >= col_limit_right) { + if constexpr (Softmax) { + tSrS_rowcol(m, n) = -INFINITY; + } else { + tSrS_rowcol(m, n) = 0.0f; + } + } + } + } + } + } else { + if constexpr (!Causal_mask && !Local_mask) { + if constexpr (Seqlenk_mask || Target_mask) { +#pragma unroll + for (int n = 0; n < size<1>(tSrS_rowcol); ++n) { + int const t0_col_idx = int(get(t0ScS_rowcol(_0{}, n))); + if constexpr (Target_mask) { + if (t0_col_idx >= uihlen_k_limit) { + bool const oob_predicate = (t0_col_idx >= seqlen_k_limit); + int const col_offset = + t0_col_idx - seqlen_k_limit + seqlen_q_limit; +#pragma unroll + for (int m = 0; m < size<0>(tSrS_rowcol); ++m) { + int const t0_row_idx = int(get(t0ScS_rowcol(m, _0{}))); + if ((t0_row_idx != col_offset) || oob_predicate) { + if constexpr (Softmax) { + tSrS_rowcol(m, n) = -INFINITY; + } else { + tSrS_rowcol(m, n) = 0.0f; + } + } + } + } + } else if constexpr (Seqlenk_mask) { + if (t0_col_idx >= seqlen_k_limit) { +#pragma unroll + for (int m = 0; m < size<0>(tSrS_rowcol); ++m) { + if constexpr (Softmax) { + tSrS_rowcol(m, n) = -INFINITY; + } else { + tSrS_rowcol(m, n) = 0.0f; + } + } + } + } + } + } + } else { // Causal_mask or Local_mask + int const causal_row_offset = 1 - n_block * BlockN - thread_kdim_offset; + if constexpr (Causal_mask) { +#pragma unroll + for (int m = 0; m < size<0>(tSrS_rowcol); ++m) { + if constexpr (Seqlenq_mask) { + if (int(get(t0ScS_rowcol(m, _0{}))) >= seqlen_q_limit) { + continue; + } + } + if constexpr (Contexual_mask) { + if (int(get(t0ScS_rowcol(m, _0{}))) < + contextual_seq_len - m_block * BlockM - thread_qdim_offset) { +#pragma unroll + for (int n = 0; n < size<1>(tSrS_rowcol); ++n) { + int const t0_col_idx = int(get(t0ScS_rowcol(_0{}, n))); + if (t0_col_idx >= uihlen_k_limit) { + if constexpr (Softmax) { + tSrS_rowcol(m, n) = -INFINITY; + } else { + tSrS_rowcol(m, n) = 0.0f; + } + } + } + continue; + } + } + int const row_idx = get(t0ScS_rowcol(m, _0{})) + + m_block * BlockM + thread_qdim_offset; + if constexpr (!Target_mask) { + int const col_limit_right = !Seqlenk_mask + ? row_idx + causal_row_offset + : __viaddmin_s32(row_idx, causal_row_offset, seqlen_k_limit); +#pragma unroll + for (int n = 0; n < size<1>(tSrS_rowcol); ++n) { + int const t0_col_idx = int(get(t0ScS_rowcol(_0{}, n))); + if (t0_col_idx >= col_limit_right) { + if constexpr (Softmax) { + tSrS_rowcol(m, n) = -INFINITY; + } else { + tSrS_rowcol(m, n) = 0.0f; + } + } + } + } else { +#pragma unroll + for (int n = 0; n < size<1>(tSrS_rowcol); ++n) { + int const t0_col_idx = int(get(t0ScS_rowcol(_0{}, n))); + int const col_idx = + t0_col_idx + n_block * BlockN + thread_kdim_offset; + bool const uih_cond = + (t0_col_idx >= row_idx + causal_row_offset) && + (row_idx < max_uih_len); + bool const target_cond = (row_idx != col_idx) && + (row_idx >= max_uih_len) && (col_idx >= max_uih_len); + bool const seqlen_k_cond = (t0_col_idx >= seqlen_k_limit); + if (uih_cond || target_cond || seqlen_k_cond) { + if constexpr (Softmax) { + tSrS_rowcol(m, n) = -INFINITY; + } else { + tSrS_rowcol(m, n) = 0.0f; + } + } + } + } + } + } else { // Local_mask + int const local_row_offset_left = + causal_row_offset - 1 - max_attn_len; + int const col_limit_sink = 0 - n_block * BlockN - thread_kdim_offset; +#pragma unroll + for (int m = 0; m < size<0>(tSrS_rowcol); ++m) { + if constexpr (Seqlenq_mask) { + if (int(get(t0ScS_rowcol(m, _0{}))) >= seqlen_q_limit) { + continue; + } + } + if constexpr (Contexual_mask) { + if (int(get(t0ScS_rowcol(m, _0{}))) < + contextual_seq_len - m_block * BlockM - thread_qdim_offset) { +#pragma unroll + for (int n = 0; n < size<1>(tSrS_rowcol); ++n) { + int const t0_col_idx = int(get(t0ScS_rowcol(_0{}, n))); + if (t0_col_idx >= uihlen_k_limit) { + if constexpr (Softmax) { + tSrS_rowcol(m, n) = -INFINITY; + } else { + tSrS_rowcol(m, n) = 0.0f; + } + } + } + continue; + } + } + int const row_idx = get(t0ScS_rowcol(m, _0{})) + + m_block * BlockM + thread_qdim_offset; + int col_limit_left = row_idx + local_row_offset_left; + if constexpr (Contexual_mask) { + // row contexual without sink + if (col_limit_left + n_block * BlockN + thread_kdim_offset < + contextual_seq_len) { + col_limit_left = 0; + } + } + if constexpr (!Target_mask) { + int const col_limit_right = !Seqlenk_mask + ? row_idx + causal_row_offset + : __viaddmin_s32(row_idx, causal_row_offset, seqlen_k_limit); +#pragma unroll + for (int n = 0; n < size<1>(tSrS_rowcol); ++n) { + int const t0_col_idx = int(get(t0ScS_rowcol(m, n))); + if (row_idx < max_uih_len - min_full_attn_seq_len) { + bool const local_left_cond = Contexual_mask + ? (t0_col_idx < col_limit_left && + t0_col_idx >= col_limit_sink) + : (t0_col_idx < col_limit_left); + if (local_left_cond) { + if constexpr (Softmax) { + tSrS_rowcol(m, n) = -INFINITY; + } else { + tSrS_rowcol(m, n) = 0.0f; + } + } + } + if (t0_col_idx >= col_limit_right) { + if constexpr (Softmax) { + tSrS_rowcol(m, n) = -INFINITY; + } else { + tSrS_rowcol(m, n) = 0.0f; + } + } + } + } else { + int const col_limit_right = row_idx + causal_row_offset; +#pragma unroll + for (int n = 0; n < size<1>(tSrS_rowcol); ++n) { + int const t0_col_idx = int(get(t0ScS_rowcol(_0{}, n))); + if (row_idx < max_uih_len) { + if (row_idx < max_uih_len - min_full_attn_seq_len) { + bool const local_left_cond = Contexual_mask + ? (t0_col_idx < col_limit_left && + t0_col_idx >= col_limit_sink) + : (t0_col_idx < col_limit_left); + if (local_left_cond) { + if constexpr (Softmax) { + tSrS_rowcol(m, n) = -INFINITY; + } else { + tSrS_rowcol(m, n) = 0.0f; + } + } + } + if (t0_col_idx >= col_limit_right) { + if constexpr (Softmax) { + tSrS_rowcol(m, n) = -INFINITY; + } else { + tSrS_rowcol(m, n) = 0.0f; + } + } + } else { + int const col_idx = + t0_col_idx + n_block * BlockN + thread_kdim_offset; + bool const target_cond = (row_idx != col_idx) && + (row_idx >= max_uih_len) && (col_idx >= max_uih_len); + bool const seqlen_k_cond = (t0_col_idx >= seqlen_k_limit); + if (target_cond || seqlen_k_cond) { + if constexpr (Softmax) { + tSrS_rowcol(m, n) = -INFINITY; + } else { + tSrS_rowcol(m, n) = 0.0f; + } + } + } + } + } + } + } + } + } + }; +}; + +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/named_barrier.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/named_barrier.h new file mode 100644 index 000000000..79dce0dd9 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/named_barrier.h @@ -0,0 +1,101 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/****************************************************************************** + * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, + *Pradeep Ramani, Tri Dao. + ******************************************************************************/ + +#pragma once + +#include "cutlass/arch/barrier.h" + +namespace hstu { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +// cutlass::arch::NamedBarrier::sync/arrive are only enabled Sm90 even though +// they work for Sm80 as well. We reimplement them here, enabled for both Sm90 +// and Sm80. + +CUTLASS_DEVICE +static void named_barrier_sync(uint32_t num_threads, uint32_t barrier_id_) { + static constexpr uint32_t ReservedNamedBarrierCount = static_cast( + cutlass::arch::ReservedNamedBarriers::FirstUserBarrier); + uint32_t barrier_id = barrier_id_ + ReservedNamedBarrierCount; + asm volatile("bar.sync %0, %1;" : : "r"(barrier_id), "r"(num_threads)); + cutlass::arch::synclog_emit_named_barrier_arrive_and_wait( + __LINE__, num_threads, barrier_id); +} + +CUTLASS_DEVICE +static void named_barrier_sync( + uint32_t num_threads, + cutlass::arch::ReservedNamedBarriers reserved_named_barriers) { + uint32_t barrier_id = static_cast(reserved_named_barriers); + asm volatile("bar.sync %0, %1;" : : "r"(barrier_id), "r"(num_threads)); + cutlass::arch::synclog_emit_named_barrier_arrive_and_wait( + __LINE__, num_threads, barrier_id); +} + +CUTLASS_DEVICE +static void named_barrier_arrive(uint32_t num_threads, uint32_t barrier_id_) { + static constexpr uint32_t ReservedNamedBarrierCount = static_cast( + cutlass::arch::ReservedNamedBarriers::FirstUserBarrier); + uint32_t barrier_id = barrier_id_ + ReservedNamedBarrierCount; + cutlass::arch::synclog_emit_named_barrier_arrive( + __LINE__, num_threads, barrier_id); + asm volatile("bar.arrive %0, %1;" : : "r"(barrier_id), "r"(num_threads)); +} + +CUTLASS_DEVICE +static void named_barrier_arrive( + uint32_t num_threads, + cutlass::arch::ReservedNamedBarriers reserved_named_barriers) { + uint32_t barrier_id = static_cast(reserved_named_barriers); + cutlass::arch::synclog_emit_named_barrier_arrive( + __LINE__, num_threads, barrier_id); + asm volatile("bar.arrive %0, %1;" : : "r"(barrier_id), "r"(num_threads)); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// +// Enumerates the reserved named barriers to avoid potential conflicts + +enum class FwdNamedBarriers { + QueryEmpty = 0, + ProducerWG = 1, + TileCountSmemEmpty = 2, + TileCountSmemFull = 3, + WarpSchedulerWG1 = 4, + WarpSchedulerWG2 = 5, + WarpSchedulerWG3 = 6, +}; + +enum class BwdNamedBarriers { + KVEmpty = 0, + PdS = 1, + // This needs to match FwdNamedBarriers::TileCountSmemEmpty since + // TileScheduler uses it + TileCountSmemEmpty = 2, + TileCountSmemFull = 3, + dQEmptyWG1 = 4, + dQEmptyWG2 = 5, + dQEmptyWG3 = 6, + dQFullWG1 = 7, + dQFullWG2 = 8, + dQFullWG3 = 9, +}; + +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/seqlen.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/seqlen.h new file mode 100644 index 000000000..c5721b272 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/seqlen.h @@ -0,0 +1,134 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/****************************************************************************** + * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, + *Pradeep Ramani, Tri Dao. + ******************************************************************************/ + +#pragma once + +#include + +namespace hstu { + +// We consolidate all the info related to sequence length here. This is so that +// we can do all the gmem reads once at the beginning of each tile, rather than +// having to repeat these reads to compute various things like n_block_min, +// n_block_max, etc. + +template +struct SeqlenInfo { + int const offset, offset_padded; + int const seqlen; + + CUTLASS_DEVICE + SeqlenInfo( + int const bidb, + int const seqlen_static, + int const* const seq_offsets) + : offset(!Jagged ? 0 : seq_offsets[bidb]), + offset_padded( + !Jagged ? 0 + : (seq_offsets[bidb] + bidb * kBlock) / kBlock * kBlock), + seqlen( + !Jagged ? seqlen_static + : (seq_offsets[bidb + 1] - seq_offsets[bidb])) {} +}; + +template +struct SeqlenInfoQKBwd { + int const offset_q, offset_k, offset_q_padded; + int const seqlen_q, seqlen_kv, uihlen_q; + + CUTLASS_DEVICE + SeqlenInfoQKBwd( + int const bidb, + int const max_q_len, + int const max_kv_len, + int const* const seq_offsets, + int const* const seq_offsets_q, + int const* const num_targets) + : offset_q( + !Jagged ? 0 : (Cross ? seq_offsets_q[bidb] : seq_offsets[bidb])), + offset_k(!Jagged ? 0 : seq_offsets[bidb]) + // If jagged, the layout for dQaccum is that we pad + // each sequence in the batch by an extra kBlockM, so that the write for + // each sequence doesn't touch the next sequence. Sequence i starts at + // seq_offsets[i] + i * kBlockM and ends at seq_offsets[i + 1] + i * + // kBlockM However, the start must align to multiples of kBlockM. + , + offset_q_padded( + !Jagged ? 0 + : Cross + ? ((seq_offsets_q[bidb] + bidb * kBlockM) / kBlockM * kBlockM) + : ((seq_offsets[bidb] + bidb * kBlockM) / kBlockM * kBlockM)), + seqlen_q( + !Jagged ? max_q_len + : (Cross ? (seq_offsets_q[bidb + 1] - seq_offsets_q[bidb]) + : (seq_offsets[bidb + 1] - seq_offsets[bidb]))), + seqlen_kv( + !Jagged ? max_kv_len : (seq_offsets[bidb + 1] - seq_offsets[bidb])), + uihlen_q( + !Jagged + ? (Has_targets ? max_q_len - num_targets[bidb] : max_q_len) + : (Has_targets + ? (Cross ? (seq_offsets_q[bidb + 1] - + seq_offsets_q[bidb] - num_targets[bidb]) + : (seq_offsets[bidb + 1] - seq_offsets[bidb] - + num_targets[bidb])) + : (Cross + ? (seq_offsets_q[bidb + 1] - seq_offsets_q[bidb]) + : (seq_offsets[bidb + 1] - seq_offsets[bidb])))) { + } +}; + +template +struct SeqlenInfoQKFwd { + int const offset_q, offset_k; + int const seqlen_q, seqlen_kv, uihlen_q; + + CUTLASS_DEVICE + SeqlenInfoQKFwd( + int const bidb, + int const max_q_len, + int const max_kv_len, + int const* const seq_offsets, + int const* const seq_offsets_q, + int const* const num_targets) + : offset_q( + !Jagged ? 0 : (Cross ? seq_offsets_q[bidb] : seq_offsets[bidb])), + offset_k(!Jagged ? 0 : seq_offsets[bidb]), + seqlen_q( + !Jagged ? max_q_len + : (Cross ? (seq_offsets_q[bidb + 1] - seq_offsets_q[bidb]) + : (seq_offsets[bidb + 1] - seq_offsets[bidb]))), + seqlen_kv( + !Jagged ? max_kv_len : (seq_offsets[bidb + 1] - seq_offsets[bidb])), + uihlen_q( + !Jagged + ? (Has_targets ? max_q_len - num_targets[bidb] : max_q_len) + : (Has_targets + ? (Cross ? (seq_offsets_q[bidb + 1] - + seq_offsets_q[bidb] - num_targets[bidb]) + : (seq_offsets[bidb + 1] - seq_offsets[bidb] - + num_targets[bidb])) + : (Cross + ? (seq_offsets_q[bidb + 1] - seq_offsets_q[bidb]) + : (seq_offsets[bidb + 1] - seq_offsets[bidb])))) { + } +}; + +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/sm90_pipeline_no_cluster.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/sm90_pipeline_no_cluster.h new file mode 100644 index 000000000..b6428e79c --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/sm90_pipeline_no_cluster.h @@ -0,0 +1,150 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/****************************************************************************** + * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, + *Pradeep Ramani, Tri Dao. + ******************************************************************************/ + +#pragma once + +#include + +namespace cutlass { + +using namespace cute; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +// As of Cutlass v3.6.0, if size(ClusterShape) == 1, PipelineTmaAsync has all +// threads signaling the barrier during consumer_release. This causes a perf +// regression in FA3 forward pass (especially hdim 128 causal). We instead +// reimplement the version of PipelineTmaAsync before v3.6.0 where only 1 out of +// 128 threads signals the barrier. +// +// Assumption: params.num_consumers % NumThreadsPerWarpGroup == 0 +template > +class PipelineTmaAsyncNoCluster : public Base { + public: + using FullBarrier = typename Base::FullBarrier; + using EmptyBarrier = typename Base::EmptyBarrier; + static constexpr uint32_t Stages = Stages_; + using PipelineState = typename Base::PipelineState; + + using SharedStorage = typename Base::SharedStorage; + using ThreadCategory = typename Base::ThreadCategory; + using Params = typename Base::Params; + + static CUTLASS_DEVICE void init_barriers( + SharedStorage& storage, + Params params) { + int warp_idx = canonical_warp_idx_sync(); + bool is_initializing_warp = (warp_idx == 0); + if (is_initializing_warp) { + // Barrier FULL and EMPTY init + constexpr int producer_arv_cnt = 1; + uint32_t const num_consumer_warpgroups_per_cluster = + params.num_consumers / NumThreadsPerWarpGroup; + uint32_t const multicast_consumer_arrival_count = + num_consumer_warpgroups_per_cluster; + + cutlass::arch::detail::initialize_barrier_array_pair_aligned< + decltype(storage.full_barrier_), + decltype(storage.empty_barrier_), + Stages>( + storage.full_barrier_, + storage.empty_barrier_, + producer_arv_cnt, + multicast_consumer_arrival_count); + } + cutlass::arch::fence_barrier_init(); + } + + template + CUTLASS_DEVICE PipelineTmaAsyncNoCluster( + SharedStorage& storage, + Params params, + ClusterShape cluster_shape, + InitBarriers = {}, + InitMasks = {}) + : Base( + storage, + params, + make_shape(_1{}, _1{}, _1{}) /*cluster_shape*/, + cute::false_type{} /*init_barriers*/, + cute::false_type{} /*init_masks*/), + empty_barrier_ptr_(&storage.empty_barrier_[0]) { + int warp_idx = canonical_warp_idx_sync(); + int lane_predicate = cute::elect_one_sync(); + + static_assert( + cute::is_same_v || + cute::is_same_v); + static_assert( + cute::is_same_v || + cute::is_same_v); + if constexpr (cute::is_same_v) { + init_barriers(storage, params); + } + } + + // Constructor + template + CUTLASS_DEVICE PipelineTmaAsyncNoCluster( + SharedStorage& storage, + Params params, + ClusterShape cluster_shape) + : PipelineTmaAsyncNoCluster( + storage, + params, + cluster_shape, + cute::true_type{}, + cute::true_type{}) {} + + template + CUTLASS_DEVICE PipelineTmaAsyncNoCluster( + SharedStorage& storage, + Params params, + ClusterShape cluster_shape, + InitBarriers = {}) + : PipelineTmaAsyncNoCluster( + storage, + params, + cluster_shape, + InitBarriers{}, + cute::true_type{}) {} + + CUTLASS_DEVICE + void consumer_release(PipelineState state) { + consumer_release(state.index()); + } + + private: + EmptyBarrier* const empty_barrier_ptr_ = nullptr; + + // Consumer signalling Producer of completion + // Ensures all blocks in the Same Row and Column get notifed. + CUTLASS_DEVICE + void consumer_release(uint32_t stage, uint32_t skip = false) { + empty_barrier_ptr_[stage].arrive( + 0 /*dst_blockid_*/, + uint32_t(threadIdx.x % cutlass::NumThreadsPerWarpGroup == 0) & + (!skip) /*is_signaling_thread*/); + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // end namespace cutlass diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/softmax.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/softmax.h new file mode 100644 index 000000000..1bd6131c6 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/softmax.h @@ -0,0 +1,256 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/****************************************************************************** + * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, + *Pradeep Ramani, Tri Dao. + ******************************************************************************/ + +#pragma once + +#include +#include +#include +#include "utils.h" + +namespace hstu { + +using namespace cute; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template < + bool zero_init = true, + typename Engine0, + typename Layout0, + typename Engine1, + typename Layout1, + typename Operator> +__device__ __forceinline__ void thread_reduce_( + Tensor const& tensor, + Tensor& summary, + Operator& op) { + static_assert(Layout0::rank == 2, "Only support 2D Tensor"); + static_assert(Layout1::rank == 1, "Only support 1D Tensor"); + CUTE_STATIC_ASSERT_V(size<0>(summary) == size<0>(tensor)); +#pragma unroll + for (int ni = 0; ni < size<1>(tensor); ni++) { +#pragma unroll + for (int mi = 0; mi < size<0>(tensor); mi++) { + summary(mi) = zero_init && ni == 0 ? tensor(mi, ni) + : op(summary(mi), tensor(mi, ni)); + } + } +} + +template < + typename Engine0, + typename Layout0, + typename Engine1, + typename Layout1, + typename Operator> +__device__ __forceinline__ void quad_allreduce_( + Tensor& dst, + Tensor& src, + Operator& op) { + CUTE_STATIC_ASSERT_V(size(dst) == size(src)); +#pragma unroll + for (int i = 0; i < size(dst); i++) { + dst(i) = Allreduce<4>::run(src(i), op); + } +} + +template < + bool zero_init = true, + typename Engine0, + typename Layout0, + typename Engine1, + typename Layout1, + typename Operator> +__device__ __forceinline__ void reduce_( + Tensor const& tensor, + Tensor& summary, + Operator& op) { + thread_reduce_(tensor, summary, op); + quad_allreduce_(summary, summary, op); +} + +template < + bool zero_init = true, + typename Engine0, + typename Layout0, + typename Engine1, + typename Layout1> +__device__ __forceinline__ void reduce_max( + Tensor const& tensor, + Tensor& max) { + MaxOp max_op; + reduce_(tensor, max, max_op); +} + +template < + bool zero_init = true, + bool warp_reduce = true, + typename Engine0, + typename Layout0, + typename Engine1, + typename Layout1> +__device__ __forceinline__ void reduce_sum( + Tensor const& tensor, + Tensor& sum) { + SumOp sum_op; + thread_reduce_(tensor, sum, sum_op); + if constexpr (warp_reduce) { + quad_allreduce_(sum, sum, sum_op); + } +} + +// Apply the exp to all the elements. +template < + bool Scale_max = true, + bool Check_inf = true, + int Max_offset = 0, + typename Engine0, + typename Layout0, + typename Engine1, + typename Layout1> +__forceinline__ __device__ void scale_apply_exp2( + Tensor& tensor, + Tensor const& max, + const float scale) { + // For FP8, we can subtract max by 8.0 so that the value after exp2 is in the + // range of [0, 256]. This lets us use more of the FP8 range (instead of just + // [0, 1]) to reduce underflow. + static constexpr float max_offset = + float(Max_offset); // We can only template on int, not float + static_assert(Layout0::rank == 2, "Only support 2D Tensor"); + static_assert(Layout1::rank == 1, "Only support 1D Tensor"); + CUTE_STATIC_ASSERT_V(size<0>(max) == size<0>(tensor)); +#pragma unroll + for (int mi = 0; mi < size<0>(tensor); ++mi) { + // If max is -inf, then all elements must have been -inf (possibly due to + // masking). We don't want (-inf - (-inf)) since that would give NaN. + const float max_scaled = Check_inf + ? (max(mi) == -INFINITY + ? 0.f + : (!Scale_max ? max(mi) : max(mi) * scale) - max_offset) + : (!Scale_max ? max(mi) : max(mi) * scale) - max_offset; +#pragma unroll + for (int ni = 0; ni < size<1>(tensor); ++ni) { + // Instead of computing exp(x - max), we compute exp2(x * log_2(e) - + // max * log_2(e)). This allows the compiler to use the ffma + // instruction instead of fadd and fmul separately. + tensor(mi, ni) = exp2f(tensor(mi, ni) * scale - max_scaled); + } + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct Softmax { + using TensorT = decltype(make_tensor(Shape>{})); + TensorT row_max, row_sum; + float const softmax_scale_log2; + + CUTLASS_DEVICE Softmax(float const softmax_scale_log2_) + : softmax_scale_log2(softmax_scale_log2_) {}; + + template + __forceinline__ __device__ TensorT max_get_scale(Tensor0& acc_s) { + // Reshape acc_s from ((2, 2, V), MMA_M, MMA_N) to (nrow=(2, MMA_M), + // ncol=(2, V, MMA_N)) + Tensor scores = make_tensor( + acc_s.data(), hstu::convert_layout_acc_rowcol(acc_s.layout())); + static_assert(CUTE_STATIC_V(size<0>(scores)) == kNRows); + TensorT scores_scale; + if constexpr (Is_first) { + hstu::template reduce_max(scores, row_max); + cute::fill(scores_scale, 1.f); + } else { + Tensor scores_max_prev = make_fragment_like(row_max); + cute::copy(row_max, scores_max_prev); + hstu::template reduce_max(scores, row_max); +#pragma unroll + for (int mi = 0; mi < size(row_max); ++mi) { + float scores_max_cur = !Check_inf + ? row_max(mi) + : (row_max(mi) == -INFINITY ? 0.0f : row_max(mi)); + scores_scale(mi) = + exp2f((scores_max_prev(mi) - scores_max_cur) * softmax_scale_log2); + row_sum(mi) *= scores_scale(mi); + } + } + return scores_scale; + }; + + template + __forceinline__ __device__ void online_softmax(Tensor0& acc_s) { + // Reshape acc_s from ((2, 2, V), MMA_M, MMA_N) to (nrow=(2, MMA_M), + // ncol=(2, V, MMA_N)) + Tensor scores = make_tensor( + acc_s.data(), hstu::convert_layout_acc_rowcol(acc_s.layout())); + static_assert(CUTE_STATIC_V(size<0>(scores)) == kNRows); + hstu::template scale_apply_exp2( + scores, row_max, softmax_scale_log2); + // We don't do the reduce across threads here since we don't need to use the + // row_sum. We do that reduce at the end when we need to normalize the + // softmax. + hstu::reduce_sum( + scores, row_sum); + }; + + __forceinline__ __device__ TensorT finalize(float const final_scale = 1.f) { + SumOp sum_op; + quad_allreduce_(row_sum, row_sum, sum_op); + TensorT scores_scale; +#pragma unroll + for (int mi = 0; mi < size(row_sum); ++mi) { + float sum = row_sum(mi); + float inv_sum = (sum == 0.f || sum != sum) ? 0.f : 1.f / sum; + scores_scale(mi) = inv_sum * final_scale; + // For FP8, we might have scaled the output of exp by 2**8 so we need to + // divide sum by that amount. + if constexpr (Max_offset != 0) { + static constexpr float sum_scale = 1.f / float(1 << Max_offset); + sum *= sum_scale; + } + row_sum(mi) = (sum == 0.f || sum != sum) + ? -INFINITY + : row_max(mi) * (softmax_scale_log2 * float(M_LN2)) + __logf(sum); + } + return scores_scale; + }; + + template + __forceinline__ __device__ void rescale_o( + Tensor1& acc_o, + TensorT const& scores_scale) { + // Reshape acc_o from (MMA=4, MMA_M, MMA_K) to (nrow=(2, MMA_M), ncol=(2, + // MMA_K)) + Tensor acc_o_rowcol = make_tensor( + acc_o.data(), hstu::convert_layout_acc_rowcol(acc_o.layout())); + static_assert(CUTE_STATIC_V(size<0>(acc_o_rowcol)) == kNRows); +#pragma unroll + for (int mi = 0; mi < size<0>(acc_o_rowcol); ++mi) { +#pragma unroll + for (int ni = 0; ni < size<1>(acc_o_rowcol); ++ni) { + acc_o_rowcol(mi, ni) *= scores_scale(mi); + } + } + }; +}; + +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/static_switch.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/static_switch.h new file mode 100644 index 000000000..c5759c9d2 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/static_switch.h @@ -0,0 +1,135 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Inspired by +// https://github.com/NVIDIA/DALI/blob/main/include/dali/core/static_switch.h +// and https://github.com/pytorch/pytorch/blob/master/aten/src/ATen/Dispatch.h + +#pragma once + +/// @param COND - a boolean expression to switch by +/// @param CONST_NAME - a name given for the constexpr bool variable. +/// @param ... - code to execute for true and false +/// +/// Usage: +/// ``` +/// BOOL_SWITCH(flag, BoolConst, [&] { +/// some_function(...); +/// }); +/// ``` +// + +#define BOOL_SWITCH(COND, CONST_NAME, ...) \ + [&] { \ + if (COND) { \ + constexpr static bool CONST_NAME = true; \ + return __VA_ARGS__(); \ + } else { \ + constexpr static bool CONST_NAME = false; \ + return __VA_ARGS__(); \ + } \ + }() + +#ifdef FLASHATTENTION_DISABLE_LOCAL +#define CAUSAL_LOCAL_SWITCH( \ + CAUSAL_COND, LOCAL_COND, CAUSAL_CONST_NAME, LOCAL_CONST_NAME, ...) \ + [&] { \ + constexpr static bool LOCAL_CONST_NAME = false; \ + if (CAUSAL_COND) { \ + constexpr static bool CAUSAL_CONST_NAME = true; \ + return __VA_ARGS__(); \ + } else { \ + constexpr static bool CAUSAL_CONST_NAME = false; \ + return __VA_ARGS__(); \ + } \ + }() +#else +#define CAUSAL_LOCAL_SWITCH( \ + CAUSAL_COND, LOCAL_COND, CAUSAL_CONST_NAME, LOCAL_CONST_NAME, ...) \ + [&] { \ + if (CAUSAL_COND) { \ + constexpr static bool CAUSAL_CONST_NAME = true; \ + constexpr static bool LOCAL_CONST_NAME = false; \ + return __VA_ARGS__(); \ + } else if (LOCAL_COND) { \ + constexpr static bool CAUSAL_CONST_NAME = false; \ + constexpr static bool LOCAL_CONST_NAME = true; \ + return __VA_ARGS__(); \ + } else { \ + constexpr static bool CAUSAL_CONST_NAME = false; \ + constexpr static bool LOCAL_CONST_NAME = false; \ + return __VA_ARGS__(); \ + } \ + }() +#endif + +#ifdef FLASHATTENTION_DISABLE_CLUSTER +#define CLUSTER_SWITCH(COND, CONST_NAME, ...) \ + [&] { \ + constexpr static bool CONST_NAME = false; \ + return __VA_ARGS__(); \ + }() +#else +#define CLUSTER_SWITCH BOOL_SWITCH +#endif + +// #ifdef FLASHATTENTION_DISABLE_SM8x +#define ARCH_SWITCH(ARCH, ARCH_NAME, ...) \ + [&] { \ + constexpr static int ARCH_NAME = 90; \ + return __VA_ARGS__(); \ + }() +// #else +// #define ARCH_SWITCH(ARCH, ARCH_NAME, ...) \ +// [&] { \ +// if (ARCH < 90) { \ +// constexpr static int ARCH_NAME = 80; \ +// return __VA_ARGS__(); \ +// } else { \ +// constexpr static int ARCH_NAME = 90; \ +// return __VA_ARGS__(); \ +// } \ +// }() +// #endif + +#ifndef FLASHATTENTION_ENABLE_VCOLMAJOR +#define VCOLMAJOR_SWITCH(COND, CONST_NAME, ...) \ + [&] { \ + constexpr static bool CONST_NAME = false; \ + return __VA_ARGS__(); \ + }() +#else +#define VCOLMAJOR_SWITCH BOOL_SWITCH +#endif + +#define HEADDIM_SWITCH(HEADDIM, ...) \ + [&] { \ + if (HEADDIM == 64) { \ + constexpr static int kHeadSize = 64; \ + return __VA_ARGS__(); \ + } else if (HEADDIM == 96) { \ + constexpr static int kHeadSize = 96; \ + return __VA_ARGS__(); \ + } else if (HEADDIM == 128) { \ + constexpr static int kHeadSize = 128; \ + return __VA_ARGS__(); \ + } else if (HEADDIM == 96) { \ + constexpr static int kHeadSize = 96; \ + return __VA_ARGS__(); \ + } else if (HEADDIM == 256) { \ + constexpr static int kHeadSize = 256; \ + return __VA_ARGS__(); \ + } \ + }() diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/tile_scheduler.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/tile_scheduler.h new file mode 100644 index 000000000..cde7837ce --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/tile_scheduler.h @@ -0,0 +1,616 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/****************************************************************************** + * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, + *Pradeep Ramani, Tri Dao. + ******************************************************************************/ + +#pragma once + +#include "cute/tensor.hpp" +#include "cutlass/arch/barrier.h" +#include "cutlass/fast_math.h" + +#include "named_barrier.h" + +namespace hstu { + +/////////////////////////////////////////////////////////////////////////////// + +// Host side kernel arguments +struct TileSchedulerArguments { + int const num_blocks, num_head, num_batch; + int const max_seq_len, headdim, + element_size; // Used to calculate L2 swizzling + int* const tile_count_semaphore = nullptr; + int* const seq_offsets = nullptr; + int* const sort_by_length_indices = nullptr; +}; + +/////////////////////////////////////////////////////////////////////////////// + +template < + bool Jagged = false, + int kBlock = 128, + bool Sort_by_length_indices = false> +class SingleTileScheduler { + public: + using SharedStorage = int; + + // Device side kernel params + struct Params { + int const num_blocks, num_head, num_batch; + int const max_seq_len; + int* const seq_offsets; + int* const sort_by_length_indices; + }; + + static Params to_underlying_arguments(TileSchedulerArguments const& args) { + return { + args.num_blocks, + args.num_head, + args.num_batch, + args.max_seq_len, + !Jagged ? nullptr : args.seq_offsets, + !Sort_by_length_indices ? nullptr : args.sort_by_length_indices}; + } + + static dim3 get_grid_shape(Params const& params, int num_sm) { +#ifdef HSTU_FLASH_ATTN_DEBUG_INFO + std::printf( + "SingleTileScheduler::get_grid_shape: %d, %d, %d\n", + params.num_blocks, + params.num_head, + params.num_batch); +#endif + return { + uint32_t(params.num_blocks), + uint32_t(params.num_head), + uint32_t(params.num_batch)}; + } + + struct WorkTileInfo { + int block_idx = 0; + int bidh = 0; + int bidb = 0; + bool is_valid_tile = false; + + CUTLASS_DEVICE + bool is_valid(Params const& params) const { + return is_valid_tile; + } + + CUTLASS_DEVICE + cute::tuple get_block_coord( + Params const& params) const { + return {block_idx, bidh, bidb, 0 /*split_idx*/}; + } + }; + + CUTLASS_DEVICE + SingleTileScheduler(SharedStorage* const smem_scheduler) {} + + template + CUTLASS_DEVICE WorkTileInfo get_initial_work(Params const& params) const { + int bidb = int(blockIdx.z); + if constexpr (Sort_by_length_indices) { + bidb = params.sort_by_length_indices[bidb]; + } + WorkTileInfo work_info{int(blockIdx.x), int(blockIdx.y), bidb, true}; + if constexpr (Jagged) { + int seqlen = + (params.seq_offsets ? params.seq_offsets[work_info.bidb + 1] - + params.seq_offsets[work_info.bidb] + : params.max_seq_len); + work_info.is_valid_tile = work_info.block_idx * kBlock < seqlen; + } + return work_info; + } + + CUTLASS_DEVICE + void init_consumer() const {} + + CUTLASS_DEVICE + void prefetch_next_work(Params const& params, WorkTileInfo& current_work) + const {} + + template + CUTLASS_DEVICE WorkTileInfo + get_next_work(Params const& params, WorkTileInfo const& current_work) const { + return {-1, -1, -1, false}; + } +}; + +/////////////////////////////////////////////////////////////////////////////// + +class StaticPersistentTileScheduler { + public: + using SharedStorage = int; + + // Device side kernel params + struct Params { + int total_blocks; + cutlass::FastDivmod m_block_divmod, head_divmod; + cutlass::FastDivmod nsplits_divmod; + }; + + static Params to_underlying_arguments(TileSchedulerArguments const& args) { + return { + args.num_blocks * args.num_head * args.num_batch, + cutlass::FastDivmod(args.num_blocks), + cutlass::FastDivmod(args.num_head), + cutlass::FastDivmod(1)}; + } + + static dim3 get_grid_shape(Params const& params, int num_sm) { +#ifdef HSTU_FLASH_ATTN_DEBUG_INFO + std::printf("StaticPersistentTileScheduler::get_grid_shape %d\n", num_sm); +#endif + return {uint32_t(num_sm)}; + } + + struct WorkTileInfo { + int tile_idx; + + CUTLASS_DEVICE + bool is_valid(Params const& params) const { + return tile_idx < params.total_blocks; + } + + CUTLASS_DEVICE + cute::tuple get_block_coord( + Params const& params) const { + int block, bidh, bidb; + bidb = params.head_divmod.divmod( + bidh, params.m_block_divmod.divmod(block, tile_idx)); + int split_idx = 0; + return {block, bidh, bidb, split_idx}; + } + }; + + CUTLASS_DEVICE + StaticPersistentTileScheduler(SharedStorage* const smem_scheduler) {}; + + template + CUTLASS_DEVICE WorkTileInfo get_initial_work(Params const& params) const { + return {int(blockIdx.x)}; + } + + CUTLASS_DEVICE + void init_consumer() const {} + + CUTLASS_DEVICE + void prefetch_next_work(Params const& params, WorkTileInfo& current_work) + const {} + + template + CUTLASS_DEVICE WorkTileInfo + get_next_work(Params const& params, WorkTileInfo const& current_work) const { + return {current_work.tile_idx + int(gridDim.x)}; + } +}; + +template < + int NumMmaThreads = 2 * cutlass::NumThreadsPerWarpGroup, + int NumProducerThreads = cutlass::NumThreadsPerWarp, + bool WarpSpecialized = true> +class DynamicPersistentTileScheduler { + // This scheduler targets the causal (or local) case where each tile takes + // different amount of time. We use longest-processing-time-first scheduling: + // the longest remaining tile is assigned to the first SM that's free. + // SM indicates they are free by incrementing a semaphore. + // However, we have to make sure K & V still fit into L2 cache, so we perform + // scheduling on "sections" of the head & batch dimension, each section + // consisting of e.g. 8 heads. This is the L2 swizzling part. The size of each + // section is precomputed based on the size of K & V and the L2 cache size. + + static_assert(WarpSpecialized || NumProducerThreads == NumMmaThreads); + static constexpr int NumThreads = + WarpSpecialized ? NumMmaThreads + NumProducerThreads : NumMmaThreads; + + public: + using SharedStorage = int; + + protected: + SharedStorage* const tile_count_smem; + + public: + // Device side kernel params + struct Params { + int const total_blocks; + cutlass::FastDivmod const m_block_divmod, head_divmod; + cutlass::FastDivmod const l2_minor_divmod, l2_major_divmod; + cutlass::FastDivmod const l2_minor_residual_divmod; + int const num_hb_quotient; + int* const tile_count_semaphore; + }; + + static Params to_underlying_arguments(TileSchedulerArguments const& args) { + int const size_one_kv_head = + args.max_seq_len * args.headdim * args.element_size * 2; + int const size_l2 = 32 * 1024 * 1024; // 32 MB for K & V + // Swizzle is the size of each "section". Round swizzle to a power of 2 + // If not PackGQA already, the size of each section can increase by + // qhead_per_khead + int const swizzle = (1 << cutlass::find_log2(size_l2 / size_one_kv_head)); + // If we're in the last section (called residual), we don't want to divide + // by swizzle. Instead we want to divide by the remainder. + int const num_hb_remainder = (args.num_head * args.num_batch) % swizzle; + int const num_split_blocks = args.num_blocks; + // printf("num_split_blocks = %d, num_head = %d, num_batch = %d, swizzle = + // %d, PackGQA = %d, qhead_per_khead = %d, num_hb_remainder = %d\n", + // num_split_blocks, args.num_head, args.num_batch, swizzle, int(PackGQA), + // args.qhead_per_khead, num_hb_remainder); + assert(args.tile_count_semaphore != nullptr); + return { + num_split_blocks * args.num_head * args.num_batch, + cutlass::FastDivmod(args.num_blocks), + cutlass::FastDivmod(args.num_head), + cutlass::FastDivmod(swizzle), + cutlass::FastDivmod(swizzle * num_split_blocks), + // don't divide by 0 + cutlass::FastDivmod(num_hb_remainder > 0 ? num_hb_remainder : 1), + (args.num_head * args.num_batch) / swizzle, + args.tile_count_semaphore}; + } + + static dim3 get_grid_shape(Params const& params, int num_sm) { +#ifdef HSTU_FLASH_ATTN_DEBUG_INFO + std::printf("DynamicPersistentTileScheduler::get_grid_shape %d\n", num_sm); +#endif + return {uint32_t(num_sm)}; + } + + struct WorkTileInfo { + int tile_idx; + + CUTLASS_DEVICE + bool is_valid(Params const& params) const { + return tile_idx < params.total_blocks; + } + + CUTLASS_DEVICE + cute::tuple get_block_coord( + Params const& params) const { + int block, bidh, bidb; + int l2_mod, bidhb, bidhb_residual; + bidhb = params.l2_major_divmod.divmod(l2_mod, tile_idx); + // If we're in the last section (called residual), we don't want to divide + // by swizzle. Instead we want to divide by the remainder. + if (bidhb < params.num_hb_quotient) { + block = params.l2_minor_divmod.divmod(bidhb_residual, l2_mod); + } else { + block = params.l2_minor_residual_divmod.divmod(bidhb_residual, l2_mod); + } + bidb = params.head_divmod.divmod( + bidh, bidhb * params.l2_minor_divmod.divisor + bidhb_residual); + int split_idx = 0; + // Longest-processing-time-first + block = params.m_block_divmod.divisor - 1 - block; + return {block, bidh, bidb, split_idx}; + } + }; + + CUTLASS_DEVICE + DynamicPersistentTileScheduler(SharedStorage* const smem_scheduler) + : tile_count_smem(smem_scheduler) {}; + + template + CUTLASS_DEVICE WorkTileInfo get_initial_work(Params const& params) const { + return {int(blockIdx.x)}; + } + + CUTLASS_DEVICE + void init_consumer() const { + if (WarpSpecialized || cutlass::canonical_warp_idx_sync() > 0) { + hstu::named_barrier_arrive( + NumThreads, + static_cast(FwdNamedBarriers::TileCountSmemEmpty) /*id*/); + } + } + + CUTLASS_DEVICE + void prefetch_next_work(Params const& params, WorkTileInfo& current_work) + const { + if (threadIdx.x % NumProducerThreads == 0) { + current_work.tile_idx = + atomicAdd(params.tile_count_semaphore, 1) + int(gridDim.x); + } + } + + template + CUTLASS_DEVICE WorkTileInfo + get_next_work(Params const& params, WorkTileInfo const& current_work) const { + if constexpr (IsProducerWarp) { + // thread 0 already has the right tile_idx, just need to broadcast to the + // rest of warp 0 + int new_tile_idx = + __shfl_sync(0xffffffff, current_work.tile_idx, 0 /*lane*/); + hstu::named_barrier_sync( + NumThreads, + static_cast(FwdNamedBarriers::TileCountSmemEmpty) /*id*/); + if (threadIdx.x % NumProducerThreads == 0) { + *tile_count_smem = current_work.tile_idx; + } + hstu::named_barrier_arrive( + NumThreads, + static_cast(FwdNamedBarriers::TileCountSmemFull) /*id*/); + return {new_tile_idx}; + } else { + hstu::named_barrier_sync( + NumThreads, + static_cast(FwdNamedBarriers::TileCountSmemFull) /*id*/); + int tile_idx = *tile_count_smem; + hstu::named_barrier_arrive( + NumThreads, + static_cast(FwdNamedBarriers::TileCountSmemEmpty) /*id*/); + return {tile_idx}; + } + } +}; + +template < + int kBlock, + int NumMmaThreads = 2 * cutlass::NumThreadsPerWarpGroup, + int NumProducerThreads = cutlass::NumThreadsPerWarp, + bool WarpSpecialized = true> +class VarlenDynamicPersistentTileScheduler { + static_assert(WarpSpecialized || NumProducerThreads == NumMmaThreads); + static constexpr int NumThreads = + WarpSpecialized ? NumMmaThreads + NumProducerThreads : NumMmaThreads; + + public: + using SharedStorage = int4; + + protected: + SharedStorage* const work_info_smem; + + public: + // Device side kernel params + struct Params { + int num_head, num_batch; + int const max_seq_len; + cutlass::FastDivmod nsplits_divmod; + int* const tile_count_semaphore; + int* const seq_offsets; + }; + + static Params to_underlying_arguments(TileSchedulerArguments const& args) { + // If Split, for the purpose of scheduling, we pretend that instead there + // are (args.num_splits * args.num_head) number of heads. + assert(args.tile_count_semaphore != nullptr); + return { + args.num_head, + args.num_batch, + args.max_seq_len, + cutlass::FastDivmod(1), + args.tile_count_semaphore, + args.seq_offsets}; + } + + static dim3 get_grid_shape(Params const& params, int num_sm) { +#ifdef HSTU_FLASH_ATTN_DEBUG_INFO + std::printf( + "VarlenDynamicPersistentTileScheduler::get_grid_shape %d\n", num_sm); +#endif + return {uint32_t(num_sm)}; + } + + struct WorkTileInfo { + int tile_idx, block, bidh, bidb; + + CUTLASS_DEVICE + bool is_valid(Params const& params) const { + // if (blockIdx.x >= 0 && (threadIdx.x == 128 || threadIdx.x == 0)) { + // printf("blockIdx.x = %d, threadIdx.x = %d, checking valid, bidb = %d, + // params.num_batch = %d\n", blockIdx.x, threadIdx.x, bidb, + // params.num_batch); } + return bidb < params.num_batch; + } + + CUTLASS_DEVICE + cute::tuple get_block_coord( + Params const& params) const { + return {block, bidh, bidb, 0 /*split_idx*/}; + } + }; + + CUTLASS_DEVICE + VarlenDynamicPersistentTileScheduler(SharedStorage* const smem_scheduler) + : work_info_smem(smem_scheduler) {}; + + CUTLASS_DEVICE + WorkTileInfo tile_idx_to_work_tile( + Params const& params, + int next_tile_idx, + WorkTileInfo const& current_work) const { + auto prefix_sum = [](int val) { + auto lane = threadIdx.x % cutlass::NumThreadsPerWarp; + CUTLASS_PRAGMA_UNROLL + for (int i = 1; i < cutlass::NumThreadsPerWarp; i <<= 1) { + int32_t partial_sum = __shfl_up_sync(0xffffffff, val, i); + if (lane >= i) { + val += partial_sum; + } + } + return val; + }; + + auto get_num_m_blocks = [&](int bidb_start) { + auto lane = threadIdx.x % cutlass::NumThreadsPerWarp; + int seqlen; + if (params.seq_offsets) { + int cur_cu_seqlen = lane + bidb_start <= params.num_batch + ? params.seq_offsets[lane + bidb_start] + : 0; + int next_cu_seqlen = __shfl_down_sync(0xffffffff, cur_cu_seqlen, 1); + seqlen = next_cu_seqlen - cur_cu_seqlen; + } else { + seqlen = params.max_seq_len; + } + return lane + bidb_start < params.num_batch && + lane < cutlass::NumThreadsPerWarp - 1 + ? cute::ceil_div(seqlen, kBlock) + : 0; + }; + + int num_m_blocks = + get_num_m_blocks(current_work.bidb); // Different for each lane + // Cumulative number of blocks for the next 31 batches + int num_m_blocks_cumulative = prefix_sum(num_m_blocks); + // Total number of blocks for the next 31 batches + int m_blocks_in_group = __shfl_sync( + 0xffffffff, num_m_blocks_cumulative, cutlass::NumThreadsPerWarp - 1); + int group_end_tile = current_work.tile_idx - current_work.block - + current_work.bidh * __shfl_sync(0xffffffff, num_m_blocks, 0 /*lane*/) + + m_blocks_in_group * params.num_head; // Same for all lanes + int bidb = current_work.bidb; + // if (blockIdx.x <= 9 && threadIdx.x == 0) { + // printf("Before while, blockIdx.x = %d, threadIdx.x = %d, bidb = %d, + // num_m_blocks = %d, next_tile_idx = %d, group_end_tile = %d, + // m_blocks_in_group = %d\n", blockIdx.x, threadIdx.x, bidb, + // num_m_blocks, next_tile_idx, group_end_tile, m_blocks_in_group); + // } + while (group_end_tile <= next_tile_idx) { + bidb += cutlass::NumThreadsPerWarp - 1; + if (bidb >= params.num_batch) { + // if (blockIdx.x <= 9 && threadIdx.x == 0) { + // printf("Returning early, blockIdx.x = %d, threadIdx.x = %d, bidb + // = %d, num_m_blocks = %d, next_tile_idx = %d, group_end_tile = %d, + // m_blocks_in_group = %d\n", blockIdx.x, threadIdx.x, bidb, + // num_m_blocks, next_tile_idx, group_end_tile, m_blocks_in_group); + // } + return {next_tile_idx, 0, 0, params.num_batch}; + } + num_m_blocks = get_num_m_blocks(bidb); + num_m_blocks_cumulative = prefix_sum(num_m_blocks); + m_blocks_in_group = __shfl_sync( + 0xffffffff, num_m_blocks_cumulative, cutlass::NumThreadsPerWarp - 1); + group_end_tile += m_blocks_in_group * params.num_head; + // if (blockIdx.x <= 9 && threadIdx.x == 0) { + // printf("Bottom of while, blockIdx.x = %d, threadIdx.x = %d, bidb = + // %d, num_m_blocks = %d, next_tile_idx = %d, group_end_tile = %d, + // m_blocks_in_group = %d\n", blockIdx.x, threadIdx.x, bidb, + // num_m_blocks, next_tile_idx, group_end_tile, m_blocks_in_group); + // } + } + int group_start_tile = group_end_tile - m_blocks_in_group * params.num_head; + // The next problem to process is the first one that does not have ending + // tile position that is greater than or equal to tile index. + int batch_idx_in_group = __popc(__ballot_sync( + 0xffffffff, + group_start_tile + num_m_blocks_cumulative * params.num_head <= + next_tile_idx)); + bidb += batch_idx_in_group; + num_m_blocks = __shfl_sync(0xffffffff, num_m_blocks, batch_idx_in_group); + int mh_block = next_tile_idx - group_start_tile - + (batch_idx_in_group == 0 ? 0 + : __shfl_sync( + 0xffffffff, + num_m_blocks_cumulative, + batch_idx_in_group - 1)) * + params.num_head; + int bidh = mh_block / num_m_blocks; + int block = mh_block - bidh * num_m_blocks; + // if (blockIdx.x <= 9 && threadIdx.x == 0) { + // printf("blockIdx.x = %d, threadIdx.x = %d, batch_idx_in_group = %d, + // bidb = %d, num_m_blocks = %d, next_tile_idx = %d, group_end_tile = + // %d, m_blocks_in_group = %d, mh_block = %d, bidh = %d, block = %d\n", + // blockIdx.x, threadIdx.x, batch_idx_in_group, bidb, num_m_blocks, + // next_tile_idx, group_end_tile, m_blocks_in_group, mh_block, bidh, + // block); + // } + return {next_tile_idx, block, bidh, bidb}; + } + + template + CUTLASS_DEVICE WorkTileInfo get_initial_work(Params const& params) const { + if constexpr (IsProducerWarp) { + WorkTileInfo work_info = + tile_idx_to_work_tile(params, int(blockIdx.x), {0, 0, 0, 0}); + if (threadIdx.x % cutlass::NumThreadsPerWarp == 0) { + *work_info_smem = make_int4( + work_info.tile_idx, + work_info.block, + work_info.bidh, + work_info.bidb); + } + hstu::named_barrier_arrive( + NumThreads, + static_cast(FwdNamedBarriers::TileCountSmemFull) /*id*/); + return work_info; + } else { + return get_next_work(params, {0, 0, 0, 0}); + } + } + + CUTLASS_DEVICE + void init_consumer() const { + // Don't arrive at the TileCountSmemEmpty barrier here, because + // get_initial_work will do that + } + + CUTLASS_DEVICE + void prefetch_next_work(Params const& params, WorkTileInfo& current_work) + const { + if (threadIdx.x % NumProducerThreads == 0) { + current_work.tile_idx = + atomicAdd(params.tile_count_semaphore, 1) + int(gridDim.x); + } + } + + template + CUTLASS_DEVICE WorkTileInfo + get_next_work(Params const& params, WorkTileInfo const& current_work) const { + if constexpr (IsProducerWarp) { + // thread 0 has the next tile_idx, just need to broadcast to the rest of + // warp 0 + int new_tile_idx = + __shfl_sync(0xffffffff, current_work.tile_idx, 0 /*lane*/); + WorkTileInfo work_info = { + __shfl_sync(0xffffffff, current_work.tile_idx, 1 /*lane*/), + current_work.block, + current_work.bidh, + current_work.bidb}; + work_info = tile_idx_to_work_tile(params, new_tile_idx, work_info); + hstu::named_barrier_sync( + NumThreads, + static_cast(FwdNamedBarriers::TileCountSmemEmpty) /*id*/); + if (threadIdx.x % cutlass::NumThreadsPerWarp == 0) { + *work_info_smem = make_int4( + work_info.tile_idx, + work_info.block, + work_info.bidh, + work_info.bidb); + } + hstu::named_barrier_arrive( + NumThreads, + static_cast(FwdNamedBarriers::TileCountSmemFull) /*id*/); + return work_info; + } else { + hstu::named_barrier_sync( + NumThreads, + static_cast(FwdNamedBarriers::TileCountSmemFull) /*id*/); + int4 work_info = *work_info_smem; + hstu::named_barrier_arrive( + NumThreads, + static_cast(FwdNamedBarriers::TileCountSmemEmpty) /*id*/); + return WorkTileInfo{work_info.x, work_info.y, work_info.z, work_info.w}; + } + } +}; + +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/tile_size.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/tile_size.h new file mode 100644 index 000000000..3c8968bda --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/tile_size.h @@ -0,0 +1,220 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/****************************************************************************** + * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, + *Pradeep Ramani, Tri Dao. + ******************************************************************************/ + +#pragma once + +#include + +namespace hstu { + +constexpr int kBlockM_bwd( + const int arch, + const int headdim, + const bool causal, + const bool is_local) { + int const kBlockM_sm90 = headdim <= 64 + ? 64 + : (headdim <= 96 + ? 64 + : (headdim <= 128 ? (causal || is_local ? 64 : 80) : 64)); + int const kBlockM_sm80 = headdim <= 64 ? 128 : 64; + int const kBlockM = arch >= 90 ? kBlockM_sm90 : kBlockM_sm80; + return kBlockM; +} + +constexpr int kBlockN_bwd(const int arch, const int headdim) { + int const kBlockN_sm90 = headdim <= 128 ? 128 : (headdim <= 192 ? 96 : 80); + int const kBlockN_sm80 = headdim <= 128 ? 128 : (headdim <= 192 ? 80 : 64); + int const kBlockN = arch >= 90 ? kBlockN_sm90 : kBlockN_sm80; + return kBlockN; +} + +constexpr int NumMmaWarpGroups_bwd(const int arch, const int headdim) { + if (headdim <= 128) { + return 2; + } else if (headdim == 192) { + return arch >= 90 ? 3 : 2; + } else { + return 2; + } +} + +constexpr bool V_in_regs_bwd(const int arch, const int headdim) { + if (arch >= 90 && headdim == 96) { + return true; + } + return false; +} + +// Stages_dO, Stages_dS_or_QSm80 +constexpr std::tuple Stages_bwd(const int arch, const int headdim) { + if (headdim <= 128) { + return {2, 2}; + } + if (headdim == 192) { + if (arch >= 90) { + return {1, 1}; + } else { + return {1, 2}; + } + } else { + return {1, 1}; + } +} + +// AtomLayoutMSdP, AtomLayoutNdKV, AtomLayoutMdQ +constexpr std::tuple AtomLayout_bwd( + const int arch, + const int headdim) { + if (headdim <= 64) { + if (arch >= 90) { + return {1, 2, 1}; + } else { + return {4, 4, 4}; + } + } else if (headdim <= 96) { + if (arch >= 90) { + return {1, 2, 1}; + } else { + return {2, 4, 2}; + } + } else if (headdim <= 128) { + if (arch >= 90) { + return {1, 2, 1}; + } else { + return {2, 2, 2}; + } + } else { + if (arch >= 90) { + return {1, 1, 1}; + } else { + return {4, 2, 2}; + } + } +} + +// SdP_swapAB, dKV_swapAB, dQ_swapAB +constexpr std::tuple swapAB_bwd( + const int arch, + const int headdim, + const bool causal, + const bool local) { + if (headdim <= 96) { + return {arch >= 90 ? true : false, false, false}; + } else if (headdim == 128) { + bool SdP_swapAB = arch >= 90 ? true : false; + bool dKV_swapAB = false; + bool dQ_swapAB = arch >= 90 ? ((causal || local) ? false : true) : false; + return {SdP_swapAB, dKV_swapAB, dQ_swapAB}; + } else if (headdim == 192) { + return {false, true, false}; + } else { + return {false, arch >= 90 ? true : false, arch >= 90 ? true : false}; + } +} + +// Return {kBlockM, kBlockN, Mma1_is_RS} +constexpr std::tuple tile_size_fwd_sm90( + int headdim, + bool is_causal, + bool is_local, + int element_size = 2, + bool v_colmajor = false, + bool Cross = false, + bool Training = true) { + // for cross attention, q is usually much smaller than k/v, so we reduce the + // BlockM size to increase parallelism + bool small_blockm = Cross && (!Training); + if (element_size == 2) { + if (headdim <= 64) { + return {small_blockm ? 64 : 192, 128, true}; + // Good for long seqlen (>= 4k) but suffers from tile quantization at + // short seqlen return {192, is_causal || is_local ? 192 : 176, true, + // false}; + } else if (headdim <= 96) { + return {small_blockm ? 64 : 192, is_local ? 128 : 144, false}; + } else if (headdim <= 128) { + return {small_blockm ? 64 : 128, is_causal || is_local ? 128 : 176, true}; + // {128, 192, false, false} and {192, 128, false, true} are quite good too + // 128 x 192 hits the limit of smem if Mma1_is_RS, 128 x 144 hits the + // limit if !Mma1_is_RS + } else if (headdim <= 192) { + return { + small_blockm ? 64 : 128, + is_local ? 96 : 112, + true}; // 128 x 112 hits the limit of smem + } else { + return { + small_blockm ? 64 : 128, + is_local ? 64 : 80, + true}; // 128 x 80 hits the limit of smem + } + } else { + if (headdim <= 64) { + return {192, 160, true}; + } else if (headdim <= 96) { + return {192, 128, true}; + } else if (headdim <= 128) { + return {128, (v_colmajor ? 192 : 224), true}; + } else if (headdim <= 192) { + return {128, 160, true}; + } else { + return {128, is_local ? 64 : 128, true}; + } + } +} + +// Return {kBlockM, kBlockN, kNWarps, kStages, Q_in_regs} +constexpr std::tuple tile_size_fwd_sm8x( + bool sm86_or_89, + int headdim, + bool is_causal, + bool is_local, + int element_size = 2) { + if (element_size == 2) { + if (headdim <= 64) { + return {128, (is_local ? 96 : 112), 4, 1, false}; + } else if (headdim <= 96) { + return {128, is_local ? 48 : 64, 4, 1, false}; + } else if (headdim <= 128) { + bool const use_8_warps = sm86_or_89; + return { + 128, + use_8_warps ? (is_local ? 96 : 128) : (is_local ? 48 : 64), + use_8_warps ? 8 : 4, + 1, + use_8_warps}; + } else if (headdim <= 192) { + bool const kBlockN_64 = is_local; + return {128, kBlockN_64 ? 64 : 96, 8, sm86_or_89 ? 1 : 2, !kBlockN_64}; + } else { + return { + 128, + sm86_or_89 ? (is_local ? 48 : 64) : (is_local ? 64 : 96), + 8, + 1, + false}; + } + } else { + // Placeholder for now + return {128, 64, 8, 2, false}; + } +} +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/utils.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/utils.h new file mode 100644 index 000000000..50a065ed4 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/utils.h @@ -0,0 +1,789 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/****************************************************************************** + * Copyright (c) 2024, Tri Dao. + ******************************************************************************/ + +#pragma once + +#include +#include +#include + +#include + +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 +#include +#endif + +#include +#include + +#include +#include +#include +#include + +#define CHECK_CUDA(call) \ + do { \ + cudaError_t status_ = call; \ + if (status_ != cudaSuccess) { \ + fprintf( \ + stderr, \ + "CUDA error (%s:%d): %s\n", \ + __FILE__, \ + __LINE__, \ + cudaGetErrorString(status_)); \ + exit(1); \ + } \ + } while (0) + +#define CHECK_CUDA_KERNEL_LAUNCH() CHECK_CUDA(cudaGetLastError()) + +#ifndef M_LOG2E +#define M_LOG2E 1.44269504088896340735992468100 /* log_2 (e) */ +#endif + +namespace hstu { + +using namespace cute; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +// A wrapper for the kernel that is used to guard against compilation on +// architectures that will never use the kernel. The purpose of this is to +// reduce the size of the compiled binary. +// Adapted from +// https://github.com/vllm-project/vllm/blob/4d29e91be84d27ca313d657eee92c067439a4c23/csrc/quantization/cutlass_w8a8/scaled_mm_c2x.cuh#L55 +template +struct enable_sm90_or_later : Kernel { + template + CUTLASS_DEVICE void operator()(Args&&... args) { +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) + Kernel::operator()(std::forward(args)...); +#endif + } +}; + +template +struct enable_sm80_to_sm89 : Kernel { + template + CUTLASS_DEVICE void operator()(Args&&... args) { +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 800) && (__CUDA_ARCH__ <= 890) + Kernel::operator()(std::forward(args)...); +#endif + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct MaxOp { + __device__ __forceinline__ T operator()(T const& x, T const& y) { + return x > y ? x : y; + } +}; + +template <> +struct MaxOp { + // This is slightly faster + __device__ __forceinline__ float operator()(float const& x, float const& y) { + return max(x, y); + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct SumOp { + __device__ __forceinline__ T operator()(T const& x, T const& y) { + return x + y; + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct SiluScaleOp { + cutlass::epilogue::thread::SiLu silu; + __device__ __forceinline__ T operator()(T const& t, T const& scale) { + float t2 = t / 2; + return t2 * (1 + cutlass::fast_tanh(t2)) * + scale; // __fdividef(t, 1.0f + cutlass::fast_exp(-t)) * scale + } +}; + +template +CUTLASS_DEVICE void inplace_silu_scale( + Tensor& tensor, + T const& scale_before, + T const& scale_after) { + SiluScaleOp silu_scale_op; +#pragma unroll + for (int i = 0; i < size(tensor); ++i) { + tensor(i) = silu_scale_op(tensor(i) * scale_before, scale_after); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct Allreduce { + static_assert(THREADS == 32 || THREADS == 16 || THREADS == 8 || THREADS == 4); + template + static __device__ __forceinline__ T run(T x, Operator& op) { + constexpr int OFFSET = THREADS / 2; + x = op(x, __shfl_xor_sync(uint32_t(-1), x, OFFSET)); + return Allreduce::run(x, op); + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <> +struct Allreduce<2> { + template + static __device__ __forceinline__ T run(T x, Operator& op) { + x = op(x, __shfl_xor_sync(uint32_t(-1), x, 1)); + return x; + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +// For SM80, convert acc_layout from (MMA=4, MMA_M, MMA_N) to (nrow=(2, +// MMA_M), ncol=(2, MMA_N)). For SM90, convert acc_layout from ((2, 2, V), +// MMA_M, MMA_N) to (nrow=(2, MMA_M), ncol=(2, V, MMA_N)) +template +CUTLASS_DEVICE auto convert_layout_acc_rowcol(Layout0 acc_layout) { + if constexpr (decltype(rank<0>(acc_layout))::value == 3) { // SM90 + static_assert(decltype(size<0, 0>(acc_layout))::value == 2); + static_assert(decltype(size<0, 1>(acc_layout))::value == 2); + static_assert(decltype(rank(acc_layout))::value == 3); + auto l = acc_layout; + if constexpr (!Transposed) { + return make_layout( + make_layout(get<0, 1>(l), get<1>(l)), + make_layout(get<0, 0>(l), get<0, 2>(l), get<2>(l))); + } else { + return make_layout( + make_layout(get<0, 0>(l), get<0, 2>(l), get<2>(l)), + make_layout(get<0, 1>(l), get<1>(l))); + } + + } else { // SM80 + static_assert(decltype(size<0>(acc_layout))::value == 4); + static_assert(decltype(rank(acc_layout))::value == 3); + auto l = logical_divide(acc_layout, Shape<_2>{}); // ((2, 2), MMA_M, MMA_N) + if constexpr (!Transposed) { + return make_layout( + make_layout(get<0, 1>(l), get<1>(l)), + make_layout(get<0, 0>(l), get<2>(l))); + } else { + return make_layout( + make_layout(get<0, 0>(l), get<2>(l)), + make_layout(get<0, 1>(l), get<1>(l))); + } + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +// For SM80, convert acc_layout from (MMA=4, MMA_M, MMA_N) to ((4, 2), MMA_M, +// MMA_N / 2) if using m16n8k16, or to (4, MMA_M, MMA_N) if using m16n8k8. For +// SM90, FP16/BF16, convert acc_layout from ((2, 2, N / 8), MMA_M, MMA_N) to +// ((2, 2, 2), MMA_M, (N / 16, MMA_N)) For SM90, FP8, convert acc_layout from +// ((2, 2, N / 8), MMA_M, MMA_N) to ((4, 2, 2), MMA_M, (N / 32, MMA_N)) +template +CUTLASS_DEVICE auto convert_layout_acc_Aregs(Layout0 acc_layout) { + using X = Underscore; + if constexpr (decltype(rank<0>(acc_layout))::value == 3) { // SM90 + static_assert(decltype(size<0, 0>(acc_layout))::value == 2); + static_assert(decltype(size<0, 1>(acc_layout))::value == 2); + static_assert(decltype(rank(acc_layout))::value == 3); + static_assert(decltype(rank(get<0>(acc_layout)))::value == 3); + if constexpr (sizeof(typename MMA_Traits::ValTypeA) == 2) { + auto l = + logical_divide(get<0, 2>(acc_layout), Tile<_2>{}); // ((2, N / 16)) + return make_layout( + make_layout( + get<0, 0>(acc_layout), get<0, 1>(acc_layout), get<0, 0>(l)), + get<1>(acc_layout), + coalesce(make_layout(get<0, 1>(l), get<2>(acc_layout)))); + } else { + static_assert(sizeof(typename MMA_Traits::ValTypeA) == 1); + static_assert(decltype(stride<0, 0>(acc_layout))::value == 1); + static_assert(decltype(stride<0, 1>(acc_layout))::value == 2); + auto l = logical_divide( + get<0, 2>(acc_layout), + Tile>>{}); // (((2, 2), N / 32)) + // This combines the first two modes (<0, 0> and <0, 1>) into one mode. + // Will require register shuffling later to be correct. + return make_layout( + make_layout(Layout<_4>{}, get<0, 0, 0>(l), get<0, 0, 1>(l)), + get<1>(acc_layout), + coalesce(make_layout( + get<0, 1>(l), + get<2>(acc_layout)))); // ((4, 2, 2), MMA_M, N / 32 * MMA_N) + // This combination is right but doesn't work with register shuffling. + // return make_layout(make_layout(coalesce(make_layout(get<0, + // 0>(acc_layout), get<0, 0, 0>(l))), get<0, 1>(acc_layout), get<0, 0, + // 1>(l)), + // get<1>(acc_layout), + // coalesce(make_layout(get<0, 1>(l), + // get<2>(acc_layout)))); + } + } else { // SM80 + static_assert(decltype(size<0>(acc_layout))::value == 4); + static_assert(decltype(rank(acc_layout))::value == 3); + constexpr int mma_shape_K = get<2>(typename MMA_Traits::Shape_MNK{}); + static_assert(mma_shape_K == 8 || mma_shape_K == 16); + if constexpr (mma_shape_K == 8) { + return acc_layout; + } else { + auto l = logical_divide( + acc_layout, Shape{}); // (4, MMA_M, (2, MMA_N / 2))) + return make_layout( + make_layout(get<0>(l), get<2, 0>(l)), get<1>(l), get<2, 1>(l)); + } + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +CUTLASS_DEVICE auto convert_type_unsafe(Tensor const& tensor) { + using From_type = typename Engine::value_type; + static constexpr int numel = decltype(size(tensor))::value; + cutlass::NumericArrayConverter convert_op; + // HACK: this requires tensor to be "contiguous" + auto frag = + convert_op(*reinterpret_cast*>( + tensor.data())); + return make_tensor(make_rmem_ptr(&frag), tensor.layout()); + // Unsafe because we're returning a tensor with memory allocated on the + // stack. If the compiler does not inline this function, then the memory + // might not be valid. +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +CUTLASS_DEVICE void convert_type_out( + Tensor const& tensor, + Tensor& out) { + // Somehow if we allocate out inside this function and return it, e2e is + // slower and the output can be wrong. + using From_type = typename Engine::value_type; + using To_type = typename EngineOut::value_type; + static constexpr int FragmentSize = std::max( + sizeof(From_type) / sizeof(To_type), sizeof(To_type) / sizeof(From_type)); + static_assert( + CUTE_STATIC_V(size(tensor)) % FragmentSize == 0, + "Fragment size does not vectorize properly"); + Tensor frag = recast const>(tensor); + Tensor out_frg = recast>(out); + static_assert(size(frag) == size(out_frg)); + cutlass::NumericArrayConverter convert_op; +#pragma unroll + for (int i = 0; i < size(frag); ++i) { + out_frg[i] = convert_op(frag[i]); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +// Blocks until all but N previous cp.async.commit_group operations have +// committed. This differs from cute::cp_async_wait in that when N = 0 we +// don't call cp.async.wait_all (which is equivalent to commit_group then +// wait_group 0). Instead we just call cp.async.wait_group 0, which is +// slightly faster. +// https://github.com/NVIDIA/cutlass/blob/master/include/cute/arch/copy_sm80.hpp#L113 +template +CUTE_HOST_DEVICE void cp_async_wait() { +#if defined(CUTE_ARCH_CP_ASYNC_SM80_ENABLED) + asm volatile("cp.async.wait_group %0;\n" ::"n"(N)); +#endif +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +CUTLASS_DEVICE auto mma_partition_fragment_AB( + Mma const& mma, + Tensor0 const& tensor0) { + if constexpr (A) { + return mma.partition_fragment_A(tensor0); + } else { + return mma.partition_fragment_B(tensor0); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template < + bool zero_init = false, + int wg_wait = 0, + bool SwapAB = false, + int M_slice = -1, + typename Tensor0, + typename Tensor1, + typename Tensor2, + typename TiledMma> +CUTLASS_DEVICE void gemm( + TiledMma& tiled_mma, + Tensor0 const& tCrA, + Tensor1 const& tCrB, + Tensor2& tCrC) { + if constexpr (M_slice >= 0) { + static constexpr int MMA_M = decltype(size<1>(tCrC))::value; + static_assert(M_slice < MMA_M); + // After logical_divide, C has shape ((2,2,V), (MMA_M, 1), MMA_N) + Tensor tCrC_slice = + cute::logical_divide(tCrC, Shape>{})( + _, make_coord(Int{}, _), _); + if constexpr (!SwapAB) { + Tensor tCrA_slice = + cute::logical_divide(tCrA, Shape>{})( + _, make_coord(Int{}, _), _); + gemm( + tiled_mma, tCrA_slice, tCrB, tCrC_slice); + } else { + Tensor tCrB_slice = + cute::logical_divide(tCrB, Shape>{})( + _, make_coord(Int{}, _), _); + gemm( + tiled_mma, tCrA, tCrB_slice, tCrC_slice); + } + } else { + constexpr bool Is_RS = !cute::is_base_of< + cute::GMMA::DescriptorIterator, + typename TiledMma::FrgTypeA>::value; + // Need to cast away const on tCrA since warpgroup_fence_operand doesn't + // take const + if constexpr (Is_RS) { + if constexpr (!SwapAB) { + warpgroup_fence_operand(const_cast(tCrA)); + } else { + warpgroup_fence_operand(const_cast(tCrB)); + } + } + warpgroup_fence_operand(tCrC); + warpgroup_arrive(); + if constexpr (zero_init) { + tiled_mma.accumulate_ = GMMA::ScaleOut::Zero; + } + // Unroll the K mode manually to set scale D to 1 + CUTLASS_PRAGMA_UNROLL + for (int k_block = 0; k_block < size<2>(tCrA); ++k_block) { + if constexpr (!SwapAB) { + cute::gemm(tiled_mma, tCrA(_, _, k_block), tCrB(_, _, k_block), tCrC); + } else { + cute::gemm(tiled_mma, tCrB(_, _, k_block), tCrA(_, _, k_block), tCrC); + } + tiled_mma.accumulate_ = GMMA::ScaleOut::One; + } + warpgroup_commit_batch(); + if constexpr (wg_wait >= 0) { + warpgroup_wait(); + } + warpgroup_fence_operand(tCrC); + if constexpr (Is_RS) { + if constexpr (!SwapAB) { + warpgroup_fence_operand(const_cast(tCrA)); + } else { + warpgroup_fence_operand(const_cast(tCrB)); + } + } + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template < + bool A_in_regs = false, + bool B_in_regs = false, + bool SwapAB = false, + typename Tensor0, + typename Tensor1, + typename Tensor2, + typename Tensor3, + typename Tensor4, + typename TiledMma, + typename TiledCopyA, + typename TiledCopyB, + typename ThrCopyA, + typename ThrCopyB, + typename Hook> +CUTLASS_DEVICE void gemm_sm80( + Tensor0& acc, + Tensor1& tCrA, + Tensor2& tCrB, + Tensor3 const& tCsA, + Tensor4 const& tCsB, + TiledMma tiled_mma, + TiledCopyA smem_tiled_copy_A, + TiledCopyB smem_tiled_copy_B, + ThrCopyA smem_thr_copy_A, + ThrCopyB smem_thr_copy_B, + Hook fn) { + if constexpr (SwapAB) { + gemm_sm80( + acc, + tCrB, + tCrA, + tCsB, + tCsA, + tiled_mma, + smem_tiled_copy_B, + smem_tiled_copy_A, + smem_thr_copy_B, + smem_thr_copy_A, + fn); + } else { + CUTE_STATIC_ASSERT_V(size<1>(tCrA) == size<1>(acc)); // MMA_M + CUTE_STATIC_ASSERT_V(size<1>(tCrB) == size<2>(acc)); // MMA_N + CUTE_STATIC_ASSERT_V(size<2>(tCrA) == size<2>(tCrB)); // MMA_K + Tensor tCrA_copy_view = smem_thr_copy_A.retile_D(tCrA); + CUTE_STATIC_ASSERT_V(size<1>(tCsA) == size<1>(tCrA_copy_view)); // M + Tensor tCrB_copy_view = smem_thr_copy_B.retile_D(tCrB); + CUTE_STATIC_ASSERT_V(size<1>(tCsB) == size<1>(tCrB_copy_view)); // N + if (!A_in_regs) { + cute::copy( + smem_tiled_copy_A, tCsA(_, _, _0{}), tCrA_copy_view(_, _, _0{})); + } + if (!B_in_regs) { + cute::copy( + smem_tiled_copy_B, tCsB(_, _, _0{}), tCrB_copy_view(_, _, _0{})); + } +#pragma unroll + for (int i = 0; i < size<2>(tCrA); ++i) { + if (i < size<2>(tCrA) - 1) { + if (!A_in_regs) { + cute::copy( + smem_tiled_copy_A, + tCsA(_, _, i + 1), + tCrA_copy_view(_, _, i + 1)); + } + if (!B_in_regs) { + cute::copy( + smem_tiled_copy_B, + tCsB(_, _, i + 1), + tCrB_copy_view(_, _, i + 1)); + } + } + if constexpr (!std::is_same_v) { + if (i == 0) { + fn(); + } + } + cute::gemm(tiled_mma, tCrA(_, _, i), tCrB(_, _, i), acc); + } + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template < + typename Tensor0, + typename Tensor1, + typename Tensor2, + typename Tensor3, + typename TiledMma, + typename TiledCopy, + typename ThrCopy> +CUTLASS_DEVICE void gemm_rs_sm80( + Tensor0& acc, + Tensor1& tCrA, + Tensor2& tCrB, + Tensor3 const& tCsB, + TiledMma tiled_mma, + TiledCopy smem_tiled_copy_B, + ThrCopy smem_thr_copy_B) { + CUTE_STATIC_ASSERT_V(size<1>(tCrA) == size<1>(acc)); // MMA_M + CUTE_STATIC_ASSERT_V(size<1>(tCrB) == size<2>(acc)); // MMA_N + CUTE_STATIC_ASSERT_V(size<2>(tCrA) == size<2>(tCrB)); // MMA_K + Tensor tCrB_copy_view = smem_thr_copy_B.retile_D(tCrB); + CUTE_STATIC_ASSERT_V(size<1>(tCsB) == size<1>(tCrB_copy_view)); // N + cute::copy(smem_tiled_copy_B, tCsB(_, _, _0{}), tCrB_copy_view(_, _, _0{})); +#pragma unroll + for (int i = 0; i < size<2>(tCrA); ++i) { + if (i < size<2>(tCrA) - 1) { + cute::copy( + smem_tiled_copy_B, tCsB(_, _, i + 1), tCrB_copy_view(_, _, i + 1)); + } + cute::gemm(tiled_mma, tCrA(_, _, i), tCrB(_, _, i), acc); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template < + bool Is_even_MN = true, + bool Is_even_K = true, + bool Clear_OOB_MN = false, + bool Clear_OOB_K = true, + class CopyAtom, + class TV, + class Tiler, + typename Engine0, + typename Layout0, + typename Engine1, + typename Layout1, + typename Engine2, + typename Layout2, + typename Engine3, + typename Layout3> +CUTLASS_DEVICE void copy( + TiledCopy const& tiled_copy, + Tensor const& S, + Tensor& D, + Tensor const& identity_MN, + Tensor const& predicate_K, + const int max_MN = 0) { + // Decay TiledCopy to CopyAtom + auto copy_atom = static_cast(tiled_copy); + CUTE_STATIC_ASSERT_V(rank(S) == Int<3>{}); + CUTE_STATIC_ASSERT_V(rank(D) == Int<3>{}); + CUTE_STATIC_ASSERT_V(size<0>(S) == size<0>(D)); // MMA + CUTE_STATIC_ASSERT_V(size<1>(S) == size<1>(D)); // MMA_M + CUTE_STATIC_ASSERT_V(size<2>(S) == size<2>(D)); // MMA_K + // There's no case where !Clear_OOB_K && Clear_OOB_MN + static_assert(!(Clear_OOB_MN && !Clear_OOB_K)); + auto has_with_bool = cute::is_valid( + [](auto t) -> void_t() + .with(true))> {}, + copy_atom); +#pragma unroll + for (int m = 0; m < size<1>(S); ++m) { + bool predicate_mn = + Is_even_MN || get<0>(identity_MN(_0{}, m, _0{})) < max_MN; + if constexpr (Is_even_MN || !Clear_OOB_MN) { + if (Is_even_MN || predicate_mn) { +#pragma unroll + for (int k = 0; k < size<2>(S); ++k) { + if constexpr (Is_even_K || !Clear_OOB_K) { + if (Is_even_K || predicate_K(k)) { + cute::copy(copy_atom, S(_, m, k), D(_, m, k)); + } + } else { // Clear_OOB_K == true && Is_even_K == false + // If copy traits can be transformed with a predicate value, do + // it, otherwise branch here + if constexpr (has_with_bool) { + cute::copy( + copy_atom.with(predicate_K(k)), S(_, m, k), D(_, m, k)); + } else { + if (predicate_K(k)) { + cute::copy(copy_atom, S(_, m, k), D(_, m, k)); + } else { + cute::clear(D(_, m, k)); + } + } + } + } + } + } else { // Clear_OOB_MN == true && Is_even_MN == false, also implies + // Clear_OOB_K == true + if constexpr (!has_with_bool) { + if (predicate_mn) { +#pragma unroll + for (int k = 0; k < size<2>(S); ++k) { + if (Is_even_K || predicate_K(k)) { + cute::copy(copy_atom, S(_, m, k), D(_, m, k)); + } else if (Clear_OOB_K) { + cute::clear(D(_, m, k)); + } + } + } else { + cute::clear(D(_, m, _)); + } + } else { // combine the mn predicate with the k predicate +#pragma unroll + for (int k = 0; k < size<2>(S); ++k) { + cute::copy( + copy_atom.with(predicate_mn && (Is_even_K || predicate_K(k))), + S(_, m, k), + D(_, m, k)); + } + } + } + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +// Byte permute and shuffle to match register layout of +// (FP8 downcasted) accumulator of GEMM-I to FP8 operand A of GEMM-II. +template +CUTLASS_DEVICE void permute_Aregs_fp8(Fragment& frag) { + // frag has shape ((4, 2, 2), MMA_M, MMA_N), each element is 8 bits + static_assert(decltype(size<0, 0>(frag))::value == 4); + static_assert(decltype(size<0, 1>(frag))::value == 2); + static_assert(decltype(stride<0, 0>(frag))::value == 1); + static_assert(decltype(stride<0, 1>(frag))::value == 4); + static_assert(sizeof(typename Fragment::value_type) == 1); + + auto quad_idx = threadIdx.x % 4; + bool lane_03 = quad_idx == 0 || quad_idx == 3; + int selector_upper = lane_03 ? 0x5410 : 0x1054; + int selector_lower = lane_03 ? 0x7632 : 0x3276; + + static constexpr int upper_map[4] = {0, 3, 1, 2}; + // static constexpr int lower_map[4] = {1, 2, 0, 3}; + + Tensor frag_64b = recast(frag); // ((1, 1, 2), MMA_M, MMA_N) +#pragma unroll + for (int i = 0; i < size(frag_64b); ++i) { + uint32_t upper = frag_64b[i].x; + uint32_t lower = frag_64b[i].y; + uint32_t upper0 = lane_03 ? upper : lower; + uint32_t lower0 = lane_03 ? lower : upper; + upper0 = __shfl_sync(uint32_t(-1), upper0, upper_map[quad_idx], 4); + // lower0 = __shfl_sync(uint32_t(-1), lower0, lower_map[quad_idx], 4); + lower0 = __shfl_sync(uint32_t(-1), lower0, upper_map[quad_idx] ^ 1, 4); + frag_64b[i].x = __byte_perm(upper0, lower0, selector_upper); + frag_64b[i].y = __byte_perm(upper0, lower0, selector_lower); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +CUTLASS_DEVICE void permute_Cregs_fp8(Fragment& frag) { + // frag has shape ((2, 2, N / 8), MMA_M, MMA_N), each element is 32 bits + static_assert(decltype(size<0, 0>(frag))::value == 2); + static_assert(decltype(size<0, 1>(frag))::value == 2); + static_assert(decltype(size<0, 2>(frag))::value % 2 == 0); + static_assert(decltype(stride<0, 0>(frag))::value == 1); + static_assert(sizeof(typename Fragment::value_type) == 4); + Tensor frag_64b = + group_modes<1, 3>(recast(frag)); // ((1, 2, N / 8), (MMA_M, MMA_N)) +#pragma unroll + for (int mi = 0; mi < size<1>(frag_64b); ++mi) { +#pragma unroll + for (int i = 0; i < size<0, 2>(frag_64b) / 2; ++i) { + cutlass::swap( + frag_64b(make_coord(_0{}, _1{}, 2 * i), mi), + frag_64b(make_coord(_0{}, _0{}, 2 * i + 1), mi)); + } + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +CUTLASS_DEVICE void permute_output_fp8(Fragment& out) { + // out has shape ((2, 2, N / 8), MMA_M, MMA_N), each element is 32 bits + static_assert(decltype(size<0, 0>(out))::value == 2); + static_assert(decltype(size<0, 1>(out))::value == 2); + static_assert(decltype(size<0, 2>(out))::value % 2 == 0); + static_assert(decltype(stride<0, 0>(out))::value == 1); + static_assert(sizeof(typename Fragment::value_type) == 4); + Tensor frag = group_modes<1, 3>(out); // ((2, 2, N / 8), (MMA_M, MMA_N)) +#pragma unroll + for (int mi = 0; mi < size<1>(frag); ++mi) { +#pragma unroll + for (int j = 0; j < size<0, 1>(frag); ++j) { +#pragma unroll + for (int i = 0; i < size<0, 2>(frag) / 2; ++i) { + cutlass::swap( + frag(make_coord(_1{}, j, 2 * i), mi), + frag(make_coord(_0{}, j, 2 * i + 1), mi)); + } + } + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +CUTLASS_DEVICE void permute_output_fp8_Vcolmajor(Fragment& frag) { + // frag has shape ((2, 2, N / 8), MMA_M, MMA_N), each element is 16 bits + static_assert(decltype(size<0, 0>(frag))::value == 2); + static_assert(decltype(size<0, 1>(frag))::value == 2); + static_assert(decltype(stride<0, 0>(frag))::value == 1); + static_assert( + sizeof(typename Fragment::value_type) == 2 || + sizeof(typename Fragment::value_type) == 4); + + auto quad_idx = threadIdx.x % 4; + bool lane_03 = quad_idx == 0 || quad_idx == 3; + + static constexpr int upper_map[4] = {0, 2, 3, 1}; + // static constexpr int lower_map[4] = {2, 0, 1, 3}; + + // if (blockIdx.x == 0 && threadIdx.x == 128) { print_tensor(frag); } + using type2 = std::conditional_t< + sizeof(typename Fragment::value_type) == 2, + uint32_t, + uint64_t>; + Tensor frag_2 = + group_modes<1, 3>(recast(frag)); // ((1, 2, N / 8), (MMA_M, MMA_N)) +// if (blockIdx.x == 0 && threadIdx.x == 128) { print(frag); printf("\n"); +// print(frag_2); } +#pragma unroll + for (int mi = 0; mi < size<1>(frag_2); ++mi) { +#pragma unroll + for (int j = 0; j < size<0, 1>(frag_2); ++j) { +#pragma unroll + for (int i = 0; i < size<0, 2>(frag_2) / 2; ++i) { + type2 upper = frag_2(make_coord(_0{}, j, 2 * i), mi); + type2 lower = frag_2(make_coord(_0{}, j, 2 * i + 1), mi); + type2 upper0 = lane_03 ? upper : lower; + type2 lower0 = lane_03 ? lower : upper; + upper0 = __shfl_sync(uint32_t(-1), upper0, upper_map[quad_idx], 4); + // lower0 = __shfl_sync(uint32_t(-1), lower0, lower_map[quad_idx], 4); + lower0 = __shfl_sync(uint32_t(-1), lower0, upper_map[quad_idx] ^ 2, 4); + frag_2(make_coord(_0{}, j, 2 * i), mi) = lane_03 ? upper0 : lower0; + frag_2(make_coord(_0{}, j, 2 * i + 1), mi) = lane_03 ? lower0 : upper0; + } + } + } + // if (blockIdx.x == 0 && threadIdx.x == 128) { print_tensor(frag); } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +CUTLASS_DEVICE void apply_softcap( + Tensor& tensor, + float const softcap) { +#pragma unroll + for (int i = 0; i < size(tensor); ++i) { + tensor(i) = cutlass::fast_tanh(tensor(i) * softcap); + } +} + +template +CUTLASS_DEVICE auto calculate_dtanh(Tensor& tensor) { + Tensor out = make_fragment_like(tensor); +#pragma unroll + for (int i = 0; i < size(tensor); ++i) { + out(i) = 1.f - (tensor(i) * tensor(i)); + } + return out; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +CUTLASS_DEVICE +int canonical_warp_group_idx_nosync() { + return threadIdx.x / cutlass::NumThreadsPerWarpGroup; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/version.txt b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/version.txt new file mode 100644 index 000000000..04d34e1a3 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/version.txt @@ -0,0 +1 @@ +5231d95 diff --git a/recommendation_v4/generative_recommenders/ops/cpp/jagged_transpose_1d.cpp b/recommendation_v4/generative_recommenders/ops/cpp/jagged_transpose_1d.cpp new file mode 100644 index 000000000..d3a2e9421 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/jagged_transpose_1d.cpp @@ -0,0 +1,130 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "fbgemm_gpu/sparse_ops.h" // @manual + +namespace hstu { + +template +void _jagged_transpose_1d_cpu_kernel( + int32_t size1, + int32_t size2, + int32_t max_len, + const at::TensorAccessor& offsets, + const at::TensorAccessor& values, + const at::TensorAccessor& lengths, + const at::TensorAccessor& trans_offsets, + at::TensorAccessor trans_values) { + for (auto i : c10::irange(size1)) { + for (auto j : c10::irange(size2)) { + auto src_idx = i * size2 + j; + auto dst_idx = j * size1 + i; + auto src_offset = offsets[src_idx]; + auto src_length = lengths[src_idx]; + auto dst_offset = trans_offsets[dst_idx]; + + for (auto k = 0; k < src_length; ++k) { + trans_values[dst_offset + k] = values[src_offset + k]; + } + } + } +} + +std::tuple jagged_transpose_1d_cpu( + const at::Tensor& values, + const at::Tensor& offsets, + const at::Tensor& lengths, + const int64_t max_len, + const int64_t size1, + const int64_t size2) { + TORCH_INTERNAL_ASSERT(values.device().type() == at::DeviceType::CPU); + TORCH_INTERNAL_ASSERT(offsets.device().type() == at::DeviceType::CPU); + TORCH_INTERNAL_ASSERT(lengths.device().type() == at::DeviceType::CPU); + TORCH_CHECK(offsets.size(0) == size1 * size2 + 1); + TORCH_CHECK(lengths.size(0) == size1 * size2); + + auto trans_lengths = + lengths.view({size1, size2}).transpose(0, 1).contiguous().view({-1}); + auto trans_offsets = + fbgemm_gpu::asynchronous_complete_cumsum_cpu(trans_lengths); + auto L_out = trans_offsets[-1].item(); + auto trans_values = at::empty({L_out}, values.options()); + + if (L_out == 0) { + return std::make_tuple(trans_values, trans_offsets, trans_lengths); + } + + AT_DISPATCH_INTEGRAL_TYPES( + lengths.scalar_type(), "jagged_transpose_1d_cpu_kernel_input1", [&] { + using index_t = scalar_t; + AT_DISPATCH_ALL_TYPES_AND2( + at::ScalarType::BFloat16, + at::ScalarType::Half, + values.scalar_type(), + "jagged_transpose_1d_cpu_kernel_input2", + [&] { + using val_t = scalar_t; + _jagged_transpose_1d_cpu_kernel( + size1, + size2, + max_len, + offsets.accessor(), + values.accessor(), + lengths.accessor(), + trans_offsets.accessor(), + trans_values.accessor()); + }); + }); + + return std::make_tuple(trans_values, trans_offsets, trans_lengths); +} + +std::tuple jagged_transpose_1d_meta( + const at::Tensor& values, + const at::Tensor& offsets, + const at::Tensor& lengths, + const int64_t max_len, + const int64_t size1, + const int64_t size2) { + auto trans_lengths = + lengths.view({size1, size2}).transpose(0, 1).contiguous().view({-1}); + auto L_out = trans_lengths.sum().item(); + + auto trans_values = at::native::empty_meta_symint( + {L_out}, + /*dtype=*/::std::make_optional(values.scalar_type()), + /*layout=*/::std::make_optional(values.layout()), + /*device=*/::std::make_optional(c10::Device(c10::kMeta)), + /*pin_memory=*/::std::nullopt); + + auto trans_offsets = at::native::empty_meta_symint( + {size1 * size2 + 1}, + /*dtype=*/::std::make_optional(lengths.scalar_type()), + /*layout=*/::std::make_optional(lengths.layout()), + /*device=*/::std::make_optional(c10::Device(c10::kMeta)), + /*pin_memory=*/::std::nullopt); + + return std::make_tuple(trans_values, trans_offsets, trans_lengths); +} +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/jagged_transpose_1d.cu b/recommendation_v4/generative_recommenders/ops/cpp/jagged_transpose_1d.cu new file mode 100644 index 000000000..380100962 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/jagged_transpose_1d.cu @@ -0,0 +1,127 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "common.h" +#include "fbgemm_gpu/sparse_ops.h" // @manual +#include "fbgemm_gpu/utils/fixed_divisor.cuh" // @manual + +namespace hstu { + +static constexpr int32_t kMaxThreads = 1024; + +template +__global__ __launch_bounds__(kMaxThreads) void _jagged_transpose_1d_cuda_kernel( + int32_t size1, + int32_t size2, + int32_t max_len, + const at::PackedTensorAccessor32 offsets, + const at::PackedTensorAccessor32 values, + const at::PackedTensorAccessor32 lengths, + const at::PackedTensorAccessor32 + trans_offsets, + at::PackedTensorAccessor32 trans_values) { + for (auto idx = blockIdx.x * blockDim.y + threadIdx.y; + idx < static_cast(size1 * size2); + idx += gridDim.x * blockDim.y) { + auto i = idx / size2; + auto j = idx % size2; + auto src_idx = i * size2 + j; + auto dst_idx = j * size1 + i; + auto src_offset = offsets[src_idx]; + auto src_length = lengths[src_idx]; + auto dst_offset = trans_offsets[dst_idx]; + + for (auto k = threadIdx.x; k < static_cast(src_length); + k += blockDim.x) { + trans_values[dst_offset + k] = values[src_offset + k]; + } + } +} + +std::tuple jagged_transpose_1d_cuda( + const at::Tensor& values, + const at::Tensor& offsets, + const at::Tensor& lengths, + const int64_t max_len, + const int64_t size1, + const int64_t size2) { + at::cuda::OptionalCUDAGuard device_guard; + device_guard.set_index(values.get_device()); + TORCH_INTERNAL_ASSERT(values.device().type() == at::DeviceType::CUDA); + TORCH_INTERNAL_ASSERT(offsets.device().type() == at::DeviceType::CUDA); + TORCH_INTERNAL_ASSERT(lengths.device().type() == at::DeviceType::CUDA); + TORCH_CHECK(offsets.size(0) == size1 * size2 + 1); + TORCH_CHECK(lengths.size(0) == size1 * size2); + TORCH_CHECK(values.get_device() == offsets.get_device()); + TORCH_CHECK(values.get_device() == lengths.get_device()); + + auto trans_lengths = + lengths.view({size1, size2}).transpose(0, 1).contiguous().view({-1}); + auto trans_offsets = + fbgemm_gpu::asynchronous_complete_cumsum_gpu(trans_lengths); + auto L_out = trans_offsets[-1].item(); + TORCH_CHECK(L_out < std::numeric_limits::max()); + auto trans_values = at::empty({L_out}, values.options()); + + if (L_out == 0) { + return std::make_tuple(trans_values, trans_offsets, trans_lengths); + } + + // Optimized thread block configuration based on benchmark results + uint32_t B_blocks = 4; + dim3 threads(256, B_blocks); + auto blocks = div_round_up(size1 * size2, B_blocks); + + AT_DISPATCH_INTEGRAL_TYPES( + lengths.scalar_type(), "jagged_transpose_1d_cuda_kernel_input1", [&] { + using index_t = scalar_t; + AT_DISPATCH_ALL_TYPES_AND2( + at::ScalarType::BFloat16, + at::ScalarType::Half, + values.scalar_type(), + "jagged_transpose_1d_cuda_kernel_input2", + [&] { + using val_t = scalar_t; + _jagged_transpose_1d_cuda_kernel<<< + blocks, + threads, + 0, + at::cuda::getCurrentCUDAStream()>>>( + size1, + size2, + max_len, + offsets + .packed_accessor32(), + values.packed_accessor32(), + lengths + .packed_accessor32(), + trans_offsets + .packed_accessor32(), + trans_values + .packed_accessor32()); + }); + }); + + return std::make_tuple(trans_values, trans_offsets, trans_lengths); +} +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/replace_last_n_with_jagged.cpp b/recommendation_v4/generative_recommenders/ops/cpp/replace_last_n_with_jagged.cpp new file mode 100644 index 000000000..fa8eaac09 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/replace_last_n_with_jagged.cpp @@ -0,0 +1,139 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "fbgemm_gpu/sparse_ops.h" // @manual + +namespace hstu { + +template +void _replace_last_n_with_jagged_cpu_kernel( + int32_t B, + const at::TensorAccessor& lengths_left, + const at::TensorAccessor& offsets_left, + const at::TensorAccessor& values_left, + const at::TensorAccessor& lengths_right, + const at::TensorAccessor& offsets_right, + const at::TensorAccessor& values_right, + const at::TensorAccessor& output_offsets, + at::TensorAccessor output) { + for (auto b : c10::irange(B)) { + auto left_start = offsets_left[b]; + auto left_len = lengths_left[b]; + auto right_start = offsets_right[b]; + auto right_len = lengths_right[b]; + auto output_start = output_offsets[b]; + + auto keep_len = left_len - right_len; + + for (auto i = 0; i < left_len; ++i) { + for (auto d = 0; d < values_left.size(1); ++d) { + if (i < keep_len) { + output[output_start + i][d] = values_left[left_start + i][d]; + } else { + auto right_idx = i - keep_len; + if (right_idx < right_len) { + output[output_start + i][d] = + values_right[right_start + right_idx][d]; + } + } + } + } + } +} + +at::Tensor replace_last_n_with_jagged_cpu( + const at::Tensor& lengths_left, + const at::Tensor& values_left, + const at::Tensor& lengths_right, + const at::Tensor& values_right) { + TORCH_INTERNAL_ASSERT(lengths_left.device().type() == at::DeviceType::CPU); + TORCH_INTERNAL_ASSERT(lengths_right.device().type() == at::DeviceType::CPU); + TORCH_INTERNAL_ASSERT(values_left.device().type() == at::DeviceType::CPU); + TORCH_INTERNAL_ASSERT(values_right.device().type() == at::DeviceType::CPU); + TORCH_CHECK(lengths_left.size(0) == lengths_right.size(0)); + TORCH_CHECK(values_left.size(1) == values_right.size(1)); + + auto B = lengths_left.size(0); + auto D = values_left.size(1); + + auto L_out = lengths_left.sum().item(); + + auto output = at::empty({L_out, D}, values_left.options()); + + if (L_out == 0) { + return output; + } + + const auto offsets_left = + fbgemm_gpu::asynchronous_complete_cumsum_cpu(lengths_left.view({-1})); + const auto offsets_right = + fbgemm_gpu::asynchronous_complete_cumsum_cpu(lengths_right.view({-1})); + const auto output_offsets = offsets_left; + + AT_DISPATCH_INTEGRAL_TYPES( + lengths_left.scalar_type(), + "replace_last_n_with_jagged_cpu_kernel_input1", + [&] { + using index_t = scalar_t; + AT_DISPATCH_ALL_TYPES_AND2( + at::ScalarType::BFloat16, + at::ScalarType::Half, + values_left.scalar_type(), + "replace_last_n_with_jagged_cpu_kernel_input2", + [&] { + using val_t = scalar_t; + _replace_last_n_with_jagged_cpu_kernel( + B, + lengths_left.accessor(), + offsets_left.accessor(), + values_left.accessor(), + lengths_right.accessor(), + offsets_right.accessor(), + values_right.accessor(), + output_offsets.accessor(), + output.accessor()); + }); + }); + + return output; +} + +at::Tensor replace_last_n_with_jagged_meta( + const at::Tensor& lengths_left, + const at::Tensor& values_left, + const at::Tensor& lengths_right, + const at::Tensor& values_right) { + auto L_out = lengths_left.sum().item(); + auto D = values_left.size(1); + + auto output = at::native::empty_meta_symint( + {L_out, D}, + /*dtype=*/::std::make_optional(values_left.scalar_type()), + /*layout=*/::std::make_optional(values_left.layout()), + /*device=*/::std::make_optional(c10::Device(c10::kMeta)), + /*pin_memory=*/::std::nullopt); + + return output; +} +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/replace_last_n_with_jagged.cu b/recommendation_v4/generative_recommenders/ops/cpp/replace_last_n_with_jagged.cu new file mode 100644 index 000000000..00a589eb9 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/replace_last_n_with_jagged.cu @@ -0,0 +1,156 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "common.h" +#include "fbgemm_gpu/sparse_ops.h" // @manual +#include "fbgemm_gpu/utils/fixed_divisor.cuh" // @manual + +namespace hstu { + +static constexpr int32_t kMaxThreads = 1024; + +template +__global__ +__launch_bounds__(kMaxThreads) void _replace_last_n_with_jagged_cuda_kernel( + int32_t B, + int32_t D, + const at::PackedTensorAccessor32 + lengths_left, + const at::PackedTensorAccessor32 + offsets_left, + const at::PackedTensorAccessor32 + values_left, + const at::PackedTensorAccessor32 + lengths_right, + const at::PackedTensorAccessor32 + offsets_right, + const at::PackedTensorAccessor32 + values_right, + at::PackedTensorAccessor32 output) { + for (auto b = blockIdx.x * blockDim.y + threadIdx.y; + b < static_cast(B); + b += gridDim.x * blockDim.y) { + auto left_start = offsets_left[b]; + auto left_len = lengths_left[b]; + auto right_start = offsets_right[b]; + auto right_len = lengths_right[b]; + auto output_start = offsets_left[b]; + auto keep_len = left_len - right_len; + + for (auto i = threadIdx.x; i < static_cast(left_len * D); + i += blockDim.x) { + auto seq_pos = i / D; + auto dim_pos = i % D; + if (seq_pos < static_cast(keep_len)) { + output[output_start + seq_pos][dim_pos] = + values_left[left_start + seq_pos][dim_pos]; + } else { + auto right_idx = seq_pos - keep_len; + if (right_idx < static_cast(right_len)) { + output[output_start + seq_pos][dim_pos] = + values_right[right_start + right_idx][dim_pos]; + } + } + } + } +} + +at::Tensor replace_last_n_with_jagged_cuda( + const at::Tensor& lengths_left, + const at::Tensor& values_left, + const at::Tensor& lengths_right, + const at::Tensor& values_right) { + at::cuda::OptionalCUDAGuard device_guard; + device_guard.set_index(values_left.get_device()); + TORCH_INTERNAL_ASSERT(lengths_left.device().type() == at::DeviceType::CUDA); + TORCH_INTERNAL_ASSERT(lengths_right.device().type() == at::DeviceType::CUDA); + TORCH_INTERNAL_ASSERT(values_left.device().type() == at::DeviceType::CUDA); + TORCH_INTERNAL_ASSERT(values_right.device().type() == at::DeviceType::CUDA); + TORCH_CHECK(lengths_left.size(0) == lengths_right.size(0)); + TORCH_CHECK(values_left.size(1) == values_right.size(1)); + + auto B = lengths_left.size(0); + auto D = values_left.size(1); + auto L_out = lengths_left.sum().item(); + TORCH_CHECK(L_out < std::numeric_limits::max()); + TORCH_CHECK(values_left.get_device() == lengths_left.get_device()); + TORCH_CHECK(values_left.get_device() == lengths_right.get_device()); + TORCH_CHECK(values_left.get_device() == values_right.get_device()); + + auto output = at::empty({L_out, D}, values_left.options()); + + if (L_out == 0) { + return output; + } + + const auto offsets_left = + fbgemm_gpu::asynchronous_complete_cumsum_gpu(lengths_left.view({-1})); + const auto offsets_right = + fbgemm_gpu::asynchronous_complete_cumsum_gpu(lengths_right.view({-1})); + + // Optimized thread block configuration based on benchmark results + uint32_t B_blocks, threads_x; + B_blocks = 4; + threads_x = 256; + + dim3 threads(threads_x, B_blocks); + auto blocks = div_round_up(B, B_blocks); + + AT_DISPATCH_INTEGRAL_TYPES( + lengths_left.scalar_type(), + "replace_last_n_with_jagged_cuda_kernel_input1", + [&] { + using index_t = scalar_t; + AT_DISPATCH_ALL_TYPES_AND2( + at::ScalarType::BFloat16, + at::ScalarType::Half, + values_left.scalar_type(), + "replace_last_n_with_jagged_cuda_kernel_input2", + [&] { + using val_t = scalar_t; + _replace_last_n_with_jagged_cuda_kernel<<< + blocks, + threads, + 0, + at::cuda::getCurrentCUDAStream()>>>( + B, + D, + lengths_left + .packed_accessor32(), + offsets_left + .packed_accessor32(), + values_left + .packed_accessor32(), + lengths_right + .packed_accessor32(), + offsets_right + .packed_accessor32(), + values_right + .packed_accessor32(), + output.packed_accessor32()); + }); + }); + + return output; +} +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/setup.py b/recommendation_v4/generative_recommenders/ops/cpp/setup.py new file mode 100644 index 000000000..2a06a9d05 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/setup.py @@ -0,0 +1,487 @@ +# pyre-unsafe +""" +Modified from https://github.com/Dao-AILab/flash-attention/blob/main/hopper/setup.py +""" + +import itertools +import os +import platform +import subprocess +import sys +import sysconfig +import warnings +from pathlib import Path + +import torch +from packaging.version import parse, Version +from setuptools import find_packages, setup +from torch.utils.cpp_extension import BuildExtension, CUDA_HOME, CUDAExtension + +# ninja build does not work unless include_dirs are abs path +this_dir = os.path.dirname(os.path.abspath(__file__)) +PACKAGE_NAME = "hstu" +# FORCE_BUILD: Force a fresh build locally, instead of attempting to find prebuilt wheels +# SKIP_CUDA_BUILD: Intended to allow CI to use a simple `python setup.py sdist` run to copy over raw files, without any cuda compilation +FORCE_BUILD = os.getenv("FLASH_ATTENTION_FORCE_BUILD", "FALSE") == "TRUE" +SKIP_CUDA_BUILD = os.getenv("FLASH_ATTENTION_SKIP_CUDA_BUILD", "FALSE") == "TRUE" +# For CI, we want the option to build with C++11 ABI since the nvcr images use C++11 ABI +FORCE_CXX11_ABI = os.getenv("FLASH_ATTENTION_FORCE_CXX11_ABI", "FALSE") == "TRUE" + +# HACK: we monkey patch pytorch's _write_ninja_file to pass +# "-gencode arch=compute_sm90a,code=sm_90a" to files ending in '_sm90.cu', +# and pass "-gencode arch=compute_sm80,code=sm_80" to files ending in '_sm80.cu' +from torch.utils.cpp_extension import ( + _is_cuda_file, + _join_cuda_home, + _join_rocm_home, + _maybe_write, + COMMON_HIP_FLAGS, + get_cxx_compiler, + IS_HIP_EXTENSION, + IS_WINDOWS, + SUBPROCESS_DECODE_ARGS, +) + +DISABLE_BACKWARD = os.getenv("FLASH_ATTENTION_DISABLE_BACKWARD", "FALSE") == "TRUE" +DISABLE_FP16 = os.getenv("FLASH_ATTENTION_DISABLE_FP16", "TRUE") == "TRUE" +DISABLE_HDIM64 = os.getenv("FLASH_ATTENTION_DISABLE_HDIM64", "TRUE") == "TRUE" +DISABLE_HDIM96 = os.getenv("FLASH_ATTENTION_DISABLE_HDIM96", "TRUE") == "TRUE" +DISABLE_HDIM128 = os.getenv("FLASH_ATTENTION_DISABLE_HDIM128", "FALSE") == "TRUE" +DISABLE_HDIM192 = os.getenv("FLASH_ATTENTION_DISABLE_HDIM192", "TRUE") == "TRUE" +DISABLE_HDIM256 = os.getenv("FLASH_ATTENTION_DISABLE_HDIM256", "TRUE") == "TRUE" +DISABLE_SM8x = os.getenv("FLASH_ATTENTION_DISABLE_SM80", "TRUE") == "TRUE" + + +def _write_ninja_file( + path, + cflags, + post_cflags, + cuda_cflags, + cuda_post_cflags, + cuda_dlink_post_cflags, + sources, + objects, + ldflags, + library_target, + with_cuda, + **kwargs, # kwargs (ignored) to absorb new flags in torch.utils.cpp_extension +) -> None: + r"""Write a ninja file that does the desired compiling and linking. + + `path`: Where to write this file + `cflags`: list of flags to pass to $cxx. Can be None. + `post_cflags`: list of flags to append to the $cxx invocation. Can be None. + `cuda_cflags`: list of flags to pass to $nvcc. Can be None. + `cuda_postflags`: list of flags to append to the $nvcc invocation. Can be None. + `sources`: list of paths to source files + `objects`: list of desired paths to objects, one per source. + `ldflags`: list of flags to pass to linker. Can be None. + `library_target`: Name of the output library. Can be None; in that case, + we do no linking. + `with_cuda`: If we should be compiling with CUDA. + """ + + def sanitize_flags(flags): + if flags is None: + return [] + else: + return [flag.strip() for flag in flags] + + cflags = sanitize_flags(cflags) + post_cflags = sanitize_flags(post_cflags) + cuda_cflags = sanitize_flags(cuda_cflags) + cuda_post_cflags = sanitize_flags(cuda_post_cflags) + cuda_dlink_post_cflags = sanitize_flags(cuda_dlink_post_cflags) + ldflags = sanitize_flags(ldflags) + + # Sanity checks... + assert len(sources) == len(objects) + assert len(sources) > 0 + + compiler = get_cxx_compiler() + + # Version 1.3 is required for the `deps` directive. + config = ["ninja_required_version = 1.3"] + config.append(f"cxx = {compiler}") + if with_cuda or cuda_dlink_post_cflags: + if IS_HIP_EXTENSION: + nvcc = _join_rocm_home("bin", "hipcc") + else: + nvcc = _join_cuda_home("bin", "nvcc") + if "PYTORCH_NVCC" in os.environ: + nvcc_from_env = os.getenv( + "PYTORCH_NVCC" + ) # user can set nvcc compiler with ccache using the environment variable here + else: + nvcc_from_env = nvcc + config.append(f"nvcc_from_env = {nvcc_from_env}") + config.append(f"nvcc = {nvcc}") + + if IS_HIP_EXTENSION: + post_cflags = COMMON_HIP_FLAGS + post_cflags + flags = [f"cflags = {' '.join(cflags)}"] + flags.append(f"post_cflags = {' '.join(post_cflags)}") + if with_cuda: + flags.append(f"cuda_cflags = {' '.join(cuda_cflags)}") + flags.append(f"cuda_post_cflags = {' '.join(cuda_post_cflags)}") + cuda_post_cflags_sm80 = [ + s if s != "arch=compute_90a,code=sm_90a" else "arch=compute_80,code=sm_80" + for s in cuda_post_cflags + ] + flags.append(f"cuda_post_cflags_sm80 = {' '.join(cuda_post_cflags_sm80)}") + cuda_post_cflags_sm80_sm90 = cuda_post_cflags + [ + "-gencode", + "arch=compute_80,code=sm_80", + ] + flags.append( + f"cuda_post_cflags_sm80_sm90 = {' '.join(cuda_post_cflags_sm80_sm90)}" + ) + cuda_post_cflags_sm100 = [ + s + if s != "arch=compute_90a,code=sm_90a" + else "arch=compute_100a,code=sm_100a" + for s in cuda_post_cflags + ] + flags.append(f"cuda_post_cflags_sm100 = {' '.join(cuda_post_cflags_sm100)}") + flags.append(f"cuda_dlink_post_cflags = {' '.join(cuda_dlink_post_cflags)}") + flags.append(f"ldflags = {' '.join(ldflags)}") + + # Turn into absolute paths so we can emit them into the ninja build + # file wherever it is. + sources = [os.path.abspath(file) for file in sources] + + # See https://ninja-build.org/build.ninja.html for reference. + compile_rule = ["rule compile"] + if IS_WINDOWS: + compile_rule.append( + " command = cl /showIncludes $cflags -c $in /Fo$out $post_cflags" + ) + compile_rule.append(" deps = msvc") + else: + compile_rule.append( + " command = $cxx -MMD -MF $out.d $cflags -c $in -o $out $post_cflags" + ) + compile_rule.append(" depfile = $out.d") + compile_rule.append(" deps = gcc") + + if with_cuda: + cuda_compile_rule = ["rule cuda_compile"] + nvcc_gendeps = "" + # --generate-dependencies-with-compile is not supported by ROCm + # Nvcc flag `--generate-dependencies-with-compile` is not supported by sccache, which may increase build time. + if ( + torch.version.cuda is not None + and os.getenv("TORCH_EXTENSION_SKIP_NVCC_GEN_DEPENDENCIES", "0") != "1" + ): + cuda_compile_rule.append(" depfile = $out.d") + cuda_compile_rule.append(" deps = gcc") + # Note: non-system deps with nvcc are only supported + # on Linux so use --generate-dependencies-with-compile + # to make this work on Windows too. + nvcc_gendeps = ( + "--generate-dependencies-with-compile --dependency-output $out.d" + ) + cuda_compile_rule_sm80 = ( + ["rule cuda_compile_sm80"] + + cuda_compile_rule[1:] + + [ + f" command = $nvcc_from_env {nvcc_gendeps} $cuda_cflags -c $in -o $out $cuda_post_cflags_sm80" + ] + ) + cuda_compile_rule_sm80_sm90 = ( + ["rule cuda_compile_sm80_sm90"] + + cuda_compile_rule[1:] + + [ + f" command = $nvcc_from_env {nvcc_gendeps} $cuda_cflags -c $in -o $out $cuda_post_cflags_sm80_sm90" + ] + ) + cuda_compile_rule_sm100 = ( + ["rule cuda_compile_sm100"] + + cuda_compile_rule[1:] + + [ + f" command = $nvcc_from_env {nvcc_gendeps} $cuda_cflags -c $in -o $out $cuda_post_cflags_sm100" + ] + ) + cuda_compile_rule.append( + f" command = $nvcc_from_env {nvcc_gendeps} $cuda_cflags -c $in -o $out $cuda_post_cflags" + ) + + # Emit one build rule per source to enable incremental build. + build = [] + for source_file, object_file in zip(sources, objects): + is_cuda_source = _is_cuda_file(source_file) and with_cuda + if is_cuda_source: + if source_file.endswith("_sm90.cu"): + rule = "cuda_compile" + elif source_file.endswith("_sm80.cu"): + rule = "cuda_compile_sm80" + elif source_file.endswith("_sm100.cu"): + rule = "cuda_compile_sm100" + else: + rule = "cuda_compile_sm80_sm90" + else: + rule = "compile" + if IS_WINDOWS: + source_file = source_file.replace(":", "$:") + object_file = object_file.replace(":", "$:") + source_file = source_file.replace(" ", "$ ") + object_file = object_file.replace(" ", "$ ") + build.append(f"build {object_file}: {rule} {source_file}") + + if cuda_dlink_post_cflags: + devlink_out = os.path.join(os.path.dirname(objects[0]), "dlink.o") + devlink_rule = ["rule cuda_devlink"] + devlink_rule.append(" command = $nvcc $in -o $out $cuda_dlink_post_cflags") + devlink = [f"build {devlink_out}: cuda_devlink {' '.join(objects)}"] + objects += [devlink_out] + else: + devlink_rule, devlink = [], [] + + if library_target is not None: + link_rule = ["rule link"] + if IS_WINDOWS: + cl_paths = ( + subprocess.check_output(["where", "cl"]) + .decode(*SUBPROCESS_DECODE_ARGS) + .split("\r\n") + ) + if len(cl_paths) >= 1: + cl_path = os.path.dirname(cl_paths[0]).replace(":", "$:") + else: + raise RuntimeError("MSVC is required to load C++ extensions") + link_rule.append( + f' command = "{cl_path}/link.exe" $in /nologo $ldflags /out:$out' + ) + else: + link_rule.append(" command = $cxx $in $ldflags -o $out") + + link = [f"build {library_target}: link {' '.join(objects)}"] + + default = [f"default {library_target}"] + else: + link_rule, link, default = [], [], [] + + # 'Blocks' should be separated by newlines, for visual benefit. + blocks = [config, flags, compile_rule] + if with_cuda: + blocks.append(cuda_compile_rule) # type: ignore[possibly-undefined] + blocks.append(cuda_compile_rule_sm80) # type: ignore[possibly-undefined] + blocks.append(cuda_compile_rule_sm80_sm90) # type: ignore[possibly-undefined] + blocks.append(cuda_compile_rule_sm100) # type: ignore[possibly-undefined] + blocks += [devlink_rule, link_rule, build, devlink, link, default] + content = "\n\n".join("\n".join(b) for b in blocks) + # Ninja requires a new lines at the end of the .ninja file + content += "\n" + _maybe_write(path, content) + + +# Monkey patching +torch.utils.cpp_extension._write_ninja_file = _write_ninja_file + + +def get_platform(): + """ + Returns the platform name as used in wheel filenames. + """ + if sys.platform.startswith("linux"): + return "linux_x86_64" + elif sys.platform == "darwin": + mac_version = ".".join(platform.mac_ver()[0].split(".")[:2]) + return f"macosx_{mac_version}_x86_64" + elif sys.platform == "win32": + return "win_amd64" + else: + raise ValueError("Unsupported platform: {}".format(sys.platform)) + + +def get_cuda_bare_metal_version(cuda_dir): + raw_output = subprocess.check_output( + [cuda_dir + "/bin/nvcc", "-V"], universal_newlines=True + ) + output = raw_output.split() + release_idx = output.index("release") + 1 + bare_metal_version = parse(output[release_idx].split(",")[0]) + + return raw_output, bare_metal_version + + +def check_if_cuda_home_none(global_option: str) -> None: + if CUDA_HOME is not None: + return + # warn instead of error because user could be downloading prebuilt wheels, so nvcc won't be necessary + # in that case. + warnings.warn( + f"{global_option} was requested, but nvcc was not found. Are you sure your environment has nvcc available? " + "If you're installing within a container from https://hub.docker.com/r/pytorch/pytorch, " + "only images whose names contain 'devel' will provide nvcc." + ) + + +def nvcc_threads_args(): + nvcc_threads = os.getenv("NVCC_THREADS") or "4" + return ["--threads", nvcc_threads] + + +exe_extension = sysconfig.get_config_var("EXE") + + +cmdclass = {} +ext_modules = [] + +# We want this even if SKIP_CUDA_BUILD because when we run python setup.py sdist we want the .hpp +# files included in the source distribution, in case the user compiles from source. +subprocess.run(["git", "submodule", "update", "--init", "cutlass"]) + +if not SKIP_CUDA_BUILD: + print("\n\ntorch.__version__ = {}\n\n".format(torch.__version__)) + TORCH_MAJOR = int(torch.__version__.split(".")[0]) + TORCH_MINOR = int(torch.__version__.split(".")[1]) + + check_if_cuda_home_none(PACKAGE_NAME) + _, bare_metal_version = get_cuda_bare_metal_version(CUDA_HOME) + if bare_metal_version < Version("12.3"): + raise RuntimeError( + f"FlashAttention-3 is only supported on CUDA 12.3 and above, get {bare_metal_version} from {CUDA_HOME}" + ) + + cc_flag = [] + cc_flag.append("-gencode") + cc_flag.append("arch=compute_90a,code=sm_90a") + + # HACK: The compiler flag -D_GLIBCXX_USE_CXX11_ABI is set to be the same as + # torch._C._GLIBCXX_USE_CXX11_ABI + # https://github.com/pytorch/pytorch/blob/8472c24e3b5b60150096486616d98b7bea01500b/torch/utils/cpp_extension.py#L920 + if FORCE_CXX11_ABI: + torch._C._GLIBCXX_USE_CXX11_ABI = True + repo_dir = Path(this_dir).parent + cutlass_dir = repo_dir / "cpp" / "cutlass" + + feature_args = ( + [] + + ["-DOSS_ENV"] + + (["-DFLASHATTENTION_DISABLE_BACKWARD"] if DISABLE_BACKWARD else []) + + (["-DFLASHATTENTION_DISABLE_FP16"] if DISABLE_FP16 else []) + + ["-DFLASHATTENTION_DISABLE_FP8"] + + (["-DFLASHATTENTION_DISABLE_HDIM64"] if DISABLE_HDIM64 else []) + + (["-DFLASHATTENTION_DISABLE_HDIM96"] if DISABLE_HDIM96 else []) + + (["-DFLASHATTENTION_DISABLE_HDIM128"] if DISABLE_HDIM128 else []) + + (["-DFLASHATTENTION_DISABLE_HDIM192"] if DISABLE_HDIM192 else []) + + (["-DFLASHATTENTION_DISABLE_HDIM256"] if DISABLE_HDIM256 else []) + + (["-DFLASHATTENTION_DISABLE_SM8x"] if DISABLE_SM8x else []) + ) + + DTYPE = ["bf16"] + (["fp16"] if not DISABLE_FP16 else []) + HEAD_DIMENSIONS = ( + [] + + ([64] if not DISABLE_HDIM64 else []) + + ([96] if not DISABLE_HDIM96 else []) + + ([128] if not DISABLE_HDIM128 else []) + + ([192] if not DISABLE_HDIM192 else []) + + ([256] if not DISABLE_HDIM256 else []) + ) + sources_fwd_sm80 = [ + f"hstu_attention/instantiations/flash_fwd_hdim{hdim}_{dtype}_sm80.cu" + for hdim, dtype in itertools.product(HEAD_DIMENSIONS, DTYPE) + ] + sources_bwd_sm80 = [ + f"hstu_attention/instantiations/flash_bwd_hdim{hdim}_{dtype}_sm80.cu" + for hdim, dtype in itertools.product(HEAD_DIMENSIONS, DTYPE) + ] + sources_fwd_sm90 = [ + f"hstu_attention/instantiations/flash_fwd_hdim{hdim}_{dtype}_sm90.cu" + for hdim, dtype in itertools.product(HEAD_DIMENSIONS, DTYPE) + ] + sources_bwd_sm90 = [ + f"hstu_attention/instantiations/flash_bwd_hdim{hdim}_{dtype}_sm90.cu" + for hdim, dtype in itertools.product(HEAD_DIMENSIONS, DTYPE) + ] + if DISABLE_BACKWARD: + sources_bwd_sm90 = [] + sources_bwd_sm80 = [] + sources = ( + [ + "hstu_attention/flash_api.cpp", + "hstu_attention/flash_common.cpp", + "hstu_attention/flash_cpu_dummy.cpp", + "hstu_attention/flash_meta.cpp", + ] + + (sources_fwd_sm80 if not DISABLE_SM8x else []) + + sources_fwd_sm90 + + (sources_bwd_sm80 if not DISABLE_SM8x else []) + + sources_bwd_sm90 + ) + nvcc_flags = [ + "-O3", + "-std=c++17", + "--ftemplate-backtrace-limit=0", # To debug template code + "--use_fast_math", + # "--keep", + # "--ptxas-options=--verbose,--register-usage-level=5,--warn-on-local-memory-usage", # printing out number of registers + "--resource-usage", # printing out number of registers + # f"--split-compile={os.getenv('NVCC_THREADS', '4')}", # split-compile is faster + "-lineinfo", # TODO: disable this for release to reduce binary size + "-DCUTE_SM90_EXTENDED_MMA_SHAPES_ENABLED", # Necessary for the WGMMA shapes that we use + "-DCUTLASS_ENABLE_GDC_FOR_SM90", # For PDL + "-DCUTLASS_DEBUG_TRACE_LEVEL=0", # Can toggle for debugging + "-DNDEBUG", # Important, otherwise performance is severely impacted + "-Xfatbin", # compress all binary sections + "-compress-all", + ] + if get_platform() == "win_amd64": + nvcc_flags.extend( + [ + "-D_USE_MATH_DEFINES", # for M_LN2 + "-Xcompiler=/Zc:__cplusplus", # sets __cplusplus correctly, CUTLASS_CONSTEXPR_IF_CXX17 needed for cutlass::gcd + ] + ) + include_dirs = [ + Path(this_dir), + cutlass_dir / "include", + ] + + ext_modules.append( + CUDAExtension( + name=f"{PACKAGE_NAME}._C", + sources=sources, + extra_compile_args={ + "cxx": ["-O3", "-std=c++17", "-DPy_LIMITED_API=0x03090000"] + + feature_args, + "nvcc": nvcc_threads_args() + nvcc_flags + cc_flag + feature_args, + }, + include_dirs=include_dirs, + py_limited_api=True, + ) + ) + + +setup( + name=PACKAGE_NAME, + version="0.1.0", + packages=find_packages( + exclude=( + "build", + "csrc", + "include", + "tests", + "dist", + "docs", + "benchmarks", + ) + ), + py_modules=["cuda_hstu_attention"], + description="FlashAttention HSTU", + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: Apache Software License", + "Operating System :: Unix", + ], + ext_modules=ext_modules, + cmdclass={"build_ext": BuildExtension}, + python_requires=">=3.8", + install_requires=[ + "torch", + "einops", + "packaging", + "ninja==1.11.1.1", + ], +) diff --git a/recommendation_v4/generative_recommenders/ops/cpp/sort_kv_pairs_cuda.cpp b/recommendation_v4/generative_recommenders/ops/cpp/sort_kv_pairs_cuda.cpp new file mode 100644 index 000000000..3925aefd7 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/sort_kv_pairs_cuda.cpp @@ -0,0 +1,40 @@ +#include "common.h" +#include "sort_kv_pairs_cuda_kernels_template.h" + +namespace hstu { + +DLL_PUBLIC std::tuple sort_kv_pairs_cuda( + const at::Tensor& keys, + const at::Tensor& values, + const std::optional& end_bit, + const bool descending = false) { + at::cuda::OptionalCUDAGuard device_guard; + device_guard.set_index(keys.get_device()); + TORCH_CHECK( + keys.dtype() == at::kInt || keys.dtype() == at::kLong || + keys.dtype() == at::kByte || keys.dtype() == at::kShort); + TORCH_CHECK(keys.numel() < std::numeric_limits::max()); + TORCH_CHECK(keys.dim() == 1); + TORCH_CHECK(values.dim() == 1); + at::Tensor sorted_keys; + at::Tensor sorted_values; + + AT_DISPATCH_INTEGRAL_TYPES(keys.scalar_type(), "sort_pairs_cuda_input1", [&] { + using key_t = scalar_t; + AT_DISPATCH_ALL_TYPES_AND2( + at::ScalarType::Half, + at::ScalarType::BFloat16, + values.scalar_type(), + "sort_pairs_cuda_input2", + [&] { + using val_t = scalar_t; + std::tie(sorted_keys, sorted_values) = + sort_kv_pairs_cuda_dispatched( + keys, values, end_bit, descending); + }); + }); + + return {std::move(sorted_keys), std::move(sorted_values)}; +} + +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/sort_kv_pairs_cuda_kernels_template.cu b/recommendation_v4/generative_recommenders/ops/cpp/sort_kv_pairs_cuda_kernels_template.cu new file mode 100644 index 000000000..8cd175c71 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/sort_kv_pairs_cuda_kernels_template.cu @@ -0,0 +1,82 @@ +#include +#include + +#include + +namespace hstu { + +template <> +DLL_PUBLIC std::tuple +sort_kv_pairs_cuda_dispatched( + const at::Tensor& keys, + const at::Tensor& values, + const std::optional& end_bit, + const bool descending) { + size_t temp_storage_bytes = 0; + auto keys_contig = keys.contiguous(); + auto values_contig = values.contiguous(); + auto sorted_keys = at::empty_like(keys_contig); + auto sorted_values = at::empty_like(values_contig); + + if (descending) { + AT_CUDA_CHECK( + cub::DeviceRadixSort::SortPairsDescending( + nullptr, + temp_storage_bytes, + keys_contig.data_ptr(), + sorted_keys.data_ptr(), + values_contig.data_ptr(), + sorted_values.data_ptr(), + keys_contig.numel(), + 0, + end_bit.has_value() ? end_bit.value() : sizeof(SUB_KEY_T) * 8, + at::cuda::getCurrentCUDAStream())); + auto temp_storage = at::empty( + {static_cast(temp_storage_bytes)}, + keys_contig.options().dtype(at::kByte)); + AT_CUDA_CHECK( + cub::DeviceRadixSort::SortPairsDescending( + temp_storage.data_ptr(), + temp_storage_bytes, + keys_contig.data_ptr(), + sorted_keys.data_ptr(), + values_contig.data_ptr(), + sorted_values.data_ptr(), + keys_contig.numel(), + 0, + end_bit.has_value() ? end_bit.value() : sizeof(SUB_KEY_T) * 8, + at::cuda::getCurrentCUDAStream())); + } else { + AT_CUDA_CHECK( + cub::DeviceRadixSort::SortPairs( + nullptr, + temp_storage_bytes, + keys_contig.data_ptr(), + sorted_keys.data_ptr(), + values_contig.data_ptr(), + sorted_values.data_ptr(), + keys_contig.numel(), + 0, + end_bit.has_value() ? end_bit.value() : sizeof(SUB_KEY_T) * 8, + at::cuda::getCurrentCUDAStream())); + auto temp_storage = at::empty( + {static_cast(temp_storage_bytes)}, + keys_contig.options().dtype(at::kByte)); + AT_CUDA_CHECK( + cub::DeviceRadixSort::SortPairs( + temp_storage.data_ptr(), + temp_storage_bytes, + keys_contig.data_ptr(), + sorted_keys.data_ptr(), + values_contig.data_ptr(), + sorted_values.data_ptr(), + keys_contig.numel(), + 0, + end_bit.has_value() ? end_bit.value() : sizeof(SUB_KEY_T) * 8, + at::cuda::getCurrentCUDAStream())); + } + + return {std::move(sorted_keys), std::move(sorted_values)}; +} + +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/sort_kv_pairs_cuda_kernels_template.h b/recommendation_v4/generative_recommenders/ops/cpp/sort_kv_pairs_cuda_kernels_template.h new file mode 100644 index 000000000..e599eccb0 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/sort_kv_pairs_cuda_kernels_template.h @@ -0,0 +1,15 @@ +#pragma once + +#include +#include + +namespace hstu { + +template +std::tuple sort_kv_pairs_cuda_dispatched( + const at::Tensor& keys_contig, + const at::Tensor& values_contig, + const std::optional& end_bit, + const bool descending); + +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/split_1d_jagged_jagged.cpp b/recommendation_v4/generative_recommenders/ops/cpp/split_1d_jagged_jagged.cpp new file mode 100644 index 000000000..c361488fa --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/split_1d_jagged_jagged.cpp @@ -0,0 +1,136 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "fbgemm_gpu/sparse_ops.h" // @manual + +namespace hstu { + +template +void _split_1d_jagged_jagged_cpu_kernel( + int32_t B, + const at::TensorAccessor& combined_offsets, + const at::TensorAccessor& combined_values, + const at::TensorAccessor& lengths_left, + const at::TensorAccessor& offsets_left, + const at::TensorAccessor& offsets_right, + at::TensorAccessor values_left, + at::TensorAccessor values_right) { + for (auto b : c10::irange(B)) { + auto combined_start = combined_offsets[b]; + auto left_len = lengths_left[b]; + auto left_start = offsets_left[b]; + auto right_start = offsets_right[b]; + + for (auto i = 0; i < left_len; ++i) { + values_left[left_start + i] = combined_values[combined_start + i]; + } + + auto right_len = combined_offsets[b + 1] - combined_offsets[b] - left_len; + for (auto i = 0; i < right_len; ++i) { + values_right[right_start + i] = + combined_values[combined_start + left_len + i]; + } + } +} + +std::tuple split_1d_jagged_jagged_cpu( + const at::Tensor& lengths_left, + const at::Tensor& lengths_right, + const at::Tensor& combined_values) { + TORCH_INTERNAL_ASSERT(lengths_left.device().type() == at::DeviceType::CPU); + TORCH_INTERNAL_ASSERT(lengths_right.device().type() == at::DeviceType::CPU); + TORCH_INTERNAL_ASSERT(combined_values.device().type() == at::DeviceType::CPU); + TORCH_CHECK(lengths_left.size(0) == lengths_right.size(0)); + auto B = lengths_left.size(0); + + auto L_left = lengths_left.sum().item(); + auto L_right = lengths_right.sum().item(); + TORCH_CHECK(L_left + L_right == combined_values.numel()); + + auto values_left = at::empty({L_left}, combined_values.options()); + auto values_right = at::empty({L_right}, combined_values.options()); + + if (L_left == 0 && L_right == 0) { + return std::make_tuple(values_left, values_right); + } + + const auto combined_lengths = lengths_left + lengths_right; + const auto combined_offsets = + fbgemm_gpu::asynchronous_complete_cumsum_cpu(combined_lengths.view({-1})); + const auto offsets_left = + fbgemm_gpu::asynchronous_complete_cumsum_cpu(lengths_left.view({-1})); + const auto offsets_right = + fbgemm_gpu::asynchronous_complete_cumsum_cpu(lengths_right.view({-1})); + + AT_DISPATCH_INTEGRAL_TYPES( + lengths_left.scalar_type(), + "split_1d_jagged_jagged_values_cpu_kernel_input1", + [&] { + using index_t = scalar_t; + AT_DISPATCH_ALL_TYPES_AND2( + at::ScalarType::BFloat16, + at::ScalarType::Half, + combined_values.scalar_type(), + "split_1d_jagged_jagged_values_cpu_kernel_input2", + [&] { + using val_t = scalar_t; + _split_1d_jagged_jagged_cpu_kernel( + B, + combined_offsets.accessor(), + combined_values.accessor(), + lengths_left.accessor(), + offsets_left.accessor(), + offsets_right.accessor(), + values_left.accessor(), + values_right.accessor()); + }); + }); + + return std::make_tuple(values_left, values_right); +} + +std::tuple split_1d_jagged_jagged_meta( + const at::Tensor& lengths_left, + const at::Tensor& lengths_right, + const at::Tensor& combined_values) { + auto L_left = lengths_left.sum().item(); + auto L_right = lengths_right.sum().item(); + + auto values_left = at::native::empty_meta_symint( + {L_left}, + /*dtype=*/::std::make_optional(combined_values.scalar_type()), + /*layout=*/::std::make_optional(combined_values.layout()), + /*device=*/::std::make_optional(c10::Device(c10::kMeta)), + /*pin_memory=*/::std::nullopt); + + auto values_right = at::native::empty_meta_symint( + {L_right}, + /*dtype=*/::std::make_optional(combined_values.scalar_type()), + /*layout=*/::std::make_optional(combined_values.layout()), + /*device=*/::std::make_optional(c10::Device(c10::kMeta)), + /*pin_memory=*/::std::nullopt); + + return std::make_tuple(values_left, values_right); +} +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/split_1d_jagged_jagged.cu b/recommendation_v4/generative_recommenders/ops/cpp/split_1d_jagged_jagged.cu new file mode 100644 index 000000000..181489bae --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/split_1d_jagged_jagged.cu @@ -0,0 +1,147 @@ +/* Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "common.h" +#include "fbgemm_gpu/sparse_ops.h" // @manual +#include "fbgemm_gpu/utils/fixed_divisor.cuh" // @manual + +namespace hstu { + +static constexpr int32_t kMaxThreads = 1024; + +template +__global__ +__launch_bounds__(kMaxThreads) void _split_1d_jagged_jagged_cuda_kernel( + int32_t B, + const at::PackedTensorAccessor32 + combined_offsets, + const at::PackedTensorAccessor32 + combined_values, + const at::PackedTensorAccessor32 + lengths_left, + const at::PackedTensorAccessor32 + offsets_left, + const at::PackedTensorAccessor32 + offsets_right, + at::PackedTensorAccessor32 values_left, + at::PackedTensorAccessor32 values_right) { + for (auto b = blockIdx.x * blockDim.y + threadIdx.y; + b < static_cast(B); + b += gridDim.x * blockDim.y) { + auto combined_start = combined_offsets[b]; + auto left_len = lengths_left[b]; + auto right_len = combined_offsets[b + 1] - combined_offsets[b] - left_len; + auto left_start = offsets_left[b]; + auto right_start = offsets_right[b]; + + for (auto i = threadIdx.x; i < static_cast(left_len + right_len); + i += blockDim.x) { + if (i < static_cast(left_len)) { + values_left[left_start + i] = combined_values[combined_start + i]; + } else { + values_right[right_start + i - left_len] = + combined_values[combined_start + i]; + } + } + } +} + +std::tuple split_1d_jagged_jagged_cuda( + const at::Tensor& lengths_left, + const at::Tensor& lengths_right, + const at::Tensor& combined_values) { + at::cuda::OptionalCUDAGuard device_guard; + device_guard.set_index(combined_values.get_device()); + TORCH_INTERNAL_ASSERT(lengths_left.device().type() == at::DeviceType::CUDA); + TORCH_INTERNAL_ASSERT(lengths_right.device().type() == at::DeviceType::CUDA); + TORCH_INTERNAL_ASSERT( + combined_values.device().type() == at::DeviceType::CUDA); + TORCH_CHECK(lengths_left.size(0) == lengths_right.size(0)); + + auto B = lengths_left.size(0); + auto L_left = lengths_left.sum().item(); + auto L_right = lengths_right.sum().item(); + TORCH_CHECK(L_left + L_right == combined_values.numel()); + TORCH_CHECK(L_left < std::numeric_limits::max()); + TORCH_CHECK(L_right < std::numeric_limits::max()); + TORCH_CHECK(combined_values.get_device() == lengths_left.get_device()); + TORCH_CHECK(combined_values.get_device() == lengths_right.get_device()); + + auto values_left = at::empty({L_left}, combined_values.options()); + auto values_right = at::empty({L_right}, combined_values.options()); + + if (L_left == 0 && L_right == 0) { + return std::make_tuple(values_left, values_right); + } + + const auto combined_lengths = lengths_left + lengths_right; + const auto combined_offsets = + fbgemm_gpu::asynchronous_complete_cumsum_gpu(combined_lengths.view({-1})); + const auto offsets_left = + fbgemm_gpu::asynchronous_complete_cumsum_gpu(lengths_left.view({-1})); + const auto offsets_right = + fbgemm_gpu::asynchronous_complete_cumsum_gpu(lengths_right.view({-1})); + + // Optimized thread block configuration based on benchmark results + uint32_t B_blocks = 4; + dim3 threads(256, B_blocks); + auto blocks = div_round_up(B, B_blocks); + + AT_DISPATCH_INTEGRAL_TYPES( + lengths_left.scalar_type(), + "split_1d_jagged_jagged_values_cuda_kernel_input1", + [&] { + using index_t = scalar_t; + AT_DISPATCH_ALL_TYPES_AND2( + at::ScalarType::BFloat16, + at::ScalarType::Half, + combined_values.scalar_type(), + "split_1d_jagged_jagged_values_cuda_kernel_input2", + [&] { + using val_t = scalar_t; + _split_1d_jagged_jagged_cuda_kernel<<< + blocks, + threads, + 0, + at::cuda::getCurrentCUDAStream()>>>( + B, + combined_offsets + .packed_accessor32(), + combined_values + .packed_accessor32(), + lengths_left + .packed_accessor32(), + offsets_left + .packed_accessor32(), + offsets_right + .packed_accessor32(), + values_left + .packed_accessor32(), + values_right + .packed_accessor32()); + }); + }); + + return std::make_tuple(values_left, values_right); +} +} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/tests/concat_1d_jagged_jagged_test.py b/recommendation_v4/generative_recommenders/ops/cpp/tests/concat_1d_jagged_jagged_test.py new file mode 100644 index 000000000..8c27a787b --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/tests/concat_1d_jagged_jagged_test.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 + +# pyre-strict + +import unittest + +import torch +from generative_recommenders.common import gpu_unavailable +from hammer.ops.jagged import concat_1D_jagged_jagged +from hypothesis import given, settings, strategies as st, Verbosity + +# buck2 test @mode/opt -c fbcode.nvcc_arch=h100 fbcode//generative_recommenders/ops/cpp/tests:concat_1d_jagged_jagged_test + +torch.ops.load_library("//generative_recommenders/ops/cpp:cpp_ops") +torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops") +torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops_cpu") + + +class OpsTest(unittest.TestCase): + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore + @given( + batch_size=st.integers(10, 500), + max_seq_len_left=st.integers(10, 1000), + max_seq_len_right=st.integers(10, 1000), + val_dtype=st.sampled_from([torch.float32, torch.float16, torch.bfloat16]), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=100, + deadline=None, + ) + def test_concat_1d_jagged_jagged( + self, + batch_size: int, + max_seq_len_left: int, + max_seq_len_right: int, + val_dtype: torch.dtype, + ) -> None: + batch_size = 3 + max_seq_len_left = 4 + max_seq_len_right = 2 + lengths_left = torch.randint( + 0, max_seq_len_left + 1, (batch_size,), device="cpu" + ) + values_left = torch.rand( + (int(torch.sum(lengths_left).cpu().item()),), dtype=val_dtype, device="cpu" + ) + offsets_left = torch.zeros( + (batch_size + 1,), + dtype=lengths_left.dtype, + device=lengths_left.device, + ) + offsets_left[1:] = torch.cumsum(lengths_left.view(-1), dim=0) + lengths_right = torch.randint( + 0, max_seq_len_right + 1, (batch_size,), device="cpu" + ) + values_right = torch.rand( + (int(torch.sum(lengths_right).cpu().item()),), dtype=val_dtype, device="cpu" + ) + offsets_right = torch.zeros( + (batch_size + 1,), + dtype=lengths_right.dtype, + device=lengths_right.device, + ) + offsets_right[1:] = torch.cumsum(lengths_right.view(-1), dim=0) + custom_cpu_result = torch.ops.hstu.concat_1d_jagged_jagged( + lengths_left=lengths_left, + values_left=values_left, + lengths_right=lengths_right, + values_right=values_right, + ) + + custom_cuda_result = torch.ops.hstu.concat_1d_jagged_jagged( + lengths_left=lengths_left.cuda(), + values_left=values_left.cuda(), + lengths_right=lengths_right.cuda(), + values_right=values_right.cuda(), + ) + torch.testing.assert_close(custom_cuda_result.cpu(), custom_cpu_result) + + @unittest.skipIf(*gpu_unavailable) + def test_concat_1d_jagged_jagged_vs_hammer(self) -> None: + torch.manual_seed(42) + batch_size = 8 + max_seq_len_left = 50 + max_seq_len_right = 30 + + lengths_left = torch.randint( + 0, max_seq_len_left + 1, (batch_size,), dtype=torch.int32 + ) + lengths_right = torch.randint( + 0, max_seq_len_right + 1, (batch_size,), dtype=torch.int32 + ) + + total_left = int(lengths_left.sum().item()) + total_right = int(lengths_right.sum().item()) + + values_left = ( + torch.randn(total_left, dtype=torch.float32) + if total_left > 0 + else torch.empty(0, dtype=torch.float32) + ) + values_right = ( + torch.randn(total_right, dtype=torch.float32) + if total_right > 0 + else torch.empty(0, dtype=torch.float32) + ) + + offsets_left = torch.zeros( + (batch_size + 1,), dtype=lengths_left.dtype, device=lengths_left.device + ) + offsets_left[1:] = torch.cumsum(lengths_left.view(-1), dim=0) + offsets_right = torch.zeros( + (batch_size + 1,), dtype=lengths_right.dtype, device=lengths_right.device + ) + offsets_right[1:] = torch.cumsum(lengths_right.view(-1), dim=0) + + combined_values_ref = concat_1D_jagged_jagged( + max_seq_len_left=max_seq_len_left, + offsets_left=offsets_left, + values_left=values_left, + max_seq_len_right=max_seq_len_right, + offsets_right=offsets_right, + values_right=values_right, + ) + + custom_cuda_result = torch.ops.hstu.concat_1d_jagged_jagged( + lengths_left=lengths_left.cuda(), + values_left=values_left.cuda(), + lengths_right=lengths_right.cuda(), + values_right=values_right.cuda(), + ) + + torch.testing.assert_close(custom_cuda_result.cpu(), combined_values_ref) diff --git a/recommendation_v4/generative_recommenders/ops/cpp/tests/hstu_mha_cpu_test.py b/recommendation_v4/generative_recommenders/ops/cpp/tests/hstu_mha_cpu_test.py new file mode 100644 index 000000000..cb787ea61 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/tests/hstu_mha_cpu_test.py @@ -0,0 +1,39 @@ +# (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +# pyre-strict + +# cmd: buck2 run @//mode/opt -c fbcode.enable_gpu_sections=true -c fbcode.platform010_cuda_version=12.8 -c fbcode.nvcc_arch=b200a //generative_recommenders/ops/cpp/tests:hstu_mha_cpu_test + +import unittest + +import torch + +torch.ops.load_library( + "//generative_recommenders/ops/cpp/hstu_attention:hstu_flash_attention" +) + + +class TestHstuMhaFwd(unittest.TestCase): + def test_hstu_mha_fwd(self) -> None: + q: torch.Tensor = torch.randn([100, 4, 64], dtype=torch.bfloat16, device="cpu") + k: torch.Tensor = torch.randn([100, 4, 64], dtype=torch.bfloat16, device="cpu") + v: torch.Tensor = torch.randn([100, 4, 64], dtype=torch.bfloat16, device="cpu") + res = torch.ops.hstu.hstu_mha_fwd( + 10, + 0.25, + q, + k, + v, + torch.empty([0], dtype=torch.int32, device="cpu"), + True, # causal + None, + None, + 0, + 0, + 0, + None, # q_descale + None, # k_descale + None, # v_descale + 0, # sm_margin + ) + self.assertIsNotNone(res) diff --git a/recommendation_v4/generative_recommenders/ops/cpp/tests/jagged_transpose_1d_test.py b/recommendation_v4/generative_recommenders/ops/cpp/tests/jagged_transpose_1d_test.py new file mode 100644 index 000000000..6a5f5997b --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/tests/jagged_transpose_1d_test.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 + +# pyre-strict + +import unittest + +import torch +from generative_recommenders.common import gpu_unavailable +from hammer.ops.jagged import jagged_transpose_1D +from hypothesis import given, settings, strategies as st, Verbosity + +# buck2 test @mode/opt -c fbcode.nvcc_arch=h100 fbcode//generative_recommenders/ops/cpp/tests:jagged_transpose_1d_test + +torch.ops.load_library("//generative_recommenders/ops/cpp:cpp_ops") +torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops") +torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops_cpu") + + +class OpsTest(unittest.TestCase): + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore + @given( + size1=st.integers(2, 10), + size2=st.integers(2, 10), + max_len=st.integers(5, 50), + val_dtype=st.sampled_from([torch.float32, torch.float16, torch.bfloat16]), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=100, + deadline=None, + ) + def test_jagged_transpose_1d( + self, + size1: int, + size2: int, + max_len: int, + val_dtype: torch.dtype, + ) -> None: + lengths = torch.randint( + 0, max_len + 1, (size1 * size2,), dtype=torch.int32, device="cpu" + ) + offsets = torch.zeros( + (size1 * size2 + 1,), dtype=lengths.dtype, device=lengths.device + ) + offsets[1:] = torch.cumsum(lengths.view(-1), dim=0) + + values = torch.randn(int(offsets[-1].item()), dtype=val_dtype, device="cpu") + + ( + custom_cpu_values, + custom_cpu_offsets, + custom_cpu_lengths, + ) = torch.ops.hstu.jagged_transpose_1d( + values=values, + offsets=offsets, + lengths=lengths, + max_len=max_len, + size1=size1, + size2=size2, + ) + + ( + custom_cuda_values, + custom_cuda_offsets, + custom_cuda_lengths, + ) = torch.ops.hstu.jagged_transpose_1d( + values=values.cuda(), + offsets=offsets.cuda(), + lengths=lengths.cuda(), + max_len=max_len, + size1=size1, + size2=size2, + ) + + torch.testing.assert_close(custom_cuda_values.cpu(), custom_cpu_values) + torch.testing.assert_close(custom_cuda_offsets.cpu(), custom_cpu_offsets) + torch.testing.assert_close(custom_cuda_lengths.cpu(), custom_cpu_lengths) + + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore + @given( + size1=st.integers(2, 10), + size2=st.integers(2, 10), + max_len=st.integers(5, 50), + val_dtype=st.sampled_from([torch.float32, torch.float16, torch.bfloat16]), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=100, + deadline=None, + ) + def test_jagged_transpose_1d_vs_hammer( + self, + size1: int, + size2: int, + max_len: int, + val_dtype: torch.dtype, + ) -> None: + lengths = torch.randint(0, max_len + 1, (size1 * size2,), dtype=torch.int32) + offsets = torch.zeros( + (size1 * size2 + 1,), dtype=lengths.dtype, device=lengths.device + ) + offsets[1:] = torch.cumsum(lengths.view(-1), dim=0) + + values = torch.randn(int(offsets[-1].item()), dtype=val_dtype) + + values_ref, offsets_ref, lengths_ref = jagged_transpose_1D( + values=values, + offsets=offsets, + lengths=lengths, + max_len=max_len, + size1=size1, + size2=size2, + ) + + ( + custom_cuda_values, + custom_cuda_offsets, + custom_cuda_lengths, + ) = torch.ops.hstu.jagged_transpose_1d( + values=values.cuda(), + offsets=offsets.cuda(), + lengths=lengths.cuda(), + max_len=max_len, + size1=size1, + size2=size2, + ) + + torch.testing.assert_close(custom_cuda_values.cpu(), values_ref) + torch.testing.assert_close(custom_cuda_offsets.cpu(), offsets_ref) + torch.testing.assert_close(custom_cuda_lengths.cpu(), lengths_ref) diff --git a/recommendation_v4/generative_recommenders/ops/cpp/tests/replace_last_n_with_jagged_test.py b/recommendation_v4/generative_recommenders/ops/cpp/tests/replace_last_n_with_jagged_test.py new file mode 100644 index 000000000..9826f199d --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/tests/replace_last_n_with_jagged_test.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 + +# pyre-strict + +import unittest + +import torch +from generative_recommenders.common import gpu_unavailable +from hammer.ops.jagged import replace_last_n_with_jagged + +# buck2 test @mode/opt -c fbcode.nvcc_arch=h100 fbcode//generative_recommenders/ops/cpp/tests:replace_last_n_with_jagged_test + +torch.ops.load_library("//generative_recommenders/ops/cpp:cpp_ops") +torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops") +torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops_cpu") + + +class OpsTest(unittest.TestCase): + @unittest.skipIf(*gpu_unavailable) + def test_replace_last_n_with_jagged(self) -> None: + torch.manual_seed(42) + batch_size = 8 + embedding_dim = 64 + max_seq_len_left = 25 + max_seq_len_right = 10 + + lengths_left = torch.randint( + max_seq_len_right, max_seq_len_left + 1, (batch_size,), dtype=torch.int32 + ) + lengths_right = torch.randint( + 1, max_seq_len_right + 1, (batch_size,), dtype=torch.int32 + ) + + lengths_right = torch.min(lengths_right, lengths_left) + + total_left = int(lengths_left.sum().item()) + total_right = int(lengths_right.sum().item()) + + values_left = torch.randn(total_left, embedding_dim, dtype=torch.float32) + values_right = torch.randn(total_right, embedding_dim, dtype=torch.float32) + + custom_cpu_result = torch.ops.hstu.replace_last_n_with_jagged( + lengths_left=lengths_left, + values_left=values_left, + lengths_right=lengths_right, + values_right=values_right, + ) + + custom_cuda_result = torch.ops.hstu.replace_last_n_with_jagged( + lengths_left=lengths_left.cuda(), + values_left=values_left.cuda(), + lengths_right=lengths_right.cuda(), + values_right=values_right.cuda(), + ) + + torch.testing.assert_close(custom_cuda_result.cpu(), custom_cpu_result) + + @unittest.skipIf(*gpu_unavailable) + def test_replace_last_n_with_jagged_vs_hammer(self) -> None: + torch.manual_seed(42) + batch_size = 8 + embedding_dim = 32 + max_seq_len_left = 20 + max_seq_len_right = 8 + + lengths_left = torch.randint( + max_seq_len_right, max_seq_len_left + 1, (batch_size,), dtype=torch.int32 + ) + lengths_right = torch.randint( + 1, max_seq_len_right + 1, (batch_size,), dtype=torch.int32 + ) + + lengths_right = torch.min(lengths_right, lengths_left) + + total_left = int(lengths_left.sum().item()) + total_right = int(lengths_right.sum().item()) + + values_left = torch.randn(total_left, embedding_dim, dtype=torch.float32) + values_right = torch.randn(total_right, embedding_dim, dtype=torch.float32) + + offsets_left = torch.zeros( + (batch_size + 1,), dtype=lengths_left.dtype, device=lengths_left.device + ) + offsets_left[1:] = torch.cumsum(lengths_left.view(-1), dim=0) + offsets_right = torch.zeros( + (batch_size + 1,), dtype=lengths_right.dtype, device=lengths_right.device + ) + offsets_right[1:] = torch.cumsum(lengths_right.view(-1), dim=0) + + result_ref = replace_last_n_with_jagged( + max_seq_len_left=max_seq_len_left, + offsets_left=offsets_left, + values_left=values_left, + offsets_right=offsets_right, + values_right=values_right, + ) + + custom_cuda_result = torch.ops.hstu.replace_last_n_with_jagged( + lengths_left=lengths_left.cuda(), + values_left=values_left.cuda(), + lengths_right=lengths_right.cuda(), + values_right=values_right.cuda(), + ) + + torch.testing.assert_close(custom_cuda_result.cpu(), result_ref) diff --git a/recommendation_v4/generative_recommenders/ops/cpp/tests/split_1d_jagged_jagged_test.py b/recommendation_v4/generative_recommenders/ops/cpp/tests/split_1d_jagged_jagged_test.py new file mode 100644 index 000000000..24f12c4a2 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/cpp/tests/split_1d_jagged_jagged_test.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 + +# pyre-strict + +import unittest + +import torch +from generative_recommenders.common import gpu_unavailable +from hammer.ops.jagged import split_1D_jagged_jagged + +# buck2 test @mode/opt -c fbcode.nvcc_arch=h100 fbcode//generative_recommenders/ops/cpp/tests:split_1d_jagged_jagged_test + +torch.ops.load_library("//generative_recommenders/ops/cpp:cpp_ops") +torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops") +torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops_cpu") + + +class OpsTest(unittest.TestCase): + @unittest.skipIf(*gpu_unavailable) + def test_split_1d_jagged_jagged(self) -> None: + torch.manual_seed(42) + batch_size = 8 + max_seq_len_left = 25 + max_seq_len_right = 20 + + lengths_left = torch.randint( + 0, max_seq_len_left + 1, (batch_size,), dtype=torch.int32 + ) + lengths_right = torch.randint( + 0, max_seq_len_right + 1, (batch_size,), dtype=torch.int32 + ) + + combined_lengths = lengths_left + lengths_right + combined_offsets = torch.zeros( + (batch_size + 1,), dtype=lengths_left.dtype, device=lengths_left.device + ) + combined_offsets[1:] = torch.cumsum(combined_lengths.view(-1), dim=0) + + combined_values = torch.randn( + int(combined_offsets[-1].item()), dtype=torch.float32 + ) + + custom_cpu_left, custom_cpu_right = torch.ops.hstu.split_1d_jagged_jagged( + lengths_left=lengths_left, + lengths_right=lengths_right, + combined_values=combined_values, + ) + + custom_cuda_left, custom_cuda_right = torch.ops.hstu.split_1d_jagged_jagged( + lengths_left=lengths_left.cuda(), + lengths_right=lengths_right.cuda(), + combined_values=combined_values.cuda(), + ) + + torch.testing.assert_close(custom_cuda_left.cpu(), custom_cpu_left) + torch.testing.assert_close(custom_cuda_right.cpu(), custom_cpu_right) + + @unittest.skipIf(*gpu_unavailable) + def test_split_1d_jagged_jagged_vs_hammer(self) -> None: + torch.manual_seed(42) + batch_size = 8 + max_seq_len_left = 25 + max_seq_len_right = 20 + + lengths_left = torch.randint( + 0, max_seq_len_left + 1, (batch_size,), dtype=torch.int32 + ) + lengths_right = torch.randint( + 0, max_seq_len_right + 1, (batch_size,), dtype=torch.int32 + ) + + offsets_left = torch.zeros( + (batch_size + 1,), dtype=lengths_left.dtype, device=lengths_left.device + ) + offsets_left[1:] = torch.cumsum(lengths_left.view(-1), dim=0) + offsets_right = torch.zeros( + (batch_size + 1,), dtype=lengths_right.dtype, device=lengths_right.device + ) + offsets_right[1:] = torch.cumsum(lengths_right.view(-1), dim=0) + + combined_offsets = offsets_left + offsets_right + combined_values = torch.randn( + int(combined_offsets[-1].item()), dtype=torch.float32 + ) + + left_ref, right_ref = split_1D_jagged_jagged( + max_seq_len=max_seq_len_left + max_seq_len_right, + values=combined_values, + offsets_left=offsets_left, + offsets_right=offsets_right, + ) + + custom_cuda_left, custom_cuda_right = torch.ops.hstu.split_1d_jagged_jagged( + lengths_left=lengths_left.cuda(), + lengths_right=lengths_right.cuda(), + combined_values=combined_values.cuda(), + ) + + torch.testing.assert_close(custom_cuda_left.cpu(), left_ref) + torch.testing.assert_close(custom_cuda_right.cpu(), right_ref) diff --git a/recommendation_v4/generative_recommenders/ops/hstu_attention.py b/recommendation_v4/generative_recommenders/ops/hstu_attention.py new file mode 100644 index 000000000..137482227 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/hstu_attention.py @@ -0,0 +1,353 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +from typing import Optional + +import torch +from generative_recommenders.common import HammerKernel, switch_to_contiguous_if_needed +from generative_recommenders.ops.pytorch.pt_hstu_attention import ( + pytorch_cached_hstu_mha, + pytorch_hstu_mha, +) +from generative_recommenders.ops.triton.triton_hstu_attention import ( + triton_cached_hstu_mha, + triton_hstu_mha, +) + +try: + # @manual=//generative_recommenders/ops/triton_aot:triton_ragged_hstu_attention + from generative_recommenders.ops.triton_aot.triton_ragged_hstu_attention import ( # pyre-ignore[21] + aot_triton_kernel_wrapper_cached_hstu_mha, + aot_triton_kernel_wrapper_ragged_hstu_mha, + ) +except ImportError: + + def aot_triton_kernel_wrapper_cached_hstu_mha( + *args: object, + **kwargs: object, + ) -> torch.Tensor: + raise ImportError( + "AOT-T is required for the TRITON_INFERENCE cached_hstu_mha kernel." + ) + + def aot_triton_kernel_wrapper_ragged_hstu_mha( + *args: object, + **kwargs: object, + ) -> torch.Tensor: + raise ImportError( + "AOT-T is required for the TRITON_INFERENCE ragged_hstu_mha kernel." + ) + + +try: + from hammer.ops.triton.cc.hstu_attention.triton_cc_hstu_attention import ( + triton_cc_hstu_mha, + ) + from hammer.v2.ops.triton.template.tlx_bw_hstu_attention import ( + tlx_bw_hstu_mha_wrapper, + ) +except ImportError: + tlx_bw_hstu_mha_wrapper = None + from generative_recommenders.ops.triton.triton_hstu_attention import ( + triton_hstu_mha as triton_cc_hstu_mha, + ) +from torch.fx._symbolic_trace import is_fx_tracing + +torch.fx.wrap("triton_hstu_mha") +torch.fx.wrap("triton_cached_hstu_mha") + + +@torch.fx.wrap +def hstu_mha_cuda( + max_seq_len: int, + alpha: float, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + seq_offsets: torch.Tensor, + num_targets: Optional[torch.Tensor] = None, + max_attn_len: int = 0, + contextual_seq_len: int = 0, +) -> torch.Tensor: + """TorchScript-friendly inference forwarder onto ``torch.ops.hstu.hstu_mha``. + + Bypasses the ``HammerKernel`` enum dispatch in :func:`hstu_mha` so the + scripted graph has a single concrete C++ op to call. Mirrors the + inference-only path of + :func:`generative_recommenders.ops.cpp.cuda_hstu_attention.cuda_hstu_mha_inference_wrapper` + with the subset of arguments :class:`STULayer` actually uses. + """ + return torch.ops.hstu.hstu_mha( + max_seq_len, + alpha, + q, + k, + v, + seq_offsets, + True, # causal + num_targets, + None, # attn_scale + max_attn_len, + 0, # min_full_attn_seq_len + contextual_seq_len, + None, # q_descale + None, # k_descale + None, # v_descale + False, # sort_by_length + False, # deterministic + 0, # sm_margin + 0, # max_q_len + None, # seq_offsets_q + 0, # num_softmax_heads + False, # training + None, # max_seq_len_tensor + None, # contextual_seq_len_tensor + None, # max_attn_len_tensor + None, # min_full_attn_seq_len_tensor + 1, # num_groups + ) + + +def hstu_mha( + max_seq_len: int, + alpha: float, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + seq_offsets: torch.Tensor, + causal: bool = True, + dropout_pr: float = 0.0, + training: bool = True, + num_targets: Optional[torch.Tensor] = None, + attn_scale: Optional[torch.Tensor] = None, + max_attn_len: int = 0, + contextual_seq_len: int = 0, + min_full_attn_seq_len: int = 0, + sort_by_length: bool = False, + kernel: HammerKernel = HammerKernel.PYTORCH, + enable_tma: bool = False, +) -> torch.Tensor: + _, H, _ = q.shape + if not is_fx_tracing(): + torch._assert(max_seq_len > 0, "max_seq_len must be larger than 0") + torch._assert(q.dim() == 3, "q must be 3-D") + torch._assert(k.shape == q.shape, "k must be the same shape as q") + torch._assert(v.dim() == 3, "v must be 3-D") + torch._assert(v.shape[0] == q.shape[0], "wrong v shape[0]") + torch._assert(v.shape[1] == H, "wrong v shape[1]") + torch._assert(causal, "only support causal attention") + + if kernel in [ + HammerKernel.TRITON, + HammerKernel.TLX, + HammerKernel.TRITON_CC, + HammerKernel.TRITON_INFERENCE, + ]: + if not is_fx_tracing() and kernel == HammerKernel.TRITON: + torch._assert(q.is_cuda, "q must be CUDA tensor") + torch._assert(k.is_cuda, "k must be CUDA tensor") + torch._assert(v.is_cuda, "v must be CUDA tensor") + torch._assert(seq_offsets.is_cuda, "seq_offsets must be CUDA tensor") + torch._assert(dropout_pr < 1e-6, "dropout for triton path not implemented") + torch._assert( + min_full_attn_seq_len == 0, "min_full_attn_seq_len not implemented" + ) + assert attn_scale is None, "attn_scale not implemented" + q = switch_to_contiguous_if_needed(q) + k = switch_to_contiguous_if_needed(k) + v = switch_to_contiguous_if_needed(v) + seq_offsets = seq_offsets.contiguous() + + if kernel == HammerKernel.TRITON: + return triton_hstu_mha( + N=max_seq_len, + alpha=alpha, + q=q, + k=k, + v=v, + seq_offsets=seq_offsets, + num_targets=num_targets, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + sort_by_length=sort_by_length, + enable_tma=enable_tma, + ) + elif kernel == HammerKernel.TLX: + if tlx_bw_hstu_mha_wrapper is None: + raise ImportError( + "hammer.v2 is required for the TLX kernel. " + "Falling back to TRITON or PYTORCH kernel instead." + ) + return tlx_bw_hstu_mha_wrapper( + max_seq_len=max_seq_len, + alpha=alpha, + q=q, + k=k, + v=v, + seq_offsets=seq_offsets, + attn_scale=torch.tensor(1.0 / max_seq_len, device=q.device), + num_targets=num_targets, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + sort_by_length=sort_by_length, + ) + elif kernel == HammerKernel.TRITON_CC: + return triton_cc_hstu_mha( + N=max_seq_len, + alpha=alpha, + q=q, + k=k, + v=v, + seq_offsets=seq_offsets, + num_targets=num_targets, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + ) + elif kernel == HammerKernel.TRITON_INFERENCE: + return aot_triton_kernel_wrapper_ragged_hstu_mha( + N=max_seq_len, + alpha=alpha, + q=q, + k=k, + v=v, + seq_offsets=seq_offsets, + invalid_attn_mask_type="causal", + num_targets=num_targets, + attn_scale=attn_scale, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + full_attn_size=min_full_attn_seq_len, + num_softmax_heads=0, + ) + else: + return pytorch_hstu_mha( + max_seq_len=max_seq_len, + alpha=alpha, + q=q, + k=k, + v=v, + seq_offsets=seq_offsets, + causal=True, + dropout_pr=dropout_pr, + training=training, + num_targets=num_targets, + attn_scale=attn_scale, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + min_full_attn_seq_len=min_full_attn_seq_len, + ) + + +def delta_hstu_mha( + max_seq_len: int, + alpha: float, + delta_q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + seq_offsets: torch.Tensor, + num_targets: Optional[torch.Tensor] = None, + max_attn_len: int = 0, + contextual_seq_len: int = 0, + kernel: HammerKernel = HammerKernel.PYTORCH, + enable_tma: bool = False, +) -> torch.Tensor: + L, H, D = delta_q.shape + B = seq_offsets.size(0) - 1 + DeltaSize = L // B + if not is_fx_tracing(): + torch._assert(max_seq_len > 0, "max_seq_len must be larger than 0") + torch._assert(delta_q.dim() == 3, "delta_q must be 3-D") + torch._assert(L % B == 0, "delta_q must be padded") + torch._assert(k.dim() == 3, "k must be 3-D") + torch._assert(k.shape[1] == H, "wrong k shape[1]") + torch._assert(k.shape[2] == D, "wrong k shape[2]") + torch._assert(v.dim() == 3, "v must be 3-D") + torch._assert(v.shape[1] == H, "wrong v shape[1]") + if kernel in [ + HammerKernel.TRITON, + HammerKernel.TRITON_CC, + HammerKernel.TRITON_INFERENCE, + ]: + if not is_fx_tracing() and kernel == HammerKernel.TRITON: + torch._assert(delta_q.is_cuda, "q must be CUDA tensor") + torch._assert(seq_offsets.is_cuda, "seq_offsets must be CUDA tensor") + if num_targets is not None: + torch._assert(num_targets.is_cuda, "num_targets must be CUDA tensor") + seq_offsets = seq_offsets.contiguous() + delta_q = switch_to_contiguous_if_needed(delta_q) + k = switch_to_contiguous_if_needed(k) + v = switch_to_contiguous_if_needed(v) + + if kernel == HammerKernel.TRITON: + return triton_cached_hstu_mha( + N=max_seq_len, + alpha=alpha, + delta_q=delta_q, + k=k, + v=v, + seq_offsets=seq_offsets, + num_targets=num_targets, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + enable_tma=enable_tma, + ) + elif kernel == HammerKernel.TRITON_CC: + return triton_cc_hstu_mha( + N=max_seq_len, + alpha=alpha, + q=delta_q, + k=k, + v=v, + seq_offsets=seq_offsets, + num_targets=num_targets, + is_delta_q=True, + delta_size=DeltaSize, + ) + elif kernel == HammerKernel.TRITON_INFERENCE: + delta_x_offsets = torch.arange( + 0, + L + 1, + DeltaSize, + device=delta_q.device, + dtype=seq_offsets.dtype, + ) + return aot_triton_kernel_wrapper_cached_hstu_mha( + N=max_seq_len, + alpha=alpha, + delta_q=delta_q, + k=k, + v=v, + delta_x_offsets=delta_x_offsets, + seq_offsets=seq_offsets, + num_targets=num_targets, + attn_scale=None, + max_attn_len=max_attn_len, + full_attn_size=0, + ) + else: + return pytorch_cached_hstu_mha( + max_seq_len=max_seq_len, + alpha=alpha, + delta_q=delta_q, + k=k, + v=v, + seq_offsets=seq_offsets, + num_targets=num_targets, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + ) diff --git a/recommendation_v4/generative_recommenders/ops/hstu_compute.py b/recommendation_v4/generative_recommenders/ops/hstu_compute.py new file mode 100644 index 000000000..7728c1454 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/hstu_compute.py @@ -0,0 +1,390 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +from typing import Optional, Tuple + +import torch +import torch.nn.functional as F +from generative_recommenders.ops.layer_norm import layer_norm +from generative_recommenders.ops.mm import addmm +from generative_recommenders.ops.pytorch.pt_hstu_linear import ( + pytorch_hstu_compute_output, +) + +try: + from hammer.ops.triton.cc.addmm.triton_cc_addmm import triton_cc_addmm + from hammer.ops.triton.cc.group_norm_mul_dropout.triton_cc_group_norm_mul_dropout import ( + triton_cc_group_norm_mul_dropout_wrapper, + ) + from hammer.ops.triton.cc.layer_norm_mul_dropout.triton_cc_layer_norm_mul_dropout import ( + triton_cc_layer_norm_mul_dropout_wrapper, + ) +except ImportError: + triton_cc_addmm = None + triton_cc_group_norm_mul_dropout_wrapper = None + triton_cc_layer_norm_mul_dropout_wrapper = None +from generative_recommenders.common import HammerKernel +from generative_recommenders.ops.hstu_attention import hstu_mha, hstu_mha_cuda +from generative_recommenders.ops.triton.triton_hstu_linear import ( + triton_hstu_compute_output, +) +from generative_recommenders.ops.triton.triton_hstu_preprocess_and_attention import ( + triton_hstu_preprocess_and_attention, +) +from torch.fx._symbolic_trace import is_fx_tracing + +try: + # @manual=//generative_recommenders/ops/triton_aot:triton_group_norm_mul_dropout + from generative_recommenders.ops.triton_aot.triton_group_norm_mul_dropout import ( # pyre-ignore[21] + aot_triton_kernel_wrapper_group_norm_mul_dropout, + ) + + # @manual=//generative_recommenders/ops/triton_aot:triton_layer_norm_mul_dropout + from generative_recommenders.ops.triton_aot.triton_layer_norm_mul_dropout import ( # pyre-ignore[21] + aot_triton_kernel_wrapper_layer_norm_mul_dropout, + ) +except ImportError: + + def aot_triton_kernel_wrapper_group_norm_mul_dropout( + *args: object, + **kwargs: object, + ) -> torch.Tensor: + raise ImportError( + "AOT-T is required for the TRITON_INFERENCE group_norm_mul_dropout kernel." + ) + + def aot_triton_kernel_wrapper_layer_norm_mul_dropout( + *args: object, + **kwargs: object, + ) -> torch.Tensor: + raise ImportError( + "AOT-T is required for the TRITON_INFERENCE layer_norm_mul_dropout kernel." + ) + + +torch.fx.wrap("triton_hstu_compute_output") + + +def hstu_compute_uqvk( + x: torch.Tensor, + norm_weight: torch.Tensor, + norm_bias: torch.Tensor, + norm_eps: float, + num_heads: int, + attn_dim: int, + hidden_dim: int, + uvqk_weight: torch.Tensor, + uvqk_bias: torch.Tensor, + kernel: HammerKernel = HammerKernel.PYTORCH, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + if torch.jit.is_scripting(): + # Script-mode fast path: pure PyTorch, no HammerKernel dispatch. + normed_x = F.layer_norm( + x, + normalized_shape=(x.shape[-1],), + weight=norm_weight, + bias=norm_bias, + eps=norm_eps, + ) + uvqk = torch.addmm(uvqk_bias, normed_x, uvqk_weight) + else: + normed_x = layer_norm( + x, + weight=norm_weight, + bias=norm_bias, + eps=norm_eps, + kernel=kernel, + ) + # NOTE: for AMD training, we go with torch.addmm instead of the triton + # version before Triton on AMD achieves on-par perf with NV GPU. + if torch.version.hip and kernel == HammerKernel.TRITON: + uvqk = torch.addmm(uvqk_bias, normed_x, uvqk_weight) + else: + uvqk = addmm(uvqk_bias, normed_x, uvqk_weight, kernel) + u, v, q, k = torch.split( + uvqk, + [ + hidden_dim * num_heads, + hidden_dim * num_heads, + attn_dim * num_heads, + attn_dim * num_heads, + ], + dim=1, + ) + u = F.silu(u) + q = q.view(-1, num_heads, attn_dim) + k = k.view(-1, num_heads, attn_dim) + v = v.view(-1, num_heads, hidden_dim) + return u, q, k, v + + +def hstu_compute_output( + attn: torch.Tensor, + u: torch.Tensor, + x: torch.Tensor, + norm_weight: torch.Tensor, + norm_bias: torch.Tensor, + norm_eps: float, + output_weight: torch.Tensor, + num_heads: int, + linear_dim: int, + dropout_ratio: float, + training: bool, + concat_u: bool, + concat_x: bool, + mul_u_activation_type: str, + group_norm: bool, + recompute_y_in_backward: bool, + kernel: HammerKernel = HammerKernel.PYTORCH, +) -> torch.Tensor: + if torch.jit.is_scripting(): + return pytorch_hstu_compute_output( + attn=attn, + u=u, + x=x, + norm_weight=norm_weight, + norm_bias=norm_bias, + output_weight=output_weight, + eps=norm_eps, + dropout_ratio=dropout_ratio, + training=training, + concat_u=concat_u, + concat_x=concat_x, + mul_u_activation_type=mul_u_activation_type, + group_norm=group_norm, + num_heads=num_heads, + linear_dim=linear_dim, + ) + if kernel == HammerKernel.TRITON: + return triton_hstu_compute_output( + attn=attn, + u=u, + x=x, + norm_weight=norm_weight, + norm_bias=norm_bias, + output_weight=output_weight, + eps=norm_eps, + dropout_ratio=dropout_ratio, + training=training, + concat_u=concat_u, + concat_x=concat_x, + mul_u_activation_type=mul_u_activation_type, + group_norm=group_norm, + num_heads=num_heads, + linear_dim=linear_dim, + seed=None, + recompute_y_in_backward=recompute_y_in_backward, + ) + elif kernel == HammerKernel.TRITON_INFERENCE: + if group_norm: + y = aot_triton_kernel_wrapper_group_norm_mul_dropout( + x=attn, + u=u, + weight=norm_weight, + bias=norm_bias, + eps=norm_eps, + silu_u=mul_u_activation_type == "silu", + concat_ux=concat_u and concat_x, + num_heads=num_heads, + linear_dim=linear_dim, + ) + else: + y = aot_triton_kernel_wrapper_layer_norm_mul_dropout( + x=attn, + u=u, + weight=norm_weight, + bias=norm_bias, + eps=norm_eps, + dropout_ratio=dropout_ratio, + training=training, + silu_u=mul_u_activation_type == "silu", + concat_ux=concat_u and concat_x, + mul_u_activation_type=mul_u_activation_type, + ) + return addmm(x, y, output_weight, kernel) + elif kernel == HammerKernel.TRITON_CC: + if triton_cc_group_norm_mul_dropout_wrapper is None or triton_cc_addmm is None: + raise ImportError( + "hammer is required for the TRITON_CC kernel in hstu_compute_output." + ) + if group_norm: + y = triton_cc_group_norm_mul_dropout_wrapper( + x=attn, + u=u, + weight=norm_weight, + bias=norm_bias, + eps=norm_eps, + dropout_ratio=dropout_ratio, + training=training, + concat_ux=concat_u and concat_x, + num_heads=num_heads, + linear_dim=linear_dim, + ) + else: + y = triton_cc_layer_norm_mul_dropout_wrapper( + x=attn, + u=u, + weight=norm_weight, + bias=norm_bias, + eps=norm_eps, + dropout_ratio=dropout_ratio, + training=training, + concat_u=concat_u, + concat_x=concat_x, + mul_u_activation_type=mul_u_activation_type, + ) + return triton_cc_addmm(x, y, output_weight) + else: + return pytorch_hstu_compute_output( + attn=attn, + u=u, + x=x, + norm_weight=norm_weight, + norm_bias=norm_bias, + output_weight=output_weight, + eps=norm_eps, + dropout_ratio=dropout_ratio, + training=training, + concat_u=concat_u, + concat_x=concat_x, + mul_u_activation_type=mul_u_activation_type, + group_norm=group_norm, + num_heads=num_heads, + linear_dim=linear_dim, + ) + + +def hstu_preprocess_and_attention( + x: torch.Tensor, + norm_weight: torch.Tensor, + norm_bias: torch.Tensor, + norm_eps: float, + num_heads: int, + attn_dim: int, + hidden_dim: int, + uvqk_weight: torch.Tensor, + uvqk_bias: torch.Tensor, + max_seq_len: int, + seq_offsets: torch.Tensor, + attn_alpha: float, + causal: bool, + num_targets: Optional[torch.Tensor], + max_attn_len: int, + contextual_seq_len: int, + recompute_uvqk_in_backward: bool, + recompute_normed_x_in_backward: bool, + sort_by_length: bool, + prefill: bool = False, + kernel: HammerKernel = HammerKernel.PYTORCH, + enable_tma: Optional[bool] = None, +) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor]]: + if not is_fx_tracing(): + torch._assert(max_seq_len > 0, "max_seq_len must be larger than 0") + torch._assert(x.dim() == 2, "x must be 2-D") + torch._assert( + x.shape[1] == uvqk_weight.shape[0], + "x.shape[1] must equal uvqk_weight.shape[0]", + ) + torch._assert( + uvqk_weight.shape[1] == 2 * num_heads * (hidden_dim + attn_dim), + "uvqk_weight.shape[1] must equal 2 * num_heads * (hidden_dim + attn_dim)", + ) + torch._assert(causal is True, "only causal attention is supported.") + if torch.jit.is_scripting(): + # Script-mode: compute uvqk via PyTorch fallback then call the + # libtorch-callable CUDA HSTU MHA op directly. Avoids both the + # HammerKernel enum dispatch and the Triton-only fused path. + u, q, k, v = hstu_compute_uqvk( + x=x, + norm_weight=norm_weight, + norm_bias=norm_bias, + norm_eps=norm_eps, + num_heads=num_heads, + attn_dim=attn_dim, + hidden_dim=hidden_dim, + uvqk_weight=uvqk_weight, + uvqk_bias=uvqk_bias, + kernel=HammerKernel.PYTORCH, + ) + attn_output = hstu_mha_cuda( + max_seq_len=max_seq_len, + alpha=attn_alpha, + q=q, + k=k, + v=v, + seq_offsets=seq_offsets, + num_targets=num_targets, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + ).view(-1, hidden_dim * num_heads) + return u, attn_output, k, v + if kernel == HammerKernel.TRITON and prefill is False: + u, attn_output = triton_hstu_preprocess_and_attention( + x=x, + norm_weight=norm_weight, + norm_bias=norm_bias, + norm_eps=norm_eps, + num_heads=num_heads, + attn_dim=attn_dim, + hidden_dim=hidden_dim, + uvqk_weight=uvqk_weight, + uvqk_bias=uvqk_bias, + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + attn_alpha=attn_alpha, + num_targets=num_targets, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + recompute_uvqk_in_backward=recompute_uvqk_in_backward, + recompute_normed_x_in_backward=recompute_normed_x_in_backward, + sort_by_length=sort_by_length, + enable_tma=enable_tma, + ) + attn_output = attn_output.view(-1, hidden_dim * num_heads) + k = None + v = None + else: + u, q, k, v = hstu_compute_uqvk( + x=x, + norm_weight=norm_weight, + norm_bias=norm_bias, + norm_eps=norm_eps, + num_heads=num_heads, + attn_dim=attn_dim, + hidden_dim=hidden_dim, + uvqk_weight=uvqk_weight, + uvqk_bias=uvqk_bias, + kernel=kernel, + ) + attn_output = hstu_mha( + max_seq_len=max_seq_len, + alpha=attn_alpha, + q=q, + k=k, + v=v, + seq_offsets=seq_offsets, + causal=causal, + dropout_pr=0.0, + training=False, + num_targets=num_targets, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + sort_by_length=sort_by_length, + kernel=kernel, + ).view(-1, hidden_dim * num_heads) + return u, attn_output, k, v diff --git a/recommendation_v4/generative_recommenders/ops/jagged_tensors.py b/recommendation_v4/generative_recommenders/ops/jagged_tensors.py new file mode 100644 index 000000000..73e3c4a73 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/jagged_tensors.py @@ -0,0 +1,451 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +from typing import Optional, Tuple + +import torch +from generative_recommenders.common import HammerKernel +from generative_recommenders.ops.pytorch.pt_jagged import pytorch_jagged_dense_bmm_add +from generative_recommenders.ops.pytorch.pt_jagged_tensors import ( + pytorch_concat_2D_jagged, + pytorch_hstu_concat_l2_embeddings, + pytorch_hstu_split_l2_embeddings, + pytorch_split_2D_jagged, +) +from generative_recommenders.ops.triton.triton_jagged import triton_jagged_dense_bmm_add +from generative_recommenders.ops.triton.triton_jagged_tensors import ( + triton_concat_2D_jagged, + triton_concat_2D_jagged_multirow, + triton_split_2D_jagged, + triton_split_2D_jagged_multirow, +) +from torch.fx._symbolic_trace import is_fx_tracing + +try: + # @manual=//generative_recommenders/ops/triton_aot:triton_concat_2d_jagged + from generative_recommenders.ops.triton_aot.triton_concat_2d_jagged import ( # pyre-ignore[21] + aot_triton_kernel_wrapper_concat_2D_jagged, + ) + + # @manual=//generative_recommenders/ops/triton_aot:triton_split_2d_jagged + from generative_recommenders.ops.triton_aot.triton_split_2d_jagged import ( # pyre-ignore[21] + aot_triton_kernel_wrapper_split_2D_jagged, + ) +except ImportError: + + def aot_triton_kernel_wrapper_concat_2D_jagged( + *args: object, + **kwargs: object, + ) -> torch.Tensor: + raise ImportError( + "AOT-T is required for the TRITON_INFERENCE concat_2D_jagged kernel." + ) + + def aot_triton_kernel_wrapper_split_2D_jagged( + *args: object, + **kwargs: object, + ) -> Tuple[torch.Tensor, torch.Tensor]: + raise ImportError( + "AOT-T is required for the TRITON_INFERENCE split_2D_jagged kernel." + ) + + +torch.fx.wrap("triton_jagged_dense_bmm_add") + +try: + from hammer.ops.triton.cc.jagged_dense_bmm.triton_cc_jagged_dense_bmm import ( + triton_cc_jagged_dense_bmm, + ) +except ImportError: + triton_cc_jagged_dense_bmm = None + + +torch.fx.wrap("triton_concat_2D_jagged") +torch.fx.wrap("triton_split_2D_jagged") +torch.fx.wrap("triton_concat_2D_jagged_multirow") +torch.fx.wrap("triton_split_2D_jagged_multirow") + + +def concat_2D_jagged( + max_seq_len: int, + values_left: torch.Tensor, + values_right: torch.Tensor, + max_len_left: Optional[int] = None, + max_len_right: Optional[int] = None, + offsets_left: Optional[torch.Tensor] = None, + offsets_right: Optional[torch.Tensor] = None, + kernel: HammerKernel = HammerKernel.PYTORCH, +) -> torch.Tensor: + if torch.jit.is_scripting(): + return pytorch_concat_2D_jagged( + values_left=values_left, + values_right=values_right, + max_len_left=max_len_left, + max_len_right=max_len_right, + offsets_left=offsets_left, + offsets_right=offsets_right, + ) + if not is_fx_tracing(): + torch._assert(values_left.dim() == 2, "values_left must be 2D") + torch._assert(values_right.dim() == 2, "values_right must be 2D") + torch._assert( + values_right.shape[1] == values_left.shape[1], + f"values_left shape[1] must be equal to values_right shape[1] {values_left.shape[1]} vs {values_right.shape[1]}", + ) + if kernel == HammerKernel.TRITON: + return triton_concat_2D_jagged( + max_seq_len=max_seq_len, + values_left=values_left, + values_right=values_right, + max_len_left=max_len_left, + max_len_right=max_len_right, + offsets_left=offsets_left, + offsets_right=offsets_right, + ) + elif kernel == HammerKernel.TRITON_INFERENCE: + aott_values_left = values_left + aott_values_right = values_right + if offsets_left is None: + assert max_len_left is not None + aott_values_left = values_left.reshape( + -1, + max_len_left, + values_left.shape[-1], + ) + if offsets_right is None: + assert max_len_right is not None + aott_values_right = values_right.reshape( + -1, + max_len_right, + values_right.shape[-1], + ) + return aot_triton_kernel_wrapper_concat_2D_jagged( + max_seq_len=max_seq_len, + values_a=aott_values_left, + values_b=aott_values_right, + offsets_a=offsets_left, + offsets_b=offsets_right, + ) + else: + return pytorch_concat_2D_jagged( + values_left=values_left, + values_right=values_right, + max_len_left=max_len_left, + max_len_right=max_len_right, + offsets_left=offsets_left, + offsets_right=offsets_right, + ) + + +def split_2D_jagged( + max_seq_len: int, + values: torch.Tensor, + total_len_left: Optional[int] = None, + total_len_right: Optional[int] = None, + max_len_left: Optional[int] = None, + max_len_right: Optional[int] = None, + offsets_left: Optional[torch.Tensor] = None, + offsets_right: Optional[torch.Tensor] = None, + kernel: HammerKernel = HammerKernel.PYTORCH, +) -> Tuple[torch.Tensor, torch.Tensor]: + if torch.jit.is_scripting(): + return pytorch_split_2D_jagged( + max_seq_len=max_seq_len, + values=values, + max_len_left=max_len_left, + max_len_right=max_len_right, + offsets_left=offsets_left, + offsets_right=offsets_right, + ) + if not is_fx_tracing(): + torch._assert(values.dim() == 2, "values must be 2D") + torch._assert( + offsets_left is not None or offsets_right is not None, + "offsets_left and offsets_right cannot be None at the same time", + ) + if offsets_left is None: + torch._assert( + max_len_left is not None, + "max_len_left must be provided when offsets_left is None", + ) + if offsets_right is None: + torch._assert( + max_len_right is not None, + "max_len_right must be provided when offsets_right is None", + ) + if offsets_left is not None and offsets_right is not None: + torch._assert( + offsets_left.shape[0] == offsets_right.shape[0], + "offsets_left shape[0] must be equal to offsets_right shape[0]", + ) + if kernel == HammerKernel.TRITON: + return triton_split_2D_jagged( + max_seq_len=max_seq_len, + values=values, + total_len_left=total_len_left, + total_len_right=total_len_right, + max_len_left=max_len_left, + max_len_right=max_len_right, + offsets_left=offsets_left, + offsets_right=offsets_right, + ) + elif kernel == HammerKernel.TRITON_INFERENCE: + dense_size = 0 + if offsets_left is None and max_len_left is not None: + dense_size = max_len_left + elif offsets_right is None and max_len_right is not None: + dense_size = max_len_right + split_left, split_right = aot_triton_kernel_wrapper_split_2D_jagged( + values=values, + max_seq_len=max_seq_len, + offsets_a=offsets_left, + offsets_b=offsets_right, + dense_size=dense_size, + ) + if offsets_left is None: + split_left = split_left.reshape(-1, split_left.shape[-1]) + if offsets_right is None: + split_right = split_right.reshape(-1, split_right.shape[-1]) + return split_left, split_right + else: + return pytorch_split_2D_jagged( + max_seq_len=max_seq_len, + values=values, + max_len_left=max_len_left, + max_len_right=max_len_right, + offsets_left=offsets_left, + offsets_right=offsets_right, + ) + + +def hstu_split_l2_embeddings( + max_seq_len: int, + x: torch.Tensor, + prefix_offsets: torch.Tensor, + l2_offsets: torch.Tensor, + contextual_seq_len: int, + kernel: HammerKernel = HammerKernel.PYTORCH, +) -> Tuple[torch.Tensor, torch.Tensor]: + if kernel == HammerKernel.TRITON: + return triton_split_2D_jagged( + max_seq_len=max_seq_len, + values=x, + total_len_right=None, + total_len_left=None, + max_len_left=None, + max_len_right=None, + offsets_left=prefix_offsets, + offsets_right=l2_offsets, + n_prefix_to_right=contextual_seq_len, + ) + else: + return pytorch_hstu_split_l2_embeddings( + max_seq_len=max_seq_len, + x=x, + prefix_offsets=prefix_offsets, + l2_offsets=l2_offsets, + contextual_seq_len=contextual_seq_len, + ) + + +def hstu_concat_l2_embeddings( + max_prefix_len: int, + prefix_x: torch.Tensor, + prefix_offsets: torch.Tensor, + max_l2_len: int, + l2_x: torch.Tensor, + l2_offsets: torch.Tensor, + contextual_seq_len: int, + kernel: HammerKernel = HammerKernel.PYTORCH, +) -> torch.Tensor: + if kernel == HammerKernel.TRITON: + return triton_concat_2D_jagged( + max_seq_len=max_prefix_len + max_l2_len, + values_left=prefix_x, + values_right=l2_x, + max_len_left=max_prefix_len, + max_len_right=max_l2_len, + offsets_left=prefix_offsets, + offsets_right=l2_offsets, + n_prefix_from_right=contextual_seq_len, + ) + else: + return pytorch_hstu_concat_l2_embeddings( + contextual_seq_len=contextual_seq_len, + max_prefix_len=max_prefix_len, + prefix_x=prefix_x, + prefix_offsets=prefix_offsets, + max_l2_len=max_l2_len, + l2_x=l2_x, + l2_offsets=l2_offsets, + ) + + +def jagged_dense_bmm_broadcast_add( + max_seq_len: int, + seq_offsets: torch.Tensor, + jagged: torch.Tensor, + dense: torch.Tensor, + bias: torch.Tensor, + kernel: HammerKernel = HammerKernel.PYTORCH, +) -> torch.Tensor: + """ + Computing out = jagged x dense + bias + jagged has shape (sum_B(M_i), K), dense has shape (B, K, N), and bias has shape (B, N) + out has shape (sum_B(M_i), N) + """ + if not is_fx_tracing(): + _, K = jagged.shape + B, _, N = dense.shape + torch._assert(dense.shape[1] == K, "wrong dense shape[1]") + torch._assert(seq_offsets.shape[0] == B + 1, "wrong seq_offsets shape[0]") + torch._assert(bias.shape[0] == B, "wrong bias shape[0]") + torch._assert(bias.shape[1] == N, "wrong bias shape[1]") + if kernel == HammerKernel.TRITON: + return triton_jagged_dense_bmm_add( + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + jagged=jagged, + dense=dense, + bias=bias, + elementwise=False, + ) + elif kernel == HammerKernel.TRITON_CC: + if triton_cc_jagged_dense_bmm is None: + raise ImportError( + "hammer is required for the TRITON_CC kernel in jagged_dense_bmm_broadcast_add." + ) + return triton_cc_jagged_dense_bmm( + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + jagged=jagged, + dense=dense, + bias=bias, + ) + else: + return pytorch_jagged_dense_bmm_add( + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + jagged=jagged, + dense=dense, + bias=bias, + ) + + +def concat_2D_jagged_multirow( + max_seq_len: int, + values_left: torch.Tensor, + values_right: torch.Tensor, + offsets_left: Optional[torch.Tensor], + offsets_right: Optional[torch.Tensor], + max_len_left: int, + max_len_right: int, + kernel: HammerKernel = HammerKernel.TRITON, +) -> torch.Tensor: + if not is_fx_tracing(): + torch._assert(values_left.dim() == 2, "values_left must be 2D") + torch._assert(values_right.dim() == 2, "values_right must be 2D") + torch._assert( + values_right.shape[1] == values_left.shape[1], + f"values_left shape[1] must be equal to values_right shape[1] {values_left.shape[1]} vs {values_right.shape[1]}", + ) + if offsets_left is not None and offsets_right is not None: + torch._assert( + offsets_left.shape[0] == offsets_right.shape[0], + "offsets_left and offsets_right must have the same batch dimension", + ) + + if kernel == HammerKernel.TRITON: + return triton_concat_2D_jagged_multirow( + max_seq_len=max_seq_len, + values_a=values_left, + values_b=values_right, + offsets_a=offsets_left, + offsets_b=offsets_right, + max_len_a=max_len_left, + max_len_b=max_len_right, + ) + else: + return concat_2D_jagged( + max_seq_len=max_seq_len, + values_left=values_left, + values_right=values_right, + max_len_left=max_len_left, + max_len_right=max_len_right, + offsets_left=offsets_left, + offsets_right=offsets_right, + kernel=kernel, + ) + + +def split_2D_jagged_multirow( + max_seq_len: int, + values: torch.Tensor, + total_len_left: Optional[int] = None, + total_len_right: Optional[int] = None, + max_len_left: Optional[int] = None, + max_len_right: Optional[int] = None, + offsets_left: Optional[torch.Tensor] = None, + offsets_right: Optional[torch.Tensor] = None, + kernel: HammerKernel = HammerKernel.TRITON, +) -> Tuple[torch.Tensor, torch.Tensor]: + if not is_fx_tracing(): + torch._assert(values.dim() == 2, "values must be 2D") + torch._assert( + offsets_left is not None or offsets_right is not None, + "offsets_left and offsets_right cannot be None at the same time", + ) + if offsets_left is None: + torch._assert( + max_len_left is not None, + "max_len_left must be provided when offsets_left is None", + ) + if offsets_right is None: + torch._assert( + max_len_right is not None, + "max_len_right must be provided when offsets_right is None", + ) + if offsets_left is not None and offsets_right is not None: + torch._assert( + offsets_left.shape[0] == offsets_right.shape[0], + "offsets_left and offsets_right must have the same batch dimension", + ) + + if kernel == HammerKernel.TRITON: + return triton_split_2D_jagged_multirow( + max_seq_len=max_seq_len, + values=values, + total_len_left=total_len_left, + total_len_right=total_len_right, + max_len_left=max_len_left, + max_len_right=max_len_right, + offsets_left=offsets_left, + offsets_right=offsets_right, + ) + else: + return split_2D_jagged( + max_seq_len=max_seq_len, + values=values, + total_len_left=total_len_left, + total_len_right=total_len_right, + max_len_left=max_len_left, + max_len_right=max_len_right, + offsets_left=offsets_left, + offsets_right=offsets_right, + kernel=kernel, + ) diff --git a/recommendation_v4/generative_recommenders/ops/layer_norm.py b/recommendation_v4/generative_recommenders/ops/layer_norm.py new file mode 100644 index 000000000..74ed377d6 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/layer_norm.py @@ -0,0 +1,330 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + + +from typing import List + +import torch +from generative_recommenders.ops.pytorch.pt_layer_norm import ( + pytorch_layer_norm, + pytorch_rms_norm, + pytorch_swish_layer_norm, +) +from generative_recommenders.ops.triton.triton_layer_norm import triton_rms_norm + +try: + from hammer.ops.triton.cc.rms_norm.triton_cc_rms_norm import triton_cc_rms_norm + from hammer.ops.triton.cc.swish_layer_norm.triton_cc_swish_layer_norm import ( + triton_cc_swish_layer_norm, + ) +except ImportError: + triton_cc_swish_layer_norm = None + triton_cc_rms_norm = None +from generative_recommenders.common import HammerKernel, HammerModule +from generative_recommenders.ops.triton.triton_layer_norm import ( + triton_layer_norm, + triton_swish_layer_norm, +) +from torch.fx._symbolic_trace import is_fx_tracing + +try: + # @manual=//generative_recommenders/ops/triton_aot:triton_layer_norm + from generative_recommenders.ops.triton_aot.triton_layer_norm import ( # pyre-ignore[21] + aot_triton_kernel_wrapper_swish_layer_norm, + ) + + # @manual=//generative_recommenders/ops/triton_aot:triton_rms_norm + from generative_recommenders.ops.triton_aot.triton_rms_norm import ( # pyre-ignore[21] + aot_triton_kernel_wrapper_rms_norm, + ) +except ImportError: + + def aot_triton_kernel_wrapper_swish_layer_norm( + x: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + eps: float, + is_swish: bool, + ) -> torch.Tensor: + raise ImportError( + "AOT-T is required for the TRITON_INFERENCE swish_layer_norm kernel." + ) + + def aot_triton_kernel_wrapper_rms_norm( + x: torch.Tensor, + weight: torch.Tensor, + eps: float, + silu: bool, + ) -> torch.Tensor: + raise ImportError("AOT-T is required for the TRITON_INFERENCE rms_norm kernel.") + + +torch.fx.wrap("triton_layer_norm") +torch.fx.wrap("triton_swish_layer_norm") +torch.fx.wrap("triton_rms_norm") + + +def layer_norm( + x: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + eps: float = 1e-5, + kernel: HammerKernel = HammerKernel.PYTORCH, +) -> torch.Tensor: + if torch.jit.is_scripting(): + # Script-mode fast path: bypass the HammerKernel ladder (which would + # drag in is_fx_tracing()'s closed-over global bool). + return torch.nn.functional.layer_norm( + x, + normalized_shape=(x.shape[-1],), + weight=weight, + bias=bias, + eps=eps, + ) + if kernel == HammerKernel.TRITON: + if not is_fx_tracing(): + torch._assert(not x.is_cpu, "x must be device tensor") + torch._assert(not weight.is_cpu, "weight must be device tensor") + torch._assert(not bias.is_cpu, "bias must be device tensor") + return triton_layer_norm(x, weight, bias, eps) + elif kernel == HammerKernel.TRITON_INFERENCE: + return aot_triton_kernel_wrapper_swish_layer_norm( + x, + weight, + bias, + eps, + is_swish=False, + ) + elif kernel == HammerKernel.TRITON_CC: + if triton_cc_swish_layer_norm is None: + raise ImportError( + "hammer is required for the TRITON_CC kernel in layer_norm." + ) + return triton_cc_swish_layer_norm( + x, + weight, + bias, + eps, + is_swish=False, + ) + else: + return pytorch_layer_norm( + x, + [ + x.shape[-1], + ], + weight, + bias, + eps, + ) + + +def rms_norm( + x: torch.Tensor, + weight: torch.Tensor, + eps: float = 1e-5, + kernel: HammerKernel = HammerKernel.PYTORCH, + silu: bool = False, +) -> torch.Tensor: + if torch.jit.is_scripting(): + # Script-mode fast path: bypass the HammerKernel ladder. + x_f = x.float() + norm = torch.rsqrt(x_f.pow(2).mean(-1, keepdim=True) + eps) + out = (x_f * norm * weight.float()).to(x.dtype) + if silu: + out = torch.nn.functional.silu(out) + return out + if kernel == HammerKernel.TRITON: + if not is_fx_tracing(): + torch._assert(not x.is_cpu, "x must be device tensor") + torch._assert(not weight.is_cpu, "weight must be device tensor") + return triton_rms_norm(x, weight, eps, silu) + elif kernel == HammerKernel.TRITON_INFERENCE: + return aot_triton_kernel_wrapper_rms_norm(x, weight, eps, silu) + elif kernel == HammerKernel.TRITON_CC: + if triton_cc_rms_norm is None: + raise ImportError( + "hammer is required for the TRITON_CC kernel in rms_norm." + ) + return triton_cc_rms_norm( + x, + weight, + eps, + silu=silu, + ) + else: + return pytorch_rms_norm( + x, + [ + x.shape[-1], + ], + weight, + eps, + silu, + ) + + +def swish_layer_norm( + x: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + eps: float = 1e-5, + kernel: HammerKernel = HammerKernel.PYTORCH, +) -> torch.Tensor: + if torch.jit.is_scripting(): + # Script-mode fast path: bypass the HammerKernel ladder (which + # otherwise drags in is_fx_tracing(), Triton/Triton_CC closures, + # etc.) and call pure PyTorch directly. + return pytorch_swish_layer_norm( + x, + [x.shape[-1]], + weight, + bias, + eps, + ) + if kernel == HammerKernel.TRITON: + if not is_fx_tracing(): + torch._assert(not x.is_cpu, "x must be device tensor") + torch._assert(not weight.is_cpu, "weight must be device tensor") + torch._assert(not bias.is_cpu, "bias must be device tensor") + return triton_swish_layer_norm(x, [x.shape[-1]], weight, bias, eps) + elif kernel == HammerKernel.TRITON_INFERENCE: + return aot_triton_kernel_wrapper_swish_layer_norm( + x, + weight, + bias, + eps, + is_swish=True, + ) + elif kernel == HammerKernel.TRITON_CC: + if triton_cc_swish_layer_norm is None: + raise ImportError( + "hammer is required for the TRITON_CC kernel in swish_layer_norm." + ) + return triton_cc_swish_layer_norm( + x, + weight, + bias, + eps, + is_swish=True, + ) + else: + return pytorch_swish_layer_norm( + x, + [ + x.shape[-1], + ], + weight, + bias, + eps, + ) + + +class LayerNorm(HammerModule): + def __init__( + self, + dim: int, + eps: float = 1e-5, + is_inference: bool = False, + ) -> None: + super().__init__(is_inference=is_inference) + self._normalized_shape: List[int] = [dim] + self._eps = eps + self.weight = torch.nn.Parameter( + torch.ones(self._normalized_shape), + ) + self.bias = torch.nn.Parameter( + torch.zeros(self._normalized_shape), + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return layer_norm( + x=x, + weight=self.weight, + bias=self.bias, + eps=self._eps, + kernel=self.hammer_kernel(), + ) + + +class RMSNorm(HammerModule): + def __init__( + self, + dim: int, + eps: float = 1e-5, + is_inference: bool = False, + ) -> None: + super().__init__(is_inference=is_inference) + self._eps = eps + self.weight = torch.nn.Parameter(torch.ones(dim)) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return rms_norm( + x, + self.weight, + self._eps, + silu=False, + kernel=self.hammer_kernel(), + ) + + +class RMSNormSilu(HammerModule): + def __init__( + self, + dim: int, + eps: float = 1e-5, + is_inference: bool = False, + ) -> None: + super().__init__(is_inference=is_inference) + self._eps = eps + self.weight = torch.nn.Parameter(torch.ones(dim)) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return rms_norm( + x, + self.weight, + self._eps, + silu=True, + kernel=self.hammer_kernel(), + ) + + +class SwishLayerNorm(HammerModule): + def __init__( + self, + dim: int, + eps: float = 1e-5, + is_inference: bool = False, + ) -> None: + super().__init__(is_inference=is_inference) + self._normalized_shape: List[int] = [dim] + self.weight = torch.nn.Parameter(torch.ones(self._normalized_shape)) + self.bias = torch.nn.Parameter(torch.zeros(self._normalized_shape)) + self._eps = eps + + def forward( + self, + x: torch.Tensor, + ) -> torch.Tensor: + return swish_layer_norm( + x=x, + weight=self.weight, + bias=self.bias, + eps=self._eps, + kernel=self.hammer_kernel(), + ) diff --git a/recommendation_v4/generative_recommenders/ops/mm.py b/recommendation_v4/generative_recommenders/ops/mm.py new file mode 100644 index 000000000..31a5c5d36 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/mm.py @@ -0,0 +1,60 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +import torch + +try: + from hammer.ops.triton.cc.addmm.triton_cc_addmm import triton_cc_addmm +except ImportError: + triton_cc_addmm = None +from generative_recommenders.common import HammerKernel +from generative_recommenders.ops.triton.triton_addmm import triton_addmm + +try: + # @manual=//generative_recommenders/ops/triton_aot:triton_addmm + from generative_recommenders.ops.triton_aot.triton_addmm import ( # pyre-ignore[21] + aot_triton_kernel_wrapper_addmm, + ) +except ImportError: + + def aot_triton_kernel_wrapper_addmm( + input: torch.Tensor, + mat1: torch.Tensor, + mat2: torch.Tensor, + ) -> torch.Tensor: + raise ImportError("AOT-T is required for the TRITON_INFERENCE addmm kernel.") + + +def addmm( + input: torch.Tensor, + mat1: torch.Tensor, + mat2: torch.Tensor, + kernel: HammerKernel = HammerKernel.PYTORCH, +) -> torch.Tensor: + if torch.jit.is_scripting(): + return torch.addmm(input, mat1, mat2) + if kernel == HammerKernel.TRITON: + return triton_addmm(input, mat1, mat2) + elif kernel == HammerKernel.TRITON_INFERENCE: + return aot_triton_kernel_wrapper_addmm(input, mat1, mat2) + elif kernel == HammerKernel.TRITON_CC: + if triton_cc_addmm is None: + raise ImportError("hammer is required for the TRITON_CC kernel in addmm.") + return triton_cc_addmm(input, mat1, mat2) + else: + return torch.addmm(input, mat1, mat2) diff --git a/recommendation_v4/generative_recommenders/ops/position.py b/recommendation_v4/generative_recommenders/ops/position.py new file mode 100644 index 000000000..e090827e3 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/position.py @@ -0,0 +1,147 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +from typing import Optional + +import torch +from generative_recommenders.ops.pytorch.pt_position import ( + pytorch_add_timestamp_positional_embeddings, +) + +try: + from hammer.ops.triton.cc.add_timestamp_position_embeddings.triton_cc_add_timestamp_position_embeddings import ( + triton_cc_add_timestamp_position_embeddings, + ) +except ImportError: + triton_cc_add_timestamp_position_embeddings = None +from generative_recommenders.common import HammerKernel +from generative_recommenders.ops.triton.triton_position import ( + triton_add_timestamp_positional_embeddings, +) + +try: + # @manual=//generative_recommenders/ops/triton_aot:triton_position + from generative_recommenders.ops.triton_aot.triton_position import ( # pyre-ignore[21] + aot_triton_kernel_wrapper_position, + ) +except ImportError: + + def aot_triton_kernel_wrapper_position( + *args: object, + **kwargs: object, + ) -> torch.Tensor: + raise ImportError("AOT-T is required for the TRITON_INFERENCE position kernel.") + + +torch.fx.wrap("triton_add_timestamp_positional_embeddings") + + +def add_timestamp_positional_embeddings( + alpha: float, + max_seq_len: int, + max_contextual_seq_len: int, + position_embeddings_weight: torch.Tensor, + timestamp_embeddings_weight: torch.Tensor, + seq_offsets: torch.Tensor, + seq_lengths: torch.Tensor, + seq_embeddings: torch.Tensor, + timestamps: torch.Tensor, + num_targets: Optional[torch.Tensor], + interleave_targets: bool, + time_bucket_fn: str = "sqrt", + kernel: HammerKernel = HammerKernel.PYTORCH, +) -> torch.Tensor: + if torch.jit.is_scripting(): + # Script-mode fast path: bypass the HammerKernel ladder. + seq_embeddings = seq_embeddings * alpha + return pytorch_add_timestamp_positional_embeddings( + seq_embeddings=seq_embeddings, + seq_offsets=seq_offsets, + pos_embeddings=position_embeddings_weight, + ts_embeddings=timestamp_embeddings_weight, + timestamps=timestamps, + max_seq_len=max_seq_len, + max_contextual_seq_len=max_contextual_seq_len, + seq_lengths=seq_lengths, + num_targets=num_targets, + interleave_targets=interleave_targets, + time_bucket_fn=time_bucket_fn, + ) + assert time_bucket_fn in ["sqrt", "log"] + seq_embeddings = seq_embeddings * alpha + if kernel == HammerKernel.TRITON: + return triton_add_timestamp_positional_embeddings( + seq_embeddings=seq_embeddings, + seq_offsets=seq_offsets, + pos_embeddings=position_embeddings_weight, + ts_embeddings=timestamp_embeddings_weight, + timestamps=timestamps, + max_seq_len=max_seq_len, + max_contextual_seq_len=max_contextual_seq_len, + seq_lengths=seq_lengths, + num_targets=num_targets, + interleave_targets=interleave_targets, + time_bucket_fn=time_bucket_fn, + ) + elif kernel == HammerKernel.TRITON_INFERENCE: + return aot_triton_kernel_wrapper_position( + alpha=1.0, + max_seq_len=max_seq_len, + max_contextual_seq_len=max_contextual_seq_len, + position_embeddings_weight=position_embeddings_weight.to(torch.float32), + timestamp_embeddings_weight=timestamp_embeddings_weight.to(torch.float32), + seq_offsets=seq_offsets, + seq_lengths=seq_lengths, + seq_embeddings=seq_embeddings, + timestamps=timestamps, + num_targets=num_targets, + interleave_targets=interleave_targets, + time_bucket_fn=time_bucket_fn, + ) + elif kernel == HammerKernel.TRITON_CC: + if triton_cc_add_timestamp_position_embeddings is None: + raise ImportError( + "hammer is required for the TRITON_CC kernel in add_timestamp_positional_embeddings." + ) + return triton_cc_add_timestamp_position_embeddings( + seq_embeddings=seq_embeddings, + seq_offsets=seq_offsets, + pos_embeddings=position_embeddings_weight, + ts_embeddings=timestamp_embeddings_weight, + timestamps=timestamps, + max_seq_len=max_seq_len, + max_contextual_seq_len=max_contextual_seq_len, + seq_lengths=seq_lengths, + num_targets=num_targets, + interleave_targets=interleave_targets, + time_bucket_fn=time_bucket_fn, + ) + else: + return pytorch_add_timestamp_positional_embeddings( + seq_embeddings=seq_embeddings, + seq_offsets=seq_offsets, + pos_embeddings=position_embeddings_weight, + ts_embeddings=timestamp_embeddings_weight, + timestamps=timestamps, + max_seq_len=max_seq_len, + max_contextual_seq_len=max_contextual_seq_len, + seq_lengths=seq_lengths, + num_targets=num_targets, + interleave_targets=interleave_targets, + time_bucket_fn=time_bucket_fn, + ) diff --git a/recommendation_v4/generative_recommenders/ops/pytorch/pt_hstu_attention.py b/recommendation_v4/generative_recommenders/ops/pytorch/pt_hstu_attention.py new file mode 100644 index 000000000..32575c4db --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/pytorch/pt_hstu_attention.py @@ -0,0 +1,251 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +from typing import Optional, Tuple + +import torch +import torch.nn.functional as F + + +try: + torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops") + torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops_cpu") +except OSError: + pass + + +@torch.fx.wrap +def _get_valid_attn_mask( + device: torch.device, + causal: bool, + N: int, + seq_lengths: torch.Tensor, + num_targets: Optional[torch.Tensor] = None, + max_attn_len: int = 0, + contextual_seq_len: int = 0, + min_full_attn_seq_len: int = 0, +) -> torch.Tensor: + ids = torch.arange(0, N, device=device).view(1, N) + max_ids = seq_lengths.view(-1, 1, 1) + if contextual_seq_len > 0: + ids = ids - contextual_seq_len + 1 + ids = torch.clamp(ids, min=0) + max_ids = max_ids - contextual_seq_len + 1 + if num_targets is not None: + max_ids = max_ids - num_targets.view(-1, 1, 1) + ids = torch.clamp( + ids, + max=max_ids, + ) + row_ids = ids.view(-1, N, 1).expand(-1, N, N) + col_ids = ids.view(-1, 1, N).expand(-1, N, N) + else: + row_ids = ids.view(N, 1).expand(N, N) + col_ids = row_ids.t() + row_ids = row_ids.view(1, N, N) + col_ids = col_ids.view(1, N, N) + row_col_dist = row_ids - col_ids + valid_attn_mask = torch.eye(N, device=device, dtype=torch.bool).view(1, N, N) + if not causal: + row_col_dist = torch.where(row_col_dist > 0, row_col_dist, -row_col_dist) + valid_attn_mask = torch.logical_or(valid_attn_mask, row_col_dist > 0) + if max_attn_len > 0: + if min_full_attn_seq_len > 0: + valid_attn_mask = torch.logical_and( + valid_attn_mask, + torch.logical_or( + row_col_dist <= max_attn_len, + row_ids >= max_ids - min_full_attn_seq_len, + ), + ) + else: + valid_attn_mask = torch.logical_and( + valid_attn_mask, row_col_dist <= max_attn_len + ) + if contextual_seq_len > 0: + valid_attn_mask = torch.logical_or( + valid_attn_mask, torch.logical_and(row_ids == 0, col_ids < max_ids) + ) + return valid_attn_mask + + +def _pad_qkv( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + seq_offsets: torch.Tensor, + N: int, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + L, H, D = q.shape + V = v.shape[2] + padded_q = ( + torch.ops.fbgemm.jagged_to_padded_dense( + values=q.reshape(L, H * D), + offsets=[seq_offsets], + max_lengths=[N], + padding_value=0.0, + ) + .view(-1, N, H, D) + .transpose(1, 2) + ) # [B, H, N, A] + padded_k = ( + torch.ops.fbgemm.jagged_to_padded_dense( + values=k.reshape(L, H * D), + offsets=[seq_offsets], + max_lengths=[N], + padding_value=0.0, + ) + .view(-1, N, H, D) + .transpose(1, 2) + ) # [B, H, N, A] + padded_v = ( + torch.ops.fbgemm.jagged_to_padded_dense( + values=v.reshape(L, H * V), + offsets=[seq_offsets], + max_lengths=[N], + padding_value=0.0, + ) + .view(-1, N, H, V) + .transpose(1, 2) + ) # [B, H, N, D] + return padded_q, padded_k, padded_v + + +@torch.fx.wrap +def pytorch_hstu_mha( + max_seq_len: int, + alpha: float, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + seq_offsets: torch.Tensor, + causal: bool = True, + dropout_pr: float = 0.0, + training: bool = True, + num_targets: Optional[torch.Tensor] = None, + attn_scale: Optional[torch.Tensor] = None, + max_attn_len: int = 0, + contextual_seq_len: int = 0, + min_full_attn_seq_len: int = 0, +) -> torch.Tensor: + L, H, _ = q.shape + V = v.shape[2] + q, k, v = _pad_qkv( + q, k, v, seq_offsets, max_seq_len + ) # [B, H, N, D) and [B, H, N, V] + qk_attn = torch.einsum("bhxa,bhya->bhxy", q, k) * alpha + if attn_scale is not None: + if attn_scale.ndim > 0: + attn_scale = ( + torch.ops.fbgemm.jagged_to_padded_dense( + values=attn_scale.unsqueeze(-1), + offsets=[seq_offsets], + max_lengths=[max_seq_len], + padding_value=0.0, + ) + .unsqueeze(1) + .to(qk_attn.dtype) + ) + + qk_attn = F.silu(qk_attn) * attn_scale + else: + qk_attn = F.silu(qk_attn) / max_seq_len + valid_attn_mask = _get_valid_attn_mask( + device=q.device, + causal=causal, + N=max_seq_len, + seq_lengths=seq_offsets[1:] - seq_offsets[:-1], + num_targets=num_targets, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + min_full_attn_seq_len=min_full_attn_seq_len, + ) + # raise NotImplementedError(valid_attn_mask[0, :, :].to(torch.int32)) + qk_attn = qk_attn * valid_attn_mask.unsqueeze(1) + if dropout_pr > 0.0: + qk_attn = F.dropout(qk_attn, p=dropout_pr, training=training) + attn_dense = torch.einsum("bhxd,bhdv->bhxv", qk_attn, v) # [B, H, N, V] + return torch.ops.fbgemm.dense_to_jagged( + attn_dense.transpose(1, 2).flatten(2, 3), # [B, N, H, V]->[B, N, H * V] + [seq_offsets], + L, + )[0].view(L, H, V) + + +@torch.fx.wrap +def pytorch_cached_hstu_mha( + max_seq_len: int, + alpha: float, + delta_q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + seq_offsets: torch.Tensor, + num_targets: Optional[torch.Tensor] = None, + max_attn_len: int = 0, + contextual_seq_len: int = 0, +) -> torch.Tensor: + L, H, D = delta_q.shape + _, _, V = v.shape + B = seq_offsets.size(0) - 1 + delta_size = L // B + delta_q = delta_q.view(B, -1, H, D).transpose(1, 2) + full_k = ( + torch.ops.fbgemm.jagged_to_padded_dense( + values=k.reshape(-1, H * D), + offsets=[seq_offsets], + max_lengths=[max_seq_len], + padding_value=0.0, + ) + .view(B, -1, H, D) + .transpose(1, 2) + ) + full_v = ( + torch.ops.fbgemm.jagged_to_padded_dense( + values=v.reshape(-1, H * V), + offsets=[seq_offsets], + max_lengths=[max_seq_len], + padding_value=0.0, + ) + .view(B, -1, H, V) + .transpose(1, 2) + ) + qk_attn = torch.einsum("bhxa,bhya->bhxy", delta_q, full_k) * alpha + qk_attn = F.silu(qk_attn) / max_seq_len + full_valid_attn_mask = _get_valid_attn_mask( + device=delta_q.device, + causal=True, + N=max_seq_len, + seq_lengths=seq_offsets[1:] - seq_offsets[:-1], + num_targets=num_targets, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + ) + seq_lengths = seq_offsets[1:] - seq_offsets[:-1] + mask = torch.arange(max_seq_len, device=delta_q.device).view(1, -1) + mask = torch.logical_and( + mask >= (seq_lengths - delta_size).view(-1, 1), + mask < seq_lengths.view(-1, 1), + ) + valid_attn_mask = ( + full_valid_attn_mask.expand(B, -1, -1) + .flatten(0, 1)[mask.view(-1), :] + .view(-1, delta_size, max_seq_len) + ) + qk_attn = qk_attn * valid_attn_mask.unsqueeze(1) + attn_output = torch.einsum("bhxd,bhdv->bhxv", qk_attn, full_v) + return attn_output.transpose(1, 2).reshape(-1, H, V) diff --git a/recommendation_v4/generative_recommenders/ops/pytorch/pt_hstu_linear.py b/recommendation_v4/generative_recommenders/ops/pytorch/pt_hstu_linear.py new file mode 100644 index 000000000..6ea94a565 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/pytorch/pt_hstu_linear.py @@ -0,0 +1,130 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +import torch +import torch.nn.functional as F + + +def pytorch_norm_mul_dropout( + x: torch.Tensor, + u: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + eps: float, + dropout_ratio: float, + training: bool, + silu_u: bool = False, + concat_u: bool = False, + concat_x: bool = False, + mul_u_activation_type: str = "none", + group_norm: bool = False, + num_heads: int = 1, + linear_dim: int = -1, +) -> torch.Tensor: + dtype = x.dtype + x = x.to(torch.float32) + u = u.to(torch.float32) + if group_norm: + if silu_u: + u = F.silu(u) + u = u.to(torch.float32) + y = u * F.group_norm( + x.view(-1, num_heads, linear_dim), + num_groups=num_heads, + weight=weight.to(torch.float32), + bias=bias.to(torch.float32), + eps=eps, + ).view(-1, num_heads * linear_dim) + if concat_u and concat_x: + y = torch.cat([u, x, y], dim=1) + else: + mul_u = u + if mul_u_activation_type == "sigmoid": + mul_u = torch.sigmoid(u) + elif mul_u_activation_type == "silu": + mul_u = F.silu(u) + y = mul_u * F.layer_norm( + x, + normalized_shape=(x.shape[-1],), + weight=weight.to(torch.float32), + bias=bias.to(torch.float32), + eps=eps, + ) + if concat_u: + if silu_u: + u = F.silu(u) + if concat_x: + y = torch.cat([u, x, y], dim=1) + else: + y = torch.cat([u, y], dim=1) + elif concat_x: + y = torch.cat([x, y], dim=1) + y = F.dropout( + y, + p=dropout_ratio, + training=training, + ) + return y.to(dtype) + + +def pytorch_hstu_compute_output( + attn: torch.Tensor, + u: torch.Tensor, + x: torch.Tensor, + norm_weight: torch.Tensor, + norm_bias: torch.Tensor, + output_weight: torch.Tensor, + eps: float, + dropout_ratio: float, + training: bool, + silu_u: bool = False, + concat_u: bool = False, + concat_x: bool = False, + mul_u_activation_type: str = "none", + group_norm: bool = False, + num_heads: int = 1, + linear_dim: int = -1, +) -> torch.Tensor: + dtype = x.dtype + y = pytorch_norm_mul_dropout( + x=attn, + u=u, + weight=norm_weight, + bias=norm_bias, + eps=eps, + dropout_ratio=dropout_ratio, + training=training, + silu_u=silu_u, + concat_u=concat_u, + concat_x=concat_x, + mul_u_activation_type=mul_u_activation_type, + group_norm=group_norm, + num_heads=num_heads, + linear_dim=linear_dim, + ) + return torch.addmm(x, y, output_weight.to(x.dtype)).to(dtype) + + +def pytorch_swiglu( + x: torch.Tensor, + w_gate: torch.Tensor, + w_up: torch.Tensor, +) -> torch.Tensor: + gate = F.silu(F.linear(x, w_gate)) + up = F.linear(x, w_up) + return gate * up diff --git a/recommendation_v4/generative_recommenders/ops/pytorch/pt_jagged.py b/recommendation_v4/generative_recommenders/ops/pytorch/pt_jagged.py new file mode 100644 index 000000000..82d82f402 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/pytorch/pt_jagged.py @@ -0,0 +1,258 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +from typing import Tuple + +import torch + + +try: + torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops") + torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops_cpu") +except OSError: + pass + + +def pytorch_jagged_dense_bmm( + max_seq_len: int, + seq_offsets: torch.Tensor, + jagged: torch.Tensor, + dense: torch.Tensor, +) -> torch.Tensor: + dtype = jagged.dtype + jagged = jagged.to(torch.float32) + dense = dense.to(torch.float32) + padded_jagged = torch.ops.fbgemm.jagged_to_padded_dense( + values=jagged, + offsets=[seq_offsets], + max_lengths=[max_seq_len], + padding_value=0.0, + ) + bmm_out = torch.bmm(padded_jagged, dense) + jagged_bmm_out = torch.ops.fbgemm.dense_to_jagged( + bmm_out, [seq_offsets], total_L=jagged.shape[0] + )[0] + jagged_bmm_out = jagged_bmm_out.to(dtype) + return jagged_bmm_out + + +def pytorch_jagged_dense_broadcast_add( + max_seq_len: int, + seq_offsets: torch.Tensor, + jagged: torch.Tensor, + dense: torch.Tensor, +) -> torch.Tensor: + dtype = jagged.dtype + jagged = jagged.to(torch.float32) + dense = dense.to(torch.float32) + padded_jagged = torch.ops.fbgemm.jagged_to_padded_dense( + values=jagged, + offsets=[seq_offsets], + max_lengths=[max_seq_len], + padding_value=0.0, + ) + out = padded_jagged + dense.unsqueeze(1) + jagged_out = torch.ops.fbgemm.dense_to_jagged( + out, [seq_offsets], total_L=jagged.shape[0] + )[0] + jagged_out = jagged_out.to(dtype) + return jagged_out + + +def pytorch_jagged_dense_bmm_add( + max_seq_len: int, + seq_offsets: torch.Tensor, + jagged: torch.Tensor, + dense: torch.Tensor, + bias: torch.Tensor, + elementwise: bool = False, +) -> torch.Tensor: + dtype = jagged.dtype + jagged = jagged.to(torch.float32) + dense = dense.to(torch.float32) + padded_jagged = torch.ops.fbgemm.jagged_to_padded_dense( + values=jagged, + offsets=[seq_offsets], + max_lengths=[max_seq_len], + padding_value=0.0, + ) + bmm_out = torch.bmm(padded_jagged, dense) + + if elementwise: + jagged_out = ( + torch.ops.fbgemm.dense_to_jagged( + bmm_out, [seq_offsets], total_L=jagged.shape[0] + )[0] + + bias + ) + else: + jagged_out = torch.ops.fbgemm.dense_to_jagged( + bmm_out + bias.unsqueeze(1), [seq_offsets], total_L=jagged.shape[0] + )[0] + + jagged_out = jagged_out.to(dtype) + return jagged_out + + +@torch.fx.wrap +def _arange(len: int, device: torch.device) -> torch.Tensor: + return torch.arange(len, device=device) + + +def pytorch_concat_2D_dense_jagged( + jagged_max_seq_len: int, + jagged_offsets: torch.Tensor, + jagged_values: torch.Tensor, + dense_values: torch.Tensor, +) -> torch.Tensor: + B, dense_size, D = dense_values.size() + jagged_dense = torch.ops.fbgemm.jagged_to_padded_dense( + values=jagged_values, + offsets=[jagged_offsets], + max_lengths=[jagged_max_seq_len], + padding_value=0.0, + ) + concatted_dense = torch.cat([dense_values, jagged_dense], dim=1) + concatted_offsets = ( + dense_size * _arange(B + 1, device=jagged_offsets.device) + jagged_offsets + ) + return torch.ops.fbgemm.dense_to_jagged( + concatted_dense, + [concatted_offsets], + total_L=jagged_values.shape[0] + dense_size * B, + )[0] + + +def pytorch_concat_2D_jagged_jagged( + max_seq_len_left: int, + offsets_left: torch.Tensor, + values_left: torch.Tensor, + max_seq_len_right: int, + offsets_right: torch.Tensor, + values_right: torch.Tensor, + is_replace: bool = False, + n_prefix_from_right: int = 0, +) -> torch.Tensor: + # is_replace with n_prefix_from_right != 0 is not supported yet (neither in triton) + if is_replace: + return pytorch_replace_last_n_with_jagged( + max_seq_len_left, + offsets_left, + values_left, + offsets_right, + values_right, + ) + + lengths_a = offsets_left[1:] - offsets_left[:-1] + lengths_b = offsets_right[1:] - offsets_right[:-1] + + # Compute output offsets via cumsum (no dynamic shapes). + output_lengths = lengths_a + lengths_b + output_offsets = torch.nn.functional.pad( + torch.cumsum(output_lengths, dim=0), (1, 0) + ) + + total_len = values_left.shape[0] + values_right.shape[0] + positions = torch.arange(total_len, device=values_left.device) + batch_idx = torch.searchsorted(output_offsets[1:], positions, right=True) + local_pos = positions - output_offsets[batch_idx] + + per_batch_lengths_a = lengths_a[batch_idx] + + # Classify each output position into prefix / left / suffix. + is_prefix = local_pos < n_prefix_from_right + is_left = (local_pos >= n_prefix_from_right) & ( + local_pos < n_prefix_from_right + per_batch_lengths_a + ) + + # Pad with a sentinel zero row so index_select works on empty tensors + values_left_safe = torch.nn.functional.pad(values_left, (0, 0, 0, 1)) + values_right_safe = torch.nn.functional.pad(values_right, (0, 0, 0, 1)) + + left_idx = (offsets_left[batch_idx] + (local_pos - n_prefix_from_right)).clamp( + min=0, max=values_left.shape[0] + ) + right_prefix_idx = offsets_right[batch_idx] + local_pos + right_suffix_idx = offsets_right[batch_idx] + (local_pos - per_batch_lengths_a) + right_idx = torch.where(is_prefix, right_prefix_idx, right_suffix_idx).clamp( + min=0, max=values_right.shape[0] + ) + + left_values = values_left_safe.index_select(0, left_idx) + right_values = values_right_safe.index_select(0, right_idx) + + return torch.where(is_left.unsqueeze(-1), left_values, right_values) + + +def pytorch_jagged_remove_first_or_last_1D( + values: torch.Tensor, + lengths: torch.Tensor, + offsets: torch.Tensor, + max_seq_len: int, +) -> Tuple[torch.Tensor, torch.Tensor]: + values = values.view(-1, 1) + shrunk_lengths = lengths - 1 + k_lengths = torch.stack([shrunk_lengths, torch.ones_like(lengths)], dim=1).view(-1) + q_lengths = torch.stack([torch.ones_like(lengths), shrunk_lengths], dim=1).view(-1) + all_indices = torch.arange( + start=0, end=q_lengths.numel(), device=values.device + ).reshape(-1, 2) + q_indices, k_indices = all_indices[:, 1], all_indices[:, 0] + values_no_first, _ = torch.ops.fbgemm.jagged_index_select( + values, q_lengths, q_indices + ) + values_no_last, _ = torch.ops.fbgemm.jagged_index_select( + values, k_lengths, k_indices + ) + return values_no_first.squeeze(), values_no_last.squeeze() + + +@torch.fx.wrap +def fx_apply_mask( + tensor: torch.Tensor, mask: torch.Tensor, fill_value: torch.Tensor +) -> torch.Tensor: + tensor[mask] = fill_value + return tensor + + +def pytorch_replace_last_n_with_jagged( + max_seq_len_left: int, + offsets_left: torch.Tensor, + values_left: torch.Tensor, + offsets_right: torch.Tensor, + values_right: torch.Tensor, +) -> torch.Tensor: + lengths_a = offsets_left[1:] - offsets_left[:-1] + lengths_b = offsets_right[1:] - offsets_right[:-1] + + total_len = values_left.shape[0] + positions = torch.arange(total_len, device=values_left.device) + batch_idx = torch.searchsorted(offsets_left[1:], positions, right=True) + local_pos = positions - offsets_left[batch_idx] + + # Positions >= (lengths_a - lengths_b) within each batch are in the replace zone. + threshold = lengths_a[batch_idx] - lengths_b[batch_idx] + in_replace_zone = local_pos >= threshold + + # Pad with a sentinel zero row so index_select works on empty tensors + values_right_safe = torch.nn.functional.pad(values_right, (0, 0, 0, 1)) + right_idx = (offsets_right[batch_idx] + (local_pos - threshold)).clamp( + min=0, max=values_right.shape[0] + ) + right_values = values_right_safe.index_select(0, right_idx) + return torch.where(in_replace_zone.unsqueeze(-1), right_values, values_left) diff --git a/recommendation_v4/generative_recommenders/ops/pytorch/pt_jagged_tensors.py b/recommendation_v4/generative_recommenders/ops/pytorch/pt_jagged_tensors.py new file mode 100644 index 000000000..27817f7fb --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/pytorch/pt_jagged_tensors.py @@ -0,0 +1,246 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +from typing import Optional, Tuple + +import torch +from generative_recommenders.common import fx_arange + +try: + torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops") + torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops_cpu") +except OSError: + pass + + +def _concat_2D_jagged_jagged( + values_left: torch.Tensor, + values_right: torch.Tensor, + max_len_left: int, + max_len_right: int, + offsets_left: torch.Tensor, + offsets_right: torch.Tensor, +) -> torch.Tensor: + max_seq_len = max_len_left + max_len_right + lengths_left = offsets_left[1:] - offsets_left[:-1] + lengths_right = offsets_right[1:] - offsets_right[:-1] + padded_left = torch.ops.fbgemm.jagged_to_padded_dense( + values=values_left, + offsets=[offsets_left], + max_lengths=[max_len_left], + padding_value=0.0, + ) + padded_right = torch.ops.fbgemm.jagged_to_padded_dense( + values=values_right, + offsets=[offsets_right], + max_lengths=[max_len_right], + padding_value=0.0, + ) + concatted_dense = torch.cat([padded_left, padded_right], dim=1) + mask = fx_arange(max_seq_len, device=offsets_left.device).view(1, -1) + mask = torch.logical_or( + mask < lengths_left.view(-1, 1), + torch.logical_and( + mask >= max_len_left, + mask < max_len_left + lengths_right.view(-1, 1), + ), + ) + return concatted_dense.flatten(0, 1)[mask.view(-1), :] + + +@torch.fx.wrap +def pytorch_concat_2D_jagged( + values_left: torch.Tensor, + values_right: torch.Tensor, + max_len_left: Optional[int], + max_len_right: Optional[int], + offsets_left: Optional[torch.Tensor], + offsets_right: Optional[torch.Tensor], +) -> torch.Tensor: + if offsets_left is None: + assert max_len_left is not None + B = values_left.shape[0] // max_len_left + offsets_left_non_optional = max_len_left * torch.arange( + B + 1, device=values_left.device + ) + else: + offsets_left_non_optional = offsets_left + if offsets_right is None: + assert max_len_right is not None + B = values_right.shape[0] // max_len_right + offsets_right_non_optional = max_len_right * torch.arange( + B + 1, device=values_left.device + ) + else: + offsets_right_non_optional = offsets_right + max_len_left = ( + int( + (offsets_left_non_optional[1:] - offsets_left_non_optional[:-1]) + .max() + .item() + ) + if max_len_left is None + else max_len_left + ) + max_len_right = ( + int( + (offsets_right_non_optional[1:] - offsets_right_non_optional[:-1]) + .max() + .item() + ) + if max_len_right is None + else max_len_right + ) + return _concat_2D_jagged_jagged( + values_left=values_left, + values_right=values_right, + max_len_left=max_len_left, + max_len_right=max_len_right, + offsets_left=offsets_left_non_optional, + offsets_right=offsets_right_non_optional, + ) + + +def _split_2D_jagged_jagged( + max_seq_len: int, + values: torch.Tensor, + offsets_left: torch.Tensor, + offsets_right: torch.Tensor, +) -> Tuple[torch.Tensor, torch.Tensor]: + offsets = offsets_left + offsets_right + padded_values = torch.ops.fbgemm.jagged_to_padded_dense( + values=values, + offsets=[offsets], + max_lengths=[max_seq_len], + padding_value=0.0, + ).flatten(0, 1) + lengths_left = offsets_left[1:] - offsets_left[:-1] + lengths_right = offsets_right[1:] - offsets_right[:-1] + mask = fx_arange(max_seq_len, device=values.device).view(1, -1) + mask_left = mask < lengths_left.view(-1, 1) + mask_right = torch.logical_and( + mask >= lengths_left.view(-1, 1), + mask < (lengths_left + lengths_right).view(-1, 1), + ) + return padded_values[mask_left.view(-1), :], padded_values[mask_right.view(-1), :] + + +@torch.fx.wrap +def pytorch_split_2D_jagged( + max_seq_len: int, + values: torch.Tensor, + max_len_left: Optional[int], + max_len_right: Optional[int], + offsets_left: Optional[torch.Tensor], + offsets_right: Optional[torch.Tensor], +) -> Tuple[torch.Tensor, torch.Tensor]: + if offsets_left is None: + assert max_len_left is not None + assert offsets_right is not None + offsets_left_non_optional = max_len_left * torch.arange( + offsets_right.shape[0], device=values.device + ) + else: + offsets_left_non_optional = offsets_left + if offsets_right is None: + assert max_len_right is not None + assert offsets_left is not None + offsets_right_non_optional = max_len_right * torch.arange( + offsets_left.shape[0], device=values.device + ) + else: + offsets_right_non_optional = offsets_right + return _split_2D_jagged_jagged( + max_seq_len=max_seq_len, + values=values, + offsets_left=offsets_left_non_optional, + offsets_right=offsets_right_non_optional, + ) + + +def pytorch_hstu_split_l2_embeddings( + max_seq_len: int, + x: torch.Tensor, + prefix_offsets: torch.Tensor, + l2_offsets: torch.Tensor, + contextual_seq_len: int, +) -> Tuple[torch.Tensor, torch.Tensor]: + x_offsets = prefix_offsets + l2_offsets + x_lengths = x_offsets[1:] - x_offsets[:-1] + padded_x = torch.ops.fbgemm.jagged_to_padded_dense( + values=x, + offsets=[x_offsets], + max_lengths=[max_seq_len], + padding_value=0.0, + ).flatten(0, 1) + prefix_lengths = prefix_offsets[1:] - prefix_offsets[:-1] + mask = fx_arange(max_seq_len, device=x_offsets.device).view(1, -1) + mask_prefix = torch.logical_and( + mask >= contextual_seq_len, + mask < prefix_lengths.view(-1, 1) + contextual_seq_len, + ) + mask_l2 = torch.logical_or( + mask < contextual_seq_len, + torch.logical_and( + mask >= prefix_lengths.view(-1, 1) + contextual_seq_len, + mask < x_lengths.view(-1, 1), + ), + ) + return padded_x[mask_prefix.view(-1), :], padded_x[mask_l2.view(-1), :] + + +def pytorch_hstu_concat_l2_embeddings( + max_prefix_len: int, + prefix_x: torch.Tensor, + prefix_offsets: torch.Tensor, + max_l2_len: int, + l2_x: torch.Tensor, + l2_offsets: torch.Tensor, + contextual_seq_len: int, +) -> torch.Tensor: + padded_prefix_x = torch.ops.fbgemm.jagged_to_padded_dense( + values=prefix_x, + offsets=[prefix_offsets], + max_lengths=[max_prefix_len], + padding_value=0.0, + ) + padded_l2_x = torch.ops.fbgemm.jagged_to_padded_dense( + values=l2_x, + offsets=[l2_offsets], + max_lengths=[max_l2_len], + padding_value=0.0, + ) + padded_x = torch.cat( + [ + padded_l2_x[:, 0:contextual_seq_len, :], + padded_prefix_x, + padded_l2_x[:, contextual_seq_len:, :], + ], + dim=1, + ) + mask = fx_arange(max_prefix_len + max_l2_len, device=prefix_x.device).view(1, -1) + prefix_lengths = prefix_offsets[1:] - prefix_offsets[:-1] + l2_lengths = l2_offsets[1:] - l2_offsets[:-1] + mask = torch.logical_or( + mask < prefix_lengths.view(-1, 1) + contextual_seq_len, + torch.logical_and( + mask >= max_prefix_len + contextual_seq_len, + mask < max_prefix_len + l2_lengths.view(-1, 1), + ), + ) + return padded_x.flatten(0, 1)[mask.view(-1), :] diff --git a/recommendation_v4/generative_recommenders/ops/pytorch/pt_layer_norm.py b/recommendation_v4/generative_recommenders/ops/pytorch/pt_layer_norm.py new file mode 100644 index 000000000..0666212ce --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/pytorch/pt_layer_norm.py @@ -0,0 +1,81 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# pyre-strict + + +from typing import List + +import torch + + +def pytorch_layer_norm( + x: torch.Tensor, + normalized_shape: List[int], + weight: torch.Tensor, + bias: torch.Tensor, + eps: float, +) -> torch.Tensor: + dtype = x.dtype + return torch.nn.functional.layer_norm( + x.to(torch.float32), + normalized_shape, + weight.to(torch.float32), + bias.to(torch.float32), + eps, + ).to(dtype) + + +def pytorch_rms_norm( + x: torch.Tensor, + normalized_shape: List[int], + weight: torch.Tensor, + eps: float, + silu: bool = False, +) -> torch.Tensor: + dtype = x.dtype + x_float = x.to(torch.float32) + normalized = torch.nn.functional.rms_norm( + x_float, + normalized_shape, + weight.to(torch.float32), + eps, + ) + if silu: + normalized = torch.nn.functional.silu(normalized) + return normalized.to(dtype) + + +def pytorch_swish_layer_norm( + x: torch.Tensor, + normalized_shape: List[int], + weight: torch.Tensor, + bias: torch.Tensor, + eps: float, +) -> torch.Tensor: + dtype = x.dtype + x = x.to(torch.float32) + return ( + x + * torch.sigmoid( + torch.nn.functional.layer_norm( + x, + normalized_shape, + weight.to(torch.float32), + bias.to(torch.float32), + eps, + ) + ) + ).to(dtype) diff --git a/recommendation_v4/generative_recommenders/ops/pytorch/pt_position.py b/recommendation_v4/generative_recommenders/ops/pytorch/pt_position.py new file mode 100644 index 000000000..fced57e4f --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/pytorch/pt_position.py @@ -0,0 +1,142 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +from typing import Optional + +import torch +from generative_recommenders.common import ( + fx_unwrap_optional_tensor, + jagged_to_padded_dense, +) + +try: + torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops") + torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops_cpu") +except OSError: + pass + + +@torch.fx.wrap +def torch_arange(end: int, device: torch.device) -> torch.Tensor: + return torch.arange(end, device=device) + + +@torch.fx.wrap +def _get_col_indices( + max_seq_len: int, + max_contextual_seq_len: int, + max_pos_ind: int, + seq_lengths: torch.Tensor, + num_targets: Optional[torch.Tensor], + interleave_targets: bool, +) -> torch.Tensor: + B = seq_lengths.size(0) + col_indices = torch.arange(max_seq_len, device=seq_lengths.device).expand( + B, max_seq_len + ) + if num_targets is not None: + if interleave_targets: + high_inds = seq_lengths - fx_unwrap_optional_tensor(num_targets) * 2 + else: + high_inds = seq_lengths - fx_unwrap_optional_tensor(num_targets) + col_indices = torch.clamp(col_indices, max=high_inds.view(-1, 1)) + col_indices = high_inds.view(-1, 1) - col_indices + else: + col_indices = seq_lengths.view(-1, 1) - col_indices + col_indices = col_indices + max_contextual_seq_len + col_indices = torch.clamp(col_indices, max=max_pos_ind - 1) + if max_contextual_seq_len > 0: + col_indices[:, :max_contextual_seq_len] = torch.arange( + 0, + max_contextual_seq_len, + device=col_indices.device, + dtype=col_indices.dtype, + ).view(1, -1) + return col_indices + + +def pytorch_add_timestamp_positional_embeddings( + seq_embeddings: torch.Tensor, + seq_offsets: torch.Tensor, + pos_embeddings: torch.Tensor, + ts_embeddings: torch.Tensor, + timestamps: torch.Tensor, + max_seq_len: int, + max_contextual_seq_len: int, + seq_lengths: torch.Tensor, + num_targets: Optional[torch.Tensor], + interleave_targets: bool, + time_bucket_fn: str, +) -> torch.Tensor: + max_pos_ind = int(pos_embeddings.size(0)) + # position encoding + pos_inds = _get_col_indices( + max_seq_len=max_seq_len, + max_contextual_seq_len=max_contextual_seq_len, + max_pos_ind=max_pos_ind, + seq_lengths=seq_lengths, + num_targets=num_targets, + interleave_targets=interleave_targets, + ) + B, _ = pos_inds.shape + # timestamp encoding + num_time_buckets = ts_embeddings.size(1) - 1 + time_bucket_increments = 60.0 + time_bucket_divisor = 1.0 + time_delta = 0 + timestamps = jagged_to_padded_dense( + values=timestamps.unsqueeze(-1), + offsets=[seq_offsets], + max_lengths=[max_seq_len], + padding_value=0.0, + ).squeeze(-1) + query_time = torch.gather( + timestamps, dim=1, index=(seq_lengths - 1).unsqueeze(1).clamp(min=0) + ) + ts = query_time - timestamps + ts = ts + time_delta + ts = ts.clamp(min=1e-6) / time_bucket_increments + if time_bucket_fn == "log": + ts = torch.log(ts) + else: + ts = torch.sqrt(ts) + ts = (ts / time_bucket_divisor).clamp(min=0).int() + ts = torch.clamp( + ts, + min=0, + max=num_time_buckets, + ) + position_embeddings = torch.index_select( + pos_embeddings, 0, pos_inds.reshape(-1) + ).view(B, max_seq_len, -1) + time_embeddings = torch.index_select(ts_embeddings, 0, ts.reshape(-1)).view( + B, max_seq_len, -1 + ) + padded_emb = torch.ops.fbgemm.jagged_to_padded_dense( + values=seq_embeddings, + offsets=[seq_offsets], + max_lengths=[max_seq_len], + padding_value=0.0, + ) + summed = padded_emb + (time_embeddings + position_embeddings).to( + seq_embeddings.dtype + ) + result, _ = torch.ops.fbgemm.dense_to_jagged( + summed, [seq_offsets], seq_embeddings.shape[0] + ) + return result diff --git a/recommendation_v4/generative_recommenders/ops/tests/fake_signature_test.py b/recommendation_v4/generative_recommenders/ops/tests/fake_signature_test.py new file mode 100644 index 000000000..fb9047454 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/tests/fake_signature_test.py @@ -0,0 +1,162 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +""" +Tests to ensure fake and real implementations of triton functions +have the same function signatures. This is critical for PT2 compile compatibility. +""" + +import inspect +import unittest +from typing import Any, Callable, List + + +def get_custom_op_params(func: Callable[..., object]) -> List[str]: + """ + Get parameter names from a function, handling custom_op decorated functions. + + For maybe_register_custom_op decorated functions, inspect.signature may return + *args, **kwargs instead of the actual parameters. In this case, we need to + access the underlying schema to get the real parameter names. + """ + sig = inspect.signature(func) + params = list(sig.parameters.keys()) + + if params == ["args", "kwargs"]: + func_any: Any = func + if hasattr(func_any, "_opoverload"): + schema = func_any._opoverload._schema + return [arg.name for arg in schema.arguments] + + return params + + +class FakeSignatureTest(unittest.TestCase): + """Test to ensure fake and real implementations have the same function signatures.""" + + def test_triton_addmm_fwd_and_fake_have_same_signature(self) -> None: + """Verify triton_addmm_fwd and triton_addmm_fwd_fake have the same arguments.""" + from generative_recommenders.ops.triton.triton_addmm import ( + triton_addmm_fwd, + triton_addmm_fwd_fake, + ) + + real_params = get_custom_op_params(triton_addmm_fwd) + fake_params = get_custom_op_params(triton_addmm_fwd_fake) + + self.assertEqual( + real_params, + fake_params, + f"triton_addmm_fwd and triton_addmm_fwd_fake have different arguments.\n" + f"Real: {real_params}\n" + f"Fake: {fake_params}", + ) + + def test_maybe_triton_addmm_fwd_and_fake_have_same_signature(self) -> None: + """Verify maybe_triton_addmm_fwd and maybe_triton_addmm_fwd_fake have the same arguments.""" + from generative_recommenders.ops.triton.triton_addmm import ( + maybe_triton_addmm_fwd, + maybe_triton_addmm_fwd_fake, + ) + + real_params = get_custom_op_params(maybe_triton_addmm_fwd) + fake_params = get_custom_op_params(maybe_triton_addmm_fwd_fake) + + self.assertEqual( + real_params, + fake_params, + f"maybe_triton_addmm_fwd and maybe_triton_addmm_fwd_fake have different arguments.\n" + f"Real: {real_params}\n" + f"Fake: {fake_params}", + ) + + def test_triton_hstu_attention_fwd_and_fake_have_same_signature(self) -> None: + """Verify triton_hstu_attention_fwd and _triton_hstu_attention_fwd_fake have the same arguments.""" + from generative_recommenders.ops.triton.triton_hstu_attention import ( + _triton_hstu_attention_fwd_fake, + triton_hstu_attention_fwd, + ) + + real_params = get_custom_op_params(triton_hstu_attention_fwd) + fake_params = get_custom_op_params(_triton_hstu_attention_fwd_fake) + + self.assertEqual( + real_params, + fake_params, + f"triton_hstu_attention_fwd and _triton_hstu_attention_fwd_fake have different arguments.\n" + f"Real: {real_params}\n" + f"Fake: {fake_params}", + ) + + def test_triton_hstu_attention_bwd_and_fake_have_same_signature(self) -> None: + """Verify triton_hstu_attention_bwd and _triton_hstu_attention_bwd_fake have the same arguments.""" + from generative_recommenders.ops.triton.triton_hstu_attention import ( + _triton_hstu_attention_bwd_fake, + triton_hstu_attention_bwd, + ) + + real_params = get_custom_op_params(triton_hstu_attention_bwd) + fake_params = get_custom_op_params(_triton_hstu_attention_bwd_fake) + + self.assertEqual( + real_params, + fake_params, + f"triton_hstu_attention_bwd and _triton_hstu_attention_bwd_fake have different arguments.\n" + f"Real: {real_params}\n" + f"Fake: {fake_params}", + ) + + def test_triton_layer_norm_mul_dropout_fwd_impl_and_fake_have_same_signature( + self, + ) -> None: + """Verify _triton_layer_norm_mul_dropout_fwd_impl and its fake have the same arguments.""" + from generative_recommenders.ops.triton.triton_hstu_linear import ( + _triton_layer_norm_mul_dropout_fwd_impl, + _triton_layer_norm_mul_dropout_fwd_impl_fake, + ) + + real_params = get_custom_op_params(_triton_layer_norm_mul_dropout_fwd_impl) + fake_params = get_custom_op_params(_triton_layer_norm_mul_dropout_fwd_impl_fake) + + self.assertEqual( + real_params, + fake_params, + f"_triton_layer_norm_mul_dropout_fwd_impl and _triton_layer_norm_mul_dropout_fwd_impl_fake have different arguments.\n" + f"Real: {real_params}\n" + f"Fake: {fake_params}", + ) + + def test_triton_layer_norm_mul_dropout_bwd_impl_and_fake_have_same_signature( + self, + ) -> None: + """Verify _triton_layer_norm_mul_dropout_bwd_impl and its fake have the same arguments.""" + from generative_recommenders.ops.triton.triton_hstu_linear import ( + _triton_layer_norm_mul_dropout_bwd_impl, + _triton_layer_norm_mul_dropout_bwd_impl_fake, + ) + + real_params = get_custom_op_params(_triton_layer_norm_mul_dropout_bwd_impl) + fake_params = get_custom_op_params(_triton_layer_norm_mul_dropout_bwd_impl_fake) + + self.assertEqual( + real_params, + fake_params, + f"_triton_layer_norm_mul_dropout_bwd_impl and _triton_layer_norm_mul_dropout_bwd_impl_fake have different arguments.\n" + f"Real: {real_params}\n" + f"Fake: {fake_params}", + ) diff --git a/recommendation_v4/generative_recommenders/ops/tests/hstu_attention_test.py b/recommendation_v4/generative_recommenders/ops/tests/hstu_attention_test.py new file mode 100644 index 000000000..ef70b8cc2 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/tests/hstu_attention_test.py @@ -0,0 +1,485 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +import random +import unittest +from typing import Optional + +import torch +from generative_recommenders.common import ( + generate_sparse_seq_len, + gpu_unavailable, + HammerKernel, + set_dev_mode, +) +from generative_recommenders.ops.jagged_tensors import split_2D_jagged +from hypothesis import given, settings, strategies as st, Verbosity + + +def test_attn( + batch_size: int, + heads: int, + max_uih_len: int, + max_targets: int, + attn_dim: int, + hidden_dim: int, + causal: bool, + has_multiple_targets: bool, + has_max_attn_len: bool, + dtype: torch.dtype, + test_backward: bool, + ref_kernel: HammerKernel, + real_kernel: HammerKernel, + skip_comparisons: bool = False, + sparsity: float = -1.0, + contextual_seq_len: int = 0, + atol: Optional[float] = None, + rtol: Optional[float] = None, + enable_tma: bool = False, +) -> None: + set_dev_mode(True) + torch.backends.cudnn.allow_tf32 = True + torch.backends.cuda.matmul.allow_tf32 = True + from generative_recommenders.ops.hstu_attention import hstu_mha + + alpha = 1.0 / (attn_dim**0.5) + if sparsity > 0.0: + lengths = generate_sparse_seq_len( + size=batch_size, + max_seq_len=max_uih_len, + sparsity=sparsity, + device=torch.device("cuda"), + ) + else: + lengths = torch.randint( + max_uih_len + 1, size=(batch_size,), device=torch.device("cuda") + ) + num_targets = torch.randint( + 1, max_targets + 1, size=(batch_size,), device=torch.device("cuda") + ) + lengths = lengths + num_targets + contextual_seq_len + max_seq_len = max_uih_len + max_targets + contextual_seq_len + if has_max_attn_len: + max_attn_len = random.randint(1, max_uih_len // 5) + else: + max_attn_len = 0 + seq_offsets = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + seq_offsets[1:] = torch.cumsum(lengths, dim=0) + + L = int(seq_offsets[-1].item()) + q = ( + torch.empty((L, heads, attn_dim), dtype=dtype, device=torch.device("cuda")) + .uniform_(-0.1, 0.1) + .requires_grad_() + ) + k = ( + torch.empty((L, heads, attn_dim), dtype=dtype, device=torch.device("cuda")) + .uniform_(-0.1, 0.1) + .requires_grad_() + ) + v = ( + torch.empty((L, heads, hidden_dim), dtype=dtype, device=torch.device("cuda")) + .uniform_(-0.1, 0.1) + .requires_grad_() + ) + + # ref implementation + ref_out = hstu_mha( + max_seq_len=max_seq_len, + alpha=alpha, + q=q, + k=k, + v=v, + seq_offsets=seq_offsets, + causal=causal, + num_targets=num_targets if has_multiple_targets else None, + dropout_pr=0.0, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + kernel=ref_kernel, + enable_tma=enable_tma, + ) + dout = torch.randn_like(ref_out) + ref_out.backward(dout) + + if skip_comparisons: + return + + # pyre-ignore + ref_dv, v.grad = v.grad.clone(), None + ref_dk, k.grad = k.grad.clone(), None + ref_dq, q.grad = q.grad.clone(), None + + # triton implementation + q = q.detach().clone().requires_grad_() + k = k.detach().clone().requires_grad_() + v = v.detach().clone().requires_grad_() + dout = dout.detach().clone() + real_out = hstu_mha( + max_seq_len=max_seq_len, + alpha=alpha, + q=q, + k=k, + v=v, + seq_offsets=seq_offsets, + causal=causal, + num_targets=num_targets if has_multiple_targets else None, + dropout_pr=0.0, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + kernel=real_kernel, + enable_tma=enable_tma, + ) + + torch.testing.assert_close( + ref_out, + real_out, + atol=atol, + rtol=rtol, + ) + if test_backward: + real_out.backward(dout) + real_dq, real_dk, real_dv = q.grad.clone(), k.grad.clone(), v.grad.clone() + torch.testing.assert_close(ref_dv, real_dv, atol=atol, rtol=rtol) + torch.testing.assert_close(ref_dk, real_dk, atol=atol, rtol=rtol) + torch.testing.assert_close(ref_dq, real_dq, atol=atol, rtol=rtol) + + +def test_delta_attn( + batch_size: int, + heads: int, + max_uih_len: int, + max_targets: int, + delta_size: int, + attn_dim: int, + hidden_dim: int, + has_multiple_targets: bool, + has_max_attn_len: bool, + dtype: torch.dtype, + ref_kernel: HammerKernel, + real_kernel: HammerKernel, + contextual_seq_len: int = 0, + atol: Optional[float] = None, + rtol: Optional[float] = None, + enable_tma: bool = False, +) -> None: + set_dev_mode(True) + torch.backends.cudnn.allow_tf32 = True + torch.backends.cuda.matmul.allow_tf32 = True + from generative_recommenders.ops.hstu_attention import delta_hstu_mha + + alpha = 1.0 / (attn_dim**0.5) + lengths = torch.randint( + max_uih_len + 1, size=(batch_size,), device=torch.device("cuda") + ) + num_targets = torch.randint( + 1, delta_size + 1, size=(batch_size,), device=torch.device("cuda") + ) + lengths = lengths + delta_size + contextual_seq_len + max_seq_len = max_uih_len + delta_size + contextual_seq_len + if has_max_attn_len: + max_attn_len = random.randint(1, max_uih_len // 5) + else: + max_attn_len = 0 + seq_offsets = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + seq_offsets[1:] = torch.cumsum(lengths, dim=0) + + L = int(seq_offsets[-1].item()) + delta_q = torch.empty( + (batch_size * delta_size, heads, attn_dim), + dtype=dtype, + device=torch.device("cuda"), + ).uniform_(-0.1, 0.1) + k = torch.empty( + (L, heads, attn_dim), dtype=dtype, device=torch.device("cuda") + ).uniform_(-0.1, 0.1) + v = torch.empty( + (L, heads, hidden_dim), dtype=dtype, device=torch.device("cuda") + ).uniform_(-0.1, 0.1) + + # ref implementation + ref_out = delta_hstu_mha( + max_seq_len=max_seq_len, + alpha=alpha, + delta_q=delta_q, + k=k, + v=v, + seq_offsets=seq_offsets, + num_targets=num_targets if has_multiple_targets else None, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + kernel=ref_kernel, + enable_tma=enable_tma, + ) + + # real implementation + real_out = delta_hstu_mha( + max_seq_len=max_seq_len, + alpha=alpha, + delta_q=delta_q, + k=k, + v=v, + seq_offsets=seq_offsets, + num_targets=num_targets if has_multiple_targets else None, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + kernel=real_kernel, + enable_tma=enable_tma, + ) + torch.testing.assert_close( + ref_out, + real_out, + atol=atol, + rtol=rtol, + ) + + +class HSTUAttentionTest(unittest.TestCase): + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore + @given( + batch_size=st.integers(4, 8), + heads=st.integers(1, 4), + max_uih_len=st.sampled_from([20, 100, 128, 256]), + max_targets=st.sampled_from([20, 512]), + attn_dim=st.sampled_from([16, 32, 64, 128]), + hidden_dim=st.sampled_from([16, 32, 64, 128]), + causal=st.sampled_from([True]), + has_multiple_targets=st.sampled_from([True, False]), + dtype=st.sampled_from( + [torch.bfloat16, torch.float32] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + has_max_attn_len=st.sampled_from([True, False]), + contextual_seq_len=st.sampled_from([0, 10]), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=200, + deadline=None, + ) + # pyre-ignore[2] + def test_attn_triton(self, *args, **kwargs) -> None: + test_attn( + *args, + **kwargs, + test_backward=True, + ref_kernel=HammerKernel.PYTORCH, + real_kernel=HammerKernel.TRITON, + ) + + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore + @given( + batch_size=st.just(64), + heads=st.just(4), + max_uih_len=st.sampled_from([32768]), + max_targets=st.sampled_from([32]), + attn_dim=st.just(128), + hidden_dim=st.just(128), + causal=st.sampled_from([True]), + has_multiple_targets=st.sampled_from([True, False]), + dtype=st.sampled_from([torch.bfloat16]), + has_max_attn_len=st.sampled_from([True, False]), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=5, + deadline=None, + ) + # pyre-ignore[2] + def test_attn_triton_long_seqs(self, *args, **kwargs) -> None: + test_attn( + *args, + **kwargs, + test_backward=True, + ref_kernel=HammerKernel.TRITON, + real_kernel=HammerKernel.TRITON, + skip_comparisons=True, + sparsity=1.0, + ) + + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore + @given( + batch_size=st.integers(4, 8), + heads=st.integers(1, 4), + max_uih_len=st.sampled_from([100, 128, 256]), + max_targets=st.sampled_from([20, 512]), + delta_size=st.sampled_from([20, 512]), + attn_dim=st.sampled_from([16, 32, 64, 128]), + hidden_dim=st.sampled_from([16, 32, 64, 128]), + has_multiple_targets=st.sampled_from([True, False]), + dtype=st.sampled_from( + [torch.bfloat16, torch.float32] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + has_max_attn_len=st.sampled_from([False, True]), + contextual_seq_len=st.sampled_from([0, 10]), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=200, + deadline=None, + ) + # pyre-ignore[2] + def test_delta_attn_triton(self, *args, **kwargs) -> None: + test_delta_attn( + *args, + **kwargs, + ref_kernel=HammerKernel.PYTORCH, + real_kernel=HammerKernel.TRITON, + ) + + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore + @given( + batch_size=st.integers(4, 8), + heads=st.integers(1, 4), + max_uih_len=st.sampled_from([20, 100, 128]), + max_targets=st.sampled_from([20, 512]), + delta_size=st.sampled_from([20, 512]), + attn_dim=st.sampled_from([16, 32, 64]), + hidden_dim=st.sampled_from([16, 32, 64]), + has_multiple_targets=st.sampled_from([True, False]), + dtype=st.sampled_from( + [torch.bfloat16, torch.float32] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + has_max_attn_len=st.sampled_from([False, True]), + contextual_seq_len=st.sampled_from([0, 10]), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=200, + deadline=None, + ) + def test_cache( + self, + batch_size: int, + heads: int, + max_uih_len: int, + max_targets: int, + delta_size: int, + attn_dim: int, + hidden_dim: int, + has_multiple_targets: bool, + dtype: torch.dtype, + has_max_attn_len: bool, + contextual_seq_len: int, + ) -> None: + set_dev_mode(True) + torch.backends.cudnn.allow_tf32 = True + torch.backends.cuda.matmul.allow_tf32 = True + from generative_recommenders.ops.hstu_attention import delta_hstu_mha, hstu_mha + + alpha = 1.0 / (attn_dim**0.5) + lengths = torch.randint( + max_uih_len + 1, size=(batch_size,), device=torch.device("cuda") + ) + num_targets = torch.randint( + 1, delta_size + 1, size=(batch_size,), device=torch.device("cuda") + ) + lengths = lengths + delta_size + contextual_seq_len + max_seq_len = max_uih_len + delta_size + contextual_seq_len + if has_max_attn_len: + max_attn_len = random.randint(1, max_uih_len // 5) + else: + max_attn_len = 0 + seq_offsets = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + seq_offsets[1:] = torch.cumsum(lengths, dim=0) + + L = int(seq_offsets[-1].item()) + q = torch.empty( + (L, heads, attn_dim), + dtype=dtype, + device=torch.device("cuda"), + ).uniform_(-0.1, 0.1) + _, delta_q = split_2D_jagged( + max_seq_len=max_seq_len, + values=q.view(-1, heads * attn_dim), + max_len_left=None, + max_len_right=delta_size, + offsets_left=torch.ops.fbgemm.asynchronous_complete_cumsum( + lengths - delta_size + ), + offsets_right=None, + kernel=HammerKernel.TRITON, + ) + delta_q = delta_q.view(-1, heads, attn_dim) + k = torch.empty( + (L, heads, attn_dim), dtype=dtype, device=torch.device("cuda") + ).uniform_(-0.1, 0.1) + v = torch.empty( + (L, heads, hidden_dim), dtype=dtype, device=torch.device("cuda") + ).uniform_(-0.1, 0.1) + prime_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum( + lengths - delta_size + ) + + # ref implementation + ref_out = hstu_mha( + max_seq_len=max_seq_len, + alpha=alpha, + q=q, + k=k, + v=v, + seq_offsets=seq_offsets, + causal=True, + num_targets=num_targets if has_multiple_targets else None, + dropout_pr=0.0, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + kernel=HammerKernel.TRITON, + ) + _, delta_out = split_2D_jagged( + max_seq_len=max_seq_len, + values=ref_out.view(-1, heads * hidden_dim), + max_len_left=None, + max_len_right=delta_size, + offsets_left=prime_offsets, + offsets_right=None, + kernel=HammerKernel.TRITON, + ) + delta_out = delta_out.view(-1, heads, hidden_dim) + + # real implementation + real_delta_out = delta_hstu_mha( + max_seq_len=max_seq_len, + alpha=alpha, + delta_q=delta_q, + k=k, + v=v, + seq_offsets=seq_offsets, + num_targets=num_targets if has_multiple_targets else None, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + ) + torch.testing.assert_close( + delta_out, + real_delta_out, + ) diff --git a/recommendation_v4/generative_recommenders/ops/tests/hstu_attention_tma_test.py b/recommendation_v4/generative_recommenders/ops/tests/hstu_attention_tma_test.py new file mode 100644 index 000000000..8bb264af6 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/tests/hstu_attention_tma_test.py @@ -0,0 +1,270 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +import random +import unittest + +import torch +from generative_recommenders.common import ( + HammerKernel, + nv_gpu_unavailable, + set_dev_mode, +) +from generative_recommenders.ops.jagged_tensors import split_2D_jagged +from generative_recommenders.ops.tests.hstu_attention_test import ( + test_attn, + test_delta_attn, +) +from hypothesis import given, settings, strategies as st, Verbosity + + +class HSTUAttentionTmaTest(unittest.TestCase): + @unittest.skipIf(*nv_gpu_unavailable) + # pyre-ignore + @given( + batch_size=st.integers(4, 8), + heads=st.integers(1, 4), + max_uih_len=st.sampled_from([20, 100, 128, 256]), + max_targets=st.sampled_from([20, 512]), + attn_dim=st.sampled_from([16, 32, 64, 128]), + hidden_dim=st.sampled_from([16, 32, 64, 128]), + causal=st.sampled_from([True]), + has_multiple_targets=st.sampled_from([True, False]), + dtype=st.sampled_from( + [torch.bfloat16, torch.float32] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + has_max_attn_len=st.sampled_from([True, False]), + contextual_seq_len=st.sampled_from([0, 10]), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=200, + deadline=None, + ) + # pyre-ignore[2] + def test_attn_triton_tma(self, *args, **kwargs) -> None: + test_attn( + *args, + **kwargs, + test_backward=True, + ref_kernel=HammerKernel.PYTORCH, + real_kernel=HammerKernel.TRITON, + enable_tma=True, + ) + + @unittest.skipIf(*nv_gpu_unavailable) + # pyre-ignore + @given( + batch_size=st.just(64), + heads=st.just(4), + max_uih_len=st.sampled_from([32768]), + max_targets=st.sampled_from([32]), + attn_dim=st.just(128), + hidden_dim=st.just(128), + causal=st.sampled_from([True]), + has_multiple_targets=st.sampled_from([True, False]), + dtype=st.sampled_from([torch.bfloat16]), + has_max_attn_len=st.sampled_from([True, False]), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=5, + deadline=None, + ) + # pyre-ignore[2] + def test_attn_triton_long_seqs_tma(self, *args, **kwargs) -> None: + test_attn( + *args, + **kwargs, + test_backward=True, + ref_kernel=HammerKernel.TRITON, + real_kernel=HammerKernel.TRITON, + skip_comparisons=True, + sparsity=1.0, + enable_tma=True, + ) + + @unittest.skipIf(*nv_gpu_unavailable) + # pyre-ignore + @given( + batch_size=st.integers(4, 8), + heads=st.integers(1, 4), + max_uih_len=st.sampled_from([100, 128, 256]), + max_targets=st.sampled_from([20, 512]), + delta_size=st.sampled_from([20, 512]), + attn_dim=st.sampled_from([16, 32, 64, 128]), + hidden_dim=st.sampled_from([16, 32, 64, 128]), + has_multiple_targets=st.sampled_from([True, False]), + dtype=st.sampled_from( + [torch.bfloat16, torch.float32] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + has_max_attn_len=st.sampled_from([False, True]), + contextual_seq_len=st.sampled_from([0, 10]), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=200, + deadline=None, + ) + # pyre-ignore[2] + def test_delta_attn_triton_tma(self, *args, **kwargs) -> None: + test_delta_attn( + *args, + **kwargs, + ref_kernel=HammerKernel.PYTORCH, + real_kernel=HammerKernel.TRITON, + enable_tma=True, + ) + + @unittest.skipIf(*nv_gpu_unavailable) + # pyre-ignore + @given( + batch_size=st.integers(4, 8), + heads=st.integers(1, 4), + max_uih_len=st.sampled_from([20, 100, 128]), + max_targets=st.sampled_from([20, 512]), + delta_size=st.sampled_from([20, 512]), + attn_dim=st.sampled_from([16, 32, 64]), + hidden_dim=st.sampled_from([16, 32, 64]), + has_multiple_targets=st.sampled_from([True, False]), + dtype=st.sampled_from( + [torch.bfloat16, torch.float32] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + has_max_attn_len=st.sampled_from([False, True]), + contextual_seq_len=st.sampled_from([0, 10]), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=200, + deadline=None, + ) + def test_cache_tma( + self, + batch_size: int, + heads: int, + max_uih_len: int, + max_targets: int, + delta_size: int, + attn_dim: int, + hidden_dim: int, + has_multiple_targets: bool, + dtype: torch.dtype, + has_max_attn_len: bool, + contextual_seq_len: int, + ) -> None: + set_dev_mode(True) + torch.backends.cudnn.allow_tf32 = True + torch.backends.cuda.matmul.allow_tf32 = True + from generative_recommenders.ops.hstu_attention import delta_hstu_mha, hstu_mha + + alpha = 1.0 / (attn_dim**0.5) + lengths = torch.randint( + max_uih_len + 1, size=(batch_size,), device=torch.device("cuda") + ) + num_targets = torch.randint( + 1, delta_size + 1, size=(batch_size,), device=torch.device("cuda") + ) + lengths = lengths + delta_size + contextual_seq_len + max_seq_len = max_uih_len + delta_size + contextual_seq_len + if has_max_attn_len: + max_attn_len = random.randint(1, max_uih_len // 5) + else: + max_attn_len = 0 + seq_offsets = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + seq_offsets[1:] = torch.cumsum(lengths, dim=0) + + L = int(seq_offsets[-1].item()) + q = torch.empty( + (L, heads, attn_dim), + dtype=dtype, + device=torch.device("cuda"), + ).uniform_(-0.1, 0.1) + _, delta_q = split_2D_jagged( + max_seq_len=max_seq_len, + values=q.view(-1, heads * attn_dim), + max_len_left=None, + max_len_right=delta_size, + offsets_left=torch.ops.fbgemm.asynchronous_complete_cumsum( + lengths - delta_size + ), + offsets_right=None, + kernel=HammerKernel.TRITON, + ) + delta_q = delta_q.view(-1, heads, attn_dim) + k = torch.empty( + (L, heads, attn_dim), dtype=dtype, device=torch.device("cuda") + ).uniform_(-0.1, 0.1) + v = torch.empty( + (L, heads, hidden_dim), dtype=dtype, device=torch.device("cuda") + ).uniform_(-0.1, 0.1) + prime_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum( + lengths - delta_size + ) + + # ref implementation + ref_out = hstu_mha( + max_seq_len=max_seq_len, + alpha=alpha, + q=q, + k=k, + v=v, + seq_offsets=seq_offsets, + causal=True, + num_targets=num_targets if has_multiple_targets else None, + dropout_pr=0.0, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + kernel=HammerKernel.TRITON, + enable_tma=True, + ) + _, delta_out = split_2D_jagged( + max_seq_len=max_seq_len, + values=ref_out.view(-1, heads * hidden_dim), + max_len_left=None, + max_len_right=delta_size, + offsets_left=prime_offsets, + offsets_right=None, + kernel=HammerKernel.TRITON, + ) + delta_out = delta_out.view(-1, heads, hidden_dim) + + # real implementation + real_delta_out = delta_hstu_mha( + max_seq_len=max_seq_len, + alpha=alpha, + delta_q=delta_q, + k=k, + v=v, + seq_offsets=seq_offsets, + num_targets=num_targets if has_multiple_targets else None, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + enable_tma=True, + ) + torch.testing.assert_close( + delta_out, + real_delta_out, + ) diff --git a/recommendation_v4/generative_recommenders/ops/tests/hstu_compute_test.py b/recommendation_v4/generative_recommenders/ops/tests/hstu_compute_test.py new file mode 100644 index 000000000..57f217895 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/tests/hstu_compute_test.py @@ -0,0 +1,503 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +import random +import unittest +from typing import Optional + +import torch +from generative_recommenders.common import ( + generate_sparse_seq_len, + gpu_unavailable, + HammerKernel, + set_dev_mode, +) +from hypothesis import given, settings, strategies as st, Verbosity + + +class HSTUComputeTest(unittest.TestCase): + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore[56] + @given( + N=st.integers(min_value=1000, max_value=1000), + D=st.integers(min_value=128, max_value=128), + L=st.integers(min_value=512, max_value=512), + concat_u=st.booleans(), + concat_x=st.booleans(), + mul_u_activation_type=st.sampled_from(["silu", "sigmoid", "none"]), + group_norm=st.booleans(), + num_heads=st.sampled_from([4]), + training=st.just(False), + recompute_y_in_backward=st.sampled_from([True, False]), + dtype=st.sampled_from( + [torch.float32, torch.bfloat16] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + ) + @settings( + deadline=None, + verbosity=Verbosity.verbose, + max_examples=20, + ) + # pyre-ignore[2] + def test_compute_output(self, *args, **kwargs) -> None: + self._test_compute_output( + *args, + **kwargs, + test_backward=True, + ref_kernel=HammerKernel.PYTORCH, + opt_kernel=HammerKernel.TRITON, + ) + + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore[56] + @given( + N=st.just(1500000), + D=st.just(512), + L=st.just(512), + concat_u=st.sampled_from([True]), + concat_x=st.sampled_from([True]), + mul_u_activation_type=st.sampled_from(["none"]), + group_norm=st.sampled_from([False]), + num_heads=st.sampled_from([4]), + training=st.just(False), + recompute_y_in_backward=st.sampled_from([False]), + dtype=st.just(torch.bfloat16), + ) + @settings( + deadline=None, + verbosity=Verbosity.verbose, + max_examples=1, + ) + # pyre-ignore[2] + def test_long_sequences_compute_output(self, *args, **kwargs) -> None: + self._test_compute_output( + *args, + **kwargs, + test_backward=False, + ref_kernel=HammerKernel.TRITON, + opt_kernel=HammerKernel.TRITON, + skip_comparisons=True, + ) + + def _test_compute_output( + self, + N: int, + D: int, + L: int, + concat_u: bool, + concat_x: bool, + mul_u_activation_type: str, + group_norm: bool, + num_heads: int, + training: bool, + recompute_y_in_backward: bool, + dtype: torch.dtype, + test_backward: bool, + ref_kernel: HammerKernel, + opt_kernel: HammerKernel, + skip_comparisons: bool = False, + atol: Optional[float] = None, + rtol: Optional[float] = None, + ) -> None: + from generative_recommenders.ops.hstu_compute import hstu_compute_output + + torch.manual_seed(0) + torch.backends.cudnn.allow_tf32 = False + torch.backends.cuda.matmul.allow_tf32 = False + dropout_ratio = 0.3 if training else 0.0 + attn = ( + torch.empty((N, L), dtype=dtype, device=torch.device("cuda")) + .uniform_(-0.1, 0.1) + .requires_grad_() + ) + u = ( + torch.empty((N, L), dtype=dtype, device=torch.device("cuda")) + .uniform_(-0.1, 0.1) + .requires_grad_() + ) + norm_weight = ( + torch.empty( + (L if not group_norm else num_heads,), + dtype=dtype, + device=torch.device("cuda"), + ) + .uniform_(-0.1, 0.1) + .requires_grad_() + ) + norm_bias = ( + torch.empty( + (L if not group_norm else num_heads,), + dtype=dtype, + device=torch.device("cuda"), + ) + .uniform_(-0.1, 0.1) + .requires_grad_() + ) + norm_eps = 1e-6 + # When group_norm=True, only concat_ux = concat_u and concat_x is supported + if group_norm: + L_mult = 3 if (concat_u and concat_x) else 1 + else: + L_mult = 1 + if concat_u: + L_mult += 1 + if concat_x: + L_mult += 1 + output_weight = ( + torch.empty((L * L_mult, D), dtype=dtype, device=torch.device("cuda")) + .uniform_(-0.1, 0.1) + .requires_grad_() + ) + x = ( + torch.empty((N, D), dtype=dtype, device=torch.device("cuda")) + .uniform_(-0.1, 0.1) + .requires_grad_() + ) + + # ref + ref_out = hstu_compute_output( + attn=attn, + u=u, + x=x, + norm_weight=norm_weight, + norm_bias=norm_bias, + norm_eps=norm_eps, + dropout_ratio=dropout_ratio, + output_weight=output_weight, + concat_u=concat_u, + concat_x=concat_x, + mul_u_activation_type=mul_u_activation_type, + group_norm=group_norm, + num_heads=num_heads, + linear_dim=L // num_heads, + training=training, + recompute_y_in_backward=recompute_y_in_backward, + kernel=ref_kernel, + ) + dout = torch.randn_like(ref_out) * 0.1 + ref_out.backward(dout) + if skip_comparisons: + return + # pyre-ignore[16] + ref_dattn, attn.grad = attn.grad.detach().clone(), None + ref_du, u.grad = u.grad.detach().clone(), None + ref_d_norm_w, norm_weight.grad = norm_weight.grad.detach().clone(), None + ref_d_norm_b, norm_bias.grad = norm_bias.grad.detach().clone(), None + ref_dx, x.grad = x.grad.detach().clone(), None + ref_d_output_w, output_weight.grad = output_weight.grad.detach().clone(), None + + # opt + attn = attn.detach().clone().requires_grad_() + u = u.detach().clone().requires_grad_() + norm_weight = norm_weight.detach().clone().requires_grad_() + norm_bias = norm_bias.detach().clone().requires_grad_() + output_weight = output_weight.detach().clone().requires_grad_() + x = x.detach().clone().requires_grad_() + opt_out = hstu_compute_output( + attn=attn, + u=u, + x=x, + norm_weight=norm_weight, + norm_bias=norm_bias, + norm_eps=norm_eps, + dropout_ratio=dropout_ratio, + output_weight=output_weight, + concat_u=concat_u, + concat_x=concat_x, + mul_u_activation_type=mul_u_activation_type, + group_norm=group_norm, + num_heads=num_heads, + linear_dim=L // num_heads, + training=training, + recompute_y_in_backward=recompute_y_in_backward, + kernel=opt_kernel, + ) + torch.testing.assert_close( + ref_out, + opt_out, + atol=atol, + rtol=rtol, + ) + + if test_backward: + dout = dout.detach().clone() + opt_out.backward(dout) + opt_dattn, attn.grad = attn.grad.detach().clone(), None + opt_du, u.grad = u.grad.detach().clone(), None + opt_d_norm_w, norm_weight.grad = norm_weight.grad.detach().clone(), None + opt_d_norm_b, norm_bias.grad = norm_bias.grad.detach().clone(), None + opt_dx, x.grad = x.grad.detach().clone(), None + opt_d_output_w, output_weight.grad = ( + output_weight.grad.detach().clone(), + None, + ) + torch.testing.assert_close(ref_du, opt_du) + torch.testing.assert_close(ref_dattn, opt_dattn) + torch.testing.assert_close(ref_d_norm_w, opt_d_norm_w) + torch.testing.assert_close(ref_d_norm_b, opt_d_norm_b) + torch.testing.assert_close(ref_dx, opt_dx) + torch.testing.assert_close(ref_d_output_w, opt_d_output_w) + + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore + @given( + batch_size=st.integers(4, 8), + heads=st.integers(1, 4), + max_uih_len=st.sampled_from([100, 128, 256, 1300]), + max_targets=st.sampled_from([20, 512]), + embedding_dim=st.sampled_from([16, 32, 64]), + attn_dim=st.sampled_from([16, 32, 64, 128]), + hidden_dim=st.sampled_from([16, 32, 64, 128]), + causal=st.sampled_from([True]), + has_multiple_targets=st.sampled_from([True, False]), + dtype=st.sampled_from( + [torch.float32] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + contextual_seq_len=st.sampled_from([0]), + has_max_attn_len=st.sampled_from([False, True]), + sort_by_length=st.sampled_from([True, False]), + recompute_uvqk_in_backward=st.sampled_from([True, False]), + recompute_normed_x_in_backward=st.sampled_from([True, False]), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=150, + deadline=None, + ) + # pyre-ignore[2] + def test_preprocess_and_attention(self, *args, **kwargs) -> None: + self._test_hstu_preprocess_and_attention( + *args, + **kwargs, + test_backward=True, + ref_kernel=HammerKernel.PYTORCH, + real_kernel=HammerKernel.TRITON, + ) + + def _test_hstu_preprocess_and_attention( + self, + batch_size: int, + heads: int, + max_uih_len: int, + max_targets: int, + embedding_dim: int, + attn_dim: int, + hidden_dim: int, + causal: bool, + has_multiple_targets: bool, + has_max_attn_len: bool, + dtype: torch.dtype, + ref_kernel: HammerKernel, + real_kernel: HammerKernel, + test_backward: bool, + contextual_seq_len: int, + sort_by_length: bool, + recompute_uvqk_in_backward: bool, + recompute_normed_x_in_backward: bool, + sparsity: float = -1.0, + atol: Optional[float] = None, + rtol: Optional[float] = None, + ) -> None: + set_dev_mode(True) + torch.backends.cudnn.allow_tf32 = False + torch.backends.cuda.matmul.allow_tf32 = False + + from generative_recommenders.ops.hstu_compute import ( + hstu_preprocess_and_attention, + ) + + alpha = 1.0 / (attn_dim**0.5) + if sparsity > 0.0: + lengths = generate_sparse_seq_len( + size=batch_size, + max_seq_len=max_uih_len, + sparsity=sparsity, + device=torch.device("cuda"), + ) + else: + lengths = torch.randint( + max_uih_len + 1, size=(batch_size,), device=torch.device("cuda") + ) + + num_targets = torch.randint( + max_targets + 1, size=(batch_size,), device=torch.device("cuda") + ) + lengths = lengths + num_targets + max_seq_len = max_uih_len + max_targets + if has_max_attn_len: + max_attn_len = random.randint(1, max_uih_len // 5) + else: + max_attn_len = 0 + seq_offsets = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + seq_offsets[1:] = torch.cumsum(lengths, dim=0) + + L = int(seq_offsets[-1].item()) + + x = ( + torch.empty((L, embedding_dim), dtype=dtype, device=torch.device("cuda")) + .uniform_(-0.1, 0.1) + .requires_grad_() + ) + norm_weight = ( + torch.empty((embedding_dim,), dtype=dtype, device=torch.device("cuda")) + .uniform_(-0.1, 0.1) + .requires_grad_() + ) + norm_bias = ( + torch.empty( + (embedding_dim,), + dtype=dtype, + device=torch.device("cuda"), + ) + .uniform_(-0.1, 0.1) + .requires_grad_() + ) + norm_eps = 1e-6 + uvqk_weight = ( + torch.empty( + ( + embedding_dim, + (hidden_dim * 2 + attn_dim * 2) * heads, + ), + dtype=dtype, + device=torch.device("cuda"), + ) + .uniform_(-0.1, 0.1) + .requires_grad_() + ) + uvqk_bias = ( + torch.empty( + (hidden_dim * 2 + attn_dim * 2) * heads, + dtype=dtype, + device=torch.device("cuda"), + ) + .uniform_(-0.1, 0.1) + .requires_grad_() + ) + + # ref implementation + ref_u, ref_attn_output, _, _ = hstu_preprocess_and_attention( + x=x, + norm_weight=norm_weight, + norm_bias=norm_bias, + norm_eps=norm_eps, + num_heads=heads, + attn_dim=attn_dim, + hidden_dim=hidden_dim, + uvqk_weight=uvqk_weight, + uvqk_bias=uvqk_bias, + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + attn_alpha=alpha, + causal=causal, + num_targets=num_targets, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + recompute_uvqk_in_backward=recompute_uvqk_in_backward, + recompute_normed_x_in_backward=recompute_normed_x_in_backward, + sort_by_length=sort_by_length, + kernel=ref_kernel, + ) + ref_out = ref_u + ref_attn_output + dout = torch.randn_like(ref_out) * 0.01 + ref_out.backward(dout) + + # pyre-ignore + ref_dx, x.grad = x.grad.clone(), None + ref_d_norm_weight, norm_weight.grad = norm_weight.grad.clone(), None + ref_d_norm_bias, norm_bias.grad = norm_bias.grad.clone(), None + ref_d_uvqk_weight, uvqk_weight.grad = uvqk_weight.grad.clone(), None + ref_d_uvqk_bias, uvqk_bias.grad = uvqk_bias.grad.clone(), None + + # real implementation + x = x.detach().clone().requires_grad_() + norm_weight = norm_weight.detach().clone().requires_grad_() + norm_bias = norm_bias.detach().clone().requires_grad_() + uvqk_weight = uvqk_weight.detach().clone().requires_grad_() + uvqk_bias = uvqk_bias.detach().clone().requires_grad_() + real_u, real_attn_output, _, _ = hstu_preprocess_and_attention( + x=x, + norm_weight=norm_weight, + norm_bias=norm_bias, + norm_eps=norm_eps, + num_heads=heads, + attn_dim=attn_dim, + hidden_dim=hidden_dim, + uvqk_weight=uvqk_weight, + uvqk_bias=uvqk_bias, + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + attn_alpha=alpha, + causal=causal, + num_targets=num_targets, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + recompute_uvqk_in_backward=recompute_uvqk_in_backward, + recompute_normed_x_in_backward=recompute_normed_x_in_backward, + sort_by_length=sort_by_length, + kernel=real_kernel, + ) + real_out = real_u + real_attn_output + torch.testing.assert_close( + ref_u, + real_u, + atol=atol, + rtol=rtol, + ) + torch.testing.assert_close( + ref_attn_output, + real_attn_output, + atol=atol, + rtol=rtol, + ) + if test_backward: + # real implementation + dout = dout.detach().clone() + real_out.backward(dout) + ( + real_dx, + real_d_norm_weight, + real_d_norm_bias, + real_d_uvqk_weight, + real_d_uvqk_bias, + ) = ( + x.grad.clone(), + norm_weight.grad.clone(), + norm_bias.grad.clone(), + uvqk_weight.grad.clone(), + uvqk_bias.grad.clone(), + ) + torch.testing.assert_close(ref_dx, real_dx, atol=atol, rtol=rtol) + torch.testing.assert_close( + ref_d_norm_weight, real_d_norm_weight, atol=atol, rtol=rtol + ) + torch.testing.assert_close( + ref_d_norm_bias, real_d_norm_bias, atol=atol, rtol=rtol + ) + torch.testing.assert_close( + ref_d_uvqk_weight, real_d_uvqk_weight, atol=atol, rtol=rtol + ) + torch.testing.assert_close( + ref_d_uvqk_bias, real_d_uvqk_bias, atol=atol, rtol=rtol + ) diff --git a/recommendation_v4/generative_recommenders/ops/tests/jagged_tensors_test.py b/recommendation_v4/generative_recommenders/ops/tests/jagged_tensors_test.py new file mode 100644 index 000000000..e03d68d0b --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/tests/jagged_tensors_test.py @@ -0,0 +1,963 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +import unittest +from typing import Optional + +import torch +from generative_recommenders.common import ( + generate_sparse_seq_len, + gpu_unavailable, + HammerKernel, + set_dev_mode, +) +from generative_recommenders.ops.jagged_tensors import ( + concat_2D_jagged, + concat_2D_jagged_multirow, + split_2D_jagged, + split_2D_jagged_multirow, +) +from hypothesis import given, settings, strategies as st, Verbosity + + +class JaggedTensorsTest(unittest.TestCase): + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore + @given( + batch_size=st.integers(2, 8), + max_len_a=st.integers(20, 100), + max_len_b=st.integers(20, 100), + D=st.integers(10, 30), + is_dense_a=st.sampled_from([True, False]), + is_dense_b=st.sampled_from([True, False]), + dtype=st.sampled_from( + [torch.bfloat16, torch.float32] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=20, + deadline=None, + ) + # pyre-ignore[2] + def test_split_2D_jagged_triton(self, *args, **kwargs) -> None: + self._test_split_2D_jagged( + *args, + **kwargs, + test_backward=True, + ref_kernel=HammerKernel.PYTORCH, + real_kernel=HammerKernel.TRITON, + ) + + def _test_split_2D_jagged( + self, + batch_size: int, + max_len_a: int, + max_len_b: int, + D: int, + is_dense_a: bool, + is_dense_b: bool, + test_backward: bool, + ref_kernel: HammerKernel, + real_kernel: HammerKernel, + dtype: torch.dtype = torch.float32, + skip_comparisons: bool = False, + ) -> None: + set_dev_mode(True) + from generative_recommenders.ops.jagged_tensors import split_2D_jagged + + max_seq_len = max_len_a + max_len_b + if not is_dense_a: + lengths_a = torch.randint( + 1, max_len_a + 1, size=(batch_size,), device=torch.device("cuda") + ) + offsets_a = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + offsets_a[1:] = torch.cumsum(lengths_a, dim=0) + total_len_a = int(offsets_a[-1].item()) + else: + offsets_a = None + total_len_a = batch_size * max_len_a + is_dense_b = False + if not is_dense_b: + lengths_b = torch.randint( + 1, max_len_b + 1, size=(batch_size,), device=torch.device("cuda") + ) + offsets_b = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + offsets_b[1:] = torch.cumsum(lengths_b, dim=0) + total_len_b = int(offsets_b[-1].item()) + else: + offsets_b = None + total_len_b = batch_size * max_len_b + values = ( + torch.empty( + (total_len_a + total_len_b, D), + dtype=dtype, + device=torch.device("cuda"), + ) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + + ref_values_a, ref_values_b = split_2D_jagged( + max_seq_len=max_len_a + max_len_b, + values=values, + max_len_left=max_len_a if is_dense_a else None, + max_len_right=max_len_b if is_dense_b else None, + offsets_left=offsets_a, + offsets_right=offsets_b, + kernel=ref_kernel, + ) + d_values_a = torch.randn_like(ref_values_a) + d_values_b = torch.randn_like(ref_values_b) + ref_values_a.backward(d_values_a, retain_graph=True) + ref_values_b.backward(d_values_b) + if skip_comparisons: + return + + assert values.grad is not None + ref_d_values, values.grad = values.grad.clone(), None + + values = values.detach().clone().requires_grad_() + real_values_a, real_values_b = split_2D_jagged( + max_seq_len=max_seq_len, + values=values, + max_len_left=max_len_a if is_dense_a else None, + max_len_right=max_len_b if is_dense_b else None, + offsets_left=offsets_a, + offsets_right=offsets_b, + kernel=real_kernel, + ) + torch.testing.assert_close(ref_values_a, real_values_a) + torch.testing.assert_close(ref_values_b, real_values_b) + + if test_backward: + d_values_a = d_values_a.detach().clone() + d_values_b = d_values_b.detach().clone() + real_values_a.backward(d_values_a, retain_graph=True) + real_values_b.backward(d_values_b) + real_d_values = values.grad.clone() + torch.testing.assert_close(ref_d_values, real_d_values) + + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore + @given( + batch_size=st.integers(2, 8), + max_len_a=st.integers(20, 100), + max_len_b=st.integers(20, 100), + D=st.integers(10, 30), + is_dense_a=st.sampled_from([True, False]), + is_dense_b=st.sampled_from([True, False]), + dtype=st.sampled_from( + [torch.bfloat16, torch.float32] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=20, + deadline=None, + ) + # pyre-ignore[2] + def test_concat_2D_jagged_triton(self, *args, **kwargs) -> None: + self._test_concat_2D_jagged( + *args, + **kwargs, + test_backward=True, + ref_kernel=HammerKernel.PYTORCH, + real_kernel=HammerKernel.TRITON, + ) + + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore + @given( + batch_size=st.sampled_from([130]), + max_len_a=st.sampled_from([32768]), + max_len_b=st.sampled_from([10]), + D=st.sampled_from([512]), + is_dense_a=st.sampled_from([True, False]), + is_dense_b=st.sampled_from([True, False]), + dtype=st.sampled_from( + [torch.bfloat16, torch.float32] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=20, + deadline=None, + ) + # pyre-ignore[2] + def test_concat_2D_jagged_large_tensor(self, *args, **kwargs) -> None: + self._test_concat_2D_jagged( + *args, + **kwargs, + test_backward=True, + skip_comparisons=True, + ref_kernel=HammerKernel.TRITON, + real_kernel=HammerKernel.TRITON, + ) + + def _test_concat_2D_jagged( + self, + batch_size: int, + max_len_a: int, + max_len_b: int, + D: int, + is_dense_a: bool, + is_dense_b: bool, + test_backward: bool, + ref_kernel: HammerKernel, + real_kernel: HammerKernel, + dtype: torch.dtype = torch.float32, + skip_comparisons: bool = False, + ) -> None: + set_dev_mode(True) + from generative_recommenders.ops.jagged_tensors import concat_2D_jagged + + if not is_dense_a: + lengths_a = torch.randint( + 1, max_len_a + 1, size=(batch_size,), device=torch.device("cuda") + ) + offsets_a = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + offsets_a[1:] = torch.cumsum(lengths_a, dim=0) + total_len_a = int(offsets_a[-1].item()) + else: + offsets_a = None + total_len_a = batch_size * max_len_a + values_a = ( + torch.empty( + (total_len_a, D), + dtype=dtype, + device=torch.device("cuda"), + ) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + + if not is_dense_b: + lengths_b = torch.randint( + 1, max_len_b + 1, size=(batch_size,), device=torch.device("cuda") + ) + offsets_b = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + offsets_b[1:] = torch.cumsum(lengths_b, dim=0) + total_len_b = int(offsets_b[-1].item()) + else: + offsets_b = None + total_len_b = batch_size * max_len_b + values_b = ( + torch.empty( + (total_len_b, D), + dtype=dtype, + device=torch.device("cuda"), + ) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + + ref_values = concat_2D_jagged( + max_seq_len=max_len_a + max_len_b, + values_left=values_a, + values_right=values_b, + max_len_left=max_len_a, + max_len_right=max_len_b, + offsets_left=offsets_a, + offsets_right=offsets_b, + kernel=ref_kernel, + ) + dout = torch.randn_like(ref_values) + ref_values.backward(dout) + if skip_comparisons: + return + + assert values_a.grad is not None + ref_d_a, values_a.grad = values_a.grad.clone(), None + assert values_b.grad is not None + ref_d_b, values_b.grad = values_b.grad.clone(), None + + values_a = values_a.detach().clone().requires_grad_() + values_b = values_b.detach().clone().requires_grad_() + dout = dout.detach().clone() + real_values = concat_2D_jagged( + max_seq_len=max_len_a + max_len_b, + values_left=values_a, + values_right=values_b, + max_len_left=max_len_a, + max_len_right=max_len_b, + offsets_left=offsets_a, + offsets_right=offsets_b, + kernel=real_kernel, + ) + torch.testing.assert_close(ref_values, real_values) + + if test_backward: + real_values.backward(dout) + real_d_a = values_a.grad.clone() + real_d_b = values_b.grad.clone() + torch.testing.assert_close(ref_d_a, real_d_a) + torch.testing.assert_close(ref_d_b, real_d_b) + + # pyre-ignore + @given( + batch_size=st.integers(2, 8), + max_uih_len=st.integers(20, 100), + max_l2_len=st.integers(10, 30), + contextual_seq_len=st.sampled_from([0, 10]), + max_targets=st.sampled_from([10, 20]), + D=st.integers(10, 30), + dtype=st.sampled_from( + [torch.bfloat16, torch.float32] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=20, + deadline=None, + ) + def test_hstu_split_l2_embeddings( + self, + batch_size: int, + max_uih_len: int, + max_l2_len: int, + contextual_seq_len: int, + max_targets: int, + D: int, + dtype: torch.dtype, + ) -> None: + set_dev_mode(True) + from generative_recommenders.ops.jagged_tensors import hstu_split_l2_embeddings + + max_seq_len = max_uih_len + max_targets + contextual_seq_len + num_targets = torch.randint( + 1, max_targets + 1, size=(batch_size,), device=torch.device("cuda") + ) + x_lengths = torch.randint( + 0, + max_uih_len + 1, + size=(batch_size,), + device=torch.device("cuda"), + ) + x_lengths = num_targets + x_lengths + contextual_seq_len + x_offsets = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + x_offsets[1:] = torch.cumsum(x_lengths, dim=0) + total_len = int(x_offsets[-1].item()) + x = ( + torch.empty( + (total_len, D), + dtype=dtype, + device=torch.device("cuda"), + ) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + prefix_lengths = x_lengths - max_l2_len - num_targets - contextual_seq_len + prefix_lengths = torch.clamp(prefix_lengths, min=0) + prefix_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum(prefix_lengths) + l2_offsets = x_offsets - prefix_offsets + ref_prefix_x, ref_l2_x = hstu_split_l2_embeddings( + max_seq_len=max_seq_len, + x=x, + prefix_offsets=prefix_offsets, + l2_offsets=l2_offsets, + contextual_seq_len=contextual_seq_len, + kernel=HammerKernel.PYTORCH, + ) + d_prefix_x = torch.randn_like(ref_prefix_x) + d_l2_x = torch.randn_like(ref_l2_x) + ref_prefix_x.backward(d_prefix_x, retain_graph=True) + ref_l2_x.backward(d_l2_x) + assert x.grad is not None + ref_d_x, x.grad = x.grad.clone(), None + x = x.detach().clone().requires_grad_() + real_prefix_x, real_l2_x = hstu_split_l2_embeddings( + max_seq_len=max_seq_len, + x=x, + prefix_offsets=prefix_offsets, + l2_offsets=l2_offsets, + contextual_seq_len=contextual_seq_len, + kernel=HammerKernel.TRITON, + ) + print(ref_prefix_x.shape, real_prefix_x.shape) + torch.testing.assert_close(ref_prefix_x, real_prefix_x) + torch.testing.assert_close(ref_l2_x, real_l2_x) + d_prefix_x = d_prefix_x.detach().clone() + d_l2_x = d_l2_x.detach().clone() + real_prefix_x.backward(d_prefix_x, retain_graph=True) + real_l2_x.backward(d_l2_x) + real_d_x = x.grad.clone() + torch.testing.assert_close(ref_d_x, real_d_x) + + # pyre-ignore + @given( + batch_size=st.integers(1, 1), + max_prefix_len=st.integers(10, 10), + max_l2_len=st.integers(5, 5), + contextual_seq_len=st.sampled_from([3]), + max_targets=st.sampled_from([2]), + D=st.integers(10, 10), + dtype=st.sampled_from( + [torch.bfloat16, torch.float32] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=20, + deadline=None, + ) + def test_hstu_concat_l2_embeddings( + self, + batch_size: int, + max_prefix_len: int, + max_l2_len: int, + contextual_seq_len: int, + max_targets: int, + D: int, + dtype: torch.dtype, + ) -> None: + set_dev_mode(True) + from generative_recommenders.ops.jagged_tensors import hstu_concat_l2_embeddings + + num_targets = torch.randint( + 1, max_targets + 1, size=(batch_size,), device=torch.device("cuda") + ) + l2_lengths = torch.randint( + 0, + max_l2_len + 1, + size=(batch_size,), + device=torch.device("cuda"), + ) + l2_lengths = num_targets + l2_lengths + contextual_seq_len + max_l2_len = max_l2_len + contextual_seq_len + max_targets + l2_offsets = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + l2_offsets[1:] = torch.cumsum(l2_lengths, dim=0) + total_l2_len = int(l2_offsets[-1].item()) + l2_x = ( + torch.empty( + (total_l2_len, D), + dtype=dtype, + device=torch.device("cuda"), + ) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + prefix_lengths = torch.randint( + 0, + max_prefix_len + 1, + size=(batch_size,), + device=torch.device("cuda"), + ) + prefix_lengths = torch.randint( + 0, + max_prefix_len + 1, + size=(batch_size,), + device=torch.device("cuda"), + ) + prefix_offsets = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + prefix_offsets[1:] = torch.cumsum(prefix_lengths, dim=0) + total_prefix_len = int(prefix_offsets[-1].item()) + prefix_x = ( + torch.empty( + (total_prefix_len, D), + dtype=dtype, + device=torch.device("cuda"), + ) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + ref_x = hstu_concat_l2_embeddings( + max_prefix_len=max_prefix_len, + prefix_x=prefix_x, + prefix_offsets=prefix_offsets, + max_l2_len=max_l2_len, + l2_x=l2_x, + l2_offsets=l2_offsets, + contextual_seq_len=contextual_seq_len, + kernel=HammerKernel.PYTORCH, + ) + dout = torch.randn_like(ref_x) + ref_x.backward(dout) + + assert prefix_x.grad is not None + ref_d_prefix_x, prefix_x.grad = prefix_x.grad.clone(), None + assert l2_x.grad is not None + ref_d_l2_x, l2_x.grad = l2_x.grad.clone(), None + + prefix_x = prefix_x.detach().clone().requires_grad_() + l2_x = l2_x.detach().clone().requires_grad_() + real_x = hstu_concat_l2_embeddings( + max_prefix_len=max_prefix_len, + prefix_x=prefix_x, + prefix_offsets=prefix_offsets, + max_l2_len=max_l2_len, + l2_x=l2_x, + l2_offsets=l2_offsets, + contextual_seq_len=contextual_seq_len, + kernel=HammerKernel.TRITON, + ) + torch.testing.assert_close(ref_x, real_x) + dout = dout.detach().clone() + real_x.backward(dout) + real_d_prefix_x = prefix_x.grad.clone() + real_d_l2_x = l2_x.grad.clone() + torch.testing.assert_close(ref_d_prefix_x, real_d_prefix_x) + torch.testing.assert_close(ref_d_l2_x, real_d_l2_x) + + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore + @given( + batch_size=st.integers(4, 8), + max_seq_len=st.integers(50, 500), + D=st.integers(20, 200), + K=st.integers(30, 200), + dtype=st.sampled_from( + [torch.float32, torch.bfloat16, torch.float16] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + contiguous=st.booleans(), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=20, + deadline=None, + ) + # pyre-ignore[2] + def test_jagged_dense_bmm_broadcast_add_triton(self, *args, **kwargs) -> None: + self._test_jagged_dense_bmm_broadcast_add( + *args, + **kwargs, + test_backward=True, + atol=None, + rtol=None, + ref_kernel=HammerKernel.PYTORCH, + real_kernel=HammerKernel.TRITON, + ) + + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore + @given( + batch_size=st.sampled_from([130]), + max_seq_len=st.sampled_from([32768]), + D=st.sampled_from([512]), + K=st.sampled_from([512]), + dtype=st.sampled_from([torch.float32, torch.bfloat16]), + contiguous=st.booleans(), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=1, + deadline=None, + ) + def test_jagged_dense_bmm_broadcast_add_triton_large_tensor( + self, + # pyre-fixme[2]: Parameter must be annotated. + *args, + **kwargs, # pyre-ignore[2] + ) -> None: + self._test_jagged_dense_bmm_broadcast_add( + *args, + **kwargs, + test_backward=True, + atol=None, + rtol=None, + ref_kernel=HammerKernel.TRITON, + real_kernel=HammerKernel.TRITON, + ) + + def _test_jagged_dense_bmm_broadcast_add( + self, + batch_size: int, + max_seq_len: int, + D: int, + K: int, + dtype: torch.dtype, + ref_kernel: HammerKernel, + real_kernel: HammerKernel, + test_backward: bool, + contiguous: bool = True, + atol: Optional[float] = None, + rtol: Optional[float] = None, + sparsity: float = -1, + ) -> None: + set_dev_mode(True) + torch.backends.cudnn.allow_tf32 = False + torch.backends.cuda.matmul.allow_tf32 = False + from generative_recommenders.ops.jagged_tensors import ( + jagged_dense_bmm_broadcast_add, + ) + + if sparsity > 0.0: + lengths = generate_sparse_seq_len( + size=batch_size, + max_seq_len=max_seq_len, + sparsity=sparsity, + device=torch.device("cuda"), + ).to(torch.int64) + else: + lengths = torch.randint( + max_seq_len + 1, size=(batch_size,), device=torch.device("cuda") + ) + # Test the edge case with an empty row + seq_offsets = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + seq_offsets[1:] = torch.cumsum(lengths, dim=0) + jagged_size = int(seq_offsets[-1].item()) + jagged = ( + torch.empty((jagged_size, D), dtype=dtype, device=torch.device("cuda")) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + dense = ( + torch.empty((batch_size, D, K), dtype=dtype, device=torch.device("cuda")) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + bias = ( + torch.empty((batch_size, K), dtype=dtype, device=torch.device("cuda")) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + + if not contiguous: + dense = ( + dense.transpose(1, 2) + .contiguous() + .transpose(1, 2) + .detach() + .clone() + .requires_grad_() + ) + + ref_out = jagged_dense_bmm_broadcast_add( + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + jagged=jagged, + dense=dense, + bias=bias, + kernel=ref_kernel, + ).to(jagged.dtype) + if test_backward: + dout = torch.randn_like(ref_out) * 0.01 + ref_out.backward(dout) + # pyre-ignore + ref_d_jagged, jagged.grad = jagged.grad.clone(), None + ref_d_dense, dense.grad = dense.grad.clone(), None + ref_d_bias, bias.grad = bias.grad.clone(), None + + jagged = jagged.detach().clone().requires_grad_() + dense = dense.detach().clone().requires_grad_() + bias = bias.detach().clone().requires_grad_() + real_out = jagged_dense_bmm_broadcast_add( + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + jagged=jagged, + dense=dense, + bias=bias, + kernel=real_kernel, + ) + torch.testing.assert_close( + ref_out, + real_out, + atol=atol, + rtol=rtol, + ) + if test_backward: + real_out.backward(dout) # pyre-ignore + real_d_jagged = jagged.grad.clone() + real_d_dense = dense.grad.clone() + real_d_bias = bias.grad.clone() + torch.testing.assert_close( + ref_d_jagged, # pyre-ignore + real_d_jagged, + atol=atol, + rtol=rtol, + ) + torch.testing.assert_close( + ref_d_dense, # pyre-ignore + real_d_dense, + atol=atol, + rtol=rtol, + ) + torch.testing.assert_close( + ref_d_bias, # pyre-ignore + real_d_bias, + atol=atol, + rtol=rtol, + ) + + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore + @given( + batch_size=st.integers(2, 8), + max_len_a=st.integers(20, 100), + max_len_b=st.integers(20, 100), + D=st.integers(10, 30), + is_dense_a=st.sampled_from([True, False]), + is_dense_b=st.sampled_from([True, False]), + dtype=st.sampled_from( + [torch.bfloat16, torch.float32] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=20, + deadline=None, + ) + # pyre-ignore[2] + def test_concat_2D_jagged_multirow_triton(self, *args, **kwargs) -> None: + self._test_concat_2D_jagged_multirow( + *args, + **kwargs, + test_backward=True, + ref_kernel=HammerKernel.PYTORCH, + real_kernel=HammerKernel.TRITON, + ) + + def _test_concat_2D_jagged_multirow( + self, + batch_size: int, + max_len_a: int, + max_len_b: int, + D: int, + is_dense_a: bool, + is_dense_b: bool, + test_backward: bool, + ref_kernel: HammerKernel, + real_kernel: HammerKernel, + dtype: torch.dtype = torch.float32, + ) -> None: + set_dev_mode(True) + + if not is_dense_a: + lengths_a = torch.randint( + 1, max_len_a + 1, size=(batch_size,), device=torch.device("cuda") + ) + offsets_a = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + offsets_a[1:] = torch.cumsum(lengths_a, dim=0) + total_len_a = int(offsets_a[-1].item()) + else: + offsets_a = None + total_len_a = batch_size * max_len_a + values_a = ( + torch.empty( + (total_len_a, D), + dtype=dtype, + device=torch.device("cuda"), + ) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + + if not is_dense_b: + lengths_b = torch.randint( + 1, max_len_b + 1, size=(batch_size,), device=torch.device("cuda") + ) + offsets_b = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + offsets_b[1:] = torch.cumsum(lengths_b, dim=0) + total_len_b = int(offsets_b[-1].item()) + else: + offsets_b = None + total_len_b = batch_size * max_len_b + values_b = ( + torch.empty( + (total_len_b, D), + dtype=dtype, + device=torch.device("cuda"), + ) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + + ref_values = concat_2D_jagged( + max_seq_len=max_len_a + max_len_b, + values_left=values_a, + values_right=values_b, + max_len_left=max_len_a, + max_len_right=max_len_b, + offsets_left=offsets_a, + offsets_right=offsets_b, + kernel=ref_kernel, + ) + dout = torch.randn_like(ref_values) * 0.1 + ref_values.backward(dout) + assert values_a.grad is not None + ref_d_a, values_a.grad = values_a.grad.clone(), None + assert values_b.grad is not None + ref_d_b, values_b.grad = values_b.grad.clone(), None + + values_a = values_a.detach().clone().requires_grad_() + values_b = values_b.detach().clone().requires_grad_() + dout = dout.detach().clone() + + real_values = concat_2D_jagged_multirow( + max_seq_len=max_len_a + max_len_b, + values_left=values_a, + values_right=values_b, + offsets_left=offsets_a, + offsets_right=offsets_b, + max_len_left=max_len_a, + max_len_right=max_len_b, + kernel=real_kernel, + ) + torch.testing.assert_close(ref_values, real_values) + if test_backward: + real_values.backward(dout) + real_d_a = values_a.grad.clone() + real_d_b = values_b.grad.clone() + torch.testing.assert_close(ref_d_a, real_d_a) + torch.testing.assert_close(ref_d_b, real_d_b) + + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore + @given( + batch_size=st.integers(2, 8), + max_len_a=st.integers(20, 100), + max_len_b=st.integers(20, 100), + D=st.integers(10, 30), + dtype=st.sampled_from( + [torch.bfloat16, torch.float32] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=20, + deadline=None, + ) + # pyre-ignore[2] + def test_split_2D_jagged_multirow_triton(self, *args, **kwargs) -> None: + self._test_split_2D_jagged_multirow( + *args, + **kwargs, + test_backward=True, + ref_kernel=HammerKernel.PYTORCH, + real_kernel=HammerKernel.TRITON, + ) + + def _test_split_2D_jagged_multirow( + self, + batch_size: int, + max_len_a: int, + max_len_b: int, + D: int, + test_backward: bool, + ref_kernel: HammerKernel, + real_kernel: HammerKernel, + dtype: torch.dtype = torch.float32, + ) -> None: + set_dev_mode(True) + + lengths_a = torch.randint( + 1, max_len_a + 1, size=(batch_size,), device=torch.device("cuda") + ) + offsets_a = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + offsets_a[1:] = torch.cumsum(lengths_a, dim=0) + + lengths_b = torch.randint( + 1, max_len_b + 1, size=(batch_size,), device=torch.device("cuda") + ) + offsets_b = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + offsets_b[1:] = torch.cumsum(lengths_b, dim=0) + + total_len = int(offsets_a[-1].item()) + int(offsets_b[-1].item()) + values = ( + torch.empty( + (total_len, D), + dtype=dtype, + device=torch.device("cuda"), + ) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + + ref_values_a, ref_values_b = split_2D_jagged( + max_seq_len=max_len_a + max_len_b, + values=values, + total_len_left=int(offsets_a[-1].item()), + total_len_right=int(offsets_b[-1].item()), + max_len_left=max_len_a, + max_len_right=max_len_b, + offsets_left=offsets_a, + offsets_right=offsets_b, + kernel=ref_kernel, + ) + d_values_a = torch.randn_like(ref_values_a) * 0.1 + d_values_b = torch.randn_like(ref_values_b) * 0.1 + ref_values_a.backward(d_values_a, retain_graph=True) + ref_values_b.backward(d_values_b) + assert values.grad is not None + ref_d_values, values.grad = values.grad.clone(), None + + values = values.detach().clone().requires_grad_() + d_values_a = d_values_a.detach().clone() + d_values_b = d_values_b.detach().clone() + + max_len_a_actual = int((offsets_a[1:] - offsets_a[:-1]).max().item()) + max_len_b_actual = int((offsets_b[1:] - offsets_b[:-1]).max().item()) + + real_values_a, real_values_b = split_2D_jagged_multirow( + max_seq_len=max_len_a + max_len_b, + values=values, + total_len_left=int(offsets_a[-1].item()), + total_len_right=int(offsets_b[-1].item()), + max_len_left=max_len_a_actual, + max_len_right=max_len_b_actual, + offsets_left=offsets_a, + offsets_right=offsets_b, + kernel=real_kernel, + ) + torch.testing.assert_close(ref_values_a, real_values_a) + torch.testing.assert_close(ref_values_b, real_values_b) + if test_backward: + real_values_a.backward(d_values_a, retain_graph=True) + real_values_b.backward(d_values_b) + real_d_values = values.grad.clone() + torch.testing.assert_close(ref_d_values, real_d_values) diff --git a/recommendation_v4/generative_recommenders/ops/tests/layer_norm_test.py b/recommendation_v4/generative_recommenders/ops/tests/layer_norm_test.py new file mode 100644 index 000000000..62540967a --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/tests/layer_norm_test.py @@ -0,0 +1,231 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +import copy +import unittest + +import torch +from generative_recommenders.common import gpu_unavailable, HammerKernel, set_dev_mode +from generative_recommenders.ops.layer_norm import ( + layer_norm, + LayerNorm, + swish_layer_norm, + SwishLayerNorm, +) +from hypothesis import given, settings, strategies as st, Verbosity + + +class LayerNormTest(unittest.TestCase): + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore[56] + @given( + N=st.sampled_from([4200000]), + D=st.sampled_from([512]), + is_swish=st.sampled_from([False]), + dtype=st.sampled_from( + [torch.bfloat16] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + ) + @settings( + deadline=None, + verbosity=Verbosity.verbose, + max_examples=1, + ) + # pyre-ignore[2] + def test_large_tensors(self, *args, **kwargs) -> None: + self._test_layernorm( + *args, + **kwargs, + ref_kernel=HammerKernel.TRITON, + real_kernel=HammerKernel.TRITON, + skip_comparisons=True, + ) + + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore[56] + @given( + N=st.integers(min_value=0, max_value=10000), + D=st.integers(min_value=32, max_value=512), + is_swish=st.sampled_from([True, False]), + dtype=st.sampled_from( + [torch.bfloat16, torch.float32] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + ) + @settings( + deadline=None, + verbosity=Verbosity.verbose, + max_examples=20, + ) + # pyre-ignore[2] + def test_ln(self, *args, **kwargs) -> None: + self._test_layernorm( + *args, + **kwargs, + ref_kernel=HammerKernel.PYTORCH, + real_kernel=HammerKernel.TRITON, + ) + + def _test_layernorm( + self, + N: int, + D: int, + is_swish: bool, + dtype: torch.dtype, + ref_kernel: HammerKernel, + real_kernel: HammerKernel, + skip_comparisons: bool = False, + ) -> None: + N = N // 4 * 4 + # enable auto-tuning to verify correctness of multi-row kernel + set_dev_mode(False) + x = ( + torch.empty((N, D), dtype=dtype, device=torch.device("cuda")) + .normal_(0.0, 1.0) + .requires_grad_() + ) + weight = ( + torch.empty((D,), device=torch.device("cuda")) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + bias = ( + torch.empty((D,), device=torch.device("cuda")) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + if is_swish: + layer_norm_func = swish_layer_norm + else: + layer_norm_func = layer_norm + # ref + ref_out = layer_norm_func(x, weight, bias, eps=1e-6, kernel=ref_kernel) + dout = torch.randn_like(ref_out) * 0.05 + ref_out.backward(dout) + if skip_comparisons: + return + # pyre-ignore[16] + ref_dx, x.grad = x.grad.detach().clone(), None + ref_dw, weight.grad = weight.grad.detach().clone(), None + ref_db, bias.grad = bias.grad.detach().clone(), None + # opt + x = x.detach().clone().requires_grad_() + weight = weight.detach().clone().requires_grad_() + bias = bias.detach().clone().requires_grad_() + opt_out = layer_norm_func(x, weight, bias, eps=1e-6, kernel=real_kernel) + dout = dout.detach().clone() + opt_out.backward(dout) + opt_dx, x.grad = x.grad.detach().clone(), None + opt_dw, weight.grad = weight.grad.detach().clone(), None + opt_db, bias.grad = bias.grad.detach().clone(), None + torch.testing.assert_close(ref_out, opt_out) + torch.testing.assert_close(ref_dx, opt_dx) + torch.testing.assert_close(ref_dw, opt_dw) + torch.testing.assert_close(ref_db, opt_db) + + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore[56] + @given( + N=st.integers(min_value=32, max_value=10000), + D=st.integers(min_value=32, max_value=512), + is_swish=st.sampled_from([True, False]), + dtype=st.sampled_from( + [torch.bfloat16, torch.float32] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + ) + @settings( + deadline=None, + verbosity=Verbosity.verbose, + max_examples=20, + ) + # pyre-ignore[2] + def test_modules(self, *args, **kwargs) -> None: + self._test_layer_norm_module( + *args, + **kwargs, + ref_kernel=HammerKernel.PYTORCH, + real_kernel=HammerKernel.TRITON, + ) + + def _test_layer_norm_module( + self, + N: int, + D: int, + is_swish: bool, + dtype: torch.dtype, + ref_kernel: HammerKernel, + real_kernel: HammerKernel, + skip_comparisons: bool = False, + ) -> None: + set_dev_mode(True) + x = ( + torch.empty((N, D), dtype=dtype, device=torch.device("cuda")) + .normal_(0.0, 1.0) + .requires_grad_() + ) + # ref + if is_swish: + ref_layer = SwishLayerNorm( + dim=D, + eps=1e-6, + ).to(device="cuda") + ref_layer._hammer_kernel = ref_kernel + else: + ref_layer = LayerNorm( + dim=D, + eps=1e-6, + ).to(device="cuda") + ref_layer._hammer_kernel = ref_kernel + opt_layer = copy.deepcopy(ref_layer) + opt_layer._hammer_kernel = real_kernel + + ref_out = ref_layer(x) + dout = torch.randn_like(ref_out) * 0.05 + ref_out.backward(dout) + if skip_comparisons: + return + # pyre-ignore[16] + ref_dx, x.grad = x.grad.detach().clone(), None + ref_dw = ref_layer.weight.grad.detach().clone() + ref_db = ref_layer.bias.grad.detach().clone() + # opt + x = x.detach().clone().requires_grad_() + opt_out = opt_layer(x) + dout = dout.detach().clone() + opt_out.backward(dout) + opt_dx, x.grad = x.grad.detach().clone(), None + opt_dw = opt_layer.weight.grad.detach().clone() + opt_db = opt_layer.bias.grad.detach().clone() + torch.testing.assert_close(ref_out, opt_out) + torch.testing.assert_close( + ref_dx, + opt_dx, + ) + torch.testing.assert_close( + ref_dw, + opt_dw, + ) + torch.testing.assert_close( + ref_db, + opt_db, + ) diff --git a/recommendation_v4/generative_recommenders/ops/tests/mm_test.py b/recommendation_v4/generative_recommenders/ops/tests/mm_test.py new file mode 100644 index 000000000..0695275e3 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/tests/mm_test.py @@ -0,0 +1,156 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +import unittest +from typing import Optional + +import torch +from generative_recommenders.common import gpu_unavailable, HammerKernel +from generative_recommenders.ops.mm import addmm +from hypothesis import given, settings, strategies as st, Verbosity + + +class MMlTest(unittest.TestCase): + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore[56] + @given( + M=st.integers(min_value=100, max_value=300), + N=st.integers(min_value=100, max_value=300), + K=st.sampled_from([128, 256]), + broadcast=st.booleans(), + dtype=st.sampled_from( + [torch.float32, torch.bfloat16, torch.float16] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=20, + deadline=None, + ) + def test_addmm( + self, + M: int, + N: int, + K: int, + broadcast: bool, + dtype: torch.dtype, + ) -> None: + self._test_addmm( + M=M, + N=N, + K=K, + broadcast=broadcast, + dtype=dtype, + kernel_type=HammerKernel.TRITON, + ) + + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore[56] + @given( + M=st.integers(min_value=100, max_value=300), + N=st.sampled_from([16, 48, 128, 144, 256]), + K=st.sampled_from([16, 48, 128, 144, 256]), + broadcast=st.booleans(), + dtype=st.sampled_from( + [torch.float32, torch.bfloat16, torch.float16] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=20, + deadline=None, + ) + def test_addmm_tma( + self, + M: int, + N: int, + K: int, + broadcast: bool, + dtype: torch.dtype, + ) -> None: + self._test_addmm( + M=M, + N=N, + K=K, + broadcast=broadcast, + dtype=dtype, + kernel_type=HammerKernel.TRITON, + ) + + def _test_addmm( + self, + M: int, + N: int, + K: int, + broadcast: bool, + dtype: torch.dtype, + kernel_type: HammerKernel, + atol: Optional[float] = None, + rtol: Optional[float] = None, + ) -> None: + # to enable more deterministic results. + torch.manual_seed(0) + + torch.backends.cudnn.allow_tf32 = False + torch.backends.cuda.matmul.allow_tf32 = False + + x: torch.Tensor = torch.rand((M, K), dtype=dtype, device="cuda").requires_grad_( + True + ) + w: torch.Tensor = torch.rand((K, N), dtype=dtype, device="cuda").requires_grad_( + True + ) + + if broadcast: + y: torch.Tensor = torch.rand( + (N), dtype=dtype, device="cuda" + ).requires_grad_(True) + else: + y: torch.Tensor = torch.rand( + (M, N), dtype=dtype, device="cuda" + ).requires_grad_(True) + + ref_z = addmm(y, x, w, kernel=HammerKernel.PYTORCH) + dz = torch.randn_like(ref_z) * 0.1 + ref_z.backward(dz) + # pyre-ignore[16] + ref_dx, x.grad = x.grad.detach().clone(), None + ref_dw, w.grad = w.grad.detach().clone(), None + ref_dy, y.grad = y.grad.detach().clone(), None + + x = x.detach().clone().requires_grad_(True) + w = w.detach().clone().requires_grad_(True) + y = y.detach().clone().requires_grad_(True) + real_z = addmm(y, x, w, kernel=kernel_type) + + torch.testing.assert_close(ref_z, real_z, atol=atol, rtol=rtol) + + # triton cc doesn't support backward + if kernel_type != HammerKernel.TRITON_CC: + real_z.backward(dz) + real_dx, x.grad = x.grad.detach().clone(), None + real_dw, w.grad = w.grad.detach().clone(), None + real_dy, y.grad = y.grad.detach().clone(), None + + torch.testing.assert_close(ref_dx, real_dx, atol=atol, rtol=rtol) + torch.testing.assert_close(ref_dw, real_dw, atol=atol, rtol=rtol) + torch.testing.assert_close(ref_dy, real_dy, atol=atol, rtol=rtol) diff --git a/recommendation_v4/generative_recommenders/ops/tests/position_test.py b/recommendation_v4/generative_recommenders/ops/tests/position_test.py new file mode 100644 index 000000000..ab9e1b415 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/tests/position_test.py @@ -0,0 +1,234 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +import unittest + +import torch +from generative_recommenders.common import ( + generate_sparse_seq_len, + gpu_unavailable, + HammerKernel, + set_dev_mode, +) +from hypothesis import given, settings, strategies as st, Verbosity + + +class PositionEmbeddingsTest(unittest.TestCase): + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore + @given( + alpha=st.sampled_from([0.5]), + max_uih_len=st.integers(50, 500), + max_contextual_seq_len=st.sampled_from([10]), + interleave_targets=st.sampled_from([True, False]), + batch_size=st.integers(16, 32), + D=st.integers(20, 200), + max_targets=st.sampled_from([10, 20]), + time_bucket_fn=st.sampled_from(["log"]), + dtype=st.sampled_from([torch.float32, torch.bfloat16, torch.float16]), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=20, + deadline=None, + ) + # pyre-ignore[2] + def test_add_timestamp_positional_embeddings_triton(self, *args, **kwargs) -> None: + self._test_add_timestamp_positional_embeddings( + *args, + **kwargs, + test_backward=True, + ref_kernel=HammerKernel.PYTORCH, + real_kernel=HammerKernel.TRITON, + ) + + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore + @given( + alpha=st.sampled_from([0.5]), + max_uih_len=st.sampled_from([32768]), + max_contextual_seq_len=st.sampled_from([10]), + interleave_targets=st.sampled_from([False]), + batch_size=st.sampled_from([130]), + D=st.sampled_from([512]), + max_targets=st.sampled_from([10]), + time_bucket_fn=st.sampled_from(["log"]), + dtype=st.sampled_from([torch.bfloat16]), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=1, + deadline=None, + ) + def test_add_timestamp_positional_embeddings_triton_large_tensor( + self, + # pyre-fixme[2]: Parameter must be annotated. + *args, + # pyre-ignore[2] + **kwargs, + ) -> None: + self._test_add_timestamp_positional_embeddings( + *args, + **kwargs, + test_backward=False, + ref_kernel=HammerKernel.TRITON, + real_kernel=HammerKernel.TRITON, + sparsity=1.0, + ) + + def _test_add_timestamp_positional_embeddings( + self, + alpha: float, + max_uih_len: int, + max_contextual_seq_len: int, + interleave_targets: bool, + batch_size: int, + D: int, + max_targets: int, + time_bucket_fn: str, + dtype: torch.dtype, + ref_kernel: HammerKernel, + real_kernel: HammerKernel, + test_backward: bool, + sparsity: float = -1, + ) -> None: + set_dev_mode(True) + from generative_recommenders.ops.position import ( + add_timestamp_positional_embeddings, + ) + + num_targets = torch.randint( + max_targets + 1, size=(batch_size,), device=torch.device("cuda") + ) + if sparsity > 0.0: + lengths = generate_sparse_seq_len( + size=batch_size, + max_seq_len=max_uih_len, + sparsity=sparsity, + device=torch.device("cuda"), + ).to(torch.int64) + else: + lengths = torch.randint( + max_uih_len + 1, size=(batch_size,), device=torch.device("cuda") + ) + seq_offsets = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + seq_offsets[1:] = torch.cumsum(lengths, dim=0) + max_seq_len = max_uih_len + max_targets + + position_embeddings_weight = ( + torch.empty( + (max_seq_len, D), dtype=torch.float32, device=torch.device("cuda") + ) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + num_time_buckets = 1000 + timestamp_embeddings_weight = ( + torch.empty( + (num_time_buckets, D), dtype=torch.float32, device=torch.device("cuda") + ) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + seq_embeddings = ( + torch.empty( + (int(seq_offsets[-1].item()), D), + dtype=dtype, + device=torch.device("cuda"), + ) + .uniform_(-0.1, 0.1) + .requires_grad_() + ) + timestamp_deltas: torch.Tensor = torch.randint( + 86400, + size=(batch_size, max_seq_len), + device="cuda", + ) + timestamps = timestamp_deltas.cumsum(dim=1) + mask = torch.arange(max_seq_len, device=timestamps.device) < lengths.unsqueeze( + 1 + ) + timestamps = timestamps[mask.view(batch_size, -1)] + + ref_out = add_timestamp_positional_embeddings( + alpha=alpha, + max_seq_len=max_seq_len, + max_contextual_seq_len=max_contextual_seq_len, + position_embeddings_weight=position_embeddings_weight, + timestamp_embeddings_weight=timestamp_embeddings_weight, + seq_offsets=seq_offsets, + seq_lengths=lengths, + seq_embeddings=seq_embeddings, + timestamps=timestamps, + num_targets=num_targets, + interleave_targets=interleave_targets, + time_bucket_fn=time_bucket_fn, + kernel=ref_kernel, + ) + dout = torch.randn_like(ref_out) * 0.01 + ref_out.backward(dout) + # pyre-ignore + ref_d_seq_embeddings, seq_embeddings.grad = seq_embeddings.grad.clone(), None + ref_d_position_embeddings_weight, position_embeddings_weight.grad = ( + position_embeddings_weight.grad.clone(), + None, + ) + ref_d_timestamp_embeddings_weight, timestamp_embeddings_weight.grad = ( + timestamp_embeddings_weight.grad.clone(), + None, + ) + + real_out = add_timestamp_positional_embeddings( + alpha=alpha, + max_seq_len=max_seq_len, + max_contextual_seq_len=max_contextual_seq_len, + position_embeddings_weight=position_embeddings_weight, + timestamp_embeddings_weight=timestamp_embeddings_weight, + seq_offsets=seq_offsets, + seq_lengths=lengths, + seq_embeddings=seq_embeddings, + timestamps=timestamps, + num_targets=num_targets, + interleave_targets=interleave_targets, + time_bucket_fn=time_bucket_fn, + kernel=real_kernel, + ) + + torch.testing.assert_close(ref_out, real_out) + if test_backward: + real_out.backward(dout) + real_d_seq_embeddings = seq_embeddings.grad.clone() + real_d_position_embeddings_weight = position_embeddings_weight.grad.clone() + real_d_timestamp_embeddings_weight = ( + timestamp_embeddings_weight.grad.clone() + ) + torch.testing.assert_close(ref_d_seq_embeddings, real_d_seq_embeddings) + torch.testing.assert_close( + ref_d_position_embeddings_weight, + real_d_position_embeddings_weight, + atol=5e-2 if dtype != torch.float32 else None, + rtol=2e-2 if dtype != torch.float32 else None, + ) + torch.testing.assert_close( + ref_d_timestamp_embeddings_weight, + real_d_timestamp_embeddings_weight, + atol=5e-2 if dtype != torch.float32 else None, + rtol=2e-2 if dtype != torch.float32 else None, + ) diff --git a/recommendation_v4/generative_recommenders/ops/tests/rms_norm_test.py b/recommendation_v4/generative_recommenders/ops/tests/rms_norm_test.py new file mode 100644 index 000000000..4e5c1a871 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/tests/rms_norm_test.py @@ -0,0 +1,229 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +import copy +import unittest + +import torch +from generative_recommenders.common import gpu_unavailable, HammerKernel, set_dev_mode +from generative_recommenders.ops.layer_norm import rms_norm, RMSNorm +from hammer.ops.triton.cc.utils import set_triton_cc_version +from hypothesis import given, settings, strategies as st, Verbosity + + +class LayerNormTest(unittest.TestCase): + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore[56] + @given( + N=st.sampled_from([2000000]), + D=st.sampled_from([512]), + dtype=st.sampled_from( + [torch.bfloat16] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + silu=st.booleans(), + ) + @settings( + deadline=None, + verbosity=Verbosity.verbose, + max_examples=1, + ) + # pyre-ignore[2] + def test_large_tensors(self, *args, **kwargs) -> None: + self._test_rms_norm( + *args, + **kwargs, + ref_kernel=HammerKernel.TRITON, + real_kernel=HammerKernel.TRITON, + skip_comparisons=True, + ) + + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore[56] + @given( + N=st.integers(min_value=0, max_value=10000), + D=st.integers(min_value=32, max_value=512), + dtype=st.sampled_from( + [torch.bfloat16, torch.float32] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + silu=st.booleans(), + ) + @settings( + deadline=None, + verbosity=Verbosity.verbose, + max_examples=50, + ) + # pyre-ignore[2] + def test_rms_norm(self, *args, **kwargs) -> None: + self._test_rms_norm( + *args, + **kwargs, + ref_kernel=HammerKernel.PYTORCH, + real_kernel=HammerKernel.TRITON, + ) + + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore[56] + @given( + N=st.integers(min_value=4, max_value=10000), + D=st.sampled_from([256, 512]), + dtype=st.sampled_from([torch.bfloat16, torch.float16]), + triton_cc_version=st.sampled_from(["", "repkg"]), + silu=st.just(False), + ) + @settings( + deadline=None, + verbosity=Verbosity.verbose, + max_examples=10, + ) + # pyre-ignore[2] + def test_rms_norm_triton_cc(self, triton_cc_version: str, *args, **kwargs) -> None: + set_triton_cc_version(triton_cc_version) + self._test_rms_norm( + *args, + **kwargs, + ref_kernel=HammerKernel.PYTORCH, + real_kernel=HammerKernel.TRITON_CC, + test_backward=False, + ) + + def _test_rms_norm( + self, + N: int, + D: int, + dtype: torch.dtype, + silu: bool, + ref_kernel: HammerKernel, + real_kernel: HammerKernel, + skip_comparisons: bool = False, + test_backward: bool = True, + ) -> None: + N = N // 4 * 4 + # enable auto-tuning to verify correctness of multi-row kernel + set_dev_mode(False) + x = ( + torch.empty((N, D), dtype=dtype, device=torch.device("cuda")) + .normal_(0.0, 1.0) + .requires_grad_() + ) + weight = ( + torch.empty((D,), device=torch.device("cuda")) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + # ref + ref_out = rms_norm(x, weight, eps=1e-6, silu=silu, kernel=ref_kernel) + opt_x = x.detach().clone().requires_grad_() + opt_weight = weight.detach().clone().requires_grad_() + opt_out = rms_norm(opt_x, opt_weight, eps=1e-6, silu=silu, kernel=real_kernel) + torch.testing.assert_close(ref_out, opt_out) + + if not test_backward: + return + + dout = torch.randn_like(ref_out) * 0.05 + ref_out.backward(dout) + if skip_comparisons: + return + # pyre-ignore[16] + ref_dx, x.grad = x.grad.detach().clone(), None + ref_dw, weight.grad = weight.grad.detach().clone(), None + # opt + dout = dout.detach().clone() + opt_out.backward(dout) + opt_dx, x.grad = opt_x.grad.detach().clone(), None + opt_dw, weight.grad = opt_weight.grad.detach().clone(), None + torch.testing.assert_close(ref_dx, opt_dx) + torch.testing.assert_close(ref_dw, opt_dw) + + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore[56] + @given( + N=st.integers(min_value=32, max_value=10000), + D=st.integers(min_value=32, max_value=512), + dtype=st.sampled_from( + [torch.bfloat16, torch.float32] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + ) + @settings( + deadline=None, + verbosity=Verbosity.verbose, + max_examples=50, + ) + # pyre-ignore[2] + def test_modules(self, *args, **kwargs) -> None: + self._test_rms_norm_module( + *args, + **kwargs, + ref_kernel=HammerKernel.PYTORCH, + real_kernel=HammerKernel.TRITON, + ) + + def _test_rms_norm_module( + self, + N: int, + D: int, + dtype: torch.dtype, + ref_kernel: HammerKernel, + real_kernel: HammerKernel, + skip_comparisons: bool = False, + ) -> None: + set_dev_mode(True) + x = ( + torch.empty((N, D), dtype=dtype, device=torch.device("cuda")) + .normal_(0.0, 1.0) + .requires_grad_() + ) + # ref + ref_layer = RMSNorm( + dim=D, + eps=1e-6, + ).to(device="cuda") + ref_layer._hammer_kernel = ref_kernel + opt_layer = copy.deepcopy(ref_layer) + opt_layer._hammer_kernel = real_kernel + + ref_out = ref_layer(x) + dout = torch.randn_like(ref_out) * 0.05 + ref_out.backward(dout) + if skip_comparisons: + return + # pyre-ignore[16] + ref_dx, x.grad = x.grad.detach().clone(), None + ref_dw = ref_layer.weight.grad.detach().clone() + # opt + x = x.detach().clone().requires_grad_() + opt_out = opt_layer(x) + dout = dout.detach().clone() + opt_out.backward(dout) + opt_dx, x.grad = x.grad.detach().clone(), None + opt_dw = opt_layer.weight.grad.detach().clone() + torch.testing.assert_close(ref_out.to(dtype), opt_out) + torch.testing.assert_close( + ref_dx, + opt_dx, + ) + torch.testing.assert_close( + ref_dw, + opt_dw, + ) diff --git a/recommendation_v4/generative_recommenders/ops/triton/triton_addmm.py b/recommendation_v4/generative_recommenders/ops/triton/triton_addmm.py new file mode 100644 index 000000000..915d85742 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton/triton_addmm.py @@ -0,0 +1,1706 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +#!/usr/bin/env python3 + + +import math +from typing import List, Optional, Tuple + +import torch + +# @manual=//triton:triton +import triton + +# @manual=//triton:triton +import triton.language as tl +from generative_recommenders.ops.utils import is_sm100_plus, maybe_register_custom_op + +try: + # @manual=//triton:triton + from triton.language.extra.subtile_ops import _split_n_2D +except ImportError: + _split_n_2D = None + +try: + # @manual=//triton:triton + import triton.language.extra.tlx as tlx # type: ignore + + HAS_TLX = True +except ImportError: + tlx = None + HAS_TLX = False + +from generative_recommenders.common import triton_autotune, triton_cc + +try: + # @manual=//triton:triton + from triton.tools.tensor_descriptor import TensorDescriptor + + TMA_AVAILABLE = True +except ImportError: + TMA_AVAILABLE = False + pass + + +ENABLE_FULL_TURNING_SPACE = False + + +def _use_meta_ws() -> bool: + """Check if Meta's warp specialization is available, enabled, and on SM100+.""" + return ( + is_sm100_plus() + and hasattr(triton, "knobs") + and hasattr(triton.knobs, "nvidia") + and triton.knobs.nvidia.use_meta_ws + ) + + +def _check_tma_alignment( + x: torch.Tensor, w: torch.Tensor, y: torch.Tensor, min_alignment: int = 16 +) -> bool: + """Check if tensors meet TMA alignment requirements. + + TMA (Tensor Memory Accelerator) on H100 requires: + 1. Base addresses to be 64-byte aligned + 2. Dimensions to be multiples of 64 for optimal performance + 3. Contiguous inner dimensions (stride=1) + + Args: + x: Input tensor [M, K] + w: Weight tensor [K, N] + y: Bias tensor [N] or [M, N] + min_alignment: Minimum alignment requirement (default: 64) + + Returns: + True if all tensors meet TMA alignment requirements + """ + _, K = x.shape + KB, N = w.shape + assert K == KB, f"incompatible dimensions {K}, {KB}" + + is_y_1d = y.dim() == 1 + NY = y.shape[0] if is_y_1d else y.shape[1] + assert N == NY, f"incompatible dimensions {N}, {NY}" + + return (K % min_alignment == 0) and (N % min_alignment == 0) + + +def _prune_persistent_autows_configs(configs, named_args, **kwargs): # noqa + if not _use_meta_ws(): + return configs + BROADCAST_Y = kwargs.get("BROADCAST_Y", False) + pruned = [] + for c in configs: + BLOCK_M = c.kwargs.get("BLOCK_M", 0) + BLOCK_N = c.kwargs.get("BLOCK_N", 0) + EPILOGUE_SUBTILE = c.kwargs.get("EPILOGUE_SUBTILE", 1) + DP = c.kwargs.get("DATA_PARTITION_FACTOR", 1) + # DATA_PARTITION_FACTOR=2 is only supported with BLOCK_M=256 + if DP == 2 and BLOCK_M != 256: + continue + if (BLOCK_N // EPILOGUE_SUBTILE) < 32: + continue + if BROADCAST_Y and (BLOCK_N // EPILOGUE_SUBTILE) < 64: + continue + pruned.append(c) + return pruned + + +def _prune_configs_for_tlx_persistent_addmm(configs, named_args, **kwargs): # noqa + M = named_args.get("M", 0) + N = named_args.get("N", 0) + BROADCAST_Y = kwargs.get("BROADCAST_Y", False) + + pruned = [] + for c in configs: + BLOCK_M = c.kwargs.get("BLOCK_M", 0) + BLOCK_N = c.kwargs.get("BLOCK_N", 0) + EPILOGUE_SUBTILE = c.kwargs.get("EPILOGUE_SUBTILE", 1) + NUM_MMA_GROUPS = c.kwargs.get("NUM_MMA_GROUPS", 1) + BLOCK_M_SPLIT = BLOCK_M // NUM_MMA_GROUPS + NUM_SMEM_BUFFERS = c.kwargs.get("NUM_SMEM_BUFFERS", 1) + + # Hardware constraint: Always make MMA tile 128. + if BLOCK_M_SPLIT != 128: + continue + + # BLOCK_N >= 64 required for PAIR_CTA + if BLOCK_N < 64: + continue + + # Subslice loads cannot be smaller than 32 + if (BLOCK_N // EPILOGUE_SUBTILE) < 32: + continue + + # TMA loads must be at least 128 bytes. With BROADCAST_Y + # this may not be met. + if BROADCAST_Y and (BLOCK_N // EPILOGUE_SUBTILE) < 64: + continue + + # Prune the support SMEM_BUFFER configurations. + if BROADCAST_Y: + if NUM_MMA_GROUPS == 1 and NUM_SMEM_BUFFERS != 5: + continue + elif NUM_MMA_GROUPS == 2 and NUM_SMEM_BUFFERS != 4: + continue + else: + if NUM_MMA_GROUPS == 1 and NUM_SMEM_BUFFERS != 4: + continue + elif NUM_MMA_GROUPS == 2 and NUM_SMEM_BUFFERS != 3: + continue + + # PAIR_CTA requires even number of M tiles and even total tiles + num_tiles_m = math.ceil(M / BLOCK_M) if BLOCK_M > 0 else 0 + num_tiles_n = math.ceil(N / BLOCK_N) if BLOCK_N > 0 else 0 + total_tiles = num_tiles_m * num_tiles_n + + # PAIR_CTA incompatible with MMA M=64 + pair_cta_compatible = ( + (num_tiles_m % 2 == 0) + and (total_tiles % 2 == 0) + and BLOCK_M == 128 + and NUM_MMA_GROUPS == 1 + ) + + c.kwargs["PAIR_CTA"] = pair_cta_compatible + # Set ctas_per_cga for CUDA-native cluster launch semantics (TLX way) + c.ctas_per_cga = (2, 1, 1) if pair_cta_compatible else None + + pruned.append(c) + return pruned + + +def get_mm_configs(pre_hook=None) -> List[triton.Config]: + if torch.version.hip: + if ENABLE_FULL_TURNING_SPACE: + block_m_range = [32, 64, 128, 256] + block_n_range = [32, 64, 128, 256] + block_k_range = [32, 64] + group_m_range = [4, 8] + matrix_instr_nonkdim_range = [16] + waves_per_eu_range = [0] + kpack_range = [1, 2] + num_warps_range = [4, 8] + num_stage_range = [2] if triton.__version__ >= "3.2.0" else [0] + else: + block_m_range = [256] + block_n_range = [256] + block_k_range = [32] + group_m_range = [8] + matrix_instr_nonkdim_range = [16] + waves_per_eu_range = [0] + kpack_range = [2] + num_warps_range = [8] + num_stage_range = [2] if triton.__version__ >= "3.2.0" else [0] + + return [ + triton.Config( + { + "BLOCK_M": block_m, + "BLOCK_N": block_n, + "BLOCK_K": block_k, + "GROUP_M": group_m, + "matrix_instr_nonkdim": matrix_instr_nonkdim, + "waves_per_eu": waves_per_eu, + "kpack": kpack, + }, + num_stages=num_stages, + num_warps=num_warps, + pre_hook=pre_hook, + ) + for block_m in block_m_range + for block_n in block_n_range + for block_k in block_k_range + for group_m in group_m_range + for matrix_instr_nonkdim in matrix_instr_nonkdim_range + for waves_per_eu in waves_per_eu_range + for kpack in kpack_range + for num_stages in num_stage_range + for num_warps in num_warps_range + ] + else: + block_m_range = [32, 64, 128, 256] + block_n_range = [32, 64, 128, 256] + block_k_range = [32, 64] + group_m_range = [4, 8] + # WARP_SPECIALIZE only works with num_warps >=4 + num_warps_range = [4, 8] if is_sm100_plus() else [2, 4, 8] + num_stage_range = [2, 3, 4, 5] + if ENABLE_FULL_TURNING_SPACE: + return [ + triton.Config( + { + "BLOCK_M": block_m, + "BLOCK_N": block_n, + "BLOCK_K": block_k, + "GROUP_M": group_m, + }, + num_stages=num_stages, + num_warps=num_warps, + pre_hook=pre_hook, + ) + for block_m in block_m_range + for block_n in block_n_range + for block_k in block_k_range + for group_m in group_m_range + for num_stages in num_stage_range + for num_warps in num_warps_range + ] + else: + configs = [ + triton.Config( + { + "BLOCK_M": 32, + "BLOCK_N": 64, + "BLOCK_K": 32, + "GROUP_M": 8, + }, + num_stages=5, + num_warps=2, + pre_hook=pre_hook, + ), + triton.Config( + { + "BLOCK_M": 128, + "BLOCK_N": 256, + "BLOCK_K": 64, + "GROUP_M": 8, + }, + num_stages=3, + num_warps=8, + pre_hook=pre_hook, + ), + triton.Config( + { + "BLOCK_M": 64, + "BLOCK_N": 256, + "BLOCK_K": 32, + "GROUP_M": 8, + }, + num_stages=4, + num_warps=4, + pre_hook=pre_hook, + ), + triton.Config( + { + "BLOCK_M": 128, + "BLOCK_N": 128, + "BLOCK_K": 32, + "GROUP_M": 8, + }, + num_stages=4, + num_warps=4, + pre_hook=pre_hook, + ), + triton.Config( + { + "BLOCK_M": 128, + "BLOCK_N": 64, + "BLOCK_K": 32, + "GROUP_M": 8, + }, + num_stages=4, + num_warps=4, + pre_hook=pre_hook, + ), + triton.Config( + { + "BLOCK_M": 64, + "BLOCK_N": 128, + "BLOCK_K": 32, + "GROUP_M": 8, + }, + num_stages=4, + num_warps=4, + pre_hook=pre_hook, + ), + triton.Config( + { + "BLOCK_M": 128, + "BLOCK_N": 32, + "BLOCK_K": 32, + "GROUP_M": 8, + }, + num_stages=4, + num_warps=4, + pre_hook=pre_hook, + ), + triton.Config( + { + "BLOCK_M": 64, + "BLOCK_N": 32, + "BLOCK_K": 32, + "GROUP_M": 8, + }, + num_stages=5, + num_warps=2, + pre_hook=pre_hook, + ), + ] + if is_sm100_plus(): + configs += [ + triton.Config( + { + "BLOCK_M": 128, + "BLOCK_N": 256, + "BLOCK_K": 64, + "GROUP_M": 8, + }, + num_stages=3, + num_warps=4, + pre_hook=pre_hook, + ), + ] + return [c for c in configs if c.num_warps >= 4] + + return configs + + +def _get_addmm_tma_ws_persistent_configs(pre_hook=None) -> List[triton.Config]: + """Get configs for _addmm_fwd_tma_ws_persistent (sm100+ TLX kernel). + + This kernel has unique requirements (warp specialization, PAIR_CTA, + EPILOGUE_SUBTILE) that don't apply to the other addmm kernels. + """ + if ENABLE_FULL_TURNING_SPACE: + block_m_range = [64, 128, 256] + block_n_range = [64, 128, 256] + block_k_range = [64, 128, 256] + group_m_range = [8] + num_warps_range = [4] + num_stage_range = [1] + epilogue_subtile_range = [1, 2, 4] + num_mma_groups_range = [1, 2] + return [ + triton.Config( + { + "BLOCK_M": block_m, + "BLOCK_N": block_n, + "BLOCK_K": block_k, + "GROUP_M": group_m, + "EPILOGUE_SUBTILE": epilogue_subtile, + "NUM_MMA_GROUPS": num_mma_groups, + "NUM_TMEM_BUFFERS": 1 if num_mma_groups == 2 else 2, + "NUM_SMEM_BUFFERS": num_smem_buffers, + }, + num_stages=num_stages, + num_warps=num_warps, + pre_hook=pre_hook, + ) + for block_m in block_m_range + for block_n in block_n_range + for block_k in block_k_range + for group_m in group_m_range + for num_stages in num_stage_range + for num_warps in num_warps_range + for epilogue_subtile in epilogue_subtile_range + for num_mma_groups in num_mma_groups_range + for num_smem_buffers in [3, 4, 5] + ] + else: + configs = [] + for block_m, block_n, block_k in [ + (128, 256, 64), + (128, 128, 64), + (64, 128, 64), + (64, 256, 64), + (128, 64, 128), + ]: + # Note: num_smem_buffers is pruned to 1 in + # the pruning function. + for num_smem_buffers in [3, 4, 5]: + configs.append( + triton.Config( + { + "BLOCK_M": block_m, + "BLOCK_N": block_n, + "BLOCK_K": block_k, + "GROUP_M": 8, + "EPILOGUE_SUBTILE": 1, + "NUM_MMA_GROUPS": 1, + "NUM_TMEM_BUFFERS": 2, + "NUM_SMEM_BUFFERS": num_smem_buffers, + }, + num_stages=1, + num_warps=4, + pre_hook=pre_hook, + ), + ) + return configs + + +def get_triton_persistent_configs(pre_hook=None) -> List[triton.Config]: + if not _use_meta_ws(): + configs = get_mm_configs(pre_hook=pre_hook) + for c in configs: + c.kwargs["DATA_PARTITION_FACTOR"] = 1 + c.kwargs["EPILOGUE_SUBTILE"] = 1 + return configs + # TODO: Prune configs to best configs. + return [ + triton.Config( # pyre-ignore[28] + { + "BLOCK_M": block_m, + "BLOCK_N": block_n, + "BLOCK_K": block_k, + "GROUP_M": 8, + "EPILOGUE_SUBTILE": subtile, + "DATA_PARTITION_FACTOR": DP, + }, + num_stages=num_stages, + num_warps=4, + pre_hook=pre_hook, + early_tma_store_lowering=1, + maxRegAutoWS=255, + ) + for block_m in [64, 128, 256] + for block_n in [64, 128, 256] + for block_k in [64, 128, 256] + for num_stages in [2, 3, 4] + for subtile in [1, 2, 4, 8] + for DP in [1, 2] + ] + + +@triton_cc( + annotations={ + "M": "i32", + "N": ("i32", 16), + "K": ("i32", 16), + "stride_xm": ("i32", 16), + "stride_xk": ("i32", 1), + "stride_wk": ("i32", 16), + "stride_wn": ("i32", 1), + "stride_ym": ("i32", 16), + "stride_yn": ("i32", 1), + "stride_zm": ("i32", 16), + "stride_zn": ("i32", 1), + }, +) +@triton_autotune( + configs=get_mm_configs(), + key=["N", "K"], +) +@triton.jit +def _addmm_fwd( + x_ptr, + w_ptr, + y_ptr, + z_ptr, + M, + N, + K, + stride_xm, + stride_xk, + stride_wk, + stride_wn, + stride_ym, + stride_yn, + stride_zm, + stride_zn, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, + GROUP_M: tl.constexpr, + ALLOW_TF32: tl.constexpr, + BROADCAST_Y: tl.constexpr, +): + pid_0, pid_1 = tl.program_id(axis=0), tl.program_id(axis=1) + pid = pid_0 * tl.num_programs(axis=1) + pid_1 + num_pid_m = tl.cdiv(M, BLOCK_M) + num_pid_n = tl.cdiv(N, BLOCK_N) + num_pid_in_group = GROUP_M * num_pid_n + group_id = pid // num_pid_in_group + first_pid_m = group_id * GROUP_M + group_size_m = min(num_pid_m - first_pid_m, GROUP_M) + pid_m = first_pid_m + (pid % group_size_m) + pid_n = (pid % num_pid_in_group) // group_size_m + + offs_m = tl.arange(0, BLOCK_M) + offs_k = tl.arange(0, BLOCK_K) + offs_n = tl.arange(0, BLOCK_N) + mask_m = (pid_m * BLOCK_M + offs_m)[:, None] < M + mask_n = (pid_n * BLOCK_N + offs_n)[None, :] < N + x_ptr += pid_m.to(tl.int64) * BLOCK_M * stride_xm + x_ptrs = x_ptr + (offs_m[:, None] * stride_xm + offs_k[None, :] * stride_xk) + w_ptr += pid_n.to(tl.int64) * BLOCK_N * stride_wn + w_ptrs = w_ptr + (offs_k[:, None] * stride_wk + offs_n[None, :] * stride_wn) + accumulator = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32) + for k in range(0, tl.cdiv(K, BLOCK_K)): + mask_k = offs_k[None, :] < K - k * BLOCK_K + x = tl.load(x_ptrs, mask=mask_k & mask_m, other=0.0) + mask_k = offs_k[:, None] < K - k * BLOCK_K + w = tl.load(w_ptrs, mask=mask_k & mask_n, other=0.0) + accumulator += tl.dot(x, w, allow_tf32=ALLOW_TF32) + x_ptrs += BLOCK_K * stride_xk + w_ptrs += BLOCK_K * stride_wk + + z_mask = mask_m & mask_n + if BROADCAST_Y: + # y is a vector, broadcast to add to z + y_ptr += pid_n.to(tl.int64) * BLOCK_N * stride_yn + y_ptrs = y_ptr + stride_yn * offs_n[None, :] + y = tl.load(y_ptrs, mask=mask_n) + else: + y_ptr += pid_m.to(tl.int64) * BLOCK_M * stride_ym + y_ptr += pid_n.to(tl.int64) * BLOCK_N * stride_yn + y_ptrs = y_ptr + stride_ym * offs_m[:, None] + stride_yn * offs_n[None, :] + y = tl.load(y_ptrs, mask=z_mask) + z = (accumulator + y.to(tl.float32)).to(z_ptr.dtype.element_ty) + z_ptr += pid_m.to(tl.int64) * BLOCK_M * stride_zm + z_ptr += pid_n.to(tl.int64) * BLOCK_N * stride_zn + z_ptrs = z_ptr + stride_zm * offs_m[:, None] + stride_zn * offs_n[None, :] + tl.store(z_ptrs, z, mask=z_mask) + + +def _addmm_tma_set_block_size_hook(nargs): + BLOCK_M = nargs["BLOCK_M"] + BLOCK_N = nargs["BLOCK_N"] + BLOCK_K = nargs["BLOCK_K"] + NUM_MMA_GROUPS = nargs.get("NUM_MMA_GROUPS", 1) + BLOCK_M_SPLIT = BLOCK_M // NUM_MMA_GROUPS + PAIR_CTA = nargs.get("PAIR_CTA", False) + nargs["x_desc"].block_shape = [BLOCK_M_SPLIT, BLOCK_K] + # In PAIR_CTA mode, each CTA loads BLOCK_N // 2 of W + if PAIR_CTA: + nargs["w_desc"].block_shape = [BLOCK_K, BLOCK_N // 2] + else: + nargs["w_desc"].block_shape = [BLOCK_K, BLOCK_N] + EPILOGUE_SUBTILE = nargs.get("EPILOGUE_SUBTILE", 1) + nargs["z_desc"].block_shape = [BLOCK_M_SPLIT, BLOCK_N // EPILOGUE_SUBTILE] + if nargs["BROADCAST_Y"]: + nargs["y_desc"].block_shape = [1, BLOCK_N // EPILOGUE_SUBTILE] + else: + nargs["y_desc"].block_shape = [BLOCK_M_SPLIT, BLOCK_N // EPILOGUE_SUBTILE] + + +@triton.jit +def _compute_pid(tile_id, num_pid_in_group, num_pid_m, GROUP_SIZE_M, NUM_SMS): + group_id = tile_id // num_pid_in_group + first_pid_m = group_id * GROUP_SIZE_M + group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M) + pid_m = first_pid_m + (tile_id % group_size_m) + pid_n = (tile_id % num_pid_in_group) // group_size_m + return pid_m, pid_n + + +@triton.jit +def _addmm_persistent_tile_body( + x_desc, + w_desc, + y_desc, + z_desc, + tile_id, + num_pid_in_group, + num_pid_m, + k_tiles, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, + GROUP_M: tl.constexpr, + ALLOW_TF32: tl.constexpr, + BROADCAST_Y: tl.constexpr, + NUM_SMS: tl.constexpr, + EPILOGUE_SUBTILE: tl.constexpr, + INNER_WARP_SPECIALIZE: tl.constexpr, +): + pid_m, pid_n = _compute_pid(tile_id, num_pid_in_group, num_pid_m, GROUP_M, NUM_SMS) + offs_xm = pid_m * BLOCK_M + offs_wn = pid_n * BLOCK_N + + accumulator = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32) + for k in tl.range(0, k_tiles, warp_specialize=INNER_WARP_SPECIALIZE): + offs_k = k * BLOCK_K + x = x_desc.load([offs_xm, offs_k]) + w = w_desc.load([offs_k, offs_wn]) + accumulator = tl.dot(x, w, accumulator, allow_tf32=ALLOW_TF32) + + # Epilogue subtiling breaks the store into multiple pieces to reduce + # shared memory consumption and allow higher stage counts. + tl.static_assert( + EPILOGUE_SUBTILE <= 8, + "EPILOGUE_SUBTILE > 8 is not supported", + ) + acc_subtiles = _split_n_2D(accumulator, EPILOGUE_SUBTILE) # pyre-ignore[16] + slice_size: tl.constexpr = BLOCK_N // EPILOGUE_SUBTILE + for i in tl.static_range(EPILOGUE_SUBTILE): + if BROADCAST_Y: + y_i = y_desc.load([0, offs_wn + i * slice_size]) + else: + y_i = y_desc.load([offs_xm, offs_wn + i * slice_size]) + z_i = (acc_subtiles[i] + y_i.to(tl.float32)).to(z_desc.dtype) + z_desc.store([offs_xm, offs_wn + i * slice_size], z_i) + + +@triton_autotune( + configs=get_triton_persistent_configs(pre_hook=_addmm_tma_set_block_size_hook), + key=["M", "N", "K", "WARP_SPECIALIZE"], + prune_configs_by={"early_config_prune": _prune_persistent_autows_configs}, +) +@triton.jit +def _addmm_fwd_tma_persistent( + x_desc, + w_desc, + y_desc, + z_desc, + M, + N, + K, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, + GROUP_M: tl.constexpr, + ALLOW_TF32: tl.constexpr, + BROADCAST_Y: tl.constexpr, + WARP_SPECIALIZE: tl.constexpr, + NUM_SMS: tl.constexpr, + EPILOGUE_SUBTILE: tl.constexpr, + DATA_PARTITION_FACTOR: tl.constexpr, + USE_META_WS: tl.constexpr, +): + start_pid = tl.program_id(axis=0) + num_pid_m = tl.cdiv(M, BLOCK_M) + num_pid_n = tl.cdiv(N, BLOCK_N) + k_tiles = tl.cdiv(K, BLOCK_K) + num_tiles = num_pid_m * num_pid_n + + num_pid_in_group = GROUP_M * num_pid_n + + if USE_META_WS: + # Some arguments are only available in FBexperimental. + # pyre-ignore[28]: smem_alloc_algo is FBexperimental + for tile_id in tl.range( + start_pid, + num_tiles, + NUM_SMS, + flatten=False, + warp_specialize=WARP_SPECIALIZE, + data_partition_factor=DATA_PARTITION_FACTOR, + smem_alloc_algo=1, + ): + _addmm_persistent_tile_body( + x_desc, + w_desc, + y_desc, + z_desc, + tile_id, + num_pid_in_group, + num_pid_m, + k_tiles, + BLOCK_M=BLOCK_M, + BLOCK_N=BLOCK_N, + BLOCK_K=BLOCK_K, + GROUP_M=GROUP_M, + ALLOW_TF32=ALLOW_TF32, + BROADCAST_Y=BROADCAST_Y, + NUM_SMS=NUM_SMS, + EPILOGUE_SUBTILE=EPILOGUE_SUBTILE, + INNER_WARP_SPECIALIZE=tl.constexpr(False), + ) + else: + # Pure OAI Triton version. + for tile_id in tl.range( + start_pid, num_tiles, NUM_SMS, flatten=True, warp_specialize=WARP_SPECIALIZE + ): + _addmm_persistent_tile_body( + x_desc, + w_desc, + y_desc, + z_desc, + tile_id, + num_pid_in_group, + num_pid_m, + k_tiles, + BLOCK_M=BLOCK_M, + BLOCK_N=BLOCK_N, + BLOCK_K=BLOCK_K, + GROUP_M=GROUP_M, + ALLOW_TF32=ALLOW_TF32, + BROADCAST_Y=BROADCAST_Y, + NUM_SMS=NUM_SMS, + EPILOGUE_SUBTILE=EPILOGUE_SUBTILE, + INNER_WARP_SPECIALIZE=WARP_SPECIALIZE, + ) + + +@triton_autotune( + configs=get_mm_configs(pre_hook=_addmm_tma_set_block_size_hook), + key=["N", "K"], +) +@triton.jit +def _addmm_fwd_tma_ws( + x_desc, + w_desc, + y_desc, + z_desc, + M, + N, + K, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, + GROUP_M: tl.constexpr, + ALLOW_TF32: tl.constexpr, + BROADCAST_Y: tl.constexpr, + NUM_SMEM_BUFFERS: tl.constexpr, +): + x_buffers = tlx.local_alloc((BLOCK_M, BLOCK_K), x_desc.dtype, NUM_SMEM_BUFFERS) + w_buffers = tlx.local_alloc((BLOCK_K, BLOCK_N), w_desc.dtype, NUM_SMEM_BUFFERS) + acc_tmem_buffer = tlx.local_alloc( + (BLOCK_M, BLOCK_N), tl.float32, tl.constexpr(1), tlx.storage_kind.tmem + ) + + if BROADCAST_Y: + y_buffer = tlx.local_alloc((1, BLOCK_N), y_desc.dtype, tl.constexpr(1)) + else: + y_buffer = tlx.local_alloc((BLOCK_M, BLOCK_N), y_desc.dtype, tl.constexpr(1)) + z_buffer = tlx.local_alloc((BLOCK_M, BLOCK_N), z_desc.dtype, tl.constexpr(1)) + + smem_full_bars = tlx.alloc_barriers(num_barriers=NUM_SMEM_BUFFERS, arrive_count=1) + smem_empty_bars = tlx.alloc_barriers(num_barriers=NUM_SMEM_BUFFERS, arrive_count=1) + y_load_barrier = tlx.alloc_barriers(num_barriers=1, arrive_count=1) + + with tlx.async_tasks(): + # Producer task: TMA loads + with tlx.async_task("default"): + pid_0, pid_1 = tl.program_id(axis=0), tl.program_id(axis=1) + pid = pid_0 * tl.num_programs(axis=1) + pid_1 + num_pid_m = tl.cdiv(M, BLOCK_M) + num_pid_n = tl.cdiv(N, BLOCK_N) + num_pid_in_group = GROUP_M * num_pid_n + group_id = pid // num_pid_in_group + first_pid_m = group_id * GROUP_M + group_size_m = min(num_pid_m - first_pid_m, GROUP_M) + pid_m = first_pid_m + (pid % group_size_m) + pid_n = (pid % num_pid_in_group) // group_size_m + + offs_xm = pid_m * BLOCK_M + offs_wn = pid_n * BLOCK_N + k_tiles = tl.cdiv(K, BLOCK_K) + + load_phase = 0 + for k in range(0, k_tiles): + buf = k % int(NUM_SMEM_BUFFERS) + + # Wait for buffer to be free + if k >= NUM_SMEM_BUFFERS: + tlx.barrier_wait(smem_empty_bars[buf], load_phase ^ 1) + + offs_k = k * BLOCK_K + tlx.barrier_expect_bytes( + smem_full_bars[buf], + 2 * (BLOCK_M * BLOCK_K + BLOCK_K * BLOCK_N), + ) + tlx.async_descriptor_load( + x_desc, x_buffers[buf], [offs_xm, offs_k], smem_full_bars[buf] + ) + tlx.async_descriptor_load( + w_desc, w_buffers[buf], [offs_k, offs_wn], smem_full_bars[buf] + ) + + load_phase = load_phase ^ (buf == NUM_SMEM_BUFFERS - 1) + + # Consumer task: async_dot MMA + with tlx.async_task(num_warps=4, num_regs=232): + pid_0, pid_1 = tl.program_id(axis=0), tl.program_id(axis=1) + pid = pid_0 * tl.num_programs(axis=1) + pid_1 + num_pid_m = tl.cdiv(M, BLOCK_M) + num_pid_n = tl.cdiv(N, BLOCK_N) + num_pid_in_group = GROUP_M * num_pid_n + group_id = pid // num_pid_in_group + first_pid_m = group_id * GROUP_M + group_size_m = min(num_pid_m - first_pid_m, GROUP_M) + pid_m = first_pid_m + (pid % group_size_m) + pid_n = (pid % num_pid_in_group) // group_size_m + + offs_xm = pid_m * BLOCK_M + offs_wn = pid_n * BLOCK_N + k_tiles = tl.cdiv(K, BLOCK_K) + + # Start async load of y early + y_buf_view = tlx.local_view(y_buffer, 0) + y_load_bar = tlx.local_view(y_load_barrier, 0) + if BROADCAST_Y: + tlx.barrier_expect_bytes(y_load_bar, 1 * BLOCK_N * 2) + tlx.async_descriptor_load(y_desc, y_buf_view, [0, offs_wn], y_load_bar) + else: + tlx.barrier_expect_bytes(y_load_bar, BLOCK_M * BLOCK_N * 2) + tlx.async_descriptor_load( + y_desc, y_buf_view, [offs_xm, offs_wn], y_load_bar + ) + + dot_phase = 0 + for k in range(0, k_tiles): + buf = k % int(NUM_SMEM_BUFFERS) + tlx.barrier_wait(smem_full_bars[buf], dot_phase) + + tlx.async_dot( + x_buffers[buf], + w_buffers[buf], + acc_tmem_buffer[0], + use_acc=k > 0, + mBarriers=[smem_empty_bars[buf]], + out_dtype=tl.float32, + ) + + dot_phase = dot_phase ^ (buf == NUM_SMEM_BUFFERS - 1) + + last_buf = (k_tiles - 1) % NUM_SMEM_BUFFERS + last_dot_phase = dot_phase ^ (last_buf == NUM_SMEM_BUFFERS - 1) + tlx.barrier_wait(smem_empty_bars[last_buf], last_dot_phase) + + tmem_result = tlx.local_load(acc_tmem_buffer[0]) + + tlx.barrier_wait(y_load_bar, 0) + y = tlx.local_load(y_buf_view) + + z = (tmem_result + y.to(tl.float32)).to(z_desc.dtype) + z_buf_view = tlx.local_view(z_buffer, 0) + tlx.local_store(z_buf_view, z) + tlx.async_descriptor_store(z_desc, z_buf_view, [offs_xm, offs_wn]) + tlx.async_descriptor_store_wait(0) + + +@triton_autotune( + configs=_get_addmm_tma_ws_persistent_configs( + pre_hook=_addmm_tma_set_block_size_hook + ), + key=["M", "N", "K"], + prune_configs_by={"early_config_prune": _prune_configs_for_tlx_persistent_addmm}, +) +@triton.jit +def _addmm_fwd_tma_ws_persistent( + x_desc, + w_desc, + y_desc, + z_desc, + M, + N, + K, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, + GROUP_M: tl.constexpr, + ALLOW_TF32: tl.constexpr, + BROADCAST_Y: tl.constexpr, + NUM_SMEM_BUFFERS: tl.constexpr, + NUM_TMEM_BUFFERS: tl.constexpr, + NUM_SMS: tl.constexpr, + EPILOGUE_SUBTILE: tl.constexpr, + PAIR_CTA: tl.constexpr, + NUM_MMA_GROUPS: tl.constexpr, +): + BLOCK_M_SPLIT: tl.constexpr = BLOCK_M // NUM_MMA_GROUPS + + # Allocate buffers once for all tiles + x_buffers = tlx.local_alloc( + (BLOCK_M_SPLIT, BLOCK_K), x_desc.dtype, NUM_SMEM_BUFFERS * NUM_MMA_GROUPS + ) + # In pair CTA mode, each CTA only needs to load half of W + if PAIR_CTA: + w_buffers = tlx.local_alloc( + (BLOCK_K, BLOCK_N // 2), w_desc.dtype, NUM_SMEM_BUFFERS + ) + else: + w_buffers = tlx.local_alloc((BLOCK_K, BLOCK_N), w_desc.dtype, NUM_SMEM_BUFFERS) + tmem_buffers = tlx.local_alloc( + (BLOCK_M_SPLIT, BLOCK_N), + tl.float32, + NUM_TMEM_BUFFERS * NUM_MMA_GROUPS, + tlx.storage_kind.tmem, + ) + slice_size: tl.constexpr = BLOCK_N // EPILOGUE_SUBTILE + + Y_Z_SHARED: tl.constexpr = NUM_MMA_GROUPS == 2 and not BROADCAST_Y + if Y_Z_SHARED: + NUM_Z_BUFFERS: tl.constexpr = EPILOGUE_SUBTILE * NUM_MMA_GROUPS + else: + NUM_Z_BUFFERS: tl.constexpr = NUM_MMA_GROUPS + + if Y_Z_SHARED: + bias_storage_alias = tlx.storage_alias_spec(storage=tlx.storage_kind.smem) + y_buffers = tlx.local_alloc( + (BLOCK_M_SPLIT, slice_size), + y_desc.dtype, + NUM_Z_BUFFERS, + reuse=bias_storage_alias, + ) + z_buffers = tlx.local_alloc( + (BLOCK_M_SPLIT, slice_size), + z_desc.dtype, + NUM_Z_BUFFERS, + reuse=bias_storage_alias, + ) + # Define y and z to share a single buffer + bias_storage_alias.set_buffer_overlap( + tlx.reuse_group( + y_buffers, + z_buffers, + group_type=tlx.reuse_group_type.shared, + ) + ) + else: + if BROADCAST_Y: + y_buffers = tlx.local_alloc( + (1, slice_size), y_desc.dtype, EPILOGUE_SUBTILE * NUM_MMA_GROUPS + ) + else: + y_buffers = tlx.local_alloc( + (BLOCK_M_SPLIT, slice_size), + y_desc.dtype, + EPILOGUE_SUBTILE * NUM_MMA_GROUPS, + ) + z_buffers = tlx.local_alloc( + (BLOCK_M_SPLIT, slice_size), z_desc.dtype, NUM_Z_BUFFERS + ) + + cluster_cta_rank = tlx.cluster_cta_rank() + pred_cta0 = cluster_cta_rank == 0 + if PAIR_CTA: + cta_bars = tlx.alloc_barriers( + num_barriers=NUM_SMEM_BUFFERS * NUM_MMA_GROUPS, arrive_count=2 + ) + + # Barriers for producer <-> MMA (separate X and W barriers) + x_smem_full_bars = tlx.alloc_barriers( + num_barriers=NUM_SMEM_BUFFERS * NUM_MMA_GROUPS, arrive_count=1 + ) + x_smem_empty_bars = tlx.alloc_barriers( + num_barriers=NUM_SMEM_BUFFERS * NUM_MMA_GROUPS, arrive_count=1 + ) + w_smem_full_bars = tlx.alloc_barriers(num_barriers=NUM_SMEM_BUFFERS, arrive_count=1) + # Barriers for MMA <-> Epilogue + tmem_full_bars = tlx.alloc_barriers( + num_barriers=NUM_TMEM_BUFFERS * NUM_MMA_GROUPS, arrive_count=1 + ) + tmem_empty_bars = tlx.alloc_barriers( + num_barriers=NUM_TMEM_BUFFERS * NUM_MMA_GROUPS, arrive_count=1 + ) + # Barriers for producer <-> Epilogue + # y_load_bar: producer signals when y data is ready + # y_empty_bar: epilogue signals when done using y buffer + y_load_bars = tlx.alloc_barriers( + num_barriers=EPILOGUE_SUBTILE * NUM_MMA_GROUPS, arrive_count=1 + ) + y_empty_bars = tlx.alloc_barriers( + num_barriers=EPILOGUE_SUBTILE * NUM_MMA_GROUPS, arrive_count=1 + ) + z_load_bars = tlx.alloc_barriers(num_barriers=NUM_Z_BUFFERS, arrive_count=1) + z_empty_bars = tlx.alloc_barriers(num_barriers=NUM_Z_BUFFERS, arrive_count=1) + + with tlx.async_tasks(): + # Epilogue consumer: waits for Y from producer, adds bias, stores to SMEM. + with tlx.async_task("default"): + start_pid = tl.program_id(axis=0) + num_pid_m = tl.cdiv(M, BLOCK_M) + if PAIR_CTA: + # Round up to even for proper CTA pairing + num_pid_m = (num_pid_m + 1) // 2 * 2 + num_pid_n = tl.cdiv(N, BLOCK_N) + num_tiles = num_pid_m * num_pid_n + + tmem_read_phase = 0 + cur_tmem_buf = 0 + y_load_phase = 0 + z_load_phase = 0 + + z_idx = 0 + for _ in range(start_pid, num_tiles, NUM_SMS): + for slice_id in tl.static_range(EPILOGUE_SUBTILE): + for group_id in tl.static_range(NUM_MMA_GROUPS): + buf_idx = group_id * NUM_TMEM_BUFFERS + cur_tmem_buf + acc_tmem = tmem_buffers[buf_idx] + if slice_id == 0: + # Wait for MMA to finish computing this group + tlx.barrier_wait(tmem_full_bars[buf_idx], tmem_read_phase) + + # Load result from TMEM and add bias + acc_subslice = tlx.subslice( + acc_tmem, slice_id * slice_size, slice_size + ) + result = tlx.local_load(acc_subslice) + if slice_id == EPILOGUE_SUBTILE - 1: + # Signal MMA that this TMEM buffer is now free + tlx.barrier_arrive(tmem_empty_bars[buf_idx], 1) + + y_idx = slice_id * NUM_MMA_GROUPS + group_id + y_buf_view = tlx.local_view(y_buffers, y_idx) + y_full = tlx.local_view(y_load_bars, y_idx) + tlx.barrier_wait(y_full, y_load_phase) + y = tlx.local_load(y_buf_view) + # If Y and Z are not shared signal we can load the next bias. + if not Y_Z_SHARED: + y_empty = tlx.local_view(y_empty_bars, y_idx) + tlx.barrier_arrive(y_empty, 1) + z = (result + y.to(tl.float32)).to(z_desc.dtype) + z_buf_view = tlx.local_view(z_buffers, z_idx) + # If Y and Z are not shared wait for Z to be empty. + # If there are shared this already guaranteed. + if not Y_Z_SHARED: + z_empty = tlx.local_view(z_empty_bars, z_idx) + tlx.barrier_wait(z_empty, z_load_phase ^ 1) + tlx.local_store(z_buf_view, z) + z_full = tlx.local_view(z_load_bars, z_idx) + tlx.barrier_arrive(z_full, 1) + z_load_phase = z_load_phase ^ (z_idx == (NUM_Z_BUFFERS - 1)) + # pyre-ignore[58] + z_idx = (z_idx + 1) % NUM_Z_BUFFERS + + tmem_read_phase = tmem_read_phase ^ ( + cur_tmem_buf == int(NUM_TMEM_BUFFERS) - 1 + ) + y_load_phase = y_load_phase ^ 1 + + cur_tmem_buf = (cur_tmem_buf + 1) % int(NUM_TMEM_BUFFERS) + + # MMA consumer: performs matrix multiplication + with tlx.async_task(num_warps=1, num_regs=24): + start_pid = tl.program_id(axis=0) + num_pid_m = tl.cdiv(M, BLOCK_M) + if PAIR_CTA: + # Round up to even for proper CTA pairing + num_pid_m = (num_pid_m + 1) // 2 * 2 + num_pid_n = tl.cdiv(N, BLOCK_N) + num_pid_in_group = GROUP_M * num_pid_n + num_tiles = num_pid_m * num_pid_n + k_tiles = tl.cdiv(K, BLOCK_K) + + dot_phase = 0 + tmem_write_phase = 1 + cur_tmem_buf = 0 + processed_k_iters = 0 + + for tile_id in range(start_pid, num_tiles, NUM_SMS): + pid_m, pid_n = _compute_pid( + tile_id, num_pid_in_group, num_pid_m, GROUP_M, NUM_SMS + ) + + # First K iteration (peeled): use_acc=False + buf = processed_k_iters % int(NUM_SMEM_BUFFERS) + tlx.barrier_wait(w_smem_full_bars[buf], dot_phase) + + for group_id in tl.static_range(NUM_MMA_GROUPS): + a_buf = group_id * NUM_SMEM_BUFFERS + buf + acc_buf = group_id * NUM_TMEM_BUFFERS + cur_tmem_buf + + tlx.barrier_wait(x_smem_full_bars[a_buf], dot_phase) + + # Wait for epilogue to finish with this TMEM buffer + tlx.barrier_wait(tmem_empty_bars[acc_buf], tmem_write_phase) + + if PAIR_CTA: + # pyre-ignore[61] + tlx.barrier_arrive(cta_bars[a_buf], 1, remote_cta_rank=0) + # pyre-ignore[61] + tlx.barrier_wait( + # pyre-ignore[61] + cta_bars[a_buf], + phase=dot_phase, + pred=pred_cta0, + ) + + tlx.async_dot( + x_buffers[a_buf], + w_buffers[buf], + tmem_buffers[acc_buf], + use_acc=False, + mBarriers=[x_smem_empty_bars[a_buf]], + two_ctas=PAIR_CTA, + out_dtype=tl.float32, + ) + + dot_phase = dot_phase ^ (buf == int(NUM_SMEM_BUFFERS) - 1) + + # Remaining K iterations: use_acc=True + for k in range(1, k_tiles): + buf = (processed_k_iters + k) % int(NUM_SMEM_BUFFERS) + tlx.barrier_wait(w_smem_full_bars[buf], dot_phase) + + for group_id in tl.static_range(NUM_MMA_GROUPS): + a_buf = group_id * NUM_SMEM_BUFFERS + buf + acc_buf = group_id * NUM_TMEM_BUFFERS + cur_tmem_buf + + tlx.barrier_wait(x_smem_full_bars[a_buf], dot_phase) + + if PAIR_CTA: + # pyre-ignore[61] + tlx.barrier_arrive(cta_bars[a_buf], 1, remote_cta_rank=0) + # pyre-ignore[61] + tlx.barrier_wait( + # pyre-ignore[61] + cta_bars[a_buf], + phase=dot_phase, + # pyre-ignore[61] + pred=pred_cta0, + ) + + tlx.async_dot( + x_buffers[a_buf], + w_buffers[buf], + tmem_buffers[acc_buf], + use_acc=True, + mBarriers=[x_smem_empty_bars[a_buf]], + two_ctas=PAIR_CTA, + out_dtype=tl.float32, + ) + + dot_phase = dot_phase ^ (buf == int(NUM_SMEM_BUFFERS) - 1) + + # Wait for last MMA to complete and signal epilogue + last_buf = (processed_k_iters + k_tiles - 1) % int(NUM_SMEM_BUFFERS) + last_dot_phase = dot_phase ^ (last_buf == int(NUM_SMEM_BUFFERS) - 1) + for group_id in tl.static_range(NUM_MMA_GROUPS): + a_buf = group_id * NUM_SMEM_BUFFERS + last_buf + tlx.barrier_wait(x_smem_empty_bars[a_buf], last_dot_phase) + acc_buf = group_id * NUM_TMEM_BUFFERS + cur_tmem_buf + # Signal epilogue that result is ready + tlx.barrier_arrive(tmem_full_bars[acc_buf], 1) + + tmem_write_phase = tmem_write_phase ^ ( + cur_tmem_buf == int(NUM_TMEM_BUFFERS) - 1 + ) + cur_tmem_buf = (cur_tmem_buf + 1) % int(NUM_TMEM_BUFFERS) + processed_k_iters += k_tiles + + # Producer: TMA loads for X, W, and Y + with tlx.async_task(num_warps=1, num_regs=24): + start_pid = tl.program_id(axis=0) + num_pid_m = tl.cdiv(M, BLOCK_M) + if PAIR_CTA: + # Round up to even for proper CTA pairing + num_pid_m = (num_pid_m + 1) // 2 * 2 + num_pid_n = tl.cdiv(N, BLOCK_N) + num_pid_in_group = GROUP_M * num_pid_n + num_tiles = num_pid_m * num_pid_n + k_tiles = tl.cdiv(K, BLOCK_K) + + load_phase = 0 + y_load_phase = 0 + processed_k_iters = 0 + + for tile_id in range(start_pid, num_tiles, NUM_SMS): + pid_m, pid_n = _compute_pid( + tile_id, num_pid_in_group, num_pid_m, GROUP_M, NUM_SMS + ) + offs_xm = pid_m * BLOCK_M + # Full tile offset for y loading (both CTAs use same y) + offs_wn_full = pid_n * BLOCK_N + # Split W into two parts so each CTA has different offset + if PAIR_CTA: + # pyre-ignore[61] + offs_wn = pid_n * BLOCK_N + cluster_cta_rank * (BLOCK_N // 2) + else: + offs_wn = pid_n * BLOCK_N + + for k in range(0, k_tiles): + buf = (processed_k_iters + k) % int(NUM_SMEM_BUFFERS) + offs_k = k * BLOCK_K + + # Load X for group 0 + a_buf = buf # 0 * NUM_SMEM_BUFFERS + buf + tlx.barrier_wait(x_smem_empty_bars[a_buf], load_phase ^ 1) + tlx.barrier_expect_bytes( + x_smem_full_bars[a_buf], + 2 * BLOCK_M_SPLIT * BLOCK_K, + ) + tlx.async_descriptor_load( + x_desc, + x_buffers[a_buf], + [offs_xm, offs_k], + x_smem_full_bars[a_buf], + ) + + # Load W (wait for last group's x_empty to know W is free) + last_a_buf = (NUM_MMA_GROUPS - 1) * NUM_SMEM_BUFFERS + buf + tlx.barrier_wait(x_smem_empty_bars[last_a_buf], load_phase ^ 1) + if PAIR_CTA: + tlx.barrier_expect_bytes( + w_smem_full_bars[buf], + 2 * BLOCK_K * (BLOCK_N // 2), + ) + else: + tlx.barrier_expect_bytes( + w_smem_full_bars[buf], + 2 * BLOCK_K * BLOCK_N, + ) + tlx.async_descriptor_load( + w_desc, + w_buffers[buf], + [offs_k, offs_wn], + w_smem_full_bars[buf], + ) + + # Load X for remaining groups + for group_id in tl.static_range(1, NUM_MMA_GROUPS): + a_buf = group_id * NUM_SMEM_BUFFERS + buf + tlx.barrier_wait(x_smem_empty_bars[a_buf], load_phase ^ 1) + offs_xm2 = offs_xm + group_id * BLOCK_M_SPLIT + tlx.barrier_expect_bytes( + x_smem_full_bars[a_buf], + 2 * BLOCK_M_SPLIT * BLOCK_K, + ) + tlx.async_descriptor_load( + x_desc, + x_buffers[a_buf], + [offs_xm2, offs_k], + x_smem_full_bars[a_buf], + ) + + load_phase = load_phase ^ (buf == int(NUM_SMEM_BUFFERS) - 1) + + for slice_id in tl.static_range(EPILOGUE_SUBTILE): + for group_id in tl.static_range(NUM_MMA_GROUPS): + y_idx = slice_id * NUM_MMA_GROUPS + group_id + y_buf_view = tlx.local_view(y_buffers, y_idx) + y_bar = tlx.local_view(y_load_bars, y_idx) + # If Y and Z are shared we need to wait for Z to be empty. + if Y_Z_SHARED: + y_empty = tlx.local_view(z_empty_bars, y_idx) + else: + y_empty = tlx.local_view(y_empty_bars, y_idx) + tlx.barrier_wait(y_empty, y_load_phase ^ 1) + if BROADCAST_Y: + tlx.barrier_expect_bytes(y_bar, 1 * slice_size * 2) + tlx.async_descriptor_load( + y_desc, + y_buf_view, + [0, offs_wn_full + slice_id * slice_size], + y_bar, + ) + else: + tlx.barrier_expect_bytes( + y_bar, BLOCK_M_SPLIT * slice_size * 2 + ) + tlx.async_descriptor_load( + y_desc, + y_buf_view, + [ + offs_xm + group_id * BLOCK_M_SPLIT, + offs_wn_full + slice_id * slice_size, + ], + y_bar, + ) + + y_load_phase = y_load_phase ^ 1 + + processed_k_iters += k_tiles + + # TMA Store consumer. Added to simplify the barrier + # logic. + with tlx.async_task(num_warps=1, num_regs=24): + start_pid = tl.program_id(axis=0) + num_pid_m = tl.cdiv(M, BLOCK_M) + if PAIR_CTA: + # Round up to even for proper CTA pairing + num_pid_m = (num_pid_m + 1) // 2 * 2 + num_pid_n = tl.cdiv(N, BLOCK_N) + num_pid_in_group = GROUP_M * num_pid_n + num_tiles = num_pid_m * num_pid_n + z_load_phase = 0 + + # Unroll the first iteration. + # This guraranteed safe from our grid size. + pid_m, pid_n = _compute_pid( + start_pid, num_pid_in_group, num_pid_m, GROUP_M, NUM_SMS + ) + offs_xm = pid_m * BLOCK_M + offs_wn = pid_n * BLOCK_N + z_idx = 0 + for slice_id in tl.static_range(EPILOGUE_SUBTILE): + for group_id in tl.static_range(NUM_MMA_GROUPS): + # Determine the base "index" to decide if we need to wait on TMA. + z_idx_unrolled = slice_id * NUM_MMA_GROUPS + group_id + if z_idx_unrolled >= NUM_Z_BUFFERS: + tlx.async_descriptor_store_wait(NUM_Z_BUFFERS - 1) + z_empty = tlx.local_view(z_empty_bars, z_idx) + tlx.barrier_arrive(z_empty, 1) + + z_full = tlx.local_view(z_load_bars, z_idx) + tlx.barrier_wait(z_full, z_load_phase) + z_buf_view = tlx.local_view(z_buffers, z_idx) + tlx.fence_async_shared() + tlx.async_descriptor_store( + z_desc, + z_buf_view, + [ + offs_xm + group_id * BLOCK_M_SPLIT, + offs_wn + slice_id * slice_size, + ], + ) + + z_load_phase = z_load_phase ^ (z_idx == (NUM_Z_BUFFERS - 1)) + # pyre-ignore[58] + z_idx = (z_idx + 1) % NUM_Z_BUFFERS + + for tile_id in range(start_pid + NUM_SMS, num_tiles, NUM_SMS): + pid_m, pid_n = _compute_pid( + tile_id, num_pid_in_group, num_pid_m, GROUP_M, NUM_SMS + ) + offs_xm = pid_m * BLOCK_M + offs_wn = pid_n * BLOCK_N + for slice_id in tl.static_range(EPILOGUE_SUBTILE): + for group_id in tl.static_range(NUM_MMA_GROUPS): + # Wait on prior store to finish. + tlx.async_descriptor_store_wait(NUM_Z_BUFFERS - 1) + z_empty = tlx.local_view(z_empty_bars, z_idx) + tlx.barrier_arrive(z_empty, 1) + # Wait for the next load to be ready + z_full = tlx.local_view(z_load_bars, z_idx) + tlx.barrier_wait(z_full, z_load_phase) + z_buf_view = tlx.local_view(z_buffers, z_idx) + tlx.async_descriptor_store( + z_desc, + z_buf_view, + [ + offs_xm + group_id * BLOCK_M_SPLIT, + offs_wn + slice_id * slice_size, + ], + ) + z_load_phase = z_load_phase ^ (z_idx == (NUM_Z_BUFFERS - 1)) + # pyre-ignore[58] + z_idx = (z_idx + 1) % NUM_Z_BUFFERS + + # Wait for the last store. + tlx.async_descriptor_store_wait(0) + + +@torch.fx.wrap +def triton_addmm_fwd_tma_persistent( + x: torch.Tensor, + w: torch.Tensor, + y: torch.Tensor, + warp_specialize: bool | None = None, +) -> torch.Tensor: + _meta_ws = _use_meta_ws() + if warp_specialize is None: + warp_specialize = _meta_ws + + M, K = x.shape + _, N = w.shape + + is_y_1d = y.dim() == 1 + + # Allocate output + z = torch.empty((M, N), device=x.device, dtype=x.dtype) + if M == 0 or N == 0: + return z + + # A dummy block value that will be overwritten when we have the real block size + dummy_block = [1, 1] + # pyre-ignore[6]: In call `TensorDescriptor.__init__`, for 2nd positional + # argument, expected `List[int]` but got `Size` + x_desc = TensorDescriptor(x, x.shape, x.stride(), dummy_block) + # pyre-ignore[6]: In call `TensorDescriptor.__init__`, for 2nd positional + # argument, expected `List[int]` but got `Size` + w_desc = TensorDescriptor(w, w.shape, w.stride(), dummy_block) + y = y.reshape(1, -1) if is_y_1d else y + # pyre-ignore[6]: In call `TensorDescriptor.__init__`, for 2nd positional + # argument, expected `List[int]` but got `Size` + y_desc = TensorDescriptor(y, y.shape, y.stride(), dummy_block) + # pyre-ignore[6]: In call `TensorDescriptor.__init__`, for 2nd positional + # argument, expected `List[int]` but got `Size` + z_desc = TensorDescriptor(z, z.shape, z.stride(), dummy_block) + NUM_SMS = torch.cuda.get_device_properties("cuda").multi_processor_count + + def grid(meta): + BLOCK_M = meta["BLOCK_M"] + BLOCK_N = meta["BLOCK_N"] + return ( + min( + NUM_SMS, + triton.cdiv(M, BLOCK_M) * triton.cdiv(N, BLOCK_N), + ), + ) + + _addmm_fwd_tma_persistent[grid]( + x_desc, + w_desc, + y_desc, + z_desc, + M, + N, + K, + ALLOW_TF32=torch.backends.cuda.matmul.allow_tf32, + BROADCAST_Y=is_y_1d, + WARP_SPECIALIZE=warp_specialize, + NUM_SMS=NUM_SMS, + USE_META_WS=_meta_ws, + ) + return z + + +@torch.fx.wrap +def triton_addmm_fwd_tma_ws_tlx( + x: torch.Tensor, + w: torch.Tensor, + y: torch.Tensor, +) -> torch.Tensor: + M, K = x.shape + _, N = w.shape + + is_y_1d = y.dim() == 1 + + # Allocate output + z = torch.empty((M, N), device=x.device, dtype=x.dtype) + if M == 0 or N == 0: + return z + + # A dummy block value that will be overwritten when we have the real block size + dummy_block = [1, 1] + # pyre-ignore[6]: In call `TensorDescriptor.__init__`, for 2nd positional + # argument, expected `List[int]` but got `Size` + x_desc = TensorDescriptor(x, x.shape, x.stride(), dummy_block) + # pyre-ignore[6]: In call `TensorDescriptor.__init__`, for 2nd positional + # argument, expected `List[int]` but got `Size` + w_desc = TensorDescriptor(w, w.shape, w.stride(), dummy_block) + y = y.reshape(1, -1) if is_y_1d else y + # pyre-ignore[6]: In call `TensorDescriptor.__init__`, for 2nd positional + # argument, expected `List[int]` but got `Size` + y_desc = TensorDescriptor(y, y.shape, y.stride(), dummy_block) + # pyre-ignore[6]: In call `TensorDescriptor.__init__`, for 2nd positional + # argument, expected `List[int]` but got `Size` + z_desc = TensorDescriptor(z, z.shape, z.stride(), dummy_block) + + def grid(meta): + BLOCK_M = meta["BLOCK_M"] + BLOCK_N = meta["BLOCK_N"] + return ( + triton.cdiv(M, BLOCK_M), + triton.cdiv(N, BLOCK_N), + ) + + _addmm_fwd_tma_ws[grid]( + x_desc, + w_desc, + y_desc, + z_desc, + M, + N, + K, + ALLOW_TF32=torch.backends.cuda.matmul.allow_tf32, + BROADCAST_Y=is_y_1d, + NUM_SMEM_BUFFERS=2, # Double buffering + ) + return z + + +@torch.fx.wrap +def triton_addmm_fwd_tma_ws_persistent_tlx( + x: torch.Tensor, + w: torch.Tensor, + y: torch.Tensor, +) -> torch.Tensor: + M, K = x.shape + _, N = w.shape + + is_y_1d = y.dim() == 1 + + # Allocate output + z = torch.empty((M, N), device=x.device, dtype=x.dtype) + if M == 0 or N == 0: + return z + + NUM_SMS = torch.cuda.get_device_properties("cuda").multi_processor_count + + # A dummy block value that will be overwritten by the hook + dummy_block = [1, 1] + # pyre-ignore[6]: In call `TensorDescriptor.__init__`, for 2nd positional + # argument, expected `List[int]` but got `Size` + x_desc = TensorDescriptor(x, x.shape, x.stride(), dummy_block) + # pyre-ignore[6]: In call `TensorDescriptor.__init__`, for 2nd positional + # argument, expected `List[int]` but got `Size` + w_desc = TensorDescriptor(w, w.shape, w.stride(), dummy_block) + y = y.reshape(1, -1) if is_y_1d else y + # pyre-ignore[6]: In call `TensorDescriptor.__init__`, for 2nd positional + # argument, expected `List[int]` but got `Size` + y_desc = TensorDescriptor(y, y.shape, y.stride(), dummy_block) + # pyre-ignore[6]: In call `TensorDescriptor.__init__`, for 2nd positional + # argument, expected `List[int]` but got `Size` + z_desc = TensorDescriptor(z, z.shape, z.stride(), dummy_block) + + def grid(meta): + BLOCK_M = meta["BLOCK_M"] + BLOCK_N = meta["BLOCK_N"] + num_pid_m = triton.cdiv(M, BLOCK_M) + num_pid_n = triton.cdiv(N, BLOCK_N) + # Round up num_pid_m to even for PAIR_CTA cluster compatibility + num_pid_m = (num_pid_m + 1) // 2 * 2 + total_tiles = num_pid_m * num_pid_n + grid_size = min(NUM_SMS, total_tiles) + # Ensure grid is even for cluster compatibility + if grid_size % 2 == 1: + grid_size = min(grid_size + 1, NUM_SMS) + # If rounding up exceeds NUM_SMS and NUM_SMS is odd, round down instead + if grid_size % 2 == 1: + grid_size = grid_size - 1 + return (grid_size,) + + _addmm_fwd_tma_ws_persistent[grid]( + x_desc, + w_desc, + y_desc, + z_desc, + M, + N, + K, + ALLOW_TF32=torch.backends.cuda.matmul.allow_tf32, + BROADCAST_Y=is_y_1d, + NUM_SMS=NUM_SMS, + ) + return z + + +@maybe_register_custom_op("generative_recommenders::triton_addmm_fwd", mutates_args=()) +def triton_addmm_fwd( + x: torch.Tensor, + w: torch.Tensor, + y: torch.Tensor, +) -> torch.Tensor: + M, K = x.shape + KB, N = w.shape + assert K == KB, f"incompatible dimensions {K}, {KB}" + + is_y_1d = y.dim() == 1 + NY = y.shape[0] if is_y_1d else y.shape[1] + assert N == NY, f"incompatible dimensions {N}, {NY}" + + # Allocate output + z = torch.empty((M, N), device=x.device, dtype=x.dtype) + if M == 0 or N == 0: + return z + + grid = lambda meta: ( # noqa E731 + triton.cdiv(M, meta["BLOCK_M"]), + triton.cdiv(N, meta["BLOCK_N"]), + ) + + _addmm_fwd[grid]( + x, + w, + y, + z, + M, + N, + K, + x.stride(0), + x.stride(1), + w.stride(0), + w.stride(1), + y.stride(0) if not is_y_1d else 0, + y.stride(1) if not is_y_1d else y.stride(0), + z.stride(0), + z.stride(1), + ALLOW_TF32=torch.backends.cuda.matmul.allow_tf32, + BROADCAST_Y=is_y_1d, + ) + return z + + +@triton_addmm_fwd.register_fake +def triton_addmm_fwd_fake( + x: torch.Tensor, + w: torch.Tensor, + y: torch.Tensor, +) -> torch.Tensor: + """Fake implementation for FakeTensor tracing.""" + M, _ = x.shape + _, N = w.shape + return torch.empty((M, N), device=x.device, dtype=x.dtype) + + +def triton_addmm_bwd( + x: torch.Tensor, + w: torch.Tensor, + dz: torch.Tensor, + is_y_1d: bool, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + if is_y_1d: + dy = torch.sum(dz, dim=0) + else: + dy = dz + dw = torch.mm(x.t(), dz) + dx = torch.mm(dz, w.t()) + + return dx, dw, dy + + +@maybe_register_custom_op( + "generative_recommenders::maybe_triton_addmm_fwd", mutates_args=() +) +def maybe_triton_addmm_fwd( + x: torch.Tensor, + w: torch.Tensor, + y: Optional[torch.Tensor], +) -> torch.Tensor: + # triton addmm is slower than torch (cublas) on AMD/Blackwell. + # Default to pytorch addmm on AMD/Blackwell for now. + if y is None: + return torch.mm(x, w) + if is_sm100_plus() or torch.version.hip is not None: + return torch.addmm(y, x, w) + else: + return triton_addmm_fwd(x=x, w=w, y=y) + + +@maybe_triton_addmm_fwd.register_fake +def maybe_triton_addmm_fwd_fake( + x: torch.Tensor, + w: torch.Tensor, + y: torch.Tensor, +) -> torch.Tensor: + """Fake implementation for FakeTensor tracing.""" + M, _ = x.shape + _, N = w.shape + return torch.empty((M, N), device=x.device, dtype=x.dtype) + + +class _AddMmFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore[14] + def forward( + ctx, + x: torch.Tensor, + w: torch.Tensor, + y: torch.Tensor, + ) -> torch.Tensor: + ctx.save_for_backward(x, w) + ctx.is_y_1d = y.dim() == 1 + if is_sm100_plus() and TMA_AVAILABLE and _check_tma_alignment(x, w, y): + if x.dtype == torch.float32 or HAS_TLX == False: + return triton_addmm_fwd_tma_persistent(x, w, y, warp_specialize=True) + else: + return triton_addmm_fwd_tma_ws_persistent_tlx( + x, w, y + ) # tlx.async_dot doesn't support fp32 inputs because of WGMMA requirements + else: + return triton_addmm_fwd(x, w, y) + + @staticmethod + # pyre-ignore[14] + def backward( + ctx, dz: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + (x, w) = ctx.saved_tensors + return triton_addmm_bwd(x, w, dz, ctx.is_y_1d) + + +def triton_addmm( + input: torch.Tensor, + mat1: torch.Tensor, + mat2: torch.Tensor, +) -> torch.Tensor: + return _AddMmFunction.apply(mat1, mat2, input) diff --git a/recommendation_v4/generative_recommenders/ops/triton/triton_attention_utils.py b/recommendation_v4/generative_recommenders/ops/triton/triton_attention_utils.py new file mode 100644 index 000000000..61fd614f3 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton/triton_attention_utils.py @@ -0,0 +1,64 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +#!/usr/bin/env python3 + + +# @manual=//triton:triton +import triton + +# @manual=//triton:triton +import triton.language as tl + + +@triton.jit +def acc_dq( + dq_ptrs_trans, + start_m, + stride_dqm, + k, + dqk_trans, + alpha, + mask_m, + MAX_SEQ_LEN, + LOCK, + BLOCK_M: tl.constexpr, + ATOMIC_ADD: tl.constexpr, + ALLOW_TF32: tl.constexpr, +): + if ATOMIC_ADD: + lock_id = start_m // BLOCK_M + stride_lock = tl.cdiv(MAX_SEQ_LEN, BLOCK_M) + lock = LOCK + tl.program_id(0) * stride_lock + lock_id + tl.debug_barrier() # add a barrier to force sync + while tl.atomic_cas(lock, 0, 1) == 1: + pass + dq_trans = tl.load( + dq_ptrs_trans + start_m * stride_dqm, + mask=mask_m[None, :], + other=0.0, + eviction_policy="evict_last", + ) + dq_trans += tl.dot(tl.trans(k), dqk_trans, allow_tf32=ALLOW_TF32) * alpha + dq_trans = dq_trans.to(k.dtype) + tl.store( + dq_ptrs_trans + start_m * stride_dqm, + dq_trans, + mask=mask_m[None, :], + eviction_policy="evict_last", + ) + if ATOMIC_ADD: + tl.atomic_xchg(lock, 0) # pyre-ignore [61] diff --git a/recommendation_v4/generative_recommenders/ops/triton/triton_hstu_attention.py b/recommendation_v4/generative_recommenders/ops/triton/triton_hstu_attention.py new file mode 100644 index 000000000..ac667e139 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton/triton_hstu_attention.py @@ -0,0 +1,3134 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#!/usr/bin/env python3 + +# pyre-unsafe + +from typing import List, Optional, Tuple + +import torch + +# @manual=//triton:triton +import triton + +# @manual=//triton:triton +import triton.language as tl +from generative_recommenders.ops.utils import ( + copy_if_different_ptr, + maybe_register_custom_op, +) + +try: + # @manual=//triton:triton + import triton.language.extra.tlx as tlx # type: ignore + + HAS_TLX = True +except ImportError: + # suppress type checking errors + tlx = None + + HAS_TLX = False + +from generative_recommenders.common import ( + autotune_max_seq_len, + prev_power_of_2, + switch_to_contiguous_if_needed, + triton_autotune, +) +from triton.language.extra.libdevice import ( # @manual=//triton:triton + fast_dividef, + fast_expf, +) + +try: + # @manual=//triton:triton + from triton.tools.tensor_descriptor import TensorDescriptor + + tensor_descriptor_tma = True +except ImportError: + tensor_descriptor_tma = False + +try: + from generative_recommenders.ops.triton.fb.triton_attention_utils import acc_dq +except ImportError: + from generative_recommenders.ops.triton.triton_attention_utils import acc_dq + + +def _host_descriptor_pre_hook(nargs): + if not tensor_descriptor_tma: + return + + if not isinstance(nargs["Q"], TensorDescriptor): + return + BLOCK_M = nargs["BLOCK_M"] + BLOCK_N = nargs["BLOCK_N"] + BLOCK_D_Q = nargs["BLOCK_D_Q"] + BLOCK_D_V = nargs["BLOCK_D_V"] + if "USE_TLX" in nargs and nargs["USE_TLX"]: + BLOCK_M = BLOCK_M // nargs["NUM_MMA_GROUPS"] + nargs["Q"].block_shape = [BLOCK_M, BLOCK_D_Q] + nargs["V"].block_shape = [BLOCK_N, BLOCK_D_V] + nargs["K"].block_shape = [BLOCK_N, BLOCK_D_Q] + + +# pyre-ignore[2] +def _early_config_prune( + configs: List[triton.Config], + named_args, + **kwargs, +) -> List[triton.Config]: + """Filter autotune configs that are incompatible with the current call. + + The TLX (warp-specialized) variant of ``_hstu_attn_fwd`` calls + ``tlx.async_descriptor_load(Q, ...)`` which requires Q/K/V to be real TMA + tensor descriptors (``tl.tensor_descriptor_base``). They are only + constructed by the host wrapper when ``ENABLE_TMA=True`` AND the host + ``TensorDescriptor`` API is importable. If the kernel is invoked without + those preconditions, raw tensors flow into the TLX path and the + ``isinstance(desc, tl.tensor_descriptor_base)`` assert in + ``triton/language/extra/tlx/mem_ops.py`` fires at compile time. + + We make autotuning robust to that mismatch by dropping any config with + ``USE_TLX=True`` whenever ENABLE_TMA is not set or TMA host descriptors + are unavailable. This is purely defensive: if the caller threads + ``enable_tma=True`` (see ``_should_enable_tma`` below) the TLX configs + remain eligible. + """ + enable_tma = kwargs.get("ENABLE_TMA", None) + if enable_tma is None: + enable_tma = named_args.get("ENABLE_TMA", False) + if enable_tma and tensor_descriptor_tma: + return configs + pruned = [c for c in configs if not c.kwargs.get("USE_TLX", False)] + # Safety: never return an empty config list. + return pruned if pruned else configs + + +def _should_enable_tma() -> bool: + """Return True iff the TMA / TLX fast path can be safely enabled. + + Conditions: + * The host ``triton.tools.tensor_descriptor.TensorDescriptor`` API is + importable (``tensor_descriptor_tma``). + * CUDA is available and the device is Hopper (compute capability 9), + which is the only architecture for which TLX configs are emitted in + ``_get_fw_configs``. + """ + if not tensor_descriptor_tma: + return False + if not torch.cuda.is_available(): + return False + try: + device_capability = torch.cuda.get_device_capability()[0] + except (RuntimeError, AssertionError): + return False + return device_capability == 9 + + +def _get_fw_configs() -> List[triton.Config]: # noqa: C901 + configs = [] + if torch.version.hip: + for BLOCK_M in [32, 64, 128]: + for BLOCK_N in [32, 64]: + for num_stages in [1, 2]: + for num_warps in [4, 8]: + for matrix_instr_nonkdim in [16, 32]: + configs.append( + triton.Config( + { + "BLOCK_M": BLOCK_M, + "BLOCK_N": BLOCK_N, + "matrix_instr_nonkdim": matrix_instr_nonkdim, + "waves_per_eu": 0, + "kpack": 2, + }, + num_stages=num_stages, + num_warps=num_warps, + ) + ) + else: + configs = [ + triton.Config( + {"BLOCK_M": 16, "BLOCK_N": 32}, + num_stages=2, + num_warps=2, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 32}, + num_stages=2, + num_warps=2, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 32}, + num_stages=4, + num_warps=2, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 32}, + num_stages=2, + num_warps=4, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 32}, + num_stages=4, + num_warps=4, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 64}, + num_stages=2, + num_warps=4, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 64}, + num_stages=4, + num_warps=4, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 64}, + num_stages=4, + num_warps=8, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 128}, + num_stages=2, + num_warps=4, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 128}, + num_stages=2, + num_warps=8, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 64, "BLOCK_N": 32}, + num_stages=4, + num_warps=2, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 64, "BLOCK_N": 32}, + num_stages=2, + num_warps=4, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 64, "BLOCK_N": 32}, + num_stages=4, + num_warps=4, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 64, "BLOCK_N": 32}, + num_stages=2, + num_warps=8, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 64, "BLOCK_N": 64}, + num_stages=2, + num_warps=2, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 64, "BLOCK_N": 64}, + num_stages=2, + num_warps=4, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 64, "BLOCK_N": 64}, + num_stages=4, + num_warps=4, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 64, "BLOCK_N": 64}, + num_stages=4, + num_warps=8, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 128, "BLOCK_N": 32}, + num_stages=2, + num_warps=2, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 128, "BLOCK_N": 32}, + num_stages=4, + num_warps=2, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 128, "BLOCK_N": 32}, + num_stages=2, + num_warps=4, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 128, "BLOCK_N": 32}, + num_stages=4, + num_warps=4, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 128, "BLOCK_N": 32}, + num_stages=2, + num_warps=8, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 128, "BLOCK_N": 32}, + num_stages=4, + num_warps=8, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 128, "BLOCK_N": 64}, + num_stages=2, + num_warps=4, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 128, "BLOCK_N": 64}, + num_stages=2, + num_warps=8, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 128, "BLOCK_N": 64}, + num_stages=4, + num_warps=8, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 128, "BLOCK_N": 128}, + num_stages=4, + num_warps=4, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 128, "BLOCK_N": 128}, + num_stages=2, + num_warps=8, + pre_hook=_host_descriptor_pre_hook, + ), + ] + + # Add 'USE_TLX' : False, 'NUM_BUFFERS': 1, 'NUM_MMA_WARPS_PER_GROUP': 1, 'NUM_MMA_GROUPS': 1 to non-TLX configs + for config in configs: + if not config.kwargs.get("USE_TLX", False): + config.kwargs["USE_TLX"] = False + config.kwargs["NUM_BUFFERS"] = 1 + config.kwargs["NUM_MMA_WARPS_PER_GROUP"] = 1 + config.kwargs["NUM_MMA_GROUPS"] = 1 + + # Add TLX configs if TLX is available + if HAS_TLX: + try: + device_capability = torch.cuda.get_device_capability()[0] + except (RuntimeError, AssertionError): + # No CUDA device available + device_capability = None + + if device_capability == 9: + # H100 configs + configs.append( + triton.Config( + { + "BLOCK_M": 128, + "BLOCK_N": 64, + "USE_TLX": True, + "NUM_BUFFERS": 2, + "NUM_MMA_WARPS_PER_GROUP": 4, + "NUM_MMA_GROUPS": 2, + }, + num_stages=0, + num_warps=4, + pre_hook=_host_descriptor_pre_hook, + ), + ) + + return configs + + +@triton.jit +def _hstu_attn_fwd_one_block( # noqa: C901 + start_n, + seq_len, + offs_m, + offs_n, + q, + K, + V, + K_block_ptr, + V_block_ptr, + offset_kh, + offset_vh, + seq_start, + n_targets, + alpha, + MAX_SEQ_LEN, + contextual_seq_len, + max_attn_len, + HAS_MULTIPLE_TARGETS: tl.constexpr, + HAS_CONTEXTUAL_SEQ_LEN: tl.constexpr, + HAS_MAX_ATTN_LEN: tl.constexpr, + ALLOW_TF32: tl.constexpr, + BLOCK_D_Q: tl.constexpr, + BLOCK_D_V: tl.constexpr, + BLOCK_N: tl.constexpr, + ENABLE_TMA: tl.constexpr, +): + start_n = tl.multiple_of(start_n, BLOCK_N) + # -- compute qk ---- + k = None + qk = None + if ENABLE_TMA: + k = K.load( + [(seq_start + start_n).to(tl.int32), offset_kh.to(tl.int32)], + ) + # tma can only be loaded in one order, use trans afterwards + qk = tl.dot(q, tl.trans(k), allow_tf32=ALLOW_TF32) * alpha + else: + k = tl.load(K_block_ptr, boundary_check=(1,), padding_option="zero") + qk = tl.dot(q, k, allow_tf32=ALLOW_TF32) * alpha + invalid_mask = offs_m[:, None] == offs_n[None, :] + max_ids = seq_len + if HAS_CONTEXTUAL_SEQ_LEN: + offs_m = offs_m - contextual_seq_len + 1 + offs_m = tl.where( + offs_m > 0, + offs_m, + 0, + ) + offs_n = offs_n - contextual_seq_len + 1 + offs_n = tl.where( + offs_n > 0, + offs_n, + 0, + ) + max_ids = max_ids - contextual_seq_len + 1 + if HAS_MULTIPLE_TARGETS: + max_ids = max_ids - n_targets + offs_m = tl.where( + offs_m < max_ids, + offs_m, + max_ids, + ) + offs_n = tl.where( + offs_n < max_ids, + offs_n, + max_ids, + ) + offs_m_minus_n = offs_m[:, None] - offs_n[None, :] + invalid_mask = invalid_mask or (offs_m_minus_n > 0) + if HAS_MAX_ATTN_LEN: + invalid_mask = invalid_mask and offs_m_minus_n <= max_attn_len + if HAS_CONTEXTUAL_SEQ_LEN: + invalid_mask = invalid_mask or ( + offs_m[:, None] == 0 and offs_n[None, :] < max_ids + ) + scale = tl.where(invalid_mask, (1.0 / MAX_SEQ_LEN), 0.0) + silu = fast_dividef(qk, 1.0 + fast_expf(-qk)) * scale + v = None + if ENABLE_TMA: + v = V.load( + [(seq_start + start_n).to(tl.int32), offset_vh.to(tl.int32)], + ) + else: + v = tl.load(V_block_ptr, boundary_check=(0,), padding_option="zero") + silu = silu.to(v.dtype) + return tl.dot(silu, v, allow_tf32=ALLOW_TF32) + + +@triton.jit +def _hstu_attn_fwd_compute( # noqa C901 + Q, + K, + V, + H, + DimQ, + DimV, + workspace_ptr, + seq_offsets, + num_targets, + Out, + stride_qm, + stride_qh, + stride_kn, + stride_kh, + stride_vn, + stride_vh, + stride_om, + stride_oh, + alpha, + MAX_SEQ_LEN, + DeltaSize, + contextual_seq_len, + max_attn_len, + off_z, + off_h, + pid, + HAS_MULTIPLE_TARGETS: tl.constexpr, + IS_DELTA_Q: tl.constexpr, + ALLOW_TF32: tl.constexpr, + BLOCK_D_Q: tl.constexpr, + BLOCK_D_V: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + HAS_CONTEXTUAL_SEQ_LEN: tl.constexpr, + HAS_MAX_ATTN_LEN: tl.constexpr, + ENABLE_TMA: tl.constexpr, + TMA_DESC_SIZE: tl.constexpr, +): + seq_start = tl.load(seq_offsets + off_z).to(tl.int64) + off_h = off_h.to(tl.int64) + off_z = off_z.to(tl.int64) + seq_end = tl.load(seq_offsets + off_z + 1) + seq_len = (seq_end - seq_start).to(tl.int32) + + if IS_DELTA_Q: + start_m_delta = pid * BLOCK_M + start_m = (start_m_delta + seq_len - DeltaSize).to(tl.int32) + else: + start_m_delta = 0 + start_m = pid * BLOCK_M + if start_m < seq_len: + if HAS_MULTIPLE_TARGETS: + n_targets = tl.load(num_targets + off_z).to(tl.int32) + else: + n_targets = None + + # initialize offsets + offs_m = start_m + tl.arange(0, BLOCK_M) + offs_n = tl.arange(0, BLOCK_N) + Q_block_ptr = None + K_block_ptr = None + V_block_ptr = None + if not ENABLE_TMA: + if IS_DELTA_Q: + Q_block_ptr = tl.make_block_ptr( + base=Q + off_h * stride_qh + off_z * DeltaSize * stride_qm, + shape=(DeltaSize, BLOCK_D_Q), + strides=(stride_qm, 1), + offsets=(start_m_delta, 0), + block_shape=(BLOCK_M, BLOCK_D_Q), + order=(1, 0), + ) + else: + Q_block_ptr = tl.make_block_ptr( + base=Q + off_h * stride_qh + seq_start * stride_qm, + shape=(seq_len, BLOCK_D_Q), + strides=(stride_qm, 1), + offsets=(start_m, 0), + block_shape=(BLOCK_M, BLOCK_D_Q), + order=(1, 0), + ) + q = tl.load(Q_block_ptr, boundary_check=(0,), padding_option="zero") + + K_block_ptr = tl.make_block_ptr( + base=K + off_h * stride_kh + seq_start * stride_kn, + shape=(BLOCK_D_Q, seq_len), + strides=(1, stride_kn), + offsets=(0, 0), + block_shape=(BLOCK_D_Q, BLOCK_N), + order=(0, 1), + ) + V_block_ptr = tl.make_block_ptr( + base=V + off_h * stride_vh + seq_start * stride_vn, + shape=(seq_len, BLOCK_D_V), + strides=(stride_vn, 1), + offsets=(0, 0), + block_shape=(BLOCK_N, BLOCK_D_V), + order=(1, 0), + ) + else: + if IS_DELTA_Q: + q = Q.load( + [ + (off_z * DeltaSize + start_m_delta).to(tl.int32), + (off_h * stride_qh).to(tl.int32), + ] + ) + else: + q = Q.load( + [ + (seq_start + start_m).to(tl.int32), + (off_h * stride_qh).to(tl.int32), + ] + ) + + acc = tl.zeros([BLOCK_M, BLOCK_D_V], dtype=tl.float32) + if HAS_MULTIPLE_TARGETS: + uih_end = seq_len - n_targets + else: + uih_end = seq_len + if HAS_CONTEXTUAL_SEQ_LEN is True and start_m < contextual_seq_len: + # uih_end must be larger than start_m + low = 0 + high = seq_len + else: + low = 0 + high = start_m + BLOCK_M + if HAS_MAX_ATTN_LEN: + if start_m > uih_end: + low = uih_end - max_attn_len + else: + low = start_m - max_attn_len + if HAS_CONTEXTUAL_SEQ_LEN: + low = low if low > contextual_seq_len else 0 + else: + low = low if low > 0 else 0 + if HAS_MULTIPLE_TARGETS: + uih_end = (uih_end + BLOCK_N - 1) // BLOCK_N * BLOCK_N + if uih_end < start_m: + high = seq_len - n_targets + + if low > 0: + if not ENABLE_TMA: + K_block_ptr = tl.advance(K_block_ptr, (0, low)) + V_block_ptr = tl.advance(V_block_ptr, (low, 0)) + end_n = low + for start_n in range(low, high, BLOCK_N): + acc += _hstu_attn_fwd_one_block( + start_n=start_n, + seq_len=seq_len, + offs_m=offs_m, + offs_n=offs_n + start_n, + q=q, + K=K, + V=V, + K_block_ptr=K_block_ptr, + V_block_ptr=V_block_ptr, + offset_kh=off_h * stride_kh, + offset_vh=off_h * stride_vh, + seq_start=seq_start, + n_targets=n_targets if HAS_MULTIPLE_TARGETS else None, + alpha=alpha, + MAX_SEQ_LEN=MAX_SEQ_LEN, + contextual_seq_len=contextual_seq_len, + max_attn_len=max_attn_len, + HAS_MULTIPLE_TARGETS=HAS_MULTIPLE_TARGETS, + HAS_CONTEXTUAL_SEQ_LEN=HAS_CONTEXTUAL_SEQ_LEN, + HAS_MAX_ATTN_LEN=HAS_MAX_ATTN_LEN, + ALLOW_TF32=ALLOW_TF32, + BLOCK_D_Q=BLOCK_D_Q, + BLOCK_D_V=BLOCK_D_V, + BLOCK_N=BLOCK_N, + ENABLE_TMA=ENABLE_TMA, + ) + if not ENABLE_TMA: + K_block_ptr = tl.advance(K_block_ptr, (0, BLOCK_N)) + V_block_ptr = tl.advance(V_block_ptr, (BLOCK_N, 0)) + end_n += BLOCK_N + + if HAS_MULTIPLE_TARGETS: + # pyre-ignore[61] + if uih_end < start_m: + low_delta = start_m + high_delta = start_m + BLOCK_M + offset = (low_delta - end_n).to(tl.int32) + if not ENABLE_TMA: + K_block_ptr = tl.advance(K_block_ptr, (0, offset)) + V_block_ptr = tl.advance(V_block_ptr, (offset, 0)) + for start_delta in tl.range( + low_delta, high_delta, BLOCK_N, num_stages=0 + ): + acc += _hstu_attn_fwd_one_block( + start_n=start_delta, + seq_len=seq_len, + offs_m=offs_m, + offs_n=offs_n + start_delta, + q=q, + K=K, + V=V, + K_block_ptr=K_block_ptr, + V_block_ptr=V_block_ptr, + offset_kh=off_h * stride_kh, + offset_vh=off_h * stride_vh, + seq_start=seq_start, + n_targets=n_targets if HAS_MULTIPLE_TARGETS else None, + alpha=alpha, + MAX_SEQ_LEN=MAX_SEQ_LEN, + contextual_seq_len=contextual_seq_len, + max_attn_len=max_attn_len, + HAS_MULTIPLE_TARGETS=HAS_MULTIPLE_TARGETS, + HAS_CONTEXTUAL_SEQ_LEN=HAS_CONTEXTUAL_SEQ_LEN, + HAS_MAX_ATTN_LEN=HAS_MAX_ATTN_LEN, + ALLOW_TF32=ALLOW_TF32, + BLOCK_D_Q=BLOCK_D_Q, + BLOCK_D_V=BLOCK_D_V, + BLOCK_N=BLOCK_N, + ENABLE_TMA=ENABLE_TMA, + ) + if not ENABLE_TMA: + K_block_ptr = tl.advance(K_block_ptr, (0, BLOCK_N)) + V_block_ptr = tl.advance(V_block_ptr, (BLOCK_N, 0)) + # Don't use TMA in Jagged case since we don't want to overwrite + # the output of another sequence + if IS_DELTA_Q: + start_m_delta = pid * BLOCK_M + offs_m_delta = start_m_delta + tl.arange(0, BLOCK_M) + offs_v_d = tl.arange(0, BLOCK_D_V) + off_o = Out + off_z * DeltaSize * stride_om + off_h * stride_oh + out_ptrs = off_o + offs_m_delta[:, None] * stride_om + offs_v_d[None, :] + tl.store(out_ptrs, acc, mask=(offs_m_delta < DeltaSize)[:, None]) + else: + # rematerialize offsets to save registers + start_m = pid * BLOCK_M + offs_m = start_m + tl.arange(0, BLOCK_M) + offs_v_d = tl.arange(0, BLOCK_D_V) + off_o = Out + seq_start * stride_om + off_h * stride_oh + out_ptrs = off_o + offs_m[:, None] * stride_om + offs_v_d[None, :] + tl.store(out_ptrs, acc, mask=(offs_m < seq_len)[:, None]) + + +@triton.jit +def _hstu_attn_fwd_compute_main_loop_tlx( # noqa C901 + low, + high, + seq_len, + offs_m, + offs_n, + acc, + q_tiles, + k_tiles, + v_tiles, + q_fulls, + k_fulls, + v_fulls, + k_empties, + v_empties, + v_dtype, + n_targets, + alpha, + end_n, + loop_trip_cnt, + max_attn_len, + HAS_MULTIPLE_TARGETS: tl.constexpr, + HAS_CONTEXTUAL_SEQ_LEN: tl.constexpr, + HAS_MAX_ATTN_LEN: tl.constexpr, + cid: tl.constexpr, + BLOCK_N: tl.constexpr, + NUM_BUFFERS: tl.constexpr, + MAX_SEQ_LEN: tl.constexpr, + WAIT_FOR_Q: tl.constexpr, +): + if WAIT_FOR_Q: + # wait for the Q buffer to be populated by the producer + q_full = tlx.local_view(q_fulls, cid) + tlx.barrier_wait(q_full, 0) + + q_tile = tlx.local_view(q_tiles, cid) + + for start in tl.range(low + BLOCK_N, high, BLOCK_N, num_stages=0): + buf_id = loop_trip_cnt % NUM_BUFFERS + # buffers in a row share the same phase + kv_phase = (loop_trip_cnt // NUM_BUFFERS) % 2 + + start_n = tl.multiple_of(start, BLOCK_N) + offs_n_start = offs_n + offs_n = offs_n_start + start_n + + # wait for the K buffer to be populated by the producer + k_full = tlx.local_view(k_fulls, buf_id) + tlx.barrier_wait(k_full, kv_phase) + k_tile = tlx.local_view(k_tiles, buf_id) + + # tma can only be loaded in one order, use trans afterwards + k_tile = tlx.local_trans(k_tile) + # second + qk = tlx.async_dot(q_tile, k_tile) + # wait for the MMA using to complete + qk = tlx.async_dot_wait(0, qk) + # release the K buffer + k_empty = tlx.local_view(k_empties, buf_id) + tlx.barrier_arrive(k_empty, 1) + + qk = qk * alpha + + invalid_mask = offs_m[:, None] == offs_n[None, :] + max_ids = seq_len + if HAS_MULTIPLE_TARGETS: + max_ids = max_ids - n_targets + offs_m = tl.where( + offs_m < max_ids, + offs_m, + max_ids, + ) + offs_n = tl.where( + offs_n < max_ids, + offs_n, + max_ids, + ) + offs_m_minus_n = offs_m[:, None] - offs_n[None, :] + invalid_mask = invalid_mask or (offs_m_minus_n > 0) + if HAS_MAX_ATTN_LEN: + invalid_mask = invalid_mask and offs_m_minus_n <= max_attn_len + if HAS_CONTEXTUAL_SEQ_LEN: + invalid_mask = invalid_mask or ( + offs_m[:, None] == 0 and offs_n[None, :] < max_ids + ) + scale = tl.where(invalid_mask, (1.0 / MAX_SEQ_LEN), 0.0) + silu = fast_dividef(qk, 1.0 + fast_expf(-qk)) * scale + silu = silu.to(v_dtype) + + # wait for the V buffer to be populated by the producer + v_full = tlx.local_view(v_fulls, buf_id) + tlx.barrier_wait(v_full, kv_phase) + v_tile = tlx.local_view(v_tiles, buf_id) + acc = tlx.async_dot(silu, v_tile, acc) + # wait for the MMA using to complete + acc = tlx.async_dot_wait(0, acc) + # release the V buffer + v_empty = tlx.local_view(v_empties, buf_id) + tlx.barrier_arrive(v_empty, 1) + + end_n += BLOCK_N + + # increment loop trip counts + loop_trip_cnt += 1 + + return acc, end_n, loop_trip_cnt + + +@triton.jit +def _hstu_attn_fwd_compute_main_loop_tlx_pipelined( # noqa C901 + low, + high, + seq_len, + offs_m, + offs_n, + acc, + q_tiles, + k_tiles, + v_tiles, + q_fulls, + k_fulls, + v_fulls, + k_empties, + v_empties, + v_dtype, + n_targets, + alpha, + end_n, + loop_trip_cnt, + max_attn_len, + HAS_MULTIPLE_TARGETS: tl.constexpr, + HAS_CONTEXTUAL_SEQ_LEN: tl.constexpr, + HAS_MAX_ATTN_LEN: tl.constexpr, + cid: tl.constexpr, + BLOCK_N: tl.constexpr, + NUM_BUFFERS: tl.constexpr, + MAX_SEQ_LEN: tl.constexpr, + WAIT_FOR_Q: tl.constexpr, +): + if WAIT_FOR_Q: + # wait for the Q buffer to be populated by the producer + q_full = tlx.local_view(q_fulls, cid) + tlx.barrier_wait(q_full, 0) + q_tile = tlx.local_view(q_tiles, cid) + + # wait for the K buffer to be populated by the producer + k_buf_id = loop_trip_cnt % NUM_BUFFERS + # buffers in a row share the same phase + k_phase = (loop_trip_cnt // NUM_BUFFERS) % 2 + + k_full = tlx.local_view(k_fulls, k_buf_id) + tlx.barrier_wait(k_full, k_phase) + k_tile = tlx.local_view(k_tiles, k_buf_id) + + # tma can only be loaded in one order, use trans afterwards + k_tile = tlx.local_trans(k_tile) + + # Pingpong + if cid == 0: + # Consumer 0 waits for Consumer 1 to reach synchronization point at barrier 9. + tlx.named_barrier_wait(9, 256) + else: + # Consumer 1 signals its arrival at barrier 9. + tlx.named_barrier_arrive(9, 256) + # Then waits at barrier 10 until Consumer 0 finishes issuing its async_dot. + tlx.named_barrier_wait(10, 256) + + qk = tlx.async_dot(q_tile, k_tile) + + if cid == 0: + # After issuing async_dot, Consumer 0 signals barrier 10 to unblock Consumer 1. + tlx.named_barrier_arrive(10, 256) + + # wait for the MMA using to complete + qk = tlx.async_dot_wait(0, qk) + # release the K buffer + k_empty = tlx.local_view(k_empties, k_buf_id) + tlx.barrier_arrive(k_empty, 1) + + qk = qk * alpha + + start_n = tl.multiple_of(low, BLOCK_N) + offs_n_start = offs_n + offs_n = offs_n_start + start_n + + invalid_mask = offs_m[:, None] == offs_n[None, :] + max_ids = seq_len + if HAS_MULTIPLE_TARGETS: + max_ids = max_ids - n_targets + offs_m = tl.where( + offs_m < max_ids, + offs_m, + max_ids, + ) + offs_n = tl.where( + offs_n < max_ids, + offs_n, + max_ids, + ) + offs_m_minus_n = offs_m[:, None] - offs_n[None, :] + invalid_mask = invalid_mask or (offs_m_minus_n > 0) + if HAS_MAX_ATTN_LEN: + invalid_mask = invalid_mask and offs_m_minus_n <= max_attn_len + if HAS_CONTEXTUAL_SEQ_LEN: + invalid_mask = invalid_mask or ( + offs_m[:, None] == 0 and offs_n[None, :] < max_ids + ) + scale = tl.where(invalid_mask, (1.0 / MAX_SEQ_LEN), 0.0) + silu = fast_dividef(qk, 1.0 + fast_expf(-qk)) * scale + silu = silu.to(v_dtype) + + loop_trip_cnt += 1 + + for start in tl.range(low + BLOCK_N, high, BLOCK_N, num_stages=0): + start_n = tl.multiple_of(start, BLOCK_N) + offs_n = offs_n_start + start_n + + k_buf_id = loop_trip_cnt % NUM_BUFFERS + # buffers in a row share the same phase + k_phase = k_phase ^ (k_buf_id == 0) + + # wait for the K buffer to be populated by the producer + k_full = tlx.local_view(k_fulls, k_buf_id) + tlx.barrier_wait(k_full, k_phase) + k_tile = tlx.local_view(k_tiles, k_buf_id) + + # tma can only be loaded in one order, use trans afterwards + k_tile = tlx.local_trans(k_tile) + + qk = tlx.async_dot(q_tile, k_tile) + # wait for the MMA using to complete + prev_silu = silu + + v_buf_id = (loop_trip_cnt - 1) % NUM_BUFFERS + # v_phase = v_phase ^ (v_buf_id == 0) + v_phase = ((loop_trip_cnt - 1) // NUM_BUFFERS) % 2 + v_full = tlx.local_view(v_fulls, v_buf_id) + tlx.barrier_wait(v_full, v_phase) + v_tile = tlx.local_view(v_tiles, v_buf_id) + acc = tlx.async_dot(prev_silu, v_tile, acc) + qk = tlx.async_dot_wait(1, qk) + + # release the K buffer + k_empty = tlx.local_view(k_empties, k_buf_id) + tlx.barrier_arrive(k_empty, 1) + + qk = qk * alpha + invalid_mask = offs_m[:, None] == offs_n[None, :] + max_ids = seq_len + if HAS_MULTIPLE_TARGETS: + max_ids = max_ids - n_targets + offs_m = tl.where( + offs_m < max_ids, + offs_m, + max_ids, + ) + offs_n = tl.where( + offs_n < max_ids, + offs_n, + max_ids, + ) + offs_m_minus_n = offs_m[:, None] - offs_n[None, :] + invalid_mask = invalid_mask or (offs_m_minus_n > 0) + if HAS_MAX_ATTN_LEN: + invalid_mask = invalid_mask and offs_m_minus_n <= max_attn_len + if HAS_CONTEXTUAL_SEQ_LEN: + invalid_mask = invalid_mask or ( + offs_m[:, None] == 0 and offs_n[None, :] < max_ids + ) + scale = tl.where(invalid_mask, (1.0 / MAX_SEQ_LEN), 0.0) + silu = fast_dividef(qk, 1.0 + fast_expf(-qk)) * scale + silu = silu.to(v_dtype) + + acc = tlx.async_dot_wait(0, acc) + # release the V buffer + v_empty = tlx.local_view(v_empties, v_buf_id) + tlx.barrier_arrive(v_empty, 1) + + end_n += BLOCK_N + + # increment loop trip counts + loop_trip_cnt += 1 + # v_buf_id = loop_trip_cnt % NUM_BUFFERS + # v_phase = (loop_trip_cnt // NUM_BUFFERS) % 2 + + # wait for the V buffer to be populated by the producer + v_buf_id = (loop_trip_cnt - 1) % NUM_BUFFERS + v_phase = ((loop_trip_cnt - 1) // NUM_BUFFERS) % 2 + v_full = tlx.local_view(v_fulls, v_buf_id) + # tlx.barrier_wait(v_full, v_buf_id) + v_tile = tlx.local_view(v_tiles, v_buf_id) + tlx.barrier_wait(v_full, v_phase) + acc = tlx.async_dot(silu, v_tile, acc) + acc = tlx.async_dot_wait(0, acc) + # release the V buffer + v_empty = tlx.local_view(v_empties, v_buf_id) + tlx.barrier_arrive(v_empty, 1) + + return acc, end_n, loop_trip_cnt + + +@triton.jit +def _hstu_attn_fwd_load_K_or_V( + K, + k_tiles, + k_empties, + k_fulls, + buf_id, + k_phase, + start_n, + seq_start, + offset_kh, + BLOCK_D_Q: tl.constexpr, + BLOCK_N: tl.constexpr, +): + # wait for the K buffer to be released by the consumer + k_empty = tlx.local_view(k_empties, buf_id) + tlx.barrier_wait(k_empty, k_phase) + # load K + k_full = tlx.local_view(k_fulls, buf_id) + k_tile = tlx.local_view(k_tiles, buf_id) + tlx.barrier_expect_bytes(k_full, 2 * BLOCK_N * BLOCK_D_Q) # float16 + tlx.async_descriptor_load( + K, + k_tile, + [(seq_start + start_n).to(tl.int32), offset_kh.to(tl.int32)], + k_full, + ) + + +@triton.jit +def _hstu_attn_fwd_load_Q( + Q, + q_tiles, + q_fulls, + cid, + off_z, + off_h, + stride_qh, + start_m, + seq_start, + DeltaSize, + IS_DELTA_Q: tl.constexpr, + BLOCK_D_Q: tl.constexpr, + BLOCK_M: tl.constexpr, +): + q_full = tlx.local_view(q_fulls, cid) + tlx.barrier_expect_bytes(q_full, 2 * BLOCK_M * BLOCK_D_Q) # float16 + q_tile = tlx.local_view(q_tiles, cid) + seq_offset = start_m + cid * BLOCK_M + if IS_DELTA_Q: + tlx.async_descriptor_load( + Q, + q_tile, + [ + (off_z * DeltaSize + start_m).to(tl.int32), + (off_h * stride_qh).to(tl.int32), + ], + q_full, + ) + else: + tlx.async_descriptor_load( + Q, + q_tile, + [ + (seq_start + seq_offset).to(tl.int32), + (off_h * stride_qh).to(tl.int32), + ], + q_full, + ) + + +@triton.jit +def _hstu_attn_fwd_caculate_range( + seq_len, + start_m, + n_targets, + contextual_seq_len, + max_attn_len, + HAS_MULTIPLE_TARGETS: tl.constexpr, + HAS_CONTEXTUAL_SEQ_LEN: tl.constexpr, + HAS_MAX_ATTN_LEN: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, +): + if HAS_MULTIPLE_TARGETS: + uih_end = seq_len - n_targets + else: + uih_end = seq_len + + if HAS_CONTEXTUAL_SEQ_LEN is True and start_m < contextual_seq_len: + # uih_end must be larger than start_m + low = 0 + high = seq_len + else: + low = 0 + high = start_m + BLOCK_M + if HAS_MAX_ATTN_LEN: + if start_m > uih_end: + low = uih_end - max_attn_len + else: + low = start_m - max_attn_len + if HAS_CONTEXTUAL_SEQ_LEN: + low = low if low > contextual_seq_len else 0 + else: + low = low if low > 0 else 0 + if HAS_MULTIPLE_TARGETS: + uih_end = (uih_end + BLOCK_N - 1) // BLOCK_N * BLOCK_N + if uih_end < start_m: + high = seq_len - n_targets + + return low, high, uih_end + + +@triton.jit +def _hstu_attn_fwd_load_Q_K_V( + Q, + K, + V, + q_tiles, + k_tiles, + v_tiles, + q_fulls, + k_fulls, + v_fulls, + k_empties, + v_empties, + stride_qh, + stride_kh, + stride_vh, + contextual_seq_len, + max_attn_len, + DeltaSize, + off_z, + off_h, + start_m, + seq_start, + seq_len, + n_targets, + HAS_MULTIPLE_TARGETS: tl.constexpr, + IS_DELTA_Q: tl.constexpr, + BLOCK_D_Q: tl.constexpr, + BLOCK_D_V: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + NUM_BUFFERS: tl.constexpr, + NUM_MMA_GROUPS: tl.constexpr, + HAS_CONTEXTUAL_SEQ_LEN: tl.constexpr, + HAS_MAX_ATTN_LEN: tl.constexpr, +): + # load q: it will stay in SRAM throughout + BLOCK_M_SPLIT: tl.constexpr = BLOCK_M // NUM_MMA_GROUPS + + _hstu_attn_fwd_load_Q( + Q=Q, + q_tiles=q_tiles, + q_fulls=q_fulls, + cid=0, + off_z=off_z, + off_h=off_h, + stride_qh=stride_qh, + start_m=start_m, + seq_start=seq_start, + DeltaSize=DeltaSize, + IS_DELTA_Q=IS_DELTA_Q, + BLOCK_D_Q=BLOCK_D_Q, + BLOCK_M=BLOCK_M_SPLIT, + ) + + off_h = off_h.to(tl.int64) + off_z = off_z.to(tl.int64) + offset_kh = off_h * stride_kh + offset_vh = off_h * stride_vh + + low, high, uih_end = _hstu_attn_fwd_caculate_range( + seq_len, + start_m, + n_targets, + contextual_seq_len, + max_attn_len, + HAS_MULTIPLE_TARGETS, + HAS_CONTEXTUAL_SEQ_LEN, + HAS_MAX_ATTN_LEN, + BLOCK_M, + BLOCK_N, + ) + + kv_phase = 0 + loop_trip_cnt = 0 + + # pyre-ignore[58] + buf_id = loop_trip_cnt % NUM_BUFFERS + # buffers in a row share the same phase + kv_phase = kv_phase ^ (buf_id == 0) + + start_n = tl.multiple_of(low, BLOCK_N) + + _hstu_attn_fwd_load_K_or_V( + K, + k_tiles, + k_empties, + k_fulls, + buf_id, + kv_phase, + start_n, + seq_start, + offset_kh, + BLOCK_D_Q, + BLOCK_N, + ) + + for cid in tl.range(1, NUM_MMA_GROUPS, loop_unroll_factor=NUM_MMA_GROUPS - 1): + _hstu_attn_fwd_load_Q( + Q, + q_tiles, + q_fulls, + cid, + off_z, + off_h, + stride_qh, + start_m, + seq_start, + DeltaSize, + IS_DELTA_Q, + BLOCK_D_Q, + BLOCK_M_SPLIT, + ) + + _hstu_attn_fwd_load_K_or_V( + V, + v_tiles, + v_empties, + v_fulls, + buf_id, + kv_phase, + start_n, + seq_start, + offset_vh, + BLOCK_D_V, + BLOCK_N, + ) + + loop_trip_cnt += 1 + + for start in range(low + BLOCK_N, high, BLOCK_N): + # pyre-ignore[58] + buf_id = loop_trip_cnt % NUM_BUFFERS + # buffers in a row share the same phase + kv_phase = kv_phase ^ (buf_id == 0) + + start_n = tl.multiple_of(start, BLOCK_N) + + _hstu_attn_fwd_load_K_or_V( + K, + k_tiles, + k_empties, + k_fulls, + buf_id, + kv_phase, + start_n, + seq_start, + offset_kh, + BLOCK_D_Q, + BLOCK_N, + ) + + _hstu_attn_fwd_load_K_or_V( + V, + v_tiles, + v_empties, + v_fulls, + buf_id, + kv_phase, + start_n, + seq_start, + offset_vh, + BLOCK_D_V, + BLOCK_N, + ) + + # increment loop trip counts + loop_trip_cnt += 1 + + # pyre-ignore[61] + if uih_end < start_m: + low_delta = start_m + high_delta = start_m + BLOCK_M + for start_delta in tl.range(low_delta, high_delta, BLOCK_N, num_stages=0): + # pyre-ignore[58] + buf_id = loop_trip_cnt % NUM_BUFFERS + # buffers in a row share the same phase + kv_phase = kv_phase ^ (buf_id == 0) + + start_n = tl.multiple_of(start_delta, BLOCK_N) + + _hstu_attn_fwd_load_K_or_V( + K, + k_tiles, + k_empties, + k_fulls, + buf_id, + kv_phase, + start_n, + seq_start, + offset_kh, + BLOCK_D_Q, + BLOCK_N, + ) + + _hstu_attn_fwd_load_K_or_V( + V, + v_tiles, + v_empties, + v_fulls, + buf_id, + kv_phase, + start_n, + seq_start, + offset_vh, + BLOCK_D_V, + BLOCK_N, + ) + + # increment loop trip counts + loop_trip_cnt += 1 + + +@triton.jit +def _hstu_attn_fwd_compute_tlx( # noqa C901 + Q, + K, + V, + H, + DimQ, + DimV, + seq_offsets, + num_targets, + Out, + stride_qh, + stride_kh, + stride_vh, + stride_om, + stride_oh, + alpha, + MAX_SEQ_LEN, + DeltaSize, + contextual_seq_len, + max_attn_len, + off_z, + off_h, + pid, + HAS_MULTIPLE_TARGETS: tl.constexpr, + IS_DELTA_Q: tl.constexpr, + ALLOW_TF32: tl.constexpr, + BLOCK_D_Q: tl.constexpr, + BLOCK_D_V: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + NUM_BUFFERS: tl.constexpr, # + NUM_MMA_WARPS_PER_GROUP: tl.constexpr, # + NUM_MMA_GROUPS: tl.constexpr, # + HAS_CONTEXTUAL_SEQ_LEN: tl.constexpr, + HAS_MAX_ATTN_LEN: tl.constexpr, +): + seq_start = tl.load(seq_offsets + off_z).to(tl.int64) + seq_end = tl.load(seq_offsets + off_z + 1) + seq_len = (seq_end - seq_start).to(tl.int32) + + if IS_DELTA_Q: + start_m = pid * BLOCK_M + start_m = (start_m + seq_len - DeltaSize).to(tl.int32) + else: + start_m = pid * BLOCK_M + + if start_m >= seq_len: + return + + if HAS_MULTIPLE_TARGETS: + n_targets = tl.load(num_targets + off_z).to(tl.int32) + else: + n_targets = None + + BLOCK_M_SPLIT: tl.constexpr = BLOCK_M // NUM_MMA_GROUPS + # allocate buffers + q_tiles = tlx.local_alloc( + (BLOCK_M_SPLIT, BLOCK_D_Q), tlx.dtype_of(Q), NUM_MMA_GROUPS + ) + k_tiles = tlx.local_alloc((BLOCK_N, BLOCK_D_Q), tlx.dtype_of(K), NUM_BUFFERS) + v_tiles = tlx.local_alloc((BLOCK_N, BLOCK_D_V), tlx.dtype_of(V), NUM_BUFFERS) + + # allocate barriers + q_fulls = tlx.alloc_barriers(num_barriers=NUM_MMA_GROUPS, arrive_count=1) + k_empties = tlx.alloc_barriers( + num_barriers=NUM_BUFFERS, arrive_count=NUM_MMA_GROUPS + ) + k_fulls = tlx.alloc_barriers(num_barriers=NUM_BUFFERS, arrive_count=1) + v_empties = tlx.alloc_barriers( + num_barriers=NUM_BUFFERS, arrive_count=NUM_MMA_GROUPS + ) + v_fulls = tlx.alloc_barriers(num_barriers=NUM_BUFFERS, arrive_count=1) + + with tlx.async_tasks(): + # producer group + with tlx.async_task("default"): + _hstu_attn_fwd_load_Q_K_V( + Q=Q, + K=K, + V=V, + q_tiles=q_tiles, + k_tiles=k_tiles, + v_tiles=v_tiles, + q_fulls=q_fulls, + k_fulls=k_fulls, + v_fulls=v_fulls, + k_empties=k_empties, + v_empties=v_empties, + stride_qh=stride_qh, + stride_kh=stride_kh, + stride_vh=stride_vh, + contextual_seq_len=contextual_seq_len, + max_attn_len=max_attn_len, + DeltaSize=DeltaSize, + off_z=off_z, + off_h=off_h, + start_m=start_m, + seq_start=seq_start, + seq_len=seq_len, + n_targets=n_targets, + HAS_MULTIPLE_TARGETS=HAS_MULTIPLE_TARGETS, + IS_DELTA_Q=IS_DELTA_Q, + BLOCK_D_Q=BLOCK_D_Q, + BLOCK_D_V=BLOCK_D_V, + BLOCK_M=BLOCK_M, + BLOCK_N=BLOCK_N, + NUM_BUFFERS=NUM_BUFFERS, + NUM_MMA_GROUPS=NUM_MMA_GROUPS, + HAS_CONTEXTUAL_SEQ_LEN=HAS_CONTEXTUAL_SEQ_LEN, + HAS_MAX_ATTN_LEN=HAS_MAX_ATTN_LEN, + ) + + # consumer groups + with tlx.async_task( + num_warps=NUM_MMA_WARPS_PER_GROUP, registers=232, replicate=NUM_MMA_GROUPS + ): + cid = tlx.async_task_replica_id() + acc = tl.zeros([BLOCK_M_SPLIT, BLOCK_D_V], dtype=tl.float32) + # initialize offsets + offs_m = start_m + tl.arange(0, BLOCK_M_SPLIT) + cid * BLOCK_M_SPLIT + offs_n = tl.arange(0, BLOCK_N) + + low, high, uih_end = _hstu_attn_fwd_caculate_range( + seq_len, + start_m, + n_targets, + contextual_seq_len, + max_attn_len, + HAS_MULTIPLE_TARGETS, + HAS_CONTEXTUAL_SEQ_LEN, + HAS_MAX_ATTN_LEN, + BLOCK_M, + BLOCK_N, + ) + + end_n = low + loop_trip_cnt = 0 + + acc, end_n, loop_trip_cnt = _hstu_attn_fwd_compute_main_loop_tlx_pipelined( + low=low, + high=high, + seq_len=seq_len, + offs_m=offs_m, + offs_n=offs_n, + acc=acc, + q_tiles=q_tiles, + k_tiles=k_tiles, + v_tiles=v_tiles, + q_fulls=q_fulls, + k_fulls=k_fulls, + v_fulls=v_fulls, + k_empties=k_empties, + v_empties=v_empties, + v_dtype=tlx.dtype_of(V), + n_targets=n_targets, + alpha=alpha, + end_n=end_n, + loop_trip_cnt=loop_trip_cnt, + max_attn_len=max_attn_len, + HAS_MULTIPLE_TARGETS=HAS_MULTIPLE_TARGETS, + HAS_CONTEXTUAL_SEQ_LEN=HAS_CONTEXTUAL_SEQ_LEN, + HAS_MAX_ATTN_LEN=HAS_MAX_ATTN_LEN, + cid=cid, + BLOCK_N=BLOCK_N, + NUM_BUFFERS=NUM_BUFFERS, + MAX_SEQ_LEN=MAX_SEQ_LEN, + WAIT_FOR_Q=1, + ) + + # pyre-ignore[61] + if uih_end < start_m: + low_delta = start_m + high_delta = start_m + BLOCK_M + acc, end_n, loop_trip_cnt = _hstu_attn_fwd_compute_main_loop_tlx( + low=low_delta, + high=high_delta, + seq_len=seq_len, + offs_m=offs_m, + offs_n=offs_n, + acc=acc, + q_tiles=q_tiles, + k_tiles=k_tiles, + v_tiles=v_tiles, + q_fulls=q_fulls, + k_fulls=k_fulls, + v_fulls=v_fulls, + k_empties=k_empties, + v_empties=v_empties, + v_dtype=tlx.dtype_of(V), + n_targets=n_targets, + alpha=alpha, + end_n=end_n, + loop_trip_cnt=loop_trip_cnt, + max_attn_len=max_attn_len, + HAS_MULTIPLE_TARGETS=HAS_MULTIPLE_TARGETS, + HAS_CONTEXTUAL_SEQ_LEN=HAS_CONTEXTUAL_SEQ_LEN, + HAS_MAX_ATTN_LEN=HAS_MAX_ATTN_LEN, + cid=cid, + BLOCK_N=BLOCK_N, + NUM_BUFFERS=NUM_BUFFERS, + MAX_SEQ_LEN=MAX_SEQ_LEN, + WAIT_FOR_Q=0, + ) + + # Don't use TMA in Jagged case since we don't want to overwrite + # the output of another sequence + if IS_DELTA_Q: + start_m_delta = pid * BLOCK_M + cid * BLOCK_M_SPLIT + offs_m_delta = start_m_delta + tl.arange(0, BLOCK_M_SPLIT) + offs_v_d = tl.arange(0, BLOCK_D_V) + off_o = Out + off_z * DeltaSize * stride_om + off_h * stride_oh + out_ptrs = off_o + offs_m_delta[:, None] * stride_om + offs_v_d[None, :] + tl.store(out_ptrs, acc, mask=(offs_m_delta < DeltaSize)[:, None]) + else: + # rematerialize offsets to save registers + start_m = pid * BLOCK_M + cid * BLOCK_M_SPLIT + offs_m = start_m + tl.arange(0, BLOCK_M_SPLIT) + offs_v_d = tl.arange(0, BLOCK_D_V) + off_o = Out + seq_start * stride_om + off_h * stride_oh + out_ptrs = off_o + offs_m[:, None] * stride_om + offs_v_d[None, :] + tl.store(out_ptrs, acc, mask=(offs_m < seq_len)[:, None]) + + +@triton_autotune( + configs=_get_fw_configs(), + key=[ + "AUTOTUNE_Z", + "H", + "AUTOTUNE_MAX_SEQ_LEN", + "DimQ", + "DimV", + "DeltaSize", + "IS_DELTA_Q", + ], + prune_configs_by={"early_config_prune": _early_config_prune}, +) +@triton.jit +def _hstu_attn_fwd( # noqa C901 + Q, + K, + V, + workspace_ptr, + sort_by_length_indices, + seq_offsets, + num_targets, + Out, + stride_qm, + stride_qh, + stride_kn, + stride_kh, + stride_vn, + stride_vh, + stride_om, + stride_oh, + alpha, + Z, + AUTOTUNE_Z, + H, + MAX_SEQ_LEN, + AUTOTUNE_MAX_SEQ_LEN, # Quantized MAX_SEQ_LEN used as an autotuning key + DimQ, + DimV, + DeltaSize, + contextual_seq_len, + max_attn_len, + HAS_MULTIPLE_TARGETS: tl.constexpr, + IS_DELTA_Q: tl.constexpr, + ALLOW_TF32: tl.constexpr, + BLOCK_D_Q: tl.constexpr, + BLOCK_D_V: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + USE_TLX: tl.constexpr, + NUM_BUFFERS: tl.constexpr, # + NUM_MMA_WARPS_PER_GROUP: tl.constexpr, # + NUM_MMA_GROUPS: tl.constexpr, # + HAS_CONTEXTUAL_SEQ_LEN: tl.constexpr, + HAS_MAX_ATTN_LEN: tl.constexpr, + HAS_SORT_BY_LENGTH_INDICES: tl.constexpr, + ENABLE_TMA: tl.constexpr, + TMA_DESC_SIZE: tl.constexpr, +): + off_hz = tl.program_id(1) + off_z = off_hz // H + if HAS_SORT_BY_LENGTH_INDICES: + off_z = tl.load(sort_by_length_indices + off_z) + off_h = off_hz % H + pid = tl.program_id(0) + if USE_TLX: + _hstu_attn_fwd_compute_tlx( + Q=Q, + K=K, + V=V, + H=H, + DimQ=DimQ, + DimV=DimV, + seq_offsets=seq_offsets, + num_targets=num_targets, + Out=Out, + stride_qh=stride_qh, + stride_kh=stride_kh, + stride_vh=stride_vh, + stride_om=stride_om, + stride_oh=stride_oh, + alpha=alpha, + MAX_SEQ_LEN=MAX_SEQ_LEN, + DeltaSize=DeltaSize, + contextual_seq_len=contextual_seq_len, + max_attn_len=max_attn_len, + off_z=off_z, + off_h=off_h, + pid=pid, + HAS_MULTIPLE_TARGETS=HAS_MULTIPLE_TARGETS, + IS_DELTA_Q=IS_DELTA_Q, + ALLOW_TF32=ALLOW_TF32, + BLOCK_D_Q=BLOCK_D_Q, + BLOCK_D_V=BLOCK_D_V, + HAS_CONTEXTUAL_SEQ_LEN=HAS_CONTEXTUAL_SEQ_LEN, + HAS_MAX_ATTN_LEN=HAS_MAX_ATTN_LEN, + BLOCK_M=BLOCK_M, + BLOCK_N=BLOCK_N, + NUM_BUFFERS=NUM_BUFFERS, + NUM_MMA_WARPS_PER_GROUP=NUM_MMA_WARPS_PER_GROUP, + NUM_MMA_GROUPS=NUM_MMA_GROUPS, + ) + else: + _hstu_attn_fwd_compute( + Q=Q, + K=K, + V=V, + H=H, + DimQ=DimQ, + DimV=DimV, + workspace_ptr=workspace_ptr, + seq_offsets=seq_offsets, + num_targets=num_targets, + Out=Out, + stride_qm=stride_qm, + stride_qh=stride_qh, + stride_kn=stride_kn, + stride_kh=stride_kh, + stride_vn=stride_vn, + stride_vh=stride_vh, + stride_om=stride_om, + stride_oh=stride_oh, + alpha=alpha, + MAX_SEQ_LEN=MAX_SEQ_LEN, + DeltaSize=DeltaSize, + contextual_seq_len=contextual_seq_len, + max_attn_len=max_attn_len, + off_z=off_z, + off_h=off_h, + pid=pid, + HAS_MULTIPLE_TARGETS=HAS_MULTIPLE_TARGETS, + IS_DELTA_Q=IS_DELTA_Q, + ALLOW_TF32=ALLOW_TF32, + BLOCK_D_Q=BLOCK_D_Q, + BLOCK_D_V=BLOCK_D_V, + HAS_CONTEXTUAL_SEQ_LEN=HAS_CONTEXTUAL_SEQ_LEN, + HAS_MAX_ATTN_LEN=HAS_MAX_ATTN_LEN, + BLOCK_M=BLOCK_M, + BLOCK_N=BLOCK_N, + ENABLE_TMA=ENABLE_TMA, + TMA_DESC_SIZE=TMA_DESC_SIZE, + ) + + +@triton_autotune( + configs=_get_fw_configs(), + key=[ + "AUTOTUNE_Z", + "H", + "AUTOTUNE_MAX_SEQ_LEN", + "DimQ", + "DimV", + "DeltaSize", + "IS_DELTA_Q", + ], + prune_configs_by={"early_config_prune": _early_config_prune}, +) +@triton.jit +def _hstu_attn_fwd_persistent( # noqa C901 + Q, + K, + V, + workspace_ptr, + sort_by_length_indices, + seq_offsets, + num_targets, + Out, + stride_qm, + stride_qh, + stride_kn, + stride_kh, + stride_vn, + stride_vh, + stride_om, + stride_oh, + alpha, + Z, + AUTOTUNE_Z, + H, + MAX_SEQ_LEN, + AUTOTUNE_MAX_SEQ_LEN, # Quantized MAX_SEQ_LEN used as an autotuning key + DimQ, + DimV, + DeltaSize, + contextual_seq_len, + max_attn_len, + HAS_MULTIPLE_TARGETS: tl.constexpr, + IS_DELTA_Q: tl.constexpr, + ALLOW_TF32: tl.constexpr, + BLOCK_D_Q: tl.constexpr, + BLOCK_D_V: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + USE_TLX: tl.constexpr, + NUM_BUFFERS: tl.constexpr, # + NUM_MMA_WARPS_PER_GROUP: tl.constexpr, # + NUM_MMA_GROUPS: tl.constexpr, # + HAS_CONTEXTUAL_SEQ_LEN: tl.constexpr, + HAS_MAX_ATTN_LEN: tl.constexpr, + HAS_SORT_BY_LENGTH_INDICES: tl.constexpr, + ENABLE_TMA: tl.constexpr, + TMA_DESC_SIZE: tl.constexpr, +): + n_tile_num = tl.cdiv(MAX_SEQ_LEN, BLOCK_M) + prog_id = tl.program_id(0) + num_progs = tl.num_programs(0) + + total_tiles = n_tile_num * Z * H + + tiles_per_sm = total_tiles // num_progs + if prog_id < total_tiles % num_progs: + tiles_per_sm += 1 + + tile_idx = prog_id + for _ in range(0, tiles_per_sm): + pid = (total_tiles - tile_idx - 1) // (Z * H) + off_hz = (total_tiles - tile_idx - 1) % (Z * H) + off_z = off_hz // H + off_h = off_hz % H + _hstu_attn_fwd_compute( + Q=Q, + K=K, + V=V, + H=H, + DimQ=DimQ, + DimV=DimV, + workspace_ptr=workspace_ptr, + seq_offsets=seq_offsets, + num_targets=num_targets, + Out=Out, + stride_qm=stride_qm, + stride_qh=stride_qh, + stride_kn=stride_kn, + stride_kh=stride_kh, + stride_vn=stride_vn, + stride_vh=stride_vh, + stride_om=stride_om, + stride_oh=stride_oh, + alpha=alpha, + MAX_SEQ_LEN=MAX_SEQ_LEN, + DeltaSize=DeltaSize, + contextual_seq_len=contextual_seq_len, + max_attn_len=max_attn_len, + off_z=off_z, + off_h=off_h, + pid=pid, + HAS_MULTIPLE_TARGETS=HAS_MULTIPLE_TARGETS, + IS_DELTA_Q=IS_DELTA_Q, + ALLOW_TF32=ALLOW_TF32, + BLOCK_D_Q=BLOCK_D_Q, + BLOCK_D_V=BLOCK_D_V, + HAS_CONTEXTUAL_SEQ_LEN=HAS_CONTEXTUAL_SEQ_LEN, + HAS_MAX_ATTN_LEN=HAS_MAX_ATTN_LEN, + BLOCK_M=BLOCK_M, + BLOCK_N=BLOCK_N, + ENABLE_TMA=ENABLE_TMA, + TMA_DESC_SIZE=TMA_DESC_SIZE, + ) + tile_idx += num_progs + + +@triton.jit +def _hstu_attn_bwd_one_block( # noqa C901 + start_m, + offs_n, + offs_m, + q_ptrs_trans, + dq_ptrs_trans, + do_ptrs, + device_desc_q, + device_desc_do, + dk, + dv, + k, + v, + pos_offs_n, + seq_len, + max_ids, + contextual_seq_len, + max_attn_len, + LOCK, + off_h, + stride_qh, + stride_doh, + stride_qm, + stride_dom, + stride_dqm, + alpha, + MAX_SEQ_LEN, + HAS_MULTIPLE_TARGETS: tl.constexpr, + HAS_CONTEXTUAL_SEQ_LEN: tl.constexpr, + HAS_MAX_ATTN_LEN: tl.constexpr, + ALLOW_TF32: tl.constexpr, + BLOCK_M: tl.constexpr, + ATOMIC_ADD: tl.constexpr, + ENABLE_TMA: tl.constexpr, + BLOCK_D_Q: tl.constexpr, + BLOCK_D_V: tl.constexpr, +): + pos_offs_m = offs_m + start_m + mask_m = pos_offs_m < seq_len + invalid_mask_trans = pos_offs_m[None, :] == offs_n[:, None] + # recompute qk and silu + if HAS_CONTEXTUAL_SEQ_LEN: + pos_offs_m = pos_offs_m - contextual_seq_len + 1 + pos_offs_m = tl.where( + pos_offs_m > 0, + pos_offs_m, + 0, + ) + if HAS_MULTIPLE_TARGETS: + pos_offs_m = tl.where( + pos_offs_m < max_ids, + pos_offs_m, + max_ids, + ) + if ENABLE_TMA: + q = device_desc_q.load( + [start_m, (off_h * stride_qh).to(tl.int32)], + ) + q_trans = tl.trans(q) + else: + q_trans = tl.load( + q_ptrs_trans + start_m * stride_qm, + mask=mask_m[None, :], + other=0.0, + ) + qk_trans = tl.dot(k, q_trans, allow_tf32=ALLOW_TF32) * alpha + sig_trans = fast_dividef(1.0, 1.0 + tl.exp(-qk_trans)) + silu_trans = qk_trans * sig_trans * (1.0 / MAX_SEQ_LEN) + pos_offs_m_minus_n = pos_offs_m[None, :] - pos_offs_n[:, None] + invalid_mask_trans = invalid_mask_trans or (pos_offs_m_minus_n > 0) + if HAS_MAX_ATTN_LEN: + invalid_mask_trans = invalid_mask_trans and pos_offs_m_minus_n <= max_attn_len + if HAS_CONTEXTUAL_SEQ_LEN: + invalid_mask_trans = invalid_mask_trans or ( + pos_offs_m[None, :] == 0 and pos_offs_n[:, None] < max_ids + ) + silu_trans = tl.where(invalid_mask_trans, silu_trans, 0) + silu_trans = silu_trans.to(k.dtype) + # compute dv + if ENABLE_TMA: + do = device_desc_do.load( + [start_m, (off_h * stride_doh).to(tl.int32)], + ) + else: + do = tl.load( + do_ptrs + start_m * stride_dom, + mask=mask_m[:, None], + other=0.0, + ) + dv += tl.dot(silu_trans, do, allow_tf32=ALLOW_TF32) + + # compute dk and dq + dqk_trans = tl.dot(v, tl.trans(do), allow_tf32=ALLOW_TF32) + dqk_trans = ( + dqk_trans * sig_trans * (1 + qk_trans * (1 - sig_trans)) * (1.0 / MAX_SEQ_LEN) + ) + dqk_trans = tl.where(invalid_mask_trans, dqk_trans, 0) + dqk_trans = dqk_trans.to(k.dtype) + + # Note: the factor `alpha` is delayed until the end of the function to reduce the cost + dk += tl.dot(dqk_trans, tl.trans(q_trans), allow_tf32=ALLOW_TF32) + acc_dq( + dq_ptrs_trans=dq_ptrs_trans, + start_m=start_m, + stride_dqm=stride_dqm, + k=k, + dqk_trans=dqk_trans, + alpha=alpha, + mask_m=mask_m, + MAX_SEQ_LEN=MAX_SEQ_LEN, + LOCK=LOCK, + BLOCK_M=BLOCK_M, + ATOMIC_ADD=ATOMIC_ADD, + ALLOW_TF32=ALLOW_TF32, + ) + return dk, dv + + +@triton.jit +def _hstu_attn_bwd_one_col_block( # noqa C901 + start_n, + seq_len, + n_targets, + contextual_seq_len, + max_attn_len, + Q, + K, + V, + DOut, + DQ, + DK, + DV, + device_desc_q, + device_desc_k, + device_desc_v, + device_desc_do, + device_desc_dk, + device_desc_dv, + LOCK, + off_h, + stride_qh, + stride_kh, + stride_vh, + stride_doh, + stride_dkh, + stride_dvh, + stride_qm, + stride_kn, + stride_vn, + stride_dom, + stride_dqm, + stride_dkn, + stride_dvn, + alpha, + MAX_SEQ_LEN, + HAS_MULTIPLE_TARGETS: tl.constexpr, + HAS_CONTEXTUAL_SEQ_LEN: tl.constexpr, + HAS_MAX_ATTN_LEN: tl.constexpr, + ALLOW_TF32: tl.constexpr, + BLOCK_D_Q: tl.constexpr, + BLOCK_D_V: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + UNROLL: tl.constexpr, + ATOMIC_ADD: tl.constexpr, + ENABLE_TMA: tl.constexpr, +): + if HAS_MULTIPLE_TARGETS: + low = start_n + if HAS_MAX_ATTN_LEN: + high = start_n + max_attn_len + BLOCK_N + high = high if high + n_targets < seq_len else seq_len + else: + high = seq_len + else: + low = start_n + if HAS_MAX_ATTN_LEN: + high = start_n + max_attn_len + BLOCK_N + high = high if high < seq_len else seq_len + else: + high = seq_len + if HAS_CONTEXTUAL_SEQ_LEN: + contextual_block_end = tl.cdiv(contextual_seq_len, BLOCK_M) * BLOCK_M + if low < contextual_block_end: + low = contextual_block_end + + offs_m = tl.arange(0, BLOCK_M) + offs_qk_d = tl.arange(0, BLOCK_D_Q) + offs_v_d = tl.arange(0, BLOCK_D_V) + offs_n = start_n + tl.arange(0, BLOCK_N) + + dq_ptrs_trans = DQ + (offs_m[None, :] * stride_dqm + offs_qk_d[:, None]) + dv = tl.zeros([BLOCK_N, BLOCK_D_V], dtype=tl.float32) + dk = tl.zeros([BLOCK_N, BLOCK_D_Q], dtype=tl.float32) + if ENABLE_TMA: + q_ptrs_trans = None + do_ptrs = None + k = device_desc_k.load( + [start_n, (off_h * stride_kh).to(tl.int32)], + ) + v = device_desc_v.load( + [start_n, (off_h * stride_vh).to(tl.int32)], + ) + else: + mask_n = offs_n < seq_len + q_ptrs_trans = Q + (offs_m[None, :] * stride_qm + offs_qk_d[:, None]) + do_ptrs = DOut + (offs_m[:, None] * stride_dom + offs_v_d[None, :]) + k_ptrs = K + (offs_n[:, None] * stride_kn + offs_qk_d[None, :]) + v_ptrs = V + (offs_n[:, None] * stride_vn + offs_v_d[None, :]) + k = tl.load(k_ptrs, mask=mask_n[:, None], other=0.0) + v = tl.load(v_ptrs, mask=mask_n[:, None], other=0.0) + max_ids = seq_len + if HAS_CONTEXTUAL_SEQ_LEN: + pos_offs_n = offs_n - contextual_seq_len + 1 + pos_offs_n = tl.where( + pos_offs_n > 0, + pos_offs_n, + 0, + ) + max_ids = max_ids - contextual_seq_len + 1 + else: + pos_offs_n = offs_n + if HAS_MULTIPLE_TARGETS: + max_ids = max_ids - n_targets + pos_offs_n = tl.where( + pos_offs_n < max_ids, + pos_offs_n, + max_ids, + ) + # loop over rows + if HAS_CONTEXTUAL_SEQ_LEN: + for start_m in range(0, contextual_seq_len, BLOCK_M): + start_m = tl.multiple_of(start_m, BLOCK_M) + dk, dv = _hstu_attn_bwd_one_block( + start_m=start_m, + offs_n=offs_n, + offs_m=offs_m, + q_ptrs_trans=q_ptrs_trans, + dq_ptrs_trans=dq_ptrs_trans, + do_ptrs=do_ptrs, + device_desc_q=device_desc_q, + device_desc_do=device_desc_do, + dk=dk, + dv=dv, + k=k, + v=v, + pos_offs_n=pos_offs_n, + seq_len=seq_len, + max_ids=max_ids, + contextual_seq_len=contextual_seq_len, + max_attn_len=max_attn_len, + LOCK=LOCK, + off_h=off_h, + stride_qh=stride_qh, + stride_doh=stride_doh, + stride_qm=stride_qm, + stride_dom=stride_dom, + stride_dqm=stride_dqm, + alpha=alpha, + MAX_SEQ_LEN=MAX_SEQ_LEN, + HAS_MULTIPLE_TARGETS=HAS_MULTIPLE_TARGETS, + HAS_CONTEXTUAL_SEQ_LEN=HAS_CONTEXTUAL_SEQ_LEN, + HAS_MAX_ATTN_LEN=HAS_MAX_ATTN_LEN, + ALLOW_TF32=ALLOW_TF32, + BLOCK_M=BLOCK_M, + ATOMIC_ADD=ATOMIC_ADD, + ENABLE_TMA=ENABLE_TMA, + BLOCK_D_Q=BLOCK_D_Q, + BLOCK_D_V=BLOCK_D_V, + ) + for start_m in tl.range(low, high, BLOCK_M, loop_unroll_factor=UNROLL): + start_m = tl.multiple_of(start_m, BLOCK_M) + dk, dv = _hstu_attn_bwd_one_block( + start_m=start_m, + offs_n=offs_n, + offs_m=offs_m, + q_ptrs_trans=q_ptrs_trans, + dq_ptrs_trans=dq_ptrs_trans, + do_ptrs=do_ptrs, + device_desc_q=device_desc_q, + device_desc_do=device_desc_do, + dk=dk, + dv=dv, + k=k, + v=v, + pos_offs_n=pos_offs_n, + seq_len=seq_len, + max_ids=max_ids, + contextual_seq_len=contextual_seq_len, + max_attn_len=max_attn_len, + LOCK=LOCK, + off_h=off_h, + stride_qh=stride_qh, + stride_doh=stride_doh, + stride_qm=stride_qm, + stride_dom=stride_dom, + stride_dqm=stride_dqm, + alpha=alpha, + MAX_SEQ_LEN=MAX_SEQ_LEN, + HAS_MULTIPLE_TARGETS=HAS_MULTIPLE_TARGETS, + HAS_CONTEXTUAL_SEQ_LEN=HAS_CONTEXTUAL_SEQ_LEN, + HAS_MAX_ATTN_LEN=HAS_MAX_ATTN_LEN, + ALLOW_TF32=ALLOW_TF32, + BLOCK_M=BLOCK_M, + ATOMIC_ADD=ATOMIC_ADD, + ENABLE_TMA=ENABLE_TMA, + BLOCK_D_Q=BLOCK_D_Q, + BLOCK_D_V=BLOCK_D_V, + ) + # write-back + dk = dk * alpha + if ENABLE_TMA: + device_desc_dv.store( + [start_n, (off_h * stride_dvh).to(tl.int32)], + dv.to(k.dtype), + ) + device_desc_dk.store( + [start_n, (off_h * stride_dkh).to(tl.int32)], + dk.to(k.dtype), + ) + else: + dv_ptrs = DV + (offs_n[:, None] * stride_dvn + offs_v_d[None, :]) + dk_ptrs = DK + (offs_n[:, None] * stride_dkn + offs_qk_d[None, :]) + tl.store(dv_ptrs, dv.to(k.dtype), mask=mask_n[:, None]) # pyre-ignore[61] + tl.store(dk_ptrs, dk.to(k.dtype), mask=mask_n[:, None]) # pyre-ignore[61] + + +def _bwd_pre_hook(nargs): + nargs["DQ"].zero_() + if nargs["SEQUENCE_PARALLEL"] is True: + nargs["LOCK"].zero_() + + +def _get_bw_configs() -> List[triton.Config]: + if torch.version.hip: + configs = [] + for BLOCK_M in [32, 64]: + for BLOCK_N in [32, 64, 128]: + for num_stages in [1, 2]: + for num_warps in [4, 8]: + for matrix_instr_nonkdim in [16, 32]: + for waves_per_eu in [0, 2, 4]: + for sp in [True, False]: + configs.append( + triton.Config( + { + "BLOCK_M": BLOCK_M, + "BLOCK_N": BLOCK_N, + "matrix_instr_nonkdim": matrix_instr_nonkdim, + "waves_per_eu": waves_per_eu, + "SEQUENCE_PARALLEL": sp, + "UNROLL": 1, + }, + num_stages=num_stages, + num_warps=num_warps, + pre_hook=_bwd_pre_hook, + ) + ) + return configs + + configs = [ + triton.Config( + {"BLOCK_M": 16, "BLOCK_N": 32, "SEQUENCE_PARALLEL": False, "UNROLL": 1}, + num_stages=2, + num_warps=2, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 16, "BLOCK_N": 16, "SEQUENCE_PARALLEL": False, "UNROLL": 1}, + num_stages=2, + num_warps=2, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 16, "BLOCK_N": 32, "SEQUENCE_PARALLEL": False, "UNROLL": 1}, + num_stages=2, + num_warps=4, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 16, "BLOCK_N": 32, "SEQUENCE_PARALLEL": False, "UNROLL": 1}, + num_stages=1, + num_warps=8, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 32, "SEQUENCE_PARALLEL": False, "UNROLL": 1}, + num_stages=1, + num_warps=4, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 32, "SEQUENCE_PARALLEL": False, "UNROLL": 1}, + num_stages=2, + num_warps=4, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 64, "SEQUENCE_PARALLEL": False, "UNROLL": 1}, + num_stages=2, + num_warps=4, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 64, "SEQUENCE_PARALLEL": False, "UNROLL": 1}, + num_stages=2, + num_warps=8, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 64, "BLOCK_N": 64, "SEQUENCE_PARALLEL": False, "UNROLL": 1}, + num_stages=1, + num_warps=4, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 64, "BLOCK_N": 64, "SEQUENCE_PARALLEL": False, "UNROLL": 1}, + num_stages=2, + num_warps=4, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 64, "BLOCK_N": 64, "SEQUENCE_PARALLEL": False, "UNROLL": 1}, + num_stages=1, + num_warps=8, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 64, "BLOCK_N": 64, "SEQUENCE_PARALLEL": False, "UNROLL": 1}, + num_stages=2, + num_warps=8, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 128, "SEQUENCE_PARALLEL": False, "UNROLL": 1}, + num_stages=2, + num_warps=8, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 128, "SEQUENCE_PARALLEL": False, "UNROLL": 1}, + num_stages=3, + num_warps=8, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 128, "SEQUENCE_PARALLEL": False, "UNROLL": 4}, + num_stages=2, + num_warps=8, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 16, "BLOCK_N": 32, "SEQUENCE_PARALLEL": True, "UNROLL": 1}, + num_stages=2, + num_warps=2, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 32, "SEQUENCE_PARALLEL": True, "UNROLL": 1}, + num_stages=1, + num_warps=4, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 32, "SEQUENCE_PARALLEL": True, "UNROLL": 1}, + num_stages=2, + num_warps=4, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 64, "BLOCK_N": 64, "SEQUENCE_PARALLEL": True, "UNROLL": 1}, + num_stages=1, + num_warps=4, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 64, "BLOCK_N": 64, "SEQUENCE_PARALLEL": True, "UNROLL": 1}, + num_stages=2, + num_warps=4, + pre_hook=_bwd_pre_hook, + ), + ] + if torch.cuda.is_available() and torch.version.cuda < "12.8": + configs += [ + triton.Config( + {"BLOCK_M": 16, "BLOCK_N": 64, "SEQUENCE_PARALLEL": False, "UNROLL": 1}, + num_stages=1, + num_warps=4, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 64, "SEQUENCE_PARALLEL": False, "UNROLL": 1}, + num_stages=1, + num_warps=4, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 64, "SEQUENCE_PARALLEL": False, "UNROLL": 1}, + num_stages=1, + num_warps=8, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 64, "SEQUENCE_PARALLEL": True, "UNROLL": 1}, + num_stages=1, + num_warps=8, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 128, "SEQUENCE_PARALLEL": True, "UNROLL": 1}, + num_stages=3, + num_warps=8, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 64, "SEQUENCE_PARALLEL": True, "UNROLL": 1}, + num_stages=1, + num_warps=4, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 64, "SEQUENCE_PARALLEL": True, "UNROLL": 1}, + num_stages=2, + num_warps=4, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + { + "BLOCK_M": 32, + "BLOCK_N": 128, + "SEQUENCE_PARALLEL": False, + "UNROLL": 2, + }, + num_stages=2, + num_warps=8, + pre_hook=_bwd_pre_hook, + ), + ] + else: + print("WARNING: temporarily disabled some autotune configs for CUDA 12.8+") + return configs + + +@triton_autotune( + configs=_get_bw_configs(), + key=[ + "AUTOTUNE_Z", + "H", + "AUTOTUNE_MAX_SEQ_LEN", + "DimQ", + "DimV", + ], +) +@triton.jit +def _hstu_attn_bwd( # noqa C901 + Q, + K, + V, + tma_workspace_ptr, + sort_by_length_indices, + seq_offsets, + num_targets, + DOut, + DQ, + DK, + DV, + LOCK, + stride_qm, + stride_qh, + stride_kn, + stride_kh, + stride_vn, + stride_vh, + stride_dom, + stride_doh, + stride_dqm, + stride_dqh, + stride_dkn, + stride_dkh, + stride_dvn, + stride_dvh, + alpha, + contextual_seq_len, + max_attn_len, + Z, + AUTOTUNE_Z, + H, + MAX_SEQ_LEN, + AUTOTUNE_MAX_SEQ_LEN, # Quantized MAX_SEQ_LEN used as an autotuning key + DimQ, + DimV, + HAS_MULTIPLE_TARGETS: tl.constexpr, + HAS_CONTEXTUAL_SEQ_LEN: tl.constexpr, + HAS_MAX_ATTN_LEN: tl.constexpr, + ALLOW_TF32: tl.constexpr, + BLOCK_D_Q: tl.constexpr, + BLOCK_D_V: tl.constexpr, + SEQUENCE_PARALLEL: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + UNROLL: tl.constexpr, + HAS_SORT_BY_LENGTH_INDICES: tl.constexpr, + ENABLE_TMA: tl.constexpr, + TMA_DESC_SIZE: tl.constexpr, + ENABLE_BUFFER_OPS_ASSUMES: tl.constexpr, +): + off_hz = tl.program_id(0) + off_z = off_hz // H + if HAS_SORT_BY_LENGTH_INDICES: + off_z = tl.load(sort_by_length_indices + off_z) + off_h = off_hz % H + off_h = off_h.to(tl.int64) + seq_start = tl.load(seq_offsets + off_z).to(tl.int64) + seq_end = tl.load(seq_offsets + off_z + 1) + seq_len = (seq_end - seq_start).to(tl.int32) + if HAS_MULTIPLE_TARGETS: + n_targets = tl.load(num_targets + off_z).to(tl.int32) + else: + n_targets = None + if ENABLE_BUFFER_OPS_ASSUMES: + tl.assume(off_hz >= 0) + tl.assume(off_z >= 0) + tl.assume(off_h >= 0) + tl.assume(seq_start >= 0) + tl.assume(stride_qm >= 0) + tl.assume(stride_qh >= 0) + tl.assume(stride_kn >= 0) + tl.assume(stride_kh >= 0) + tl.assume(stride_vn >= 0) + tl.assume(stride_vh >= 0) + tl.assume(stride_dom >= 0) + tl.assume(stride_doh >= 0) + tl.assume(stride_dqm >= 0) + tl.assume(stride_dqh >= 0) + tl.assume(stride_dkn >= 0) + tl.assume(stride_dkh >= 0) + tl.assume(stride_dvn >= 0) + tl.assume(stride_dvh >= 0) + + # offset pointers for batch/head + Q = Q + seq_start * stride_qm + K = K + seq_start * stride_kn + V = V + seq_start * stride_vn + DOut = DOut + seq_start * stride_dom + DQ = DQ + seq_start * stride_dqm + off_h * stride_dqh + DK = DK + seq_start * stride_dkn + DV = DV + seq_start * stride_dvn + device_desc_q = None + device_desc_k = None + device_desc_v = None + device_desc_do = None + device_desc_dk = None + device_desc_dv = None + if ENABLE_TMA: + device_desc_q = tl.make_tensor_descriptor( + Q, + shape=[seq_len, H * DimQ], + strides=[H * DimQ, 1], + block_shape=[BLOCK_M, BLOCK_D_Q], + ) + device_desc_do = tl.make_tensor_descriptor( + DOut, + shape=[seq_len, H * DimV], + strides=[H * DimV, 1], + block_shape=[BLOCK_M, BLOCK_D_V], + ) + device_desc_k = tl.make_tensor_descriptor( + K, + shape=[seq_len, H * DimQ], + strides=[H * DimQ, 1], + block_shape=[BLOCK_N, BLOCK_D_Q], + ) + device_desc_dk = tl.make_tensor_descriptor( + DK, + shape=[seq_len, H * DimQ], + strides=[H * DimQ, 1], + block_shape=[BLOCK_N, BLOCK_D_Q], + ) + device_desc_v = tl.make_tensor_descriptor( + V, + shape=[seq_len, H * DimV], + strides=[H * DimV, 1], + block_shape=[BLOCK_N, BLOCK_D_V], + ) + device_desc_dv = tl.make_tensor_descriptor( + DV, + shape=[seq_len, H * DimV], + strides=[H * DimV, 1], + block_shape=[BLOCK_N, BLOCK_D_V], + ) + else: + Q += off_h * stride_qh + K += off_h * stride_kh + V += off_h * stride_vh + DOut += off_h * stride_doh + DK += off_h * stride_dkh + DV += off_h * stride_dvh + if SEQUENCE_PARALLEL: + start_n = tl.program_id(1) * BLOCK_N + if start_n >= seq_len: + return + _hstu_attn_bwd_one_col_block( + start_n=start_n, + seq_len=seq_len, + n_targets=n_targets, + contextual_seq_len=contextual_seq_len, + max_attn_len=max_attn_len, + Q=Q, + K=K, + V=V, + DOut=DOut, + DQ=DQ, + DK=DK, + DV=DV, + device_desc_q=device_desc_q, + device_desc_k=device_desc_k, + device_desc_v=device_desc_v, + device_desc_do=device_desc_do, + device_desc_dk=device_desc_dk, + device_desc_dv=device_desc_dv, + LOCK=LOCK, + off_h=off_h, + stride_qh=stride_qh, + stride_kh=stride_kh, + stride_vh=stride_vh, + stride_doh=stride_doh, + stride_dkh=stride_dkh, + stride_dvh=stride_dvh, + stride_qm=stride_qm, + stride_kn=stride_kn, + stride_vn=stride_vn, + stride_dom=stride_dom, + stride_dqm=stride_dqm, + stride_dkn=stride_dkn, + stride_dvn=stride_dvn, + alpha=alpha, + MAX_SEQ_LEN=MAX_SEQ_LEN, + HAS_MULTIPLE_TARGETS=HAS_MULTIPLE_TARGETS, + HAS_CONTEXTUAL_SEQ_LEN=HAS_CONTEXTUAL_SEQ_LEN, + HAS_MAX_ATTN_LEN=HAS_MAX_ATTN_LEN, + ALLOW_TF32=ALLOW_TF32, + BLOCK_D_Q=BLOCK_D_Q, + BLOCK_D_V=BLOCK_D_V, + BLOCK_M=BLOCK_M, + BLOCK_N=BLOCK_N, + UNROLL=UNROLL, + ATOMIC_ADD=True, + ENABLE_TMA=ENABLE_TMA, + ) + else: + for start_n in range(0, seq_len, BLOCK_N): + _hstu_attn_bwd_one_col_block( + start_n=start_n, + seq_len=seq_len, + n_targets=n_targets, + contextual_seq_len=contextual_seq_len, + max_attn_len=max_attn_len, + Q=Q, + K=K, + V=V, + DOut=DOut, + DQ=DQ, + DK=DK, + DV=DV, + device_desc_q=device_desc_q, + device_desc_k=device_desc_k, + device_desc_v=device_desc_v, + device_desc_do=device_desc_do, + device_desc_dk=device_desc_dk, + device_desc_dv=device_desc_dv, + LOCK=LOCK, + off_h=off_h, + stride_qh=stride_qh, + stride_kh=stride_kh, + stride_vh=stride_vh, + stride_doh=stride_doh, + stride_dkh=stride_dkh, + stride_dvh=stride_dvh, + stride_qm=stride_qm, + stride_kn=stride_kn, + stride_vn=stride_vn, + stride_dom=stride_dom, + stride_dqm=stride_dqm, + stride_dkn=stride_dkn, + stride_dvn=stride_dvn, + alpha=alpha, + MAX_SEQ_LEN=MAX_SEQ_LEN, + HAS_MULTIPLE_TARGETS=HAS_MULTIPLE_TARGETS, + HAS_CONTEXTUAL_SEQ_LEN=HAS_CONTEXTUAL_SEQ_LEN, + HAS_MAX_ATTN_LEN=HAS_MAX_ATTN_LEN, + ALLOW_TF32=ALLOW_TF32, + BLOCK_D_Q=BLOCK_D_Q, + BLOCK_D_V=BLOCK_D_V, + BLOCK_M=BLOCK_M, + BLOCK_N=BLOCK_N, + UNROLL=UNROLL, + ATOMIC_ADD=False, + ENABLE_TMA=ENABLE_TMA, + ) + + +@maybe_register_custom_op( + "generative_recommenders::triton_hstu_attention_fwd", mutates_args=() +) +def triton_hstu_attention_fwd( + N: int, + alpha: float, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + seq_offsets: torch.Tensor, + num_targets: Optional[torch.Tensor], + max_attn_len: int, + contextual_seq_len: int, + sort_by_length_indices: Optional[torch.Tensor], + enable_tma: bool, + num_softmax_heads: int, +) -> torch.Tensor: + Z = seq_offsets.numel() - 1 + AUTOTUNE_Z = prev_power_of_2(Z) + L, H, DimQ = q.shape + _, _, DimV = v.shape + out = torch.empty_like(v) + has_multiple_targets = num_targets is not None + has_contextual_seq_len = contextual_seq_len > 0 + has_max_attn_len = max_attn_len > 0 + has_sort_by_length_indices = sort_by_length_indices is not None + if L == 0: + return out + + TMA_DESC_SIZE = 128 + workspace = None + desc_q = q + desc_k = k + desc_v = v + + if enable_tma and tensor_descriptor_tma: + dummy_block = [1, 1] + desc_q = TensorDescriptor( + q, + shape=[L, H * DimQ], + strides=[H * DimQ, 1], + block_shape=dummy_block, + ) + desc_v = TensorDescriptor( + v, + shape=[L, H * DimV], + strides=[H * DimV, 1], + block_shape=dummy_block, + ) + desc_k = TensorDescriptor( + k, + shape=[L, H * DimQ], + strides=[H * DimQ, 1], + block_shape=dummy_block, + ) + + def alloc_fn(size: int, align: int, stream: Optional[int]): + assert align == TMA_DESC_SIZE + return torch.empty(size, dtype=torch.int8, device="cuda") + + # pyre-ignore [6] + triton.set_allocator(alloc_fn) + grid = lambda meta: ( # noqa E731 + triton.cdiv(N, meta["BLOCK_M"]), + Z * H, + ) + + _hstu_attn_fwd[grid]( + Q=desc_q, + K=desc_k, + V=desc_v, + workspace_ptr=workspace, + sort_by_length_indices=sort_by_length_indices, + seq_offsets=seq_offsets, + num_targets=num_targets, + Out=out, + stride_qm=q.stride(0), + stride_qh=q.stride(1), + stride_kn=k.stride(0), + stride_kh=k.stride(1), + stride_vn=v.stride(0), + stride_vh=v.stride(1), + stride_om=out.stride(0), + stride_oh=out.stride(1), + alpha=alpha, + Z=Z, + AUTOTUNE_Z=AUTOTUNE_Z, + H=H, + MAX_SEQ_LEN=N, + AUTOTUNE_MAX_SEQ_LEN=autotune_max_seq_len(N), + DimQ=DimQ, + DimV=DimV, + DeltaSize=0, + contextual_seq_len=contextual_seq_len, + max_attn_len=max_attn_len, + HAS_MULTIPLE_TARGETS=has_multiple_targets, + IS_DELTA_Q=False, + ALLOW_TF32=torch.backends.cuda.matmul.allow_tf32, + BLOCK_D_Q=DimQ, + BLOCK_D_V=DimV, + HAS_CONTEXTUAL_SEQ_LEN=has_contextual_seq_len, + HAS_MAX_ATTN_LEN=has_max_attn_len, + HAS_SORT_BY_LENGTH_INDICES=has_sort_by_length_indices, + ENABLE_TMA=enable_tma, + TMA_DESC_SIZE=TMA_DESC_SIZE, + ) + return out + + +@maybe_register_custom_op( + "generative_recommenders::triton_hstu_attention_bwd", + mutates_args=("dq", "dk", "dv"), +) +def triton_hstu_attention_bwd( + dout: torch.Tensor, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + dq: torch.Tensor, + dk: torch.Tensor, + dv: torch.Tensor, + seq_offsets: torch.Tensor, + num_targets: Optional[torch.Tensor], + N: int, + alpha: float, + max_attn_len: int, + contextual_seq_len: int, + sort_by_length_indices: Optional[torch.Tensor], + enable_tma: bool, + num_softmax_heads: int, +) -> None: + orig_dq, orig_dk, orig_dv = dq, dk, dv + dout = switch_to_contiguous_if_needed(dout) + dq = switch_to_contiguous_if_needed(dq) + dk = switch_to_contiguous_if_needed(dk) + dv = switch_to_contiguous_if_needed(dv) + if dout.shape[0] == 0: + orig_dq.zero_() + orig_dk.zero_() + orig_dv.zero_() + return + Z = seq_offsets.numel() - 1 + _, H, DimQ = q.shape + _, _, DimV = v.shape + grid = lambda meta: ( # noqa E731 + Z * H, + (triton.cdiv(N, meta["BLOCK_N"]) if meta["SEQUENCE_PARALLEL"] else 1), + ) + # The minimum size of BLOCK_M used in `_get_bw_configs`. + # TODO (linjianma): avoid hardcoding the value. + MIN_BLOCK_M = 16 + lock = torch.empty( + (Z * H, triton.cdiv(N, MIN_BLOCK_M)), + dtype=torch.int32, + device=q.device, + ) + AUTOTUNE_Z = prev_power_of_2(Z) + TMA_DESC_SIZE = 128 + tma_workspace = None + + def alloc_fn(size: int, align: int, stream: Optional[int]): + assert align == TMA_DESC_SIZE + return torch.empty(size, dtype=torch.int8, device="cuda") + + # pyre-ignore [6] + triton.set_allocator(alloc_fn) + + # Enable BufferOps on AMD + ENABLE_BUFFER_OPS_ASSUMES = torch.version.hip is not None + _hstu_attn_bwd[grid]( + Q=q, + K=k, + V=v, + tma_workspace_ptr=tma_workspace, + sort_by_length_indices=sort_by_length_indices, + seq_offsets=seq_offsets, + num_targets=num_targets, + DOut=dout, + DQ=dq, + DK=dk, + DV=dv, + LOCK=lock, + stride_qm=q.stride(0), + stride_qh=q.stride(1), + stride_kn=k.stride(0), + stride_kh=k.stride(1), + stride_vn=v.stride(0), + stride_vh=v.stride(1), + stride_dom=dout.stride(0), + stride_doh=dout.stride(1), + stride_dqm=dq.stride(0), + stride_dqh=dq.stride(1), + stride_dkn=dk.stride(0), + stride_dkh=dk.stride(1), + stride_dvn=dv.stride(0), + stride_dvh=dv.stride(1), + alpha=alpha, + contextual_seq_len=contextual_seq_len, + max_attn_len=max_attn_len, + Z=Z, + AUTOTUNE_Z=AUTOTUNE_Z, + H=H, + MAX_SEQ_LEN=N, + AUTOTUNE_MAX_SEQ_LEN=autotune_max_seq_len(N), + DimQ=DimQ, + DimV=DimV, + HAS_MULTIPLE_TARGETS=num_targets is not None, + HAS_CONTEXTUAL_SEQ_LEN=contextual_seq_len > 0, + HAS_MAX_ATTN_LEN=max_attn_len > 0, + ALLOW_TF32=torch.backends.cuda.matmul.allow_tf32, + BLOCK_D_Q=DimQ, + BLOCK_D_V=DimV, + HAS_SORT_BY_LENGTH_INDICES=sort_by_length_indices is not None, + ENABLE_TMA=enable_tma, + TMA_DESC_SIZE=TMA_DESC_SIZE, + ENABLE_BUFFER_OPS_ASSUMES=ENABLE_BUFFER_OPS_ASSUMES, + ) + + copy_if_different_ptr(orig_dq, dq) + copy_if_different_ptr(orig_dk, dk) + copy_if_different_ptr(orig_dv, dv) + + +@triton_hstu_attention_fwd.register_fake +def _triton_hstu_attention_fwd_fake( + N: int, + alpha: float, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + seq_offsets: torch.Tensor, + num_targets: Optional[torch.Tensor], + max_attn_len: int, + contextual_seq_len: int, + sort_by_length_indices: Optional[torch.Tensor], + enable_tma: bool, + num_softmax_heads: int, +) -> torch.Tensor: + L, H, _ = q.shape + _, _, DimV = v.shape + out = torch.empty((L, H, DimV), dtype=v.dtype, device=v.device) + return out + + +@triton_hstu_attention_bwd.register_fake +def _triton_hstu_attention_bwd_fake( + dout: torch.Tensor, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + dq: torch.Tensor, + dk: torch.Tensor, + dv: torch.Tensor, + seq_offsets: torch.Tensor, + num_targets: Optional[torch.Tensor], + N: int, + alpha: float, + max_attn_len: int, + contextual_seq_len: int, + sort_by_length_indices: Optional[torch.Tensor], + enable_tma: bool, + num_softmax_heads: int, +) -> None: + return None + + +class _AttentionFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore[14] + def forward( + ctx, + N: int, + alpha: float, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + seq_offsets: torch.Tensor, + num_targets: Optional[torch.Tensor], + max_attn_len: int, + contextual_seq_len: int, + sort_by_length: bool, + enable_tma: bool, + ) -> torch.Tensor: + sort_by_length_indices = None + if sort_by_length: + seq_lengths = seq_offsets[1:] - seq_offsets[:-1] + _, sort_by_length_indices = torch.sort( + seq_lengths, descending=True, stable=False + ) + saved_tensors = [q, k, v, seq_offsets] + if num_targets is not None: + saved_tensors.append(num_targets) + if sort_by_length_indices is not None: + saved_tensors.append(sort_by_length_indices) + ctx.save_for_backward(*saved_tensors) + ctx.alpha = alpha + ctx.has_multiple_targets = num_targets is not None + ctx.max_attn_len = max_attn_len + ctx.N = N + ctx.contextual_seq_len = contextual_seq_len + ctx.sort_by_length = sort_by_length + ctx.enable_tma = enable_tma + return triton_hstu_attention_fwd( + N=N, + alpha=alpha, + q=q, + k=k, + v=v, + seq_offsets=seq_offsets, + num_targets=num_targets, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + sort_by_length_indices=sort_by_length_indices, + enable_tma=enable_tma, + num_softmax_heads=0, + ) + + @staticmethod + # pyre-ignore[14] + def backward( + ctx, dout: torch.Tensor + ) -> Tuple[ + None, + None, + torch.Tensor, + torch.Tensor, + torch.Tensor, + None, + None, + None, + None, + None, + None, + ]: + with torch.inference_mode(): + q, k, v, seq_offsets = ctx.saved_tensors[:4] + idx = 4 + if ctx.has_multiple_targets: + num_targets = ctx.saved_tensors[idx] + idx += 1 + else: + num_targets = None + if ctx.sort_by_length: + sort_by_length_indices = ctx.saved_tensors[idx] + else: + sort_by_length_indices = None + + dq = torch.empty_like(q) + dk = torch.empty_like(k) + dv = torch.empty_like(v) + triton_hstu_attention_bwd( + dout=dout, + q=q, + k=k, + v=v, + dq=dq, + dk=dk, + dv=dv, + seq_offsets=seq_offsets, + num_targets=num_targets, + N=ctx.N, + alpha=ctx.alpha, + max_attn_len=ctx.max_attn_len, + contextual_seq_len=ctx.contextual_seq_len, + sort_by_length_indices=sort_by_length_indices, + enable_tma=ctx.enable_tma, + num_softmax_heads=0, + ) + return ( + None, + None, + dq, + dk, + dv, + None, + None, + None, + None, + None, + None, + ) + + +@torch.jit.unused +@torch.fx.wrap +def triton_hstu_mha( + N: int, + alpha: float, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + seq_offsets: torch.Tensor, + num_targets: Optional[torch.Tensor] = None, + max_attn_len: int = 0, + contextual_seq_len: int = 0, + sort_by_length: bool = False, + enable_tma: bool = False, +) -> torch.Tensor: + return _AttentionFunction.apply( + N, + alpha, + q, + k, + v, + seq_offsets, + num_targets, + max_attn_len, + contextual_seq_len, + sort_by_length, + enable_tma, + ) + + +@torch.jit.unused +@torch.fx.wrap +def triton_cached_hstu_mha( + N: int, + alpha: float, + delta_q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + seq_offsets: torch.Tensor, + num_targets: Optional[torch.Tensor] = None, + max_attn_len: int = 0, + contextual_seq_len: int = 0, + enable_tma: bool = False, +) -> torch.Tensor: + Z = seq_offsets.size(0) - 1 + AUTOTUNE_Z = prev_power_of_2(Z) + DELTA_L, H, DimQ = delta_q.shape + DeltaSize = DELTA_L // Z + L, _, DimV = v.shape + out = torch.empty((DELTA_L, H, DimV), dtype=delta_q.dtype, device=delta_q.device) + + TMA_DESC_SIZE = 128 + desc_q = delta_q + desc_k = k + desc_v = v + + if enable_tma and tensor_descriptor_tma: + dummy_block = [1, 1] + desc_q = TensorDescriptor( + delta_q, + shape=[DELTA_L, H * DimQ], + strides=[H * DimQ, 1], + block_shape=dummy_block, + ) + desc_v = TensorDescriptor( + v, + shape=[L, H * DimV], + strides=[H * DimV, 1], + block_shape=dummy_block, + ) + desc_k = TensorDescriptor( + k, + shape=[L, H * DimQ], + strides=[H * DimQ, 1], + block_shape=dummy_block, + ) + + def alloc_fn(size: int, align: int, stream: Optional[int]): + assert align == TMA_DESC_SIZE + return torch.empty(size, dtype=torch.int8, device="cuda") + + # pyre-ignore [6] + triton.set_allocator(alloc_fn) + grid = lambda meta: ( # noqa E731 + triton.cdiv(DeltaSize, meta["BLOCK_M"]), + Z * H, + ) + + has_contextual_seq_len = contextual_seq_len > 0 + has_max_attn_len = max_attn_len > 0 + _hstu_attn_fwd[grid]( + Q=desc_q, + K=desc_k, + V=desc_v, + workspace_ptr=None, + sort_by_length_indices=None, + seq_offsets=seq_offsets, + num_targets=num_targets, + Out=out, + stride_qm=delta_q.stride(0), + stride_qh=delta_q.stride(1), + stride_kn=k.stride(0), + stride_kh=k.stride(1), + stride_vn=v.stride(0), + stride_vh=v.stride(1), + stride_om=out.stride(0), + stride_oh=out.stride(1), + alpha=alpha, + contextual_seq_len=contextual_seq_len, + max_attn_len=max_attn_len, + Z=Z, + AUTOTUNE_Z=AUTOTUNE_Z, + H=H, + MAX_SEQ_LEN=N, + AUTOTUNE_MAX_SEQ_LEN=autotune_max_seq_len(N), + DimQ=DimQ, + DimV=DimV, + DeltaSize=DeltaSize, + HAS_MULTIPLE_TARGETS=num_targets is not None, + IS_DELTA_Q=True, + ALLOW_TF32=torch.backends.cuda.matmul.allow_tf32, + BLOCK_D_Q=DimQ, + BLOCK_D_V=DimV, + HAS_CONTEXTUAL_SEQ_LEN=has_contextual_seq_len, + HAS_MAX_ATTN_LEN=has_max_attn_len, + HAS_SORT_BY_LENGTH_INDICES=False, + ENABLE_TMA=enable_tma, + TMA_DESC_SIZE=TMA_DESC_SIZE, + ) + return out diff --git a/recommendation_v4/generative_recommenders/ops/triton/triton_hstu_linear.py b/recommendation_v4/generative_recommenders/ops/triton/triton_hstu_linear.py new file mode 100644 index 000000000..ff04dde40 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton/triton_hstu_linear.py @@ -0,0 +1,3042 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +#!/usr/bin/env python3 + + +from typing import List, Optional, Tuple + +import torch + +# @manual=//triton:triton +import triton + +# @manual=//triton:triton +import triton.language as tl +from generative_recommenders.common import ( + switch_to_contiguous_if_needed, + triton_autotune, +) +from generative_recommenders.ops.triton.triton_addmm import maybe_triton_addmm_fwd +from generative_recommenders.ops.utils import maybe_register_custom_op + + +def _get_layer_norm_mul_dropout_fwd_multirow_configs() -> List[triton.Config]: + """Generate autotune configs for multi-row LayerNorm multiplication with dropout kernels.""" + configs = [] + for BLOCK_N in [1, 2, 4, 8, 16]: + for num_warps in [1, 2, 4]: + configs.append( + triton.Config( + {"BLOCK_N": BLOCK_N}, + num_warps=num_warps, + ) + ) + return configs + + +from generative_recommenders.ops.utils import is_sm100_plus + +# @manual=//triton:triton +from triton.language.extra import libdevice + +try: + # @manual=//triton:triton + from triton.language.extra.libdevice import fast_dividef +except ImportError: + try: + # @manual=//triton:triton + from triton.language.extra.cuda.libdevice import fast_dividef + except ImportError: + # pyre-ignore: Undefined import [21] + # @manual=//triton:triton + from triton.language.math import fast_dividef + + +COMPUTE_OUTPUT_LN_FAST_DROPOUT = False + + +def set_compute_output_ln_fast_dropout(value: bool) -> None: + global COMPUTE_OUTPUT_LN_FAST_DROPOUT + COMPUTE_OUTPUT_LN_FAST_DROPOUT = value + + +FUSE_OUTPUT_LN_RNG_BLACKWELL = False + + +# Only impact B200 training when CONCAT_UX is False +def set_fuse_output_ln_rng_blackwell(value: bool) -> None: + global FUSE_OUTPUT_LN_RNG_BLACKWELL + FUSE_OUTPUT_LN_RNG_BLACKWELL = value + + +@triton.jit +def rand3x(seed, offsets, n_rounds: tl.constexpr = 10): # pyre-ignore [9] + i1, i2, i3, _ = tl.randint4x(seed, offsets, n_rounds) + u1 = tl.uint_to_uniform_float(i1) + u2 = tl.uint_to_uniform_float(i2) + u3 = tl.uint_to_uniform_float(i3) + return u1, u2, u3 + + +@triton.jit +def _generate_random_mask( + MASK_BUFFER, + N, + dropout_ratio, + seed, + D: tl.constexpr, + STRIDE: tl.constexpr, + BLOCK_D: tl.constexpr, + NUM_MASKS: tl.constexpr, +): + """Generate bit-packed dropout masks for (N, D) tensors. Outputs int8. + + Processes 4 rows per program using rand4x. Mask j occupies bit j. + Extraction: y = val & 1, x = val & 2, u = val & 4. + """ + pid = tl.program_id(0) + cols = tl.arange(0, BLOCK_D) + col_mask = cols < D + start_row = pid.to(tl.int64) * 4 + + base_ptr = MASK_BUFFER + start_row * STRIDE + cols + row0_mask = (start_row < N) & col_mask + row1_mask = ((start_row + 1) < N) & col_mask + row2_mask = ((start_row + 2) < N) & col_mask + row3_mask = ((start_row + 3) < N) & col_mask + + # Each pid uses NUM_MASKS consecutive BLOCK_D chunks for Philox offsets + rand_offset = pid * (NUM_MASKS * BLOCK_D) + cols + + packed0 = tl.zeros([BLOCK_D], dtype=tl.int8) + packed1 = tl.zeros([BLOCK_D], dtype=tl.int8) + packed2 = tl.zeros([BLOCK_D], dtype=tl.int8) + packed3 = tl.zeros([BLOCK_D], dtype=tl.int8) + + for j in tl.static_range(NUM_MASKS): + r0, r1, r2, r3 = tl.rand4x(seed, rand_offset) + packed0 |= (r0 > dropout_ratio).to(tl.int8) << j + packed1 |= (r1 > dropout_ratio).to(tl.int8) << j + packed2 |= (r2 > dropout_ratio).to(tl.int8) << j + packed3 |= (r3 > dropout_ratio).to(tl.int8) << j + rand_offset += BLOCK_D + + tl.store(base_ptr, packed0, mask=row0_mask) + tl.store(base_ptr + STRIDE, packed1, mask=row1_mask) + tl.store(base_ptr + 2 * STRIDE, packed2, mask=row2_mask) + tl.store(base_ptr + 3 * STRIDE, packed3, mask=row3_mask) + + +@triton_autotune( + configs=_get_layer_norm_mul_dropout_fwd_multirow_configs(), + key=["BLOCK_D"], +) +@triton.jit +def _ln_mul_dropout_fwd_rng( + X, + U, + Y, + W, + B, + Mean, + Rstd, + RANDOM_MASK, + N, + D, + eps, + dropout_ratio, + stride_x, + stride_u, + stride_y, + stride_mask, + SILU_U: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_N: tl.constexpr, + TRAINING: tl.constexpr, + CONCAT_U: tl.constexpr, + CONCAT_X: tl.constexpr, + MUL_U_ACTIVATION_TYPE: tl.constexpr, +): + block_id = tl.program_id(0) + start_row = block_id * BLOCK_N + + # Create block pointers for X, U, and Y + X_block_ptr = tl.make_block_ptr( + base=X, + shape=(N, D), + strides=(stride_x, 1), + offsets=(start_row, 0), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + + U_block_ptr = tl.make_block_ptr( + base=U, + shape=(N, D), + strides=(stride_u, 1), + offsets=(start_row, 0), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + + # Load data blocks + x_block = tl.load(X_block_ptr, boundary_check=(0, 1), padding_option="zero").to( + tl.float32 + ) + u_block = tl.load(U_block_ptr, boundary_check=(0, 1), padding_option="zero").to( + tl.float32 + ) + + cols = tl.arange(0, BLOCK_D) + col_mask = cols < D + rows = start_row + tl.arange(0, BLOCK_N) + row_mask = rows < N + # Pre-compute 2D mask for reuse in dropout and masked operations + mask_2d = row_mask[:, None] & col_mask[None, :] + + # Pre-compute inv_D to replace divisions with multiplications (optimization) + inv_D = 1.0 / D + + mean = tl.sum(x_block, axis=1) * inv_D + tl.store(Mean + rows, mean, mask=row_mask) + mean = tl.expand_dims(mean, 1) + + x_mean = x_block - mean + x_mean = tl.where(mask_2d, x_mean, 0.0) + _var = x_mean * x_mean + var = tl.sum(_var, axis=1) * inv_D + rstd = 1 / tl.sqrt(var + eps) + tl.store(Rstd + rows, rstd, mask=row_mask) + rstd = tl.expand_dims(rstd, 1) + + y = x_mean * rstd + w = tl.load(W + cols, mask=col_mask).to(tl.float32) + b = tl.load(B + cols, mask=col_mask).to(tl.float32) + y = y * w[None, :] + b[None, :] + + # Pre-compute sigmoid once to avoid redundant computation + sigmoid_u_block = tl.sigmoid(u_block) + silu_u_block = u_block * sigmoid_u_block + + if MUL_U_ACTIVATION_TYPE == "silu": + y = y * silu_u_block + elif MUL_U_ACTIVATION_TYPE == "sigmoid": + y = y * sigmoid_u_block + else: + y = y * u_block + + if CONCAT_U and SILU_U: + # pyre-fixme[16] + u_block = silu_u_block + + if TRAINING: + # Reuse rows (as int64 for pointer arithmetic) and pre-computed mask_2d + row_offsets_i64 = rows.to(tl.int64) + # Pre-compute loop-invariant values + dropout_scale = 1.0 / (1.0 - dropout_ratio) + offsets = row_offsets_i64[:, None] * stride_mask + cols[None, :] + + if CONCAT_U or CONCAT_X: + # All 2+ mask cases use compressed int8 format - load once + compressed = tl.load(RANDOM_MASK + offsets, mask=mask_2d, other=0).to( + tl.int32 + ) + # Bit 0 is always y_mask + y_keep = (compressed & 1) != 0 + + if CONCAT_U and CONCAT_X: + # 3-mask: (u_mask << 2) | (x_mask << 1) | y_mask + x_keep = (compressed & 2) != 0 + u_keep = (compressed & 4) != 0 + u_block = tl.where(u_keep, u_block * dropout_scale, 0.0) + x_block = tl.where(x_keep, x_block * dropout_scale, 0.0) + elif CONCAT_U: + # 2-mask: (u_mask << 1) | y_mask + u_keep = (compressed & 2) != 0 + u_block = tl.where(u_keep, u_block * dropout_scale, 0.0) + else: # CONCAT_X + # 2-mask: (x_mask << 1) | y_mask + x_keep = (compressed & 2) != 0 + x_block = tl.where(x_keep, x_block * dropout_scale, 0.0) + + y = tl.where(y_keep, y * dropout_scale, 0.0) + else: + # 1-mask: y_mask at bit 0 + y_keep = tl.load(RANDOM_MASK + offsets, mask=mask_2d, other=True) + y = tl.where(y_keep, y * dropout_scale, 0.0) + + if CONCAT_U and CONCAT_X: + Y_block_ptr_u = tl.make_block_ptr( + base=Y, + shape=(N, 3 * D), + strides=(stride_y, 1), + offsets=(start_row, 0), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + + Y_block_ptr_x = tl.make_block_ptr( + base=Y, + shape=(N, 3 * D), + strides=(stride_y, 1), + offsets=(start_row, D), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + + Y_block_ptr_y = tl.make_block_ptr( + base=Y, + shape=(N, 3 * D), + strides=(stride_y, 1), + offsets=(start_row, 2 * D), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + + tl.store(Y_block_ptr_u, u_block.to(Y.dtype.element_ty), boundary_check=(0, 1)) + tl.store(Y_block_ptr_x, x_block.to(Y.dtype.element_ty), boundary_check=(0, 1)) + tl.store(Y_block_ptr_y, y.to(Y.dtype.element_ty), boundary_check=(0, 1)) + elif CONCAT_U: + Y_block_ptr_u = tl.make_block_ptr( + base=Y, + shape=(N, 2 * D), + strides=(stride_y, 1), + offsets=(start_row, 0), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + Y_block_ptr_y = tl.make_block_ptr( + base=Y, + shape=(N, 2 * D), + strides=(stride_y, 1), + offsets=(start_row, D), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + tl.store(Y_block_ptr_u, u_block.to(Y.dtype.element_ty), boundary_check=(0, 1)) + tl.store(Y_block_ptr_y, y.to(Y.dtype.element_ty), boundary_check=(0, 1)) + elif CONCAT_X: + Y_block_ptr_x = tl.make_block_ptr( + base=Y, + shape=(N, 2 * D), + strides=(stride_y, 1), + offsets=(start_row, 0), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + Y_block_ptr_y = tl.make_block_ptr( + base=Y, + shape=(N, 2 * D), + strides=(stride_y, 1), + offsets=(start_row, D), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + tl.store(Y_block_ptr_x, x_block.to(Y.dtype.element_ty), boundary_check=(0, 1)) + tl.store(Y_block_ptr_y, y.to(Y.dtype.element_ty), boundary_check=(0, 1)) + else: + Y_block_ptr = tl.make_block_ptr( + base=Y, + shape=(N, D), + strides=(stride_y, 1), + offsets=(start_row, 0), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + + tl.store(Y_block_ptr, y.to(Y.dtype.element_ty), boundary_check=(0, 1)) + + +@triton.jit +def _ln_mul_dropout_fwd( + X, + U, + Y, + W, + B, + Mean, + Rstd, + D, + eps, + seed, + dropout_ratio, + stride_x, + stride_u, + stride_y, + SILU_U: tl.constexpr, + BLOCK_D: tl.constexpr, + TRAINING: tl.constexpr, + CONCAT_U: tl.constexpr, + CONCAT_X: tl.constexpr, + MUL_U_ACTIVATION_TYPE: tl.constexpr, + FAST_DROPOUT: tl.constexpr, +): + row = tl.program_id(0) + X += row.to(tl.int64) * stride_x + U += row.to(tl.int64) * stride_u + Y += row.to(tl.int64) * stride_y + cols = tl.arange(0, BLOCK_D) + + # Compute mean + mean = 0.0 + x = tl.load(X + cols, mask=cols < D, other=0.0).to(tl.float32) + mean = tl.sum(x, axis=0) / D + + # Compute variance + _var = tl.zeros([BLOCK_D], dtype=tl.float32) + x_mean = tl.where(cols < D, x - mean, 0.0) + _var += x_mean * x_mean + var = tl.sum(_var, axis=0) / D + rstd = 1 / tl.sqrt(var + eps) + tl.store(Mean + row, mean) + tl.store(Rstd + row, rstd) + + # Normalize and apply linear transformation + mask = cols < D + y = x_mean * rstd + w = tl.load(W + cols, mask=mask).to(tl.float32) + b = tl.load(B + cols, mask=mask).to(tl.float32) + y = y * w + b + u = tl.load(U + cols, mask=cols < D, other=0.0).to(tl.float32) + sigmoid_u = tl.sigmoid(u) + silu_u = u * sigmoid_u + + if MUL_U_ACTIVATION_TYPE == "silu": + y = y * silu_u + elif MUL_U_ACTIVATION_TYPE == "sigmoid": + y = y * sigmoid_u + else: + y = y * u + + if CONCAT_U and SILU_U: + u = silu_u + + if TRAINING: + random_offsets = 3 * row * BLOCK_D + cols + if CONCAT_U and CONCAT_X: + # apply dropout on u + if FAST_DROPOUT: + random_u, random_x, random_y = rand3x(seed, random_offsets) + else: + random_u = tl.rand(seed, random_offsets) + u_keep = random_u > dropout_ratio + u = tl.where(u_keep, u / (1.0 - dropout_ratio), 0.0) + # apply dropout on x + if not FAST_DROPOUT: + random_x = tl.rand(seed, random_offsets + D) + x_keep = random_x > dropout_ratio # pyre-ignore [61] + x = tl.where(x_keep, x / (1.0 - dropout_ratio), 0.0) + # apply dropout on y + if not FAST_DROPOUT: + random_y = tl.rand(seed, random_offsets + 2 * D) + y_keep = random_y > dropout_ratio # pyre-ignore [61] + y = tl.where(y_keep, y / (1.0 - dropout_ratio), 0.0) + elif CONCAT_U: + # apply dropout on u + if FAST_DROPOUT: + random_u, random_y, _ = rand3x(seed, random_offsets) + else: + random_u = tl.rand(seed, random_offsets) + u_keep = random_u > dropout_ratio + u = tl.where(u_keep, u / (1.0 - dropout_ratio), 0.0) + # apply dropout on y + if not FAST_DROPOUT: + random_y = tl.rand(seed, random_offsets + D) + y_keep = random_y > dropout_ratio # pyre-ignore [61] + y = tl.where(y_keep, y / (1.0 - dropout_ratio), 0.0) + elif CONCAT_X: + # apply dropout on x + if FAST_DROPOUT: + random_x, random_y, _ = rand3x(seed, random_offsets) + else: + random_x = tl.rand(seed, random_offsets) + x_keep = random_x > dropout_ratio + x = tl.where(x_keep, x / (1.0 - dropout_ratio), 0.0) + # apply dropout on y + if not FAST_DROPOUT: + random_y = tl.rand(seed, random_offsets + D) + y_keep = random_y > dropout_ratio # pyre-ignore [61] + y = tl.where(y_keep, y / (1.0 - dropout_ratio), 0.0) + else: + random = tl.rand(seed, random_offsets) + y_keep = random > dropout_ratio + # write-back + y = tl.where(y_keep, y / (1.0 - dropout_ratio), 0.0) + + # Write output + if CONCAT_U and CONCAT_X: + tl.store(Y + cols, u.to(Y.dtype.element_ty), mask=mask) + tl.store(Y + D + cols, x.to(Y.dtype.element_ty), mask=mask) + tl.store(Y + 2 * D + cols, y.to(Y.dtype.element_ty), mask=mask) + elif CONCAT_U: + tl.store(Y + cols, u.to(Y.dtype.element_ty), mask=mask) + tl.store(Y + D + cols, y.to(Y.dtype.element_ty), mask=mask) + elif CONCAT_X: + tl.store(Y + cols, x.to(Y.dtype.element_ty), mask=mask) + tl.store(Y + D + cols, y.to(Y.dtype.element_ty), mask=mask) + else: + tl.store(Y + cols, y.to(Y.dtype.element_ty), mask=mask) + + +@triton.jit +def _ln_mul_dropout_bwd_dx_du_rng( + DX, + DU, + DY, + DW, + DB, + X, + U, + Y, + W, + B, + Mean, + Rstd, + RANDOM_MASK, + stride_dx, + stride_du, + stride_dy, + stride_x, + stride_u, + stride_y, + stride_mask, + D, + eps, + dropout_ratio, + N, + SILU_U: tl.constexpr, + BLOCK_D: tl.constexpr, + TRAINING: tl.constexpr, + CONCAT_U: tl.constexpr, + CONCAT_X: tl.constexpr, + MUL_U_ACTIVATION_TYPE: tl.constexpr, + COMPUTE_Y: tl.constexpr, +): + pid = tl.program_id(0) + tile_num = tl.num_programs(0) + rows_per_tile = N // tile_num + if pid < N % tile_num: + rows_per_tile += 1 + + if rows_per_tile == 0: + return + + cols = tl.arange(0, BLOCK_D) + mask = cols < D + + row = pid + # Pre-compute row and pid as int64 once for initial pointer setup + row_i64 = row.to(tl.int64) + pid_i64 = pid.to(tl.int64) + X += row_i64 * stride_x + U += row_i64 * stride_u + if COMPUTE_Y: + Y += row_i64 * stride_y + DY += row_i64 * stride_dy + DX += row_i64 * stride_dx + DU += row_i64 * stride_du + DW = DW + pid_i64 * D + cols + DB = DB + pid_i64 * D + cols + + # Pre-compute mask pointer offset (all cases use stride_mask for (N, D) shape) + RANDOM_MASK += row_i64 * stride_mask + + partial_dw = tl.zeros((BLOCK_D,), dtype=tl.float32) + partial_db = tl.zeros((BLOCK_D,), dtype=tl.float32) + w = tl.load(W + cols, mask=mask).to(tl.float32) + b = tl.load(B + cols, mask=mask).to(tl.float32) + + dropout_scale = 0.0 + if TRAINING: + dropout_scale = 1.0 / (1.0 - dropout_ratio) + + # Pre-compute inv_D to replace divisions with multiplications (optimization) + inv_D = 1.0 / D + + # Pre-compute tile_num as int64 to avoid repeated conversion in the loop + tile_num_i64 = tile_num.to(tl.int64) + for _ in range(0, rows_per_tile): + # Load data to SRAM + x = tl.load(X + cols, mask=mask, other=0).to(tl.float32) + if CONCAT_U and CONCAT_X: + du = tl.load(DY + cols, mask=mask, other=0).to(tl.float32) + dx = tl.load(DY + D + cols, mask=mask, other=0).to(tl.float32) + dy = tl.load(DY + 2 * D + cols, mask=mask, other=0).to(tl.float32) + elif CONCAT_U: + du = tl.load(DY + cols, mask=mask, other=0).to(tl.float32) + dx = tl.zeros([BLOCK_D], dtype=tl.float32) + dy = tl.load(DY + D + cols, mask=mask, other=0).to(tl.float32) + elif CONCAT_X: + du = tl.zeros([BLOCK_D], dtype=tl.float32) + dx = tl.load(DY + cols, mask=mask, other=0).to(tl.float32) + dy = tl.load(DY + D + cols, mask=mask, other=0).to(tl.float32) + else: + du = tl.zeros([BLOCK_D], dtype=tl.float32) + dx = tl.zeros([BLOCK_D], dtype=tl.float32) + dy = tl.load(DY + cols, mask=mask, other=0).to(tl.float32) + if TRAINING: + if CONCAT_U or CONCAT_X: + # All 2+ mask cases use compressed int8 format - load once + compressed = tl.load(RANDOM_MASK + cols, mask=mask, other=0).to( + tl.int32 + ) + dy_keep = (compressed & 1) != 0 # Bit 0 always y_mask + + if CONCAT_U and CONCAT_X: + # Format: (u_mask << 2) | (x_mask << 1) | y_mask + dx_keep = (compressed & 2) != 0 + du_keep = (compressed & 4) != 0 + du = tl.where(du_keep, du * dropout_scale, 0.0) + dx = tl.where(dx_keep, dx * dropout_scale, 0.0) + elif CONCAT_U: + # Format: (u_mask << 1) | y_mask + du_keep = (compressed & 2) != 0 + du = tl.where(du_keep, du * dropout_scale, 0.0) + else: # CONCAT_X + # Format: (x_mask << 1) | y_mask + dx_keep = (compressed & 2) != 0 + dx = tl.where(dx_keep, dx * dropout_scale, 0.0) + dy = tl.where(dy_keep, dy * dropout_scale, 0.0) + else: + # 1-mask: y_mask at bit 0 + dy_keep = tl.load(RANDOM_MASK + cols, mask=mask, other=True) + dy = tl.where(dy_keep, dy * dropout_scale, 0.0) + + mean = tl.load(Mean + row) + rstd = tl.load(Rstd + row) + + # Compute dx + xhat = (x - mean) * rstd + u = tl.load(U + cols, mask=mask, other=0).to(tl.float32) + ln = xhat * w + b + du_y = dy * ln + mul_u = u + sig_u = tl.sigmoid(u) + + # Pre-compute commonly used expressions to avoid redundant computation + silu_u = u * sig_u # silu(u) - used multiple times + dsig_u = sig_u * (1.0 - sig_u) # sigmoid derivative - used multiple times + dsilu_u = sig_u + silu_u * ( + 1.0 - sig_u + ) # silu derivative - used multiple times + + if MUL_U_ACTIVATION_TYPE == "silu": + mul_u = silu_u + du_y = dy * ln * dsilu_u + dy = dy * silu_u + elif MUL_U_ACTIVATION_TYPE == "sigmoid": + mul_u = sig_u + du_y = dy * ln * dsig_u + dy = dy * sig_u + else: + dy = dy * u + + du_u = du + if CONCAT_U and SILU_U: + du_u *= dsilu_u + u = silu_u + + du = du_y + du_u + + tl.store(DU + cols, du.to(DU.dtype.element_ty), mask=mask) + + wdy = w * dy + if COMPUTE_Y: + y = ln * mul_u + if TRAINING: + if CONCAT_U: + u = tl.where( + du_keep, # pyre-ignore [61] + u * dropout_scale, + 0.0, + ) + if CONCAT_X: + x = tl.where( + dx_keep, # pyre-ignore [61] + x * dropout_scale, + 0.0, + ) + y = tl.where( + dy_keep, # pyre-ignore [61] + y * dropout_scale, + 0.0, + ) + if CONCAT_U and CONCAT_X: + tl.store(Y + cols, u.to(Y.dtype.element_ty), mask=mask) + tl.store(Y + D + cols, x.to(Y.dtype.element_ty), mask=mask) + tl.store(Y + 2 * D + cols, y.to(Y.dtype.element_ty), mask=mask) + elif CONCAT_U: + tl.store(Y + cols, u.to(Y.dtype.element_ty), mask=mask) + tl.store(Y + D + cols, y.to(Y.dtype.element_ty), mask=mask) + elif CONCAT_X: + tl.store(Y + cols, x.to(Y.dtype.element_ty), mask=mask) + tl.store(Y + D + cols, y.to(Y.dtype.element_ty), mask=mask) + else: + tl.store(Y + cols, y.to(Y.dtype.element_ty), mask=mask) + Y += tile_num_i64 * stride_y + + # Note: xhat and wdy are already 0 outside valid range due to masked loads, + # so no additional tl.where masking is needed before reduction + c1 = tl.sum(xhat * wdy, axis=0) * inv_D + c2 = tl.sum(wdy, axis=0) * inv_D + dx += (wdy - (xhat * c1 + c2)) * rstd + # Write dx + tl.store(DX + cols, dx, mask=mask) + + # Accumulate partial sums for dw/db + partial_dw += dy * xhat + partial_db += dy + X += tile_num_i64 * stride_x + U += tile_num_i64 * stride_u + DY += tile_num_i64 * stride_dy + DX += tile_num_i64 * stride_dx + DU += tile_num_i64 * stride_du + # Increment mask pointer + RANDOM_MASK += tile_num_i64 * stride_mask + row += tile_num + tl.store(DW, partial_dw, mask=mask) + tl.store(DB, partial_db, mask=mask) + + +@triton.jit +def _ln_mul_dropout_bwd_dx_du( + DX, + DU, + DY, + DW, + DB, + X, + U, + Y, + W, + B, + Mean, + Rstd, + stride_dx, + stride_du, + stride_dy, + stride_x, + stride_u, + stride_y, + D, + eps, + seed, + dropout_ratio, + N, + SILU_U: tl.constexpr, + BLOCK_D: tl.constexpr, + TRAINING: tl.constexpr, + CONCAT_U: tl.constexpr, + CONCAT_X: tl.constexpr, + MUL_U_ACTIVATION_TYPE: tl.constexpr, + COMPUTE_Y: tl.constexpr, + FAST_DROPOUT: tl.constexpr, +): + pid = tl.program_id(0) + tile_num = tl.num_programs(0) + rows_per_tile = N // tile_num + if pid < N % tile_num: + rows_per_tile += 1 + + if rows_per_tile == 0: + return + + cols = tl.arange(0, BLOCK_D) + mask = cols < D + + row = pid + X += row.to(tl.int64) * stride_x + U += row.to(tl.int64) * stride_u + if COMPUTE_Y: + Y += row.to(tl.int64) * stride_y + DY += row.to(tl.int64) * stride_dy + DX += row.to(tl.int64) * stride_dx + DU += row.to(tl.int64) * stride_du + DW = DW + pid * D + cols + DB = DB + pid * D + cols + + partial_dw = tl.zeros((BLOCK_D,), dtype=tl.float32) + partial_db = tl.zeros((BLOCK_D,), dtype=tl.float32) + w = tl.load(W + cols, mask=mask).to(tl.float32) + b = tl.load(B + cols, mask=mask).to(tl.float32) + for _idx in range(0, rows_per_tile): + # Load data to SRAM + x = tl.load(X + cols, mask=mask, other=0).to(tl.float32) + if CONCAT_U and CONCAT_X: + du = tl.load(DY + cols, mask=mask, other=0).to(tl.float32) + dx = tl.load(DY + D + cols, mask=mask, other=0).to(tl.float32) + dy = tl.load(DY + 2 * D + cols, mask=mask, other=0).to(tl.float32) + elif CONCAT_U: + du = tl.load(DY + cols, mask=mask, other=0).to(tl.float32) + dx = tl.zeros([BLOCK_D], dtype=tl.float32) + dy = tl.load(DY + D + cols, mask=mask, other=0).to(tl.float32) + elif CONCAT_X: + du = tl.zeros([BLOCK_D], dtype=tl.float32) + dx = tl.load(DY + cols, mask=mask, other=0).to(tl.float32) + dy = tl.load(DY + D + cols, mask=mask, other=0).to(tl.float32) + else: + du = tl.zeros([BLOCK_D], dtype=tl.float32) + dx = tl.zeros([BLOCK_D], dtype=tl.float32) + dy = tl.load(DY + cols, mask=mask, other=0).to(tl.float32) + if TRAINING: + random_offsets = 3 * row * BLOCK_D + cols + if CONCAT_U and CONCAT_X: + # apply dropout on du + if FAST_DROPOUT: + random_du, random_dx, random_dy = rand3x(seed, random_offsets) + else: + random_du = tl.rand(seed, random_offsets) + du_keep = random_du > dropout_ratio + du = tl.where(du_keep, du / (1.0 - dropout_ratio), 0.0) + # apply dropout on dx + if not FAST_DROPOUT: + random_dx = tl.rand(seed, random_offsets + D) + dx_keep = random_dx > dropout_ratio # pyre-ignore [61] + dx = tl.where(dx_keep, dx / (1.0 - dropout_ratio), 0.0) + # apply dropout on dy + if not FAST_DROPOUT: + random_dy = tl.rand(seed, random_offsets + 2 * D) + dy_keep = random_dy > dropout_ratio # pyre-ignore [61] + dy = tl.where(dy_keep, dy / (1.0 - dropout_ratio), 0.0) + elif CONCAT_U: + # apply dropout on du + if FAST_DROPOUT: + random_du, _, random_dy = rand3x(seed, random_offsets) + else: + random_du = tl.rand(seed, random_offsets) + du_keep = random_du > dropout_ratio + du = tl.where(du_keep, du / (1.0 - dropout_ratio), 0.0) + # apply dropout on dy + if not FAST_DROPOUT: + random_dy = tl.rand(seed, random_offsets + D) + dy_keep = random_dy > dropout_ratio # pyre-ignore [61] + dy = tl.where(dy_keep, dy / (1.0 - dropout_ratio), 0.0) + elif CONCAT_X: + # apply dropout on dx + if FAST_DROPOUT: + _, random_dx, random_dy = rand3x(seed, random_offsets) + else: + random_dx = tl.rand(seed, random_offsets) + dx_keep = random_dx > dropout_ratio # pyre-ignore [61] + dx = tl.where(dx_keep, dx / (1.0 - dropout_ratio), 0.0) + # apply dropout on dy + if not FAST_DROPOUT: + random_dy = tl.rand(seed, random_offsets + D) + dy_keep = random_dy > dropout_ratio # pyre-ignore [61] + dy = tl.where(dy_keep, dy / (1.0 - dropout_ratio), 0.0) + else: + random = tl.rand(seed, random_offsets) + dy_keep = random > dropout_ratio + # write-back + dy = tl.where(dy_keep, dy / (1.0 - dropout_ratio), 0.0) + + mean = tl.load(Mean + row) + rstd = tl.load(Rstd + row) + + # Compute dx + xhat = (x - mean) * rstd + u = tl.load(U + cols, mask=mask, other=0).to(tl.float32) + ln = xhat * w + b + du_y = dy * ln + mul_u = u + sig_u = tl.sigmoid(u) + + if MUL_U_ACTIVATION_TYPE == "silu": + mul_u = u * sig_u + du_y = dy * ln * (sig_u + u * sig_u * (1.0 - sig_u)) + dy = dy * u * sig_u + elif MUL_U_ACTIVATION_TYPE == "sigmoid": + mul_u = sig_u + du_y = dy * ln * (sig_u * (1.0 - sig_u)) + dy = dy * sig_u + else: + dy = dy * u + + du_u = du + if CONCAT_U: + if SILU_U: + du_u *= sig_u + u * sig_u * (1.0 - sig_u) + u = u * sig_u + + du = du_y + du_u + + tl.store(DU + cols, du.to(DU.dtype.element_ty), mask=mask) + wdy = w * dy + if COMPUTE_Y: + y = ln * mul_u + if TRAINING: + if CONCAT_U: + u = tl.where( + du_keep, # pyre-ignore [61] + u / (1.0 - dropout_ratio), + 0.0, + ) + if CONCAT_X: + x = tl.where( + dx_keep, # pyre-ignore [61] + x / (1.0 - dropout_ratio), + 0.0, + ) + y = tl.where( + dy_keep, # pyre-ignore [61] + y / (1.0 - dropout_ratio), + 0.0, + ) + if CONCAT_U and CONCAT_X: + tl.store(Y + cols, u.to(Y.dtype.element_ty), mask=mask) + tl.store(Y + D + cols, x.to(Y.dtype.element_ty), mask=mask) + tl.store(Y + 2 * D + cols, y.to(Y.dtype.element_ty), mask=mask) + elif CONCAT_U: + tl.store(Y + cols, u.to(Y.dtype.element_ty), mask=mask) + tl.store(Y + D + cols, y.to(Y.dtype.element_ty), mask=mask) + elif CONCAT_X: + tl.store(Y + cols, x.to(Y.dtype.element_ty), mask=mask) + tl.store(Y + D + cols, y.to(Y.dtype.element_ty), mask=mask) + else: + tl.store(Y + cols, y.to(Y.dtype.element_ty), mask=mask) + Y += tile_num.to(tl.int64) * stride_y + + xhat = tl.where(mask, xhat, 0.0) + wdy = tl.where(mask, wdy, 0.0) + c1 = tl.sum(xhat * wdy, axis=0) / D + c2 = tl.sum(wdy, axis=0) / D + dx += (wdy - (xhat * c1 + c2)) * rstd + # Write dx + tl.store(DX + cols, dx, mask=mask) + + # Accumulate partial sums for dw/db + partial_dw += dy * xhat + partial_db += dy + X += tile_num.to(tl.int64) * stride_x + U += tile_num.to(tl.int64) * stride_u + DY += tile_num.to(tl.int64) * stride_dy + DX += tile_num.to(tl.int64) * stride_dx + DU += tile_num.to(tl.int64) * stride_du + row += tile_num + tl.store(DW, partial_dw, mask=mask) + tl.store(DB, partial_db, mask=mask) + + +def _get_bwd_dwdb_configs() -> List[triton.Config]: + configs = [] + for BLOCK_N in [32, 64, 128, 256]: + for num_warps in [8, 16] + ([] if torch.ops.hip else [32]): + configs.append( + triton.Config( + {"BLOCK_N": BLOCK_N}, + num_warps=num_warps, + ) + ) + return configs + + +@triton_autotune( + configs=_get_bwd_dwdb_configs(), + key=["D"], +) +@triton.jit +def _ln_mul_dropout_bwd_dwdb( + DW, + DB, + FINAL_DW, + FINAL_DB, + N, + D, + BLOCK_N: tl.constexpr, + BLOCK_D: tl.constexpr, +): + pid = tl.program_id(0).to(tl.int64) + cols = pid * BLOCK_D + tl.arange(0, BLOCK_D) + dw = tl.zeros((BLOCK_N, BLOCK_D), dtype=tl.float32) + db = tl.zeros((BLOCK_N, BLOCK_D), dtype=tl.float32) + + for i in range(0, N, BLOCK_N): + rows = i + tl.arange(0, BLOCK_N) + # pyre-fixme[16]: `int` has no attribute `__getitem__`. + mask = (rows[:, None] < N) & (cols[None, :] < D) + offs = rows[:, None] * D + cols[None, :] + dw += tl.load(DW + offs, mask=mask, other=0.0) + db += tl.load(DB + offs, mask=mask, other=0.0) + + sum_dw = tl.sum(dw, axis=0) + sum_db = tl.sum(db, axis=0) + tl.store(FINAL_DW + cols, sum_dw.to(FINAL_DW.dtype.element_ty), mask=cols < D) + tl.store(FINAL_DB + cols, sum_db.to(FINAL_DB.dtype.element_ty), mask=cols < D) + + +def _create_dropout_mask( + N: int, + D: int, + BLOCK_D: int, + concat_u: bool, + concat_x: bool, + dropout_ratio: float, + seed: int, + device: torch.device, +) -> torch.Tensor: + """Create dropout mask tensor for layer norm mul dropout. + + Args: + N: Number of rows + D: Feature dimension + BLOCK_D: Block size for D dimension + concat_u: Whether to concatenate u + concat_x: Whether to concatenate x + dropout_ratio: Dropout ratio + seed: Random seed + device: Device to create tensor on + + Returns: + random_mask: (N, D) int8 tensor. Mask j at bit j. + + Bit layout: y = val & 1, x = val & 2, u = val & 4. + """ + num_masks = 1 + int(concat_u) + int(concat_x) + # Torch uses 1 byte for bool internally, same as int8, so always use int8. + random_mask = torch.empty([N, D], dtype=torch.int8, device=device) + _generate_random_mask[(triton.cdiv(N, 4),)]( + random_mask, + N, + dropout_ratio, + seed, + D, # pyre-ignore[6] + random_mask.stride(0), # pyre-ignore[6] + BLOCK_D, # pyre-fixme[6]: Triton constexpr param + num_masks, # pyre-ignore[6]: NUM_MASKS constexpr + ) + return random_mask + + +@maybe_register_custom_op( + "generative_recommenders::_triton_layer_norm_mul_dropout_fwd_impl", mutates_args=() +) +def _triton_layer_norm_mul_dropout_fwd_impl( + x: torch.Tensor, + u: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + eps: float, + dropout_ratio: float, + training: bool, + silu_u: bool, + concat_u: bool, + concat_x: bool, + mul_u_activation_type: str, + seed: int, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Internal implementation that returns only tensors for custom_op compatibility. + + Returns (y, mean, rstd, random_mask) where random_mask is empty when not used. + """ + N, D = x.shape + + if concat_u and concat_x: + y = torch.empty((N, 3 * D), dtype=x.dtype, device=x.device) + elif concat_u: + y = torch.empty((N, 2 * D), dtype=x.dtype, device=x.device) + elif concat_x: + y = torch.empty((N, 2 * D), dtype=x.dtype, device=x.device) + else: + y = torch.empty_like(x) + mean = torch.empty((N,), dtype=torch.float32, device=x.device) + rstd = torch.empty((N,), dtype=torch.float32, device=x.device) + if N == 0: + return y, mean, rstd, torch.empty(0, dtype=x.dtype, device=x.device) + # Less than 64KB per feature: enqueue fused kernel + MAX_FUSED_SIZE = 65536 // x.element_size() + BLOCK_D: int = min(MAX_FUSED_SIZE, triton.next_power_of_2(D)) + if D > BLOCK_D: + raise RuntimeError("This layer norm doesn't support feature dim >= 64KB.") + + num_warps: int = min(max(BLOCK_D // 256, 1), 8) + random_mask: torch.Tensor = torch.empty(0, dtype=x.dtype, device=x.device) + # Benchmark shows separating RNG from ln_mul_dropout kernel only benefits on + # blackwell when CONCAT_UX is enabled. (fused RNG kernel can benefit from rand3x fast + # dropout) + # Extended to support concat_u + concat_x for mask reuse optimization + if not FUSE_OUTPUT_LN_RNG_BLACKWELL and is_sm100_plus() and training: + random_mask = _create_dropout_mask( + N=N, + D=D, + BLOCK_D=BLOCK_D, + concat_u=concat_u, + concat_x=concat_x, + dropout_ratio=dropout_ratio, + seed=seed, + device=x.device, + ) + + def grid(META): + return (triton.cdiv(N, META["BLOCK_N"]),) + + # pyre-ignore[28] + _ln_mul_dropout_fwd_rng[grid]( + x, + u, + y, + weight, + bias, + mean, + rstd, + random_mask, + N, + D, + eps, + dropout_ratio, + x.stride(0), + u.stride(0), + y.stride(0), + random_mask.stride(0), + SILU_U=silu_u, + BLOCK_D=BLOCK_D, + TRAINING=training, + CONCAT_U=concat_u, + CONCAT_X=concat_x, + MUL_U_ACTIVATION_TYPE=mul_u_activation_type, + ) + + else: + # Default path: fused RNG generation + # Mask cannot be saved with fused RNG - it's generated inline in the kernel + # pyre-ignore[28] + _ln_mul_dropout_fwd[(N,)]( + x, + u, + y, + weight, + bias, + mean, + rstd, + D, + eps, + seed, + dropout_ratio, + x.stride(0), + u.stride(0), + y.stride(0), + SILU_U=silu_u, + BLOCK_D=BLOCK_D, + TRAINING=training, + CONCAT_U=concat_u, + CONCAT_X=concat_x, + MUL_U_ACTIVATION_TYPE=mul_u_activation_type, + FAST_DROPOUT=COMPUTE_OUTPUT_LN_FAST_DROPOUT, + num_warps=num_warps, + ) + return y, mean, rstd, random_mask + + +@_triton_layer_norm_mul_dropout_fwd_impl.register_fake +def _triton_layer_norm_mul_dropout_fwd_impl_fake( + x: torch.Tensor, + u: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + eps: float, + dropout_ratio: float, + training: bool, + silu_u: bool, + concat_u: bool, + concat_x: bool, + mul_u_activation_type: str, + seed: int, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Fake implementation for FakeTensor tracing.""" + N, D = x.shape + if concat_u and concat_x: + y = torch.empty((N, 3 * D), dtype=x.dtype, device=x.device) + elif concat_u: + y = torch.empty((N, 2 * D), dtype=x.dtype, device=x.device) + elif concat_x: + y = torch.empty((N, 2 * D), dtype=x.dtype, device=x.device) + else: + y = torch.empty_like(x) + mean = torch.empty((N,), dtype=torch.float32, device=x.device) + rstd = torch.empty((N,), dtype=torch.float32, device=x.device) + random_mask = torch.empty(0, dtype=x.dtype, device=x.device) + return y, mean, rstd, random_mask + + +def triton_layer_norm_mul_dropout_fwd( + x: torch.Tensor, + u: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + eps: float, + dropout_ratio: float, + training: bool, + silu_u: bool = False, + concat_u: bool = False, + concat_x: bool = False, + mul_u_activation_type: str = "none", + seed: Optional[int] = None, +) -> Tuple[ + torch.Tensor, torch.Tensor, torch.Tensor, int, int, int, Optional[torch.Tensor] +]: # y, mean, rstd, BLOCK_D, num_warps, seed, random_mask + """Forward pass for layer norm + mul + dropout. + + Args: + x: Input tensor of shape (N, D) + u: Second input tensor of shape (N, D) + weight: Layer norm weight of shape (D,) + bias: Layer norm bias of shape (D,) + eps: Layer norm epsilon + dropout_ratio: Dropout probability + training: Whether in training mode + silu_u: Whether to apply SiLU to u before concatenation + concat_u: Whether to concatenate u to output + concat_x: Whether to concatenate x to output + mul_u_activation_type: Activation type for u multiplication + seed: Random seed for dropout + + Returns: + Tuple of (y, mean, rstd, BLOCK_D, num_warps, seed, random_mask) + - random_mask is None when using fused RNG path (non-SM100+) + - random_mask is always returned when using separate RNG path (SM100+) + for reuse in backward pass (avoids redundant mask generation) + """ + assert x.dim() == 2 + x = switch_to_contiguous_if_needed(x) + N, D = x.shape + assert weight.dim() == 1 + assert bias.dim() == 1 + assert weight.numel() == D + assert bias.numel() == D + + if N == 0: + D = x.shape[1] + if concat_u and concat_x: + y = torch.empty((0, 3 * D), dtype=x.dtype, device=x.device) + elif concat_u or concat_x: + y = torch.empty((0, 2 * D), dtype=x.dtype, device=x.device) + else: + y = torch.empty_like(x) + return ( + y, + torch.empty((N,), dtype=torch.float32, device=x.device), + torch.empty((N,), dtype=torch.float32, device=x.device), + 0, + 0, + 0, + None, + ) + + # Less than 64KB per feature: enqueue fused kernel + MAX_FUSED_SIZE = 65536 // x.element_size() + BLOCK_D: int = min(MAX_FUSED_SIZE, triton.next_power_of_2(D)) + if D > BLOCK_D: + raise RuntimeError("This layer norm doesn't support feature dim >= 64KB.") + + if seed is None and training: + # pyre-ignore[9]: torch.randint with dtype=int64 always returns int + seed = torch.randint(low=0, high=2**62, size=(1,), dtype=torch.int64).item() + num_warps: int = min(max(BLOCK_D // 256, 1), 8) + + # Call internal implementation + y, mean, rstd, random_mask_tensor = _triton_layer_norm_mul_dropout_fwd_impl( + x, + u, + weight, + bias, + eps, + dropout_ratio, + training, + silu_u, + concat_u, + concat_x, + mul_u_activation_type, + seed if seed is not None else 0, + ) + + # Convert empty tensor back to None + random_mask: Optional[torch.Tensor] = ( + random_mask_tensor if random_mask_tensor.numel() > 0 else None + ) + + return y, mean, rstd, BLOCK_D, num_warps, seed, random_mask # pyre-ignore[7] + + +@maybe_register_custom_op( + "generative_recommenders::_triton_layer_norm_mul_dropout_bwd_impl", mutates_args=() +) +def _triton_layer_norm_mul_dropout_bwd_impl( + dy: torch.Tensor, + x: torch.Tensor, + u: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + mean: torch.Tensor, + rstd: torch.Tensor, + BLOCK_D: int, + num_warps: int, + eps: float, + training: bool, + dropout_ratio: float, + seed: int, + silu_u: bool, + concat_u: bool, + concat_x: bool, + mul_u_activation_type: str, + compute_y: bool, + random_mask: torch.Tensor, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Internal implementation that returns only tensors for custom_op compatibility. + + When compute_y is False, y is returned as an empty tensor. + random_mask with numel() == 0 means no mask (fused RNG path). + """ + N, D = x.shape + if compute_y: + if concat_u and concat_x: + y = torch.empty((N, 3 * D), dtype=x.dtype, device=x.device) + elif concat_u: + y = torch.empty((N, 2 * D), dtype=x.dtype, device=x.device) + elif concat_x: + y = torch.empty((N, 2 * D), dtype=x.dtype, device=x.device) + else: + y = torch.empty_like(x) + else: + y = torch.empty(0, dtype=x.dtype, device=x.device) + + if N == 0: + return ( + torch.zeros_like(x), + torch.zeros_like(u), + torch.zeros((D,), dtype=weight.dtype, device=x.device), + torch.zeros((D,), dtype=weight.dtype, device=x.device), + y, + ) + dx = torch.empty_like(x) + du = torch.empty_like(u) + sms = torch.cuda.get_device_properties(x.device).multi_processor_count + tile_num = max(1, min(sms * 64, N // 4)) + _dweight = torch.empty((tile_num, D), dtype=torch.float32, device=x.device) + _dbias = torch.empty((tile_num, D), dtype=torch.float32, device=x.device) + dweight = torch.empty((D,), dtype=weight.dtype, device=x.device) + dbias = torch.empty((D,), dtype=weight.dtype, device=x.device) + + # Use separated RNG when random_mask is provided (from forward pass on SM100+ path) + has_random_mask = random_mask.numel() > 0 + if has_random_mask: + # pyre-ignore[28] + _ln_mul_dropout_bwd_dx_du_rng[(tile_num,)]( + dx, + du, + dy, + _dweight, + _dbias, + x, + u, + y if compute_y else None, + weight, + bias, + mean, + rstd, + random_mask, + dx.stride(0), + du.stride(0), + dy.stride(0), + x.stride(0), + u.stride(0), + y.stride(0) if compute_y else 0, + random_mask.stride(0), + D, + eps, + dropout_ratio, + N=N, + SILU_U=silu_u, + BLOCK_D=BLOCK_D, + TRAINING=training, + CONCAT_U=concat_u, + CONCAT_X=concat_x, + MUL_U_ACTIVATION_TYPE=mul_u_activation_type, + COMPUTE_Y=compute_y, + num_warps=num_warps, + ) + + else: + # pyre-ignore[28] + _ln_mul_dropout_bwd_dx_du[(tile_num,)]( + dx, + du, + dy, + _dweight, + _dbias, + x, + u, + y if compute_y else None, + weight, + bias, + mean, + rstd, + dx.stride(0), + du.stride(0), + dy.stride(0), + x.stride(0), + u.stride(0), + y.stride(0) if compute_y else 0, + D, + eps, + seed, + dropout_ratio, + N=N, + SILU_U=silu_u, + BLOCK_D=BLOCK_D, + TRAINING=training, + CONCAT_U=concat_u, + CONCAT_X=concat_x, + MUL_U_ACTIVATION_TYPE=mul_u_activation_type, + COMPUTE_Y=compute_y, + FAST_DROPOUT=COMPUTE_OUTPUT_LN_FAST_DROPOUT, + num_warps=num_warps, + ) + + def grid(META): + return (triton.cdiv(D, META["BLOCK_D"]),) + + blocks = triton.next_power_of_2(sms * 4) + BLOCK_D_bwd = triton.next_power_of_2(triton.cdiv(D, blocks)) + BLOCK_D_bwd = min(max(BLOCK_D_bwd, 4), 128) + _ln_mul_dropout_bwd_dwdb[grid]( + _dweight, + _dbias, + dweight, + dbias, + tile_num, + D, + BLOCK_D=BLOCK_D_bwd, + ) + return dx, du, dweight, dbias, y + + +@_triton_layer_norm_mul_dropout_bwd_impl.register_fake +def _triton_layer_norm_mul_dropout_bwd_impl_fake( + dy: torch.Tensor, + x: torch.Tensor, + u: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + mean: torch.Tensor, + rstd: torch.Tensor, + BLOCK_D: int, + num_warps: int, + eps: float, + training: bool, + dropout_ratio: float, + seed: int, + silu_u: bool, + concat_u: bool, + concat_x: bool, + mul_u_activation_type: str, + compute_y: bool, + random_mask: torch.Tensor, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Fake implementation for FakeTensor tracing.""" + N, D = x.shape + dx = torch.empty_like(x) + du = torch.empty_like(u) + dweight = torch.empty((D,), dtype=weight.dtype, device=x.device) + dbias = torch.empty((D,), dtype=weight.dtype, device=x.device) + if compute_y: + if concat_u and concat_x: + y = torch.empty((N, 3 * D), dtype=x.dtype, device=x.device) + elif concat_u: + y = torch.empty((N, 2 * D), dtype=x.dtype, device=x.device) + elif concat_x: + y = torch.empty((N, 2 * D), dtype=x.dtype, device=x.device) + else: + y = torch.empty_like(x) + else: + y = torch.empty(0, dtype=x.dtype, device=x.device) + return dx, du, dweight, dbias, y + + +def triton_layer_norm_mul_dropout_bwd( + dy: torch.Tensor, + x: torch.Tensor, + u: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + mean: torch.Tensor, + rstd: torch.Tensor, + BLOCK_D: int, + num_warps: int, + eps: float, + training: bool, + dropout_ratio: float, + seed: Optional[int] = None, + silu_u: bool = False, + concat_u: bool = False, + concat_x: bool = False, + mul_u_activation_type: str = "none", + compute_y: bool = False, + random_mask: Optional[torch.Tensor] = None, +) -> Tuple[ + torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, Optional[torch.Tensor] +]: + N, D = x.shape + + # Use empty tensor as sentinel for no random_mask + random_mask_tensor = ( + random_mask + if random_mask is not None + else torch.empty(0, dtype=x.dtype, device=x.device) + ) + + dx, du, dweight, dbias, y_tensor = _triton_layer_norm_mul_dropout_bwd_impl( + dy, + x, + u, + weight, + bias, + mean, + rstd, + BLOCK_D, + num_warps, + eps, + training, + dropout_ratio, + seed if seed is not None else 0, + silu_u, + concat_u, + concat_x, + mul_u_activation_type, + compute_y, + random_mask_tensor, + ) + + # Convert empty tensor back to None + y: Optional[torch.Tensor] = y_tensor if compute_y else None + return dx, du, dweight, dbias, y + + +class LayerNormMulDropoutFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore[14] + def forward( + ctx, + x: torch.Tensor, + u: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + eps: float, + dropout_ratio: float, + training: bool, + silu_u: bool = False, + concat_ux: bool = False, + seed: Optional[int] = None, + ) -> torch.Tensor: + if dropout_ratio == 0.0: + # skip dropout computation if dropout ratio is 0 + training = False + # skipping supporting concat_u and concat_x separately here because seems like this code path is only used in v1 of hstu_linear + concat_u, concat_x = concat_ux, concat_ux + + # Call forward function which generates and returns random_mask + # On SM100+ path, random_mask is always returned for backward reuse + # On fused RNG path, random_mask is None (mask generated inline in kernel) + y, mean, rstd, BLOCK_D, num_warps, returned_seed, random_mask = ( + triton_layer_norm_mul_dropout_fwd( + x=x, + u=u, + weight=weight, + bias=bias, + eps=eps, + dropout_ratio=dropout_ratio, + training=training, + silu_u=silu_u, + concat_u=concat_u, + concat_x=concat_x, + seed=seed, + ) + ) + + # Save tensors for backward pass + # When random_mask is generated (SM100+ path), always save it for reuse + # in backward pass. This avoids redundant _generate_random_mask execution. + if random_mask is not None: + ctx.save_for_backward(x, u, weight, bias, mean, rstd, random_mask) + ctx.has_random_mask = True + else: + ctx.save_for_backward(x, u, weight, bias, mean, rstd) + ctx.has_random_mask = False + + ctx.BLOCK_D = BLOCK_D + ctx.num_warps = num_warps + ctx.eps = eps + ctx.seed = returned_seed + ctx.training = training + ctx.concat_ux = concat_ux + ctx.silu_u = silu_u + ctx.dropout_ratio = dropout_ratio + return y + + @staticmethod + # pyre-ignore[14] + def backward( + ctx, dy: torch.Tensor + ) -> Tuple[ + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + None, + None, + None, + None, + None, + None, + ]: + # Extract saved tensors including optional random mask + if ctx.has_random_mask: + x, u, weight, bias, mean, rstd, random_mask = ctx.saved_tensors + else: + x, u, weight, bias, mean, rstd = ctx.saved_tensors + random_mask = None + + dx, du, dweight, dbias, _ = triton_layer_norm_mul_dropout_bwd( + dy=dy, + x=x, + u=u, + weight=weight, + bias=bias, + mean=mean, + rstd=rstd, + BLOCK_D=ctx.BLOCK_D, + num_warps=ctx.num_warps, + eps=ctx.eps, + training=ctx.training, + dropout_ratio=ctx.dropout_ratio, + seed=ctx.seed, + silu_u=ctx.silu_u, + concat_u=ctx.concat_ux, + concat_x=ctx.concat_ux, + compute_y=False, + random_mask=random_mask, # Pass saved mask to backward + ) + return dx, du, dweight, dbias, None, None, None, None, None, None + + +@triton.jit +def _group_norm_mul_dropout_fwd( + X, + U, + Y, + W, + B, + Mean, + Rstd, + D, + Heads, + eps, + seed, + dropout_ratio, + stride_x, + stride_u, + stride_y, + SILU_U: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_H: tl.constexpr, + TRAINING: tl.constexpr, + CONCAT_UX: tl.constexpr, +): + row = tl.program_id(0) + X += row.to(tl.int64) * stride_x + U += row.to(tl.int64) * stride_u + Y += row.to(tl.int64) * stride_y + cols = tl.arange(0, BLOCK_D) + heads = tl.arange(0, BLOCK_H) + offsets = heads[:, None] * D + cols[None, :] + mask_h = heads < Heads + mask_c = cols < D + mask = mask_c[None, :] & mask_h[:, None] + + # Compute mean + mean = 0.0 + x = tl.load(X + offsets, mask=mask, other=0.0).to(tl.float32) + mean = tl.sum(x, axis=1) / D + mean = tl.ravel(mean) + + # Compute variance + _var = tl.zeros([BLOCK_H, BLOCK_D], dtype=tl.float32) + x_mean = tl.where(mask, x - mean[:, None], 0.0) + _var += x_mean * x_mean + var = tl.sum(_var, axis=1) / D + var = tl.ravel(var) + rstd = 1 / tl.sqrt(var + eps) + tl.store(Mean + row * Heads + heads, mean, mask=mask_h) + tl.store(Rstd + row * Heads + heads, rstd, mask=mask_h) + + # Normalize and apply linear transformation + y = x_mean * rstd[:, None] # pyre-ignore [16] + w = tl.load(W + heads, mask=mask_h).to(tl.float32) + b = tl.load(B + heads, mask=mask_h).to(tl.float32) + y = y * w[:, None] + b[:, None] + u = tl.load(U + offsets, mask=mask, other=0.0).to(tl.float32) + if SILU_U: + u = u * tl.sigmoid(u) + y = y * u + + if TRAINING: + if CONCAT_UX: + random_offsets = row * 3 * D * Heads + offsets + # apply dropout on u + random_u = tl.rand(seed, random_offsets) + u_keep = random_u > dropout_ratio + u = tl.where(u_keep, u / (1.0 - dropout_ratio), 0.0) + # apply dropout on x + random_x = tl.rand(seed, random_offsets + Heads * D) + x_keep = random_x > dropout_ratio + x = tl.where(x_keep, x / (1.0 - dropout_ratio), 0.0) + # apply dropout on y + random_y = tl.rand(seed, random_offsets + 2 * Heads * D) + y_keep = random_y > dropout_ratio + y = tl.where(y_keep, y / (1.0 - dropout_ratio), 0.0) + else: + random_offsets = row * D * Heads + offsets + random = tl.rand(seed, random_offsets) + y_keep = random > dropout_ratio + # write-back + y = tl.where(y_keep, y / (1.0 - dropout_ratio), 0.0) + + # Write output + if CONCAT_UX: + tl.store(Y + offsets, u.to(Y.dtype.element_ty), mask=mask) + tl.store(Y + Heads * D + offsets, x.to(Y.dtype.element_ty), mask=mask) + tl.store(Y + 2 * Heads * D + offsets, y.to(Y.dtype.element_ty), mask=mask) + else: + tl.store(Y + offsets, y.to(Y.dtype.element_ty), mask=mask) + + +@triton.jit +def _group_norm_mul_dropout_bwd_dx_du( + DX, + DU, + DY, + DW, + DB, + X, + U, + Y, + W, + B, + Mean, + Rstd, + stride_dx, + stride_du, + stride_dy, + stride_x, + stride_u, + stride_y, + D, + Heads, + eps, + seed, + dropout_ratio, + SILU_U: tl.constexpr, + GROUP_N: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_H: tl.constexpr, + TRAINING: tl.constexpr, + CONCAT_UX: tl.constexpr, + COMPUTE_Y: tl.constexpr, +): + row = tl.program_id(0) + cols = tl.arange(0, BLOCK_D) + off_heads = tl.arange(0, BLOCK_H) + mask_c = cols < D + mask_h = off_heads < Heads + mask = mask_c[None, :] & mask_h[:, None] + X += row.to(tl.int64) * stride_x + U += row.to(tl.int64) * stride_u + DY += row.to(tl.int64) * stride_dy + DX += row.to(tl.int64) * stride_dx + DU += row.to(tl.int64) * stride_du + offsets = off_heads[:, None] * D + cols[None, :] + + # Load data to SRAM + x = tl.load(X + offsets, mask=mask, other=0).to(tl.float32) + if CONCAT_UX: + du = tl.load(DY + offsets, mask=mask, other=0).to(tl.float32) + dx = tl.load(DY + Heads * D + offsets, mask=mask, other=0).to(tl.float32) + dy = tl.load(DY + 2 * Heads * D + offsets, mask=mask, other=0).to(tl.float32) + else: + du = tl.zeros([BLOCK_H, BLOCK_D], dtype=tl.float32) + dx = tl.zeros([BLOCK_H, BLOCK_D], dtype=tl.float32) + dy = tl.load(DY + offsets, mask=mask, other=0).to(tl.float32) + if TRAINING: + if CONCAT_UX: + random_offsets = row * 3 * D * Heads + offsets + # apply dropout on du + random_du = tl.rand(seed, random_offsets) + du_keep = random_du > dropout_ratio + du = tl.where(du_keep, du / (1.0 - dropout_ratio), 0.0) + # apply dropout on dx + random_dx = tl.rand(seed, random_offsets + Heads * D) + dx_keep = random_dx > dropout_ratio + dx = tl.where(dx_keep, dx / (1.0 - dropout_ratio), 0.0) + # apply dropout on dy + random_dy = tl.rand(seed, random_offsets + 2 * Heads * D) + dy_keep = random_dy > dropout_ratio + dy = tl.where(dy_keep, dy / (1.0 - dropout_ratio), 0.0) + else: + random_offsets = row * D * Heads + offsets + random = tl.rand(seed, random_offsets) + dy_keep = random > dropout_ratio + # write-back + dy = tl.where(dy_keep, dy / (1.0 - dropout_ratio), 0.0) + + mean = tl.load(Mean + row * Heads + off_heads) + rstd = tl.load(Rstd + row * Heads + off_heads) + + # Compute dx + xhat = (x - mean[:, None]) * rstd[:, None] + w = tl.load(W + off_heads, mask=mask_h).to(tl.float32) + b = tl.load(B + off_heads, mask=mask_h).to(tl.float32) + u = tl.load(U + offsets, mask=mask, other=0).to(tl.float32) + ln = xhat * w[:, None] + b[:, None] + du += dy * ln + if SILU_U: + sig_u = tl.sigmoid(u) + silu_u = u * sig_u + du = du * sig_u * (1 + u - silu_u) + u = silu_u + tl.store(DU + offsets, du.to(DU.dtype.element_ty), mask=mask) + dy = dy * u + wdy = w[:, None] * dy + if COMPUTE_Y: + Y += row.to(tl.int64) * stride_y + y = ln * u + if TRAINING: + if CONCAT_UX: + u = tl.where( + du_keep, # pyre-ignore [61] + u / (1.0 - dropout_ratio), + 0.0, + ) + x = tl.where( + dx_keep, # pyre-ignore [61] + x / (1.0 - dropout_ratio), + 0.0, + ) + y = tl.where( + dy_keep, # pyre-ignore [61] + y / (1.0 - dropout_ratio), + 0.0, + ) + else: + y = tl.where( + dy_keep, # pyre-ignore [61] + y / (1.0 - dropout_ratio), + 0.0, + ) + if CONCAT_UX: + tl.store(Y + offsets, u.to(Y.dtype.element_ty), mask=mask) + tl.store(Y + Heads * D + offsets, x.to(Y.dtype.element_ty), mask=mask) + tl.store(Y + 2 * Heads * D + offsets, y.to(Y.dtype.element_ty), mask=mask) + else: + tl.store(Y + offsets, y.to(Y.dtype.element_ty), mask=mask) + + xhat = tl.where(mask, xhat, 0.0) + wdy = tl.where(mask, wdy, 0.0) + c1 = tl.sum(xhat * wdy, axis=1) / D + c2 = tl.sum(wdy, axis=1) / D + dx += (wdy - (xhat * c1[:, None] + c2[:, None])) * rstd[:, None] + # Write dx + tl.store(DX + offsets, dx, mask=mask) + + # Offset locks and weights/biases gradient pointer for parallel reduction + lock_id = row % GROUP_N + DW = DW + lock_id * Heads + off_heads + DB = DB + lock_id * Heads + off_heads + # Accumulate partial sums for dw/db + partial_dw = tl.sum(dy * xhat, axis=1) + partial_dw = tl.ravel(partial_dw) + partial_db = tl.sum(dy, axis=1) + partial_db = tl.ravel(partial_db) + tl.atomic_add( + DW, + partial_dw, + mask=mask_h, + sem="relaxed", + ) + tl.atomic_add( + DB, + partial_db, + mask=mask_h, + sem="relaxed", + ) + + +def triton_group_norm_mul_dropout_fwd( + x: torch.Tensor, + u: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + eps: float, + dropout_ratio: float, + training: bool, + silu_u: bool = False, + concat_ux: bool = False, + num_heads: int = 1, + linear_dim: int = -1, + seed: Optional[int] = None, +) -> Tuple[ + torch.Tensor, torch.Tensor, torch.Tensor, int, int, int, int +]: # y, mean, rstd, BLOCK_D, BLOCK_H, num_warps, seed + assert x.dim() == 2 + assert x.shape == u.shape + assert x.shape[1] == num_heads * linear_dim + x = switch_to_contiguous_if_needed(x) + u = switch_to_contiguous_if_needed(u) + N, _ = x.shape + assert weight.dim() == 1 + assert bias.dim() == 1 + assert weight.numel() == num_heads + assert bias.numel() == num_heads + + if concat_ux: + y = torch.empty((N, 3 * num_heads * linear_dim), dtype=x.dtype, device=x.device) + else: + y = torch.empty((N, num_heads * linear_dim), dtype=x.dtype, device=x.device) + mean = torch.empty((N * num_heads,), dtype=torch.float32, device=x.device) + rstd = torch.empty((N * num_heads,), dtype=torch.float32, device=x.device) + if N == 0: + return y, mean, rstd, 0, 0, 0, 0 + # Less than 64KB per feature: enqueue fused kernel + MAX_FUSED_SIZE = 65536 // x.element_size() + BLOCK_D: int = triton.next_power_of_2(linear_dim) + BLOCK_H: int = triton.next_power_of_2(num_heads) + if BLOCK_D * BLOCK_H > MAX_FUSED_SIZE: + raise RuntimeError( + "This group norm doesn't support num_heads * linear_dim >= 64KB." + ) + + if seed is None: + seed = torch.randint(low=0, high=2**62, size=(1,), dtype=torch.int64).item() + num_warps: int = min(max(BLOCK_D * BLOCK_H // 256, 1), 8) + # pyre-ignore[28] + _group_norm_mul_dropout_fwd[(N,)]( + x, + u, + y, + weight, + bias, + mean, + rstd, + linear_dim, + num_heads, + eps, + seed, + dropout_ratio, + x.stride(0), + u.stride(0), + y.stride(0), + SILU_U=silu_u, + BLOCK_D=BLOCK_D, + BLOCK_H=BLOCK_H, + TRAINING=training, + CONCAT_UX=concat_ux, + num_warps=num_warps, + ) + return y, mean, rstd, BLOCK_D, BLOCK_H, num_warps, seed # pyre-ignore [7] + + +def triton_group_norm_mul_dropout_bwd( + dy: torch.Tensor, + x: torch.Tensor, + u: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + mean: torch.Tensor, + rstd: torch.Tensor, + BLOCK_D: int, + BLOCK_H: int, + num_warps: int, + eps: float, + training: bool, + dropout_ratio: float, + seed: Optional[int] = None, + silu_u: bool = False, + concat_ux: bool = False, + num_heads: int = 1, + linear_dim: int = -1, + compute_y: bool = False, +) -> Tuple[ + torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, Optional[torch.Tensor] +]: + y = None + N, dim = x.shape + if compute_y: + if concat_ux: + y = torch.empty( + (N, 3 * num_heads * linear_dim), dtype=x.dtype, device=x.device + ) + else: + y = torch.empty((N, num_heads * linear_dim), dtype=x.dtype, device=x.device) + if N == 0: + return ( + torch.zeros_like(x), + torch.zeros_like(u), + torch.zeros_like(weight), + torch.zeros_like(bias), + y, + ) + dx = torch.empty_like(x) + du = torch.empty_like(u) + if dim <= 1024: + GROUP_N = 256 * 8 + elif dim <= 4096: + GROUP_N = 128 * 8 + elif dim <= 8192: + GROUP_N = 96 * 8 + else: + GROUP_N = 64 * 8 + GROUP_N = N if GROUP_N > N else GROUP_N + _dweight = torch.zeros((GROUP_N, num_heads), dtype=torch.float32, device=x.device) + _dbias = torch.zeros((GROUP_N, num_heads), dtype=torch.float32, device=x.device) + dweight = torch.empty((num_heads,), dtype=weight.dtype, device=x.device) + dbias = torch.empty((num_heads,), dtype=weight.dtype, device=x.device) + # pyre-ignore[28] + _group_norm_mul_dropout_bwd_dx_du[(N,)]( + dx, + du, + dy, + _dweight, + _dbias, + x, + u, + y, + weight, + bias, + mean, + rstd, + dx.stride(0), + du.stride(0), + dy.stride(0), + x.stride(0), + u.stride(0), + y.stride(0) if compute_y else 0, # pyre-ignore [16] + linear_dim, + num_heads, + eps, + seed, + dropout_ratio, + SILU_U=silu_u, + GROUP_N=GROUP_N, + BLOCK_D=BLOCK_D, + BLOCK_H=BLOCK_H, + TRAINING=training, + CONCAT_UX=concat_ux, + COMPUTE_Y=compute_y, + num_warps=num_warps, + ) + _group_norm_bwd_dwdb[(num_heads,)]( + _dweight, + _dbias, + dweight, + dbias, + GROUP_N, + ) + return dx, du, dweight, dbias, y + + +def _get_bwd_dwdb_configs() -> List[triton.Config]: + configs = [] + for BLOCK_N in [32, 64, 128, 256]: + for num_warps in [8, 16] + ([] if torch.ops.hip else [32]): + configs.append( + triton.Config( + {"BLOCK_N": BLOCK_N}, + num_warps=num_warps, + ) + ) + return configs + + +@triton_autotune( + configs=_get_bwd_dwdb_configs(), + key=[], +) +@triton.jit +def _group_norm_bwd_dwdb( + DW, + DB, + FINAL_DW, + FINAL_DB, + N, + BLOCK_N: tl.constexpr, +): + col = tl.program_id(0) + num_heads = tl.num_programs(0) + dw = tl.zeros((BLOCK_N,), dtype=tl.float32) + db = tl.zeros((BLOCK_N,), dtype=tl.float32) + + for i in range(0, N, BLOCK_N): + rows = i + tl.arange(0, BLOCK_N) + mask = rows < N + offs = rows * num_heads + col + dw += tl.load(DW + offs, mask=mask, other=0.0) + db += tl.load(DB + offs, mask=mask, other=0.0) + + sum_dw = tl.sum(dw, axis=0) + sum_db = tl.sum(db, axis=0) + tl.store(FINAL_DW + col, sum_dw.to(FINAL_DW.dtype.element_ty)) + tl.store(FINAL_DB + col, sum_db.to(FINAL_DB.dtype.element_ty)) + + +class GroupNormMulDropoutFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore[14] + def forward( + ctx, + x: torch.Tensor, + u: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + eps: float, + dropout_ratio: float, + training: bool, + silu_u: bool = False, + concat_ux: bool = False, + num_heads: int = 1, + linear_dim: int = -1, + seed: Optional[int] = None, + ) -> torch.Tensor: + y, mean, rstd, BLOCK_D, BLOCK_H, num_warps, seed = ( + triton_group_norm_mul_dropout_fwd( + x=x, + u=u, + weight=weight, + bias=bias, + eps=eps, + dropout_ratio=dropout_ratio, + training=training, + silu_u=silu_u, + concat_ux=concat_ux, + num_heads=num_heads, + linear_dim=linear_dim, + seed=seed, + ) + ) + ctx.save_for_backward(x, u, weight, bias, mean, rstd) + ctx.BLOCK_D = BLOCK_D + ctx.BLOCK_H = BLOCK_H + ctx.num_warps = num_warps + ctx.eps = eps + ctx.seed = seed + ctx.training = training + ctx.silu_u = silu_u + ctx.concat_ux = concat_ux + ctx.dropout_ratio = dropout_ratio + ctx.num_heads = num_heads + ctx.linear_dim = linear_dim + return y + + @staticmethod + # pyre-ignore[14] + def backward( + ctx, dy: torch.Tensor + ) -> Tuple[ + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + None, + None, + None, + None, + None, + None, + None, + None, + ]: + x, u, weight, bias, mean, rstd = ctx.saved_tensors + dx, du, dweight, dbias, _ = triton_group_norm_mul_dropout_bwd( + dy=dy, + x=x, + u=u, + weight=weight, + bias=bias, + mean=mean, + rstd=rstd, + BLOCK_D=ctx.BLOCK_D, + BLOCK_H=ctx.BLOCK_H, + num_warps=ctx.num_warps, + eps=ctx.eps, + training=ctx.training, + dropout_ratio=ctx.dropout_ratio, + seed=ctx.seed, + silu_u=ctx.silu_u, + concat_ux=ctx.concat_ux, + num_heads=ctx.num_heads, + linear_dim=ctx.linear_dim, + compute_y=False, + ) + return ( + dx, + du, + dweight, + dbias, + None, + None, + None, + None, + None, + None, + None, + None, + ) + + +class HSTUComputeOutputFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore[14] + def forward( + ctx, + attn: torch.Tensor, + u: torch.Tensor, + x: torch.Tensor, + norm_weight: torch.Tensor, + norm_bias: torch.Tensor, + output_weight: torch.Tensor, + eps: float, + dropout_ratio: float, + training: bool, + silu_u: bool = False, + concat_u: bool = False, + concat_x: bool = False, + mul_u_activation_type: str = "none", + group_norm: bool = False, + num_heads: int = 1, + linear_dim: int = -1, + seed: Optional[int] = None, + recompute_y_in_backward: bool = False, + ) -> torch.Tensor: + if dropout_ratio == 0.0: + training = False + + if group_norm: + y, mean, rstd, BLOCK_D, BLOCK_H, num_warps, seed = ( + triton_group_norm_mul_dropout_fwd( + x=attn, + u=u, + weight=norm_weight, + bias=norm_bias, + eps=eps, + dropout_ratio=dropout_ratio, + training=training, + silu_u=silu_u, + concat_ux=concat_u and concat_x, + num_heads=num_heads, + linear_dim=linear_dim, + seed=seed, + ) + ) + ctx.BLOCK_H = BLOCK_H + random_mask = None + else: + y, mean, rstd, BLOCK_D, num_warps, seed, random_mask = ( + triton_layer_norm_mul_dropout_fwd( + x=attn, + u=u, + weight=norm_weight, + bias=norm_bias, + eps=eps, + dropout_ratio=dropout_ratio, + training=training, + silu_u=silu_u, + concat_u=concat_u, + concat_x=concat_x, + seed=seed, + ) + ) + + out = maybe_triton_addmm_fwd(x=y, w=output_weight, y=x) + + saved_tensors = [attn, u, norm_weight, norm_bias, mean, rstd, output_weight] + if not recompute_y_in_backward: + saved_tensors.append(y) + # Save random_mask for reuse in backward pass (avoids regenerating mask) + # When random_mask is available (SM100+ path), always save it. + if random_mask is not None: + saved_tensors.append(random_mask) + ctx.has_random_mask = True + else: + ctx.has_random_mask = False + ctx.save_for_backward(*saved_tensors) + ctx.BLOCK_D = BLOCK_D + ctx.num_warps = num_warps + ctx.eps = eps + ctx.seed = seed + ctx.training = training + ctx.concat_u = concat_u + ctx.concat_x = concat_x + ctx.dropout_ratio = dropout_ratio + ctx.num_heads = num_heads + ctx.linear_dim = linear_dim + ctx.group_norm = group_norm + ctx.recompute_y_in_backward = recompute_y_in_backward + ctx.silu_u = silu_u + ctx.mul_u_activation_type = mul_u_activation_type + return out + + @staticmethod + # pyre-ignore[14] + def backward( + ctx, dout: torch.Tensor + ) -> Tuple[ + torch.Tensor, # dattn + torch.Tensor, # du + torch.Tensor, # dx + torch.Tensor, # d_norm_weight + torch.Tensor, # d_norm_bias + torch.Tensor, # d_output_weight + None, # eps + None, # dropout_ratio + None, # training + None, # silu_u + None, # concat_u + None, # concat_x + None, # mul_u_activation_type + None, # group_norm + None, # num_heads + None, # linear_dim + None, # seed + None, # recompute_y_in_backward + ]: + attn, u, norm_weight, norm_bias, mean, rstd, output_weight = ctx.saved_tensors[ + :7 + ] + # Extract optional saved tensors based on flags + next_idx = 7 + if not ctx.recompute_y_in_backward: + saved_y = ctx.saved_tensors[next_idx] + next_idx += 1 + else: + saved_y = None + if ctx.has_random_mask: + random_mask = ctx.saved_tensors[next_idx] + else: + random_mask = None + dy = torch.mm(dout, output_weight.t()) + + if ctx.group_norm: + dattn, du, d_norm_weight, d_norm_bias, y = ( + triton_group_norm_mul_dropout_bwd( + dy=dy, + x=attn, + u=u, + weight=norm_weight, + bias=norm_bias, + mean=mean, + rstd=rstd, + BLOCK_D=ctx.BLOCK_D, + BLOCK_H=ctx.BLOCK_H, + num_warps=ctx.num_warps, + eps=ctx.eps, + training=ctx.training, + dropout_ratio=ctx.dropout_ratio, + seed=ctx.seed, + silu_u=ctx.silu_u, + concat_ux=ctx.concat_u and ctx.concat_x, + num_heads=ctx.num_heads, + linear_dim=ctx.linear_dim, + compute_y=ctx.recompute_y_in_backward, + ) + ) + else: + dattn, du, d_norm_weight, d_norm_bias, y = ( + triton_layer_norm_mul_dropout_bwd( + dy=dy, + x=attn, + u=u, + weight=norm_weight, + bias=norm_bias, + mean=mean, + rstd=rstd, + BLOCK_D=ctx.BLOCK_D, + num_warps=ctx.num_warps, + eps=ctx.eps, + training=ctx.training, + dropout_ratio=ctx.dropout_ratio, + seed=ctx.seed, + silu_u=ctx.silu_u, + concat_u=ctx.concat_u, + concat_x=ctx.concat_x, + mul_u_activation_type=ctx.mul_u_activation_type, + compute_y=ctx.recompute_y_in_backward, + random_mask=random_mask, + ) + ) + if not ctx.recompute_y_in_backward: + y = saved_y + d_output_weight = torch.mm(y.t(), dout) + return ( + dattn, + du, + dout, + d_norm_weight, + d_norm_bias, + d_output_weight, + None, # eps + None, # dropout_ratio + None, # training + None, # silu_u + None, # concat_u + None, # concat_x + None, # mul_u_activation_type + None, # group_norm + None, # num_heads + None, # linear_dim + None, # seed + None, # recompute_y_in_backward + ) + + +@triton.jit +def _helion_ln_mul_dropout_fwd( + x, + weight, + bias, + u, + y, + mean, + rstd, + eps, + seed, + dropout_ratio, + D: tl.constexpr, + stride_x: tl.constexpr, + stride_u: tl.constexpr, + stride_y: tl.constexpr, + BLOCK_D: tl.constexpr, + CONCAT_UX: tl.constexpr, + SILU_U: tl.constexpr, + TRAINING: tl.constexpr, +): + row = tl.program_id(0) + x += row.to(tl.int64) * stride_x + u += row.to(tl.int64) * stride_u + y += row.to(tl.int64) * stride_y + cols = tl.arange(0, BLOCK_D) + mask = cols < D + + # Load input + x_val = tl.load(x + cols, mask=mask, other=0.0).to(tl.float32) + + # Precompute inverse of D for faster computation + inv_D = 1.0 / D + + # Compute mean + mean_val = tl.sum(x_val, axis=0) * inv_D + + # Center the data + x_mean = tl.where(mask, x_val - mean_val, 0.0) + + # Compute variance + var = tl.sum(x_mean * x_mean, axis=0) * inv_D + + # Compute reciprocal standard deviation + # pyre-fixme[16] + rstd_val = libdevice.rsqrt(var + eps) + + # Normalize + y_norm = x_mean * rstd_val + + # Apply weight and bias + w = tl.load(weight + cols, mask=mask, other=0.0).to(tl.float32) + b = tl.load(bias + cols, mask=mask, other=0.0).to(tl.float32) + y_ln = y_norm * w + b + + # Load u and optionally apply SiLU activation + u_val = tl.load(u + cols, mask=mask, other=0.0).to(tl.float32) + if SILU_U: + u_processed = u_val * tl.sigmoid(u_val) + else: + u_processed = u_val + + y_out = y_ln * u_processed + + if TRAINING: + # Compute dropout scale + # pyre-fixme[16] + dropout_scale = fast_dividef(1.0, 1.0 - dropout_ratio) + + if CONCAT_UX: + # Generate dropout masks + random_offsets = 3 * row * BLOCK_D + cols + random_u, random_x, random_y = rand3x(seed, random_offsets) + + u_keep = random_u > dropout_ratio + x_keep = random_x > dropout_ratio + y_keep = random_y > dropout_ratio + + # Apply dropout to u, x, y + u_output = tl.where(u_keep, u_processed * dropout_scale, 0.0) + x_output = tl.where(x_keep, x_val * dropout_scale, 0.0) + y_output = tl.where(y_keep, y_out * dropout_scale, 0.0) + else: + # Generate dropout mask for y + random_offsets = row * BLOCK_D + cols + random_y = tl.rand(seed, random_offsets) + y_keep = random_y > dropout_ratio + + # Apply dropout to y + y_output = tl.where(y_keep, y_out * dropout_scale, 0.0) + else: + if CONCAT_UX: + u_output = u_processed + x_output = x_val + y_output = y_out + + # Store outputs + if CONCAT_UX: + tl.store(y + cols, u_output.to(y.dtype.element_ty), mask=mask) + tl.store(y + D + cols, x_output.to(y.dtype.element_ty), mask=mask) + tl.store(y + 2 * D + cols, y_output.to(y.dtype.element_ty), mask=mask) + else: + tl.store(y + cols, y_output.to(y.dtype.element_ty), mask=mask) + + # Store mean and rstd + tl.store(mean + row, mean_val) + tl.store(rstd + row, rstd_val) + + +def helion_layer_norm_mul_dropout_fwd( + x: torch.Tensor, + u: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + eps: float, + dropout_ratio: float, + training: bool, + silu_u: bool = False, + concat_ux: bool = False, + seed: Optional[int] = None, +) -> Tuple[ + torch.Tensor, torch.Tensor, torch.Tensor, int, int, int +]: # y, mean, rstd, BLOCK_D, num_warps, seed + N, D = x.shape + + if seed is None: + seed = torch.randint(low=0, high=2**62, size=(1,), dtype=torch.int64).item() + + if concat_ux: + y = torch.empty([N, 3 * D], dtype=x.dtype, device=x.device) + else: + y = torch.empty([N, D], dtype=x.dtype, device=x.device) + mean = torch.empty([N], dtype=torch.float32, device=x.device) + rstd = torch.empty([N], dtype=torch.float32, device=x.device) + + BLOCK_D = triton.next_power_of_2(D) + # pyre-ignore[28] + _helion_ln_mul_dropout_fwd[(N,)]( + x, + weight, + bias, + u, + y, + mean, + rstd, + eps, + seed, + dropout_ratio, + D, + x.stride(0), + u.stride(0), + y.stride(0), + BLOCK_D, + CONCAT_UX=concat_ux, + SILU_U=silu_u, + TRAINING=training, + num_warps=1, + ) + + return y, mean, rstd, BLOCK_D, 1, seed # pyre-ignore [7] + + +@triton.jit +def _helion_ln_mul_dropout_bwd_dx_du( + DX, + DU, + DY, + DW, + DB, + X, + U, + Y, + W, + B, + Mean, + Rstd, + stride_dx, + stride_du, + stride_dy, + stride_x, + stride_u, + stride_y, + D: tl.constexpr, + eps, + seed, + dropout_ratio, + N, + SILU_U: tl.constexpr, + BLOCK_D: tl.constexpr, + TRAINING: tl.constexpr, + CONCAT_UX: tl.constexpr, + COMPUTE_Y: tl.constexpr, +): + pid = tl.program_id(0) + tile_num = tl.num_programs(0) + rows_per_tile = N // tile_num + if pid < N % tile_num: + rows_per_tile += 1 + + if rows_per_tile == 0: + return + + cols = tl.arange(0, BLOCK_D) + mask = cols < D + + # precompute inverse of D + inv_D: tl.constexpr = 1.0 / D + + row = pid + X += row.to(tl.int64) * stride_x + U += row.to(tl.int64) * stride_u + if COMPUTE_Y: + Y += row.to(tl.int64) * stride_y + DY += row.to(tl.int64) * stride_dy + DX += row.to(tl.int64) * stride_dx + DU += row.to(tl.int64) * stride_du + DW = DW + pid * D + cols + DB = DB + pid * D + cols + + partial_dw = tl.zeros((BLOCK_D,), dtype=tl.float32) + partial_db = tl.zeros((BLOCK_D,), dtype=tl.float32) + w = tl.load(W + cols, mask=mask).to(tl.float32) + b = tl.load(B + cols, mask=mask).to(tl.float32) + + for _idx in range(0, rows_per_tile): + x = tl.load(X + cols, mask=mask, other=0).to(tl.float32) + if CONCAT_UX: + du = tl.load(DY + cols, mask=mask, other=0).to(tl.float32) + dx = tl.load(DY + D + cols, mask=mask, other=0).to(tl.float32) + dy = tl.load(DY + 2 * D + cols, mask=mask, other=0).to(tl.float32) + else: + du = tl.zeros([BLOCK_D], dtype=tl.float32) + dx = tl.zeros([BLOCK_D], dtype=tl.float32) + dy = tl.load(DY + cols, mask=mask, other=0).to(tl.float32) + + if TRAINING: + # pyre-fixme[16] + dropout_scale = fast_dividef(1.0, 1.0 - dropout_ratio) + if CONCAT_UX: + random_offsets = 3 * row * BLOCK_D + cols + # apply dropout on du + random_du, random_dx, random_dy = rand3x(seed, random_offsets) + du_keep = random_du > dropout_ratio + du = tl.where(du_keep, du * dropout_scale, 0.0) + # apply dropout on dx + dx_keep = random_dx > dropout_ratio + dx = tl.where(dx_keep, dx * dropout_scale, 0.0) + # apply dropout on dy + dy_keep = random_dy > dropout_ratio + dy = tl.where(dy_keep, dy * dropout_scale, 0.0) + else: + random_offsets = row * BLOCK_D + cols + random = tl.rand(seed, random_offsets) + dy_keep = random > dropout_ratio + dy = tl.where(dy_keep, dy * dropout_scale, 0.0) + + mean = tl.load(Mean + row) + rstd = tl.load(Rstd + row) + + # Compute dx + xhat = (x - mean) * rstd + u = tl.load(U + cols, mask=mask, other=0).to(tl.float32) + ln = xhat * w + b + du += dy * ln + + if SILU_U: + sig_u = tl.sigmoid(u) + silu_u = u * sig_u + du = du * sig_u * (1 + u - silu_u) + u = silu_u + + tl.store(DU + cols, du.to(DU.dtype.element_ty), mask=mask) + dy = dy * u + wdy = w * dy + + if COMPUTE_Y: + y = ln * u + if TRAINING: + # pyre-fixme[16] + dropout_scale_y = fast_dividef(1.0, 1.0 - dropout_ratio) + if CONCAT_UX: + u = tl.where(du_keep, u * dropout_scale_y, 0.0) # pyre-ignore [61] + x = tl.where(dx_keep, x * dropout_scale_y, 0.0) # pyre-ignore [61] + y = tl.where(dy_keep, y * dropout_scale_y, 0.0) # pyre-ignore [61] + else: + y = tl.where(dy_keep, y * dropout_scale_y, 0.0) # pyre-ignore [61] + if CONCAT_UX: + tl.store(Y + cols, u.to(Y.dtype.element_ty), mask=mask) + tl.store(Y + D + cols, x.to(Y.dtype.element_ty), mask=mask) + tl.store(Y + 2 * D + cols, y.to(Y.dtype.element_ty), mask=mask) + else: + tl.store(Y + cols, y.to(Y.dtype.element_ty), mask=mask) + Y += tile_num.to(tl.int64) * stride_y + + xhat = tl.where(mask, xhat, 0.0) + wdy = tl.where(mask, wdy, 0.0) + # multiply by inv_D + c1 = tl.sum(xhat * wdy, axis=0) * inv_D + c2 = tl.sum(wdy, axis=0) * inv_D + dx += (wdy - (xhat * c1 + c2)) * rstd + + # Write dx + tl.store(DX + cols, dx, mask=mask) + + # Accumulate partial sums for dw/db + partial_dw += dy * xhat + partial_db += dy + + X += tile_num.to(tl.int64) * stride_x + U += tile_num.to(tl.int64) * stride_u + DY += tile_num.to(tl.int64) * stride_dy + DX += tile_num.to(tl.int64) * stride_dx + DU += tile_num.to(tl.int64) * stride_du + row += tile_num + + tl.store(DW, partial_dw, mask=mask) + tl.store(DB, partial_db, mask=mask) + + +@triton_autotune( + configs=_get_bwd_dwdb_configs(), + key=["D"], +) +@triton.jit +def _helion_ln_mul_dropout_bwd_dwdb( + DW, + DB, + FINAL_DW, + FINAL_DB, + N, + D, + BLOCK_N: tl.constexpr, + BLOCK_D: tl.constexpr, +): + pid = tl.program_id(0) + cols = pid * BLOCK_D + tl.arange(0, BLOCK_D) + dw = tl.zeros((BLOCK_N, BLOCK_D), dtype=tl.float32) + db = tl.zeros((BLOCK_N, BLOCK_D), dtype=tl.float32) + + for i in range(0, N, BLOCK_N): + rows = i + tl.arange(0, BLOCK_N) + # pyre-fixme[16]: `int` has no attribute `__getitem__`. + off_mask = (rows[:, None] < N) & (cols[None, :] < D) + offs = rows[:, None] * D + cols[None, :] + dw += tl.load(DW + offs, mask=off_mask, other=0.0) + db += tl.load(DB + offs, mask=off_mask, other=0.0) + + sum_dw = tl.sum(dw, axis=0) + sum_db = tl.sum(db, axis=0) + tl.store(FINAL_DW + cols, sum_dw.to(FINAL_DW.dtype.element_ty), mask=cols < D) + tl.store(FINAL_DB + cols, sum_db.to(FINAL_DB.dtype.element_ty), mask=cols < D) + + +def helion_layer_norm_mul_dropout_bwd( + dy: torch.Tensor, + x: torch.Tensor, + u: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + mean: torch.Tensor, + rstd: torch.Tensor, + BLOCK_D: int, + num_warps: int, + eps: float, + training: bool, + dropout_ratio: float, + seed: Optional[int] = None, + silu_u: bool = False, + concat_ux: bool = False, + compute_y: bool = False, +) -> Tuple[ + torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, Optional[torch.Tensor] +]: + y = None + N, D = x.shape + if compute_y: + if concat_ux: + y = torch.empty((N, 3 * D), dtype=x.dtype, device=x.device) + else: + y = torch.empty_like(x) + if N == 0: + return ( + torch.zeros_like(x), + torch.zeros_like(u), + torch.zeros((D,), dtype=weight.dtype, device=x.device), + torch.zeros((D,), dtype=weight.dtype, device=x.device), + y, + ) + dx = torch.empty_like(x) + du = torch.empty_like(u) + sms = torch.cuda.get_device_properties(x.device).multi_processor_count + tile_num = max(1, min(sms * 64, N // 4)) + _dweight = torch.empty((tile_num, D), dtype=torch.float32, device=x.device) + _dbias = torch.empty((tile_num, D), dtype=torch.float32, device=x.device) + dweight = torch.empty((D,), dtype=weight.dtype, device=x.device) + dbias = torch.empty((D,), dtype=weight.dtype, device=x.device) + + # pyre-ignore[28] + _helion_ln_mul_dropout_bwd_dx_du[(tile_num,)]( + dx, + du, + dy, + _dweight, + _dbias, + x, + u, + y, + weight, + bias, + mean, + rstd, + dx.stride(0), + du.stride(0), + dy.stride(0), + x.stride(0), + u.stride(0), + y.stride(0) if compute_y else 0, # pyre-ignore [16] + D, + eps, + seed, + dropout_ratio, + N=N, + SILU_U=silu_u, + BLOCK_D=BLOCK_D, + TRAINING=training, + CONCAT_UX=concat_ux, + COMPUTE_Y=compute_y, + num_warps=num_warps, + ) + + blocks = triton.next_power_of_2(sms * 4) + BLOCK_D_DWDB = triton.next_power_of_2(triton.cdiv(D, blocks)) + BLOCK_D_DWDB = min(max(BLOCK_D_DWDB, 4), 128) + _helion_ln_mul_dropout_bwd_dwdb[(triton.cdiv(D, BLOCK_D_DWDB),)]( + _dweight, + _dbias, + dweight, + dbias, + tile_num, + D, + BLOCK_D=BLOCK_D_DWDB, + ) + return dx, du, dweight, dbias, y + + +class HelionLayerNormMulDropoutFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore[14] + def forward( + ctx, + x: torch.Tensor, + u: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + eps: float, + dropout_ratio: float, + training: bool, + silu_u: bool = False, + concat_ux: bool = False, + seed: Optional[int] = None, + ) -> torch.Tensor: + if dropout_ratio == 0.0: + # skip dropout computation if dropout ratio is 0 + training = False + y, mean, rstd, BLOCK_D, num_warps, seed = helion_layer_norm_mul_dropout_fwd( + x=x, + u=u, + weight=weight, + bias=bias, + eps=eps, + dropout_ratio=dropout_ratio, + training=training, + silu_u=silu_u, + concat_ux=concat_ux, + seed=seed, + ) + ctx.save_for_backward(x, u, weight, bias, mean, rstd) + ctx.BLOCK_D = BLOCK_D + ctx.num_warps = num_warps + ctx.eps = eps + ctx.seed = seed + ctx.training = training + ctx.silu_u = silu_u + ctx.concat_ux = concat_ux + ctx.dropout_ratio = dropout_ratio + return y + + @staticmethod + # pyre-ignore[14] + def backward( + ctx, dy: torch.Tensor + ) -> Tuple[ + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + None, + None, + None, + None, + None, + None, + ]: + x, u, weight, bias, mean, rstd = ctx.saved_tensors + dx, du, dweight, dbias, _ = helion_layer_norm_mul_dropout_bwd( + dy=dy, + x=x, + u=u, + weight=weight, + bias=bias, + mean=mean, + rstd=rstd, + BLOCK_D=ctx.BLOCK_D, + num_warps=ctx.num_warps, + eps=ctx.eps, + training=ctx.training, + dropout_ratio=ctx.dropout_ratio, + seed=ctx.seed, + silu_u=ctx.silu_u, + concat_ux=ctx.concat_ux, + compute_y=False, + ) + return dx, du, dweight, dbias, None, None, None, None, None, None + + +@torch.fx.wrap +def helion_norm_mul_dropout( + x: torch.Tensor, + u: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + eps: float, + dropout_ratio: float, + training: bool, + silu_u: bool = False, + concat_ux: bool = False, + seed: Optional[int] = None, +) -> torch.Tensor: + return HelionLayerNormMulDropoutFunction.apply( + x, u, weight, bias, eps, dropout_ratio, training, silu_u, concat_ux, seed + ) + + +@torch.fx.wrap +def triton_norm_mul_dropout( + x: torch.Tensor, + u: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + eps: float, + dropout_ratio: float, + training: bool, + silu_u: bool = False, + concat_u: bool = False, + concat_x: bool = False, + group_norm: bool = False, + num_heads: int = 1, + linear_dim: int = -1, + seed: Optional[int] = None, +) -> torch.Tensor: + if group_norm: + return GroupNormMulDropoutFunction.apply( + x, + u, + weight, + bias, + eps, + dropout_ratio, + training, + silu_u, + concat_u and concat_x, + num_heads, + linear_dim, + seed, + ) + else: + return LayerNormMulDropoutFunction.apply( + x, + u, + weight, + bias, + eps, + dropout_ratio, + training, + silu_u, + concat_u and concat_x, + seed, + ) + + +@torch.jit.unused +@torch.fx.wrap +def triton_hstu_compute_output( + attn: torch.Tensor, + u: torch.Tensor, + x: torch.Tensor, + norm_weight: torch.Tensor, + norm_bias: torch.Tensor, + output_weight: torch.Tensor, + eps: float, + dropout_ratio: float, + training: bool, + silu_u: bool = False, + concat_u: bool = False, + concat_x: bool = False, + mul_u_activation_type: str = "none", + group_norm: bool = False, + num_heads: int = 1, + linear_dim: int = -1, + seed: Optional[int] = None, + recompute_y_in_backward: bool = False, +) -> torch.Tensor: + return HSTUComputeOutputFunction.apply( + attn, + u, + x, + norm_weight, + norm_bias, + output_weight, + eps, + dropout_ratio, + training, + silu_u, + concat_u, + concat_x, + mul_u_activation_type, + group_norm, + num_heads, + linear_dim, + seed, + recompute_y_in_backward, + ) diff --git a/recommendation_v4/generative_recommenders/ops/triton/triton_hstu_preprocess_and_attention.py b/recommendation_v4/generative_recommenders/ops/triton/triton_hstu_preprocess_and_attention.py new file mode 100644 index 000000000..bda97ff96 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton/triton_hstu_preprocess_and_attention.py @@ -0,0 +1,342 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +from typing import Optional, Tuple + +import torch +from generative_recommenders.ops.triton.triton_addmm import ( + maybe_triton_addmm_fwd, + triton_addmm_bwd, + triton_addmm_fwd, +) +from generative_recommenders.ops.triton.triton_hstu_attention import ( + _should_enable_tma, + triton_hstu_attention_bwd, + triton_hstu_attention_fwd, +) +from generative_recommenders.ops.triton.triton_layer_norm import ( + compute_BLOCK_D, + triton_weighted_layer_norm_bwd, + triton_weighted_layer_norm_fwd, +) +from torch.nn import functional as F + + +class _HSTUPreprocessAndAttentionFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore [14] + def forward( + ctx, # pyre-ignore [2] + x: torch.Tensor, + norm_weight: torch.Tensor, + norm_bias: torch.Tensor, + norm_eps: float, + num_heads: int, + attn_dim: int, + hidden_dim: int, + uvqk_weight: torch.Tensor, + uvqk_bias: torch.Tensor, + max_seq_len: int, + seq_offsets: torch.Tensor, + attn_alpha: float, + num_targets: Optional[torch.Tensor], + max_attn_len: int, + contextual_seq_len: int, + recompute_uvqk_in_backward: bool, + recompute_normed_x_in_backward: bool, + sort_by_length: bool, + enable_tma: bool, + num_softmax_heads: int = 0, + ) -> Tuple[torch.Tensor, torch.Tensor]: + assert num_softmax_heads == 0, "Softmax attention is not supported" + normed_x, x_mean, x_rstd = triton_weighted_layer_norm_fwd( + x=x, + weight=norm_weight, + bias=norm_bias, + eps=norm_eps, + ) + BLOCK_D = compute_BLOCK_D(x) + uvqk = maybe_triton_addmm_fwd( + x=normed_x, w=uvqk_weight, y=uvqk_bias + ).contiguous() + u, v, q, k = uvqk.split( + [ + hidden_dim * num_heads, + hidden_dim * num_heads, + attn_dim * num_heads, + attn_dim * num_heads, + ], + dim=1, + ) + q = q.view(-1, num_heads, attn_dim) + k = k.view(-1, num_heads, attn_dim) + v = v.view(-1, num_heads, hidden_dim) + silu_u = F.silu(u) + sort_by_length_indices = None + if sort_by_length: + seq_lengths = seq_offsets[1:] - seq_offsets[:-1] + _, sort_by_length_indices = torch.sort( + seq_lengths, descending=True, stable=False + ) + out = triton_hstu_attention_fwd( + N=max_seq_len, + alpha=attn_alpha, + q=q, + k=k, + v=v, + seq_offsets=seq_offsets, + num_targets=num_targets, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + sort_by_length_indices=sort_by_length_indices, + enable_tma=enable_tma, + num_softmax_heads=num_softmax_heads, + ) + # update ctx + saved_tensors = [ + x, + norm_weight, + norm_bias, + x_mean, + x_rstd, + uvqk_weight, + seq_offsets, + ] + if num_targets is not None: + saved_tensors.append(num_targets) + if not recompute_normed_x_in_backward: + saved_tensors.append(normed_x) + if recompute_uvqk_in_backward: + saved_tensors.append(uvqk_bias) + else: + saved_tensors.append(uvqk) + if sort_by_length: + saved_tensors.append(sort_by_length_indices) + ctx.save_for_backward(*saved_tensors) + ctx.attn_alpha = attn_alpha + ctx.has_multiple_targets = num_targets is not None + ctx.max_seq_len = max_seq_len + ctx.max_attn_len = max_attn_len + ctx.recompute_normed_x_in_backward = recompute_normed_x_in_backward + ctx.recompute_uvqk_in_backward = recompute_uvqk_in_backward + ctx.hidden_dim = hidden_dim + ctx.attn_dim = attn_dim + ctx.num_heads = num_heads + ctx.uvqk_bias_1d = uvqk_bias.dim() == 1 + ctx.norm_eps = norm_eps + ctx.norm_BLOCK_D = BLOCK_D + ctx.contextual_seq_len = contextual_seq_len + ctx.sort_by_length = sort_by_length + ctx.enable_tma = enable_tma + ctx.num_softmax_heads = num_softmax_heads + return silu_u, out + + @staticmethod + # pyre-ignore[14] + def backward( + ctx, # pyre-ignore[2] + dsilu_u: torch.Tensor, + dout: torch.Tensor, + ) -> Tuple[ + torch.Tensor, # d_x + torch.Tensor, # d_norm_weight + torch.Tensor, # d_norm_bias + None, + None, + None, + None, + torch.Tensor, # d_uvqk_weight + torch.Tensor, # d_uvqk_bias + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + ]: + x, norm_weight, norm_bias, x_mean, x_rstd, uvqk_weight, seq_offsets = ( + ctx.saved_tensors[:7] + ) + idx = 7 + if ctx.has_multiple_targets: + num_targets = ctx.saved_tensors[idx] + idx += 1 + else: + num_targets = None + if ctx.recompute_normed_x_in_backward: + normed_x, _, _ = triton_weighted_layer_norm_fwd( + x=x, + weight=norm_weight, + bias=norm_bias, + eps=ctx.norm_eps, + mean=x_mean, + rstd=x_rstd, + ) + else: + normed_x = ctx.saved_tensors[idx] + idx += 1 + if ctx.recompute_uvqk_in_backward: + uvqk_bias = ctx.saved_tensors[idx] + uvqk = maybe_triton_addmm_fwd(x=normed_x, w=uvqk_weight, y=uvqk_bias) + idx += 1 + else: + uvqk = ctx.saved_tensors[idx] + idx += 1 + if ctx.sort_by_length: + sort_by_length_indices = ctx.saved_tensors[idx] + else: + sort_by_length_indices = None + + duvqk = torch.empty_like(uvqk) + du, dv, dq, dk = duvqk.split( + [ + ctx.hidden_dim * ctx.num_heads, + ctx.hidden_dim * ctx.num_heads, + ctx.attn_dim * ctx.num_heads, + ctx.attn_dim * ctx.num_heads, + ], + dim=1, + ) + u, v, q, k = uvqk.split( + [ + ctx.hidden_dim * ctx.num_heads, + ctx.hidden_dim * ctx.num_heads, + ctx.attn_dim * ctx.num_heads, + ctx.attn_dim * ctx.num_heads, + ], + dim=1, + ) + q = q.view(-1, ctx.num_heads, ctx.attn_dim) + k = k.view(-1, ctx.num_heads, ctx.attn_dim) + v = v.view(-1, ctx.num_heads, ctx.hidden_dim) + dq = dq.view(-1, ctx.num_heads, ctx.attn_dim) + dk = dk.view(-1, ctx.num_heads, ctx.attn_dim) + dv = dv.view(-1, ctx.num_heads, ctx.hidden_dim) + # Note: the operation below updates duvqk in place + triton_hstu_attention_bwd( + dout=dout, + q=q, + k=k, + v=v, + dq=dq, + dk=dk, + dv=dv, + seq_offsets=seq_offsets, + num_targets=num_targets, + N=ctx.max_seq_len, + max_attn_len=ctx.max_attn_len, + alpha=ctx.attn_alpha, + contextual_seq_len=ctx.contextual_seq_len, + sort_by_length_indices=sort_by_length_indices, + enable_tma=ctx.enable_tma, + num_softmax_heads=ctx.num_softmax_heads, + ) + torch.ops.aten.silu_backward(dsilu_u, u, grad_input=du) + d_normed_x, d_uvqk_weight, d_uvqk_bias = triton_addmm_bwd( + x=normed_x, + w=uvqk_weight, + dz=duvqk, + is_y_1d=ctx.uvqk_bias_1d, + ) + d_x, d_norm_weight, d_norm_bias = triton_weighted_layer_norm_bwd( + dy=d_normed_x, + x=x, + weight=norm_weight, + bias=norm_bias, + mean=x_mean, + rstd=x_rstd, + learnable=True, + eps=ctx.norm_eps, + BLOCK_D=ctx.norm_BLOCK_D, + ) + # pyre-ignore[7] + return ( + d_x, + d_norm_weight, + d_norm_bias, + None, + None, + None, + None, + d_uvqk_weight, + d_uvqk_bias, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + ) + + +def triton_hstu_preprocess_and_attention( + x: torch.Tensor, + norm_weight: torch.Tensor, + norm_bias: torch.Tensor, + norm_eps: float, + num_heads: int, + attn_dim: int, + hidden_dim: int, + uvqk_weight: torch.Tensor, + uvqk_bias: torch.Tensor, + max_seq_len: int, + seq_offsets: torch.Tensor, + attn_alpha: float, + num_targets: Optional[torch.Tensor], + max_attn_len: int = 0, + contextual_seq_len: int = 0, + recompute_uvqk_in_backward: bool = False, + recompute_normed_x_in_backward: bool = False, + sort_by_length: bool = False, + enable_tma: Optional[bool] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + # When the caller does not specify enable_tma, auto-detect whether the + # TMA / TLX fast path is safe on this device. Resolving here (vs inside + # the autograd Function.forward) keeps a concrete bool flowing through + # ctx.save_for_backward / ctx attributes. + if enable_tma is None: + enable_tma = _should_enable_tma() + return _HSTUPreprocessAndAttentionFunction.apply( + x, + norm_weight, + norm_bias, + norm_eps, + num_heads, + attn_dim, + hidden_dim, + uvqk_weight, + uvqk_bias, + max_seq_len, + seq_offsets, + attn_alpha, + num_targets, + max_attn_len, + contextual_seq_len, + recompute_uvqk_in_backward, + recompute_normed_x_in_backward, + sort_by_length, + enable_tma, + ) diff --git a/recommendation_v4/generative_recommenders/ops/triton/triton_jagged.py b/recommendation_v4/generative_recommenders/ops/triton/triton_jagged.py new file mode 100644 index 000000000..7a4e82cf4 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton/triton_jagged.py @@ -0,0 +1,2533 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +#!/usr/bin/env python3 + + +import os +from typing import List, Optional, Tuple + +import torch + +# @manual=//triton:triton +import triton + +# @manual=//triton:triton +import triton.language as tl +from generative_recommenders.common import ( + autotune_max_seq_len, + fine_grained_autotune_max_seq_len, + switch_to_contiguous_if_needed, + triton_autotune, +) +from generative_recommenders.ops.utils import is_sm100_plus, is_sm90 +from torch._inductor.runtime import triton_helpers + +try: + torch.ops.load_library( + "//generative_recommenders/fb/ultra/ops/hopper/jagged_dense_bmm_add:jagged_dense_bmm_add" + ) +except OSError: + pass + +CUDA_JAGGED_DENSE_BMM_FWD = False +CUDA_JAGGED_DENSE_BMM_BWD = False + +SPLIT_2D_JAGGED_KERNEL = None +GLN_MUL_DROPOUT_KERNEL = None +CONCAT_2D_JAGGED_KERNEL = None + + +def set_cuda_jagged_dense_bmm_fwd(value: bool) -> None: + global CUDA_JAGGED_DENSE_BMM_FWD + CUDA_JAGGED_DENSE_BMM_FWD = value + + +def get_cuda_jagged_dense_bmm_fwd() -> bool: + # currently only supports H100 + return CUDA_JAGGED_DENSE_BMM_FWD and is_sm90() + + +def set_cuda_jagged_dense_bmm_bwd(value: bool) -> None: + global CUDA_JAGGED_DENSE_BMM_BWD + CUDA_JAGGED_DENSE_BMM_BWD = value + + +def get_cuda_jagged_dense_bmm_bwd() -> bool: + # currently only supports H100 + return CUDA_JAGGED_DENSE_BMM_BWD and is_sm90() + + +def set_split_2d_jagged_kernel(value: Optional[str]) -> None: + global SPLIT_2D_JAGGED_KERNEL + SPLIT_2D_JAGGED_KERNEL = value + + +def get_split_2d_jagged_kernel() -> Optional[str]: + # only override during training + if torch.is_grad_enabled(): + return SPLIT_2D_JAGGED_KERNEL + return None + + +def set_concat_2d_jagged_kernel(value: Optional[str]) -> None: + global CONCAT_2D_JAGGED_KERNEL + CONCAT_2D_JAGGED_KERNEL = value + + +def get_concat_2d_jagged_kernel() -> Optional[str]: + # only override during training + if torch.is_grad_enabled(): + return CONCAT_2D_JAGGED_KERNEL + return None + + +def _should_use_multirow() -> bool: + """Check if multirow kernel should be used based on current hardware. + + Can be overridden via the JAGGED_USE_MULTIROW_MI350 environment variable: + JAGGED_USE_MULTIROW_MI350=1 -> force multirow on + JAGGED_USE_MULTIROW_MI350=0 -> force multirow off + unset -> auto-detect based on hardware (SM100+ or MI350) + """ + env = os.environ.get("JAGGED_USE_MULTIROW_MI350") + if env is not None: + return env == "1" + return is_sm100_plus() + + +def set_gln_mul_dropout_kernel(value: Optional[str]) -> None: + global GLN_MUL_DROPOUT_KERNEL + GLN_MUL_DROPOUT_KERNEL = value + + +def get_gln_mul_dropout_kernel() -> Optional[str]: + # only override during training + return GLN_MUL_DROPOUT_KERNEL + + +def _triton_concat_2D_jagged_internal( + values_a: torch.Tensor, + values_b: torch.Tensor, + values_out: torch.Tensor, + max_seq_len: int, + B: int, + offsets_a: Optional[torch.Tensor], + offsets_b: Optional[torch.Tensor], + D: int, + dense_size: int, + stride_dense_batch: int, + n_prefix: int, + is_dense_a: bool, + is_dense_b: bool, + is_replace: bool, + BLOCK_D: int, +) -> None: + use_multirow = _should_use_multirow() + if n_prefix != 0: + if use_multirow: + + def grid(meta): + return (triton.cdiv(max_seq_len, meta["BLOCK_N"]), B) + + concat_2D_jagged_jagged_w_prefix_multirow[grid]( + OffsetsA=offsets_a, + ValuesA=values_a, + OffsetsB=offsets_b, + ValuesB=values_b, + Out=values_out, + D=D, + stride_ad=values_a.stride(-2), + stride_bd=values_b.stride(-2), + stride_od=values_out.stride(0), + n_prefix_from_B=n_prefix, + BLOCK_D=BLOCK_D, + ) + else: + concat_2D_jagged_jagged_w_prefix[(max_seq_len, B)]( + OffsetsA=offsets_a, + ValuesA=values_a, + OffsetsB=offsets_b, + ValuesB=values_b, + Out=values_out, + D=D, + stride_ad=values_a.stride(-2), + stride_bd=values_b.stride(-2), + stride_od=values_out.stride(0), + n_prefix_from_B=n_prefix, + BLOCK_D=BLOCK_D, # pyre-ignore[6] + ) + else: + if use_multirow: + + def grid(meta): + return (triton.cdiv(max_seq_len, meta["BLOCK_N"]), B) + + concat_2D_jagged_multirow[grid]( + OffsetsA=offsets_a, + ValuesA=values_a, + OffsetsB=offsets_b, + ValuesB=values_b, + DenseSize=dense_size, + Out=values_out, + D=D, + stride_ad=values_a.stride(-2), + stride_bd=values_b.stride(-2), + stride_dense_batch=stride_dense_batch, + stride_od=values_out.stride(0), + IS_DENSE_A=is_dense_a, # pyre-ignore[6] + IS_DENSE_B=is_dense_b, # pyre-ignore[6] + BLOCK_D=BLOCK_D, # pyre-ignore[6] + IS_REPLACE=is_replace, # pyre-ignore[6] + ) + else: + concat_2D_jagged[(max_seq_len, B)]( + OffsetsA=offsets_a, + ValuesA=values_a, + OffsetsB=offsets_b, + ValuesB=values_b, + DenseSize=dense_size, + Out=values_out, + D=D, + stride_ad=values_a.stride(-2), + stride_bd=values_b.stride(-2), + stride_dense_batch=stride_dense_batch, + stride_od=values_out.stride(0), + IS_DENSE_A=is_dense_a, # pyre-ignore[6] + IS_DENSE_B=is_dense_b, # pyre-ignore[6] + BLOCK_D=BLOCK_D, # pyre-ignore[6] + IS_REPLACE=is_replace, # pyre-ignore[6] + ) + + +def _get_split_concat_2d_jagged_multirow_configs() -> List[triton.Config]: + configs = [] + for BLOCK_N in [1, 2, 4, 8]: + for num_warps in [1, 2, 4]: + configs.append( + triton.Config( + {"BLOCK_N": BLOCK_N}, + num_warps=num_warps, + ) + ) + return configs + + +def _get_split_concat_2d_jagged_multirow_configs_wrapper() -> List[triton.Config]: + # Use extended config space only when JAGGED_USE_MULTIROW_MI350 is explicitly set, + # otherwise fall back to the existing configs to avoid breaking autotune. + if os.environ.get("JAGGED_USE_MULTIROW_MI350") is not None: + configs = [] + # Extended config space for MI350 tuning + # - BLOCK_N: number of rows processed per block + # - num_warps: number of warps (AMD wavefront = 64 threads) + # - num_stages: software pipeline depth for memory latency hiding + # NOTE: num_stages=0 is invalid for AMD GPUs, start from 1 + # - waves_per_eu: AMD-specific, controls occupancy (waves per execution unit) + for BLOCK_N in [1, 2, 4, 8, 16, 32]: + for num_warps in [1, 2, 4, 8, 16, 32]: + for num_stages in [1, 2, 3, 4]: + for waves_per_eu in [0, 1, 2, 4]: + configs.append( + triton.Config( + {"BLOCK_N": BLOCK_N, "waves_per_eu": waves_per_eu}, + num_warps=num_warps, + num_stages=num_stages, + ) + ) + return configs + return _get_split_concat_2d_jagged_multirow_configs() + + +def _get_bmm_configs() -> List[triton.Config]: + configs = [] + for BLOCK_M in [64, 128]: + for BLOCK_N in [64, 128, 256]: + for BLOCK_K in [32, 64]: + for num_stages in [3, 5]: + for num_warps in [4, 8]: + configs.append( + triton.Config( + { + "BLOCK_M": BLOCK_M, + "BLOCK_N": BLOCK_N, + "BLOCK_K": BLOCK_K, + }, + num_stages=num_stages, + num_warps=num_warps, + ) + ) + return configs + + +@triton_autotune( + configs=_get_bmm_configs(), + key=["AUTOTUNE_MAX_SEQ_LEN", "N", "K", "ELEMENTWISE", "HAS_BIAS"], +) +@triton.jit +def jagged_dense_bmm_broadcast_add_kernel( + seq_offsets, + Jagged, + Dense, + Bias, + Out, + AUTOTUNE_MAX_SEQ_LEN, + N, + K, + stride_jm, + stride_db, + stride_dk, + stride_dn, + stride_bias_b, + stride_om, + HAS_BIAS: tl.constexpr, + ALLOW_TF32: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, + ELEMENTWISE: tl.constexpr, +): + """ + Computing bmm Out = Jagged x Dense + Bias + M is the jagged dimension + Jagged has shape (sum_B(M_i), K), Dense has shape (B, K, N), Bias has shape (B, N), and Out has shape (sum_B(M_i), N) + """ + + off_n = tl.program_id(0) + off_m = tl.program_id(1).to(tl.int64) + off_b = tl.program_id(2) + + seq_start = tl.load(seq_offsets + off_b).to(tl.int64) + seq_end = tl.load(seq_offsets + off_b + 1) + seq_len = seq_end - seq_start + start_m = off_m * BLOCK_M + start_n = off_n * BLOCK_N + if start_m >= seq_len: + return + + Jagged += (seq_start + start_m) * stride_jm + Dense += off_b.to(tl.int64) * stride_db + Out += seq_start * stride_om + + offs_m = tl.arange(0, BLOCK_M) + offs_n = start_n + tl.arange(0, BLOCK_N) + offs_k = tl.arange(0, BLOCK_K) + jg_ptrs = Jagged + offs_m[:, None] * stride_jm + offs_k[None, :] + dn_ptrs = Dense + offs_k[:, None] * stride_dk + offs_n[None, :] * stride_dn + + accumulator = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32) + for k in range(0, K, BLOCK_K): + jg = tl.load( + jg_ptrs, + # pyre-fixme[16]: `int` has no attribute `__getitem__`. + mask=(offs_m[:, None] < (seq_len - start_m)) & ((k + offs_k)[None, :] < K), + other=0.0, + ) + dn = tl.load( + dn_ptrs, + mask=((k + offs_k)[:, None] < K) & (offs_n[None, :] < N), + other=0.0, + ) + accumulator += tl.dot(jg, dn, allow_tf32=ALLOW_TF32) + jg_ptrs += BLOCK_K + dn_ptrs += BLOCK_K * stride_dk + + if HAS_BIAS: + if ELEMENTWISE: + Bias += (seq_start + start_m) * stride_bias_b + bias_ptrs = Bias + offs_m[:, None] * stride_bias_b + offs_n[None, :] + bias = tl.load( + bias_ptrs, + mask=(offs_m[:, None] < (seq_len - start_m)) & (offs_n[None, :] < N), + other=0.0, + ) + accumulator += bias.to(tl.float32) + else: + bias_ptrs = Bias + off_b.to(tl.int64) * stride_bias_b + offs_n + bias = tl.load(bias_ptrs, mask=offs_n < N) + accumulator += bias[None, :].to(tl.float32) + + out = accumulator.to(Out.dtype.element_ty) + + offs_m = tl.arange(0, BLOCK_M) + offs_n = start_n + tl.arange(0, BLOCK_N) + Out += start_m * stride_om + out_ptrs = Out + offs_m[:, None] * stride_om + offs_n[None, :] + tl.store( + out_ptrs, + out, + mask=(offs_m[:, None] < (seq_len - start_m)) & (offs_n[None, :] < N), + ) + + +def _get_bmm_reduce_sum_configs() -> List[triton.Config]: + configs = [] + for BLOCK_M in [64, 128]: + for BLOCK_N in [64, 128]: + for BLOCK_K in [64, 128]: + for num_stages in [3, 4]: + for num_warps in [4, 8]: + configs.append( + triton.Config( + { + "BLOCK_M": BLOCK_M, + "BLOCK_N": BLOCK_N, + "BLOCK_K": BLOCK_K, + }, + num_stages=num_stages, + num_warps=num_warps, + ) + ) + return configs + + +@triton_autotune( + configs=_get_bmm_reduce_sum_configs(), + key=["M", "N", "AUTOTUNE_MAX_SEQ_LEN"], +) +@triton.jit +def _jagged_jagged_bmm_reduce_sum( + seq_offsets, + JaggedA, + JaggedB, + Out, + ReduceOut, + M, + N, + AUTOTUNE_MAX_SEQ_LEN, + stride_ak, + stride_bk, + stride_ob, + stride_om, + stride_on, + stride_orb, + stride_orn, + REDUCE_JAGGEDB: tl.constexpr, + ALLOW_TF32: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, +): + """ + Computing bmm Out = Jagged x Jagged + K is the jagged dimension + JaggedA has shape (sum_B(K_i), M), JaggedB has shape (sum_B(K_i), N), and Out has shape (B, M, N) + """ + + off_m = tl.program_id(0).to(tl.int64) + off_n = tl.program_id(1) + off_b = tl.program_id(2) + + seq_start = tl.load(seq_offsets + off_b).to(tl.int64) + seq_end = tl.load(seq_offsets + off_b + 1) + seq_len = seq_end - seq_start + + start_m = off_m * BLOCK_M + start_n = off_n * BLOCK_N + + accumulator = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32) + Out += off_b.to(tl.int64) * stride_ob + offs_m = tl.arange(0, BLOCK_M) + offs_n = start_n + tl.arange(0, BLOCK_N) + Out += start_m * stride_om + out_ptrs = Out + offs_m[:, None] * stride_om + offs_n[None, :] * stride_on + if REDUCE_JAGGEDB: + out_reduce_ptrs = ( + ReduceOut + off_b.to(tl.int64) * stride_orb + offs_n * stride_orn + ) + acc_reduce = tl.zeros((BLOCK_N,), dtype=tl.float32) + if seq_len == 0: + out = accumulator.to(Out.dtype.element_ty) + tl.store( + out_ptrs, + out, + mask=(offs_m[:, None] < (M - start_m)) & (offs_n[None, :] < N), + ) + if REDUCE_JAGGEDB: + if off_m == 0: + tl.store( + out_reduce_ptrs, # pyre-ignore [61] + acc_reduce.to(ReduceOut.dtype.element_ty), + mask=(offs_n < N), + ) + return + + JaggedA += seq_start * stride_ak + JaggedB += seq_start * stride_bk + offs_k = tl.arange(0, BLOCK_K) + jg_a_ptrs = JaggedA + offs_k[None, :] * stride_ak + (start_m + offs_m)[:, None] + jg_b_ptrs = JaggedB + offs_k[:, None] * stride_bk + offs_n[None, :] + + for k in range(0, seq_len, BLOCK_K): + jg_a = tl.load( + jg_a_ptrs, + # pyre-fixme[16]: `int` has no attribute `__getitem__`. + mask=(offs_m[:, None] < (M - start_m)) & ((k + offs_k)[None, :] < seq_len), + other=0.0, + ) + jg_b = tl.load( + jg_b_ptrs, + mask=(offs_n[None, :] < N) & ((k + offs_k)[:, None] < seq_len), + other=0.0, + ) + + accumulator += tl.dot(jg_a, jg_b, allow_tf32=ALLOW_TF32) + if REDUCE_JAGGEDB: + if off_m == 0: + acc_reduce += tl.sum(jg_b.to(tl.float32), axis=0) + + jg_a_ptrs += BLOCK_K * stride_ak + jg_b_ptrs += BLOCK_K * stride_bk + + out = accumulator.to(Out.dtype.element_ty) + tl.store( + out_ptrs, + out, + mask=(offs_m[:, None] < (M - start_m)) & (offs_n[None, :] < N), + ) + if REDUCE_JAGGEDB: + if off_m == 0: + tl.store( + out_reduce_ptrs, # pyre-ignore [61] + acc_reduce.to(ReduceOut.dtype.element_ty), + mask=(offs_n < N), + ) + + +class _JaggedDenseBmmFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore[14] + def forward( + ctx, + max_seq_len: int, + seq_offsets: torch.Tensor, + jagged: torch.Tensor, + dense: torch.Tensor, + ): + jagged = switch_to_contiguous_if_needed(jagged) + L, D = jagged.shape + B, _, K = dense.shape + bmm_out = torch.empty((L, K), dtype=jagged.dtype, device=jagged.device) + + grid = lambda meta: ( # noqa E731 + triton.cdiv(K, meta["BLOCK_N"]), + triton.cdiv(max_seq_len, meta["BLOCK_M"]), + B, + ) + + jagged_dense_bmm_broadcast_add_kernel[grid]( + seq_offsets=seq_offsets, + Jagged=jagged, + Dense=dense, + Bias=0, + Out=bmm_out, + AUTOTUNE_MAX_SEQ_LEN=autotune_max_seq_len(max_seq_len), + N=K, + K=D, + stride_jm=jagged.stride(0), + stride_db=dense.stride(0), + stride_dk=dense.stride(1), + stride_dn=dense.stride(2), + stride_bias_b=0, + stride_om=bmm_out.stride(0), + HAS_BIAS=False, + ALLOW_TF32=torch.backends.cuda.matmul.allow_tf32, + ELEMENTWISE=False, + ) + + ctx.save_for_backward(seq_offsets, jagged, dense) + ctx.B = B + ctx.max_seq_len = max_seq_len + ctx.K = K + ctx.D = D + return bmm_out + + @staticmethod + # pyre-ignore[14] + def backward( + ctx, d_bmm_out: torch.Tensor + ) -> Tuple[None, None, torch.Tensor, torch.Tensor]: + seq_offsets, jagged, dense = ctx.saved_tensors + d_jagged = torch.empty_like(jagged) + d_dense = torch.empty_like(dense) + + grid = lambda meta: ( # noqa E731 + triton.cdiv(ctx.D, meta["BLOCK_N"]), + triton.cdiv(ctx.max_seq_len, meta["BLOCK_M"]), + ctx.B, + ) + jagged_dense_bmm_broadcast_add_kernel[grid]( + seq_offsets=seq_offsets, + Jagged=d_bmm_out, + Dense=dense, + Bias=None, + Out=d_jagged, + AUTOTUNE_MAX_SEQ_LEN=autotune_max_seq_len(ctx.max_seq_len), + N=ctx.D, + K=ctx.K, + stride_jm=d_bmm_out.stride(0), + stride_db=dense.stride(0), + stride_dk=dense.stride(2), + stride_dn=dense.stride(1), + stride_bias_b=0, + stride_om=d_jagged.stride(0), + HAS_BIAS=False, + ALLOW_TF32=torch.backends.cuda.matmul.allow_tf32, + ELEMENTWISE=False, + ) + + grid = lambda meta: ( # noqa E731 + triton.cdiv(ctx.D, meta["BLOCK_M"]), + triton.cdiv(ctx.K, meta["BLOCK_N"]), + ctx.B, + ) + _jagged_jagged_bmm_reduce_sum[grid]( + seq_offsets=seq_offsets, + JaggedA=jagged, + JaggedB=d_bmm_out, + Out=d_dense, + ReduceOut=None, + M=ctx.D, + N=ctx.K, + AUTOTUNE_MAX_SEQ_LEN=autotune_max_seq_len(ctx.max_seq_len), + stride_ak=jagged.stride(0), + stride_bk=d_bmm_out.stride(0), + stride_ob=d_dense.stride(0), + stride_om=d_dense.stride(1), + stride_on=d_dense.stride(2), + stride_orb=0, + stride_orn=0, + REDUCE_JAGGEDB=False, + ALLOW_TF32=torch.backends.cuda.matmul.allow_tf32, + ) + + return None, None, d_jagged, d_dense + + +def _get_jagged_dense_broadcast_add_configs() -> List[triton.Config]: + configs = [] + for BLOCK_N in [16, 32, 64]: + for num_stages in [1, 2]: + for num_warps in [2, 4, 8]: + configs.append( + triton.Config( + { + "BLOCK_N": BLOCK_N, + }, + num_stages=num_stages, + num_warps=num_warps, + ) + ) + return configs + + +@triton_autotune( + configs=_get_jagged_dense_broadcast_add_configs(), + key=["AUTOTUNE_MAX_SEQ_LEN"], +) +@triton.jit +def jagged_dense_broadcast_add_kernel( + seq_offsets, + Jagged, + Dense, + Out, + AUTOTUNE_MAX_SEQ_LEN, + D, + stride_jn, + stride_db, + stride_on, + BLOCK_N: tl.constexpr, + BLOCK_D: tl.constexpr, +): + """ + Computing Out = Jagged + Dense + JaggedA has shape (sum_B(N_i), D), Dense has shape (B, D), and Out has shape (sum_B(N_i), D) + """ + + off_b = tl.program_id(0) + off_n = tl.program_id(1) + seq_start = tl.load(seq_offsets + off_b) + seq_end = tl.load(seq_offsets + off_b + 1) + seq_len = seq_end - seq_start + start_n = off_n * BLOCK_N + if start_n >= seq_len: + return + Jagged += seq_start * stride_jn + Dense += off_b * stride_db + Out += seq_start * stride_on + offs_n = start_n + tl.arange(0, BLOCK_N) + offs_d = tl.arange(0, BLOCK_D) + jagged_ptrs = Jagged + offs_n[:, None] * stride_jn + offs_d[None, :] + dense_ptrs = Dense + offs_d + out_ptrs = Out + offs_n[:, None] * stride_jn + offs_d[None, :] + for d in range(0, D, BLOCK_D): + jg = tl.load( + jagged_ptrs, + # pyre-fixme[16]: `int` has no attribute `__getitem__`. + mask=(offs_n[:, None] < seq_len) & ((d + offs_d)[None, :] < D), + ) + dn = tl.load(dense_ptrs, mask=d + offs_d < D) + out = jg + dn[None, :] + tl.store( + out_ptrs, + out, + mask=(offs_n[:, None] < seq_len) & ((d + offs_d)[None, :] < D), + ) + dense_ptrs += BLOCK_D + jagged_ptrs += BLOCK_D + out_ptrs += BLOCK_D + + +@triton.jit +def jagged_reduce_sum( + seq_offsets, + Jagged, + Out, + D, + stride_jn, + stride_ob, + BLOCK_D: tl.constexpr, +): + """ + Computing Out = Jagged + Dense + JaggedA has shape (sum_B(N_i), D), Dense has shape (B, D), and Out has shape (sum_B(N_i), D) + """ + off_b = tl.program_id(0) + off_d = tl.program_id(1) * BLOCK_D + seq_start = tl.load(seq_offsets + off_b) + seq_end = tl.load(seq_offsets + off_b + 1) + seq_len = seq_end - seq_start + Jagged += seq_start * stride_jn + Out += off_b * stride_ob + offs_d = off_d + tl.arange(0, BLOCK_D) + jagged_ptrs = Jagged + offs_d + out_ptrs = Out + offs_d + accumulator = tl.zeros((BLOCK_D,), dtype=tl.float32) + for _ in range(0, seq_len): + jg = tl.load( + jagged_ptrs, + mask=offs_d < D, + ) + accumulator += jg + jagged_ptrs += stride_jn + out = accumulator.to(Out.dtype.element_ty) + tl.store( + out_ptrs, + out, + mask=offs_d < D, + ) + + +class _JaggedDenseBroadcastAddFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore[14] + def forward( + ctx, + max_seq_len: int, + seq_offsets: torch.Tensor, + jagged: torch.Tensor, + dense: torch.Tensor, + ): + jagged = switch_to_contiguous_if_needed(jagged) + dense = switch_to_contiguous_if_needed(dense) + L, D = jagged.shape + B, _ = dense.shape + out = torch.empty_like(jagged) + + grid = lambda meta: ( # noqa E731 + B, + triton.cdiv(max_seq_len, meta["BLOCK_N"]), + ) + BLOCK_D = triton.next_power_of_2(D) if D < 64 else 64 + jagged_dense_broadcast_add_kernel[grid]( + seq_offsets=seq_offsets, + Jagged=jagged, + Dense=dense, + Out=out, + AUTOTUNE_MAX_SEQ_LEN=autotune_max_seq_len(max_seq_len), + D=D, + stride_jn=jagged.stride(0), + stride_db=dense.stride(0), + stride_on=out.stride(0), + BLOCK_D=BLOCK_D, + ) + + ctx.save_for_backward(seq_offsets) + ctx.max_seq_len = max_seq_len + ctx.B = B + ctx.D = D + return out + + @staticmethod + # pyre-ignore[14] + def backward( + ctx, d_out: torch.Tensor + ) -> Tuple[None, None, torch.Tensor, torch.Tensor]: + seq_offsets = ctx.saved_tensors[0] + d_dense = torch.empty((ctx.B, ctx.D), device=d_out.device, dtype=d_out.dtype) + BLOCK_D = triton.next_power_of_2(ctx.D) if ctx.D < 64 else 64 + jagged_reduce_sum[(ctx.B, triton.cdiv(ctx.D, BLOCK_D))]( + seq_offsets=seq_offsets, + Jagged=d_out, + Out=d_dense, + D=ctx.D, + stride_jn=d_out.stride(0), + stride_ob=d_dense.stride(0), + BLOCK_D=BLOCK_D, + ) + return None, None, d_out, d_dense + + +def triton_jagged_dense_bmm_add_fwd( + max_seq_len: int, + seq_offsets: torch.Tensor, + jagged: torch.Tensor, + dense: torch.Tensor, + bias: torch.Tensor, + elementwise: bool = False, +) -> Tuple[torch.Tensor, int, int, int]: + jagged = switch_to_contiguous_if_needed(jagged) + bias = switch_to_contiguous_if_needed(bias) + L, K = jagged.shape + B, _, N = dense.shape + out = torch.empty((L, N), dtype=jagged.dtype, device=jagged.device) + + grid = lambda meta: ( # noqa E731 + triton.cdiv(N, meta["BLOCK_N"]), + triton.cdiv(max_seq_len, meta["BLOCK_M"]), + B, + ) + + jagged_dense_bmm_broadcast_add_kernel[grid]( + seq_offsets=seq_offsets, + Jagged=jagged, + Dense=dense, + Bias=bias, + Out=out, + AUTOTUNE_MAX_SEQ_LEN=fine_grained_autotune_max_seq_len(max_seq_len), + N=N, + K=K, + stride_jm=jagged.stride(0), + stride_db=dense.stride(0), + stride_dk=dense.stride(1), + stride_dn=dense.stride(2), + stride_bias_b=bias.stride(0), + stride_om=out.stride(0), + HAS_BIAS=True, + ALLOW_TF32=torch.backends.cuda.matmul.allow_tf32, + ELEMENTWISE=elementwise, + ) + + return out, B, K, N + + +def triton_jagged_dense_bmm_add_bwd_jagged( + max_seq_len: int, + seq_offsets: torch.Tensor, + d_jagged: torch.Tensor, + dense: torch.Tensor, + d_out: torch.Tensor, + K: int, + B: int, + N: int, +) -> torch.Tensor: + grid = lambda meta: ( # noqa E731 + triton.cdiv(K, meta["BLOCK_N"]), + triton.cdiv(max_seq_len, meta["BLOCK_M"]), + B, + ) + jagged_dense_bmm_broadcast_add_kernel[grid]( + seq_offsets=seq_offsets, + Jagged=d_out, + Dense=dense, + Bias=None, + Out=d_jagged, + AUTOTUNE_MAX_SEQ_LEN=autotune_max_seq_len(max_seq_len), + N=K, + K=N, + stride_jm=d_out.stride(0), + stride_db=dense.stride(0), + stride_dk=dense.stride(2), + stride_dn=dense.stride(1), + stride_bias_b=0, + stride_om=d_jagged.stride(0), + HAS_BIAS=False, + ALLOW_TF32=torch.backends.cuda.matmul.allow_tf32, + ELEMENTWISE=False, + ) + + return d_jagged + + +def triton_jagged_dense_bmm_add_bwd_dense_bias( + max_seq_len: int, + seq_offsets: torch.Tensor, + jagged: torch.Tensor, + d_dense: torch.Tensor, + B: int, + K: int, + N: int, + d_out: torch.Tensor, + elementwise: bool, +) -> Tuple[torch.Tensor, torch.Tensor]: + d_bias = torch.empty((B, N), device=d_out.device, dtype=d_out.dtype) + + grid = lambda meta: ( # noqa E731 + triton.cdiv(K, meta["BLOCK_M"]), + triton.cdiv(N, meta["BLOCK_N"]), + B, + ) + + if elementwise: + d_bias = d_out + reduce_out = None + stride_orb = 0 + stride_orn = 0 + reduce_jaggedb = False + else: + reduce_out = d_bias + stride_orb = d_bias.stride(0) + stride_orn = d_bias.stride(1) + reduce_jaggedb = True + + _jagged_jagged_bmm_reduce_sum[grid]( + seq_offsets=seq_offsets, + JaggedA=jagged, + JaggedB=d_out, + Out=d_dense, + ReduceOut=reduce_out, + M=K, + N=N, + AUTOTUNE_MAX_SEQ_LEN=autotune_max_seq_len(max_seq_len), + stride_ak=jagged.stride(0), + stride_bk=d_out.stride(0), + stride_ob=d_dense.stride(0), + stride_om=d_dense.stride(1), + stride_on=d_dense.stride(2), + stride_orb=stride_orb, + stride_orn=stride_orn, + REDUCE_JAGGEDB=reduce_jaggedb, + ALLOW_TF32=torch.backends.cuda.matmul.allow_tf32, + ) + + return d_dense, d_bias + + +class _JaggedDenseBmmAddFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore[14] + def forward( + ctx, + max_seq_len: int, + seq_offsets: torch.Tensor, + jagged: torch.Tensor, + dense: torch.Tensor, + bias: torch.Tensor, + elementwise: bool = False, + ): + if get_cuda_jagged_dense_bmm_fwd(): + jagged = switch_to_contiguous_if_needed(jagged) + bias = switch_to_contiguous_if_needed(bias) + # Ensure bias has same dtype as jagged (required by CUDA kernel) + bias = bias.to(jagged.dtype) + # Ensure seq_offsets is int64 (required by CUDA kernel) + seq_offsets = seq_offsets.to(torch.int64) + _, K = jagged.shape + B, _, N = dense.shape + out = torch.ops.jagged_dense_bmm_broadcast_add.jagged_dense_bmm_broadcast_add_fwd( + max_seq_len, seq_offsets, jagged, dense, bias, elementwise + ) + else: + out, B, K, N = triton_jagged_dense_bmm_add_fwd( + max_seq_len, seq_offsets, jagged, dense, bias, elementwise + ) + + ctx.save_for_backward(seq_offsets, jagged, dense) + ctx.B = B + ctx.max_seq_len = max_seq_len + ctx.K = K + ctx.N = N + ctx.elementwise = elementwise + return out + + @staticmethod + # pyre-ignore[14] + def backward( + ctx, d_out: torch.Tensor + ) -> Tuple[None, None, torch.Tensor, torch.Tensor, torch.Tensor, None]: + seq_offsets, jagged, dense = ctx.saved_tensors + if get_cuda_jagged_dense_bmm_bwd(): + d_jagged, d_dense, d_bias = ( + torch.ops.jagged_dense_bmm_broadcast_add.jagged_dense_bmm_broadcast_add_bwd( + ctx.max_seq_len, + d_out, + seq_offsets.to(torch.int64), + jagged, + dense, + ctx.elementwise, + ) + ) + else: + d_jagged = triton_jagged_dense_bmm_add_bwd_jagged( + ctx.max_seq_len, + seq_offsets, + torch.empty_like(jagged), + dense, + d_out, + ctx.K, + ctx.B, + ctx.N, + ) + d_dense, d_bias = triton_jagged_dense_bmm_add_bwd_dense_bias( + ctx.max_seq_len, + seq_offsets, + jagged, + torch.empty_like(dense), + ctx.B, + ctx.K, + ctx.N, + d_out, + ctx.elementwise, + ) + + return None, None, d_jagged, d_dense, d_bias, None + + +@triton.jit +def concat_2D_jagged_w_prefix( + OffsetsA, + ValuesA, + OffsetsB, + ValuesB, + DenseSize, + Out, + D, + stride_ad, + stride_bd, + stride_dense_batch, + stride_od, + n_prefix_from_B, # nonzero is not supported when IS_REPLACE=True + IS_DENSE_A: tl.constexpr, + IS_DENSE_B: tl.constexpr, + BLOCK_D: tl.constexpr, + IS_REPLACE: tl.constexpr, +): + off_z = tl.program_id(1) + off_n = tl.program_id(0) + if IS_DENSE_A: + seq_start_a = off_z * DenseSize + seq_len_a = DenseSize + seq_start_b = tl.load(OffsetsB + off_z) + seq_end_b = tl.load(OffsetsB + off_z + 1) + seq_len_b = seq_end_b - seq_start_b + elif IS_DENSE_B: + seq_start_a = tl.load(OffsetsA + off_z) + seq_end_a = tl.load(OffsetsA + off_z + 1) + seq_len_a = seq_end_a - seq_start_a + seq_start_b = off_z * DenseSize + seq_len_b = DenseSize + else: + seq_start_a = tl.load(OffsetsA + off_z) + seq_end_a = tl.load(OffsetsA + off_z + 1) + seq_len_a = seq_end_a - seq_start_a + seq_start_b = tl.load(OffsetsB + off_z) + seq_end_b = tl.load(OffsetsB + off_z + 1) + seq_len_b = seq_end_b - seq_start_b + + if IS_REPLACE: + seq_len = seq_len_a + else: + seq_len = seq_len_a + seq_len_b + if off_n >= seq_len: + return + + offs_d = tl.arange(0, BLOCK_D) + if IS_REPLACE: + out_seq_start = seq_start_a + off_n + out_seq_b_start = seq_len_a - seq_len_b + else: + out_seq_start = seq_start_a + seq_start_b + off_n + out_seq_b_start = seq_len_a + n_prefix_from_B + + out_ptrs = Out + out_seq_start.to(tl.int64) * stride_od + offs_d + if off_n < out_seq_b_start and off_n >= n_prefix_from_B: + off_a = off_n - n_prefix_from_B + if IS_DENSE_A: + in_ptrs = ( + ValuesA + + off_a.to(tl.int64) * stride_ad + + off_z.to(tl.int64) * stride_dense_batch + + offs_d + ) + else: + in_ptrs = ValuesA + (off_a + seq_start_a).to(tl.int64) * stride_ad + offs_d + else: + off_b = off_n - out_seq_b_start + n_prefix_from_B + if off_n < n_prefix_from_B: + off_b += out_seq_b_start - n_prefix_from_B + if IS_DENSE_B: + in_ptrs = ( + ValuesB + + off_b.to(tl.int64) * stride_bd + + off_z.to(tl.int64) * stride_dense_batch + + offs_d + ) + else: + in_ptrs = ValuesB + (off_b + seq_start_b).to(tl.int64) * stride_bd + offs_d + v = tl.load(in_ptrs, mask=offs_d < D) + tl.store(out_ptrs, v, mask=offs_d < D) + + +@triton.jit +def concat_2D_jagged( + OffsetsA, + ValuesA, + OffsetsB, + ValuesB, + DenseSize, + Out, + D, + stride_ad, + stride_bd, + stride_dense_batch, + stride_od, + IS_DENSE_A: tl.constexpr, + IS_DENSE_B: tl.constexpr, + BLOCK_D: tl.constexpr, + IS_REPLACE: tl.constexpr, +): + concat_2D_jagged_w_prefix( + OffsetsA, + ValuesA, + OffsetsB, + ValuesB, + DenseSize, + Out, + D, + stride_ad, + stride_bd, + stride_dense_batch, + stride_od, + 0, + IS_DENSE_A, + IS_DENSE_B, + BLOCK_D, + IS_REPLACE, + ) + + +@triton.jit +def concat_2D_jagged_jagged_w_prefix( + OffsetsA, + ValuesA, + OffsetsB, + ValuesB, + Out, + D, + stride_ad, + stride_bd, + stride_od, + n_prefix_from_B, + BLOCK_D: tl.constexpr, +): + concat_2D_jagged_w_prefix( + OffsetsA, + ValuesA, + OffsetsB, + ValuesB, + 0, + Out, + D, + stride_ad, + stride_bd, + 0, + stride_od, + n_prefix_from_B, + IS_DENSE_A=False, + IS_DENSE_B=False, + BLOCK_D=BLOCK_D, + IS_REPLACE=False, + ) + + +@triton.jit +def split_2D_jagged_w_prefix( + JaggedIn, + DenseSize, + OffsetsA, + OffsetsB, + OutA, + OutB, + D, + stride_id, + stride_ad, + stride_bd, + n_prefix_to_B, + IS_DENSE_A: tl.constexpr, + IS_DENSE_B: tl.constexpr, + BLOCK_D: tl.constexpr, + IS_REPLACE: tl.constexpr, +): + off_z = tl.program_id(1) + off_n = tl.program_id(0) + if IS_DENSE_A: + seq_start_b = tl.load(OffsetsB + off_z) + seq_end_b = tl.load(OffsetsB + off_z + 1) + seq_start_a = off_z * DenseSize + seq_len_a = DenseSize + seq_len_b = seq_end_b - seq_start_b + elif IS_DENSE_B: + seq_start_a = tl.load(OffsetsA + off_z) + seq_end_a = tl.load(OffsetsA + off_z + 1) + seq_len_a = seq_end_a - seq_start_a + seq_start_b = off_z * DenseSize + seq_len_b = DenseSize + else: + seq_start_a = tl.load(OffsetsA + off_z) + seq_end_a = tl.load(OffsetsA + off_z + 1) + seq_len_a = seq_end_a - seq_start_a + seq_start_b = tl.load(OffsetsB + off_z) + seq_end_b = tl.load(OffsetsB + off_z + 1) + seq_len_b = seq_end_b - seq_start_b + if IS_REPLACE: + seq_len = seq_len_a + else: + seq_len = seq_len_a + seq_len_b + if off_n >= seq_len: + return + + if IS_REPLACE: + seq_start = seq_start_a + out_seq_b_start = seq_len_a - seq_len_b + else: + seq_start = seq_start_a + seq_start_b + out_seq_b_start = seq_len_a + n_prefix_to_B + + offs_d = tl.arange(0, BLOCK_D) + in_ptrs = JaggedIn + (seq_start + off_n).to(tl.int64) * stride_id + offs_d + if off_n < out_seq_b_start and off_n >= n_prefix_to_B: + off_a = off_n - n_prefix_to_B + out_ptrs = OutA + (off_a + seq_start_a).to(tl.int64) * stride_ad + offs_d + else: + off_b = off_n - out_seq_b_start + n_prefix_to_B + if off_n < n_prefix_to_B: + off_b += out_seq_b_start - n_prefix_to_B + out_ptrs = OutB + (off_b + seq_start_b).to(tl.int64) * stride_bd + offs_d + v = tl.load(in_ptrs, mask=offs_d < D) + tl.store(out_ptrs, v, mask=offs_d < D) + + +@triton.jit +def split_2D_jagged( + JaggedIn, + DenseSize, + OffsetsA, + OffsetsB, + OutA, + OutB, + D, + stride_id, + stride_ad, + stride_bd, + IS_DENSE_A: tl.constexpr, + IS_DENSE_B: tl.constexpr, + BLOCK_D: tl.constexpr, + IS_REPLACE: tl.constexpr, +): + split_2D_jagged_w_prefix( + JaggedIn, + DenseSize, + OffsetsA, + OffsetsB, + OutA, + OutB, + D, + stride_id, + stride_ad, + stride_bd, + 0, + IS_DENSE_A, + IS_DENSE_B, + BLOCK_D, + IS_REPLACE, + ) + + +@triton.jit +def split_2D_jagged_jagged_w_prefix( + JaggedIn, + OffsetsA, + OffsetsB, + OutA, + OutB, + D, + stride_id, + stride_ad, + stride_bd, + n_prefix_to_B, + BLOCK_D: tl.constexpr, +): + split_2D_jagged_w_prefix( + JaggedIn, + 0, + OffsetsA, + OffsetsB, + OutA, + OutB, + D, + stride_id, + stride_ad, + stride_bd, + n_prefix_to_B, + IS_DENSE_A=False, + IS_DENSE_B=False, + BLOCK_D=BLOCK_D, + IS_REPLACE=False, + ) + + +def _triton_split_2D_jagged_internal( + jagged_in: torch.Tensor, + max_seq_len: int, + B: int, + offsets_a: Optional[torch.Tensor], + offsets_b: Optional[torch.Tensor], + out_a: torch.Tensor, + out_b: torch.Tensor, + D: int, + dense_size: int, + n_prefix: int, + is_dense_a: bool, + is_dense_b: bool, + is_replace: bool, + BLOCK_D: int, +) -> None: + use_multirow = _should_use_multirow() + if n_prefix != 0: + if use_multirow: + + def grid(meta): + return (triton.cdiv(max_seq_len, meta["BLOCK_N"]), B) + + split_2D_jagged_jagged_w_prefix_multirow[grid]( + JaggedIn=jagged_in, + OffsetsA=offsets_a, + OffsetsB=offsets_b, + OutA=out_a, + OutB=out_b, + D=D, + stride_id=jagged_in.stride(0), + stride_ad=out_a.stride(0), + stride_bd=out_b.stride(0), + n_prefix_to_B=n_prefix, + BLOCK_D=BLOCK_D, + ) + else: + split_2D_jagged_jagged_w_prefix[(max_seq_len, B)]( + JaggedIn=jagged_in, + OffsetsA=offsets_a, + OffsetsB=offsets_b, + OutA=out_a, + OutB=out_b, + D=D, + stride_id=jagged_in.stride(0), + stride_ad=out_a.stride(0), + stride_bd=out_b.stride(0), + n_prefix_to_B=n_prefix, + BLOCK_D=BLOCK_D, # pyre-ignore[6] + ) + else: + if use_multirow: + + def grid(meta): + return (triton.cdiv(max_seq_len, meta["BLOCK_N"]), B) + + split_2D_jagged_multirow[grid]( + JaggedIn=jagged_in, + DenseSize=dense_size, + OffsetsA=offsets_a, + OffsetsB=offsets_b, + OutA=out_a, + OutB=out_b, + D=D, + stride_id=jagged_in.stride(0), + stride_ad=out_a.stride(0), + stride_bd=out_b.stride(0), + IS_DENSE_A=is_dense_a, # pyre-ignore[6] + IS_DENSE_B=is_dense_b, # pyre-ignore[6] + BLOCK_D=BLOCK_D, # pyre-ignore[6] + IS_REPLACE=is_replace, # pyre-ignore[6] + ) + else: + split_2D_jagged[(max_seq_len, B)]( + JaggedIn=jagged_in, + DenseSize=dense_size, + OffsetsA=offsets_a, + OffsetsB=offsets_b, + OutA=out_a, + OutB=out_b, + D=D, + stride_id=jagged_in.stride(0), + stride_ad=out_a.stride(0), + stride_bd=out_b.stride(0), + IS_DENSE_A=is_dense_a, # pyre-ignore[6] + IS_DENSE_B=is_dense_b, # pyre-ignore[6] + BLOCK_D=BLOCK_D, # pyre-ignore[6] + IS_REPLACE=is_replace, # pyre-ignore[6] + ) + + +class _Concat2DJaggedFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore[14] + def forward( + ctx, + max_seq_len: int, + values_a: torch.Tensor, + values_b: torch.Tensor, + offsets_a: Optional[torch.Tensor] = None, + offsets_b: Optional[torch.Tensor] = None, + is_replace: bool = False, + n_prefix_from_right: int = 0, + ): + values_a = switch_to_contiguous_if_needed(values_a) + values_b = switch_to_contiguous_if_needed(values_b) + is_dense_a = offsets_a is None + is_dense_b = offsets_b is None + dense_size: int = 0 + if is_dense_a: + assert offsets_b is not None + B, dense_size, D = values_a.shape + seq_len_a = dense_size * B + seq_len_b, _ = values_b.shape + device = values_b.device + dtype = values_b.dtype + stride_dense_batch = values_a.stride(0) + elif is_dense_b: + assert offsets_a is not None + B, dense_size, D = values_b.shape + seq_len_a, _ = values_a.shape + seq_len_b = dense_size * B + device = values_a.device + dtype = values_a.dtype + stride_dense_batch = values_b.stride(0) + else: + assert offsets_a is not None and offsets_b is not None + B = offsets_a.shape[0] - 1 + seq_len_a, D = values_a.shape + seq_len_b, _ = values_b.shape + device = values_a.device + dtype = values_a.dtype + stride_dense_batch = 0 + + BLOCK_D = triton.next_power_of_2(D) + if is_replace: + values_out = torch.empty_like(values_a) + else: + values_out = torch.empty( + (seq_len_a + seq_len_b, D), device=device, dtype=dtype + ) + _triton_concat_2D_jagged_internal( + values_a=values_a, + values_b=values_b, + values_out=values_out, + max_seq_len=max_seq_len, + B=B, + offsets_a=offsets_a, + offsets_b=offsets_b, + D=D, + dense_size=dense_size, + stride_dense_batch=stride_dense_batch, + n_prefix=n_prefix_from_right, + is_dense_a=is_dense_a, + is_dense_b=is_dense_b, + is_replace=is_replace, + BLOCK_D=BLOCK_D, + ) + ctx.save_for_backward(offsets_a, offsets_b) + ctx.max_seq_len = max_seq_len + ctx.seq_len_a = seq_len_a + ctx.seq_len_b = seq_len_b + ctx.is_dense_a = is_dense_a + ctx.is_dense_b = is_dense_b + ctx.dense_size = dense_size + ctx.is_replace = is_replace + ctx.n_prefix_from_right = n_prefix_from_right + return values_out + + @staticmethod + # pyre-ignore[14] + def backward( + ctx, d_out: torch.Tensor + ) -> Tuple[None, torch.Tensor, torch.Tensor, None, None, None, None]: + offsets_a, offsets_b = ctx.saved_tensors + is_dense_a, is_dense_b, is_replace = ( + ctx.is_dense_a, + ctx.is_dense_b, + ctx.is_replace, + ) + dense_size = ctx.dense_size + if is_dense_a: + B = offsets_b.shape[0] - 1 + else: + B = offsets_a.shape[0] - 1 + _, D = d_out.shape + BLOCK_D = triton.next_power_of_2(D) + values_a = torch.zeros( + (ctx.seq_len_a, D), device=d_out.device, dtype=d_out.dtype + ) + values_b = torch.empty( + (ctx.seq_len_b, D), device=d_out.device, dtype=d_out.dtype + ) + _triton_split_2D_jagged_internal( + jagged_in=d_out, + max_seq_len=ctx.max_seq_len, + B=B, + offsets_a=offsets_a, + offsets_b=offsets_b, + out_a=values_a, + out_b=values_b, + D=D, + dense_size=dense_size, + n_prefix=ctx.n_prefix_from_right, + is_dense_a=is_dense_a, + is_dense_b=is_dense_b, + is_replace=is_replace, + BLOCK_D=BLOCK_D, + ) + + if is_dense_a: + values_a = values_a.reshape((B, dense_size, D)) + elif is_dense_b: + values_b = values_b.reshape((B, dense_size, D)) + return None, values_a, values_b, None, None, None, None + + +class _HelionConcat2DJaggedFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore[14] + def forward( + ctx, + max_seq_len: int, + values_a: torch.Tensor, + values_b: torch.Tensor, + offsets_a: Optional[torch.Tensor] = None, + offsets_b: Optional[torch.Tensor] = None, + ): + values_a = switch_to_contiguous_if_needed(values_a) + values_b = switch_to_contiguous_if_needed(values_b) + + assert offsets_a is not None and offsets_b is not None + B = offsets_a.shape[0] - 1 + seq_len_a, D = values_a.shape + seq_len_b, _ = values_b.shape + device = values_a.device + dtype = values_a.dtype + + BLOCK_D = triton.next_power_of_2(D) + values_out = torch.empty((seq_len_a + seq_len_b, D), device=device, dtype=dtype) + _triton_concat_2D_jagged_internal( + values_a=values_a, + values_b=values_b, + values_out=values_out, + max_seq_len=max_seq_len, + B=B, + offsets_a=offsets_a, + offsets_b=offsets_b, + D=D, + dense_size=0, + stride_dense_batch=0, + n_prefix=0, + is_dense_a=False, + is_dense_b=False, + is_replace=False, + BLOCK_D=BLOCK_D, + ) + ctx.save_for_backward(offsets_a, offsets_b) + ctx.max_seq_len = max_seq_len + ctx.seq_len_a = seq_len_a + ctx.seq_len_b = seq_len_b + return values_out + + @staticmethod + # pyre-ignore[14] + def backward( + ctx, d_out: torch.Tensor + ) -> Tuple[None, torch.Tensor, torch.Tensor, None, None, None, None]: + offsets_a, offsets_b = ctx.saved_tensors + d_out = switch_to_contiguous_if_needed(d_out) + values_a, values_b = _helion_split_2D_jagged_impl( + values=d_out, + max_seq_len=ctx.max_seq_len, + offsets_a=offsets_a, + offsets_b=offsets_b, + dense_size=0, + total_len_a=ctx.seq_len_a, + total_len_b=ctx.seq_len_b, + ) + + return None, values_a, values_b, None, None, None, None + + +class _Split2DJaggedFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore[14] + def forward( + ctx, + values: torch.Tensor, + max_seq_len: int, + offsets_a: Optional[torch.Tensor] = None, + offsets_b: Optional[torch.Tensor] = None, + dense_size: int = 0, + n_prefix_to_right: int = 0, + seq_len_a: Optional[int] = None, + seq_len_b: Optional[int] = None, + total_seq_len: Optional[int] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + values = switch_to_contiguous_if_needed(values) + is_dense_a: bool = offsets_a is None + is_dense_b: bool = offsets_b is None + if is_dense_a: + L, _ = values.shape + assert offsets_b is not None + B = offsets_b.shape[0] - 1 + seq_len_a = dense_size * B + seq_len_b = L - seq_len_a + offsets_a = offsets_b.new_empty(0) + elif is_dense_b: + L, _ = values.shape + assert offsets_a is not None + B = offsets_a.shape[0] - 1 + seq_len_b = dense_size * B + seq_len_a = L - seq_len_b + offsets_b = offsets_a.new_empty(0) + else: + assert offsets_a is not None and offsets_b is not None + B = offsets_a.shape[0] - 1 + + # Select the last offset item using torch.index_select instead of + # "int(offsets_a[-1].item())" so that it won't cause "Cannot cast + # FakeTensor to python number" error for AOTI. + if torch.compiler.is_compiling(): + offsets_b_last_idx = torch.tensor(offsets_b.size(0) - 1).to( + offsets_b.device, non_blocking=True + ) + if seq_len_b is None: + seq_len_b = offsets_b.index_select(dim=0, index=offsets_b_last_idx) + if seq_len_a is None and total_seq_len is None: + offsets_a_last_idx = torch.tensor(offsets_a.size(0) - 1).to( + offsets_a.device, non_blocking=True + ) + seq_len_a = offsets_a.index_select(dim=0, index=offsets_a_last_idx) + else: + if seq_len_b is None: + seq_len_b = int(offsets_b[-1].item()) + if seq_len_a is None and total_seq_len is None: + seq_len_a = int(offsets_a[-1].item()) + _, D = values.shape + BLOCK_D = triton.next_power_of_2(D) + # pyre-ignore[6] Incompatible parameter type + values_b = torch.empty((seq_len_b, D), device=values.device, dtype=values.dtype) + if seq_len_a is None: + # Derive seq_len_a from total_seq_len and values_b.size(0). + # values_b.size(0) is a SymInt (from the torch.empty above), + # so this is SymInt arithmetic — no new unbacked SymInt. + assert total_seq_len is not None + seq_len_a = total_seq_len - values_b.size(0) + # pyre-ignore[6] Incompatible parameter type + values_a = torch.empty((seq_len_a, D), device=values.device, dtype=values.dtype) + _triton_split_2D_jagged_internal( + jagged_in=values, + max_seq_len=max_seq_len, + B=B, + offsets_a=offsets_a, + offsets_b=offsets_b, + out_a=values_a, + out_b=values_b, + D=D, + dense_size=dense_size, + n_prefix=n_prefix_to_right, + is_dense_a=is_dense_a, + is_dense_b=is_dense_b, + is_replace=False, + BLOCK_D=BLOCK_D, + ) + if is_dense_a: + values_a = values_a.reshape(B, dense_size, D) + if is_dense_b: + values_b = values_b.reshape(B, dense_size, D) + ctx.save_for_backward(offsets_a, offsets_b) + ctx.max_seq_len = max_seq_len + ctx.seq_len_a = seq_len_a + ctx.seq_len_b = seq_len_b + ctx.is_dense_a = is_dense_a + ctx.is_dense_b = is_dense_b + ctx.dense_size = dense_size + ctx.B = B + ctx.D = D + ctx.n_prefix_to_right = n_prefix_to_right + return values_a, values_b + + @staticmethod + def backward( + ctx, *d_values + ) -> Tuple[torch.Tensor, None, None, None, None, None, None, None, None]: + offsets_a, offsets_b = ctx.saved_tensors + is_dense_a, is_dense_b = ctx.is_dense_a, ctx.is_dense_b + values_a, values_b = d_values + if is_dense_a: + stride_dense_batch = values_a.stride(0) + elif is_dense_b: + stride_dense_batch = values_b.stride(0) + else: + stride_dense_batch = 0 + + BLOCK_D = triton.next_power_of_2(ctx.D) + dvalues = torch.empty( + (ctx.seq_len_a + ctx.seq_len_b, ctx.D), + device=values_a.device, + dtype=values_b.dtype, + ) + _triton_concat_2D_jagged_internal( + values_a=values_a, + values_b=values_b, + values_out=dvalues, + max_seq_len=ctx.max_seq_len, + B=ctx.B, + offsets_a=offsets_a, + offsets_b=offsets_b, + D=ctx.D, + dense_size=ctx.dense_size, + stride_dense_batch=stride_dense_batch, + n_prefix=ctx.n_prefix_to_right, + is_dense_a=is_dense_a, + is_dense_b=is_dense_b, + is_replace=False, + BLOCK_D=BLOCK_D, + ) + + return dvalues, None, None, None, None, None, None, None, None + + +@torch.jit.unused +@torch.fx.wrap +def triton_jagged_dense_bmm_add( + max_seq_len: int, + seq_offsets: torch.Tensor, + jagged: torch.Tensor, + dense: torch.Tensor, + bias: torch.Tensor, + elementwise: bool = False, +) -> torch.Tensor: + """ + Computing bmm Out = Jagged x Dense + Bias + M is the jagged dimension + Jagged has shape (sum_B(M_i), K), Dense has shape (B, K, N), Bias has shape (B, N) or (sum_B(M_i), N) depending on Elementwise, and Out has shape (sum_B(M_i), N) + """ + return _JaggedDenseBmmAddFunction.apply( + max_seq_len, seq_offsets, jagged, dense, bias, elementwise + ) + + +@torch.fx.wrap +def triton_concat_2D_jagged( + max_seq_len: int, + values_a: torch.Tensor, + values_b: torch.Tensor, + offsets_a: Optional[torch.Tensor] = None, + offsets_b: Optional[torch.Tensor] = None, + is_replace: bool = False, + n_prefix_from_right: int = 0, +) -> torch.Tensor: + return _Concat2DJaggedFunction.apply( + max_seq_len, + values_a, + values_b, + offsets_a, + offsets_b, + is_replace, + n_prefix_from_right, + ) + + +@torch.fx.wrap +def triton_concat_2D_jagged_jagged( + max_seq_len_left: int, + offsets_left: torch.Tensor, + values_left: torch.Tensor, + max_seq_len_right: int, + offsets_right: torch.Tensor, + values_right: torch.Tensor, + is_replace: bool, + n_prefix_from_right: int, +) -> torch.Tensor: + return triton_concat_2D_jagged( + max_seq_len=max_seq_len_left + max_seq_len_right, + values_a=values_left, + values_b=values_right, + offsets_a=offsets_left, + offsets_b=offsets_right, + is_replace=is_replace, + n_prefix_from_right=n_prefix_from_right, + ) + + +@torch.fx.wrap +def helion_concat_2D_jagged( + max_seq_len: int, + values_a: torch.Tensor, + values_b: torch.Tensor, + offsets_a: Optional[torch.Tensor] = None, + offsets_b: Optional[torch.Tensor] = None, +) -> torch.Tensor: + return _HelionConcat2DJaggedFunction.apply( + max_seq_len, + values_a, + values_b, + offsets_a, + offsets_b, + ) + + +@torch.fx.wrap +def triton_concat_2D_dense_jagged( + jagged_max_seq_len: int, + jagged_offsets: torch.Tensor, + jagged_values: torch.Tensor, + dense_values: torch.Tensor, +) -> torch.Tensor: + B, dense_size, D = dense_values.size() + max_seq_len = jagged_max_seq_len + dense_size + return triton_concat_2D_jagged( + max_seq_len=max_seq_len, + values_a=dense_values, + values_b=jagged_values, + offsets_a=None, + offsets_b=jagged_offsets, + ) + + +def triton_jagged_dense_bmm( + max_seq_len: int, + seq_offsets: torch.Tensor, + jagged: torch.Tensor, + dense: torch.Tensor, +) -> torch.Tensor: + return _JaggedDenseBmmFunction.apply(max_seq_len, seq_offsets, jagged, dense) + + +@torch.jit.unused +def triton_split_2D_jagged( + values: torch.Tensor, + max_seq_len: int, + offsets_a: Optional[torch.Tensor] = None, + offsets_b: Optional[torch.Tensor] = None, + dense_size: int = 0, + n_prefix_to_right: int = 0, + seq_len_a: Optional[int] = None, + seq_len_b: Optional[int] = None, + total_seq_len: Optional[int] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + return _Split2DJaggedFunction.apply( + values, + max_seq_len, + offsets_a, + offsets_b, + dense_size, + n_prefix_to_right, + seq_len_a, + seq_len_b, + total_seq_len, + ) + + +@torch.jit.unused +def helion_split_2D_jagged( + values: torch.Tensor, + max_seq_len: int, + offsets_a: Optional[torch.Tensor] = None, + offsets_b: Optional[torch.Tensor] = None, + dense_size: int = 0, +) -> Tuple[torch.Tensor, torch.Tensor]: + return _HelionSplit2DJaggedFunction.apply( + values, + max_seq_len, + offsets_a, + offsets_b, + dense_size, + ) + + +@triton.jit +def concat_2D_jagged_w_prefix_multirow( + OffsetsA, + ValuesA, + OffsetsB, + ValuesB, + DenseSize, + Out, + D, + stride_ad, + stride_bd, + stride_dense_batch, + stride_od, + n_prefix_from_B, + IS_DENSE_A: tl.constexpr, + IS_DENSE_B: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_N: tl.constexpr, + IS_REPLACE: tl.constexpr, +): + off_z = tl.program_id(1) + off_block_n = tl.program_id(0) + + if IS_DENSE_A: + seq_start_a = off_z * DenseSize + seq_len_a = DenseSize + seq_start_b = tl.load(OffsetsB + off_z) + seq_end_b = tl.load(OffsetsB + off_z + 1) + seq_len_b = seq_end_b - seq_start_b + elif IS_DENSE_B: + seq_start_a = tl.load(OffsetsA + off_z) + seq_end_a = tl.load(OffsetsA + off_z + 1) + seq_len_a = seq_end_a - seq_start_a + seq_start_b = off_z * DenseSize + seq_len_b = DenseSize + else: + seq_start_a = tl.load(OffsetsA + off_z) + seq_end_a = tl.load(OffsetsA + off_z + 1) + seq_len_a = seq_end_a - seq_start_a + seq_start_b = tl.load(OffsetsB + off_z) + seq_end_b = tl.load(OffsetsB + off_z + 1) + seq_len_b = seq_end_b - seq_start_b + + if IS_REPLACE: + seq_len = seq_len_a + out_seq_start = seq_start_a + out_seq_b_start = seq_len_a - seq_len_b + else: + seq_len = seq_len_a + seq_len_b + out_seq_start = seq_start_a + seq_start_b + out_seq_b_start = seq_len_a + n_prefix_from_B + + start_n = off_block_n * BLOCK_N + offs_n = start_n + tl.arange(0, BLOCK_N) + offs_d = tl.arange(0, BLOCK_D) + if start_n >= seq_len: + return + valid_mask = offs_n < seq_len + + out_ptrs = ( + Out + + (out_seq_start + offs_n[:, None]).to(tl.int64) * stride_od + + offs_d[None, :] + ) + + to_a_mask = (offs_n < out_seq_b_start) & (offs_n >= n_prefix_from_B) & valid_mask + to_b_mask = ~to_a_mask & valid_mask + + off_a = offs_n - n_prefix_from_B + if IS_DENSE_A: + in_a_ptrs = ( + ValuesA + + off_a[:, None].to(tl.int64) * stride_ad + + off_z.to(tl.int64) * stride_dense_batch + + offs_d[None, :] + ) + else: + in_a_ptrs = ( + ValuesA + + (off_a[:, None] + seq_start_a).to(tl.int64) * stride_ad + + offs_d[None, :] + ) + + v_a = tl.load(in_a_ptrs, mask=to_a_mask[:, None] & (offs_d[None, :] < D), other=0.0) + tl.store(out_ptrs, v_a, mask=to_a_mask[:, None] & (offs_d[None, :] < D)) + + prefix_mask = offs_n < n_prefix_from_B + + off_b = tl.where(prefix_mask, offs_n, offs_n - out_seq_b_start + n_prefix_from_B) + if IS_DENSE_B: + in_b_ptrs = ( + ValuesB + + off_b[:, None].to(tl.int64) * stride_bd + + off_z.to(tl.int64) * stride_dense_batch + + offs_d[None, :] + ) + else: + in_b_ptrs = ( + ValuesB + + (off_b[:, None] + seq_start_b).to(tl.int64) * stride_bd + + offs_d[None, :] + ) + + v_b = tl.load(in_b_ptrs, mask=to_b_mask[:, None] & (offs_d[None, :] < D), other=0.0) + tl.store(out_ptrs, v_b, mask=to_b_mask[:, None] & (offs_d[None, :] < D)) + + +@triton_autotune( + configs=_get_split_concat_2d_jagged_multirow_configs_wrapper(), + key=["BLOCK_D"], +) +@triton.jit +def concat_2D_jagged_multirow( + OffsetsA, + ValuesA, + OffsetsB, + ValuesB, + DenseSize, + Out, + D, + stride_ad, + stride_bd, + stride_dense_batch, + stride_od, + IS_DENSE_A: tl.constexpr, + IS_DENSE_B: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_N: tl.constexpr, + IS_REPLACE: tl.constexpr, +): + concat_2D_jagged_w_prefix_multirow( + OffsetsA, + ValuesA, + OffsetsB, + ValuesB, + DenseSize, + Out, + D, + stride_ad, + stride_bd, + stride_dense_batch, + stride_od, + 0, + IS_DENSE_A, + IS_DENSE_B, + BLOCK_D, + BLOCK_N, + IS_REPLACE, + ) + + +@triton_autotune( + configs=_get_split_concat_2d_jagged_multirow_configs(), + key=["BLOCK_D"], +) +@triton.jit +def concat_2D_jagged_jagged_w_prefix_multirow( + OffsetsA, + ValuesA, + OffsetsB, + ValuesB, + Out, + D, + stride_ad, + stride_bd, + stride_od, + n_prefix_from_B, + BLOCK_D: tl.constexpr, + BLOCK_N: tl.constexpr, +): + concat_2D_jagged_w_prefix_multirow( + OffsetsA, + ValuesA, + OffsetsB, + ValuesB, + 0, + Out, + D, + stride_ad, + stride_bd, + 0, + stride_od, + n_prefix_from_B, + IS_DENSE_A=False, + IS_DENSE_B=False, + BLOCK_D=BLOCK_D, + BLOCK_N=BLOCK_N, + IS_REPLACE=False, + ) + + +@triton.jit +def split_2D_jagged_w_prefix_multirow( + JaggedIn, + DenseSize, + OffsetsA, + OffsetsB, + OutA, + OutB, + D, + stride_id, + stride_ad, + stride_bd, + n_prefix_to_B, + IS_DENSE_A: tl.constexpr, + IS_DENSE_B: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_N: tl.constexpr, + IS_REPLACE: tl.constexpr, +): + off_z = tl.program_id(1) + off_block_n = tl.program_id(0) + + if IS_DENSE_A: + seq_start_b = tl.load(OffsetsB + off_z) + seq_end_b = tl.load(OffsetsB + off_z + 1) + seq_start_a = off_z * DenseSize + seq_len_a = DenseSize + seq_len_b = seq_end_b - seq_start_b + elif IS_DENSE_B: + seq_start_a = tl.load(OffsetsA + off_z) + seq_end_a = tl.load(OffsetsA + off_z + 1) + seq_len_a = seq_end_a - seq_start_a + seq_start_b = off_z * DenseSize + seq_len_b = DenseSize + else: + seq_start_a = tl.load(OffsetsA + off_z) + seq_end_a = tl.load(OffsetsA + off_z + 1) + seq_len_a = seq_end_a - seq_start_a + seq_start_b = tl.load(OffsetsB + off_z) + seq_end_b = tl.load(OffsetsB + off_z + 1) + seq_len_b = seq_end_b - seq_start_b + + if IS_REPLACE: + seq_len = seq_len_a + else: + seq_len = seq_len_a + seq_len_b + + if IS_REPLACE: + seq_start = seq_start_a + out_seq_b_start = seq_len_a - seq_len_b + else: + seq_start = seq_start_a + seq_start_b + out_seq_b_start = seq_len_a + n_prefix_to_B + + start_n = off_block_n * BLOCK_N + offs_n = start_n + tl.arange(0, BLOCK_N) + offs_d = tl.arange(0, BLOCK_D) + if start_n >= seq_len: + return + valid_mask = offs_n < seq_len + + in_ptrs = ( + JaggedIn + + (seq_start + offs_n[:, None]).to(tl.int64) * stride_id + + offs_d[None, :] + ) + + v = tl.load(in_ptrs, mask=valid_mask[:, None] & (offs_d[None, :] < D), other=0.0) + + to_a_mask = (offs_n < out_seq_b_start) & (offs_n >= n_prefix_to_B) & valid_mask + to_b_mask = ~to_a_mask & valid_mask + + off_a = offs_n - n_prefix_to_B + out_a_ptrs = ( + OutA + (off_a[:, None] + seq_start_a).to(tl.int64) * stride_ad + offs_d[None, :] + ) + tl.store(out_a_ptrs, v, mask=to_a_mask[:, None] & (offs_d[None, :] < D)) + + prefix_mask = offs_n < n_prefix_to_B + + off_b = tl.where(prefix_mask, offs_n, offs_n - out_seq_b_start + n_prefix_to_B) + out_b_ptrs = ( + OutB + (off_b[:, None] + seq_start_b).to(tl.int64) * stride_bd + offs_d[None, :] + ) + tl.store(out_b_ptrs, v, mask=to_b_mask[:, None] & (offs_d[None, :] < D)) + + +@triton_autotune( + configs=_get_split_concat_2d_jagged_multirow_configs_wrapper(), + key=["BLOCK_D"], +) +@triton.jit +def split_2D_jagged_multirow( + JaggedIn, + DenseSize, + OffsetsA, + OffsetsB, + OutA, + OutB, + D, + stride_id, + stride_ad, + stride_bd, + IS_DENSE_A: tl.constexpr, + IS_DENSE_B: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_N: tl.constexpr, + IS_REPLACE: tl.constexpr, +): + split_2D_jagged_w_prefix_multirow( + JaggedIn, + DenseSize, + OffsetsA, + OffsetsB, + OutA, + OutB, + D, + stride_id, + stride_ad, + stride_bd, + 0, + IS_DENSE_A, + IS_DENSE_B, + BLOCK_D, + BLOCK_N, + IS_REPLACE, + ) + + +@triton_autotune( + configs=_get_split_concat_2d_jagged_multirow_configs(), + key=["BLOCK_D"], +) +@triton.jit +def split_2D_jagged_jagged_w_prefix_multirow( + JaggedIn, + OffsetsA, + OffsetsB, + OutA, + OutB, + D, + stride_id, + stride_ad, + stride_bd, + n_prefix_to_B, + BLOCK_D: tl.constexpr, + BLOCK_N: tl.constexpr, +): + split_2D_jagged_w_prefix_multirow( + JaggedIn, + 0, + OffsetsA, + OffsetsB, + OutA, + OutB, + D, + stride_id, + stride_ad, + stride_bd, + n_prefix_to_B, + IS_DENSE_A=False, + IS_DENSE_B=False, + BLOCK_D=BLOCK_D, + BLOCK_N=BLOCK_N, + IS_REPLACE=False, + ) + + +def triton_jagged_dense_broadcast_add( + max_seq_len: int, + seq_offsets: torch.Tensor, + jagged: torch.Tensor, + dense: torch.Tensor, +) -> torch.Tensor: + return _JaggedDenseBroadcastAddFunction.apply( + max_seq_len, seq_offsets, jagged, dense + ) + + +@triton.jit +def _helion_split_2d_jagged_kernel( + offsets_a, + offsets_b, + values_flat, + out_a_flat, + out_b_flat, + max_seq_len, + D: tl.constexpr, + _BLOCK_SIZE_0: tl.constexpr, + _BLOCK_SIZE_1: tl.constexpr, +) -> None: + # Get program ID and decompose to batch and sequence block coordinates + program_id = tl.program_id(0) + flat_program_id = program_id + batch_id = triton_helpers.div_floor_integer( + flat_program_id, + triton_helpers.div_floor_integer( + -1 + _BLOCK_SIZE_0 + max_seq_len, _BLOCK_SIZE_0 + ), + ) + seq_block_id = triton_helpers.remainder_integer( # noqa: F841 + flat_program_id, + triton_helpers.div_floor_integer( + -1 + _BLOCK_SIZE_0 + max_seq_len, _BLOCK_SIZE_0 + ), + ) + # Load output boundaries for part A + out_a_start = tl.load(offsets_a + batch_id * 1, None, eviction_policy="evict_last") + batch_id_plus_1 = 1 + triton_helpers.div_floor_integer( + flat_program_id, + triton_helpers.div_floor_integer( + -1 + _BLOCK_SIZE_0 + max_seq_len, _BLOCK_SIZE_0 + ), + ) + out_a_end = tl.load( + offsets_a + batch_id_plus_1 * 1, None, eviction_policy="evict_last" + ) + len_a = out_a_end - out_a_start + # Load output boundaries for part B + out_b_start = tl.load(offsets_b + batch_id * 1, None) + out_b_end = tl.load( + offsets_b + batch_id_plus_1 * 1, None, eviction_policy="evict_last" + ) + len_b = out_b_end - out_b_start + # Compute input start and total length for this batch + input_start = out_a_start + out_b_start + total_len = len_a + len_b + # Calculate sequence offset for this block + seq_offset = _BLOCK_SIZE_0 * triton_helpers.remainder_integer( + flat_program_id, + triton_helpers.div_floor_integer( + -1 + _BLOCK_SIZE_0 + max_seq_len, _BLOCK_SIZE_0 + ), + ) + has_work = total_len > seq_offset + if has_work: + # Generate row indices for this sequence block + seq_range = tl.arange(0, _BLOCK_SIZE_0) + seq_offset_i32 = tl.cast(seq_offset, tl.int32) + row_indices = seq_range + seq_offset_i32 + + # Create masks for valid rows and parts A/B + total_len_i32 = tl.cast(total_len[None], tl.int32) + len_a_i32 = tl.cast(len_a[None], tl.int32) + valid_mask = row_indices < total_len_i32 + is_part_a = row_indices < len_a_i32 + is_part_b = (row_indices >= len_a_i32) & valid_mask + + # Extract scalar values once + input_start_i32 = tl.cast(input_start[None, None], tl.int32) + out_a_start_i32 = tl.cast(out_a_start[None, None], tl.int32) + out_b_start_i32 = tl.cast(out_b_start[None, None], tl.int32) + + # Process features in smaller tiles + for feature_offset in tl.range( + 0, + D, + _BLOCK_SIZE_1, + loop_unroll_factor=1, + num_stages=4, + disallow_acc_multi_buffer=True, + flatten=True, + ): + feature_indices = feature_offset + tl.arange(0, _BLOCK_SIZE_1).to(tl.int32) + + # Compute D constant and feature mask once per feature iteration + D_const = tl.full([], tl.cast(D, tl.int32), tl.int32) + D_i32 = tl.cast(D, tl.int32) + feature_mask = feature_indices < D_i32 + + # Compute indices for part A + row_subscript = row_indices[:, None] + input_row_a = input_start_i32 + row_subscript + input_idx_a = ( + tl.cast(input_row_a * D_const, tl.int32) + feature_indices[None, :] + ) + + out_a_row = out_a_start_i32 + row_subscript + out_a_idx = ( + tl.cast(out_a_row * D_const, tl.int32) + feature_indices[None, :] + ) + + mask_a = is_part_a[:, None] & valid_mask[:, None] & feature_mask[None, :] + + # Load and store part A data + slice_a = tl.load( + values_flat + input_idx_a * 1, + mask_a, + other=0, + eviction_policy="evict_first", + ) + tl.store(out_a_flat + out_a_idx * 1, slice_a, mask_a) + + # Compute indices for part B + input_idx_b = ( + tl.cast((input_start_i32 + row_subscript) * D_const, tl.int32) + + feature_indices[None, :] + ) + + row_minus_len_a = row_subscript - len_a_i32 + out_b_row = out_b_start_i32 + row_minus_len_a + out_b_idx = ( + tl.cast(out_b_row * D_const, tl.int32) + feature_indices[None, :] + ) + + mask_b = is_part_b[:, None] & feature_mask[None, :] + + # Load and store part B data + slice_b = tl.load( + values_flat + input_idx_b * 1, + mask_b, + other=0, + eviction_policy="evict_first", + ) + tl.store(out_b_flat + out_b_idx * 1, slice_b, mask_b) + + +class _HelionSplit2DJaggedFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore[14] + def forward( + ctx, + values: torch.Tensor, + max_seq_len: int, + offsets_a: torch.Tensor, + offsets_b: torch.Tensor, + dense_size: int = 0, # noqa: F841 + ) -> Tuple[torch.Tensor, torch.Tensor]: + values = switch_to_contiguous_if_needed(values) + B = offsets_a.shape[0] - 1 + D = values.size(1) + + # TODO: maybe check if torch.compiler.is_compiling() and use index_select instead + seq_len_a = int(offsets_a[-1].item()) + seq_len_b = int(offsets_b[-1].item()) + + values_a, values_b = _helion_split_2D_jagged_impl( + values=values, + max_seq_len=max_seq_len, + offsets_a=offsets_a, + offsets_b=offsets_b, + dense_size=dense_size, + total_len_a=seq_len_a, + total_len_b=seq_len_b, + ) + + ctx.save_for_backward(offsets_a, offsets_b) + ctx.max_seq_len = max_seq_len + ctx.seq_len_a = seq_len_a + ctx.seq_len_b = seq_len_b + ctx.dense_size = dense_size + ctx.B = B + ctx.D = D + return values_a, values_b + + @staticmethod + def backward(ctx, *d_values) -> Tuple[torch.Tensor, None, None, None, None]: + offsets_a, offsets_b = ctx.saved_tensors + values_a, values_b = d_values + BLOCK_D = triton.next_power_of_2(ctx.D) + + dvalues = torch.empty( + (ctx.seq_len_a + ctx.seq_len_b, ctx.D), + device=values_a.device, + dtype=values_a.dtype, + ) + _triton_concat_2D_jagged_internal( + values_a=values_a, + values_b=values_b, + values_out=dvalues, + max_seq_len=ctx.max_seq_len, + B=ctx.B, + offsets_a=offsets_a, + offsets_b=offsets_b, + D=ctx.D, + dense_size=0, + stride_dense_batch=0, + n_prefix=0, + is_dense_a=False, + is_dense_b=False, + is_replace=False, + BLOCK_D=BLOCK_D, + ) + return dvalues, None, None, None, None + + +def _helion_split_2D_jagged_impl( + values: torch.Tensor, + max_seq_len: int, + offsets_a: torch.Tensor, + offsets_b: torch.Tensor, + dense_size: int = 0, # noqa: F841 + total_len_a: Optional[int] = None, + total_len_b: Optional[int] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + D = values.size(1) + + # Select dtype-specific optimal parameters + if values.dtype == torch.float32: + # FP32-optimized parameters + block_size_0 = 64 + block_size_1 = 64 + num_warps = 4 + num_stages = 4 + else: + # BF16/FP16-optimized parameters + block_size_0 = 128 + block_size_1 = triton.next_power_of_2(D) + num_warps = 32 + num_stages = 7 + + return _helion_split_2d_jagged( + values, + max_seq_len, + offsets_a, + offsets_b, + dense_size, + block_size_0=block_size_0, + block_size_1=block_size_1, + num_warps=num_warps, + num_stages=num_stages, + total_len_a=total_len_a, + total_len_b=total_len_b, + ) + + +def _helion_split_2d_jagged( + values: torch.Tensor, + max_seq_len: int, + offsets_a: torch.Tensor, + offsets_b: torch.Tensor, + dense_size: int, # noqa: F841 + block_size_0: int = 64, + block_size_1: int = 64, + num_warps: int = 4, + num_stages: int = 4, + total_len_a: Optional[int] = None, + total_len_b: Optional[int] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + values = values.contiguous() + num_batches = offsets_a.size(0) - 1 + D = values.size(1) + num_seq_blocks = (max_seq_len + block_size_0 - 1) // block_size_0 + if total_len_a is None: + total_len_a = int(offsets_a[-1].item()) + if total_len_b is None: + total_len_b = int(offsets_b[-1].item()) + out_a = torch.empty([total_len_a, D], dtype=values.dtype, device=values.device) + out_b = torch.empty([total_len_b, D], dtype=values.dtype, device=values.device) + values_flat = values.view(-1) + out_a_flat = out_a.view(-1) + out_b_flat = out_b.view(-1) + total_programs = num_batches * num_seq_blocks + + # pyre-ignore[28] + _helion_split_2d_jagged_kernel[(total_programs,)]( + offsets_a, + offsets_b, + values_flat, + out_a_flat, + out_b_flat, + max_seq_len, + D, + block_size_0, + block_size_1, + num_warps=num_warps, + num_stages=num_stages, + ) + return (out_a, out_b) diff --git a/recommendation_v4/generative_recommenders/ops/triton/triton_jagged_tensors.py b/recommendation_v4/generative_recommenders/ops/triton/triton_jagged_tensors.py new file mode 100644 index 000000000..3488e308a --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton/triton_jagged_tensors.py @@ -0,0 +1,1067 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +#!/usr/bin/env python3 + + +from typing import Optional, Tuple + +import torch + +# @manual=//triton:triton +import triton + +# @manual=//triton:triton +import triton.language as tl +from generative_recommenders.common import ( + switch_to_contiguous_if_needed, + triton_autotune, +) +from generative_recommenders.ops.utils import is_sm100_plus + + +def _triton_concat_2D_jagged_internal( + values_a: torch.Tensor, + values_b: torch.Tensor, + values_out: torch.Tensor, + max_seq_len: int, + B: int, + offsets_a: Optional[torch.Tensor], + offsets_b: Optional[torch.Tensor], + max_len_a: Optional[int], + max_len_b: Optional[int], + D: int, + n_prefix_from_B: int, + is_dense_a: bool, + is_dense_b: bool, + BLOCK_D: int, +) -> None: + if is_sm100_plus(): + + def grid(meta): + return (triton.cdiv(max_seq_len, meta["BLOCK_N"]), B) + + concat_2D_jagged_multirow[grid]( + ValuesA=values_a, + ValuesB=values_b, + OffsetsA=offsets_a, + OffsetsB=offsets_b, + MaxLenA=max_len_a, + MaxLenB=max_len_b, + Out=values_out, + D=D, + stride_ad=values_a.stride(-2), + stride_bd=values_b.stride(-2), + stride_od=values_out.stride(-2), + n_prefix_from_B=n_prefix_from_B, + IS_DENSE_A=is_dense_a, # pyre-ignore[6] + IS_DENSE_B=is_dense_b, # pyre-ignore[6] + BLOCK_D=BLOCK_D, # pyre-ignore[6] + ) + else: + _concat_2D_jagged[(max_seq_len, B)]( + ValuesA=values_a, + ValuesB=values_b, + OffsetsA=offsets_a, + OffsetsB=offsets_b, + MaxLenA=max_len_a, + MaxLenB=max_len_b, + Out=values_out, + D=D, + stride_ad=values_a.stride(-2), + stride_bd=values_b.stride(-2), + stride_od=values_out.stride(-2), + n_prefix_from_B=n_prefix_from_B, + IS_DENSE_A=is_dense_a, # pyre-ignore[6] + IS_DENSE_B=is_dense_b, # pyre-ignore[6] + BLOCK_D=BLOCK_D, # pyre-ignore[6] + ) + + +def _triton_split_2D_jagged_internal( + jagged_in: torch.Tensor, + max_seq_len: int, + B: int, + offsets_a: Optional[torch.Tensor], + offsets_b: Optional[torch.Tensor], + max_len_a: Optional[int], + max_len_b: Optional[int], + out_a: torch.Tensor, + out_b: torch.Tensor, + D: int, + n_prefix_to_B: int, + is_dense_a: bool, + is_dense_b: bool, + BLOCK_D: int, +) -> None: + if is_sm100_plus(): + + def grid(meta): + return (triton.cdiv(max_seq_len, meta["BLOCK_N"]), B) + + split_2D_jagged_multirow[grid]( + JaggedIn=jagged_in, + OffsetsA=offsets_a, + OffsetsB=offsets_b, + MaxLenA=max_len_a, + MaxLenB=max_len_b, + OutA=out_a, + OutB=out_b, + D=D, + stride_id=jagged_in.stride(0), + stride_ad=out_a.stride(0), + stride_bd=out_b.stride(0), + n_prefix_to_B=n_prefix_to_B, + IS_DENSE_A=is_dense_a, # pyre-ignore[6] + IS_DENSE_B=is_dense_b, # pyre-ignore[6] + BLOCK_D=BLOCK_D, # pyre-ignore[6] + ) + else: + _split_2D_jagged[(max_seq_len, B)]( + JaggedIn=jagged_in, + OffsetsA=offsets_a, + OffsetsB=offsets_b, + MaxLenA=max_len_a, + MaxLenB=max_len_b, + OutA=out_a, + OutB=out_b, + D=D, + stride_id=jagged_in.stride(0), + stride_ad=out_a.stride(0), + stride_bd=out_b.stride(0), + n_prefix_to_B=n_prefix_to_B, + IS_DENSE_A=is_dense_a, # pyre-ignore[6] + IS_DENSE_B=is_dense_b, # pyre-ignore[6] + BLOCK_D=BLOCK_D, # pyre-ignore[6] + ) + + +def _get_concat_split_2d_jagged_multirow_configs(): + configs = [] + for BLOCK_N in [1, 2, 4, 8]: + for num_warps in [1, 2, 4]: + configs.append( + triton.Config( + {"BLOCK_N": BLOCK_N}, + num_warps=num_warps, + ) + ) + return configs + + +@triton.jit +def _concat_2D_jagged_multirow( + ValuesA, + ValuesB, + OffsetsA, + OffsetsB, + MaxLenA, + MaxLenB, + Out, + D, + stride_ad, + stride_bd, + stride_od, + n_prefix_from_B, + IS_DENSE_A: tl.constexpr, + IS_DENSE_B: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_N: tl.constexpr, +): + off_z = tl.program_id(1) + block_n = tl.program_id(0) + + if IS_DENSE_A: + seq_start_a = off_z * MaxLenA + seq_len_a = MaxLenA + else: + seq_start_a = tl.load(OffsetsA + off_z) + seq_end_a = tl.load(OffsetsA + off_z + 1) + seq_len_a = seq_end_a - seq_start_a + if IS_DENSE_B: + seq_start_b = off_z * MaxLenB + seq_len_b = MaxLenB + else: + seq_start_b = tl.load(OffsetsB + off_z) + seq_end_b = tl.load(OffsetsB + off_z + 1) + seq_len_b = seq_end_b - seq_start_b + seq_len = seq_len_a + seq_len_b + + start_n = block_n * BLOCK_N + offs_n = start_n + tl.arange(0, BLOCK_N) + offs_d = tl.arange(0, BLOCK_D) + + valid_mask = offs_n < seq_len + + out_seq_start = seq_start_a + seq_start_b + offs_n + out_ptrs = Out + out_seq_start[:, None].to(tl.int64) * stride_od + offs_d[None, :] + + from_prefix_b_mask = (offs_n < n_prefix_from_B) & valid_mask + from_a_mask = ( + (offs_n >= n_prefix_from_B) + & (offs_n < seq_len_a + n_prefix_from_B) + & valid_mask + ) + from_suffix_b_mask = (offs_n >= seq_len_a + n_prefix_from_B) & valid_mask + + in_b1_ptrs = ( + ValuesB + + (offs_n[:, None] + seq_start_b).to(tl.int64) * stride_bd + + offs_d[None, :] + ) + v_b1 = tl.load( + in_b1_ptrs, mask=from_prefix_b_mask[:, None] & (offs_d[None, :] < D), other=0.0 + ) + tl.store(out_ptrs, v_b1, mask=from_prefix_b_mask[:, None] & (offs_d[None, :] < D)) + + off_a = offs_n - n_prefix_from_B + in_a_ptrs = ( + ValuesA + + (off_a[:, None] + seq_start_a).to(tl.int64) * stride_ad + + offs_d[None, :] + ) + v_a = tl.load( + in_a_ptrs, mask=from_a_mask[:, None] & (offs_d[None, :] < D), other=0.0 + ) + tl.store(out_ptrs, v_a, mask=from_a_mask[:, None] & (offs_d[None, :] < D)) + + off_b = offs_n - seq_len_a + in_b2_ptrs = ( + ValuesB + + (off_b[:, None] + seq_start_b).to(tl.int64) * stride_bd + + offs_d[None, :] + ) + v_b2 = tl.load( + in_b2_ptrs, mask=from_suffix_b_mask[:, None] & (offs_d[None, :] < D), other=0.0 + ) + tl.store(out_ptrs, v_b2, mask=from_suffix_b_mask[:, None] & (offs_d[None, :] < D)) + + +@triton_autotune( + configs=_get_concat_split_2d_jagged_multirow_configs(), + key=["BLOCK_D"], +) +@triton.jit +def concat_2D_jagged_multirow( + ValuesA, + ValuesB, + OffsetsA, + OffsetsB, + MaxLenA, + MaxLenB, + Out, + D, + stride_ad, + stride_bd, + stride_od, + n_prefix_from_B, + IS_DENSE_A: tl.constexpr, + IS_DENSE_B: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_N: tl.constexpr, +): + _concat_2D_jagged_multirow( + ValuesA, + ValuesB, + OffsetsA, + OffsetsB, + MaxLenA, + MaxLenB, + Out, + D, + stride_ad, + stride_bd, + stride_od, + n_prefix_from_B, + IS_DENSE_A, + IS_DENSE_B, + BLOCK_D, + BLOCK_N, + ) + + +@triton.jit +def _split_2D_jagged_multirow( + JaggedIn, + OffsetsA, + OffsetsB, + MaxLenA, + MaxLenB, + OutA, + OutB, + D, + stride_id, + stride_ad, + stride_bd, + n_prefix_to_B, + IS_DENSE_A: tl.constexpr, + IS_DENSE_B: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_N: tl.constexpr, +): + off_z = tl.program_id(1) + block_n = tl.program_id(0) + + if IS_DENSE_A: + seq_start_a = off_z * MaxLenA + seq_len_a = MaxLenA + else: + seq_start_a = tl.load(OffsetsA + off_z) + seq_end_a = tl.load(OffsetsA + off_z + 1) + seq_len_a = seq_end_a - seq_start_a + if IS_DENSE_B: + seq_start_b = off_z * MaxLenB + seq_len_b = MaxLenB + else: + seq_start_b = tl.load(OffsetsB + off_z) + seq_end_b = tl.load(OffsetsB + off_z + 1) + seq_len_b = seq_end_b - seq_start_b + seq_len = seq_len_a + seq_len_b + seq_start = seq_start_a + seq_start_b + + start_n = block_n * BLOCK_N + offs_n = start_n + tl.arange(0, BLOCK_N) + offs_d = tl.arange(0, BLOCK_D) + + valid_mask = offs_n < seq_len + + in_ptrs = ( + JaggedIn + + (seq_start + offs_n[:, None]).to(tl.int64) * stride_id + + offs_d[None, :] + ) + + v = tl.load(in_ptrs, mask=valid_mask[:, None] & (offs_d[None, :] < D), other=0.0) + + to_prefix_b_mask = (offs_n < n_prefix_to_B) & valid_mask + to_a_mask = ( + (offs_n >= n_prefix_to_B) & (offs_n < seq_len_a + n_prefix_to_B) & valid_mask + ) + to_suffix_b_mask = (offs_n >= seq_len_a + n_prefix_to_B) & valid_mask + + out_b1_ptrs = ( + OutB + + (offs_n[:, None] + seq_start_b).to(tl.int64) * stride_bd + + offs_d[None, :] + ) + tl.store(out_b1_ptrs, v, mask=to_prefix_b_mask[:, None] & (offs_d[None, :] < D)) + + off_a = offs_n - n_prefix_to_B + out_a_ptrs = ( + OutA + (off_a[:, None] + seq_start_a).to(tl.int64) * stride_ad + offs_d[None, :] + ) + tl.store(out_a_ptrs, v, mask=to_a_mask[:, None] & (offs_d[None, :] < D)) + + off_b = offs_n - seq_len_a + out_b2_ptrs = ( + OutB + (off_b[:, None] + seq_start_b).to(tl.int64) * stride_bd + offs_d[None, :] + ) + tl.store(out_b2_ptrs, v, mask=to_suffix_b_mask[:, None] & (offs_d[None, :] < D)) + + +@triton_autotune( + configs=_get_concat_split_2d_jagged_multirow_configs(), + key=["BLOCK_D"], +) +@triton.jit +def split_2D_jagged_multirow( + JaggedIn, + OffsetsA, + OffsetsB, + MaxLenA, + MaxLenB, + OutA, + OutB, + D, + stride_id, + stride_ad, + stride_bd, + n_prefix_to_B, + IS_DENSE_A: tl.constexpr, + IS_DENSE_B: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_N: tl.constexpr, +): + _split_2D_jagged_multirow( + JaggedIn, + OffsetsA, + OffsetsB, + MaxLenA, + MaxLenB, + OutA, + OutB, + D, + stride_id, + stride_ad, + stride_bd, + n_prefix_to_B, + IS_DENSE_A, + IS_DENSE_B, + BLOCK_D, + BLOCK_N, + ) + + +@triton.jit +def _concat_2D_jagged( + ValuesA, + ValuesB, + OffsetsA, + OffsetsB, + MaxLenA, + MaxLenB, + Out, + D, + stride_ad, + stride_bd, + stride_od, + n_prefix_from_B, + IS_DENSE_A: tl.constexpr, + IS_DENSE_B: tl.constexpr, + BLOCK_D: tl.constexpr, +): + off_z = tl.program_id(1) + off_n = tl.program_id(0) + if IS_DENSE_A: + seq_start_a = off_z * MaxLenA + seq_len_a = MaxLenA + else: + seq_start_a = tl.load(OffsetsA + off_z) + seq_end_a = tl.load(OffsetsA + off_z + 1) + seq_len_a = seq_end_a - seq_start_a + if IS_DENSE_B: + seq_start_b = off_z * MaxLenB + seq_len_b = MaxLenB + else: + seq_start_b = tl.load(OffsetsB + off_z) + seq_end_b = tl.load(OffsetsB + off_z + 1) + seq_len_b = seq_end_b - seq_start_b + seq_len = seq_len_a + seq_len_b + if off_n >= seq_len: + return + offs_d = tl.arange(0, BLOCK_D) + out_seq_start = seq_start_a + seq_start_b + off_n + out_ptrs = Out + out_seq_start.to(tl.int64) * stride_od + offs_d + if off_n < n_prefix_from_B: + in_ptrs = ValuesB + (off_n + seq_start_b).to(tl.int64) * stride_bd + offs_d + elif off_n < seq_len_a + n_prefix_from_B: + in_ptrs = ( + ValuesA + + (off_n - n_prefix_from_B + seq_start_a).to(tl.int64) * stride_ad + + offs_d + ) + else: + in_ptrs = ( + ValuesB + + (off_n - seq_len_a + seq_start_b).to(tl.int64) * stride_bd + + offs_d + ) + v = tl.load(in_ptrs, mask=offs_d < D) + tl.store(out_ptrs, v, mask=offs_d < D) + + +@triton.jit +def _split_2D_jagged( + JaggedIn, + OffsetsA, + OffsetsB, + MaxLenA, + MaxLenB, + OutA, + OutB, + D, + stride_id, + stride_ad, + stride_bd, + n_prefix_to_B, + IS_DENSE_A: tl.constexpr, + IS_DENSE_B: tl.constexpr, + BLOCK_D: tl.constexpr, +): + off_z = tl.program_id(1) + off_n = tl.program_id(0) + if IS_DENSE_A: + seq_start_a = off_z * MaxLenA + seq_len_a = MaxLenA + else: + seq_start_a = tl.load(OffsetsA + off_z) + seq_end_a = tl.load(OffsetsA + off_z + 1) + seq_len_a = seq_end_a - seq_start_a + if IS_DENSE_B: + seq_start_b = off_z * MaxLenB + seq_len_b = MaxLenB + else: + seq_start_b = tl.load(OffsetsB + off_z) + seq_end_b = tl.load(OffsetsB + off_z + 1) + seq_len_b = seq_end_b - seq_start_b + seq_len = seq_len_a + seq_len_b + if off_n >= seq_len: + return + seq_start = seq_start_a + seq_start_b + offs_d = tl.arange(0, BLOCK_D) + in_ptrs = JaggedIn + (seq_start + off_n).to(tl.int64) * stride_id + offs_d + if off_n < n_prefix_to_B: + out_ptrs = OutB + (off_n + seq_start_b).to(tl.int64) * stride_bd + offs_d + elif off_n < seq_len_a + n_prefix_to_B: + out_ptrs = ( + OutA + + (off_n - n_prefix_to_B + seq_start_a).to(tl.int64) * stride_ad + + offs_d + ) + else: + out_ptrs = ( + OutB + (off_n - seq_len_a + seq_start_b).to(tl.int64) * stride_bd + offs_d + ) + v = tl.load(in_ptrs, mask=offs_d < D) + tl.store(out_ptrs, v, mask=offs_d < D) + + +class _Concat2DJaggedFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore[14] + def forward( + ctx, + max_seq_len: int, + values_a: torch.Tensor, + values_b: torch.Tensor, + max_len_a: Optional[int], + max_len_b: Optional[int], + offsets_a: Optional[torch.Tensor], + offsets_b: Optional[torch.Tensor], + n_prefix_from_B: int, + ): + values_a = switch_to_contiguous_if_needed(values_a) + values_b = switch_to_contiguous_if_needed(values_b) + is_dense_a = offsets_a is None + is_dense_b = offsets_b is None + total_len_a, D = values_a.shape + total_len_b, _ = values_b.shape + if is_dense_a: + assert max_len_a is not None + B = total_len_a // max_len_a + else: + assert offsets_a is not None + B = offsets_a.shape[0] - 1 + if is_dense_b: + assert max_len_b is not None + B = total_len_b // max_len_b + else: + assert offsets_b is not None + B = offsets_b.shape[0] - 1 + total_seq_len = total_len_a + total_len_b + BLOCK_D = triton.next_power_of_2(D) + values_out = torch.empty( + (total_seq_len, D), device=values_a.device, dtype=values_a.dtype + ) + _triton_concat_2D_jagged_internal( + values_a=values_a, + values_b=values_b, + values_out=values_out, + max_seq_len=max_seq_len, + B=B, + offsets_a=offsets_a, + offsets_b=offsets_b, + max_len_a=max_len_a, + max_len_b=max_len_b, + D=D, + n_prefix_from_B=n_prefix_from_B, + is_dense_a=is_dense_a, + is_dense_b=is_dense_b, + BLOCK_D=BLOCK_D, + ) + ctx.save_for_backward(offsets_a, offsets_b) + ctx.max_seq_len = max_seq_len + ctx.total_len_a = total_len_a + ctx.total_len_b = total_len_b + ctx.is_dense_a = is_dense_a + ctx.is_dense_b = is_dense_b + ctx.max_len_a = max_len_a + ctx.max_len_b = max_len_b + ctx.B = B + ctx.n_prefix_from_B = n_prefix_from_B + return values_out + + @staticmethod + # pyre-ignore[14] + def backward( + ctx, d_out: torch.Tensor + ) -> Tuple[None, torch.Tensor, torch.Tensor, None, None, None, None, None]: + offsets_a, offsets_b = ctx.saved_tensors + _, D = d_out.shape + BLOCK_D = triton.next_power_of_2(D) + d_values_a = torch.zeros( + (ctx.total_len_a, D), device=d_out.device, dtype=d_out.dtype + ) + d_values_b = torch.empty( + (ctx.total_len_b, D), device=d_out.device, dtype=d_out.dtype + ) + _split_2D_jagged[(ctx.max_seq_len, ctx.B)]( + JaggedIn=d_out, + OffsetsA=offsets_a, + OffsetsB=offsets_b, + MaxLenA=ctx.max_len_a, + MaxLenB=ctx.max_len_b, + OutA=d_values_a, + OutB=d_values_b, + D=D, + stride_id=d_out.stride(-2), + stride_ad=d_values_a.stride(-2), + stride_bd=d_values_b.stride(-2), + n_prefix_to_B=ctx.n_prefix_from_B, + BLOCK_D=BLOCK_D, + IS_DENSE_A=ctx.is_dense_a, + IS_DENSE_B=ctx.is_dense_b, + ) + return None, d_values_a, d_values_b, None, None, None, None, None + + +class _Split2DJaggedFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore[14] + def forward( + ctx, + max_seq_len: int, + values: torch.Tensor, + total_len_left: Optional[int], + total_len_right: Optional[int], + max_len_a: Optional[int], + max_len_b: Optional[int], + offsets_a: Optional[torch.Tensor], + offsets_b: Optional[torch.Tensor], + n_prefix_to_B: int, + ) -> Tuple[torch.Tensor, torch.Tensor]: + values = switch_to_contiguous_if_needed(values) + is_dense_a: bool = offsets_a is None + is_dense_b: bool = offsets_b is None + total_seq_len, D = values.shape + if is_dense_a: + assert is_dense_b is False + assert offsets_b is not None + assert max_len_a is not None + B = offsets_b.shape[0] - 1 + total_len_a = max_len_a * B + total_len_b = total_seq_len - total_len_a + elif is_dense_b: + assert is_dense_a is False + assert offsets_a is not None + assert max_len_b is not None + B = offsets_a.shape[0] - 1 + total_len_b = max_len_b * B + total_len_a = total_seq_len - total_len_b + else: + assert offsets_a is not None and offsets_b is not None + B = offsets_a.shape[0] - 1 + if total_len_left is not None and total_len_right is not None: + assert total_len_left + total_len_right == total_seq_len + total_len_a = total_len_left + total_len_b = total_len_right + else: + total_len_a = int(offsets_a[-1].item()) + total_len_b = values.size(0) - total_len_a + _, D = values.shape + BLOCK_D = triton.next_power_of_2(D) + values_a = torch.empty( + (total_len_a, D), device=values.device, dtype=values.dtype + ) + values_b = torch.empty( + (total_len_b, D), device=values.device, dtype=values.dtype + ) + _triton_split_2D_jagged_internal( + jagged_in=values, + max_seq_len=max_seq_len, + B=B, + offsets_a=offsets_a, + offsets_b=offsets_b, + max_len_a=max_len_a, + max_len_b=max_len_b, + out_a=values_a, + out_b=values_b, + D=D, + n_prefix_to_B=n_prefix_to_B, + is_dense_a=is_dense_a, + is_dense_b=is_dense_b, + BLOCK_D=BLOCK_D, + ) + ctx.save_for_backward(offsets_a, offsets_b) + ctx.max_seq_len = max_seq_len + ctx.total_seq_len = total_seq_len + ctx.max_len_a = max_len_a + ctx.max_len_b = max_len_b + ctx.is_dense_a = is_dense_a + ctx.is_dense_b = is_dense_b + ctx.B = B + ctx.D = D + ctx.n_prefix_to_B = n_prefix_to_B + return values_a, values_b + + @staticmethod + def backward( + ctx, *d_values + ) -> Tuple[None, torch.Tensor, None, None, None, None, None, None, None]: + offsets_a, offsets_b = ctx.saved_tensors + d_values_a, d_values_b = d_values + BLOCK_D = triton.next_power_of_2(ctx.D) + d_jagged_in = torch.empty( + (ctx.total_seq_len, ctx.D), + device=d_values_a.device, + dtype=d_values_a.dtype, + ) + _triton_concat_2D_jagged_internal( + values_a=d_values_a, + values_b=d_values_b, + values_out=d_jagged_in, + max_seq_len=ctx.max_seq_len, + B=ctx.B, + offsets_a=offsets_a, + offsets_b=offsets_b, + max_len_a=ctx.max_len_a, + max_len_b=ctx.max_len_b, + D=ctx.D, + n_prefix_from_B=ctx.n_prefix_to_B, + is_dense_a=ctx.is_dense_a, + is_dense_b=ctx.is_dense_b, + BLOCK_D=BLOCK_D, + ) + + return None, d_jagged_in, None, None, None, None, None, None, None + + +@torch.jit.unused +@torch.fx.wrap +def triton_concat_2D_jagged( + max_seq_len: int, + values_left: torch.Tensor, + values_right: torch.Tensor, + max_len_left: Optional[int], + max_len_right: Optional[int], + offsets_left: Optional[torch.Tensor], + offsets_right: Optional[torch.Tensor], + n_prefix_from_right: int = 0, +) -> torch.Tensor: + return _Concat2DJaggedFunction.apply( + max_seq_len, + values_left, + values_right, + max_len_left, + max_len_right, + offsets_left, + offsets_right, + n_prefix_from_right, + ) + + +@torch.jit.unused +@torch.fx.wrap +def triton_split_2D_jagged( + max_seq_len: int, + values: torch.Tensor, + total_len_left: Optional[int], + total_len_right: Optional[int], + max_len_left: Optional[int], + max_len_right: Optional[int], + offsets_left: Optional[torch.Tensor], + offsets_right: Optional[torch.Tensor], + n_prefix_to_right: int = 0, +) -> Tuple[torch.Tensor, torch.Tensor]: + return _Split2DJaggedFunction.apply( + max_seq_len, + values, + total_len_left, + total_len_right, + max_len_left, + max_len_right, + offsets_left, + offsets_right, + n_prefix_to_right, + ) + + +class _Concat2DJaggedMultirowFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore[14] + def forward( + ctx, + max_seq_len: int, + values_left: torch.Tensor, + values_right: torch.Tensor, + max_len_left: Optional[int], + max_len_right: Optional[int], + offsets_left: Optional[torch.Tensor], + offsets_right: Optional[torch.Tensor], + n_prefix_from_right: int, + ) -> torch.Tensor: + values_left = switch_to_contiguous_if_needed(values_left) + values_right = switch_to_contiguous_if_needed(values_right) + is_dense_left = offsets_left is None + is_dense_right = offsets_right is None + total_len_left, D = values_left.shape + total_len_right, _ = values_right.shape + if is_dense_left: + assert max_len_left is not None + B = total_len_left // max_len_left + else: + assert offsets_left is not None + B = offsets_left.shape[0] - 1 + if is_dense_right: + assert max_len_right is not None + B = total_len_right // max_len_right + else: + assert offsets_right is not None + B = offsets_right.shape[0] - 1 + total_seq_len = total_len_left + total_len_right + BLOCK_D = triton.next_power_of_2(D) + values_out = torch.empty( + (total_seq_len, D), device=values_left.device, dtype=values_left.dtype + ) + + def grid(meta): + return (triton.cdiv(max_seq_len, meta["BLOCK_N"]), B) + + concat_2D_jagged_multirow[grid]( + ValuesA=values_left, + ValuesB=values_right, + OffsetsA=offsets_left, + OffsetsB=offsets_right, + MaxLenA=max_len_left, + MaxLenB=max_len_right, + Out=values_out, + D=D, + stride_ad=values_left.stride(-2), + stride_bd=values_right.stride(-2), + stride_od=values_out.stride(-2), + n_prefix_from_B=n_prefix_from_right, + IS_DENSE_A=is_dense_left, + IS_DENSE_B=is_dense_right, + BLOCK_D=BLOCK_D, + ) + ctx.save_for_backward(offsets_left, offsets_right) + ctx.max_seq_len = max_seq_len + ctx.total_len_left = total_len_left + ctx.total_len_right = total_len_right + ctx.is_dense_left = is_dense_left + ctx.is_dense_right = is_dense_right + ctx.max_len_left = max_len_left + ctx.max_len_right = max_len_right + ctx.B = B + ctx.n_prefix_from_right = n_prefix_from_right + return values_out + + @staticmethod + # pyre-ignore[14] + def backward( + ctx, d_out: torch.Tensor + ) -> Tuple[None, torch.Tensor, torch.Tensor, None, None, None, None, None]: + offsets_left, offsets_right = ctx.saved_tensors + _, D = d_out.shape + BLOCK_D = triton.next_power_of_2(D) + d_values_left = torch.zeros( + (ctx.total_len_left, D), device=d_out.device, dtype=d_out.dtype + ) + d_values_right = torch.empty( + (ctx.total_len_right, D), device=d_out.device, dtype=d_out.dtype + ) + + def grid(meta): + return (triton.cdiv(ctx.max_seq_len, meta["BLOCK_N"]), ctx.B) + + split_2D_jagged_multirow[grid]( + JaggedIn=d_out, + OffsetsA=offsets_left, + OffsetsB=offsets_right, + MaxLenA=ctx.max_len_left, + MaxLenB=ctx.max_len_right, + OutA=d_values_left, + OutB=d_values_right, + D=D, + stride_id=d_out.stride(-2), + stride_ad=d_values_left.stride(-2), + stride_bd=d_values_right.stride(-2), + n_prefix_to_B=ctx.n_prefix_from_right, + IS_DENSE_A=ctx.is_dense_left, + IS_DENSE_B=ctx.is_dense_right, + BLOCK_D=BLOCK_D, + ) + return None, d_values_left, d_values_right, None, None, None, None, None + + +class _Split2DJaggedMultirowFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore[14] + def forward( + ctx, + max_seq_len: int, + values: torch.Tensor, + total_len_left: Optional[int], + total_len_right: Optional[int], + max_len_left: Optional[int], + max_len_right: Optional[int], + offsets_left: Optional[torch.Tensor], + offsets_right: Optional[torch.Tensor], + ) -> Tuple[torch.Tensor, torch.Tensor]: + values = switch_to_contiguous_if_needed(values) + is_dense_left: bool = offsets_left is None + is_dense_right: bool = offsets_right is None + total_seq_len, D = values.shape + + if is_dense_left: + assert is_dense_right is False + assert offsets_right is not None + assert max_len_left is not None + B = offsets_right.shape[0] - 1 + total_len_a = max_len_left * B + total_len_b = total_seq_len - total_len_a + elif is_dense_right: + assert is_dense_left is False + assert offsets_left is not None + assert max_len_right is not None + B = offsets_left.shape[0] - 1 + total_len_b = max_len_right * B + total_len_a = total_seq_len - total_len_b + else: + assert offsets_left is not None and offsets_right is not None + B = offsets_left.shape[0] - 1 + if total_len_left is not None and total_len_right is not None: + assert total_len_left + total_len_right == total_seq_len + total_len_a = total_len_left + total_len_b = total_len_right + else: + total_len_a = int(offsets_left[-1].item()) + total_len_b = values.size(0) - total_len_a + + BLOCK_D = triton.next_power_of_2(D) + values_a = torch.empty( + (total_len_a, D), device=values.device, dtype=values.dtype + ) + values_b = torch.empty( + (total_len_b, D), device=values.device, dtype=values.dtype + ) + + def grid(meta): + return (triton.cdiv(max_seq_len, meta["BLOCK_N"]), B) + + split_2D_jagged_multirow[grid]( + JaggedIn=values, + OffsetsA=offsets_left, + OffsetsB=offsets_right, + MaxLenA=max_len_left, + MaxLenB=max_len_right, + OutA=values_a, + OutB=values_b, + D=D, + stride_id=values.stride(-2), + stride_ad=values_a.stride(-2), + stride_bd=values_b.stride(-2), + n_prefix_to_B=0, + IS_DENSE_A=is_dense_left, + IS_DENSE_B=is_dense_right, + BLOCK_D=BLOCK_D, + ) + + ctx.save_for_backward(offsets_left, offsets_right) + ctx.max_seq_len = max_seq_len + ctx.total_seq_len = total_seq_len + ctx.max_len_left = max_len_left + ctx.max_len_right = max_len_right + ctx.is_dense_left = is_dense_left + ctx.is_dense_right = is_dense_right + ctx.B = B + ctx.D = D + + return values_a, values_b + + @staticmethod + # pyre-ignore[14] + def backward( + ctx, *d_values + ) -> Tuple[None, torch.Tensor, None, None, None, None, None, None]: + offsets_left, offsets_right = ctx.saved_tensors + d_values_a, d_values_b = d_values + BLOCK_D = triton.next_power_of_2(ctx.D) + d_jagged_in = torch.empty( + (ctx.total_seq_len, ctx.D), + device=d_values_a.device, + dtype=d_values_a.dtype, + ) + + def grid(meta): + return (triton.cdiv(ctx.max_seq_len, meta["BLOCK_N"]), ctx.B) + + concat_2D_jagged_multirow[grid]( + ValuesA=d_values_a, + ValuesB=d_values_b, + OffsetsA=offsets_left, + OffsetsB=offsets_right, + MaxLenA=ctx.max_len_left, + MaxLenB=ctx.max_len_right, + Out=d_jagged_in, + D=ctx.D, + stride_ad=d_values_a.stride(-2), + stride_bd=d_values_b.stride(-2), + stride_od=d_jagged_in.stride(-2), + n_prefix_from_B=0, + IS_DENSE_A=ctx.is_dense_left, + IS_DENSE_B=ctx.is_dense_right, + BLOCK_D=BLOCK_D, + ) + + return None, d_jagged_in, None, None, None, None, None, None + + +@torch.jit.unused +@torch.fx.wrap +def triton_concat_2D_jagged_multirow( + max_seq_len: int, + values_a: torch.Tensor, + values_b: torch.Tensor, + offsets_a: Optional[torch.Tensor], + offsets_b: Optional[torch.Tensor], + max_len_a: int, + max_len_b: int, +) -> torch.Tensor: + return _Concat2DJaggedMultirowFunction.apply( + max_seq_len, + values_a, + values_b, + max_len_a, + max_len_b, + offsets_a, + offsets_b, + 0, + ) + + +@torch.jit.unused +@torch.fx.wrap +def triton_split_2D_jagged_multirow( + max_seq_len: int, + values: torch.Tensor, + total_len_left: Optional[int] = None, + total_len_right: Optional[int] = None, + max_len_left: Optional[int] = None, + max_len_right: Optional[int] = None, + offsets_left: Optional[torch.Tensor] = None, + offsets_right: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + return _Split2DJaggedMultirowFunction.apply( + max_seq_len, + values, + total_len_left, + total_len_right, + max_len_left, + max_len_right, + offsets_left, + offsets_right, + ) diff --git a/recommendation_v4/generative_recommenders/ops/triton/triton_layer_norm.py b/recommendation_v4/generative_recommenders/ops/triton/triton_layer_norm.py new file mode 100644 index 000000000..1e997fd40 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton/triton_layer_norm.py @@ -0,0 +1,1327 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +#!/usr/bin/env python3 + + +from typing import List, Optional, Tuple + +import torch + +# @manual=//triton:triton +import triton + +# @manual=//triton:triton +import triton.language as tl +from generative_recommenders.common import ( + switch_to_contiguous_if_needed, + triton_autotune, +) +from generative_recommenders.ops.utils import ( + is_sm100_plus, + is_sm90, + maybe_register_custom_op, +) + +try: + # @manual=//triton:triton + from triton.language.extra.libdevice import fast_dividef, rsqrt as libdevice_rsqrt +except ImportError: + try: + # @manual=//triton:triton + from triton.language.extra.cuda.libdevice import ( + fast_dividef, + rsqrt as libdevice_rsqrt, + ) + except ImportError: + # pyre-ignore: Undefined import [21] + # @manual=//triton:triton + from triton.language.math import fast_dividef, rsqrt as libdevice_rsqrt + + +def _get_layer_norm_fwd_configs() -> List[triton.Config]: + """Generate autotune configs for multi-row LayerNorm kernels.""" + configs = [] + block_ns = [4, 8, 16] if is_sm100_plus() else [1, 2, 4, 8] + for BLOCK_N in block_ns: + for num_warps in [1, 2, 4, 8]: + configs.append( + triton.Config( + {"BLOCK_N": BLOCK_N}, + num_warps=num_warps, + ) + ) + return configs + + +def _bwd_pre_hook(nargs): + nargs["DW"].zero_() + if "DB" in nargs: + nargs["DB"].zero_() + + +def _get_norm_bwd_configs() -> List[triton.Config]: + """Generate autotune configs for multi-row LayerNorm kernels.""" + configs = [] + if is_sm100_plus(): + block_ns = [8, 16] + num_shards_list = [8, 16] + num_warps_list = [2, 4] + elif is_sm90(): + block_ns = [2, 4] + num_shards_list = [8] + num_warps_list = [2, 4] + else: + block_ns = [1, 2] + num_shards_list = [8] + num_warps_list = [2, 4] + for BLOCK_N in block_ns: + for num_warps in num_warps_list: + for num_shards in num_shards_list: + configs.append( + triton.Config( + {"BLOCK_N": BLOCK_N, "SHARDS_PER_SM": num_shards}, + num_warps=num_warps, + ) + ) + return configs + + +@triton_autotune( + configs=_get_layer_norm_fwd_configs(), + key=["BLOCK_D"], +) +@triton.jit +def _layer_norm_fwd( + X, + Y, + Mean, + Rstd, + N, + D, + eps, + stride_x, + stride_y, + TRAINING: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_N: tl.constexpr, + COMPUTE_MEAN_AND_RSTD: tl.constexpr, +): + block_id = tl.program_id(0) + start_row = block_id * BLOCK_N + + X_block_ptr = tl.make_block_ptr( + base=X, + shape=(N, D), + strides=(stride_x, 1), + offsets=(start_row, 0), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + + Y_block_ptr = tl.make_block_ptr( + base=Y, + shape=(N, D), + strides=(stride_y, 1), + offsets=(start_row, 0), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + + x_block = tl.load(X_block_ptr, boundary_check=(0, 1), padding_option="zero").to( + tl.float32 + ) + + cols = tl.arange(0, BLOCK_D) + col_mask = cols < D + rows = start_row + tl.arange(0, BLOCK_N) + row_mask = rows < N + + if COMPUTE_MEAN_AND_RSTD: + mean = tl.sum(x_block, axis=1) / D + if TRAINING: + tl.store(Mean + rows, mean, row_mask) + mean = tl.expand_dims(mean, 1) + else: + mean = tl.load(Mean + rows, row_mask, other=0.0) + mean = tl.expand_dims(mean, 1) + + x_mean = x_block - mean + x_mean = tl.where(row_mask[:, None] & col_mask[None, :], x_mean, 0.0) + + if COMPUTE_MEAN_AND_RSTD: + _var = x_mean * x_mean + var = tl.sum(_var, axis=1) / D + rstd = 1 / tl.sqrt(var + eps) + if TRAINING: + tl.store(Rstd + rows, rstd, row_mask) + else: + rstd = tl.load(Rstd + rows, row_mask, other=0.0) + + rstd = tl.expand_dims(rstd, 1) + y = x_mean * rstd + + tl.store(Y_block_ptr, y.to(Y.dtype.element_ty), boundary_check=(0, 1)) + + +@triton_autotune( + configs=_get_layer_norm_fwd_configs(), + key=["BLOCK_D"], +) +@triton.jit +def _weighted_layer_norm_fwd( + X, + Y, + W, + B, + Mean, + Rstd, + N, + D, + eps, + stride_x, + stride_y, + IS_SWISH: tl.constexpr, + TRAINING: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_N: tl.constexpr, + COMPUTE_MEAN_AND_RSTD: tl.constexpr, +): + # Get the block ID and calculate starting row + block_id = tl.program_id(0) + start_row = block_id * BLOCK_N + + # Load weight and bias once (shared across all rows in this block) + cols = tl.arange(0, BLOCK_D) + col_mask = cols < D + w = tl.load(W + cols, mask=col_mask, other=0.0).to(tl.float32) + b = tl.load(B + cols, mask=col_mask, other=0.0).to(tl.float32) + + # Create block pointers for X and Y + X_block_ptr = tl.make_block_ptr( + base=X, + shape=(N, D), + strides=(stride_x, 1), + offsets=(start_row, 0), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + + Y_block_ptr = tl.make_block_ptr( + base=Y, + shape=(N, D), + strides=(stride_y, 1), + offsets=(start_row, 0), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + + x_block = tl.load(X_block_ptr, boundary_check=(0, 1), padding_option="zero").to( + tl.float32 + ) + + rows = start_row + tl.arange(0, BLOCK_N) + row_mask = rows < N + + if COMPUTE_MEAN_AND_RSTD: + mean = tl.sum(x_block, axis=1) / D + if TRAINING: + tl.store(Mean + rows, mean, row_mask) + mean = tl.expand_dims(mean, 1) + else: + mean = tl.load(Mean + rows, row_mask, other=0.0) + mean = tl.expand_dims(mean, 1) + + x_mean = x_block - mean + x_mean = tl.where(row_mask[:, None] & col_mask[None, :], x_mean, 0.0) + + if COMPUTE_MEAN_AND_RSTD: + _var = x_mean * x_mean + var = tl.sum(_var, axis=1) / D + rstd = libdevice_rsqrt(var + eps) + if TRAINING: + tl.store(Rstd + rows, rstd, row_mask) + else: + rstd = tl.load(Rstd + rows, row_mask, other=0.0) + + rstd = tl.expand_dims(rstd, 1) + y = x_mean * rstd + y = y * w[None, :] + b[None, :] + + if IS_SWISH: + y = tl.sigmoid(y) * x_block + + tl.store(Y_block_ptr, y.to(Y.dtype.element_ty), boundary_check=(0, 1)) + + +@triton.jit +def _layer_norm_bwd_dx( + DX, + DY, + X, + Mean, + Rstd, + stride_dx, + stride_dy, + stride_x, + D, + eps, + BLOCK_D: tl.constexpr, +): + row = tl.program_id(0) + cols = tl.arange(0, BLOCK_D) + mask = cols < D + X += row.to(tl.int64) * stride_x + DY += row.to(tl.int64) * stride_dy + DX += row.to(tl.int64) * stride_dx + + # Load data to SRAM + x = tl.load(X + cols, mask=mask, other=0).to(tl.float32) + dy = tl.load(DY + cols, mask=mask, other=0).to(tl.float32) + mean = tl.load(Mean + row) + rstd = tl.load(Rstd + row) + + # Compute dx + xhat = (x - mean) * rstd + xhat = tl.where(mask, xhat, 0.0) + dy = tl.where(mask, dy, 0.0) + c1 = tl.sum(xhat * dy, axis=0) / D + c2 = tl.sum(dy, axis=0) / D + dx = (dy - (xhat * c1 + c2)) * rstd + # Write dx + tl.store(DX + cols, dx, mask=mask) + + +@triton_autotune( + configs=_get_layer_norm_fwd_configs(), + key=["BLOCK_D"], +) +@triton.jit +def _weighted_layer_norm_bwd_dx( + DX, + DY, + DW, + DB, + X, + W, + B, + Mean, + Rstd, + stride_dx, + stride_dy, + stride_x, + D, + eps, + IS_SWISH: tl.constexpr, + N, + BLOCK_D: tl.constexpr, + BLOCK_N: tl.constexpr, +): + pid = tl.program_id(0) + tile_num = tl.num_programs(0) + num_blocks = tl.cdiv(N, BLOCK_N) + blocks_per_tile = num_blocks // tile_num + if pid < num_blocks % tile_num: + blocks_per_tile += 1 + + cols = tl.arange(0, BLOCK_D) + col_mask = cols < D + w = tl.load(W + cols, mask=col_mask, other=0.0).to(tl.float32) + + acc_dw = tl.zeros([BLOCK_D], dtype=tl.float32) + acc_db = tl.zeros([BLOCK_D], dtype=tl.float32) + + start_block = pid + + for idx in range(blocks_per_tile): + current_block = start_block + idx * tile_num + start_row = current_block * BLOCK_N + + X_block_ptr = tl.make_block_ptr( + base=X, + shape=(N, D), + strides=(stride_x, 1), + offsets=(start_row, 0), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + + DX_block_ptr = tl.make_block_ptr( + base=DX, + shape=(N, D), + strides=(stride_dx, 1), + offsets=(start_row, 0), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + + DY_block_ptr = tl.make_block_ptr( + base=DY, + shape=(N, D), + strides=(stride_dy, 1), + offsets=(start_row, 0), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + + # Load data blocks + x_block = tl.load(X_block_ptr, boundary_check=(0, 1), padding_option="zero").to( + tl.float32 + ) + dy_block = tl.load( + DY_block_ptr, boundary_check=(0, 1), padding_option="zero" + ).to(tl.float32) + + # Load mean and rstd for all rows in this block + rows = start_row + tl.arange(0, BLOCK_N) + row_mask = rows < N + mean = tl.load(Mean + rows, row_mask, other=0.0) + rstd = tl.load(Rstd + rows, row_mask, other=0.0) + + # Expand dimensions for broadcasting + mean = tl.expand_dims(mean, 1) + rstd = tl.expand_dims(rstd, 1) + + xhat = (x_block - mean) * rstd + + xhat = tl.where(row_mask[:, None] & col_mask[None, :], xhat, 0.0) + wdy = w[None, :] * dy_block + wdy = tl.where(row_mask[:, None] & col_mask[None, :], wdy, 0.0) + + # Compute dx + if IS_SWISH: + b = tl.load(B + cols, mask=col_mask, other=0.0).to(tl.float32) + sigmoid_layer_norm = tl.sigmoid(xhat * w[None, :] + b[None, :]) + sigmoid_layer_norm = tl.where( + row_mask[:, None] & col_mask[None, :], sigmoid_layer_norm, 0.0 + ) + + sigmoid_deriv = sigmoid_layer_norm * (1 - sigmoid_layer_norm) + x_ = wdy * x_block * sigmoid_deriv + x_ = tl.where(row_mask[:, None] & col_mask[None, :], x_, 0.0) + + c1 = tl.sum(xhat * x_, axis=1) / D + c2 = tl.sum(x_, axis=1) / D + c1 = tl.expand_dims(c1, 1) + c2 = tl.expand_dims(c2, 1) + dx = (x_ - (xhat * c1 + c2)) * rstd + + dx = dy_block * sigmoid_layer_norm + dx + # Write dx + tl.store(DX_block_ptr, dx.to(DX.dtype.element_ty), boundary_check=(0, 1)) + partial_dw = tl.sum(dy_block * x_block * xhat * sigmoid_deriv, axis=0) + partial_db = tl.sum(dy_block * x_block * sigmoid_deriv, axis=0) + else: + c1 = tl.sum(xhat * wdy, axis=1) / D + c2 = tl.sum(wdy, axis=1) / D + c1 = tl.expand_dims(c1, 1) + c2 = tl.expand_dims(c2, 1) + dx = (wdy - (xhat * c1 + c2)) * rstd + # Write dx + tl.store(DX_block_ptr, dx.to(DX.dtype.element_ty), boundary_check=(0, 1)) + partial_dw = tl.sum(dy_block * xhat, axis=0) + partial_db = tl.sum(dy_block, axis=0) + + # Accumulate partial sums in shared memory + acc_dw += partial_dw + acc_db += partial_db + + # Store accumulated sums back to global memory + dw_ptrs = DW + pid.to(tl.int64) * D + cols + db_ptrs = DB + pid.to(tl.int64) * D + cols + tl.store(dw_ptrs, acc_dw, mask=col_mask) + tl.store(db_ptrs, acc_db, mask=col_mask) + + +def _get_bwd_dwdb_configs() -> List[triton.Config]: + configs = [] + BLOCK_N_CHOICES = [32, 64, 128, 256] + if is_sm100_plus(): + BLOCK_N_CHOICES = [128, 256, 512, 1024] + for BLOCK_N in BLOCK_N_CHOICES: + for num_warps in [8, 16] + ([] if torch.ops.hip else [32]): + configs.append( + triton.Config( + {"BLOCK_N": BLOCK_N}, + num_warps=num_warps, + ) + ) + return configs + + +@triton_autotune( + configs=_get_bwd_dwdb_configs(), + key=["D"], +) +@triton.jit +def _layer_norm_bwd_dwdb( + DW, + DB, + FINAL_DW, + FINAL_DB, + N, + D, + BLOCK_N: tl.constexpr, + BLOCK_D: tl.constexpr, +): + pid = tl.program_id(0) + cols = pid * BLOCK_D + tl.arange(0, BLOCK_D) + dw = tl.zeros((BLOCK_N, BLOCK_D), dtype=tl.float32) + db = tl.zeros((BLOCK_N, BLOCK_D), dtype=tl.float32) + + for i in range(0, N, BLOCK_N): + rows = i + tl.arange(0, BLOCK_N) + # pyre-fixme[16]: `int` has no attribute `__getitem__`. + mask = (rows[:, None] < N) & (cols[None, :] < D) + offs = rows[:, None] * D + cols[None, :] + dw += tl.load(DW + offs, mask=mask, other=0.0) + db += tl.load(DB + offs, mask=mask, other=0.0) + + sum_dw = tl.sum(dw, axis=0) + sum_db = tl.sum(db, axis=0) + tl.store(FINAL_DW + cols, sum_dw.to(FINAL_DW.dtype.element_ty), mask=cols < D) + tl.store(FINAL_DB + cols, sum_db.to(FINAL_DB.dtype.element_ty), mask=cols < D) + + +def compute_BLOCK_D(x: torch.Tensor) -> int: + """Compute the BLOCK_D parameter for layer norm kernels.""" + D = x.shape[-1] + MAX_FUSED_SIZE = 65536 // x.element_size() + return min(MAX_FUSED_SIZE, triton.next_power_of_2(D)) + + +@maybe_register_custom_op( + "generative_recommenders::triton_weighted_layer_norm_fwd", mutates_args=() +) +def triton_weighted_layer_norm_fwd( + x: torch.Tensor, + weight: Optional[torch.Tensor], + bias: Optional[torch.Tensor], + eps: float, + mean: Optional[torch.Tensor] = None, + rstd: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + assert x.dim() == 2, f"x.dim() == {x.dim()}, expected 2" + x = switch_to_contiguous_if_needed(x) + N, D = x.shape + learnable = weight is not None + if learnable: + assert bias is not None and weight is not None + assert weight.dim() == 1 + assert bias.dim() == 1 + assert weight.numel() == D + assert bias.numel() == D + + y = torch.empty_like(x) + compute_mean_and_rstd = mean is None or rstd is None + # Always allocate new tensors to avoid aliasing inputs with outputs + out_mean = torch.empty((N,), dtype=torch.float32, device=x.device) + out_rstd = torch.empty((N,), dtype=torch.float32, device=x.device) + if not compute_mean_and_rstd: + assert mean is not None and rstd is not None + out_mean.copy_(mean) + out_rstd.copy_(rstd) + + # Less than 64KB per feature: enqueue fused kernel + MAX_FUSED_SIZE = 65536 // x.element_size() + BLOCK_D: int = min(MAX_FUSED_SIZE, triton.next_power_of_2(D)) + if D > BLOCK_D: + raise RuntimeError("This layer norm doesn't support feature dim >= 64KB.") + + if N == 0: + return y, out_mean, out_rstd + + # pyre-ignore[28] + grid = lambda meta: (triton.cdiv(N, meta["BLOCK_N"]),) # noqa E731 + if learnable: + _weighted_layer_norm_fwd[grid]( + x, + y, + weight, + bias, + out_mean, + out_rstd, + N, + D, + eps, + x.stride(0), + y.stride(0), + IS_SWISH=False, + TRAINING=True, + BLOCK_D=BLOCK_D, + COMPUTE_MEAN_AND_RSTD=compute_mean_and_rstd, + ) + else: + _layer_norm_fwd[grid]( + x, + y, + out_mean, + out_rstd, + N, + D, + eps, + x.stride(0), + y.stride(0), + TRAINING=True, + BLOCK_D=BLOCK_D, + COMPUTE_MEAN_AND_RSTD=compute_mean_and_rstd, + ) + + return y, out_mean, out_rstd + + +@triton_weighted_layer_norm_fwd.register_fake +def _( + x: torch.Tensor, + weight: Optional[torch.Tensor], + bias: Optional[torch.Tensor], + eps: float, + mean: Optional[torch.Tensor] = None, + rstd: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + N = x.shape[0] + y = torch.empty_like(x) + # Always allocate new tensors to avoid aliasing inputs with outputs + out_mean = torch.empty((N,), dtype=torch.float32, device=x.device) + out_rstd = torch.empty((N,), dtype=torch.float32, device=x.device) + return y, out_mean, out_rstd + + +@maybe_register_custom_op( + "generative_recommenders::triton_weighted_layer_norm_bwd", mutates_args=() +) +def _triton_weighted_layer_norm_bwd_impl( + dy: torch.Tensor, + x: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + mean: torch.Tensor, + rstd: torch.Tensor, + learnable: bool, + eps: float, + BLOCK_D: int, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + num_warps: int = min(max(BLOCK_D // 256, 1), 8) + if learnable: + N, D = x.shape + dx = torch.empty_like(x) + sms = torch.cuda.get_device_properties(x.device).multi_processor_count + tile_num = max(1, min(sms * 8, N // 4)) + _dweight = torch.empty((tile_num, D), dtype=torch.float32, device=x.device) + _dbias = torch.empty((tile_num, D), dtype=torch.float32, device=x.device) + dweight = torch.empty((D,), dtype=weight.dtype, device=x.device) + dbias = torch.empty((D,), dtype=weight.dtype, device=x.device) + if N == 0: + dweight.zero_() + dbias.zero_() + return dx, dweight, dbias + # pyre-ignore[28] + _weighted_layer_norm_bwd_dx[(tile_num,)]( + dx, + dy, + _dweight, + _dbias, + x, + weight, + bias, + mean, + rstd, + dx.stride(0), + dy.stride(0), + x.stride(0), + D, + eps, + IS_SWISH=False, + N=N, + BLOCK_D=BLOCK_D, + ) + + def grid(META): + return (triton.cdiv(D, META["BLOCK_D"]),) + + blocks = triton.next_power_of_2(sms * 4) + BLOCK_D = triton.next_power_of_2(triton.cdiv(D, blocks)) + BLOCK_D = min(max(BLOCK_D, 4), 128) + _layer_norm_bwd_dwdb[grid]( + _dweight, + _dbias, + dweight, + dbias, + tile_num, + D, + BLOCK_D=BLOCK_D, + ) + + return dx, dweight, dbias + else: + N, D = x.shape + dx = torch.empty_like(x) + # Return empty tensors as sentinels for None + dweight = torch.empty(0, dtype=x.dtype, device=x.device) + dbias = torch.empty(0, dtype=x.dtype, device=x.device) + if N == 0: + return dx, dweight, dbias + # pyre-ignore[28] + _layer_norm_bwd_dx[(N,)]( + dx, + dy, + x, + mean, + rstd, + dx.stride(0), + dy.stride(0), + x.stride(0), + D, + eps, + BLOCK_D=BLOCK_D, + num_warps=num_warps, + ) + return dx, dweight, dbias + + +@_triton_weighted_layer_norm_bwd_impl.register_fake +def _( + dy: torch.Tensor, + x: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + mean: torch.Tensor, + rstd: torch.Tensor, + learnable: bool, + eps: float, + BLOCK_D: int, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + dx = torch.empty_like(x) + if learnable: + D = x.shape[-1] + dweight = torch.empty((D,), dtype=weight.dtype, device=x.device) + dbias = torch.empty((D,), dtype=weight.dtype, device=x.device) + else: + dweight = torch.empty(0, dtype=x.dtype, device=x.device) + dbias = torch.empty(0, dtype=x.dtype, device=x.device) + return dx, dweight, dbias + + +def triton_weighted_layer_norm_bwd( + dy: torch.Tensor, + x: torch.Tensor, + weight: Optional[torch.Tensor], + bias: Optional[torch.Tensor], + mean: torch.Tensor, + rstd: torch.Tensor, + learnable: bool, + eps: float, + BLOCK_D: int, +) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor]]: + # Use sentinel tensors for custom_op compatibility (can't return Optional[Tensor]) + _weight = ( + weight if weight is not None else torch.empty(0, dtype=x.dtype, device=x.device) + ) + _bias = bias if bias is not None else torch.empty(0, dtype=x.dtype, device=x.device) + dx, dweight, dbias = _triton_weighted_layer_norm_bwd_impl( + dy=dy, + x=x, + weight=_weight, + bias=_bias, + mean=mean, + rstd=rstd, + learnable=learnable, + eps=eps, + BLOCK_D=BLOCK_D, + ) + if not learnable: + return dx, None, None + return dx, dweight, dbias + + +class LayerNormFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore[14] + def forward( + ctx, + x: torch.Tensor, + weight: Optional[torch.Tensor], + bias: Optional[torch.Tensor], + eps: float, + ) -> torch.Tensor: + y, mean, rstd = triton_weighted_layer_norm_fwd( + x=x, + weight=weight, + bias=bias, + eps=eps, + ) + BLOCK_D = compute_BLOCK_D(x) + learnable = weight is not None + if learnable: + ctx.save_for_backward(x, weight, bias, mean, rstd) + else: + ctx.save_for_backward(x, mean, rstd) + ctx.BLOCK_D = BLOCK_D + ctx.eps = eps + ctx.learnable = learnable + return y + + @staticmethod + # pyre-ignore[14] + def backward( + ctx, dy: torch.Tensor + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor], None]: + if ctx.learnable: + x, weight, bias, mean, rstd = ctx.saved_tensors + else: + x, mean, rstd = ctx.saved_tensors + weight, bias = None, None + dx, dweight, dbias = triton_weighted_layer_norm_bwd( + dy=dy, + x=x, + weight=weight, + bias=bias, + mean=mean, + rstd=rstd, + learnable=ctx.learnable, + eps=ctx.eps, + BLOCK_D=ctx.BLOCK_D, + ) + return dx, dweight, dbias, None + + +def _get_rms_norm_fwd_configs() -> List[triton.Config]: + """Generate autotune configs for multi-row RMSNorm kernels.""" + configs = [] + for BLOCK_N in [1, 4, 16]: + for num_warps in [2, 4]: + configs.append( + triton.Config( + {"BLOCK_N": BLOCK_N}, + num_warps=num_warps, + ) + ) + return configs + + +@triton.autotune( + configs=_get_rms_norm_fwd_configs(), + key=["BLOCK_D", "SILU"], +) +@triton.jit +def _weighted_rms_norm_fwd( + X, + Y, + W, + Rstd, + N, + D: tl.constexpr, + eps, + stride_x, + stride_y, + SILU: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_N: tl.constexpr, +): + block_id = tl.program_id(0) + start_row = block_id * BLOCK_N + + # Load weight once (shared across all rows in this block) + cols = tl.arange(0, BLOCK_D) + col_mask = cols < D + w = tl.load(W + cols, mask=col_mask, other=0.0).to(tl.float32) + + # Create block pointers for X and Y + X_block_ptr = tl.make_block_ptr( + base=X, + shape=(N, D), + strides=(stride_x, 1), + offsets=(start_row, 0), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + + Y_block_ptr = tl.make_block_ptr( + base=Y, + shape=(N, D), + strides=(stride_y, 1), + offsets=(start_row, 0), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + + x_block = tl.load(X_block_ptr, boundary_check=(0, 1), padding_option="zero").to( + tl.float32 + ) + + rows = start_row + tl.arange(0, BLOCK_N) + row_mask = rows < N + + # Compute variance (RMS norm uses x directly, not x - mean) + x_masked = tl.where(row_mask[:, None] & col_mask[None, :], x_block, 0.0) + _var = x_masked * x_masked + var = tl.sum(_var, axis=1) / D + rstd = 1 / tl.sqrt(var + eps) + tl.store(Rstd + rows, rstd, row_mask) + + # Normalize and apply linear transformation + rstd = tl.expand_dims(rstd, 1) + y = x_block * rstd + y = y * w[None, :] + + if SILU: + # pyre-ignore[16]: Module `triton.language.math` has no attribute `fast_dividef` + y = fast_dividef(y, 1.0 + tl.exp(-y)) + + tl.store(Y_block_ptr, y.to(Y.dtype.element_ty), boundary_check=(0, 1)) + + +@triton.jit +def _weighted_rms_norm_bwd_dx( + DX, + DY, + DW, + X, + W, + Rstd, + Lock, + stride_dx, + stride_dy, + stride_x, + D: tl.constexpr, + eps, + GROUP_N, + BLOCK_D: tl.constexpr, +): + row = tl.program_id(0) + cols = tl.arange(0, BLOCK_D) + mask = cols < D + X += row.to(tl.int64) * stride_x + DY += row.to(tl.int64) * stride_dy + DX += row.to(tl.int64) * stride_dx + + # Load data to SRAM + x = tl.load(X + cols, mask=mask, other=0).to(tl.float32) + dy = tl.load(DY + cols, mask=mask, other=0).to(tl.float32) + rstd = tl.load(Rstd + row) + + # Compute dx + xhat = x * rstd + w = tl.load(W + cols, mask=mask).to(tl.float32) + wdy = w * dy + + xhat = tl.where(mask, xhat, 0.0) + wdy = tl.where(mask, wdy, 0.0) + c1 = tl.sum(xhat * wdy, axis=0) / D + dx = (wdy - (xhat * c1)) * rstd + # Write dx + tl.store(DX + cols, dx, mask=mask) + + # Offset locks and weights/biases gradient pointer for parallel reduction + lock_id = row % GROUP_N + Lock += lock_id + Count = Lock + GROUP_N + DW = DW + lock_id * D + cols + # Accumulate partial sums for dw/db + partial_dw = dy * xhat + while tl.atomic_cas(Lock, 0, 1) == 1: + pass + count = tl.load(Count) + # First store doesn't accumulate + if count == 0: + tl.atomic_xchg(Count, 1) + else: + partial_dw += tl.load(DW, mask=mask) + tl.store(DW, partial_dw, mask=mask) + # Release the lock + tl.atomic_xchg(Lock, 0) + + +@triton_autotune( + configs=_get_norm_bwd_configs(), + key=["BLOCK_D", "SILU"], + reset_to_zero=["DW"], +) +@triton.jit +def _weighted_rms_norm_bwd( + DX, + DY, + DW, + X, + W, + Rstd, + stride_dx, + stride_dy, + stride_x, + D, + eps, + N, + SILU: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_N: tl.constexpr, + SHARDS_PER_SM: tl.constexpr, +): + pid = tl.program_id(0) + tile_num = tl.num_programs(0) + num_blocks = tl.cdiv(N, BLOCK_N) + blocks_per_tile = num_blocks // tile_num + if pid < num_blocks % tile_num: + blocks_per_tile += 1 + + cols = tl.arange(0, BLOCK_D) + col_mask = cols < D + w = tl.load(W + cols, mask=col_mask, other=0.0).to(tl.float32) + + start_block = pid + + acc_dw = tl.zeros([BLOCK_D], dtype=tl.float32) + + for idx in range(blocks_per_tile): + current_block = start_block + idx * tile_num + start_row = current_block * BLOCK_N + + X_block_ptr = tl.make_block_ptr( + base=X, + shape=(N, D), + strides=(stride_x, 1), + offsets=(start_row, 0), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + + DX_block_ptr = tl.make_block_ptr( + base=DX, + shape=(N, D), + strides=(stride_dx, 1), + offsets=(start_row, 0), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + + DY_block_ptr = tl.make_block_ptr( + base=DY, + shape=(N, D), + strides=(stride_dy, 1), + offsets=(start_row, 0), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + + # Load data blocks + x_block = tl.load(X_block_ptr, boundary_check=(0, 1), padding_option="zero").to( + tl.float32 + ) + dy_block = tl.load( + DY_block_ptr, boundary_check=(0, 1), padding_option="zero" + ).to(tl.float32) + + # Load rstd for all rows in this block + rows = start_row + tl.arange(0, BLOCK_N) + row_mask = rows < N + rstd = tl.load(Rstd + rows, row_mask, other=0.0) + + # Expand dimensions for broadcasting + rstd = tl.expand_dims(rstd, 1) + + # Compute dx + xhat = x_block * rstd + + # Apply SILU backward if enabled + if SILU: + y_before_silu = xhat * w[None, :] + # pyre-fixme[16] + sig_y = fast_dividef(1.0, 1.0 + tl.exp(-y_before_silu)) + # SILU derivative: sigmoid(y) + y * sigmoid(y) * (1 - sigmoid(y)) + dy_block = dy_block * (sig_y + y_before_silu * sig_y * (1.0 - sig_y)) + + wdy = w[None, :] * dy_block + + c1 = tl.sum(xhat * wdy, axis=1) / D + c1 = tl.expand_dims(c1, 1) + dx = (wdy - (xhat * c1)) * rstd + + # Write dx + tl.store(DX_block_ptr, dx.to(DX.dtype.element_ty), boundary_check=(0, 1)) + + # Accumulate partial sums for dw + # Compute dw for all rows, then sum locally before atomic operation + partial_dw_block = dy_block * xhat + # Local reduction: sum across all rows in this block + partial_dw = tl.sum(partial_dw_block, axis=0) + acc_dw += partial_dw + + DW_ptr = DW + cols + tl.atomic_add(DW_ptr, acc_dw, col_mask) + + +@triton_autotune( + configs=_get_bwd_dwdb_configs(), + key=["D"], +) +@triton.jit +def _rms_norm_bwd_dwdb( + DW, + FINAL_DW, + N, + D, + BLOCK_N: tl.constexpr, + BLOCK_D: tl.constexpr, +): + pid = tl.program_id(0) + cols = pid * BLOCK_D + tl.arange(0, BLOCK_D) + dw = tl.zeros((BLOCK_N, BLOCK_D), dtype=tl.float32) + + for i in range(0, N, BLOCK_N): + rows = i + tl.arange(0, BLOCK_N) + # pyre-fixme[16]: `int` has no attribute `__getitem__`. + mask = (rows[:, None] < N) & (cols[None, :] < D) + offs = rows[:, None] * D + cols[None, :] + dw += tl.load(DW + offs, mask=mask, other=0.0) + + sum_dw = tl.sum(dw, axis=0) + tl.store(FINAL_DW + cols, sum_dw.to(FINAL_DW.dtype.element_ty), mask=cols < D) + + +class RMSNormFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore[14] + def forward( + ctx, + x: torch.Tensor, + weight: torch.Tensor, + eps: float, + silu: bool, + ) -> torch.Tensor: + assert x.dim() == 2 + x = switch_to_contiguous_if_needed(x) + N, D = x.shape + assert weight.dim() == 1 + assert weight.numel() == D + + y = torch.empty_like(x) + rstd = torch.empty((N,), dtype=torch.float32, device=x.device) + + # Less than 64KB per feature: enqueue fused kernel + MAX_FUSED_SIZE = 65536 // x.element_size() + BLOCK_D = min(MAX_FUSED_SIZE, triton.next_power_of_2(D)) + if D > BLOCK_D: + raise RuntimeError("This layer norm doesn't support feature dim >= 64KB.") + + ctx.save_for_backward(x, weight, rstd) + ctx.silu = silu + if N == 0: + return y + + # pyre-ignore[28] + grid = lambda meta: (triton.cdiv(N, meta["BLOCK_N"]),) # noqa E731 + _weighted_rms_norm_fwd[grid]( + x, + y, + weight, + rstd, + N, + D, + eps, + x.stride(0), + y.stride(0), + SILU=silu, + BLOCK_D=BLOCK_D, + ) + + ctx.BLOCK_D = BLOCK_D + ctx.eps = eps + return y + + @staticmethod + # pyre-ignore[14] + def backward( + ctx, dy: torch.Tensor + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], None, None]: + x, weight, rstd = ctx.saved_tensors + N, D = x.shape + dx = torch.empty_like(x) + dweight = torch.zeros((D,), dtype=weight.dtype, device=x.device) + if N == 0: + dweight.zero_() + return dx, dweight, None, None + + sms = torch.cuda.get_device_properties(x.device).multi_processor_count + + # pyre-ignore[28] + grid = lambda meta: ( # noqa E731 + max(1, min(sms * meta["SHARDS_PER_SM"], N // 4)), + ) + _weighted_rms_norm_bwd[grid]( + dx, + dy, + dweight, + x, + weight, + rstd, + dx.stride(0), + dy.stride(0), + x.stride(0), + D, + ctx.eps, + N=N, + SILU=ctx.silu, + BLOCK_D=ctx.BLOCK_D, + ) + + return dx, dweight, None, None + + +class SwishLayerNormFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore[14] + def forward( + ctx, + x: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + eps: float, + ) -> torch.Tensor: + assert x.dim() == 2, f"x.dim() == {x.dim()}, expected 2" + x = switch_to_contiguous_if_needed(x) + N, D = x.shape + + assert bias is not None and weight is not None + assert weight.dim() == 1 + assert bias.dim() == 1 + assert weight.numel() == D + assert bias.numel() == D + + y = torch.empty_like(x) + mean = torch.empty((N,), dtype=torch.float32, device=x.device) + rstd = torch.empty((N,), dtype=torch.float32, device=x.device) + + BLOCK_D = triton.next_power_of_2(D) + num_warps = min(max(BLOCK_D // 256, 1), 8) + + ctx.save_for_backward(x, weight, bias, mean, rstd) + ctx.BLOCK_D = BLOCK_D + ctx.num_warps = num_warps + ctx.eps = eps + if N == 0: + return y + + # pyre-ignore[28] + grid = lambda meta: (triton.cdiv(N, meta["BLOCK_N"]),) # noqa E731 + _weighted_layer_norm_fwd[grid]( + x, + y, + weight, + bias, + mean, + rstd, + N, + D, + eps, + x.stride(0), + y.stride(0), + IS_SWISH=True, + TRAINING=True, + BLOCK_D=BLOCK_D, + COMPUTE_MEAN_AND_RSTD=True, + ) + + return y + + @staticmethod + # pyre-ignore[14] + def backward( + ctx, dy: torch.Tensor + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor], None]: + x, weight, bias, mean, rstd = ctx.saved_tensors + N, D = x.shape + dx = torch.empty_like(x) + sms = torch.cuda.get_device_properties(x.device).multi_processor_count + tile_num = max(1, min(sms * 8, N // 4)) + _dweight = torch.empty((tile_num, D), dtype=torch.float32, device=x.device) + _dbias = torch.empty((tile_num, D), dtype=torch.float32, device=x.device) + dweight = torch.empty((D,), dtype=weight.dtype, device=x.device) + dbias = torch.empty((D,), dtype=weight.dtype, device=x.device) + if N == 0: + dweight.zero_() + dbias.zero_() + return dx, dweight, dbias, None + # pyre-ignore[28] + _weighted_layer_norm_bwd_dx[(tile_num,)]( + dx, + dy, + _dweight, + _dbias, + x, + weight, + bias, + mean, + rstd, + dx.stride(0), + dy.stride(0), + x.stride(0), + D, + ctx.eps, + IS_SWISH=True, + N=N, + BLOCK_D=ctx.BLOCK_D, + ) + + def grid(META): + return (triton.cdiv(D, META["BLOCK_D"]),) + + blocks = triton.next_power_of_2(sms * 4) + BLOCK_D = triton.next_power_of_2(triton.cdiv(D, blocks)) + BLOCK_D = min(max(BLOCK_D, 4), 128) + _layer_norm_bwd_dwdb[grid]( + _dweight, + _dbias, + dweight, + dbias, + tile_num, + D, + BLOCK_D=BLOCK_D, + ) + + return dx, dweight, dbias, None + + +@torch.jit.unused +@torch.fx.wrap +def triton_layer_norm( + x: torch.Tensor, + weight: Optional[torch.Tensor], + bias: Optional[torch.Tensor], + eps: float, +) -> torch.Tensor: + return LayerNormFunction.apply(x, weight, bias, eps) + + +@torch.jit.unused +@torch.fx.wrap +def triton_rms_norm( + x: torch.Tensor, + weight: Optional[torch.Tensor], + eps: float, + silu: bool = False, +) -> torch.Tensor: + return RMSNormFunction.apply(x, weight, eps, silu) + + +@torch.jit.unused +@torch.fx.wrap +def triton_swish_layer_norm( + x: torch.Tensor, + normalized_shape: List[int], + weight: Optional[torch.Tensor], + bias: Optional[torch.Tensor], + eps: float, +) -> torch.Tensor: + return SwishLayerNormFunction.apply(x, weight, bias, eps) diff --git a/recommendation_v4/generative_recommenders/ops/triton/triton_position.py b/recommendation_v4/generative_recommenders/ops/triton/triton_position.py new file mode 100644 index 000000000..72c43f9ac --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton/triton_position.py @@ -0,0 +1,438 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +#!/usr/bin/env python3 + + +from typing import List, Optional, Tuple + +import torch + +# @manual=//triton:triton +import triton + +# @manual=//triton:triton +import triton.language as tl + +try: + torch.ops.load_library("//hammer/ops/cuda:cuda_ops") +except OSError: + pass + +from generative_recommenders.common import ( + autotune_max_seq_len, + prev_power_of_2, + switch_to_contiguous_if_needed, + triton_autotune, +) + + +def _autotune_configs() -> List[triton.Config]: + configs = [] + for BLOCK_N in [16, 32, 64]: + for num_stages in [1, 2]: + for num_warps in [2, 4, 8]: + configs.append( + triton.Config( + { + "BLOCK_N": BLOCK_N, + }, + num_stages=num_stages, + num_warps=num_warps, + ) + ) + return configs + + +@triton_autotune( + configs=_autotune_configs(), + key=["AUTOTUNE_MAX_SEQ_LEN"], +) +@triton.jit +def _add_timestamp_position_embeddings_kernel( + SeqEmb, + Offsets, + Lengths, + PosEmb, + TsEmb, + Out, + TS, + PosInds, + TsInds, + NumTargets, + AUTOTUNE_MAX_SEQ_LEN, + D, + num_time_buckets, + time_bucket_increments, + time_bucket_scale, + time_delta, + max_contextual_seq_len, + max_pos_ind, + stride_sn, + stride_pn, + stride_tn, + stride_on, + TRAINING: tl.constexpr, + HAS_MULTIPLE_TARGETS: tl.constexpr, + INTERLEAVE_TARGETS: tl.constexpr, + TIME_BUCKET_FN: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_N: tl.constexpr, +): + """ + SeqEmb has shape (sum_B(N_i), D), + PosEmb has shape (N_p, D), + TsEmb has shape (N_t, D), + Out has shape (sum_B(N_i), D) + """ + + off_b = tl.program_id(0) + off_n = tl.program_id(1) + seq_start = tl.load(Offsets + off_b) + seq_end = tl.load(Offsets + off_b + 1) + seq_len = seq_end - seq_start + start_n = off_n * BLOCK_N + if start_n >= seq_len: + return + offs_n = start_n + tl.arange(0, BLOCK_N) + offs_d = tl.arange(0, BLOCK_D) + seq_emb_offsets = offs_n[:, None] * stride_sn + offs_d[None, :] + SeqEmb += seq_start.to(tl.int64) * stride_sn + mask_n = offs_n < seq_len + # position encoding + seq_len = tl.load(Lengths + off_b) + if HAS_MULTIPLE_TARGETS: + num_targets = tl.load(NumTargets + off_b) + if INTERLEAVE_TARGETS: + high_ind = seq_len - num_targets * 2 + else: + high_ind = seq_len - num_targets + else: + high_ind = seq_len + pos_inds = tl.where(offs_n < high_ind, offs_n, high_ind) + pos_inds = high_ind - pos_inds + max_contextual_seq_len + pos_inds = tl.where(pos_inds < max_pos_ind - 1, pos_inds, max_pos_ind - 1) + pos_inds = tl.where(offs_n < max_contextual_seq_len, offs_n, pos_inds) + if TRAINING: + tl.store(PosInds + seq_start + offs_n, pos_inds, mask=mask_n) + pos_emb_offsets = pos_inds[:, None] * stride_pn + offs_d[None, :] + # timestamp encoding + ts = tl.load(TS + seq_start + offs_n, mask=mask_n) + query_time = tl.load(TS + seq_end - 1) + ts = query_time - ts + time_delta + ts = tl.where(ts > 1e-6, ts, 1e-6) / time_bucket_increments + if TIME_BUCKET_FN == "log": + ts = tl.log(ts) + else: + ts = tl.sqrt(ts) + ts = ts * time_bucket_scale + ts = ts.to(tl.int32) + ts = tl.where(ts > 0, ts, 0) + ts = tl.where(ts < num_time_buckets, ts, num_time_buckets) + if TRAINING: + tl.store(TsInds + seq_start + offs_n, ts, mask=mask_n) + ts_emb_offsets = ts[:, None] * stride_tn + offs_d[None, :] + Out += seq_start.to(tl.int64) * stride_on + out_offsets = Out + offs_n[:, None] * stride_on + offs_d[None, :] + for _d in range(0, D, BLOCK_D): + mask = (offs_n[:, None] < seq_len) and offs_d[None, :] < D + seq_emb = tl.load(SeqEmb + seq_emb_offsets, mask=mask) + pos_emb = tl.load(PosEmb + pos_emb_offsets, mask=mask) + ts_emb = tl.load(TsEmb + ts_emb_offsets, mask=mask) + tl.store(out_offsets, seq_emb + (pos_emb + ts_emb).to(seq_emb.dtype), mask=mask) + seq_emb_offsets += BLOCK_D + pos_emb_offsets += BLOCK_D + ts_emb_offsets += BLOCK_D + out_offsets += BLOCK_D + offs_d += BLOCK_D + + +def bwd_pre_hook(nargs): + nargs["Out"].zero_() + + +def _add_embeddings_bwd_configs() -> List[triton.Config]: + configs = [] + for BLOCK in [32, 64, 128]: + for num_stages in [2, 3, 4]: + for num_warps in [2, 4, 8]: + configs.append( + triton.Config( + { + "BLOCK": BLOCK, + }, + num_stages=num_stages, + num_warps=num_warps, + pre_hook=bwd_pre_hook, + ) + ) + return configs + + +@triton_autotune( + configs=_add_embeddings_bwd_configs(), + key=["AUTOTUNE_MAX_SEQ_LEN", "AUTOTUNE_B", "D"], +) +@triton.jit +def _add_embeddings_bwd_kernel( + In, + KeyInds, + ValueInds, + Out, + AUTOTUNE_MAX_SEQ_LEN, + AUTOTUNE_B, + D, + jagged_size, + stride_in, + stride_on, + BLOCK_D: tl.constexpr, + BLOCK: tl.constexpr, +): + off_block = tl.program_id(0) + offs_d = tl.arange(0, BLOCK_D) + mask_d = offs_d < D + key_ind = -1 + key_ind = key_ind.to(KeyInds.dtype.element_ty) # pyre-ignore[16] + accumulator = tl.zeros((BLOCK_D,), dtype=In.dtype.element_ty) + for off_i in range(0, BLOCK): + off = off_block * BLOCK + off_i + if off < jagged_size: + value_ind = tl.load(ValueInds + off) + in_offset = In + value_ind.to(tl.int64) * stride_in + jagged_in = tl.load(in_offset + offs_d, mask=mask_d) + key_ind_new = tl.load(KeyInds + off) + if key_ind == key_ind_new: + accumulator += jagged_in + else: + if key_ind >= 0: + out_offset = Out + key_ind.to(tl.int64) * stride_on + tl.atomic_add( + out_offset + offs_d, + accumulator.to(Out.dtype.element_ty), + mask=mask_d, + sem="relaxed", + ) + key_ind = key_ind_new + accumulator = jagged_in + if key_ind >= 0: + out_offset = Out + key_ind.to(tl.int64) * stride_on + tl.atomic_add( + out_offset + offs_d, + accumulator.to(Out.dtype.element_ty), + mask=mask_d, + sem="relaxed", + ) + + +class _AddTimestampPositionEmbeddingsFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore[14] + def forward( + ctx, + seq_embeddings: torch.Tensor, + seq_offsets: torch.Tensor, + pos_embeddings: torch.Tensor, + ts_embeddings: torch.Tensor, + timestamps: torch.Tensor, + max_seq_len: int, + max_contextual_seq_len: int, + seq_lengths: torch.Tensor, + num_targets: Optional[torch.Tensor], + interleave_targets: bool, + time_bucket_fn: str, + ): + seq_embeddings = switch_to_contiguous_if_needed(seq_embeddings) + pos_embeddings = switch_to_contiguous_if_needed(pos_embeddings) + ts_embeddings = switch_to_contiguous_if_needed(ts_embeddings) + + max_pos_ind = pos_embeddings.shape[0] + B = seq_lengths.shape[0] + N, D = seq_embeddings.shape + assert len(pos_embeddings.shape) == 2 + assert len(ts_embeddings.shape) == 2 + assert pos_embeddings.shape[1] == D, ( + "shape[1] of pos_embeddings much match seq_embeddings" + ) + assert ts_embeddings.shape[1] == D, ( + "shape[1] of ts_embeddings much match seq_embeddings" + ) + out = torch.empty_like(seq_embeddings) + + timestamps = switch_to_contiguous_if_needed(timestamps) + ts_inds = torch.empty_like(seq_embeddings[:, 0], dtype=torch.int32) + pos_inds = torch.empty_like(seq_embeddings[:, 0], dtype=torch.int32) + ts_emb_size = ts_embeddings.shape[0] + + grid = lambda meta: ( # noqa E731 + B, + triton.cdiv(max_seq_len, meta["BLOCK_N"]), + ) + BLOCK_D = triton.next_power_of_2(D) if D < 64 else 64 + _add_timestamp_position_embeddings_kernel[grid]( + SeqEmb=seq_embeddings, + Offsets=seq_offsets, + Lengths=seq_lengths, + PosEmb=pos_embeddings, + TsEmb=ts_embeddings, + Out=out, + TS=timestamps, + PosInds=pos_inds, + TsInds=ts_inds, + NumTargets=num_targets, + AUTOTUNE_MAX_SEQ_LEN=autotune_max_seq_len(max_seq_len), + D=D, + num_time_buckets=ts_emb_size - 1, + time_bucket_increments=60.0, + time_bucket_scale=1.0, + time_delta=0, + max_contextual_seq_len=max_contextual_seq_len, + max_pos_ind=max_pos_ind, + stride_sn=seq_embeddings.stride(0), + stride_pn=pos_embeddings.stride(0), + stride_tn=ts_embeddings.stride(0), + stride_on=out.stride(0), + TRAINING=True, + HAS_MULTIPLE_TARGETS=num_targets is not None, + INTERLEAVE_TARGETS=interleave_targets, + TIME_BUCKET_FN=time_bucket_fn, + BLOCK_D=BLOCK_D, + ) + try: + values = torch.arange(0, N, dtype=torch.int32, device=timestamps.device) + sorted_ts_key_inds, sorted_ts_value_inds = torch.ops.hammer.sort_kv_pairs( + ts_inds, values + ) + sorted_pos_key_inds, sorted_pos_value_inds = torch.ops.hammer.sort_kv_pairs( + pos_inds, values + ) + except Exception: + sorted_ts_key_inds, sorted_ts_value_inds = torch.sort(ts_inds) + sorted_pos_key_inds, sorted_pos_value_inds = torch.sort(pos_inds) + ctx.save_for_backward( + sorted_pos_key_inds, + sorted_pos_value_inds, + sorted_ts_key_inds, + sorted_ts_value_inds, + ) + ctx.B = B + ctx.D = D + ctx.max_seq_len = max_seq_len + ctx.pos_emb_size = pos_embeddings.shape[0] + ctx.ts_emb_size = ts_emb_size + ctx.pos_dtype = pos_embeddings.dtype + ctx.ts_dtype = ts_embeddings.dtype + return out + + @staticmethod + # pyre-ignore[14] + def backward( + ctx, d_out: torch.Tensor + ) -> Tuple[ + torch.Tensor, + None, + torch.Tensor, + torch.Tensor, + None, + None, + None, + None, + None, + None, + None, + ]: + ( + sorted_pos_key_inds, + sorted_pos_value_inds, + sorted_ts_key_inds, + sorted_ts_value_inds, + ) = ctx.saved_tensors + d_pos_embeddings = torch.empty( + (ctx.pos_emb_size, ctx.D), device=d_out.device, dtype=torch.float32 + ) + d_ts_embeddings = torch.empty( + (ctx.ts_emb_size, ctx.D), device=d_out.device, dtype=torch.float32 + ) + grid = lambda meta: (triton.cdiv(d_out.shape[0], meta["BLOCK"]),) # noqa E731 + AUTOTUNE_B = prev_power_of_2(ctx.B) + _add_embeddings_bwd_kernel[grid]( + In=d_out, + KeyInds=sorted_pos_key_inds, + ValueInds=sorted_pos_value_inds, + Out=d_pos_embeddings, + AUTOTUNE_MAX_SEQ_LEN=autotune_max_seq_len(ctx.max_seq_len), + AUTOTUNE_B=AUTOTUNE_B, + D=ctx.D, + jagged_size=d_out.shape[0], + stride_in=d_out.stride(0), + stride_on=d_pos_embeddings.stride(0), + BLOCK_D=triton.next_power_of_2(ctx.D), + ) + _add_embeddings_bwd_kernel[grid]( + In=d_out, + KeyInds=sorted_ts_key_inds, + ValueInds=sorted_ts_value_inds, + Out=d_ts_embeddings, + AUTOTUNE_MAX_SEQ_LEN=autotune_max_seq_len(ctx.max_seq_len), + AUTOTUNE_B=AUTOTUNE_B, + D=ctx.D, + jagged_size=d_out.shape[0], + stride_in=d_out.stride(0), + stride_on=d_ts_embeddings.stride(0), + BLOCK_D=triton.next_power_of_2(ctx.D), + ) + return ( + d_out, + None, + d_pos_embeddings.to(ctx.pos_dtype), + d_ts_embeddings.to(ctx.ts_dtype), + None, + None, + None, + None, + None, + None, + None, + ) + + +@torch.jit.unused +@torch.fx.wrap +def triton_add_timestamp_positional_embeddings( + seq_embeddings: torch.Tensor, + seq_offsets: torch.Tensor, + pos_embeddings: torch.Tensor, + ts_embeddings: torch.Tensor, + timestamps: torch.Tensor, + max_seq_len: int, + max_contextual_seq_len: int, + seq_lengths: torch.Tensor, + num_targets: Optional[torch.Tensor], + interleave_targets: bool, + time_bucket_fn: str, +) -> torch.Tensor: + return _AddTimestampPositionEmbeddingsFunction.apply( + seq_embeddings, + seq_offsets, + pos_embeddings, + ts_embeddings, + timestamps, + max_seq_len, + max_contextual_seq_len, + seq_lengths, + num_targets, + interleave_targets, + time_bucket_fn, + ) diff --git a/recommendation_v4/generative_recommenders/ops/triton/triton_swiglu.py b/recommendation_v4/generative_recommenders/ops/triton/triton_swiglu.py new file mode 100644 index 000000000..5f30da53b --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton/triton_swiglu.py @@ -0,0 +1,753 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-unsafe + +from typing import List + +import torch + +# @manual=//triton:triton +import triton + +# @manual=//triton:triton +import triton.language as tl +from generative_recommenders.common import triton_autotune +from generative_recommenders.ops.utils import is_sm100_plus + +TMA_AVAILABLE = False +try: + # @manual=//triton:triton + from triton.tools.tensor_descriptor import TensorDescriptor + + TMA_AVAILABLE = True +except ImportError: + pass + +HAS_TLX = False +try: + # @manual=//triton:triton + import triton.language.extra.tlx as tlx # type: ignore + + HAS_TLX = True +except ImportError: + pass + + +def is_blackwell_triton_swiglu_supported() -> bool: + return is_sm100_plus() and TMA_AVAILABLE and HAS_TLX + + +def _swiglu_tma_set_block_size_hook(nargs) -> None: + BLOCK_M = nargs["BLOCK_M"] + BLOCK_N = nargs["BLOCK_N"] + BLOCK_K = nargs["BLOCK_K"] + EPILOGUE_SUBTILE = nargs.get("EPILOGUE_SUBTILE", 1) + + nargs["x_desc"].block_shape = [BLOCK_M, BLOCK_K] + nargs["w_gate_desc"].block_shape = [BLOCK_N, BLOCK_K] + nargs["w_up_desc"].block_shape = [BLOCK_N, BLOCK_K] + nargs["out_desc"].block_shape = [BLOCK_M, BLOCK_N // EPILOGUE_SUBTILE] + + +def get_swiglu_configs(pre_hook) -> List[triton.Config]: + return [ + triton.Config( + { + "BLOCK_M": 128, + "BLOCK_N": 128, + "BLOCK_K": 64, + "GROUP_M": 8, + }, + num_stages=1, + num_warps=4, + pre_hook=pre_hook, + ), + triton.Config( + { + "BLOCK_M": 64, + "BLOCK_N": 128, + "BLOCK_K": 64, + "GROUP_M": 8, + }, + num_stages=1, + num_warps=4, + pre_hook=pre_hook, + ), + triton.Config( + { + "BLOCK_M": 128, + "BLOCK_N": 64, + "BLOCK_K": 64, + "GROUP_M": 8, + }, + num_stages=1, + num_warps=4, + pre_hook=pre_hook, + ), + ] + + +@triton.jit +def _compute_pid_swiglu( + tile_id, + num_pid_in_group, + num_pid_m, + GROUP_M: tl.constexpr, + NUM_SMS: tl.constexpr, +): + group_id = tile_id // num_pid_in_group + first_pid_m = group_id * GROUP_M + group_size_m = min(num_pid_m - first_pid_m, GROUP_M) + pid_m = first_pid_m + (tile_id % group_size_m) + pid_n = (tile_id % num_pid_in_group) // group_size_m + return pid_m, pid_n + + +@triton_autotune( + configs=get_swiglu_configs(pre_hook=_swiglu_tma_set_block_size_hook), + key=["M_BLOCK", "N", "K"], +) +@triton.jit +def _swiglu_fwd_tma_ws_persistent( + x_desc, + w_gate_desc, + w_up_desc, + out_desc, + M, + N, + K, + M_BLOCK, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, + GROUP_M: tl.constexpr, + NUM_SMEM_BUFFERS: tl.constexpr, + NUM_TMEM_BUFFERS: tl.constexpr, + NUM_SMS: tl.constexpr, + EPILOGUE_SUBTILE: tl.constexpr, +): + # Allocate SMEM buffers + x_buffers = tlx.local_alloc((BLOCK_M, BLOCK_K), x_desc.dtype, NUM_SMEM_BUFFERS) + + # Allocate SMEM buffers for W_gate and W_up + w_gate_buffers = tlx.local_alloc( + (BLOCK_N, BLOCK_K), w_gate_desc.dtype, NUM_SMEM_BUFFERS + ) + w_up_buffers = tlx.local_alloc( + (BLOCK_N, BLOCK_K), w_up_desc.dtype, NUM_SMEM_BUFFERS + ) + + # Allocate TMEM for accumulators + tmem_gate_buffers = tlx.local_alloc( + (BLOCK_M, BLOCK_N), tl.float32, NUM_TMEM_BUFFERS, tlx.storage_kind.tmem + ) + tmem_up_buffers = tlx.local_alloc( + (BLOCK_M, BLOCK_N), tl.float32, NUM_TMEM_BUFFERS, tlx.storage_kind.tmem + ) + + # Barriers for Producer <-> MMA synchronization + smem_full_bars_x_gate = tlx.alloc_barriers( + num_barriers=NUM_SMEM_BUFFERS, + arrive_count=1, # pyre-ignore[6] + ) + smem_full_bars_up = tlx.alloc_barriers( + num_barriers=NUM_SMEM_BUFFERS, + arrive_count=1, # pyre-ignore[6] + ) + # Empty barriers: arrive_count=2 because both GEMM1 and GEMM2 signal completion + smem_empty_bars = tlx.alloc_barriers( + num_barriers=NUM_SMEM_BUFFERS, + arrive_count=2, # pyre-ignore[6] + ) + + # Barriers for MMA <-> Epilogue synchronization + # pyre-ignore[6] + tmem_full_bars = tlx.alloc_barriers(num_barriers=NUM_TMEM_BUFFERS, arrive_count=1) + # pyre-ignore[6] + tmem_empty_bars = tlx.alloc_barriers(num_barriers=NUM_TMEM_BUFFERS, arrive_count=1) + + with tlx.async_tasks(): + # Epilogue Consumer: Reads from TMEM, applies SwiGLU, and stores to output + with tlx.async_task("default"): + start_pid = tl.program_id(axis=0) + num_pid_m = tl.cdiv(M, BLOCK_M) + num_pid_n = tl.cdiv(N, BLOCK_N) + num_pid_in_group = GROUP_M * num_pid_n + num_tiles = num_pid_m * num_pid_n + + # Initialize buffer tracking + processed_tiles = 0 + + for tile_id in range(start_pid, num_tiles, NUM_SMS): + pid_m, pid_n = _compute_pid_swiglu( + tile_id, num_pid_in_group, num_pid_m, GROUP_M, NUM_SMS + ) + offs_m = pid_m * BLOCK_M + offs_n = pid_n * BLOCK_N + + cur_tmem_buf = processed_tiles % int(NUM_TMEM_BUFFERS) + tmem_read_phase = (processed_tiles // int(NUM_TMEM_BUFFERS)) & 1 + + # Wait for MMA to finish writing to TMEM + # pyre-ignore[16] + tlx.barrier_wait(tmem_full_bars[cur_tmem_buf], tmem_read_phase) + + # Load gate and up results from TMEM + # pyre-ignore[16] + gate_tmem = tmem_gate_buffers[cur_tmem_buf] + up_tmem = tmem_up_buffers[cur_tmem_buf] + + if EPILOGUE_SUBTILE > 1: + # Process tile in subtiles + slice_size: tl.constexpr = BLOCK_N // EPILOGUE_SUBTILE + for slice_id in tl.static_range(EPILOGUE_SUBTILE): + gate_subslice = tlx.local_slice( + gate_tmem, + [0, slice_id * slice_size], + # pyre-ignore[6] + [BLOCK_M, slice_size], + ) + up_subslice = tlx.local_slice( + up_tmem, + [0, slice_id * slice_size], + # pyre-ignore[6] + [BLOCK_M, slice_size], + ) + + gate = tlx.local_load(gate_subslice).to(out_desc.dtype) + up = tlx.local_load(up_subslice).to(out_desc.dtype) + + gate_fp32 = gate.to(tl.float32) + silu_gate = (gate_fp32 * tl.sigmoid(gate_fp32)).to( + out_desc.dtype + ) + result = silu_gate * up + + out_desc.store([offs_m, offs_n + slice_id * slice_size], result) + else: + # Process full tile + gate = tlx.local_load(gate_tmem).to(out_desc.dtype) + up = tlx.local_load(up_tmem).to(out_desc.dtype) + + gate_fp32 = gate.to(tl.float32) + silu_gate = (gate_fp32 * tl.sigmoid(gate_fp32)).to(out_desc.dtype) + result = silu_gate * up + + out_desc.store([offs_m, offs_n], result) + + # Signal MMA that TMEM buffer is free + # pyre-ignore[6] + tlx.barrier_arrive(tmem_empty_bars[cur_tmem_buf], 1) + + processed_tiles += 1 + + # MMA Consumer: Computes both GEMMs: gate = X @ W_gate, up = X @ W_up + with tlx.async_task(num_warps=4, num_regs=232): + start_pid = tl.program_id(axis=0) + num_pid_m = tl.cdiv(M, BLOCK_M) + num_pid_n = tl.cdiv(N, BLOCK_N) + num_pid_in_group = GROUP_M * num_pid_n + num_tiles = num_pid_m * num_pid_n + k_tiles = tl.cdiv(K, BLOCK_K) + + processed_k_iters = 0 + processed_tiles = 0 + + for tile_id in range(start_pid, num_tiles, NUM_SMS): + pid_m, pid_n = _compute_pid_swiglu( + tile_id, num_pid_in_group, num_pid_m, GROUP_M, NUM_SMS + ) + + cur_tmem_buf = processed_tiles % int(NUM_TMEM_BUFFERS) + tmem_write_phase = (processed_tiles // int(NUM_TMEM_BUFFERS)) & 1 + + # Wait for epilogue to finish + tlx.barrier_wait(tmem_empty_bars[cur_tmem_buf], tmem_write_phase ^ 1) + + # Perform K-dimension reduction for both GEMMs + for k in range(0, k_tiles): + buf = (processed_k_iters + k) % int(NUM_SMEM_BUFFERS) + + total_iters = processed_k_iters + k + dot_phase = (total_iters // int(NUM_SMEM_BUFFERS)) & 1 + + # Wait for x and w_gate to be loaded, then start GEMM1 + tlx.barrier_wait(smem_full_bars_x_gate[buf], dot_phase) + + # Transpose weight buffer for MMA + w_gate_trans = tlx.local_trans(w_gate_buffers[buf]) + + # GEMM 1: gate = X @ W_gate.T + tlx.async_dot( + x_buffers[buf], + w_gate_trans, + tmem_gate_buffers[cur_tmem_buf], + # pyre-ignore[6] + use_acc=(k > 0), + mBarriers=[smem_empty_bars[buf]], + out_dtype=tl.float32, + ) + + # Wait for w_up to be loaded before starting GEMM2 + tlx.barrier_wait(smem_full_bars_up[buf], dot_phase) + + w_up_trans = tlx.local_trans(w_up_buffers[buf]) + + # GEMM 2: up = X @ W_up.T + tlx.async_dot( + x_buffers[buf], + w_up_trans, + tmem_up_buffers[cur_tmem_buf], + # pyre-ignore[6] + use_acc=(k > 0), + mBarriers=[smem_empty_bars[buf]], + out_dtype=tl.float32, + ) + + # Wait for last MMA to complete + last_buf = (processed_k_iters + k_tiles - 1) % int(NUM_SMEM_BUFFERS) + last_total_iters = processed_k_iters + k_tiles - 1 + last_dot_phase = (last_total_iters // int(NUM_SMEM_BUFFERS)) & 1 + tlx.barrier_wait(smem_empty_bars[last_buf], last_dot_phase) + + # Signal epilogue that results are ready + # pyre-ignore[6] + tlx.barrier_arrive(tmem_full_bars[cur_tmem_buf], 1) + + processed_tiles += 1 + processed_k_iters += k_tiles + + # Producer: TMA loads for X, W_gate, W_up + with tlx.async_task(num_warps=1, num_regs=24): + start_pid = tl.program_id(axis=0) + num_pid_m = tl.cdiv(M, BLOCK_M) + num_pid_n = tl.cdiv(N, BLOCK_N) + num_pid_in_group = GROUP_M * num_pid_n + num_tiles = num_pid_m * num_pid_n + k_tiles = tl.cdiv(K, BLOCK_K) + + # Initialize phase tracking + processed_k_iters = 0 + + for tile_id in range(start_pid, num_tiles, NUM_SMS): + pid_m, pid_n = _compute_pid_swiglu( + tile_id, num_pid_in_group, num_pid_m, GROUP_M, NUM_SMS + ) + offs_m = pid_m * BLOCK_M + offs_n = pid_n * BLOCK_N + + for k in range(0, k_tiles): + buf = (processed_k_iters + k) % int(NUM_SMEM_BUFFERS) + + total_iters = processed_k_iters + k + load_phase = (total_iters // int(NUM_SMEM_BUFFERS)) & 1 + + # Wait for buffer to be free + tlx.barrier_wait(smem_empty_bars[buf], load_phase ^ 1) + + offs_k = k * BLOCK_K + + # Set expected bytes for x+w_gate barrier + tlx.barrier_expect_bytes( + smem_full_bars_x_gate[buf], + # pyre-ignore[6] + 2 * (BLOCK_M * BLOCK_K + BLOCK_K * BLOCK_N), + ) + + # Set expected bytes for w_up barrier + tlx.barrier_expect_bytes( + smem_full_bars_up[buf], + # pyre-ignore[6] + 2 * (BLOCK_K * BLOCK_N), + ) + + # Load x and w_gate first, signal smem_full_bars_x_gate + tlx.async_descriptor_load( + x_desc, + x_buffers[buf], + [offs_m, offs_k], + smem_full_bars_x_gate[buf], + ) + + # Weights are in [N, K] layout, load with [offs_n, offs_k] + tlx.async_descriptor_load( + w_gate_desc, + w_gate_buffers[buf], + [offs_n, offs_k], + smem_full_bars_x_gate[buf], + ) + + # Load w_up separately, signal smem_full_bars_up + tlx.async_descriptor_load( + w_up_desc, + w_up_buffers[buf], + [offs_n, offs_k], + smem_full_bars_up[buf], + ) + + processed_k_iters += k_tiles + + +@torch.fx.wrap +def triton_swiglu_fwd_tma_ws_persistent_tlx( + x: torch.Tensor, + w_gate: torch.Tensor, + w_up: torch.Tensor, +) -> torch.Tensor: + M, K = x.shape + N, K_gate = w_gate.shape + N_up, K_up = w_up.shape + + # Only bf16/fp16 supported by the kernel + supported_dtypes = (torch.bfloat16, torch.float16) + assert x.dtype in supported_dtypes, ( + f"x.dtype must be bfloat16 or float16, got {x.dtype}" + ) + assert w_gate.dtype in supported_dtypes, ( + f"w_gate.dtype must be bfloat16 or float16, got {w_gate.dtype}" + ) + assert w_up.dtype in supported_dtypes, ( + f"w_up.dtype must be bfloat16 or float16, got {w_up.dtype}" + ) + + assert K == K_gate, f"Incompatible dimensions: x.K={K}, w_gate.K={K_gate}" + assert K == K_up, f"Incompatible dimensions: x.K={K}, w_up.K={K_up}" + assert N == N_up, f"Incompatible dimensions: w_gate.N={N}, w_up.N={N_up}" + + # Allocate output + out = torch.empty((M, N), device=x.device, dtype=x.dtype) + if M == 0 or N == 0: + return out + + M_BLOCK = triton.next_power_of_2(M) + + NUM_SMS = torch.cuda.get_device_properties("cuda").multi_processor_count + + # A dummy block value that will be overwritten by the hook + dummy_block = [1, 1] + + # pyre-ignore[6] + x_desc = TensorDescriptor(x, x.shape, x.stride(), dummy_block) + # pyre-ignore[6] + w_gate_desc = TensorDescriptor(w_gate, w_gate.shape, w_gate.stride(), dummy_block) + # pyre-ignore[6] + w_up_desc = TensorDescriptor(w_up, w_up.shape, w_up.stride(), dummy_block) + # pyre-ignore[6] + out_desc = TensorDescriptor(out, out.shape, out.stride(), dummy_block) + + def grid(meta): + BLOCK_M = meta["BLOCK_M"] + BLOCK_N = meta["BLOCK_N"] + return ( + min( + NUM_SMS, + triton.cdiv(M, BLOCK_M) * triton.cdiv(N, BLOCK_N), + ), + ) + + _swiglu_fwd_tma_ws_persistent[grid]( + x_desc, + w_gate_desc, + w_up_desc, + out_desc, + M, + N, + K, + M_BLOCK, + NUM_SMS=NUM_SMS, + NUM_SMEM_BUFFERS=4, + NUM_TMEM_BUFFERS=2, + EPILOGUE_SUBTILE=2, + ) + return out + + +# ============================================================================= +# Standard fused SwiGLU kernel for A100/H100 (non-TLX path). +# +# Fuses silu(x @ W_gate^T) * (x @ W_up^T) into a single kernel launch. +# Uses standard Triton pointer arithmetic (no TMA), works on SM80+. +# +# Key optimization: x is loaded from HBM ONCE and reused for both GEMMs. +# Activation (silu * up) is computed in float32 registers, no HBM round-trip. +# +# Weight layout: expects [N, K] (nn.Linear native format). +# The wrapper transposes to [K, N] for the GEMM internally. +# ============================================================================= + + +def _get_swiglu_fwd_configs() -> List[triton.Config]: + """ + Autotune configs for the standard (non-TLX) fused SwiGLU kernel. + + Two float32 accumulators (gate + up) double register pressure vs single + GEMM, so smaller block sizes are included. + """ + configs = [ + triton.Config( + {"BLOCK_M": 64, "BLOCK_N": 64, "BLOCK_K": 32, "GROUP_M": 8}, + num_stages=3, + num_warps=4, + ), + triton.Config( + {"BLOCK_M": 64, "BLOCK_N": 64, "BLOCK_K": 64, "GROUP_M": 8}, + num_stages=4, + num_warps=4, + ), + triton.Config( + {"BLOCK_M": 128, "BLOCK_N": 64, "BLOCK_K": 32, "GROUP_M": 8}, + num_stages=3, + num_warps=4, + ), + triton.Config( + {"BLOCK_M": 128, "BLOCK_N": 64, "BLOCK_K": 32, "GROUP_M": 8}, + num_stages=3, + num_warps=8, + ), + triton.Config( + {"BLOCK_M": 64, "BLOCK_N": 128, "BLOCK_K": 32, "GROUP_M": 8}, + num_stages=3, + num_warps=8, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 64, "BLOCK_K": 32, "GROUP_M": 8}, + num_stages=4, + num_warps=4, + ), + triton.Config( + {"BLOCK_M": 64, "BLOCK_N": 32, "BLOCK_K": 32, "GROUP_M": 8}, + num_stages=4, + num_warps=4, + ), + triton.Config( + {"BLOCK_M": 64, "BLOCK_N": 64, "BLOCK_K": 64, "GROUP_M": 8}, + num_stages=3, + num_warps=8, + ), + ] + if torch.version.hip: + hip_num_stages = 2 if triton.__version__ >= "3.2.0" else 0 + configs.extend( + [ + triton.Config( + {"BLOCK_M": 128, "BLOCK_N": 128, "BLOCK_K": 32, "GROUP_M": 8}, + num_stages=hip_num_stages, + num_warps=4, + ), + triton.Config( + {"BLOCK_M": 64, "BLOCK_N": 128, "BLOCK_K": 32, "GROUP_M": 8}, + num_stages=hip_num_stages, + num_warps=4, + ), + triton.Config( + {"BLOCK_M": 128, "BLOCK_N": 64, "BLOCK_K": 32, "GROUP_M": 8}, + num_stages=hip_num_stages, + num_warps=4, + ), + triton.Config( + {"BLOCK_M": 64, "BLOCK_N": 64, "BLOCK_K": 32, "GROUP_M": 8}, + num_stages=hip_num_stages, + num_warps=4, + ), + ] + ) + return configs + + +@triton_autotune( + configs=_get_swiglu_fwd_configs(), + key=["M_BLOCK", "N", "K"], +) +@triton.jit +def _swiglu_fwd_kernel( + # Pointers to input/output tensors + x_ptr, # [M, K] input activation + w_gate_ptr, # [K, N] gate weight (already transposed from [N, K]) + w_up_ptr, # [K, N] up weight (already transposed from [N, K]) + out_ptr, # [M, N] output = silu(x @ w_gate) * (x @ w_up) + # Matrix dimensions + M, # rows in x (batch_size * seq_len) + N, # output dimension (hidden_dim) + K, # input/reduction dimension (input_dim) + M_BLOCK, # next_power_of_2(M) for stable autotuning + # Strides + stride_xm, + stride_xk, + stride_wgk, + stride_wgn, + stride_wuk, + stride_wun, + stride_om, + stride_on, + # Compile-time constants + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, + GROUP_M: tl.constexpr, + ALLOW_TF32: tl.constexpr, +): + """ + Fused SwiGLU forward: out = silu(x @ W_gate) * (x @ W_up). + + Each thread block computes one [BLOCK_M, BLOCK_N] output tile. + Two accumulators share the same x tile loads (the fusion benefit). + """ + # -- Step 1: Compute tile coordinates with grouped ordering (L2 reuse) -- + pid = tl.program_id(axis=0) + num_pid_m = tl.cdiv(M, BLOCK_M) + num_pid_n = tl.cdiv(N, BLOCK_N) + num_pid_in_group = GROUP_M * num_pid_n + group_id = pid // num_pid_in_group + first_pid_m = group_id * GROUP_M + + group_size_m = min(num_pid_m - first_pid_m, GROUP_M) + pid_in_group = pid % num_pid_in_group + pid_m = first_pid_m + (pid_in_group % group_size_m) + pid_n = pid_in_group // group_size_m + + # -- Step 2: Set up pointers for x, w_gate, w_up tiles -- + offs_m = tl.arange(0, BLOCK_M) + offs_n = tl.arange(0, BLOCK_N) + offs_k = tl.arange(0, BLOCK_K) + mask_m = (pid_m * BLOCK_M + offs_m)[:, None] < M + mask_n = (pid_n * BLOCK_N + offs_n)[None, :] < N + # [BLOCK_M, BLOCK_K] + x_ptrs = ( + x_ptr + + (pid_m.to(tl.int64) * BLOCK_M + offs_m)[:, None] * stride_xm + + offs_k[None, :] * stride_xk + ) + # [BLOCK_K, BLOCK_N] + wg_ptrs = ( + w_gate_ptr + + offs_k[:, None] * stride_wgk + + (pid_n.to(tl.int64) * BLOCK_N + offs_n)[None, :] * stride_wgn + ) + + # [BLOCK_K, BLOCK_N] + wu_ptrs = ( + w_up_ptr + + offs_k[:, None] * stride_wuk + + (pid_n.to(tl.int64) * BLOCK_N + offs_n)[None, :] * stride_wun + ) + + # -- Step 3: K-loop - two GEMMs sharing the same x tile -- + acc_gate = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32) + acc_up = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32) + + for k in range(0, tl.cdiv(K, BLOCK_K)): + mask_k = offs_k[None, :] < K - k * BLOCK_K + x = tl.load(x_ptrs, mask=mask_m & mask_k, other=0.0) + mask_k = offs_k[:, None] < K - k * BLOCK_K + wg = tl.load(wg_ptrs, mask=mask_k & mask_n, other=0.0) + wu = tl.load(wu_ptrs, mask=mask_k & mask_n, other=0.0) + + acc_gate += tl.dot(x, wg, allow_tf32=ALLOW_TF32) + acc_up += tl.dot(x, wu, allow_tf32=ALLOW_TF32) + + x_ptrs += BLOCK_K * stride_xk + wg_ptrs += BLOCK_K * stride_wgk + wu_ptrs += BLOCK_K * stride_wuk + + # -- Step 4: Apply SwiGLU activation in registers (no HBM round-trip) -- + gate_activated = acc_gate * tl.sigmoid(acc_gate) # silu + result = (gate_activated * acc_up).to(out_ptr.dtype.element_ty) + + # -- Step 5: Store result -- + offs_m = pid_m * BLOCK_M + offs_m + offs_n = pid_n * BLOCK_N + offs_n + out_ptrs = out_ptr + offs_m[:, None] * stride_om + offs_n[None, :] * stride_on + tl.store(out_ptrs, result, mask=mask_m & mask_n) + + +def triton_swiglu_fwd( + x: torch.Tensor, + w_gate: torch.Tensor, + w_up: torch.Tensor, +) -> torch.Tensor: + """ + Forward pass of fused SwiGLU (non-TLX path). Works on A100/H100/MI300X. + + Computes: silu(x @ w_gate^T) * (x @ w_up^T) + + Args: + x: [M, K] input tensor + w_gate: [N, K] gate weight (nn.Linear format) + w_up: [N, K] up weight (nn.Linear format) + + Returns: + [M, N] output tensor + """ + M, K = x.shape + N, K_gate = w_gate.shape + N_up, K_up = w_up.shape + assert K == K_gate, f"x.K={K} != w_gate.K={K_gate}" + assert K == K_up, f"x.K={K} != w_up.K={K_up}" + assert N == N_up, f"w_gate.N={N} != w_up.N={N_up}" + + out = torch.empty((M, N), device=x.device, dtype=x.dtype) + if M == 0 or N == 0: + return out + + M_BLOCK = triton.next_power_of_2(M) + + # Transpose weights from [N, K] to [K, N] for the GEMM kernel + w_gate_t = w_gate.t().contiguous() + w_up_t = w_up.t().contiguous() + + grid = lambda meta: ( # noqa E731 + triton.cdiv(M, meta["BLOCK_M"]) * triton.cdiv(N, meta["BLOCK_N"]), + ) + + _swiglu_fwd_kernel[grid]( + x, + w_gate_t, + w_up_t, + out, + M, + N, + K, + M_BLOCK, + x.stride(0), + x.stride(1), + w_gate_t.stride(0), + w_gate_t.stride(1), + w_up_t.stride(0), + w_up_t.stride(1), + out.stride(0), + out.stride(1), + ALLOW_TF32=torch.backends.cuda.matmul.allow_tf32, + ) + return out + + +def triton_swiglu( + x: torch.Tensor, + w_gate: torch.Tensor, + w_up: torch.Tensor, +) -> torch.Tensor: + if is_sm100_plus() and TMA_AVAILABLE and HAS_TLX: + # Blackwell: use the fast TLX persistent kernel with TMA + _, K = x.shape + N, _ = w_gate.shape + assert K % 16 == 0 and N % 16 == 0, ( + f"K ({K}) and N ({N}) must be divisible by 16 for TMA alignment" + ) + return triton_swiglu_fwd_tma_ws_persistent_tlx(x, w_gate, w_up) + else: + # A100/H100: use the standard fused kernel + return triton_swiglu_fwd(x, w_gate, w_up) diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/README.md b/recommendation_v4/generative_recommenders/ops/triton_aot/README.md new file mode 100644 index 000000000..2b0b1a834 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton_aot/README.md @@ -0,0 +1,54 @@ +# Local Triton AOT Support + +This package is a minimal local copy of the Triton AOT pieces needed by the +DLRM v3 HSTU inference end-to-end test. It avoids depending on the standalone +`fbcode/triton_aot` package while preserving the compile, transform, and +runtime-loading flow used by `generative_recommenders`. + +This is not intended to be a full fork of `fbcode/triton_aot`. Keep changes +scoped to the GR inference use case unless a broader migration plan exists. + +## Code Structure + +- `types.py`: local `TritonAOT` registration object and `triton_aot` helper used + by GR AOT wrapper modules. +- `preprocess.py`: FX graph preprocessing helpers, including wrapper-node + unwrapping before compile/transform. +- `triton_*.py`: GR kernel-specific AOT wrapper modules for addmm, jagged + concat/split, layer norm variants, HSTU attention, and timestamp position + embeddings. +- `compile/`: compile-time state, Triton signature/spec processing, generated + C++ codegen, and the `TritonAOTCompile` context manager. +- `transform/`: FX graph transformation and generated Python wrapper code that + swaps Python AOT wrappers for `torch.ops.triton_aot.*` calls backed by built + shared libraries. +- `build/`: extension builders and CUBIN embedding utilities used to create + loadable kernel libraries from compiled Triton artifacts. +- `templates/`: C++ template files used by the compile/codegen path for kernel + entry points, embedded CUBIN data, and Torch operator registration. +- `shared/`: compatibility helpers and type/spec conversion utilities shared by + compile and transform code. + +## Runtime Flow + +1. GR `triton_*.py` wrappers expose Triton kernels through local `triton_aot` + descriptors. +2. `TritonAOTCompile` runs representative CUDA inputs, records kernel specs, and + compiles the collected Triton kernels into shared libraries. +3. `transform_kernels` rewrites the FX graph so wrapper calls dispatch through + `torch.ops.triton_aot.*`. +4. The e2e test copies the generated libraries into its workdir and passes them + to the C++ runner before executing the scripted sparse/dense modules. + + +## Authors + +- Chang Pan +- Zhiyong Wang (MRS) +- Chenzhi Yu +- Runming Lu +- Chun-Wei Chen +- Michael He +- Linjian Ma +- Xing Liu +- Zhuoran Zhao diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/compile/arg_descriptor.py b/recommendation_v4/generative_recommenders/ops/triton_aot/compile/arg_descriptor.py new file mode 100644 index 000000000..bc5963674 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton_aot/compile/arg_descriptor.py @@ -0,0 +1,146 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. + +# pyre-strict + +"""ArgDescriptor — per-arg codegen descriptor for AOT-T. + +Centralises arg classification (pointer / scalar / constant) so every +``gen_*`` function in ``codegen.py`` iterates descriptors instead of +doing its own dict lookups into ``OpsUnit`` fields. + +Also provides type-mapping helpers that convert ``ArgDescriptor`` +metadata into context-specific C++ / TorchScript type strings. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +from generative_recommenders.ops.triton_aot.compile.spec_processing import OpsUnit +from triton.runtime.jit import JITFunction + + +# --------------------------------------------------------------------------- +# Type-mapping helpers +# --------------------------------------------------------------------------- + +CONSTANT_SELECTOR_CTYPE: dict[type[Any], str] = { + bool: "bool", + int: "int", + str: "const std::string&", +} + +CONSTANT_CPP_OP_CTYPE: dict[type[Any], str] = { + bool: "bool", + int: "int64_t", + str: "const std::string&", +} + +CONSTANT_TORCH_SCHEMA: dict[type[Any], str] = { + bool: "bool", + int: "int", + str: "str", +} + + +def scalar_cpp_op_ctype(triton_dtype: str) -> str: + """Triton scalar dtype → widened C++ type for cpp_op / torch_op params.""" + if triton_dtype.startswith("i"): + return "int64_t" + if triton_dtype.startswith("f"): + return "double" + if triton_dtype == "bool": + return "bool" + raise ValueError(f"Unsupported scalar dtype for cpp_op: {triton_dtype}") + + +def scalar_torch_schema(triton_dtype: str) -> str: + """Triton scalar dtype → TorchScript schema type string.""" + if triton_dtype.startswith("i"): + return "int" + if triton_dtype.startswith("f"): + return "float" + if triton_dtype == "bool": + return "bool" + raise ValueError(f"Unsupported scalar dtype for torch schema: {triton_dtype}") + + +# --------------------------------------------------------------------------- +# ArgDescriptor hierarchy +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class ArgDescriptor: + """Base class for per-arg codegen descriptors. + + Built once by ``build_arg_descriptors`` and consumed by all ``gen_*`` + functions. Use ``isinstance`` to dispatch on arg kind: + + - ``PointerArg`` — tensor pointer (required or optional) + - ``ScalarArg`` — non-pointer signature arg with a Triton dtype + - ``ConstantArg`` — compile-time constant with a Python type + """ + + name: str + index: int + + +@dataclass(frozen=True) +class PointerArg(ArgDescriptor): + """Tensor pointer arg (required or optional).""" + + is_optional: bool + + +@dataclass(frozen=True) +class ScalarArg(ArgDescriptor): + """Non-pointer signature arg with a Triton dtype (e.g., ``"i32"``, ``"fp32"``). + + ``triton_dtype`` is the **widest** type across all specs for this + position, computed by ``_compute_invariants`` via ``_wider_type``. + Individual specs may use a narrower type (e.g., ``"i32"`` when + ``triton_dtype`` is ``"i64"``); codegen adds ``fits_i32`` guards + and ``static_cast`` for narrowing. + """ + + triton_dtype: str + + +@dataclass(frozen=True) +class ConstantArg(ArgDescriptor): + """Compile-time constant arg with a Python type (``int``, ``str``, ``bool``).""" + + python_type: type[Any] + + +def build_arg_descriptors( + func: JITFunction[list[Any]], + unit: OpsUnit, +) -> list[ArgDescriptor]: + """Build ordered arg descriptors from func arg names + OpsUnit invariants. + + Single source of truth for arg classification. Called once in + ``compile_to_cpp`` and passed to all downstream codegen functions. + """ + result: list[ArgDescriptor] = [] + for i, name in enumerate(func.arg_names): + if i in unit.pointer_args: + result.append( + PointerArg(name=name, index=i, is_optional=i in unit.optional) + ) + elif i in unit.scalar_dtypes: + result.append( + ScalarArg(name=name, index=i, triton_dtype=unit.scalar_dtypes[i]) + ) + elif i in unit.constant_types: + result.append( + ConstantArg(name=name, index=i, python_type=unit.constant_types[i]) + ) + else: + raise ValueError( + f"Arg {name} (index {i}) not classified as pointer, scalar, " + f"or constant in OpsUnit" + ) + return result diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/compile/codegen.py b/recommendation_v4/generative_recommenders/ops/triton_aot/compile/codegen.py new file mode 100644 index 000000000..f1b03848f --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton_aot/compile/codegen.py @@ -0,0 +1,780 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. + +# pyre-strict + +"""C++ and Python code generation for AOT-T compiled kernels. + +Generates: + - kernel.h (header with gridDims, tuner meta, selector proto) + - kernel.cpp (cubin externs, loaders, launchers, selector) + - _torch_op.cpp (torch op registration) + - _meta.py (Python autotuner meta function) +""" + +import textwrap +from collections import Counter +from typing import Any + +# @manual=//triton:triton +import triton +from generative_recommenders.ops.triton_aot.compile.arg_descriptor import ( + ArgDescriptor, + CONSTANT_CPP_OP_CTYPE, + CONSTANT_SELECTOR_CTYPE, + CONSTANT_TORCH_SCHEMA, + ConstantArg, + PointerArg, + scalar_cpp_op_ctype, + scalar_torch_schema, + ScalarArg, +) +from generative_recommenders.ops.triton_aot.compile.spec_processing import ( + KernelSpec, + OpsUnit, +) +from generative_recommenders.ops.triton_aot.compile.stable_types import ( + PY_TYPES_TO_CPP_TYPES, + SCALAR_TYPES, +) +from generative_recommenders.ops.triton_aot.compile.utils import ( + hash_kernel_name, + unwrap_heuristic, +) +from generative_recommenders.ops.triton_aot.shared.compat import get_scratch_parameters +from generative_recommenders.ops.triton_aot.shared.types import AUTOTUNE_ATTRs, CTYPES +from generative_recommenders.ops.triton_aot.templates.template_utils import ( + load_template, + render_template, +) +from triton.runtime.jit import JITFunction + + +# --------------------------------------------------------------------------- +# Kernel naming and binary generation +# --------------------------------------------------------------------------- + + +def gen_kernel_name( + fn: Any, + spec: KernelSpec, + cc: int | str, +) -> str: + name = fn.__name__ + sig = "_".join([p.replace("*", "p") for p in spec.signature.values()]) + const = "_".join(map(str, spec.constants.values())) + cc_str = f"sm{cc}" + autotune_configs = [] + autotune_configs.append(f"w{spec.num_warps}") + autotune_configs.append(f"s{spec.num_stages}") + # AMD only + autotune_configs.append(f"matrix{spec.matrix_instr_nonkdim}") + autotune_configs.append(f"wave{spec.waves_per_eu}") + autotune_configs.append(f"kpack{spec.kpack}") + # See kernel_suffix in triton/compiler/code_generator.py + suffix = "" + for i, _ in enumerate(spec.signature): + suffix += str(i) + if i in spec.divisible_by_16: + suffix += "d" + if i in spec.divisible_by_8: + suffix += "e" + return "_".join([name, cc_str, sig, const] + autotune_configs + [suffix]) + + +def gen_cubin(kernel_name: str, kernel: Any, install_dir: str, backend: str) -> str: + """Generate kernel binary file (.cubin or .hsaco) and return extern declaration. + + Args: + kernel_name: Full kernel name including specialization suffix. + kernel: Compiled Triton kernel object containing binary in kernel.asm. + install_dir: Directory to write binary file. + backend: GPU backend ("cuda" or "hip"). + + Returns: + C++ extern declaration for the kernel binary array. + """ + hashed = hash_kernel_name(kernel_name) + if backend == "hip": + binary_file = f"{install_dir}/{hashed}.hsaco" + with open(binary_file, "wb") as hsaco: + hsaco.write(kernel.asm["hsaco"]) + target_symbol_name = f"{kernel_name}_cubin" + else: + binary_file = f"{install_dir}/{hashed}.cubin" + with open(binary_file, "wb") as cubin: + cubin.write(kernel.asm["cubin"]) + target_symbol_name = f"{kernel_name}_cubin" + + # We return extern declarations for both the array and its pointer. + # The pointer is used by gen_loader() to generate R_X86_64_64 relocations + # instead of R_X86_64_32, which allows the .triton section to be placed + # beyond the 4GB address limit in large binaries. + # Note: The pointer is volatile to prevent optimizer constant-propagation. + return f'extern "C" {{ extern unsigned char {target_symbol_name}[]; extern const void* volatile {target_symbol_name}_ptr; }}' + + +def gen_loader(kernel_name: str, cubin_name: str, shared: int) -> str: + # TODO(changpan): Extract inline cuModuleLoadData/cuModuleGetFunction error + # handling into a shared helper to reduce generated code size. + return textwrap.dedent( + f""" + CUfunction load_{kernel_name}(void) + {{ + thread_local std::unordered_map cache; + auto idx = torch::stable::accelerator::getCurrentDeviceIndex(); + auto res = cache.find(idx); + if (res != cache.end()) {{ + return res->second; + }} + CUfunction func; + CUmodule mod_ptr; + CUresult err; + // Use pointer to cubin data to generate R_X86_64_64 relocation + // instead of R_X86_64_32, allowing cubin data to be placed beyond 4GB + const void *image = {kernel_name}_cubin_ptr; + + err = cuModuleLoadData(&mod_ptr, image); + if (err != 0) {{ + const char* errStr; + cuGetErrorString(err, &errStr); + throw std::runtime_error("cuModuleLoadData failed for {kernel_name}: error " + std::to_string(err) + " (" + (errStr ? errStr : "unknown") + ")"); + }} + + err = cuModuleGetFunction(&func, mod_ptr, "{cubin_name}"); + if (err != 0) {{ + const char* errStr; + cuGetErrorString(err, &errStr); + throw std::runtime_error("cuModuleGetFunction failed for {kernel_name}: error " + std::to_string(err) + " (" + (errStr ? errStr : "unknown") + ")"); + }} + + check_errors({shared}, func); + cache.emplace(idx, func); + return func; + }} + """ + ) + + +# --------------------------------------------------------------------------- +# Launcher codegen (per-spec) +# --------------------------------------------------------------------------- + + +def gen_launcher_params( + descriptors: list[ArgDescriptor], + signature: dict[int, str], +) -> str: + args = ["gridDims grid"] + for d in descriptors: + if d.index in signature: + if isinstance(d, PointerArg): + ctype = "void*" + else: + ctype = CTYPES[signature[d.index]] + args.append(f"{ctype} {d.name}") + return ", ".join(args) + + +def gen_launch_args( + func: JITFunction[list[Any]], + spec: KernelSpec, +) -> list[str]: + """Generate kernel launch argument list (pointers to non-constant arguments).""" + args = [] + for i, arg in enumerate(func.arg_names): + if i in spec.constants: + continue + assert i in spec.signature, f"Argument {i} ({arg}) does not appear in signature" + args.append(f"&{arg}") + return args + + +def gen_launcher( + kernel_name: str, + func: JITFunction[list[Any]], + kernel: Any, + shared: int, + warp_size: int, + spec: KernelSpec, + descriptors: list[ArgDescriptor], +) -> str: + params = gen_launcher_params(descriptors, spec.signature) + args = gen_launch_args(func, spec) + + scratch_declarations, scratch_args = get_scratch_parameters(kernel) + args.extend(scratch_args) + + args_str = ", ".join(args) + + return textwrap.dedent( + f""" + void {kernel_name}({params}) {{ + CUfunction func = load_{kernel_name}(); + cudaStream_t stream = grid.stream ? grid.stream : triton_aot_get_current_stream(); + {scratch_declarations} + void *args[] = {{ {args_str} }}; + auto res = cuLaunchKernel(func, grid.x, grid.y, grid.z, {warp_size} * {spec.num_warps}, 1, 1, {shared}, stream, args, NULL); + TRITON_AOT_CU_CHECK(res); + }} + """ + ) + + +# --------------------------------------------------------------------------- +# Selector codegen (invariant) +# --------------------------------------------------------------------------- + + +def gen_selector_params( + descriptors: list[ArgDescriptor], +) -> str: + """Generate C++ selector function parameter list.""" + args = ["gridDims grid"] + for d in descriptors: + if isinstance(d, PointerArg): + args.append(f"const std::optional& {d.name}") + elif isinstance(d, ScalarArg): + args.append(f"{CTYPES[d.triton_dtype]} {d.name}") + elif isinstance(d, ConstantArg): + args.append(f"{CONSTANT_SELECTOR_CTYPE[d.python_type]} {d.name}") + + for name, value in AUTOTUNE_ATTRs.items(): + args.append(f"{type(value).__name__} {name}") + return ", ".join(args) + + +def gen_launcher_call_args( + descriptors: list[ArgDescriptor], + signature: dict[int, str], +) -> str: + args = ["grid"] + for d in descriptors: + if d.index in signature: + if isinstance(d, PointerArg): + args.append(f"{d.name}.value().data_ptr()") + elif isinstance(d, ScalarArg) and signature[d.index] != d.triton_dtype: + args.append(f"static_cast<{CTYPES[signature[d.index]]}>({d.name})") + else: + args.append(d.name) + return ", ".join(args) + + +def gen_guarded_calls( # noqa: C901 + func: JITFunction[list[Any]], + unit: OpsUnit, + descriptors: list[ArgDescriptor], +) -> str: + desc_by_idx: dict[int, ArgDescriptor] = {d.index: d for d in descriptors} + calls = [] + for spec in unit.specs: + kernel_name = gen_kernel_name(func, spec, unit.cc) + args = gen_launcher_call_args(descriptors, spec.signature) + guards = "" + + # Guard on tensor dtypes (per-spec: different specs may have different dtypes) + for i, ttype in spec.signature.items(): + d = desc_by_idx[i] + if not isinstance(d, PointerArg): + continue + arg = d.name + atype = SCALAR_TYPES[ttype] + guards += f"if ({arg}.has_value()) " + guards += f"if ({arg}.value().scalar_type() == {atype}) " + + # Guard on int range (spec uses narrower type than selector) + for i, dtype in spec.signature.items(): + d = desc_by_idx[i] + if isinstance(d, ScalarArg) and dtype != d.triton_dtype: + if dtype == "i32": + guards += f"if (fits_i32({d.name})) " + + # Guard on constant values. + for i, val in spec.constants.items(): + arg = desc_by_idx[i].name + if isinstance(val, bool): + guards += f"if ({arg}) " if val else f"if (!({arg})) " + elif isinstance(val, str): + guards += f'if ({arg} == "{val}") ' + elif val is None: + guards += f"if (!{arg}.has_value()) " + else: + guards += f"if ({arg} == {val}) " + + # Guard on special constants + for name in AUTOTUNE_ATTRs.keys(): + guards += f"if ({name} == {getattr(spec, name)}) " + + # Guard on divisible_by_16 + for i in spec.divisible_by_16: + arg = desc_by_idx[i].name + if i in spec.signature: + ttype = spec.signature[i] + if ttype.startswith("*"): + guards += f"if ((((uintptr_t){arg}.value().data_ptr()) % 16) == 0) " + else: + guards += f"if (({arg} % 16) == 0) " + elif i in spec.constants: + assert (spec.constants[i] % 16) == 0 + + # Guard on divisible_by_8 + for i in spec.divisible_by_8: + arg = desc_by_idx[i].name + if i in spec.signature: + ttype = spec.signature[i] + # divisible_by_8 is only applied to int + if not ttype.startswith("*"): + guards += f"if (({arg} % 8) == 0) " + elif i in spec.constants: + assert (spec.constants[i] % 8) == 0 + + # Call the specialization. + calls.append(f"{guards}return {kernel_name}({args});\n") + return "".join(calls) + + +def gen_selector_proto( + descriptors: list[ArgDescriptor], + func_name: str, +) -> str: + params = gen_selector_params(descriptors) + # Add Triton's default values for num warps/stages, etc + for name, value in AUTOTUNE_ATTRs.items(): + params = params.replace(name, f"{name}={value}") + return f"void {func_name}({params});" + + +def gen_failure_msg( + descriptors: list[ArgDescriptor], +) -> str: + """Generate C++ ``<<``-chain for the dispatch-failure error message. + + Groups parameters by category (Tensors / Scalars / Constants / + Autotune / Device). Tensor entries include aligned16 status. + """ + tensors: list[str] = [] + scalars: list[str] = [] + constants: list[str] = [] + + for d in descriptors: + if isinstance(d, PointerArg): + dtype_expr = ( + f"({d.name}.has_value()" + f" ? c10::toString({d.name}.value().scalar_type())" + f' : "nullptr")' + ) + align_expr = ( + f"(({d.name}.has_value()" + f" && (((uintptr_t){d.name}.value().data_ptr()) % 16) == 0)" + f' ? "true" : "false")' + ) + tensors.append( + f'" {d.name}=" << {dtype_expr} << "(aligned16=" << {align_expr} << ")"' + ) + elif isinstance(d, ScalarArg): + scalars.append(f'" {d.name}=" << {d.name}') + elif isinstance(d, ConstantArg): + constants.append(f'" {d.name}=" << {d.name}') + + autotune: list[str] = [f'" {n}=" << {n}' for n in AUTOTUNE_ATTRs] + + sections: list[str] = [] + if tensors: + sections.append('"\\n Tensors:" << ' + " << ".join(tensors)) + if scalars: + sections.append('"\\n Scalars:" << ' + " << ".join(scalars)) + if constants: + sections.append('"\\n Constants:" << ' + " << ".join(constants)) + sections.append('"\\n Autotune:" << ' + " << ".join(autotune)) + sections.append('"\\n Device: cc=" << cc') + + return " << ".join(sections) + + +def gen_selector( + func: JITFunction[list[Any]], + unit: OpsUnit, + descriptors: list[ArgDescriptor], +) -> str: + params = gen_selector_params(descriptors) + guarded_calls = gen_guarded_calls(func, unit, descriptors) + failure_msg = gen_failure_msg(descriptors) + return f""" + void {func.__name__}({params}) {{ + auto cc = compute_capability(); + if (grid.x * grid.y * grid.z > 0) {{ + {guarded_calls} + std::stringstream ss; + ss << "[TritonAOT] No implementation found for {func.__name__}" << {failure_msg}; + throw std::runtime_error(ss.str()); + }} + }} + """ + + +# --------------------------------------------------------------------------- +# Torch op codegen (invariant) +# --------------------------------------------------------------------------- + + +def gen_cpp_op_params( + descriptors: list[ArgDescriptor], +) -> str: + args = [] + for d in descriptors: + if isinstance(d, PointerArg): + args.append(f"std::optional {d.name}") + elif isinstance(d, ScalarArg): + args.append(f"{scalar_cpp_op_ctype(d.triton_dtype)} {d.name}") + elif isinstance(d, ConstantArg): + args.append(f"{CONSTANT_CPP_OP_CTYPE[d.python_type]} {d.name}") + for name, value in AUTOTUNE_ATTRs.items(): + args.append(f"{PY_TYPES_TO_CPP_TYPES[type(value)]} {name}") + return ", ".join(args) + + +def gen_torch_op_params( + descriptors: list[ArgDescriptor], + default_values: dict[str, Any], +) -> str: + args = [] + + def gen_str_wrap(value: Any) -> Any: + return f'\\"{value}\\"' if isinstance(value, str) else value + + def gen_default_str(arg: str) -> str: + return ( + f" = {gen_str_wrap(default_values[arg])}" if arg in default_values else "" + ) + + for d in descriptors: + df_str = gen_default_str(d.name) + if isinstance(d, PointerArg): + t = chr(ord("a") + d.index) + args.append(f"Tensor({t}!)? {d.name}") + elif isinstance(d, ScalarArg): + args.append(f"{scalar_torch_schema(d.triton_dtype)} {d.name}{df_str}") + elif isinstance(d, ConstantArg): + args.append(f"{CONSTANT_TORCH_SCHEMA[d.python_type]} {d.name}{df_str}") + for name, value in AUTOTUNE_ATTRs.items(): + args.append(f"{type(value).__name__} {name}={value}") + return ", ".join(args) + + +def gen_torch_op( + func: JITFunction[list[Any]], + descriptors: list[ArgDescriptor], + default_values: dict[str, Any], +) -> str: + cpp_params = gen_cpp_op_params(descriptors) + torch_params = gen_torch_op_params(descriptors, default_values) + arg_names = list(func.arg_names) + list(AUTOTUNE_ATTRs.keys()) + args = ", ".join(arg_names) + + # Generate a comment noting which tensor params are non-optional but + # promoted to Tensor? for TorchScript compatibility. + promoted = [ + d.name for d in descriptors if isinstance(d, PointerArg) and not d.is_optional + ] + type_comment = "" + if promoted: + type_comment = ( + f"// Note: {', '.join(promoted)} are non-optional but use Tensor? " + "for TorchScript compatibility.\n" + "// Dispatch uses HAS_XXX constexpr ints, not tensor presence.\n" + ) + return textwrap.dedent( + f""" + namespace {{ + triton::aot::gridDims dims_from_vec( + const std::vector& grid + ) {{ + return triton::aot::gridDims( + grid.size() > 0 ? grid[0] : 1, + grid.size() > 1 ? grid[1] : 1, + grid.size() > 2 ? grid[2] : 1 + ); + }} + + {type_comment}void {func.__name__}_op( + std::vector grid, + {cpp_params} + ) {{ + triton::aot::{func.__name__}( + dims_from_vec(grid), + {args} + ); + }} + + void {func.__name__}_dummy_op( + std::vector grid, + {cpp_params} + ) {{ + // Do nothing. The op is a dummy for model transform, + // processing, and splitting services. + }} + }} + + STABLE_TORCH_LIBRARY_FRAGMENT(triton_aot, m) {{ + m.def("{func.__name__}(int[] grid, {torch_params}) -> ()"); + }} + STABLE_TORCH_LIBRARY_IMPL(triton_aot, CUDA, m) {{ + m.impl("{func.__name__}", TORCH_BOX(&{func.__name__}_op)); + }} + + STABLE_TORCH_LIBRARY_IMPL(triton_aot, CPU, m) {{ + m.impl("{func.__name__}", TORCH_BOX(&{func.__name__}_dummy_op)); + }} + + STABLE_TORCH_LIBRARY_IMPL(triton_aot, Meta, m) {{ + m.impl("{func.__name__}", TORCH_BOX(&{func.__name__}_dummy_op)); + }} + """ + ) + + +# --------------------------------------------------------------------------- +# Tuner meta codegen +# --------------------------------------------------------------------------- + + +def key_names_and_idx(func: Any) -> tuple[list[str], list[int]]: + if hasattr(func, "key_idx"): + arg_names = [func.arg_names[idx] for idx in func.key_idx] + key_idx = func.key_idx + else: + arg_names = func.keys + key_idx = [func.arg_names.index(arg) for arg in arg_names] + return arg_names, key_idx + + +def is_non_empty_mapping_of_type(obj: object, value_type: type[Any]) -> bool: + """Check if object is a non-empty dict with all values of specific type""" + if not obj or not isinstance(obj, dict): + return False + + return all(isinstance(value, value_type) for value in obj.values()) + + +_LAUNCH_PARAM_NAMES: list[str] = ["num_warps", "num_stages"] + + +def gen_tuner_meta_py( + func: Any, + tuner_fallback: bool, + unit: OpsUnit, +) -> str: + vals = [] + + guard_list = [] + + # Use custom meta generation function if available + if hasattr(func, "gen_autotune_select_meta_src"): + return func.gen_autotune_select_meta_src(unit.constant_types) + + if hasattr(func, "cache") and is_non_empty_mapping_of_type( + func.cache, triton.runtime.autotuner.Config + ): + # auto tuned configs + arg_names, key_idx = key_names_and_idx(func) + + in_args = ", ".join( + [ + f"{name}: {unit.constant_types[idx].__name__ if idx in unit.constant_types else 'int'}" + for idx, name in zip(key_idx, arg_names) + ] + ) + + cfg_first = next(iter(func.cache.values())) + return_names = list(cfg_first.kwargs.keys()) + _LAUNCH_PARAM_NAMES + + for key, cfg in func.cache.items(): + val = list(cfg.kwargs.values()) + [cfg.num_warps, cfg.num_stages] + val = tuple(val) + vals.append(val) + equations = [] + for arg, value in zip(arg_names, key): + if isinstance(value, str): + equations.append(f"{arg} == '{value}'") + elif isinstance(value, bool): + equations.append(f"{arg} == {int(value)}") + else: + equations.append(f"{arg} == {value}") + guard_list.append(f"if {' and '.join(equations)}: return {val}") + + else: + # default configs — single spec, use specs[0] + in_args = "" + arg_names = list(_LAUNCH_PARAM_NAMES) + return_names = list(_LAUNCH_PARAM_NAMES) + val = unit.specs[0].num_warps, unit.specs[0].num_stages + vals.append(val) + + name = unwrap_heuristic(func, JITFunction).__name__ + meta = name + "_meta" + + guards = "\n ".join(guard_list) + + fmt_args = ", ".join([f"{{{arg_name}}}" for arg_name in arg_names]) + + raise_runtime_error_str = ( + f"""raise RuntimeError(f"No autotuning config found for {name}({fmt_args})")""" + ) + fallback_str = f"""return {Counter(vals).most_common(1)[0][0]}""" + + returns_comment = f"# Returns: ({', '.join(return_names)})" + + return textwrap.dedent( + f""" + def {meta}({in_args}): + {returns_comment} + {guards} + {fallback_str if tuner_fallback else raise_runtime_error_str} + """ + ) + + +def gen_tuner_meta_cpp( + func: Any, + tuner_fallback: bool, + constant_types: dict[int, type[Any]], +) -> str: + # TODO(changpan): This C++ inline _meta is currently dead code — no C++ caller + # invokes it. The Python _meta.py (gen_tuner_meta_py) is the only consumer. + # Double check, try remove this and the TUNER_META_CPP template region. + def infer_arg_type(idx: int) -> str: + if idx in constant_types: + return PY_TYPES_TO_CPP_TYPES[constant_types[idx]] + else: + return "int64_t" + + arg_names, key_idx = key_names_and_idx(func) + + in_args = ", ".join( + [f"{infer_arg_type(idx)} {name}" for idx, name in zip(key_idx, arg_names)] + ) + + vals = [] + guard_list = [] + for key, cfg in func.cache.items(): + val = list(cfg.kwargs.values()) + [cfg.num_warps, cfg.num_stages] + val = tuple(val) + vals.append(val) + equations = [] + for arg, value in zip(arg_names, key): + if isinstance(value, str): + equations.append(f'{arg} == "{value}"') + elif isinstance(value, bool): + equations.append(f"{arg} == {int(value)}") + else: + equations.append(f"{arg} == {value}") + guard_list.append(f"if ({' && '.join(equations)}) return std::make_tuple{val};") + guards = "\n ".join(guard_list) + name = unwrap_heuristic(func, JITFunction).__name__ + meta = name + "_meta" + fmt_args = ", ".join([f"{arg_name}" for arg_name in arg_names]) + raise_runtime_error_str = f"""throw std::runtime_error("No autotuning config found for {name}({fmt_args})");""" + fallback_str = f"""return std::make_tuple{Counter(vals).most_common(1)[0][0]};""" + # Infer the return type from the actual values + return_type = _infer_return_type(vals[0]) + return textwrap.dedent( + f""" + inline std::tuple<{return_type}> {meta}({in_args}) {{ + {guards} + {fallback_str if tuner_fallback else raise_runtime_error_str} + }} + """ + ) + + +def _infer_return_type(vals: tuple[Any, ...]) -> str: + types = [PY_TYPES_TO_CPP_TYPES.get(type(val)) for val in vals] + try: + # pyre-fixme[6]: For 1st argument expected + # `Iterable[typing_extensions.LiteralString]` but got `List[Optional[str]]`. + return ", ".join(types) + except TypeError: # one of the types cannot be inferred, e.g. `None` + raise ValueError("Cannot infer return type from `vals`") + + +# --------------------------------------------------------------------------- +# Top-level codegen entry points +# --------------------------------------------------------------------------- + + +def generate_header_content( + tuned_func: triton.runtime.autotuner.Autotuner | None, + func: JITFunction[list[Any]], + unit: OpsUnit, + descriptors: list[ArgDescriptor], + tuner_fallback: bool, +) -> str: + """Generate the content of the .h header file.""" + h_template = load_template("kernel.h") + tuner_meta_cpp = ( + gen_tuner_meta_cpp(tuned_func, tuner_fallback, unit.constant_types) + if tuned_func + else "" + ) + selector_proto = gen_selector_proto(descriptors, func.__name__) + return render_template( + h_template, + { + "TUNER_META_CPP": tuner_meta_cpp, + "SELECTOR_PROTO": selector_proto, + }, + ) + + +def generate_kernel_cpp_content( + func: JITFunction[list[Any]], + unit: OpsUnit, + descriptors: list[ArgDescriptor], + prefix: str, + generated_specs: list[str], + backend: str, +) -> str: + """Generate the content of the kernel .cpp file. + + All tensor params use Tensor? for TorchScript compatibility. + Dispatch relies on HAS_XXX constexpr ints, not tensor presence. + """ + cpp_template = load_template("kernel.cpp") + kernel_specs = "\n".join(generated_specs) + selector = gen_selector(func, unit, descriptors) + + # On AMD, apply hipification to generated code (KERNEL_SPECS, SELECTOR) + # Templates are already hipified at load time (from hip/ subdirectory) + if backend == "hip": + from torch._inductor.codegen.aoti_hipify_utils import maybe_hipify_code_wrapper + + kernel_specs = maybe_hipify_code_wrapper(kernel_specs, force_hipify=True) + selector = maybe_hipify_code_wrapper(selector, force_hipify=True) + + cpp_content = render_template( + cpp_template, + { + "HEADER_INCLUDE": f'#include "{prefix}.h"\n', + "KERNEL_SPECS": kernel_specs, + "SELECTOR": selector, + }, + ) + return cpp_content + + +def generate_torch_op_content( + func: JITFunction[list[Any]], + descriptors: list[ArgDescriptor], + prefix: str, + default_values: dict[str, Any], +) -> str: + """Generate the content of the torch_op .cpp file.""" + torch_template = load_template("torch_op.cpp") + torch_op_content = gen_torch_op(func, descriptors, default_values) + torch_content = render_template( + torch_template, + { + "HEADER_INCLUDE": f'#include "{prefix}.h"\n', + "TORCH_OP": torch_op_content, + }, + ) + return torch_content diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/compile/compile_state.py b/recommendation_v4/generative_recommenders/ops/triton_aot/compile/compile_state.py new file mode 100644 index 000000000..231cb52f1 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton_aot/compile/compile_state.py @@ -0,0 +1,409 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# pyre-strict + +#!/usr/bin/env python3 + +from __future__ import annotations + +import hashlib +import json +import os +import tempfile +from inspect import getcallargs, Parameter, signature +from typing import Any, Callable, Dict, List, Optional, Set + +import torch + +# @manual=//triton:triton +import triton.language as tl +from generative_recommenders.ops.triton_aot.compile.stable_types import SCALAR_TYPES +from generative_recommenders.ops.triton_aot.compile.utils import is_autotuner +from generative_recommenders.ops.triton_aot.types import ( + Annotation, + AnnotationHint, + TritonAOT, +) + +# @manual=//triton:triton +from triton.runtime.jit import KernelInterface, mangle_type + + +class CustomEncoder(json.JSONEncoder): + # pyre-ignore[14]: Inconsistent override + def default(self, obj: object) -> Any: + if isinstance(obj, set): + return {"__set__": True, "items": sorted(obj)} + # Handle other non-serializable types + return super().default(obj) + + +def hash_spec(spec: Dict[str, Any]) -> str: + serialized_dict = json.dumps(spec, cls=CustomEncoder, sort_keys=True) + return hashlib.sha256(serialized_dict.encode("utf-8")).hexdigest() + + +class AOTTCompileState: + """ + Singleton state container for Triton AOT compilation. + + Description: + This singleton pattern enables state sharing between code loaded via + torch.package (which creates isolated module namespaces) and the regular + Python import system. Without this pattern, the packaged module would have + its own copy of global state, leading to inconsistencies. + + Usage: + # Normal usage - get the singleton instance + state = AOTTCompileState.get_instance() + + # For torch.package integration - inject shared instance into packaged module + packaged_module = package_importer.import_module("triton_aot.compile.compile_state") + packaged_module.AOTTCompileState.set_instance(AOTTCompileState.get_instance()) + """ + + _instance: Optional["AOTTCompileState"] = None + + kernel_specs: Dict[KernelInterface[List[Any]], List[Dict[str, List[Any]]]] = {} + specs_hashset: Dict[KernelInterface[List[Any]], Set[str]] = {} + enable_aott_compile: bool = False + compile_base_dir: str = "" + compile_path: str = "" + + def __new__(cls) -> "AOTTCompileState": + if cls._instance is None: + instance = super().__new__(cls) + instance._initialize() + cls._instance = instance + return cls._instance + + def _initialize(self) -> None: + """Initialize the singleton state. Called only once.""" + self.kernel_specs: Dict[ + KernelInterface[List[Any]], List[Dict[str, List[Any]]] + ] = {} + self.specs_hashset: Dict[KernelInterface[List[Any]], Set[str]] = {} + self.enable_aott_compile: bool = False + self.compile_base_dir: str = os.getenv("TRITON_AOT_PATH_PREFIX", "/var/tmp") + self.compile_path: str = tempfile.mkdtemp( + dir=self.compile_base_dir, prefix="triton_aot_compile_" + ) + + @classmethod + def get_instance(cls) -> "AOTTCompileState": + """Get the singleton instance, creating it if necessary.""" + if cls._instance is None: + cls._instance = cls() + return cls._instance + + @classmethod + def set_instance(cls, instance: "AOTTCompileState") -> None: + """ + Set the singleton instance. Used for torch.package integration. + + When code is loaded via torch.package, it creates a separate module + namespace with its own class objects. This method allows injecting + a shared instance from the main module into the packaged module. + """ + cls._instance = instance + + def reset(self) -> None: + """Reset all state to initial values.""" + self.kernel_specs = {} + self.specs_hashset = {} + self.disable() + self.compile_base_dir = os.getenv("TRITON_AOT_PATH_PREFIX", "/var/tmp") + self.compile_path = tempfile.mkdtemp( + dir=self.compile_base_dir, prefix="triton_aot_compile_" + ) + + def add_kernel_spec( + self, + fn: KernelInterface[List[Any]], + spec: Dict[str, List[Any]], + hashed_spec: str, + ) -> None: + """Add a kernel spec if not already present (based on hash). + If the same Triton kernel is used at multiple locations in a model: + - All calls share one spec list under the same kernel function key + - Specs with identical signatures (same dtypes, shapes) are deduplicated via hash + - Specs with different signatures (e.g., fp32 vs bf16) are recorded separately + + Example: + # Two call sites using the same kernel: + my_kernel[grid](tensor_fp32, ...) # Records spec with "*fp32" + my_kernel[grid](tensor_bf16, ...) # Records spec with "*bf16" + my_kernel[grid](tensor_fp32, ...) # Deduplicated, same hash as first call + + # Result: kernel_specs[my_kernel] = [fp32_spec, bf16_spec] + """ + if fn not in self.kernel_specs: + self.kernel_specs[fn] = [] + self.specs_hashset[fn] = set() + if hashed_spec not in self.specs_hashset[fn]: + self.kernel_specs[fn].append(spec) + self.specs_hashset[fn].add(hashed_spec) + + def _collect_spec( + self, + fn: KernelInterface[List[Any]], + annotations: Dict[str, Annotation], + *args: Any, + **kwargs: Any, + ) -> None: + """Spec collection callback registered on TritonAOT during compile. + + Always collects the annotated spec (which equals the inferred spec + when no annotations are present). Also collects the inferred spec + when it differs and either: + - annotations conflict with sample (fallback for safety), or + - inferred has perf hints the annotation lacks (perf variant). + """ + spec = infer_spec(fn, annotations, *args, **kwargs) + annotated_hash = hash_spec(spec) + self.add_kernel_spec(fn, spec, annotated_hash) + + if annotations: + inferred = infer_spec(fn, {}, *args, **kwargs) + inferred_hash = hash_spec(inferred) + if inferred_hash == annotated_hash: + return + if _annotation_conflicts_with_sample( + fn, annotations, *args, **kwargs + ) or _inferred_has_perf_advantage(spec, inferred): + self.add_kernel_spec(fn, inferred, inferred_hash) + + def enable(self) -> None: + """Enable AOT compile and register the spec collection hook.""" + self.enable_aott_compile = True + TritonAOT.set_spec_collector(self._collect_spec) + + def disable(self) -> None: + """Disable AOT compile and unregister the spec collection hook.""" + self.enable_aott_compile = False + TritonAOT.set_spec_collector(None) + + +def get_aott_compile_state() -> AOTTCompileState: + """Get the current AOTTCompileState singleton. + + Uses get_instance() so injected instances (via set_instance() for + torch.package integration) are respected. + """ + return AOTTCompileState.get_instance() + + +######## +# Module-level global accessors that delegate to singleton +######## + + +def get_triton_aot_kernel_specs() -> Dict[ + KernelInterface[List[Any]], List[Dict[str, List[Any]]] +]: + return get_aott_compile_state().kernel_specs + + +def get_triton_aot_specs_hashset() -> Dict[KernelInterface[List[Any]], Set[str]]: + return get_aott_compile_state().specs_hashset + + +def get_aott_compile_path() -> str: + return get_aott_compile_state().compile_path + + +def add_kernel_spec( + fn: KernelInterface[List[Any]], spec: Dict[str, List[Any]], hashed_spec: str +) -> None: + get_aott_compile_state().add_kernel_spec(fn, spec, hashed_spec) + + +def _unwrap_triton_fn( + fn: KernelInterface[List[Any]], +) -> Callable[..., Any]: + while isinstance(fn, KernelInterface): + # pyre-ignore[16]: KernelInterface has `fn` attribute at runtime + fn = fn.fn + return fn + + +def _inferred_has_perf_advantage( + annotated_spec: Dict[str, List[Any]], + inferred_spec: Dict[str, List[Any]], +) -> bool: + """True if inferred spec has alignment/divisibility hints the annotated lacks. + + A tuple element ``(type, N)`` carries alignment or divisibility info + that a bare string does not. When inference adds such hints (e.g., + tensor alignment from ``data_ptr() % 16 == 0``), the inferred spec + produces a more optimized cubin worth keeping as a perf variant. + """ + for ann_elem, inf_elem in zip( + annotated_spec["signature"], inferred_spec["signature"] + ): + if isinstance(inf_elem, tuple) and not isinstance(ann_elem, tuple): + return True + return False + + +# Triton-internal kwargs injected by KernelInterface.__getitem__ +# (triton/runtime/jit.py). These are not kernel parameters and must +# be stripped before getcallargs. +_TRITON_INTERNAL_KWARGS: frozenset[str] = frozenset({"warmup", "grid"}) + + +def _resolve_call_args( + fn: KernelInterface[List[Any]], + *args: Any, + **kwargs: Any, +) -> tuple[Callable[..., Any], dict[str, Any]]: + """Unwrap kernel and resolve call args with autotune placeholder fill.""" + triton_fn = _unwrap_triton_fn(fn) + # Filter Triton-internal kwargs injected by KernelInterface.__getitem__ + # (triton/runtime/jit.py) — not part of the kernel signature. + clean_kwargs = {k: v for k, v in kwargs.items() if k not in _TRITON_INTERNAL_KWARGS} + if is_autotuner(fn): + # pyre-ignore[16]: Attributes checked by is_autotuner + for arg_name in fn.configs[0].kwargs.keys(): + if arg_name not in clean_kwargs: + clean_kwargs[arg_name] = -1 + return triton_fn, getcallargs(triton_fn, *args, **clean_kwargs) + + +_I32_MIN: int = -(2**31) +_I32_MAX: int = 2**31 - 1 + + +def _sample_satisfies_int_type(sample: int, ann_type: str) -> bool: + """True if sample int fits the annotated type range.""" + if ann_type == "i32": + return _I32_MIN <= sample <= _I32_MAX + return True + + +def _sample_satisfies_annotation(sample: Any, ann: Annotation) -> bool: + """True if a single sample value satisfies its annotation constraint.""" + if isinstance(ann, AnnotationHint): + if isinstance(sample, torch.Tensor): + return sample.data_ptr() % ann.hint == 0 + if isinstance(sample, int): + if ann.hint == 1: + return sample == 1 + if not _sample_satisfies_int_type(sample, ann.dtype): + return False + if ann.hint > 1: + return sample % ann.hint == 0 + return True + if isinstance(ann, str) and not ann.startswith("*") and isinstance(sample, int): + return _sample_satisfies_int_type(sample, ann) + return True + + +def _annotation_conflicts_with_sample( + fn: KernelInterface[List[Any]], + annotations: Dict[str, Annotation], + *args: Any, + **kwargs: Any, +) -> bool: + """True if any annotated param's sample value doesn't satisfy the annotation. + + Used by ``_collect_spec`` to decide whether to generate an inferred + fallback spec. When the sample satisfies all annotations, only the + annotated spec is needed (the user's constraints hold for this input). + """ + _, sample_args = _resolve_call_args(fn, *args, **kwargs) + + for param_name, ann in annotations.items(): + sample = sample_args.get(param_name) + if sample is None: + continue + if not _sample_satisfies_annotation(sample, ann): + return True + + return False + + +def _infer_spec_entry( + arg_name: str, + arg: Any, + arg_annotation: Any, + annotations: Dict[str, Annotation], +) -> Any: + if arg_annotation != Parameter.empty: + if arg_annotation == tl.constexpr: + return arg + raise RuntimeError( + f"TritonAOT: unsupported scalar annotation {arg_annotation}." + ) + + if arg_name in annotations: + ann = annotations[arg_name] + # Convert to tuple for raw spec format (shared/spec_conversion + # processes plain tuples). + return ann.to_tuple() if isinstance(ann, AnnotationHint) else ann + + if arg is None: + return None + + if isinstance(arg, torch.Tensor): + # Reject dtypes SCALAR_TYPES can't render (e.g. *u1, *u16, *fp8e5) + # so codegen doesn't KeyError downstream. + type_str = mangle_type(arg) + if type_str not in SCALAR_TYPES: + raise RuntimeError( + f"TritonAOT: unsupported tensor type for {arg_name}: " + f"{arg.dtype} (Triton mangled to {type_str!r}). " + f"Supported tensor dtypes: {sorted(SCALAR_TYPES.keys())}." + ) + return (type_str, 16) if arg.data_ptr() % 16 == 0 else type_str + + if isinstance(arg, bool): + # bool is subclass of int; must check before int. + # Non-constexpr bools have no CTYPES entry for codegen. + raise RuntimeError( + f"TritonAOT: parameter {arg_name} is a bool without " + f"tl.constexpr annotation. Add `{arg_name}: tl.constexpr` " + f"to the kernel signature." + ) + + if isinstance(arg, int): + # Always i64 for safety; users annotate "i32" for narrower variant via + # annotation-as-variant. + if not -(2**63) <= arg <= 2**63 - 1: + raise RuntimeError( + f"TritonAOT: unsupported int value for {arg_name}: " + f"value exceeds i64 range. Use a smaller value or tl.constexpr." + ) + return "i64" + + if isinstance(arg, float): + return "fp32" + + raise RuntimeError(f"TritonAOT: parameter {arg_name} needs annotation.") + + +def infer_spec( + fn: KernelInterface[List[Any]], + annotations: Dict[str, Annotation], + *args: Any, + **kwargs: Any, +) -> Dict[str, List[Any]]: + """Infer kernel spec from sample args. + + Tensor dtype: ``mangle_type``, alignment: ``data_ptr() % 16``. + Scalar int: always ``"i64"`` (safe default; user can annotate ``"i32"`` + to get a narrower variant via annotation-as-variant). + Float: ``mangle_type`` → fp32. + """ + triton_fn, call_args = _resolve_call_args(fn, *args, **kwargs) + fn_sig = signature(triton_fn) + arg_annotations = { + name: param.annotation for name, param in fn_sig.parameters.items() + } + spec = [] + + for arg_name in fn_sig.parameters.keys(): + arg = call_args[arg_name] + spec.append( + _infer_spec_entry(arg_name, arg, arg_annotations[arg_name], annotations) + ) + return {"signature": spec} diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/compile/pipeline.py b/recommendation_v4/generative_recommenders/ops/triton_aot/compile/pipeline.py new file mode 100644 index 000000000..cdf4d1821 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton_aot/compile/pipeline.py @@ -0,0 +1,300 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. + +# pyre-strict + +"""AOT-T compilation pipeline. + +Orchestrates: spec processing → Triton native compile → C++ / Python codegen. +""" + +from __future__ import annotations + +import logging +import multiprocessing as mp +import os +import signal +import threading +from concurrent.futures import ThreadPoolExecutor +from types import FrameType, ModuleType +from typing import Any, Callable + +# @manual=//triton:triton +import triton +import triton.compiler +from generative_recommenders.ops.triton_aot.compile.arg_descriptor import ( + ArgDescriptor, + build_arg_descriptors, +) +from generative_recommenders.ops.triton_aot.compile.codegen import ( + gen_cubin, + gen_kernel_name, + gen_launcher, + gen_loader, + gen_tuner_meta_py, + generate_header_content, + generate_kernel_cpp_content, + generate_torch_op_content, +) +from generative_recommenders.ops.triton_aot.compile.spec_processing import ( + gen_compile_arg, + KernelSpec, + OpsUnit, + RawKernelSpec, +) +from generative_recommenders.ops.triton_aot.compile.utils import ( + is_autotuner, + unwrap_heuristic, +) +from generative_recommenders.ops.triton_aot.shared.types import AUTOTUNE_ATTRs +from triton.backends.compiler import GPUTarget +from triton.runtime.jit import JITFunction, KernelInterface + +logger: logging.Logger = logging.getLogger(__name__) + + +def compile_specs_parallel( + specs: list[KernelSpec], + install_dir: str, + module: str, + name: str, + gpu_target: GPUTarget, + import_module: Callable[[str], ModuleType], + descriptors: list[ArgDescriptor], +) -> list[str]: + """Compile kernel specs in parallel using multiprocessing. + + When TRITON_AOT_DEBUG=1 is set, compiles sequentially for easier debugging. + + Args: + specs: List of kernel specifications to compile + install_dir: Directory to install generated files + module: The module name of the function + name: The function name + gpu_target: GPU target for compilation + import_module: Function to import modules (e.g., importlib.import_module or PackageImporter.import_module) + + Returns: + List of generated code strings for each spec (cubin, loader, launcher) + """ + + debug = os.environ.get("TRITON_AOT_DEBUG", "0") == "1" + if debug: + outputs = [ + spec_gen( + install_dir, + spec, + module, + name, + gpu_target, + import_module, + descriptors, + ) + for spec in specs + ] + else: + max_workers = mp.cpu_count() // 2 + 1 + with ThreadPoolExecutor(max_workers=min(len(specs), max_workers)) as executor: + outputs = list( + executor.map( + lambda spec: spec_gen( + install_dir, + spec, + module, + name, + gpu_target, + import_module, + descriptors, + ), + specs, + ) + ) + return outputs + + +# For each spec, generate a kernel: +# - cubin +# - loader +# - launcher +def spec_gen( + install_dir: str, + spec: KernelSpec, + module: str, + name: str, + gpu_target: GPUTarget, + import_module: Callable[[str], ModuleType], + descriptors: list[ArgDescriptor], +) -> str: + # To run this function with multiprocessing, we need to import the function by name, + # since JITFunction cannot be pickled. + # we have the case where the func name is injected with a suffix, like "_cuda" or "_amd", + # we should use the original name to import the func in such case + original_name = name + splits = name.split("_") + end_idx = len(splits) + + while end_idx > 0: + original_name = "_".join(splits[:end_idx]) + if hasattr(import_module(module), original_name): + break + end_idx -= 1 + func = unwrap_heuristic(getattr(import_module(module), original_name), JITFunction) + func.__name__ = name + + # Generate cubin. + kernel_name = gen_kernel_name(func, spec, gpu_target.arch) + + compile_arg = gen_compile_arg(spec, func) + options = {name: getattr(spec, name) for name in AUTOTUNE_ATTRs.keys()} + compile_kwargs = { + "target": gpu_target, + "options": options, + } + kernel = triton.compiler.compile(*compile_arg, **compile_kwargs) + if getattr(kernel.metadata, "global_scratch_size", 0) > 0: + raise RuntimeError(f"{kernel_name=} with global scratch is not supported.") + + metadata_name = kernel.metadata.name + metadata_shared = kernel.metadata.shared + + cubin = gen_cubin(kernel_name, kernel, install_dir, gpu_target.backend) + out = [ + cubin, + # Generate loader. + gen_loader(kernel_name, metadata_name, metadata_shared), + # Generate launcher. + gen_launcher( + kernel_name, + func, + kernel, + metadata_shared, + gpu_target.warp_size, + spec, + descriptors, + ), + ] + return "".join(out) + + +def sigchld_handler(signum: int, frame: FrameType | None) -> None: + sketchy_signals = map(int, [signal.SIGSEGV, signal.SIGABRT, signal.SIGBUS]) + try: + # Consume all pending SIGCHLDs, looking for unexpected failures + while True: + pid, status = os.waitpid(-1, os.WNOHANG) + if pid == 0: + break + if os.WIFSIGNALED(status) and os.WTERMSIG(status) in sketchy_signals: + logger.error( + f"Child process {pid} exited catastrophically with signal {os.WTERMSIG(status)}, terminating!" + ) + + # Avoid triggering atexit etc which can get stuck and behave improperly + # because multiprocessing sets up an atexit handler to join workers + # (sigh). We want to exit, now, so use os._exit instead of sys.exit. + os._exit(1) + except ChildProcessError: + pass + + +def compile_to_cpp( + func: KernelInterface[list[Any]] | triton.runtime.autotuner.Autotuner, + base_specs: list[RawKernelSpec], + install_dir: str, + prefix: str, + *, + gpu_target: GPUTarget, + import_module: Callable[[str], ModuleType], + default_values: dict[str, Any] | None = None, + tuner_fallback: bool = False, +) -> None: + """Compile a Triton kernel into .cpp, .h, _torch_op.cpp, _meta.py files. + + Args: + func: Triton JITFunction or Autotuner to compile. + base_specs: List of kernel specialization specs. + install_dir: Directory to output generated files. + prefix: Kernel name prefix, e.g., "_addmm_fwd". + gpu_target: GPU target for compilation. + import_module: torch.package importer for loading kernels source code. + default_values: Default values for kernel arguments. + tuner_fallback: If True, generate fallback tuner code. + """ + tuned_func = func if is_autotuner(func) else None + # pyre-ignore[6]: Attributes verified by is_autotuner + unit = OpsUnit.from_raw_specs(base_specs, gpu_target, tuned_func) + default_values = {} if default_values is None else default_values + + func_unwrapped = unwrap_heuristic(func, JITFunction) + descriptors = build_arg_descriptors(func_unwrapped, unit) + + # Python's multiprocessing.Pool class is not great at handling unexpected child + # failures such as segfaults. Account for this by temporarily installing a signal + # handler that considers such signals a catastrophic compilation failure. If not + # for this, the Pool will deadlock. + if threading.current_thread() is threading.main_thread(): + previous_child_handler = signal.signal(signal.SIGCHLD, sigchld_handler) + else: + previous_child_handler = None + + func = func_unwrapped + + # sanity check to make sure args with default values are always at the end + has_default_value_arg = False + for name in func.arg_names: + if name in default_values: + has_default_value_arg = True + elif has_default_value_arg: + raise RuntimeError( + f"default values must be at the end of the argument list. {func.arg_names=} {default_values=}" + ) + + h_out = f"{install_dir}/{prefix}.h" + cu_out = f"{install_dir}/{prefix}.cpp" + torch_out = f"{install_dir}/{prefix}_torch_op.cpp" + py_out = f"{install_dir}/{prefix}_meta.py" + + # Generate kernel.h file + h_content = generate_header_content( + tuned_func, # pyre-ignore[6]: Autotuner when set (verified by is_autotuner) + func, + unit, + descriptors, + tuner_fallback, + ) + + with open(h_out, "w") as fp: + fp.write(h_content) + + generated_specs = compile_specs_parallel( + unit.specs, + install_dir, + func.__module__, + func.__name__, + gpu_target, + import_module, + descriptors, + ) + # Generate kernel.cpp file + cu_content = generate_kernel_cpp_content( + func, unit, descriptors, prefix, generated_specs, gpu_target.backend + ) + with open(cu_out, "w") as fp: + fp.write(cu_content) + + # Generate torch_op.cpp file + torch_op_content = generate_torch_op_content( + func, descriptors, prefix, default_values + ) + + with open(torch_out, "w") as fp: + fp.write(torch_op_content) + + if tuned_func: + with open(py_out, "w") as fp: + fp.write(gen_tuner_meta_py(tuned_func, tuner_fallback, unit)) + else: + with open(py_out, "w") as fp: + fp.write(gen_tuner_meta_py(func, tuner_fallback, unit)) + + if previous_child_handler is not None: + signal.signal(signal.SIGCHLD, previous_child_handler) diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/compile/spec_processing.py b/recommendation_v4/generative_recommenders/ops/triton_aot/compile/spec_processing.py new file mode 100644 index 000000000..e8c181121 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton_aot/compile/spec_processing.py @@ -0,0 +1,593 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. + +# pyre-strict + +"""Kernel spec processing for AOT-T compilation. + +Transforms raw kernel specs (from infer_spec) into compiled specs ready +for Triton native compile and C++ codegen. +""" + +from __future__ import annotations + +import copy +import dataclasses +import logging +from dataclasses import dataclass +from typing import Any, cast + +# @manual=//triton:triton +import triton +from generative_recommenders.ops.triton_aot.compile.compile_state import hash_spec +from generative_recommenders.ops.triton_aot.shared.spec_conversion import ( + collect_constraints, + extract_constants, + get_fp8_replacement_signature_for_amd, + get_fp8_replacement_signature_for_sm80, + signature_list_to_dict, + SignatureElement, +) +from generative_recommenders.ops.triton_aot.shared.types import AUTOTUNE_ATTRs +from triton.backends.compiler import BaseBackend, GPUTarget +from triton.compiler.compiler import ASTSource +from triton.runtime.jit import JITFunction + +logger: logging.Logger = logging.getLogger(__name__) + +TRITON_VERSION: str = triton.__version__ + +# A raw kernel spec produced by infer_spec. The only key is "signature". +RawKernelSpec = dict[str, list[SignatureElement]] + + +@dataclass +class KernelSpec: + """A single compilation variant for a kernel. + + Each variant represents one combination of dtypes, constant values, + alignment constraints, and autotune configuration. Multiple variants + are grouped together in an ``OpsUnit``. + + Attributes: + signature: Non-constant arg index → dtype string (e.g., ``{0: "*fp32", 4: "i32"}``). + constants: Arg index → compile-time constant value. Includes bare + literals (128, ``"leaky_relu"``), absent optional tensors (None), + and equal-to-1 specializations (stride=1 → constexpr folding). + divisible_by_16: Indices of args whose values are divisible by 16. + For pointers this means the address is 16-byte aligned; + for scalars it means the value itself is a multiple of 16. + divisible_by_8: Indices of args whose values are divisible by 8. + Only meaningful for scalars (pointer alignment is always ≥16). + num_warps: Number of warps per block. + num_stages: Number of pipeline stages. + matrix_instr_nonkdim: AMD matrix instruction non-K dimension. + waves_per_eu: AMD waves per execution unit. + kpack: AMD kpack factor. + """ + + signature: dict[int, str] + constants: dict[int, Any] + divisible_by_16: set[int] + divisible_by_8: set[int] + num_warps: int = 4 + num_stages: int = 3 + matrix_instr_nonkdim: int = 0 + waves_per_eu: int = 1 + kpack: int = 1 + + +@dataclass +class OpsUnit: + """All compilation variants for a single kernel op. + + Groups per-kernel invariants with the list of ``KernelSpec`` variants. + Use ``OpsUnit.from_raw_specs()`` to build — it performs the complete + spec processing pipeline (convert → detect optional → validate → + autotune → dedup → compute invariants). + + Attributes: + cc: Compute capability (int for NVIDIA, str for AMD). + optional: Indices of optional tensor args (unified across all call sites). + pointer_args: Indices of all tensor pointer args (required + optional). + Invariant across specs — a pointer arg never becomes a non-pointer. + scalar_dtypes: Non-pointer signature arg index → widest dtype string + across all specs (e.g., ``"i32"``, ``"i64"``, ``"fp32"``). + Computed by ``_wider_type`` — individual specs may use narrower types. + constant_types: Python type per constant arg position (e.g., ``{15: int, 19: bool}``). + Excludes optional tensor args (None constants). + Invariant across specs — same Python type for each position. + specs: Per-variant compilation specs. + """ + + cc: int | str + optional: set[int] + pointer_args: set[int] + scalar_dtypes: dict[int, str] + constant_types: dict[int, type[Any]] + specs: list[KernelSpec] + + @classmethod + def from_raw_specs( + cls, + base_specs: list[RawKernelSpec], + gpu_target: GPUTarget, + tuned_func: triton.runtime.autotuner.Autotuner | None = None, + ) -> OpsUnit: + """Build an OpsUnit from raw kernel specs. + + Performs the complete spec processing pipeline: + 1. Convert raw specs to KernelSpecs + 2. Detect optional tensor args (cross-spec + 3-tuple) + 3. Validate consistency across converted specs + 4. Apply autotuning (if tuned_func provided) + 5. Deduplicate specs + 6. Compute shared invariants (pointer_args, scalar_dtypes, constant_types) + """ + # Validate raw specs upfront, before any rewriting. + num_params = _check_uniform_signature_length(base_specs) + specs, three_tuple_optional = _convert_raw_specs(base_specs, gpu_target) + optional = _detect_optional_args(specs) | three_tuple_optional + + _validate_converted_specs(specs, optional, num_params) + + # Plain @triton.jit kernels (no @triton.autotune) skip config expansion. + if tuned_func is not None: + specs = _autotune_specs(tuned_func, gpu_target, specs) + + specs = _dedup_specs(specs) + + pointer_args, scalar_dtypes, constant_types = _compute_invariants( + specs, optional + ) + + return cls( + cc=gpu_target.arch, + optional=optional, + pointer_args=pointer_args, + scalar_dtypes=scalar_dtypes, + constant_types=constant_types, + specs=specs, + ) + + +# --------------------------------------------------------------------------- +# Public helpers (used outside spec processing) +# --------------------------------------------------------------------------- + + +def gen_compile_arg( + spec: KernelSpec, + func: JITFunction[list[Any]], +) -> tuple[ASTSource]: + # ASTSource expects tuple-keyed dicts: {(idx,): value} for constants, + # {(idx,): [[attr_name, attr_val], ...]} for attrs. Tuple keys support + # nested paths into structured types (asserted by ASTSource.__init__). + new_signature = {} + new_constants = {} + param_names = list(func.signature.parameters.keys()) + for idx, param in enumerate(param_names): + if idx in spec.signature: + new_signature[param] = spec.signature[idx] + if idx in spec.constants: + new_constants[(idx,)] = spec.constants[idx] + new_signature[param] = "constexpr" + + # parse_attr("D") returns a fresh [["tt.divisibility", 16]] each call. + new_attrs = {(idx,): BaseBackend.parse_attr("D") for idx in spec.divisible_by_16} + + return ( + ASTSource( + func, + new_signature, + constexprs=new_constants, + attrs=new_attrs, + ), + ) + + +# --------------------------------------------------------------------------- +# Int width helpers +# --------------------------------------------------------------------------- + +_INT_WIDTH_RANK: dict[str, int] = {"i32": 0, "i64": 1} + + +def _wider_type(t1: str, t2: str) -> str: + """Return the wider of two scalar dtypes. + + Only i32/i64 widening is supported. All other types must match exactly. + """ + if t1 == t2: + return t1 + r1 = _INT_WIDTH_RANK.get(t1) + r2 = _INT_WIDTH_RANK.get(t2) + if r1 is not None and r2 is not None: + return t1 if r1 >= r2 else t2 + raise ValueError(f"Cannot widen incompatible types: {t1!r} vs {t2!r}") + + +# --------------------------------------------------------------------------- +# Private helpers — called by OpsUnit.from_raw_specs +# --------------------------------------------------------------------------- + + +def _detect_optional_args(specs: list[KernelSpec]) -> set[int]: + """Detect optional tensor args by cross-spec comparison. + + An arg at index ``i`` is optional if: + - Some specs have ``i`` in ``signature`` as a pointer type (``*...``) + - Other specs have ``constants[i] = None`` + + Single-spec None args (always-absent tensors) are NOT detected here + but are handled by ``_compute_invariants`` which adds any + ``constants[i] = None`` to ``pointer_args``. + """ + if len(specs) <= 1: + return set() + optional: set[int] = set() + all_indices: set[int] = set() + for spec in specs: + all_indices |= spec.signature.keys() + all_indices |= spec.constants.keys() + for i in all_indices: + has_pointer = any( + i in s.signature and s.signature[i].startswith("*") for s in specs + ) + has_none_const = any(i in s.constants and s.constants[i] is None for s in specs) + if has_pointer and has_none_const: + optional.add(i) + return optional + + +def _check_uniform_signature_length(base_specs: list[RawKernelSpec]) -> int: + """All raw specs must declare the same param count; return that count. + + Each raw spec is one ``infer_spec`` call site for the same kernel, + so all should have ``len(fn.signature.parameters)`` entries. Differing + lengths means upstream bug (mixed kernels, truncated spec, etc.) and + would surface later as silent IndexError or wrong bound checks. + """ + if not base_specs: + return 0 + sig_lens = {len(spec["signature"]) for spec in base_specs} + if len(sig_lens) != 1: + raise ValueError( + f"Raw specs declare inconsistent signature lengths: " + f"{sorted(sig_lens)}. All specs for the same kernel must have " + f"one entry per declared param." + ) + return sig_lens.pop() + + +def _check_arg_indices_in_range( + specs: list[KernelSpec], + num_params: int, +) -> None: + """Every spec arg index must be in ``[0, num_params)``. + + Out-of-range indices would silently drop in ``gen_compile_arg``'s + ``enumerate(param_names)`` loop. ``num_params <= 0`` disables the check. + """ + if num_params <= 0: + return + for idx, spec in enumerate(specs): + all_indices = ( + spec.signature.keys() + | spec.constants.keys() + | spec.divisible_by_16 + | spec.divisible_by_8 + ) + for i in all_indices: + if not 0 <= i < num_params: + raise ValueError( + f"Spec {idx}: arg index {i} out of range " + f"[0, {num_params}) — kernel has {num_params} declared params" + ) + + +def _collect_pointer_args( + specs: list[KernelSpec], + optional: set[int], +) -> set[int]: + """Collect all tensor pointer indices across all specs. + + Includes optional args (from _detect_optional_args) AND any arg + whose constant value is None (single-spec optional tensor case + where _detect_optional_args didn't fire). + """ + pointer_args: set[int] = set(optional) + for spec in specs: + for i, dtype in spec.signature.items(): + if dtype.startswith("*"): + pointer_args.add(i) + for i, val in spec.constants.items(): + if val is None: + pointer_args.add(i) + return pointer_args + + +def _collect_scalar_dtypes( + specs: list[KernelSpec], + pointer_args: set[int], +) -> dict[int, str]: + """Collect non-pointer signature arg dtypes, widening compatible int types. + + Invariant across specs (validated by _validate_converted_specs). + """ + scalar_dtypes: dict[int, str] = {} + for spec in specs: + for i, dtype in spec.signature.items(): + if i not in pointer_args: + if i in scalar_dtypes: + scalar_dtypes[i] = _wider_type(scalar_dtypes[i], dtype) + else: + scalar_dtypes[i] = dtype + return scalar_dtypes + + +def _collect_constant_types( + specs: list[KernelSpec], +) -> dict[int, type[Any]]: + """Collect Python type per constant position. + + Excludes None constants (optional tensor args — already in pointer_args). + """ + constant_types: dict[int, type[Any]] = {} + for spec in specs: + for i, val in spec.constants.items(): + if val is not None and i not in constant_types: + constant_types[i] = type(val) + return constant_types + + +def _compute_invariants( + specs: list[KernelSpec], + optional: set[int], +) -> tuple[set[int], dict[int, str], dict[int, type[Any]]]: + """Compute shared invariants from processed specs. + + Returns (pointer_args, scalar_dtypes, constant_types). + + When annotation-as-variant produces mixed partitions (arg in + ``signature`` in some specs, ``constants`` in others), the arg + appears in both ``scalar_dtypes`` and ``constant_types``. The + selector must receive it as a runtime parameter for dispatch, + so ``scalar_dtypes`` wins and the arg is removed from + ``constant_types``. + """ + pointer_args = _collect_pointer_args(specs, optional) + scalar_dtypes = _collect_scalar_dtypes(specs, pointer_args) + constant_types = _collect_constant_types(specs) + + # Resolve overlap: if any spec has the arg in signature (scalar), + # the selector needs it as a runtime parameter → not a constant. + for i in scalar_dtypes: + constant_types.pop(i, None) + + return pointer_args, scalar_dtypes, constant_types + + +def _validate_converted_specs( + specs: list[KernelSpec], + optional: set[int], + num_params: int = 0, +) -> None: + """Validate that converted specs are consistent before further processing. + + Checks that all specs produce identical C++ function signatures: + - All arg indices are in ``[0, num_params)`` (when ``num_params > 0``) + - Optional args: each spec has either a pointer in signature or None in constants + - Non-optional scalar args: same dtype (or compatible int widths) + - Non-optional constant args: same Python type + + Called after _convert_raw_specs + _detect_optional_args, before autotuning. + """ + _check_arg_indices_in_range(specs, num_params) + if len(specs) <= 1: + return + ref = specs[0] + for idx, spec in enumerate(specs[1:], 1): + _check_optional_consistency(ref, spec, idx, optional) + _check_signature_consistency(ref, spec, idx, optional) + _check_constants_consistency(ref, spec, idx, optional) + + +def _check_optional_consistency( + ref: KernelSpec, + spec: KernelSpec, + idx: int, + optional: set[int], +) -> None: + """Optional positions must be pointer-in-signature or None-in-constants. + + Validates that optional tensor args are not misclassified as scalars + or non-None constants, which would produce incompatible C++ types. + """ + for i in optional: + for label, s in [("spec 0", ref), (f"spec {idx}", spec)]: + if i in s.signature: + if not s.signature[i].startswith("*"): + raise ValueError( + f"Arg {i}: optional position has non-pointer type " + f"'{s.signature[i]}' in {label}" + ) + elif i in s.constants: + if s.constants[i] is not None: + raise ValueError( + f"Arg {i}: optional position has non-None constant " + f"{s.constants[i]!r} in {label}" + ) + + +def _check_signature_consistency( + ref: KernelSpec, + spec: KernelSpec, + idx: int, + optional: set[int], +) -> None: + """Non-optional, non-pointer scalar args must have compatible dtypes. + + Pointer args are skipped (different tensor dtypes are dispatched by + the dtype guard in ``gen_guarded_calls``). Compatible int widths + (i32/i64) are allowed — handled by ``_wider_type`` and int range guards. + Optional positions are validated by ``_check_optional_consistency``. + + Partition differences are allowed: an arg may be in ``signature`` in + one spec and in ``constants`` in another (e.g., annotation-as-variant + where stride=1 is constexpr in one spec but a runtime parameter in + another). The per-spec codegen handles this correctly. + """ + for i in ref.signature.keys() | spec.signature.keys(): + if i in optional: + continue + if (i in ref.signature and ref.signature[i].startswith("*")) or ( + i in spec.signature and spec.signature[i].startswith("*") + ): + continue + # Allow partition differences: arg in signature in one spec, + # in constants in another (annotation-as-variant pattern). + if i not in ref.signature or i not in spec.signature: + continue + if ref.signature[i] != spec.signature[i]: + r1 = _INT_WIDTH_RANK.get(ref.signature[i]) + r2 = _INT_WIDTH_RANK.get(spec.signature[i]) + if r1 is not None and r2 is not None: + continue + raise ValueError( + f"Arg {i}: dtype mismatch '{ref.signature[i]}' vs " + f"'{spec.signature[i]}' (spec 0 vs spec {idx})" + ) + + +def _check_constants_consistency( + ref: KernelSpec, + spec: KernelSpec, + idx: int, + optional: set[int], +) -> None: + """Non-optional constant args must have the same Python type across specs. + + C++ codegen uses one type per constant arg position (``PY_TYPES_TO_CPP_TYPES``), + so ``BLOCK_M=64`` (int) and ``BLOCK_M=64.0`` (float) would produce + incompatible launchers. Optional positions are validated separately + by ``_check_optional_consistency``. + """ + for i in ref.constants.keys() | spec.constants.keys(): + if i in optional: + continue + if ref.constants.get(i) is None or spec.constants.get(i) is None: + continue + if type(ref.constants[i]) is not type(spec.constants[i]): + raise ValueError( + f"Arg {i}: constant type mismatch " + f"{type(ref.constants[i]).__name__} vs " + f"{type(spec.constants[i]).__name__} (spec 0 vs spec {idx})" + ) + + +def _convert_raw_specs( + base_specs: list[RawKernelSpec], + gpu_target: GPUTarget, +) -> tuple[list[KernelSpec], set[int]]: + """Convert raw specs to KernelSpecs. + + Returns (specs, three_tuple_optional) where three_tuple_optional is the + union of optional_args detected from 3-tuple signature elements across + all specs (backward compat with ``collect_constraints``). + """ + raw_specs = cast(list[dict[str, Any]], copy.deepcopy(base_specs)) + is_amd = gpu_target.backend == "hip" + + result: list[KernelSpec] = [] + three_tuple_optional: set[int] = set() + for raw_spec in raw_specs: + constraints = collect_constraints(raw_spec["signature"]) + constants = extract_constants(raw_spec["signature"], constraints) + signature: dict[int, str] = signature_list_to_dict( + raw_spec["signature"], constants + ) + three_tuple_optional |= constraints.optional_args + + spec = KernelSpec( + signature=signature, + constants=constants, + divisible_by_16=constraints.divisible_by_16, + divisible_by_8=constraints.divisible_by_8, + ) + + if constraints.has_fp8: + if is_amd: + spec.signature = get_fp8_replacement_signature_for_amd( + {"signature": spec.signature}, {str(gpu_target.arch)} + ) + elif gpu_target.arch == 80: + spec.signature = get_fp8_replacement_signature_for_sm80( + {"signature": spec.signature} + ) + + result.append(spec) + + return result, three_tuple_optional + + +def _autotune_specs( + func: triton.runtime.autotuner.Autotuner, + target: GPUTarget, + specs: list[KernelSpec], +) -> list[KernelSpec]: + tuned_specs: list[KernelSpec] = [] + for spec in specs: + for cfg in func.cache.values(): + constants = spec.constants.copy() + for arg_name, arg_val in cfg.kwargs.items(): + if arg_name in AUTOTUNE_ATTRs: + continue + arg_idx = func.arg_names.index(arg_name) + if constants.get(arg_idx, -1) == -1: + constants[arg_idx] = arg_val + + autotune_values: dict[str, int] = {} + for name, default in AUTOTUNE_ATTRs.items(): + if name in cfg.kwargs: + autotune_values[name] = cfg.kwargs[name] + else: + autotune_values[name] = getattr(cfg, name, default) + # AMD has changed their software pipeliner in Triton + # It now expects num_stages == 2 instead of 0 + # see: https://github.com/pytorch/pytorch/pull/139881 + # if we see someone try to set num_stages == 0, set it to the default (2) instead + # We can't use the Triton hook to get the default value because it requires the AMD runtime to be loaded + if ( + target.backend == "hip" + and name == "num_stages" + and autotune_values[name] == 0 + and TRITON_VERSION >= "3.2.0" + ): + autotune_values[name] = 2 + + tuned_spec = dataclasses.replace( + spec, + constants=constants, + # pyrefly: ignore [bad-argument-type] + **autotune_values, + ) + tuned_specs.append(tuned_spec) + return tuned_specs + + +def _dedup_specs(specs: list[KernelSpec]) -> list[KernelSpec]: + deduped_specs: list[KernelSpec] = [] + duplicated_specs: list[KernelSpec] = [] + hash_spec_ids: set[str] = set() + for spec in specs: + id = hash_spec(dataclasses.asdict(spec)) + if id in hash_spec_ids: + duplicated_specs.append(spec) + else: + hash_spec_ids.add(id) + deduped_specs.append(spec) + + logger.debug( + f"[TritonAOT Dedup] {len(specs)=} {len(deduped_specs)=} {len(duplicated_specs)=}" + ) + return deduped_specs diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/compile/stable_types.py b/recommendation_v4/generative_recommenders/ops/triton_aot/compile/stable_types.py new file mode 100644 index 000000000..33038a534 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton_aot/compile/stable_types.py @@ -0,0 +1,35 @@ +# pyre-strict + +"""AOTT-local type mappings for stable ABI codegen. + +These replace ``shared.types.ATYPES`` and ``shared.types.PY_TYPES_TO_CPP_TYPES`` +with versions that have zero link dependency on ATen. The shared dicts are kept +unchanged so TritonCC is not affected. +""" + +from typing import Any + +# Stable ABI scalar type mapping: Triton pointer dtype → c10::ScalarType enum. +# Uses c10::ScalarType:: (from torch/headeronly/core/ScalarType.h) instead of +# at::kFloat aliases (which require ATen headers). +SCALAR_TYPES: dict[str, str] = { + "*i1": "c10::ScalarType::Bool", + "*u8": "c10::ScalarType::Byte", + "*i8": "c10::ScalarType::Char", + "*i16": "c10::ScalarType::Short", + "*i32": "c10::ScalarType::Int", + "*i64": "c10::ScalarType::Long", + "*fp16": "c10::ScalarType::Half", + "*fp32": "c10::ScalarType::Float", + "*fp64": "c10::ScalarType::Double", + "*bf16": "c10::ScalarType::BFloat16", + "*fp8e4nv": "c10::ScalarType::Float8_e4m3fn", + "*fp8e4b8": "c10::ScalarType::Float8_e4m3fnuz", +} + +# Stable ABI override: str → "std::string" instead of "at::string". +PY_TYPES_TO_CPP_TYPES: dict[type[Any], str] = { + int: "int64_t", + str: "std::string", + float: "double", +} diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/compile/triton_aot_compile.py b/recommendation_v4/generative_recommenders/ops/triton_aot/compile/triton_aot_compile.py new file mode 100644 index 000000000..74179435a --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton_aot/compile/triton_aot_compile.py @@ -0,0 +1,149 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# pyre-strict + +import importlib +import logging +import os +import pickle +from types import ModuleType, TracebackType +from typing import Any, Callable, Optional, Type + +from generative_recommenders.ops.triton_aot.build.extension_builder import ( + build_triton_aot_extension, +) +from generative_recommenders.ops.triton_aot.compile.codegen import ( + is_non_empty_mapping_of_type, +) +from generative_recommenders.ops.triton_aot.compile.compile_state import ( + get_aott_compile_path, + get_aott_compile_state, + get_triton_aot_kernel_specs, +) +from generative_recommenders.ops.triton_aot.compile.pipeline import compile_to_cpp +from generative_recommenders.ops.triton_aot.compile.utils import unwrap_heuristic +from torch import package +from triton.backends.compiler import GPUTarget +from triton.runtime import driver, JITFunction + +# @manual=//triton:triton +from triton.runtime.autotuner import Config + +logger: logging.Logger = logging.getLogger(__name__) + + +class TritonAOTCompile: + """ + Context manager to compile Triton kernels to C++ and build a shared library. + The compiled kernels are cached in a temporary directory. + + - package_importer: + torch.package importer for loading kernels source code (aott/ops). + If not provided, the default importlib is used (for local use cases) + - gpu_target: + GPU target to compile for (default: active GPU target, determined by Triton driver) + This local copy intentionally omits Manifold autotune-cache overrides. The + HSTU e2e path only needs representative-input autotuning captured during + the compile context. + """ + + def __init__( + self, + package_importer: Optional[package.PackageImporter] = None, + gpu_target: Optional[GPUTarget] = None, + auto_tune_cache_override_path: Optional[str] = None, + ) -> None: + self._import_module: Callable[[str], ModuleType] = ( + package_importer.import_module + if package_importer is not None + else importlib.import_module + ) + self.gpu_target: GPUTarget = gpu_target or driver.active.get_current_target() + self.auto_tune_cache_override_path: Optional[str] = ( + auto_tune_cache_override_path + ) + + def _load_autotune_cache_overrides( + self, + ) -> dict[str, Any]: + if self.auto_tune_cache_override_path is None: + return {} + raise NotImplementedError( + "Local generative_recommenders AOT-T compile does not support " + "auto_tune_cache_override_path." + ) + + def __enter__(self) -> None: + state = get_aott_compile_state() + state.reset() + state.enable() + logger.info( + f"Start AOTT compile, output dir: {get_aott_compile_path()}, gpu_target: {self.gpu_target}" + ) + + def _resolve_autotune_cache( + self, + fn: Any, + fn_name: str, + fn_dir: str, + overrides: dict[str, Any], + ) -> None: + """Apply override (if matched) and dump the autotune cache to fn_dir.""" + override = overrides.get(fn_name) + if override is not None: + logger.info( + f"[AOTT]: Overriding autotune cache for {fn_name} " + f"from {self.auto_tune_cache_override_path}" + ) + fn.cache = override + + # cache are dumped just for testing + if hasattr(fn, "cache") and is_non_empty_mapping_of_type(fn.cache, Config): + with open(f"{fn_dir}/{fn_name}_autotune_cache", "wb") as data: + # @lint-ignore PYTHONPICKLEISBAD + pickle.dump(fn.cache, data) + + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_value: Optional[BaseException], + traceback: Optional[TracebackType], + ) -> None: + compile_path = get_aott_compile_path() + if not os.path.exists(compile_path): + os.makedirs(compile_path) + + kernel_specs = get_triton_aot_kernel_specs() + auto_tune_overrides = self._load_autotune_cache_overrides() + + logger.info(f"[AOTT]: compiling {len(kernel_specs)} kernels") + + for fn, specs in kernel_specs.items(): + jit_fn = unwrap_heuristic(fn, JITFunction) + fn_name = jit_fn.__name__ + + logger.info(f"[AOTT]: compiling {fn_name} with specs: {specs}") + + module_suffix = jit_fn.__module__.rsplit(".", 1)[-1] + fn_dir = f"{compile_path}/{module_suffix}_{fn_name}" + if not os.path.exists(fn_dir): + os.makedirs(fn_dir) + + self._resolve_autotune_cache(fn, fn_name, fn_dir, auto_tune_overrides) + + compile_to_cpp( + func=fn, + base_specs=specs, + install_dir=f"{fn_dir}", + prefix=f"{fn_name}", + gpu_target=self.gpu_target, + tuner_fallback=True, + import_module=self._import_module, + ) + + build_triton_aot_extension( + source_dir=fn_dir, + kernel_name=fn_name, + output_dir=fn_dir, + ) + + get_aott_compile_state().disable() diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/compile/utils.py b/recommendation_v4/generative_recommenders/ops/triton_aot/compile/utils.py new file mode 100644 index 000000000..e3848fd2e --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton_aot/compile/utils.py @@ -0,0 +1,47 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. + +# pyre-strict + +import hashlib +from typing import Any, Type, TypeVar + +T = TypeVar("T") + + +def unwrap_heuristic(func: Any, return_type: Type[T]) -> T: + while not isinstance(func, return_type): + func = func.fn + if not hasattr(func, "fn"): + # pyre-fixme[7]: Incompatible return type [7]: Expected `Variable[T]` but got `None`. + return None + return func + + +def is_autotuner(obj: Any) -> bool: + """Check whether *obj* is a Triton Autotuner using duck typing. + + In Buck builds the ``Autotuner`` class can be loaded from multiple module + paths (e.g. via ``torch.package`` re-imports), causing ``isinstance`` to + return ``False`` for genuine Autotuner instances. We combine a class-name + check with duck-typing on the attributes that callers actually need + (``cache``, ``configs``, ``arg_names``), making detection robust against + module-path aliasing. + """ + return "Autotuner" in type(obj).__name__ and all( + hasattr(obj, attr) for attr in ("cache", "configs", "arg_names") + ) + + +def hash_kernel_name(kernel_name: str) -> str: + """Hash kernel name to create shorter, filesystem-safe names. + + Args: + kernel_name: Full kernel name (can be very long with specialization suffixes). + e.g., "_addmm_fwd_sm80_pfp32_pfp32_pfp32_pfp32_i32_..." + + Returns: + Hashed name in format "kernel_". + e.g., "kernel_a1b2c3d4e5f6..." + + """ + return "kernel_" + hashlib.sha256(kernel_name.encode("utf-8")).hexdigest() diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/preprocess.py b/recommendation_v4/generative_recommenders/ops/triton_aot/preprocess.py new file mode 100644 index 000000000..ce2e63d43 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton_aot/preprocess.py @@ -0,0 +1,76 @@ +# (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +# pyre-strict + +""" +Preprocessing utilities for triton_aot models before AOT compilation. +""" + +import logging + +from tgif.fx.tgif_tracer import TGIFTracer +from torch.fx import GraphModule + +logger: logging.Logger = logging.getLogger(__name__) + +# "aot_triton_kernel_wrapper_" is a pre-defined prefix for +# AOT-T triton kernel wrapper functions. This is required for +# AOT-T backend to recognize and trace correctly for ops transformation. +AOTT_WRAPPER_PREFIX: str = "aot_triton_kernel_wrapper_" + + +def unwrap_aott_wrapper_nodes(fx_m: GraphModule, tracer: TGIFTracer) -> GraphModule: + """Mark ``aot_triton_kernel_wrapper_*`` FX nodes as unwrapped and re-trace. + + In the traced FX graph, outer wrapper functions (prefixed with + ``aot_triton_kernel_wrapper_``) are ``@torch.fx.wrap`` leaves. + Setting ``node.meta["is_wrapped"] = False`` causes a subsequent + ``symbolic_trace`` to trace *through* them, exposing the inner + ``@torch.fx.wrap`` functions (e.g., ``_triton_aot_grouped_gemm``) + that contain the actual kernel calls. + + Any ``_body_transformer`` hook (e.g. one registered by + ``early_return_fx_code_transform``) is temporarily removed before + re-tracing to avoid injecting un-traceable control flow + (``if Proxy: …``) into the generated ``forward``. After re-trace + the hook is restored on the new module. See P2266562545. + + Args: + fx_m: The FX GraphModule to modify **in-place** before re-trace. + tracer: Tracer instance used for the re-trace step. + + Returns: + The re-traced ``GraphModule`` with AOTT wrappers expanded. + """ + logger.info("Re-trace to get the AOTT node exposed.") + + # Save and clear the body transformer so that re-trace does not hit + # ``if Proxy:`` from code-level hooks like early_return_fx_code_transform. + saved_body_transformer = fx_m.graph._codegen._body_transformer + fx_m.graph._codegen._body_transformer = None + + unwrap_count = 0 + for node in fx_m.graph.nodes: + if node.op == "call_function": + target = node.target + if hasattr(target, "__name__") and target.__name__.startswith( + AOTT_WRAPPER_PREFIX + ): + logger.info(f"[AOTT] Found inference wrapper node: {node=}") + node.meta["is_wrapped"] = False + unwrap_count += 1 + + if unwrap_count > 0: + logger.info(f"[AOTT] Found {unwrap_count} inference wrapper nodes.") + fx_m.recompile() + else: + logger.warning("[AOTT] No inference wrapper node found. Skip re-compile.") + + result = tracer.symbolic_trace(fx_m) + + # Restore the body transformer on the new module. + if saved_body_transformer is not None: + result.graph._codegen._body_transformer = saved_body_transformer + result.recompile() + + return result diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/shared/compat.py b/recommendation_v4/generative_recommenders/ops/triton_aot/shared/compat.py new file mode 100644 index 000000000..be6235701 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton_aot/shared/compat.py @@ -0,0 +1,91 @@ +# pyre-strict +""" +This module provides shared utilities that handle differences between +Triton versions. +""" + +from typing import Any + +# @manual=//triton:triton +import triton +from packaging.version import Version +from triton.runtime.jit import JITFunction + +TRITON_VERSION: str = triton.__version__ + + +def version_gte(version: str, target: str) -> bool: + """ + Check if version >= target using semantic version comparison. + Simple string comparison fails for versions like "3.10" vs "3.5" + """ + return Version(version) >= Version(target) + + +def get_kernel_name(jit_fn: JITFunction[Any]) -> str: + """ + Get the simple kernel name from a JITFunction. + + In Triton 3.5+, JITFunction._fn_name returns the full qualified name + (e.g., "generative_recommenders.ops.triton_aot.triton_addmm._addmm_fwd"). + In older versions, it returns just the simple name (e.g., "_addmm_fwd"). + + This function normalizes the behavior to always return the simple name. + + Args: + jit_fn: A Triton JITFunction + + Returns: + The simple kernel name (e.g., "_addmm_fwd") + """ + fn_name = jit_fn._fn_name + if version_gte(TRITON_VERSION, "3.5"): + # Triton 3.5+ uses get_full_name(fn) which returns qualified name + return fn_name.rsplit(".", 1)[-1] + else: + # Older versions use fn.__name__ which is already simple + return fn_name + + +def get_scratch_parameters(kernel: Any) -> tuple[str, list[str]]: + """ + Get scratch parameter declarations and argument pointers for the kernel launcher. + + Scratch parameters are backend and version-specific features for profiling + and global memory management. + + Detection Strategy: + 1. Check metadata first for each parameter + 2. Fall back to version-based detection if metadata unavailable + + Version Requirements (fallback): + - v3.4+: both global_scratch and profile_scratch + - v3.3: only global_scratch + - v3.2 and earlier: no scratch parameters + + Args: + kernel: Compiled Triton kernel with metadata attribute + + Returns: + Tuple of (declarations, arg_pointers): + - declarations: C++ variable declarations for scratch parameters + - arg_pointers: List of argument pointers to append to kernel args + """ + declarations = [] + arg_pointers = [] + + if hasattr(kernel.metadata, "global_scratch_size"): + declarations.append("CUdeviceptr global_scratch = 0;") + arg_pointers.append("&global_scratch") + elif version_gte(TRITON_VERSION, "3.3"): + declarations.append("CUdeviceptr global_scratch = 0;") + arg_pointers.append("&global_scratch") + + if hasattr(kernel.metadata, "profile_scratch_size"): + declarations.append("CUdeviceptr profile_scratch = 0;") + arg_pointers.append("&profile_scratch") + elif version_gte(TRITON_VERSION, "3.4"): + declarations.append("CUdeviceptr profile_scratch = 0;") + arg_pointers.append("&profile_scratch") + + return ("\n ".join(declarations), arg_pointers) diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/shared/spec_conversion.py b/recommendation_v4/generative_recommenders/ops/triton_aot/shared/spec_conversion.py new file mode 100644 index 000000000..4a1ebb133 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton_aot/shared/spec_conversion.py @@ -0,0 +1,389 @@ +# pyre-strict + +"""Functions for converting kernel specs to architecture-specific formats. + +A "spec" (specification) describes how to compile a Triton kernel for a specific +set of input shapes and types. Users provide "base specs" in a human-friendly +format that describes kernel arguments: + + {"signature": [("*fp32", 16), ("*bf16", 16), ("i32", None), 128]} + +This format encodes dtypes, alignment hints, and constant values together. +Before compilation, base specs must be converted to "compiled specs" that +separate this information into distinct fields the compiler understands: + + {"signature": {0: "*fp32", 1: "*bf16"}, + "constants": {2: None, 3: 128}, + "configs": (instance_descriptor(...),), + "cc": 80} + +This module provides the functionality to perform this transformation, +extracting constraints, identifying constants, and preparing specs for each +target GPU architecture. +""" + +from collections import namedtuple +from dataclasses import dataclass +from typing import Any, TypeAlias + +from generative_recommenders.ops.triton_aot.shared.types import CTYPES + +# Compile-time constant values that can appear in signatures or be returned +# by constexpr(). These are values the compiler can fold into generated code. +ConstantValue: TypeAlias = str | int | float | bool | None + +# A single element in a kernel signature list. +# Can be: dtype string, (dtype, alignment) tuple, (dtype, alignment, has_value) +# triple for optional args, or a bare literal constant. +SignatureElement: TypeAlias = ( + ConstantValue | tuple[str, int | None] | tuple[str, int | None, bool] +) + + +instance_descriptor = namedtuple( + "instance_descriptor", + [ + "divisible_by_16", + "equal_to_1", + "ids_of_folded_args", + "divisible_by_8", + ], +) + + +def constexpr(s: SignatureElement) -> ConstantValue: + """Identify compile-time constant expressions in signature elements. + + Args: + s: A signature element. + + Returns: + The constant value if s is a compile-time constant, None otherwise. + Constants are: int, float, bool, or strings that aren't dtype names. + """ + expr = s[0] if isinstance(s, tuple) and len(s) > 1 else s + + if expr is None: + return expr + + try: + ret = int(expr) + return ret + except (ValueError, TypeError): + pass + try: + ret = float(expr) + return ret + except (ValueError, TypeError): + pass + + if isinstance(expr, bool): + return expr + if isinstance(expr, str) and expr not in CTYPES and not expr.startswith("*"): + return expr + return None + + +@dataclass +class SignatureConstraints: + """Constraints extracted from parsing a kernel signature. + + When compiling a Triton kernel, the compiler can generate more efficient + code if it knows certain properties about the arguments: + + - Pointer alignment: If a pointer is always 16-byte aligned, the compiler + can use faster aligned memory operations. + - Constant values: Arguments known at compile time can be folded into the + generated code, eliminating runtime checks. + - FP8 dtypes: Some GPU architectures require dtype substitutions for FP8 + types (e.g., gfx942 needs fp8e4b8 instead of fp8e4nv). + + This dataclass collects all these constraints from a single pass over the + signature, so downstream code can use them without re-parsing. + + Attributes: + divisible_by_16: Indices of args with values divisible by 16. + divisible_by_8: Indices of args with values divisible by 8. + equal_to_1: Indices of args with value equal to 1. + none_args: Indices of args that are None (not provided). + optional_args: Indices of optional arguments. + has_fp8: Whether any argument has an FP8 dtype. + """ + + divisible_by_16: set[int] + divisible_by_8: set[int] + equal_to_1: set[int] + none_args: set[int] + optional_args: set[int] + has_fp8: bool + + +def collect_constraints(signature: list[SignatureElement]) -> SignatureConstraints: + """Collect divisibility and type constraints from a signature list. + + Iterates through signature elements and identifies: + - Arguments divisible by 16 or 8 (for memory alignment) + - Arguments equal to 1 (for optimization) + - Optional arguments and those not provided (None) + - Whether any FP8 dtypes are present + + Args: + signature: List of signature elements. The input format is unfortunately + variable; each element can be one of several types: + + 1. Plain string (dtype only, no alignment info): + "*fp32" - A float32 pointer + "i32" - A 32-bit integer scalar + + 2. Tuple of (dtype, value) where value indicates alignment or constness: + ("*fp32", 16) - Float32 pointer, 16-byte aligned + ("i32", None) - Integer arg not provided (becomes constant None) + ("*bf16", 1) - Pointer with value=1 (folded as constant) + + 3. Triple of (dtype, value, has_value) for optional arguments: + ("*fp32", 16, True) - Optional arg that IS provided, 16-byte aligned + ("*fp32", 16, False) - Optional arg NOT provided (becomes None) + + 4. Bare literals (become compile-time constants): + 128 - Integer constant + "leaky_relu" - String constant (e.g., activation name) + + Returns: + SignatureConstraints with all constraint sets populated. + + Example: + >>> sig = [("*fp32", 16), ("i32", None), ("*fp8e4nv", 8)] + >>> c = collect_constraints(sig) + >>> 0 in c.divisible_by_16 + True + >>> c.has_fp8 + True + """ + divisible_by_16: set[int] = set() + divisible_by_8: set[int] = set() + equal_to_1: set[int] = set() + none_args: set[int] = set() + optional_args: set[int] = set() + has_fp8: bool = False + + for i, s in enumerate(signature): + # Handle optional tensor case: tuple with 3 elements where s[2] indicates + # whether the optional arg has a value + if isinstance(s, tuple) and len(s) > 2: + optional_args.add(i) + # pyrefly: ignore [bad-index] + if not s[2]: # has_value is False + none_args.add(i) + continue + + # Extract dtype + dtype = s[0] if isinstance(s, tuple) else s + + # Check for FP8 types + if isinstance(dtype, str) and ("fp8e4nv" in dtype or "fp8e4b8" in dtype): + has_fp8 = True + + # Extract value (alignment or constant) + value = s[1] if isinstance(s, tuple) else s + + # Check divisibility and equality constraints + if isinstance(value, int): + if value % 16 == 0: + divisible_by_16.add(i) + if value % 8 == 0: + divisible_by_8.add(i) + if value == 1: + equal_to_1.add(i) + + if value is None: + none_args.add(i) + + return SignatureConstraints( + divisible_by_16=divisible_by_16, + divisible_by_8=divisible_by_8, + equal_to_1=equal_to_1, + none_args=none_args, + optional_args=optional_args, + has_fp8=has_fp8, + ) + + +def make_instance_descriptor( + constraints: SignatureConstraints, +) -> tuple[instance_descriptor]: + """Create an instance_descriptor tuple from constraints. + + Args: + constraints: The collected signature constraints. + + Returns: + A tuple containing a single instance_descriptor namedtuple with + divisible_by_16, equal_to_1, ids_of_folded_args, and divisible_by_8. + """ + ids_of_folded_args = constraints.equal_to_1 | constraints.none_args + return ( + instance_descriptor( + divisible_by_16=constraints.divisible_by_16, + equal_to_1=constraints.equal_to_1, + ids_of_folded_args=ids_of_folded_args, + divisible_by_8=constraints.divisible_by_8, + ), + ) + + +def extract_constants( + signature: list[SignatureElement], + constraints: SignatureConstraints, +) -> dict[int, ConstantValue]: + """Extract compile-time constant values from signature elements. + + Identifies arguments that can be folded into generated code at compile time. + Constants come from three sources: + + 1. Bare literals in the signature (e.g., 128 for block size, "leaky_relu" + for activation type, True for a boolean flag) + 2. Arguments with value=1 (tracked in constraints.equal_to_1) + 3. Arguments not provided (tracked in constraints.none_args) + + Args: + signature: List of signature elements in input format. + constraints: The collected signature constraints. + + Returns: + Dict mapping argument indices to their constant values. + """ + # Use constexpr to identify constant expressions + constexprs = {i: constexpr(s) for i, s in enumerate(signature)} + constants: dict[int, ConstantValue] = { + k: v for k, v in constexprs.items() if v is not None + } + + # Add equal_to_1 args with value 1 + for k in constraints.equal_to_1: + constants[k] = 1 + + # Add none_args with value None + for k in constraints.none_args: + constants[k] = None + + return constants + + +def signature_list_to_dict( + signature: list[SignatureElement], + constants: dict[int, ConstantValue], +) -> dict[int, str]: + """Convert signature from list format to dict format. + + Transforms the input signature list into a dict mapping argument + indices to dtype strings. Arguments that are constants are excluded + since they don't need runtime type information. + + Args: + signature: List of signature elements in input format. + constants: Dict of constant argument indices to exclude. + + Returns: + Dict mapping non-constant argument indices to their dtype strings. + """ + result: dict[int, str] = {} + for i, s in enumerate(signature): + if i in constants: + continue + # After filtering out constants, remaining elements are dtype declarations. + # For tuples like ("*fp32", 16), s[0] is the dtype string. + # For plain strings like "*fp32", the element itself is the dtype. + if isinstance(s, tuple) and len(s) > 1: + dtype = s[0] + else: + dtype = s + assert isinstance(dtype, str) + result[i] = dtype + return result + + +# CC (compute capability) to AMD GPU architecture mapping +# CC is a 2-digit shorthand: 94 -> gfx942, 95 -> gfx950 +HIP_CC_TO_ARCH_INFO: dict[int, str] = { + 90: "gfx90a", + 94: "gfx942", + 95: "gfx950", +} + +# Reverse mapping: architecture string -> CC string +HIP_ARCH_TO_CC: dict[str, str] = {v: str(k) for k, v in HIP_CC_TO_ARCH_INFO.items()} + +HIP_CC_MI350X: str = "95" # CC string for gfx950 (MI350X/MI355X) + + +def _normalize_cc(cc: set[str]) -> set[str]: + """Normalize CC values to 2-digit format for internal comparison. + + Accepts both tritoncc format ("94", "95") and Triton driver format + ("gfx942", "gfx950"). Returns 2-digit CC strings. + """ + return {HIP_ARCH_TO_CC.get(c, c) for c in cc} + + +def get_fp8_replacement_signature_for_amd( + spec: dict[str, Any], cc: set[str] +) -> dict[int, str]: + """Replace FP8 dtypes in signature for AMD architectures. + + Args: + spec: Compiled spec dict with 'signature' in dict format. + cc: Set of CC strings in either format: + - 2-digit tritoncc format: {"94"} for gfx942 + - Triton driver format: {"gfx942"} + See HIP_CC_TO_ARCH_INFO. + + Returns: + Dict mapping argument indices to dtype strings with FP8 types replaced. + """ + normalized_cc: set[str] = _normalize_cc(cc) + + def replace_fp8_type(dtype_str: str) -> str: + if "fp8e4nv" in dtype_str: + if HIP_CC_MI350X not in normalized_cc: + return dtype_str.replace("fp8e4nv", "fp8e4b8") + elif "fp8e4b8" in dtype_str and HIP_CC_MI350X in normalized_cc: + return dtype_str.replace("fp8e4b8", "fp8e4nv") + return dtype_str + + replace_fp8_signatures: dict[int, str] = {} + for key, value in spec["signature"].items(): + if isinstance(value, str): + replace_fp8_signatures[key] = replace_fp8_type(value) + else: + replace_fp8_signatures[key] = value + + return replace_fp8_signatures + + +def get_fp8_replacement_signature_for_sm80( + spec: dict[str, Any], +) -> dict[int, Any]: + """Replace FP8 dtypes with bf16 for SM80 (A100) which lacks native FP8 support. + + Args: + spec: Compiled spec dict with 'signature' in dict format. + + Returns: + Dict mapping argument indices to dtype strings with FP8 types replaced by bf16. + """ + + def replace_fp8_type(dtype_str: str) -> str: + if "fp8e4nv" in dtype_str: + return dtype_str.replace("fp8e4nv", "bf16") + return dtype_str + + replace_fp8_signatures: dict[int, Any] = {} + for key, value in spec["signature"].items(): + if isinstance(value, tuple) and isinstance(value[0], str): + replace_fp8_signatures[key] = (replace_fp8_type(value[0]), value[1]) + elif isinstance(value, str): + replace_fp8_signatures[key] = replace_fp8_type(value) + else: + replace_fp8_signatures[key] = value + + return replace_fp8_signatures diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/shared/types.py b/recommendation_v4/generative_recommenders/ops/triton_aot/shared/types.py new file mode 100644 index 000000000..6a870fc4d --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton_aot/shared/types.py @@ -0,0 +1,58 @@ +# pyre-strict + +"""Shared type definitions for AOTT and Triton CC. + +This module contains fundamental type mappings used across the compiler. +""" + +from typing import Any + +# Mapping from Triton dtype names to C type names +CTYPES: dict[str, str] = { + "i1": "bool", + "u8": "uint8_t", + "i8": "int8_t", + "i16": "int16_t", + "i32": "int32_t", + "i64": "int64_t", + "fp16": "half", + "fp32": "float", + "fp64": "double", + "bf16": "__nv_bfloat16", + "fp8e4nv": "__nv_fp8_e4m3", + "fp8e4b8": "__hip_fp8_e4m3_fnuz", +} + +# Mapping from Triton pointer dtype names to ATen scalar types +ATYPES: dict[str, str] = { + "*i1": "at::kBool", + "*u8": "at::kByte", + "*i8": "at::kChar", + "*i16": "at::kShort", + "*i32": "at::kInt", + "*i64": "at::kLong", + "*fp16": "at::kHalf", + "*fp32": "at::kFloat", + "*fp64": "at::kDouble", + "*bf16": "at::kBFloat16", + "*fp8e4nv": "at::kFloat8_e4m3fn", + "*fp8e4b8": "at::kFloat8_e4m3fnuz", +} + +# Mapping from Python types to C++ type names +PY_TYPES_TO_CPP_TYPES: dict[type[Any], str] = { + int: "int64_t", + str: "at::string", + float: "double", +} + +# Default values for autotuning attributes. +# These are used as default kernel launch parameters. +AUTOTUNE_ATTRs: dict[str, int] = { + "num_warps": 4, + "num_stages": 3, + # AMD only + "matrix_instr_nonkdim": 0, + "waves_per_eu": 1, + "kpack": 1, +} diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/templates/embedded_cubins.cpp b/recommendation_v4/generative_recommenders/ops/triton_aot/templates/embedded_cubins.cpp new file mode 100644 index 000000000..d269c3555 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton_aot/templates/embedded_cubins.cpp @@ -0,0 +1,7 @@ +#include + +extern "C" { +// __TRITON_AOT_GENERATE_BEGIN__ CUBIN_ARRAYS +// placeholder +// __TRITON_AOT_GENERATE_END__ CUBIN_ARRAYS +} diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/templates/kernel.cpp b/recommendation_v4/generative_recommenders/ops/triton_aot/templates/kernel.cpp new file mode 100644 index 000000000..7f882f11f --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton_aot/templates/kernel.cpp @@ -0,0 +1,104 @@ +// __TRITON_AOT_GENERATE_BEGIN__ HEADER_INCLUDE +#include "kernel.h" +// __TRITON_AOT_GENERATE_END__ HEADER_INCLUDE +// These headers are used by code generated at runtime in KERNEL_SPECS blocks +#include +#include // NOLINT(facebook-unused-include-check) + +inline void triton_aot_cu_check(CUresult err, const char* file, int line) { + if (err != CUDA_SUCCESS) { + const char* err_str; + cuGetErrorString(err, &err_str); + throw std::runtime_error( + std::string(file) + ":" + std::to_string(line) + + " CUDA driver error: " + (err_str ? err_str : "unknown")); + } +} +#define TRITON_AOT_CU_CHECK(EXPR) triton_aot_cu_check(EXPR, __FILE__, __LINE__) + +// NOLINTNEXTLINE(facebook-hte-NullableReturn): error path throws +inline cudaStream_t triton_aot_get_current_stream() { + auto device_idx = torch::stable::accelerator::getCurrentDeviceIndex(); + void* stream_ptr = nullptr; + // TODO: No torch::stable op provides the same functionality + // today. Revisit if torch exposes a proper stable::accelerator stream API. + if (aoti_torch_get_current_cuda_stream(device_idx, &stream_ptr) != 0) { + throw std::runtime_error("Failed to get current CUDA stream"); + } + return reinterpret_cast(stream_ptr); +} + +namespace triton { +namespace aot { + +namespace { +[[maybe_unused]] int compute_capability() { + // Cached: AOTT hosts use homogeneous GPUs. + static int cc = 0; + if (cc == 0) { + CUdevice device; + TRITON_AOT_CU_CHECK(cuCtxGetDevice(&device)); + int major, minor; + TRITON_AOT_CU_CHECK(cuDeviceGetAttribute( + &major, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, device)); + TRITON_AOT_CU_CHECK(cuDeviceGetAttribute( + &minor, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, device)); + cc = major * 10 + minor; + } + return cc; +} +} // namespace + +namespace { +#ifdef USE_ROCM +[[maybe_unused]] void check_errors(int shared, hipFunction_t func) { + // HIP doesn't need the same shared memory configuration as CUDA + return; +} +#else +[[maybe_unused]] void check_errors(int shared, CUfunction func) { + int shared_optin; + int device = 0; + TRITON_AOT_CU_CHECK(cuDeviceGetAttribute( + &shared_optin, + CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_BLOCK_OPTIN, + device)); + if (shared > 49152 && shared_optin > 49152) { + // If requested/shared_optin exceed 48 KB, it switches cache to prefer + // shared memory and sets the max dynamic shared memory so the kernel can + // allocate the larger amount needed. + TRITON_AOT_CU_CHECK( + cuFuncSetCacheConfig(func, CU_FUNC_CACHE_PREFER_SHARED)); + int shared_total, shared_static; + TRITON_AOT_CU_CHECK(cuDeviceGetAttribute( + &shared_total, + CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_MULTIPROCESSOR, + device)); + TRITON_AOT_CU_CHECK(cuFuncGetAttribute( + &shared_static, CU_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES, func)); + TRITON_AOT_CU_CHECK(cuFuncSetAttribute( + func, + CU_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES, + shared_optin - shared_static)); + } +} +#endif +} // namespace + +// __TRITON_AOT_GENERATE_BEGIN__ KERNEL_SPECS +// __TRITON_AOT_GENERATE_END__ KERNEL_SPECS + +// __TRITON_AOT_GENERATE_BEGIN__ SELECTOR +// __TRITON_AOT_GENERATE_END__ SELECTOR + +} // namespace aot +} // namespace triton + +// Anchor: keeps the inline `triton_aot_get_current_stream` (and its reference +// to `aoti_torch_get_current_cuda_stream`) from being dead-stripped at +// buck-build time, where KERNEL_SPECS is empty. `weak` dedups the symbol +// across the per-op .so files generated by the runtime template substitution. +extern "C" __attribute__((weak, visibility("default"))) cudaStream_t +__triton_aot_anchor_get_stream() { + return triton_aot_get_current_stream(); +} diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/templates/kernel.h b/recommendation_v4/generative_recommenders/ops/triton_aot/templates/kernel.h new file mode 100644 index 000000000..6b6f76e17 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton_aot/templates/kernel.h @@ -0,0 +1,36 @@ +#pragma once + +#include + +#include +#include + +namespace triton { +namespace aot { + +#ifndef GRID_DIM_DEFINED_MACRO +struct gridDims { + int x = 1; + int y = 1; + int z = 1; + cudaStream_t stream = nullptr; + gridDims(int _x = 1, int _y = 1, int _z = 1, cudaStream_t _stream = nullptr) + : x(_x), y(_y), z(_z), stream(_stream) {} +}; +#define GRID_DIM_DEFINED_MACRO +#endif + +#ifndef FITS_I32_DEFINED_MACRO +constexpr bool fits_i32(int64_t v) { + return v >= INT32_MIN && v <= INT32_MAX; +} +#define FITS_I32_DEFINED_MACRO +#endif + +// __TRITON_AOT_GENERATE_BEGIN__ TUNER_META_CPP +// __TRITON_AOT_GENERATE_END__ TUNER_META_CPP +// __TRITON_AOT_GENERATE_BEGIN__ SELECTOR_PROTO +// __TRITON_AOT_GENERATE_END__ SELECTOR_PROTO + +} // namespace aot +} // namespace triton diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/templates/template_utils.py b/recommendation_v4/generative_recommenders/ops/triton_aot/templates/template_utils.py new file mode 100644 index 000000000..d89bbe8b6 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton_aot/templates/template_utils.py @@ -0,0 +1,96 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. + +# pyre-strict + +""" +Common utilities for template loading and rendering. + +This module provides functions to load template files from the templates +directory and render them by replacing marker blocks with actual values. +""" + +import re +from collections import Counter +from importlib import resources + + +def load_template(name: str) -> str: + """Load template file content from Buck resources. + + Templates are loaded from the resources bundled with this package via Buck's + select_accelerator mechanism: + - AMD builds: hipified templates with HIP APIs + - NVIDIA builds: original templates with CUDA APIs + + Args: + name: Template filename (e.g., 'kernel.cpp', 'embedded_cubins.cpp'). + + Returns: + The template file content as a string. + """ + return resources.files(__package__).joinpath(name).read_text() + + +def render_template(template: str, replacements: dict[str, str]) -> str: + """Replace block markers in template with actual values. + + Replaces content between "// __TRITON_AOT_GENERATE_BEGIN__ NAME" + and "// __TRITON_AOT_GENERATE_END__ NAME" with the value for key "NAME". + Each key must have exactly one BEGIN/END pair in the template. + The markers are preserved for easier debugging. + + Args: + template: Template string containing marker blocks. + replacements: Dict mapping marker names to replacement values. + + Returns: + Rendered template with all marker blocks replaced. + + Raises: + AssertionError: If markers are duplicated, mismatched, or keys don't match. + """ + BEGIN_PREFIX = "// __TRITON_AOT_GENERATE_BEGIN__ " + END_PREFIX = "// __TRITON_AOT_GENERATE_END__ " + + begin_keys = re.findall(r"// __TRITON_AOT_GENERATE_BEGIN__ (\w+)", template) + end_keys = re.findall(r"// __TRITON_AOT_GENERATE_END__ (\w+)", template) + + # Check for duplicate keys + begin_key_counts = Counter(begin_keys) + end_key_counts = Counter(end_keys) + for key, count in begin_key_counts.items(): + assert count == 1, f"Duplicate BEGIN marker for key: {key}" + for key, count in end_key_counts.items(): + assert count == 1, f"Duplicate END marker for key: {key}" + + # Check BEGIN and END keys match + template_keys = set(begin_keys) + assert template_keys == set(end_keys), ( + f"Mismatched BEGIN/END markers: BEGIN={template_keys}, END={set(end_keys)}" + ) + + # Validate keys match between template and replacements + replacement_keys = set(replacements.keys()) + assert template_keys == replacement_keys, ( + f"Keys mismatch: in template but not in replacements: {template_keys - replacement_keys}, " + f"in replacements but not in template: {replacement_keys - template_keys}" + ) + + # Do the replacements + result = template + for key, value in replacements.items(): + begin_marker = f"{BEGIN_PREFIX}{key}" + end_marker = f"{END_PREFIX}{key}" + + begin_idx = result.find(begin_marker) + newline_idx = result.find("\n", begin_idx) + assert newline_idx != -1, ( + f"BEGIN marker for key '{key}' must be followed by newline" + ) + content_start = newline_idx + 1 + end_idx = result.find(end_marker, begin_idx) + + result = result[:content_start] + value + result[end_idx:] + + return result diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/templates/torch_op.cpp b/recommendation_v4/generative_recommenders/ops/triton_aot/templates/torch_op.cpp new file mode 100644 index 000000000..c2e5a4063 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton_aot/templates/torch_op.cpp @@ -0,0 +1,22 @@ +// __TRITON_AOT_GENERATE_BEGIN__ HEADER_INCLUDE +#include "kernel.h" +// __TRITON_AOT_GENERATE_END__ HEADER_INCLUDE +#include +#include // NOLINT(facebook-unused-include-check) + +// __TRITON_AOT_GENERATE_BEGIN__ TORCH_OP +namespace { +// no-op, force link StableLibrary +torch::stable::Tensor _triton_aot_placeholder_noop( + torch::stable::Tensor input) { + return input; +} +} // namespace + +STABLE_TORCH_LIBRARY_FRAGMENT(triton_aot, m) { + m.def("_placeholder_noop(Tensor input) -> Tensor"); +} +STABLE_TORCH_LIBRARY_IMPL(triton_aot, CPU, m) { + m.impl("_placeholder_noop", TORCH_BOX(&_triton_aot_placeholder_noop)); +} +// __TRITON_AOT_GENERATE_END__ TORCH_OP diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/transform/import_utils.py b/recommendation_v4/generative_recommenders/ops/triton_aot/transform/import_utils.py new file mode 100644 index 000000000..e3c4f1955 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton_aot/transform/import_utils.py @@ -0,0 +1,89 @@ +# pyre-strict + +""" +Import-header utilities for triton_aot codegen. +""" + +import ast + +from torch import package + + +def get_original_import_header(source_code: str) -> str: + """Extract all import statements from *source_code* as a single string.""" + tree = ast.parse(source_code) + import_header = "" + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom): + import_header += ast.unparse(node) + "\n" + elif isinstance(node, ast.Import): + import_header += ast.unparse(node) + "\n" + return import_header + + +def _is_extern_module(module_name: str, extern_modules: set[str]) -> bool: + """Return True if *module_name* (or a parent) is in the extern set.""" + if module_name in extern_modules: + return True + parts = module_name.split(".") + for i in range(1, len(parts)): + if ".".join(parts[:i]) in extern_modules: + return True + return False + + +def rewrite_package_imports( + import_header: str, + package_importer: package.PackageImporter, +) -> str: + """Rewrite interned imports to use ``_package_importer``. + + Extern modules (``torch``, ``typing``, …) keep regular ``import`` + statements. Interned modules (for example, local + ``generative_recommenders.*`` modules) are rewritten to:: + + _pkg_mod = _package_importer.import_module( + 'generative_recommenders.ops.triton.triton_utils' + ) + helper = _pkg_mod.helper + + The ``_package_importer`` object is injected into the wrapper module's + namespace by ``replace_kernels`` before ``exec_module`` is called. + """ + extern_modules = set(package_importer.extern_modules) + header_tree = ast.parse(import_header) + + regular: list[str] = [] + from_package: list[str] = [] + + for node in header_tree.body: + if isinstance(node, ast.Import): + for alias in node.names: + if _is_extern_module(alias.name, extern_modules): + regular.append(ast.unparse(node)) + else: + local = alias.asname or alias.name + from_package.append( + f"{local} = _package_importer.import_module('{alias.name}')" + ) + elif isinstance(node, ast.ImportFrom): + mod = node.module or "" + if _is_extern_module(mod, extern_modules): + regular.append(ast.unparse(node)) + else: + var = f"_pkg_{mod.replace('.', '_')}" + from_package.append(f"{var} = _package_importer.import_module('{mod}')") + for alias in node.names: + local = alias.asname or alias.name + from_package.append(f"{local} = {var}.{alias.name}") + else: + # Non-import statement (should not appear, but preserve if it does) + regular.append(ast.unparse(node)) + + parts: list[str] = [] + if regular: + parts.append("\n".join(regular)) + if from_package: + parts.append("# Imports resolved from torch package via _package_importer") + parts.append("\n".join(from_package)) + return "\n".join(parts) + "\n" if parts else "" diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/transform/kernel_wrapper_codegen.py b/recommendation_v4/generative_recommenders/ops/triton_aot/transform/kernel_wrapper_codegen.py new file mode 100644 index 000000000..51d144f1b --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton_aot/transform/kernel_wrapper_codegen.py @@ -0,0 +1,500 @@ +# pyre-strict +import ast +import inspect +import os +from typing import Any, Callable, Dict, List, Optional + +from generative_recommenders.ops.triton_aot.compile.compile_state import ( + get_aott_compile_path, + get_triton_aot_kernel_specs, +) +from generative_recommenders.ops.triton_aot.compile.utils import unwrap_heuristic +from generative_recommenders.ops.triton_aot.shared.compat import get_kernel_name +from generative_recommenders.ops.triton_aot.transform.import_utils import ( + get_original_import_header, + rewrite_package_imports, +) +from generative_recommenders.ops.triton_aot.types import TritonAOT +from pyre_extensions import none_throws +from torch import package +from torch.fx import GraphModule + +# @manual=//triton:triton +from triton.runtime.autotuner import Autotuner +from triton.runtime.jit import JITFunction, KernelInterface + + +def _is_torch_package_module(module_name: str) -> bool: + """Check if a module name is from torch.package namespace.""" + return module_name.startswith(" str: + """Strip the torch.package namespace prefix from a module name. + + Example: + '.generative_recommenders.ops.triton_aot.triton_layer_norm' + -> 'generative_recommenders.ops.triton_aot.triton_layer_norm' + """ + if _is_torch_package_module(module_name): + # Remove '.' prefix + return module_name.split(".", 1)[1] + return module_name + + +def _get_clean_module_basename(module_name: str) -> str: + """Get the basename of a module, stripping torch.package prefix if present. + + Example: + '.generative_recommenders.ops.triton_aot.triton_layer_norm' + -> 'triton_layer_norm' + 'generative_recommenders.ops.triton_aot.triton_layer_norm' + -> 'triton_layer_norm' + """ + clean_name = _strip_torch_package_prefix(module_name) + return clean_name.rsplit(".", 1)[-1] + + +def _extract_function_source(module_source: str, fn_name: str) -> str: + """Extract a function's source code from module source. + + Parses the module source and extracts just the function definition. + """ + tree = ast.parse(module_source) + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef) and node.name == fn_name: + return ast.unparse(node) + raise ValueError(f"Function '{fn_name}' not found in module source") + + +def _get_module_and_source( + target: Callable[..., Any], + package_importer: Optional[package.PackageImporter], +) -> tuple[Any, str, str]: + """Get module, module source, and function source for a callable. + + Handles both regular modules and torch.package loaded modules. + + Args: + target: The callable (function) to get source for + package_importer: Optional PackageImporter for torch.package modules + + Returns: + Tuple of (module, module_source, function_source) + """ + module_name = target.__module__ + fn_name = target.__name__ + + if _is_torch_package_module(module_name) and package_importer is not None: + # Handle torch.package namespace + real_module_name = _strip_torch_package_prefix(module_name) + assert real_module_name.startswith( + "generative_recommenders.ops.triton_aot" + ) or real_module_name.startswith("prime_perf_optimizer"), ( + f"Expected module under 'generative_recommenders.ops.triton_aot' or 'prime_perf_optimizer', got: {real_module_name}" + ) + + # Get module source from package + module_source = package_importer.get_source(real_module_name) + + # Import the module through the package importer + fn_module = package_importer.import_module(real_module_name) + + # Extract function source from module source + fn_source = _extract_function_source(module_source, fn_name) + + return fn_module, module_source, fn_source + else: + # Standard module handling + fn_module = inspect.getmodule(target) + module_source = inspect.getsource(none_throws(fn_module)) + fn_source = inspect.getsource(target) + + return fn_module, module_source, fn_source + + +def _calls_triton_aot_kernel(node: ast.FunctionDef, kernel_name: str) -> bool: + """ + kernel_name is the JIT function name (e.g. "_weighted_layer_norm_fwd"), + which may differ from the wrapper function name (e.g. + "_triton_aot_swish_layer_norm"). We match by looking for a + Subscript-call ``kernel_name[grid](...)`` inside the function body. + """ + for child in ast.walk(node): + if ( + isinstance(child, ast.Call) + and isinstance(child.func, ast.Subscript) + and isinstance(child.func.value, ast.Name) + and child.func.value.id == kernel_name + ): + return True + return False + + +def _is_torch_jit_unused(d: ast.expr) -> bool: + """Check if a decorator AST node represents @torch.jit.unused.""" + return ( + isinstance(d, ast.Attribute) + and d.attr == "unused" + and isinstance(d.value, ast.Attribute) + and d.value.attr == "jit" + and isinstance(d.value.value, ast.Name) + and d.value.value.id == "torch" + ) + + +def strip_jit_unused_decorator( + node: ast.FunctionDef, kernel_name: str +) -> ast.FunctionDef: + """Strip @torch.jit.unused if the function body calls ``kernel_name[grid](...)``. + + kernel_name is the TritonAOT kernel's JIT function name (e.g. + ``_weighted_layer_norm_fwd``), not the wrapper function name. This avoids + relying on a naming convention on the wrapper function itself. + """ + if _calls_triton_aot_kernel(node, kernel_name): + node.decorator_list = [ + d for d in node.decorator_list if not _is_torch_jit_unused(d) + ] + return node + + +class TritonAOTOperatorTransform(ast.NodeTransformer): + def __init__(self, kernel: Any) -> None: + super().__init__() + self._kernel: Any = kernel + self._kernel_jit_fn: JITFunction[List[Any]] = unwrap_heuristic( + kernel, return_type=JITFunction + ) + self._kernel_autotuner: Optional[Autotuner] = unwrap_heuristic( + kernel, return_type=Autotuner + ) + self._kernel_name: str = get_kernel_name(self._kernel_jit_fn) + # Only transform the function body + self._autotune_params: List[str] = ( + list(list(self._kernel_autotuner.cache.values())[0].kwargs.keys()) + if self._kernel_autotuner is not None + else [] + ) + self._autotune_params += ["num_warps", "num_stages"] + + self._lambda_arg_name: Optional[str] = None + self._grid_name: Optional[str] = None + self._autotune_key_id: Optional[Dict[str, int]] = None + self._autotune_key_map: Optional[Dict[str, ast.expr]] = None + self._kernel_meta: Optional[ast.Assign] = None + + if self._kernel_autotuner is not None: + autotune_key_id: Dict[str, int] = {} + self._autotune_key_id = autotune_key_id + # pyre-ignore[16]: JITFunction has arg_names at runtime + for key in self._kernel_autotuner.keys: + autotune_key_id[key] = self._kernel_jit_fn.arg_names.index(key) + + def generate_function_meta(self) -> None: + targets = [ + ast.Name(id=param, ctx=ast.Store()) for param in self._autotune_params + ] + autotune_key_map = self._autotune_key_map + kernel_autotuner = self._kernel_autotuner + call = ast.Call( + func=ast.Name(id=f"{self._kernel_name}_meta", ctx=ast.Load()), + args=[ + none_throws(autotune_key_map)[key] + for key in none_throws(kernel_autotuner).keys + ] + if kernel_autotuner is not None + else [], + keywords=[], + ) + self._kernel_meta = ast.Assign( + # pyre-ignore[6]: ast.Assign targets type + targets=[ast.Tuple(elts=targets, ctx=ast.Store())], + value=call, + ) + + def visit_FunctionDef(self, node: ast.FunctionDef) -> ast.FunctionDef: + strip_jit_unused_decorator(node, self._kernel_name) + + new_body: List[ast.stmt] = [] + stmts = node.body + for stmt in stmts: + if isinstance(stmt, ast.Assign): + for target in stmt.targets: + if isinstance(target, ast.Name) and target.id == self._grid_name: + assert self._kernel_meta is not None + self._kernel_meta.lineno = stmt.lineno + new_body.append(self._kernel_meta) + new_body.append(self.visit(stmt)) + node.body = new_body + return node + + def visit_Assign(self, node: ast.Assign) -> ast.Assign: + for target in node.targets: + if isinstance(target, ast.Name) and isinstance(node.value, ast.Lambda): + lambda_node = node.value + self._lambda_arg_name = lambda_node.args.args[0].arg + lambda_body = lambda_node.body + assert isinstance(lambda_body, ast.Tuple) + new_elts: List[ast.expr] = [] + for elt in lambda_body.elts: + new_elts.append(self.visit(elt)) + node.value = ast.Tuple(elts=new_elts, ctx=ast.Load()) + self._lambda_arg_name = None + return node + + def visit_Subscript(self, node: ast.Subscript) -> ast.expr: + if isinstance(node.value, ast.Name) and node.value.id == self._lambda_arg_name: + assert isinstance(node.slice, ast.Constant) + assert isinstance(node.slice.value, str) + var_name = node.slice.value + # pyre-ignore + node = ast.Name(id=var_name, ctx=ast.Load()) + return node + + def visit_Expr(self, node: ast.Expr) -> ast.Expr: + if isinstance(node.value, ast.Call): + call = node.value + if ( + isinstance(call.func, ast.Subscript) + and isinstance(call.func.value, ast.Name) + and call.func.value.id == self._kernel_name + ): + grid_arg = call.func.slice + new_func = ast.Attribute( + value=ast.Attribute( + value=ast.Attribute( + value=ast.Name(id="torch", ctx=ast.Load()), + attr="ops", + ctx=ast.Load(), + ), + attr="triton_aot", + ctx=ast.Load(), + ), + attr=self._kernel_name, + ctx=ast.Load(), + ) + new_args = [grid_arg] + call.args + new_keywords = call.keywords + [ + ast.keyword(arg=param, value=ast.Name(id=param, ctx=ast.Load())) + for param in self._autotune_params + ] + node.value = ast.Call( + func=new_func, + args=new_args, + keywords=new_keywords, + ) + return node + + def contains_triton_call(self, node: ast.AST) -> bool: + for child in ast.walk(node): + if ( + isinstance(child, ast.Call) + and isinstance(child.func, ast.Subscript) + # pyre-ignore[16]: ast.expr may have `id` attribute at runtime + and child.func.value.id == self._kernel_name + ): + # pyrefly: ignore [missing-attribute] + self._grid_name = child.func.slice.id + + if self._kernel_autotuner is not None: + autotune_key_map: Dict[str, ast.expr] = {} + self._autotune_key_map = autotune_key_map + # pyre-ignore[16]: Autotuner has keys at runtime + for key in self._kernel_autotuner.keys: + found_key = False + for keyword in child.keywords: + if keyword.arg == key: + autotune_key_map[key] = keyword.value + found_key = True + break + + if not found_key: + autotune_key_id = self._autotune_key_id + assert autotune_key_id is not None + assert key in autotune_key_id + key_id = autotune_key_id[key] + autotune_key_map[key] = child.args[key_id] + + self.generate_function_meta() + return True + return False + + def contains_lambda(self, node: ast.AST) -> bool: + for child in ast.walk(node): + if isinstance(child, ast.Lambda): + return True + return False + + def _get_grid_name(self, node: ast.AST) -> Optional[str]: + for child in ast.walk(node): + if ( + isinstance(child, ast.Call) + and isinstance(child.func, ast.Subscript) + # pyre-ignore[16]: ast.expr may have `id` attribute at runtime + and child.func.value.id == self._kernel_name + ): + # pyrefly: ignore [missing-attribute] + return child.func.slice.id + return None + + def generate_so_loading_code( + self, + node: ast.AST, + abs_triton_aot_path: str, + ) -> str: + """Return auto-generated code to load the compiled kernel at runtime. + + If *node* contains a call to this transformer's kernel, returns + ``import importlib.util`` + meta-module loading + ``torch.ops.load_library`` + code. Otherwise returns an empty string. + + This method also sets up internal transformer state (grid name, + autotune key map, etc.) via ``contains_triton_call`` as a side effect. + + Example for _addmm_fwd kernel: + kernel_dir = "triton_addmm__addmm_fwd" + meta_module_path = "/path/to/triton_aot_compile/triton_addmm__addmm_fwd/_addmm_fwd_meta.py" + so_path = "/path/to/triton_aot_compile/triton_addmm__addmm_fwd/addmm_fwd.so" + """ + if not self.contains_triton_call(node): + return "" + + kernel_dir = f"{_get_clean_module_basename(self._kernel_jit_fn.__module__)}_{self._kernel_name}" + + meta_module_path = os.path.join( + abs_triton_aot_path, kernel_dir, f"{self._kernel_name}_meta.py" + ) + + so_path = os.path.join( + abs_triton_aot_path, + kernel_dir, + f"{self._kernel_name.lstrip('_')}.so", + ) + + return f""" +# Auto-generated by triton_aot.kernel_wrapper_codegen +import importlib.util +_meta_spec = importlib.util.spec_from_file_location("{self._kernel_name}_meta", "{meta_module_path}") +_meta_module = importlib.util.module_from_spec(_meta_spec) +_meta_spec.loader.exec_module(_meta_module) +{self._kernel_name}_meta = _meta_module.{self._kernel_name}_meta + +torch.ops.load_library("{so_path}") +""" + + +def _find_triton_aot_kernel( + node_target: Any, + kernel_specs: Dict[KernelInterface[List[Any]], List[Dict[str, List[Any]]]], +) -> Optional[TritonAOT]: + """Find the single TritonAOT kernel referenced in a node target's globals. + + Scans ``node_target.__globals__`` for ``TritonAOT`` instances, validates + that every instance appears in *kernel_specs*, and asserts at most one + kernel is present (per the one-kernel-per-wrapper invariant). + + Returns the kernel, or ``None`` if the function references no kernels. + """ + kernels: set[TritonAOT] = set() + for _, var in node_target.__globals__.items(): + if isinstance(var, TritonAOT): + if var.fn in kernel_specs: + kernels.add(var) + else: + raise RuntimeError( + f"Cannot find TritonAOT kernel {var.fn} in TRITON_AOT_KERNEL_SPECS" + ) + + if len(kernels) == 0: + return None + + fn_name = node_target.__name__ + assert len(kernels) == 1, ( + f"Expected exactly 1 kernel per wrapper function '{fn_name}', " + f"got {len(kernels)}" + ) + (kernel_obj,) = kernels + return kernel_obj + + +def _generate_wrapper_files( + node_target: Any, + kernel: TritonAOT, + compile_path: str, + package_importer: Optional[package.PackageImporter], +) -> None: + """Generate ``_original.py`` and ``_wrapper.py`` for a single kernel. + + Creates a per-kernel subdirectory under *compile_path*, writes the + original function source, then AST-transforms the wrapper to replace + ``kernel[grid](...)`` with ``torch.ops.triton_aot.*`` calls. + """ + fn_name = node_target.__name__ + + jit_fn = none_throws( + unwrap_heuristic(kernel, return_type=JITFunction), + f"Failed to unwrap kernel to JITFunction: {kernel}", + ) + kernel_dir = ( + f"{_get_clean_module_basename(jit_fn.__module__)}_{get_kernel_name(jit_fn)}" + ) + output_dir = os.path.join(compile_path, kernel_dir) + os.makedirs(output_dir, exist_ok=True) + + _, module_code, wrapper_code = _get_module_and_source(node_target, package_importer) + import_header = get_original_import_header(module_code) + + with open(os.path.join(output_dir, f"{fn_name}_original.py"), "w") as f: + f.write(import_header) + f.write(wrapper_code) + + # When source comes from a torch package, rewrite interned + # imports to use _package_importer, which + # is injected by replace_kernels at load time. Must happen + # before auto-generated code is appended (so stdlib imports + # like ``import importlib.util`` are not touched). + if package_importer is not None: + import_header = rewrite_package_imports(import_header, package_importer) + + tree = ast.parse(wrapper_code) + transformer = TritonAOTOperatorTransform(kernel=kernel) + import_header += transformer.generate_so_loading_code(tree, compile_path) + tree = transformer.visit(tree) + + new_source_code = ast.unparse(tree) + + with open(os.path.join(output_dir, f"{fn_name}_wrapper.py"), "w") as f: + f.write(import_header) + f.write(new_source_code) + + +def kernel_wrapper_codegen( + module: GraphModule, packageImporter: package.PackageImporter | None = None +) -> None: + """ + Generate wrapper files for TritonAOT kernels. + Requirement: under wrapper.py, @triton.jit kernel/func is imported without 'as' alias. + + For each function containing TritonAOT kernels, generates: + - {fn_name}_original.py: Original source code with imports + - {fn_name}_wrapper.py: Transformed wrapper that uses torch.ops.triton_aot + """ + compile_path = get_aott_compile_path() + if not os.path.exists(compile_path): + os.makedirs(compile_path) + + transformed_ops: set[Callable[..., Any]] = set() + kernel_specs = get_triton_aot_kernel_specs() + for node in module.graph.nodes: + if node.op == "call_function" and hasattr(node.target, "__globals__"): + if node.target not in transformed_ops: + transformed_ops.add(node.target) + else: + continue + + kernel = _find_triton_aot_kernel(node.target, kernel_specs) + if kernel is not None: + _generate_wrapper_files( + node.target, kernel, compile_path, packageImporter + ) diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/transform/replace_kernels.py b/recommendation_v4/generative_recommenders/ops/triton_aot/transform/replace_kernels.py new file mode 100644 index 000000000..53e5d83e9 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton_aot/transform/replace_kernels.py @@ -0,0 +1,137 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files +# (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, +# publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +#!/usr/bin/env python3 + +# pyre-strict + +import importlib.util +import logging +import os +import sys +from typing import Any, Dict, Optional + +from generative_recommenders.ops.triton_aot.compile.compile_state import ( + get_aott_compile_path, +) +from torch import package +from torch.fx import GraphModule + +logger: logging.Logger = logging.getLogger(__name__) + + +def _find_wrapper_files( + compile_path: str, +) -> list[tuple[str, str, str]]: + """Find all ``*_wrapper.py`` files under *compile_path*. + + Walks one level deep into kernel subdirectories and returns a list of + ``(wrapper_name, fn_name, wrapper_path)`` tuples. + """ + results: list[tuple[str, str, str]] = [] + for dirpath, dirnames, filenames in os.walk(compile_path): + if dirpath != compile_path: + dirnames.clear() # only recurse one level into kernel subdirs + for item in filenames: + if item.endswith("_wrapper.py"): + wrapper_name = item.removesuffix(".py") + fn_name = wrapper_name.removesuffix("_wrapper") + wrapper_path = os.path.join(dirpath, item) + results.append((wrapper_name, fn_name, wrapper_path)) + return results + + +def _load_wrapper_module( + wrapper_name: str, + fn_name: str, + wrapper_path: str, + package_importer: Optional[package.PackageImporter], +) -> Optional[Any]: + """Dynamically import a single ``*_wrapper.py`` and return its wrapper callable. + + Returns ``None`` if the module does not expose a function named *fn_name*. + """ + spec = importlib.util.spec_from_file_location(wrapper_name, wrapper_path) + assert spec is not None, f"Failed to create spec for {wrapper_path}" + assert spec.loader is not None, f"Spec has no loader for {wrapper_path}" + + loader = spec.loader + wrapper_module = importlib.util.module_from_spec(spec) + + sys.modules[wrapper_name] = wrapper_module + + if package_importer is not None: + wrapper_module._package_importer = package_importer # type: ignore[attr-defined] + + loader.exec_module(wrapper_module) + + if hasattr(wrapper_module, fn_name): + return getattr(wrapper_module, fn_name) + return None + + +def replace_kernels( + fx_m: GraphModule, + eager: bool = False, + package_importer: Optional[package.PackageImporter] = None, +) -> GraphModule: + if eager: + raise NotImplementedError( + "Local generative_recommenders AOT-T transform does not support " + "eager replacement." + ) + + compile_path = get_aott_compile_path() + assert os.path.exists(compile_path), "triton_aot_compile dir does not exist" + + wrapper_dict: Dict[str, Any] = {} + for wrapper_name, fn_name, wrapper_path in _find_wrapper_files(compile_path): + wrapper_fn = _load_wrapper_module( + wrapper_name, fn_name, wrapper_path, package_importer + ) + if wrapper_fn is not None: + wrapper_dict[fn_name] = wrapper_fn + + logger.info(f"replace_kernels: {wrapper_dict=}") + + # Phase 2: Replace FX graph nodes + # Walk the FX graph, find call_function nodes whose target name + # matches a loaded wrapper, and swap the target so that + # kernel[grid](...) calls become torch.ops.triton_aot.* calls. + replaced_count = 0 + for nodes in fx_m.graph.nodes: + if nodes.op == "call_function" and nodes.target.__name__ in wrapper_dict.keys(): + logger.info( + f"Replaced node: {nodes.op} {nodes.target} -> {wrapper_dict[nodes.target.__name__]} {nodes.meta}" + ) + nodes.target = wrapper_dict[nodes.target.__name__] + replaced_count += 1 + + assert replaced_count > 0, ( + f"No ops were replaced with triton_aot wrappers. " + f"wrapper_dict={wrapper_dict}, compile_path={compile_path}" + ) + logger.info( + f"Successfully replaced {replaced_count} op(s) with triton_aot wrappers." + ) + + fx_m.recompile() + return fx_m diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/transform/transform_kernels.py b/recommendation_v4/generative_recommenders/ops/triton_aot/transform/transform_kernels.py new file mode 100644 index 000000000..c2be78989 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton_aot/transform/transform_kernels.py @@ -0,0 +1,29 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# pyre-strict + +from typing import Optional + +from generative_recommenders.ops.triton_aot.transform.kernel_wrapper_codegen import ( + kernel_wrapper_codegen, +) +from generative_recommenders.ops.triton_aot.transform.replace_kernels import ( + replace_kernels, +) +from torch import package +from torch.fx import GraphModule + + +def transform_kernels( + fx_m: GraphModule, + eager: bool = False, + package_importer: Optional[package.PackageImporter] = None, +) -> GraphModule: + """Generate AOT wrappers and replace FX graph nodes in one step. + + 1. kernel_wrapper_codegen: AST-transforms wrapper functions, + rewrites kernel[grid](...) -> torch.ops.triton_aot.kernel(...), + writes {fn}_wrapper.py + 2. replace_kernels: loads wrappers and replaces graph node targets + """ + kernel_wrapper_codegen(fx_m, package_importer) + return replace_kernels(fx_m, eager=eager, package_importer=package_importer) diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/triton_addmm.py b/recommendation_v4/generative_recommenders/ops/triton_aot/triton_addmm.py new file mode 100644 index 000000000..b71ec8144 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton_aot/triton_addmm.py @@ -0,0 +1,347 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files +# (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, +# publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +# pyre-strict +# pyre-ignore-all-errors[2]: Triton has its own type system on func's input + +#!/usr/bin/env python3 + + +from typing import Any, List, Tuple + +import torch + +# @manual=//triton:triton +import triton + +# @manual=//triton:triton +import triton.language as tl +from generative_recommenders.common import ( + BACKEND_ALLOW_TF32, + cdiv, + should_trigger_eager_impl, +) +from generative_recommenders.ops.triton_aot.types import triton_aot + + +def get_mm_configs() -> List[triton.Config]: + return [ + triton.Config( + { + "BLOCK_M": 32, + "BLOCK_N": 64, + "BLOCK_K": 32, + "GROUP_M": 8, + }, + num_stages=5, + num_warps=2, + ), + triton.Config( + { + "BLOCK_M": 128, + "BLOCK_N": 256, + "BLOCK_K": 64, + "GROUP_M": 8, + }, + num_stages=3, + num_warps=8, + ), + triton.Config( + { + "BLOCK_M": 64, + "BLOCK_N": 256, + "BLOCK_K": 32, + "GROUP_M": 8, + }, + num_stages=4, + num_warps=4, + ), + triton.Config( + { + "BLOCK_M": 128, + "BLOCK_N": 128, + "BLOCK_K": 32, + "GROUP_M": 8, + }, + num_stages=4, + num_warps=4, + ), + triton.Config( + { + "BLOCK_M": 128, + "BLOCK_N": 64, + "BLOCK_K": 32, + "GROUP_M": 8, + }, + num_stages=4, + num_warps=4, + ), + triton.Config( + { + "BLOCK_M": 64, + "BLOCK_N": 128, + "BLOCK_K": 32, + "GROUP_M": 8, + }, + num_stages=4, + num_warps=4, + ), + triton.Config( + { + "BLOCK_M": 128, + "BLOCK_N": 32, + "BLOCK_K": 32, + "GROUP_M": 8, + }, + num_stages=4, + num_warps=4, + ), + triton.Config( + { + "BLOCK_M": 64, + "BLOCK_N": 32, + "BLOCK_K": 32, + "GROUP_M": 8, + }, + num_stages=5, + num_warps=2, + ), + ] + + +@triton_aot( + annotations={ + "M": "i32", + "N": ("i32", 16), + "K": ("i32", 16), + "stride_xm": ("i32", 16), + "stride_xk": ("i32", 1), + "stride_wk": ("i32", 16), + "stride_wn": ("i32", 1), + "stride_ym": ("i32", 16), + "stride_yn": ("i32", 1), + "stride_zm": ("i32", 16), + "stride_zn": ("i32", 1), + }, +) +# pyre-ignore[56]: Pyre cannot infer triton.autotune decorator type +@triton.autotune( + configs=get_mm_configs(), + key=["N", "K"], +) +@triton.jit +def _addmm_fwd( + x_ptr, + w_ptr, + y_ptr, + z_ptr, + M, + N, + K, + stride_xm, + stride_xk, + stride_wk, + stride_wn, + stride_ym, + stride_yn, + stride_zm, + stride_zn, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, + GROUP_M: tl.constexpr, + ALLOW_TF32: tl.constexpr, + BROADCAST_Y: tl.constexpr, +) -> None: + pid_0, pid_1 = tl.program_id(axis=0), tl.program_id(axis=1) + pid = pid_0 * tl.num_programs(axis=1) + pid_1 + num_pid_m = tl.cdiv(M, BLOCK_M) + num_pid_n = tl.cdiv(N, BLOCK_N) + num_pid_in_group = GROUP_M * num_pid_n + group_id = pid // num_pid_in_group + first_pid_m = group_id * GROUP_M + group_size_m = min(num_pid_m - first_pid_m, GROUP_M) + pid_m = first_pid_m + (pid % group_size_m) + pid_n = (pid % num_pid_in_group) // group_size_m + + offs_m = tl.arange(0, BLOCK_M) + offs_k = tl.arange(0, BLOCK_K) + offs_n = tl.arange(0, BLOCK_N) + mask_m = (pid_m * BLOCK_M + offs_m)[:, None] < M + mask_n = (pid_n * BLOCK_N + offs_n)[None, :] < N + x_ptr += pid_m.to(tl.int64) * BLOCK_M * stride_xm + x_ptrs = x_ptr + (offs_m[:, None] * stride_xm + offs_k[None, :] * stride_xk) + w_ptr += pid_n.to(tl.int64) * BLOCK_N * stride_wn + w_ptrs = w_ptr + (offs_k[:, None] * stride_wk + offs_n[None, :] * stride_wn) + accumulator = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32) + for k in range(0, tl.cdiv(K, BLOCK_K)): + mask_k = offs_k[None, :] < K - k * BLOCK_K + x = tl.load(x_ptrs, mask=mask_k & mask_m, other=0.0) + mask_k = offs_k[:, None] < K - k * BLOCK_K + w = tl.load(w_ptrs, mask=mask_k & mask_n, other=0.0) + accumulator += tl.dot(x, w, allow_tf32=ALLOW_TF32) + x_ptrs += BLOCK_K * stride_xk + w_ptrs += BLOCK_K * stride_wk + + z_mask = mask_m & mask_n + if BROADCAST_Y: + # y is a vector, broadcast to add to z + y_ptr += pid_n.to(tl.int64) * BLOCK_N * stride_yn + y_ptrs = y_ptr + stride_yn * offs_n[None, :] + y = tl.load(y_ptrs, mask=mask_n) + else: + y_ptr += pid_m.to(tl.int64) * BLOCK_M * stride_ym + y_ptr += pid_n.to(tl.int64) * BLOCK_N * stride_yn + y_ptrs = y_ptr + stride_ym * offs_m[:, None] + stride_yn * offs_n[None, :] + y = tl.load(y_ptrs, mask=z_mask) + z = (accumulator + y.to(tl.float32)).to(z_ptr.dtype.element_ty) + z_ptr += pid_m.to(tl.int64) * BLOCK_M * stride_zm + z_ptr += pid_n.to(tl.int64) * BLOCK_N * stride_zn + z_ptrs = z_ptr + stride_zm * offs_m[:, None] + stride_zn * offs_n[None, :] + tl.store(z_ptrs, z, mask=z_mask) + + +@torch.jit.unused +@torch.fx.wrap +def _triton_aot_addmm_fwd( + x: torch.Tensor, + w: torch.Tensor, + y: torch.Tensor, + allow_tf32: bool = BACKEND_ALLOW_TF32, +) -> torch.Tensor: + M, K = x.shape + KB, N = w.shape + assert K == KB, f"incompatible dimensions {K}, {KB}" + + is_y_1d = y.dim() == 1 + NY = y.shape[0] if is_y_1d else y.shape[1] + assert N == NY, f"incompatible dimensions {N}, {NY}" + + # Allocate output + z = torch.empty((M, N), device=x.device, dtype=x.dtype) + if M == 0 or N == 0: + return z + + grid = lambda meta: ( # noqa E731 + cdiv(M, meta["BLOCK_M"]), + cdiv(N, meta["BLOCK_N"]), + ) + + _addmm_fwd[grid]( + x, + w, + y, + z, + M, + N, + K, + x.stride(0), + x.stride(1), + w.stride(0), + w.stride(1), + y.stride(0) if not is_y_1d else 0, + y.stride(1) if not is_y_1d else y.stride(0), + z.stride(0), + z.stride(1), + ALLOW_TF32=allow_tf32, + BROADCAST_Y=is_y_1d, + ) + return z + + +def _triton_aot_addmm_fwd_eager( + x: torch.Tensor, + w: torch.Tensor, + y: torch.Tensor, +) -> torch.Tensor: + return torch.addmm(y, x, w) + + +@torch.fx.wrap +def _triton_aot_addmm_fwd_maybe_eager( + x: torch.Tensor, + w: torch.Tensor, + y: torch.Tensor, +) -> torch.Tensor: + if torch.jit.is_scripting(): + # call eager + return torch.addmm(y, x, w) + else: + return _triton_aot_addmm_fwd(x, w, y) + + +def triton_addmm_bwd( + x: torch.Tensor, + w: torch.Tensor, + dz: torch.Tensor, + is_y_1d: bool, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + if is_y_1d: + dy = torch.sum(dz, dim=0) + else: + dy = dz + dw = torch.mm(x.t(), dz) + dx = torch.mm(dz, w.t()) + + return dx, dw, dy + + +class _AddMmFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore[14]: autograd.Function signature override + def forward( + ctx: Any, + x: torch.Tensor, + w: torch.Tensor, + y: torch.Tensor, + ) -> torch.Tensor: + ctx.save_for_backward(x, w) + ctx.is_y_1d = y.dim() == 1 + return _triton_aot_addmm_fwd(x, w, y) + + @staticmethod + # pyre-ignore[14]: autograd.Function signature override + def backward( + ctx: Any, dz: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + (x, w) = ctx.saved_tensors + return triton_addmm_bwd(x, w, dz, ctx.is_y_1d) + + +def triton_addmm( + input: torch.Tensor, + mat1: torch.Tensor, + mat2: torch.Tensor, +) -> torch.Tensor: + return _AddMmFunction.apply(mat1, mat2, input) + + +@torch.fx.wrap +def aot_triton_kernel_wrapper_addmm( + input: torch.Tensor, + mat1: torch.Tensor, + mat2: torch.Tensor, + allow_tf32: bool = BACKEND_ALLOW_TF32, +) -> torch.Tensor: + if should_trigger_eager_impl(): + return torch.addmm(input, mat1, mat2) + else: + return _triton_aot_addmm_fwd(mat1, mat2, input, allow_tf32=allow_tf32) diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/triton_concat_2d_jagged.py b/recommendation_v4/generative_recommenders/ops/triton_aot/triton_concat_2d_jagged.py new file mode 100644 index 000000000..8afc3c18e --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton_aot/triton_concat_2d_jagged.py @@ -0,0 +1,183 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files +# (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, +# publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +# pyre-strict + +from typing import Optional + +import torch +from generative_recommenders.common import ( + fx_unwrap_optional_tensor, + next_power_of_2, + should_trigger_eager_impl, +) +from generative_recommenders.ops.pytorch.pt_jagged import ( + pytorch_replace_last_n_with_jagged, +) +from generative_recommenders.ops.pytorch.pt_jagged_tensors import ( + pytorch_concat_2D_jagged, +) +from generative_recommenders.ops.triton.triton_jagged import concat_2D_jagged +from generative_recommenders.ops.triton_aot.types import triton_aot + + +concat_2D_jagged = triton_aot( + annotations={ + "DenseSize": "i32", + "D": "i32", + "stride_ad": "i32", + "stride_bd": "i32", + "stride_dense_batch": "i32", + "stride_od": "i32", + }, + # pyrefly: ignore [bad-argument-type] +)(concat_2D_jagged) + + +@torch.jit.unused +@torch.fx.wrap +def _triton_aot_concat_2D_jagged( + max_seq_len: int, + values_a: torch.Tensor, + values_b: torch.Tensor, + offsets_a: Optional[torch.Tensor] = None, + offsets_b: Optional[torch.Tensor] = None, + is_replace: bool = False, +) -> torch.Tensor: + is_dense_a = offsets_a is None + is_dense_b = offsets_b is None + + dense_size: int = 0 + if is_dense_a: + B, dense_size, D = values_a.size() + offsets_b = fx_unwrap_optional_tensor(offsets_b) + jagged_seq_len, _ = values_b.shape + values_out = torch.empty( + (dense_size * B + jagged_seq_len, D), + device=values_b.device, + dtype=values_b.dtype, + ) + offsets_a = offsets_b.new_empty(0) + stride_dense_batch = values_a.stride(0) + elif is_dense_b: + B, dense_size, D = values_b.size() + offsets_a = fx_unwrap_optional_tensor(offsets_a) + jagged_seq_len, _ = values_a.shape + values_out = torch.empty( + (jagged_seq_len + dense_size * B, D), + device=values_a.device, + dtype=values_a.dtype, + ) + offsets_b = offsets_a.new_empty(0) + stride_dense_batch = values_b.stride(0) + else: + offsets_a = fx_unwrap_optional_tensor(offsets_a) + offsets_b = fx_unwrap_optional_tensor(offsets_b) + B = offsets_a.size(0) - 1 + seq_len_a, D = values_a.shape + seq_len_b, _ = values_b.shape + if is_replace: + values_out = torch.empty_like(values_a) + else: + values_out = torch.empty( + (seq_len_a + seq_len_b, D), device=values_a.device, dtype=values_a.dtype + ) + stride_dense_batch = 0 + + # Make sure offsets are alignted on 16-byte to match AOTT spec + if ( + offsets_a is not None + and (offsets_a.storage_offset() * offsets_a.element_size()) % 16 != 0 + ): + offsets_a = offsets_a.clone() + if ( + offsets_b is not None + and (offsets_b.storage_offset() * offsets_b.element_size()) % 16 != 0 + ): + offsets_b = offsets_b.clone() + + BLOCK_D = next_power_of_2(D) + + grid = (max_seq_len, B) + # pyrefly: ignore [not-callable] + concat_2D_jagged[grid]( + OffsetsA=offsets_a, + ValuesA=values_a, + OffsetsB=offsets_b, + ValuesB=values_b, + DenseSize=dense_size, + Out=values_out, + D=D, + stride_ad=(values_a.stride(1) if is_dense_a else values_a.stride(0)), + stride_bd=(values_b.stride(1) if is_dense_b else values_b.stride(0)), + stride_dense_batch=stride_dense_batch, + stride_od=values_out.stride(0), + # pyrefly: ignore [bad-argument-type] + IS_DENSE_A=is_dense_a, + # pyrefly: ignore [bad-argument-type] + IS_DENSE_B=is_dense_b, + # pyrefly: ignore [bad-argument-type] + BLOCK_D=BLOCK_D, + # pyrefly: ignore [bad-argument-type] + IS_REPLACE=is_replace, + ) + return values_out + + +@torch.fx.wrap +# "aot_triton_kernel_wrapper_" is a pre-defined prefix for +# AOT-T triton kernel wrapper functions. This is required for +# AOT-T backend to recognize and trace correctly for ops transformation. +def aot_triton_kernel_wrapper_concat_2D_jagged( + max_seq_len: int, + values_a: torch.Tensor, + values_b: torch.Tensor, + offsets_a: Optional[torch.Tensor] = None, + offsets_b: Optional[torch.Tensor] = None, + is_replace: bool = False, +) -> torch.Tensor: + if should_trigger_eager_impl(): + if is_replace: + assert offsets_a is not None and offsets_b is not None + return pytorch_replace_last_n_with_jagged( + max_seq_len_left=max_seq_len, + offsets_left=offsets_a, + values_left=values_a, + offsets_right=offsets_b, + values_right=values_b, + ) + return pytorch_concat_2D_jagged( + values_left=values_a, + values_right=values_b, + max_len_left=max_seq_len if offsets_a is None else None, + max_len_right=max_seq_len if offsets_b is None else None, + offsets_left=offsets_a, + offsets_right=offsets_b, + ) + else: + return _triton_aot_concat_2D_jagged( + max_seq_len=max_seq_len, + values_a=values_a, + values_b=values_b, + offsets_a=offsets_a, + offsets_b=offsets_b, + is_replace=is_replace, + ) diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/triton_group_norm_mul_dropout.py b/recommendation_v4/generative_recommenders/ops/triton_aot/triton_group_norm_mul_dropout.py new file mode 100644 index 000000000..15c609a3c --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton_aot/triton_group_norm_mul_dropout.py @@ -0,0 +1,124 @@ +# pyre-strict + +import torch +from generative_recommenders.common import next_power_of_2, should_trigger_eager_impl +from generative_recommenders.ops.pytorch.pt_hstu_linear import pytorch_norm_mul_dropout +from generative_recommenders.ops.triton.triton_hstu_linear import ( + _group_norm_mul_dropout_fwd, +) +from generative_recommenders.ops.triton_aot.types import triton_aot + +_group_norm_mul_dropout_fwd = triton_aot( + annotations={ + "D": ("i32", 16), + "eps": "fp32", + "seed": "i64", + "dropout_ratio": "fp32", + "stride_x": ("i32", 16), + "stride_u": ("i32", 16), + "stride_y": ("i32", 16), + }, + # pyrefly: ignore [bad-argument-type] +)(_group_norm_mul_dropout_fwd) + + +@torch.jit.unused +@torch.fx.wrap +def _triton_aot_group_norm_mul_dropout( + x: torch.Tensor, + u: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + eps: float, + silu_u: bool, + concat_ux: bool, + num_heads: int, + linear_dim: int, +) -> torch.Tensor: + x = x.contiguous() + u = u.contiguous() + N, _ = x.shape + if concat_ux: + y = torch.empty((N, 3 * num_heads * linear_dim), dtype=x.dtype, device=x.device) + else: + y = torch.empty((N, num_heads * linear_dim), dtype=x.dtype, device=x.device) + mean = torch.empty((N * num_heads,), dtype=x.dtype, device=x.device) + rstd = torch.empty((N * num_heads,), dtype=x.dtype, device=x.device) + + BLOCK_D = next_power_of_2(linear_dim) + BLOCK_H = next_power_of_2(num_heads) + + seed = 0 + dropout_ratio = 0.0 + + grid = (N,) + # pyrefly: ignore [not-callable] + _group_norm_mul_dropout_fwd[grid]( + x, # X + u, # U + y, # Y + weight, # W + bias, # B + mean, # Mean + rstd, # Rstd + linear_dim, # D + num_heads, # Heads + eps, # eps + seed, # seed + dropout_ratio, # dropout_ratio + x.stride(0), # stride_x + u.stride(0), # stride_u + y.stride(0), # stride_y + # pyrefly: ignore [bad-argument-type] + SILU_U=silu_u, + # pyrefly: ignore [bad-argument-type] + BLOCK_D=BLOCK_D, + # pyrefly: ignore [bad-argument-type] + BLOCK_H=BLOCK_H, + # pyrefly: ignore [bad-argument-type] + TRAINING=False, + # pyrefly: ignore [bad-argument-type] + CONCAT_UX=concat_ux, + ) + return y + + +@torch.fx.wrap +def aot_triton_kernel_wrapper_group_norm_mul_dropout( + x: torch.Tensor, + u: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + eps: float, + silu_u: bool, + concat_ux: bool, + num_heads: int, + linear_dim: int, +) -> torch.Tensor: + if should_trigger_eager_impl(): + return pytorch_norm_mul_dropout( + x=x, + u=u, + weight=weight, + bias=bias, + eps=eps, + dropout_ratio=0.0, + training=False, + silu_u=silu_u, + concat_u=concat_ux, + concat_x=concat_ux, + group_norm=True, + num_heads=num_heads, + linear_dim=linear_dim, + ) + return _triton_aot_group_norm_mul_dropout( + x=x, + u=u, + weight=weight, + bias=bias, + eps=eps, + silu_u=silu_u, + concat_ux=concat_ux, + num_heads=num_heads, + linear_dim=linear_dim, + ) diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/triton_layer_norm.py b/recommendation_v4/generative_recommenders/ops/triton_aot/triton_layer_norm.py new file mode 100644 index 000000000..c5339033f --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton_aot/triton_layer_norm.py @@ -0,0 +1,119 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files +# (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, +# publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +# pyre-strict + +#!/usr/bin/env python3 + +import torch +from generative_recommenders.common import ( + cdiv, + next_power_of_2, + should_trigger_eager_impl, + switch_to_contiguous_if_needed, +) +from generative_recommenders.ops.pytorch.pt_layer_norm import ( + pytorch_layer_norm, + pytorch_swish_layer_norm, +) +from generative_recommenders.ops.triton.triton_layer_norm import ( + _weighted_layer_norm_fwd, +) +from generative_recommenders.ops.triton_aot.types import triton_aot + + +_weighted_layer_norm_fwd = triton_aot( + annotations={ + "N": "i32", + "D": ("i32", 16), + "stride_x": ("i32", 16), + "stride_y": ("i32", 16), + }, +)(_weighted_layer_norm_fwd) + + +@torch.jit.unused +@torch.fx.wrap +def _triton_aot_swish_layer_norm( + x: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + eps: float, + is_swish: bool, +) -> torch.Tensor: + assert x.dim() == 2, f"x.dim() == {x.dim()}, expected 2" + x = switch_to_contiguous_if_needed(x) + N, D = x.shape + + assert weight.dim() == 1 + assert bias.dim() == 1 + assert weight.numel() == D + assert bias.numel() == D + + y = torch.empty_like(x) + + BLOCK_D = next_power_of_2(D) + + grid = lambda meta: ( # noqa E731 + cdiv(N, meta["BLOCK_N"]), + ) + # pyrefly: ignore [not-callable] + _weighted_layer_norm_fwd[grid]( + x, + y, + weight, + bias, + torch.empty(0, dtype=torch.float32), + torch.empty(0, dtype=torch.float32), + N, + D, + eps, + stride_x=x.stride(0), + stride_y=y.stride(0), + IS_SWISH=is_swish, + TRAINING=False, + BLOCK_D=BLOCK_D, + COMPUTE_MEAN_AND_RSTD=True, + ) + + return y + + +@torch.fx.wrap +# "aot_triton_kernel_wrapper_" is a pre-defined prefix for +# AOT-T triton kernel wrapper functions. This is required for +# AOT-T backend to recognize and trace correctly for ops transformation. +def aot_triton_kernel_wrapper_swish_layer_norm( + x: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + eps: float, + is_swish: bool, +) -> torch.Tensor: + if should_trigger_eager_impl(): + if is_swish: + return pytorch_swish_layer_norm(x, [x.shape[1]], weight, bias, eps).to( + x.dtype + ) + else: + return pytorch_layer_norm(x, [x.shape[1]], weight, bias, eps).to(x.dtype) + else: + return _triton_aot_swish_layer_norm(x, weight, bias, eps, is_swish) diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/triton_layer_norm_mul_dropout.py b/recommendation_v4/generative_recommenders/ops/triton_aot/triton_layer_norm_mul_dropout.py new file mode 100644 index 000000000..7f7a6c743 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton_aot/triton_layer_norm_mul_dropout.py @@ -0,0 +1,162 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files +# (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, +# publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +# pyre-strict + +#!/usr/bin/env python3 + + +import torch +from generative_recommenders.common import next_power_of_2, should_trigger_eager_impl +from generative_recommenders.ops.pytorch.pt_hstu_linear import pytorch_norm_mul_dropout +from generative_recommenders.ops.triton.triton_hstu_linear import _ln_mul_dropout_fwd +from generative_recommenders.ops.triton_aot.types import triton_aot + +_ln_mul_dropout_fwd = triton_aot( + annotations={ + "D": ("i32", 16), + "stride_x": ("i32", 16), + "stride_u": ("i32", 16), + "stride_y": ("i32", 16), + }, + # pyrefly: ignore [bad-argument-type] +)(_ln_mul_dropout_fwd) + + +@torch.jit.unused +@torch.fx.wrap +def _triton_aot_layer_norm_mul_dropout( + x: torch.Tensor, + u: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + eps: float, + dropout_ratio: float, + training: bool, + silu_u: bool, + concat_ux: bool, + mul_u_activation_type: str, +) -> torch.Tensor: + assert x.dim() == 2 + if x.stride(1) != 1: + x = x.contiguous() + N, D = x.shape + assert weight.dim() == 1 + assert bias.dim() == 1 + assert weight.numel() == D + assert bias.numel() == D + + if concat_ux: + y = torch.empty((N, 3 * D), dtype=x.dtype, device=x.device) + else: + y = torch.empty_like(x) + if N == 0: + return y + mean = x.new_empty((N,)) + rstd = x.new_empty((N,)) + + # Less than 64KB per feature: enqueue fused kernel + MAX_FUSED_SIZE = 65536 // x.element_size() + BLOCK_D = min(MAX_FUSED_SIZE, next_power_of_2(D)) + if D > BLOCK_D: + raise RuntimeError("This layer norm doesn't support feature dim >= 64KB.") + + seed = 0 + # num_warps = min(max(BLOCK_D // 256, 1), 8) + grid = (N,) + # pyrefly: ignore [not-callable] + _ln_mul_dropout_fwd[grid]( + x, + u, + y, + weight, + bias, + mean, + rstd, + D, + eps, + seed, + dropout_ratio, + x.stride(0), + u.stride(0), + y.stride(0), + # pyrefly: ignore [bad-argument-type] + SILU_U=silu_u, + # pyrefly: ignore [bad-argument-type] + BLOCK_D=BLOCK_D, + # pyrefly: ignore [bad-argument-type] + TRAINING=training, + # pyrefly: ignore [bad-argument-type] + CONCAT_U=concat_ux, + # pyrefly: ignore [bad-argument-type] + CONCAT_X=concat_ux, + # pyrefly: ignore [bad-argument-type] + MUL_U_ACTIVATION_TYPE=mul_u_activation_type, + # pyrefly: ignore [bad-argument-type] + FAST_DROPOUT=False, + ) + return y + + +@torch.fx.wrap +# "aot_triton_kernel_wrapper_" is a pre-defined prefix for +# AOT-T triton kernel wrapper functions. This is required for +# AOT-T backend to recognize and trace correctly for ops transformation. +def aot_triton_kernel_wrapper_layer_norm_mul_dropout( + x: torch.Tensor, + u: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + eps: float, + dropout_ratio: float, + training: bool, + silu_u: bool, + concat_ux: bool, + mul_u_activation_type: str, +) -> torch.Tensor: + if should_trigger_eager_impl(): + return pytorch_norm_mul_dropout( + x=x, + u=u, + weight=weight, + bias=bias, + eps=eps, + dropout_ratio=dropout_ratio, + training=training, + silu_u=silu_u, + concat_u=concat_ux, + concat_x=concat_ux, + mul_u_activation_type=mul_u_activation_type, + group_norm=False, + ) + else: + return _triton_aot_layer_norm_mul_dropout( + x=x, + u=u, + weight=weight, + bias=bias, + eps=eps, + dropout_ratio=dropout_ratio, + training=training, + silu_u=silu_u, + concat_ux=concat_ux, + mul_u_activation_type=mul_u_activation_type, + ) diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/triton_position.py b/recommendation_v4/generative_recommenders/ops/triton_aot/triton_position.py new file mode 100644 index 000000000..828a4f4d4 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton_aot/triton_position.py @@ -0,0 +1,176 @@ +# (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +# pyre-strict + +from typing import Optional + +import torch +from generative_recommenders.common import ( + cdiv, + fx_unwrap_optional_tensor, + next_power_of_2, + prev_power_of_2, + should_trigger_eager_impl, +) +from generative_recommenders.ops.pytorch.pt_position import ( + pytorch_add_timestamp_positional_embeddings, +) +from generative_recommenders.ops.triton.triton_position import ( + _add_timestamp_position_embeddings_kernel, +) +from generative_recommenders.ops.triton_aot.types import triton_aot + + +_add_timestamp_position_embeddings_kernel = triton_aot( + annotations={ + "SeqEmb": ("*bf16", 16), + "Offsets": ("*i64", 16), + "Lengths": ("*i64", 16), + "PosEmb": ("*fp32", 16), + "TsEmb": ("*fp32", 16), + "Out": ("*bf16", 16), + "TS": ("*i64", 16), + "PosInds": ("*i32", 16), + "TsInds": ("*i32", 16), + "NumTargets": ("*i64", 16), + "AUTOTUNE_MAX_SEQ_LEN": "i32", + "D": "i32", + "num_time_buckets": "i32", + "time_bucket_increments": "fp32", + "time_bucket_scale": "fp32", + "time_delta": "i32", + "max_contextual_seq_len": "i32", + "max_pos_ind": "i32", + "stride_sn": ("i32", 16), + "stride_pn": ("i32", 16), + "stride_tn": ("i32", 16), + "stride_on": ("i32", 16), + }, +)(_add_timestamp_position_embeddings_kernel) + + +@torch.jit.unused +@torch.fx.wrap +def _triton_aot_position( + seq_embeddings: torch.Tensor, + seq_offsets: torch.Tensor, + pos_embeddings: torch.Tensor, + ts_embeddings: torch.Tensor, + timestamps: torch.Tensor, + max_seq_len: int, + max_contextual_seq_len: int, + seq_lengths: torch.Tensor, + num_targets: Optional[torch.Tensor], + interleave_targets: bool, + time_bucket_fn: str, +) -> torch.Tensor: + has_multiple_targets = num_targets is not None + if not has_multiple_targets: + num_targets_resolved = torch.empty( + 0, dtype=torch.int64, device=seq_embeddings.device + ) + else: + num_targets_resolved = fx_unwrap_optional_tensor(num_targets).to(torch.int64) + + seq_embeddings = seq_embeddings.contiguous() + pos_embeddings = pos_embeddings.contiguous() + ts_embeddings = ts_embeddings.contiguous() + + max_pos_ind = pos_embeddings.shape[0] + B = seq_lengths.shape[0] + + N, D = seq_embeddings.shape + out = torch.empty_like(seq_embeddings) + + timestamps = timestamps.contiguous() + ts_inds = torch.empty((N,), device=timestamps.device, dtype=torch.int32) + pos_inds = torch.empty((N,), device=timestamps.device, dtype=torch.int32) + + autotune_max_seq_len = prev_power_of_2(max_seq_len) + BLOCK_D = next_power_of_2(D) if D < 64 else 64 + + grid = lambda meta: ( # noqa E731 + B, + cdiv(max_seq_len, meta["BLOCK_N"]), + ) + # pyrefly: ignore [not-callable] + _add_timestamp_position_embeddings_kernel[grid]( + SeqEmb=seq_embeddings, + Offsets=seq_offsets, + Lengths=seq_lengths, + PosEmb=pos_embeddings, + TsEmb=ts_embeddings, + Out=out, + TS=timestamps, + PosInds=pos_inds, + TsInds=ts_inds, + NumTargets=num_targets_resolved, + AUTOTUNE_MAX_SEQ_LEN=autotune_max_seq_len, + D=D, + num_time_buckets=2048, + time_bucket_increments=60.0, + time_bucket_scale=1.0, + time_delta=0, + max_contextual_seq_len=max_contextual_seq_len, + max_pos_ind=max_pos_ind, + stride_sn=seq_embeddings.stride(0), + stride_pn=pos_embeddings.stride(0), + stride_tn=ts_embeddings.stride(0), + stride_on=out.stride(0), + TRAINING=False, + HAS_MULTIPLE_TARGETS=has_multiple_targets, + INTERLEAVE_TARGETS=interleave_targets, + TIME_BUCKET_FN=time_bucket_fn, + BLOCK_D=BLOCK_D, + ) + + return out + + +@torch.fx.wrap +# "aot_triton_kernel_wrapper_" is a pre-defined prefix for +# AOT-T triton kernel wrapper functions. This is required for +# AOT-T backend to recognize and trace correctly for ops transformation. +def aot_triton_kernel_wrapper_position( + alpha: float, + max_seq_len: int, + max_contextual_seq_len: int, + position_embeddings_weight: torch.Tensor, + timestamp_embeddings_weight: torch.Tensor, + seq_offsets: torch.Tensor, + seq_lengths: torch.Tensor, + seq_embeddings: torch.Tensor, + timestamps: torch.Tensor, + num_targets: Optional[torch.Tensor], + interleave_targets: bool, + time_bucket_fn: str, +) -> torch.Tensor: + seq_embeddings = seq_embeddings * alpha + if should_trigger_eager_impl(): + return pytorch_add_timestamp_positional_embeddings( + seq_embeddings=seq_embeddings, + seq_offsets=seq_offsets, + pos_embeddings=position_embeddings_weight, + ts_embeddings=timestamp_embeddings_weight, + timestamps=timestamps, + max_seq_len=max_seq_len, + max_contextual_seq_len=max_contextual_seq_len, + seq_lengths=seq_lengths, + num_targets=num_targets, + interleave_targets=interleave_targets, + time_bucket_fn=time_bucket_fn, + ) + else: + return _triton_aot_position( + seq_embeddings=seq_embeddings, + seq_offsets=seq_offsets, + pos_embeddings=position_embeddings_weight, + ts_embeddings=timestamp_embeddings_weight, + timestamps=timestamps, + max_seq_len=max_seq_len, + max_contextual_seq_len=max_contextual_seq_len, + seq_lengths=seq_lengths, + num_targets=num_targets, + interleave_targets=interleave_targets, + time_bucket_fn=time_bucket_fn, + ) diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/triton_ragged_hstu_attention.py b/recommendation_v4/generative_recommenders/ops/triton_aot/triton_ragged_hstu_attention.py new file mode 100644 index 000000000..4fa11dddc --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton_aot/triton_ragged_hstu_attention.py @@ -0,0 +1,366 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files +# (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, +# publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +# pyre-strict + +#!/usr/bin/env python3 + +from typing import Optional + +import torch +from generative_recommenders.common import ( + autotune_max_seq_len, + BACKEND_ALLOW_TF32, + cdiv, + prev_power_of_2, + should_trigger_eager_impl, +) +from generative_recommenders.ops.pytorch.pt_hstu_attention import ( + pytorch_cached_hstu_mha, + pytorch_hstu_mha, +) +from generative_recommenders.ops.triton.triton_hstu_attention import _hstu_attn_fwd +from generative_recommenders.ops.triton_aot.types import triton_aot + + +for _config in _hstu_attn_fwd.configs: + if isinstance(_config.kwargs.get("USE_TLX"), bool): + _config.kwargs["USE_TLX"] = int(_config.kwargs["USE_TLX"]) + + +_hstu_attn_fwd = triton_aot( + annotations={ + "stride_qm": ("i32", 16), + "stride_qh": ("i32", 16), + "stride_kn": ("i32", 16), + "stride_kh": ("i32", 16), + "stride_vn": ("i32", 16), + "stride_vh": ("i32", 16), + "stride_om": ("i32", 16), + "stride_oh": ("i32", 16), + "contextual_seq_len": "i32", + "max_attn_len": "i32", + "Z": "i32", + "AUTOTUNE_Z": "i32", + "H": "i32", + "MAX_SEQ_LEN": "i32", + "AUTOTUNE_MAX_SEQ_LEN": "i32", + "DimQ": "i32", + "DimV": "i32", + "DeltaSize": "i32", + "workspace_ptr": "*i8", + "sort_by_length_indices": "*i64", + } +)(_hstu_attn_fwd) + + +def _check_common_args( + invalid_attn_mask_type: str, + attn_scale: Optional[torch.Tensor], + full_attn_size: int, + num_softmax_heads: int, +) -> None: + assert invalid_attn_mask_type in ("causal", "lower_triangular"), ( + f"unsupported invalid_attn_mask_type: {invalid_attn_mask_type}" + ) + assert attn_scale is None, "attn_scale is not implemented for AOT-T HSTU MHA" + assert full_attn_size == 0, "full_attn_size is not implemented for AOT-T HSTU MHA" + assert num_softmax_heads == 0, ( + "num_softmax_heads is not implemented for AOT-T HSTU MHA" + ) + + +@torch.jit.unused +@torch.fx.wrap +def _triton_aot_ragged_hstu_mha( + N: int, + alpha: float, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + seq_offsets: torch.Tensor, + invalid_attn_mask_type: str, + num_targets: Optional[torch.Tensor], + attn_scale: Optional[torch.Tensor], + max_attn_len: int, + contextual_seq_len: int, + full_attn_size: int, + num_softmax_heads: int = 0, + allow_tf32: bool = BACKEND_ALLOW_TF32, +) -> torch.Tensor: + assert invalid_attn_mask_type in ("causal", "lower_triangular"), ( + f"unsupported invalid_attn_mask_type: {invalid_attn_mask_type}" + ) + assert attn_scale is None, "attn_scale is not implemented for AOT-T HSTU MHA" + assert full_attn_size == 0, "full_attn_size is not implemented for AOT-T HSTU MHA" + assert num_softmax_heads == 0, ( + "num_softmax_heads is not implemented for AOT-T HSTU MHA" + ) + Z = seq_offsets.numel() - 1 + L, H, DimQ = q.shape + DimV = v.shape[2] + + q = q.contiguous() + k = k.contiguous() + v = v.contiguous() + seq_offsets = seq_offsets.contiguous() + + out = torch.empty_like(v) + if L == 0: + return out + workspace = torch.empty(0, dtype=torch.int8, device=q.device) + sort_by_length_indices = torch.empty( + 0, dtype=torch.int64, device=seq_offsets.device + ) + + grid = lambda meta: ( # noqa E731 + cdiv(N, meta["BLOCK_M"]), + Z * H, + ) + # pyrefly: ignore [not-callable] + _hstu_attn_fwd[grid]( + Q=q, + K=k, + V=v, + workspace_ptr=workspace, + sort_by_length_indices=sort_by_length_indices, + seq_offsets=seq_offsets, + num_targets=num_targets, + Out=out, + stride_qm=q.stride(0), + stride_qh=q.stride(1), + stride_kn=k.stride(0), + stride_kh=k.stride(1), + stride_vn=v.stride(0), + stride_vh=v.stride(1), + stride_om=out.stride(0), + stride_oh=out.stride(1), + alpha=alpha, + contextual_seq_len=contextual_seq_len, + max_attn_len=max_attn_len, + Z=Z, + AUTOTUNE_Z=prev_power_of_2(Z), + H=H, + MAX_SEQ_LEN=N, + AUTOTUNE_MAX_SEQ_LEN=autotune_max_seq_len(N), + DimQ=DimQ, + DimV=DimV, + DeltaSize=0, + HAS_MULTIPLE_TARGETS=num_targets is not None, + IS_DELTA_Q=False, + ALLOW_TF32=allow_tf32, + BLOCK_D_Q=DimQ, + BLOCK_D_V=DimV, + HAS_CONTEXTUAL_SEQ_LEN=contextual_seq_len > 0, + HAS_MAX_ATTN_LEN=max_attn_len > 0, + HAS_SORT_BY_LENGTH_INDICES=False, + ENABLE_TMA=False, + TMA_DESC_SIZE=128, + ) + return out + + +@torch.fx.wrap +def aot_triton_kernel_wrapper_ragged_hstu_mha( + N: int, + alpha: float, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + seq_offsets: torch.Tensor, + invalid_attn_mask_type: str, + num_targets: Optional[torch.Tensor], + attn_scale: Optional[torch.Tensor], + max_attn_len: int, + contextual_seq_len: int, + full_attn_size: int, + num_softmax_heads: int, + allow_tf32: bool = BACKEND_ALLOW_TF32, +) -> torch.Tensor: + _check_common_args( + invalid_attn_mask_type=invalid_attn_mask_type, + attn_scale=attn_scale, + full_attn_size=full_attn_size, + num_softmax_heads=num_softmax_heads, + ) + if should_trigger_eager_impl(): + return pytorch_hstu_mha( + max_seq_len=N, + alpha=alpha, + q=q, + k=k, + v=v, + seq_offsets=seq_offsets, + causal=True, + dropout_pr=0.0, + training=False, + num_targets=num_targets, + attn_scale=attn_scale, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + min_full_attn_seq_len=full_attn_size, + ) + return _triton_aot_ragged_hstu_mha( + N=N, + alpha=alpha, + q=q, + k=k, + v=v, + seq_offsets=seq_offsets, + invalid_attn_mask_type=invalid_attn_mask_type, + num_targets=num_targets, + attn_scale=attn_scale, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + full_attn_size=full_attn_size, + num_softmax_heads=num_softmax_heads, + allow_tf32=allow_tf32, + ) + + +@torch.jit.unused +@torch.fx.wrap +def _triton_aot_cached_hstu_mha( + N: int, + alpha: float, + delta_q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + delta_x_offsets: torch.Tensor, + seq_offsets: torch.Tensor, + num_targets: Optional[torch.Tensor], + attn_scale: Optional[torch.Tensor], + max_attn_len: int, + full_attn_size: int, + allow_tf32: bool = BACKEND_ALLOW_TF32, +) -> torch.Tensor: + assert attn_scale is None, "attn_scale is not implemented for AOT-T HSTU MHA" + assert full_attn_size == 0, "full_attn_size is not implemented for AOT-T HSTU MHA" + Z = seq_offsets.size(0) - 1 + DELTA_L, H, DimQ = delta_q.shape + DeltaSize = DELTA_L // Z + DimV = v.shape[2] + + delta_q = delta_q.contiguous() + k = k.contiguous() + v = v.contiguous() + seq_offsets = seq_offsets.contiguous() + + out = torch.empty((DELTA_L, H, DimV), dtype=delta_q.dtype, device=delta_q.device) + if DELTA_L == 0: + return out + workspace = torch.empty(0, dtype=torch.int8, device=delta_q.device) + sort_by_length_indices = torch.empty( + 0, dtype=torch.int64, device=seq_offsets.device + ) + + grid = lambda meta: ( # noqa E731 + cdiv(DeltaSize, meta["BLOCK_M"]), + Z * H, + ) + # pyrefly: ignore [not-callable] + _hstu_attn_fwd[grid]( + Q=delta_q, + K=k, + V=v, + workspace_ptr=workspace, + sort_by_length_indices=sort_by_length_indices, + seq_offsets=seq_offsets, + num_targets=num_targets, + Out=out, + stride_qm=delta_q.stride(0), + stride_qh=delta_q.stride(1), + stride_kn=k.stride(0), + stride_kh=k.stride(1), + stride_vn=v.stride(0), + stride_vh=v.stride(1), + stride_om=out.stride(0), + stride_oh=out.stride(1), + alpha=alpha, + contextual_seq_len=0, + max_attn_len=max_attn_len, + Z=Z, + AUTOTUNE_Z=prev_power_of_2(Z), + H=H, + MAX_SEQ_LEN=N, + AUTOTUNE_MAX_SEQ_LEN=autotune_max_seq_len(N), + DimQ=DimQ, + DimV=DimV, + DeltaSize=DeltaSize, + HAS_MULTIPLE_TARGETS=num_targets is not None, + IS_DELTA_Q=True, + ALLOW_TF32=allow_tf32, + BLOCK_D_Q=DimQ, + BLOCK_D_V=DimV, + HAS_CONTEXTUAL_SEQ_LEN=False, + HAS_MAX_ATTN_LEN=max_attn_len > 0, + HAS_SORT_BY_LENGTH_INDICES=False, + ENABLE_TMA=False, + TMA_DESC_SIZE=128, + ) + return out + + +@torch.fx.wrap +def aot_triton_kernel_wrapper_cached_hstu_mha( + N: int, + alpha: float, + delta_q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + delta_x_offsets: torch.Tensor, + seq_offsets: torch.Tensor, + num_targets: Optional[torch.Tensor], + attn_scale: Optional[torch.Tensor], + max_attn_len: int, + full_attn_size: int, +) -> torch.Tensor: + _check_common_args( + invalid_attn_mask_type="causal", + attn_scale=attn_scale, + full_attn_size=full_attn_size, + num_softmax_heads=0, + ) + if should_trigger_eager_impl(): + return pytorch_cached_hstu_mha( + max_seq_len=N, + alpha=alpha, + delta_q=delta_q, + k=k, + v=v, + seq_offsets=seq_offsets, + num_targets=num_targets, + max_attn_len=max_attn_len, + contextual_seq_len=0, + ) + return _triton_aot_cached_hstu_mha( + N=N, + alpha=alpha, + delta_q=delta_q, + k=k, + v=v, + delta_x_offsets=delta_x_offsets, + seq_offsets=seq_offsets, + num_targets=num_targets, + attn_scale=attn_scale, + max_attn_len=max_attn_len, + full_attn_size=full_attn_size, + ) diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/triton_rms_norm.py b/recommendation_v4/generative_recommenders/ops/triton_aot/triton_rms_norm.py new file mode 100644 index 000000000..e5d9e093e --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton_aot/triton_rms_norm.py @@ -0,0 +1,114 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files +# (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, +# publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +# pyre-strict + +#!/usr/bin/env python3 + +import torch +from generative_recommenders.common import ( + cdiv, + next_power_of_2, + should_trigger_eager_impl, + switch_to_contiguous_if_needed, +) +from generative_recommenders.ops.pytorch.pt_layer_norm import pytorch_rms_norm +from generative_recommenders.ops.triton.triton_layer_norm import _weighted_rms_norm_fwd +from generative_recommenders.ops.triton_aot.types import triton_aot + +_weighted_rms_norm_fwd = triton_aot( + annotations={ + "N": "i32", + "D": ("i32", 16), + "stride_x": ("i32", 16), + "stride_y": ("i32", 16), + }, +)(_weighted_rms_norm_fwd) + + +@torch.jit.unused +@torch.fx.wrap +def _triton_aot_rms_norm( + x: torch.Tensor, + weight: torch.Tensor, + eps: float, + silu: bool, +) -> torch.Tensor: + """Internal AOTT kernel function for RMS norm.""" + assert x.dim() == 2, f"x.dim() == {x.dim()}, expected 2" + x = switch_to_contiguous_if_needed(x) + N, D = x.shape + + assert weight.dim() == 1 + assert weight.numel() == D + + y = torch.empty_like(x) + rstd = torch.empty(N, dtype=torch.float32, device=x.device) + + BLOCK_D = next_power_of_2(D) + + grid = lambda meta: ( # noqa E731 + cdiv(N, meta["BLOCK_N"]), + ) + # pyrefly: ignore [not-callable] + _weighted_rms_norm_fwd[grid]( + x, + y, + weight, + rstd, + N, + D, + eps, + stride_x=x.stride(0), + stride_y=y.stride(0), + SILU=silu, + BLOCK_D=BLOCK_D, + ) + + return y + + +def _pytorch_rms_norm_fallback( + x: torch.Tensor, + weight: torch.Tensor, + eps: float, + silu: bool, +) -> torch.Tensor: + """PyTorch fallback for RMS norm in eager mode.""" + + return pytorch_rms_norm(x, [x.shape[-1]], weight, eps, silu) + + +@torch.fx.wrap +def aot_triton_kernel_wrapper_rms_norm( + x: torch.Tensor, + weight: torch.Tensor, + eps: float, + silu: bool, +) -> torch.Tensor: + """AOT-T wrapper for RMS norm. + + Routes between PyTorch fallback (for tracing/serialization) and AOTT kernel path. + """ + if should_trigger_eager_impl(): + return _pytorch_rms_norm_fallback(x, weight, eps, silu) + else: + return _triton_aot_rms_norm(x, weight, eps, silu) diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/triton_split_2d_jagged.py b/recommendation_v4/generative_recommenders/ops/triton_aot/triton_split_2d_jagged.py new file mode 100644 index 000000000..9aa0c655b --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton_aot/triton_split_2d_jagged.py @@ -0,0 +1,138 @@ +# (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +# pyre-strict + +from typing import Optional, Tuple + +import torch +from generative_recommenders.common import ( + fx_unwrap_optional_tensor, + next_power_of_2, + should_trigger_eager_impl, +) +from generative_recommenders.ops.pytorch.pt_jagged_tensors import ( + pytorch_split_2D_jagged, +) +from generative_recommenders.ops.triton.triton_jagged import split_2D_jagged +from generative_recommenders.ops.triton_aot.types import triton_aot + + +split_2D_jagged = triton_aot( + annotations={ + "DenseSize": "i32", + "D": ("i32", 16), + "stride_id": ("i32", 16), + "stride_ad": ("i32", 16), + "stride_bd": ("i32", 16), + }, + # pyrefly: ignore [bad-argument-type] +)(split_2D_jagged) + + +@torch.jit.unused +@torch.fx.wrap +def _triton_aot_split_2D_jagged( + values: torch.Tensor, + max_seq_len: int, + offsets_a: torch.Tensor, + offsets_b: torch.Tensor, + dense_size: int = 0, + is_dense_a: bool = False, + is_dense_b: bool = False, +) -> Tuple[torch.Tensor, torch.Tensor]: + _, D = values.shape + BLOCK_D = next_power_of_2(D) + + if is_dense_a: + L, _ = values.shape + B = offsets_b.size(0) - 1 + seq_len_a = dense_size * B + seq_len_b = L - seq_len_a + elif is_dense_b: + L, _ = values.shape + B = offsets_a.size(0) - 1 + seq_len_b = dense_size * B + seq_len_a = L - seq_len_b + else: + B = offsets_a.size(0) - 1 + seq_len_a = int(offsets_a[-1].item()) + seq_len_b = int(offsets_b[-1].item()) + + values_a = torch.empty((seq_len_a, D), device=values.device, dtype=values.dtype) + values_b = torch.empty((seq_len_b, D), device=values.device, dtype=values.dtype) + + grid = (max_seq_len, B) + # pyre-ignore[29]: TritonAOT.__getitem__ is callable at runtime + split_2D_jagged[grid]( + JaggedIn=values, + DenseSize=dense_size, + OffsetsA=offsets_a, + OffsetsB=offsets_b, + OutA=values_a, + OutB=values_b, + D=D, + stride_id=values.stride(0), + stride_ad=values_a.stride(0), + stride_bd=values_b.stride(0), + # pyrefly: ignore [bad-argument-type] + IS_DENSE_A=is_dense_a, + # pyrefly: ignore [bad-argument-type] + IS_DENSE_B=is_dense_b, + # pyrefly: ignore [bad-argument-type] + BLOCK_D=BLOCK_D, + # pyrefly: ignore [bad-argument-type] + IS_REPLACE=False, + ) + + if is_dense_a: + values_a = values_a.reshape(B, dense_size, D) + if is_dense_b: + values_b = values_b.reshape(B, dense_size, D) + + return values_a, values_b + + +@torch.fx.wrap +def aot_triton_kernel_wrapper_split_2D_jagged( + values: torch.Tensor, + max_seq_len: int, + offsets_a: Optional[torch.Tensor] = None, + offsets_b: Optional[torch.Tensor] = None, + dense_size: int = 0, +) -> Tuple[torch.Tensor, torch.Tensor]: + if should_trigger_eager_impl(): + assert offsets_a is not None and offsets_b is not None, ( + "Eager fallback requires both offsets_a and offsets_b" + ) + return pytorch_split_2D_jagged( + max_seq_len=max_seq_len, + values=values, + max_len_left=None, + max_len_right=None, + offsets_left=offsets_a, + offsets_right=offsets_b, + ) + else: + is_dense_a: bool = offsets_a is None + is_dense_b: bool = offsets_b is None + resolved_offsets_a: torch.Tensor = values.new_empty(0) + resolved_offsets_b: torch.Tensor = values.new_empty(0) + if is_dense_a: + resolved_offsets_b = fx_unwrap_optional_tensor(offsets_b) + resolved_offsets_a = resolved_offsets_b.new_empty(0) + elif is_dense_b: + resolved_offsets_a = fx_unwrap_optional_tensor(offsets_a) + resolved_offsets_b = resolved_offsets_a.new_empty(0) + else: + resolved_offsets_a = fx_unwrap_optional_tensor(offsets_a) + resolved_offsets_b = fx_unwrap_optional_tensor(offsets_b) + + return _triton_aot_split_2D_jagged( + values=values, + max_seq_len=max_seq_len, + offsets_a=resolved_offsets_a, + offsets_b=resolved_offsets_b, + dense_size=dense_size, + is_dense_a=is_dense_a, + is_dense_b=is_dense_b, + ) diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/types.py b/recommendation_v4/generative_recommenders/ops/triton_aot/types.py new file mode 100644 index 000000000..bd7b2a1ed --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton_aot/types.py @@ -0,0 +1,181 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. + +# pyre-strict + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from typing import Any, Callable, ClassVar, Dict, List, Optional, Protocol, Union + +from generative_recommenders.ops.triton_aot.compile.utils import is_autotuner + +# @manual=//triton:triton +from triton.runtime.jit import KernelInterface +# triton.fb.triton_util depends on torch +# @dep=//caffe2:_torch + + +_VALID_HINTS: frozenset[int] = frozenset({1, 8, 16}) +_VALID_POINTER_HINTS: frozenset[int] = frozenset({16}) + + +@dataclass(frozen=True) +class AnnotationHint: + """Annotation with a value hint (dtype + divisibility/alignment). + + Valid hints: 16 (divisible_by_16), 8 (divisible_by_8), 1 (equal_to_1). + For pointers (dtype starts with ``*``), only 16 is valid — other values + would cause incorrect codegen (e.g. alignment=1 folds the pointer as a + constexpr constant, causing a segfault at launch). + """ + + dtype: str + hint: int + + def __post_init__(self) -> None: + if self.hint not in _VALID_HINTS: + raise RuntimeError( + f"TritonAOT: invalid annotation hint {self.hint!r} for " + f"dtype {self.dtype!r}. Valid hints: {sorted(_VALID_HINTS)}." + ) + if self.dtype.startswith("*") and self.hint not in _VALID_POINTER_HINTS: + raise RuntimeError( + f"TritonAOT: invalid pointer alignment {self.hint!r} for " + f"dtype {self.dtype!r}. Pointer annotations only support " + f"alignment={sorted(_VALID_POINTER_HINTS)}." + ) + + def to_tuple(self) -> tuple[str, int]: + """Convert to plain tuple for raw spec format.""" + return (self.dtype, self.hint) + + +# Internal annotation type (after normalization). +Annotation = Union[str, AnnotationHint] + +# User-facing input type (also accepts raw tuples). +AnnotationInput = Union[str, tuple[str, int], AnnotationHint] + + +def _normalize_annotation(ann: AnnotationInput) -> Annotation: + """Convert a raw tuple to AnnotationHint (triggers validation).""" + if isinstance(ann, AnnotationHint): + return ann + if isinstance(ann, tuple): + return AnnotationHint(ann[0], ann[1]) + return ann + + +class SpecCollector(Protocol): + """Callback invoked by TritonAOT.run() to collect kernel specs during AOT compile.""" + + def __call__( + self, + fn: KernelInterface[List[Any]], + annotations: Dict[str, Annotation], + *args: Any, + **kwargs: Any, + ) -> None: ... + + +logger: logging.Logger = logging.getLogger(__name__) + + +class TritonAOTMeta(type): + # TODO consider merge with AOTTCompileState + def __init__(cls, name, bases, attrs): # pyre-ignore [2,3] + super().__init__(name, bases, attrs) + # Initialize an empty list for each new class created + cls._instances: List["TritonAOT"] = [] + + def __call__(cls, *args, **kwargs): # pyre-ignore [2,3] + # Create the instance using the default behavior + instance = super().__call__(*args, **kwargs) + # Store the instance in the class-specific list + cls._instances.append(instance) + return instance + + def get_instances(cls) -> List["TritonAOT"]: + return cls._instances + + +class TritonAOT(KernelInterface[List[Any]], metaclass=TritonAOTMeta): + """Wraps a Triton kernel for ahead-of-time compilation. + + Annotations specify dtype and optional value hints for kernel parameters: + + - Scalar: ``"i32"``, ``"fp32"``, or ``AnnotationHint("i32", 16)`` + where 16 means the runtime value is divisible by 16. + - Pointer: ``AnnotationHint("*fp32", 16)`` for 16-byte aligned tensors. + Only alignment=16 is valid for pointers. + - Tensor: typically inferred from runtime ``torch.Tensor.dtype``. + - Optional tensor: auto-detected when the same kernel is called + with a tensor at one site and ``None`` at another. + """ + + _spec_collector: ClassVar[Optional[SpecCollector]] = None + + def __init__( + self, + fn: KernelInterface[List[Any]], + annotations: Dict[str, AnnotationInput], + ) -> None: + self.fn: KernelInterface[List[Any]] = fn + self.annotations: Dict[str, Annotation] = { + k: _normalize_annotation(v) for k, v in annotations.items() + } + + @classmethod + def set_spec_collector(cls, collector: Optional[SpecCollector]) -> None: + """Register or unregister the spec collection callback. + + When a collector is registered (not None), TritonAOT.run() will call + it to collect kernel specs for AOT compilation. When None, run() + simply delegates to the underlying Triton kernel (normal JIT path). + """ + cls._spec_collector = collector + + # pyrefly: ignore [bad-override] + def run(self, *args: Any, **kwargs: Any) -> Any: + if self._spec_collector is not None: + self._spec_collector(self.fn, self.annotations, *args, **kwargs) + # pyre-ignore[29]: KernelInterface.run is callable at runtime + return self.fn.run(*args, **kwargs) + + +def triton_aot( + annotations: Dict[str, AnnotationInput], +) -> Callable[[KernelInterface[List[Any]]], TritonAOT]: + def decorator(fn: KernelInterface[List[Any]]) -> TritonAOT: + return TritonAOT(fn, annotations) + + return decorator + + +def get_all_triton_aot_instances() -> List[TritonAOT]: + """Return all triton aot function instances (e.g. decorated with @triton_aot).""" + return TritonAOT.get_instances() + + +def reset_all_triton_aot_autotune_cache() -> bool: + """Reset triton autotune cache for all triton aot kernels. + + If triton aot compile is not enabled, this function is no op. Return True if any + kernel's autotune cache is reset. Else return False. + + """ + if TritonAOT._spec_collector is None: + return False + + reset = False + for triton_aot_kernel in get_all_triton_aot_instances(): + if is_autotuner(triton_aot_kernel.fn): + autotune_fn = triton_aot_kernel.fn + autotune_fn.cache.clear() # pyre-ignore [16] + logger.info( + f"Reset autotune cache for triton kernel {autotune_fn.fn.__name__}" # pyre-ignore [16] + ) + reset = True + + return reset diff --git a/recommendation_v4/generative_recommenders/ops/utils.py b/recommendation_v4/generative_recommenders/ops/utils.py new file mode 100644 index 000000000..94ab69e30 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/utils.py @@ -0,0 +1,93 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-ignore-all-errors + +import functools +import os + +import torch + + +class _PlainFuncWrapper: + """Thin wrapper around a plain function that provides no-op register_fake + and register_kernel methods, mirroring the CustomOpDef API so that + downstream @func.register_fake / func.register_kernel("cpu") calls + don't break when the function is not wrapped as a custom op.""" + + def __init__(self, func): + self._func = func + functools.update_wrapper(self, func) + + def __call__(self, *args, **kwargs): + return self._func(*args, **kwargs) + + def register_fake(self, fake_func): + return fake_func + + def register_kernel(self, device): + def inner(func): + return func + + return inner + + +def maybe_register_custom_op(op_name, mutates_args): + """ + Conditionally registers a function as a torch custom op. + + When AOTI_LOWER is set in the environment, the function is returned + unwrapped so that torch.export / Dynamo can trace through the plain + Python implementation instead of treating the custom op as opaque. + """ + + def decorator(func): + if os.environ.get("AOTI_LOWER"): + return _PlainFuncWrapper(func) + return torch.library.custom_op(op_name, func, mutates_args=mutates_args) + + return decorator + + +def is_sm100_plus() -> bool: + """ + Check if this is a Blackwell Datacenter GPU. + These are between 100 and 103 for B200-GB300. + """ + if not torch.cuda.is_available(): + return False + props = torch.cuda.get_device_properties(0) + return props.major == 10 and (props.minor >= 0 and props.minor <= 3) + + +def is_sm90() -> bool: + if not torch.cuda.is_available(): + return False + props = torch.cuda.get_device_properties(0) + return props.major == 9 and props.minor == 0 + + +def is_sm90_plus() -> bool: + return is_sm100_plus() or is_sm90() + + +def copy_if_different_ptr(dst: torch.Tensor, src: torch.Tensor) -> None: + if torch.compiler.is_compiling(): + # .data_ptr() will break PT2 + dst.copy_(src) + else: + if dst.data_ptr() != src.data_ptr(): + dst.copy_(src) diff --git a/recommendation_v4/generative_recommenders/research/data/dataset.py b/recommendation_v4/generative_recommenders/research/data/dataset.py new file mode 100644 index 000000000..09a18ae01 --- /dev/null +++ b/recommendation_v4/generative_recommenders/research/data/dataset.py @@ -0,0 +1,248 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +import csv +import linecache +from typing import Dict, List, Optional, Tuple + +import numpy as np +import pandas as pd +import torch + + +class DatasetV2(torch.utils.data.Dataset): + """In reverse chronological order.""" + + def __init__( + self, + ratings_file: str, + padding_length: int, + ignore_last_n: int, # used for creating train/valid/test sets + shift_id_by: int = 0, + chronological: bool = False, + sample_ratio: float = 1.0, + ) -> None: + """ + Args: + csv_file (string): Path to the csv file. + """ + super().__init__() + + self.ratings_frame: pd.DataFrame = pd.read_csv( + ratings_file, + delimiter=",", + # iterator=True, + ) + self._padding_length: int = padding_length + self._ignore_last_n: int = ignore_last_n + self._cache: Dict[int, Dict[str, torch.Tensor]] = dict() + self._shift_id_by: int = shift_id_by + self._chronological: bool = chronological + self._sample_ratio: float = sample_ratio + + def __len__(self) -> int: + return len(self.ratings_frame) + + def __getitem__(self, idx: int) -> Dict[str, torch.Tensor]: + if idx in self._cache.keys(): + return self._cache[idx] + data = self.ratings_frame.iloc[idx] + sample = self.load_item(data) + self._cache[idx] = sample + return sample + + def load_item(self, data) -> Dict[str, torch.Tensor]: + user_id = data.user_id + + def eval_as_list(x: str, ignore_last_n: int) -> List[int]: + y = eval(x) + y_list = [y] if type(y) == int else list(y) + if ignore_last_n > 0: + # for training data creation + y_list = y_list[:-ignore_last_n] + return y_list + + def eval_int_list( + x: str, + target_len: int, + ignore_last_n: int, + shift_id_by: int, + sampling_kept_mask: Optional[List[bool]], + ) -> Tuple[List[int], int]: + y = eval_as_list(x, ignore_last_n=ignore_last_n) + if sampling_kept_mask is not None: + y = [x for x, kept in zip(y, sampling_kept_mask) if kept] + y_len = len(y) + y.reverse() + if shift_id_by > 0: + y = [x + shift_id_by for x in y] + return y, y_len + + if self._sample_ratio < 1.0: + raw_length = len(eval_as_list(data.sequence_item_ids, self._ignore_last_n)) + sampling_kept_mask = ( + torch.rand((raw_length,), dtype=torch.float32) < self._sample_ratio + ).tolist() + else: + sampling_kept_mask = None + + movie_history, movie_history_len = eval_int_list( + data.sequence_item_ids, + self._padding_length, + self._ignore_last_n, + shift_id_by=self._shift_id_by, + sampling_kept_mask=sampling_kept_mask, + ) + movie_history_ratings, ratings_len = eval_int_list( + data.sequence_ratings, + self._padding_length, + self._ignore_last_n, + 0, + sampling_kept_mask=sampling_kept_mask, + ) + movie_timestamps, timestamps_len = eval_int_list( + data.sequence_timestamps, + self._padding_length, + self._ignore_last_n, + 0, + sampling_kept_mask=sampling_kept_mask, + ) + assert movie_history_len == timestamps_len, ( + f"history len {movie_history_len} differs from timestamp len {timestamps_len}." + ) + assert movie_history_len == ratings_len, ( + f"history len {movie_history_len} differs from ratings len {ratings_len}." + ) + + def _truncate_or_pad_seq( + y: List[int], target_len: int, chronological: bool + ) -> List[int]: + y_len = len(y) + if y_len < target_len: + y = y + [0] * (target_len - y_len) + else: + if not chronological: + y = y[:target_len] + else: + y = y[-target_len:] + assert len(y) == target_len + return y + + historical_ids = movie_history[1:] + historical_ratings = movie_history_ratings[1:] + historical_timestamps = movie_timestamps[1:] + target_ids = movie_history[0] + target_ratings = movie_history_ratings[0] + target_timestamps = movie_timestamps[0] + if self._chronological: + historical_ids.reverse() + historical_ratings.reverse() + historical_timestamps.reverse() + + max_seq_len = self._padding_length - 1 + history_length = min(len(historical_ids), max_seq_len) + historical_ids = _truncate_or_pad_seq( + historical_ids, + max_seq_len, + self._chronological, + ) + historical_ratings = _truncate_or_pad_seq( + historical_ratings, + max_seq_len, + self._chronological, + ) + historical_timestamps = _truncate_or_pad_seq( + historical_timestamps, + max_seq_len, + self._chronological, + ) + # moved to features.py + # if self._chronological: + # historical_ids.append(0) + # historical_ratings.append(0) + # historical_timestamps.append(0) + # print(historical_ids, historical_ratings, historical_timestamps, target_ids, target_ratings, target_timestamps) + ret = { + "user_id": user_id, + "historical_ids": torch.tensor(historical_ids, dtype=torch.int64), + "historical_ratings": torch.tensor(historical_ratings, dtype=torch.int64), + "historical_timestamps": torch.tensor( + historical_timestamps, dtype=torch.int64 + ), + "history_lengths": history_length, + "target_ids": target_ids, + "target_ratings": target_ratings, + "target_timestamps": target_timestamps, + } + return ret + + +class MultiFileDatasetV2(DatasetV2, torch.utils.data.Dataset): + def __init__( + self, + file_prefix: str, + num_files: int, + padding_length: int, + ignore_last_n: int, # used for creating train/valid/test sets + shift_id_by: int = 0, + chronological: bool = False, + sample_ratio: float = 1.0, + ) -> None: + torch.utils.data.Dataset().__init__() + self._file_prefix: str = file_prefix + self._num_files: int = num_files + with open(f"{file_prefix}_users.csv", "r") as file: + reader = csv.reader(file) + self.users_cumsum: List[int] = np.cumsum( + [int(row[1]) for row in reader] + ).tolist() + self._padding_length: int = padding_length + self._ignore_last_n: int = ignore_last_n + self._shift_id_by: int = shift_id_by + self._chronological: bool = chronological + self._sample_ratio: float = sample_ratio + + def __len__(self) -> int: + return self.users_cumsum[-1] + + def _process_line(self, line: str) -> pd.Series: + reader = csv.reader([line]) + parsed_line = next(reader) + user_id = int(parsed_line[0]) + sequence_item_ids = parsed_line[1] + sequence_ratings = parsed_line[2] + return pd.Series( + data={ + "user_id": user_id, + "sequence_item_ids": sequence_item_ids, + "sequence_ratings": sequence_ratings, + "sequence_timestamps": sequence_item_ids, # placeholder + } + ) + + def __getitem__(self, idx) -> Dict[str, torch.Tensor]: + assert idx < self.users_cumsum[-1] + file_idx: int = 0 + while self.users_cumsum[file_idx] <= idx: + file_idx += 1 + if file_idx == 0: + local_idx = idx + else: + local_idx = idx - self.users_cumsum[file_idx - 1] + line = linecache.getline(f"{self._file_prefix}_{file_idx}.csv", local_idx + 1) + data = self._process_line(line) + sample = self.load_item(data) + return sample diff --git a/recommendation_v4/generative_recommenders/research/data/eval.py b/recommendation_v4/generative_recommenders/research/data/eval.py new file mode 100644 index 000000000..16026e5c7 --- /dev/null +++ b/recommendation_v4/generative_recommenders/research/data/eval.py @@ -0,0 +1,263 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +import logging +import sys +from dataclasses import dataclass +from typing import Callable, Dict, List, Optional, Set, Union + +import torch +import torch.distributed as dist +from generative_recommenders.research.indexing.candidate_index import ( + CandidateIndex, + TopKModule, +) +from generative_recommenders.research.modeling.sequential.features import ( + SequentialFeatures, +) +from generative_recommenders.research.rails.similarities.module import SimilarityModule +from torch.utils.tensorboard import SummaryWriter + + +logging.basicConfig(stream=sys.stdout, level=logging.INFO) + + +@dataclass +class EvalState: + all_item_ids: Set[int] + candidate_index: CandidateIndex + top_k_module: TopKModule + + +def get_eval_state( + model: SimilarityModule, + all_item_ids: List[int], # [X] + negatives_sampler: torch.nn.Module, + top_k_module_fn: Callable[[torch.Tensor, torch.Tensor], TopKModule], + device: int, + float_dtype: Optional[torch.dtype] = None, +) -> EvalState: + # Exhaustively eval all items (incl. seen ids). + eval_negatives_ids = torch.as_tensor(all_item_ids).to(device).unsqueeze(0) # [1, X] + # pyre-fixme[29]: `Union[Tensor, Module]` is not a function. + eval_negative_embeddings = negatives_sampler.normalize_embeddings( + # pyre-fixme[29]: `Union[Tensor, Module]` is not a function. + model.get_item_embeddings(eval_negatives_ids) + ) + if float_dtype is not None: + eval_negative_embeddings = eval_negative_embeddings.to(float_dtype) + candidates = CandidateIndex( + ids=eval_negatives_ids, + embeddings=eval_negative_embeddings, + ) + return EvalState( + all_item_ids=set(all_item_ids), + candidate_index=candidates, + top_k_module=top_k_module_fn(eval_negative_embeddings, eval_negatives_ids), + ) + + +@torch.inference_mode # pyre-ignore [56] +def eval_metrics_v2_from_tensors( + eval_state: EvalState, + model: SimilarityModule, + seq_features: SequentialFeatures, + target_ids: torch.Tensor, # [B, 1] + min_positive_rating: int = 4, + target_ratings: Optional[torch.Tensor] = None, # [B, 1] + epoch: Optional[str] = None, + filter_invalid_ids: bool = True, + user_max_batch_size: Optional[int] = None, + dtype: Optional[torch.dtype] = None, +) -> Dict[str, Union[float, torch.Tensor]]: + """ + Args: + eval_negatives_ids: Optional[Tensor]. If not present, defaults to eval over + the entire corpus (`num_items`) excluding all the items that users have + seen in the past (historical_ids, target_ids). This is consistent with + papers like SASRec and TDM but may not be fair in practice as retrieval + modules don't have access to read state during the initial fetch stage. + filter_invalid_ids: bool. If true, filters seen ids by default. + Returns: + keyed metric -> list of values for each example. + """ + B, _ = target_ids.shape + device = target_ids.device + + for target_id in target_ids: + target_id = int(target_id) + if target_id not in eval_state.all_item_ids: + print(f"missing target_id {target_id}") + + # computes ro- part exactly once. + # pyre-fixme[29]: `Union[Tensor, Module]` is not a function. + shared_input_embeddings = model.encode( + past_lengths=seq_features.past_lengths, + past_ids=seq_features.past_ids, + # pyre-fixme[29]: `Union[Tensor, Module]` is not a function. + past_embeddings=model.get_item_embeddings(seq_features.past_ids), + past_payloads=seq_features.past_payloads, + ) + if dtype is not None: + shared_input_embeddings = shared_input_embeddings.to(dtype) + + MAX_K = 2500 + k = min(MAX_K, eval_state.candidate_index.ids.size(1)) + user_max_batch_size = user_max_batch_size or shared_input_embeddings.size(0) + num_batches = ( + shared_input_embeddings.size(0) + user_max_batch_size - 1 + ) // user_max_batch_size + eval_top_k_ids_all = [] + eval_top_k_prs_all = [] + for mb in range(num_batches): + eval_top_k_ids, eval_top_k_prs, _ = ( + eval_state.candidate_index.get_top_k_outputs( + query_embeddings=shared_input_embeddings[ + mb * user_max_batch_size : (mb + 1) * user_max_batch_size, ... + ], + top_k_module=eval_state.top_k_module, + k=k, + invalid_ids=( + seq_features.past_ids[ + mb * user_max_batch_size : (mb + 1) * user_max_batch_size, : + ] + if filter_invalid_ids + else None + ), + return_embeddings=False, + ) + ) + eval_top_k_ids_all.append(eval_top_k_ids) + eval_top_k_prs_all.append(eval_top_k_prs) + + if num_batches == 1: + eval_top_k_ids = eval_top_k_ids_all[0] + eval_top_k_prs = eval_top_k_prs_all[0] + else: + eval_top_k_ids = torch.cat(eval_top_k_ids_all, dim=0) + eval_top_k_prs = torch.cat(eval_top_k_prs_all, dim=0) + + assert eval_top_k_ids.size(1) == k + _, eval_rank_indices = torch.max( + torch.cat( + [eval_top_k_ids, target_ids], + dim=1, + ) + == target_ids, + dim=1, + ) + eval_ranks = torch.where(eval_rank_indices == k, MAX_K + 1, eval_rank_indices + 1) + + output = { + "ndcg@1": torch.where( + eval_ranks <= 1, + torch.div(1.0, torch.log2(eval_ranks + 1)), + torch.zeros(1, dtype=torch.float32, device=device), + ), + "ndcg@10": torch.where( + eval_ranks <= 10, + torch.div(1.0, torch.log2(eval_ranks + 1)), + torch.zeros(1, dtype=torch.float32, device=device), + ), + "ndcg@50": torch.where( + eval_ranks <= 50, + torch.div(1.0, torch.log2(eval_ranks + 1)), + torch.zeros(1, dtype=torch.float32, device=device), + ), + "ndcg@100": torch.where( + eval_ranks <= 100, + torch.div(1.0, torch.log2(eval_ranks + 1)), + torch.zeros(1, dtype=torch.float32, device=device), + ), + "ndcg@200": torch.where( + eval_ranks <= 200, + torch.div(1.0, torch.log2(eval_ranks + 1)), + torch.zeros(1, dtype=torch.float32, device=device), + ), + "hr@1": (eval_ranks <= 1), + "hr@10": (eval_ranks <= 10), + "hr@50": (eval_ranks <= 50), + "hr@100": (eval_ranks <= 100), + "hr@200": (eval_ranks <= 200), + "hr@500": (eval_ranks <= 500), + "hr@1000": (eval_ranks <= 1000), + "mrr": torch.div(1.0, eval_ranks), + } + if target_ratings is not None: + target_ratings = target_ratings.squeeze(1) # [B] + output["ndcg@10_>=4"] = torch.where( + eval_ranks[target_ratings >= 4] <= 10, + torch.div(1.0, torch.log2(eval_ranks[target_ratings >= 4] + 1)), + torch.zeros(1, dtype=torch.float32, device=device), + ) + output[f"hr@10_>={min_positive_rating}"] = ( + eval_ranks[target_ratings >= min_positive_rating] <= 10 + ) + output[f"hr@50_>={min_positive_rating}"] = ( + eval_ranks[target_ratings >= min_positive_rating] <= 50 + ) + output[f"mrr_>={min_positive_rating}"] = torch.div( + 1.0, eval_ranks[target_ratings >= min_positive_rating] + ) + + return output # pyre-ignore [7] + + +def eval_recall_metrics_from_tensors( + eval_state: EvalState, + model: SimilarityModule, + seq_features: SequentialFeatures, + user_max_batch_size: Optional[int] = None, + dtype: Optional[torch.dtype] = None, +) -> Dict[str, torch.Tensor]: + target_ids = seq_features.past_ids[:, -1].unsqueeze(1) + filtered_past_ids = seq_features.past_ids.detach().clone() + filtered_past_ids[:, -1] = torch.zeros_like(target_ids.squeeze(1)) + return eval_metrics_v2_from_tensors( + eval_state=eval_state, + model=model, + seq_features=SequentialFeatures( + past_lengths=seq_features.past_lengths - 1, + past_ids=filtered_past_ids, + past_embeddings=seq_features.past_embeddings, + past_payloads=seq_features.past_payloads, + ), + target_ids=target_ids, + user_max_batch_size=user_max_batch_size, + dtype=dtype, + ) + + +def _avg(x: torch.Tensor, world_size: int) -> torch.Tensor: + _sum_and_numel = torch.tensor( + [x.sum(), x.numel()], dtype=torch.float32, device=x.device + ) + if world_size > 1: + dist.all_reduce(_sum_and_numel, op=dist.ReduceOp.SUM) + return _sum_and_numel[0] / _sum_and_numel[1] + + +def add_to_summary_writer( + writer: Optional[SummaryWriter], + batch_id: int, + metrics: Dict[str, torch.Tensor], + prefix: str, + world_size: int, +) -> None: + for key, values in metrics.items(): + avg_value = _avg(values, world_size) + if writer is not None: + writer.add_scalar(f"{prefix}/{key}", avg_value, batch_id) diff --git a/recommendation_v4/generative_recommenders/research/data/item_features.py b/recommendation_v4/generative_recommenders/research/data/item_features.py new file mode 100644 index 000000000..8ecb6ea6a --- /dev/null +++ b/recommendation_v4/generative_recommenders/research/data/item_features.py @@ -0,0 +1,29 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +from dataclasses import dataclass +from typing import List + +import torch + + +@dataclass +class ItemFeatures: + num_items: int + max_jagged_dimension: int + max_ind_range: List[int] # [(,)] x num_features + lengths: List[torch.Tensor] # [(num_items,)] x num_features + values: List[torch.Tensor] # [(num_items, max_jagged_dimension)] x num_features diff --git a/recommendation_v4/generative_recommenders/research/data/preprocessor.py b/recommendation_v4/generative_recommenders/research/data/preprocessor.py new file mode 100644 index 000000000..bf52f41da --- /dev/null +++ b/recommendation_v4/generative_recommenders/research/data/preprocessor.py @@ -0,0 +1,474 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +import abc +import logging +import os +import sys +import tarfile +from typing import Dict, Optional, Union +from urllib.request import urlretrieve +from zipfile import ZipFile + +import numpy as np +import pandas as pd + + +logging.basicConfig(stream=sys.stdout, level=logging.INFO) + + +class DataProcessor: + """ + This preprocessor does not remap item_ids. This is intended so that we can easily join other + side-information based on item_ids later. + """ + + def __init__( + self, + prefix: str, + expected_num_unique_items: Optional[int], + expected_max_item_id: Optional[int], + ) -> None: + self._prefix: str = prefix + self._expected_num_unique_items = expected_num_unique_items + self._expected_max_item_id = expected_max_item_id + + @abc.abstractmethod + def expected_num_unique_items(self) -> Optional[int]: + return self._expected_num_unique_items + + @abc.abstractmethod + def expected_max_item_id(self) -> Optional[int]: + return self._expected_max_item_id + + @abc.abstractmethod + def processed_item_csv(self) -> str: + pass + + def output_format_csv(self) -> str: + return f"tmp/{self._prefix}/sasrec_format.csv" + + def to_seq_data( + self, + ratings_data: pd.DataFrame, + user_data: Optional[pd.DataFrame] = None, + ) -> pd.DataFrame: + if user_data is not None: + ratings_data_transformed = ratings_data.join( + user_data.set_index("user_id"), on="user_id" + ) + else: + ratings_data_transformed = ratings_data + ratings_data_transformed.item_ids = ratings_data_transformed.item_ids.apply( + lambda x: ",".join([str(v) for v in x]) + ) + ratings_data_transformed.ratings = ratings_data_transformed.ratings.apply( + lambda x: ",".join([str(v) for v in x]) + ) + ratings_data_transformed.timestamps = ratings_data_transformed.timestamps.apply( + lambda x: ",".join([str(v) for v in x]) + ) + ratings_data_transformed.rename( + columns={ + "item_ids": "sequence_item_ids", + "ratings": "sequence_ratings", + "timestamps": "sequence_timestamps", + }, + inplace=True, + ) + return ratings_data_transformed + + def file_exists(self, name: str) -> bool: + return os.path.isfile("%s/%s" % (os.getcwd(), name)) + + +class MovielensSyntheticDataProcessor(DataProcessor): + def __init__( + self, + prefix: str, + expected_num_unique_items: Optional[int] = None, + expected_max_item_id: Optional[int] = None, + ) -> None: + super().__init__(prefix, expected_num_unique_items, expected_max_item_id) + + def preprocess_rating(self) -> None: + return + + +class MovielensDataProcessor(DataProcessor): + def __init__( + self, + download_path: str, + saved_name: str, + prefix: str, + convert_timestamp: bool, + expected_num_unique_items: Optional[int] = None, + expected_max_item_id: Optional[int] = None, + ) -> None: + super().__init__(prefix, expected_num_unique_items, expected_max_item_id) + self._download_path = download_path + self._saved_name = saved_name + self._convert_timestamp: bool = convert_timestamp + + def download(self) -> None: + if not self.file_exists(self._saved_name): + urlretrieve(self._download_path, self._saved_name) + if self._saved_name[-4:] == ".zip": + ZipFile(self._saved_name, "r").extractall(path="tmp/") + else: + with tarfile.open(self._saved_name, "r:*") as tar_ref: + tar_ref.extractall("tmp/") + + def processed_item_csv(self) -> str: + return f"tmp/processed/{self._prefix}/movies.csv" + + def sasrec_format_csv_by_user_train(self) -> str: + return f"tmp/{self._prefix}/sasrec_format_by_user_train.csv" + + def sasrec_format_csv_by_user_test(self) -> str: + return f"tmp/{self._prefix}/sasrec_format_by_user_test.csv" + + def preprocess_rating(self) -> int: + self.download() + + if self._prefix == "ml-1m": + users = pd.read_csv( + f"tmp/{self._prefix}/users.dat", + sep="::", + names=["user_id", "sex", "age_group", "occupation", "zip_code"], + ) + ratings = pd.read_csv( + f"tmp/{self._prefix}/ratings.dat", + sep="::", + names=["user_id", "movie_id", "rating", "unix_timestamp"], + ) + movies = pd.read_csv( + f"tmp/{self._prefix}/movies.dat", + sep="::", + names=["movie_id", "title", "genres"], + encoding="iso-8859-1", + ) + elif self._prefix == "ml-20m": + # ml-20m + # ml-20m doesn't have user data. + users = None + # ratings: userId,movieId,rating,timestamp + ratings = pd.read_csv( + f"tmp/{self._prefix}/ratings.csv", + sep=",", + ) + ratings.rename( + columns={ + "userId": "user_id", + "movieId": "movie_id", + "timestamp": "unix_timestamp", + }, + inplace=True, + ) + # movieId,title,genres + # 1,Toy Story (1995),Adventure|Animation|Children|Comedy|Fantasy + # 2,Jumanji (1995),Adventure|Children|Fantasy + movies = pd.read_csv( + f"tmp/{self._prefix}/movies.csv", + sep=",", + encoding="iso-8859-1", + ) + movies.rename(columns={"movieId": "movie_id"}, inplace=True) + else: + assert self._prefix == "ml-20mx16x32" + # ml-1b + user_ids = [] + movie_ids = [] + for i in range(16): + train_file = f"tmp/{self._prefix}/trainx16x32_{i}.npz" + with np.load(train_file) as data: + user_ids.extend([x[0] for x in data["arr_0"]]) + movie_ids.extend([x[1] for x in data["arr_0"]]) + ratings = pd.DataFrame( + data={ + "user_id": user_ids, + "movie_id": movie_ids, + "rating": user_ids, # placeholder + "unix_timestamp": movie_ids, # placeholder + } + ) + users = None + movies = None + + if movies is not None: + # ML-1M and ML-20M only + movies["year"] = movies["title"].apply(lambda x: x[-5:-1]) + movies["cleaned_title"] = movies["title"].apply(lambda x: x[:-7]) + # movies.year = pd.Categorical(movies.year) + # movies["year"] = movies.year.cat.codes + + if users is not None: + ## Users (ml-1m only) + users.sex = pd.Categorical(users.sex) + users["sex"] = users.sex.cat.codes + + users.age_group = pd.Categorical(users.age_group) + users["age_group"] = users.age_group.cat.codes + + users.occupation = pd.Categorical(users.occupation) + users["occupation"] = users.occupation.cat.codes + + users.zip_code = pd.Categorical(users.zip_code) + users["zip_code"] = users.zip_code.cat.codes + + # Normalize movie ids to speed up training + print( + f"{self._prefix} #item before normalize: {len(set(ratings['movie_id'].values))}" + ) + print( + f"{self._prefix} max item id before normalize: {max(set(ratings['movie_id'].values))}" + ) + # print(f"ratings.movie_id.cat.categories={ratings.movie_id.cat.categories}; {type(ratings.movie_id.cat.categories)}") + # print(f"ratings.movie_id.cat.codes={ratings.movie_id.cat.codes}; {type(ratings.movie_id.cat.codes)}") + # print(movie_id_to_cat) + # ratings["movie_id"] = ratings.movie_id.cat.codes + # print(f"{self._prefix} #item after normalize: {len(set(ratings['movie_id'].values))}") + # print(f"{self._prefix} max item id after normalize: {max(set(ratings['movie_id'].values))}") + # movies["remapped_id"] = movies["movie_id"].apply(lambda x: movie_id_to_cat[x]) + + if self._convert_timestamp: + ratings["unix_timestamp"] = pd.to_datetime( + ratings["unix_timestamp"], unit="s" + ) + + # Save primary csv's + if not os.path.exists(f"tmp/processed/{self._prefix}"): + os.makedirs(f"tmp/processed/{self._prefix}") + if users is not None: + users.to_csv(f"tmp/processed/{self._prefix}/users.csv", index=False) + if movies is not None: + movies.to_csv(f"tmp/processed/{self._prefix}/movies.csv", index=False) + ratings.to_csv(f"tmp/processed/{self._prefix}/ratings.csv", index=False) + + num_unique_users = len(set(ratings["user_id"].values)) + num_unique_items = len(set(ratings["movie_id"].values)) + + # SASRec version + ratings_group = ratings.sort_values(by=["unix_timestamp"]).groupby("user_id") + seq_ratings_data = pd.DataFrame( + data={ + "user_id": list(ratings_group.groups.keys()), + "item_ids": list(ratings_group.movie_id.apply(list)), + "ratings": list(ratings_group.rating.apply(list)), + "timestamps": list(ratings_group.unix_timestamp.apply(list)), + } + ) + + result = pd.DataFrame([[]]) + for col in ["item_ids"]: + result[col + "_mean"] = seq_ratings_data[col].apply(len).mean() + result[col + "_min"] = seq_ratings_data[col].apply(len).min() + result[col + "_max"] = seq_ratings_data[col].apply(len).max() + print(self._prefix) + print(result) + + seq_ratings_data = self.to_seq_data(seq_ratings_data, users) + seq_ratings_data.sample(frac=1).reset_index().to_csv( + self.output_format_csv(), index=False, sep="," + ) + + # Split by user ids (not tested yet) + user_id_split = int(num_unique_users * 0.9) + seq_ratings_data_train = seq_ratings_data[ + seq_ratings_data["user_id"] <= user_id_split + ] + seq_ratings_data_train.sample(frac=1).reset_index().to_csv( + self.sasrec_format_csv_by_user_train(), + index=False, + sep=",", + ) + seq_ratings_data_test = seq_ratings_data[ + seq_ratings_data["user_id"] > user_id_split + ] + seq_ratings_data_test.sample(frac=1).reset_index().to_csv( + self.sasrec_format_csv_by_user_test(), index=False, sep="," + ) + print( + f"{self._prefix}: train num user: {len(set(seq_ratings_data_train['user_id'].values))}" + ) + print( + f"{self._prefix}: test num user: {len(set(seq_ratings_data_test['user_id'].values))}" + ) + + # print(seq_ratings_data) + if self.expected_num_unique_items() is not None: + assert self.expected_num_unique_items() == num_unique_items, ( + f"Expected items: {self.expected_num_unique_items()}, got: {num_unique_items}" + ) + + return num_unique_items + + +class AmazonDataProcessor(DataProcessor): + def __init__( + self, + download_path: str, + saved_name: str, + prefix: str, + expected_num_unique_items: Optional[int], + ) -> None: + super().__init__( + prefix, + expected_num_unique_items=expected_num_unique_items, + expected_max_item_id=None, + ) + self._download_path = download_path + self._saved_name = saved_name + self._prefix = prefix + + def download(self) -> None: + if not self.file_exists(self._saved_name): + urlretrieve(self._download_path, self._saved_name) + + def preprocess_rating(self) -> int: + self.download() + + ratings = pd.read_csv( + self._saved_name, + sep=",", + names=["user_id", "item_id", "rating", "timestamp"], + ) + print(f"{self._prefix} #data points before filter: {ratings.shape[0]}") + print( + f"{self._prefix} #user before filter: {len(set(ratings['user_id'].values))}" + ) + print( + f"{self._prefix} #item before filter: {len(set(ratings['item_id'].values))}" + ) + + # filter users and items with presence < 5 + item_id_count = ( + ratings["item_id"] + .value_counts() + .rename_axis("unique_values") + .reset_index(name="item_count") + ) + user_id_count = ( + ratings["user_id"] + .value_counts() + .rename_axis("unique_values") + .reset_index(name="user_count") + ) + ratings = ratings.join(item_id_count.set_index("unique_values"), on="item_id") + ratings = ratings.join(user_id_count.set_index("unique_values"), on="user_id") + ratings = ratings[ratings["item_count"] >= 5] + ratings = ratings[ratings["user_count"] >= 5] + print(f"{self._prefix} #data points after filter: {ratings.shape[0]}") + + # categorize user id and item id + ratings["item_id"] = pd.Categorical(ratings["item_id"]) + ratings["item_id"] = ratings["item_id"].cat.codes + ratings["user_id"] = pd.Categorical(ratings["user_id"]) + ratings["user_id"] = ratings["user_id"].cat.codes + print( + f"{self._prefix} #user after filter: {len(set(ratings['user_id'].values))}" + ) + print( + f"{self._prefix} #item ater filter: {len(set(ratings['item_id'].values))}" + ) + + num_unique_items = len(set(ratings["item_id"].values)) + + # SASRec version + ratings_group = ratings.sort_values(by=["timestamp"]).groupby("user_id") + + seq_ratings_data = pd.DataFrame( + data={ + "user_id": list(ratings_group.groups.keys()), + "item_ids": list(ratings_group.item_id.apply(list)), + "ratings": list(ratings_group.rating.apply(list)), + "timestamps": list(ratings_group.timestamp.apply(list)), + } + ) + + seq_ratings_data = seq_ratings_data[ + seq_ratings_data["item_ids"].apply(len) >= 5 + ] + + result = pd.DataFrame([[]]) + for col in ["item_ids"]: + result[col + "_mean"] = seq_ratings_data[col].apply(len).mean() + result[col + "_min"] = seq_ratings_data[col].apply(len).min() + result[col + "_max"] = seq_ratings_data[col].apply(len).max() + print(self._prefix) + print(result) + + if not os.path.exists(f"tmp/{self._prefix}"): + os.makedirs(f"tmp/{self._prefix}") + + seq_ratings_data = self.to_seq_data(seq_ratings_data) + seq_ratings_data.sample(frac=1).reset_index().to_csv( + self.output_format_csv(), index=False, sep="," + ) + + if self.expected_num_unique_items() is not None: + assert self.expected_num_unique_items() == num_unique_items, ( + f"expected: {self.expected_num_unique_items()}, actual: {num_unique_items}" + ) + logging.info(f"{self.expected_num_unique_items()} unique items.") + + return num_unique_items + + +def get_common_preprocessors() -> Dict[ + str, + Union[AmazonDataProcessor, MovielensDataProcessor, MovielensSyntheticDataProcessor], +]: + ml_1m_dp = MovielensDataProcessor( # pyre-ignore [45] + "http://files.grouplens.org/datasets/movielens/ml-1m.zip", + "tmp/movielens1m.zip", + prefix="ml-1m", + convert_timestamp=False, + expected_num_unique_items=3706, + expected_max_item_id=3952, + ) + ml_20m_dp = MovielensDataProcessor( # pyre-ignore [45] + "http://files.grouplens.org/datasets/movielens/ml-20m.zip", + "tmp/movielens20m.zip", + prefix="ml-20m", + convert_timestamp=False, + expected_num_unique_items=26744, + expected_max_item_id=131262, + ) + ml_1b_dp = MovielensDataProcessor( # pyre-ignore [45] + "https://files.grouplens.org/datasets/movielens/ml-20mx16x32.tar", + "tmp/movielens1b.tar", + prefix="ml-20mx16x32", + convert_timestamp=False, + ) + ml_3b_dp = MovielensSyntheticDataProcessor( # pyre-ignore [45] + prefix="ml-3b", + expected_num_unique_items=26743 * 32, + expected_max_item_id=26743 * 32, + ) + amzn_books_dp = AmazonDataProcessor( # pyre-ignore [45] + "http://snap.stanford.edu/data/amazon/productGraph/categoryFiles/ratings_Books.csv", + "tmp/ratings_Books.csv", + prefix="amzn_books", + expected_num_unique_items=695762, + ) + return { + "ml-1m": ml_1m_dp, + "ml-20m": ml_20m_dp, + "ml-1b": ml_1b_dp, + "ml-3b": ml_3b_dp, + "amzn-books": amzn_books_dp, + } diff --git a/recommendation_v4/generative_recommenders/research/data/reco_dataset.py b/recommendation_v4/generative_recommenders/research/data/reco_dataset.py new file mode 100644 index 000000000..eedcdc08a --- /dev/null +++ b/recommendation_v4/generative_recommenders/research/data/reco_dataset.py @@ -0,0 +1,176 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +from dataclasses import dataclass +from typing import List + +import pandas as pd +import torch +from generative_recommenders.research.data.dataset import DatasetV2, MultiFileDatasetV2 +from generative_recommenders.research.data.item_features import ItemFeatures +from generative_recommenders.research.data.preprocessor import get_common_preprocessors + + +@dataclass +class RecoDataset: + max_sequence_length: int + num_unique_items: int + max_item_id: int + all_item_ids: List[int] + train_dataset: torch.utils.data.Dataset + eval_dataset: torch.utils.data.Dataset + + +def get_reco_dataset( + dataset_name: str, + max_sequence_length: int, + chronological: bool, + positional_sampling_ratio: float = 1.0, +) -> RecoDataset: + if dataset_name == "ml-1m": + dp = get_common_preprocessors()[dataset_name] + train_dataset = DatasetV2( + ratings_file=dp.output_format_csv(), + padding_length=max_sequence_length + 1, # target + ignore_last_n=1, + chronological=chronological, + sample_ratio=positional_sampling_ratio, + ) + eval_dataset = DatasetV2( + ratings_file=dp.output_format_csv(), + padding_length=max_sequence_length + 1, # target + ignore_last_n=0, + chronological=chronological, + sample_ratio=1.0, # do not sample + ) + elif dataset_name == "ml-20m": + dp = get_common_preprocessors()[dataset_name] + train_dataset = DatasetV2( + ratings_file=dp.output_format_csv(), + padding_length=max_sequence_length + 1, # target + ignore_last_n=1, + chronological=chronological, + ) + eval_dataset = DatasetV2( + ratings_file=dp.output_format_csv(), + padding_length=max_sequence_length + 1, # target + ignore_last_n=0, + chronological=chronological, + ) + elif dataset_name == "ml-3b": + dp = get_common_preprocessors()[dataset_name] + train_dataset = MultiFileDatasetV2( + file_prefix="tmp/ml-3b/16x32", + num_files=16, + padding_length=max_sequence_length + 1, # target + ignore_last_n=1, + chronological=chronological, + ) + eval_dataset = MultiFileDatasetV2( + file_prefix="tmp/ml-3b/16x32", + num_files=16, + padding_length=max_sequence_length + 1, # target + ignore_last_n=0, + chronological=chronological, + ) + elif dataset_name == "amzn-books": + dp = get_common_preprocessors()[dataset_name] + train_dataset = DatasetV2( + ratings_file=dp.output_format_csv(), + padding_length=max_sequence_length + 1, # target + ignore_last_n=1, + shift_id_by=1, # [0..n-1] -> [1..n] + chronological=chronological, + ) + eval_dataset = DatasetV2( + ratings_file=dp.output_format_csv(), + padding_length=max_sequence_length + 1, # target + ignore_last_n=0, + shift_id_by=1, # [0..n-1] -> [1..n] + chronological=chronological, + ) + else: + raise ValueError(f"Unknown dataset {dataset_name}") + + if dataset_name == "ml-1m" or dataset_name == "ml-20m": + items = pd.read_csv(dp.processed_item_csv(), delimiter=",") + max_jagged_dimension = 16 + expected_max_item_id = dp.expected_max_item_id() + assert expected_max_item_id is not None + item_features: ItemFeatures = ItemFeatures( + max_ind_range=[63, 16383, 511], + num_items=expected_max_item_id + 1, + max_jagged_dimension=max_jagged_dimension, + lengths=[ + torch.zeros((expected_max_item_id + 1,), dtype=torch.int64), + torch.zeros((expected_max_item_id + 1,), dtype=torch.int64), + torch.zeros((expected_max_item_id + 1,), dtype=torch.int64), + ], + values=[ + torch.zeros( + (expected_max_item_id + 1, max_jagged_dimension), + dtype=torch.int64, + ), + torch.zeros( + (expected_max_item_id + 1, max_jagged_dimension), + dtype=torch.int64, + ), + torch.zeros( + (expected_max_item_id + 1, max_jagged_dimension), + dtype=torch.int64, + ), + ], + ) + all_item_ids = [] + for df_index, row in items.iterrows(): + # print(f"index {df_index}: {row}") + movie_id = int(row["movie_id"]) + genres = row["genres"].split("|") + titles = row["cleaned_title"].split(" ") + # print(f"{index}: genres{genres}, title{titles}") + genres_vector = [hash(x) % item_features.max_ind_range[0] for x in genres] + titles_vector = [hash(x) % item_features.max_ind_range[1] for x in titles] + years_vector = [hash(row["year"]) % item_features.max_ind_range[2]] + item_features.lengths[0][movie_id] = min( + len(genres_vector), max_jagged_dimension + ) + item_features.lengths[1][movie_id] = min( + len(titles_vector), max_jagged_dimension + ) + item_features.lengths[2][movie_id] = min( + len(years_vector), max_jagged_dimension + ) + for f, f_values in enumerate([genres_vector, titles_vector, years_vector]): + for j in range(min(len(f_values), max_jagged_dimension)): + item_features.values[f][movie_id][j] = f_values[j] + all_item_ids.append(movie_id) + max_item_id = dp.expected_max_item_id() + for x in all_item_ids: + assert x > 0, "x in all_item_ids should be positive" + else: + # expected_max_item_id and item_features are not set for Amazon datasets. + item_features = None + max_item_id = dp.expected_num_unique_items() + all_item_ids = [x + 1 for x in range(max_item_id)] # pyre-ignore [6] + + return RecoDataset( + max_sequence_length=max_sequence_length, + num_unique_items=dp.expected_num_unique_items(), # pyre-ignore [6] + max_item_id=max_item_id, # pyre-ignore [6] + all_item_ids=all_item_ids, + train_dataset=train_dataset, + eval_dataset=eval_dataset, + ) diff --git a/recommendation_v4/generative_recommenders/research/indexing/candidate_index.py b/recommendation_v4/generative_recommenders/research/indexing/candidate_index.py new file mode 100644 index 000000000..fee763eaa --- /dev/null +++ b/recommendation_v4/generative_recommenders/research/indexing/candidate_index.py @@ -0,0 +1,179 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +from typing import Optional, Tuple + +import torch +from generative_recommenders.research.rails.indexing.candidate_index import TopKModule + + +class CandidateIndex(object): + def __init__( + self, + ids: torch.Tensor, + embeddings: torch.Tensor, + invalid_ids: Optional[torch.Tensor] = None, + debug_path: Optional[str] = None, + ) -> None: + super().__init__() + + self._ids: torch.Tensor = ids + self._embeddings: torch.Tensor = embeddings + self._invalid_ids: Optional[torch.Tensor] = invalid_ids + self._debug_path: Optional[str] = debug_path + + @property + def ids(self) -> torch.Tensor: + """ + Returns: + (1, X) or (B, X), where valid ids are positive integers. + """ + return self._ids + + @property + def num_objects(self) -> int: + return self._ids.size(1) + + @property + def embeddings(self) -> torch.Tensor: + """ + Returns: + (1, X, D) or (B, X, D) with the same shape as `ids'. + """ + return self._embeddings + + def filter_invalid_ids( + self, + invalid_ids: torch.Tensor, + ) -> "CandidateIndex": + """ + Filters invalid_ids (batch dimension dependent) from the current index. + + Args: + invalid_ids: (B, N) x int64. + + Returns: + CandidateIndex with invalid_ids filtered. + """ + X = self._ids.size(1) + if self._ids.size(0) == 1: + # ((1, X, 1) == (B, 1, N)) -> (B, X) + invalid_mask, _ = (self._ids.unsqueeze(2) == invalid_ids.unsqueeze(1)).max( + dim=2 + ) + lengths = (~invalid_mask).int().sum(-1) # (B,) + valid_1d_mask = (~invalid_mask).view(-1) + B: int = lengths.size(0) + D: int = self._embeddings.size(-1) + jagged_ids = self._ids.expand(B, -1).reshape(-1)[valid_1d_mask] + jagged_embeddings = self._embeddings.expand(B, -1, -1).reshape(-1, D)[ + valid_1d_mask + ] + X_prime: int = lengths.max(-1)[0].item() + jagged_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum(lengths) + return CandidateIndex( + ids=torch.ops.fbgemm.jagged_to_padded_dense( + values=jagged_ids.unsqueeze(-1), + offsets=[jagged_offsets], + max_lengths=[X_prime], + padding_value=0, + ).squeeze(-1), + embeddings=torch.ops.fbgemm.jagged_to_padded_dense( + values=jagged_embeddings, + offsets=[jagged_offsets], + max_lengths=[X_prime], + padding_value=0.0, + ), + debug_path=self._debug_path, + ) + else: + assert self._invalid_ids == None + return CandidateIndex( + ids=self.ids, + embeddings=self.embeddings, + invalid_ids=invalid_ids, + debug_path=self._debug_path, + ) + + def get_top_k_outputs( + self, + query_embeddings: torch.Tensor, + k: int, + top_k_module: TopKModule, + invalid_ids: Optional[torch.Tensor], + r: int = 1, + return_embeddings: bool = False, + ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: + """ + Gets top-k outputs specified by `policy_fn', while filtering out + invalid ids per row as specified by `invalid_ids'. + + Args: + k: int. top k to return. + policy_fn: lambda that takes in item-side embeddings (B, X, D,) and user-side + embeddings (B * r, ...), and returns predictions (unnormalized logits) + of shape (B * r, X,). + invalid_ids: (B * r, N_0) x int64. The list of ids (if > 0) to filter from + results if present. Expect N_0 to be a small constant. + return_embeddings: bool if we should additionally return embeddings for the + top k results. + + Returns: + A tuple of (top_k_ids, top_k_prs, top_k_embeddings) of shape (B * r, k, ...). + """ + B: int = query_embeddings.size(0) + max_num_invalid_ids = 0 + if invalid_ids is not None: + max_num_invalid_ids = invalid_ids.size(1) + + k_prime = min(k + max_num_invalid_ids, self.num_objects) + top_k_prime_scores, top_k_prime_ids = top_k_module( + query_embeddings=query_embeddings, k=k_prime + ) + # Masks out invalid items rowwise. + if invalid_ids is not None: + id_is_valid = ~( + (top_k_prime_ids.unsqueeze(2) == invalid_ids.unsqueeze(1)).max(2)[0] + ) # [B, K + N_0] + id_is_valid = torch.logical_and( + id_is_valid, torch.cumsum(id_is_valid.int(), dim=1) <= k + ) + # [[1, 0, 1, 0], [0, 1, 1, 1]], k=2 -> [[0, 2], [1, 2]] + top_k_rowwise_offsets = torch.nonzero(id_is_valid, as_tuple=True)[1].view( + -1, k + ) + top_k_scores = torch.gather( + top_k_prime_scores, dim=1, index=top_k_rowwise_offsets + ) + top_k_ids = torch.gather( + top_k_prime_ids, dim=1, index=top_k_rowwise_offsets + ) + else: + top_k_scores = top_k_prime_scores + top_k_ids = top_k_prime_ids + + # TODO: this should be decoupled from candidate_index. + if return_embeddings: + raise ValueError("return_embeddings not supported yet.") + else: + top_k_embeddings = None + return top_k_ids, top_k_scores, top_k_embeddings + + def apply_object_filter(self) -> "CandidateIndex": + """ + Applies general per batch filters. + """ + raise NotImplementedError("not implemented.") diff --git a/recommendation_v4/generative_recommenders/research/indexing/utils.py b/recommendation_v4/generative_recommenders/research/indexing/utils.py new file mode 100644 index 000000000..972d3c2e7 --- /dev/null +++ b/recommendation_v4/generative_recommenders/research/indexing/utils.py @@ -0,0 +1,43 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +import torch +from generative_recommenders.research.rails.indexing.candidate_index import TopKModule +from generative_recommenders.research.rails.indexing.mips_top_k import ( + MIPSBruteForceTopK, +) +from generative_recommenders.research.rails.indexing.mol_top_k import MoLBruteForceTopK + + +def get_top_k_module( + top_k_method: str, + model: torch.nn.Module, + item_embeddings: torch.Tensor, + item_ids: torch.Tensor, +) -> TopKModule: + if top_k_method == "MIPSBruteForceTopK": + top_k_module = MIPSBruteForceTopK( + item_embeddings=item_embeddings, + item_ids=item_ids, + ) + elif top_k_method == "MoLBruteForceTopK": + top_k_module = MoLBruteForceTopK( # pyre-ignore [20] + item_embeddings=item_embeddings, + item_ids=item_ids, + ) + else: + raise ValueError(f"Invalid top-k method {top_k_method}") + return top_k_module diff --git a/recommendation_v4/generative_recommenders/research/modeling/initialization.py b/recommendation_v4/generative_recommenders/research/modeling/initialization.py new file mode 100644 index 000000000..c80d60075 --- /dev/null +++ b/recommendation_v4/generative_recommenders/research/modeling/initialization.py @@ -0,0 +1,35 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +import torch + + +def truncated_normal(x: torch.Tensor, mean: float, std: float) -> torch.Tensor: + with torch.no_grad(): + size = x.shape + tmp = x.new_empty(size + (4,)).normal_() + valid = (tmp < 2) & (tmp > -2) + ind = valid.max(-1, keepdim=True)[1] + x.data.copy_(tmp.gather(-1, ind).squeeze(-1)) + x.data.mul_(std).add_(mean) + return x + + +def init_mlp_xavier_weights_zero_bias(m: torch.nn.Module) -> None: + if isinstance(m, torch.nn.Linear): + torch.nn.init.xavier_uniform(m.weight) + if getattr(m, "bias", None) is not None: + m.bias.data.fill_(0.0) diff --git a/recommendation_v4/generative_recommenders/research/modeling/sequential/autoregressive_losses.py b/recommendation_v4/generative_recommenders/research/modeling/sequential/autoregressive_losses.py new file mode 100644 index 000000000..c32bedf0e --- /dev/null +++ b/recommendation_v4/generative_recommenders/research/modeling/sequential/autoregressive_losses.py @@ -0,0 +1,477 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +import abc +from collections import OrderedDict +from typing import List, Tuple + +import torch +import torch.nn.functional as F +from generative_recommenders.research.rails.similarities.module import SimilarityModule +from torch.utils.checkpoint import checkpoint + + +class NegativesSampler(torch.nn.Module): + def __init__(self, l2_norm: bool, l2_norm_eps: float) -> None: + super().__init__() + + self._l2_norm: bool = l2_norm + self._l2_norm_eps: float = l2_norm_eps + + def normalize_embeddings(self, x: torch.Tensor) -> torch.Tensor: + return self._maybe_l2_norm(x) + + def _maybe_l2_norm(self, x: torch.Tensor) -> torch.Tensor: + if self._l2_norm: + x = x / torch.clamp( + torch.linalg.norm(x, ord=2, dim=-1, keepdim=True), + min=self._l2_norm_eps, + ) + return x + + @abc.abstractmethod + def debug_str(self) -> str: + pass + + @abc.abstractmethod + def process_batch( + self, + ids: torch.Tensor, + presences: torch.Tensor, + embeddings: torch.Tensor, + ) -> None: + pass + + @abc.abstractmethod + def forward( + self, + positive_ids: torch.Tensor, + num_to_sample: int, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Returns: + A tuple of (sampled_ids, sampled_negative_embeddings). + """ + pass + + +class LocalNegativesSampler(NegativesSampler): + def __init__( + self, + num_items: int, + item_emb: torch.nn.Embedding, + all_item_ids: List[int], + l2_norm: bool, + l2_norm_eps: float, + ) -> None: + super().__init__(l2_norm=l2_norm, l2_norm_eps=l2_norm_eps) + + self._num_items: int = len(all_item_ids) + self._item_emb: torch.nn.Embedding = item_emb + self.register_buffer("_all_item_ids", torch.tensor(all_item_ids)) + + def debug_str(self) -> str: + sampling_debug_str = ( + f"local{f'-l2-eps{self._l2_norm_eps}' if self._l2_norm else ''}" + ) + return sampling_debug_str + + def process_batch( + self, + ids: torch.Tensor, + presences: torch.Tensor, + embeddings: torch.Tensor, + ) -> None: + pass + + def forward( + self, + positive_ids: torch.Tensor, + num_to_sample: int, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Returns: + A tuple of (sampled_ids, sampled_negative_embeddings). + """ + # assert torch.max(torch.abs(self._item_emb(positive_ids) - positive_embeddings)) < 1e-4 + output_shape = positive_ids.size() + (num_to_sample,) + sampled_offsets = torch.randint( + low=0, + high=self._num_items, + size=output_shape, + dtype=positive_ids.dtype, + device=positive_ids.device, + ) + sampled_ids = self._all_item_ids[sampled_offsets.view(-1)].reshape(output_shape) + return sampled_ids, self.normalize_embeddings(self._item_emb(sampled_ids)) + + +class InBatchNegativesSampler(NegativesSampler): + def __init__( + self, + l2_norm: bool, + l2_norm_eps: float, + dedup_embeddings: bool, + ) -> None: + super().__init__(l2_norm=l2_norm, l2_norm_eps=l2_norm_eps) + + self._dedup_embeddings: bool = dedup_embeddings + + def debug_str(self) -> str: + sampling_debug_str = ( + f"in-batch{f'-l2-eps{self._l2_norm_eps}' if self._l2_norm else ''}" + ) + if self._dedup_embeddings: + sampling_debug_str += "-dedup" + return sampling_debug_str + + def process_batch( + self, + ids: torch.Tensor, + presences: torch.Tensor, + embeddings: torch.Tensor, + ) -> None: + """ + Args: + ids: (N') or (B, N) x int64 + presences: (N') or (B, N) x bool + embeddings: (N', D) or (B, N, D) x float + """ + assert ids.size() == presences.size() + assert ids.size() == embeddings.size()[:-1] + if self._dedup_embeddings: + valid_ids = ids[presences] + unique_ids, unique_ids_inverse_indices = torch.unique( + input=valid_ids, sorted=False, return_inverse=True + ) + device = unique_ids.device + unique_embedding_offsets = torch.empty( + (unique_ids.numel(),), + dtype=torch.int64, + device=device, + ) + unique_embedding_offsets[unique_ids_inverse_indices] = torch.arange( + valid_ids.numel(), dtype=torch.int64, device=device + ) + unique_embeddings = embeddings[presences][unique_embedding_offsets, :] + self._cached_embeddings = self._maybe_l2_norm( # pyre-ignore [16] + unique_embeddings + ) + self._cached_ids = unique_ids # pyre-ignore [16] + else: + self._cached_embeddings = self._maybe_l2_norm(embeddings[presences]) + self._cached_ids = ids[presences] + + def get_all_ids_and_embeddings(self) -> Tuple[torch.Tensor, torch.Tensor]: + return self._cached_ids, self._cached_embeddings # pyre-ignore [7] + + def forward( + self, + positive_ids: torch.Tensor, + num_to_sample: int, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Returns: + A tuple of (sampled_ids, sampled_negative_embeddings,). + """ + X = self._cached_ids.size(0) + sampled_offsets = torch.randint( + low=0, + high=X, + size=positive_ids.size() + (num_to_sample,), + dtype=positive_ids.dtype, + device=positive_ids.device, + ) + return ( + self._cached_ids[sampled_offsets], # pyre-ignore [29] + self._cached_embeddings[sampled_offsets], # pyre-ignore [29] + ) + + +class AutoregressiveLoss(torch.nn.Module): + @abc.abstractmethod + def jagged_forward( + self, + output_embeddings: torch.Tensor, + supervision_ids: torch.Tensor, + supervision_embeddings: torch.Tensor, + supervision_weights: torch.Tensor, + negatives_sampler: NegativesSampler, + ) -> torch.Tensor: + """ + Variant of forward() when the tensors are already in jagged format. + + Args: + output_embeddings: [N', D] x float, embeddings for the current + input sequence. + supervision_ids: [N'] x int64, (positive) supervision ids. + supervision_embeddings: [N', D] x float. + supervision_weights: Optional [N'] x float. Optional weights for + masking out invalid positions, or reweighting supervision labels. + negatives_sampler: sampler used to obtain negative examples paired with + positives. + + Returns: + (1), loss for the current engaged sequence. + """ + pass + + @abc.abstractmethod + def forward( + self, + lengths: torch.Tensor, + output_embeddings: torch.Tensor, + supervision_ids: torch.Tensor, + supervision_embeddings: torch.Tensor, + supervision_weights: torch.Tensor, + negatives_sampler: NegativesSampler, + ) -> torch.Tensor: + """ + Args: + lengths: [B] x int32 representing number of non-zero elements per row. + output_embeddings: [B, N, D] x float, embeddings for the current + input sequence. + supervision_ids: [B, N] x int64, (positive) supervision ids. + supervision_embeddings: [B, N, D] x float. + supervision_weights: Optional [B, N] x float. Optional weights for + masking out invalid positions, or reweighting supervision labels. + negatives_sampler: sampler used to obtain negative examples paired with + positives. + + Returns: + (1), loss for the current engaged sequence. + """ + pass + + +class BCELoss(AutoregressiveLoss): + def __init__( + self, + temperature: float, + model: SimilarityModule, + ) -> None: + super().__init__() + self._temperature: float = temperature + self._model = model + + def jagged_forward( + self, + output_embeddings: torch.Tensor, + supervision_ids: torch.Tensor, + supervision_embeddings: torch.Tensor, + supervision_weights: torch.Tensor, + negatives_sampler: NegativesSampler, + ) -> torch.Tensor: + assert output_embeddings.size() == supervision_embeddings.size() + assert supervision_ids.size() == supervision_embeddings.size()[:-1] + assert supervision_ids.size() == supervision_weights.size() + + sampled_ids, sampled_negative_embeddings = negatives_sampler( + positive_ids=supervision_ids, + num_to_sample=1, + ) + + positive_logits = ( + self._model.interaction( # pyre-ignore [29] + input_embeddings=output_embeddings, # [B, D] = [N', D] + target_ids=supervision_ids.unsqueeze(1), # [N', 1] + target_embeddings=supervision_embeddings.unsqueeze( + 1 + ), # [N', D] -> [N', 1, D] + )[0].squeeze(1) + / self._temperature + ) # [N'] + + sampled_negatives_logits = ( + self._model.interaction( # pyre-ignore [29] + input_embeddings=output_embeddings, # [N', D] + target_ids=sampled_ids, # [N', 1] + target_embeddings=sampled_negative_embeddings, # [N', 1, D] + )[0].squeeze(1) + / self._temperature + ) # [N'] + sampled_negatives_valid_mask = ( + supervision_ids != sampled_ids.squeeze(1) + ).float() # [N'] + loss_weights = supervision_weights * sampled_negatives_valid_mask + weighted_losses = ( + ( + F.binary_cross_entropy_with_logits( + input=positive_logits, + target=torch.ones_like(positive_logits), + reduction="none", + ) + + F.binary_cross_entropy_with_logits( + input=sampled_negatives_logits, + target=torch.zeros_like(sampled_negatives_logits), + reduction="none", + ) + ) + * loss_weights + * 0.5 + ) + return weighted_losses.sum() / loss_weights.sum() + + def forward( + self, + lengths: torch.Tensor, + output_embeddings: torch.Tensor, + supervision_ids: torch.Tensor, + supervision_embeddings: torch.Tensor, + supervision_weights: torch.Tensor, + negatives_sampler: NegativesSampler, + ) -> torch.Tensor: + """ + Args: + lengths: [B] x int32 representing number of non-zero elements per row. + output_embeddings: [B, N, D] x float, embeddings for the current + input sequence. + supervision_ids: [B, N] x int64, (positive) supervision ids. + supervision_embeddings: [B, N, D] x float. + supervision_weights: Optional [B, N] x float. Optional weights for + masking out invalid positions, or reweighting supervision labels. + negatives_sampler: sampler used to obtain negative examples paired with + positives. + Returns: + (1), loss for the current engaged sequence. + """ + assert output_embeddings.size() == supervision_embeddings.size() + assert supervision_ids.size() == supervision_embeddings.size()[:-1] + jagged_id_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum(lengths) + jagged_supervision_ids = ( + torch.ops.fbgemm.dense_to_jagged( + supervision_ids.unsqueeze(-1).float(), [jagged_id_offsets] + )[0] + .squeeze(1) + .long() + ) + jagged_supervision_weights = torch.ops.fbgemm.dense_to_jagged( + supervision_weights.unsqueeze(-1), + [jagged_id_offsets], + )[0].squeeze(1) + return self.jagged_forward( + output_embeddings=torch.ops.fbgemm.dense_to_jagged( + output_embeddings, + [jagged_id_offsets], + )[0], + supervision_ids=jagged_supervision_ids, + supervision_embeddings=torch.ops.fbgemm.dense_to_jagged( + supervision_embeddings, + [jagged_id_offsets], + )[0], + supervision_weights=jagged_supervision_weights, + negatives_sampler=negatives_sampler, + ) + + +class BCELossWithRatings(AutoregressiveLoss): + def __init__( + self, + temperature: float, + model: SimilarityModule, + ) -> None: + super().__init__() + self._temperature: float = temperature + self._model = model + + def jagged_forward( + self, + output_embeddings: torch.Tensor, + supervision_ids: torch.Tensor, + supervision_embeddings: torch.Tensor, + supervision_weights: torch.Tensor, + supervision_ratings: torch.Tensor, + negatives_sampler: NegativesSampler, + ) -> torch.Tensor: + assert output_embeddings.size() == supervision_embeddings.size() + assert supervision_ids.size() == supervision_embeddings.size()[:-1] + assert supervision_ids.size() == supervision_weights.size() + + target_logits = ( + self._model.interaction( # pyre-ignore [29] + input_embeddings=output_embeddings, # [B, D] = [N', D] + target_ids=supervision_ids.unsqueeze(1), # [N', 1] + target_embeddings=supervision_embeddings.unsqueeze( + 1 + ), # [N', D] -> [N', 1, D] + )[0].squeeze(1) + / self._temperature + ) # [N', 1] + + weighted_losses = ( + F.binary_cross_entropy_with_logits( + input=target_logits, + target=supervision_ratings.to(dtype=target_logits.dtype), + reduction="none", + ) + ) * supervision_weights + return weighted_losses.sum() / supervision_weights.sum() + + def forward( + self, + lengths: torch.Tensor, + output_embeddings: torch.Tensor, + supervision_ids: torch.Tensor, + supervision_embeddings: torch.Tensor, + supervision_weights: torch.Tensor, + supervision_ratings: torch.Tensor, + negatives_sampler: NegativesSampler, + ) -> torch.Tensor: + """ + Args: + lengths: [B] x int32 representing number of non-zero elements per row. + output_embeddings: [B, N, D] x float, embeddings for the current + input sequence. + supervision_ids: [B, N] x int64, (positive) supervision ids. + supervision_embeddings: [B, N, D] x float. + supervision_weights: Optional [B, N] x float. Optional weights for + masking out invalid positions, or reweighting supervision labels. + negatives_sampler: sampler used to obtain negative examples paired with + positives. + Returns: + (1), loss for the current engaged sequence. + """ + assert output_embeddings.size() == supervision_embeddings.size() + assert supervision_ids.size() == supervision_embeddings.size()[:-1] + jagged_id_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum(lengths) + jagged_supervision_ids = ( + torch.ops.fbgemm.dense_to_jagged( + supervision_ids.unsqueeze(-1).float(), [jagged_id_offsets] + )[0] + .squeeze(1) + .long() + ) + jagged_supervision_weights = torch.ops.fbgemm.dense_to_jagged( + supervision_weights.unsqueeze(-1), + [jagged_id_offsets], + )[0].squeeze(1) + return self.jagged_forward( + output_embeddings=torch.ops.fbgemm.dense_to_jagged( + output_embeddings, + [jagged_id_offsets], + )[0], + supervision_ids=jagged_supervision_ids, + supervision_embeddings=torch.ops.fbgemm.dense_to_jagged( + supervision_embeddings, + [jagged_id_offsets], + )[0], + supervision_weights=jagged_supervision_weights, + supervision_ratings=torch.ops.fbgemm.dense_to_jagged( + supervision_ratings.unsqueeze(-1), + [jagged_id_offsets], + )[0].squeeze(1), + negatives_sampler=negatives_sampler, + ) diff --git a/recommendation_v4/generative_recommenders/research/modeling/sequential/embedding_modules.py b/recommendation_v4/generative_recommenders/research/modeling/sequential/embedding_modules.py new file mode 100644 index 000000000..6e85a62dd --- /dev/null +++ b/recommendation_v4/generative_recommenders/research/modeling/sequential/embedding_modules.py @@ -0,0 +1,108 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +import abc + +import torch +from generative_recommenders.research.modeling.initialization import truncated_normal + + +class EmbeddingModule(torch.nn.Module): + @abc.abstractmethod + def debug_str(self) -> str: + pass + + @abc.abstractmethod + def get_item_embeddings(self, item_ids: torch.Tensor) -> torch.Tensor: + pass + + @property + @abc.abstractmethod + def item_embedding_dim(self) -> int: + pass + + +class LocalEmbeddingModule(EmbeddingModule): + def __init__( + self, + num_items: int, + item_embedding_dim: int, + ) -> None: + super().__init__() + + self._item_embedding_dim: int = item_embedding_dim + self._item_emb = torch.nn.Embedding( + num_items + 1, item_embedding_dim, padding_idx=0 + ) + self.reset_params() + + def debug_str(self) -> str: + return f"local_emb_d{self._item_embedding_dim}" + + def reset_params(self) -> None: + for name, params in self.named_parameters(): + if "_item_emb" in name: + print( + f"Initialize {name} as truncated normal: {params.data.size()} params" + ) + truncated_normal(params, mean=0.0, std=0.02) + else: + print(f"Skipping initializing params {name} - not configured") + + def get_item_embeddings(self, item_ids: torch.Tensor) -> torch.Tensor: + return self._item_emb(item_ids) + + @property + def item_embedding_dim(self) -> int: + return self._item_embedding_dim + + +class CategoricalEmbeddingModule(EmbeddingModule): + def __init__( + self, + num_items: int, + item_embedding_dim: int, + item_id_to_category_id: torch.Tensor, + ) -> None: + super().__init__() + + self._item_embedding_dim: int = item_embedding_dim + self._item_emb: torch.nn.Embedding = torch.nn.Embedding( + num_items + 1, item_embedding_dim, padding_idx=0 + ) + self.register_buffer("_item_id_to_category_id", item_id_to_category_id) + self.reset_params() + + def debug_str(self) -> str: + return f"cat_emb_d{self._item_embedding_dim}" + + def reset_params(self) -> None: + for name, params in self.named_parameters(): + if "_item_emb" in name: + print( + f"Initialize {name} as truncated normal: {params.data.size()} params" + ) + truncated_normal(params, mean=0.0, std=0.02) + else: + print(f"Skipping initializing params {name} - not configured") + + def get_item_embeddings(self, item_ids: torch.Tensor) -> torch.Tensor: + item_ids = self._item_id_to_category_id[(item_ids - 1).clamp(min=0)] + 1 + return self._item_emb(item_ids) + + @property + def item_embedding_dim(self) -> int: + return self._item_embedding_dim diff --git a/recommendation_v4/generative_recommenders/research/modeling/sequential/encoder_utils.py b/recommendation_v4/generative_recommenders/research/modeling/sequential/encoder_utils.py new file mode 100644 index 000000000..dc64aa2cf --- /dev/null +++ b/recommendation_v4/generative_recommenders/research/modeling/sequential/encoder_utils.py @@ -0,0 +1,150 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +import gin +from generative_recommenders.research.modeling.sequential.embedding_modules import ( + EmbeddingModule, +) +from generative_recommenders.research.modeling.sequential.hstu import HSTU +from generative_recommenders.research.modeling.sequential.input_features_preprocessors import ( + InputFeaturesPreprocessorModule, +) +from generative_recommenders.research.modeling.sequential.output_postprocessors import ( + OutputPostprocessorModule, +) +from generative_recommenders.research.modeling.sequential.sasrec import SASRec +from generative_recommenders.research.modeling.similarity_module import ( + SequentialEncoderWithLearnedSimilarityModule, +) +from generative_recommenders.research.rails.similarities.module import SimilarityModule + + +@gin.configurable +def sasrec_encoder( + max_sequence_length: int, + max_output_length: int, + embedding_module: EmbeddingModule, + similarity_module: SimilarityModule, + input_preproc_module: InputFeaturesPreprocessorModule, + output_postproc_module: OutputPostprocessorModule, + activation_checkpoint: bool, + verbose: bool, + ffn_hidden_dim: int = 64, + ffn_activation_fn: str = "relu", + ffn_dropout_rate: float = 0.2, + num_blocks: int = 2, + num_heads: int = 1, +) -> SequentialEncoderWithLearnedSimilarityModule: + return SASRec( + embedding_module=embedding_module, + max_sequence_len=max_sequence_length, + max_output_len=max_output_length, + embedding_dim=embedding_module.item_embedding_dim, + ffn_hidden_dim=ffn_hidden_dim, + ffn_activation_fn=ffn_activation_fn, + ffn_dropout_rate=ffn_dropout_rate, + num_blocks=num_blocks, + num_heads=num_heads, + similarity_module=similarity_module, # pyre-ignore [6] + input_features_preproc_module=input_preproc_module, + output_postproc_module=output_postproc_module, + activation_checkpoint=activation_checkpoint, + verbose=verbose, + ) + + +@gin.configurable +def hstu_encoder( + max_sequence_length: int, + max_output_length: int, + embedding_module: EmbeddingModule, + similarity_module: SimilarityModule, + input_preproc_module: InputFeaturesPreprocessorModule, + output_postproc_module: OutputPostprocessorModule, + activation_checkpoint: bool, + verbose: bool, + num_blocks: int = 2, + num_heads: int = 1, + dqk: int = 64, + dv: int = 64, + linear_dropout_rate: float = 0.0, + attn_dropout_rate: float = 0.0, + normalization: str = "rel_bias", + linear_config: str = "uvqk", + linear_activation: str = "silu", + concat_ua: bool = False, + enable_relative_attention_bias: bool = True, +) -> SequentialEncoderWithLearnedSimilarityModule: + return HSTU( + embedding_module=embedding_module, + similarity_module=similarity_module, # pyre-ignore [6] + input_features_preproc_module=input_preproc_module, + output_postproc_module=output_postproc_module, + max_sequence_len=max_sequence_length, + max_output_len=max_output_length, + embedding_dim=embedding_module.item_embedding_dim, + num_blocks=num_blocks, + num_heads=num_heads, + attention_dim=dqk, + linear_dim=dv, + linear_dropout_rate=linear_dropout_rate, + attn_dropout_rate=attn_dropout_rate, + linear_config=linear_config, + linear_activation=linear_activation, + normalization=normalization, + concat_ua=concat_ua, + enable_relative_attention_bias=enable_relative_attention_bias, + verbose=verbose, + ) + + +@gin.configurable +def get_sequential_encoder( + module_type: str, + max_sequence_length: int, + max_output_length: int, + embedding_module: EmbeddingModule, + interaction_module: SimilarityModule, + input_preproc_module: InputFeaturesPreprocessorModule, + output_postproc_module: OutputPostprocessorModule, + verbose: bool, + activation_checkpoint: bool = False, +) -> SequentialEncoderWithLearnedSimilarityModule: + if module_type == "SASRec": + model = sasrec_encoder( + max_sequence_length=max_sequence_length, + max_output_length=max_output_length, + embedding_module=embedding_module, + similarity_module=interaction_module, + input_preproc_module=input_preproc_module, + output_postproc_module=output_postproc_module, + activation_checkpoint=activation_checkpoint, + verbose=verbose, + ) + elif module_type == "HSTU": + model = hstu_encoder( + max_sequence_length=max_sequence_length, + max_output_length=max_output_length, + embedding_module=embedding_module, + similarity_module=interaction_module, + input_preproc_module=input_preproc_module, + output_postproc_module=output_postproc_module, + activation_checkpoint=activation_checkpoint, + verbose=verbose, + ) + else: + raise ValueError(f"Unsupported module_type {module_type}") + return model diff --git a/recommendation_v4/generative_recommenders/research/modeling/sequential/features.py b/recommendation_v4/generative_recommenders/research/modeling/sequential/features.py new file mode 100644 index 000000000..70bf80cc0 --- /dev/null +++ b/recommendation_v4/generative_recommenders/research/modeling/sequential/features.py @@ -0,0 +1,94 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +from typing import Dict, NamedTuple, Optional, Tuple + +import torch + + +class SequentialFeatures(NamedTuple): + # (B,) x int64. Requires past_lengths[i] > 0 \forall i. + past_lengths: torch.Tensor + # (B, N,) x int64. 0 denotes valid ids. + past_ids: torch.Tensor + # (B, N, D) x float. + past_embeddings: Optional[torch.Tensor] + # Implementation-specific payloads. + # e.g., past timestamps, past event_types (e.g., clicks, likes), etc. + past_payloads: Dict[str, torch.Tensor] + + +def movielens_seq_features_from_row( + row: Dict[str, torch.Tensor], + device: int, + max_output_length: int, +) -> Tuple[SequentialFeatures, torch.Tensor, torch.Tensor]: + historical_lengths = row["history_lengths"].to(device) # [B] + historical_ids = row["historical_ids"].to(device) # [B, N] + historical_ratings = row["historical_ratings"].to(device) + historical_timestamps = row["historical_timestamps"].to(device) + target_ids = row["target_ids"].to(device).unsqueeze(1) # [B, 1] + target_ratings = row["target_ratings"].to(device).unsqueeze(1) + target_timestamps = row["target_timestamps"].to(device).unsqueeze(1) + if max_output_length > 0: + B = historical_lengths.size(0) + historical_ids = torch.cat( + [ + historical_ids, + torch.zeros( + (B, max_output_length), dtype=historical_ids.dtype, device=device + ), + ], + dim=1, + ) + historical_ratings = torch.cat( + [ + historical_ratings, + torch.zeros( + (B, max_output_length), + dtype=historical_ratings.dtype, + device=device, + ), + ], + dim=1, + ) + historical_timestamps = torch.cat( + [ + historical_timestamps, + torch.zeros( + (B, max_output_length), + dtype=historical_timestamps.dtype, + device=device, + ), + ], + dim=1, + ) + historical_timestamps.scatter_( + dim=1, + index=historical_lengths.view(-1, 1), + src=target_timestamps.view(-1, 1), + ) + # print(f"historical_ids.size()={historical_ids.size()}, historical_timestamps.size()={historical_timestamps.size()}") + features = SequentialFeatures( + past_lengths=historical_lengths, + past_ids=historical_ids, + past_embeddings=None, + past_payloads={ + "timestamps": historical_timestamps, + "ratings": historical_ratings, + }, + ) + return features, target_ids, target_ratings diff --git a/recommendation_v4/generative_recommenders/research/modeling/sequential/hstu.py b/recommendation_v4/generative_recommenders/research/modeling/sequential/hstu.py new file mode 100644 index 000000000..3c89245a2 --- /dev/null +++ b/recommendation_v4/generative_recommenders/research/modeling/sequential/hstu.py @@ -0,0 +1,808 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +""" +Implements HSTU (Hierarchical Sequential Transduction Unit) in +Actions Speak Louder than Words: Trillion-Parameter Sequential Transducers for Generative Recommendations +(https://arxiv.org/abs/2402.17152, ICML'24). +""" + +import abc +import math +from typing import Callable, Dict, List, Optional, Tuple, Union + +import torch +import torch.nn.functional as F +from generative_recommenders.research.modeling.sequential.embedding_modules import ( + EmbeddingModule, +) +from generative_recommenders.research.modeling.sequential.input_features_preprocessors import ( + InputFeaturesPreprocessorModule, +) +from generative_recommenders.research.modeling.sequential.output_postprocessors import ( + OutputPostprocessorModule, +) +from generative_recommenders.research.modeling.sequential.utils import ( + get_current_embeddings, +) +from generative_recommenders.research.modeling.similarity_module import ( + SequentialEncoderWithLearnedSimilarityModule, +) +from generative_recommenders.research.rails.similarities.module import SimilarityModule + + +TIMESTAMPS_KEY = "timestamps" + + +class RelativeAttentionBiasModule(torch.nn.Module): + @abc.abstractmethod + def forward( + self, + all_timestamps: torch.Tensor, + ) -> torch.Tensor: + """ + Args: + all_timestamps: [B, N] x int64 + Returns: + torch.float tensor broadcastable to [B, N, N] + """ + pass + + +class RelativePositionalBias(RelativeAttentionBiasModule): + def __init__(self, max_seq_len: int) -> None: + super().__init__() + + self._max_seq_len: int = max_seq_len + self._w = torch.nn.Parameter( + torch.empty(2 * max_seq_len - 1).normal_(mean=0, std=0.02), + ) + + def forward( + self, + all_timestamps: torch.Tensor, + ) -> torch.Tensor: + del all_timestamps + n: int = self._max_seq_len + t = F.pad(self._w[: 2 * n - 1], [0, n]).repeat(n) + t = t[..., :-n].reshape(1, n, 3 * n - 2) + r = (2 * n - 1) // 2 + return t[..., r:-r] + + +class RelativeBucketedTimeAndPositionBasedBias(RelativeAttentionBiasModule): + """ + Bucketizes timespans based on ts(next-item) - ts(current-item). + """ + + def __init__( + self, + max_seq_len: int, + num_buckets: int, + bucketization_fn: Callable[[torch.Tensor], torch.Tensor], + ) -> None: + super().__init__() + + self._max_seq_len: int = max_seq_len + self._ts_w = torch.nn.Parameter( + torch.empty(num_buckets + 1).normal_(mean=0, std=0.02), + ) + self._pos_w = torch.nn.Parameter( + torch.empty(2 * max_seq_len - 1).normal_(mean=0, std=0.02), + ) + self._num_buckets: int = num_buckets + self._bucketization_fn: Callable[[torch.Tensor], torch.Tensor] = ( + bucketization_fn + ) + + def forward( + self, + all_timestamps: torch.Tensor, + ) -> torch.Tensor: + """ + Args: + all_timestamps: (B, N). + Returns: + (B, N, N). + """ + B = all_timestamps.size(0) + N = self._max_seq_len + t = F.pad(self._pos_w[: 2 * N - 1], [0, N]).repeat(N) + t = t[..., :-N].reshape(1, N, 3 * N - 2) + r = (2 * N - 1) // 2 + + # [B, N + 1] to simplify tensor manipulations. + ext_timestamps = torch.cat( + [all_timestamps, all_timestamps[:, N - 1 : N]], dim=1 + ) + # causal masking. Otherwise [:, :-1] - [:, 1:] works + bucketed_timestamps = torch.clamp( + self._bucketization_fn( + ext_timestamps[:, 1:].unsqueeze(2) - ext_timestamps[:, :-1].unsqueeze(1) + ), + min=0, + max=self._num_buckets, + ).detach() + rel_pos_bias = t[:, :, r:-r] + rel_ts_bias = torch.index_select( + self._ts_w, dim=0, index=bucketed_timestamps.view(-1) + ).view(B, N, N) + return rel_pos_bias + rel_ts_bias + + +HSTUCacheState = Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor] + + +def _hstu_attention_maybe_from_cache( + num_heads: int, + attention_dim: int, + linear_dim: int, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + cached_q: Optional[torch.Tensor], + cached_k: Optional[torch.Tensor], + delta_x_offsets: Optional[Tuple[torch.Tensor, torch.Tensor]], + x_offsets: torch.Tensor, + all_timestamps: Optional[torch.Tensor], + invalid_attn_mask: torch.Tensor, + rel_attn_bias: RelativeAttentionBiasModule, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + B: int = x_offsets.size(0) - 1 + n: int = invalid_attn_mask.size(-1) + if delta_x_offsets is not None: + padded_q, padded_k = cached_q, cached_k + flattened_offsets = delta_x_offsets[1] + torch.arange( + start=0, + end=B * n, + step=n, + device=delta_x_offsets[1].device, + dtype=delta_x_offsets[1].dtype, + ) + assert isinstance(padded_q, torch.Tensor) + assert isinstance(padded_k, torch.Tensor) + padded_q = ( + padded_q.view(B * n, -1) + .index_copy_( + dim=0, + index=flattened_offsets, + source=q, + ) + .view(B, n, -1) + ) + padded_k = ( + padded_k.view(B * n, -1) + .index_copy_( + dim=0, + index=flattened_offsets, + source=k, + ) + .view(B, n, -1) + ) + else: + padded_q = torch.ops.fbgemm.jagged_to_padded_dense( + values=q, offsets=[x_offsets], max_lengths=[n], padding_value=0.0 + ) + padded_k = torch.ops.fbgemm.jagged_to_padded_dense( + values=k, offsets=[x_offsets], max_lengths=[n], padding_value=0.0 + ) + + qk_attn = torch.einsum( + "bnhd,bmhd->bhnm", + padded_q.view(B, n, num_heads, attention_dim), + padded_k.view(B, n, num_heads, attention_dim), + ) + if all_timestamps is not None: + qk_attn = qk_attn + rel_attn_bias(all_timestamps).unsqueeze(1) + qk_attn = F.silu(qk_attn) / n + qk_attn = qk_attn * invalid_attn_mask.unsqueeze(0).unsqueeze(0) + attn_output = torch.ops.fbgemm.dense_to_jagged( + torch.einsum( + "bhnm,bmhd->bnhd", + qk_attn, + torch.ops.fbgemm.jagged_to_padded_dense(v, [x_offsets], [n]).reshape( + B, n, num_heads, linear_dim + ), + ).reshape(B, n, num_heads * linear_dim), + [x_offsets], + )[0] + return attn_output, padded_q, padded_k + + +class SequentialTransductionUnitJagged(torch.nn.Module): + def __init__( + self, + embedding_dim: int, + linear_hidden_dim: int, + attention_dim: int, + dropout_ratio: float, + attn_dropout_ratio: float, + num_heads: int, + linear_activation: str, + relative_attention_bias_module: Optional[RelativeAttentionBiasModule] = None, + normalization: str = "rel_bias", + linear_config: str = "uvqk", + concat_ua: bool = False, + epsilon: float = 1e-6, + max_length: Optional[int] = None, + ) -> None: + super().__init__() + self._embedding_dim: int = embedding_dim + self._linear_dim: int = linear_hidden_dim + self._attention_dim: int = attention_dim + self._dropout_ratio: float = dropout_ratio + self._attn_dropout_ratio: float = attn_dropout_ratio + self._num_heads: int = num_heads + self._rel_attn_bias: Optional[RelativeAttentionBiasModule] = ( + relative_attention_bias_module + ) + self._normalization: str = normalization + self._linear_config: str = linear_config + if self._linear_config == "uvqk": + self._uvqk: torch.nn.Parameter = torch.nn.Parameter( + torch.empty( + ( + embedding_dim, + linear_hidden_dim * 2 * num_heads + + attention_dim * num_heads * 2, + ) + ).normal_(mean=0, std=0.02), + ) + else: + raise ValueError(f"Unknown linear_config {self._linear_config}") + self._linear_activation: str = linear_activation + self._concat_ua: bool = concat_ua + self._o = torch.nn.Linear( + in_features=linear_hidden_dim * num_heads * (3 if concat_ua else 1), + out_features=embedding_dim, + ) + torch.nn.init.xavier_uniform_(self._o.weight) + self._eps: float = epsilon + + def _norm_input(self, x: torch.Tensor) -> torch.Tensor: + return F.layer_norm(x, normalized_shape=[self._embedding_dim], eps=self._eps) + + def _norm_attn_output(self, x: torch.Tensor) -> torch.Tensor: + return F.layer_norm( + x, normalized_shape=[self._linear_dim * self._num_heads], eps=self._eps + ) + + def forward( # pyre-ignore [3] + self, + x: torch.Tensor, + x_offsets: torch.Tensor, + all_timestamps: Optional[torch.Tensor], + invalid_attn_mask: torch.Tensor, + delta_x_offsets: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + cache: Optional[HSTUCacheState] = None, + return_cache_states: bool = False, + ): + """ + Args: + x: (\sum_i N_i, D) x float. + x_offsets: (B + 1) x int32. + all_timestamps: optional (B, N) x int64. + invalid_attn_mask: (B, N, N) x float, each element in {0, 1}. + delta_x_offsets: optional 2-tuple ((B,) x int32, (B,) x int32). + For the 1st element in the tuple, each element is in [0, x_offsets[-1]). For the + 2nd element in the tuple, each element is in [0, N). + cache: Optional 4-tuple of (v, padded_q, padded_k, output) from prior runs, + where all except padded_q, padded_k are jagged. + Returns: + x' = f(x), (\sum_i N_i, D) x float. + """ + n: int = invalid_attn_mask.size(-1) + cached_q = None + cached_k = None + if delta_x_offsets is not None: + # In this case, for all the following code, x, u, v, q, k become restricted to + # [delta_x_offsets[0], :]. + assert cache is not None + x = x[delta_x_offsets[0], :] + cached_v, cached_q, cached_k, cached_outputs = cache + + normed_x = self._norm_input(x) + + if self._linear_config == "uvqk": + batched_mm_output = torch.mm(normed_x, self._uvqk) + if self._linear_activation == "silu": + batched_mm_output = F.silu(batched_mm_output) + elif self._linear_activation == "none": + batched_mm_output = batched_mm_output + u, v, q, k = torch.split( + batched_mm_output, + [ + self._linear_dim * self._num_heads, + self._linear_dim * self._num_heads, + self._attention_dim * self._num_heads, + self._attention_dim * self._num_heads, + ], + dim=1, + ) + else: + raise ValueError(f"Unknown self._linear_config {self._linear_config}") + + if delta_x_offsets is not None: + v = cached_v.index_copy_(dim=0, index=delta_x_offsets[0], source=v) + + B: int = x_offsets.size(0) - 1 + if self._normalization == "rel_bias" or self._normalization == "hstu_rel_bias": + assert self._rel_attn_bias is not None + attn_output, padded_q, padded_k = _hstu_attention_maybe_from_cache( + num_heads=self._num_heads, + attention_dim=self._attention_dim, + linear_dim=self._linear_dim, + q=q, + k=k, + v=v, + cached_q=cached_q, + cached_k=cached_k, + delta_x_offsets=delta_x_offsets, + x_offsets=x_offsets, + all_timestamps=all_timestamps, + invalid_attn_mask=invalid_attn_mask, + rel_attn_bias=self._rel_attn_bias, + ) + elif self._normalization == "softmax_rel_bias": + if delta_x_offsets is not None: + B = x_offsets.size(0) - 1 + padded_q, padded_k = cached_q, cached_k + flattened_offsets = delta_x_offsets[1] + torch.arange( + start=0, + end=B * n, + step=n, + device=delta_x_offsets[1].device, + dtype=delta_x_offsets[1].dtype, + ) + assert padded_q is not None + assert padded_k is not None + padded_q = ( + padded_q.view(B * n, -1) + .index_copy_( + dim=0, + index=flattened_offsets, + source=q, + ) + .view(B, n, -1) + ) + padded_k = ( + padded_k.view(B * n, -1) + .index_copy_( + dim=0, + index=flattened_offsets, + source=k, + ) + .view(B, n, -1) + ) + else: + padded_q = torch.ops.fbgemm.jagged_to_padded_dense( + values=q, offsets=[x_offsets], max_lengths=[n], padding_value=0.0 + ) + padded_k = torch.ops.fbgemm.jagged_to_padded_dense( + values=k, offsets=[x_offsets], max_lengths=[n], padding_value=0.0 + ) + + qk_attn = torch.einsum("bnd,bmd->bnm", padded_q, padded_k) + if self._rel_attn_bias is not None: + qk_attn = qk_attn + self._rel_attn_bias(all_timestamps) + qk_attn = F.softmax(qk_attn / math.sqrt(self._attention_dim), dim=-1) + qk_attn = qk_attn * invalid_attn_mask + attn_output = torch.ops.fbgemm.dense_to_jagged( + torch.bmm( + qk_attn, + torch.ops.fbgemm.jagged_to_padded_dense(v, [x_offsets], [n]), + ), + [x_offsets], + )[0] + else: + raise ValueError(f"Unknown normalization method {self._normalization}") + + attn_output = ( + attn_output + if delta_x_offsets is None + else attn_output[delta_x_offsets[0], :] + ) + if self._concat_ua: + a = self._norm_attn_output(attn_output) + o_input = torch.cat([u, a, u * a], dim=-1) + else: + o_input = u * self._norm_attn_output(attn_output) + + new_outputs = ( + self._o( + F.dropout( + o_input, + p=self._dropout_ratio, + training=self.training, + ) + ) + + x + ) + + if delta_x_offsets is not None: + new_outputs = cached_outputs.index_copy_( + dim=0, index=delta_x_offsets[0], source=new_outputs + ) + + if return_cache_states and delta_x_offsets is None: + v = v.contiguous() + + return new_outputs, (v, padded_q, padded_k, new_outputs) + + +class HSTUJagged(torch.nn.Module): + def __init__( + self, + modules: List[SequentialTransductionUnitJagged], + autocast_dtype: Optional[torch.dtype], + ) -> None: + super().__init__() + + self._attention_layers: torch.nn.ModuleList = torch.nn.ModuleList( + modules=modules + ) + self._autocast_dtype: Optional[torch.dtype] = autocast_dtype + + def jagged_forward( + self, + x: torch.Tensor, + x_offsets: torch.Tensor, + all_timestamps: Optional[torch.Tensor], + invalid_attn_mask: torch.Tensor, + delta_x_offsets: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + cache: Optional[List[HSTUCacheState]] = None, + return_cache_states: bool = False, + ) -> Tuple[torch.Tensor, List[HSTUCacheState]]: + """ + Args: + x: (\sum_i N_i, D) x float + x_offsets: (B + 1) x int32 + all_timestamps: (B, 1 + N) x int64 + invalid_attn_mask: (B, N, N) x float, each element in {0, 1} + return_cache_states: bool. True if we should return cache states. + + Returns: + x' = f(x), (\sum_i N_i, D) x float + """ + cache_states: List[HSTUCacheState] = [] + + with torch.autocast( + "cuda", + enabled=self._autocast_dtype is not None, + dtype=self._autocast_dtype or torch.float16, + ): + for i, layer in enumerate(self._attention_layers): + x, cache_states_i = layer( + x=x, + x_offsets=x_offsets, + all_timestamps=all_timestamps, + invalid_attn_mask=invalid_attn_mask, + delta_x_offsets=delta_x_offsets, + cache=cache[i] if cache is not None else None, + return_cache_states=return_cache_states, + ) + if return_cache_states: + cache_states.append(cache_states_i) + + return x, cache_states + + def forward( + self, + x: torch.Tensor, + x_offsets: torch.Tensor, + all_timestamps: Optional[torch.Tensor], + invalid_attn_mask: torch.Tensor, + delta_x_offsets: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + cache: Optional[List[HSTUCacheState]] = None, + return_cache_states: bool = False, + ) -> Tuple[torch.Tensor, List[HSTUCacheState]]: + """ + Args: + x: (B, N, D) x float. + x_offsets: (B + 1) x int32. + all_timestamps: (B, 1 + N) x int64 + invalid_attn_mask: (B, N, N) x float, each element in {0, 1}. + Returns: + x' = f(x), (B, N, D) x float + """ + if len(x.size()) == 3: + x = torch.ops.fbgemm.dense_to_jagged(x, [x_offsets])[0] + + jagged_x, cache_states = self.jagged_forward( + x=x, + x_offsets=x_offsets, + all_timestamps=all_timestamps, + invalid_attn_mask=invalid_attn_mask, + delta_x_offsets=delta_x_offsets, + cache=cache, + return_cache_states=return_cache_states, + ) + y = torch.ops.fbgemm.jagged_to_padded_dense( + values=jagged_x, + offsets=[x_offsets], + max_lengths=[invalid_attn_mask.size(1)], + padding_value=0.0, + ) + return y, cache_states + + +class HSTU(SequentialEncoderWithLearnedSimilarityModule): + """ + Implements HSTU (Hierarchical Sequential Transduction Unit) in + Actions Speak Louder than Words: Trillion-Parameter Sequential Transducers for Generative Recommendations, + https://arxiv.org/abs/2402.17152. + + Note that this implementation is intended for reproducing experiments in + the traditional sequential recommender setting (Section 4.1.1), and does + not yet use optimized kernels discussed in the paper. + """ + + def __init__( + self, + max_sequence_len: int, + max_output_len: int, + embedding_dim: int, + num_blocks: int, + num_heads: int, + linear_dim: int, + attention_dim: int, + normalization: str, + linear_config: str, + linear_activation: str, + linear_dropout_rate: float, + attn_dropout_rate: float, + embedding_module: EmbeddingModule, + similarity_module: SimilarityModule, + input_features_preproc_module: InputFeaturesPreprocessorModule, + output_postproc_module: OutputPostprocessorModule, + enable_relative_attention_bias: bool = True, + concat_ua: bool = False, + verbose: bool = True, + ) -> None: + super().__init__(ndp_module=similarity_module) + + self._embedding_dim: int = embedding_dim + self._item_embedding_dim: int = embedding_module.item_embedding_dim + self._max_sequence_length: int = max_sequence_len + self._embedding_module: EmbeddingModule = embedding_module + self._input_features_preproc: InputFeaturesPreprocessorModule = ( + input_features_preproc_module + ) + self._output_postproc: OutputPostprocessorModule = output_postproc_module + self._num_blocks: int = num_blocks + self._num_heads: int = num_heads + self._dqk: int = attention_dim + self._dv: int = linear_dim + self._linear_activation: str = linear_activation + self._linear_dropout_rate: float = linear_dropout_rate + self._attn_dropout_rate: float = attn_dropout_rate + self._enable_relative_attention_bias: bool = enable_relative_attention_bias + self._hstu = HSTUJagged( + modules=[ + SequentialTransductionUnitJagged( + embedding_dim=self._embedding_dim, + linear_hidden_dim=linear_dim, + attention_dim=attention_dim, + normalization=normalization, + linear_config=linear_config, + linear_activation=linear_activation, + num_heads=num_heads, + # TODO: change to lambda x. + relative_attention_bias_module=( + RelativeBucketedTimeAndPositionBasedBias( + max_seq_len=max_sequence_len + + max_output_len, # accounts for next item. + num_buckets=128, + bucketization_fn=lambda x: ( + torch.log(torch.abs(x).clamp(min=1)) / 0.301 + ).long(), + ) + if enable_relative_attention_bias + else None + ), + dropout_ratio=linear_dropout_rate, + attn_dropout_ratio=attn_dropout_rate, + concat_ua=concat_ua, + ) + for _ in range(num_blocks) + ], + autocast_dtype=None, + ) + # causal forward, w/ +1 for padding. + self.register_buffer( + "_attn_mask", + torch.triu( + torch.ones( + ( + self._max_sequence_length + max_output_len, + self._max_sequence_length + max_output_len, + ), + dtype=torch.bool, + ), + diagonal=1, + ), + ) + self._verbose: bool = verbose + self.reset_params() + + def reset_params(self) -> None: + for name, params in self.named_parameters(): + if ("_hstu" in name) or ("_embedding_module" in name): + if self._verbose: + print(f"Skipping init for {name}") + continue + try: + torch.nn.init.xavier_normal_(params.data) + if self._verbose: + print( + f"Initialize {name} as xavier normal: {params.data.size()} params" + ) + except: + if self._verbose: + print(f"Failed to initialize {name}: {params.data.size()} params") + + def get_item_embeddings(self, item_ids: torch.Tensor) -> torch.Tensor: + return self._embedding_module.get_item_embeddings(item_ids) + + def debug_str(self) -> str: + debug_str = ( + f"HSTU-b{self._num_blocks}-h{self._num_heads}-dqk{self._dqk}-dv{self._dv}" + + f"-l{self._linear_activation}d{self._linear_dropout_rate}" + + f"-ad{self._attn_dropout_rate}" + ) + if not self._enable_relative_attention_bias: + debug_str += "-norab" + return debug_str + + def generate_user_embeddings( + self, + past_lengths: torch.Tensor, + past_ids: torch.Tensor, + past_embeddings: torch.Tensor, + past_payloads: Dict[str, torch.Tensor], + delta_x_offsets: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + cache: Optional[List[HSTUCacheState]] = None, + return_cache_states: bool = False, + ) -> Tuple[torch.Tensor, List[HSTUCacheState]]: + """ + [B, N] -> [B, N, D]. + """ + device = past_lengths.device + float_dtype = past_embeddings.dtype + B, N, _ = past_embeddings.size() + + past_lengths, user_embeddings, _ = self._input_features_preproc( + past_lengths=past_lengths, + past_ids=past_ids, + past_embeddings=past_embeddings, + past_payloads=past_payloads, + ) + + float_dtype = user_embeddings.dtype + user_embeddings, cached_states = self._hstu( + x=user_embeddings, + x_offsets=torch.ops.fbgemm.asynchronous_complete_cumsum(past_lengths), + all_timestamps=( + past_payloads[TIMESTAMPS_KEY] + if TIMESTAMPS_KEY in past_payloads + else None + ), + invalid_attn_mask=1.0 - self._attn_mask.to(float_dtype), + delta_x_offsets=delta_x_offsets, + cache=cache, + return_cache_states=return_cache_states, + ) + return self._output_postproc(user_embeddings), cached_states + + def forward( + self, + past_lengths: torch.Tensor, + past_ids: torch.Tensor, + past_embeddings: torch.Tensor, + past_payloads: Dict[str, torch.Tensor], + batch_id: Optional[int] = None, + ) -> torch.Tensor: + """ + Runs the main encoder. + + Args: + past_lengths: (B,) x int64 + past_ids: (B, N,) x int64 where the latest engaged ids come first. In + particular, past_ids[i, past_lengths[i] - 1] should correspond to + the latest engaged values. + past_embeddings: (B, N, D) x float or (\sum_b N_b, D) x float. + past_payloads: implementation-specific keyed tensors of shape (B, N, ...). + + Returns: + encoded_embeddings of [B, N, D]. + """ + encoded_embeddings, _ = self.generate_user_embeddings( + past_lengths=past_lengths, + past_ids=past_ids, + past_embeddings=past_embeddings, + past_payloads=past_payloads, + ) + return encoded_embeddings + + def _encode( + self, + past_lengths: torch.Tensor, + past_ids: torch.Tensor, + past_embeddings: torch.Tensor, + past_payloads: Dict[str, torch.Tensor], + delta_x_offsets: Optional[Tuple[torch.Tensor, torch.Tensor]], + cache: Optional[List[HSTUCacheState]], + return_cache_states: bool, + ) -> Union[torch.Tensor, Tuple[torch.Tensor, List[HSTUCacheState]]]: + """ + Args: + past_lengths: (B,) x int64. + past_ids: (B, N,) x int64. + past_embeddings: (B, N, D,) x float. + past_payloads: implementation-specific keyed tensors of shape (B, N, ...). + return_cache_states: bool. + + Returns: + (B, D) x float, representing embeddings for the current state. + """ + encoded_seq_embeddings, cache_states = self.generate_user_embeddings( + past_lengths=past_lengths, + past_ids=past_ids, + past_embeddings=past_embeddings, + past_payloads=past_payloads, + delta_x_offsets=delta_x_offsets, + cache=cache, + return_cache_states=return_cache_states, + ) # [B, N, D] + current_embeddings = get_current_embeddings( + lengths=past_lengths, encoded_embeddings=encoded_seq_embeddings + ) + if return_cache_states: + return current_embeddings, cache_states + else: + return current_embeddings + + def encode( + self, + past_lengths: torch.Tensor, + past_ids: torch.Tensor, + past_embeddings: torch.Tensor, + past_payloads: Dict[str, torch.Tensor], + delta_x_offsets: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + cache: Optional[List[HSTUCacheState]] = None, + return_cache_states: bool = False, + ) -> Union[torch.Tensor, Tuple[torch.Tensor, List[HSTUCacheState]]]: + """ + Runs encoder to obtain the current hidden states. + + Args: + past_lengths: (B,) x int. + past_ids: (B, N,) x int. + past_embeddings: (B, N, D) x float. + past_payloads: implementation-specific keyed tensors of shape (B, N, ...). + + Returns: + (B, D,) x float, representing encoded states at the most recent time step. + """ + return self._encode( + past_lengths=past_lengths, + past_ids=past_ids, + past_embeddings=past_embeddings, + past_payloads=past_payloads, + delta_x_offsets=delta_x_offsets, + cache=cache, + return_cache_states=return_cache_states, + ) diff --git a/recommendation_v4/generative_recommenders/research/modeling/sequential/input_features_preprocessors.py b/recommendation_v4/generative_recommenders/research/modeling/sequential/input_features_preprocessors.py new file mode 100644 index 000000000..a461ab879 --- /dev/null +++ b/recommendation_v4/generative_recommenders/research/modeling/sequential/input_features_preprocessors.py @@ -0,0 +1,259 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +import abc +import math +from typing import Dict, Tuple + +import torch +from generative_recommenders.research.modeling.initialization import truncated_normal + + +class InputFeaturesPreprocessorModule(torch.nn.Module): + @abc.abstractmethod + def debug_str(self) -> str: + pass + + @abc.abstractmethod + def forward( + self, + past_lengths: torch.Tensor, + past_ids: torch.Tensor, + past_embeddings: torch.Tensor, + past_payloads: Dict[str, torch.Tensor], + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + pass + + +class LearnablePositionalEmbeddingInputFeaturesPreprocessor( + InputFeaturesPreprocessorModule +): + def __init__( + self, + max_sequence_len: int, + embedding_dim: int, + dropout_rate: float, + ) -> None: + super().__init__() + + self._embedding_dim: int = embedding_dim + self._pos_emb: torch.nn.Embedding = torch.nn.Embedding( + max_sequence_len, + self._embedding_dim, + ) + self._dropout_rate: float = dropout_rate + self._emb_dropout = torch.nn.Dropout(p=dropout_rate) + self.reset_state() + + def debug_str(self) -> str: + return f"posi_d{self._dropout_rate}" + + def reset_state(self) -> None: + truncated_normal( + self._pos_emb.weight.data, + mean=0.0, + std=math.sqrt(1.0 / self._embedding_dim), + ) + + def forward( + self, + past_lengths: torch.Tensor, + past_ids: torch.Tensor, + past_embeddings: torch.Tensor, + past_payloads: Dict[str, torch.Tensor], + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + B, N = past_ids.size() + D = past_embeddings.size(-1) + + user_embeddings = past_embeddings * (self._embedding_dim**0.5) + self._pos_emb( + torch.arange(N, device=past_ids.device).unsqueeze(0).repeat(B, 1) + ) + user_embeddings = self._emb_dropout(user_embeddings) + + valid_mask = (past_ids != 0).unsqueeze(-1).float() # [B, N, 1] + user_embeddings *= valid_mask + return past_lengths, user_embeddings, valid_mask + + +class LearnablePositionalEmbeddingRatedInputFeaturesPreprocessor( + InputFeaturesPreprocessorModule +): + def __init__( + self, + max_sequence_len: int, + item_embedding_dim: int, + dropout_rate: float, + rating_embedding_dim: int, + num_ratings: int, + ) -> None: + super().__init__() + + self._embedding_dim: int = item_embedding_dim + rating_embedding_dim + self._pos_emb: torch.nn.Embedding = torch.nn.Embedding( + max_sequence_len, + self._embedding_dim, + ) + self._dropout_rate: float = dropout_rate + self._emb_dropout = torch.nn.Dropout(p=dropout_rate) + self._rating_emb: torch.nn.Embedding = torch.nn.Embedding( + num_ratings, + rating_embedding_dim, + ) + self.reset_state() + + def debug_str(self) -> str: + return f"posir_d{self._dropout_rate}" + + def reset_state(self) -> None: + truncated_normal( + self._pos_emb.weight.data, + mean=0.0, + std=math.sqrt(1.0 / self._embedding_dim), + ) + truncated_normal( + self._rating_emb.weight.data, + mean=0.0, + std=math.sqrt(1.0 / self._embedding_dim), + ) + + def forward( + self, + past_lengths: torch.Tensor, + past_ids: torch.Tensor, + past_embeddings: torch.Tensor, + past_payloads: Dict[str, torch.Tensor], + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + B, N = past_ids.size() + + user_embeddings = torch.cat( + [past_embeddings, self._rating_emb(past_payloads["ratings"].int())], + dim=-1, + ) * (self._embedding_dim**0.5) + self._pos_emb( + torch.arange(N, device=past_ids.device).unsqueeze(0).repeat(B, 1) + ) + user_embeddings = self._emb_dropout(user_embeddings) + + valid_mask = (past_ids != 0).unsqueeze(-1).float() # [B, N, 1] + user_embeddings *= valid_mask + return past_lengths, user_embeddings, valid_mask + + +class CombinedItemAndRatingInputFeaturesPreprocessor(InputFeaturesPreprocessorModule): + def __init__( + self, + max_sequence_len: int, + item_embedding_dim: int, + dropout_rate: float, + num_ratings: int, + ) -> None: + super().__init__() + + self._embedding_dim: int = item_embedding_dim + # Due to [item_0, rating_0, item_1, rating_1, ...] + self._pos_emb: torch.nn.Embedding = torch.nn.Embedding( + max_sequence_len * 2, + self._embedding_dim, + ) + self._dropout_rate: float = dropout_rate + self._emb_dropout = torch.nn.Dropout(p=dropout_rate) + self._rating_emb: torch.nn.Embedding = torch.nn.Embedding( + num_ratings, + item_embedding_dim, + ) + self.reset_state() + + def debug_str(self) -> str: + return f"combir_d{self._dropout_rate}" + + def reset_state(self) -> None: + truncated_normal( + self._pos_emb.weight.data, + mean=0.0, + std=math.sqrt(1.0 / self._embedding_dim), + ) + truncated_normal( + self._rating_emb.weight.data, + mean=0.0, + std=math.sqrt(1.0 / self._embedding_dim), + ) + + def get_preprocessed_ids( + self, + past_lengths: torch.Tensor, + past_ids: torch.Tensor, + past_embeddings: torch.Tensor, + past_payloads: Dict[str, torch.Tensor], + ) -> torch.Tensor: + """ + Returns (B, N * 2,) x int64. + """ + B, N = past_ids.size() + return torch.cat( + [ + past_ids.unsqueeze(2), # (B, N, 1) + past_payloads["ratings"].to(past_ids.dtype).unsqueeze(2), + ], + dim=2, + ).reshape(B, N * 2) + + def get_preprocessed_masks( + self, + past_lengths: torch.Tensor, + past_ids: torch.Tensor, + past_embeddings: torch.Tensor, + past_payloads: Dict[str, torch.Tensor], + ) -> torch.Tensor: + """ + Returns (B, N * 2,) x bool. + """ + B, N = past_ids.size() + return (past_ids != 0).unsqueeze(2).expand(-1, -1, 2).reshape(B, N * 2) + + def forward( + self, + past_lengths: torch.Tensor, + past_ids: torch.Tensor, + past_embeddings: torch.Tensor, + past_payloads: Dict[str, torch.Tensor], + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + B, N = past_ids.size() + D = past_embeddings.size(-1) + + user_embeddings = torch.cat( + [ + past_embeddings, # (B, N, D) + self._rating_emb(past_payloads["ratings"].int()), + ], + dim=2, + ) * (self._embedding_dim**0.5) + user_embeddings = user_embeddings.view(B, N * 2, D) + user_embeddings = user_embeddings + self._pos_emb( + torch.arange(N * 2, device=past_ids.device).unsqueeze(0).repeat(B, 1) + ) + user_embeddings = self._emb_dropout(user_embeddings) + + valid_mask = ( + self.get_preprocessed_masks( + past_lengths, + past_ids, + past_embeddings, + past_payloads, + ) + .unsqueeze(2) + .float() + ) # (B, N * 2, 1,) + user_embeddings *= valid_mask + return past_lengths * 2, user_embeddings, valid_mask diff --git a/recommendation_v4/generative_recommenders/research/modeling/sequential/losses/sampled_softmax.py b/recommendation_v4/generative_recommenders/research/modeling/sequential/losses/sampled_softmax.py new file mode 100644 index 000000000..8e2195783 --- /dev/null +++ b/recommendation_v4/generative_recommenders/research/modeling/sequential/losses/sampled_softmax.py @@ -0,0 +1,193 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +from collections import OrderedDict +from typing import Dict, Optional, Tuple + +import torch +import torch.nn.functional as F +from generative_recommenders.research.modeling.sequential.autoregressive_losses import ( + AutoregressiveLoss, + NegativesSampler, +) +from torch.utils.checkpoint import checkpoint + + +class SampledSoftmaxLoss(AutoregressiveLoss): + def __init__( + self, + num_to_sample: int, + softmax_temperature: float, + model, + activation_checkpoint: bool = False, + ) -> None: + super().__init__() + + self._num_to_sample: int = num_to_sample + self._softmax_temperature: float = softmax_temperature + self._model = model + self._activation_checkpoint: bool = activation_checkpoint + + def jagged_forward( # pyre-ignore [15] + self, + output_embeddings: torch.Tensor, + supervision_ids: torch.Tensor, + supervision_embeddings: torch.Tensor, + supervision_weights: torch.Tensor, + negatives_sampler: NegativesSampler, + **kwargs, + ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: + assert output_embeddings.size() == supervision_embeddings.size() + assert supervision_ids.size() == supervision_embeddings.size()[:-1] + assert supervision_ids.size() == supervision_weights.size() + + sampled_ids, sampled_negative_embeddings = negatives_sampler( + positive_ids=supervision_ids, + num_to_sample=self._num_to_sample, + ) + positive_embeddings = negatives_sampler.normalize_embeddings( + supervision_embeddings + ) + positive_logits, aux_losses = self._model.similarity_fn( + query_embeddings=output_embeddings, # [B, D] = [N', D] + item_ids=supervision_ids.unsqueeze(1), # [N', 1] + item_embeddings=positive_embeddings.unsqueeze(1), # [N', D] -> [N', 1, D] + **kwargs, + ) + positive_logits = positive_logits / self._softmax_temperature # [0] + sampled_negatives_logits, _ = self._model.similarity_fn( + query_embeddings=output_embeddings, # [N', D] + item_ids=sampled_ids, # [N', R] + item_embeddings=sampled_negative_embeddings, # [N', R, D] + **kwargs, + ) # [N', R] # [0] + sampled_negatives_logits = torch.where( + supervision_ids.unsqueeze(1) == sampled_ids, # [N', R] + -5e4, + sampled_negatives_logits / self._softmax_temperature, + ) + jagged_loss = -F.log_softmax( + torch.cat([positive_logits, sampled_negatives_logits], dim=1), dim=1 + )[:, 0] + return ( + jagged_loss * supervision_weights + ).sum() / supervision_weights.sum(), aux_losses + + def forward( # pyre-ignore [15] + self, + lengths: torch.Tensor, + output_embeddings: torch.Tensor, + supervision_ids: torch.Tensor, + supervision_embeddings: torch.Tensor, + supervision_weights: torch.Tensor, + negatives_sampler: NegativesSampler, + **kwargs, + ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: + """ + Args: + lengths: [B] x int32 representing number of non-zero elements per row. + output_embeddings: [B, N, D] x float, embeddings for the current + input sequence. + supervision_ids: [B, N] x int64, (positive) supervision ids. + supervision_embeddings: [B, N, D] x float. + supervision_weights: Optional [B, N] x float. Optional weights for + masking out invalid positions, or reweighting supervision labels. + negatives_sampler: sampler used to obtain negative examples paired with + positives. + + Returns: + Tuple of (loss for the current engaged sequence, str-keyed aux_losses). + """ + torch._assert( + output_embeddings.size() == supervision_embeddings.size(), + "Invalid supervision embeddings size.", + ) + torch._assert( + supervision_ids.size() == supervision_embeddings.size()[:-1], + "Invalid supervision ids size.", + ) + + jagged_id_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum(lengths) + jagged_supervision_ids = ( + torch.ops.fbgemm.dense_to_jagged( + supervision_ids.unsqueeze(-1).float(), [jagged_id_offsets] + )[0] + .squeeze(1) + .long() + ) + if "user_ids" in kwargs: + # expand to jagged. + max_length: int = int(lengths.max()) + kwargs["user_ids"] = torch.ops.fbgemm.dense_to_jagged( + kwargs["user_ids"] + .unsqueeze(1) + .expand(-1, max_length) + .unsqueeze(2), # (B, max_length, 1) + [jagged_id_offsets], + )[0].squeeze(1) + + args = OrderedDict( + [ + ( + "output_embeddings", + torch.ops.fbgemm.dense_to_jagged( + output_embeddings, + [jagged_id_offsets], + )[0], + ), + ("supervision_ids", jagged_supervision_ids), + ( + "supervision_embeddings", + torch.ops.fbgemm.dense_to_jagged( + supervision_embeddings, + [jagged_id_offsets], + )[0], + ), + ( + "supervision_weights", + torch.ops.fbgemm.dense_to_jagged( + supervision_weights.unsqueeze(-1), + [jagged_id_offsets], + )[0].squeeze(1), + ), + ("negatives_sampler", negatives_sampler), + ] + ) + args.update(kwargs) + if self._activation_checkpoint: + return checkpoint( + self.jagged_forward, + *args.values(), + use_reentrant=False, + ) + else: + return self.jagged_forward( + output_embeddings=torch.ops.fbgemm.dense_to_jagged( + output_embeddings, + [jagged_id_offsets], + )[0], + supervision_ids=jagged_supervision_ids, + supervision_embeddings=torch.ops.fbgemm.dense_to_jagged( + supervision_embeddings, + [jagged_id_offsets], + )[0], + supervision_weights=torch.ops.fbgemm.dense_to_jagged( + supervision_weights.unsqueeze(-1), + [jagged_id_offsets], + )[0].squeeze(1), + negatives_sampler=negatives_sampler, + **kwargs, + ) diff --git a/recommendation_v4/generative_recommenders/research/modeling/sequential/output_postprocessors.py b/recommendation_v4/generative_recommenders/research/modeling/sequential/output_postprocessors.py new file mode 100644 index 000000000..3319dfd93 --- /dev/null +++ b/recommendation_v4/generative_recommenders/research/modeling/sequential/output_postprocessors.py @@ -0,0 +1,82 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +import abc + +import torch +import torch.nn.functional as F + + +class OutputPostprocessorModule(torch.nn.Module): + @abc.abstractmethod + def debug_str(self) -> str: + pass + + @abc.abstractmethod + def forward( + self, + output_embeddings: torch.Tensor, + ) -> torch.Tensor: + pass + + +class L2NormEmbeddingPostprocessor(OutputPostprocessorModule): + def __init__( + self, + embedding_dim: int, + eps: float = 1e-6, + ) -> None: + super().__init__() + self._embedding_dim: int = embedding_dim + self._eps: float = eps + + def debug_str(self) -> str: + return "l2" + + def forward( + self, + output_embeddings: torch.Tensor, + ) -> torch.Tensor: + output_embeddings = output_embeddings[..., : self._embedding_dim] + return output_embeddings / torch.clamp( + torch.linalg.norm(output_embeddings, ord=None, dim=-1, keepdim=True), + min=self._eps, + ) + + +class LayerNormEmbeddingPostprocessor(OutputPostprocessorModule): + def __init__( + self, + embedding_dim: int, + eps: float = 1e-6, + ) -> None: + super().__init__() + self._embedding_dim: int = embedding_dim + self._eps: float = eps + + def debug_str(self) -> str: + return "ln" + + def forward( + self, + output_embeddings: torch.Tensor, + ) -> torch.Tensor: + output_embeddings = output_embeddings[..., : self._embedding_dim] + return F.layer_norm( + output_embeddings, + normalized_shape=(self._embedding_dim,), + eps=self._eps, + ) diff --git a/recommendation_v4/generative_recommenders/research/modeling/sequential/sasrec.py b/recommendation_v4/generative_recommenders/research/modeling/sequential/sasrec.py new file mode 100644 index 000000000..2709ddb08 --- /dev/null +++ b/recommendation_v4/generative_recommenders/research/modeling/sequential/sasrec.py @@ -0,0 +1,316 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +""" +Implements SASRec (Self-Attentive Sequential Recommendation, https://arxiv.org/abs/1808.09781, ICDM'18). + +Compared with the original paper which used BCE loss, this implementation is modified so that +we can utilize a Sampled Softmax loss proposed in Revisiting Neural Retrieval on Accelerators +(https://arxiv.org/abs/2306.04039, KDD'23) and Turning Dross Into Gold Loss: is BERT4Rec really +better than SASRec? (https://arxiv.org/abs/2309.07602, RecSys'23), where the authors showed +sampled softmax loss to significantly improved SASRec model quality. +""" + +from typing import Dict, Optional, Tuple + +import torch +import torch.nn.functional as F +from generative_recommenders.research.modeling.sequential.embedding_modules import ( + EmbeddingModule, +) +from generative_recommenders.research.modeling.sequential.input_features_preprocessors import ( + InputFeaturesPreprocessorModule, +) +from generative_recommenders.research.modeling.sequential.output_postprocessors import ( + OutputPostprocessorModule, +) +from generative_recommenders.research.modeling.sequential.utils import ( + get_current_embeddings, +) +from generative_recommenders.research.modeling.similarity_module import ( + SequentialEncoderWithLearnedSimilarityModule, +) +from generative_recommenders.research.rails.similarities.module import SimilarityModule + + +class StandardAttentionFF(torch.nn.Module): + def __init__( + self, + embedding_dim: int, + hidden_dim: int, + activation_fn: str, + dropout_rate: float, + ) -> None: + super().__init__() + + assert activation_fn == "relu" or activation_fn == "gelu", ( + f"Invalid activation_fn {activation_fn}" + ) + + self._conv1d = torch.nn.Sequential( + torch.nn.Conv1d( + in_channels=embedding_dim, + out_channels=hidden_dim, + kernel_size=1, + ), + torch.nn.GELU() if activation_fn == "gelu" else torch.nn.ReLU(), + torch.nn.Dropout(p=dropout_rate), + torch.nn.Conv1d( + in_channels=hidden_dim, + out_channels=embedding_dim, + kernel_size=1, + ), + torch.nn.Dropout(p=dropout_rate), + ) + + def forward(self, inputs: torch.Tensor) -> torch.Tensor: + # Conv1D requires (B, D, N) + return self._conv1d(inputs.transpose(-1, -2)).transpose(-1, -2) + inputs + + +class SASRec(SequentialEncoderWithLearnedSimilarityModule): + """ + Implements SASRec (Self-Attentive Sequential Recommendation, https://arxiv.org/abs/1808.09781, ICDM'18). + + Compared with the original paper which used BCE loss, this implementation is modified so that + we can utilize a Sampled Softmax loss proposed in Revisiting Neural Retrieval on Accelerators + (https://arxiv.org/abs/2306.04039, KDD'23) and Turning Dross Into Gold Loss: is BERT4Rec really + better than SASRec? (https://arxiv.org/abs/2309.07602, RecSys'23), where the authors showed + sampled softmax loss to significantly improved SASRec model quality. + """ + + def __init__( + self, + max_sequence_len: int, + max_output_len: int, + embedding_dim: int, + num_blocks: int, + num_heads: int, + ffn_hidden_dim: int, + ffn_activation_fn: str, + ffn_dropout_rate: float, + embedding_module: EmbeddingModule, + similarity_module: SimilarityModule, + input_features_preproc_module: InputFeaturesPreprocessorModule, + output_postproc_module: OutputPostprocessorModule, + activation_checkpoint: bool = False, + verbose: bool = False, + ) -> None: + super().__init__(ndp_module=similarity_module) + + self._embedding_module: EmbeddingModule = embedding_module + self._embedding_dim: int = embedding_dim + self._item_embedding_dim: int = embedding_module.item_embedding_dim + self._max_sequence_length: int = max_sequence_len + max_output_len + self._input_features_preproc: InputFeaturesPreprocessorModule = ( + input_features_preproc_module + ) + self._output_postproc: OutputPostprocessorModule = output_postproc_module + self._activation_checkpoint: bool = activation_checkpoint + self._verbose: bool = verbose + + self.attention_layers = torch.nn.ModuleList() + self.forward_layers = torch.nn.ModuleList() + self._num_blocks: int = num_blocks + self._num_heads: int = num_heads + self._ffn_hidden_dim: int = ffn_hidden_dim + self._ffn_activation_fn: str = ffn_activation_fn + self._ffn_dropout_rate: float = ffn_dropout_rate + + for _ in range(num_blocks): + self.attention_layers.append( + torch.nn.MultiheadAttention( + embed_dim=self._embedding_dim, + num_heads=num_heads, + dropout=ffn_dropout_rate, + batch_first=True, + ) + ) + self.forward_layers.append( + StandardAttentionFF( + embedding_dim=self._embedding_dim, + hidden_dim=ffn_hidden_dim, + activation_fn=ffn_activation_fn, + dropout_rate=self._ffn_dropout_rate, + ) + ) + + self.register_buffer( + "_attn_mask", + torch.triu( + torch.ones( + (self._max_sequence_length, self._max_sequence_length), + dtype=torch.bool, + ), + diagonal=1, + ), + ) + self.reset_state() + + def reset_state(self) -> None: + for name, params in self.named_parameters(): + if ( + "_input_features_preproc" in name + or "_embedding_module" in name + or "_output_postproc" in name + ): + if self._verbose: + print(f"Skipping initialization for {name}") + continue + try: + torch.nn.init.xavier_normal_(params.data) + if self._verbose: + print( + f"Initialize {name} as xavier normal: {params.data.size()} params" + ) + except: + if self._verbose: + print(f"Failed to initialize {name}: {params.data.size()} params") + + def get_item_embeddings(self, item_ids: torch.Tensor) -> torch.Tensor: + return self._embedding_module.get_item_embeddings(item_ids) + + def debug_str(self) -> str: + return ( + f"SASRec-d{self._item_embedding_dim}-b{self._num_blocks}-h{self._num_heads}" + + "-" + + self._input_features_preproc.debug_str() + + "-" + + self._output_postproc.debug_str() + + f"-ffn{self._ffn_hidden_dim}-{self._ffn_activation_fn}-d{self._ffn_dropout_rate}" + + f"{'-ac' if self._activation_checkpoint else ''}" + ) + + def _run_one_layer( + self, + i: int, + user_embeddings: torch.Tensor, + valid_mask: torch.Tensor, + ) -> torch.Tensor: + Q = F.layer_norm( + user_embeddings, + normalized_shape=(self._embedding_dim,), + eps=1e-8, + ) + mha_outputs, _ = self.attention_layers[i]( + query=Q, + key=user_embeddings, + value=user_embeddings, + attn_mask=self._attn_mask, + ) + user_embeddings = self.forward_layers[i]( + F.layer_norm( + Q + mha_outputs, + normalized_shape=(self._embedding_dim,), + eps=1e-8, + ) + ) + user_embeddings *= valid_mask + return user_embeddings + + def generate_user_embeddings( + self, + past_lengths: torch.Tensor, + past_ids: torch.Tensor, + past_embeddings: torch.Tensor, + past_payloads: Dict[str, torch.Tensor], + ) -> torch.Tensor: + """ + Args: + past_ids: (B, N,) x int + + Returns: + (B, N, D,) x float + """ + past_lengths, user_embeddings, valid_mask = self._input_features_preproc( + past_lengths=past_lengths, + past_ids=past_ids, + past_embeddings=past_embeddings, + past_payloads=past_payloads, + ) + + for i in range(len(self.attention_layers)): + if self._activation_checkpoint: + user_embeddings = torch.utils.checkpoint.checkpoint( + self._run_one_layer, + i, + user_embeddings, + valid_mask, + use_reentrant=False, + ) + else: + user_embeddings = self._run_one_layer(i, user_embeddings, valid_mask) + + return self._output_postproc(user_embeddings) + + def forward( + self, + past_lengths: torch.Tensor, + past_ids: torch.Tensor, + past_embeddings: torch.Tensor, + past_payloads: Dict[str, torch.Tensor], + batch_id: Optional[int] = None, + ) -> torch.Tensor: + """ + Args: + past_ids: [B, N] x int64 where the latest engaged ids come first. In + particular, [:, 0] should correspond to the last engaged values. + past_ratings: [B, N] x int64. + past_timestamps: [B, N] x int64. + + Returns: + encoded_embeddings of [B, N, D]. + """ + encoded_embeddings = self.generate_user_embeddings( + past_lengths, + past_ids, + past_embeddings, + past_payloads, + ) + return encoded_embeddings + + def encode( + self, + past_lengths: torch.Tensor, + past_ids: torch.Tensor, # [B, N] x int64 + past_embeddings: torch.Tensor, + past_payloads: Dict[str, torch.Tensor], + ) -> torch.Tensor: + encoded_seq_embeddings = self.generate_user_embeddings( + past_lengths, past_ids, past_embeddings, past_payloads + ) # [B, N, D] + return get_current_embeddings( + lengths=past_lengths, encoded_embeddings=encoded_seq_embeddings + ) + + def predict( + self, + past_ids: torch.Tensor, + past_ratings: torch.Tensor, + past_timestamps: torch.Tensor, + next_timestamps: torch.Tensor, + target_ids: torch.Tensor, + batch_id: Optional[int] = None, + ) -> torch.Tensor: + return self.interaction( # pyre-ignore [29] + self.encode( + past_ids, + past_ratings, + past_timestamps, + next_timestamps, # pyre-ignore [6] + ), + target_ids, + ) # [B, X] diff --git a/recommendation_v4/generative_recommenders/research/modeling/sequential/utils.py b/recommendation_v4/generative_recommenders/research/modeling/sequential/utils.py new file mode 100644 index 000000000..60dfb8e44 --- /dev/null +++ b/recommendation_v4/generative_recommenders/research/modeling/sequential/utils.py @@ -0,0 +1,129 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +import torch + + +def batch_gather_embeddings( + rowwise_indices: torch.Tensor, + embeddings: torch.Tensor, +) -> torch.Tensor: + """ + Args: + rowwise_indices: (B, N) x int, where each entry is in [0, X). + embeddings: (B, X, D,) x float. + + Returns: + (B, N, D,) x float, embeddings corresponding to rowwise_indices. + """ + _, N = rowwise_indices.size() + B, X, D = embeddings.size() + flattened_indices = ( + rowwise_indices + + torch.arange( + start=0, + end=B, + step=1, + dtype=rowwise_indices.dtype, + device=rowwise_indices.device, + ) + .unsqueeze(1) + .expand(-1, N) + * X + ) + return embeddings.view(-1, D)[flattened_indices, :].reshape( + rowwise_indices.size() + (D,) + ) + + +def batch_scatter_embeddings( + dst_embeddings: torch.Tensor, + rowwise_indices: torch.Tensor, + src_embeddings: torch.Tensor, +) -> None: + """ + Args: + dst_embeddings: (B, N, D,) x float. + rowwise_indices: (B,) x int, where each entry is in [0, N - 1). + source_embeddings: (B, D,) x float. + """ + B, N, D = dst_embeddings.size() + flattened_indices = rowwise_indices + torch.arange( + start=0, + end=B * N, + step=N, + dtype=rowwise_indices.dtype, + device=rowwise_indices.device, + ) + dst_embeddings.view(B * N, D)[flattened_indices, :] = src_embeddings + + +def get_current_embeddings( + lengths: torch.Tensor, + encoded_embeddings: torch.Tensor, +) -> torch.Tensor: + """ + Args: + lengths: (B,) x int + seq_embeddings: (B, N, D,) x float + + Returns: + (B, D,) x float, where [i, :] == encoded_embeddings[i, lengths[i] - 1, :] + """ + B, N, D = encoded_embeddings.size() + flattened_offsets = (lengths - 1) + torch.arange( + start=0, end=B, step=1, dtype=lengths.dtype, device=lengths.device + ) * N + return encoded_embeddings.reshape(-1, D)[flattened_offsets, :].reshape(B, D) + + +def jagged_or_dense_repeat_interleave_dim0( + x: torch.Tensor, lengths: torch.Tensor, repeats: int +) -> torch.Tensor: + if len(x.size()) == 3: + return x.repeat_interleave(repeats, dim=0) + else: + assert len(x.size()) == 2, f"x.size() = {x.size()}" + padded_x = torch.ops.fbgemm.jagged_to_padded_dense( + values=x, + offsets=[torch.ops.fbgemm.asynchronous_complete_cumsum(lengths)], + max_lengths=[lengths.max()], + padding_value=0.0, + ) + lengths = lengths.repeat_interleave(repeats, dim=0) + return torch.ops.fbgemm.dense_to_jagged( + padded_x.repeat_interleave(repeats, dim=0), + [torch.ops.fbgemm.asynchronous_complete_cumsum(lengths)], + )[0] + + +def jagged_or_dense_index_select_dim0( + x: torch.Tensor, lengths: torch.Tensor, indices: torch.Tensor +) -> torch.Tensor: + if len(x.size()) == 3: + return x[indices, :, :] + else: + assert len(x.size()) == 2, f"x.size() = {x.size()}" + padded_x = torch.ops.fbgemm.jagged_to_padded_dense( + values=x, + offsets=[torch.ops.fbgemm.asynchronous_complete_cumsum(lengths)], + max_lengths=[lengths.max()], + padding_value=0.0, + ) + return torch.ops.fbgemm.dense_to_jagged( + padded_x[indices, :], + [torch.ops.fbgemm.asynchronous_complete_cumsum(lengths[indices])], + )[0] diff --git a/recommendation_v4/generative_recommenders/research/modeling/similarity_module.py b/recommendation_v4/generative_recommenders/research/modeling/similarity_module.py new file mode 100644 index 000000000..3ba32d239 --- /dev/null +++ b/recommendation_v4/generative_recommenders/research/modeling/similarity_module.py @@ -0,0 +1,68 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +import abc +from typing import Optional + +import torch +from generative_recommenders.research.rails.similarities.module import SimilarityModule + + +class SequentialEncoderWithLearnedSimilarityModule(torch.nn.Module): + """ + Interface enabling using various similarity functions (besides inner products) + as part of a sequential encoder/decoder. + + See rails/ for more details. + """ + + def __init__( + self, + ndp_module: SimilarityModule, + ) -> None: + super().__init__() + + self._ndp_module: SimilarityModule = ndp_module + + @abc.abstractmethod + def debug_str( + self, + ) -> str: + pass + + def similarity_fn( + self, + query_embeddings: torch.Tensor, + item_ids: torch.Tensor, + item_embeddings: Optional[torch.Tensor] = None, + **kwargs, + ) -> torch.Tensor: + torch._assert( + len(query_embeddings.size()) == 2, "len(query_embeddings.size()) must be 2" + ) + torch._assert(len(item_ids.size()) == 2, "len(item_ids.size()) must be 2") + if item_embeddings is None: + item_embeddings = self.get_item_embeddings(item_ids) # pyre-ignore [29] + torch._assert( + len(item_embeddings.size()) == 3, "len(item_embeddings.size()) must be 3" + ) + + return self._ndp_module( + query_embeddings=query_embeddings, # (B, query_embedding_dim) + item_embeddings=item_embeddings, # (1/B, X, item_embedding_dim) + item_ids=item_ids, + **kwargs, + ) diff --git a/recommendation_v4/generative_recommenders/research/modeling/similarity_utils.py b/recommendation_v4/generative_recommenders/research/modeling/similarity_utils.py new file mode 100644 index 000000000..7fd870b4b --- /dev/null +++ b/recommendation_v4/generative_recommenders/research/modeling/similarity_utils.py @@ -0,0 +1,222 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +from typing import List, Optional, Tuple + +import gin +import torch +from generative_recommenders.research.rails.similarities.dot_product_similarity_fn import ( + DotProductSimilarity, +) +from generative_recommenders.research.rails.similarities.layers import SwiGLU +from generative_recommenders.research.rails.similarities.mol.item_embeddings_fn import ( + RecoMoLItemEmbeddingsFn, +) +from generative_recommenders.research.rails.similarities.mol.query_embeddings_fn import ( + RecoMoLQueryEmbeddingsFn, +) +from generative_recommenders.research.rails.similarities.mol.similarity_fn import ( + MoLSimilarity, + SoftmaxDropoutCombiner, +) + + +def init_mlp_xavier_weights_zero_bias(m) -> None: + if isinstance(m, torch.nn.Linear): + torch.nn.init.xavier_uniform(m.weight) + if getattr(m, "bias", None) is not None: + m.bias.data.fill_(0.0) + + +@gin.configurable +def create_mol_interaction_module( + query_embedding_dim: int, + item_embedding_dim: int, + dot_product_dimension: int, + query_dot_product_groups: int, + item_dot_product_groups: int, + temperature: float, + query_dropout_rate: float, + query_hidden_dim: int, + item_dropout_rate: float, + item_hidden_dim: int, + gating_query_hidden_dim: int, + gating_qi_hidden_dim: int, + gating_item_hidden_dim: int, + softmax_dropout_rate: float, + bf16_training: bool, + gating_query_fn: bool = True, + gating_item_fn: bool = True, + dot_product_l2_norm: bool = True, + query_nonlinearity: str = "geglu", + item_nonlinearity: str = "geglu", + uid_dropout_rate: float = 0.5, + uid_embedding_hash_sizes: Optional[List[int]] = None, + uid_embedding_level_dropout: bool = False, + gating_combination_type: str = "glu_silu", + gating_item_dropout_rate: float = 0.0, + gating_qi_dropout_rate: float = 0.0, + eps: float = 1e-6, +) -> Tuple[MoLSimilarity, str]: + """ + Gin wrapper for creating MoL learned similarity. + """ + mol_module = MoLSimilarity( + query_embedding_dim=query_embedding_dim, + item_embedding_dim=item_embedding_dim, + dot_product_dimension=dot_product_dimension, + query_dot_product_groups=query_dot_product_groups, + item_dot_product_groups=item_dot_product_groups, + temperature=temperature, + dot_product_l2_norm=dot_product_l2_norm, + query_embeddings_fn=RecoMoLQueryEmbeddingsFn( + query_embedding_dim=query_embedding_dim, + query_dot_product_groups=query_dot_product_groups, + dot_product_dimension=dot_product_dimension, + dot_product_l2_norm=dot_product_l2_norm, + proj_fn=lambda input_dim, output_dim: ( + torch.nn.Sequential( + torch.nn.Dropout(p=query_dropout_rate), + SwiGLU( + in_features=input_dim, + out_features=query_hidden_dim, + ), + torch.nn.Linear( + in_features=query_hidden_dim, + out_features=output_dim, + ), + ).apply(init_mlp_xavier_weights_zero_bias) + ), + eps=eps, + ), + item_embeddings_fn=RecoMoLItemEmbeddingsFn( + item_embedding_dim=item_embedding_dim, + item_dot_product_groups=item_dot_product_groups, + dot_product_dimension=dot_product_dimension, + dot_product_l2_norm=dot_product_l2_norm, + proj_fn=lambda input_dim, output_dim: ( + torch.nn.Sequential( + torch.nn.Dropout(p=item_dropout_rate), + SwiGLU(in_features=input_dim, out_features=item_hidden_dim), + torch.nn.Linear( + in_features=item_hidden_dim, + out_features=output_dim, + ), + ).apply(init_mlp_xavier_weights_zero_bias) + ), + eps=eps, + ), + gating_query_only_partial_fn=lambda input_dim, output_dim: ( # pyre-ignore [6] + torch.nn.Sequential( + torch.nn.Linear( + in_features=input_dim, + out_features=gating_query_hidden_dim, + ), + torch.nn.SiLU(), + torch.nn.Linear( + in_features=gating_query_hidden_dim, + out_features=output_dim, + bias=False, + ), + ).apply(init_mlp_xavier_weights_zero_bias) + if gating_query_fn + else None + ), + gating_item_only_partial_fn=lambda input_dim, output_dim: ( # pyre-ignore [6] + torch.nn.Sequential( + torch.nn.Dropout(p=gating_item_dropout_rate), + torch.nn.Linear( + in_features=input_dim, + out_features=gating_item_hidden_dim, + ), + torch.nn.SiLU(), + torch.nn.Linear( + in_features=gating_item_hidden_dim, + out_features=output_dim, + bias=False, + ), + ).apply(init_mlp_xavier_weights_zero_bias) + if gating_item_fn + else None + ), + gating_qi_partial_fn=lambda input_dim, output_dim: ( # pyre-ignore [6] + torch.nn.Sequential( + torch.nn.Dropout(p=gating_qi_dropout_rate), + torch.nn.Linear( + in_features=input_dim, + out_features=gating_qi_hidden_dim, + ), + torch.nn.SiLU(), + torch.nn.Linear( + in_features=gating_qi_hidden_dim, + out_features=output_dim, + ), + ).apply(init_mlp_xavier_weights_zero_bias) + if gating_qi_hidden_dim > 0 + else torch.nn.Sequential( + torch.nn.Dropout(p=gating_qi_dropout_rate), + torch.nn.Linear( + in_features=input_dim, + out_features=output_dim, + ), + ).apply(init_mlp_xavier_weights_zero_bias) + ), + gating_combination_type=gating_combination_type, + gating_normalization_fn=lambda _: SoftmaxDropoutCombiner( + dropout_rate=softmax_dropout_rate, eps=1e-6 + ), + eps=eps, + autocast_bf16=bf16_training, + ) + interaction_module_debug_str = ( + f"MoL-{query_dot_product_groups}x{item_dot_product_groups}x{dot_product_dimension}" + + f"-t{temperature}-d{softmax_dropout_rate}" + + f"{'-l2' if dot_product_l2_norm else ''}" + + f"-q{query_hidden_dim}d{query_dropout_rate}{query_nonlinearity}" + + f"-i{item_hidden_dim}d{item_dropout_rate}{item_nonlinearity}" + + (f"-gq{gating_query_hidden_dim}" if gating_query_fn else "") + + ( + f"-gi{gating_item_hidden_dim}d{gating_item_dropout_rate}" + if gating_item_fn + else "" + ) + + f"-gqi{gating_qi_hidden_dim}d{gating_qi_dropout_rate}-x-{gating_combination_type}" + ) + return mol_module, interaction_module_debug_str + + +@gin.configurable +def get_similarity_function( + module_type: str, + query_embedding_dim: int, + item_embedding_dim: int, + bf16_training: bool = False, + activation_checkpoint: bool = False, +) -> Tuple[torch.nn.Module, str]: + if module_type == "DotProduct": + interaction_module = DotProductSimilarity() + interaction_module_debug_str = "DotProduct" + elif module_type == "MoL": + interaction_module, interaction_module_debug_str = ( + create_mol_interaction_module( + query_embedding_dim=query_embedding_dim, + item_embedding_dim=item_embedding_dim, + bf16_training=bf16_training, + ) + ) + else: + raise ValueError(f"Unknown interaction_module_type {module_type}") + return interaction_module, interaction_module_debug_str diff --git a/recommendation_v4/generative_recommenders/research/rails/indexing/candidate_index.py b/recommendation_v4/generative_recommenders/research/rails/indexing/candidate_index.py new file mode 100644 index 000000000..f628468ce --- /dev/null +++ b/recommendation_v4/generative_recommenders/research/rails/indexing/candidate_index.py @@ -0,0 +1,41 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +import abc +from typing import Tuple + +import torch + + +class TopKModule(torch.nn.Module): + @abc.abstractmethod + def forward( + self, + query_embeddings: torch.Tensor, + k: int, + sorted: bool = True, + **kwargs, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Args: + query_embeddings: (B, X, ...). Implementation-specific. + k: int. top k to return. + sorted: bool. + + Returns: + Tuple of (top_k_scores, top_k_ids), both of shape (B, K,) + """ + pass diff --git a/recommendation_v4/generative_recommenders/research/rails/indexing/mips_top_k.py b/recommendation_v4/generative_recommenders/research/rails/indexing/mips_top_k.py new file mode 100644 index 000000000..810b24c42 --- /dev/null +++ b/recommendation_v4/generative_recommenders/research/rails/indexing/mips_top_k.py @@ -0,0 +1,80 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +from typing import Tuple + +import torch +from generative_recommenders.research.rails.indexing.candidate_index import TopKModule + + +class MIPSTopKModule(TopKModule): + def __init__( + self, + item_embeddings: torch.Tensor, + item_ids: torch.Tensor, + ) -> None: + """ + Args: + item_embeddings: (1, X, D) + item_ids: (1, X,) + """ + super().__init__() + + self._item_embeddings: torch.Tensor = item_embeddings + self._item_ids: torch.Tensor = item_ids + + +class MIPSBruteForceTopK(MIPSTopKModule): + def __init__( + self, + item_embeddings: torch.Tensor, + item_ids: torch.Tensor, + ) -> None: + super().__init__( + item_embeddings=item_embeddings, + item_ids=item_ids, + ) + del self._item_embeddings + self._item_embeddings_t: torch.Tensor = item_embeddings.permute( + 2, 1, 0 + ).squeeze(2) + + def forward( + self, + query_embeddings: torch.Tensor, + k: int, + sorted: bool = True, + **kwargs, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Args: + query_embeddings: (B, ...). Implementation-specific. + k: int. final top-k to return. + sorted: bool. whether to sort final top-k results or not. + + Returns: + Tuple of (top_k_scores x float, top_k_ids x int), both of shape (B, K,) + """ + # (B, X,) + all_logits = torch.mm(query_embeddings, self._item_embeddings_t) + top_k_logits, top_k_indices = torch.topk( + all_logits, + dim=1, + k=k, + sorted=sorted, + largest=True, + ) # (B, k,) + return top_k_logits, self._item_ids.squeeze(0)[top_k_indices] diff --git a/recommendation_v4/generative_recommenders/research/rails/indexing/mol_top_k.py b/recommendation_v4/generative_recommenders/research/rails/indexing/mol_top_k.py new file mode 100644 index 000000000..fe88ca919 --- /dev/null +++ b/recommendation_v4/generative_recommenders/research/rails/indexing/mol_top_k.py @@ -0,0 +1,132 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +""" +Defines exact- and approximate- Top-K modules for Mixture-of-Logits (MoL), +discussed in Retrieval with Learned Similarities (https://arxiv.org/abs/2407.15462). + +Forked from bailuding/rails @ 664fdb9. +""" + +from typing import Tuple + +import torch +from generative_recommenders.research.rails.indexing.candidate_index import TopKModule +from generative_recommenders.research.rails.similarities.mol.similarity_fn import ( + MoLSimilarity, +) + + +class MoLTopKModule(TopKModule): + def __init__( + self, + mol_module: MoLSimilarity, + item_embeddings: torch.Tensor, + item_ids: torch.Tensor, + flatten_item_ids_and_embeddings: bool, + keep_component_level_item_embeddings: bool, + component_level_item_embeddings_dtype: torch.dtype = torch.bfloat16, + ) -> None: + """ + Args: + mol_module: MoLSimilarity. + item_embeddings: (1, X, D) if mol_module._apply_item_embeddings_fn is True, + (1, X, P_X, D_P) otherwise. + item_ids: (1, X,) representing the item ids. + flatten_item_ids_and_embeddings: bool. If true, do not keep the extra (1,) + dimension at size(0). + keep_component_level_item_embeddings: bool. If true, keep P_x component-level + embeddings in `self._mol_item_embeddings` for downstream applications. + component_level_item_embeddings_dtype: torch.dtype. If set, the dtype + to keep component-level item embeddings in. By default we use bfloat16. + """ + super().__init__() + + self._mol_module: MoLSimilarity = mol_module + self._item_embeddings: torch.Tensor = ( + item_embeddings + if not flatten_item_ids_and_embeddings + else item_embeddings.squeeze(0) + ) + + if keep_component_level_item_embeddings: + self._mol_item_embeddings: torch.Tensor = ( + mol_module.get_item_component_embeddings( + ( + self._item_embeddings.squeeze(0) + if not flatten_item_ids_and_embeddings + else self._item_embeddings + ), + decoupled_inference=True, + )[0] # (X, D) -> (X, P_X, D_P) + ).to(component_level_item_embeddings_dtype) + + self._item_ids: torch.Tensor = ( + item_ids if not flatten_item_ids_and_embeddings else item_ids.squeeze(0) + ) + + @property + def mol_module(self) -> MoLSimilarity: + return self._mol_module + + +class MoLBruteForceTopK(MoLTopKModule): + def __init__( + self, + mol_module: MoLSimilarity, + item_embeddings: torch.Tensor, + item_ids: torch.Tensor, + ) -> None: + super().__init__( + mol_module=mol_module, + item_embeddings=item_embeddings, + item_ids=item_ids, + flatten_item_ids_and_embeddings=False, + keep_component_level_item_embeddings=False, + ) + + def forward( + self, + query_embeddings: torch.Tensor, + k: int, + sorted: bool = True, + **kwargs, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Args: + query_embeddings: (B, X, D) if mol_module._apply_query_embeddings_fn is True, + (B, X, P_Q, D_P) otherwise. + k: int. final top-k to return. + sorted: bool. whether to sort final top-k results or not. + **kwargs: Implementation-specific keys/values. + + Returns: + Tuple of (top_k_scores x float, top_k_ids x int), both of shape (B, K,) + """ + # (B, X,) + all_logits, _ = self.mol_module( + query_embeddings, + self._item_embeddings, + **kwargs, + ) + top_k_logits, top_k_indices = torch.topk( + all_logits, + dim=1, + k=k, + sorted=sorted, + largest=True, + ) # (B, k,) + return top_k_logits, self._item_ids.squeeze(0)[top_k_indices] diff --git a/recommendation_v4/generative_recommenders/research/rails/similarities/dot_product_similarity_fn.py b/recommendation_v4/generative_recommenders/research/rails/similarities/dot_product_similarity_fn.py new file mode 100644 index 000000000..9357fd0e4 --- /dev/null +++ b/recommendation_v4/generative_recommenders/research/rails/similarities/dot_product_similarity_fn.py @@ -0,0 +1,68 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +from typing import Dict, Tuple + +import torch +from generative_recommenders.research.rails.similarities.module import SimilarityModule + + +class DotProductSimilarity(SimilarityModule): + def __init__( + self, + ) -> None: + super().__init__() + + def debug_str(self) -> str: + return "dp" + + def forward( + self, + query_embeddings: torch.Tensor, + item_embeddings: torch.Tensor, + **kwargs, + ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: + """ + Args: + query_embeddings: (B, D,) or (B * r, D) x float. + item_embeddings: (1, X, D) or (B, X, D) x float. + + Returns: + (B, X) x float. + """ + + B_I, X, D = item_embeddings.size() + if B_I == 1: + # [B, D] x ([1, X, D] -> [D, X]) => [B, X] + return ( + torch.mm(query_embeddings, item_embeddings.squeeze(0).t()), + {}, + ) # [B, X] + elif query_embeddings.size(0) != B_I: + # (B * r, D) x (B, X, D). + return ( + torch.bmm( + query_embeddings.view(B_I, -1, D), + item_embeddings.permute(0, 2, 1), + ).view(-1, X), + {}, + ) + else: + # [B, X, D] x ([B, D] -> [B, D, 1]) => [B, X, 1] -> [B, X] + return ( + torch.bmm(item_embeddings, query_embeddings.unsqueeze(2)).squeeze(2), + {}, + ) diff --git a/recommendation_v4/generative_recommenders/research/rails/similarities/layers.py b/recommendation_v4/generative_recommenders/research/rails/similarities/layers.py new file mode 100644 index 000000000..3f838bc48 --- /dev/null +++ b/recommendation_v4/generative_recommenders/research/rails/similarities/layers.py @@ -0,0 +1,82 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +""" +Defines network architectures used in constructing various learned similarities. + +Forked from bailuding/rails @ 664fdb9. +""" + +import torch +import torch.nn.functional as F + + +class GeGLU(torch.nn.Module): + def __init__( + self, + in_features: int, + out_features: int, + ) -> None: + super().__init__() + + self._in_features = in_features + self._out_features = out_features + self._w = torch.nn.Parameter( + torch.empty((in_features, out_features * 2)).normal_(mean=0, std=0.02), + ) + self._b = torch.nn.Parameter( + torch.zeros((1, out_features * 2)), + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + bs = x.size()[:-1] + lhs, rhs = torch.split( + torch.mm(x.reshape(-1, self._in_features), self._w) + self._b, + [self._out_features, self._out_features], + dim=-1, + ) + return (F.gelu(lhs) * rhs).reshape(bs + (self._out_features,)) + + +class SwiGLU(torch.nn.Module): + """ + SwiGLU from https://arxiv.org/abs/2002.05202. + """ + + def __init__( + self, + in_features: int, + out_features: int, + ) -> None: + super().__init__() + + self._in_features = in_features + self._out_features = out_features + self._w = torch.nn.Parameter( + torch.empty((in_features, out_features * 2)).normal_(mean=0, std=0.02), + ) + self._b = torch.nn.Parameter( + torch.zeros((1, out_features * 2)), + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + bs = x.size()[:-1] + lhs, rhs = torch.split( + torch.mm(x.reshape(-1, self._in_features), self._w) + self._b, + [self._out_features, self._out_features], + dim=-1, + ) + return (F.silu(lhs) * rhs).reshape(bs + (self._out_features,)) diff --git a/recommendation_v4/generative_recommenders/research/rails/similarities/module.py b/recommendation_v4/generative_recommenders/research/rails/similarities/module.py new file mode 100644 index 000000000..e4061fa74 --- /dev/null +++ b/recommendation_v4/generative_recommenders/research/rails/similarities/module.py @@ -0,0 +1,55 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +import abc +from typing import Dict, Tuple + +import torch + + +class SimilarityModule(torch.nn.Module): + """ + Interface enabling interfacing with various similarity functions. + + While the discussions in our initial ICML'24 paper are based on inner products + for simplicity, we provide this interface (SimilarityModule) to support various + learned similarities at the retrieval stage, such as MLPs, Factorization Machines + (FMs), and Mixture-of-Logits (MoL), which we discussed in + - Revisiting Neural Retrieval on Accelerators (KDD'23), and + - Retrieval with Learned Similarities (https://arxiv.org/abs/2407.15462). + """ + + @abc.abstractmethod + def forward( + self, + query_embeddings: torch.Tensor, + item_embeddings: torch.Tensor, + **kwargs, + ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: + """ + Args: + query_embeddings: (B, input_embedding_dim) x float. + item_embeddings: (1/B, X, item_embedding_dim) x float. + **kwargs: Implementation-specific keys/values (e.g., + item ids / sideinfo, etc.) + + Returns: + A tuple of ( + (B, X,) similarity values, + keyed outputs representing auxiliary losses at training time. + ). + """ + pass diff --git a/recommendation_v4/generative_recommenders/research/rails/similarities/mol/embeddings_fn.py b/recommendation_v4/generative_recommenders/research/rails/similarities/mol/embeddings_fn.py new file mode 100644 index 000000000..fd94e6f22 --- /dev/null +++ b/recommendation_v4/generative_recommenders/research/rails/similarities/mol/embeddings_fn.py @@ -0,0 +1,52 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +""" +Defines interface for generating query- and item-side embeddings for MoL. + +Forked from bailuding/rails @ 664fdb9. +""" + +import abc +from typing import Dict, Tuple + +import torch + + +class MoLEmbeddingsFn(torch.nn.Module): + """ + Generates K_Q query-side (K_I item-side) embeddings for MoL based on + input embeddings and other optional implementation-specific tensors. + """ + + @abc.abstractmethod + def forward( + self, + input_embeddings: torch.Tensor, + **kwargs, + ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: + """ + Args: + input_embeddings: (B, ...) x float where B is the batch size. + kwargs: implementation-specific. + + Returns: + Tuple of ( + (B, query_dot_product_groups/item_dot_product_groups, dot_product_embedding_dim) x float, + str-keyed auxiliary losses. + ). + """ + pass diff --git a/recommendation_v4/generative_recommenders/research/rails/similarities/mol/item_embeddings_fn.py b/recommendation_v4/generative_recommenders/research/rails/similarities/mol/item_embeddings_fn.py new file mode 100644 index 000000000..237cd8942 --- /dev/null +++ b/recommendation_v4/generative_recommenders/research/rails/similarities/mol/item_embeddings_fn.py @@ -0,0 +1,99 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +""" +Defines functions to generate item-side embeddings for MoL. + +Forked from bailuding/rails @ 664fdb9. +""" + +from typing import Callable, Dict, Tuple + +import torch +from generative_recommenders.research.rails.similarities.mol.embeddings_fn import ( + MoLEmbeddingsFn, +) + + +def init_mlp_xavier_weights_zero_bias(m) -> None: + if isinstance(m, torch.nn.Linear): + torch.nn.init.xavier_uniform_(m.weight) + if getattr(m, "bias", None) is not None: + m.bias.data.fill_(0.0) + + +class RecoMoLItemEmbeddingsFn(MoLEmbeddingsFn): + """ + Generates P_X query-side embeddings for MoL based on input embeddings and other + optional tensors for recommendation models. Tested for sequential retrieval + scenarios. + """ + + def __init__( + self, + item_embedding_dim: int, + item_dot_product_groups: int, + dot_product_dimension: int, + dot_product_l2_norm: bool, + proj_fn: Callable[[int, int], torch.nn.Module], + eps: float, + ) -> None: + super().__init__() + + self._item_emb_based_dot_product_groups: int = item_dot_product_groups + self._item_emb_proj_module: torch.nn.Module = proj_fn( + item_embedding_dim, + dot_product_dimension * self._item_emb_based_dot_product_groups, + ) + self._dot_product_dimension: int = dot_product_dimension + self._dot_product_l2_norm: bool = dot_product_l2_norm + self._eps: float = eps + + def forward( + self, + input_embeddings: torch.Tensor, + **kwargs, + ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: + """ + Args: + input_embeddings: (B, item_embedding_dim,) x float where B is the batch size. + kwargs: str-keyed tensors. Implementation-specific. + + Returns: + Tuple of ( + (B, item_dot_product_groups, dot_product_embedding_dim) x float, + str-keyed aux_losses, + ). + """ + split_item_embeddings = self._item_emb_proj_module(input_embeddings).reshape( + input_embeddings.size()[:-1] + + ( + self._item_emb_based_dot_product_groups, + self._dot_product_dimension, + ) + ) + + if self._dot_product_l2_norm: + split_item_embeddings = split_item_embeddings / torch.clamp( + torch.linalg.norm( + split_item_embeddings, + ord=None, + dim=-1, + keepdim=True, + ), + min=self._eps, + ) + return split_item_embeddings, {} diff --git a/recommendation_v4/generative_recommenders/research/rails/similarities/mol/query_embeddings_fn.py b/recommendation_v4/generative_recommenders/research/rails/similarities/mol/query_embeddings_fn.py new file mode 100644 index 000000000..8fe28ee11 --- /dev/null +++ b/recommendation_v4/generative_recommenders/research/rails/similarities/mol/query_embeddings_fn.py @@ -0,0 +1,164 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +""" +Defines functions to generate query-side embeddings for MoL. + +Forked from bailuding/rails @ 664fdb9. +""" + +from typing import Callable, Dict, List, Optional, Tuple + +import torch +import torch.nn.functional as F +from generative_recommenders.research.rails.similarities.mol.embeddings_fn import ( + MoLEmbeddingsFn, +) + + +def init_mlp_xavier_weights_zero_bias(m) -> None: + if isinstance(m, torch.nn.Linear): + torch.nn.init.xavier_uniform_(m.weight) + if getattr(m, "bias", None) is not None: + m.bias.data.fill_(0.0) + + +class RecoMoLQueryEmbeddingsFn(MoLEmbeddingsFn): + """ + Generates P_Q query-side embeddings for MoL based on input embeddings and other + optional tensors for recommendation models. Tested for sequential retrieval + scenarios. + + The current implementation accesses user_ids associated with the query from + `user_ids' in kwargs. + """ + + def __init__( + self, + query_embedding_dim: int, + query_dot_product_groups: int, + dot_product_dimension: int, + dot_product_l2_norm: bool, + proj_fn: Callable[[int, int], torch.nn.Module], + eps: float, + uid_embedding_hash_sizes: Optional[List[int]] = None, + uid_dropout_rate: float = 0.0, + uid_embedding_level_dropout: bool = False, + ) -> None: + super().__init__() + self._uid_embedding_hash_sizes: List[int] = uid_embedding_hash_sizes or [] + self._query_emb_based_dot_product_groups: int = query_dot_product_groups - len( + self._uid_embedding_hash_sizes + ) + self._query_emb_proj_module: torch.nn.Module = proj_fn( + query_embedding_dim, + dot_product_dimension * self._query_emb_based_dot_product_groups, + ) + self._dot_product_dimension: int = dot_product_dimension + self._dot_product_l2_norm: bool = dot_product_l2_norm + if len(self._uid_embedding_hash_sizes) > 0: + for i, hash_size in enumerate(self._uid_embedding_hash_sizes): + setattr( + self, + f"_uid_embeddings_{i}", + torch.nn.Embedding( + hash_size + 1, dot_product_dimension, padding_idx=0 + ), + ) + self._uid_dropout_rate: float = uid_dropout_rate + self._uid_embedding_level_dropout: bool = uid_embedding_level_dropout + self._eps: float = eps + + def forward( + self, + input_embeddings: torch.Tensor, + **kwargs, + ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: + """ + Args: + input_embeddings: (B, query_embedding_dim,) x float where B is the batch size. + kwargs: str-keyed tensors. Implementation-specific. + + Returns: + Tuple of ( + (B, query_dot_product_groups, dot_product_embedding_dim) x float, + str-keyed aux_losses, + ). + """ + split_query_embeddings = self._query_emb_proj_module(input_embeddings).reshape( + ( + input_embeddings.size(0), + self._query_emb_based_dot_product_groups, + self._dot_product_dimension, + ) + ) + + aux_losses: Dict[str, torch.Tensor] = {} + + if len(self._uid_embedding_hash_sizes) > 0: + all_uid_embeddings = [] + for i, hash_size in enumerate(self._uid_embedding_hash_sizes): + # TODO: decouple this from MoLQueryEmbeddingFn. + uid_embeddings = getattr(self, f"_uid_embeddings_{i}")( + (kwargs["user_ids"] % hash_size) + 1 + ) + if self.training: + l2_norm = (uid_embeddings * uid_embeddings).sum(-1).mean() + if i == 0: + aux_losses["uid_embedding_l2_norm"] = l2_norm + else: + aux_losses["uid_embedding_l2_norm"] = ( + aux_losses["uid_embedding_l2_norm"] + l2_norm + ) + + if self._uid_dropout_rate > 0.0: + if self._uid_embedding_level_dropout: + # conditionally dropout the entire embedding. + if self.training: + uid_dropout_mask = ( + torch.rand( + uid_embeddings.size()[:-1], + device=uid_embeddings.device, + ) + > self._uid_dropout_rate + ) + uid_embeddings = ( + uid_embeddings + * uid_dropout_mask.unsqueeze(-1) + / (1.0 - self._uid_dropout_rate) + ) + else: + uid_embeddings = F.dropout( + uid_embeddings, + p=self._uid_dropout_rate, + training=self.training, + ) + all_uid_embeddings.append(uid_embeddings.unsqueeze(1)) + split_query_embeddings = torch.cat( + [split_query_embeddings] + all_uid_embeddings, dim=1 + ) + + if self._dot_product_l2_norm: + split_query_embeddings = split_query_embeddings / torch.clamp( + torch.linalg.norm( + split_query_embeddings, + ord=None, + dim=-1, + keepdim=True, + ), + min=self._eps, + ) + return split_query_embeddings, aux_losses diff --git a/recommendation_v4/generative_recommenders/research/rails/similarities/mol/similarity_fn.py b/recommendation_v4/generative_recommenders/research/rails/similarities/mol/similarity_fn.py new file mode 100644 index 000000000..34e4c4a23 --- /dev/null +++ b/recommendation_v4/generative_recommenders/research/rails/similarities/mol/similarity_fn.py @@ -0,0 +1,388 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +""" +Implements MoL (Mixture-of-Logits) with load balancing regularization loss, as discussed in: +- Revisiting Neural Retrieval on Accelerators (https://arxiv.org/abs/2306.04039, KDD'23). +- Retrieval with Learned Similarities (https://arxiv.org/abs/2407.15462). + +Forked from bailuding/rails @ 664fdb9. +""" + +from typing import Callable, Dict, Optional, Tuple + +import torch +import torch.nn.functional as F +from generative_recommenders.research.rails.similarities.module import SimilarityModule +from generative_recommenders.research.rails.similarities.mol.embeddings_fn import ( + MoLEmbeddingsFn, +) + + +@torch.compile(dynamic=True) +def _softmax_dropout_combiner_fn( + x: torch.Tensor, + y: torch.Tensor, + dropout_pr: float, + eps: float, + training: bool, +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Computes (_softmax_dropout_fn(x) * y).sum(-1). + """ + x = F.softmax(x, dim=-1) + if dropout_pr > 0.0: + x = F.dropout(x, p=dropout_pr, training=training) + x = x / torch.clamp(x.sum(-1, keepdims=True), min=eps) # pyre-ignore [19] + return x, (x * y).sum(-1) + + +@torch.compile +def _load_balancing_mi_loss_fn( + gating_prs: torch.Tensor, + eps: float, +) -> torch.Tensor: + """ + See Retrieval with Learned Similarities (RAILS, https://arxiv.org/abs/2407.15462) for discussions. + """ + B, X, E = gating_prs.size() + expert_util_prs = gating_prs.view(B * X, E).sum(0, keepdim=False) / (1.0 * B * X) + expert_util_entropy = -(expert_util_prs * torch.log(expert_util_prs + eps)).sum() + per_example_expert_entropy = -(gating_prs * torch.log(gating_prs + eps)).sum() / ( + 1.0 * B * X + ) + return -expert_util_entropy + per_example_expert_entropy + + +class SoftmaxDropoutCombiner(torch.nn.Module): + def __init__( + self, + dropout_rate: float, + eps: float, + ) -> None: + super().__init__() + + self._dropout_rate: float = dropout_rate + self._eps: float = eps + + def forward( + self, + gating_weights: torch.Tensor, + x: torch.Tensor, + ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: + gating_prs, combined_logits = _softmax_dropout_combiner_fn( + x=gating_weights, + y=x, + dropout_pr=self._dropout_rate, + eps=self._eps, + training=self.training, + ) + + aux_losses = {} + if self.training: + aux_losses["mi_loss"] = _load_balancing_mi_loss_fn( + gating_prs, eps=self._eps + ) + + return combined_logits, aux_losses + + +class MoLGatingFn(torch.nn.Module): + """ + Implements the gating function for MoL, used to compute pi_p(q, x) for a given (p, x) pair. + """ + + def __init__( + self, + num_logits: int, + query_embedding_dim: int, + item_embedding_dim: int, + query_only_partial_fn: Optional[Callable[[int, int], torch.nn.Module]], + item_only_partial_fn: Optional[Callable[[int, int], torch.nn.Module]], + qi_partial_fn: Optional[Callable[[int, int], torch.nn.Module]], + combination_type: str, + normalization_fn: Callable[[int], torch.nn.Module], + ) -> None: + super().__init__() + + self._query_only_partial_module: Optional[torch.nn.Module] = ( + query_only_partial_fn(query_embedding_dim, num_logits) + if query_only_partial_fn + else None + ) + self._item_only_partial_module: Optional[torch.nn.Module] = ( + item_only_partial_fn(item_embedding_dim, num_logits) + if item_only_partial_fn + else None + ) + self._qi_partial_module: Optional[torch.nn.Module] = ( + qi_partial_fn( + num_logits, + num_logits, + ) + if qi_partial_fn is not None + else None + ) + if ( + self._query_only_partial_module is None + and self._item_only_partial_module is None + and self._qi_partial_module is None + ): + raise ValueError( + "At least one of query_only_partial_fn, item_only_partial_fn, " + "and qi_partial_fn must not be None." + ) + self._num_logits: int = num_logits + self._combination_type: str = combination_type + self._normalization_fn: torch.nn.Module = normalization_fn(num_logits) + + def forward( + self, + logits: torch.Tensor, + query_embeddings: torch.Tensor, + item_embeddings: torch.Tensor, + ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: + """ + Args: + logits: (B, X, P_Q * P_X) x float; + query_embeddings: (B, D) x float; + item_embeddings: (1/B, X, D') x float; + + Returns: + (B, X) x float, Dict[str, Tensor] representing auxiliary losses. + """ + B, X, _ = logits.size() + # [B, 1, F], [1/B, X, F], [B, X, F] + query_partial_inputs, item_partial_inputs, qi_partial_inputs = None, None, None + if self._query_only_partial_module is not None: + query_partial_inputs = self._query_only_partial_module( + query_embeddings + ).unsqueeze(1) + if self._item_only_partial_module is not None: + item_partial_inputs = self._item_only_partial_module(item_embeddings) + if self._qi_partial_module is not None: + qi_partial_inputs = self._qi_partial_module(logits) + + if self._combination_type == "glu_silu": + gating_inputs = ( + query_partial_inputs * item_partial_inputs + qi_partial_inputs + ) + gating_weights = gating_inputs * F.sigmoid(gating_inputs) + elif self._combination_type == "glu_silu_ln": + gating_inputs = ( + query_partial_inputs * item_partial_inputs + qi_partial_inputs + ) + gating_weights = gating_inputs * F.sigmoid( + F.layer_norm(gating_inputs, normalized_shape=[self._num_logits]) + ) + elif self._combination_type == "none": + gating_inputs = query_partial_inputs + if gating_inputs is None: + gating_inputs = item_partial_inputs + elif item_partial_inputs is not None: + gating_inputs += item_partial_inputs + if gating_inputs is None: + gating_inputs = qi_partial_inputs + elif qi_partial_inputs is not None: + gating_inputs += qi_partial_inputs + gating_weights = gating_inputs + else: + raise ValueError(f"Unknown combination_type {self._combination_type}") + + return self._normalization_fn(gating_weights, logits) + + +class MoLSimilarity(SimilarityModule): + def __init__( + self, + query_embedding_dim: int, + item_embedding_dim: int, + dot_product_dimension: int, + query_dot_product_groups: int, + item_dot_product_groups: int, + temperature: float, + dot_product_l2_norm: bool, + query_embeddings_fn: MoLEmbeddingsFn, + item_embeddings_fn: Optional[MoLEmbeddingsFn], + gating_query_only_partial_fn: Optional[Callable[[int, int], torch.nn.Module]], + gating_item_only_partial_fn: Optional[Callable[[int, int], torch.nn.Module]], + gating_qi_partial_fn: Optional[Callable[[int], torch.nn.Module]], + gating_combination_type: str, + gating_normalization_fn: Callable[[int], torch.nn.Module], + eps: float, + apply_query_embeddings_fn: bool = True, + apply_item_embeddings_fn: bool = True, + autocast_bf16: bool = False, + ) -> None: + """ + Args: + apply_query_embeddings_fn: bool. If true, compute query_embeddings_fn + to input during forward(). Otherwise, we assume the caller will + invoke get_query_component_embeddings() separately before + calling forward(). + apply_item_embeddings_fn: bool. If true, compute item_embeddings_fn + to input during forward(). Otherwise, we assume the caller will + invoke get_item_component_embeddings() separately before + calling forward(). + """ + super().__init__() + + self._gating_fn: MoLGatingFn = MoLGatingFn( + num_logits=query_dot_product_groups * item_dot_product_groups, + query_embedding_dim=query_embedding_dim, + item_embedding_dim=item_embedding_dim, + query_only_partial_fn=gating_query_only_partial_fn, + item_only_partial_fn=gating_item_only_partial_fn, + qi_partial_fn=gating_qi_partial_fn, # pyre-ignore [6] + combination_type=gating_combination_type, + normalization_fn=gating_normalization_fn, + ) + self._query_embeddings_fn: MoLEmbeddingsFn = query_embeddings_fn + self._item_embeddings_fn: MoLEmbeddingsFn = ( # pyre-ignore [8] + item_embeddings_fn + ) + self._apply_query_embeddings_fn: bool = apply_query_embeddings_fn + self._apply_item_embeddings_fn: bool = apply_item_embeddings_fn + self._dot_product_l2_norm: bool = dot_product_l2_norm + self._query_dot_product_groups: int = query_dot_product_groups + self._item_dot_product_groups: int = item_dot_product_groups + self._dot_product_dimension: int = dot_product_dimension + self._temperature: float = temperature + self._eps: float = eps + self._autocast_bf16: bool = autocast_bf16 + + def get_query_component_embeddings( + self, + input_embeddings: torch.Tensor, + decoupled_inference: bool = False, + **kwargs, + ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: + """ + Args: + input_embeddings: (B, self._input_embedding_dim,) x float + or (B, P_Q, self._dot_product_dimension) x float. + decoupled_inference: bool. If true, the call represents an attempt to run + forward() in decoupled mode at inference time (e.g., to pre-compute + component-level query embeddings for filtering, etc.). We simulate + the logic in forward() in this case (e.g., if forward() doesn't apply + query_embeddings_fn, then this call won't either). + kwargs: additional implementation-specific arguments. + + Returns: + (B, query_dot_product_groups, dot_product_embedding_dim) x float. + """ + if decoupled_inference and not self._apply_query_embeddings_fn: + return input_embeddings, {} + return self._query_embeddings_fn(input_embeddings, **kwargs) + + def get_item_component_embeddings( + self, + input_embeddings: torch.Tensor, + decoupled_inference: bool = False, + **kwargs, + ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: + """ + Args: + input_embeddings: (..., self._input_embedding_dim,) x float + or (..., P_X, self._dot_product_dimension) x float. + decoupled_inference: bool. If true, the call represents an attempt to run + forward() in decoupled mode at inference time (e.g., to pre-compute + component-level item embeddings for filtering, etc.). We simulate + the logic in forward() in this case (e.g., if forward() doesn't apply + item_embeddings_fn, then this call won't either). + kwargs: additional implementation-specific arguments. + + Returns: + (..., item_dot_product_groups, dot_product_embedding_dim) x float. + """ + if decoupled_inference and not self._apply_item_embeddings_fn: + return input_embeddings, {} + + return self._item_embeddings_fn(input_embeddings, **kwargs) + + def forward( + self, + query_embeddings: torch.Tensor, + item_embeddings: torch.Tensor, + **kwargs, + ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: + """ + Args: + query_embeddings: (B, self._input_embedding_dim) x float or + (B, P_Q, self._dot_product_dimension) x float (when query_embeddings_fn + is applied externally). + item_embeddings: (1/B, X, self._item_embedding_dim) x float or + (1/B, X, P_X, self._dot_product_dimension) x float (when item_embeddings_fn + is applied externally). + kwargs: additional implementation-specific arguments. + + Returns: + (B, X) x float, Dict[str, Tensor] representing auxiliary losses. + """ + with torch.autocast( + enabled=self._autocast_bf16, dtype=torch.bfloat16, device_type="cuda" + ): + B = query_embeddings.size(0) + B_prime = item_embeddings.shape[0] # 1 or B + X = item_embeddings.shape[1] + + if self._apply_query_embeddings_fn: + ( + split_query_embeddings, + query_aux_losses, + ) = self.get_query_component_embeddings( + query_embeddings, + **kwargs, + ) + else: + split_query_embeddings, query_aux_losses = query_embeddings, {} + + if self._apply_item_embeddings_fn: + ( + split_item_embeddings, + item_aux_losses, + ) = self.get_item_component_embeddings( + input_embeddings=item_embeddings, + **kwargs, + ) + else: + split_item_embeddings, item_aux_losses = item_embeddings, {} + + if B_prime == 1: + logits = torch.einsum( + "bnd,xmd->bxnm", + split_query_embeddings, + split_item_embeddings.squeeze(0), + ).reshape( + B, X, self._query_dot_product_groups * self._item_dot_product_groups + ) + else: + logits = torch.einsum( + "bnd,bxmd->bxnm", split_query_embeddings, split_item_embeddings + ).reshape( + B, X, self._query_dot_product_groups * self._item_dot_product_groups + ) + + gated_outputs, gating_aux_losses = self._gating_fn( + logits=logits / self._temperature, # [B, X, L] + query_embeddings=query_embeddings, # [B, D] + item_embeddings=item_embeddings, # [1/B, X, D'] + ) + return gated_outputs, { + **gating_aux_losses, + **query_aux_losses, + **item_aux_losses, + } diff --git a/recommendation_v4/generative_recommenders/research/trainer/data_loader.py b/recommendation_v4/generative_recommenders/research/trainer/data_loader.py new file mode 100644 index 000000000..390b04bdb --- /dev/null +++ b/recommendation_v4/generative_recommenders/research/trainer/data_loader.py @@ -0,0 +1,57 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +import os +from typing import Optional, Tuple + +import gin +import torch + + +@gin.configurable +def create_data_loader( + dataset: torch.utils.data.Dataset, + batch_size: int, + world_size: int, + rank: int, + shuffle: bool, + prefetch_factor: int = 128, + num_workers: Optional[int] = os.cpu_count(), + drop_last: bool = False, +) -> Tuple[ + Optional[torch.utils.data.distributed.DistributedSampler[torch.utils.data.Dataset]], + torch.utils.data.DataLoader, +]: + if shuffle: + sampler = torch.utils.data.distributed.DistributedSampler( + dataset, + num_replicas=world_size, + rank=rank, + shuffle=True, + seed=0, + drop_last=drop_last, + ) + else: + sampler = None + data_loader = torch.utils.data.DataLoader( + dataset, + batch_size=batch_size, + # shuffle=True, cannot use with sampler + num_workers=num_workers or 0, + sampler=sampler, + prefetch_factor=prefetch_factor, + ) + return sampler, data_loader diff --git a/recommendation_v4/generative_recommenders/research/trainer/train.py b/recommendation_v4/generative_recommenders/research/trainer/train.py new file mode 100644 index 000000000..6d2da5be7 --- /dev/null +++ b/recommendation_v4/generative_recommenders/research/trainer/train.py @@ -0,0 +1,532 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +import logging +import os +import random +import time +from datetime import date +from typing import Dict, Optional + +import gin +import torch +import torch.distributed as dist +from generative_recommenders.research.data.eval import ( + _avg, + add_to_summary_writer, + eval_metrics_v2_from_tensors, + get_eval_state, +) +from generative_recommenders.research.data.reco_dataset import get_reco_dataset +from generative_recommenders.research.indexing.utils import get_top_k_module +from generative_recommenders.research.modeling.sequential.autoregressive_losses import ( + BCELoss, + InBatchNegativesSampler, + LocalNegativesSampler, +) +from generative_recommenders.research.modeling.sequential.embedding_modules import ( + EmbeddingModule, + LocalEmbeddingModule, +) +from generative_recommenders.research.modeling.sequential.encoder_utils import ( + get_sequential_encoder, +) +from generative_recommenders.research.modeling.sequential.features import ( + movielens_seq_features_from_row, +) +from generative_recommenders.research.modeling.sequential.input_features_preprocessors import ( + LearnablePositionalEmbeddingInputFeaturesPreprocessor, +) +from generative_recommenders.research.modeling.sequential.losses.sampled_softmax import ( + SampledSoftmaxLoss, +) +from generative_recommenders.research.modeling.sequential.output_postprocessors import ( + L2NormEmbeddingPostprocessor, + LayerNormEmbeddingPostprocessor, +) +from generative_recommenders.research.modeling.similarity_utils import ( + get_similarity_function, +) +from generative_recommenders.research.trainer.data_loader import create_data_loader +from torch.nn.parallel import DistributedDataParallel as DDP +from torch.utils.tensorboard import SummaryWriter + + +def setup(rank: int, world_size: int, master_port: int) -> None: + os.environ["MASTER_ADDR"] = "localhost" + os.environ["MASTER_PORT"] = str(master_port) + + # initialize the process group + dist.init_process_group("nccl", rank=rank, world_size=world_size) + + +def cleanup() -> None: + dist.destroy_process_group() + + +@gin.configurable +def get_weighted_loss( + main_loss: torch.Tensor, + aux_losses: Dict[str, torch.Tensor], + weights: Dict[str, float], +) -> torch.Tensor: + weighted_loss = main_loss + for key, weight in weights.items(): + cur_weighted_loss = aux_losses[key] * weight + weighted_loss = weighted_loss + cur_weighted_loss + return weighted_loss + + +@gin.configurable +def train_fn( + rank: int, + world_size: int, + master_port: int, + dataset_name: str = "ml-20m", + max_sequence_length: int = 200, + positional_sampling_ratio: float = 1.0, + local_batch_size: int = 128, + eval_batch_size: int = 128, + eval_user_max_batch_size: Optional[int] = None, + main_module: str = "SASRec", + main_module_bf16: bool = False, + dropout_rate: float = 0.2, + user_embedding_norm: str = "l2_norm", + sampling_strategy: str = "in-batch", + loss_module: str = "SampledSoftmaxLoss", + loss_weights: Optional[Dict[str, float]] = {}, + num_negatives: int = 1, + loss_activation_checkpoint: bool = False, + item_l2_norm: bool = False, + temperature: float = 0.05, + num_epochs: int = 101, + learning_rate: float = 1e-3, + num_warmup_steps: int = 0, + weight_decay: float = 1e-3, + top_k_method: str = "MIPSBruteForceTopK", + eval_interval: int = 100, + full_eval_every_n: int = 1, + save_ckpt_every_n: int = 1000, + partial_eval_num_iters: int = 32, + embedding_module_type: str = "local", + item_embedding_dim: int = 240, + interaction_module_type: str = "", + gr_output_length: int = 10, + l2_norm_eps: float = 1e-6, + enable_tf32: bool = False, + random_seed: int = 42, +) -> None: + # to enable more deterministic results. + random.seed(random_seed) + torch.backends.cuda.matmul.allow_tf32 = enable_tf32 + torch.backends.cudnn.allow_tf32 = enable_tf32 + logging.info(f"cuda.matmul.allow_tf32: {enable_tf32}") + logging.info(f"cudnn.allow_tf32: {enable_tf32}") + logging.info(f"Training model on rank {rank}.") + setup(rank, world_size, master_port) + + dataset = get_reco_dataset( + dataset_name=dataset_name, + max_sequence_length=max_sequence_length, + chronological=True, + positional_sampling_ratio=positional_sampling_ratio, + ) + + train_data_sampler, train_data_loader = create_data_loader( + dataset.train_dataset, + batch_size=local_batch_size, + world_size=world_size, + rank=rank, + shuffle=True, + drop_last=world_size > 1, + ) + eval_data_sampler, eval_data_loader = create_data_loader( + dataset.eval_dataset, + batch_size=eval_batch_size, + world_size=world_size, + rank=rank, + shuffle=True, # needed for partial eval + drop_last=world_size > 1, + ) + + model_debug_str = main_module + if embedding_module_type == "local": + embedding_module: EmbeddingModule = LocalEmbeddingModule( + num_items=dataset.max_item_id, + item_embedding_dim=item_embedding_dim, + ) + else: + raise ValueError(f"Unknown embedding_module_type {embedding_module_type}") + model_debug_str += f"-{embedding_module.debug_str()}" + + interaction_module, interaction_module_debug_str = get_similarity_function( + module_type=interaction_module_type, + query_embedding_dim=item_embedding_dim, + item_embedding_dim=item_embedding_dim, + ) + + assert user_embedding_norm == "l2_norm" or user_embedding_norm == "layer_norm", ( + f"Not implemented for {user_embedding_norm}" + ) + output_postproc_module = ( + L2NormEmbeddingPostprocessor( + embedding_dim=item_embedding_dim, + eps=1e-6, + ) + if user_embedding_norm == "l2_norm" + else LayerNormEmbeddingPostprocessor( + embedding_dim=item_embedding_dim, + eps=1e-6, + ) + ) + input_preproc_module = LearnablePositionalEmbeddingInputFeaturesPreprocessor( + max_sequence_len=dataset.max_sequence_length + gr_output_length + 1, + embedding_dim=item_embedding_dim, + dropout_rate=dropout_rate, + ) + + model = get_sequential_encoder( + module_type=main_module, + max_sequence_length=dataset.max_sequence_length, + max_output_length=gr_output_length + 1, + embedding_module=embedding_module, + interaction_module=interaction_module, + input_preproc_module=input_preproc_module, + output_postproc_module=output_postproc_module, + verbose=True, + ) + model_debug_str = model.debug_str() + + # loss + loss_debug_str = loss_module + if loss_module == "BCELoss": + loss_debug_str = loss_debug_str[:-4] + assert temperature == 1.0 + ar_loss = BCELoss(temperature=temperature, model=model) + elif loss_module == "SampledSoftmaxLoss": + loss_debug_str = "ssl" + if temperature != 1.0: + loss_debug_str += f"-t{temperature}" + ar_loss = SampledSoftmaxLoss( + num_to_sample=num_negatives, + softmax_temperature=temperature, + model=model, + activation_checkpoint=loss_activation_checkpoint, + ) + loss_debug_str += ( + f"-n{num_negatives}{'-ac' if loss_activation_checkpoint else ''}" + ) + else: + raise ValueError(f"Unrecognized loss module {loss_module}.") + + # sampling + if sampling_strategy == "in-batch": + negatives_sampler = InBatchNegativesSampler( + l2_norm=item_l2_norm, + l2_norm_eps=l2_norm_eps, + dedup_embeddings=True, + ) + sampling_debug_str = ( + f"in-batch{f'-l2-eps{l2_norm_eps}' if item_l2_norm else ''}-dedup" + ) + elif sampling_strategy == "local": + negatives_sampler = LocalNegativesSampler( + num_items=dataset.max_item_id, + item_emb=model._embedding_module._item_emb, + all_item_ids=dataset.all_item_ids, + l2_norm=item_l2_norm, + l2_norm_eps=l2_norm_eps, + ) + else: + raise ValueError(f"Unrecognized sampling strategy {sampling_strategy}.") + sampling_debug_str = negatives_sampler.debug_str() + + # Creates model and moves it to GPU with id rank + device = rank + if main_module_bf16: + model = model.to(torch.bfloat16) + model = model.to(device) + ar_loss = ar_loss.to(device) + negatives_sampler = negatives_sampler.to(device) + model = DDP(model, device_ids=[rank], broadcast_buffers=False) + + # TODO: wrap in create_optimizer. + opt = torch.optim.AdamW( + model.parameters(), + lr=learning_rate, + betas=(0.9, 0.98), + weight_decay=weight_decay, + ) + + date_str = date.today().strftime("%Y-%m-%d") + model_subfolder = f"{dataset_name}-l{max_sequence_length}" + model_desc = ( + f"{model_subfolder}" + + f"/{model_debug_str}_{interaction_module_debug_str}_{sampling_debug_str}_{loss_debug_str}" + + f"{f'-ddp{world_size}' if world_size > 1 else ''}-b{local_batch_size}-lr{learning_rate}-wu{num_warmup_steps}-wd{weight_decay}{'' if enable_tf32 else '-notf32'}-{date_str}" + ) + if full_eval_every_n > 1: + model_desc += f"-fe{full_eval_every_n}" + if positional_sampling_ratio is not None and positional_sampling_ratio < 1: + model_desc += f"-d{positional_sampling_ratio}" + # creates subfolders. + os.makedirs(f"./exps/{model_subfolder}", exist_ok=True) + os.makedirs(f"./ckpts/{model_subfolder}", exist_ok=True) + log_dir = f"./exps/{model_desc}" + if rank == 0: + writer = SummaryWriter(log_dir=log_dir) + logging.info(f"Rank {rank}: writing logs to {log_dir}") + else: + writer = None + logging.info(f"Rank {rank}: disabling summary writer") + + last_training_time = time.time() + torch.autograd.set_detect_anomaly(True) + + batch_id = 0 + epoch = 0 + for epoch in range(num_epochs): + if train_data_sampler is not None: + train_data_sampler.set_epoch(epoch) + if eval_data_sampler is not None: + eval_data_sampler.set_epoch(epoch) + model.train() + for row in iter(train_data_loader): + seq_features, target_ids, target_ratings = movielens_seq_features_from_row( + row, + device=device, + max_output_length=gr_output_length + 1, + ) + + if (batch_id % eval_interval) == 0: + model.eval() + + eval_state = get_eval_state( + model=model.module, + all_item_ids=dataset.all_item_ids, + negatives_sampler=negatives_sampler, + top_k_module_fn=lambda item_embeddings, item_ids: get_top_k_module( + top_k_method=top_k_method, + model=model.module, + item_embeddings=item_embeddings, + item_ids=item_ids, + ), + device=device, + float_dtype=torch.bfloat16 if main_module_bf16 else None, + ) + eval_dict = eval_metrics_v2_from_tensors( + eval_state, + model.module, + seq_features, + target_ids=target_ids, + target_ratings=target_ratings, + user_max_batch_size=eval_user_max_batch_size, + dtype=torch.bfloat16 if main_module_bf16 else None, + ) + add_to_summary_writer( + writer, batch_id, eval_dict, prefix="eval", world_size=world_size + ) + logging.info( + f"rank {rank}: batch-stat (eval): iter {batch_id} (epoch {epoch}): " + + f"NDCG@10 {_avg(eval_dict['ndcg@10'], world_size):.4f}, " + f"HR@10 {_avg(eval_dict['hr@10'], world_size):.4f}, " + f"HR@50 {_avg(eval_dict['hr@50'], world_size):.4f}, " + + f"MRR {_avg(eval_dict['mrr'], world_size):.4f} " + ) + model.train() + + # TODO: consider separating this out? + B, N = seq_features.past_ids.shape + seq_features.past_ids.scatter_( + dim=1, + index=seq_features.past_lengths.view(-1, 1), + src=target_ids.view(-1, 1), + ) + + opt.zero_grad() + input_embeddings = model.module.get_item_embeddings(seq_features.past_ids) + seq_embeddings = model( + past_lengths=seq_features.past_lengths, + past_ids=seq_features.past_ids, + past_embeddings=input_embeddings, + past_payloads=seq_features.past_payloads, + ) # [B, X] + + supervision_ids = seq_features.past_ids + + if sampling_strategy == "in-batch": + # get_item_embeddings currently assume 1-d tensor. + in_batch_ids = supervision_ids.view(-1) + negatives_sampler.process_batch( + ids=in_batch_ids, + presences=(in_batch_ids != 0), + embeddings=model.module.get_item_embeddings(in_batch_ids), + ) + else: + # pyre-fixme[16]: `InBatchNegativesSampler` has no attribute + # `_item_emb`. + negatives_sampler._item_emb = model.module._embedding_module._item_emb + + ar_mask = supervision_ids[:, 1:] != 0 + loss, aux_losses = ar_loss( + lengths=seq_features.past_lengths, # [B], + output_embeddings=seq_embeddings[:, :-1, :], # [B, N-1, D] + supervision_ids=supervision_ids[:, 1:], # [B, N-1] + supervision_embeddings=input_embeddings[:, 1:, :], # [B, N - 1, D] + supervision_weights=ar_mask.float(), + negatives_sampler=negatives_sampler, + **seq_features.past_payloads, + ) # [B, N] + + main_loss = loss.detach().clone() + loss = get_weighted_loss(loss, aux_losses, weights=loss_weights or {}) + + if rank == 0: + assert writer is not None + writer.add_scalar("losses/ar_loss", loss, batch_id) + writer.add_scalar("losses/main_loss", main_loss, batch_id) + + loss.backward() + + # Optional linear warmup. + if batch_id < num_warmup_steps: + lr_scalar = min(1.0, float(batch_id + 1) / num_warmup_steps) + for pg in opt.param_groups: + pg["lr"] = lr_scalar * learning_rate + lr = lr_scalar * learning_rate + else: + lr = learning_rate + + if (batch_id % eval_interval) == 0: + logging.info( + f" rank: {rank}, batch-stat (train): step {batch_id} " + f"(epoch {epoch} in {time.time() - last_training_time:.2f}s): {loss:.6f}" + ) + last_training_time = time.time() + if rank == 0: + assert writer is not None + writer.add_scalar("loss/train", loss, batch_id) + writer.add_scalar("lr", lr, batch_id) + + opt.step() + + batch_id += 1 + + def is_full_eval(epoch: int) -> bool: + return (epoch % full_eval_every_n) == 0 + + # eval per epoch + eval_dict_all = None + eval_start_time = time.time() + model.eval() + eval_state = get_eval_state( + model=model.module, + all_item_ids=dataset.all_item_ids, + negatives_sampler=negatives_sampler, + top_k_module_fn=lambda item_embeddings, item_ids: get_top_k_module( + top_k_method=top_k_method, + model=model.module, + item_embeddings=item_embeddings, + item_ids=item_ids, + ), + device=device, + float_dtype=torch.bfloat16 if main_module_bf16 else None, + ) + for eval_iter, row in enumerate(iter(eval_data_loader)): + seq_features, target_ids, target_ratings = movielens_seq_features_from_row( + row, device=device, max_output_length=gr_output_length + 1 + ) + eval_dict = eval_metrics_v2_from_tensors( + eval_state, + model.module, + seq_features, + target_ids=target_ids, + target_ratings=target_ratings, + user_max_batch_size=eval_user_max_batch_size, + dtype=torch.bfloat16 if main_module_bf16 else None, + ) + + if eval_dict_all is None: + eval_dict_all = {} + for k, v in eval_dict.items(): + eval_dict_all[k] = [] + + for k, v in eval_dict.items(): + eval_dict_all[k] = eval_dict_all[k] + [v] + del eval_dict + + if (eval_iter + 1 >= partial_eval_num_iters) and (not is_full_eval(epoch)): + logging.info( + f"Truncating epoch {epoch} eval to {eval_iter + 1} iters to save cost.." + ) + break + + assert eval_dict_all is not None + for k, v in eval_dict_all.items(): + eval_dict_all[k] = torch.cat(v, dim=-1) + + ndcg_10 = _avg(eval_dict_all["ndcg@10"], world_size=world_size) + ndcg_50 = _avg(eval_dict_all["ndcg@50"], world_size=world_size) + hr_10 = _avg(eval_dict_all["hr@10"], world_size=world_size) + hr_50 = _avg(eval_dict_all["hr@50"], world_size=world_size) + mrr = _avg(eval_dict_all["mrr"], world_size=world_size) + + add_to_summary_writer( + writer, + batch_id=epoch, + metrics=eval_dict_all, + prefix="eval_epoch", + world_size=world_size, + ) + if full_eval_every_n > 1 and is_full_eval(epoch): + add_to_summary_writer( + writer, + batch_id=epoch, + metrics=eval_dict_all, + prefix="eval_epoch_full", + world_size=world_size, + ) + if rank == 0 and epoch > 0 and (epoch % save_ckpt_every_n) == 0: + torch.save( + { + "epoch": epoch, + "model_state_dict": model.state_dict(), + "optimizer_state_dict": opt.state_dict(), + }, + f"./ckpts/{model_desc}_ep{epoch}", + ) + + logging.info( + f"rank {rank}: eval @ epoch {epoch} in {time.time() - eval_start_time:.2f}s: " + f"NDCG@10 {ndcg_10:.4f}, NDCG@50 {ndcg_50:.4f}, HR@10 {hr_10:.4f}, HR@50 {hr_50:.4f}, MRR {mrr:.4f}" + ) + last_training_time = time.time() + + if rank == 0: + if writer is not None: + writer.flush() + writer.close() + + torch.save( + { + "epoch": epoch, + "model_state_dict": model.state_dict(), + "optimizer_state_dict": opt.state_dict(), + }, + f"./ckpts/{model_desc}_ep{epoch}", + ) + + cleanup() diff --git a/recommendation_v4/generative_recommenders/tests/test_common.py b/recommendation_v4/generative_recommenders/tests/test_common.py new file mode 100644 index 000000000..be3823d67 --- /dev/null +++ b/recommendation_v4/generative_recommenders/tests/test_common.py @@ -0,0 +1,41 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +import unittest + +import torch +from generative_recommenders.common import switch_to_contiguous_if_needed + + +class SwitchToContiguousIfNeededTest(unittest.TestCase): + def test_torchscript_does_not_compile_fx_tracing_helper(self) -> None: + class ContiguousModule(torch.nn.Module): + def forward(self, x: torch.Tensor) -> torch.Tensor: + return switch_to_contiguous_if_needed(x) + + scripted = torch.jit.script(ContiguousModule()) + x = torch.arange(12).reshape(3, 4).transpose(0, 1) + + out = scripted(x) + + self.assertTrue(torch.equal(out, x)) + self.assertTrue(out.is_contiguous()) + + +if __name__ == "__main__": + unittest.main() diff --git a/recommendation_v4/main.py b/recommendation_v4/main.py new file mode 100644 index 000000000..445f25820 --- /dev/null +++ b/recommendation_v4/main.py @@ -0,0 +1,82 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +""" +Main entry point for model training. Please refer to README.md for usage instructions. +""" + +import logging +import os +from typing import List, Optional + +os.environ["TF_CPP_MIN_LOG_LEVEL"] = "1" # Hide excessive tensorflow debug messages +import sys + +import fbgemm_gpu # noqa: F401, E402 +import gin +import torch +import torch.multiprocessing as mp +from absl import app, flags +from generative_recommenders.research.trainer.train import train_fn + +logging.basicConfig(stream=sys.stdout, level=logging.INFO) + + +def delete_flags(FLAGS, keys_to_delete: List[str]) -> None: # pyre-ignore [2] + keys = [key for key in FLAGS._flags()] + for key in keys: + if key in keys_to_delete: + delattr(FLAGS, key) + + +delete_flags(flags.FLAGS, ["gin_config_file", "master_port"]) +flags.DEFINE_string("gin_config_file", None, "Path to the config file.") +flags.DEFINE_integer("master_port", 12355, "Master port.") +FLAGS = flags.FLAGS # pyre-ignore [5] + + +def mp_train_fn( + rank: int, + world_size: int, + master_port: int, + gin_config_file: Optional[str], +) -> None: + if gin_config_file is not None: + # Hack as absl doesn't support flag parsing inside multiprocessing. + logging.info(f"Rank {rank}: loading gin config from {gin_config_file}") + gin.parse_config_file(gin_config_file) + + train_fn(rank, world_size, master_port) + + +def _main(argv) -> None: # pyre-ignore [2] + world_size = torch.cuda.device_count() + + mp.set_start_method("forkserver") + mp.spawn( + mp_train_fn, + args=(world_size, FLAGS.master_port, FLAGS.gin_config_file), + nprocs=world_size, + join=True, + ) + + +def main() -> None: + app.run(_main) + + +if __name__ == "__main__": + main() diff --git a/recommendation_v4/preprocess_public_data.py b/recommendation_v4/preprocess_public_data.py new file mode 100644 index 000000000..927ccf4c6 --- /dev/null +++ b/recommendation_v4/preprocess_public_data.py @@ -0,0 +1,32 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +""" +Usage: mkdir -p tmp/ && python3 preprocess_public_data.py +""" + +from generative_recommenders.research.data.preprocessor import get_common_preprocessors + + +def main() -> None: + get_common_preprocessors()["ml-1m"].preprocess_rating() + get_common_preprocessors()["ml-20m"].preprocess_rating() + # get_common_preprocessors()["ml-1b"].preprocess_rating() + get_common_preprocessors()["amzn-books"].preprocess_rating() + + +if __name__ == "__main__": + main() diff --git a/recommendation_v4/requirements.txt b/recommendation_v4/requirements.txt new file mode 100644 index 000000000..023c22332 --- /dev/null +++ b/recommendation_v4/requirements.txt @@ -0,0 +1,7 @@ +torch>=2.6.0 +fbgemm_gpu>=1.1.0 +torchrec>=1.1.0 +gin_config>=0.5.0 +pandas>=2.2.0 +tensorboard>=2.19.0 +pybind11 diff --git a/recommendation_v4/run_fractal_expansion.py b/recommendation_v4/run_fractal_expansion.py new file mode 100644 index 000000000..308eadea2 --- /dev/null +++ b/recommendation_v4/run_fractal_expansion.py @@ -0,0 +1,588 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +""" +Run fractal expansion introduced in https://arxiv.org/abs/1901.08910. +Implementation adapted from the scripts used to generate MovieLens-1B +(https://grouplens.org/datasets/movielens/movielens-1b/). +""" + +# Generate a 3B dataset (takes around 50 minutes): +# python run_fractal_expansion.py --input-csv-file ~/data/ml-20m/ratings.csv --write-dataset True --output-prefix ~/data/ml-3b/ +# Generate a 13B dataset with 440M item size: +# python run_fractal_expansion.py --input-csv-file ~/data/ml-20m/ratings.csv --write-dataset True --output-prefix ~/data/ml-13b/ --num-row-multiplier 16 --num-col-multiplier 16384 --element-sample-rate 0.2 --block-sample-rate 0.05 +# Generate a 18B dataset with 1B item size: +# python run_fractal_expansion.py --input-csv-file ~/data/ml-20m/ratings.csv --write-dataset True --output-prefix ~/data/ml-18b/ --num-row-multiplier 20 --num-col-multiplier 36864 --element-sample-rate 0.08 --block-sample-rate 0.05 + +import csv +import linecache +import logging +import os +import pickle +from dataclasses import dataclass + +import click +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +import scipy.linalg +import skimage.transform as transform +from scipy import sparse +from scipy.sparse import linalg +from sklearn.utils import shuffle +from tqdm import tqdm + + +logging.basicConfig() +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + + +@dataclass +class SparseMatrixMetadata: + num_interactions: int = 0 + num_rows: int = 0 + num_cols: int = 0 + + +def _dropout_sparse_coo_matrix( + sparse_matrix, rate, min_dropout_rate=0.005, max_dropout_rate=0.999 +): + assert min_dropout_rate <= max_dropout_rate + sampling_rate = 1.0 - rate + + sampled_fraction = min( + max(sampling_rate, 1.0 - max_dropout_rate), 1.0 - min_dropout_rate + ) + if sampled_fraction != sampling_rate: + logger.warning( + f"Desired sampling rate {sampling_rate} clipped to {sampled_fraction}." + ) + num_sampled = min( + max(int(sparse_matrix.nnz * sampled_fraction), 1), sparse_matrix.nnz + ) + sampled_indices = np.random.choice( + sparse_matrix.nnz, size=num_sampled, replace=False + ) + return sparse.coo_matrix( + ( + sparse_matrix.data[sampled_indices], + (sparse_matrix.row[sampled_indices], sparse_matrix.col[sampled_indices]), + ), + shape=sparse_matrix.shape, + ) + + +def shuffle_sparse_matrix( + sparse_matrix, dropout_rate=0.0, min_dropout_rate=0.005, max_dropout_rate=0.999 +): + """ + Shuffle sparse matrix encoded as a SciPy csr matrix. + """ + + assert dropout_rate >= 0.0 and dropout_rate <= 1.0 + (num_rows, num_cols) = sparse_matrix.shape + shuffled_rows = shuffle(np.arange(num_rows)) + shuffled_cols = shuffle(np.arange(num_cols)) + sparse_matrix = _dropout_sparse_coo_matrix( + sparse_matrix, dropout_rate, min_dropout_rate, max_dropout_rate + ) + new_row = np.take(shuffled_rows, sparse_matrix.row) + new_col = np.take(shuffled_cols, sparse_matrix.col) + return sparse.csr_matrix( + (sparse_matrix.data, (new_row, new_col)), shape=(num_rows, num_cols) + ) + + +def graph_reduce(usv, num_rows, num_cols): + """Apply algorithm 2 in https://arxiv.org/pdf/1901.08910.pdf.""" + + def _closest_column_orthogonal_matrix(matrix): + return np.matmul( + matrix, np.linalg.inv(scipy.linalg.sqrtm(np.matmul(matrix.T, matrix))) + ) + + u, s, v = usv + k = min(num_rows, num_cols) + u_random_proj = transform.resize(u[:, :k], (num_rows, k)) + v_random_proj = transform.resize(v[:k, :], (k, num_cols)) + u_random_proj_orth = _closest_column_orthogonal_matrix(u_random_proj) + v_random_proj_orth = _closest_column_orthogonal_matrix(v_random_proj.T).T + return np.matmul(u_random_proj_orth, np.matmul(np.diag(s[:k]), v_random_proj_orth)) + + +def rescale(matrix, rescale_w_abs=False, element_sample_rate=1.0): + """Rescale all values of the matrix into [0, 1].""" + if rescale_w_abs: + abs_matrix = np.abs(matrix.copy()) + out = abs_matrix / abs_matrix.max() + else: + out = (matrix - matrix.min()) / (matrix.max() - matrix.min()) + assert out.min() >= 0 and out.max() <= 1 + return out * element_sample_rate + + +def _compute_row_block( + i, left_matrix, right_matrix, block_sample_rate, indices_out_path, remove_empty_rows +): + """Compute row block of expansion for row i of the left_matrix.""" + + kron_blocks = [] + num_rows = 0 + num_removed_rows = 0 + num_interactions = 0 + + for j in range(left_matrix.shape[1]): + if np.random.random() <= block_sample_rate: + dropout_rate = 1.0 - left_matrix[i, j] + kron_block = shuffle_sparse_matrix(right_matrix, dropout_rate).tocsr() + num_interactions += kron_block.nnz + kron_blocks.append(kron_block) + logger.info(f"Kronecker block ({i}, {j}) processed.") + else: + kron_blocks.append(sparse.csr_matrix(right_matrix.shape)) + logger.info(f"Kronecker block ({i}, {j}) skipped.") + + rows_to_write = sparse.hstack(kron_blocks).tocsr() + logger.info("Writing dataset row by row.") + + # Write Kronecker product line per line. + filepath = f"{indices_out_path}_{i}.csv" + os.makedirs(os.path.dirname(filepath), exist_ok=True) + with open(filepath, "w", newline="") as file: + writer = csv.writer(file) + for k in range(right_matrix.shape[0]): + items_to_write = rows_to_write.getrow(k).indices + ratings_to_write = rows_to_write.getrow(k).data + num = items_to_write.shape[0] + if remove_empty_rows and (not num): + logger.info(f"Removed empty output row {i * left_matrix.shape[0] + k}.") + num_removed_rows += 1 + continue + num_rows += 1 + writer.writerow( + [ + i * right_matrix.shape[0] + k, + ",".join([str(x) for x in items_to_write]), + ",".join([str(x) for x in ratings_to_write]), + ] + ) + if k % 100000 == 0: + logger.info(f"Done producing data set row {k}.") + + num_cols = rows_to_write.shape[1] + metadata = SparseMatrixMetadata( + num_interactions=num_interactions, num_rows=num_rows, num_cols=num_cols + ) + logger.info( + f"Done with left matrix row {i}, {num_interactions} interactions written in shard, {num_removed_rows} rows removed in shard." + ) + return (num_removed_rows, metadata) + + +def visualize_samples( + right_matrix, + visualize_num_samples, + expanded_file_name, + output_prefix, +): + # Note: only the rows of the first Kronecker block are visualized. + logger.info("visualize dataset row by row.") + fig, axs = plt.subplots(1, 2, figsize=(12, 5)) + axs[0].set_title("Original data Histogram") + axs[0].set_xlabel("Value") + axs[0].set_ylabel("Frequency") + axs[1].set_title("Expended Row Histogram") + axs[1].set_xlabel("Value") + axs[1].set_ylabel("Frequency") + for k in range(visualize_num_samples): + original_row = right_matrix.getrow(k).data + line = linecache.getline(expanded_file_name, k + 1) + reader = csv.reader([line]) + parsed_line = next(reader) + expended_row = eval(parsed_line[2]) + original_hist_counts, original_bin_edges = np.histogram(original_row, bins=9) + expended_hist_counts, expended_bin_edges = np.histogram(expended_row, bins=9) + axs[0].plot(original_bin_edges[:-1], original_hist_counts, alpha=0.2) + axs[1].plot(expended_bin_edges[:-1], expended_hist_counts, alpha=0.2) + axs[0].fill_between(original_bin_edges[:-1], original_hist_counts, alpha=0.2) + axs[1].fill_between(expended_bin_edges[:-1], expended_hist_counts, alpha=0.2) + plt.tight_layout() + plt.savefig(f"{output_prefix}_sample_distribution.png") + logger.info("Sample visualization finished.") + + +def build_randomized_kronecker( + left_matrix, + right_matrix, + block_sample_rate, + indices_out_path, + metadata_out_path=None, + remove_empty_rows=True, +): + """Compute randomized Kronecker product and dump it on the fly based on https://arxiv.org/pdf/1901.08910.pdf.""" + logger.info(f"Writing item sequences to pickle files {metadata_out_path}.") + + num_rows = 0 + num_removed_rows = 0 + num_cols = left_matrix.shape[1] * right_matrix.shape[1] + num_interactions = 0 + + filepath = f"{indices_out_path}_users.csv" + os.makedirs(os.path.dirname(filepath), exist_ok=True) + with open(filepath, "w", newline="") as file: + writer = csv.writer(file) + for i in tqdm(range(left_matrix.shape[0])): + (shard_num_removed_rows, shard_metadata) = _compute_row_block( + i, + left_matrix, + right_matrix, + block_sample_rate, + indices_out_path, + remove_empty_rows, + ) + writer.writerow([i, shard_metadata.num_rows]) + file.flush() + num_rows += shard_metadata.num_rows + num_removed_rows += shard_num_removed_rows + num_interactions += shard_metadata.num_interactions + + logger.info(f"{num_interactions / num_rows} average sequence length") + logger.info(f"{num_interactions} total interactions written.") + logger.info(f"{num_removed_rows} total rows removed.") + + metadata = SparseMatrixMetadata( + num_interactions=num_interactions, num_rows=num_rows, num_cols=num_cols + ) + if metadata_out_path is not None: + logger.info(f"Writing metadata file to {metadata_out_path}") + with open(metadata_out_path, "wb") as output_file: + pickle.dump(metadata, output_file) + return metadata + + +def _preprocess_movie_lens(ratings_df, binary=False): + """ + Filters out users with less than three distinct timestamps. + """ + + def _create_index(df, colname): + value_set = sorted(set(df[colname].values)) + num_unique = len(value_set) + return dict(zip(value_set, range(num_unique))) + + if not binary: + ratings_df["data"] = ratings_df["rating"] + else: + ratings_df["data"] = 1.0 + ratings_df["binary_data"] = 1.0 + num_timestamps = ratings_df[["userId", "timestamp"]].groupby("userId").nunique() + ratings_df["numberOfTimestamps"] = ratings_df["userId"].apply( + lambda x: num_timestamps["timestamp"][x] + ) + ratings_df = ratings_df[ratings_df["numberOfTimestamps"] > 2] + user_id_to_user_idx = _create_index(ratings_df, "userId") + item_id_to_item_idx = _create_index(ratings_df, "movieId") + ratings_df["row"] = ratings_df["userId"].apply(lambda x: user_id_to_user_idx[x]) + ratings_df["col"] = ratings_df["movieId"].apply(lambda x: item_id_to_item_idx[x]) + return ratings_df + + +def normalize(matrix): + norm_matrix = matrix.copy() + if isinstance(norm_matrix, np.ndarray): + norm_matrix -= norm_matrix.mean() + else: + norm_matrix.data -= norm_matrix.mean() + max_val = norm_matrix.max() + min_val = norm_matrix.min() + if isinstance(norm_matrix, np.ndarray): + norm_matrix /= max(abs(max_val), abs(min_val)) + else: + norm_matrix.data /= max(abs(max_val), abs(min_val)) + return norm_matrix + + +def plot_distribution(user_wise_sum, item_wise_sum, s, title_prefix, normalized=False): + y_label = "rating sums" if normalized else "number of ratings" + fig, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize=(15, 5)) + ax1.loglog( + np.arange(len(user_wise_sum)) + 1, + np.sort(user_wise_sum)[::-1], + linestyle="-", + color="blue", + marker="", + ) + ax1.set_title(f"{title_prefix} matrix user-wise rating sums") + ax1.set_xlabel("User rank") + ax1.set_ylabel(y_label) + ax1.grid(True) + ax2.loglog( + np.arange(len(item_wise_sum)) + 1, + np.sort(item_wise_sum)[::-1], + linestyle="-", + color="green", + marker="", + ) + ax2.set_title(f"{title_prefix} matrix item-wise rating sums") + ax2.set_xlabel("Item rank") + ax2.set_ylabel(y_label) + ax2.grid(True) + ax3.loglog( + np.arange(len(s)) + 1, np.sort(s)[::-1], linestyle="-", color="red", marker="" + ) + ax3.set_title(f"{title_prefix} matrix singular values") + ax3.set_xlabel("Singular value Rank") + ax3.set_ylabel("Magnitude") + ax3.grid(True) + plt.tight_layout() + plt.savefig(f"{title_prefix}_distribution.png") + + +def visualize_distribution(mat, reduced_mat, s, reduced_s, normalized=False, title=""): + user_wise_sum = np.asarray(mat.sum(axis=1)).flatten() + item_wise_sum = np.asarray(mat.sum(axis=0)).flatten() + assert len(user_wise_sum) == mat.shape[0] + assert len(item_wise_sum) == mat.shape[1] + plot_distribution( + user_wise_sum, + item_wise_sum, + s, + title_prefix=f"{title}_Original", + normalized=normalized, + ) + + reduced_user_wise_sum = np.asarray(reduced_mat.sum(axis=1)).flatten() + reduced_item_wise_sum = np.asarray(reduced_mat.sum(axis=0)).flatten() + assert len(reduced_user_wise_sum) == reduced_mat.shape[0] + assert len(reduced_item_wise_sum) == reduced_mat.shape[1] + plot_distribution( + reduced_user_wise_sum, + reduced_item_wise_sum, + reduced_s, + title_prefix=f"{title}_Reduced", + normalized=normalized, + ) + + expanded_s = np.einsum("i,j->ij", reduced_s, s).flatten() + expanded_user_wise_sum = np.einsum("ij,k->ik", reduced_mat, user_wise_sum).flatten() + expanded_item_wise_sum = np.einsum("ij,k->jk", reduced_mat, item_wise_sum).flatten() + assert len(expanded_user_wise_sum) == reduced_mat.shape[0] * mat.shape[0] + assert len(expanded_item_wise_sum) == reduced_mat.shape[1] * mat.shape[1] + plot_distribution( + expanded_user_wise_sum, + expanded_item_wise_sum, + expanded_s, + title_prefix=f"{title}_Expanded", + normalized=normalized, + ) + + +def expand_dataset( + ratings_matrix, + binary_ratings_matrix, + num_users, + num_items, + reduced_num_rows, + reduced_num_cols, + rescale_w_abs, + element_sample_rate, + block_sample_rate, + visualize, + write_dataset, + output_prefix, +): + k = min(reduced_num_rows, reduced_num_cols) + norm_rating_matrix = normalize(ratings_matrix) + (u, s, v) = linalg.svds( + norm_rating_matrix, k=k, maxiter=None, return_singular_vectors=True + ) + + logger.info( + f"Creating reduced rating matrix (size {reduced_num_rows}, {reduced_num_cols})" + ) + reduced_matrix = graph_reduce((u, s, v), reduced_num_rows, reduced_num_cols) + norm_reduced_matrix = normalize(reduced_matrix) + (_, s_reduce, _) = linalg.svds( + norm_reduced_matrix, k=k - 1, maxiter=None, return_singular_vectors=True + ) + reduced_matrix = rescale( + reduced_matrix, + rescale_w_abs=rescale_w_abs, + element_sample_rate=element_sample_rate, + ) + logger.info(f"largest singular value of the reduced matrix is {s_reduce[-1]}") + logger.info( + f"Sampling rate mean is {reduced_matrix.mean()}, var is {reduced_matrix.var()}, min is {reduced_matrix.min()}, max is {reduced_matrix.max()}" + ) + samples = reduced_matrix.sum() * ratings_matrix.nnz * block_sample_rate + logger.info( + f"Expected number of synthetic samples: {samples}, sparsity is {samples / (num_users * num_items * reduced_num_rows * reduced_num_cols)}, average seqlen is {samples / (num_users * reduced_num_rows)}" + ) + + if visualize: + s = linalg.svds( + norm_rating_matrix, k=20 * k, maxiter=None, return_singular_vectors=False + ) + visualize_distribution( + norm_rating_matrix, + norm_reduced_matrix, + s, + s_reduce, + normalized=True, + title="Normalized", + ) + visualize_distribution( + binary_ratings_matrix, + reduced_matrix, + s, + s_reduce, + normalized=False, + title="Binary", + ) + if write_dataset: + output_file = ( + output_prefix + str(reduced_num_rows) + "x" + str(reduced_num_cols) + ) + output_file_metadata = None + + logger.info(f"Creating synthetic dataset and dumping to {output_file}.") + build_randomized_kronecker( + left_matrix=reduced_matrix, + right_matrix=ratings_matrix.tocoo(), + block_sample_rate=block_sample_rate, + indices_out_path=output_file, + metadata_out_path=output_file_metadata, + ) + + +@click.command() +@click.option( + "--random-seed", + type=int, + default=0, +) +@click.option( + "--input-csv-file", + type=str, + default="ratings.csv", +) +@click.option( + "--output-prefix", + type=str, + default="", +) +@click.option( + "--num-row-multiplier", + type=int, + default=16, +) +@click.option( + "--num-col-multiplier", + type=int, + default=32, +) +@click.option( + "--element-sample-rate", + type=float, + default=1.0, +) +@click.option( + "--block-sample-rate", + type=float, + default=1.0, +) +@click.option( + "--visualize", + type=bool, + default=False, +) +@click.option( + "--write-dataset", + type=bool, + default=False, +) +@click.option( + "--visualize-num-samples", + type=int, + default=0, +) +def main( + random_seed: int, + input_csv_file: str, + output_prefix: str, + num_row_multiplier: int, + num_col_multiplier: int, + element_sample_rate: float, + block_sample_rate: float, + visualize: bool, + write_dataset: bool, + visualize_num_samples: int, +): + np.random.seed(random_seed) + + logger.info(f"Loading and preprocessing MovieLens-20m from {input_csv_file}") + with open(input_csv_file, "r") as infile: + ratings_df = pd.read_csv(infile, sep=",", header=0) + ratings_df = _preprocess_movie_lens(ratings_df, binary=False) + num_ratings = len(ratings_df) + num_users = len(set(ratings_df["row"].values)) + num_items = len(set(ratings_df["col"].values)) + logger.info( + f"number of ratings of input dataset is {num_ratings}, number of users is {num_users}, number of items is {num_items}, sparsity is {num_ratings / (num_users * num_items)}, average seqlen is {num_ratings / num_users}" + ) + + ratings_matrix = sparse.csr_matrix( + ( + ratings_df["data"].values, + (ratings_df["row"].values, ratings_df["col"].values), + ), + shape=(num_users, num_items), + ) + binary_ratings_matrix = sparse.csr_matrix( + ( + ratings_df["binary_data"].values, + (ratings_df["row"].values, ratings_df["col"].values), + ), + shape=(num_users, num_items), + ) + if write_dataset or visualize: + expand_dataset( + ratings_matrix=ratings_matrix, + binary_ratings_matrix=binary_ratings_matrix, + num_users=num_users, + num_items=num_items, + reduced_num_rows=num_row_multiplier, + reduced_num_cols=num_col_multiplier, + rescale_w_abs=False, + element_sample_rate=element_sample_rate, + block_sample_rate=block_sample_rate, + visualize=visualize, + write_dataset=write_dataset, + output_prefix=output_prefix, + ) + if visualize_num_samples > 0: + logger.info(f"Visualizing {visualize_num_samples} samples.") + visualize_samples( + right_matrix=ratings_matrix.tocoo(), + visualize_num_samples=visualize_num_samples, + expanded_file_name=f"{output_prefix}{num_row_multiplier}x{num_col_multiplier}_0.csv", + output_prefix="Sample_Histogram", + ) + + +if __name__ == "__main__": + main() diff --git a/recommendation_v4/scripts/launch_smoke_8gpu.sh b/recommendation_v4/scripts/launch_smoke_8gpu.sh new file mode 100755 index 000000000..92daa6ef8 --- /dev/null +++ b/recommendation_v4/scripts/launch_smoke_8gpu.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# 8-GPU yambda-5b run. Resolves the package root from this script's location, +# so it works from any container mount point. Dataset path is in the gin file +# (generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin). +set -uo pipefail + +REPO_ROOT=$(cd "$(dirname "$0")/.." && pwd) +cd "$REPO_ROOT" + +LOG=${LOG:-/apps/chcai/yambda_5b_8gpu.log} +echo "[$(date)] REPO_ROOT=$REPO_ROOT" | tee "$LOG" + +# polars-u64-idx (NOT stock polars) — yambda parquet's flat-explode overruns +# 32-bit row index. Reserved node has no outbound DNS, so we install from a +# pre-staged tarball under /apps/chcai/. Override PIP_LOCAL_TGZ for other hosts. +PIP_LOCAL_TGZ=${PIP_LOCAL_TGZ:-/apps/chcai/pip_local_yambda.tgz} +PIP_LOCAL_DIR=${PIP_LOCAL_DIR:-/tmp/pip_local} +if [ ! -f "$PIP_LOCAL_DIR/lib/python3.12/site-packages/polars/__init__.py" ]; then + rm -rf "$PIP_LOCAL_DIR" + mkdir -p "$PIP_LOCAL_DIR" && tar xzf "$PIP_LOCAL_TGZ" -C "$(dirname "$PIP_LOCAL_DIR")" 2>&1 | tail -3 | tee -a "$LOG" +fi + +export PYTHONPATH="$PIP_LOCAL_DIR/lib/python3.12/site-packages:$REPO_ROOT:${PYTHONPATH:-}" +export HOME=${HOME:-/tmp} +echo "[$(date)] PYTHONPATH=$PYTHONPATH" | tee -a "$LOG" +python -c "import torch, fbgemm_gpu, torchrec, polars, xxhash, gin; print('imports OK,', torch.__version__, torch.cuda.device_count(),'gpus')" 2>&1 | tee -a "$LOG" + +export HIP_VISIBLE_DEVICES=${HIP_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7} +export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7} +export WORLD_SIZE=$(python -c "import torch; print(torch.cuda.device_count())") +# AMD/ROCm: Triton HSTU kernel hits PassManager errors on some shapes; force +# PYTORCH backend. On CUDA, unset this to default to TRITON for ~3-5x speedup. +export HSTU_HAMMER_KERNEL=${HSTU_HAMMER_KERNEL:-PYTORCH} +export PYTORCH_CUDA_ALLOC_CONF=${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True} +echo "[$(date)] launching train_ranker with WORLD_SIZE=$WORLD_SIZE" | tee -a "$LOG" + +python -m generative_recommenders.dlrm_v3.train.train_ranker \ + --dataset yambda-5b --mode train-eval 2>&1 | tee -a "$LOG" diff --git a/recommendation_v4/setup.py b/recommendation_v4/setup.py new file mode 100644 index 000000000..bdab528f4 --- /dev/null +++ b/recommendation_v4/setup.py @@ -0,0 +1,41 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +from setuptools import find_packages, setup + +setup( + name="generative_recommenders", + version="0.1.0", + description="Library for generative recommendation algorithms.", + packages=find_packages(exclude=["configs"]), + python_requires=">=3.10", + install_requires=[ + "torch>=2.6.0", + "fbgemm_gpu>=1.1.0", + "torchrec>=1.1.0", + "gin_config>=0.5.0", + "pandas>=2.2.0", + "tensorboard>=2.19.0", + "pybind11", + "click", + "pandas", + "matplotlib", + ], + long_description=open("README.md").read(), + long_description_content_type="text/markdown", + url="https://github.com/meta-recsys/generative-recommenders", + license="Apache-2.0", +) From 9b56d4f35dec4a7799e7cbf0d1ec9c927506c732 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Fri, 29 May 2026 17:54:39 -0500 Subject: [PATCH 002/127] Enable Triton HSTU kernels on AMD/ROCm (gfx950 MI350X) Four fixes unlocking the HSTU_HAMMER_KERNEL=TRITON path on MI350X: 1. triton_hstu_attention.py _should_enable_tma(): add HIP early-out. torch.cuda.get_device_capability() on gfx950 returns (9, 5) which would pass the major==9 Hopper check and trick the kernel into the TMA path, producing kernels that don't compile on ROCm. 2. triton_hstu_attention.py _get_fw_configs(): hoist the USE_TLX/NUM_BUFFERS/ NUM_MMA_WARPS_PER_GROUP/NUM_MMA_GROUPS defaults loop out of the CUDA-only else: branch. The _hstu_attn_fwd signature requires these constexprs regardless of backend; missing them on HIP triggered TypeError: dynamic_func() missing N required positional arguments at autotune. Also gate the H100 TLX configs append on `not torch.version.hip`. 3. triton_jagged_tensors.py concat/split dispatch: route AMD/ROCm through *_2D_jagged_multirow instead of the basic _concat_2D_jagged / _split_2D_jagged kernels. The basic kernels fail PassManager::run at make_ttgir (TritonAMDGPUCanonicalizePointers pass) on ROCm; multirow compiles fine. NVIDIA non-Blackwell paths (H100/A100) are unchanged. 4. triton_jagged_tensors.py _Concat2DJaggedFunction.backward: replace the raw _split_2D_jagged[grid] call with _triton_split_2D_jagged_internal so the backward pass benefits from the same AMD multirow routing as the forward. Verified end-to-end on 8x MI350X: yambda-5b bs=32 seq=4k at 782 global_sps vs PYTORCH backend 547 sps -- 1.43x throughput, 75% peak VRAM vs 92%. Co-Authored-By: Claude Opus 4.7 --- .../ops/triton/triton_hstu_attention.py | 47 +++++++++++++---- .../ops/triton/triton_jagged_tensors.py | 51 +++++++++++++------ 2 files changed, 74 insertions(+), 24 deletions(-) diff --git a/recommendation_v4/generative_recommenders/ops/triton/triton_hstu_attention.py b/recommendation_v4/generative_recommenders/ops/triton/triton_hstu_attention.py index ac667e139..03a1f8f67 100644 --- a/recommendation_v4/generative_recommenders/ops/triton/triton_hstu_attention.py +++ b/recommendation_v4/generative_recommenders/ops/triton/triton_hstu_attention.py @@ -129,6 +129,19 @@ def _should_enable_tma() -> bool: return False if not torch.cuda.is_available(): return False + # NVIDIA-only gate: TMA (Tensor Memory Accelerator) is Hopper-specific + # hardware. On ROCm/HIP, `torch.cuda.get_device_capability()` mirrors the + # gfx name into a major.minor tuple — gfx950 (MI350X) returns (9, 5), which + # would otherwise pass the `device_capability == 9` check below and trick + # the kernel into taking the TMA path. The TMA path uses + # `triton.tools.tensor_descriptor.TensorDescriptor` and `TensorDescriptor.load` + # which lower to PTX `cp.async.bulk.tensor.*`; on AMD this either fails to + # compile or produces a kernel with mismatched reduction-dim shapes for + # `tl.dot(silu, v)` in `_hstu_attn_fwd_one_block` (see WARNING in + # `_hstu_attn_fwd` for the cascade). Bail out early on HIP so the + # non-TMA path is selected and AMD gets a working kernel. + if torch.version.hip: + return False try: device_capability = torch.cuda.get_device_capability()[0] except (RuntimeError, AssertionError): @@ -335,15 +348,31 @@ def _get_fw_configs() -> List[triton.Config]: # noqa: C901 ), ] - # Add 'USE_TLX' : False, 'NUM_BUFFERS': 1, 'NUM_MMA_WARPS_PER_GROUP': 1, 'NUM_MMA_GROUPS': 1 to non-TLX configs - for config in configs: - if not config.kwargs.get("USE_TLX", False): - config.kwargs["USE_TLX"] = False - config.kwargs["NUM_BUFFERS"] = 1 - config.kwargs["NUM_MMA_WARPS_PER_GROUP"] = 1 - config.kwargs["NUM_MMA_GROUPS"] = 1 - - # Add TLX configs if TLX is available + # The `_hstu_attn_fwd` kernel signature unconditionally declares the four + # constexprs `USE_TLX`, `NUM_BUFFERS`, `NUM_MMA_WARPS_PER_GROUP`, + # `NUM_MMA_GROUPS` (introduced for the Hopper TLX warp-specialized variant). + # Triton requires every constexpr be bound at autotune time; missing any one + # of them triggers `TypeError: dynamic_func() missing N required positional + # arguments` during kernel dispatch. This loop populates the non-TLX defaults + # so the kernel call site doesn't have to know about TLX at all. + # + # IMPORTANT: this loop must apply to BOTH the HIP branch and the CUDA branch + # above. It used to live inside the CUDA `else:` block which meant HIP + # configs reached `_hstu_attn_fwd[grid](...)` without these defaults and + # crashed at dispatch. Keep this hoisted (outside the if/else) when + # editing — see commit message for the symptom. + for config in configs: + if not config.kwargs.get("USE_TLX", False): + config.kwargs["USE_TLX"] = False + config.kwargs["NUM_BUFFERS"] = 1 + config.kwargs["NUM_MMA_WARPS_PER_GROUP"] = 1 + config.kwargs["NUM_MMA_GROUPS"] = 1 + + # TLX (Triton Language Extension) warp-specialized configs are Hopper-only. + # Guard with `not torch.version.hip` so AMD never sees them — the TLX code + # path inside `_hstu_attn_fwd` calls `tlx.async_descriptor_load(...)` which + # requires real TMA tensor descriptors and only compiles on CUDA. + if not torch.version.hip: if HAS_TLX: try: device_capability = torch.cuda.get_device_capability()[0] diff --git a/recommendation_v4/generative_recommenders/ops/triton/triton_jagged_tensors.py b/recommendation_v4/generative_recommenders/ops/triton/triton_jagged_tensors.py index 3488e308a..2c3728c0a 100644 --- a/recommendation_v4/generative_recommenders/ops/triton/triton_jagged_tensors.py +++ b/recommendation_v4/generative_recommenders/ops/triton/triton_jagged_tensors.py @@ -49,7 +49,19 @@ def _triton_concat_2D_jagged_internal( is_dense_b: bool, BLOCK_D: int, ) -> None: - if is_sm100_plus(): + if is_sm100_plus() or (torch.cuda.is_available() and torch.version.hip): + # Route AMD/ROCm through the multirow kernel. + # + # The basic `_concat_2D_jagged` kernel below issues one program per + # output row (grid = `(max_seq_len, B)`). On ROCm Triton this fails to + # lower in the `TritonAMDGPUCanonicalizePointers` pass with + # `RuntimeError: PassManager::run failed` at `make_ttgir`. The + # multirow variant tiles rows with a tunable `BLOCK_N` (grid = + # `(cdiv(max_seq_len, BLOCK_N), B)`) and compiles cleanly on ROCm. + # The original `is_sm100_plus()` gate was conservative — only Blackwell + # was opted in. Adding HIP keeps NVIDIA H100/A100 on the basic kernel + # they were validated against and unblocks AMD without behavior change + # on existing NVIDIA paths. def grid(meta): return (triton.cdiv(max_seq_len, meta["BLOCK_N"]), B) @@ -107,7 +119,11 @@ def _triton_split_2D_jagged_internal( is_dense_b: bool, BLOCK_D: int, ) -> None: - if is_sm100_plus(): + if is_sm100_plus() or (torch.cuda.is_available() and torch.version.hip): + # Route AMD/ROCm through the multirow kernel for the same reason as + # `_triton_concat_2D_jagged_internal` above: basic `_split_2D_jagged` + # hits `PassManager::run failed` in `TritonAMDGPUCanonicalizePointers` + # on ROCm; multirow lowers cleanly. def grid(meta): return (triton.cdiv(max_seq_len, meta["BLOCK_N"]), B) @@ -608,22 +624,27 @@ def backward( d_values_b = torch.empty( (ctx.total_len_b, D), device=d_out.device, dtype=d_out.dtype ) - _split_2D_jagged[(ctx.max_seq_len, ctx.B)]( - JaggedIn=d_out, - OffsetsA=offsets_a, - OffsetsB=offsets_b, - MaxLenA=ctx.max_len_a, - MaxLenB=ctx.max_len_b, - OutA=d_values_a, - OutB=d_values_b, + # Go through `_triton_split_2D_jagged_internal` (not raw + # `_split_2D_jagged[grid]`) so this backward pass benefits from the same + # AMD-routing-through-multirow workaround as the forward. Calling the + # raw kernel directly would hit `PassManager::run failed` on ROCm at + # `TritonAMDGPUCanonicalizePointers`. If you refactor this, do not + # collapse it back to `_split_2D_jagged[(ctx.max_seq_len, ctx.B)](...)`. + _triton_split_2D_jagged_internal( + jagged_in=d_out, + max_seq_len=ctx.max_seq_len, + B=ctx.B, + offsets_a=offsets_a, + offsets_b=offsets_b, + max_len_a=ctx.max_len_a, + max_len_b=ctx.max_len_b, + out_a=d_values_a, + out_b=d_values_b, D=D, - stride_id=d_out.stride(-2), - stride_ad=d_values_a.stride(-2), - stride_bd=d_values_b.stride(-2), n_prefix_to_B=ctx.n_prefix_from_B, + is_dense_a=ctx.is_dense_a, + is_dense_b=ctx.is_dense_b, BLOCK_D=BLOCK_D, - IS_DENSE_A=ctx.is_dense_a, - IS_DENSE_B=ctx.is_dense_b, ) return None, d_values_a, d_values_b, None, None, None, None, None From 751f2f0ec3c14bc7a41bb71993c2bc0c67c190e4 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Fri, 29 May 2026 23:14:14 +0000 Subject: [PATCH 003/127] Fix AttributeError on triton.knobs.nvidia.use_meta_ws The attribute is absent in some Triton builds (e.g. nvcr.io/nvidia/pytorch:26.01-py3), causing import-time AttributeError before any training step runs. Use getattr with a False default so _use_meta_ws() gracefully reports disabled on those builds. --- .../generative_recommenders/ops/triton/triton_addmm.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/recommendation_v4/generative_recommenders/ops/triton/triton_addmm.py b/recommendation_v4/generative_recommenders/ops/triton/triton_addmm.py index 915d85742..487aae189 100644 --- a/recommendation_v4/generative_recommenders/ops/triton/triton_addmm.py +++ b/recommendation_v4/generative_recommenders/ops/triton/triton_addmm.py @@ -65,7 +65,9 @@ def _use_meta_ws() -> bool: is_sm100_plus() and hasattr(triton, "knobs") and hasattr(triton.knobs, "nvidia") - and triton.knobs.nvidia.use_meta_ws + # `use_meta_ws` is absent in some Triton builds (e.g. nvcr.io/nvidia/pytorch:26.01-py3); + # use getattr so import doesn't crash on AttributeError before any step runs. + and getattr(triton.knobs.nvidia, "use_meta_ws", False) ) From 0f1fcf05b6fa7c3d69f4bb45df1d2887ec76a683 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Fri, 29 May 2026 19:12:08 -0500 Subject: [PATCH 004/127] Make HSTU model arch + dataset history_length gin-tunable Three small changes so you can sweep model size and per-sample sequence length from a gin file without editing configs.py. configs.py: - get_hstu_configs is now @gin.configurable. Accepts optional overrides for max_seq_len, max_num_candidates, hstu_embedding_table_dim, hstu_transducer_embedding_dim, hstu_num_heads, hstu_attn_num_layers, hstu_attn_linear_dim, hstu_attn_qk_dim, hstu_input_dropout_ratio, hstu_linear_dropout_rate. Per-dataset defaults still apply unless explicitly overridden in gin. - get_embedding_table_config is now @gin.configurable with an embedding_dim override that uniformly sets the dim for all tables of the chosen dataset. - Drop the YAMBDA_EMBEDDING_DIM constant (was a duplicate of HSTU_EMBEDDING_DIM=512). Yambda branch now uses HSTU_EMBEDDING_DIM directly. Add a comment noting the model+table dim must stay aligned when overriding either via gin. utils.py: - get_dataset accepts an optional history_length kwarg that wins over the yambda dataset's hardcoded default of 4096. Caches are still keyed on disk under hstu_cache_L/ so switching L between previously built values is free. train/gin/yambda_5b.gin: - Pin history_length=2048 and max_seq_len=2048 for the seq-2k smoke config. Both lines have inline comments explaining the +9 overhead (uid + 7 cross + 1 candidate) so total per-sample seq is ~2046, within the 2048 budget. Verified: default codepath unchanged, gin overrides apply consistently to both get_hstu_configs (model) and get_embedding_table_config (tables). Co-Authored-By: Claude Opus 4.7 --- .../dlrm_v3/configs.py | 116 +++++++++++++----- .../dlrm_v3/train/gin/yambda_5b.gin | 8 ++ .../generative_recommenders/dlrm_v3/utils.py | 6 +- 3 files changed, 94 insertions(+), 36 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/configs.py b/recommendation_v4/generative_recommenders/dlrm_v3/configs.py index 2981f01e3..1fd7f07a9 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/configs.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/configs.py @@ -19,7 +19,9 @@ This module provides configuration functions for the HSTU model architecture and embedding table configurations. """ -from typing import Dict +from typing import Dict, Optional + +import gin from generative_recommenders.modules.dlrm_hstu import DlrmHSTUConfig from generative_recommenders.modules.multitask_module import ( @@ -32,8 +34,6 @@ HASH_SIZE = 10_000_000 HASH_SIZE_1B = 1_000_000_000 -YAMBDA_EMBEDDING_DIM = 512 - # (name, keys, num_embeddings, salt) — single source of truth for both # get_embedding_table_config("yambda-5b") and the dataset's cross-hash inputs. # Sizes mirror Primus-DLRM/configs/bench_onetrans_large_5b_cross_feat_shampoo.yaml. @@ -48,7 +48,20 @@ ] -def get_hstu_configs(dataset: str = "debug") -> DlrmHSTUConfig: +@gin.configurable +def get_hstu_configs( + dataset: str = "debug", + max_seq_len: Optional[int] = None, + max_num_candidates: Optional[int] = None, + hstu_embedding_table_dim: Optional[int] = None, + hstu_transducer_embedding_dim: Optional[int] = None, + hstu_num_heads: Optional[int] = None, + hstu_attn_num_layers: Optional[int] = None, + hstu_attn_linear_dim: Optional[int] = None, + hstu_attn_qk_dim: Optional[int] = None, + hstu_input_dropout_ratio: Optional[float] = None, + hstu_linear_dropout_rate: Optional[float] = None, +) -> DlrmHSTUConfig: """ Create and return HSTU model configuration. @@ -333,9 +346,11 @@ def get_hstu_configs(dataset: str = "debug") -> DlrmHSTUConfig: elif "yambda" in dataset: assert dataset in ["yambda-5b"] cross_names = [name for (name, _k, _n, _s) in YAMBDA_5B_CROSS_SPECS] - # Smaller per-table dim for yambda (see YAMBDA_EMBEDDING_DIM); transducer - # still projects to 512. - hstu_config.hstu_embedding_table_dim = YAMBDA_EMBEDDING_DIM + # Per-table dim defaults to HSTU_EMBEDDING_DIM (512); override via the + # `get_hstu_configs.hstu_embedding_table_dim = N` gin binding if needed. + # Note: the embedding tables in get_embedding_table_config also use + # HSTU_EMBEDDING_DIM and must stay aligned with this value. + hstu_config.hstu_embedding_table_dim = HSTU_EMBEDDING_DIM hstu_config.hstu_transducer_embedding_dim = 512 hstu_config.max_seq_len = 8192 hstu_config.max_num_candidates = 1 @@ -468,10 +483,37 @@ def get_hstu_configs(dataset: str = "debug") -> DlrmHSTUConfig: task_type=MultitaskTaskType.BINARY_CLASSIFICATION, ) ] + + # Apply gin overrides last so a value set in the gin file wins over the + # per-dataset defaults above. Anything left as None inherits the default + # the dataset branch (or DlrmHSTUConfig) chose. Example in a gin file: + # get_hstu_configs.max_seq_len = 4096 + # get_hstu_configs.hstu_embedding_table_dim = 256 + _gin_overrides = { + "max_seq_len": max_seq_len, + "max_num_candidates": max_num_candidates, + "max_num_candidates_inference": max_num_candidates, + "hstu_embedding_table_dim": hstu_embedding_table_dim, + "hstu_transducer_embedding_dim": hstu_transducer_embedding_dim, + "hstu_num_heads": hstu_num_heads, + "hstu_attn_num_layers": hstu_attn_num_layers, + "hstu_attn_linear_dim": hstu_attn_linear_dim, + "hstu_attn_qk_dim": hstu_attn_qk_dim, + "hstu_input_dropout_ratio": hstu_input_dropout_ratio, + "hstu_linear_dropout_rate": hstu_linear_dropout_rate, + } + for _name, _val in _gin_overrides.items(): + if _val is not None: + setattr(hstu_config, _name, _val) + return hstu_config -def get_embedding_table_config(dataset: str = "debug") -> Dict[str, EmbeddingConfig]: +@gin.configurable +def get_embedding_table_config( + dataset: str = "debug", + embedding_dim: Optional[int] = None, +) -> Dict[str, EmbeddingConfig]: """ Create and return embedding table configurations. @@ -480,10 +522,16 @@ def get_embedding_table_config(dataset: str = "debug") -> Dict[str, EmbeddingCon Args: dataset: Dataset identifier (currently unused, reserved for dataset-specific configs). + embedding_dim: Per-table embedding width override. When set via gin + (e.g. `get_embedding_table_config.embedding_dim = 256`), wins over + `HSTU_EMBEDDING_DIM`. Keep in sync with the matching gin override on + `get_hstu_configs.hstu_embedding_table_dim` — the model and the + tables must agree on dim or sharding will reject the plan. Returns: Dict mapping table names to their EmbeddingConfig objects. """ + DIM = embedding_dim if embedding_dim is not None else HSTU_EMBEDDING_DIM if "movielens" in dataset: assert dataset in [ "movielens-1m", @@ -495,42 +543,42 @@ def get_embedding_table_config(dataset: str = "debug") -> Dict[str, EmbeddingCon { "movie_id": EmbeddingConfig( num_embeddings=HASH_SIZE, - embedding_dim=HSTU_EMBEDDING_DIM, + embedding_dim=DIM, name="movie_id", data_type=DataType.FP16, feature_names=["movie_id", "item_movie_id"], ), "user_id": EmbeddingConfig( num_embeddings=HASH_SIZE, - embedding_dim=HSTU_EMBEDDING_DIM, + embedding_dim=DIM, name="user_id", data_type=DataType.FP16, feature_names=["user_id"], ), "sex": EmbeddingConfig( num_embeddings=HASH_SIZE, - embedding_dim=HSTU_EMBEDDING_DIM, + embedding_dim=DIM, name="sex", data_type=DataType.FP16, feature_names=["sex"], ), "age_group": EmbeddingConfig( num_embeddings=HASH_SIZE, - embedding_dim=HSTU_EMBEDDING_DIM, + embedding_dim=DIM, name="age_group", data_type=DataType.FP16, feature_names=["age_group"], ), "occupation": EmbeddingConfig( num_embeddings=HASH_SIZE, - embedding_dim=HSTU_EMBEDDING_DIM, + embedding_dim=DIM, name="occupation", data_type=DataType.FP16, feature_names=["occupation"], ), "zip_code": EmbeddingConfig( num_embeddings=HASH_SIZE, - embedding_dim=HSTU_EMBEDDING_DIM, + embedding_dim=DIM, name="zip_code", data_type=DataType.FP16, feature_names=["zip_code"], @@ -540,14 +588,14 @@ def get_embedding_table_config(dataset: str = "debug") -> Dict[str, EmbeddingCon else { "movie_id": EmbeddingConfig( num_embeddings=HASH_SIZE_1B, - embedding_dim=HSTU_EMBEDDING_DIM, + embedding_dim=DIM, name="movie_id", data_type=DataType.FP16, feature_names=["movie_id", "item_movie_id"], ), "user_id": EmbeddingConfig( num_embeddings=3_000_000, - embedding_dim=HSTU_EMBEDDING_DIM, + embedding_dim=DIM, name="user_id", data_type=DataType.FP16, feature_names=["user_id"], @@ -558,14 +606,14 @@ def get_embedding_table_config(dataset: str = "debug") -> Dict[str, EmbeddingCon return { "item_id": EmbeddingConfig( num_embeddings=HASH_SIZE_1B, - embedding_dim=HSTU_EMBEDDING_DIM, + embedding_dim=DIM, name="item_id", data_type=DataType.FP16, feature_names=["item_id", "item_candidate_id"], ), "item_category_id": EmbeddingConfig( num_embeddings=128, - embedding_dim=HSTU_EMBEDDING_DIM, + embedding_dim=DIM, name="item_category_id", data_type=DataType.FP16, weight_init_max=1.0, @@ -574,7 +622,7 @@ def get_embedding_table_config(dataset: str = "debug") -> Dict[str, EmbeddingCon ), "user_id": EmbeddingConfig( num_embeddings=10_000_000, - embedding_dim=HSTU_EMBEDDING_DIM, + embedding_dim=DIM, name="user_id", data_type=DataType.FP16, feature_names=["user_id"], @@ -584,49 +632,49 @@ def get_embedding_table_config(dataset: str = "debug") -> Dict[str, EmbeddingCon return { "video_id": EmbeddingConfig( num_embeddings=HASH_SIZE, - embedding_dim=HSTU_EMBEDDING_DIM, + embedding_dim=DIM, name="video_id", data_type=DataType.FP16, feature_names=["video_id", "item_video_id"], ), "user_id": EmbeddingConfig( num_embeddings=HASH_SIZE, - embedding_dim=HSTU_EMBEDDING_DIM, + embedding_dim=DIM, name="user_id", data_type=DataType.FP16, feature_names=["user_id"], ), "user_active_degree": EmbeddingConfig( num_embeddings=8, - embedding_dim=HSTU_EMBEDDING_DIM, + embedding_dim=DIM, name="user_active_degree", data_type=DataType.FP16, feature_names=["user_active_degree"], ), "follow_user_num_range": EmbeddingConfig( num_embeddings=9, - embedding_dim=HSTU_EMBEDDING_DIM, + embedding_dim=DIM, name="follow_user_num_range", data_type=DataType.FP16, feature_names=["follow_user_num_range"], ), "fans_user_num_range": EmbeddingConfig( num_embeddings=9, - embedding_dim=HSTU_EMBEDDING_DIM, + embedding_dim=DIM, name="fans_user_num_range", data_type=DataType.FP16, feature_names=["fans_user_num_range"], ), "friend_user_num_range": EmbeddingConfig( num_embeddings=8, - embedding_dim=HSTU_EMBEDDING_DIM, + embedding_dim=DIM, name="friend_user_num_range", data_type=DataType.FP16, feature_names=["friend_user_num_range"], ), "register_days_range": EmbeddingConfig( num_embeddings=8, - embedding_dim=HSTU_EMBEDDING_DIM, + embedding_dim=DIM, name="register_days_range", data_type=DataType.FP16, feature_names=["register_days_range"], @@ -637,28 +685,28 @@ def get_embedding_table_config(dataset: str = "debug") -> Dict[str, EmbeddingCon tables: Dict[str, EmbeddingConfig] = { "item_id": EmbeddingConfig( num_embeddings=9_390_000, - embedding_dim=YAMBDA_EMBEDDING_DIM, + embedding_dim=DIM, name="item_id", data_type=DataType.FP32, feature_names=["item_id", "item_candidate_id"], ), "artist_id": EmbeddingConfig( num_embeddings=1_290_000, - embedding_dim=YAMBDA_EMBEDDING_DIM, + embedding_dim=DIM, name="artist_id", data_type=DataType.FP32, feature_names=["artist_id", "item_candidate_artist_id"], ), "album_id": EmbeddingConfig( num_embeddings=3_370_000, - embedding_dim=YAMBDA_EMBEDDING_DIM, + embedding_dim=DIM, name="album_id", data_type=DataType.FP32, feature_names=["album_id", "item_candidate_album_id"], ), "uid": EmbeddingConfig( num_embeddings=1_000_000, - embedding_dim=YAMBDA_EMBEDDING_DIM, + embedding_dim=DIM, name="uid", data_type=DataType.FP32, feature_names=["uid"], @@ -667,7 +715,7 @@ def get_embedding_table_config(dataset: str = "debug") -> Dict[str, EmbeddingCon for name, _keys, num_embeddings, _salt in YAMBDA_5B_CROSS_SPECS: tables[name] = EmbeddingConfig( num_embeddings=num_embeddings, - embedding_dim=YAMBDA_EMBEDDING_DIM, + embedding_dim=DIM, name=name, data_type=DataType.FP32, feature_names=[name], @@ -677,7 +725,7 @@ def get_embedding_table_config(dataset: str = "debug") -> Dict[str, EmbeddingCon return { "post_id": EmbeddingConfig( num_embeddings=HASH_SIZE, - embedding_dim=HSTU_EMBEDDING_DIM, + embedding_dim=DIM, name="post_id", data_type=DataType.FP16, feature_names=[ @@ -689,14 +737,14 @@ def get_embedding_table_config(dataset: str = "debug") -> Dict[str, EmbeddingCon ), "viewer_id": EmbeddingConfig( num_embeddings=HASH_SIZE, - embedding_dim=HSTU_EMBEDDING_DIM, + embedding_dim=DIM, name="viewer_id", data_type=DataType.FP16, feature_names=["viewer_id"], ), "dummy_contexual": EmbeddingConfig( num_embeddings=HASH_SIZE, - embedding_dim=HSTU_EMBEDDING_DIM, + embedding_dim=DIM, name="dummy_contexual", data_type=DataType.FP16, feature_names=["dummy_contexual"], diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index a483f8766..0a8da31ab 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -34,6 +34,14 @@ make_train_test_dataloaders.num_blocks = 1 get_dataset.name = %dataset get_dataset.new_path_prefix = "/apps/chcai/dlrm_data" +# Per-pool truncation cap. Cache is keyed by L on disk under +# /hstu_cache_L/; switching L reuses an existing cache +# (L=2048 was built in a prior session, no rebuild needed). +get_dataset.history_length = 2048 + +# Model-side attention budget. Dataset truncates UIH to fit this value if +# `history_length + contextual + candidate` would overflow. +get_hstu_configs.max_seq_len = 2048 # train-eval loop variables (yambda is non-streaming) train_eval_loop.num_epochs = 1 diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py index 52091d8dd..43d641a0c 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py @@ -509,7 +509,7 @@ def reset(self, mode: str = "train"): @gin.configurable -def get_dataset(name: str, new_path_prefix: str = ""): +def get_dataset(name: str, new_path_prefix: str = "", history_length: Optional[int] = None): """ Get dataset class and configuration by name. @@ -630,7 +630,9 @@ def get_dataset(name: str, new_path_prefix: str = ""): # all ranks on a node share the same physical pages. "processed_dir": os.path.join(new_path_prefix, "processed_5b"), "metadata_dir": os.path.join(new_path_prefix, "shared_metadata"), - "history_length": 4096, + # Per-pool truncation cap; total interleaved UIH ~ 3*L/3 = L. + # Override via `get_dataset.history_length = N` in gin. + "history_length": history_length if history_length is not None else 4096, "scan_window": 20000, "cross_specs": YAMBDA_5B_CROSS_SPECS, }, From b7f8b2d56cdc5c58774c9d02233be648648650fb Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Fri, 29 May 2026 22:43:21 -0500 Subject: [PATCH 005/127] Make EmbeddingShardingPlanner hbm_cap_gb gin-tunable, set ddr_cap=0 make_optimizer_and_shard now accepts hbm_cap_gb (default 260, the MI350X value) via @gin.configurable. The yambda gin pins the same default so sweeps just change the number in the gin file instead of editing utils.py. ddr_cap dropped from 32 GiB to 0: with all 11 yambda 5b embedding tables fitting on 8x MI350X HBM, allowing host DRAM offload only invites the planner to pick slower per-lookup-PCIe-traffic plans. Verified gin binding flows through to the Topology: a probe with hbm_cap_gb=100 produced Topology(hbm_cap=107374182400) and the planner correctly raised insufficient-storage error at that tightness. Co-Authored-By: Claude Opus 4.7 --- .../dlrm_v3/train/gin/yambda_5b.gin | 3 +++ .../generative_recommenders/dlrm_v3/train/utils.py | 9 ++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index 0a8da31ab..939b42b78 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -32,6 +32,9 @@ make_train_test_dataloaders.num_workers = %num_workers make_train_test_dataloaders.prefetch_factor = %prefetch_factor make_train_test_dataloaders.num_blocks = 1 +# embedding planner +make_optimizer_and_shard.hbm_cap_gb = 260 + get_dataset.name = %dataset get_dataset.new_path_prefix = "/apps/chcai/dlrm_data" # Per-pool truncation cap. Cache is keyed by L on disk under diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py index 21d2baa6e..90669c1a6 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -295,10 +295,12 @@ def sparse_optimizer_factory_and_class( return optimizer_cls, kwargs, optimizer_factory +@gin.configurable def make_optimizer_and_shard( model: torch.nn.Module, device: torch.device, world_size: int, + hbm_cap_gb: int = 260, ) -> Tuple[DistributedModelParallel, torch.optim.Optimizer]: dense_opt_cls, dense_opt_args, dense_opt_factory = ( dense_optimizer_factory_and_class() @@ -316,16 +318,13 @@ def make_optimizer_and_shard( sparse_opt_cls, [param], sparse_opt_args ) sharders = get_default_sharders() - # MI350X has 288 GiB HBM3e per GPU; the 160 GiB cap was sized for older parts. - # Matches Primus-DLRM (hbm_cap_gb: 260) which runs the same 5b cross-feat - # table set on the same hardware without host materialization. planner = EmbeddingShardingPlanner( topology=Topology( local_world_size=world_size, world_size=world_size, compute_device="cuda", - hbm_cap=260 * 1024 * 1024 * 1024, - ddr_cap=32 * 1024 * 1024 * 1024, + hbm_cap=hbm_cap_gb * 1024 * 1024 * 1024, + ddr_cap=0, ) ) pg = dist.GroupMember.WORLD From 5ba9b074c7eccb74b58e8104cffca9b0bfbbb092 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Sat, 30 May 2026 00:13:27 -0500 Subject: [PATCH 006/127] Add yambda 50m/500m/5b preprocessor + DLRM_DATA_PATH env override preprocess_public_data.py: - Add DLRMYambdaProcessor: downloads Yambda multi_event + catalog metadata from the yandex/yambda HuggingFace repo, then runs a temporal split (300 train days / 30 min gap / 1 test day), builds per-user sessions (1800s inactivity threshold), and writes the layout DLRMv3YambdaDataset expects: /raw//multi_event.parquet /shared_metadata/{artist,album,embeddings}.parquet /processed_/{train_sessions,test_events, session_index}.parquet /processed_/item_popularity.npy /processed_/split_meta.json - 5b variant uses chunked polars load (10M rows/chunk) to keep peak RAM under control (single-shot read of the 50 GB parquet OOMs ~150 GB systems). - SUPPORTED_DATASETS adds yambda-50m, yambda-500m, yambda-5b. - main() takes --data-path for custom output root. - Verified end-to-end: 50m run completes in ~2 min, 5b in ~53 min (download dominates), output is byte-compatible with the dataset cache builder; TRITON training reaches steady state on the fresh data at 2050 sps. utils.py: - Add env_path(key, default) @gin.configurable helper. Used as a gin macro so any string-valued binding can be overridden by an env var without editing the gin file. train/gin/yambda_5b.gin: - Declare DATA_PATH = @env_path() macro with key="DLRM_DATA_PATH" and default="/apps/chcai/dlrm_data". Both new_path_prefix bindings (make_train_test_dataloaders and get_dataset) now consume %DATA_PATH. Setting DLRM_DATA_PATH=/some/path at run time redirects the dataset without a gin edit. datasets/yambda.py: - Strip stale references to upstream-internal preprocessing in docstrings/comments; point at preprocess_public_data.py instead. Co-Authored-By: Claude Opus 4.7 --- .../dlrm_v3/datasets/yambda.py | 14 +- .../dlrm_v3/preprocess_public_data.py | 352 +++++++++++++++++- .../dlrm_v3/train/gin/yambda_5b.gin | 10 +- .../generative_recommenders/dlrm_v3/utils.py | 15 + 4 files changed, 368 insertions(+), 23 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/datasets/yambda.py b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/yambda.py index fb8b212b1..5a13ac034 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/datasets/yambda.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/yambda.py @@ -10,15 +10,12 @@ """ Yambda dataset for the DLRMv3 HSTU `modules/` path. -Reads the same parquets produced by Primus-DLRM's preprocessing (no runtime -dep on Primus). Each sample is one anchor LISTEN event with: +Reads the parquets produced by `dlrm_v3/preprocess_public_data.py +--dataset yambda-`. Each sample is one anchor LISTEN event with: * label = (played_ratio >= LISTEN_PLUS_THRESHOLD) — the listen_plus bit * a chronologically interleaved 3-pool history (listen+/like/skip), with pool identity tagged per-position in `action_weight` (bits 1/2/4) * 7 pre-hashed cross-feature ids exposed as length-1 contextual entries - -Hash formula is byte-identical to `primus_dlrm.data.hashing.cross_hash_nway` -so embedding rows are interchangeable. """ import logging @@ -60,7 +57,7 @@ def _load_npy_readonly(path: Union[str, Path]) -> np.ndarray: arr.flags.writeable = False return arr -# Match primus_dlrm.data.preprocessing.EVENT_TYPE_MAP / dataset.LISTEN_PLUS_THRESHOLD +# Yambda event-type encoding written by preprocess_public_data.py. LISTEN_TYPE = 0 LIKE_TYPE = 1 LISTEN_PLUS_THRESHOLD = 50 @@ -72,15 +69,14 @@ def _load_npy_readonly(path: Union[str, Path]) -> np.ndarray: class _FlatEventStore: - """Minimal port of Primus-DLRM's FlatEventStore. + """Per-user flat event index built from the preprocessed sessions parquet. Reads `train_sessions.parquet` and explodes per-session arrays into flat numpy columns + per-user `(start, end)` index arrays. Cache-compatible layout, but writes nothing (rebuilds from parquet each construction). """ - # On-disk column layout written by Primus-DLRM's FlatEventStore.save_mmap. - # Bit-identical to that schema so the cache is interchangeable. + # On-disk column layout. _MMAP_COLS = ( "flat_uid", "flat_item_ids", "flat_timestamps", "flat_event_types", "flat_played_ratio", diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/preprocess_public_data.py b/recommendation_v4/generative_recommenders/dlrm_v3/preprocess_public_data.py index 488af712d..06b43aa48 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/preprocess_public_data.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/preprocess_public_data.py @@ -14,10 +14,12 @@ # pyre-unsafe import argparse +import json import logging import os import tarfile -from typing import Dict, List +from pathlib import Path +from typing import Dict, List, Optional, Tuple from urllib.request import urlretrieve import numpy as np @@ -28,10 +30,20 @@ log = logging.getLogger("main") """ -Usage: mkdir -p data/ && python3 preprocess_public_data.py --dataset kuairand-1k +Usage: + mkdir -p data/ && python3 preprocess_public_data.py --dataset kuairand-1k + python3 preprocess_public_data.py --dataset yambda-5b --data-path + python3 preprocess_public_data.py --dataset yambda-500m --data-path + python3 preprocess_public_data.py --dataset yambda-50m --data-path """ -SUPPORTED_DATASETS = ["kuairand-1k", "kuairand-27k"] +SUPPORTED_DATASETS = [ + "kuairand-1k", + "kuairand-27k", + "yambda-50m", + "yambda-500m", + "yambda-5b", +] def get_feature_merge_weights(dataset: str = "debug") -> Dict[str, int]: @@ -185,26 +197,342 @@ def _one_hot_encode(row): log.info(f"Processed file saved to {self._output_file}") +# ---------------------------------------------------------------------------- +# Yambda processor +# ---------------------------------------------------------------------------- +# +# Yambda is hosted on HuggingFace at `yandex/yambda` and comes in three sizes: +# 50m, 500m, 5b. Each size shares the same catalog metadata (embeddings, +# artist/album mappings); only the interaction stream differs. +# +# This processor: +# 1) Downloads `multi_event.parquet` for the chosen size + the catalog +# metadata files via the `datasets` library. +# 2) Encodes event_type strings into uint8. +# 3) Splits temporally into train + test (Global Temporal Split, GTS). +# 4) Builds per-user sessions by inactivity gap. +# 5) Computes item popularity counts. +# 6) Writes the layout expected by `DLRMv3YambdaDataset`: +# +# /processed_/ +# train_sessions.parquet +# test_events.parquet +# session_index.parquet +# item_popularity.npy +# split_meta.json +# +# /shared_metadata/ +# artist_item_mapping.parquet +# album_item_mapping.parquet +# embeddings.parquet (optional; not used by HSTU training) +# +# The HSTU training path then auto-builds an `hstu_cache_L/` mmap under +# `processed_/` on first use. +# ---------------------------------------------------------------------------- + +YAMBDA_HF_REPO = "yandex/yambda" +YAMBDA_SIZES = {"yambda-50m": "50m", "yambda-500m": "500m", "yambda-5b": "5b"} +YAMBDA_METADATA_FILES = ( + "artist_item_mapping", + "album_item_mapping", + "embeddings", +) + +# Yambda timestamps are seconds (rounded to 5s boundaries). +SECONDS_PER_DAY = 86400 +# Polars chunk size for streaming the 5b parquet (~150 GB on disk). +YAMBDA_CHUNK_SIZE = 10_000_000 +EVENT_TYPE_MAP = {"listen": 0, "like": 1, "dislike": 2, "unlike": 3, "undislike": 4} + + +class DLRMYambdaProcessor(DataProcessor): + """Download + preprocess Yambda (50m / 500m / 5b) for DLRMv3YambdaDataset.""" + + def __init__( + self, + data_path: str, + size: str, + session_gap_seconds: int = 1800, + train_days: int = 300, + gap_minutes: int = 30, + test_days: int = 1, + ) -> None: + assert size in {"50m", "500m", "5b"}, f"unknown yambda size {size}" + super().__init__( + download_url="", # download is via HuggingFace `datasets` lib + data_path=data_path.rstrip("/") + "/", + file_name=f"{size}/multi_event.parquet", + prefix=f"yambda-{size}", + ) + self._size: str = size + self._raw_dir: Path = Path(self._data_path) / "raw" + self._processed_dir: Path = Path(self._data_path) / f"processed_{size}" + self._shared_dir: Path = Path(self._data_path) / "shared_metadata" + self._session_gap_seconds: int = session_gap_seconds + self._train_days: int = train_days + self._gap_minutes: int = gap_minutes + self._test_days: int = test_days + + def download(self) -> None: + try: + from datasets import DatasetDict, load_dataset + except ImportError as e: + raise ImportError( + "Downloading Yambda requires the `datasets` package " + "(`pip install datasets`)." + ) from e + + self._raw_dir.mkdir(parents=True, exist_ok=True) + self._shared_dir.mkdir(parents=True, exist_ok=True) + + # Size-specific interaction stream. + event_path = self._raw_dir / self._size / "multi_event.parquet" + if not event_path.exists(): + event_path.parent.mkdir(parents=True, exist_ok=True) + log.info( + f"Downloading multi_event.parquet for {self._size} " + f"from {YAMBDA_HF_REPO} ..." + ) + ds = load_dataset( + YAMBDA_HF_REPO, + data_dir=f"flat/{self._size}", + data_files="multi_event.parquet", + ) + assert isinstance(ds, DatasetDict) + ds["train"].to_parquet(str(event_path)) + log.info(f"Saved {event_path}") + else: + log.info(f"Already exists: {event_path}") + + # Catalog metadata files (shared across sizes). + for name in YAMBDA_METADATA_FILES: + shared_path = self._shared_dir / f"{name}.parquet" + if shared_path.exists(): + log.info(f"Already exists: {shared_path}") + continue + log.info(f"Downloading {name}.parquet from {YAMBDA_HF_REPO} ...") + ds = load_dataset(YAMBDA_HF_REPO, data_files=f"{name}.parquet") + assert isinstance(ds, DatasetDict) + ds["train"].to_parquet(str(shared_path)) + log.info(f"Saved {shared_path}") + + def preprocess(self) -> None: + self.download() + try: + import polars as pl + except ImportError as e: + raise ImportError( + "Yambda preprocessing requires polars " + "(`pip install polars-u64-idx` is recommended for the 5b " + "variant — stock polars overflows its 32-bit row index)." + ) from e + + self._processed_dir.mkdir(parents=True, exist_ok=True) + event_path = self._raw_dir / self._size / "multi_event.parquet" + + log.info(f"Loading multi_event from {event_path} ...") + events = self._load_events(pl, event_path) + log.info(f"Loaded {len(events):,} events") + + events = self._encode_event_types(pl, events) + t_min = int(events["timestamp"].min()) + t_max = int(events["timestamp"].max()) + log.info( + f"Timestamp range: {t_min}..{t_max} " + f"({(t_max - t_min) / SECONDS_PER_DAY:.1f} days)" + ) + + train_start, train_end, test_start, test_end = self._split_boundaries(t_max) + log.info( + f"GTS train=[{train_start},{train_end}) gap=[{train_end},{test_start}) " + f"test=[{test_start},{test_end})" + ) + train_events, test_events = self._temporal_split( + pl, events, train_start, train_end, test_start, test_end + ) + log.info( + f"Train: {len(train_events):,} events, Test: {len(test_events):,} events" + ) + + gap_units = self._session_gap_seconds # 1 unit = 1 second + sessions = self._build_sessions(pl, train_events, gap_units) + log.info(f"Built {len(sessions):,} sessions") + + session_index = self._build_session_index(pl, sessions) + log.info(f"Session index covers {len(session_index):,} users") + + item_popularity = self._compute_item_popularity(train_events) + + sessions.write_parquet(str(self._processed_dir / "train_sessions.parquet")) + test_events.write_parquet(str(self._processed_dir / "test_events.parquet")) + session_index.write_parquet(str(self._processed_dir / "session_index.parquet")) + np.save(self._processed_dir / "item_popularity.npy", item_popularity) + + with open(self._processed_dir / "split_meta.json", "w") as f: + json.dump( + { + "size": self._size, + "t_min": t_min, + "t_max": t_max, + "train_start": train_start, + "train_end": train_end, + "test_start": test_start, + "test_end": test_end, + "train_days": self._train_days, + "gap_minutes": self._gap_minutes, + "test_days": self._test_days, + "session_gap_seconds": self._session_gap_seconds, + "num_train_events": int(len(train_events)), + "num_test_events": int(len(test_events)), + "num_sessions": int(len(sessions)), + "num_users": int(len(session_index)), + }, + f, + indent=2, + ) + log.info(f"Preprocessing complete: {self._processed_dir}") + + # ------- helpers -------- + + def _load_events(self, pl, parquet_path: Path): + # 5b is too large to load in one polars pass on most boxes (~150 GB + # peak in-RAM with eager read). Stream in 10M-row chunks for safety. + if self._size == "5b": + log.info(f"Streaming load (chunk_size={YAMBDA_CHUNK_SIZE:,})...") + lf = pl.scan_parquet(parquet_path) + n = lf.select(pl.len()).collect().item() + log.info(f"Total rows: {n:,}") + chunks = [] + for off in range(0, n, YAMBDA_CHUNK_SIZE): + chunk = lf.slice(off, YAMBDA_CHUNK_SIZE).collect() + chunks.append(chunk) + log.info(f" loaded {off:,}..{off + len(chunk):,}") + return pl.concat(chunks) + return pl.read_parquet(parquet_path) + + def _encode_event_types(self, pl, events): + dt = events["event_type"].dtype + if dt == pl.Utf8 or isinstance(dt, (pl.Categorical, pl.Enum)): + events = events.with_columns( + pl.col("event_type") + .cast(pl.Utf8) + .replace_strict(EVENT_TYPE_MAP) + .cast(pl.UInt8) + .alias("event_type") + ) + return events + + def _split_boundaries(self, t_max: int) -> Tuple[int, int, int, int]: + test_end = t_max + test_start = test_end - self._test_days * SECONDS_PER_DAY + train_end = test_start - self._gap_minutes * 60 + train_start = train_end - self._train_days * SECONDS_PER_DAY + return train_start, train_end, test_start, test_end + + def _temporal_split(self, pl, events, train_start, train_end, test_start, test_end): + train = events.filter( + (pl.col("timestamp") >= train_start) & (pl.col("timestamp") < train_end) + ) + test_all = events.filter( + (pl.col("timestamp") >= test_start) & (pl.col("timestamp") < test_end) + ) + # Test users must also appear in train (next-item prediction setup). + train_users = train.select("uid").unique() + test = test_all.join(train_users, on="uid", how="inner") + return train, test + + def _build_sessions(self, pl, events, session_gap_units: int): + sorted_events = events.sort(["uid", "timestamp"]) + return ( + sorted_events + .with_columns( + ( + (pl.col("timestamp").diff().fill_null(0) > session_gap_units) + .cast(pl.UInt32) + .cum_sum() + ) + .over("uid") + .alias("session_id") + ) + .group_by(["uid", "session_id"]) + .agg( + pl.col("item_id").alias("item_ids"), + pl.col("timestamp").alias("timestamps"), + pl.col("event_type").alias("event_types"), + pl.col("is_organic").alias("is_organic"), + pl.col("played_ratio_pct").alias("played_ratio_pct"), + pl.col("track_length_seconds").alias("track_length_seconds"), + ) + .sort(["uid", "session_id"]) + ) + + def _build_session_index(self, pl, sessions): + return ( + sessions + .with_columns(pl.col("item_ids").list.len().alias("session_len")) + .group_by("uid") + .agg( + pl.col("session_id").alias("session_ids"), + pl.col("session_len").alias("session_lens"), + pl.col("session_len").cum_sum().alias("session_offsets"), + ) + .sort("uid") + ) + + def _compute_item_popularity(self, train_events) -> np.ndarray: + counts = ( + train_events + .group_by("item_id") + .len() + .sort("item_id") + ) + max_item = int(counts["item_id"].max()) + popularity = np.zeros(max_item + 1, dtype=np.int64) + popularity[counts["item_id"].to_numpy()] = counts["len"].to_numpy() + return popularity + + def main() -> None: parser = argparse.ArgumentParser() - parser.add_argument("--dataset", choices=SUPPORTED_DATASETS, help="dataset") + parser.add_argument( + "--dataset", + choices=SUPPORTED_DATASETS, + required=True, + help="dataset", + ) + parser.add_argument( + "--data-path", + default="data/", + help=( + "Root directory for raw + processed data. KuaiRand defaults to " + "the existing `data/` convention; Yambda defaults to `data/` too " + "but is commonly overridden to a shared filesystem location with " + "enough space for the 5b variant (~500 GB)." + ), + ) args = parser.parse_args() + + data_path = args.data_path.rstrip("/") + "/" + if args.dataset == "kuairand-1k": - kuairand_processor = DLRMKuaiRandProcessor( + DLRMKuaiRandProcessor( download_url="https://zenodo.org/records/10439422/files/KuaiRand-1K.tar.gz", - data_path="data/", + data_path=data_path, file_name="KuaiRand-1K.tar.gz", prefix="KuaiRand-1K", - ) - kuairand_processor.preprocess() + ).preprocess() elif args.dataset == "kuairand-27k": - kuairand_processor = DLRMKuaiRandProcessor( + DLRMKuaiRandProcessor( download_url="https://zenodo.org/records/10439422/files/KuaiRand-27K.tar.gz", - data_path="data/", + data_path=data_path, file_name="KuaiRand-27K.tar.gz", prefix="KuaiRand-27K", - ) - kuairand_processor.preprocess() + ).preprocess() + elif args.dataset in YAMBDA_SIZES: + DLRMYambdaProcessor( + data_path=data_path, + size=YAMBDA_SIZES[args.dataset], + ).preprocess() if __name__ == "__main__": diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index 939b42b78..afc7581ee 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -22,12 +22,18 @@ sparse_optimizer_factory_and_class.weight_decay = 0 sparse_optimizer_factory_and_class.eps = 1e-8 sparse_optimizer_factory_and_class.betas = (0.95, 0.999) +# Data root: resolved at runtime from $DLRM_DATA_PATH if set, else the literal +# below. Used by both make_train_test_dataloaders and get_dataset. +DATA_PATH = @env_path() +env_path.key = "DLRM_DATA_PATH" +env_path.default = "/apps/chcai/dlrm_data" + # dataloader configs make_train_test_dataloaders.batch_size = %batch_size make_train_test_dataloaders.eval_batch_size = 32 make_train_test_dataloaders.dataset_type = %dataset make_train_test_dataloaders.train_split_percentage = 0.90 -make_train_test_dataloaders.new_path_prefix = "/apps/chcai/dlrm_data" +make_train_test_dataloaders.new_path_prefix = %DATA_PATH make_train_test_dataloaders.num_workers = %num_workers make_train_test_dataloaders.prefetch_factor = %prefetch_factor make_train_test_dataloaders.num_blocks = 1 @@ -36,7 +42,7 @@ make_train_test_dataloaders.num_blocks = 1 make_optimizer_and_shard.hbm_cap_gb = 260 get_dataset.name = %dataset -get_dataset.new_path_prefix = "/apps/chcai/dlrm_data" +get_dataset.new_path_prefix = %DATA_PATH # Per-pool truncation cap. Cache is keyed by L on disk under # /hstu_cache_L/; switching L reuses an existing cache # (L=2048 was built in a prior session, no rebuild needed). diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py index 43d641a0c..9f60c162a 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py @@ -508,6 +508,21 @@ def reset(self, mode: str = "train"): ] +@gin.configurable +def env_path(key: str = "", default: str = "") -> str: + """Resolve a path from os.environ[key], falling back to `default`. + + Intended as a gin macro so paths can be overridden via env vars without + editing the gin file. Example gin usage: + + DATA_PATH = @env_path() + env_path.key = "DLRM_DATA_PATH" + env_path.default = "/some/default/path" + make_train_test_dataloaders.new_path_prefix = %DATA_PATH + """ + return os.environ.get(key, default) if key else default + + @gin.configurable def get_dataset(name: str, new_path_prefix: str = "", history_length: Optional[int] = None): """ From c639f6a9de5093300540f657f1e7fe9e28137af8 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Sat, 30 May 2026 05:37:25 +0000 Subject: [PATCH 007/127] Fix NCCL init: set CUDA device before init_process_group Every rank's first CUDA context was landing on GPU 0 (the default device), so NCCL bound its communicators there before set_device switched to the correct GPU. This leaked allocations on GPU 0 across all 8 ranks and caused spurious OOMs during embedding-table init at high HBM caps. Moving set_device above init_process_group and passing device_id ensures each rank's NCCL state is created on its own GPU. --- .../generative_recommenders/dlrm_v3/train/utils.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py index 90669c1a6..acf2f14dd 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -79,18 +79,22 @@ def setup( BACKEND = dist.Backend.NCCL TIMEOUT = 1800 + # set device BEFORE init_process_group so NCCL binds this rank to its + # own GPU; otherwise every rank's first CUDA context lands on GPU 0, + # leaving stale allocations and triggering OOMs on rank 0. + torch.cuda.set_device(device) + # initialize the process group if not dist.is_initialized(): - dist.init_process_group("nccl", rank=rank, world_size=world_size) + dist.init_process_group( + "nccl", rank=rank, world_size=world_size, device_id=device + ) pg = dist.new_group( backend=BACKEND, timeout=timedelta(seconds=TIMEOUT), ) - # set device - torch.cuda.set_device(device) - return pg From d76f96b6cac67569829fee37e962655253edae27 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Sat, 30 May 2026 23:04:25 -0500 Subject: [PATCH 008/127] Profiler: write traces to local disk under /results// dlrm_v3/utils.py: - Replace the hardcoded manifold:// URL in _on_trace_ready_fn with a local trace_dir (default /tmp/dlrm_v3_traces). Filename now follows trace_step{step}_rank{rank}.json so per-rank captures don't collide. - Add _multi_window_schedule helper: a torch.profiler schedule that fires around each step in trace_steps=[...] (warmup before, active after, RECORD_AND_SAVE at the last active step). Lets one run capture multiple windows (e.g. early-step + steady-state) without re-running. - Make Profiler @gin.configurable. New knobs: trace_dir, trace_steps, wait, warmup, repeat, record_shapes, profile_memory, with_stack, with_flops, with_modules. Defaults preserve the prior single-window behavior (wait=10, warmup=20, active=50, repeat=1) so existing callers are unaffected. - Add run_results_dir(run_name) gin macro: resolves to /results//. Used as the canonical output prefix for traces (and any future per-run artifacts). recommendation_v4/ is bind-mounted into the training container, so files written through this helper persist on the host. train/gin/yambda_5b.gin: - Wire RUN_NAME env override -> run_results_dir(run_name=%RUN_NAME) -> Profiler.trace_dir. Sets trace_steps=[52], warmup=5, active=5 (capture the 5-step window 52-56 on every rank). - Toggle train_eval_loop.output_trace = True so the profiler actually instantiates. .gitignore: - Add results/ alongside the existing tmp/exps/ckpts/ runtime directories so per-run trace dumps don't show up in git status. Verified: 8x MI350X TRITON yambda-5b run at bs=32 seq=2k drops 8 well-formed trace_step62_rank{0..7}.json files (~37 MB each) into recommendation_v4/results/default/; visible on the host immediately. Co-Authored-By: Claude Opus 4.7 --- recommendation_v4/.gitignore | 1 + .../dlrm_v3/train/gin/yambda_5b.gin | 13 ++ .../generative_recommenders/dlrm_v3/utils.py | 157 ++++++++++++------ 3 files changed, 122 insertions(+), 49 deletions(-) diff --git a/recommendation_v4/.gitignore b/recommendation_v4/.gitignore index 560f823c4..5edddc5b3 100644 --- a/recommendation_v4/.gitignore +++ b/recommendation_v4/.gitignore @@ -2,6 +2,7 @@ tmp/ exps/ ckpts/ +results/ # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index afc7581ee..01cb92ed6 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -59,6 +59,19 @@ train_eval_loop.metric_log_frequency = 50 train_eval_loop.eval_frequency = 5000 train_eval_loop.num_eval_batches = 500 train_eval_loop.checkpoint_frequency = 1000000000 # disable mid-training checkpoints (disk-full guard) +train_eval_loop.output_trace = True + +# Run name → recommendation_v4/results// (override via $RUN_NAME env). +RUN_NAME = @env_path() +env_path.key = "RUN_NAME" +env_path.default = "default" +run_results_dir.run_name = %RUN_NAME + +# profiler: capture a 5-step window starting at step 52, every rank. +Profiler.trace_steps = [52] +Profiler.warmup = 5 +Profiler.active = 5 +Profiler.trace_dir = @run_results_dir() # logger variables MetricsLogger.tensorboard_log_path = "/tmp/tb/yambda_5b/" diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py index 9f60c162a..3c534dafc 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py @@ -21,6 +21,7 @@ import logging import os import time +from pathlib import Path from typing import Callable, Dict, List, Optional import gin @@ -81,54 +82,41 @@ def _compute(self) -> List[MetricComputationReport]: def _on_trace_ready_fn( rank: Optional[int] = None, + trace_dir: str = "/tmp/dlrm_v3_traces", ) -> Callable[[torch.profiler.profile], None]: - """ - Create a callback function for handling profiler trace output. - - Args: - rank: Optional process rank for distributed training (included in filename). + """Create the on_trace_ready callback that exports a chrome trace to disk. - Returns: - A callback function that exports profiler traces to Manifold storage. + Filename follows the convention ``trace_step{step}_rank{rank}.json`` so + multi-rank captures don't collide and ``scripts/stitch_traces.py`` can + merge them by step number. """ def handle_fn(p: torch.profiler.profile) -> None: - bucket_name = "hammer_gpu_traces" - pid = os.getpid() - rank_str = f"_rank_{rank}" if rank is not None else "" - file_name = f"libkineto_activities_{pid}_{rank_str}.json" - manifold_path = "tree/dlrm_v3_bench" - target_object_name = manifold_path + "/" + file_name + ".gz" - path = f"manifold://{bucket_name}/{manifold_path}/{file_name}" + os.makedirs(trace_dir, exist_ok=True) + step = getattr(p, "step_num", 0) + rank_str = f"_rank{rank}" if rank is not None else "" + file_name = f"trace_step{step}{rank_str}.json" + path = os.path.join(trace_dir, file_name) logger.warning( p.key_averages(group_by_input_shape=True).table( sort_by="self_cuda_time_total" ) ) - logger.warning( - f"trace url: https://www.internalfb.com/intern/perfdoctor/trace_view?filepath={target_object_name}&bucket={bucket_name}" - ) p.export_chrome_trace(path) + logger.warning(f"Trace written to: {path}") return handle_fn -def profiler_or_nullcontext(enabled: bool, with_stack: bool): - """ - Create a profiler context manager or null context based on enabled flag. - - Args: - enabled: Whether to enable profiling. - with_stack: Whether to include stack traces in profile. - - Returns: - Either a torch.profiler.profile context manager or nullcontext. - """ +def profiler_or_nullcontext( + enabled: bool, with_stack: bool, trace_dir: str = "/tmp/dlrm_v3_traces" +): + """One-shot profile context for ad-hoc captures (no scheduling).""" return ( profile( # pyre-fixme[16]: Module `profiler` has no attribute `ProfilerActivity`. activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA], - on_trace_ready=_on_trace_ready_fn(), + on_trace_ready=_on_trace_ready_fn(trace_dir=trace_dir), with_stack=with_stack, ) if enabled @@ -136,35 +124,85 @@ def profiler_or_nullcontext(enabled: bool, with_stack: bool): ) -class Profiler: +def _multi_window_schedule( + trace_steps, + warmup: int, + active: int, +): + """Custom schedule that profiles around each step in ``trace_steps``. + + Step s gets: + [s - warmup, s) -> WARMUP + [s, s + active - 1) -> RECORD + s + active - 1 -> RECORD_AND_SAVE """ - Wrapper around PyTorch profiler with scheduled profiling. + windows = [(s - warmup, s, s + active) for s in sorted(trace_steps)] - Implements a wait-warmup-active schedule for controlled profiling that - avoids startup noise and captures representative performance data. + def schedule_fn(step: int) -> torch.profiler.ProfilerAction: + for warmup_start, active_start, active_end in windows: + if warmup_start <= step < active_start: + return torch.profiler.ProfilerAction.WARMUP + if active_start <= step < active_end - 1: + return torch.profiler.ProfilerAction.RECORD + if step == active_end - 1: + return torch.profiler.ProfilerAction.RECORD_AND_SAVE + return torch.profiler.ProfilerAction.NONE - Args: - rank: Process rank for trace file naming. - active: Number of active profiling steps (default: 50). + return schedule_fn + + +@gin.configurable +class Profiler: + """Scheduled torch.profiler wrapper that writes Chrome traces to disk. + + Two modes (set via gin): + + * Single window (default): ``wait=10, warmup=20, active=50, repeat=1``. + Captures one contiguous window starting after ``wait`` steps. + * Multi-window: ``trace_steps=[500, 1000, 5000]`` (overrides wait+repeat). + Captures a separate window around each listed step. + + All knobs are gin-tunable, e.g. in a gin file:: + + Profiler.trace_dir = "/apps/chcai/dlrm_runs/exp42/trace" + Profiler.trace_steps = [500, 1000, 5000] + Profiler.warmup = 5 + Profiler.active = 10 """ - def __init__(self, rank, active: int = 50) -> None: + def __init__( + self, + rank: int, + active: int = 50, + wait: int = 10, + warmup: int = 20, + repeat: int = 1, + trace_steps: Optional[List[int]] = None, + trace_dir: str = "/tmp/dlrm_v3_traces", + record_shapes: bool = True, + profile_memory: bool = False, + with_stack: bool = False, + with_flops: bool = False, + with_modules: bool = False, + ) -> None: self.rank = rank + self.trace_dir = trace_dir + if trace_steps: + sched = _multi_window_schedule(trace_steps, warmup, active) + else: + sched = torch.profiler.schedule( + wait=wait, warmup=warmup, active=active, repeat=repeat + ) self._profiler: profiler.profile = torch.profiler.profile( - schedule=torch.profiler.schedule( - wait=10, - warmup=20, - active=active, - repeat=1, - ), - on_trace_ready=_on_trace_ready_fn(self.rank), + schedule=sched, + on_trace_ready=_on_trace_ready_fn(self.rank, trace_dir), # pyre-fixme[16]: Module `profiler` has no attribute `ProfilerActivity`. activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA], - record_shapes=True, - profile_memory=False, - with_stack=False, - with_flops=False, - with_modules=False, + record_shapes=record_shapes, + profile_memory=profile_memory, + with_stack=with_stack, + with_flops=with_flops, + with_modules=with_modules, ) def step(self) -> None: @@ -523,6 +561,27 @@ def env_path(key: str = "", default: str = "") -> str: return os.environ.get(key, default) if key else default +@gin.configurable +def run_results_dir(run_name: str = "default", subdir: str = "results") -> str: + """Resolve ``//`` from this file's location. + + Used as a gin macro to give per-run output directories that persist on the + host (recommendation_v4 is bind-mounted into the training container). + + Example gin usage:: + + RUN_NAME = @env_path() + env_path.key = "RUN_NAME" + env_path.default = "default" + run_results_dir.run_name = %RUN_NAME + Profiler.trace_dir = @run_results_dir() + """ + # utils.py lives at /generative_recommenders/dlrm_v3/utils.py; + # parents[2] climbs to /. + repo_root = Path(__file__).resolve().parents[2] + return str(repo_root / subdir / run_name) + + @gin.configurable def get_dataset(name: str, new_path_prefix: str = "", history_length: Optional[int] = None): """ From 4a2a7bde225191e28f66817762a127fe139d8793 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Sun, 31 May 2026 00:09:01 -0500 Subject: [PATCH 009/127] Tracing fixes: gin scoping, drop active=10 override, intuitive filenames, trim_warmup dlrm_v3/utils.py * Add run_results_dir(run_name) gin macro (resolves to /results//) so trace artifacts persist on the host via the bind-mount. * Add _trim_warmup_from_trace post-processor: dedupes ProfilerStep spans by name first, then keeps only the last N unique steps' worth of events. Drops WARMUP-phase events that torch.profiler otherwise includes in the chrome trace. * Add trim_warmup kwarg (default True) on Profiler; auto-invokes the trimmer with N=active so the exported file matches the user-requested active window. * Filename now uses trace_steps[i] (the user-requested step) as the {step} label when multi-window mode is in use, instead of torch.profiler's internal step_num (which is off by ~warmup+active from the schedule trigger and confused everyone). train/utils.py * Drop hardcoded `active=10` from the four `Profiler(rank, active=10)` call sites in train_loop / train_eval_loop. Positional args block gin overrides; once removed, Profiler.active in gin (default 50) and user gin bindings actually take effect. train/gin/yambda_5b.gin * Fix env_path scoping collision: both DATA_PATH and RUN_NAME used the unscoped @env_path() configurable, which made the second binding's `env_path.key = "RUN_NAME"` overwrite the first's `env_path.key = "DLRM_DATA_PATH"`. Both names then resolved via the same env var (whichever was last), pointing DATA_PATH at trace_run2/ and breaking dataset loads. Fixed by giving each call site its own scope: @data/env_path() and @run/env_path(), each with independent .key/.default bindings. * Set Profiler.trace_steps=[52], warmup=1, active=5; let trim_warmup default to True so the exported trace contains exactly 5 active ProfilerStep events. Verified end-to-end: - Run with RUN_NAME=trace_run2 writes results/trace_run2/trace_step52_ rank{0..7}.json (~19 MB each), step labels match trace_steps gin. - Triton cache persisted across runs: cold start ~6 min -> warm start ~2 min for autotune-to-first-step. Co-Authored-By: Claude Opus 4.7 --- .../dlrm_v3/train/gin/yambda_5b.gin | 20 ++-- .../dlrm_v3/train/utils.py | 8 +- .../generative_recommenders/dlrm_v3/utils.py | 104 +++++++++++++++++- 3 files changed, 115 insertions(+), 17 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index 01cb92ed6..b090695dc 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -24,9 +24,12 @@ sparse_optimizer_factory_and_class.betas = (0.95, 0.999) # Data root: resolved at runtime from $DLRM_DATA_PATH if set, else the literal # below. Used by both make_train_test_dataloaders and get_dataset. -DATA_PATH = @env_path() -env_path.key = "DLRM_DATA_PATH" -env_path.default = "/apps/chcai/dlrm_data" +# Scoped (`data/env_path`) so this binding doesn't collide with the RUN_NAME +# env_path binding below — every distinct env_path() call site needs its own +# scope or the later `env_path.key=...` overrides earlier ones. +DATA_PATH = @data/env_path() +data/env_path.key = "DLRM_DATA_PATH" +data/env_path.default = "/apps/chcai/dlrm_data" # dataloader configs make_train_test_dataloaders.batch_size = %batch_size @@ -62,14 +65,17 @@ train_eval_loop.checkpoint_frequency = 1000000000 # disable mid-training checkp train_eval_loop.output_trace = True # Run name → recommendation_v4/results// (override via $RUN_NAME env). -RUN_NAME = @env_path() -env_path.key = "RUN_NAME" -env_path.default = "default" +RUN_NAME = @run/env_path() +run/env_path.key = "RUN_NAME" +run/env_path.default = "default" run_results_dir.run_name = %RUN_NAME # profiler: capture a 5-step window starting at step 52, every rank. +# `trim_warmup = True` (default) post-filters the chrome trace so only the 5 +# `active` steps appear; the 1 WARMUP step still runs (lets CUPTI/ROCprof +# settle) but its events are dropped from the output file. Profiler.trace_steps = [52] -Profiler.warmup = 5 +Profiler.warmup = 1 Profiler.active = 5 Profiler.trace_dir = @run_results_dir() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py index acf2f14dd..2ff025394 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -498,7 +498,7 @@ def train_loop( ) -> None: model.train() batch_idx: int = start_batch_idx - profiler = Profiler(rank, active=10) if output_trace else None + profiler = Profiler(rank) if output_trace else None for epoch in range(num_epochs): dataloader.sampler.set_epoch(epoch) # pyre-ignore [16] @@ -567,7 +567,7 @@ def eval_loop( ) -> None: model.eval() batch_idx: int = 0 - profiler = Profiler(rank, active=10) if output_trace else None + profiler = Profiler(rank) if output_trace else None metric_logger.reset(mode="eval") with torch.no_grad(): for sample in dataloader: @@ -627,7 +627,7 @@ def train_eval_loop( ) -> None: train_batch_idx: int = start_train_batch_idx eval_batch_idx: int = start_eval_batch_idx - profiler = Profiler(rank, active=10) if output_trace else None + profiler = Profiler(rank) if output_trace else None assert train_dataloader is not None and eval_dataloader is not None eval_data_iterator = iter(eval_dataloader) @@ -752,7 +752,7 @@ def streaming_train_eval_loop( metric_log_frequency: int = 1, checkpoint_frequency: int = 100, ) -> None: - profiler = Profiler(rank, active=10) if output_trace else None + profiler = Profiler(rank) if output_trace else None dataset_class, kwargs = get_dataset() kwargs["embedding_config"] = embedding_table_configs dataset = HammerToTorchDataset( diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py index 3c534dafc..508f894a9 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py @@ -80,22 +80,108 @@ def _compute(self) -> List[MetricComputationReport]: logger = logging.getLogger("utils") +def _trim_warmup_from_trace(path: str, keep_n_active: int) -> None: + """Post-process a chrome trace to drop events from WARMUP-phase steps. + + torch.profiler captures events during BOTH the WARMUP and RECORD phases + of a schedule and writes them all to the exported trace. There is no + built-in flag to exclude WARMUP from the export. We approximate it by: + + 1) Finding all ``ProfilerStep#N`` spans in the file. + 2) Keeping only the last ``keep_n_active`` of them (sorted by start + timestamp) as the "active" range. + 3) Filtering ``traceEvents`` to events whose ``ts`` falls inside that + range. Metadata events (``ph='M'``) are always preserved. + + Mutates the file in place. + """ + import json as _json + with open(path) as f: + d = _json.load(f) + events = d.get("traceEvents", []) + + # ProfilerStep spans mark training-step boundaries; we filter by their + # time ranges rather than by name index because step numbering can offset + # between schedule_fn argument and the value printed in the trace. + # torch.profiler emits one ProfilerStep#N span per CPU thread that ran + # during that step, so dedupe by name first so "5 active steps" means + # 5 distinct step numbers, not 5 spans. + name_to_span: Dict[str, tuple] = {} + for e in events: + nm = e.get("name", "") + if "ProfilerStep" not in nm or e.get("ph") != "X" or "ts" not in e: + continue + ts = e["ts"] + end = ts + e.get("dur", 0) + prev = name_to_span.get(nm) + if prev is None: + name_to_span[nm] = (ts, end) + else: + name_to_span[nm] = (min(prev[0], ts), max(prev[1], end)) + if len(name_to_span) <= keep_n_active: + return + sorted_spans = sorted(name_to_span.values()) + active = sorted_spans[-keep_n_active:] + t_start = min(s for s, _ in active) + t_end = max(e for _, e in active) + + def _keep(e: dict) -> bool: + if e.get("ph") == "M": + return True + ts = e.get("ts") + if ts is None: + return True + return t_start <= ts < t_end + + kept = [e for e in events if _keep(e)] + d["traceEvents"] = kept + with open(path, "w") as f: + _json.dump(d, f) + logger.warning( + f"Trimmed WARMUP events from {path}: {len(events):,} -> {len(kept):,} " + f"(kept active range [{t_start:.0f}, {t_end:.0f}] us)" + ) + + def _on_trace_ready_fn( rank: Optional[int] = None, trace_dir: str = "/tmp/dlrm_v3_traces", + keep_n_active: Optional[int] = None, + trace_steps: Optional[List[int]] = None, ) -> Callable[[torch.profiler.profile], None]: """Create the on_trace_ready callback that exports a chrome trace to disk. - Filename follows the convention ``trace_step{step}_rank{rank}.json`` so - multi-rank captures don't collide and ``scripts/stitch_traces.py`` can - merge them by step number. + Filename follows ``trace_step{step}_rank{rank}.json`` so multi-rank + captures don't collide and ``scripts/stitch_traces.py`` can merge them + by step number. + + The ``{step}`` label: + + * If ``trace_steps`` is provided (multi-window mode), the Nth callback + invocation labels its file with ``trace_steps[N]`` -- i.e. the + user-requested step that triggered the window. This is the most + intuitive labelling. + * Otherwise falls back to ``p.step_num`` (torch.profiler's internal + counter at trigger time, off by ~warmup+active from the schedule + arg). + + If ``keep_n_active`` is set, the exported file is post-processed to keep + only the last N ProfilerStep-spans worth of events (i.e. drop WARMUP). """ + state = {"fire_count": 0} def handle_fn(p: torch.profiler.profile) -> None: os.makedirs(trace_dir, exist_ok=True) - step = getattr(p, "step_num", 0) + if trace_steps: + i = state["fire_count"] + step_label = ( + trace_steps[i] if i < len(trace_steps) else getattr(p, "step_num", 0) + ) + else: + step_label = getattr(p, "step_num", 0) + state["fire_count"] += 1 rank_str = f"_rank{rank}" if rank is not None else "" - file_name = f"trace_step{step}{rank_str}.json" + file_name = f"trace_step{step_label}{rank_str}.json" path = os.path.join(trace_dir, file_name) logger.warning( p.key_averages(group_by_input_shape=True).table( @@ -104,6 +190,8 @@ def handle_fn(p: torch.profiler.profile) -> None: ) p.export_chrome_trace(path) logger.warning(f"Trace written to: {path}") + if keep_n_active is not None and keep_n_active > 0: + _trim_warmup_from_trace(path, keep_n_active) return handle_fn @@ -179,6 +267,7 @@ def __init__( repeat: int = 1, trace_steps: Optional[List[int]] = None, trace_dir: str = "/tmp/dlrm_v3_traces", + trim_warmup: bool = True, record_shapes: bool = True, profile_memory: bool = False, with_stack: bool = False, @@ -193,9 +282,12 @@ def __init__( sched = torch.profiler.schedule( wait=wait, warmup=warmup, active=active, repeat=repeat ) + keep_n = active if trim_warmup else None self._profiler: profiler.profile = torch.profiler.profile( schedule=sched, - on_trace_ready=_on_trace_ready_fn(self.rank, trace_dir), + on_trace_ready=_on_trace_ready_fn( + self.rank, trace_dir, keep_n, trace_steps + ), # pyre-fixme[16]: Module `profiler` has no attribute `ProfilerActivity`. activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA], record_shapes=record_shapes, From 9b0ae8b3607e1f306c3c19a173e65029c437930e Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Mon, 1 Jun 2026 02:04:49 -0500 Subject: [PATCH 010/127] =?UTF-8?q?gin:=20history=5Flength=202048=20?= =?UTF-8?q?=E2=86=92=202039=20+=20expanded=20per-pool=20comment?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 2048 was chosen for "round number near max_seq_len" but it slightly overflows the per-sample budget: 3 * (2048//3) + 9 = 2055 > 2048, so the dataset truncates ~7 UIH events to fit. 2039 makes the math exact (3 * 679 + 9 = 2046 ≤ 2048) so no truncation. Comment block expanded to document: - The 3-pool gather semantic (L//3 events per pool, interleaved chronologically). - The like-pool under-fill observation: like events are only 1.9% of yambda corpus and max user lifetime is ~28k events, so the like pool fills to ~105 events per anchor on average (not 679). TRITON's jagged attention skips the unfilled slots, so under-fill costs sequence budget but not GPU compute. No code change. Cache for L=2039 already built and reused. Co-Authored-By: Claude Opus 4.7 --- .../dlrm_v3/train/gin/yambda_5b.gin | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index b090695dc..f9b451f33 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -46,10 +46,20 @@ make_optimizer_and_shard.hbm_cap_gb = 260 get_dataset.name = %dataset get_dataset.new_path_prefix = %DATA_PATH -# Per-pool truncation cap. Cache is keyed by L on disk under -# /hstu_cache_L/; switching L reuses an existing cache -# (L=2048 was built in a prior session, no rebuild needed). -get_dataset.history_length = 2048 +# Total user-interaction-history (UIH) budget per sample, distributed evenly +# across 3 behaviour pools (listen+ / like / skip) at L//3 events each. +# Per-sample sequence the model sees = +# 3 × (L // 3) + 8 contextual + 1 candidate +# Choosing 2039 makes 3 × 679 + 9 = 2046, the largest value that fits +# get_hstu_configs.max_seq_len = 2048 with no dataset-side truncation. +# Larger L overflows the budget; the dataset truncates UIH events to fit. +# Note: like events are only 1.9% of the yambda corpus and max user lifetime +# is ~28k events, so the like pool fills to ~105 events per anchor on +# average (not 679) — TRITON's jagged attention skips the unfilled slots, +# so the under-fill costs sequence budget but not GPU compute. +# Cache is keyed by L on disk under /hstu_cache_L/; +# switching L reuses an existing cache or builds a new one (~5 min). +get_dataset.history_length = 2039 # Model-side attention budget. Dataset truncates UIH to fit this value if # `history_length + contextual + candidate` would overflow. From 2b58a63f113869a3d78f98bfc0fef415b10ffaa8 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Mon, 1 Jun 2026 02:17:35 -0500 Subject: [PATCH 011/127] =?UTF-8?q?README:=20rewrite=20for=20yambda-5b=20f?= =?UTF-8?q?ork=20=E2=80=94=20upstream=20link,=20data=20prep,=20per-pool=20?= =?UTF-8?q?gather?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Documents the fork's scope (yambda-5b on HSTU dlrm_v3 path), per-pool gather strategy with effective fill table, and dataset statistics. Sections indexed 1–5 for navigation. Co-Authored-By: Claude Opus 4.7 --- recommendation_v4/README.MD | 185 ++++++++++++++++++------------------ 1 file changed, 93 insertions(+), 92 deletions(-) diff --git a/recommendation_v4/README.MD b/recommendation_v4/README.MD index a60d0d3d7..f22f1e165 100644 --- a/recommendation_v4/README.MD +++ b/recommendation_v4/README.MD @@ -1,135 +1,136 @@ -# Generative Recommenders +# Recommendation v4 — HSTU + Yambda-5b -Repository hosting code for ``Actions Speak Louder than Words: Trillion-Parameter Sequential Transducers for Generative Recommendations`` ([ICML'24 paper](https://proceedings.mlr.press/v235/zhai24a.html)) and related code, where we demonstrate that the ubiquitously used classical deep learning recommendation paradigm (DLRMs) can be reformulated as a generative modeling problem (Generative Recommenders or GRs) to overcome known compute scaling bottlenecks, propose efficient algorithms such as HSTU and M-FALCON to accelerate training and inference for large-scale sequential models by 10x-1000x, and demonstrate scaling law for the first-time in deployed, billion-user scale recommendation systems. +This is a fork of [meta-recsys/generative-recommenders](https://github.com/meta-recsys/generative-recommenders) extended to train HSTU (Hierarchical Sequential Transducer Units) on the [Yambda-5b](https://huggingface.co/datasets/yandex/yambda) music-recommendation dataset, sized as an MLPerf-style training benchmark inside the `mlcommons/training` tree. -## Getting started +For the original repository and the underlying ICML'24 paper (*Actions Speak Louder than Words*), see the upstream README at the link above. This README focuses on what this fork adds: the Yambda data pipeline, the per-pool gather strategy, and how the data feeds into the HSTU `modules/` (dlrm_v3) path. -We recommend using `requirements.txt`. This has been tested with Ubuntu 22.04, CUDA 12.4, and Python 3.10. +## 1. Quick start (8-GPU Yambda) ```bash -pip3 install -r requirements.txt +docker exec yambda_8gpu bash -c \ + 'cd /workspace/recommendation_v4 && bash scripts/launch_smoke_8gpu.sh' ``` -Alternatively, you can manually install PyTorch based on official instructions. Then, +Override the data path or run name without editing the gin: ```bash -pip3 install gin-config pandas fbgemm_gpu torchrec tensorboard +DLRM_DATA_PATH=/apps/chcai/dlrm_data \ +RUN_NAME=my_experiment \ +bash scripts/launch_smoke_8gpu.sh ``` -## Experiments +Data path resolves at runtime via `env_path` gin macros (see [yambda_5b.gin](generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin)). Traces and any per-run outputs land in `results//`. -### Public Experiments - -To reproduce the public experiments in our paper (traditional sequential recommender setting, Section 4.1.1) on MovieLens and Amazon Reviews in the paper, please follow these steps: - -#### Download and preprocess data. - -```bash -mkdir -p tmp/ && python3 preprocess_public_data.py -``` - -A GPU with 24GB or more HBM should work for most datasets. +## 2. Data preparation ```bash -CUDA_VISIBLE_DEVICES=0 python3 main.py --gin_config_file=configs/ml-1m/hstu-sampled-softmax-n128-large-final.gin --master_port=12345 +python3 -m generative_recommenders.dlrm_v3.preprocess_public_data \ + --dataset yambda-5b --data-path /apps/chcai/dlrm_data ``` -Other configurations are included in configs/ml-1m, configs/ml-20m, and configs/amzn-books to make reproducing these experiments easier. - -#### Verify results. +This downloads the 5b variant of [yandex/yambda](https://huggingface.co/datasets/yandex/yambda) from HuggingFace, then: -By default we write experimental logs to exps/. We can launch tensorboard with something like the following: +1. **Encodes** the raw `event_type` string column into a uint8 lookup (listen=0, like=1, dislike=2, unlike=3, undislike=4). +2. **Splits** events temporally — 300 train days, 30-min gap, 1 test day — by `Global Temporal Split` (GTS). +3. **Segments** per-user event timelines into sessions on a 30-min inactivity gap. +4. **Computes** per-item popularity for downstream metric weighting. +5. **Writes** the layout `DLRMv3YambdaDataset` expects: -```bash -tensorboard --logdir ~/generative-recommenders/exps/ml-1m-l200/ --port 24001 --bind_all -tensorboard --logdir ~/generative-recommenders/exps/ml-20m-l200/ --port 24001 --bind_all -tensorboard --logdir ~/generative-recommenders/exps/amzn-books-l50/ --port 24001 --bind_all +``` +/ +├── raw/5b/multi_event.parquet 50 GB (downloaded) +├── shared_metadata/ +│ ├── artist_item_mapping.parquet 60 MB +│ ├── album_item_mapping.parquet 76 MB +│ └── embeddings.parquet 18 GB (unused by HSTU training) +└── processed_5b/ + ├── train_sessions.parquet 47 GB ← main training input + ├── test_events.parquet 152 MB + ├── session_index.parquet 600 MB + ├── item_popularity.npy 75 MB + └── split_meta.json anchor + boundary stats ``` -With the provided configuration (.gin) files, you should be able to reproduce the following results (verified as of 04/15/2024): - -**MovieLens-1M (ML-1M)**: - -| Method | HR@10 | NDCG@10 | HR@50 | NDCG@50 | HR@200 | NDCG@200 | -| ------------- | ---------------- | ----------------| --------------- | --------------- | --------------- | --------------- | -| SASRec | 0.2853 | 0.1603 | 0.5474 | 0.2185 | 0.7528 | 0.2498 | -| BERT4Rec | 0.2843 (-0.4%) | 0.1537 (-4.1%) | | | | | -| GRU4Rec | 0.2811 (-1.5%) | 0.1648 (+2.8%) | | | | | -| HSTU | 0.3097 (+8.6%) | 0.1720 (+7.3%) | 0.5754 (+5.1%) | 0.2307 (+5.6%) | 0.7716 (+2.5%) | 0.2606 (+4.3%) | -| HSTU-large | **0.3294 (+15.5%)** | **0.1893 (+18.1%)** | **0.5935 (+8.4%)** | **0.2481 (+13.5%)** | **0.7839 (+4.1%)** | **0.2771 (+10.9%)** | +For smaller variants (yambda-50m / yambda-500m) substitute the dataset name. The preprocessor takes ~2 min for 50m and ~53 min for 5b end-to-end. -**MovieLens-20M (ML-20M)**: +## 3. Yambda dataset statistics -| Method | HR@10 | NDCG@10 | HR@50 | NDCG@50 | HR@200 | NDCG@200 | -| ------------- | ---------------- | --------------- | --------------- | --------------- | --------------- | --------------- | -| SASRec | 0.2889 | 0.1621 | 0.5503 | 0.2199 | 0.7661 | 0.2527 | -| BERT4Rec | 0.2816 (-2.5%) | 0.1703 (+5.1%) | | | | | -| GRU4Rec | 0.2813 (-2.6%) | 0.1730 (+6.7%) | | | | | -| HSTU | 0.3273 (+13.3%) | 0.1895 (+16.9%) | 0.5889 (+7.0%) | 0.2473 (+12.5%) | 0.7952 (+3.8%) | 0.2787 (+10.3%) | -| HSTU-large | **0.3556 (+23.1%)** | **0.2098 (+29.4%)** | **0.6143 (+11.6%)** | **0.2671 (+21.5%)** | **0.8074 (+5.4%)** | **0.2965 (+17.4%)** | +Numbers from the 5b variant, after preprocessing: -**Amazon Reviews (Books)**: +| | | +|---|---| +| Total interaction events | **4.76 B** | +| Unique users | **1.00 M** | +| Max events per user | 27,738 | +| Median events per user | 2,695 | +| Mean events per user | 4,763 | +| Train events (300d) | 4.76 B | +| Test events (1d) | 22.4 M | +| Training positions (≥2039 prior events filter) | **3.23 B** | +| Item catalog size | 9.39 M | -| Method | HR@10 | NDCG@10 | HR@50 | NDCG@50 | HR@200 | NDCG@200 | -| ------------- | ---------------- | ----------------|---------------- | --------------- | --------------- | --------------- | -| SASRec | 0.0306 | 0.0164 | 0.0754 | 0.0260 | 0.1431 | 0.0362 | -| HSTU | 0.0416 (+36.4%) | 0.0227 (+39.3%) | 0.0957 (+27.1%) | 0.0344 (+32.3%) | 0.1735 (+21.3%) | 0.0461 (+27.7%) | -| HSTU-large | **0.0478 (+56.7%)** | **0.0262 (+60.7%)** | **0.1082 (+43.7%)** | **0.0393 (+51.2%)** | **0.1908 (+33.4%)** | **0.0517 (+43.2%)** | +### 3.1 Per-event-type distribution (across the full 4.76 B corpus) -for all three tables above, the ``SASRec`` rows are based on [Self-Attentive Sequential Recommendation](https://arxiv.org/abs/1808.09781) but with the original binary cross entropy loss -replaced with sampled softmax losses proposed in [Revisiting Neural Retrieval on Accelerators](https://arxiv.org/abs/2306.04039). These rows are reproducible with ``configs/*/sasrec-*-final.gin``. -The ``BERT4Rec`` and ``GRU4Rec`` rows are based on results reported by [Turning Dross Into Gold Loss: is BERT4Rec really better than SASRec?](https://arxiv.org/abs/2309.07602) - -note that the comparison slightly favors these two, due to them using full negatives whereas the other rows used 128/512 sampled negatives. The ``HSTU`` and ``HSTU-large`` rows are based on [Actions Speak Louder than Words: Trillion-Parameter Sequential Transducers for Generative Recommendations](https://arxiv.org/abs/2402.17152); in particular, HSTU rows utilize identical configurations as SASRec. ``HSTU`` and ``HSTU-large`` results can be reproduced with ``configs/*/hstu-*-final.gin``. +| Pool | Definition | Count | Share | +|---|---|---|---| +| **listen_plus (lp)** | `is_listen AND played_ratio ≥ 50%` | 2.92 B | **61.3%** | +| **skip** | `is_listen AND played_ratio < 50%` | 1.71 B | **35.9%** | +| **like** | explicit thumbs-up action | 89 M | **1.9%** | +| other | dislike / unlike / undislike | 47 M | 1.0% | -### Synthetic Dataset / MovieLens-3B +The `like` pool is roughly **30× rarer** than `lp` — important context for the gather strategy below. -We support generating synthetic dataset with fractal expansion introduced in https://arxiv.org/abs/1901.08910. This allows us to expand the current 20 million real-world ratings in ML-20M to 3 billion. +## 4. How data is fed to HSTU -To download the pre-generated synthetic dataset: +For every training anchor (a LISTEN event with ≥ `history_length` prior events), the dataset builds a `(uih_kjt, candidate_kjt)` pair: -```bash -pip3 install gdown -mkdir -p tmp/ && cd tmp/ -gdown https://drive.google.com/uc?id=1-jZ6k0el7e7PyFnwqMLfqUTRh_Qdumt- -unzip ml-3b.zip && rm ml-3b.zip ``` - -To generate the synthetic dataset on your own: - -```bash -python3 run_fractal_expansion.py --input-csv-file tmp/ml-20m/ratings.csv --write-dataset True --output-prefix tmp/ml-3b/ +UIH (User Interaction History): + ┌─ Sequence features (chronologically interleaved across 3 pools) + │ item_id, artist_id, album_id ← per-position + │ action_weight ← per-position (LP_BIT/LIKE_BIT/SKIP_BIT) + │ action_timestamp, dummy_watch_time ← per-position + └─ Contextual features (length 1 each) + uid + 7 cross-feature hashes (user_x_artist, item_x_hour, …) + = 8 contextual entries + +CANDIDATE (the LISTEN event at the anchor): + item_id, artist_id, album_id, item_query_time, + item_action_weight (LP_BIT if listen_plus, else 0), + item_dummy_watchtime ``` -### Efficiency experiments +The candidate's `action_weight` is **the supervision label**: HSTU's `_get_supervision_labels_and_weights` masks BCE training to `(supervision_bitmask & task_weight) > 0`, with `task_weight = 1` (LP bit) for the single `listen_plus` task — so only listen_plus candidates supervise. -``ops/triton`` contains triton kernels needed for efficiency experiments. ``ops/cpp`` contains efficient CUDA kernels. In particular, ``ops/cpp/hstu_attention`` contains the attention implementation based on [FlashAttention V3](https://github.com/Dao-AILab/flash-attention) with state-of-the-art efficiency on H100 GPUs. +### 4.1 Per-pool gather (the cap = L // 3 strategy) -## DLRM-v3 +The UIH is built by `DLRMv3YambdaDataset._gather_interleaved_history`. For each anchor, it: -We have created a DLRM model using HSTU and have developed benchmarks for both training and inference to faciliate production RecSys use cases. +1. Scans the most recent `scan_window` (default 20,000) events of any type before the anchor, **clipped to user_start** so users with shorter history get a smaller window. +2. From those, takes **the last `L // 3` events** from each of the three pools (lp, like, skip) independently. +3. Concatenates the three streams and **re-sorts chronologically** to produce an interleaved sequence. +4. Tags each event's pool identity into `action_weight` via OR'd bitmask (LP=1, LIKE=2, SKIP=4). -#### Run model training with 4 GPUs +With `L = 2039` and `max_seq_len = 2048`: +- Per-pool cap = `L // 3 = 679` +- Maximum total UIH = `3 × 679 = 2037` events +- Plus `8 contextual + 1 candidate = 9` overhead → 2046 ≤ 2048 model budget (no truncation) -```bash -LOCAL_WORLD_SIZE=4 WORLD_SIZE=4 python3 generative_recommenders/dlrm_v3/train/train_ranker.py --dataset debug --mode train -``` +### 4.2 Effective per-anchor fill on real data -#### Run model inference with 4 GPUs +Because the `like` pool is rare (1.9% of events) and the average user has only ~4,763 lifetime events: -```bash -git clone --recurse-submodules https://github.com/mlcommons/inference.git mlperf_inference -cd mlperf_inference/loadgen -CFLAGS="-std=c++14 -O3" python -m pip install . - -LOCAL_WORLD_SIZE=4 WORLD_SIZE=4 python3 generative_recommenders/dlrm_v3/inference/main.py --dataset debug -``` +| Pool | per-pool cap (L//3) | actual avg fill per anchor | fill rate | +|---|---|---|---| +| lp | 679 | ~673 | **99%** | +| like | 679 | ~105 | **15%** (data-bounded, not cap-bounded) | +| skip | 679 | ~624 | 92% | +| **total UIH** | 2037 max | **~1402** | 69% | -## License -This codebase is Apache 2.0 licensed, as found in the [LICENSE](LICENSE) file. +The `like` cap of 679 is unreachable for yambda data — at the 1.9% global like rate, filling 679 likes would require a user to have ~36k prior events, but the **longest user in the dataset has only 27,738 events total** (and the median user has 2,695). So under-fill on `like` is fundamental to the data. -## Contributors -The overall project is made possible thanks to the joint work from many technical contributors (listed in alphabetical order): +This means the model sees on average ~1,402 UIH events per sample, not the theoretical 2,037. With the TRITON jagged-attention backend the GPU only does work for the actual events, so the under-fill costs **sequence budget but not GPU compute** — no wasted attention work, just less context per sample than the budget suggests. -Adnan Akhundov, Bugra Akyildiz, Shabab Ayub, Alex Bao, Renqin Cai, Jennifer Cao, Xuan Cao, Guoqiang Jerry Chen, Lei Chen, Li Chen, Sean Chen, Xianjie Chen, Huihui Cheng, Weiwei Chu, Ted Cui, Shiyan Deng, Nimit Desai, Fei Ding, Shilin Ding, Francois Fagan, Lu Fang, Leon Gao, Zhaojie Gong, Fangda Gu, Liang Guo, Liz Guo, Jeevan Gyawali, Yuchen Hao, Daisy Shi He, Michael Jiayuan He, Yu He, Samuel Hsia, Jie Hua, Yanzun Huang, Hongyi Jia, Rui Jian, Jian Jin, Rafay Khurram, Rahul Kindi, Changkyu Kim, Yejin Lee, Fu Li, Han Li, Hong Li, Shen Li, Rui Li, Wei Li, Zhijing Li, Lucy Liao, Xueting Liao, Emma Lin, Hao Lin, Chloe Liu, Jingzhou Liu, Xing Liu, Xingyu Liu, Kai Londenberg, Yinghai Lu, Liang Luo, Linjian Ma, Matt Ma, Yun Mao, Bert Maher, Ajit Mathews, Matthew Murphy, Satish Nadathur, Min Ni, Jongsoo Park, Colin Peppler, Jing Qian, Lijing Qin, Jing Shan, Alex Singh, Timothy Shi, Yu Shi, Dennis van der Staay, Xiao Sun, Colin Taylor, Shin-Yeh Tsai, Rohan Varma, Omkar Vichare, Alyssa Wang, Pengchao Wang, Shengzhi Wang, Wenting Wang, Xiaolong Wang, Yueming Wang, Zhiyong Wang, Wei Wei, Bin Wen, Carole-Jean Wu, Yanhong Wu, Eric Xu, Bi Xue, Hong Yan, Zheng Yan, Chao Yang, Junjie Yang, Wen-Yun Yang, Ze Yang, Zimeng Yang, Yuanjun Yao, Chunxing Yin, Daniel Yin, Yiling You, Jiaqi Zhai, Keke Zhai, Yanli Zhao, Zhuoran Zhao, Hui Zhang, Jingjing Zhang, Lu Zhang, Lujia Zhang, Na Zhang, Rui Zhang, Xiong Zhang, Ying Zhang, Zhiyun Zhang, Charles Zheng, Erheng Zhong, Zhao Zhu, Xin Zhuang. +## 5. License -For the initial paper describing the Generative Recommender problem formulation and the algorithms used, including HSTU and M-FALCON, please refer to ``Actions Speak Louder than Words: Trillion-Parameter Sequential Transducers for Generative Recommendations``([ICML'24 paper](https://dl.acm.org/doi/10.5555/3692070.3694484), [slides](https://icml.cc/media/icml-2024/Slides/32684.pdf)). +Apache 2.0 (inherited from upstream). From 03662de7ed538b7189915fd8981fa397389b6b16 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Mon, 1 Jun 2026 02:39:55 -0500 Subject: [PATCH 012/127] gin: make hbm_cap_gb overridable via \$HBM_CAP_GB Adds env_int gin macro (companion to env_path) and wires make_optimizer_and_shard.hbm_cap_gb through it so the per-rank HBM ceiling can be tuned without editing the gin file. Co-Authored-By: Claude Opus 4.7 --- .../dlrm_v3/train/gin/yambda_5b.gin | 7 +++++-- .../generative_recommenders/dlrm_v3/utils.py | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index f9b451f33..cc2c1000d 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -41,8 +41,11 @@ make_train_test_dataloaders.num_workers = %num_workers make_train_test_dataloaders.prefetch_factor = %prefetch_factor make_train_test_dataloaders.num_blocks = 1 -# embedding planner -make_optimizer_and_shard.hbm_cap_gb = 260 +# embedding planner: per-rank HBM ceiling the torchrec sharder targets. +# Override via $HBM_CAP_GB (e.g. lower to 150 to force more CW sharding). +make_optimizer_and_shard.hbm_cap_gb = @env_int() +env_int.key = "HBM_CAP_GB" +env_int.default = 260 get_dataset.name = %dataset get_dataset.new_path_prefix = %DATA_PATH diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py index 508f894a9..f276780c2 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py @@ -653,6 +653,20 @@ def env_path(key: str = "", default: str = "") -> str: return os.environ.get(key, default) if key else default +@gin.configurable +def env_int(key: str = "", default: int = 0) -> int: + """Resolve an int from os.environ[key], falling back to `default`. + + Companion to `env_path` for numeric overrides. Example gin usage: + + make_optimizer_and_shard.hbm_cap_gb = @env_int() + env_int.key = "HBM_CAP_GB" + env_int.default = 260 + """ + raw = os.environ.get(key) if key else None + return int(raw) if raw else default + + @gin.configurable def run_results_dir(run_name: str = "default", subdir: str = "results") -> str: """Resolve ``//`` from this file's location. From bb012a2cfb301a1aa388171d5bce496bec9f2216 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Mon, 1 Jun 2026 19:17:02 +0000 Subject: [PATCH 013/127] docs: add B200 training recipe for yambda-5b Document the container image, dependency versions (native NGC torch 2.10, triton 3.6, source-built fbgemm_gpu, torchrec 1.4.0, polars-u64-idx), gin training configuration, and env vars needed to reproduce the 8x B200 run. --- recommendation_v4/docs/training_recipe.md | 93 +++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 recommendation_v4/docs/training_recipe.md diff --git a/recommendation_v4/docs/training_recipe.md b/recommendation_v4/docs/training_recipe.md new file mode 100644 index 000000000..155dc9afd --- /dev/null +++ b/recommendation_v4/docs/training_recipe.md @@ -0,0 +1,93 @@ +# Training Recipe + +Reproducible environment + configuration for training HSTU / DLRM-v3 on the +`yambda-5b` dataset. + +--- + +## B200 + +Single-node, 8× NVIDIA **B200** (Blackwell, `sm_100`, ~183 GiB HBM each), HSTU +ranker on `yambda-5b` with the **TRITON** HSTU kernel and **bf16** mixed-precision +training. + +### Hardware / host + +| item | value | +|---|---| +| GPUs | 8× NVIDIA B200 (`sm_100`, compute capability 10.0) | +| Host driver | 580.159.03 (reports CUDA 13.2) | +| Forward-compat userspace driver | `libcuda.so.595.45.04` (engaged automatically by the NGC image) | + +### Container image + +``` +nvcr.io/nvidia/pytorch:26.01-py3 +``` + +Digest: `sha256:38ed2ecb2c16d10677006d73fb0a150855d6ec81db8fc66e800b5ae92741007e` + +The image's native PyTorch is kept as-is and must not be reinstalled (so CUPTI +stays matched to the driver and `sm_100` support is preserved). + +### Dependency versions + +| package | version | notes | +|---|---|---| +| **torch** | `2.10.0a0+a36e1d39eb.nv26.01.42222806` (CUDA 13.1) | native to the image; not reinstalled | +| **triton** | `3.6.0` | native to the image; provides `triton.language.make_tensor_descriptor` (required by the TRITON HSTU path) | +| **fbgemm_gpu** | `fbgemm_gpu_nightly-2026.6.1` (CUDA 13.1, `sm_100`) | built from source against the native torch, from FBGEMM commit `939f2da156b05d2f1bcba8c037d613c1098d0db5` (2026-04-29); public wheels are ABI-incompatible with the NGC torch | +| **torchrec** | `1.4.0` | installed with `--no-deps` | +| **polars-u64-idx** | `1.33.1` | 64-bit row index — `yambda-5b` has > 4.29 B rows (overflows stock polars' 32-bit index) | +| CUPTI (for `torch.profiler`) | 13.1 (native) | matches the driver; the `+cu128` stack's CUPTI 12.8 fails on B200 (`CUPTI_ERROR_INVALID_DEVICE`) | + +Additional Python deps: +`xxhash`, `gin-config`, `absl-py`, `pandas`, `tensorboard`, `pyarrow`, `pyyaml`, +`tqdm`, `psutil`, `torchmetrics==1.0.3`, `tensordict`, `pyre-extensions`, +`iopath`, `typing-inspect`. + +### Training configuration + +From `generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin`: + +| parameter | value | gin binding | +|---|---|---| +| batch_size (train) | 32 | `make_train_test_dataloaders.batch_size` | +| eval_batch_size | 32 | `make_train_test_dataloaders.eval_batch_size` | +| num_workers (dataloader) | 4 | `make_train_test_dataloaders.num_workers` | +| prefetch_factor | 8 | `make_train_test_dataloaders.prefetch_factor` | +| num_blocks | 1 | `make_train_test_dataloaders.num_blocks` | +| train_split_percentage | 0.90 | `make_train_test_dataloaders.train_split_percentage` | +| history_length (per-sample UIH budget) | 2039 | `get_dataset.history_length` | +| max_seq_len (attention budget) | 2048 | `get_hstu_configs.max_seq_len` | +| bf16 training | True | `make_model.bf16_training` | +| HBM cap (per GPU) | 150 GiB | `make_optimizer_and_shard.hbm_cap_gb` (env `HBM_CAP_GB`) | +| dense optimizer | Adam, lr 1e-3, betas (0.95, 0.999), eps 1e-8 | `dense_optimizer_factory_and_class.*` | +| sparse optimizer | RowWiseAdagrad, lr 1e-3, betas (0.95, 0.999), eps 1e-8 | `sparse_optimizer_factory_and_class.*` | +| world_size | 8 | `MetricsLogger.world_size` | + +Effective global batch = `batch_size × world_size = 32 × 8 = 256` samples/step. + +### Environment variables + +| var | value | purpose | +|---|---|---| +| `HSTU_HAMMER_KERNEL` | `TRITON` | fast HSTU kernel (vs `PYTORCH` fallback) | +| `TORCH_CUDA_ARCH_LIST` | `10.0` | target `sm_100` for JIT / Triton compilation | +| `DLRM_DATA_PATH` | dataset root | overrides gin default `/apps/chcai/dlrm_data` | +| `HBM_CAP_GB` | `150` | embedding planner HBM budget per GPU | +| `RUN_NAME` | run id | results dir → `results//` | +| `PYTORCH_CUDA_ALLOC_CONF` | `expandable_segments:True` | allocator headroom | +| `TRITON_CACHE_DIR` | cache path | persist compiled Triton kernels across runs | +| `WORLD_SIZE` / `LOCAL_WORLD_SIZE` | `8` | mp.spawn rank count | + +### Known pitfalls + +- Never reinstall torch in this image — a cu12x wheel breaks CUPTI and may drop + `sm_100`. +- The `+cu128` stack (`torch==2.7.1+cu128` + `fbgemm-gpu==1.2.0+cu128` + + `torchrec==1.2.0+cu128`) runs on B200 but cannot profile GPU activity (CUPTI + 12.8 vs the 13.2 driver). +- Stock `polars` silently overflows on `yambda-5b` (> 4.29 B rows); always use + `polars-u64-idx`. +- `EmbeddingBoundsCheck ... Setting idx to zero` warnings are benign data clamps. From 8d8844b4d8ab35fbbd8aad632e76c873de1a8784 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Mon, 1 Jun 2026 16:37:22 -0500 Subject: [PATCH 014/127] bf16 + triton autotune pinning with gin-driven full-tune override MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds three knobs, all driven from the gin file: - make_model.bf16_training: enable bf16 autocast for the DlrmHSTU model. - env_int macro: lets numeric gin values come from env vars (used by the existing hbm_cap_gb binding). - apply_env_bootstrap.TRITON_FULL_AUTOTUNE: when False (default), three layer-norm/jagged triton kernels are pinned to a single Config so cold starts land at the same steady-state deterministically. When True, the full autotune search runs again — use this when changing shape, GPU, or triton/torch version, then re-pin from the discovered winners. train_ranker._main_func now parses gin in two phases (skip_unknown=True early, full pass after the heavy imports) so the bootstrap env var is set BEFORE the triton kernel modules evaluate their @triton.autotune decorators at module load time. Co-Authored-By: Claude Opus 4.7 --- .../dlrm_v3/train/_env_bootstrap.py | 28 +++++++++++ .../dlrm_v3/train/gin/yambda_5b.gin | 13 ++++- .../dlrm_v3/train/train_ranker.py | 50 +++++++++++++------ .../dlrm_v3/train/utils.py | 7 ++- .../ops/triton/_autotune_pinning.py | 27 ++++++++++ .../ops/triton/triton_jagged.py | 6 ++- .../ops/triton/triton_layer_norm.py | 11 +++- 7 files changed, 122 insertions(+), 20 deletions(-) create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/train/_env_bootstrap.py create mode 100644 recommendation_v4/generative_recommenders/ops/triton/_autotune_pinning.py diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/_env_bootstrap.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/_env_bootstrap.py new file mode 100644 index 000000000..2890851de --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/_env_bootstrap.py @@ -0,0 +1,28 @@ +"""Gin-driven env-var bootstrap. + +Some env vars must be set *before* certain modules import (e.g. Triton's +`@triton.autotune` decorator reads `TRITON_FULL_AUTOTUNE` at module load +time, well before `gin.parse_config_file` runs in the default ordering). + +`apply_env_bootstrap()` is `@gin.configurable`, so the gin file becomes the +canonical source of truth. `train_ranker.py` parses gin with +`skip_unknown=True` early in `_main_func`, calls this function to push the +bindings into `os.environ`, then does the heavy imports. +""" + +import logging +import os +from typing import Optional + +import gin + +logger: logging.Logger = logging.getLogger(__name__) + + +@gin.configurable +def apply_env_bootstrap( + TRITON_FULL_AUTOTUNE: Optional[bool] = None, +) -> None: + if TRITON_FULL_AUTOTUNE is not None: + os.environ["TRITON_FULL_AUTOTUNE"] = "1" if TRITON_FULL_AUTOTUNE else "0" + logger.info("env bootstrap: TRITON_FULL_AUTOTUNE=%s", os.environ["TRITON_FULL_AUTOTUNE"]) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index cc2c1000d..da5727de3 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -1,10 +1,19 @@ batch_size = 32 -num_workers = 1 -prefetch_factor = 2 +num_workers = 4 +prefetch_factor = 8 dataset = "yambda-5b" # model parameters make_model.dataset = %dataset +make_model.bf16_training = True + +# False = use pinned triton kernel configs (deterministic; whether that's +# the fast or slow equilibrium depends on which config was pinned for the +# current training shape + GPU). For a NEW training config (new shape, +# new GPU, new triton/torch version), set True and run with +# TRITON_PRINT_AUTOTUNING=1 to discover the fast configs, then update the +# pinned constants in ops/triton/_autotune_pinning.py call sites. +apply_env_bootstrap.TRITON_FULL_AUTOTUNE = False # dense model optimizer dense_optimizer_factory_and_class.learning_rate = 0.001 diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py index d17e2992d..c6a90c2b7 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py @@ -23,22 +23,14 @@ import gin import torch -from generative_recommenders.dlrm_v3.checkpoint import load_dmp_checkpoint -from generative_recommenders.dlrm_v3.train.utils import ( - cleanup, - eval_loop, - make_model, - make_optimizer_and_shard, - make_train_test_dataloaders, - setup, - streaming_train_eval_loop, - train_eval_loop, - train_loop, -) -from generative_recommenders.dlrm_v3.utils import MetricsLogger from torch import multiprocessing as mp from torchrec.test_utils import get_free_port +# NOTE: heavy imports of generative_recommenders.dlrm_v3.* are deferred to +# inside _main_func so that gin-driven env-var bootstrap (see +# _env_bootstrap.apply_env_bootstrap) can run BEFORE the triton kernel +# modules evaluate their `@triton.autotune` decorators at module-load time. + logger: logging.Logger = logging.getLogger(__name__) @@ -65,13 +57,43 @@ def _main_func( ) -> None: device = torch.device(f"cuda:{rank}") logger.info(f"rank: {rank}, world_size: {world_size}, device: {device}") + # Phase 1: parse gin early with skip_unknown=True so env-bootstrap + # bindings take effect BEFORE any module-level @gin.configurable + # discovers itself. This is required because triton @triton.autotune + # decorators in generative_recommenders.ops.triton.* read env vars at + # module import time, and the heavy imports below pull those in. + from generative_recommenders.dlrm_v3.train._env_bootstrap import apply_env_bootstrap + + gin.parse_config_file(gin_file, skip_unknown=True) + apply_env_bootstrap() + + # Phase 2: heavy imports. Triton kernel modules evaluate their autotune + # decorators here, using the env vars set above. + from generative_recommenders.dlrm_v3.checkpoint import load_dmp_checkpoint + from generative_recommenders.dlrm_v3.train.utils import ( + cleanup, + eval_loop, + make_model, + make_optimizer_and_shard, + make_train_test_dataloaders, + setup, + streaming_train_eval_loop, + train_eval_loop, + train_loop, + ) + from generative_recommenders.dlrm_v3.utils import MetricsLogger + setup( rank=rank, world_size=world_size, master_port=master_port, device=device, ) - # parse all arguments + # Phase 3: re-parse to bind the @gin.configurables now that they are + # registered. The earlier skip_unknown pass already consumed the + # env-bootstrap binding, but bindings are idempotent so re-applying is + # fine, and this pass is the one that actually wires up make_model, + # make_train_test_dataloaders, etc. gin.parse_config_file(gin_file) model, model_configs, embedding_table_configs = make_model() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py index 2ff025394..a957122a3 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -204,15 +204,20 @@ def set_epoch(self, epoch: int) -> None: @gin.configurable def make_model( dataset: str, + bf16_training: bool = False, ) -> Tuple[torch.nn.Module, DlrmHSTUConfig, Dict[str, EmbeddingConfig]]: hstu_config = get_hstu_configs(dataset) table_config = get_embedding_table_config(dataset) + # bf16 autocast is off by default: on the PYTORCH attn backend the + # pt_hstu_attention QK einsum backward overflows in bf16 at long + # sequences (NaN at step 1 when N>1k). Safe with TRITON; flip via + # `make_model.bf16_training = True` in the gin. model = DlrmHSTU( hstu_configs=hstu_config, embedding_tables=table_config, is_inference=False, - bf16_training=False, + bf16_training=bf16_training, ) # Triton on ROCm fails to compile some jagged kernels at our shapes diff --git a/recommendation_v4/generative_recommenders/ops/triton/_autotune_pinning.py b/recommendation_v4/generative_recommenders/ops/triton/_autotune_pinning.py new file mode 100644 index 000000000..5aa24eb85 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton/_autotune_pinning.py @@ -0,0 +1,27 @@ +"""Triton autotune pinning helper. + +A handful of Triton kernels in this directory have two stable autotune +equilibria on MI350X gfx950 at our yambda bs=32 L=2039 shape: a fast one +(~52 ms/step) and a slow one (~71 ms/step). The autotuner's measurement +noise puts the choice on a coin flip per cold start. We pin the winning +config for these kernels so every cold start lands at the fast equilibrium +deterministically. + +Set `TRITON_FULL_AUTOTUNE=1` to bypass the pin and re-enable the full +autotune search (useful when validating a new shape, GPU, or Triton version +before re-capturing winners). +""" + +import os +from typing import Callable, List + +import triton + + +def pinned_or_full( + pinned: List[triton.Config], + full_configs_fn: Callable[[], List[triton.Config]], +) -> List[triton.Config]: + if os.environ.get("TRITON_FULL_AUTOTUNE", "0") == "1": + return full_configs_fn() + return pinned diff --git a/recommendation_v4/generative_recommenders/ops/triton/triton_jagged.py b/recommendation_v4/generative_recommenders/ops/triton/triton_jagged.py index 7a4e82cf4..8172a0f7b 100644 --- a/recommendation_v4/generative_recommenders/ops/triton/triton_jagged.py +++ b/recommendation_v4/generative_recommenders/ops/triton/triton_jagged.py @@ -33,6 +33,7 @@ switch_to_contiguous_if_needed, triton_autotune, ) +from generative_recommenders.ops.triton._autotune_pinning import pinned_or_full from generative_recommenders.ops.utils import is_sm100_plus, is_sm90 from torch._inductor.runtime import triton_helpers @@ -2150,7 +2151,10 @@ def split_2D_jagged_w_prefix_multirow( @triton_autotune( - configs=_get_split_concat_2d_jagged_multirow_configs_wrapper(), + configs=pinned_or_full( + [triton.Config({"BLOCK_N": 1}, num_warps=2)], + _get_split_concat_2d_jagged_multirow_configs_wrapper, + ), key=["BLOCK_D"], ) @triton.jit diff --git a/recommendation_v4/generative_recommenders/ops/triton/triton_layer_norm.py b/recommendation_v4/generative_recommenders/ops/triton/triton_layer_norm.py index 1e997fd40..5ed508108 100644 --- a/recommendation_v4/generative_recommenders/ops/triton/triton_layer_norm.py +++ b/recommendation_v4/generative_recommenders/ops/triton/triton_layer_norm.py @@ -30,6 +30,7 @@ switch_to_contiguous_if_needed, triton_autotune, ) +from generative_recommenders.ops.triton._autotune_pinning import pinned_or_full from generative_recommenders.ops.utils import ( is_sm100_plus, is_sm90, @@ -306,7 +307,10 @@ def _layer_norm_bwd_dx( @triton_autotune( - configs=_get_layer_norm_fwd_configs(), + configs=pinned_or_full( + [triton.Config({"BLOCK_N": 8}, num_warps=1)], + _get_layer_norm_fwd_configs, + ), key=["BLOCK_D"], ) @triton.jit @@ -463,7 +467,10 @@ def _get_bwd_dwdb_configs() -> List[triton.Config]: @triton_autotune( - configs=_get_bwd_dwdb_configs(), + configs=pinned_or_full( + [triton.Config({"BLOCK_N": 128}, num_warps=8)], + _get_bwd_dwdb_configs, + ), key=["D"], ) @triton.jit From 1c9315ae9eb5136584ed2238105b93f932d8535c Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Mon, 1 Jun 2026 16:40:18 -0500 Subject: [PATCH 015/127] docs: add MI350X training recipe section Mirrors the B200 layout with MI350X (gfx950, ROCm 7.2.1) specifics: container image (rocm/primus:v26.3), fbgemm_gpu rebuild requirement (HEAD nightly_rocm-2026.6.1 for ~30% step-time win over the shipped 2026.5.14), the gin-driven TRITON_FULL_AUTOTUNE knob, and the measured perf ladder from fp32/PYTORCH baseline (~28 d/epoch) down to the pinned bf16/TRITON fast equilibrium (~7.6 d/epoch). Co-Authored-By: Claude Opus 4.7 --- recommendation_v4/docs/training_recipe.md | 103 ++++++++++++++++++++++ 1 file changed, 103 insertions(+) diff --git a/recommendation_v4/docs/training_recipe.md b/recommendation_v4/docs/training_recipe.md index 155dc9afd..f6744be39 100644 --- a/recommendation_v4/docs/training_recipe.md +++ b/recommendation_v4/docs/training_recipe.md @@ -5,6 +5,109 @@ Reproducible environment + configuration for training HSTU / DLRM-v3 on the --- +## MI350X + +Single-node, 8× AMD **Instinct MI350X** (`gfx950`, ~288 GiB HBM3e each), HSTU +ranker on `yambda-5b` with the **TRITON** HSTU kernel and **bf16** +mixed-precision training. + +### Hardware / host + +| item | value | +|---|---| +| GPUs | 8× AMD Instinct MI350X (`gfx950`, ROCm 7.2.1) | +| Host CPU | AMD EPYC 9655 96-Core (192 cores × 2 threads) | + +### Container image + +``` +rocm/primus:v26.3 +``` + +The image's native PyTorch is kept as-is and must not be reinstalled — it is +the ROCm-matched build used by triton/fbgemm. + +### Dependency versions + +| package | version | notes | +|---|---|---| +| **torch** | `2.10.0+git94c6e04` | native to the image; not reinstalled | +| **triton** | `3.6.0` | native to the image; same major as B200 path | +| **fbgemm_gpu** | `fbgemm_gpu_nightly_rocm-2026.6.1` (built from FBGEMM commit `1509423`, 2026-06-01) for `gfx950` | image ships `2026.5.14`; rebuild from source gives a measurable boost from the TBE-forward V2 grid-striding (#5669) + warpSize 32/64 unified build (#5739) + `__syncthreads` cleanup (#5744). Build command: `python setup.py -j 32 bdist_wheel --build-target=default --build-variant=rocm -DHIP_ROOT_DIR=/opt/rocm -DAMDGPU_TARGETS=gfx950` | +| **torchrec** | `1.4.0` | matches B200 | +| **polars-u64-idx** | `1.33.1` | 64-bit row index — `yambda-5b` has > 4.29 B rows. Installed from a pre-staged local tarball by `scripts/launch_smoke_8gpu.sh` (reserved nodes have no outbound DNS) | + +### Training configuration + +From `generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin`: + +| parameter | value | gin binding | +|---|---|---| +| batch_size (train) | 32 | `make_train_test_dataloaders.batch_size` | +| eval_batch_size | 32 | `make_train_test_dataloaders.eval_batch_size` | +| num_workers (dataloader) | 4 | `make_train_test_dataloaders.num_workers` | +| prefetch_factor | 8 | `make_train_test_dataloaders.prefetch_factor` | +| num_blocks | 1 | `make_train_test_dataloaders.num_blocks` | +| train_split_percentage | 0.90 | `make_train_test_dataloaders.train_split_percentage` | +| history_length (per-sample UIH budget) | 2039 | `get_dataset.history_length` | +| max_seq_len (attention budget) | 2048 | `get_hstu_configs.max_seq_len` | +| bf16 training | True | `make_model.bf16_training` | +| HBM cap (per GPU) | 260 GiB | `make_optimizer_and_shard.hbm_cap_gb` (env `HBM_CAP_GB`) | +| **triton autotune pinning** | **False (pinned)** | `apply_env_bootstrap.TRITON_FULL_AUTOTUNE` | +| dense optimizer | Adam, lr 1e-3, betas (0.95, 0.999), eps 1e-8 | `dense_optimizer_factory_and_class.*` | +| sparse optimizer | RowWiseAdagrad, lr 1e-3, betas (0.95, 0.999), eps 1e-8 | `sparse_optimizer_factory_and_class.*` | +| world_size | 8 | `MetricsLogger.world_size` | + +Effective global batch = `batch_size × world_size = 32 × 8 = 256` samples/step. + +### Environment variables + +| var | value | purpose | +|---|---|---| +| `HSTU_HAMMER_KERNEL` | `TRITON` | fast HSTU kernel (vs `PYTORCH` fallback) | +| `DLRM_DATA_PATH` | dataset root | overrides gin default `/apps/chcai/dlrm_data` | +| `HBM_CAP_GB` | (optional) | embedding planner HBM budget per GPU | +| `RUN_NAME` | run id | results dir → `results//` | +| `PYTORCH_CUDA_ALLOC_CONF` | `expandable_segments:True` | allocator headroom | +| `HIP_VISIBLE_DEVICES` / `CUDA_VISIBLE_DEVICES` | `0,1,2,3,4,5,6,7` | rank visibility | + +`TRITON_FULL_AUTOTUNE` is set automatically by the gin-driven bootstrap +(`generative_recommenders.dlrm_v3.train._env_bootstrap.apply_env_bootstrap`), +which runs in `train_ranker._main_func` BEFORE the triton kernel modules +import — so the gin file is the source of truth. + +### Measured performance + +| variant | steady-state ms/step | global sps | epoch ETA (3.23B anchors) | +|---|---|---|---| +| nightly + fp32 + PYTORCH attn (baseline) | ~190 | ~1340 | ~28 d | +| nightly + bf16 + TRITON attn | ~93 | ~2787 | ~13.4 d | +| primus + bf16 + TRITON attn | ~67.5 | ~3793 | ~9.9 d | +| primus + fbgemm HEAD + bf16 + TRITON, autotune drift | ~53 fast / ~70 slow | 3700–4860 | 7.7–10.2 d | +| **primus + fbgemm HEAD + bf16 + TRITON + pinning (default)** | **~52** | **~4970** | **~7.6 d** | + +The "pinning" line is the deterministic per-cold-start equilibrium — +three layer-norm / jagged triton kernels have two stable autotune winners +and the pin forces the fast one every run. + +### Known pitfalls + +- The image ships `fbgemm_gpu==2026.5.14`. The wheel built from FBGEMM HEAD + (`2026.6.1`) is required for the 70 → 52 ms step. Build inside the + container so the wheel links against the image's native torch. +- Stock `polars` silently overflows on `yambda-5b` (> 4.29 B rows); always + use `polars-u64-idx`. +- When changing shape (batch size, history length), GPU, or triton/torch + version, flip `apply_env_bootstrap.TRITON_FULL_AUTOTUNE = True` and run + with `TRITON_PRINT_AUTOTUNING=1` to re-capture winners, then update the + pinned configs at the `pinned_or_full(...)` call sites in + `generative_recommenders/ops/triton/`. +- Do not run with bf16 on the `PYTORCH` HSTU attention backend at our + sequence length — `pt_hstu_attention`'s QK einsum backward overflows in + bf16 at N > 1k and produces NaN at step 1. bf16 is only safe with TRITON. + +--- + ## B200 Single-node, 8× NVIDIA **B200** (Blackwell, `sm_100`, ~183 GiB HBM each), HSTU From 30275194c1e3c0a735b4a0161360eedf6c3bd54b Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Mon, 1 Jun 2026 16:48:54 -0500 Subject: [PATCH 016/127] scripts: add stitch_traces.py Merges per-rank chrome traces (results//trace_step{N}_rank{R}.json) into a single Perfetto-loadable file, remapping pid/flow ids so cross-rank events land on distinct tracks instead of collapsing onto one. Used to produce the bf16 + pinned-autotune step-52 trace (results/verify_rename/trace_step52.json.gz). Co-Authored-By: Claude Opus 4.7 --- recommendation_v4/scripts/stitch_traces.py | 329 +++++++++++++++++++++ 1 file changed, 329 insertions(+) create mode 100644 recommendation_v4/scripts/stitch_traces.py diff --git a/recommendation_v4/scripts/stitch_traces.py b/recommendation_v4/scripts/stitch_traces.py new file mode 100644 index 000000000..54d7963d6 --- /dev/null +++ b/recommendation_v4/scripts/stitch_traces.py @@ -0,0 +1,329 @@ +#!/usr/bin/env python3 +"""Stitch per-rank Chrome traces from a dlrm_v3 run into one merged file. + +When ``Profiler`` runs on multiple ranks, each rank writes its own file: + + /trace_step{step}_rank{rank}.json + +Each per-rank trace uses overlapping ``pid`` namespaces (CPU pid = OS pid; +GPU streams pid = 0..N), so concatenating the raw event lists would collapse +multiple ranks onto the same Perfetto track. This script: + +* Identifies each pid as ``CPU`` / ``GPU`` / ``Spans`` (and other torch.profiler + string-pid tracks) using the per-rank ``process_labels`` metadata events. +* Always drops the ``Spans`` track (low-signal in this codebase, large in + visual clutter). +* Optionally filters to just ``cpu`` or ``gpu`` events via ``--include``. +* Sorts the surviving tracks into contiguous Perfetto sections: + **all CPU tracks (rank 0..N) first, then all GPU tracks (rank 0..N, stream + 0..K)**. +* Remaps every event's ``pid`` and flow ``id`` so cross-rank events never + collide on the same track or flow arrow. + +Because torch.profiler emits ``baseTimeNanoseconds`` from the same node clock, +timestamps line up directly across ranks — no time-shift needed for single-node +runs (multi-node would need clock-skew correction, not implemented here). + +Examples +-------- +Stitch step 52, default (CPU + GPU, drop Spans), gzip output:: + + python scripts/stitch_traces.py --step 52 --gzip + +GPU-only view (skip CPU thread tree entirely — useful for kernel-level analysis):: + + python scripts/stitch_traces.py --step 52 --include gpu --gzip + +CPU-only view (host-side ops, profiler annotations, comm scheduling):: + + python scripts/stitch_traces.py --step 52 --include cpu --gzip +""" +from __future__ import annotations + +import argparse +import gzip +import json +import re +import sys +from collections import defaultdict +from pathlib import Path + +# trace_step52_rank3.json or trace_3_rank0.json (legacy filename) +_RANK_RE = re.compile(r"trace_(?:step)?(\d+)_rank(\d+)\.json$") +_KEY_RE = re.compile(r"trace_(.+?)_rank\d+\.json$") + +# Per-rank pid offset. Picked large enough that no real OS pid collides +# (Linux pids fit in 22 bits; 1e6 per rank gives ~10 ranks of headroom). +_PID_STRIDE = 1_000_000 + +# Per-rank flow-id offset. torch.profiler flow ids are int32/int64 — pack rank +# into the high bits so cross-rank flows can never link by accident. +_FLOW_ID_STRIDE = 1 << 40 + +# Sort-index sections in Perfetto. Lower = appears higher in the timeline UI. +# Each section reserves a wide range so within-section ordering (rank, stream) +# fits comfortably without overlapping the next section. +_SORT_BASE = { + "cpu": 0, + "gpu": 1_000_000, + "other": 10_000_000, # Traces / "" misc string-pid tracks +} + +# `Spans` carries no useful content in our workloads (one X event per trace) +# and clutters the timeline — always dropped. +_ALWAYS_DROP_PIDS_STR = {"Spans"} + + +def _classify_pid(pid_to_label: dict, pid_to_name: dict) -> dict: + """Map original pid -> ('cpu'|'gpu'|'spans'|'other', stream_idx_or_0). + + Classification order, first match wins: + 1. pid (as a string) is in the always-drop set -> 'spans' + 2. process_name is in the always-drop set -> 'spans' + 3. process_labels == 'CPU' -> 'cpu' + 4. process_labels starts with 'GPU ' -> 'gpu', stream id + 5. anything else (including unlabeled pids) -> 'other' + """ + all_pids = set(pid_to_label) | set(pid_to_name) + out: dict = {} + for pid in all_pids: + label = pid_to_label.get(pid, "") + name = pid_to_name.get(pid, "") + if isinstance(pid, str) and pid in _ALWAYS_DROP_PIDS_STR: + out[pid] = ("spans", 0) + continue + if name in _ALWAYS_DROP_PIDS_STR: + out[pid] = ("spans", 0) + continue + if label == "CPU": + out[pid] = ("cpu", 0) + elif label.startswith("GPU"): + try: + stream_idx = int(label.split()[1]) + except (IndexError, ValueError): + stream_idx = 0 + out[pid] = ("gpu", stream_idx) + else: + out[pid] = ("other", 0) + return out + + +def _scan_pid_metadata(events: list[dict]) -> tuple[dict, dict]: + """First pass: collect per-pid label and name from ``ph='M'`` events.""" + label: dict = {} + name: dict = {} + for e in events: + if e.get("ph") != "M": + continue + pid = e.get("pid") + if pid is None: + continue + if e.get("name") == "process_labels": + label[pid] = e.get("args", {}).get("labels", "") + elif e.get("name") == "process_name": + name[pid] = e.get("args", {}).get("name", "") + return label, name + + +def _new_sort_index(kind: str, rank: int, stream_idx: int) -> int: + """Compute Perfetto sort_index so tracks group as: CPU(rank0..N), GPU(rank0..N, stream0..K), other.""" + base = _SORT_BASE.get(kind, _SORT_BASE["other"]) + return base + rank * 100 + stream_idx + + +def _new_pid(orig_pid, rank: int) -> object: + """Remap a single pid into a per-rank namespace, preserving int vs str.""" + if isinstance(orig_pid, int): + return orig_pid + rank * _PID_STRIDE + if isinstance(orig_pid, str): + try: + return int(orig_pid) + rank * _PID_STRIDE + except ValueError: + return f"rank{rank}_{orig_pid}" if orig_pid else f"rank{rank}_misc" + return orig_pid + + +def _process_one_rank( + events: list[dict], + rank: int, + include: set[str], +) -> list[dict]: + """Filter + remap one rank's events. ``include`` is a subset of {'cpu','gpu','other'}.""" + label, name = _scan_pid_metadata(events) + classify = _classify_pid(label, name) + + out: list[dict] = [] + for e in events: + pid = e.get("pid") + if pid is None: + out.append(e) + continue + # Always-drop check on the raw pid value first - Spans events in our + # workloads have NO process_name/process_labels metadata, so the + # classifier table doesn't list them. Catch them here directly. + if isinstance(pid, str) and pid in _ALWAYS_DROP_PIDS_STR: + continue + kind, stream_idx = classify.get(pid, ("other", 0)) + if kind == "spans": # always dropped + continue + if kind not in include: # filtered by --include + continue + + # Remap pid + flow id (per-rank namespace). + e["pid"] = _new_pid(pid, rank) + if "id" in e and e.get("ph") in ("s", "t", "f"): + try: + e["id"] = int(e["id"]) + rank * _FLOW_ID_STRIDE + except (TypeError, ValueError): + pass + + # Rewrite metadata: section-aware sort_index + rank-prefixed name. + if e.get("ph") == "M": + args = e.setdefault("args", {}) + if e.get("name") == "process_sort_index": + args["sort_index"] = _new_sort_index(kind, rank, stream_idx) + elif e.get("name") == "process_name": + orig = args.get("name", "python") + args["name"] = f"[Rank {rank}] {orig}" + + out.append(e) + + return out + + +def _group_by_step(trace_dir: Path) -> dict[str, dict[int, Path]]: + """Map step-key (e.g. ``"step52"`` or ``"3"``) -> {rank: path}.""" + groups: dict[str, dict[int, Path]] = defaultdict(dict) + for p in sorted(trace_dir.glob("trace_*_rank*.json")): + m = _RANK_RE.search(p.name) + if not m: + continue + prefix_match = _KEY_RE.match(p.name) + key = prefix_match.group(1) if prefix_match else m.group(1) + groups[key][int(m.group(2))] = p + return dict(groups) + + +def stitch_one(rank_to_path: dict[int, Path], out_path: Path, *, + include: set[str], gzip_out: bool, verbose: bool) -> None: + """Merge one (step, rank->path) group into a single trace file.""" + merged_events: list[dict] = [] + base: dict | None = None + + for rank in sorted(rank_to_path): + path = rank_to_path[rank] + if verbose: + sz_mb = path.stat().st_size / (1 << 20) + print(f" rank {rank}: {path.name} ({sz_mb:.1f} MB)", file=sys.stderr) + with path.open() as f: + trace = json.load(f) + if base is None: + base = {k: v for k, v in trace.items() if k != "traceEvents"} + base["distributedInfo"] = { + **trace.get("distributedInfo", {}), + "stitched_ranks": sorted(rank_to_path), + "stitched_files": [p.name for p in rank_to_path.values()], + "stitched_include": sorted(include), + } + merged_events.extend( + _process_one_rank(trace.get("traceEvents", []), rank, include) + ) + + assert base is not None, "no input traces provided" + base["traceEvents"] = merged_events + + out_path.parent.mkdir(parents=True, exist_ok=True) + if gzip_out: + with gzip.open(out_path, "wt") as f: + json.dump(base, f) + else: + with out_path.open("w") as f: + json.dump(base, f) + if verbose: + sz_mb = out_path.stat().st_size / (1 << 20) + print( + f" -> {out_path} ({len(merged_events):,} events, {sz_mb:.1f} MB)", + file=sys.stderr, + ) + + +def main() -> int: + ap = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + ap.add_argument("trace_dir", type=Path, + help="Directory containing trace_*_rank*.json files.") + ap.add_argument("--step", type=str, default=None, + help="Stitch only the given step key (e.g. '52' or 'step52'). " + "Default: stitch every step group found.") + ap.add_argument("--out", type=Path, default=None, + help="Output path. Only valid when --step selects exactly " + "one group. Default: /trace_.json[.gz] " + "(or trace__cpu/_gpu when --include filters).") + ap.add_argument("--include", choices=("cpu", "gpu", "both"), default="both", + help="Which sections to keep: cpu-only tracks, gpu-only " + "tracks, or both (default). 'Spans' is always dropped.") + ap.add_argument("--gzip", action="store_true", + help="Write gzip-compressed JSON (Perfetto auto-detects).") + ap.add_argument("-q", "--quiet", action="store_true") + args = ap.parse_args() + + if not args.trace_dir.is_dir(): + print(f"error: {args.trace_dir} is not a directory", file=sys.stderr) + return 2 + + if args.include == "both": + # 'other' covers torch.profiler string-pid tracks (Traces / misc) that + # carry low-volume but legitimate annotations. Dropped under cpu/gpu + # so each filtered view is clean. + include = {"cpu", "gpu", "other"} + else: + include = {args.include} + + groups = _group_by_step(args.trace_dir) + if not groups: + print(f"error: no trace_*_rank*.json files under {args.trace_dir}", + file=sys.stderr) + return 2 + + if args.step is not None: + wanted = args.step if args.step.startswith("step") else f"step{args.step}" + if wanted not in groups and args.step in groups: + wanted = args.step + if wanted not in groups: + print( + f"error: step {args.step!r} not found. " + f"Available: {sorted(groups)}", + file=sys.stderr, + ) + return 2 + groups = {wanted: groups[wanted]} + + if args.out is not None and len(groups) != 1: + print("error: --out requires --step to select exactly one group", + file=sys.stderr) + return 2 + + for key, rank_map in sorted(groups.items()): + if not args.quiet: + print( + f"stitching {key} ({len(rank_map)} ranks, include={args.include}):", + file=sys.stderr, + ) + if args.out is not None: + out = args.out + else: + ext = ".json.gz" if args.gzip else ".json" + # Default mode ("both") gets the bare filename; explicit cpu/gpu + # filters tag the output so they can coexist in one directory. + suffix = "" if args.include == "both" else f"_{args.include}" + out = args.trace_dir / f"trace_{key}{suffix}{ext}" + stitch_one(rank_map, out, include=include, + gzip_out=args.gzip, verbose=not args.quiet) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 8cbbab0ff25c034af48fa04b0192a257059000d3 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Mon, 1 Jun 2026 22:49:02 +0000 Subject: [PATCH 017/127] docs: update B200 recipe deps to NGC 26.04 (torch 2.12 / CUDA 13.2) Refresh the B200 dependency versions to the latest validated stack (torch 2.12.0a0 / CUDA 13.2, fbgemm_gpu built for sm_100+CUDA 13.2, CUPTI 13.2), note 26.01 as an equivalent alternative, and record the TRITON_FULL_AUTOTUNE=True setting for B200. --- recommendation_v4/docs/training_recipe.md | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/recommendation_v4/docs/training_recipe.md b/recommendation_v4/docs/training_recipe.md index f6744be39..bd6fee159 100644 --- a/recommendation_v4/docs/training_recipe.md +++ b/recommendation_v4/docs/training_recipe.md @@ -125,24 +125,29 @@ training. ### Container image ``` -nvcr.io/nvidia/pytorch:26.01-py3 +nvcr.io/nvidia/pytorch:26.04-py3 ``` -Digest: `sha256:38ed2ecb2c16d10677006d73fb0a150855d6ec81db8fc66e800b5ae92741007e` +Digest: `sha256:192d749b4d773610ec9e01c0443a9df545d196c412b7b8fd33bfa3da362a49e7` The image's native PyTorch is kept as-is and must not be reinstalled (so CUPTI stays matched to the driver and `sm_100` support is preserved). +`nvcr.io/nvidia/pytorch:26.01-py3` (torch `2.10.0a0` / CUDA 13.1, digest +`sha256:38ed2ecb2c16d10677006d73fb0a150855d6ec81db8fc66e800b5ae92741007e`) is +also validated and performance-equivalent — rebuild `fbgemm_gpu` against +whichever image's torch you run. + ### Dependency versions | package | version | notes | |---|---|---| -| **torch** | `2.10.0a0+a36e1d39eb.nv26.01.42222806` (CUDA 13.1) | native to the image; not reinstalled | +| **torch** | `2.12.0a0+0291f960b6.nv26.04.48445190` (CUDA 13.2) | native to the image; not reinstalled | | **triton** | `3.6.0` | native to the image; provides `triton.language.make_tensor_descriptor` (required by the TRITON HSTU path) | -| **fbgemm_gpu** | `fbgemm_gpu_nightly-2026.6.1` (CUDA 13.1, `sm_100`) | built from source against the native torch, from FBGEMM commit `939f2da156b05d2f1bcba8c037d613c1098d0db5` (2026-04-29); public wheels are ABI-incompatible with the NGC torch | +| **fbgemm_gpu** | `fbgemm_gpu_nightly-2026.6.1` (CUDA 13.2, `sm_100`) | built from source against the native torch, from FBGEMM commit `939f2da156b05d2f1bcba8c037d613c1098d0db5` (2026-04-29); public wheels are ABI-incompatible with the NGC torch. Build command: `TORCH_CUDA_ARCH_LIST=10.0 python setup.py bdist_wheel --build-target default --build-variant cuda --package_channel nightly --nvml_lib_path /usr/lib/x86_64-linux-gnu/libnvidia-ml.so` (~55 min — the `sm_100` TBE-forward kernels dominate via `ptxas`) | | **torchrec** | `1.4.0` | installed with `--no-deps` | | **polars-u64-idx** | `1.33.1` | 64-bit row index — `yambda-5b` has > 4.29 B rows (overflows stock polars' 32-bit index) | -| CUPTI (for `torch.profiler`) | 13.1 (native) | matches the driver; the `+cu128` stack's CUPTI 12.8 fails on B200 (`CUPTI_ERROR_INVALID_DEVICE`) | +| CUPTI (for `torch.profiler`) | 13.2 (native) | matches the driver; the `+cu128` stack's CUPTI 12.8 fails on B200 (`CUPTI_ERROR_INVALID_DEVICE`) | Additional Python deps: `xxhash`, `gin-config`, `absl-py`, `pandas`, `tensorboard`, `pyarrow`, `pyyaml`, @@ -165,6 +170,7 @@ From `generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin`: | max_seq_len (attention budget) | 2048 | `get_hstu_configs.max_seq_len` | | bf16 training | True | `make_model.bf16_training` | | HBM cap (per GPU) | 150 GiB | `make_optimizer_and_shard.hbm_cap_gb` (env `HBM_CAP_GB`) | +| **triton autotune pinning** | **True (full autotune)** | `apply_env_bootstrap.TRITON_FULL_AUTOTUNE` — the pinned configs are MI350X-specific, so B200 runs full autotune to find its own `sm_100` winners | | dense optimizer | Adam, lr 1e-3, betas (0.95, 0.999), eps 1e-8 | `dense_optimizer_factory_and_class.*` | | sparse optimizer | RowWiseAdagrad, lr 1e-3, betas (0.95, 0.999), eps 1e-8 | `sparse_optimizer_factory_and_class.*` | | world_size | 8 | `MetricsLogger.world_size` | From fc4d09268d0a7695fdbe747cde1ff95e7c499c3a Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Tue, 2 Jun 2026 00:00:08 +0000 Subject: [PATCH 018/127] docs: refresh B200 recipe deps (fbgemm HEAD, torchrec 1.7 nightly, driver) Point fbgemm at the latest validated source commit (10b77573, 2026-06-01), record the tested torchrec 1.7.0.dev nightly (1.4.0 stable fallback), clarify the fbgemm wheel version string is the build date, and correct the host/forward-compat driver CUDA versions (13.0 host / 595.58.03 compat). --- recommendation_v4/docs/training_recipe.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/recommendation_v4/docs/training_recipe.md b/recommendation_v4/docs/training_recipe.md index bd6fee159..47ef0577e 100644 --- a/recommendation_v4/docs/training_recipe.md +++ b/recommendation_v4/docs/training_recipe.md @@ -119,8 +119,8 @@ training. | item | value | |---|---| | GPUs | 8× NVIDIA B200 (`sm_100`, compute capability 10.0) | -| Host driver | 580.159.03 (reports CUDA 13.2) | -| Forward-compat userspace driver | `libcuda.so.595.45.04` (engaged automatically by the NGC image) | +| Host driver | 580.159.03 (reports CUDA 13.0) | +| Forward-compat userspace driver | `libcuda.so.595.58.03` (CUDA 13.2.1; engaged automatically by the NGC image) | ### Container image @@ -144,8 +144,8 @@ whichever image's torch you run. |---|---|---| | **torch** | `2.12.0a0+0291f960b6.nv26.04.48445190` (CUDA 13.2) | native to the image; not reinstalled | | **triton** | `3.6.0` | native to the image; provides `triton.language.make_tensor_descriptor` (required by the TRITON HSTU path) | -| **fbgemm_gpu** | `fbgemm_gpu_nightly-2026.6.1` (CUDA 13.2, `sm_100`) | built from source against the native torch, from FBGEMM commit `939f2da156b05d2f1bcba8c037d613c1098d0db5` (2026-04-29); public wheels are ABI-incompatible with the NGC torch. Build command: `TORCH_CUDA_ARCH_LIST=10.0 python setup.py bdist_wheel --build-target default --build-variant cuda --package_channel nightly --nvml_lib_path /usr/lib/x86_64-linux-gnu/libnvidia-ml.so` (~55 min — the `sm_100` TBE-forward kernels dominate via `ptxas`) | -| **torchrec** | `1.4.0` | installed with `--no-deps` | +| **fbgemm_gpu** | FBGEMM commit `10b775730212923f65f7b78f79b6a01d80cf3c29` (2026-06-01 `main`, CUDA 13.2, `sm_100`) | built from source against the native torch; public wheels are ABI-incompatible with the NGC torch. The built wheel is named `fbgemm_gpu_nightly-2026.6.1` — that version is the build date, not the source date, so always identify the build by the commit above. Build command: `TORCH_CUDA_ARCH_LIST=10.0 python setup.py bdist_wheel --build-target default --build-variant cuda --package_channel nightly --nvml_lib_path /usr/lib/x86_64-linux-gnu/libnvidia-ml.so` (~55 min — the `sm_100` TBE-forward kernels dominate via `ptxas`) | +| **torchrec** | `1.7.0.dev20260601+cu130` (nightly, tested) | installed `--no-deps` from `https://download.pytorch.org/whl/nightly/cu130`. Perf-neutral vs stable `1.4.0`; use `1.4.0` (latest stable) if you prefer a non-pre-release | | **polars-u64-idx** | `1.33.1` | 64-bit row index — `yambda-5b` has > 4.29 B rows (overflows stock polars' 32-bit index) | | CUPTI (for `torch.profiler`) | 13.2 (native) | matches the driver; the `+cu128` stack's CUPTI 12.8 fails on B200 (`CUPTI_ERROR_INVALID_DEVICE`) | From 194fc9bee96c94ed91cc79310a9e1caf91affe09 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Mon, 1 Jun 2026 19:22:54 -0500 Subject: [PATCH 019/127] MI350X: re-pin 2 triton configs for the torch 2.12 + torchrec 1.7 stack After upgrading to torch 2.12 / torchrec 1.7 (B200-aligned), the pinned configs from the torch 2.10 stack stopped landing on the fast equilibrium because the torchrec 1.7 code path invokes these kernels at different shape keys. Re-captured winners via a fresh autotune run and updated the pin sites: - _weighted_layer_norm_bwd_dx: BLOCK_N 8 -> 1 (num_warps 1 unchanged) - split_2D_jagged_multirow: BLOCK_N 1 / num_warps 2 -> BLOCK_N 8 / num_warps 1 - _layer_norm_bwd_dwdb: BLOCK_N 128, num_warps 8 (unchanged - same winner on both stacks) Verified: 3 consecutive checkpoints (steps 151/201/251) at 52.75-53.36 ms deterministic on the new stack. Same equilibrium band as the torch 2.10 stack (51.5-53.0 ms). Also adds a Stack B section to docs/training_recipe.md (MI350X) documenting the torch 2.12 swap recipe (torch + torchvision + torchaudio + fbgemm rebuild + torchrec git tag) so the MI350X recipe is dependency-aligned with the B200 path. Co-Authored-By: Claude Opus 4.7 --- recommendation_v4/docs/training_recipe.md | 27 ++++++++++++++++--- .../ops/triton/triton_jagged.py | 2 +- .../ops/triton/triton_layer_norm.py | 2 +- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/recommendation_v4/docs/training_recipe.md b/recommendation_v4/docs/training_recipe.md index 47ef0577e..d18221cb3 100644 --- a/recommendation_v4/docs/training_recipe.md +++ b/recommendation_v4/docs/training_recipe.md @@ -29,14 +29,35 @@ the ROCm-matched build used by triton/fbgemm. ### Dependency versions +Two stacks validated; both land at the same ~52 ms/step steady state with the +pinned triton autotune configs. + +**Stack A — image-native torch (default, no torch swap):** + | package | version | notes | |---|---|---| | **torch** | `2.10.0+git94c6e04` | native to the image; not reinstalled | -| **triton** | `3.6.0` | native to the image; same major as B200 path | -| **fbgemm_gpu** | `fbgemm_gpu_nightly_rocm-2026.6.1` (built from FBGEMM commit `1509423`, 2026-06-01) for `gfx950` | image ships `2026.5.14`; rebuild from source gives a measurable boost from the TBE-forward V2 grid-striding (#5669) + warpSize 32/64 unified build (#5739) + `__syncthreads` cleanup (#5744). Build command: `python setup.py -j 32 bdist_wheel --build-target=default --build-variant=rocm -DHIP_ROOT_DIR=/opt/rocm -DAMDGPU_TARGETS=gfx950` | -| **torchrec** | `1.4.0` | matches B200 | +| **triton** | `3.6.0` | native to the image | +| **fbgemm_gpu** | `fbgemm_gpu_nightly_rocm-2026.6.1` (built from FBGEMM commit `1509423`, 2026-06-01 `main`) for `gfx950` | image ships `2026.5.14`; rebuild from source gives a measurable boost from the TBE-forward V2 grid-striding (#5669) + warpSize 32/64 unified build (#5739) + `__syncthreads` cleanup (#5744). Build command: `python setup.py -j 32 bdist_wheel --build-target=default --build-variant=rocm -DHIP_ROOT_DIR=/opt/rocm -DAMDGPU_TARGETS=gfx950` | +| **torchrec** | `1.4.0` | image native | | **polars-u64-idx** | `1.33.1` | 64-bit row index — `yambda-5b` has > 4.29 B rows. Installed from a pre-staged local tarball by `scripts/launch_smoke_8gpu.sh` (reserved nodes have no outbound DNS) | +**Stack B — torch 2.12 / torchrec 1.7 (B200-aligned):** + +| package | version | install | +|---|---|---| +| **torch** | `2.12.0+rocm7.2` | `pip install --upgrade --no-deps --index-url https://download.pytorch.org/whl/rocm7.2 torch==2.12.0+rocm7.2` | +| **torchvision** | `0.27.0+rocm7.2` | `pip install --upgrade --no-deps --index-url https://download.pytorch.org/whl/rocm7.2 torchvision` — ABI must match torch 2.12 | +| **torchaudio** | `2.11.0+rocm7.2` | `pip install --upgrade --no-deps --index-url https://download.pytorch.org/whl/rocm7.2 torchaudio` — ABI must match torch 2.12 | +| **triton** | `3.6.0` | image native, unchanged | +| **fbgemm_gpu** | `fbgemm_gpu_nightly_rocm-2026.6.1` (rebuilt against torch 2.12) | same FBGEMM commit `1509423`, same build command as Stack A — must rebuild after the torch swap | +| **torchrec** | `1.7.0a0+bf55480` (git tag `v2026.06.01.00`) | `pip install --force-reinstall --no-deps "git+https://github.com/pytorch/torchrec.git@v2026.06.01.00"` | +| **polars-u64-idx** | `1.33.1` | as above | + +Both stacks use the same pinned triton configs; perf is parity. Stack A is +the lower-risk path; Stack B aligns the active torch / torchrec versions +with the B200 path below. + ### Training configuration From `generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin`: diff --git a/recommendation_v4/generative_recommenders/ops/triton/triton_jagged.py b/recommendation_v4/generative_recommenders/ops/triton/triton_jagged.py index 8172a0f7b..3f5609d75 100644 --- a/recommendation_v4/generative_recommenders/ops/triton/triton_jagged.py +++ b/recommendation_v4/generative_recommenders/ops/triton/triton_jagged.py @@ -2152,7 +2152,7 @@ def split_2D_jagged_w_prefix_multirow( @triton_autotune( configs=pinned_or_full( - [triton.Config({"BLOCK_N": 1}, num_warps=2)], + [triton.Config({"BLOCK_N": 8}, num_warps=1)], _get_split_concat_2d_jagged_multirow_configs_wrapper, ), key=["BLOCK_D"], diff --git a/recommendation_v4/generative_recommenders/ops/triton/triton_layer_norm.py b/recommendation_v4/generative_recommenders/ops/triton/triton_layer_norm.py index 5ed508108..62fe626de 100644 --- a/recommendation_v4/generative_recommenders/ops/triton/triton_layer_norm.py +++ b/recommendation_v4/generative_recommenders/ops/triton/triton_layer_norm.py @@ -308,7 +308,7 @@ def _layer_norm_bwd_dx( @triton_autotune( configs=pinned_or_full( - [triton.Config({"BLOCK_N": 8}, num_warps=1)], + [triton.Config({"BLOCK_N": 1}, num_warps=1)], _get_layer_norm_fwd_configs, ), key=["BLOCK_D"], From 671e7e680d97bc2609e472e03a7d35d0857cde1e Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Mon, 1 Jun 2026 19:54:33 -0500 Subject: [PATCH 020/127] docs: update MI350X Stack B to fbgemm @ B200 commit + caveat Bumps the Stack B (torch 2.12 / torchrec 1.7) section to: - fbgemm commit 10b77573 (same SHA as the B200 path) instead of 1509423 (one cosmetic commit behind). Wheel rename 2026.6.1 -> 2026.6.2. - Note that Stack A and Stack B use different pinned triton configs (already merged) and explain why (torchrec 1.7 invokes the kernels at different shape keys). - Caveat: HSTU_HAMMER_KERNEL=PYTORCH fallback regresses to ~169 ms on Stack B (vs 107 ms on Stack A). TRITON is unaffected and remains the default; this only matters for PYTORCH-backend debugging. Co-Authored-By: Claude Opus 4.7 --- recommendation_v4/docs/training_recipe.md | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/recommendation_v4/docs/training_recipe.md b/recommendation_v4/docs/training_recipe.md index d18221cb3..c4995b6b7 100644 --- a/recommendation_v4/docs/training_recipe.md +++ b/recommendation_v4/docs/training_recipe.md @@ -42,7 +42,7 @@ pinned triton autotune configs. | **torchrec** | `1.4.0` | image native | | **polars-u64-idx** | `1.33.1` | 64-bit row index — `yambda-5b` has > 4.29 B rows. Installed from a pre-staged local tarball by `scripts/launch_smoke_8gpu.sh` (reserved nodes have no outbound DNS) | -**Stack B — torch 2.12 / torchrec 1.7 (B200-aligned):** +**Stack B — torch 2.12 / torchrec 1.7 / fbgemm @ B200 commit (B200-aligned, validated):** | package | version | install | |---|---|---| @@ -50,13 +50,24 @@ pinned triton autotune configs. | **torchvision** | `0.27.0+rocm7.2` | `pip install --upgrade --no-deps --index-url https://download.pytorch.org/whl/rocm7.2 torchvision` — ABI must match torch 2.12 | | **torchaudio** | `2.11.0+rocm7.2` | `pip install --upgrade --no-deps --index-url https://download.pytorch.org/whl/rocm7.2 torchaudio` — ABI must match torch 2.12 | | **triton** | `3.6.0` | image native, unchanged | -| **fbgemm_gpu** | `fbgemm_gpu_nightly_rocm-2026.6.1` (rebuilt against torch 2.12) | same FBGEMM commit `1509423`, same build command as Stack A — must rebuild after the torch swap | +| **fbgemm_gpu** | `fbgemm_gpu_nightly_rocm-2026.6.2` (built from FBGEMM commit `10b77573`, same SHA as the B200 path) | rebuild against torch 2.12. Build command unchanged from Stack A: `python setup.py -j 32 bdist_wheel --build-target=default --build-variant=rocm -DHIP_ROOT_DIR=/opt/rocm -DAMDGPU_TARGETS=gfx950` | | **torchrec** | `1.7.0a0+bf55480` (git tag `v2026.06.01.00`) | `pip install --force-reinstall --no-deps "git+https://github.com/pytorch/torchrec.git@v2026.06.01.00"` | | **polars-u64-idx** | `1.33.1` | as above | -Both stacks use the same pinned triton configs; perf is parity. Stack A is -the lower-risk path; Stack B aligns the active torch / torchrec versions -with the B200 path below. +Both stacks land at the same ~52 ms/step steady state with the TRITON HSTU +backend + pinned triton configs. Pinned configs differ between Stack A and +Stack B — the torchrec 1.7 code path invokes layer-norm / jagged kernels at +different shape keys than torchrec 1.4, so Stack B uses a re-captured pin +set (already merged in this repo; flip Stack via Container A vs B above). + +Stack A is the lower-risk path (no torch swap). Stack B aligns the active +torch / torchrec / fbgemm SHA exactly with the B200 path below, useful +for cross-platform A/B comparisons. + +**Caveat:** on Stack B the `HSTU_HAMMER_KERNEL=PYTORCH` fallback regresses +to ~169 ms/step (vs ~107 ms on Stack A). Only the TRITON HSTU path is +performance-parity across stacks. Default config uses TRITON so this only +matters if you intentionally force PYTORCH for debugging. ### Training configuration From b3d17641cb8c75cb9b2b4da404bd3307b1b90dee Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Mon, 1 Jun 2026 19:55:20 -0500 Subject: [PATCH 021/127] docs: drop Stack A; MI350X recipe is now single-stack (B200-aligned) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Collapses the two-stack MI350X section into one canonical dependency table: torch 2.12 / torchrec 1.7 / fbgemm @ 10b77573 — the same SHAs as the B200 path. The image-native torch 2.10 / torchrec 1.4 / fbgemm 2026.5.14 path still works for development but the recipe doc now documents the validated production stack only. PYTORCH-backend caveat preserved. Co-Authored-By: Claude Opus 4.7 --- recommendation_v4/docs/training_recipe.md | 41 ++++++----------------- 1 file changed, 10 insertions(+), 31 deletions(-) diff --git a/recommendation_v4/docs/training_recipe.md b/recommendation_v4/docs/training_recipe.md index c4995b6b7..f1ed651f2 100644 --- a/recommendation_v4/docs/training_recipe.md +++ b/recommendation_v4/docs/training_recipe.md @@ -29,20 +29,9 @@ the ROCm-matched build used by triton/fbgemm. ### Dependency versions -Two stacks validated; both land at the same ~52 ms/step steady state with the -pinned triton autotune configs. - -**Stack A — image-native torch (default, no torch swap):** - -| package | version | notes | -|---|---|---| -| **torch** | `2.10.0+git94c6e04` | native to the image; not reinstalled | -| **triton** | `3.6.0` | native to the image | -| **fbgemm_gpu** | `fbgemm_gpu_nightly_rocm-2026.6.1` (built from FBGEMM commit `1509423`, 2026-06-01 `main`) for `gfx950` | image ships `2026.5.14`; rebuild from source gives a measurable boost from the TBE-forward V2 grid-striding (#5669) + warpSize 32/64 unified build (#5739) + `__syncthreads` cleanup (#5744). Build command: `python setup.py -j 32 bdist_wheel --build-target=default --build-variant=rocm -DHIP_ROOT_DIR=/opt/rocm -DAMDGPU_TARGETS=gfx950` | -| **torchrec** | `1.4.0` | image native | -| **polars-u64-idx** | `1.33.1` | 64-bit row index — `yambda-5b` has > 4.29 B rows. Installed from a pre-staged local tarball by `scripts/launch_smoke_8gpu.sh` (reserved nodes have no outbound DNS) | - -**Stack B — torch 2.12 / torchrec 1.7 / fbgemm @ B200 commit (B200-aligned, validated):** +Aligned with the B200 path: same torch major.minor, same torchrec commit, +same fbgemm SHA. The image's native torch / torchvision / torchaudio / +torchrec / fbgemm_gpu are all replaced; only the image's triton stays. | package | version | install | |---|---|---| @@ -50,24 +39,14 @@ pinned triton autotune configs. | **torchvision** | `0.27.0+rocm7.2` | `pip install --upgrade --no-deps --index-url https://download.pytorch.org/whl/rocm7.2 torchvision` — ABI must match torch 2.12 | | **torchaudio** | `2.11.0+rocm7.2` | `pip install --upgrade --no-deps --index-url https://download.pytorch.org/whl/rocm7.2 torchaudio` — ABI must match torch 2.12 | | **triton** | `3.6.0` | image native, unchanged | -| **fbgemm_gpu** | `fbgemm_gpu_nightly_rocm-2026.6.2` (built from FBGEMM commit `10b77573`, same SHA as the B200 path) | rebuild against torch 2.12. Build command unchanged from Stack A: `python setup.py -j 32 bdist_wheel --build-target=default --build-variant=rocm -DHIP_ROOT_DIR=/opt/rocm -DAMDGPU_TARGETS=gfx950` | +| **fbgemm_gpu** | `fbgemm_gpu_nightly_rocm-2026.6.2` (built from FBGEMM commit `10b77573`, same SHA as the B200 path) for `gfx950` | rebuild from source against the replaced torch. Build command: `python setup.py -j 32 bdist_wheel --build-target=default --build-variant=rocm -DHIP_ROOT_DIR=/opt/rocm -DAMDGPU_TARGETS=gfx950` | | **torchrec** | `1.7.0a0+bf55480` (git tag `v2026.06.01.00`) | `pip install --force-reinstall --no-deps "git+https://github.com/pytorch/torchrec.git@v2026.06.01.00"` | -| **polars-u64-idx** | `1.33.1` | as above | - -Both stacks land at the same ~52 ms/step steady state with the TRITON HSTU -backend + pinned triton configs. Pinned configs differ between Stack A and -Stack B — the torchrec 1.7 code path invokes layer-norm / jagged kernels at -different shape keys than torchrec 1.4, so Stack B uses a re-captured pin -set (already merged in this repo; flip Stack via Container A vs B above). - -Stack A is the lower-risk path (no torch swap). Stack B aligns the active -torch / torchrec / fbgemm SHA exactly with the B200 path below, useful -for cross-platform A/B comparisons. - -**Caveat:** on Stack B the `HSTU_HAMMER_KERNEL=PYTORCH` fallback regresses -to ~169 ms/step (vs ~107 ms on Stack A). Only the TRITON HSTU path is -performance-parity across stacks. Default config uses TRITON so this only -matters if you intentionally force PYTORCH for debugging. +| **polars-u64-idx** | `1.33.1` | 64-bit row index — `yambda-5b` has > 4.29 B rows. Installed from a pre-staged local tarball by `scripts/launch_smoke_8gpu.sh` | + +**Caveat:** the `HSTU_HAMMER_KERNEL=PYTORCH` fallback path regresses on +torch 2.12 (~169 ms/step vs ~107 ms on torch 2.10). The default TRITON +HSTU backend is unaffected — only matters if you intentionally force +PYTORCH for debugging. ### Training configuration From 7f6553e07d2e7dfbaef46072ae3ad0ec590785f4 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Mon, 1 Jun 2026 19:55:41 -0500 Subject: [PATCH 022/127] docs: drop PYTORCH-fallback caveat from MI350X recipe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Not relevant — TRITON is the documented default backend. Co-Authored-By: Claude Opus 4.7 --- recommendation_v4/docs/training_recipe.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/recommendation_v4/docs/training_recipe.md b/recommendation_v4/docs/training_recipe.md index f1ed651f2..34297fc40 100644 --- a/recommendation_v4/docs/training_recipe.md +++ b/recommendation_v4/docs/training_recipe.md @@ -43,11 +43,6 @@ torchrec / fbgemm_gpu are all replaced; only the image's triton stays. | **torchrec** | `1.7.0a0+bf55480` (git tag `v2026.06.01.00`) | `pip install --force-reinstall --no-deps "git+https://github.com/pytorch/torchrec.git@v2026.06.01.00"` | | **polars-u64-idx** | `1.33.1` | 64-bit row index — `yambda-5b` has > 4.29 B rows. Installed from a pre-staged local tarball by `scripts/launch_smoke_8gpu.sh` | -**Caveat:** the `HSTU_HAMMER_KERNEL=PYTORCH` fallback path regresses on -torch 2.12 (~169 ms/step vs ~107 ms on torch 2.10). The default TRITON -HSTU backend is unaffected — only matters if you intentionally force -PYTORCH for debugging. - ### Training configuration From `generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin`: From cb68c8ea4013c0ffe3dab58a51aba23666a24e07 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Mon, 1 Jun 2026 21:39:24 -0500 Subject: [PATCH 023/127] MI350X: fit-entity embedding sizes, bs=1024 default, batch-agnostic recipe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Embedding sizes match the true entity counts in yambda-5b: item_id 9_390_000 -> 9_390_624 artist_id 1_290_000 -> 1_293_395 album_id 3_370_000 -> 3_367_692 uid 1_000_000 -> 1_000_001 This eliminates the recurring "EmbeddingBoundsCheck ... Setting idx to zero" warnings at training time. Gin default raised to batch_size=1024 / eval_batch_size=1024. Measured steady-state on the torch 2.12 + torchrec 1.7 + fbgemm HEAD stack with TRITON HSTU + pinned triton configs: ~635 ms/step, ~12.9K sps, ~2.92 days/epoch vs ~7.6 days at bs=32. bs=2048 is feasible but only +3% throughput at much higher autotune cost, so bs=1024 is the sweet spot. Triton autotune pin for _weighted_layer_norm_bwd_dx now ships TWO configs in the pinned list — BLOCK_N=1 (bs=32 winner) and BLOCK_N=8 (bs=1024 winner). Triton's autotune key=[BLOCK_D] dispatches the right one per shape in <5 sec on cold start (vs ~30 sec from the full pool). The other two pinned kernels (_layer_norm_bwd_dwdb, split_2D_jagged_multirow) have identical winners at bs=32 and bs=1024 so they stay single-config. Training-recipe doc drops the batch_size rows from both MI350X and B200 config tables — the recipe is intentionally batch-size-agnostic now that the pin set covers a range. Co-Authored-By: Claude Opus 4.7 --- recommendation_v4/docs/training_recipe.md | 2 -- .../generative_recommenders/dlrm_v3/configs.py | 8 ++++---- .../dlrm_v3/train/gin/yambda_5b.gin | 4 ++-- .../ops/triton/triton_layer_norm.py | 5 ++++- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/recommendation_v4/docs/training_recipe.md b/recommendation_v4/docs/training_recipe.md index 34297fc40..28c88ded2 100644 --- a/recommendation_v4/docs/training_recipe.md +++ b/recommendation_v4/docs/training_recipe.md @@ -49,8 +49,6 @@ From `generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin`: | parameter | value | gin binding | |---|---|---| -| batch_size (train) | 32 | `make_train_test_dataloaders.batch_size` | -| eval_batch_size | 32 | `make_train_test_dataloaders.eval_batch_size` | | num_workers (dataloader) | 4 | `make_train_test_dataloaders.num_workers` | | prefetch_factor | 8 | `make_train_test_dataloaders.prefetch_factor` | | num_blocks | 1 | `make_train_test_dataloaders.num_blocks` | diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/configs.py b/recommendation_v4/generative_recommenders/dlrm_v3/configs.py index 1fd7f07a9..1b6ecf62f 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/configs.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/configs.py @@ -684,28 +684,28 @@ def get_embedding_table_config( assert dataset in ["yambda-5b"] tables: Dict[str, EmbeddingConfig] = { "item_id": EmbeddingConfig( - num_embeddings=9_390_000, + num_embeddings=9_390_624, embedding_dim=DIM, name="item_id", data_type=DataType.FP32, feature_names=["item_id", "item_candidate_id"], ), "artist_id": EmbeddingConfig( - num_embeddings=1_290_000, + num_embeddings=1_293_395, embedding_dim=DIM, name="artist_id", data_type=DataType.FP32, feature_names=["artist_id", "item_candidate_artist_id"], ), "album_id": EmbeddingConfig( - num_embeddings=3_370_000, + num_embeddings=3_367_692, embedding_dim=DIM, name="album_id", data_type=DataType.FP32, feature_names=["album_id", "item_candidate_album_id"], ), "uid": EmbeddingConfig( - num_embeddings=1_000_000, + num_embeddings=1_000_001, embedding_dim=DIM, name="uid", data_type=DataType.FP32, diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index da5727de3..491cc853a 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -1,4 +1,4 @@ -batch_size = 32 +batch_size = 1024 num_workers = 4 prefetch_factor = 8 dataset = "yambda-5b" @@ -42,7 +42,7 @@ data/env_path.default = "/apps/chcai/dlrm_data" # dataloader configs make_train_test_dataloaders.batch_size = %batch_size -make_train_test_dataloaders.eval_batch_size = 32 +make_train_test_dataloaders.eval_batch_size = 1024 make_train_test_dataloaders.dataset_type = %dataset make_train_test_dataloaders.train_split_percentage = 0.90 make_train_test_dataloaders.new_path_prefix = %DATA_PATH diff --git a/recommendation_v4/generative_recommenders/ops/triton/triton_layer_norm.py b/recommendation_v4/generative_recommenders/ops/triton/triton_layer_norm.py index 62fe626de..cc513433b 100644 --- a/recommendation_v4/generative_recommenders/ops/triton/triton_layer_norm.py +++ b/recommendation_v4/generative_recommenders/ops/triton/triton_layer_norm.py @@ -308,7 +308,10 @@ def _layer_norm_bwd_dx( @triton_autotune( configs=pinned_or_full( - [triton.Config({"BLOCK_N": 1}, num_warps=1)], + [ + triton.Config({"BLOCK_N": 1}, num_warps=1), # bs=32 winner + triton.Config({"BLOCK_N": 8}, num_warps=1), # bs=1024 winner + ], _get_layer_norm_fwd_configs, ), key=["BLOCK_D"], From 2e95a4d24786798b5f71cb83a52f3c85d9e21481 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Tue, 2 Jun 2026 01:32:45 -0500 Subject: [PATCH 024/127] MI350X: separated-RNG LN-dropout + attention autotune pin + clock guard Enable the multi-row, separated-RNG _ln_mul_dropout path on AMD MI350 (gfx950), previously Blackwell-only. Batches rows per program and reuses a precomputed dropout mask in the backward instead of one-program-per-row fused RNG; +5.6% end-to-end (-> 14,222 global sps) at bs=1024 on yambda-5b. - ops/utils.py: add is_amd_mi350() + use_separated_rng_ln_mul_dropout() gate. - ops/triton/triton_hstu_linear.py: dispatch the fwd LN-dropout to the separated-RNG path via the new gate. - ops/triton/triton_hstu_attention.py: pin fast nonkdim:16 fwd/persistent/bwd configs via pinned_or_full (TRITON_FULL_AUTOTUNE=1 still bypasses). Multi-config lists with an inline "add a new batch size" guide. - scripts/launch_smoke_8gpu.sh: GPU clock sanity guard - log perf level + sclk, auto-restore 'auto' if a perf_determinism/manual/low lock is found (a half-clock lock uniformly slowed every Triton kernel ~1.9x and masked perf changes). - docs/perf_opt.md: document the LN-dropout fix and the clock-lock caveat. Co-authored-by: Cursor --- recommendation_v4/docs/perf_opt.md | 73 ++++++++++++++++ .../ops/triton/triton_hstu_attention.py | 85 ++++++++++++++++++- .../ops/triton/triton_hstu_linear.py | 15 ++-- .../generative_recommenders/ops/utils.py | 31 +++++++ .../scripts/launch_smoke_8gpu.sh | 20 +++++ 5 files changed, 216 insertions(+), 8 deletions(-) create mode 100644 recommendation_v4/docs/perf_opt.md diff --git a/recommendation_v4/docs/perf_opt.md b/recommendation_v4/docs/perf_opt.md new file mode 100644 index 000000000..627ab74aa --- /dev/null +++ b/recommendation_v4/docs/perf_opt.md @@ -0,0 +1,73 @@ +# Performance Optimizations — MI350X HSTU / OneTrans (yambda-5b, bs=1024, TRITON) + +Performance work for the 8× MI350X HSTU ranker on `yambda-5b` at `batch_size=1024` +with the **TRITON** HSTU kernel and bf16 training. Companion to +[`training_recipe.md`](./training_recipe.md) (environment + reproduction). + +Throughput numbers are global samples/sec across 8 GPUs (`global_sps`), measured +at steady state (instantaneous, computed from consecutive logged steps). + +--- + +## LN-dropout: multi-row, separated-RNG path on MI350 + +### What + +`_ln_mul_dropout_*` has two kernel variants: + +- **legacy** — single program per row, RNG fused inline (`_ln_mul_dropout_fwd`). +- **separated-RNG** — multiple rows per program, dropout mask precomputed once + and reused by the backward (`_ln_mul_dropout_fwd_rng` / + `_ln_mul_dropout_bwd_dx_du_rng`). + +The separated path was previously gated to Blackwell only (`is_sm100_plus()`). +MI350X (`gfx950`) benefits from the same structure, so the gate now also enables +it on MI350. + +### Where + +| file | change | +|---|---| +| `ops/utils.py` | `is_amd_mi350()` (gfx950 detect) + `use_separated_rng_ln_mul_dropout()` gate | +| `ops/triton/triton_hstu_linear.py` | dispatch LN-dropout fwd to the separated-RNG path when the gate is true | + +```python +# ops/utils.py +def use_separated_rng_ln_mul_dropout() -> bool: + return is_sm100_plus() or is_amd_mi350() +``` + +### Perf + +**+5.6% end-to-end → 14,222 global sps** (separated-RNG vs legacy fused, identical +config, full boost clocks — see the caveat below). + +--- + +## Caveat — GPU clock lock can mask all perf changes + +A node-level GPU clock lock will silently invalidate any benchmark on this +machine, so check it before trusting numbers. + +During this work all 8 GPUs were stuck in **`perf_determinism`** performance +level at **sclk 1093 MHz** (DPM level 1) while the real max is **2200 MHz** +(level 2) — despite 100% utilization, ~370 W of power headroom (629 / 1000 W), +and low temps (~50 °C). This was **not** thermal/power throttling; it was +leftover node state from a prior job. + +Effect: a **uniform ~1.87× slowdown of every Triton compute kernel** +(`2200 / 1093 ≈ 2.0×`), including kernels unrelated to any code change. It made +the LN-dropout fix above look like a regression until the clock state was found. + +### Detect + fix + +```bash +rocm-smi --showperflevel # expect "auto", not perf_determinism/manual/low +rocm-smi -d 0 --showclocks # expect sclk ~2000+ MHz under load +rocm-smi --setperflevel auto # restore boost +``` + +`scripts/launch_smoke_8gpu.sh` now logs the perf level + a live `sclk` sample on +every launch, auto-restores `auto` if it finds a `perf_determinism`/`manual`/`low` +lock, and warns (to reset from the host) if it lacks permission inside the +container. **Always sanity-check `sclk ≈ 2000+ MHz` before trusting a benchmark.** diff --git a/recommendation_v4/generative_recommenders/ops/triton/triton_hstu_attention.py b/recommendation_v4/generative_recommenders/ops/triton/triton_hstu_attention.py index 03a1f8f67..768ef0013 100644 --- a/recommendation_v4/generative_recommenders/ops/triton/triton_hstu_attention.py +++ b/recommendation_v4/generative_recommenders/ops/triton/triton_hstu_attention.py @@ -46,6 +46,7 @@ switch_to_contiguous_if_needed, triton_autotune, ) +from generative_recommenders.ops.triton._autotune_pinning import pinned_or_full from triton.language.extra.libdevice import ( # @manual=//triton:triton fast_dividef, fast_expf, @@ -1585,8 +1586,50 @@ def _hstu_attn_fwd_compute_tlx( # noqa C901 tl.store(out_ptrs, acc, mask=(offs_m < seq_len)[:, None]) +def _get_fw_pinned_configs() -> List[triton.Config]: + # Pinned forward-attention configs for MI350X gfx950. The full search is a + # coin flip between matrix_instr_nonkdim 16 (fast) and 32 (slow); pinning the + # known winners makes cold starts deterministic. See _autotune_pinning. + # + # This is a LIST, one entry per training shape we've tuned (currently just + # bs=1024). With >1 entry the autotuner still runs a tiny benchmark over only + # these candidates and caches the winner per `key` (AUTOTUNE_Z / H / + # AUTOTUNE_MAX_SEQ_LEN / DimQ / DimV / ...), so each batch size automatically + # picks its own config — same pattern as the layer-norm pins. + # + # TO ADD A NEW BATCH SIZE / SHAPE: + # 1. Run once with TRITON_FULL_AUTOTUNE=1 TRITON_PRINT_AUTOTUNING=1. + # 2. Grep the log for "best config selected:" under "_hstu_attn_fwd". + # 3. Append that config below (copy BLOCK_M/BLOCK_N/matrix_instr_nonkdim/ + # waves_per_eu/kpack/num_stages/num_warps verbatim). + # The four USE_TLX/NUM_* defaults below are required by the kernel signature + # (see the USE_TLX-default loop in _get_fw_configs); the pinned path bypasses + # that loop, so every pinned entry must set them explicitly. + if torch.version.hip: + return [ + # --- yambda bs=1024, L=2048 winner (from capture log) --- + triton.Config( + { + "BLOCK_M": 128, + "BLOCK_N": 32, + "matrix_instr_nonkdim": 16, + "waves_per_eu": 0, + "kpack": 2, + "USE_TLX": False, + "NUM_BUFFERS": 1, + "NUM_MMA_WARPS_PER_GROUP": 1, + "NUM_MMA_GROUPS": 1, + }, + num_stages=2, + num_warps=8, + ), + # --- add more (bs, L) winners here; see "TO ADD A NEW BATCH SIZE" --- + ] + return _get_fw_configs() + + @triton_autotune( - configs=_get_fw_configs(), + configs=pinned_or_full(_get_fw_pinned_configs(), _get_fw_configs), key=[ "AUTOTUNE_Z", "H", @@ -1730,7 +1773,7 @@ def _hstu_attn_fwd( # noqa C901 @triton_autotune( - configs=_get_fw_configs(), + configs=pinned_or_full(_get_fw_pinned_configs(), _get_fw_configs), key=[ "AUTOTUNE_Z", "H", @@ -2390,8 +2433,44 @@ def _get_bw_configs() -> List[triton.Config]: return configs +def _get_bw_pinned_configs() -> List[triton.Config]: + # Pinned backward-attention configs for MI350X gfx950. Pins the fast + # matrix_instr_nonkdim=16 winner(s) to avoid the 16-vs-32 autotune lottery. + # + # LIST, one entry per tuned shape (currently just bs=1024). With >1 entry the + # autotuner benchmarks only these candidates and caches the winner per `key` + # (AUTOTUNE_Z / H / AUTOTUNE_MAX_SEQ_LEN / DimQ / DimV), so each batch size + # picks its own config automatically — same pattern as the layer-norm pins. + # + # TO ADD A NEW BATCH SIZE / SHAPE: + # 1. Run once with TRITON_FULL_AUTOTUNE=1 TRITON_PRINT_AUTOTUNING=1. + # 2. Grep the log for "best config selected:" under "_hstu_attn_bwd". + # 3. Append that config below (verbatim BLOCK_M/BLOCK_N/matrix_instr_nonkdim/ + # waves_per_eu/SEQUENCE_PARALLEL/UNROLL/num_stages/num_warps). + # Keep pre_hook=_bwd_pre_hook on every entry (the bwd configs require it). + if torch.version.hip: + return [ + # --- yambda bs=1024, L=2048 winner (from capture log) --- + triton.Config( + { + "BLOCK_M": 32, + "BLOCK_N": 128, + "matrix_instr_nonkdim": 16, + "waves_per_eu": 0, + "SEQUENCE_PARALLEL": False, + "UNROLL": 1, + }, + num_stages=1, + num_warps=4, + pre_hook=_bwd_pre_hook, + ), + # --- add more (bs, L) winners here; see "TO ADD A NEW BATCH SIZE" --- + ] + return _get_bw_configs() + + @triton_autotune( - configs=_get_bw_configs(), + configs=pinned_or_full(_get_bw_pinned_configs(), _get_bw_configs), key=[ "AUTOTUNE_Z", "H", diff --git a/recommendation_v4/generative_recommenders/ops/triton/triton_hstu_linear.py b/recommendation_v4/generative_recommenders/ops/triton/triton_hstu_linear.py index ff04dde40..516a15664 100644 --- a/recommendation_v4/generative_recommenders/ops/triton/triton_hstu_linear.py +++ b/recommendation_v4/generative_recommenders/ops/triton/triton_hstu_linear.py @@ -48,7 +48,7 @@ def _get_layer_norm_mul_dropout_fwd_multirow_configs() -> List[triton.Config]: return configs -from generative_recommenders.ops.utils import is_sm100_plus +from generative_recommenders.ops.utils import use_separated_rng_ln_mul_dropout # @manual=//triton:triton from triton.language.extra import libdevice @@ -1064,11 +1064,16 @@ def _triton_layer_norm_mul_dropout_fwd_impl( num_warps: int = min(max(BLOCK_D // 256, 1), 8) random_mask: torch.Tensor = torch.empty(0, dtype=x.dtype, device=x.device) - # Benchmark shows separating RNG from ln_mul_dropout kernel only benefits on - # blackwell when CONCAT_UX is enabled. (fused RNG kernel can benefit from rand3x fast - # dropout) + # Separating RNG from the ln_mul_dropout kernel lets us batch multiple rows per + # program (autotuned _ln_mul_dropout_fwd_rng) and reuse the precomputed mask in the + # backward, instead of launching one program per row with fused RNG. This is a large + # win on Blackwell (sm_100) and AMD MI350 (gfx950); other GPUs keep the fused path. # Extended to support concat_u + concat_x for mask reuse optimization - if not FUSE_OUTPUT_LN_RNG_BLACKWELL and is_sm100_plus() and training: + if ( + not FUSE_OUTPUT_LN_RNG_BLACKWELL + and use_separated_rng_ln_mul_dropout() + and training + ): random_mask = _create_dropout_mask( N=N, D=D, diff --git a/recommendation_v4/generative_recommenders/ops/utils.py b/recommendation_v4/generative_recommenders/ops/utils.py index 94ab69e30..16edd99a9 100644 --- a/recommendation_v4/generative_recommenders/ops/utils.py +++ b/recommendation_v4/generative_recommenders/ops/utils.py @@ -84,6 +84,37 @@ def is_sm90_plus() -> bool: return is_sm100_plus() or is_sm90() +@functools.lru_cache(maxsize=None) +def is_amd_mi350() -> bool: + """Detect an AMD Instinct MI350-class GPU (gfx950) running under ROCm. + + MI350 benefits from the same multi-row, separated-RNG layer-norm-mul-dropout + path as Blackwell datacenter parts (sm_100), so it is gated together with + is_sm100_plus() at the kernel dispatch sites. + """ + if not torch.cuda.is_available(): + return False + if getattr(torch.version, "hip", None) is None: + return False + try: + arch = torch.cuda.get_device_properties(0).gcnArchName or "" + except (AssertionError, RuntimeError, AttributeError): + return False + return "gfx950" in arch + + +def use_separated_rng_ln_mul_dropout() -> bool: + """Hardware that should use the autotuned, multi-row ``_ln_mul_dropout_fwd_rng`` + kernel with a precomputed dropout mask instead of the legacy single-row, + fused-RNG ``_ln_mul_dropout_fwd`` kernel. + + Blackwell datacenter GPUs (sm_100-103) and AMD MI350 (gfx950) both prefer the + separated-RNG path: it batches rows per program and lets the backward reuse the + same mask, which is a large win over launching one program per row. + """ + return is_sm100_plus() or is_amd_mi350() + + def copy_if_different_ptr(dst: torch.Tensor, src: torch.Tensor) -> None: if torch.compiler.is_compiling(): # .data_ptr() will break PT2 diff --git a/recommendation_v4/scripts/launch_smoke_8gpu.sh b/recommendation_v4/scripts/launch_smoke_8gpu.sh index 92daa6ef8..ad363cfaa 100755 --- a/recommendation_v4/scripts/launch_smoke_8gpu.sh +++ b/recommendation_v4/scripts/launch_smoke_8gpu.sh @@ -32,6 +32,26 @@ export WORLD_SIZE=$(python -c "import torch; print(torch.cuda.device_count())") # PYTORCH backend. On CUDA, unset this to default to TRITON for ~3-5x speedup. export HSTU_HAMMER_KERNEL=${HSTU_HAMMER_KERNEL:-PYTORCH} export PYTORCH_CUDA_ALLOC_CONF=${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True} + +# --- GPU clock sanity guard --------------------------------------------------- +# Leftover node state once pinned all 8 GPUs into `perf_determinism` at half +# clock (1093 vs 2200 MHz max). That uniformly slowed every Triton kernel ~1.9x +# and silently masked real perf changes for an entire debugging session. Always +# log the perf level + a live sclk sample so a capped run is obvious from the +# log, and try to restore boost. Fully non-fatal (rocm-smi may be absent or +# lack permission inside the container — in that case reset from the host). +if command -v rocm-smi >/dev/null 2>&1; then + echo "[$(date)] GPU perf-level check:" | tee -a "$LOG" + rocm-smi --showperflevel 2>/dev/null | grep -iE "GPU\[[0-9]+\]" | tee -a "$LOG" || true + if rocm-smi --showperflevel 2>/dev/null | grep -iqE "Performance Level: *(perf_determinism|manual|low)"; then + echo "[$(date)] WARNING: GPUs not in 'auto' perf level — attempting --setperflevel auto" | tee -a "$LOG" + rocm-smi --setperflevel auto 2>/dev/null | grep -iE "set to auto" | tee -a "$LOG" \ + || echo "[$(date)] WARNING: could not set perf level (no permission?). Run 'rocm-smi --setperflevel auto' on the HOST before benchmarking — clocks may be capped." | tee -a "$LOG" + fi + echo "[$(date)] sclk sample (GPU0):$(rocm-smi -d 0 --showclocks 2>/dev/null | grep -i 'sclk clock level' | sed -E 's/.*sclk clock level//')" | tee -a "$LOG" || true +fi +# ----------------------------------------------------------------------------- + echo "[$(date)] launching train_ranker with WORLD_SIZE=$WORLD_SIZE" | tee -a "$LOG" python -m generative_recommenders.dlrm_v3.train.train_ranker \ From 17e04af320676b25b56252d1a5ee2fa5d13c8e76 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Tue, 2 Jun 2026 16:17:56 -0500 Subject: [PATCH 025/127] dlrmv4: TorchRec 3-stage sparse-dist pipeline + gin-selectable HSTU kernel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an opt-in TrainPipelineSparseDist path that overlaps the embedding input-distribution all-to-all with dense fwd/bwd. To make the embedding collection pipelineable, the merged sparse KJT is now pre-built in the dataloader (Samples.merged_sparse_features) and the model consumes it via a _pipeline_mode forward that takes the batch as a single arg, so TorchRec's tracer resolves the lookup input as a plain getattr off the batch. - dataset.py: Samples.merged_sparse_features + merge_uih_candidate_kjts, built in collate_fn; wired into to()/record_stream()/pin_memory(). - dlrm_hstu.py: _pipeline_mode flag; forward unpacks the batch and preprocess accepts the prebuilt merged KJT (falls back to building it when absent). - utils.py: _PipelineModelWrapper, build_train_pipeline, train_eval_loop use_pipeline branch + eval batch-arg; seed all RNGs in setup() for reproducible weight init. - gin/launch: make_model.hammer_kernel selects TRITON vs PYTORCH (env override still honored); launch script defers to the gin default. use_pipeline defaults to False. Validated on MI350/ROCm 8-GPU: embedding collection is pipelined (input-dist a2a moves to hidden); model quality and throughput match the sequential path (seeded A/B). The exposed embedding-output a2a still dominates the step, so throughput is unchanged — pipelining is quality- and perf-neutral here. Co-authored-by: Cursor --- .../dlrm_v3/datasets/dataset.py | 81 ++++++- .../dlrm_v3/train/gin/yambda_5b.gin | 9 + .../dlrm_v3/train/utils.py | 219 +++++++++++++++--- .../modules/dlrm_hstu.py | 67 ++++-- .../scripts/launch_smoke_8gpu.sh | 9 +- 5 files changed, 323 insertions(+), 62 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/datasets/dataset.py b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/dataset.py index a1cbb33fa..204c06df1 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/datasets/dataset.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/dataset.py @@ -28,6 +28,7 @@ import torch from generative_recommenders.modules.dlrm_hstu import DlrmHSTUConfig from torchrec.sparse.jagged_tensor import KeyedJaggedTensor +from torchrec.streamable import Pipelineable logging.basicConfig(level=logging.INFO) @@ -35,7 +36,7 @@ @dataclass -class Samples: +class Samples(Pipelineable): """ Container for batched samples with user interaction history and candidate features. @@ -46,16 +47,48 @@ class Samples: uih_features_kjt: KeyedJaggedTensor candidates_features_kjt: KeyedJaggedTensor - - def to(self, device: torch.device) -> None: + # UIH + candidate features concatenated into the single KJT that the model's + # sharded EmbeddingCollection consumes. Pre-built here (dataloader/CPU) rather + # than inside DlrmHSTU.forward so the embedding lookup's input is a plain + # attribute of the batch — which lets TorchRec's TrainPipelineSparseDist hoist + # its input_dist into the prefetch stage (otherwise the runtime cat + + # from_lengths_sync counts as an "input modification" and the embedding + # collection is left un-pipelined). + merged_sparse_features: KeyedJaggedTensor + + def to(self, device: torch.device, non_blocking: bool = False) -> "Samples": """ - Move all tensors to the specified device. + Move all tensors to the specified device (in place) and return self. - Args: - device: Target device to move tensors to. + Returning ``self`` (rather than ``None``) and accepting ``non_blocking`` + makes ``Samples`` conform to TorchRec's ``Pipelineable`` protocol so it + can be driven by ``TrainPipelineSparseDist``. Existing call sites that + use ``sample.to(device)`` for its side effect continue to work unchanged. """ for attr in vars(self): - setattr(self, attr, getattr(self, attr).to(device=device)) + setattr( + self, + attr, + getattr(self, attr).to(device=device, non_blocking=non_blocking), + ) + return self + + def record_stream(self, stream: torch.Stream) -> None: + """Record the contained KJTs on ``stream`` (Pipelineable protocol). + + Required by ``TrainPipelineSparseDist`` so the prefetched batch's H2D + copy on the side stream is not freed before compute consumes it. + """ + self.uih_features_kjt.record_stream(stream) + self.candidates_features_kjt.record_stream(stream) + self.merged_sparse_features.record_stream(stream) + + def pin_memory(self) -> "Samples": + """Pin the contained KJTs' host memory (Pipelineable protocol).""" + self.uih_features_kjt = self.uih_features_kjt.pin_memory() + self.candidates_features_kjt = self.candidates_features_kjt.pin_memory() + self.merged_sparse_features = self.merged_sparse_features.pin_memory() + return self def batch_size(self) -> int: """ @@ -67,6 +100,31 @@ def batch_size(self) -> int: return self.uih_features_kjt.stride() +def merge_uih_candidate_kjts( + uih_features: KeyedJaggedTensor, + candidates_features: KeyedJaggedTensor, +) -> KeyedJaggedTensor: + """Concatenate the UIH and candidate KJTs into the single KJT consumed by the + model's ``EmbeddingCollection``. + + Must mirror ``DlrmHSTU.preprocess`` exactly (key order = uih + candidates, + values/lengths concatenated in that order). Built on the dataloader side so + the model can read it straight off the batch and TorchRec can pipeline the + embedding ``input_dist``. + """ + return KeyedJaggedTensor.from_lengths_sync( + keys=uih_features.keys() + candidates_features.keys(), + values=torch.cat( + [uih_features.values(), candidates_features.values()], + dim=0, + ), + lengths=torch.cat( + [uih_features.lengths(), candidates_features.lengths()], + dim=0, + ), + ) + + def collate_fn( samples: List[Tuple[KeyedJaggedTensor, KeyedJaggedTensor]], ) -> Samples: @@ -84,9 +142,14 @@ def collate_fn( candidates_features_kjt_list, ) = list(zip(*samples)) + uih_features_kjt = kjt_batch_func(uih_features_kjt_list) + candidates_features_kjt = kjt_batch_func(candidates_features_kjt_list) return Samples( - uih_features_kjt=kjt_batch_func(uih_features_kjt_list), - candidates_features_kjt=kjt_batch_func(candidates_features_kjt_list), + uih_features_kjt=uih_features_kjt, + candidates_features_kjt=candidates_features_kjt, + merged_sparse_features=merge_uih_candidate_kjts( + uih_features_kjt, candidates_features_kjt + ), ) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index 491cc853a..9715f0b78 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -6,6 +6,11 @@ dataset = "yambda-5b" # model parameters make_model.dataset = %dataset make_model.bf16_training = True +# HSTU attention/compute backend: "TRITON" (fused, flash-style — low HBM) or +# "PYTORCH" (unfused; materializes the dense [B,H,N,N] score tensor, ~32 GiB at +# N=2048/bs=1024). TRITON validated on MI350/ROCm. The HSTU_HAMMER_KERNEL env +# var, if set, overrides this binding for one-off runs. +make_model.hammer_kernel = "TRITON" # False = use pinned triton kernel configs (deterministic; whether that's # the fast or slow equilibrium depends on which config was pinned for the @@ -85,6 +90,10 @@ train_eval_loop.eval_frequency = 5000 train_eval_loop.num_eval_batches = 500 train_eval_loop.checkpoint_frequency = 1000000000 # disable mid-training checkpoints (disk-full guard) train_eval_loop.output_trace = True +# 3-stage TorchRec pipeline: overlaps the embedding all-to-all (the dominant +# exposed-comm bottleneck) with dense fwd/bwd. Set False to fall back to the +# sequential fwd/bwd loop. +train_eval_loop.use_pipeline = False # Run name → recommendation_v4/results// (override via $RUN_NAME env). RUN_NAME = @run/env_path() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py index a957122a3..94237f947 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -38,7 +38,11 @@ get_embedding_table_config, get_hstu_configs, ) -from generative_recommenders.dlrm_v3.datasets.dataset import collate_fn, Dataset +from generative_recommenders.dlrm_v3.datasets.dataset import ( + collate_fn, + Dataset, + Samples, +) from generative_recommenders.dlrm_v3.utils import get_dataset, MetricsLogger, Profiler from generative_recommenders.common import HammerKernel from generative_recommenders.modules.dlrm_hstu import DlrmHSTU, DlrmHSTUConfig @@ -84,6 +88,21 @@ def setup( # leaving stale allocations and triggering OOMs on rank 0. torch.cuda.set_device(device) + # Seed all RNGs so weight init (make_model, called after setup) is + # reproducible across runs. Same seed on every rank → dense params are + # initialized identically across ranks; sharded embeddings are init'd from + # the meta device by DMP. Fixed seed makes pipeline-vs-non-pipeline an + # init-matched A/B (data order is already deterministic via the sampler). + import random + + import numpy as np + + _SEED = 1 + random.seed(_SEED) + np.random.seed(_SEED) + torch.manual_seed(_SEED) + torch.cuda.manual_seed_all(_SEED) + # initialize the process group if not dist.is_initialized(): dist.init_process_group( @@ -205,6 +224,7 @@ def set_epoch(self, epoch: int) -> None: def make_model( dataset: str, bf16_training: bool = False, + hammer_kernel: Optional[str] = None, ) -> Tuple[torch.nn.Module, DlrmHSTUConfig, Dict[str, EmbeddingConfig]]: hstu_config = get_hstu_configs(dataset) table_config = get_embedding_table_config(dataset) @@ -220,13 +240,21 @@ def make_model( bf16_training=bf16_training, ) - # Triton on ROCm fails to compile some jagged kernels at our shapes - # (PassManager::run failed at make_ttgir). Allow the PyTorch backend as a - # global override so AMD smoke runs end-to-end. CUDA paths default to TRITON. - kernel_override = os.environ.get("HSTU_HAMMER_KERNEL", "").upper() - if kernel_override: - model.set_hammer_kernel(HammerKernel[kernel_override]) - logger.warning(f"HSTU_HAMMER_KERNEL override: {kernel_override}") + # HSTU attention/compute kernel backend. Precedence: + # HSTU_HAMMER_KERNEL env var > make_model.hammer_kernel gin > model default. + # The env var stays as an ad-hoc override (e.g. forcing PYTORCH for a one-off + # debug run) without editing the gin. Note: the fused TRITON path avoids + # materializing the dense [B, H, N, N] attention-score tensor that the PYTORCH + # path allocates (~32 GiB at N=2048, bs=1024), so TRITON is both faster and + # far lighter on HBM. On older ROCm, TRITON could hit PassManager errors at + # some shapes (make_ttgir) — fall back to PYTORCH via the gin/env if so. + kernel_choice = ( + os.environ.get("HSTU_HAMMER_KERNEL", "").upper() + or (hammer_kernel.upper() if hammer_kernel else "") + ) + if kernel_choice: + model.set_hammer_kernel(HammerKernel[kernel_choice]) + logger.warning(f"HSTU hammer kernel set to: {kernel_choice}") return ( model, @@ -610,6 +638,97 @@ def eval_loop( print(f"{k}: {v}") +class _PipelineModelWrapper(torch.nn.Module): + """Adapt ``DlrmHSTU.forward`` to the ``(loss, output)`` contract that + ``TrainPipelineSparseDist`` expects. + + The wrapped ``model`` is the same DMP instance handed to the pipeline as + ``model=``; the pipeline rewrites its sharded ``EmbeddingCollection`` in + place, so calling it here is what lets the embedding all-to-all overlap the + dense forward/backward compute. + """ + + def __init__(self, model: torch.nn.Module) -> None: + super().__init__() + self._model = model + + def forward( + self, batch: Samples + ) -> Tuple[torch.Tensor, Tuple[Any, ...]]: + # The model runs in `_pipeline_mode`: it takes the whole batch as its + # single arg and reads the pre-merged sparse KJT off it. This keeps the + # EmbeddingCollection input a plain getattr on the batch placeholder so + # TorchRec pipelines its input_dist (instead of skipping it for "input + # modifications"). + ( + _, + _, + aux_losses, + mt_target_preds, + mt_target_labels, + mt_target_weights, + ) = self._model(batch) + loss = sum(aux_losses.values()) + num_candidates = batch.candidates_features_kjt.lengths().view( + len(batch.candidates_features_kjt.keys()), -1 + )[0] + output = ( + aux_losses, + mt_target_preds, + mt_target_labels, + mt_target_weights, + num_candidates, + ) + return loss, output + + +def build_train_pipeline( + model: torch.nn.Module, + optimizer: Optimizer, + device: torch.device, + grad_clip_norm: float = 1.0, +) -> Any: + """Build a ``TrainPipelineSparseDist`` for the DMP-wrapped HSTU model. + + The 3-stage pipeline overlaps (1) H2D transfer of batch N+2, (2) the sparse + data-dist all-to-all of batch N+1's embedding lookup, and (3) dense fwd/bwd + of batch N, on separate CUDA streams. Requires the model to be wrapped with + ``DistributedModelParallel`` (see ``make_optimizer_and_shard``). + """ + # Lazy import: keeps module import working on torchrec builds that move or + # rename the pipeline, and matches the reference Primus-DLRM setup. + from torchrec.distributed.train_pipeline import TrainPipelineSparseDist + + # Switch the (DMP-wrapped) HSTU model into pipeline mode so both the fx trace + # and the live forward consume the batch as a single arg and read the + # pre-merged sparse KJT off it — required for the embedding input_dist to be + # pipelined. Eval call sites pass the batch the same way (see train_eval_loop). + underlying = model.module if hasattr(model, "module") else model + underlying._pipeline_mode = True + + # The pipeline calls backward()+optimizer.step() internally inside + # progress(), leaving no in-loop hook point for gradient clipping. Clip via + # a full-backward hook (fires after autograd populates dense grads, before + # the optimizer step) to preserve parity with the sequential path's + # clip_grad_norm_(model.parameters(), max_norm=1.0). + if grad_clip_norm and grad_clip_norm > 0: + + def _clip_grads(_m: torch.nn.Module, _gi: Any, _go: Any) -> None: + torch.nn.utils.clip_grad_norm_( + model.parameters(), max_norm=grad_clip_norm + ) + + model.register_full_backward_hook(_clip_grads) + + return TrainPipelineSparseDist( + model=model, + optimizer=optimizer, + device=device, + execute_all_batches=True, + custom_model_fwd=_PipelineModelWrapper(model), + ) + + @gin.configurable def train_eval_loop( rank: int, @@ -628,6 +747,7 @@ def train_eval_loop( eval_frequency: int = 1, start_train_batch_idx: int = 0, start_eval_batch_idx: int = 0, + use_pipeline: bool = False, # lr_scheduler: to-do: Add a scheduler ) -> None: train_batch_idx: int = start_train_batch_idx @@ -638,40 +758,61 @@ def train_eval_loop( eval_data_iterator = iter(eval_dataloader) train_data_iterator = iter(train_dataloader) + # 3-stage TorchRec pipeline (overlaps embedding a2a with dense compute). + # When enabled, progress() owns H2D copy, sparse-dist, fwd/bwd and the + # optimizer step; grad clipping moves to a full-backward hook (see builder). + train_pipeline = ( + build_train_pipeline(model, optimizer, device) if use_pipeline else None + ) + for epoch in range(num_epochs): train_dataloader.sampler.set_epoch(epoch) # pyre-ignore [16] while True: model.train() - try: - sample = next(train_data_iterator) - except StopIteration: - train_data_iterator = iter(train_dataloader) - break - optimizer.zero_grad() - sample.to(device) - ( - _, - _, - aux_losses, - mt_target_preds, - mt_target_labels, - mt_target_weights, - ) = model.forward( - sample.uih_features_kjt, - sample.candidates_features_kjt, - ) - # pyre-ignore - sum(aux_losses.values()).backward() - torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0) - optimizer.step() + if train_pipeline is not None: + try: + ( + aux_losses, + mt_target_preds, + mt_target_labels, + mt_target_weights, + num_candidates, + ) = train_pipeline.progress(train_data_iterator) + except StopIteration: + train_data_iterator = iter(train_dataloader) + break + else: + try: + sample = next(train_data_iterator) + except StopIteration: + train_data_iterator = iter(train_dataloader) + break + optimizer.zero_grad() + sample.to(device) + ( + _, + _, + aux_losses, + mt_target_preds, + mt_target_labels, + mt_target_weights, + ) = model.forward( + sample.uih_features_kjt, + sample.candidates_features_kjt, + ) + # pyre-ignore + sum(aux_losses.values()).backward() + torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0) + optimizer.step() + num_candidates = sample.candidates_features_kjt.lengths().view( + len(sample.candidates_features_kjt.keys()), -1 + )[0] metric_logger.update( mode="train", predictions=mt_target_preds, labels=mt_target_labels, weights=mt_target_weights, - num_candidates=sample.candidates_features_kjt.lengths().view( - len(sample.candidates_features_kjt.keys()), -1 - )[0], + num_candidates=num_candidates, ) if train_batch_idx % metric_log_frequency == 0: metric_logger.compute_and_log( @@ -710,9 +851,15 @@ def train_eval_loop( mt_target_preds, mt_target_labels, mt_target_weights, - ) = model.forward( - sample.uih_features_kjt, - sample.candidates_features_kjt, + ) = ( + # In pipeline mode the model takes the batch as one + # arg (see _PipelineModelWrapper / DlrmHSTU.forward). + model.forward(sample) + if use_pipeline + else model.forward( + sample.uih_features_kjt, + sample.candidates_features_kjt, + ) ) metric_logger.update( mode="eval", diff --git a/recommendation_v4/generative_recommenders/modules/dlrm_hstu.py b/recommendation_v4/generative_recommenders/modules/dlrm_hstu.py index af2edc998..3a35df3ba 100644 --- a/recommendation_v4/generative_recommenders/modules/dlrm_hstu.py +++ b/recommendation_v4/generative_recommenders/modules/dlrm_hstu.py @@ -52,7 +52,18 @@ logger: logging.Logger = logging.getLogger(__name__) +def fx_total_targets(num_candidates: torch.Tensor) -> int: + """Sum a per-sample candidate-count tensor to a Python int. + + Wrapped with ``torch.fx.wrap`` so ``TrainPipelineSparseDist``'s symbolic + trace treats it as an opaque leaf instead of recursing into the data- + dependent ``int(Proxy.sum().item())`` (which raises during tracing). + """ + return int(num_candidates.sum().item()) + + torch.fx.wrap("fx_infer_max_len") +torch.fx.wrap("fx_total_targets") torch.fx.wrap("len") @@ -129,6 +140,12 @@ def __init__( # noqa C901 ) -> None: super().__init__(is_inference=is_inference) logger.info(f"Initialize HSTU module with configs {hstu_configs}") + # When True, forward() takes the whole `Samples` batch as its single + # positional arg and reads the pre-merged sparse KJT off it. This keeps + # the EmbeddingCollection's input a plain getattr on the batch placeholder + # so TorchRec's TrainPipelineSparseDist can pipeline its input_dist. Set + # by build_train_pipeline(); leave False for eager / inference / eval. + self._pipeline_mode: bool = False self._hstu_configs = hstu_configs self._bf16_training: bool = bf16_training set_static_max_seq_lens([self._hstu_configs.max_seq_len]) @@ -345,7 +362,7 @@ def _user_forward( kernel=self.hammer_kernel(), ).squeeze(-1) if total_targets is None: - total_targets = int(num_candidates.sum().item()) + total_targets = fx_total_targets(num_candidates) if total_uih_len is None: total_uih_len = source_timestamps.numel() - total_targets embedding = seq_embeddings[ @@ -415,6 +432,7 @@ def preprocess( self, uih_features: KeyedJaggedTensor, candidates_features: KeyedJaggedTensor, + merged_sparse_features: Optional[KeyedJaggedTensor] = None, ) -> Tuple[ Dict[str, SequenceEmbedding], Dict[str, torch.Tensor], @@ -423,18 +441,25 @@ def preprocess( int, torch.Tensor, ]: - # embedding lookup for uih and candidates - merged_sparse_features = KeyedJaggedTensor.from_lengths_sync( - keys=uih_features.keys() + candidates_features.keys(), - values=torch.cat( - [uih_features.values(), candidates_features.values()], - dim=0, - ), - lengths=torch.cat( - [uih_features.lengths(), candidates_features.lengths()], - dim=0, - ), - ) + # Embedding lookup for uih + candidates. When the caller (the pipeline + # path) supplies the pre-merged KJT from the batch, feed it straight to + # the EmbeddingCollection: that keeps the lookup's input a plain getattr + # off the batch so TorchRec's TrainPipelineSparseDist can hoist its + # input_dist into the prefetch stage. Building it here (cat + + # from_lengths_sync's .sync()) is an "input modification" that makes + # TorchRec skip pipelining the embedding collection. + if merged_sparse_features is None: + merged_sparse_features = KeyedJaggedTensor.from_lengths_sync( + keys=uih_features.keys() + candidates_features.keys(), + values=torch.cat( + [uih_features.values(), candidates_features.values()], + dim=0, + ), + lengths=torch.cat( + [uih_features.lengths(), candidates_features.lengths()], + dim=0, + ), + ) seq_embeddings_dict = self._embedding_collection(merged_sparse_features) num_candidates = fx_mark_length_features( candidates_features.lengths().view(len(candidates_features.keys()), -1) @@ -593,7 +618,8 @@ def main_forward( def forward( self, uih_features: KeyedJaggedTensor, - candidates_features: KeyedJaggedTensor, + candidates_features: Optional[KeyedJaggedTensor] = None, + merged_sparse_features: Optional[KeyedJaggedTensor] = None, ) -> Tuple[ torch.Tensor, torch.Tensor, @@ -602,6 +628,18 @@ def forward( Optional[torch.Tensor], Optional[torch.Tensor], ]: + # Pipeline mode: TorchRec fx-traces this forward (via DMP.module) and the + # pipeline calls it with the single `Samples` batch. Unpacking the KJTs + # here — rather than in the wrapper — makes the EmbeddingCollection's + # input `batch.merged_sparse_features` a getattr off the batch placeholder, + # which is what lets TrainPipelineSparseDist hoist the embedding input_dist + # into the prefetch stage. Guarded from TorchScript (inference path). + if not torch.jit.is_scripting() and self._pipeline_mode: + batch = uih_features + uih_features = batch.uih_features_kjt + candidates_features = batch.candidates_features_kjt + merged_sparse_features = batch.merged_sparse_features + with record_function("## preprocess ##"): ( seq_embeddings, @@ -613,6 +651,7 @@ def forward( ) = self.preprocess( uih_features=uih_features, candidates_features=candidates_features, + merged_sparse_features=merged_sparse_features, ) with record_function("## main_forward ##"): diff --git a/recommendation_v4/scripts/launch_smoke_8gpu.sh b/recommendation_v4/scripts/launch_smoke_8gpu.sh index ad363cfaa..09a3af2ba 100755 --- a/recommendation_v4/scripts/launch_smoke_8gpu.sh +++ b/recommendation_v4/scripts/launch_smoke_8gpu.sh @@ -28,9 +28,12 @@ python -c "import torch, fbgemm_gpu, torchrec, polars, xxhash, gin; print('impor export HIP_VISIBLE_DEVICES=${HIP_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7} export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7} export WORLD_SIZE=$(python -c "import torch; print(torch.cuda.device_count())") -# AMD/ROCm: Triton HSTU kernel hits PassManager errors on some shapes; force -# PYTORCH backend. On CUDA, unset this to default to TRITON for ~3-5x speedup. -export HSTU_HAMMER_KERNEL=${HSTU_HAMMER_KERNEL:-PYTORCH} +# HSTU attention backend is selected in the gin (make_model.hammer_kernel), +# defaulting to TRITON — fused/flash-style, so it avoids the dense [B,H,N,N] +# score tensor the PYTORCH path materializes (~32 GiB at N=2048/bs=1024) and is +# both faster and far lighter on HBM. Only export HSTU_HAMMER_KERNEL=PYTORCH +# before launch for a one-off fallback (e.g. a ROCm Triton PassManager error). +export HSTU_HAMMER_KERNEL=${HSTU_HAMMER_KERNEL:-} export PYTORCH_CUDA_ALLOC_CONF=${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True} # --- GPU clock sanity guard --------------------------------------------------- From 123c55c3aea87d66ad2c7d7ebbee60461b1ba137 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Tue, 2 Jun 2026 22:13:17 -0500 Subject: [PATCH 026/127] dlrmv4: streaming (temporal-order) training for yambda-5b Add a forward-in-time streaming path: slice the timeline into fixed-duration windows (default 1 day), train window T then eval window T+1, enforcing no future leakage (across-window + causal-history guarantees). Make it the default mode in launch_smoke_8gpu.sh. Window-reset overhead is hidden via a persistent worker pool + double buffering (next window's index mask and first-batch prefetch overlap compute on a background thread) and eval-window prefetch one window ahead, dropping train/eval first-batch waits to ~1-3ms with no steady-state regression. Window selection uses a lazily-built, mmap'd anchor-timestamp cache so the default non-streaming path is unaffected. Also harden trace export (best-effort: IO/permission failures warn instead of crashing training) now that streaming enables output_trace by default, and document the path + knobs in the README. Co-authored-by: Cursor --- recommendation_v4/README.MD | 87 +++++- .../dlrm_v3/datasets/yambda.py | 106 +++++++ .../dlrm_v3/train/gin/yambda_5b.gin | 52 +++- .../dlrm_v3/train/utils.py | 264 +++++++++++++++++- .../generative_recommenders/dlrm_v3/utils.py | 27 +- .../scripts/launch_smoke_8gpu.sh | 2 +- 6 files changed, 516 insertions(+), 22 deletions(-) diff --git a/recommendation_v4/README.MD b/recommendation_v4/README.MD index f22f1e165..5c78f6627 100644 --- a/recommendation_v4/README.MD +++ b/recommendation_v4/README.MD @@ -131,6 +131,91 @@ The `like` cap of 679 is unreachable for yambda data — at the 1.9% global like This means the model sees on average ~1,402 UIH events per sample, not the theoretical 2,037. With the TRITON jagged-attention backend the GPU only does work for the actual events, so the under-fill costs **sequence budget but not GPU compute** — no wasted attention work, just less context per sample than the budget suggests. -## 5. License +## 5. Streaming (temporal-order) training + +`scripts/launch_smoke_8gpu.sh` defaults to `--mode streaming-train-eval`, which +trains Yambda in strict wall-clock order instead of shuffling the whole corpus. +The timeline is sliced into fixed-duration **windows** (default 1 day, +`get_dataset.streaming_window_seconds = 86400`), and the loop walks them forward: + +``` +window T: train window T+1: eval (then train) window T+2: eval (then train) ... + └─ train window T ─┐ + └─ eval window T+1 ─┐ + └─ train window T+1 ─┐ + └─ eval window T+2 ... +``` + +i.e. for each step it **trains window T, then evaluates window T+1** before +advancing — always predicting the immediate future from the past. + +### 5.1 Temporal guarantee + +The streaming path enforces **no future leakage** at two levels: + +1. **Across windows** — a window is the set of anchors whose *target/candidate* + timestamp falls in `[t_min + T·W, t_min + (T+1)·W)`. Training only ever sees + windows `≤ T`; the evaluation window `T+1` is strictly in the future of every + training anchor it is scored against. Eval always leads train by exactly one + window, so reported eval NE/AUC is genuine next-period generalization, never + an in-sample measurement. +2. **Within an anchor** — history is still gathered **causally**: the UIH scan + is `scan_start:flat_pos` (events strictly before the anchor), so even though a + long user history may reach back across earlier windows, no event at or after + the anchor's timestamp can enter its features. Forward-time windowing and + causal history are independent guarantees, and both hold simultaneously. + +Note this is a *temporal* split on the training stream — distinct from the +preprocessing GTS split (§2) that carves off the final test day. Windows are +indexed off the per-anchor target timestamp via a lazily-built, mmap'd +`anchor_ts_L{H}.npy` cache (built once on first use; the default non-streaming +path never touches it). + +### 5.2 Knobs + +All configurable via gin ([yambda_5b.gin](generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin)) +with env overrides: + +| env | gin | default | meaning | +|---|---|---|---| +| `START_TS` | `streaming_train_eval_loop.start_ts` | 150 | first window (early windows are near-empty warm-up; start dense) | +| `NUM_TRAIN_TS` | `streaming_train_eval_loop.num_train_ts` | 30 | number of train windows (clamped to available) | +| `PERSISTENT_LOADER` | `streaming_train_eval_loop.persistent_loader` | 1 | reuse one worker pool across windows (no per-window respawn) | +| `DOUBLE_BUFFER` | `streaming_train_eval_loop.double_buffer` | 1 | prepare the next window in a background thread during compute | +| `EVAL_EACH_WINDOW` | `streaming_train_eval_loop.eval_each_window` | 1 | eval window T+1 after training window T | +| — | `streaming_train_eval_loop.num_train_batches` / `num_eval_batches` | unset | cap per-window steps (unset = consume full window) | + +### 5.3 Hiding the window-reset overhead + +Advancing to a new window has a fixed cost — selecting the window's anchor +indices and warming the dataloader's first batch — that, done naively, stalls +training at every window boundary. Three layers drive it to ~0: + +1. **Persistent loader** (`persistent_loader=1`). The naive path recreates a + `DataLoader` per window, re-forking workers and paying first-batch warmup + each time (~11 s/window). Instead we build **one** `DataLoader` backed by a + stateful `StreamingWindowSampler` whose index set is swapped per window + (`set_window`), so workers fork once and persist. This removes the respawn + but still pays the index-mask + first-batch stall (~3.6 s/window). +2. **Double buffering** (`double_buffer=1`). Two pre-forked worker pools + ping-pong: while the current window trains on pool A, the *next* window's + index mask (`window_indices`, a GIL-releasing NumPy `np.where`) and + first-batch prefetch are prepared on pool B in a **background thread**, so + that work overlaps GPU compute. The boundary train batch then arrives warm — + measured train first-batch data-wait drops to **~1–3 ms**. Pools are forked + up front on the main thread (never inside the background thread), so a forking + worker can never race a thread holding a lock. +3. **Eval prefetch one window ahead.** With `eval_each_window=1` the eval window + (`T+1`) is prepared *before* training window `T` runs, so the idle eval pool + prefetches its first batches concurrently with train compute. This hides the + eval-side first-batch stall (**~0.55 s → ~2 ms**). It is safe because a + sample's content depends only on the sampler's window indices, not on any + train/eval flag. + +Net effect: steady-state throughput matches the non-streaming baseline and the +per-window reset is effectively free; the only remaining one-time cost is the +process cold start (CUDA-graph capture + the first lazy `anchor_ts` mmap). + +## 6. License Apache 2.0 (inherited from upstream). diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/datasets/yambda.py b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/yambda.py index 5a13ac034..bed1aafb8 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/datasets/yambda.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/yambda.py @@ -185,6 +185,8 @@ def __init__( cross_specs: Optional[Sequence[Tuple[str, Sequence[str], int, int]]] = None, cache_dir: Optional[str] = None, is_inference: bool = False, + streaming_window_seconds: int = 86400, + streaming_sort_within_window: bool = False, *args, **kwargs, ) -> None: @@ -193,6 +195,17 @@ def __init__( self._metadata_dir: str = metadata_dir self._history_length: int = history_length self._scan_window: int = scan_window + # Streaming/temporal-order state. Everything here is LAZY: nothing is + # built or read until the first set_ts()/num_windows() call (only the + # streaming-train-eval loop does that), so the default train-eval path + # is byte-for-byte unaffected. + self._streaming_window_seconds: int = streaming_window_seconds + self._streaming_sort_within_window: bool = streaming_sort_within_window + self._active: Optional[np.ndarray] = None + self.is_eval: bool = False + self._anchor_ts: Optional[np.ndarray] = None + self._t_min: Optional[int] = None + self._t_max: Optional[int] = None self._cache_dir: Optional[str] = cache_dir self._cross_specs: List[Tuple[str, Tuple[str, ...], int, int]] = [ (name, tuple(keys), n, s) for (name, keys, n, s) in (cross_specs or []) @@ -409,11 +422,104 @@ def _load_metadata(self, metadata_dir: str) -> None: self.num_items: int = n_items def get_item_count(self) -> int: + # Streaming mode restricts the active set to the current time window; + # otherwise the full (user-major) anchor list is used (train-eval). + if self._active is not None: + return int(len(self._active)) return int(len(self._positions)) def iloc(self, idx: int) -> int: + if self._active is not None: + return int(self._positions[self._active[idx]]) return int(self._positions[idx]) + def _ensure_streaming_index(self) -> None: + """Lazily build + mmap the per-anchor target-timestamp array used for + time-windowed streaming. + + Built only on the first ``set_ts()``/``num_windows()`` call, so the + default train-eval path never reads timestamps or writes a new file. + Multi-rank safe via an exclusive file lock + atomic rename; all ranks + then mmap the result read-only (shared physical pages, ~0 anon). + """ + if self._anchor_ts is not None: + return + import fcntl + + assert self._cache_dir is not None + anchor_path = os.path.join( + self._cache_dir, f"anchor_ts_L{self._history_length}.npy" + ) + if not os.path.exists(anchor_path): + lock_path = os.path.join(self._cache_dir, "_anchor_ts_lock") + with open(lock_path, "w") as lf: + logger.info(f"Acquiring anchor-ts build lock for {anchor_path}...") + fcntl.flock(lf, fcntl.LOCK_EX) + if not os.path.exists(anchor_path): + logger.info( + f"Building {anchor_path}: target ts for " + f"{len(self._positions):,} anchors" + ) + anchor_ts = self.store.flat_timestamps[self._positions] + tmp = anchor_path + ".tmp.npy" + np.save(tmp, anchor_ts) + os.replace(tmp, anchor_path) + del anchor_ts + self._anchor_ts = _load_npy_readonly(anchor_path) + self._t_min = int(self._anchor_ts.min()) + self._t_max = int(self._anchor_ts.max()) + + def num_windows(self) -> int: + """Number of fixed-duration windows spanning [t_min, t_max].""" + self._ensure_streaming_index() + assert self._t_min is not None and self._t_max is not None + span = self._t_max - self._t_min + 1 + w = self._streaming_window_seconds + return int((span + w - 1) // w) + + def window_indices( + self, ts: int, sort_by_time: Optional[bool] = None + ) -> np.ndarray: + """Global anchor indices (into ``_positions``) whose target timestamp is + in window ``ts``: ``[t_min + ts*W, t_min + (ts+1)*W)``. + + Returned in ascending global-index order (user-major), which keeps the + per-sample history scans page-local in the mmap'd event arrays. Used by + the per-window path (via ``set_ts``) and the persistent path (shipped to + workers through the sampler). ``sort_by_time`` defaults to + ``streaming_sort_within_window``. + + Note: an O(log N) variant using a cached argsort of the timestamps was + evaluated but rejected — it doubles resident mmap (sorted-ts + order + permutation, ~52 GB) and that extra residency evicts the event-array + page cache, stalling dataloader workers (NCCL watchdog timeouts). The + O(N) mask here keeps only one ~26 GB array resident and is robust. + """ + self._ensure_streaming_index() + assert self._anchor_ts is not None and self._t_min is not None + w = self._streaming_window_seconds + lo = self._t_min + ts * w + hi = lo + w + idx = np.where((self._anchor_ts >= lo) & (self._anchor_ts < hi))[0] + do_sort = ( + self._streaming_sort_within_window if sort_by_time is None else sort_by_time + ) + if do_sort and idx.size > 0: + idx = idx[np.argsort(self._anchor_ts[idx], kind="stable")] + logger.warning(f"window_indices({ts}): [{lo}, {hi}) -> {idx.size:,} anchors") + return idx.astype(np.int64) + + def set_ts(self, ts: int) -> None: + """Restrict the active sample set to anchors in window ``ts`` (used by + the per-window-DataLoader path, where ``iloc``/``get_item_count`` index + through ``_active``). + + Forward-only temporal slicing for streaming train/eval. History for any + anchor is still gathered causally (``scan_start:flat_pos``) and may span + earlier windows, so there is no feature leakage from future events. + """ + self._active = self.window_indices(ts) + def load_query_samples(self, sample_list) -> None: max_num_candidates = ( self._max_num_candidates_inference diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index 9715f0b78..38cdd9e3d 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -82,12 +82,56 @@ get_dataset.history_length = 2039 # `history_length + contextual + candidate` would overflow. get_hstu_configs.max_seq_len = 2048 -# train-eval loop variables (yambda is non-streaming) +# --- streaming (temporal-order) training ------------------------------------- +# Only consumed under `--mode streaming-train-eval`; the default train-eval +# path above is unaffected. Trains time window T then evals window T+1, +# advancing forward in wall-clock time (no future leakage). Window size is the +# temporal-ordering granularity knob (default 1 day). num_train_ts is clamped +# to the dataset's available window count at runtime; override via $NUM_TRAIN_TS. +get_dataset.streaming_window_seconds = 86400 +get_dataset.streaming_sort_within_window = False +make_streaming_dataloader.batch_size = %batch_size +make_streaming_dataloader.num_workers = %num_workers +make_streaming_dataloader.prefetch_factor = %prefetch_factor +make_persistent_streaming_dataloader.batch_size = %batch_size +make_persistent_streaming_dataloader.num_workers = %num_workers +make_persistent_streaming_dataloader.prefetch_factor = %prefetch_factor +streaming_train_eval_loop.num_train_ts = @nts/env_int() +nts/env_int.key = "NUM_TRAIN_TS" +nts/env_int.default = 30 +# Anchors need >= history_length prior events, so the first ~130 daily windows +# are near-empty warm-up; start at a dense window. Override via $START_TS. +streaming_train_eval_loop.start_ts = @sts/env_int() +sts/env_int.key = "START_TS" +sts/env_int.default = 150 +streaming_train_eval_loop.metric_log_frequency = 50 +# Trace on by default: reuses the shared Profiler.* bindings below (5-step +# window at step 52). The streaming step counter advances across train+eval +# batches, so step 52 lands in the first (train) window's compute. +streaming_train_eval_loop.output_trace = True +# Reuse one DataLoader (persistent workers) across windows instead of respawning +# per window. Skip eval to isolate window-reset cost. Override via env. +streaming_train_eval_loop.persistent_loader = @pl/env_int() +pl/env_int.key = "PERSISTENT_LOADER" +pl/env_int.default = 1 +streaming_train_eval_loop.eval_each_window = @ev/env_int() +ev/env_int.key = "EVAL_EACH_WINDOW" +ev/env_int.default = 1 +# Double-buffer windows: prepare the next window (index mask + first-batch +# prefetch) in a background thread during the current window's compute, hiding +# the per-window reset. Needs persistent_loader=1. Override via env. +streaming_train_eval_loop.double_buffer = @db/env_int() +db/env_int.key = "DOUBLE_BUFFER" +db/env_int.default = 1 +# num_train_batches / num_eval_batches unset => consume each full window. +# Set them (e.g. via gin) to cap per-window steps for short experiments. + +# Default (non-streaming) train-eval loop variables; used unless +# `--mode streaming-train-eval` selects the temporal-order path configured above. train_eval_loop.num_epochs = 1 -train_eval_loop.output_trace = False train_eval_loop.metric_log_frequency = 50 -train_eval_loop.eval_frequency = 5000 -train_eval_loop.num_eval_batches = 500 +train_eval_loop.eval_frequency = 500 +train_eval_loop.num_eval_batches = 200 train_eval_loop.checkpoint_frequency = 1000000000 # disable mid-training checkpoints (disk-full guard) train_eval_loop.output_trace = True # 3-stage TorchRec pipeline: overlaps the embedding all-to-all (the dominant diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py index 94237f947..890144e1d 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -15,6 +15,8 @@ # pyre-strict import logging import os +import threading +import time from collections.abc import Iterator from datetime import timedelta from typing import ( @@ -424,6 +426,12 @@ def make_streaming_dataloader( dataset.dataset.set_ts(ts) # pyre-ignore [16] total_items = dataset.dataset.get_item_count() subset = torch.utils.data.Subset(dataset, range(total_items)) + # shuffle=False keeps temporal order within the window: a non-shuffling + # DistributedSampler hands rank r the strided slice indices[r::num_replicas] + # (round-robin), so all ranks stay on the same time front and consume the + # window in index order. Fork ctx mirrors the train path (COW-share the + # mmap'd store instead of pickling it into every worker). + mp_ctx = "fork" if num_workers and num_workers > 0 else None dataloader = DataLoader( dataset=subset, batch_size=batch_size, @@ -432,11 +440,128 @@ def make_streaming_dataloader( drop_last=True, num_workers=num_workers, prefetch_factor=prefetch_factor, - sampler=DistributedSampler(subset, drop_last=True), + sampler=DistributedSampler(subset, shuffle=False, drop_last=True), + multiprocessing_context=mp_ctx, ) return dataloader +class StreamingWindowSampler(torch.utils.data.Sampler): + """Per-rank sampler whose index list is swapped each window. + + Yields this rank's round-robin slice of the active window's GLOBAL anchor + indices (into the dataset's ``_positions``). Because indices are global, a + single DataLoader with ``persistent_workers=True`` can be reused across all + windows: the main process re-iterates this sampler each window and ships the + new indices to the already-forked workers, which map any global index via + the shared mmap. No per-window worker respawn / dataset re-pickle. + + Round-robin striding (rank r gets ``indices[r::world_size]``) over the + time-sorted window keeps every rank on the same time front; the window is + truncated to a multiple of ``world_size`` so all ranks get equal counts + (required for DDP collective lockstep). + """ + + def __init__(self, rank: int, world_size: int) -> None: + self._rank: int = rank + self._world_size: int = world_size + self._indices: List[int] = [] + + def set_window(self, global_indices) -> None: + n = (len(global_indices) // self._world_size) * self._world_size + self._indices = global_indices[:n][self._rank :: self._world_size].tolist() + + def __iter__(self): + return iter(self._indices) + + def __len__(self) -> int: + return len(self._indices) + + +@gin.configurable +def make_persistent_streaming_dataloader( + dataset: HammerToTorchDataset, + sampler: StreamingWindowSampler, + batch_size: int, + num_workers: int, + prefetch_factor: int, +) -> DataLoader: + """One reusable DataLoader for the whole streaming run. ``sampler`` is + mutated per window via ``set_window``; workers persist across windows.""" + use_workers = bool(num_workers and num_workers > 0) + return DataLoader( + dataset=dataset, + batch_size=batch_size, + shuffle=False, + collate_fn=collate_fn, + drop_last=True, + num_workers=num_workers, + prefetch_factor=prefetch_factor if use_workers else None, + sampler=sampler, + persistent_workers=use_workers, + multiprocessing_context="fork" if use_workers else None, + ) + + +class _PrefetchingWindowLoader: + """Double-buffered window loader for the persistent streaming path. + + Holds ``n_buffers`` pre-forked persistent worker pools that ping-pong: while + the current window trains on one pool, the *next* window's index selection + (``window_indices``) and first-batch prefetch are prepared on another pool + in a background thread. By the time training advances, that window is warm, + so the per-window reset (mask + first-batch stall) is hidden behind GPU + compute (~0 dead time at the boundary). + + Worker pools are forked once on the main thread at the start of ``stream``; + afterwards only iterator resets happen (no forks), so background-thread + preparation cannot fork while other threads hold locks. + """ + + def __init__( + self, + dataset: "HammerToTorchDataset", + sampler_factory, + dl_factory, + n_buffers: int = 2, + ) -> None: + self._dataset = dataset + self._n = n_buffers + self._samplers = [sampler_factory() for _ in range(n_buffers)] + self._dls = [dl_factory(s) for s in self._samplers] + self._iters: List[Optional[object]] = [None] * n_buffers + + def _prepare(self, buf: int, ts: int) -> None: + # window_indices() is the O(N) mask; numpy releases the GIL for it, so it + # overlaps the main thread's GPU dispatch. iter() then kicks off this + # pool's background prefetch. + self._samplers[buf].set_window(self._dataset.dataset.window_indices(ts)) + self._iters[buf] = iter(self._dls[buf]) + + def stream(self, ts_list: List[int]): + n = len(ts_list) + if n == 0: + return + threads: List[Optional[threading.Thread]] = [None] * self._n + # Prime the first n_buffers windows on the main thread (forks all pools). + for b in range(min(self._n, n)): + self._prepare(b, ts_list[b]) + for i in range(n): + buf = i % self._n + if threads[buf] is not None: + threads[buf].join() + threads[buf] = None + yield ts_list[i], self._iters[buf] + # This pool is now free; prefetch the window n_buffers ahead. + j = i + self._n + if j < n: + th = threading.Thread( + target=self._prepare, args=(buf, ts_list[j]), daemon=True + ) + th.start() + threads[buf] = th + + @gin.configurable def make_train_test_dataloaders( batch_size: int, @@ -903,6 +1028,10 @@ def streaming_train_eval_loop( output_trace: bool = False, metric_log_frequency: int = 1, checkpoint_frequency: int = 100, + start_ts: int = 0, + persistent_loader: bool = False, + eval_each_window: bool = True, + double_buffer: bool = False, ) -> None: profiler = Profiler(rank) if output_trace else None dataset_class, kwargs = get_dataset() @@ -910,16 +1039,55 @@ def streaming_train_eval_loop( dataset = HammerToTorchDataset( dataset=dataset_class(hstu_config=hstu_config, is_inference=False, **kwargs) ) - for train_ts in range(num_train_ts): - train_batch_idx: int = 0 - train_dataloader = make_streaming_dataloader(dataset=dataset, ts=train_ts) - train_data_iterator = iter(train_dataloader) + # Persistent path: build ONE DataLoader + a stateful sampler whose indices + # are swapped per window, so workers fork once and are reused across all + # windows (eliminates the per-window dataloader respawn + first-batch + # warmup). The non-persistent path recreates a DataLoader per window. + window_sampler: Optional[StreamingWindowSampler] = None + persistent_dl: Optional[DataLoader] = None + if persistent_loader: + world_size = ( + torch.distributed.get_world_size() + if torch.distributed.is_initialized() + else 1 + ) + window_sampler = StreamingWindowSampler(rank=rank, world_size=world_size) + persistent_dl = make_persistent_streaming_dataloader( + dataset=dataset, sampler=window_sampler + ) + + def _window_iter(ts: int): + if persistent_loader: + assert window_sampler is not None and persistent_dl is not None + window_sampler.set_window(dataset.dataset.window_indices(ts)) # pyre-ignore [16] + return iter(persistent_dl) + return iter(make_streaming_dataloader(dataset=dataset, ts=ts)) + # Windows are [start_ts, start_ts + num_train_ts); each step trains window T + # then evals window T+1, so the last eval window is start_ts + num_train_ts, + # which must be < num_windows(). Anchors require >= history_length prior + # events, so the earliest windows are near-empty warm-up — use start_ts to + # begin at a dense window. Clamp instead of failing. + if hasattr(dataset.dataset, "num_windows"): + available = dataset.dataset.num_windows() # pyre-ignore [16] + max_count = max(0, available - 1 - start_ts) + if num_train_ts > max_count: + logger.warning( + f"start_ts={start_ts} + num_train_ts={num_train_ts} exceeds " + f"available windows ({available}); clamping num_train_ts to {max_count}." + ) + num_train_ts = max_count + def _run_train_window(train_data_iterator, label: Optional[str] = None) -> None: + train_batch_idx = 0 + first_wait: Optional[float] = None while True: model.train() + _t_next = time.perf_counter() if (label and rank == 0) else None try: sample = next(train_data_iterator) except StopIteration: break + if _t_next is not None and first_wait is None: + first_wait = time.perf_counter() - _t_next optimizer.zero_grad() sample.to(device) ( @@ -958,18 +1126,25 @@ def streaming_train_eval_loop( profiler.step() if num_train_batches is not None and train_batch_idx >= num_train_batches: break - eval_ts = train_ts + 1 - dataset.dataset.is_eval = True # pyre-ignore [16] + if label and rank == 0 and first_wait is not None: + logger.info( + f"[boundary] {label} train first-batch data-wait={first_wait * 1000:.1f}ms" + ) + + def _run_eval_window(eval_data_iterator, label: Optional[str] = None) -> None: model.eval() - eval_batch_idx: int = 0 - eval_dataloader = make_streaming_dataloader(dataset=dataset, ts=eval_ts) - eval_data_iterator = iter(eval_dataloader) + eval_batch_idx = 0 + first_wait: Optional[float] = None + _t_enter = time.perf_counter() if (label and rank == 0) else None with torch.no_grad(): while True: + _t_next = time.perf_counter() if (label and rank == 0) else None try: sample = next(eval_data_iterator) except StopIteration: break + if _t_next is not None and first_wait is None: + first_wait = time.perf_counter() - _t_next sample.to(device) ( _, @@ -1001,9 +1176,18 @@ def streaming_train_eval_loop( break for k, v in metric_logger.compute(mode="eval").items(): print(f"{k}: {v}") + if label and rank == 0 and _t_enter is not None: + _eval_total = time.perf_counter() - _t_enter + _fw = (first_wait * 1000) if first_wait is not None else float("nan") + logger.info( + f"[boundary] {label} eval first-batch data-wait={_fw:.1f}ms " + f"total_eval={_eval_total * 1000:.1f}ms batches={eval_batch_idx}" + ) + + def _maybe_checkpoint(train_ts: int) -> None: if ( train_ts % checkpoint_frequency == 0 and train_ts > 0 - ) or train_ts == num_train_ts - 1: + ) or train_ts == start_ts + num_train_ts - 1: save_dmp_checkpoint( model=model, optimizer=optimizer, @@ -1012,6 +1196,64 @@ def streaming_train_eval_loop( batch_idx=train_ts, ) + train_ts_list = list(range(start_ts, start_ts + num_train_ts)) + if persistent_loader and double_buffer: + # Double-buffered: next window prepared in the background during the + # current window's compute. Eval (if enabled) uses its own pre-forked + # pool, primed up front on the main thread so no fork races a bg thread. + prefetcher = _PrefetchingWindowLoader( + dataset=dataset, + sampler_factory=lambda: StreamingWindowSampler(rank, world_size), + dl_factory=lambda s: make_persistent_streaming_dataloader( + dataset=dataset, sampler=s + ), + ) + eval_sampler: Optional[StreamingWindowSampler] = None + eval_dl: Optional[DataLoader] = None + # Eval iterator is built one window ahead: the eval pool (idle while the + # current train window runs) prefetches the eval window's first batches + # concurrently with train compute, so eval starts warm (hides the + # ~0.5s eval first-batch stall). yambda's sample content depends only on + # the sampler window, not is_eval, so prefetching during train is safe. + eval_iter: Optional[Iterator] = None + if eval_each_window and len(train_ts_list) > 0: + eval_sampler = StreamingWindowSampler(rank, world_size) + eval_dl = make_persistent_streaming_dataloader( + dataset=dataset, sampler=eval_sampler + ) + # Fork the eval pool now (main thread, before any prefetch thread) + # and kick off prefetch of the first eval window (train_ts_list[0]+1). + eval_sampler.set_window( + dataset.dataset.window_indices(train_ts_list[0] + 1) # pyre-ignore [16] + ) + eval_iter = iter(eval_dl) + n_train = len(train_ts_list) + for i, (train_ts, train_data_iterator) in enumerate( + prefetcher.stream(train_ts_list) + ): + dataset.dataset.is_eval = False # pyre-ignore [16] + _run_train_window(train_data_iterator, label=f"train_ts={train_ts}") + if eval_each_window: + dataset.dataset.is_eval = True # pyre-ignore [16] + assert eval_sampler is not None and eval_dl is not None + _run_eval_window(eval_iter, label=f"eval_ts={train_ts + 1}") + # Re-arm the eval pool for the next window so it prefetches + # during the upcoming train window. + if i + 1 < n_train: + eval_sampler.set_window( + dataset.dataset.window_indices(train_ts + 2) # pyre-ignore [16] + ) + eval_iter = iter(eval_dl) + _maybe_checkpoint(train_ts) + else: + for train_ts in train_ts_list: + dataset.dataset.is_eval = False # pyre-ignore [16] + _run_train_window(_window_iter(train_ts)) + if eval_each_window: + dataset.dataset.is_eval = True # pyre-ignore [16] + _run_eval_window(_window_iter(train_ts + 1)) + _maybe_checkpoint(train_ts) + eval_ts = num_train_ts dataset.dataset.is_eval = True model.eval() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py index f276780c2..f261cda97 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py @@ -188,10 +188,17 @@ def handle_fn(p: torch.profiler.profile) -> None: sort_by="self_cuda_time_total" ) ) - p.export_chrome_trace(path) - logger.warning(f"Trace written to: {path}") - if keep_n_active is not None and keep_n_active > 0: - _trim_warmup_from_trace(path, keep_n_active) + # Tracing is best-effort: a write/trim failure (permissions, disk full, + # malformed export) must never crash the training run. Degrade to a + # warning so the loop continues — especially important since streaming + # enables output_trace by default. + try: + p.export_chrome_trace(path) + logger.warning(f"Trace written to: {path}") + if keep_n_active is not None and keep_n_active > 0: + _trim_warmup_from_trace(path, keep_n_active) + except Exception as exc: + logger.warning(f"Trace export/trim failed for {path}: {exc!r} (skipping)") return handle_fn @@ -689,7 +696,13 @@ def run_results_dir(run_name: str = "default", subdir: str = "results") -> str: @gin.configurable -def get_dataset(name: str, new_path_prefix: str = "", history_length: Optional[int] = None): +def get_dataset( + name: str, + new_path_prefix: str = "", + history_length: Optional[int] = None, + streaming_window_seconds: int = 86400, + streaming_sort_within_window: bool = False, +): """ Get dataset class and configuration by name. @@ -815,6 +828,10 @@ def get_dataset(name: str, new_path_prefix: str = "", history_length: Optional[i "history_length": history_length if history_length is not None else 4096, "scan_window": 20000, "cross_specs": YAMBDA_5B_CROSS_SPECS, + # Temporal-streaming knobs (only used under --mode + # streaming-train-eval; ignored by the default train-eval path). + "streaming_window_seconds": streaming_window_seconds, + "streaming_sort_within_window": streaming_sort_within_window, }, ) if name == "sampled-streaming-100b": diff --git a/recommendation_v4/scripts/launch_smoke_8gpu.sh b/recommendation_v4/scripts/launch_smoke_8gpu.sh index 09a3af2ba..94886dc87 100755 --- a/recommendation_v4/scripts/launch_smoke_8gpu.sh +++ b/recommendation_v4/scripts/launch_smoke_8gpu.sh @@ -58,4 +58,4 @@ fi echo "[$(date)] launching train_ranker with WORLD_SIZE=$WORLD_SIZE" | tee -a "$LOG" python -m generative_recommenders.dlrm_v3.train.train_ranker \ - --dataset yambda-5b --mode train-eval 2>&1 | tee -a "$LOG" + --dataset yambda-5b --mode "${MODE:-streaming-train-eval}" 2>&1 | tee -a "$LOG" From d017bcc15064c647574b0bcb8347945bcb69b393 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Wed, 3 Jun 2026 12:30:01 -0500 Subject: [PATCH 027/127] dlrmv4: disable checkpointing by default; fix recipe torch note MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit save_dmp_checkpoint.path now resolves from $CKPT_PATH and defaults to empty, so checkpoints (a full DMP is ~100s of GB, and the streaming loop always saves the final window) are off unless explicitly enabled. Also drop the stale training-recipe sentence claiming native torch is kept — it contradicts the dependency table, which replaces torch and keeps only the image's triton. Co-authored-by: Cursor --- recommendation_v4/docs/training_recipe.md | 3 --- .../dlrm_v3/train/gin/yambda_5b.gin | 7 ++++++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/recommendation_v4/docs/training_recipe.md b/recommendation_v4/docs/training_recipe.md index 28c88ded2..cf31a9fff 100644 --- a/recommendation_v4/docs/training_recipe.md +++ b/recommendation_v4/docs/training_recipe.md @@ -24,9 +24,6 @@ mixed-precision training. rocm/primus:v26.3 ``` -The image's native PyTorch is kept as-is and must not be reinstalled — it is -the ROCm-matched build used by triton/fbgemm. - ### Dependency versions Aligned with the B200 path: same torch major.minor, same torchrec commit, diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index 38cdd9e3d..53066ef9a 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -158,4 +158,9 @@ Profiler.trace_dir = @run_results_dir() MetricsLogger.tensorboard_log_path = "/tmp/tb/yambda_5b/" MetricsLogger.world_size = 8 MetricsLogger.auc_threshold = 0.80275 -save_dmp_checkpoint.path = "/apps/chcai/ckpts/yambda_5b/" +# Checkpointing disabled by default — a full DMP checkpoint is ~100s of GB and +# the streaming loop always saves on the final window. save_dmp_checkpoint +# no-ops on the empty path. Set $CKPT_PATH to a directory to re-enable. +save_dmp_checkpoint.path = @ckpt/env_path() +ckpt/env_path.key = "CKPT_PATH" +ckpt/env_path.default = "" From 7d11c1771f8a6ec8ab980e3eed18a1d099644e49 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Wed, 3 Jun 2026 17:54:07 -0500 Subject: [PATCH 028/127] dlrmv4: ROCm-only Perfetto trace render fixes at export time Add in-process trace postprocessing in the profiler on_trace_ready callback to fix two ROCm/roctracer rendering artifacts that make MI350X traces look wrong in Perfetto (the timing is correct, only the layout): - _normalize_profilerstep_layout: collapse the fragmented GPU-side ProfilerStep#N spans (roctracer splits a step across the HIP null + compute streams) into one full-width span per step on the busiest compute stream, matching the CUDA look. - _deoverlap_gpu_slices: pull back sub-us kernel end timestamps so back-to-back kernels don't touch/overlap; Perfetto otherwise nests the later (long) kernel inside the tiny epilogue and clips it to zero width, hiding kernels like _hstu_attn_bwd. Leaves a ~1ns gap (exact end==start is just as fatal as an overlap) and leaves real nesting untouched. Both passes are gated behind _is_rocm() (torch.version.hip) so they are complete no-ops on CUDA/B200, which don't have these artifacts. All best-effort: failures degrade to a warning and never crash training. Co-authored-by: Cursor --- .../generative_recommenders/dlrm_v3/utils.py | 232 ++++++++++++++++++ 1 file changed, 232 insertions(+) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py index f261cda97..008e41f07 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py @@ -143,6 +143,227 @@ def _keep(e: dict) -> bool: ) +# GPU activity categories used to detect GPU stream rows and their busy time. +_GPU_KERNEL_CATS = frozenset({"kernel", "gpu_memcpy", "gpu_memset"}) + + +def _is_rocm() -> bool: + """True on ROCm/AMD builds (``torch.version.hip`` set), False on CUDA/B200. + + The ProfilerStep-layout normalization and the sub-us kernel de-overlap are + workarounds for how roctracer projects annotations/kernels onto HIP streams; + CUDA/CUPTI traces don't have those artifacts, so these passes must be skipped + on NVIDIA to avoid touching otherwise-correct traces. + """ + return getattr(torch.version, "hip", None) is not None + + +def _normalize_profilerstep_layout(path: str) -> None: + """Collapse fragmented GPU-side ``ProfilerStep#N`` spans into one span/step. + + ``torch.profiler`` emits ``ProfilerStep#N`` as a CPU ``user_annotation`` that + Kineto projects onto the GPU timeline as ``gpu_user_annotation`` spans. On + CUDA the blocking H2D copy shares the compute stream, so each step projects + onto a single GPU stream and renders as one full-width span. On ROCm a + blocking H2D copy lands on HIP's null stream (a different stream than the + non-null compute stream), so the step splits across two GPU rows and looks + truncated in Perfetto — a pure rendering artifact (every kernel is still + captured, and the underlying GPU is busy for the whole step). + + This rewrites each per-step GPU ``ProfilerStep`` annotation to a single span + on the rank's busiest (compute) GPU stream, covering the kernel extent inside + that step's CPU window. Works on a raw per-rank trace (GPU streams are tids + under one pid) by keying the busiest stream on ``(pid, tid)``. No-op when the + annotation already lives on a single GPU stream (the CUDA case), so it is + safe to run on every platform. Mutates the file in place. + """ + import json as _json + + with open(path) as f: + d = _json.load(f) + events = d.get("traceEvents", []) + + # Per (pid,tid) GPU busy time -> identify the busiest = compute stream. + stream_busy: Dict[tuple, int] = {} + for e in events: + if e.get("ph") == "X" and e.get("cat") in _GPU_KERNEL_CATS: + dur = e.get("dur", 0) + if dur > 0: + key = (e.get("pid"), e.get("tid")) + stream_busy[key] = stream_busy.get(key, 0) + dur + if not stream_busy: + return + busiest = max(stream_busy, key=lambda k: stream_busy[k]) + + # Existing GPU-side ProfilerStep spans and the streams they sit on. + gpu_ps_streams = set() + template = None + for e in events: + if e.get("cat") == "gpu_user_annotation" and str( + e.get("name", "") + ).startswith("ProfilerStep"): + gpu_ps_streams.add((e.get("pid"), e.get("tid"))) + if template is None: + template = e + # No fragmentation (single stream or none) -> leave the trace untouched. + if len(gpu_ps_streams) <= 1: + return + + # CPU ProfilerStep windows: step name -> [min ts, max end]. + cpu_win: Dict[str, list] = {} + for e in events: + if ( + e.get("cat") == "user_annotation" + and e.get("ph") == "X" + and str(e.get("name", "")).startswith("ProfilerStep") + ): + ts = e.get("ts", 0) + end = ts + e.get("dur", 0) + w = cpu_win.get(e["name"]) + if w is None: + cpu_win[e["name"]] = [ts, end] + else: + w[0] = min(w[0], ts) + w[1] = max(w[1], end) + + # GPU kernel extents (any stream) for clamping each step's span. + gpu_kernels = [ + (e.get("ts", 0), e.get("ts", 0) + e.get("dur", 0)) + for e in events + if e.get("ph") == "X" + and e.get("cat") in _GPU_KERNEL_CATS + and e.get("dur", 0) > 0 + ] + + new_spans = [] + for sname, (cs, ce) in cpu_win.items(): + ks = [(ts, end) for ts, end in gpu_kernels if end > cs and ts < ce] + if not ks: + continue + gmin = min(ts for ts, _ in ks) + gmax = max(end for _, end in ks) + span = dict(template) if template else {} + span.update( + { + "ph": "X", + "cat": "gpu_user_annotation", + "name": sname, + "pid": busiest[0], + "tid": busiest[1], + "ts": gmin, + "dur": gmax - gmin, + "args": {"normalized_profilerstep": True}, + } + ) + new_spans.append(span) + + if not new_spans: + return + + out = [ + e + for e in events + if not ( + e.get("cat") == "gpu_user_annotation" + and str(e.get("name", "")).startswith("ProfilerStep") + ) + ] + dropped = len(events) - len(out) + out.extend(new_spans) + d["traceEvents"] = out + with open(path, "w") as f: + _json.dump(d, f) + logger.warning( + f"Normalized GPU ProfilerStep layout in {path}: dropped {dropped} " + f"fragmented span(s) across {len(gpu_ps_streams)} stream(s), wrote " + f"{len(new_spans)} span(s) on busiest stream pid={busiest[0]} " + f"tid={busiest[1]}" + ) + + +def _deoverlap_gpu_slices(path: str, max_snap_us: float = 5.0) -> None: + """Remove sub-microsecond kernel overlaps that break Perfetto's renderer. + + Perfetto draws all ``ph=="X"`` slices on a single track (one ``(pid, tid)``) + as a strict nested stack ordered by start time: a slice that *opens* while a + previous slice on the same track is still open is treated as that slice's + child and is **clipped to the parent's end**. ROCm's roctracer reports + per-stream kernel timestamps at ns granularity, so two back-to-back kernels + on the same compute stream occasionally overlap by a fraction of a + microsecond (e.g. an 88 ns ``elementwise`` epilogue ending 0.075 us *after* + the next 21 ms ``_hstu_attn_bwd`` kernel begins). Perfetto then nests the + long kernel inside the tiny one and clips it to a sub-pixel sliver, so the + kernel "disappears" from the timeline even though it is fully present in the + JSON. + + This pulls each slice's end back to just *before* the next slice's start + whenever they overlap by less than ``max_snap_us`` (a measurement artifact, + not real concurrency — kernels on one stream are serialized), leaving genuine + nesting (a small kernel fully contained in a larger one) untouched. The + adjustment is sub-microsecond and does not change any reported duration + meaningfully. Mutates the file in place; best-effort. + + Critically, the slices are separated by a tiny ``_GAP_US`` (~1 ns) rather + than snapped to an *exactly equal* end==start timestamp. A coincident + end==start is just as fatal as an overlap in Perfetto: it nests the next + slice inside the previous one and clips it to zero width (this is the ~1 ns + gap that roctracer leaves between cleanly-rendered back-to-back kernels). So + we also fix exact-touch (``a_end == b.ts``) boundaries, not just overlaps. + """ + import json as _json + from collections import defaultdict + + # ~1 ns. Matches the natural inter-kernel gap roctracer leaves between + # back-to-back kernels that Perfetto already renders correctly. Must be + # strictly > 0 so end != start after the nudge. + _GAP_US = 0.001 + + with open(path) as f: + d = _json.load(f) + events = d.get("traceEvents", []) + + tracks: Dict[tuple, list] = defaultdict(list) + for e in events: + if ( + e.get("ph") == "X" + and e.get("cat") in _GPU_KERNEL_CATS + and e.get("dur", 0) > 0 + ): + tracks[(e.get("pid"), e.get("tid"))].append(e) + + snapped = 0 + max_clip = 0.0 + for sl in tracks.values(): + # Sort by start, then longest-first so a container precedes the slices + # it nests; consecutive pairs are then either disjoint, properly nested, + # or a tiny artifact overlap. + sl.sort(key=lambda e: (e["ts"], -e["dur"])) + for i in range(len(sl) - 1): + a = sl[i] + b = sl[i + 1] + a_end = a["ts"] + a["dur"] + b_end = b["ts"] + b["dur"] + # Touching (a_end == b.ts) or partial overlap (a ends inside b) both + # break rendering; true containment (a_end >= b_end) is valid nesting + # and is left alone. + if b["ts"] <= a_end < b_end: + desired_end = b["ts"] - _GAP_US + clip = a_end - desired_end + if a["ts"] < desired_end and 0 < clip < max_snap_us: + a["dur"] = desired_end - a["ts"] + snapped += 1 + if clip > max_clip: + max_clip = clip + + if snapped: + with open(path, "w") as f: + _json.dump(d, f) + logger.warning( + f"De-overlapped GPU slices in {path}: snapped {snapped} sub-us " + f"overlap(s) (max {max_clip:.3f}us) so Perfetto renders every kernel" + ) + + def _on_trace_ready_fn( rank: Optional[int] = None, trace_dir: str = "/tmp/dlrm_v3_traces", @@ -197,6 +418,17 @@ def handle_fn(p: torch.profiler.profile) -> None: logger.warning(f"Trace written to: {path}") if keep_n_active is not None and keep_n_active > 0: _trim_warmup_from_trace(path, keep_n_active) + # ROCm/AMD-only rendering fixes. CUDA/CUPTI (e.g. B200) traces don't + # exhibit the fragmented-ProfilerStep or sub-us kernel-overlap + # artifacts, so skip entirely on NVIDIA to avoid touching otherwise + # correct traces. Best-effort like trim above. + if _is_rocm(): + # Normalize the GPU-side ProfilerStep layout so ROCm traces + # render with one full-width step span per stream like CUDA. + _normalize_profilerstep_layout(path) + # Snap roctracer's sub-us kernel overlaps so Perfetto doesn't + # mis-nest and hide long kernels. + _deoverlap_gpu_slices(path) except Exception as exc: logger.warning(f"Trace export/trim failed for {path}: {exc!r} (skipping)") From e0242179a1d6494fc01d85f9afd6295212704993 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Wed, 3 Jun 2026 19:17:23 -0500 Subject: [PATCH 029/127] dlrmv4: ROCm annotation de-overlap so phase spans render full width Add _deoverlap_gpu_annotations to the trace-export postprocessing, the annotation-boundary analog of the kernel de-overlap. Kineto projects the forward/backward phase annotations (## user_forward ##, ## item_forward ##, ## stu_* ##, ...) onto the GPU stream as a chain of end-to-end siblings. The absolute step timestamps are ~5.4e12 us, where a float64's quantum is ~1 ns, so a sibling boundary that should be coincident lands a few ns off; when the earlier sibling ends at/after the next one's start, Perfetto nests and clips the next span to a sliver -- e.g. the 100+ ms ## user_forward ## vanishes on some ranks/steps purely by rounding luck. Since annotations form a real nesting hierarchy (user_forward contains the stu_* spans and their kernels), this walks the per-track slice stack and only snaps a slice back when the next slice extends beyond it (siblings, not parent/child), guarding against trimming into a span's own descendants. It also snaps kernel tails that straddle an annotation boundary. Gated by _is_rocm() (no-op on B200/CUDA) and best-effort like the other passes. Verified end-to-end on an 8-rank MI350X run: ## user_forward ## renders 40/40 (was 9/40), total clipped annotations 1352 -> ~5. Co-authored-by: Cursor --- .../generative_recommenders/dlrm_v3/utils.py | 125 ++++++++++++++++++ 1 file changed, 125 insertions(+) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py index 008e41f07..5a1e7fb9d 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py @@ -364,6 +364,128 @@ def _deoverlap_gpu_slices(path: str, max_snap_us: float = 5.0) -> None: ) +def _deoverlap_gpu_annotations(path: str, max_snap_us: float = 5.0) -> None: + """Separate touching/overlapping *sibling* GPU annotations so Perfetto draws + each one full width (the B200-style stacked layout). + + Same root cause as :func:`_deoverlap_gpu_slices`, but at the annotation + boundary instead of the kernel boundary. The forward/backward phase + annotations Kineto projects onto the GPU stream (``## item_forward ##``, + ``## user_forward ##``, ``## multitask_module ##``, the ``## stu_* ##`` + pairs, ...) are emitted as a chain of siblings laid end-to-end: each is meant + to end exactly where the next begins. Perfetto stores timestamps as int64 ns, + and the absolute step timestamps are ~5.4e12 us where a float64's quantum is + already ~1 ns, so a sibling boundary that should be coincident instead lands + a few ns off. When the earlier sibling's end falls *at or after* the next + sibling's start, Perfetto nests the next sibling inside it and clips it to a + sub-pixel sliver — so e.g. the 100+ ms ``## user_forward ##`` span vanishes on + some ranks/steps and renders on others purely by rounding luck. + + Unlike kernels (all flat on one stream), annotations form a real nesting + hierarchy — ``## user_forward ##`` legitimately *contains* the ``## stu_* ##`` + spans and their kernels — so this cannot blindly snap consecutive slices. It + walks the per-track slice stack (sorted by start, longest-first) and only + snaps a slice ``a`` back when the next slice ``b`` is **not** contained in it + (``b`` extends beyond ``a``'s end), i.e. they are siblings rather than + parent/child. Real containment is left untouched, and a snap is skipped if it + would clip into ``a``'s own descendants (kernels or child annotations). + Mutates the file in place; best-effort. Run after :func:`_deoverlap_gpu_slices` + so kernel boundaries are already clean. + """ + import json as _json + from collections import defaultdict + + # ~2 ns. The annotation boundaries sit at ~5.4e12 us where a float64's + # quantum is ~0.98 ns, so a 1 ns nudge can round back onto the neighbour's + # timestamp (an exact touch, which Perfetto still nests+clips). 2 ns (~2 + # quanta) reliably separates them and is still far below any visible width. + _GAP_US = 0.002 + + with open(path) as f: + d = _json.load(f) + events = d.get("traceEvents", []) + + # Stack the full per-track hierarchy over BOTH kernels and annotations so a + # parent annotation knows the extent of its descendants (the snap guard), + # but only annotation slices are ever trimmed. + _ANN = "gpu_user_annotation" + tracks: Dict[tuple, list] = defaultdict(list) + for e in events: + if ( + e.get("ph") == "X" + and e.get("dur", 0) > 0 + and (e.get("cat") in _GPU_KERNEL_CATS or e.get("cat") == _ANN) + ): + tracks[(e.get("pid"), e.get("tid"))].append(e) + + snapped = 0 + max_clip = 0.0 + for sl in tracks.values(): + # Longest-first on ties so a container precedes the slices it nests. + sl.sort(key=lambda e: (e["ts"], -e["dur"])) + # Each frame: [event, max_descendant_end]. The stack holds the chain of + # currently-open ancestors for the slice being placed. + stack: list = [] + for b in sl: + b_ts = b["ts"] + b_end = b_ts + b["dur"] + while stack: + a = stack[-1][0] + a_end = a["ts"] + a["dur"] + if a_end < b_ts: + # a closed strictly before b begins -> disjoint sibling, pop. + frame = stack.pop() + eff = frame[0]["ts"] + frame[0]["dur"] + if stack: + stack[-1][1] = max(stack[-1][1], eff, frame[1]) + continue + if a_end < b_end: + # b starts at/inside a but extends past a's end => they are + # siblings (not parent/child), and a's tail nests+clips b in + # Perfetto. Snap a's end to just before b. This fires for both + # annotation tails (## item_forward ## overhanging + # ## user_forward ##) and kernel tails that straddle an + # annotation boundary (a layer-norm kernel ending a few ns + # past the start of the next phase span) -- both are sub-us + # roctracer/rounding artifacts, since kernels on one stream + # are serialized and phase spans are sequential. + desired_end = b_ts - _GAP_US + clip = a_end - desired_end + # Guard: only snap when a's deepest descendant ends at or + # before b's start. If a child (kernel or nested span) + # actually extends *past* b.ts, trimming a wouldn't fix b's + # clipping (the child would still nest b) and could drop a + # real child into b's territory, so leave it. A descendant + # ending exactly at the boundary is itself rounding noise and + # is clipped by <=1 ns, which is fine. + if ( + a["ts"] < desired_end + and stack[-1][1] <= b_ts + and 0 < clip < max_snap_us + ): + a["dur"] = desired_end - a["ts"] + snapped += 1 + if clip > max_clip: + max_clip = clip + frame = stack.pop() + eff = frame[0]["ts"] + frame[0]["dur"] + if stack: + stack[-1][1] = max(stack[-1][1], eff, frame[1]) + continue + # a_end >= b_end: a fully contains b -> b is a child, stop. + break + stack.append([b, b_ts]) + + if snapped: + with open(path, "w") as f: + _json.dump(d, f) + logger.warning( + f"De-overlapped GPU annotations in {path}: snapped {snapped} sub-us " + f"sibling overlap(s) (max {max_clip:.3f}us) so Perfetto renders every " + f"annotation full width" + ) + + def _on_trace_ready_fn( rank: Optional[int] = None, trace_dir: str = "/tmp/dlrm_v3_traces", @@ -429,6 +551,9 @@ def handle_fn(p: torch.profiler.profile) -> None: # Snap roctracer's sub-us kernel overlaps so Perfetto doesn't # mis-nest and hide long kernels. _deoverlap_gpu_slices(path) + # Same fix at the annotation-sibling boundary so phase spans + # (## user_forward ##, ## stu_* ##, ...) render full width. + _deoverlap_gpu_annotations(path) except Exception as exc: logger.warning(f"Trace export/trim failed for {path}: {exc!r} (skipping)") From f447075180abb932193f57a736199dff937ccece Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Wed, 3 Jun 2026 23:11:02 -0500 Subject: [PATCH 030/127] dlrmv4: streaming checkpoint resume + step/time checkpoint cadences Make streaming-train-eval crash-resumable and add general checkpoint cadence controls: - Atomic checkpoint saves (.tmp dir + rename), keep_last_n pruning, and swap-aside .old overwrite so a save can safely replace an existing train_ts dir; stale .tmp/.old swept on the next save. - Per-rank RNG snapshot/restore for bit-equal dropout replay on resume; auto-latest-subdir resolution + (train_ts, batch_idx_in_window) resume hint so a run re-enters a partial window and skips already-trained batches exact-once. - Three independent in-window checkpoint cadences via a pure, testable decision helper: per-window batch count, monotonic global step (e.g. every 1000 steps), and wall-clock interval (e.g. hourly, rank-0-decided + broadcast to keep the save barrier in lockstep). - gin/env bindings for all cadences + a test-only die_at_step hook. Tests: checkpoint_cadence_test.py (cadence precedence/triggers) and an end-to-end baseline/interrupt/resume harness (streaming_resume_test.{sh,py}) that gates on functional invariants (RNG restored, correct resumed step, atomic save, keep_last_n) plus a loose trajectory-closeness bound. Co-authored-by: Cursor --- .../dlrm_v3/checkpoint.py | 274 ++++++++++++-- .../dlrm_v3/train/gin/yambda_5b.gin | 48 ++- .../train/tests/checkpoint_cadence_test.py | 131 +++++++ .../train/tests/streaming_resume_test.py | 166 ++++++++ .../dlrm_v3/train/train_ranker.py | 14 +- .../dlrm_v3/train/utils.py | 358 ++++++++++++++++-- .../generative_recommenders/dlrm_v3/utils.py | 15 + .../scripts/streaming_resume_test.sh | 242 ++++++++++++ 8 files changed, 1187 insertions(+), 61 deletions(-) create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/train/tests/checkpoint_cadence_test.py create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.py create mode 100755 recommendation_v4/scripts/streaming_resume_test.sh diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/checkpoint.py b/recommendation_v4/generative_recommenders/dlrm_v3/checkpoint.py index 33445bce9..aa3c9daa0 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/checkpoint.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/checkpoint.py @@ -21,17 +21,29 @@ """ import gc +import logging import os +import random +import shutil from datetime import datetime -from typing import Any, Dict, Optional, Set +from typing import Any, Dict, Optional, Set, Tuple import gin +import numpy as np import torch from generative_recommenders.dlrm_v3.utils import MetricsLogger from torch.distributed.checkpoint.stateful import Stateful from torch.optim.optimizer import Optimizer from torchrec.distributed.types import ShardedTensor +logger: logging.Logger = logging.getLogger(__name__) + +# Sentinel meaning "the saved window completed in full" — when the loop reads +# this back it advances start_ts past the saved train_ts. Anything >=0 means the +# saved checkpoint stopped mid-window after K batches; resume continues that +# window at batch K. +WINDOW_COMPLETE: int = -1 + class SparseState(Stateful): """ @@ -86,6 +98,114 @@ def load_dense_state_dict(model: torch.nn.Module, state_dict: Dict[str, Any]) -> own_state[name].copy_(param) +def _rng_state(device: torch.device) -> Dict[str, Any]: + """Snapshot every RNG source bit-equal training depends on. + + HSTU has stochastic dropout (input_dropout=0.2, linear_dropout_rate=0.1) + consuming the per-device CUDA RNG cycle each step. Without round-tripping + these, a resumed run draws different dropout masks and the resumed AUC + trajectory diverges from the uninterrupted run within a few steps. + """ + return { + "cpu": torch.get_rng_state(), + "cuda": torch.cuda.get_rng_state(device), + "numpy": np.random.get_state(), + "random": random.getstate(), + } + + +def _restore_rng_state(state: Dict[str, Any], device: torch.device) -> None: + torch.set_rng_state(state["cpu"]) + torch.cuda.set_rng_state(state["cuda"], device) + np.random.set_state(state["numpy"]) + random.setstate(state["random"]) + + +def _list_numeric_subdirs(base_path: str) -> list[str]: + """Return subdir names of `base_path` that look like an int, sorted ascending. + + Filters out `*.tmp` (orphaned in-progress saves), `*.sparse/` and any other + non-numeric entries. + """ + if not os.path.isdir(base_path): + return [] + out: list[str] = [] + for name in os.listdir(base_path): + if name.isdigit(): + out.append(name) + return sorted(out, key=int) + + +def _resolve_latest_subdir(path: str) -> str: + """Map a base ckpt dir → its highest-numbered numeric subdir. + + Used so users can set `load_dmp_checkpoint.path = ""` (or + `CKPT_PATH=`) and automatically pick up the most recent save without + needing to know which step number to point at. If `path` already names a leaf save (numeric basename) it's returned + unchanged. If the base dir has no numeric subdirs yet — the cold-start case + where ``CKPT_PATH`` is configured but nothing has been saved (e.g. the + interrupt phase of the resume test starts from a freshly-cleaned dir) — we + return ``""`` so ``load_*_checkpoint`` no-ops instead of asserting on a + missing ``sparse/.metadata``. + """ + if not path: + return path + base = path.rstrip("/") + leaf = os.path.basename(base) + if leaf.isdigit(): + return base # already a leaf, caller knows what it wants + subs = _list_numeric_subdirs(base) + if not subs: + logger.info("No checkpoint subdirs under %s — cold start (no load).", base) + return "" # nothing to load → load_*_checkpoint short-circuits + resolved = os.path.join(base, subs[-1]) + logger.info("Auto-latest checkpoint: %s → %s", base, resolved) + return resolved + + +def _prune_old_checkpoints(base_path: str, keep_last_n: int, just_saved_subdir: str) -> None: + """Delete numeric subdirs older than the keep_last_n most recent. + + Defensive: never prune `just_saved_subdir` even if it would be evicted by + the keep_last_n window (shouldn't happen since we just wrote it, but + catches off-by-one bugs). Skipped entirely when keep_last_n<=0. + """ + if keep_last_n <= 0: + return + subs = _list_numeric_subdirs(base_path) + if len(subs) <= keep_last_n: + return + to_prune = subs[:-keep_last_n] + for name in to_prune: + full = os.path.join(base_path, name) + if os.path.realpath(full) == os.path.realpath(just_saved_subdir): + continue + try: + shutil.rmtree(full) + logger.info("Pruned old checkpoint: %s", full) + except OSError as e: + logger.warning("Failed to prune %s: %s", full, e) + + +def _cleanup_stale_tmps(base_path: str) -> None: + """Remove `*.tmp`/`*.old` subdirs left by a crashed prior save attempt. + + `*.tmp` = an interrupted write; `*.old` = an interrupted atomic-overwrite + swap (see the promotion step in save_dmp_checkpoint). Both are non-numeric + so `_resolve_latest_subdir` already ignores them; this just reclaims disk. + """ + if not os.path.isdir(base_path): + return + for name in os.listdir(base_path): + if name.endswith(".tmp") or name.endswith(".old"): + full = os.path.join(base_path, name) + try: + shutil.rmtree(full) + logger.warning("Removed stale checkpoint dir: %s", full) + except OSError as e: + logger.warning("Failed to remove stale dir %s: %s", full, e) + + @gin.configurable def save_dmp_checkpoint( model: torch.nn.Module, @@ -94,32 +214,64 @@ def save_dmp_checkpoint( rank: int, batch_idx: int, path: str = "", + keep_last_n: int = 1, + train_ts: Optional[int] = None, + batch_idx_in_window: int = WINDOW_COMPLETE, + device: Optional[torch.device] = None, ) -> None: """ Save a distributed model checkpoint including sparse and dense components. - Saves the model's sparse tensors using distributed checkpointing and dense - tensors, optimizer state, and metrics using standard PyTorch serialization. + Writes into a per-rank-coordinated atomic layout: + /.tmp/ ← directory written into during save + // ← atomically renamed from .tmp on success + + A crash mid-save leaves the `.tmp/` orphan, which `_cleanup_stale_tmps` + sweeps on the next save attempt and which `_resolve_latest_subdir` ignores + (non-numeric basename). The previous successful `/` remains valid. Args: model: The model to checkpoint. optimizer: The optimizer whose state should be saved. metric_logger: The metrics logger containing training/eval metrics. rank: The current process rank in distributed training. - batch_idx: The current batch index (used for checkpoint naming). + batch_idx: Subdir name (for streaming we set this == train_ts so the + on-disk layout monotonically increases). path: Base path for saving the checkpoint. If empty, no checkpoint is saved. + keep_last_n: Number of most-recent numeric subdirs to retain after a + successful save. Set 1 (default) for disk-bounded long runs; + <=0 disables pruning. + train_ts: For streaming-train-eval, the current train timestamp. + Stored in non_sparse.ckpt so resume knows which window to enter. + batch_idx_in_window: For streaming-train-eval, batches completed within + train_ts. WINDOW_COMPLETE (-1) means the window finished; resume + advances to train_ts+1. >=0 means crash happened mid-window; resume + re-enters train_ts at batch_idx_in_window. + device: CUDA device for the per-rank RNG snapshot. Required for + bit-equal trajectories across resume (HSTU dropout consumes the + per-device RNG cycle). """ if path == "": return - now = datetime.now() - formatted_datetime = now.strftime("%Y_%m_%d_%H_%M_%S") - path = f"{path}/{batch_idx}" - if not os.path.exists(path) and rank == 0: - os.makedirs(path) - sparse_path = f"{path}/sparse/" - if not os.path.exists(sparse_path) and rank == 0: - os.makedirs(sparse_path) - non_sparse_ckpt = f"{path}/non_sparse.ckpt" + base_path = path + # Atomic-save layout: write to .tmp, rename to final, prune older. + tmp_subdir = f"{base_path}/{batch_idx}.tmp" + final_subdir = f"{base_path}/{batch_idx}" + + if rank == 0: + _cleanup_stale_tmps(base_path) + # Always (re)write into a fresh .tmp. An existing `final_subdir` with the + # same batch_idx (e.g. a later in-window save for the same train_ts, or a + # deterministic re-run at the same step) is overwritten atomically at the + # promotion step below — NOT skipped here. Skipping would desync ranks: + # the collective barrier/checkpoint.save calls below run on *every* rank, + # so a rank-0-only early return deadlocks ranks 1..N on the next barrier. + shutil.rmtree(tmp_subdir, ignore_errors=True) + os.makedirs(tmp_subdir, exist_ok=True) + os.makedirs(f"{tmp_subdir}/sparse/", exist_ok=True) + torch.distributed.barrier() + sparse_path = f"{tmp_subdir}/sparse/" + non_sparse_ckpt = f"{tmp_subdir}/non_sparse.ckpt" sparse_tensor_keys = { k for k, v in model.state_dict().items() if isinstance(v, ShardedTensor) @@ -148,9 +300,20 @@ def save_dmp_checkpoint( "reg_metrics": regression_metric_state_dict, "global_step": metric_logger.global_step, "sparse_tensor_keys": sparse_tensor_keys, + # Streaming resume fields. Defaulted on load so old checkpoints + # (pre-streaming-resume) still load as a normal restart. + "train_ts": train_ts, + "batch_idx_in_window": batch_idx_in_window, }, non_sparse_ckpt, ) + + # Per-rank RNG snapshot. Written even on a single rank because dropout's + # randomness comes from the CUDA generator which differs across devices. + if device is not None: + rng_path = f"{tmp_subdir}/rng_rank{rank}.pt" + torch.save(_rng_state(device), rng_path) + torch.distributed.barrier() sparse_dict = {"sparse_dict": SparseState(model, sparse_tensor_keys)} torch.distributed.checkpoint.save( @@ -158,7 +321,25 @@ def save_dmp_checkpoint( storage_writer=torch.distributed.checkpoint.FileSystemWriter(sparse_path), ) torch.distributed.barrier() - print("checkpoint successfully saved") + # Promote .tmp → final, then prune. Done on rank 0 only since the directory + # operations are global filesystem state. + if rank == 0: + if os.path.exists(final_subdir): + # POSIX rename() refuses to replace a non-empty directory, so we + # can't os.replace(tmp, final) directly. Swap the old snapshot aside + # (instant rename), move the new one into place, then delete the old. + # The `.old` name is non-numeric → ignored by _resolve_latest_subdir + # and swept by _cleanup_stale_tmps on the next save if we crash mid-swap. + old_aside = f"{final_subdir}.old" + shutil.rmtree(old_aside, ignore_errors=True) + os.replace(final_subdir, old_aside) + os.replace(tmp_subdir, final_subdir) + shutil.rmtree(old_aside, ignore_errors=True) + else: + os.replace(tmp_subdir, final_subdir) + _prune_old_checkpoints(base_path, keep_last_n, final_subdir) + logger.info("checkpoint successfully saved → %s", final_subdir) + torch.distributed.barrier() @gin.configurable @@ -190,24 +371,30 @@ def load_nonsparse_checkpoint( optimizer: Optional[Optimizer] = None, metric_logger: Optional[MetricsLogger] = None, path: str = "", -) -> None: + rank: int = 0, +) -> Tuple[Optional[int], int]: """ Load non-sparse (dense) components from a checkpoint. Loads dense model parameters, and optionally optimizer state and metrics. + Also restores per-rank RNG state if a matching `rng_rank{rank}.pt` is found + next to `non_sparse.ckpt`. - Args: - model: The model to load dense parameters into. - device: The device to load tensors onto. - optimizer: Optional optimizer to restore state for. - metric_logger: Optional metrics logger to restore state for. - path: Base path of the checkpoint. If empty, no loading is performed. + Returns: + (train_ts, batch_idx_in_window) — the streaming resume hint stored at + save time. `(None, WINDOW_COMPLETE)` if not a streaming checkpoint or + no path supplied. """ if path == "": - return + return None, WINDOW_COMPLETE non_sparse_ckpt = f"{path}/non_sparse.ckpt" - non_sparse_state_dict = torch.load(non_sparse_ckpt, map_location=device) + # weights_only=False: these are our own trusted checkpoints, and they hold + # non-tensor objects (optimizer/metric state dicts, numpy-backed RNG state) + # that PyTorch>=2.6's weights_only=True default refuses to unpickle. + non_sparse_state_dict = torch.load( + non_sparse_ckpt, map_location=device, weights_only=False + ) load_dense_state_dict(model, non_sparse_state_dict["dense_dict"]) print("dense checkpoint successfully loaded") if optimizer is not None: @@ -226,6 +413,21 @@ def load_nonsparse_checkpoint( for i, m in enumerate(metric_logger.regression_metrics["eval"]): m.load_state_dict(regression_metric_state_dict["eval"][i]) + # Per-rank RNG restore. Missing file = bit-equal trajectory not requested at + # save time; we silently continue (the test harness checks for both). + rng_path = f"{path}/rng_rank{rank}.pt" + if os.path.exists(rng_path): + # weights_only=False: RNG state is numpy/Python tuples, not tensors. + rng_state = torch.load(rng_path, map_location="cpu", weights_only=False) + _restore_rng_state(rng_state, device) + logger.info("RNG state restored from %s", rng_path) + + train_ts = non_sparse_state_dict.get("train_ts") + batch_idx_in_window = non_sparse_state_dict.get( + "batch_idx_in_window", WINDOW_COMPLETE + ) + return train_ts, batch_idx_in_window + @gin.configurable def load_dmp_checkpoint( @@ -234,25 +436,27 @@ def load_dmp_checkpoint( metric_logger: MetricsLogger, device: torch.device, path: str = "", -) -> None: + rank: int = 0, +) -> Tuple[Optional[int], int]: """ Load a complete distributed model checkpoint (both sparse and dense components). - This is a convenience function that calls both load_sparse_checkpoint and - load_nonsparse_checkpoint. + `path` is auto-resolved: if it points at a directory containing numeric + subdirs (e.g. CKPT_PATH=/), the highest-numbered subdir is used. If it + already names a leaf save (e.g. /300), it's used as-is. Empty string = + no load. - Args: - model: The model to load the checkpoint into. - optimizer: The optimizer to restore state for. - metric_logger: The metrics logger to restore state for. - device: The device to load tensors onto. - path: Base path of the checkpoint. If empty, no loading is performed. + Returns: + (train_ts, batch_idx_in_window) — streaming resume hint. Callers that + don't need it can ignore. """ - load_sparse_checkpoint(model=model, path=path) - load_nonsparse_checkpoint( + resolved = _resolve_latest_subdir(path) + load_sparse_checkpoint(model=model, path=resolved) + return load_nonsparse_checkpoint( model=model, optimizer=optimizer, metric_logger=metric_logger, - path=path, + path=resolved, device=device, + rank=rank, ) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index 53066ef9a..747e3876f 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -104,7 +104,12 @@ nts/env_int.default = 30 streaming_train_eval_loop.start_ts = @sts/env_int() sts/env_int.key = "START_TS" sts/env_int.default = 150 -streaming_train_eval_loop.metric_log_frequency = 50 +# Per-step metric logging cadence. Default 50 (one compute_and_log GPU->CPU +# sync per 50 batches). The streaming-resume test sets METRIC_LOG_FREQ=1 so +# every step emits a parseable "Step N metrics" line for trajectory comparison. +streaming_train_eval_loop.metric_log_frequency = @mlf/env_int() +mlf/env_int.key = "METRIC_LOG_FREQ" +mlf/env_int.default = 50 # Trace on by default: reuses the shared Profiler.* bindings below (5-step # window at step 52). The streaming step counter advances across train+eval # batches, so step 52 lands in the first (train) window's compute. @@ -160,7 +165,46 @@ MetricsLogger.world_size = 8 MetricsLogger.auc_threshold = 0.80275 # Checkpointing disabled by default — a full DMP checkpoint is ~100s of GB and # the streaming loop always saves on the final window. save_dmp_checkpoint -# no-ops on the empty path. Set $CKPT_PATH to a directory to re-enable. +# no-ops on the empty path. Set $CKPT_PATH to a directory to re-enable; the +# load path auto-resolves to the highest-numbered numeric subdir inside it. save_dmp_checkpoint.path = @ckpt/env_path() ckpt/env_path.key = "CKPT_PATH" ckpt/env_path.default = "" +load_dmp_checkpoint.path = @ckpt/env_path() +# Retention: keep only the most-recent N numeric subdirs after each successful +# save (atomic rename + prune-older). Override via $KEEP_LAST_N. +save_dmp_checkpoint.keep_last_n = @kln/env_int() +kln/env_int.key = "KEEP_LAST_N" +kln/env_int.default = 1 +# In-window checkpoint cadence for streaming-train-eval. 0 (default) = end-of- +# window only; on crash mid-window the partial progress is lost. Set N>0 (e.g. +# via $IN_WINDOW_CKPT_FREQ) for batch-granularity exact-once-on-resume — paired +# with the auto-latest load above, the resumed run skips the K already-trained +# batches of the partial window and continues with bit-equal trajectory. +streaming_train_eval_loop.in_window_checkpoint_frequency = @iwcf/env_int() +iwcf/env_int.key = "IN_WINDOW_CKPT_FREQ" +iwcf/env_int.default = 0 +# Global-step checkpoint cadence: save whenever the monotonic train global_step +# crosses a multiple of N (a true "every 1000 steps" trigger that spans windows +# and survives resume). 0 (default) = off. Override via $CKPT_STEP_FREQ. +streaming_train_eval_loop.checkpoint_step_frequency = @csf/env_int() +csf/env_int.key = "CKPT_STEP_FREQ" +csf/env_int.default = 0 +# Wall-clock checkpoint cadence in seconds: save when >= this many seconds have +# elapsed since the last save (e.g. 3600 for hourly). Rank 0 owns the clock and +# broadcasts the decision so all ranks save together. 0.0 (default) = off. +# Override via $CKPT_TIME_INTERVAL_S. +streaming_train_eval_loop.checkpoint_time_interval_s = @ctis/env_float() +ctis/env_float.key = "CKPT_TIME_INTERVAL_S" +ctis/env_float.default = 0.0 +# Cap each train_ts window's batch count (mostly for the resume test driver). +# Unset / 0 = use the full window. +streaming_train_eval_loop.num_train_batches = @ntb/env_int() +ntb/env_int.key = "NUM_TRAIN_BATCHES" +ntb/env_int.default = 0 +# Test-only failure injection: when >=0 and metric_logger.global_step['train'] +# reaches this, the process exits with code 42 right after the in-window save +# fires. Used by `scripts/streaming_resume_test.sh` to verify resume. +streaming_train_eval_loop.die_at_step = @das/env_int() +das/env_int.key = "DIE_AT_STEP" +das/env_int.default = -1 diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/checkpoint_cadence_test.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/checkpoint_cadence_test.py new file mode 100644 index 000000000..4a483c262 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/checkpoint_cadence_test.py @@ -0,0 +1,131 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-strict +"""Unit tests for `select_in_window_checkpoint_reason` — the pure decision that +drives the streaming loop's three fine-grained checkpoint cadences: + + * `in_window_checkpoint_frequency` — per-window-local batch count + * `checkpoint_step_frequency` — monotonic global step ("every 1000 steps") + * `checkpoint_time_interval_s` — wall-clock ("hourly") + +These run without a GPU / distributed init: the loop broadcasts a single +`elapsed_since_last_save` from rank 0 and then calls this pure function, so +exercising the function directly fully covers the trigger semantics. +""" +import unittest + +from generative_recommenders.dlrm_v3.train.utils import ( + select_in_window_checkpoint_reason, +) + + +def _reason( + *, + batch: int = 1, + step: int = 1, + elapsed: float = 0.0, + in_window: int = 0, + step_freq: int = 0, + time_s: float = 0.0, +) -> str | None: + return select_in_window_checkpoint_reason( + train_batch_idx=batch, + global_step=step, + elapsed_since_last_save=elapsed, + in_window_checkpoint_frequency=in_window, + checkpoint_step_frequency=step_freq, + checkpoint_time_interval_s=time_s, + ) + + +class CheckpointCadenceTest(unittest.TestCase): + def test_all_disabled_never_fires(self) -> None: + for batch in (1, 100, 1000): + for step in (1, 1000, 5000): + self.assertIsNone(_reason(batch=batch, step=step, elapsed=1e9)) + + def test_step_based_every_1000(self) -> None: + # Fires exactly on multiples of the step frequency. + self.assertEqual(_reason(step=1000, step_freq=1000), "global_step") + self.assertEqual(_reason(step=2000, step_freq=1000), "global_step") + # Does not fire just off a boundary. + self.assertIsNone(_reason(step=999, step_freq=1000)) + self.assertIsNone(_reason(step=1001, step_freq=1000)) + + def test_step_zero_does_not_trigger(self) -> None: + # global_step==0 must not trivially satisfy `0 % N == 0`. + self.assertIsNone(_reason(step=0, step_freq=1000)) + + def test_time_based_interval(self) -> None: + # At/over the interval -> fires; under -> no save. + self.assertEqual( + _reason(step=3, elapsed=3600.0, time_s=3600.0), "time_interval" + ) + self.assertEqual( + _reason(step=3, elapsed=4000.0, time_s=3600.0), "time_interval" + ) + self.assertIsNone(_reason(step=3, elapsed=3599.9, time_s=3600.0)) + + def test_in_window_batch_cadence(self) -> None: + self.assertEqual(_reason(batch=5, in_window=5), "in_window_batch") + self.assertEqual(_reason(batch=10, in_window=5), "in_window_batch") + self.assertIsNone(_reason(batch=4, in_window=5)) + + def test_precedence_in_window_over_step_over_time(self) -> None: + # All three would fire this batch; precedence picks in_window first. + self.assertEqual( + _reason( + batch=5, + step=1000, + elapsed=9999.0, + in_window=5, + step_freq=1000, + time_s=3600.0, + ), + "in_window_batch", + ) + # in_window not due this batch -> step wins over time. + self.assertEqual( + _reason( + batch=4, + step=1000, + elapsed=9999.0, + in_window=5, + step_freq=1000, + time_s=3600.0, + ), + "global_step", + ) + # Neither batch nor step due -> time wins. + self.assertEqual( + _reason( + batch=4, + step=999, + elapsed=9999.0, + in_window=5, + step_freq=1000, + time_s=3600.0, + ), + "time_interval", + ) + + def test_step_and_time_combined_independent(self) -> None: + # Step frequency enabled, time disabled: only step boundaries fire. + self.assertEqual(_reason(step=1000, step_freq=1000, time_s=0.0), "global_step") + self.assertIsNone(_reason(step=1000, elapsed=1e9, step_freq=0, time_s=0.0)) + + +if __name__ == "__main__": + unittest.main() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.py new file mode 100644 index 000000000..2774e8c03 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.py @@ -0,0 +1,166 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""End-to-end failure-injection test for streaming resume. + +Validates the four resume features end-to-end on the yambda-5b stack: + 1. Mid-window save (in_window_checkpoint_frequency) + 2. Within-window exact-once skip (StreamingWindowSampler.set_window skip) + 3. Auto-detect-latest checkpoint subdir + 4. keep_last_n retention (default 1) + +Test flow (driven by `scripts/streaming_resume_test.sh`): + Phase 1 (baseline): Run streaming-train-eval for N=2 train_ts × K batches/window + with die_at_step=-1. Capture per-batch window_ne / window_auc into traj_baseline.json. + Phase 2 (interrupt): Same config but die_at_step=M (M mid-window-2). Expect + process to exit(42) after the in-window checkpoint at step M lands. + Phase 3 (resume): Re-launch with same CKPT_PATH (auto-latest picks the + in-window save). Continue to the same total step count. Capture + traj_resumed.json (which only contains the post-resume steps). + + Correctness is proven by the FUNCTIONAL INVARIANTS checked in the shell + driver (resumed at exactly batch_idx_in_window, per-rank RNG restored, atomic + save + keep_last_n), NOT by bit-equal trajectory matching. The training stack + is nondeterministic across runs (non-deterministic atomic scatter-add in the + embedding/attention backward on ROCm): two independent *cold* runs already + drift ~7e-4 in window_ne over 20 steps, and early-training chaos amplifies + it, so resume-vs-baseline can differ by a few percent even when resume is + perfect. The trajectory comparison here is therefore a LOOSE closeness bound + (default atol below) that only flags gross divergence — wrong data slice or + unrestored model state — while tolerating nondeterministic drift. + +This module also provides a CLI entry point used by the shell driver to (a) +parse a train.log into a step-keyed dict of metrics, and (b) compare two such +dicts and fail loudly on mismatch. +""" + +import argparse +import json +import re +import sys +from typing import Dict, Tuple + +# Per-step metrics from MetricsLogger.compute_and_log are emitted like: +# "train - Step 51 metrics: {'metric/lifetime_ne/listen_plus': tensor(1.0954, ...) +# 'metric/window_ne/listen_plus': tensor(0.9940, ...), +# 'metric/window_accuracy/listen_plus': tensor(0.6231, ...) ..." +_STEP_RE = re.compile(r"train - Step (\d+) metrics:") +_WNE_RE = re.compile(r"window_ne/listen_plus.*?tensor\(([0-9.]+)") +_WAUC_RE = re.compile(r"window_auc/listen_plus.*?tensor\(([0-9.]+)") +_WACC_RE = re.compile(r"window_accuracy/listen_plus.*?tensor\(([0-9.]+)") + + +def parse_trajectory(log_path: str) -> Dict[int, Dict[str, float]]: + """Extract a {step: {window_ne, window_auc, window_accuracy}} dict from a + train.log. The grep is loose on the metric line itself — we accept the + very long truncated form MetricsLogger prints.""" + out: Dict[int, Dict[str, float]] = {} + with open(log_path, "r", errors="replace") as f: + for line in f: + m = _STEP_RE.search(line) + if not m: + continue + step = int(m.group(1)) + wne = _WNE_RE.search(line) + wauc = _WAUC_RE.search(line) + wacc = _WACC_RE.search(line) + if not (wne and wauc and wacc): + continue + # Only keep ONE entry per step — log can have duplicate per-rank + # prints; first one wins (they're identical). + if step in out: + continue + out[step] = { + "window_ne": float(wne.group(1)), + "window_auc": float(wauc.group(1)), + "window_accuracy": float(wacc.group(1)), + } + return out + + +def compare_trajectories( + baseline: Dict[int, Dict[str, float]], + resumed: Dict[int, Dict[str, float]], + min_resume_step: int, + atol: float = 0.15, +) -> Tuple[bool, str]: + """Compare baseline vs resumed trajectories for steps >= min_resume_step. + + This is a LOOSE closeness bound, not a bit-equality check — see the module + docstring. `atol` defaults to a value that tolerates the nondeterministic + cross-run drift of this stack while still catching gross resume bugs. + Returns (ok, message). `ok=False` on any divergence outside `atol`.""" + steps = sorted(s for s in resumed if s >= min_resume_step) + if not steps: + return False, f"No resumed steps >= {min_resume_step}" + mismatches = [] + for s in steps: + if s not in baseline: + mismatches.append(f"step {s}: missing from baseline") + continue + b = baseline[s] + r = resumed[s] + for key in ("window_ne", "window_auc", "window_accuracy"): + if abs(b[key] - r[key]) > atol: + mismatches.append( + f"step {s} {key}: baseline={b[key]:.6f} " + f"resumed={r[key]:.6f} diff={b[key]-r[key]:+.6f}" + ) + if mismatches: + return False, ( + f"{len(mismatches)} mismatches across {len(steps)} resumed steps " + f"(atol={atol}):\n " + "\n ".join(mismatches[:10]) + ) + return True, ( + f"{len(steps)} resumed steps match baseline within atol={atol} " + f"(range: step {steps[0]}..{steps[-1]})" + ) + + +def main() -> int: + ap = argparse.ArgumentParser() + sub = ap.add_subparsers(dest="cmd", required=True) + + p_parse = sub.add_parser("parse", help="Parse a train.log → traj JSON") + p_parse.add_argument("log") + p_parse.add_argument("out") + + p_cmp = sub.add_parser("compare", help="Compare baseline vs resumed traj JSONs") + p_cmp.add_argument("baseline") + p_cmp.add_argument("resumed") + p_cmp.add_argument("--min-resume-step", type=int, required=True) + p_cmp.add_argument("--atol", type=float, default=0.15) + + args = ap.parse_args() + if args.cmd == "parse": + traj = parse_trajectory(args.log) + with open(args.out, "w") as f: + json.dump(traj, f, indent=2) + print(f"Wrote {len(traj)} step entries to {args.out}", file=sys.stderr) + return 0 + if args.cmd == "compare": + with open(args.baseline) as f: + baseline = {int(k): v for k, v in json.load(f).items()} + with open(args.resumed) as f: + resumed = {int(k): v for k, v in json.load(f).items()} + ok, msg = compare_trajectories( + baseline, resumed, args.min_resume_step, atol=args.atol + ) + print(msg) + return 0 if ok else 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py index c6a90c2b7..19490b840 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py @@ -111,8 +111,16 @@ def _main_func( device=device, rank=rank, ) - load_dmp_checkpoint( - model=model, optimizer=optimizer, metric_logger=metrics, device=device + # Capture streaming resume hint (None for cold start / non-streaming + # checkpoints). For the streaming-train-eval mode, we forward this into + # streaming_train_eval_loop so it can advance past the last completed + # window OR re-enter the partial window and skip already-trained batches. + resume_train_ts, resume_batch_idx_in_window = load_dmp_checkpoint( + model=model, + optimizer=optimizer, + metric_logger=metrics, + device=device, + rank=rank, ) # train loop @@ -161,6 +169,8 @@ def _main_func( device=device, hstu_config=model_configs, embedding_table_configs=embedding_table_configs, + resume_train_ts=resume_train_ts, + resume_batch_idx_in_window=resume_batch_idx_in_window, ) except Exception as e: logger.info(traceback.format_exc()) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py index 890144e1d..6d50064b6 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -35,7 +35,7 @@ import gin import torch import torchrec -from generative_recommenders.dlrm_v3.checkpoint import save_dmp_checkpoint +from generative_recommenders.dlrm_v3.checkpoint import save_dmp_checkpoint, WINDOW_COMPLETE from generative_recommenders.dlrm_v3.configs import ( get_embedding_table_config, get_hstu_configs, @@ -467,9 +467,27 @@ def __init__(self, rank: int, world_size: int) -> None: self._world_size: int = world_size self._indices: List[int] = [] - def set_window(self, global_indices) -> None: + def set_window(self, global_indices, skip_samples: int = 0) -> None: + """Install this window's per-rank index list, optionally fast-forwarding. + + ``skip_samples`` drops the first N per-rank samples from the list so the + next ``__iter__`` starts at sample N+1 in this rank's slice. Used on + resume to skip batches that were already trained: pass + ``skip_samples = batch_size * batches_completed`` and the dataloader + emits batches starting at exactly the next unseen batch. + + The skip is safe because the sample order is fully deterministic given + (global_indices, rank, world_size): we re-derive the same per-rank list + as the pre-crash run, just hand back a tail slice of it. + """ n = (len(global_indices) // self._world_size) * self._world_size - self._indices = global_indices[:n][self._rank :: self._world_size].tolist() + per_rank = global_indices[:n][self._rank :: self._world_size].tolist() + if skip_samples < 0 or skip_samples > len(per_rank): + raise ValueError( + f"skip_samples={skip_samples} out of [0, {len(per_rank)}] " + f"for rank={self._rank} world_size={self._world_size}" + ) + self._indices = per_rank[skip_samples:] def __iter__(self): return iter(self._indices) @@ -531,21 +549,29 @@ def __init__( self._dls = [dl_factory(s) for s in self._samplers] self._iters: List[Optional[object]] = [None] * n_buffers - def _prepare(self, buf: int, ts: int) -> None: + def _prepare(self, buf: int, ts: int, skip_samples: int = 0) -> None: # window_indices() is the O(N) mask; numpy releases the GIL for it, so it # overlaps the main thread's GPU dispatch. iter() then kicks off this # pool's background prefetch. - self._samplers[buf].set_window(self._dataset.dataset.window_indices(ts)) + # `skip_samples` is non-zero only for the very first window after a + # mid-window resume; subsequent windows always start at 0. + self._samplers[buf].set_window( + self._dataset.dataset.window_indices(ts), skip_samples=skip_samples + ) self._iters[buf] = iter(self._dls[buf]) - def stream(self, ts_list: List[int]): + def stream(self, ts_list: List[int], first_skip_samples: int = 0): + """Stream (ts, iterator) pairs. `first_skip_samples` is applied ONLY to + the first ts in ``ts_list`` (the mid-window-resumed window); every + subsequent window starts at sample 0 of its own per-rank list.""" n = len(ts_list) if n == 0: return threads: List[Optional[threading.Thread]] = [None] * self._n # Prime the first n_buffers windows on the main thread (forks all pools). for b in range(min(self._n, n)): - self._prepare(b, ts_list[b]) + skip = first_skip_samples if b == 0 else 0 + self._prepare(b, ts_list[b], skip_samples=skip) for i in range(n): buf = i % self._n if threads[buf] is not None: @@ -553,6 +579,8 @@ def stream(self, ts_list: List[int]): threads[buf] = None yield ts_list[i], self._iters[buf] # This pool is now free; prefetch the window n_buffers ahead. + # No skip on subsequent windows — only the first prepared window + # carries `first_skip_samples`. j = i + self._n if j < n: th = threading.Thread( @@ -1009,10 +1037,56 @@ def train_eval_loop( for k, v in metric_logger.compute(mode="eval").items(): print(f"{k}: {v}") model.train() - if num_train_batches is not None and train_batch_idx >= num_train_batches: + # `num_train_batches` cap: None or 0 = run the whole window. >0 caps + # batches per window (mostly the streaming-resume test driver uses + # this to keep test windows short). + if num_train_batches and train_batch_idx >= num_train_batches: break +def select_in_window_checkpoint_reason( + *, + train_batch_idx: int, + global_step: int, + elapsed_since_last_save: float, + in_window_checkpoint_frequency: int, + checkpoint_step_frequency: int, + checkpoint_time_interval_s: float, +) -> Optional[str]: + """Decide which (if any) in-window checkpoint cadence fires this batch. + + Pure / distributed-agnostic so it can be unit-tested without a real run. + The caller computes `elapsed_since_last_save` (broadcast from rank 0 in the + streaming loop) so all ranks pass the same value and reach the same verdict. + + Precedence (at most one save per batch): per-window-local batch count > + monotonic global step > wall-clock interval. Returns the trigger reason + string, or None when no cadence fires. A cadence is disabled when its + frequency/interval is 0 / 0.0. + + Counter conventions match the loop: `train_batch_idx` is already + post-incremented (>=1 on the first batch), and `global_step` is guarded + >0 so step 0 doesn't trivially satisfy `% N == 0`. + """ + if ( + in_window_checkpoint_frequency > 0 + and train_batch_idx % in_window_checkpoint_frequency == 0 + ): + return "in_window_batch" + if ( + checkpoint_step_frequency > 0 + and global_step > 0 + and global_step % checkpoint_step_frequency == 0 + ): + return "global_step" + if ( + checkpoint_time_interval_s > 0 + and elapsed_since_last_save >= checkpoint_time_interval_s + ): + return "time_interval" + return None + + @gin.configurable def streaming_train_eval_loop( rank: int, @@ -1032,7 +1106,55 @@ def streaming_train_eval_loop( persistent_loader: bool = False, eval_each_window: bool = True, double_buffer: bool = False, + # --- resume / mid-window-exact-once knobs --- + resume_train_ts: Optional[int] = None, + resume_batch_idx_in_window: int = WINDOW_COMPLETE, + in_window_checkpoint_frequency: int = 0, + # --- global step / wall-clock checkpoint cadences --- + checkpoint_step_frequency: int = 0, + checkpoint_time_interval_s: float = 0.0, + # --- test-only failure injection knob --- + die_at_step: int = -1, ) -> None: + """Streaming train+eval loop with per-window (and optionally mid-window) + checkpoints. + + Resume semantics (set by train_ranker after `load_dmp_checkpoint` returns): + - resume_train_ts=None: cold start; honor `start_ts` as-is. + - resume_train_ts=N, resume_batch_idx_in_window=WINDOW_COMPLETE(-1): + previous run finished window N cleanly. Start at N+1 from sample 0. + - resume_train_ts=N, resume_batch_idx_in_window=K (K>=0): previous run + crashed mid-window after K completed batches. Re-enter window N and + skip the first K batches of THIS rank's per-rank sample list (deterministic + slice since `window_indices(N)` is a pure function of the anchor_ts cache). + + Checkpoint cadences (all independent; any combination may be enabled): + - `checkpoint_frequency`: window-granularity. End-of-window save every + Nth train_ts (and always on the final window). Uses WINDOW_COMPLETE. + - `in_window_checkpoint_frequency`: per-window-local batch count. Fires + every N batches *within* a window (counter resets each window). + - `checkpoint_step_frequency`: global-step granularity. Fires whenever + the monotonic `metric_logger.global_step['train']` hits a multiple of + N — i.e. a true "every 1000 steps" trigger that spans windows and + survives resume (global_step is restored from the checkpoint). + - `checkpoint_time_interval_s`: wall-clock granularity. Fires when at + least this many seconds have elapsed since the last save (e.g. 3600 + for hourly). Rank 0 owns the clock and broadcasts the decision so all + ranks save together (avoids the collective barrier in + `save_dmp_checkpoint` deadlocking on a split decision). + + All in-window triggers (`in_window_checkpoint_frequency`, + `checkpoint_step_frequency`, `checkpoint_time_interval_s`) route through + `_save_mid_window`, which stamps `batch_idx_in_window=K` so a crash leaves + a resumable partial-window checkpoint. End-of-window saves + (`checkpoint_frequency`) always use the WINDOW_COMPLETE sentinel. 0 / 0.0 + disables a given cadence (the default for all three fine-grained ones). + + `die_at_step` is a test-only hook: when `metric_logger.global_step['train']` + reaches this value, the process exits with code 42 right after the in-window + save fires. Used by the failure-injection test to crash at a deterministic + boundary and then resume. + """ profiler = Profiler(rank) if output_trace else None dataset_class, kwargs = get_dataset() kwargs["embedding_config"] = embedding_table_configs @@ -1045,22 +1167,73 @@ def streaming_train_eval_loop( # warmup). The non-persistent path recreates a DataLoader per window. window_sampler: Optional[StreamingWindowSampler] = None persistent_dl: Optional[DataLoader] = None + world_size = ( + torch.distributed.get_world_size() + if torch.distributed.is_initialized() + else 1 + ) if persistent_loader: - world_size = ( - torch.distributed.get_world_size() - if torch.distributed.is_initialized() - else 1 - ) window_sampler = StreamingWindowSampler(rank=rank, world_size=world_size) persistent_dl = make_persistent_streaming_dataloader( dataset=dataset, sampler=window_sampler ) - def _window_iter(ts: int): + # Apply resume hint: advance start_ts past the last completed window, or + # re-enter the partial window with a per-rank skip on its first iter. + # Shrink num_train_ts by the same amount so the resumed run finishes at + # the same final timestamp (start_ts + num_train_ts) as a fresh run would + # — i.e. resumed and uninterrupted produce identical total work. + first_skip_samples = 0 + if resume_train_ts is not None: + original_end_ts = start_ts + num_train_ts + if resume_batch_idx_in_window == WINDOW_COMPLETE: + new_start = resume_train_ts + 1 + if rank == 0: + logger.info( + "Resuming from completed train_ts=%d → start_ts=%d " + "(num_train_ts %d → %d)", + resume_train_ts, new_start, + num_train_ts, max(0, original_end_ts - new_start), + ) + start_ts = new_start + else: + if rank == 0: + logger.info( + "Resuming mid-window at train_ts=%d batch_idx_in_window=%d " + "(skipping batches already trained)", + resume_train_ts, + resume_batch_idx_in_window, + ) + start_ts = resume_train_ts + # `batch_size` is per-rank from the persistent dataloader (set via + # gin `make_persistent_streaming_dataloader.batch_size`). The + # skip-samples-per-rank below maps "K batches done" → "K * bs + # samples in this rank's index list", since each batch draws bs + # samples from this rank's deterministic round-robin slice. + assert persistent_dl is not None, ( + "Mid-window resume requires persistent_loader=True" + ) + first_skip_samples = resume_batch_idx_in_window * persistent_dl.batch_size + num_train_ts = max(0, original_end_ts - start_ts) + if num_train_ts == 0 and rank == 0: + logger.info( + "Resume target already reached (end_ts=%d, start_ts=%d) — " + "no further training windows; skipping straight to final eval.", + original_end_ts, start_ts, + ) + + def _window_iter(ts: int, skip_samples: int = 0): if persistent_loader: assert window_sampler is not None and persistent_dl is not None - window_sampler.set_window(dataset.dataset.window_indices(ts)) # pyre-ignore [16] + window_sampler.set_window( + dataset.dataset.window_indices(ts), # pyre-ignore [16] + skip_samples=skip_samples, + ) return iter(persistent_dl) + if skip_samples != 0: + raise NotImplementedError( + "skip_samples>0 requires persistent_loader=True" + ) return iter(make_streaming_dataloader(dataset=dataset, ts=ts)) # Windows are [start_ts, start_ts + num_train_ts); each step trains window T # then evals window T+1, so the last eval window is start_ts + num_train_ts, @@ -1076,8 +1249,54 @@ def _window_iter(ts: int): f"available windows ({available}); clamping num_train_ts to {max_count}." ) num_train_ts = max_count - def _run_train_window(train_data_iterator, label: Optional[str] = None) -> None: - train_batch_idx = 0 + # Wall-clock anchor for time-based checkpointing. Mutable single-element + # list so the nested train loop can reset it after each save. Starts at + # loop entry so the first time-trigger fires ~interval seconds in. + last_ckpt_time = [time.time()] + + def _broadcast_elapsed() -> float: + """Seconds since the last save, owned by rank 0 and broadcast to all + ranks. save_dmp_checkpoint runs a collective barrier, so every rank must + feed the same wall-clock value into the cadence decision — otherwise a + split verdict (rank 0 saves, rank 1 doesn't) would deadlock. Broadcasting + rank 0's elapsed keeps the (pure) decision identical everywhere.""" + elapsed = time.time() - last_ckpt_time[0] + if torch.distributed.is_initialized() and world_size > 1: + t = torch.tensor([elapsed], device=device, dtype=torch.float64) + torch.distributed.broadcast(t, src=0) + elapsed = float(t.item()) + return elapsed + + def _save_mid_window(train_ts: int, batch_idx_in_window: int) -> None: + """In-window checkpoint helper. Snapshots the same state as the + end-of-window save but stamps `batch_idx_in_window=K` instead of + WINDOW_COMPLETE so the resume path knows to skip K batches. + Uses train_ts as the numeric subdir name — every save into the same + train_ts overwrites the previous in-window snapshot (via atomic + replace), so disk stays bounded to keep_last_n train_ts dirs.""" + save_dmp_checkpoint( + model=model, + optimizer=optimizer, + metric_logger=metric_logger, + rank=rank, + batch_idx=train_ts, + train_ts=train_ts, + batch_idx_in_window=batch_idx_in_window, + device=device, + ) + + def _run_train_window( + train_data_iterator, + train_ts: int, + start_batch_idx: int = 0, + label: Optional[str] = None, + ) -> None: + # `start_batch_idx` is set when we're re-entering a window that was + # interrupted mid-way (in_window resume); the dataloader iterator was + # already advanced past those batches via the sampler skip, and we + # account for them in the local counter so in-window saves and the + # die_at_step hook fire at the right relative offsets. + train_batch_idx = start_batch_idx first_wait: Optional[float] = None while True: model.train() @@ -1124,7 +1343,65 @@ def _run_train_window(train_data_iterator, label: Optional[str] = None) -> None: if output_trace: assert profiler is not None profiler.step() - if num_train_batches is not None and train_batch_idx >= num_train_batches: + # Fine-grained in-window checkpoint triggers. All stamp + # batch_idx_in_window so a crash here leaves a resumable partial + # checkpoint, and all fire AFTER the metric update so restored + # state reflects the just-completed batch. Triggers are mutually + # short-circuited (one save per batch max) but evaluated on the + # same deterministic counters across all ranks, so the collective + # inside save_dmp_checkpoint stays in lockstep. + gstep = metric_logger.global_step["train"] + # Wall-clock elapsed is broadcast from rank 0 so every rank feeds + # the same value into the (otherwise pure) cadence decision. + elapsed = ( + _broadcast_elapsed() if checkpoint_time_interval_s > 0 else 0.0 + ) + save_reason = select_in_window_checkpoint_reason( + train_batch_idx=train_batch_idx, + global_step=gstep, + elapsed_since_last_save=elapsed, + in_window_checkpoint_frequency=in_window_checkpoint_frequency, + checkpoint_step_frequency=checkpoint_step_frequency, + checkpoint_time_interval_s=checkpoint_time_interval_s, + ) + if save_reason is not None: + if rank == 0: + logger.info( + "checkpoint trigger=%s train_ts=%d batch=%d global_step=%d", + save_reason, + train_ts, + train_batch_idx, + gstep, + ) + _save_mid_window(train_ts, train_batch_idx) + # Reset the wall-clock anchor on ANY save so the next time + # trigger is measured from the most recent checkpoint. + last_ckpt_time[0] = time.time() + # Test-only: deterministic crash for the failure-injection test. + # Triggered AFTER the save above, so on resume we re-enter at + # batch_idx_in_window=train_batch_idx and emit batches [K+1, end). + if ( + die_at_step >= 0 + and metric_logger.global_step["train"] >= die_at_step + ): + if rank == 0: + logger.warning( + "die_at_step=%d hit at train_ts=%d batch=%d global_step=%d " + "→ sys.exit(42)", + die_at_step, + train_ts, + train_batch_idx, + metric_logger.global_step["train"], + ) + # Distributed barrier so all ranks exit together rather than + # leaving a few ranks hanging on NCCL ops. + torch.distributed.barrier() + import sys + sys.exit(42) + # `num_train_batches` cap: None or 0 = run the whole window. >0 caps + # batches per window (mostly the streaming-resume test driver uses + # this to keep test windows short). + if num_train_batches and train_batch_idx >= num_train_batches: break if label and rank == 0 and first_wait is not None: logger.info( @@ -1188,14 +1465,24 @@ def _maybe_checkpoint(train_ts: int) -> None: if ( train_ts % checkpoint_frequency == 0 and train_ts > 0 ) or train_ts == start_ts + num_train_ts - 1: + # End-of-window save: stamp WINDOW_COMPLETE so resume advances past + # this train_ts. `device` enables per-rank RNG snapshot for + # bit-equal resume of dropout-bearing modules. save_dmp_checkpoint( model=model, optimizer=optimizer, metric_logger=metric_logger, rank=rank, batch_idx=train_ts, + train_ts=train_ts, + batch_idx_in_window=WINDOW_COMPLETE, + device=device, ) + last_ckpt_time[0] = time.time() + # Apply start_ts shift from resume (may have moved past the original start). + # num_train_ts is the requested *count*; preserve it so the loop runs for + # the same total number of windows post-resume as a fresh run would have. train_ts_list = list(range(start_ts, start_ts + num_train_ts)) if persistent_loader and double_buffer: # Double-buffered: next window prepared in the background during the @@ -1229,10 +1516,27 @@ def _maybe_checkpoint(train_ts: int) -> None: eval_iter = iter(eval_dl) n_train = len(train_ts_list) for i, (train_ts, train_data_iterator) in enumerate( - prefetcher.stream(train_ts_list) + # Only the FIRST window after a mid-window resume needs the skip + # (handed via prefetcher.stream's first_skip_samples). The skip is + # zero on cold start (resume_train_ts is None → first_skip_samples=0) + # and on completed-window resume (mid-window slice is 0 too). + prefetcher.stream(train_ts_list, first_skip_samples=first_skip_samples) ): dataset.dataset.is_eval = False # pyre-ignore [16] - _run_train_window(train_data_iterator, label=f"train_ts={train_ts}") + # First iteration after a mid-window resume carries + # resume_batch_idx_in_window so in-window saves and the die_at_step + # hook keep accurate counters; otherwise count from 0. + start_batch = ( + resume_batch_idx_in_window + if i == 0 and resume_batch_idx_in_window > 0 + else 0 + ) + _run_train_window( + train_data_iterator, + train_ts=train_ts, + start_batch_idx=start_batch, + label=f"train_ts={train_ts}", + ) if eval_each_window: dataset.dataset.is_eval = True # pyre-ignore [16] assert eval_sampler is not None and eval_dl is not None @@ -1246,9 +1550,19 @@ def _maybe_checkpoint(train_ts: int) -> None: eval_iter = iter(eval_dl) _maybe_checkpoint(train_ts) else: - for train_ts in train_ts_list: + for i, train_ts in enumerate(train_ts_list): dataset.dataset.is_eval = False # pyre-ignore [16] - _run_train_window(_window_iter(train_ts)) + skip = first_skip_samples if i == 0 else 0 + start_batch = ( + resume_batch_idx_in_window + if i == 0 and resume_batch_idx_in_window > 0 + else 0 + ) + _run_train_window( + _window_iter(train_ts, skip_samples=skip), + train_ts=train_ts, + start_batch_idx=start_batch, + ) if eval_each_window: dataset.dataset.is_eval = True # pyre-ignore [16] _run_eval_window(_window_iter(train_ts + 1)) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py index 5a1e7fb9d..51e35d90e 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py @@ -1031,6 +1031,21 @@ def env_int(key: str = "", default: int = 0) -> int: return int(raw) if raw else default +@gin.configurable +def env_float(key: str = "", default: float = 0.0) -> float: + """Resolve a float from os.environ[key], falling back to `default`. + + Companion to `env_int` for fractional/duration overrides (e.g. a + checkpoint time interval in seconds). Example gin usage: + + streaming_train_eval_loop.checkpoint_time_interval_s = @env_float() + env_float.key = "CKPT_TIME_INTERVAL_S" + env_float.default = 3600.0 + """ + raw = os.environ.get(key) if key else None + return float(raw) if raw else default + + @gin.configurable def run_results_dir(run_name: str = "default", subdir: str = "results") -> str: """Resolve ``//`` from this file's location. diff --git a/recommendation_v4/scripts/streaming_resume_test.sh b/recommendation_v4/scripts/streaming_resume_test.sh new file mode 100755 index 000000000..043d5b926 --- /dev/null +++ b/recommendation_v4/scripts/streaming_resume_test.sh @@ -0,0 +1,242 @@ +#!/bin/bash +# End-to-end failure-injection + resume test for streaming-train-eval. +# +# Validates exact-once mid-window resume on the yambda-5b stack: +# Phase 1 (baseline): uninterrupted run for N=2 train_ts × K batches/window +# Phase 2 (interrupted): same config but die_at_step=M → exits at step M +# after the in-window checkpoint lands +# Phase 3 (resume): re-launch with same CKPT_PATH → auto-latest picks +# the in-window save → finishes the partial window +# and the rest of the requested train_ts list +# Assertion: traj_resumed[step].window_ne / window_auc / window_accuracy match +# traj_baseline bit-equal (np.allclose atol=1e-4) for all step > die_at_step. +# +# Driven entirely via env-driven gin knobs defined in yambda_5b.gin: +# NUM_TRAIN_TS / NUM_TRAIN_BATCHES / IN_WINDOW_CKPT_FREQ / DIE_AT_STEP / +# CKPT_PATH / KEEP_LAST_N / EVAL_EACH_WINDOW +# +# Usage: +# bash scripts/streaming_resume_test.sh --jobid +# [--container yambda_primus] +# [--num-train-batches 200] +# [--die-at-step 350] +# [--keep] # retain LOG_DIR + CKPT after run for inspection + +set -uo pipefail + +JOBID="" +CONTAINER="yambda_primus" +NUM_TRAIN_BATCHES=200 +DIE_AT_STEP=350 +IN_WINDOW_FREQ=50 +KEEP=0 +# Trajectory closeness bound — NOT a bit-equality check. The ROCm training stack +# is nondeterministic across runs (non-deterministic atomic scatter-add in the +# embedding/attention backward): two independent *cold* runs already drift +# ~7e-4 in window_ne over 20 steps, and early-training chaos (AUC~0.5) amplifies +# any seed difference. So resume-vs-baseline can legitimately differ by a few +# percent. This bound just catches GROSS divergence (wrong data skip, totally +# unrestored state) while tolerating nondeterministic drift. The HARD resume +# correctness gates are the functional-invariant checks below (RNG restored, +# resumed-at-correct-step, atomic/keep_last_n), not this number. +ATOL=0.15 +CKPT_ROOT=/apps/chcai/ckpts_resume_test +LOG_DIR=/apps/chcai/streaming_resume_test +REPO=/home/chcai/training/recommendation_v4 + +while [[ $# -gt 0 ]]; do + case $1 in + --jobid) JOBID="$2"; shift 2;; + --container) CONTAINER="$2"; shift 2;; + --num-train-batches) NUM_TRAIN_BATCHES="$2"; shift 2;; + --die-at-step) DIE_AT_STEP="$2"; shift 2;; + --in-window-freq) IN_WINDOW_FREQ="$2"; shift 2;; + --atol) ATOL="$2"; shift 2;; + --keep) KEEP=1; shift;; + *) echo "Unknown arg: $1"; exit 1;; + esac +done +[[ -z "$JOBID" ]] && { echo "Error: --jobid required"; exit 1; } + +mkdir -p "$LOG_DIR" + +# Single-window mid-window resume: NUM_TRAIN_TS=1, so the whole test runs inside +# train_ts=START_TS. die_at_step must land strictly inside that window, AT a +# multiple of IN_WINDOW_FREQ so an in-window checkpoint is saved right before +# the crash (resume then skips exactly DIE_AT_STEP already-trained batches). +if (( DIE_AT_STEP <= 0 || DIE_AT_STEP >= NUM_TRAIN_BATCHES )); then + echo "Warning: die_at_step=$DIE_AT_STEP not strictly inside window (0, $NUM_TRAIN_BATCHES)" >&2 +fi +if (( DIE_AT_STEP % IN_WINDOW_FREQ != 0 )); then + echo "Warning: die_at_step=$DIE_AT_STEP not a multiple of in_window_freq=$IN_WINDOW_FREQ; no save lands exactly at crash" >&2 +fi + +cleanup_workers() { + srun --jobid="$JOBID" --overlap docker exec "$CONTAINER" bash -lc \ + "pkill -9 -f generative_recommenders 2>/dev/null; sleep 2; \ + pkill -9 -f spawn_main 2>/dev/null; sleep 3; true" 2>/dev/null || true +} +clean_ckpt() { + srun --jobid="$JOBID" --overlap docker exec "$CONTAINER" rm -rf "$CKPT_ROOT" 2>/dev/null || true +} + +# Wait for a log line to appear OR a crash sentinel. Returns 0 if target found, +# 1 if crash sentinel found first. +wait_for_log() { + local log="$1"; local target_re="$2"; local timeout_s="${3:-1500}" + local elapsed=0 + while (( elapsed < timeout_s )); do + if grep -qE "$target_re" "$log" 2>/dev/null; then + return 0 + fi + if grep -qE "Traceback|RuntimeError|OutOfMemoryError" "$log" 2>/dev/null; then + return 1 + fi + sleep 5 + elapsed=$((elapsed + 5)) + done + return 2 +} + +# Single train window of NUM_TRAIN_BATCHES steps → last train step == NUM_TRAIN_BATCHES. +LAST_STEP=$NUM_TRAIN_BATCHES + +run_phase() { + local name="$1"; shift + local log="$LOG_DIR/${name}.log" + # Join the per-phase env overrides into ONE word. Using `$*` (not `$@`) is + # essential: `$@` embedded mid-string in the double-quoted `bash -lc "..."` + # expands to *multiple* arguments, so bash -lc would only run up to the + # first override and treat the rest as positional params — launch_smoke + # would never execute (silent 0-byte log). + local env_overrides="$*" + : > "$log" + echo "[$(date)] === phase '$name' ===" + cleanup_workers + srun --jobid="$JOBID" --overlap docker exec -d "$CONTAINER" bash -lc " + cd $REPO && + HSTU_HAMMER_KERNEL=TRITON \ + $env_overrides \ + RUN_NAME=resume_test_$name \ + LOG=$log \ + bash scripts/launch_smoke_8gpu.sh + " +} + +# === Phase 1: baseline === +clean_ckpt +run_phase baseline \ + "NUM_TRAIN_TS=1" \ + "EVAL_EACH_WINDOW=0" \ + "METRIC_LOG_FREQ=1" \ + "NUM_TRAIN_BATCHES=$NUM_TRAIN_BATCHES" \ + "DIE_AT_STEP=-1" +wait_for_log "$LOG_DIR/baseline.log" "train - Step $LAST_STEP metrics" 1500 +rc=$? +cleanup_workers +[[ $rc -ne 0 ]] && { echo "FAIL: baseline didn't finish"; tail -20 "$LOG_DIR/baseline.log"; exit 1; } + +# === Phase 2: interrupted === +clean_ckpt +run_phase interrupt \ + "NUM_TRAIN_TS=1" \ + "EVAL_EACH_WINDOW=0" \ + "METRIC_LOG_FREQ=1" \ + "NUM_TRAIN_BATCHES=$NUM_TRAIN_BATCHES" \ + "IN_WINDOW_CKPT_FREQ=$IN_WINDOW_FREQ" \ + "KEEP_LAST_N=1" \ + "DIE_AT_STEP=$DIE_AT_STEP" \ + "CKPT_PATH=$CKPT_ROOT" +wait_for_log "$LOG_DIR/interrupt.log" "die_at_step=$DIE_AT_STEP hit" 1500 +rc=$? +cleanup_workers +[[ $rc -ne 0 ]] && { echo "FAIL: interrupt didn't hit die_at_step"; tail -20 "$LOG_DIR/interrupt.log"; exit 1; } + +SAVED=$(srun --jobid="$JOBID" --overlap docker exec "$CONTAINER" ls "$CKPT_ROOT" 2>/dev/null | tr '\n' ' ') +echo "Saved checkpoints after interrupt: $SAVED" + +# === Phase 3: resume === +run_phase resume \ + "NUM_TRAIN_TS=1" \ + "EVAL_EACH_WINDOW=0" \ + "METRIC_LOG_FREQ=1" \ + "NUM_TRAIN_BATCHES=$NUM_TRAIN_BATCHES" \ + "IN_WINDOW_CKPT_FREQ=$IN_WINDOW_FREQ" \ + "KEEP_LAST_N=1" \ + "DIE_AT_STEP=-1" \ + "CKPT_PATH=$CKPT_ROOT" +wait_for_log "$LOG_DIR/resume.log" "train - Step $LAST_STEP metrics" 1500 +rc=$? +[[ $rc -ne 0 ]] && { cleanup_workers; echo "FAIL: resume didn't finish"; tail -20 "$LOG_DIR/resume.log"; exit 1; } +# The resume run performs an end-of-window checkpoint save AFTER the final +# step's metric line. That save (hundreds of GB) writes .tmp and then +# atomically renames it onto , logging "checkpoint successfully saved" only +# once the rename completes. If we kill workers right after the step line we'd +# orphan a half-written .tmp and trip the stale-dir gate below — a harness +# race, not a resume bug. Wait for the save to finish before tearing down. +wait_for_log "$LOG_DIR/resume.log" "checkpoint successfully saved" 1500 +save_rc=$? +cleanup_workers +[[ $save_rc -ne 0 ]] && { echo "FAIL: resume end-of-window checkpoint save did not complete"; tail -20 "$LOG_DIR/resume.log"; exit 1; } + +# === HARD resume-correctness gates (functional invariants) === +# These — not the trajectory closeness check below — are the authoritative +# proof the resume path is correct, because they're deterministic and immune +# to the GPU nondeterminism that perturbs the metric trajectory. + +# (1) Re-entered the partial window at exactly the saved batch_idx_in_window. +if ! grep -qE "Resuming mid-window at train_ts=[0-9]+ batch_idx_in_window=$DIE_AT_STEP\b" "$LOG_DIR/resume.log" 2>/dev/null; then + echo "FAIL: resume did not re-enter mid-window at batch_idx_in_window=$DIE_AT_STEP" + grep -E "Resuming" "$LOG_DIR/resume.log" 2>/dev/null | head -2 + exit 1 +fi +# (2) Per-rank RNG state was actually restored (dropout determinism path). +RNG_RESTORED=$(grep -c "RNG state restored from" "$LOG_DIR/resume.log" 2>/dev/null || echo 0) +echo "RNG state restored on $RNG_RESTORED ranks" +[[ "$RNG_RESTORED" -lt 1 ]] && { echo "FAIL: no RNG state restored on resume"; exit 1; } +# (3) The FIRST training step after resume is exactly die_at_step+1, i.e. the +# skip-already-trained-batches logic emitted the next unseen batch (not a +# restart from step 1, and not a gap). +FIRST_RESUMED=$(grep -oE 'train - Step [0-9]+ metrics: \{.metric' "$LOG_DIR/resume.log" 2>/dev/null \ + | grep -oE 'Step [0-9]+' | awk '{print $2}' | sort -n | head -1) +echo "First resumed train step: $FIRST_RESUMED (expect $((DIE_AT_STEP + 1)))" +[[ "$FIRST_RESUMED" != "$((DIE_AT_STEP + 1))" ]] && { + echo "FAIL: resume did not continue at step $((DIE_AT_STEP + 1)) (got $FIRST_RESUMED)"; exit 1; } + +# === Final on-disk state checks (atomic save + retention) === +NUM_CKPT=$(srun --jobid="$JOBID" --overlap docker exec "$CONTAINER" bash -lc \ + "ls $CKPT_ROOT 2>/dev/null | grep -E '^[0-9]+$' | wc -l" | tr -d ' ') +# Both .tmp (interrupted write) and .old (interrupted atomic-overwrite swap) +# must be absent — their presence means a save crashed without clean recovery. +STALE_CKPT=$(srun --jobid="$JOBID" --overlap docker exec "$CONTAINER" bash -lc \ + "ls $CKPT_ROOT 2>/dev/null | grep -E '\\.(tmp|old)$' | wc -l" | tr -d ' ') +echo "Final: $NUM_CKPT numeric ckpt subdirs, $STALE_CKPT stale (.tmp/.old) dirs (expect 1, 0)" +[[ "$NUM_CKPT" != "1" ]] && { echo "FAIL: keep_last_n=1 violated"; exit 1; } +[[ "$STALE_CKPT" != "0" ]] && { echo "FAIL: stale .tmp/.old dirs left behind"; exit 1; } +echo "=== Resume functional invariants: ALL PASS ===" + +# === Trajectory closeness (sanity bound, NOT bit-equality) === +# Catches gross resume bugs (wrong data slice, unrestored model) that throw the +# metric trajectory far off. Small drift is expected & tolerated (see ATOL note +# at top). The functional invariants above are the real correctness proof. +python3 $REPO/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.py parse \ + "$LOG_DIR/baseline.log" "$LOG_DIR/traj_baseline.json" +python3 $REPO/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.py parse \ + "$LOG_DIR/resume.log" "$LOG_DIR/traj_resumed.json" + +python3 $REPO/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.py compare \ + "$LOG_DIR/traj_baseline.json" "$LOG_DIR/traj_resumed.json" \ + --min-resume-step $((DIE_AT_STEP + 1)) --atol $ATOL +RC=$? + +if [[ "$KEEP" != "1" ]]; then + rm -rf "$LOG_DIR" + clean_ckpt +fi + +if [[ $RC -eq 0 ]]; then + echo "=== PASS: resume validated (functional invariants + trajectory within $ATOL of baseline) ===" +else + echo "=== FAIL: trajectory diverged beyond $ATOL — likely a real resume bug (wrong data slice / unrestored state), not nondeterminism ===" +fi +exit $RC From 4a18b954c43604aa6dbc45276eb3871898b3bbb4 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Wed, 3 Jun 2026 23:14:59 -0500 Subject: [PATCH 031/127] dlrmv4: move streaming resume test harness into train/tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It's a test driver, not a general script — colocate the shell harness with its Python comparator under train/tests/ and fix the stale path references. Co-authored-by: Cursor --- .../generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin | 2 +- .../dlrm_v3/train/tests/streaming_resume_test.py | 2 +- .../dlrm_v3/train/tests}/streaming_resume_test.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename recommendation_v4/{scripts => generative_recommenders/dlrm_v3/train/tests}/streaming_resume_test.sh (99%) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index 747e3876f..6a1e9d762 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -204,7 +204,7 @@ ntb/env_int.key = "NUM_TRAIN_BATCHES" ntb/env_int.default = 0 # Test-only failure injection: when >=0 and metric_logger.global_step['train'] # reaches this, the process exits with code 42 right after the in-window save -# fires. Used by `scripts/streaming_resume_test.sh` to verify resume. +# fires. Used by the streaming resume test harness (train/tests/) to verify resume. streaming_train_eval_loop.die_at_step = @das/env_int() das/env_int.key = "DIE_AT_STEP" das/env_int.default = -1 diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.py index 2774e8c03..b46da936b 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.py @@ -20,7 +20,7 @@ 3. Auto-detect-latest checkpoint subdir 4. keep_last_n retention (default 1) -Test flow (driven by `scripts/streaming_resume_test.sh`): +Test flow (driven by the sibling `streaming_resume_test.sh`): Phase 1 (baseline): Run streaming-train-eval for N=2 train_ts × K batches/window with die_at_step=-1. Capture per-batch window_ne / window_auc into traj_baseline.json. Phase 2 (interrupt): Same config but die_at_step=M (M mid-window-2). Expect diff --git a/recommendation_v4/scripts/streaming_resume_test.sh b/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh similarity index 99% rename from recommendation_v4/scripts/streaming_resume_test.sh rename to recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh index 043d5b926..c3690e652 100755 --- a/recommendation_v4/scripts/streaming_resume_test.sh +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh @@ -16,7 +16,7 @@ # CKPT_PATH / KEEP_LAST_N / EVAL_EACH_WINDOW # # Usage: -# bash scripts/streaming_resume_test.sh --jobid +# bash generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh --jobid # [--container yambda_primus] # [--num-train-batches 200] # [--die-at-step 350] From f89da0e25eae73a23afd0246f25aa882c14058d1 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Thu, 4 Jun 2026 01:43:33 -0500 Subject: [PATCH 032/127] dlrmv4: sparse full-holdout eval cadence + eval-pool fork-race fix Add eval_every_n_windows (env EVAL_EVERY_N_WINDOWS, default 1 = no change) so the heavy full next-day eval window can run every Nth train window (and always the final one) instead of every window, amortizing its cost on the long run. Fix a deadlock this exposed in the double-buffer path: the persistent eval worker pool's first iter() (its only fork) must happen on the main thread BEFORE the prefetcher's background prep thread starts. Deferring that first fork into the loop (as the sparse cadence naively did) forks while the bg thread holds an allocator/GIL-released lock and hangs the run. Always pre-fork the eval pool before the loop; in-loop re-arms only reset the persistent workers (no fork) and target the next window that will actually eval. Also normalize NUM_TRAIN_BATCHES/NUM_EVAL_BATCHES <=0 to None (full window / full-holdout eval) and bind NUM_EVAL_BATCHES in gin so eval can be capped for fast validation without affecting the full run. Co-authored-by: Cursor --- .../dlrm_v3/train/gin/yambda_5b.gin | 13 ++++ .../dlrm_v3/train/utils.py | 64 +++++++++++++++---- 2 files changed, 64 insertions(+), 13 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index 6a1e9d762..fe05db74d 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -122,6 +122,13 @@ pl/env_int.default = 1 streaming_train_eval_loop.eval_each_window = @ev/env_int() ev/env_int.key = "EVAL_EACH_WINDOW" ev/env_int.default = 1 +# Full-holdout eval cadence: run eval every Nth train window (and always on the +# final window) instead of every window. 1 (default) = eval every window (no +# behavior change). Set >1 (e.g. 5 via $EVAL_EVERY_N_WINDOWS) to amortize the +# cost of consuming the full next-day eval window over several train windows. +streaming_train_eval_loop.eval_every_n_windows = @evn/env_int() +evn/env_int.key = "EVAL_EVERY_N_WINDOWS" +evn/env_int.default = 1 # Double-buffer windows: prepare the next window (index mask + first-batch # prefetch) in a background thread during the current window's compute, hiding # the per-window reset. Needs persistent_loader=1. Override via env. @@ -202,6 +209,12 @@ ctis/env_float.default = 0.0 streaming_train_eval_loop.num_train_batches = @ntb/env_int() ntb/env_int.key = "NUM_TRAIN_BATCHES" ntb/env_int.default = 0 +# Cap each eval (full-holdout) window's batch count. Unset / <=0 = consume the +# full eval window (the genuine full-holdout NE/AUC; this is what the long run +# uses). Set >0 via $NUM_EVAL_BATCHES to subsample eval for fast validation. +streaming_train_eval_loop.num_eval_batches = @neb/env_int() +neb/env_int.key = "NUM_EVAL_BATCHES" +neb/env_int.default = 0 # Test-only failure injection: when >=0 and metric_logger.global_step['train'] # reaches this, the process exits with code 42 right after the in-window save # fires. Used by the streaming resume test harness (train/tests/) to verify resume. diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py index 6d50064b6..e437aca60 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -1105,6 +1105,7 @@ def streaming_train_eval_loop( start_ts: int = 0, persistent_loader: bool = False, eval_each_window: bool = True, + eval_every_n_windows: int = 1, double_buffer: bool = False, # --- resume / mid-window-exact-once knobs --- resume_train_ts: Optional[int] = None, @@ -1156,6 +1157,14 @@ def streaming_train_eval_loop( boundary and then resume. """ profiler = Profiler(rank) if output_trace else None + # Normalize the per-window caps: <=0 (the env-binding default) means "no cap + # = consume the full window". The eval-break check below is `is not None and + # eval_batch_idx >= num_eval_batches`, so a literal 0 would (wrongly) break + # after the first batch — map it to None instead for the full-holdout eval. + if num_eval_batches is not None and num_eval_batches <= 0: + num_eval_batches = None + if num_train_batches is not None and num_train_batches <= 0: + num_train_batches = None dataset_class, kwargs = get_dataset() kwargs["embedding_config"] = embedding_table_configs dataset = HammerToTorchDataset( @@ -1484,6 +1493,21 @@ def _maybe_checkpoint(train_ts: int) -> None: # num_train_ts is the requested *count*; preserve it so the loop runs for # the same total number of windows post-resume as a fresh run would have. train_ts_list = list(range(start_ts, start_ts + num_train_ts)) + n_train = len(train_ts_list) + + def _should_eval(i: int) -> bool: + """Whether to run the full-holdout eval after training window index `i`. + + `eval_every_n_windows<=1` (default) preserves the per-window cadence. + For K>1 we eval on windows 0, K, 2K, ... and ALWAYS on the final window + so the trajectory ends with an eval point. Gated by `eval_each_window`. + """ + if not eval_each_window: + return False + if eval_every_n_windows <= 1: + return True + return i % eval_every_n_windows == 0 or i == n_train - 1 + if persistent_loader and double_buffer: # Double-buffered: next window prepared in the background during the # current window's compute. Eval (if enabled) uses its own pre-forked @@ -1498,23 +1522,30 @@ def _maybe_checkpoint(train_ts: int) -> None: eval_sampler: Optional[StreamingWindowSampler] = None eval_dl: Optional[DataLoader] = None # Eval iterator is built one window ahead: the eval pool (idle while the - # current train window runs) prefetches the eval window's first batches - # concurrently with train compute, so eval starts warm (hides the - # ~0.5s eval first-batch stall). yambda's sample content depends only on - # the sampler window, not is_eval, so prefetching during train is safe. + # current train window runs) prefetches the next eval window's first + # batches concurrently with train compute, so eval starts warm. yambda's + # sample content depends only on the sampler window, not is_eval, so + # prefetching during train is safe. eval_iter: Optional[Iterator] = None if eval_each_window and len(train_ts_list) > 0: eval_sampler = StreamingWindowSampler(rank, world_size) eval_dl = make_persistent_streaming_dataloader( dataset=dataset, sampler=eval_sampler ) - # Fork the eval pool now (main thread, before any prefetch thread) - # and kick off prefetch of the first eval window (train_ts_list[0]+1). + # CRITICAL: fork the eval worker pool HERE, on the main thread, + # BEFORE prefetcher.stream() below spins up its background prep + # thread. The pool is persistent_workers=True, so this first iter() + # is the ONLY fork; every later iter() merely resets and reuses these + # workers (no fork), so it can never deadlock against the background + # thread holding an allocator/GIL-released lock. (Deferring this + # first fork into the loop — as a sparse-eval cadence naively might — + # hangs the run.) _should_eval(0) is always True when eval is enabled + # (0 % K == 0), so the first eval window is always train_ts_list[0]+1; + # arm it now so it prefetches during the i=0 train window. eval_sampler.set_window( dataset.dataset.window_indices(train_ts_list[0] + 1) # pyre-ignore [16] ) eval_iter = iter(eval_dl) - n_train = len(train_ts_list) for i, (train_ts, train_data_iterator) in enumerate( # Only the FIRST window after a mid-window resume needs the skip # (handed via prefetcher.stream's first_skip_samples). The skip is @@ -1537,15 +1568,22 @@ def _maybe_checkpoint(train_ts: int) -> None: start_batch_idx=start_batch, label=f"train_ts={train_ts}", ) - if eval_each_window: + if _should_eval(i): dataset.dataset.is_eval = True # pyre-ignore [16] assert eval_sampler is not None and eval_dl is not None _run_eval_window(eval_iter, label=f"eval_ts={train_ts + 1}") - # Re-arm the eval pool for the next window so it prefetches - # during the upcoming train window. - if i + 1 < n_train: + # Re-arm the (already-forked) eval pool for the NEXT window that + # will eval (i+1 in dense mode, i+K in sparse mode), so it warms + # up during the upcoming train window(s). iter() reuses the + # persistent workers — no fork, safe alongside the bg thread. + next_eval_i = next( + (j for j in range(i + 1, n_train) if _should_eval(j)), None + ) + if next_eval_i is not None: eval_sampler.set_window( - dataset.dataset.window_indices(train_ts + 2) # pyre-ignore [16] + dataset.dataset.window_indices( # pyre-ignore [16] + train_ts_list[next_eval_i] + 1 + ) ) eval_iter = iter(eval_dl) _maybe_checkpoint(train_ts) @@ -1563,7 +1601,7 @@ def _maybe_checkpoint(train_ts: int) -> None: train_ts=train_ts, start_batch_idx=start_batch, ) - if eval_each_window: + if _should_eval(i): dataset.dataset.is_eval = True # pyre-ignore [16] _run_eval_window(_window_iter(train_ts + 1)) _maybe_checkpoint(train_ts) From 039f7c9c1ac1308375ee00556b660bc9d309fe87 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Thu, 4 Jun 2026 01:43:43 -0500 Subject: [PATCH 033/127] dlrmv4: self-healing streaming-e2e supervisor + NE/AUC trajectory builder run_streaming_e2e.sh: head-node supervisor that keeps a multi-day yambda-5b streaming train+eval alive across (1) trainer crash/OOM, (2) silent SIGKILL, and (3) node loss. Relaunches from the latest checkpoint each time (exact-once resume handles continuity). Node failover salloc's a fresh exclusive node on the partition, provisions the container on it, and resumes from shared NFS; allocations it creates are released on success (never the user's own --jobid). Includes disk guard + stale .tmp sweep, keep_last_n retention, an exit-sentinel + stall watchdog for crash detection, and a node-health watchdog. Heavily documented inline. build_ne_auc_trajectory.py: parse train+eval NE/AUC (+perf) from a run log and emit combined CSV/JSON plus an NE/AUC-vs-step trajectory plot. Co-authored-by: Cursor --- .../scripts/build_ne_auc_trajectory.py | 225 ++++++++++ .../scripts/run_streaming_e2e.sh | 416 ++++++++++++++++++ 2 files changed, 641 insertions(+) create mode 100644 recommendation_v4/scripts/build_ne_auc_trajectory.py create mode 100755 recommendation_v4/scripts/run_streaming_e2e.sh diff --git a/recommendation_v4/scripts/build_ne_auc_trajectory.py b/recommendation_v4/scripts/build_ne_auc_trajectory.py new file mode 100644 index 000000000..bba1bdec6 --- /dev/null +++ b/recommendation_v4/scripts/build_ne_auc_trajectory.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python3 +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Build a combined train+eval NE/AUC trajectory from a streaming-train-eval log. + +The streaming loop (generative_recommenders/dlrm_v3/train/utils.py) emits, via +MetricsLogger.compute(), one line per logged step of the form: + + INFO:utils:train - Step 201 metrics: {'metric/lifetime_ne/listen_plus': + tensor(1.0182, dtype=torch.float64), 'metric/window_ne/listen_plus': + tensor(0.9846, ...), ..., 'metric/window_auc/listen_plus': tensor(0.5912), + 'metric/lifetime_auc/listen_plus': tensor(0.5480)} + +and the analogous `eval - Step N metrics:` lines during each (full-holdout) eval +window, plus throughput lines: + + INFO:utils:train - Step 201 perf: local_sps=97.0 global_sps=776.2 + step_ms=10553.89 elapsed_sec=680.6 total_samples=205824 + +This script parses all three, for a chosen task (default listen_plus), and writes: + * /trajectory.json — {"train": {step: {...}}, "eval": {...}, "perf": [...]} + * /trajectory.csv — long-form rows (mode, step, metric, value) + * /trajectory_ne_auc.png — NE and AUC vs train step, train + eval overlaid + (skipped gracefully if matplotlib is absent) + +It is dependency-light (stdlib + optional matplotlib) so it runs anywhere the +log is readable, including the head node. + +Usage: + python3 scripts/build_ne_auc_trajectory.py LOG [--out DIR] [--task listen_plus] +""" + +import argparse +import csv +import json +import os +import re +import sys +from typing import Dict, List, Optional, Tuple + +# `train - Step 201 metrics: {...}` / `eval - Step 17 metrics: {...}` +_STEP_RE = re.compile(r"(train|eval) - Step (\d+) metrics: \{(.*)\}") +# `metric//': tensor(` — value may be int/float/sci. +_METRIC_RE = re.compile( + r"metric/([A-Za-z0-9_]+)/([A-Za-z0-9_+]+)'?\s*:\s*tensor\(\s*([-0-9.eE+]+)" +) +# `train - Step 201 perf: local_sps=97.0 global_sps=776.2 step_ms=10553.89 ` +# `elapsed_sec=680.6 total_samples=205824` +_PERF_RE = re.compile( + r"train - Step (\d+) perf: local_sps=([-0-9.eE+]+) global_sps=([-0-9.eE+]+) " + r"step_ms=([-0-9.eE+]+) elapsed_sec=([-0-9.eE+]+) total_samples=(\d+)" +) + +# Metrics we surface in the trajectory (others are still captured if present). +_KEEP = ("window_ne", "lifetime_ne", "window_auc", "lifetime_auc", + "window_accuracy", "lifetime_accuracy", "window_gauc", "lifetime_gauc") + + +def parse_log( + log_path: str, task: str +) -> Tuple[Dict[str, Dict[int, Dict[str, float]]], List[Dict[str, float]]]: + """Return ({'train': {step: {metric: val}}, 'eval': {...}}, perf_rows). + + For a given (mode, step) the LAST occurrence wins — duplicate per-rank prints + are identical, and within an eval window later steps carry more aggregation. + """ + out: Dict[str, Dict[int, Dict[str, float]]] = {"train": {}, "eval": {}} + perf: List[Dict[str, float]] = [] + with open(log_path, "r", errors="replace") as f: + for line in f: + pm = _PERF_RE.search(line) + if pm: + perf.append({ + "step": int(pm.group(1)), + "local_sps": float(pm.group(2)), + "global_sps": float(pm.group(3)), + "step_ms": float(pm.group(4)), + "elapsed_sec": float(pm.group(5)), + "total_samples": int(pm.group(6)), + }) + continue + m = _STEP_RE.search(line) + if not m: + continue + mode, step_s, body = m.group(1), m.group(2), m.group(3) + step = int(step_s) + row: Dict[str, float] = {} + for name, tname, val in _METRIC_RE.findall(body): + if tname != task: + continue + try: + row[name] = float(val) + except ValueError: + continue + if row: + out[mode][step] = row # last write wins + return out, perf + + +def write_outputs( + traj: Dict[str, Dict[int, Dict[str, float]]], + perf: List[Dict[str, float]], + out_dir: str, + task: str, +) -> None: + os.makedirs(out_dir, exist_ok=True) + + json_path = os.path.join(out_dir, "trajectory.json") + with open(json_path, "w") as f: + json.dump( + { + "task": task, + "train": {str(k): v for k, v in sorted(traj["train"].items())}, + "eval": {str(k): v for k, v in sorted(traj["eval"].items())}, + "perf": perf, + }, + f, + indent=2, + ) + + csv_path = os.path.join(out_dir, "trajectory.csv") + with open(csv_path, "w", newline="") as f: + w = csv.writer(f) + w.writerow(["mode", "step", "metric", "value"]) + for mode in ("train", "eval"): + for step in sorted(traj[mode]): + for metric, val in traj[mode][step].items(): + w.writerow([mode, step, metric, val]) + + n_train = len(traj["train"]) + n_eval = len(traj["eval"]) + print(f"Parsed {n_train} train points, {n_eval} eval points, " + f"{len(perf)} perf points (task={task}).", file=sys.stderr) + print(f"Wrote {json_path}", file=sys.stderr) + print(f"Wrote {csv_path}", file=sys.stderr) + + _maybe_plot(traj, out_dir, task) + + +def _maybe_plot( + traj: Dict[str, Dict[int, Dict[str, float]]], out_dir: str, task: str +) -> None: + try: + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + except Exception as e: # noqa: BLE001 + print(f"matplotlib unavailable ({e}); skipping plot.", file=sys.stderr) + return + + def series(mode: str, metric: str) -> Tuple[List[int], List[float]]: + steps = sorted(s for s in traj[mode] if metric in traj[mode][s]) + return steps, [traj[mode][s][metric] for s in steps] + + fig, (ax_ne, ax_auc) = plt.subplots(2, 1, figsize=(11, 9), sharex=True) + + for metric, style in (("window_ne", "-"), ("lifetime_ne", "--")): + xs, ys = series("train", metric) + if xs: + ax_ne.plot(xs, ys, style, label=f"train/{metric}", alpha=0.85) + for metric, marker in (("window_ne", "o"), ("lifetime_ne", "s")): + xs, ys = series("eval", metric) + if xs: + ax_ne.plot(xs, ys, marker, ms=4, ls="", label=f"eval/{metric}") + ax_ne.set_ylabel("NE (normalized entropy)") + ax_ne.set_title(f"yambda-5b streaming train+eval trajectory — task={task}") + ax_ne.grid(True, alpha=0.3) + ax_ne.legend(fontsize=8, ncol=2) + + for metric, style in (("window_auc", "-"), ("lifetime_auc", "--")): + xs, ys = series("train", metric) + if xs: + ax_auc.plot(xs, ys, style, label=f"train/{metric}", alpha=0.85) + for metric, marker in (("window_auc", "o"), ("lifetime_auc", "s")): + xs, ys = series("eval", metric) + if xs: + ax_auc.plot(xs, ys, marker, ms=4, ls="", label=f"eval/{metric}") + ax_auc.set_ylabel("AUC") + ax_auc.set_xlabel("train global step") + ax_auc.grid(True, alpha=0.3) + ax_auc.legend(fontsize=8, ncol=2) + + png_path = os.path.join(out_dir, "trajectory_ne_auc.png") + fig.tight_layout() + fig.savefig(png_path, dpi=120) + print(f"Wrote {png_path}", file=sys.stderr) + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("log", help="Path to the streaming train.log") + ap.add_argument("--out", default=None, + help="Output dir (default: /_trajectory)") + ap.add_argument("--task", default="listen_plus", + help="Task name to extract (default: listen_plus)") + args = ap.parse_args() + + if not os.path.exists(args.log): + print(f"Log not found: {args.log}", file=sys.stderr) + return 2 + out_dir = args.out + if out_dir is None: + stem = os.path.splitext(os.path.basename(args.log))[0] + out_dir = os.path.join(os.path.dirname(os.path.abspath(args.log)), + f"{stem}_trajectory") + + traj, perf = parse_log(args.log, args.task) + write_outputs(traj, perf, out_dir, args.task) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/recommendation_v4/scripts/run_streaming_e2e.sh b/recommendation_v4/scripts/run_streaming_e2e.sh new file mode 100755 index 000000000..64fb1274e --- /dev/null +++ b/recommendation_v4/scripts/run_streaming_e2e.sh @@ -0,0 +1,416 @@ +#!/bin/bash +# ============================================================================= +# run_streaming_e2e.sh — self-healing supervisor for the long-run yambda-5b +# streaming train+eval (NE/AUC over the full ~5B dataset) +# ============================================================================= +# +# WHAT IT DOES +# Owns a multi-day "streaming-train-eval" run and keeps it alive unattended +# across the three failure modes that actually kill long runs: +# 1. trainer process crash / OOM / nonzero exit +# 2. silent death (the whole process group gets SIGKILLed — no exit code) +# 3. the SLURM node itself going away (down / drained / job ended) +# In every case it relaunches the trainer from the latest on-disk checkpoint +# (failing over to a brand-new node for case 3) until the run finishes. +# +# WHY A RELAUNCH "JUST WORKS" (resume model) +# The training stack already implements exact-once resume: on startup it picks +# the latest numeric checkpoint subdir under $CKPT_PATH, restores model + +# optimizer + per-rank RNG, and (for mid-window in-window saves) skips the +# batches already trained in the partially-done window. So relaunching with +# the SAME --ckpt-path transparently continues from where it died — no manual +# bookkeeping here beyond pointing every attempt at the same base dir. +# +# WHERE IT RUNS / HOW IT DRIVES WORK +# This script runs on the SLURM HEAD node. The trainer runs inside a long- +# lived docker container ($CONTAINER) on the compute node held by a SLURM +# allocation ($JOBID). All control flow is `srun --jobid --overlap +# docker exec ...` into that container. The container bind-mounts shared NFS +# (/home/chcai = code, /apps/chcai = checkpoints+logs), which is what makes +# node failover possible: any node in $PARTITION sees the same code+state. +# +# MAIN LOOP (state machine, up to --max-relaunch attempts) +# for each attempt: +# ensure_ready — guarantee a healthy allocation whose container is up, +# failing over to a freshly-provisioned node if not. +# disk_guard — sweep crash-orphaned *.tmp/*.old saves; abort if the +# ckpt volume has < --min-free-gib free. +# cleanup_workers— kill any stragglers from a previous attempt. +# launch — detached `docker exec -d` of the trainer; a trailing +# echo appends an `E2E_RUN_EXIT=` sentinel to the log +# when the trainer returns (clean OR crash). +# monitor loop (every --poll-s): +# * node watchdog — if $JOBID stops being healthy mid-run, break and +# let the next attempt fail over. +# * exit sentinel — E2E_RUN_EXIT=0 => success (done); nonzero => relaunch. +# * stall watchdog — if the log stops growing AND no trainer process is +# alive for --stall-s, treat as silent death=>relaunch. +# (Long blocking saves keep the process alive, so they +# never false-trip this.) +# +# NODE FAILOVER (case 3, the --allow-failover path) +# ensure_ready -> acquire_node: `salloc --no-shell --exclusive` a fresh node on +# $PARTITION, wait for RUNNING, then provision_node runs $PROVISION_SCRIPT on +# it (docker pull + container create + dep install; ~15 min on a cold node). +# Allocations WE create are tracked and `scancel`ed (container removed first) +# on success via release_acquired; the user's original --jobid is never +# cancelled. Checkpoints on shared NFS make the resume seamless. +# +# CHECKPOINTS / DISK +# The trainer saves atomically (write to .tmp, fsync, rename to ) and +# prunes to keep_last_n newest. One checkpoint is ~560 GB; a save blocks the +# step it fires on for ~83 s (measured, no NFS contention). Cadence is driven +# by --ckpt-time-interval (time-based) and optional --in-window-freq. +# +# ARGS (all optional; defaults target the full production run) +# run shape: --jobid --container --start-ts --num-train-ts --eval-every +# ckpt: --ckpt-path --keep-last-n --ckpt-time-interval --in-window-freq +# logging: --run-name --log +# resilience: --max-relaunch --min-free-gib --stall-s +# failover: --partition --alloc-time --allow-failover --provision-script +# validation: --num-train-batches --num-eval-batches (>0 caps batches/window +# for fast tests; 0 = full window / full-holdout eval) +# test-only: --die-at-step (>=0 injects a crash at that global step) +# +# EXIT CODES +# 0 run completed (E2E_RUN_EXIT=0 — all windows + final eval done) +# 1 exhausted --max-relaunch without completing +# 3 disk guard tripped (insufficient free space) +# 4 could not secure a healthy allocation (failover failed / disabled) +# +# OUTPUTS (next to --log) +# trainer stdout/stderr + E2E_RUN_EXIT sentinels +# .supervisor.log this supervisor's own timeline +# .provision.log node-provisioning output (failover only) +# +# EXAMPLE +# nohup bash scripts/run_streaming_e2e.sh \ +# --ckpt-path /apps/chcai/ckpts/yambda_5b_e2e \ +# --run-name yambda_5b_e2e --log /apps/chcai/yambda_5b_e2e.log \ +# --start-ts 150 --num-train-ts 149 --eval-every 10 \ +# --ckpt-time-interval 7200 --keep-last-n 2 --max-relaunch 50 \ +# > /apps/chcai/yambda_5b_e2e.supervisor.console.log 2>&1 & +# ============================================================================= + +set -uo pipefail + +JOBID=11367 +CONTAINER=yambda_primus +REPO=/home/chcai/training/recommendation_v4 + +# Defaults are sized from measurement: ~560 GB/checkpoint, ~83 s/save (blocking, +# attributed to the step it fires on), ~650 ms/train step @ global batch 8192, +# ~1465 steps (~16 min) per full ~12M-anchor window, full-holdout eval +# ~6-7 min/window. A ~2h time-based checkpoint interval keeps save overhead ~1% +# while bounding crash-loss to ~2h of compute; eval every N windows +# (EVAL_EVERY_N_WINDOWS) amortizes the full-holdout eval cost. +NUM_TRAIN_TS=149 +START_TS=150 +EVAL_EVERY=5 +CKPT_TIME_INTERVAL=7200 +KEEP_LAST_N=2 +CKPT_PATH=/apps/chcai/ckpts/yambda_5b_e2e +RUN_NAME=yambda_5b_e2e +LOG=/apps/chcai/yambda_5b_e2e.log +MAX_RELAUNCH=50 +NUM_TRAIN_BATCHES=0 # 0 = full window (only capped for validation/tests) +NUM_EVAL_BATCHES=0 # 0 = full holdout eval (only capped for validation) +DIE_AT_STEP=-1 # >=0 = test-only failure injection +IN_WINDOW_FREQ=0 # >0 = also save every N batches within a window + +# --- node failover ---------------------------------------------------------- +# If the current allocation/node goes away, acquire a FRESH node, (re)provision +# the container on it, and resume — checkpoints + code live on shared NFS +# (/apps/chcai, /home/chcai), so any node in the partition can continue. +PARTITION=meta64 +ALLOC_TIME=7-00:00:00 # SLURM --time for a failover allocation +ALLOW_FAILOVER=1 # 0 = never acquire a new node +PROVISION_SCRIPT=/home/chcai/_provision_yambda_primus.sh + +# Disk guard: require at least this many GiB free on the ckpt volume before a +# (re)launch. One checkpoint is ~600 GB; with keep_last_n the existing copies +# are already counted as used, so we only need room for one new in-flight .tmp +# plus margin (~800 GiB). The volume has ~3.7 TB free. +MIN_FREE_GIB=800 +# Stall watchdog: if the log hasn't grown AND no trainer process is alive for +# this many seconds with no exit sentinel, treat it as a silent death. Comfortably +# exceeds one blocking checkpoint save (~83 s); and because a save keeps the +# trainer process alive, an in-progress save never trips the watchdog anyway. +STALL_S=1200 +POLL_S=30 + +while [[ $# -gt 0 ]]; do + case $1 in + --jobid) JOBID="$2"; shift 2;; + --container) CONTAINER="$2"; shift 2;; + --num-train-ts) NUM_TRAIN_TS="$2"; shift 2;; + --start-ts) START_TS="$2"; shift 2;; + --eval-every) EVAL_EVERY="$2"; shift 2;; + --ckpt-time-interval) CKPT_TIME_INTERVAL="$2"; shift 2;; + --keep-last-n) KEEP_LAST_N="$2"; shift 2;; + --ckpt-path) CKPT_PATH="$2"; shift 2;; + --run-name) RUN_NAME="$2"; shift 2;; + --log) LOG="$2"; shift 2;; + --max-relaunch) MAX_RELAUNCH="$2"; shift 2;; + --num-train-batches) NUM_TRAIN_BATCHES="$2"; shift 2;; + --num-eval-batches) NUM_EVAL_BATCHES="$2"; shift 2;; + --die-at-step) DIE_AT_STEP="$2"; shift 2;; + --in-window-freq) IN_WINDOW_FREQ="$2"; shift 2;; + --min-free-gib) MIN_FREE_GIB="$2"; shift 2;; + --stall-s) STALL_S="$2"; shift 2;; + --partition) PARTITION="$2"; shift 2;; + --alloc-time) ALLOC_TIME="$2"; shift 2;; + --allow-failover) ALLOW_FAILOVER="$2"; shift 2;; + --provision-script) PROVISION_SCRIPT="$2"; shift 2;; + *) echo "Unknown arg: $1"; exit 1;; + esac +done + +ORIGINAL_JOBID="$JOBID" # never scancel the user's own hold allocation +ACQUIRED_JOBIDS=() # failover allocations WE created (released on success) + +SUP_LOG="${LOG%.log}.supervisor.log" + +sup() { echo "[$(date '+%F %T')] [supervisor] $*" | tee -a "$SUP_LOG"; } + +# Run a command inside the allocation's container, capturing its stdout. +cexec() { srun --jobid="$JOBID" --overlap docker exec "$CONTAINER" bash -lc "$1" 2>/dev/null; } + +cleanup_workers() { + # The trainer spawns 8 rank processes + dataloader workers whose cmdlines + # don't all match `train_ranker`/`spawn_main`, so target them, then fall + # back to `pkill python` — safe because this container is dedicated to this + # training (only the trainer runs python here during a supervised run). + srun --jobid="$JOBID" --overlap docker exec "$CONTAINER" bash -lc \ + "pkill -9 -f train_ranker 2>/dev/null; pkill -9 -f multiprocessing 2>/dev/null; \ + sleep 2; pkill -9 python 2>/dev/null; sleep 3; true" 2>/dev/null || true +} + +# --- node-failover helpers --------------------------------------------------- + +# Healthy = the job is RUNNING and its node is not down/drained/failing. +alloc_healthy() { + local jid="$1" + [[ -z "$jid" ]] && return 1 + local st node nstate + st=$(squeue -h -j "$jid" -o '%T' 2>/dev/null | head -1) + [[ "$st" != "RUNNING" ]] && return 1 + node=$(squeue -h -j "$jid" -o '%N' 2>/dev/null | head -1) + [[ -z "$node" ]] && return 1 + nstate=$(sinfo -h -n "$node" -o '%t' 2>/dev/null | head -1) + case "$nstate" in + *down*|*drain*|*fail*|*unk*|*boot*|"") return 1;; + esac + return 0 +} + +# Can we actually exec in the training container on this allocation? +container_up() { + srun --jobid="$1" --overlap docker exec "$CONTAINER" true >/dev/null 2>&1 +} + +# (Re)create + dep-install the container on the given allocation's node. +provision_node() { + local jid="$1" node + node=$(squeue -h -j "$jid" -o '%N' 2>/dev/null | head -1) + sup "provisioning container '$CONTAINER' on job $jid (node ${node:-?}) — cold node can take ~15 min" + srun --jobid="$jid" --overlap bash "$PROVISION_SCRIPT" >> "${LOG%.log}.provision.log" 2>&1 + container_up "$jid" +} + +# Acquire a fresh exclusive node on $PARTITION; sets global JOBID on success. +acquire_node() { + if [[ "$ALLOW_FAILOVER" != "1" ]]; then + sup "failover disabled (--allow-failover 0); cannot acquire a new node"; return 1 + fi + sup "requesting a fresh node on partition=$PARTITION (exclusive, time=$ALLOC_TIME)" + local out jid + out=$(salloc --no-shell --partition="$PARTITION" --nodes=1 --exclusive \ + --time="$ALLOC_TIME" --job-name=e2e_failover 2>&1) + jid=$(echo "$out" | grep -oiE "Granted job allocation [0-9]+" | grep -oE "[0-9]+" | head -1) + if [[ -z "$jid" ]]; then + sup "FATAL: salloc did not grant a node: $out"; return 1 + fi + ACQUIRED_JOBIDS+=("$jid") + sup "granted new allocation jobid=$jid; waiting for RUNNING" + local waited=0 + while (( waited < 600 )); do + [[ "$(squeue -h -j "$jid" -o '%T' 2>/dev/null | head -1)" == "RUNNING" ]] && break + sleep 10; waited=$((waited + 10)) + done + if [[ "$(squeue -h -j "$jid" -o '%T' 2>/dev/null | head -1)" != "RUNNING" ]]; then + sup "FATAL: new allocation $jid never reached RUNNING (waited ${waited}s)"; return 1 + fi + JOBID="$jid" + sup "new node ready: jobid=$JOBID node=$(squeue -h -j "$JOBID" -o '%N' 2>/dev/null | head -1)" + return 0 +} + +# Ensure $JOBID is a healthy allocation with the container up, failing over to a +# fresh provisioned node if not. Resume is automatic: the latest checkpoint is +# on shared NFS, reachable from whatever node we end up on. +ensure_ready() { + if alloc_healthy "$JOBID"; then + if container_up "$JOBID"; then return 0; fi + sup "alloc $JOBID healthy but container '$CONTAINER' not up — (re)provisioning" + provision_node "$JOBID" && return 0 + sup "provisioning on $JOBID failed; will try a fresh node" + else + sup "current allocation $JOBID unavailable (job not RUNNING or node down/drained)" + fi + acquire_node || return 1 + provision_node "$JOBID" || { sup "provisioning new node $JOBID failed"; return 1; } + sup "failover complete — now running on jobid=$JOBID" + return 0 +} + +release_acquired() { + local jid + for jid in "${ACQUIRED_JOBIDS[@]:-}"; do + [[ -n "$jid" && "$jid" != "$ORIGINAL_JOBID" ]] || continue + # docker is independent of SLURM, so remove the container before freeing + # the node, otherwise it lingers for the next tenant. + srun --jobid="$jid" --overlap docker rm -f "$CONTAINER" >/dev/null 2>&1 || true + scancel "$jid" 2>/dev/null && sup "released failover allocation $jid (container removed)" + done +} + +# Returns 0 (true) if a trainer process is alive in the container. +trainer_alive() { + local n + n=$(cexec "pgrep -f generative_recommenders | wc -l" | tr -d ' ') + [[ "${n:-0}" -gt 0 ]] +} + +disk_guard() { + # Sweep crash-orphaned partial saves, then check free space. + cexec "for d in '$CKPT_PATH'/*.tmp '$CKPT_PATH'/*.old; do [ -e \"\$d\" ] && rm -rf \"\$d\" && echo swept \"\$d\"; done; true" + local free_gib + free_gib=$(cexec "df -BG --output=avail '$CKPT_PATH' 2>/dev/null | tail -1 | tr -dc '0-9'") + free_gib=${free_gib:-0} + sup "disk guard: ${free_gib} GiB free on $CKPT_PATH (min ${MIN_FREE_GIB})" + if (( free_gib < MIN_FREE_GIB )); then + sup "FATAL: insufficient free space (${free_gib} < ${MIN_FREE_GIB} GiB). Aborting." + return 1 + fi + return 0 +} + +launch() { + # Detached launch. The trailing echo appends a definitive exit sentinel to + # the log once the trainer returns (clean finish OR crash with nonzero rc). + srun --jobid="$JOBID" --overlap docker exec -d "$CONTAINER" bash -lc " + cd $REPO && + HSTU_HAMMER_KERNEL=TRITON \ + MODE=streaming-train-eval \ + START_TS=$START_TS \ + NUM_TRAIN_TS=$NUM_TRAIN_TS \ + EVAL_EACH_WINDOW=1 \ + EVAL_EVERY_N_WINDOWS=$EVAL_EVERY \ + CKPT_PATH=$CKPT_PATH \ + KEEP_LAST_N=$KEEP_LAST_N \ + CKPT_TIME_INTERVAL_S=$CKPT_TIME_INTERVAL \ + IN_WINDOW_CKPT_FREQ=$IN_WINDOW_FREQ \ + NUM_TRAIN_BATCHES=$NUM_TRAIN_BATCHES \ + NUM_EVAL_BATCHES=$NUM_EVAL_BATCHES \ + DIE_AT_STEP=$DIE_AT_STEP \ + METRIC_LOG_FREQ=50 \ + RUN_NAME=$RUN_NAME \ + LOG=$LOG \ + bash scripts/launch_smoke_8gpu.sh; + echo \"E2E_RUN_EXIT=\$? \$(date '+%F %T')\" >> $LOG + " +} + +# Returns the exit code from the most recent E2E_RUN_EXIT sentinel APPENDED +# since `since_marker` bytes, or empty if none yet. +last_exit_since() { + local since_line="$1" + cexec "tail -n +$since_line '$LOG' 2>/dev/null | grep -aoE 'E2E_RUN_EXIT=[0-9]+' | tail -1 | cut -d= -f2" +} + +sup "=== streaming e2e supervisor start ===" +sup "jobid=$JOBID container=$CONTAINER repo=$REPO" +sup "start_ts=$START_TS num_train_ts=$NUM_TRAIN_TS eval_every=$EVAL_EVERY" +sup "ckpt_path=$CKPT_PATH keep_last_n=$KEEP_LAST_N ckpt_time_interval=${CKPT_TIME_INTERVAL}s in_window_freq=$IN_WINDOW_FREQ" +sup "log=$LOG num_train_batches=$NUM_TRAIN_BATCHES die_at_step=$DIE_AT_STEP max_relaunch=$MAX_RELAUNCH" + +cexec "mkdir -p '$CKPT_PATH'" + +attempt=0 +while (( attempt < MAX_RELAUNCH )); do + attempt=$((attempt + 1)) + sup "--- attempt $attempt/$MAX_RELAUNCH ---" + + # Make sure we have a live, container-ready node (failover + provision if the + # current allocation/node has gone away). + if ! ensure_ready; then + sup "FATAL: could not secure a healthy allocation (failover failed)." + exit 4 + fi + if ! disk_guard; then exit 3; fi + cleanup_workers + + # Mark current end of log so we only read sentinels produced by THIS attempt. + start_line=$(cexec "wc -l < '$LOG' 2>/dev/null" | tr -d ' '); start_line=${start_line:-0} + start_line=$((start_line + 1)) + + sup "launching (reading sentinels from log line $start_line)" + launch + sleep 15 # let docker exec spin up the process + + # Monitor loop. + last_size=0 + stall_accum=0 + hb=0 + while true; do + # Node/allocation watchdog: if the node we're on goes down/drains or the + # job ends, bail out of the monitor — the next attempt's ensure_ready + # will fail over to a fresh node and resume from the latest checkpoint. + hb=$((hb + 1)) + if (( hb % 4 == 0 )) && ! alloc_healthy "$JOBID"; then + sup "allocation $JOBID lost mid-run (node down/job ended) — relaunching with failover." + break + fi + + rc=$(last_exit_since "$start_line") + if [[ -n "$rc" ]]; then + if [[ "$rc" == "0" ]]; then + sup "RUN COMPLETED CLEANLY (E2E_RUN_EXIT=0) on attempt $attempt." + cleanup_workers + final_ckpts=$(cexec "ls '$CKPT_PATH' 2>/dev/null | grep -E '^[0-9]+$' | tr '\n' ' '") + sup "final checkpoints retained: ${final_ckpts:-}" + release_acquired + sup "=== streaming e2e supervisor done (success) ===" + exit 0 + fi + sup "trainer exited nonzero (E2E_RUN_EXIT=$rc). Will relaunch from latest checkpoint." + break + fi + + # Stall watchdog: track log growth; if frozen and no trainer alive, die. + cur_size=$(cexec "wc -c < '$LOG' 2>/dev/null" | tr -d ' '); cur_size=${cur_size:-0} + if [[ "$cur_size" == "$last_size" ]]; then + if trainer_alive; then + stall_accum=0 # alive but quiet (e.g. long save / eval) — ok + else + stall_accum=$((stall_accum + POLL_S)) + if (( stall_accum >= STALL_S )); then + sup "STALL: log frozen ${stall_accum}s and no trainer alive — silent death. Relaunching." + break + fi + fi + else + stall_accum=0 + last_size=$cur_size + fi + sleep "$POLL_S" + done + + cleanup_workers + sleep $(( attempt < 5 ? 20 : 60 )) # small backoff +done + +sup "FATAL: exhausted MAX_RELAUNCH=$MAX_RELAUNCH without completion." +sup "=== streaming e2e supervisor done (failure) ===" +exit 1 From 0be8a71d7bd542800366f2b3f2ff7468c47fe999 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Thu, 4 Jun 2026 01:57:34 -0500 Subject: [PATCH 034/127] dlrmv4: durable streaming-run metrics (append log + TensorBoard on NFS) Make the full-run NE/AUC record survive relaunches and node failover: - launch_smoke_8gpu.sh now appends to $LOG (tee -a) instead of truncating, so a supervised run that relaunches many times into the same log keeps its full metrics history. The supervisor initializes the log once at run start. - run_streaming_e2e.sh: truncate $LOG once at start, create the per-run TB dir, and export TENSORBOARD_LOG_PATH=/apps/chcai/tb/$RUN_NAME/ into the launch env. - yambda_5b.gin: MetricsLogger.tensorboard_log_path now reads $TENSORBOARD_LOG_PATH (via the existing env_path helper) defaulting to /apps/chcai/tb/yambda_5b/ on shared NFS, instead of container-local /tmp (which is wiped on failover). Co-authored-by: Cursor --- .../dlrm_v3/train/gin/yambda_5b.gin | 8 +++++++- recommendation_v4/scripts/launch_smoke_8gpu.sh | 6 +++++- recommendation_v4/scripts/run_streaming_e2e.sh | 10 +++++++++- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index fe05db74d..febca9d77 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -167,7 +167,13 @@ Profiler.active = 5 Profiler.trace_dir = @run_results_dir() # logger variables -MetricsLogger.tensorboard_log_path = "/tmp/tb/yambda_5b/" +# TensorBoard event dir. Default lives on shared NFS (not container-local /tmp, +# which is wiped on node failover) so the NE/AUC scalars survive relaunches and +# failover. Override per-run via $TENSORBOARD_LOG_PATH (the supervisor sets it +# to /apps/chcai/tb/$RUN_NAME/). +MetricsLogger.tensorboard_log_path = @tbp/env_path() +tbp/env_path.key = "TENSORBOARD_LOG_PATH" +tbp/env_path.default = "/apps/chcai/tb/yambda_5b/" MetricsLogger.world_size = 8 MetricsLogger.auc_threshold = 0.80275 # Checkpointing disabled by default — a full DMP checkpoint is ~100s of GB and diff --git a/recommendation_v4/scripts/launch_smoke_8gpu.sh b/recommendation_v4/scripts/launch_smoke_8gpu.sh index 94886dc87..eaa0aa19b 100755 --- a/recommendation_v4/scripts/launch_smoke_8gpu.sh +++ b/recommendation_v4/scripts/launch_smoke_8gpu.sh @@ -8,7 +8,11 @@ REPO_ROOT=$(cd "$(dirname "$0")/.." && pwd) cd "$REPO_ROOT" LOG=${LOG:-/apps/chcai/yambda_5b_8gpu.log} -echo "[$(date)] REPO_ROOT=$REPO_ROOT" | tee "$LOG" +# Append (not truncate): under the streaming-e2e supervisor a run may relaunch +# many times into the SAME $LOG, and we want the full NE/AUC history preserved +# across attempts. The supervisor initializes ($LOG) once at run start. For a +# standalone invocation, set a fresh $LOG (or truncate it yourself) per run. +echo "[$(date)] REPO_ROOT=$REPO_ROOT" | tee -a "$LOG" # polars-u64-idx (NOT stock polars) — yambda parquet's flat-explode overruns # 32-bit row index. Reserved node has no outbound DNS, so we install from a diff --git a/recommendation_v4/scripts/run_streaming_e2e.sh b/recommendation_v4/scripts/run_streaming_e2e.sh index 64fb1274e..1d5953328 100755 --- a/recommendation_v4/scripts/run_streaming_e2e.sh +++ b/recommendation_v4/scripts/run_streaming_e2e.sh @@ -316,6 +316,7 @@ launch() { DIE_AT_STEP=$DIE_AT_STEP \ METRIC_LOG_FREQ=50 \ RUN_NAME=$RUN_NAME \ + TENSORBOARD_LOG_PATH=/apps/chcai/tb/$RUN_NAME/ \ LOG=$LOG \ bash scripts/launch_smoke_8gpu.sh; echo \"E2E_RUN_EXIT=\$? \$(date '+%F %T')\" >> $LOG @@ -335,7 +336,14 @@ sup "start_ts=$START_TS num_train_ts=$NUM_TRAIN_TS eval_every=$EVAL_EVERY" sup "ckpt_path=$CKPT_PATH keep_last_n=$KEEP_LAST_N ckpt_time_interval=${CKPT_TIME_INTERVAL}s in_window_freq=$IN_WINDOW_FREQ" sup "log=$LOG num_train_batches=$NUM_TRAIN_BATCHES die_at_step=$DIE_AT_STEP max_relaunch=$MAX_RELAUNCH" -cexec "mkdir -p '$CKPT_PATH'" +cexec "mkdir -p '$CKPT_PATH' '/apps/chcai/tb/$RUN_NAME'" +# Initialize this run's metrics log ONCE. launch_smoke_8gpu.sh appends (tee -a), +# so every relaunch attempt accumulates into this single file — the full-run +# NE/AUC history survives crashes and node failover instead of being truncated +# on each relaunch. (Starting the supervisor = starting a fresh run.) +cexec ": > '$LOG'" +sup "metrics log initialized (relaunch-append): $LOG" +sup "tensorboard (NFS): /apps/chcai/tb/$RUN_NAME/" attempt=0 while (( attempt < MAX_RELAUNCH )); do From da8a9ebbe71c6faf6696f6c67a75b505a1136a4b Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Thu, 4 Jun 2026 03:10:37 -0500 Subject: [PATCH 035/127] dlrmv4: raise streaming-run disk guard for keep_last_n=1 saves A checkpoint save writes a fresh ~560 GB .tmp before the old copy is pruned, so peak transient usage is (keep_last_n + 1) copies (~1120 GB at keep_last_n=1). Bump MIN_FREE_GIB 800 -> 1200 so a (re)launch never wedges mid-save on a near-full shared NFS volume. Co-authored-by: Cursor --- recommendation_v4/scripts/run_streaming_e2e.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/recommendation_v4/scripts/run_streaming_e2e.sh b/recommendation_v4/scripts/run_streaming_e2e.sh index 1d5953328..4a0a0c7f0 100755 --- a/recommendation_v4/scripts/run_streaming_e2e.sh +++ b/recommendation_v4/scripts/run_streaming_e2e.sh @@ -128,10 +128,11 @@ ALLOW_FAILOVER=1 # 0 = never acquire a new node PROVISION_SCRIPT=/home/chcai/_provision_yambda_primus.sh # Disk guard: require at least this many GiB free on the ckpt volume before a -# (re)launch. One checkpoint is ~600 GB; with keep_last_n the existing copies -# are already counted as used, so we only need room for one new in-flight .tmp -# plus margin (~800 GiB). The volume has ~3.7 TB free. -MIN_FREE_GIB=800 +# (re)launch. One checkpoint is ~560 GB. A save writes a fresh .tmp BEFORE the +# old copy is pruned, so peak transient usage is (keep_last_n + 1) copies. With +# keep_last_n=1 that is ~1120 GB; require ~1200 GiB free at launch so the run +# never wedges mid-save on a near-full shared NFS volume. +MIN_FREE_GIB=1200 # Stall watchdog: if the log hasn't grown AND no trainer process is alive for # this many seconds with no exit sentinel, treat it as a silent death. Comfortably # exceeds one blocking checkpoint save (~83 s); and because a save keeps the From 3c896c927e8a30b29b3031c61e375032f8e2175c Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Thu, 4 Jun 2026 13:30:19 -0500 Subject: [PATCH 036/127] dlrmv4: anchor eval points to train global step in NE/AUC trajectory Eval used a per-window-resetting internal step counter, so eval markers bunched at the left of the x-axis instead of overlaying the train curve. Parse the log sequentially, collapse each eval window to its final full-holdout metrics, and anchor it to the train global step it ran at (tagging eval_ts via the [boundary] marker, which can interleave around the eval's metric lines). Eval points now overlay train on a shared axis. Co-authored-by: Cursor --- .../scripts/build_ne_auc_trajectory.py | 92 ++++++++++++++++--- 1 file changed, 77 insertions(+), 15 deletions(-) diff --git a/recommendation_v4/scripts/build_ne_auc_trajectory.py b/recommendation_v4/scripts/build_ne_auc_trajectory.py index bba1bdec6..edc57cfbf 100644 --- a/recommendation_v4/scripts/build_ne_auc_trajectory.py +++ b/recommendation_v4/scripts/build_ne_auc_trajectory.py @@ -62,22 +62,63 @@ r"train - Step (\d+) perf: local_sps=([-0-9.eE+]+) global_sps=([-0-9.eE+]+) " r"step_ms=([-0-9.eE+]+) elapsed_sec=([-0-9.eE+]+) total_samples=(\d+)" ) +# `[boundary] eval_ts=181 eval first-batch ...` — marks the start of a full-holdout +# eval block; the eval runs at whatever the latest train global step was, so we use +# it to anchor each eval's metrics onto the shared train-global-step x-axis. +_EVAL_BOUNDARY_RE = re.compile(r"\[boundary\] eval_ts=(\d+) eval first-batch") # Metrics we surface in the trajectory (others are still captured if present). _KEEP = ("window_ne", "lifetime_ne", "window_auc", "lifetime_auc", "window_accuracy", "lifetime_accuracy", "window_gauc", "lifetime_gauc") +def _parse_metrics(body: str, task: str) -> Dict[str, float]: + row: Dict[str, float] = {} + for name, tname, val in _METRIC_RE.findall(body): + if tname != task: + continue + try: + row[name] = float(val) + except ValueError: + continue + return row + + def parse_log( log_path: str, task: str ) -> Tuple[Dict[str, Dict[int, Dict[str, float]]], List[Dict[str, float]]]: """Return ({'train': {step: {metric: val}}, 'eval': {...}}, perf_rows). - For a given (mode, step) the LAST occurrence wins — duplicate per-rank prints - are identical, and within an eval window later steps carry more aggregation. + Train is keyed by train global step (last write wins — duplicate per-rank + prints are identical). Eval uses a per-rank-resetting internal step counter + that restarts every eval window, so we instead anchor each eval window onto + the *train global step at which it ran* (the loop trains window T then evals + window T+1, so the eval's anchor is the last train step before it). Each eval + window collapses to a single point carrying its final, most-aggregated + full-holdout metrics, plus `eval_window` (the eval_ts) for reference. """ out: Dict[str, Dict[int, Dict[str, float]]] = {"train": {}, "eval": {}} perf: List[Dict[str, float]] = [] + + last_train_step = 0 + cur_anchor: Optional[int] = None # train global step this eval block runs at + cur_ts: Optional[int] = None # eval window id (eval_ts) + cur_row: Optional[Dict[str, float]] = None # final row of the current block + cur_internal: Optional[int] = None # last eval internal step (reset detection) + + def flush_eval() -> None: + nonlocal cur_anchor, cur_ts, cur_row, cur_internal + if cur_row: + anchor = cur_anchor if cur_anchor is not None else last_train_step + row = dict(cur_row) + if cur_ts is not None: + row["eval_window"] = float(cur_ts) + key = anchor + while key in out["eval"]: # keep distinct evals from colliding + key += 1 + out["eval"][key] = row + cur_anchor = cur_ts = cur_row = cur_internal = None + with open(log_path, "r", errors="replace") as f: for line in f: pm = _PERF_RE.search(line) @@ -91,21 +132,40 @@ def parse_log( "total_samples": int(pm.group(6)), }) continue + bm = _EVAL_BOUNDARY_RE.search(line) + if bm: + # The boundary line (a different logger) can interleave before OR + # after this eval's metric lines, so don't use it to delimit the + # block — just tag the current block with its eval_ts. Block + # boundaries come from eval-step resets / training resuming. + if cur_anchor is None: + cur_anchor = last_train_step + cur_ts = int(bm.group(1)) + continue m = _STEP_RE.search(line) if not m: continue mode, step_s, body = m.group(1), m.group(2), m.group(3) step = int(step_s) - row: Dict[str, float] = {} - for name, tname, val in _METRIC_RE.findall(body): - if tname != task: - continue - try: - row[name] = float(val) - except ValueError: - continue - if row: - out[mode][step] = row # last write wins + row = _parse_metrics(body, task) + if mode == "train": + last_train_step = step + if cur_anchor is not None or cur_row is not None: + flush_eval() # an eval block ends when training resumes + if row: + out["train"][step] = row # last write wins + else: # eval — accumulate into the current block (last = most aggregated) + # Fallback for logs without a boundary marker: a drop in the eval + # internal step counter signals a fresh eval window. + if (cur_internal is not None and step < cur_internal + and cur_anchor is None): + flush_eval() + if cur_anchor is None: + cur_anchor = last_train_step + cur_internal = step + if row: + cur_row = row + flush_eval() return out, perf @@ -173,7 +233,8 @@ def series(mode: str, metric: str) -> Tuple[List[int], List[float]]: for metric, marker in (("window_ne", "o"), ("lifetime_ne", "s")): xs, ys = series("eval", metric) if xs: - ax_ne.plot(xs, ys, marker, ms=4, ls="", label=f"eval/{metric}") + ax_ne.plot(xs, ys, marker=marker, ms=5, ls="-", lw=1.0, alpha=0.9, + label=f"eval/{metric}") ax_ne.set_ylabel("NE (normalized entropy)") ax_ne.set_title(f"yambda-5b streaming train+eval trajectory — task={task}") ax_ne.grid(True, alpha=0.3) @@ -186,9 +247,10 @@ def series(mode: str, metric: str) -> Tuple[List[int], List[float]]: for metric, marker in (("window_auc", "o"), ("lifetime_auc", "s")): xs, ys = series("eval", metric) if xs: - ax_auc.plot(xs, ys, marker, ms=4, ls="", label=f"eval/{metric}") + ax_auc.plot(xs, ys, marker=marker, ms=5, ls="-", lw=1.0, alpha=0.9, + label=f"eval/{metric}") ax_auc.set_ylabel("AUC") - ax_auc.set_xlabel("train global step") + ax_auc.set_xlabel("train global step (eval points anchored to the step they ran at)") ax_auc.grid(True, alpha=0.3) ax_auc.legend(fontsize=8, ncol=2) From 04dc53a0cfaa2dac61d9eaea0f257299e7c5faf5 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Thu, 4 Jun 2026 15:14:16 -0500 Subject: [PATCH 037/127] dlrmv4: supervisor tolerates control-plane outages + attach mode A SLURM controller outage made the supervisor's squeue/sinfo health check report "node lost", trigger failover, and then FATAL-exit when salloc couldn't reach the controller either - abandoning a run whose trainer was in fact still alive (slurmctld outages don't kill RUNNING jobs). Harden it: - controller_up()/wait_for_controller(): treat an unreachable controller as transient; wait for recovery (up to --ctrl-wait-max) instead of failing over. - Direct-SSH fallback (dexec via cached LAST_NODE) so trainer_alive() and the mid-run watchdog verify liveness even while the controller is down; only fail over if the trainer is genuinely gone, not on a control-plane blip. - timeout-guard all srun/squeue/sinfo calls so a hung control plane / NFS can't wedge the supervisor. - --attach mode: adopt an already-running trainer (one that outlived a killed supervisor) without truncating its log, sweeping its in-flight .tmp, killing it, or relaunching - just resume monitoring in place. Co-authored-by: Cursor --- .../scripts/run_streaming_e2e.sh | 134 +++++++++++++++--- 1 file changed, 115 insertions(+), 19 deletions(-) diff --git a/recommendation_v4/scripts/run_streaming_e2e.sh b/recommendation_v4/scripts/run_streaming_e2e.sh index 4a0a0c7f0..40dc5fe81 100755 --- a/recommendation_v4/scripts/run_streaming_e2e.sh +++ b/recommendation_v4/scripts/run_streaming_e2e.sh @@ -98,6 +98,12 @@ JOBID=11367 CONTAINER=yambda_primus REPO=/home/chcai/training/recommendation_v4 +# Direct-SSH fallback so the supervisor can probe the node even while the SLURM +# control plane is unreachable — a transient controller outage must NOT be +# mistaken for node death (which would needlessly tear down a healthy run). +SSH_OPTS="-o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=no" +LAST_NODE="" # last known node hostname for $JOBID (cached for direct probes) + # Defaults are sized from measurement: ~560 GB/checkpoint, ~83 s/save (blocking, # attributed to the step it fires on), ~650 ms/train step @ global batch 8192, # ~1465 steps (~16 min) per full ~12M-anchor window, full-holdout eval @@ -117,6 +123,12 @@ NUM_TRAIN_BATCHES=0 # 0 = full window (only capped for validation/tests) NUM_EVAL_BATCHES=0 # 0 = full holdout eval (only capped for validation) DIE_AT_STEP=-1 # >=0 = test-only failure injection IN_WINDOW_FREQ=0 # >0 = also save every N batches within a window +ATTACH=0 # 1 = (re)attach to an already-running trainer without + # killing it or truncating its log — used to restore + # supervision over a trainer that outlived a previous + # supervisor (e.g. one a control-plane outage killed). +CTRL_WAIT_MAX=3600 # max seconds to wait for an unreachable SLURM controller + # to recover before concluding failover is needed. # --- node failover ---------------------------------------------------------- # If the current allocation/node goes away, acquire a FRESH node, (re)provision @@ -157,6 +169,8 @@ while [[ $# -gt 0 ]]; do --num-eval-batches) NUM_EVAL_BATCHES="$2"; shift 2;; --die-at-step) DIE_AT_STEP="$2"; shift 2;; --in-window-freq) IN_WINDOW_FREQ="$2"; shift 2;; + --attach) ATTACH="$2"; shift 2;; + --ctrl-wait-max) CTRL_WAIT_MAX="$2"; shift 2;; --min-free-gib) MIN_FREE_GIB="$2"; shift 2;; --stall-s) STALL_S="$2"; shift 2;; --partition) PARTITION="$2"; shift 2;; @@ -174,8 +188,46 @@ SUP_LOG="${LOG%.log}.supervisor.log" sup() { echo "[$(date '+%F %T')] [supervisor] $*" | tee -a "$SUP_LOG"; } -# Run a command inside the allocation's container, capturing its stdout. -cexec() { srun --jobid="$JOBID" --overlap docker exec "$CONTAINER" bash -lc "$1" 2>/dev/null; } +# Run a command inside the allocation's container, capturing its stdout. Wrapped +# in `timeout` so a hung control plane / NFS can never wedge the supervisor. +cexec() { timeout 90 srun --jobid="$JOBID" --overlap docker exec "$CONTAINER" bash -lc "$1" 2>/dev/null; } + +# Is the SLURM control plane reachable right now? +controller_up() { timeout 12 sinfo -h -o '%P' >/dev/null 2>&1; } + +# Refresh + echo the node hostname for $JOBID (cached in LAST_NODE for direct +# probes that must work even while the controller is down). +refresh_node() { + local n; n=$(timeout 12 squeue -h -j "$JOBID" -o '%N' 2>/dev/null | head -1) + [[ -n "$n" ]] && LAST_NODE="$n" + echo "$LAST_NODE" +} + +# Run a (simple) command in the container by SSHing the node DIRECTLY, bypassing +# SLURM — the only way to observe the trainer during a controller outage. Needs a +# previously-cached LAST_NODE. Keep "$1" free of embedded double quotes. +dexec() { + [[ -z "$LAST_NODE" ]] && return 1 + timeout 40 ssh $SSH_OPTS "$LAST_NODE" "docker exec $CONTAINER bash -lc '$1'" 2>/dev/null +} + +# Block (with backoff) until the controller is reachable again, up to +# CTRL_WAIT_MAX. A controller outage leaves RUNNING jobs running, so waiting it +# out is almost always preferable to abandoning a healthy node. +wait_for_controller() { + local waited=0 + controller_up && return 0 + while ! controller_up; do + if (( waited >= CTRL_WAIT_MAX )); then + sup "controller still unreachable after ${waited}s (max ${CTRL_WAIT_MAX}s) — proceeding." + return 1 + fi + sup "SLURM controller unreachable; waiting for recovery (${waited}s/${CTRL_WAIT_MAX}s)…" + sleep 30; waited=$((waited + 30)) + done + sup "SLURM controller reachable again after ${waited}s." + return 0 +} cleanup_workers() { # The trainer spawns 8 rank processes + dataloader workers whose cmdlines @@ -207,7 +259,7 @@ alloc_healthy() { # Can we actually exec in the training container on this allocation? container_up() { - srun --jobid="$1" --overlap docker exec "$CONTAINER" true >/dev/null 2>&1 + timeout 30 srun --jobid="$1" --overlap docker exec "$CONTAINER" true >/dev/null 2>&1 } # (Re)create + dep-install the container on the given allocation's node. @@ -251,7 +303,11 @@ acquire_node() { # fresh provisioned node if not. Resume is automatic: the latest checkpoint is # on shared NFS, reachable from whatever node we end up on. ensure_ready() { + # A controller outage leaves RUNNING jobs running; wait it out before deciding + # anything is wrong, so we never abandon a healthy node over a transient blip. + wait_for_controller || true if alloc_healthy "$JOBID"; then + refresh_node >/dev/null if container_up "$JOBID"; then return 0; fi sup "alloc $JOBID healthy but container '$CONTAINER' not up — (re)provisioning" provision_node "$JOBID" && return 0 @@ -276,10 +332,16 @@ release_acquired() { done } -# Returns 0 (true) if a trainer process is alive in the container. +# Returns 0 (true) if a trainer process is alive in the container. Uses SLURM +# (srun) when the controller is up, else falls back to a direct SSH probe so a +# control-plane outage can't make a live trainer look dead. trainer_alive() { local n - n=$(cexec "pgrep -f generative_recommenders | wc -l" | tr -d ' ') + if controller_up; then + n=$(cexec "pgrep -f generative_recommenders | wc -l" | tr -d ' ') + else + n=$(dexec "pgrep -f generative_recommenders | wc -l" | tr -d ' ') + fi [[ "${n:-0}" -gt 0 ]] } @@ -341,9 +403,14 @@ cexec "mkdir -p '$CKPT_PATH' '/apps/chcai/tb/$RUN_NAME'" # Initialize this run's metrics log ONCE. launch_smoke_8gpu.sh appends (tee -a), # so every relaunch attempt accumulates into this single file — the full-run # NE/AUC history survives crashes and node failover instead of being truncated -# on each relaunch. (Starting the supervisor = starting a fresh run.) -cexec ": > '$LOG'" -sup "metrics log initialized (relaunch-append): $LOG" +# on each relaunch. (Starting the supervisor = starting a fresh run.) In ATTACH +# mode we are adopting an already-running trainer, so we KEEP its existing log. +if [[ "$ATTACH" == "1" ]]; then + sup "ATTACH mode: adopting existing run — keeping metrics log intact: $LOG" +else + cexec ": > '$LOG'" + sup "metrics log initialized (relaunch-append): $LOG" +fi sup "tensorboard (NFS): /apps/chcai/tb/$RUN_NAME/" attempt=0 @@ -357,16 +424,33 @@ while (( attempt < MAX_RELAUNCH )); do sup "FATAL: could not secure a healthy allocation (failover failed)." exit 4 fi - if ! disk_guard; then exit 3; fi - cleanup_workers - - # Mark current end of log so we only read sentinels produced by THIS attempt. - start_line=$(cexec "wc -l < '$LOG' 2>/dev/null" | tr -d ' '); start_line=${start_line:-0} - start_line=$((start_line + 1)) + refresh_node >/dev/null # cache LAST_NODE for direct probes during outages + + # ATTACH (first attempt only): if a trainer is already running for this run, + # adopt it in place — DON'T disk-guard (its sweep would delete an in-flight + # .tmp save), DON'T cleanup_workers (would kill it), DON'T launch. Just begin + # monitoring. Any subsequent relaunch is a normal launch from the checkpoint. + adopt=0 + if [[ "$ATTACH" == "1" ]] && trainer_alive; then + adopt=1; ATTACH=0 + sup "ATTACH mode: trainer already alive on ${LAST_NODE:-node} — monitoring in place (no relaunch/kill/sweep)." + fi - sup "launching (reading sentinels from log line $start_line)" - launch - sleep 15 # let docker exec spin up the process + if (( adopt )); then + # Mark current end of log so we only read sentinels produced from here on. + start_line=$(cexec "wc -l < '$LOG' 2>/dev/null" | tr -d ' '); start_line=${start_line:-0} + start_line=$((start_line + 1)) + sup "monitoring adopted run (reading sentinels from log line $start_line)" + else + if ! disk_guard; then exit 3; fi + cleanup_workers + # Mark current end of log so we only read sentinels produced by THIS attempt. + start_line=$(cexec "wc -l < '$LOG' 2>/dev/null" | tr -d ' '); start_line=${start_line:-0} + start_line=$((start_line + 1)) + sup "launching (reading sentinels from log line $start_line)" + launch + sleep 15 # let docker exec spin up the process + fi # Monitor loop. last_size=0 @@ -378,8 +462,20 @@ while (( attempt < MAX_RELAUNCH )); do # will fail over to a fresh node and resume from the latest checkpoint. hb=$((hb + 1)) if (( hb % 4 == 0 )) && ! alloc_healthy "$JOBID"; then - sup "allocation $JOBID lost mid-run (node down/job ended) — relaunching with failover." - break + if ! controller_up; then + # Control plane unreachable != node down. If the trainer is still + # alive on the node (direct SSH probe), this is a transient blip — + # keep monitoring rather than tearing down a healthy run. + if trainer_alive; then + sup "control plane unreachable but trainer still alive on ${LAST_NODE:-node} — transient; continuing to monitor." + else + sup "control plane unreachable AND trainer absent on ${LAST_NODE:-node} — relaunching with failover." + break + fi + else + sup "allocation $JOBID lost mid-run (node down/job ended) — relaunching with failover." + break + fi fi rc=$(last_exit_since "$start_line") From 45e0daf4b44e8943afb1a9d047154fa1857c348a Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Thu, 4 Jun 2026 23:31:37 -0500 Subject: [PATCH 038/127] dlrmv4: TFLOPS/MFU/HFU reporting + MAX_SEQ_LEN/HISTORY_LENGTH gin knobs Adds per-step TFLOPS, MFU and HFU to the training perf line + TensorBoard scalars, plus env-driven gin knobs for sequence-length sweeps. The two changes ship together because TFLOPS reporting is what makes the 4k vs 2k comparison interpretable (a sps drop alone doesn't say whether the GPU is doing more work per sample or less work overall). TFLOPS reporting HSTU is structurally different from a standard transformer: the UVQK projection fuses Q/K/V/U into one matmul, and SiLU(U)*y elementwise gating replaces the FFN block. So a TorchTitan-style "count matmuls with factor 6, attention with factor 12" template gives the wrong per-sample FLOPs unless it's coded against HSTU's specific shapes. DlrmHSTU.get_num_flops_per_sample() implements the HSTU-specific dense formula (UVQK projection + Q.K^T + att.V + output projection, per layer, times n_layers). Multitask head adds a trivial constant. Embedding lookups excluded because they're memory-bound and would otherwise pollute MFU. This is the dense yardstick: what the FLOPs would be if every sample's UIH filled max_seq_len. It's the standard MFU denominator (matches Primus-DLRM's OneTrans accounting style). Yambda's per-user history is jagged, so the actual GPU work is significantly less than the dense estimate. main_forward stashes _last_jagged_flops_per_sample after computing each batch's mean(s) and mean(s^2), and MetricsLogger reads + .item()s it once per metric_log_frequency (one D->H sync per logging interval, not per step). When present, the perf line splits into: tflops_algo/gpu mfu - dense yardstick, MFU denominator tflops_real/gpu hfu - actual jagged work, hardware utilization fill - real / algo, padding-skipped fraction When the jagged stash is absent (other model types, or before the first main_forward), only tflops_algo/mfu print. When the model doesn't expose get_num_flops_per_sample at all, the perf line is byte-for-byte unchanged (backward compatible). get_gpu_peak_flops("bf16"/"fp32") consults a per-GPU peak table (MI355X/MI350X=2300 TF, MI300X/MI325X=1300, B200=2250, H100=990, A100=312 for bf16) and warns + defaults to MI350X for unknown device names. train_ranker pulls "bf16" when bf16_training=True else "fp32"; the dtype string drives only the denominator, not anything else. TensorBoard scalars added alongside the existing perf/* group: perf/train_tflops_algo_gpu, perf/train_mfu_pct, perf/train_tflops_real_gpu, perf/train_hfu_pct, perf/train_fill_pct. Validated on 8x MI350X yambda-5b at the 2k baseline: 241.6 GFLOP/sample (dense), GPU peak 2300 TFLOPS steady-state ~13500 sps -> mfu 17.7-17.9%, hfu 9.7-10.2%, fill 55-57% (yambda users average ~1170 events vs 2046 max). HSTU's MFU is higher than the OneTrans baseline on the same GPU (Primus-DLRM hit 5.7%) because HSTU does less compute per token, so the FLOPs it does execute run at higher hardware utilization. MAX_SEQ_LEN / HISTORY_LENGTH env knobs Adds two env-driven gin macros so sequence-length sweeps don't require editing yambda_5b.gin (which a running e2e job has parsed -- editing the file would change behavior on a supervisor restart): get_hstu_configs.max_seq_len = @msl/env_int() default 2048 get_dataset.history_length = @hl/env_int() default 2039 Defaults are the current production values, so unset env is a no-op. Used the 4k validation run with MAX_SEQ_LEN=4096 HISTORY_LENGTH=4096 (reuses hstu_cache_L4096/ on disk; ~8 events of trailing UIH truncation per sample, negligible). Co-Authored-By: Claude Opus 4.7 --- .../dlrm_v3/train/gin/yambda_5b.gin | 17 +++- .../dlrm_v3/train/train_ranker.py | 21 ++++- .../generative_recommenders/dlrm_v3/utils.py | 81 ++++++++++++++++ .../modules/dlrm_hstu.py | 92 +++++++++++++++++++ 4 files changed, 206 insertions(+), 5 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index febca9d77..19c23e5b6 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -75,12 +75,21 @@ get_dataset.new_path_prefix = %DATA_PATH # average (not 679) — TRITON's jagged attention skips the unfilled slots, # so the under-fill costs sequence budget but not GPU compute. # Cache is keyed by L on disk under /hstu_cache_L/; -# switching L reuses an existing cache or builds a new one (~5 min). -get_dataset.history_length = 2039 +# switching L reuses an existing cache or builds a new one (~5 min). Override +# via $HISTORY_LENGTH (default 2039 keeps the existing single-task cache hot). +get_dataset.history_length = @hl/env_int() +hl/env_int.key = "HISTORY_LENGTH" +hl/env_int.default = 2039 # Model-side attention budget. Dataset truncates UIH to fit this value if -# `history_length + contextual + candidate` would overflow. -get_hstu_configs.max_seq_len = 2048 +# `history_length + contextual + candidate` would overflow. Override via +# $MAX_SEQ_LEN (default 2048 preserves the production single-task shape). +# Pair MAX_SEQ_LEN=4096 with HISTORY_LENGTH=4086 for the 4k-no-truncation +# analog (3*1362+9=4095 ≤ 4096); pair with HISTORY_LENGTH=4096 to reuse the +# existing hstu_cache_L4096/ cache with ~8 events of trailing truncation. +get_hstu_configs.max_seq_len = @msl/env_int() +msl/env_int.key = "MAX_SEQ_LEN" +msl/env_int.default = 2048 # --- streaming (temporal-order) training ------------------------------------- # Only consumed under `--mode streaming-train-eval`; the default train-eval diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py index 19490b840..59520bd16 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py @@ -81,7 +81,10 @@ def _main_func( train_eval_loop, train_loop, ) - from generative_recommenders.dlrm_v3.utils import MetricsLogger + from generative_recommenders.dlrm_v3.utils import ( + MetricsLogger, + get_gpu_peak_flops, + ) setup( rank=rank, @@ -104,12 +107,28 @@ def _main_func( hstu_config=model_configs, embedding_table_configs=embedding_table_configs, ) + # TFLOPS/MFU reporting: query the model's static dense estimate + + # current GPU's peak FLOPS. Both default to 0 if the model doesn't + # expose get_num_flops_per_sample, in which case MetricsLogger silently + # drops the tflops fields from the perf line. + inner_model = model.module if hasattr(model, "module") else model + num_flops_per_sample = ( + float(inner_model.get_num_flops_per_sample()) + if hasattr(inner_model, "get_num_flops_per_sample") + else 0.0 + ) + gpu_peak_flops = get_gpu_peak_flops( + "bf16" if getattr(model_configs, "bf16_training", True) else "fp32" + ) metrics = MetricsLogger( multitask_configs=model_configs.multitask_configs, batch_size=train_dataloader.batch_size, window_size=2500, device=device, rank=rank, + num_flops_per_sample=num_flops_per_sample, + gpu_peak_flops=gpu_peak_flops, + model=model, ) # Capture streaming resume hint (None for cold start / non-streaming # checkpoints). For the streaming-train-eval mode, we forward this into diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py index 51e35d90e..be94af667 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py @@ -693,7 +693,22 @@ def __init__( tensorboard_log_path: str = "", world_size: int = 1, auc_threshold: Optional[float] = None, + num_flops_per_sample: float = 0.0, + gpu_peak_flops: float = 0.0, + model: Optional[torch.nn.Module] = None, ) -> None: + # tflops/mfu reporting state (optional — when both num_flops_per_sample + # and gpu_peak_flops are set, the train perf line gains tflops_algo/gpu, + # mfu, tflops_real/gpu, hfu, fill. The jagged ("real") numbers come + # from `model._last_jagged_flops_per_sample` stashed by DlrmHSTU.main_forward. + self._num_flops_per_sample: float = max(0.0, float(num_flops_per_sample)) + self._gpu_peak_flops: float = max(0.0, float(gpu_peak_flops)) + self._model_ref: Optional[torch.nn.Module] = model + if rank == 0 and self._num_flops_per_sample > 0 and self._gpu_peak_flops > 0: + logger.info( + f"FLOPS reporting enabled: {self._num_flops_per_sample / 1e9:.1f} " + f"GFLOP/sample (dense fwd+bwd), GPU peak {self._gpu_peak_flops / 1e12:.0f} TFLOPS" + ) self.multitask_configs: List[TaskConfig] = multitask_configs all_classification_tasks: List[str] = [ task.task_name @@ -940,10 +955,44 @@ def compute_and_log( self.tb_logger.add_scalar( "perf/train_elapsed_sec", elapsed, global_step=step ) + # TFLOPS / MFU reporting (algo = dense yardstick, real = jagged). + # tflops_algo/gpu, mfu — uses max_seq_len^2 attention work (the + # MFU yardstick: the FLOPs the workload would do if every + # user's UIH filled the padded seq length). + # tflops_real/gpu, hfu — uses this batch's mean(s_i^2) (actual + # GPU work; hardware utilization). + # fill — real / algo as a percent; how much of + # the algo budget the model actually executed this batch. + # The jagged stash is read from the inner model; the model ref may + # be a DMP wrapper, so unwrap via .module if present. + tflops_str = "" + if self._num_flops_per_sample > 0 and self._gpu_peak_flops > 0: + local_flops = self._num_flops_per_sample * local_sps + tflops_algo = local_flops / 1e12 + mfu = 100.0 * local_flops / self._gpu_peak_flops + self.tb_logger.add_scalar("perf/train_tflops_algo_gpu", tflops_algo, global_step=step) + self.tb_logger.add_scalar("perf/train_mfu_pct", mfu, global_step=step) + tflops_str = f" tflops_algo/gpu={tflops_algo:.1f} mfu={mfu:.1f}%" + jagged_t = None + m = self._model_ref + if m is not None: + inner = m.module if hasattr(m, "module") else m + jagged_t = getattr(inner, "_last_jagged_flops_per_sample", None) + if jagged_t is not None: + jagged = float(jagged_t.item()) + if 0 < jagged < self._num_flops_per_sample: + tflops_real = jagged * local_sps / 1e12 + hfu = 100.0 * jagged * local_sps / self._gpu_peak_flops + fill = 100.0 * jagged / self._num_flops_per_sample + self.tb_logger.add_scalar("perf/train_tflops_real_gpu", tflops_real, global_step=step) + self.tb_logger.add_scalar("perf/train_hfu_pct", hfu, global_step=step) + self.tb_logger.add_scalar("perf/train_fill_pct", fill, global_step=step) + tflops_str += f" tflops_real/gpu={tflops_real:.1f} hfu={hfu:.1f}% fill={fill:.1f}%" logger.info( f"train - Step {step} perf: local_sps={local_sps:.1f} " f"global_sps={global_sps:.1f} step_ms={step_ms:.2f} " f"elapsed_sec={elapsed:.1f} total_samples={self._perf_total_samples}" + + tflops_str ) self._perf_t_window = now self._perf_steps_in_window = 0 @@ -1046,6 +1095,38 @@ def env_float(key: str = "", default: float = 0.0) -> float: return float(raw) if raw else default +_GPU_PEAK_FLOPS_TABLE: Dict[str, Dict[str, float]] = { + # Per-GPU peak TFLOPS by dtype. Values from vendor datasheets / Primus-DLRM + # peak_table. Used as the denominator in MFU/HFU. Keyed by case-insensitive + # substring of torch.cuda.get_device_name(0). + "MI355X": {"bf16": 2300e12, "fp32": 575e12}, + "MI350X": {"bf16": 2300e12, "fp32": 575e12}, + "MI300X": {"bf16": 1300e12, "fp32": 653e12}, + "MI325X": {"bf16": 1300e12, "fp32": 653e12}, + "B200": {"bf16": 2250e12, "fp32": 1125e12}, + "H100": {"bf16": 990e12, "fp32": 67e12}, + "A100": {"bf16": 312e12, "fp32": 19.5e12}, +} + + +def get_gpu_peak_flops(dtype: str = "bf16") -> float: + """Peak FLOPS for the current GPU at the given dtype. + + Falls back to MI350X's number with a warning when the device name doesn't + match any table entry — better to over-report MFU than to silently skip. + """ + if not torch.cuda.is_available(): + return 0.0 + name = torch.cuda.get_device_name(0) + for gpu_key, peaks in _GPU_PEAK_FLOPS_TABLE.items(): + if gpu_key in name: + return peaks.get(dtype, peaks["bf16"]) + logger.warning( + f"Unknown GPU for peak FLOPS: {name}; defaulting to MI350X bf16 (2300 TF)" + ) + return _GPU_PEAK_FLOPS_TABLE["MI350X"]["bf16"] + + @gin.configurable def run_results_dir(run_name: str = "default", subdir: str = "results") -> str: """Resolve ``//`` from this file's location. diff --git a/recommendation_v4/generative_recommenders/modules/dlrm_hstu.py b/recommendation_v4/generative_recommenders/modules/dlrm_hstu.py index 3a35df3ba..f11bb226a 100644 --- a/recommendation_v4/generative_recommenders/modules/dlrm_hstu.py +++ b/recommendation_v4/generative_recommenders/modules/dlrm_hstu.py @@ -148,6 +148,11 @@ def __init__( # noqa C901 self._pipeline_mode: bool = False self._hstu_configs = hstu_configs self._bf16_training: bool = bf16_training + # Last batch's jagged FLOPs/sample (0-d tensor on GPU). Populated by + # main_forward; MetricsLogger reads + .item()s on each compute_and_log + # to compute tflops_real/gpu and hfu (vs dense yardstick from + # get_num_flops_per_sample()). + self._last_jagged_flops_per_sample: Optional[torch.Tensor] = None set_static_max_seq_lens([self._hstu_configs.max_seq_len]) if not is_dense: @@ -284,6 +289,81 @@ def __init__( # noqa C901 LayerNorm(hstu_configs.hstu_transducer_embedding_dim), ).apply(init_mlp_weights_optional_bias) + # -- FLOPs estimation ----------------------------------------------------- + # Convention matches TorchTitan / Primus-DLRM: matmul = 6 × M × N × K + # (×3 fwd+bwd, ×2 FMA), attention = 2 matmuls (Q·K^T + att·V). + # Embedding lookups excluded — they're memory-bound, not compute. + # + # HSTU vs OneTrans: HSTU collapses attention + FFN into a single UVQK + # projection plus SiLU(U) ⊙ y elementwise gating. There is NO separate + # FFN block (which dominates FLOPs in a standard transformer), so HSTU + # is intentionally compute-leaner per layer for the same N. + def _hstu_layer_flops( + self, n_tokens_linear: float, n_tokens_attn_sq: float + ) -> float: + """Per-layer FLOPs given linear-op token count and attention-token² + count. Dense estimate uses ``N`` and ``N²``; jagged estimate + substitutes ``mean(s_i)`` and ``mean(s_i²)``.""" + cfg = self._hstu_configs + D = cfg.hstu_embedding_table_dim + H = cfg.hstu_num_heads + hd = cfg.hstu_attn_linear_dim # V/U head dim + qd = cfg.hstu_attn_qk_dim # Q/K head dim + uvqk = 6 * n_tokens_linear * D * (2 * hd + 2 * qd) * H + attn = 6 * n_tokens_attn_sq * H * (qd + hd) # Q·K^T + att·V + out = 6 * n_tokens_linear * (3 * H * hd) * D + return uvqk + attn + out + + def get_num_flops_per_sample(self) -> float: + """Dense-equivalent fwd+bwd FLOPs per sample at ``max_seq_len``. + + Used as the MFU yardstick (peak utilization the workload could + theoretically reach if every sample's sequence were the full padded + length). The actual ``tflops_real``/``hfu`` reported per step uses + the jagged estimate stashed by ``main_forward``. + """ + cfg = self._hstu_configs + N = float(cfg.max_seq_len) + n_layers = cfg.hstu_attn_num_layers + flops = n_layers * self._hstu_layer_flops( + n_tokens_linear=N, n_tokens_attn_sq=N * N + ) + # Multitask head (Linear(D, n_tasks)) — negligible but cheap to add. + n_tasks = len(self._multitask_configs) + if n_tasks > 0: + flops += 6 * n_tasks * cfg.hstu_embedding_table_dim + return float(flops) + + def _compute_jagged_flops_per_sample( + self, + uih_seq_lengths: torch.Tensor, + num_candidates: torch.Tensor, + ) -> torch.Tensor: + """Jagged fwd+bwd FLOPs per sample for THIS batch's actual lengths. + + Per-sample merged sequence length s_i = uih_seq_lengths[i] + + num_candidates[i]. Returns a 0-d tensor on the batch's device; + caller should ``.item()`` it (one D→H sync per logging interval). + """ + s = (uih_seq_lengths + num_candidates).float() + mean_s = s.mean() + mean_s_sq = (s * s).mean() + cfg = self._hstu_configs + n_layers = cfg.hstu_attn_num_layers + flops = n_layers * ( + 6 * mean_s * cfg.hstu_embedding_table_dim + * (2 * cfg.hstu_attn_linear_dim + 2 * cfg.hstu_attn_qk_dim) + * cfg.hstu_num_heads + + 6 * mean_s_sq * cfg.hstu_num_heads + * (cfg.hstu_attn_qk_dim + cfg.hstu_attn_linear_dim) + + 6 * mean_s * (3 * cfg.hstu_num_heads * cfg.hstu_attn_linear_dim) + * cfg.hstu_embedding_table_dim + ) + n_tasks = len(self._multitask_configs) + if n_tasks > 0: + flops = flops + 6 * n_tasks * cfg.hstu_embedding_table_dim + return flops + def _construct_payload( self, payload_features: Dict[str, torch.Tensor], @@ -542,6 +622,18 @@ def main_forward( Optional[torch.Tensor], Optional[torch.Tensor], ]: + # Stash this batch's jagged FLOPs/sample for MetricsLogger to read. + # No D->H sync: the .item() happens once per metric_log_frequency in + # the trainer, not on every step. Eval-mode batches also produce a + # stash but the trainer only consumes it on train batches. + if not torch.jit.is_scripting(): + self._last_jagged_flops_per_sample = ( + self._compute_jagged_flops_per_sample( + uih_seq_lengths=uih_seq_lengths, + num_candidates=num_candidates, + ) + ) + # merge uih and candidates embeddings for ( uih_feature_name, From e794c0a1703850937a040f5eca6c18c2f2cf61df Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Mon, 8 Jun 2026 20:52:18 -0500 Subject: [PATCH 039/127] dlrmv4: configurable lifetime-AUC backend + fixed-holdout streaming eval Add a dual-set streaming eval (fresh per-pass "window_*" + cumulative "lifetime_*") for NE/Accuracy/GAUC/AUC, with a gin-selectable lifetime-AUC backend for both train and eval: "binned" (BinnedCumulativeAUC, exact cumulative AUC from an O(bins) histogram, default) or "capped" (LifetimeAUCMetricComputation, trailing per-rank buffer). Backend, bins, and window are MetricsLogger gin bindings (env-overridable). Persist per-rank cumulative metric state in metricbuf_rank{rank}.pt for both backends across train/eval/eval_cum; keep per-rank state out of the shared rank-0 blob (strip capped buffers, zero binned histograms) so a resume never inherits rank-0's counts. Eval set is a stable user-hash holdout over a fixed window range, validated against a checkpoint split contract on resume. Co-authored-by: Cursor --- .../dlrm_v3/checkpoint.py | 227 ++++++++- .../dlrm_v3/datasets/synthetic_streaming.py | 5 +- .../dlrm_v3/datasets/yambda.py | 109 ++++- .../dlrm_v3/tests/test_lifetime_auc_resume.py | 146 ++++++ .../dlrm_v3/train/gin/yambda_5b.gin | 54 ++- .../dlrm_v3/train/train_ranker.py | 24 +- .../dlrm_v3/train/utils.py | 278 ++++++++--- .../generative_recommenders/dlrm_v3/utils.py | 430 ++++++++++++++---- .../scripts/run_streaming_e2e.sh | 14 +- 9 files changed, 1106 insertions(+), 181 deletions(-) create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/tests/test_lifetime_auc_resume.py diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/checkpoint.py b/recommendation_v4/generative_recommenders/dlrm_v3/checkpoint.py index aa3c9daa0..0ef223b23 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/checkpoint.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/checkpoint.py @@ -31,7 +31,11 @@ import gin import numpy as np import torch -from generative_recommenders.dlrm_v3.utils import MetricsLogger +from generative_recommenders.dlrm_v3.utils import ( + BinnedCumulativeAUC, + LifetimeAUCMetricComputation, + MetricsLogger, +) from torch.distributed.checkpoint.stateful import Stateful from torch.optim.optimizer import Optimizer from torchrec.distributed.types import ShardedTensor @@ -44,6 +48,93 @@ # window at batch K. WINDOW_COMPLETE: int = -1 +# Filename (per-rank) holding the lifetime-AUC trailing buffers, mirroring the +# rng_rank{rank}.pt pattern. The buffers are per-rank-local, so a single +# rank-0 copy in non_sparse.ckpt would (wrongly) restore 1/world_size of the +# true history to every rank — hence a dedicated per-rank artifact. +METRICBUF_FILE_FMT: str = "metricbuf_rank{rank}.pt" + + +def _metric_blob_state_dict(m: torch.nn.Module) -> Dict[str, Any]: + """State dict for the shared (rank-0) non_sparse.ckpt metric blob. + + Both lifetime-AUC backends carry per-rank-local state that is persisted + authoritatively per-rank in ``metricbuf_rank{rank}.pt``; we must keep it out + of the shared blob so a rank's load doesn't inherit rank-0's counts: + + - ``LifetimeAUCMetricComputation``: drop the explicitly-serialized trailing + buffer keys (the rest of the blob keys are the parent's persistent state). + - ``BinnedCumulativeAUC``: zero the histogram buffers (they are persistent so + the keys must remain for a strict load, but the values are neutralized). + + All other metrics serialize normally. In both cases the per-rank file is + loaded afterward and is authoritative. + """ + sd = m.state_dict() + if isinstance(m, LifetimeAUCMetricComputation): + prefix = LifetimeAUCMetricComputation._LIFETIME_KEY_PREFIX + sd = {k: v for k, v in sd.items() if not k.startswith(prefix)} + elif isinstance(m, BinnedCumulativeAUC): + sd = { + k: (torch.zeros_like(v) if torch.is_tensor(v) else v) + for k, v in sd.items() + } + return sd + + +def _collect_perrank_metric_state( + metric_logger: "MetricsLogger", +) -> Dict[str, Dict[str, Any]]: + """Map "||" -> state_dict for every metric whose + cumulative state is per-rank-local and must be restored per-rank: + + - lifetime-AUC instances (`LifetimeAUCMetricComputation` trailing buffer, or + `BinnedCumulativeAUC` histograms) in class_metrics train/eval. Covers the + train lifetime AUC and, in legacy single-set eval, the eval lifetime AUC, + under either configured backend. + - the ENTIRE cumulative eval set (`eval_cum`, both class + regression) used + by the streaming dual-set eval: the lifetime-AUC backend state plus the + persistent cumulative scalar sums of NE/Accuracy/GAUC/MSE/MAE. + + Selected by structure/isinstance (not a hard index) since metric positions + depend on the configured tasks/mode. + """ + out: Dict[str, Dict[str, Any]] = {} + for mode in ("train", "eval"): + for idx, m in enumerate(metric_logger.class_metrics.get(mode, [])): + if isinstance(m, (LifetimeAUCMetricComputation, BinnedCumulativeAUC)): + out[f"class_metrics|{mode}|{idx}"] = m.state_dict() + for coll in ("class_metrics", "regression_metrics"): + for idx, m in enumerate(getattr(metric_logger, coll).get("eval_cum", [])): + out[f"{coll}|eval_cum|{idx}"] = m.state_dict() + return out + + +def _restore_perrank_metric_state( + metric_logger: "MetricsLogger", state: Dict[str, Dict[str, Any]] +) -> None: + for key, sd in state.items(): + coll, mode, idx_str = key.split("|") + getattr(metric_logger, coll)[mode][int(idx_str)].load_state_dict(sd) + + +def _perrank_sample_counts(metric_logger: "MetricsLogger") -> Dict[str, int]: + out: Dict[str, int] = {} + + def _count(m: torch.nn.Module) -> Optional[int]: + if isinstance(m, LifetimeAUCMetricComputation): + return m.lifetime_sample_count() + if isinstance(m, BinnedCumulativeAUC): + return m.cumulative_sample_count() + return None + + for mode in ("train", "eval", "eval_cum"): + for idx, m in enumerate(metric_logger.class_metrics.get(mode, [])): + n = _count(m) + if n is not None: + out[f"class|{mode}|{idx}"] = n + return out + class SparseState(Stateful): """ @@ -218,6 +309,7 @@ def save_dmp_checkpoint( train_ts: Optional[int] = None, batch_idx_in_window: int = WINDOW_COMPLETE, device: Optional[torch.device] = None, + split_contract: Optional[Dict[str, Any]] = None, ) -> None: """ Save a distributed model checkpoint including sparse and dense components. @@ -283,8 +375,14 @@ def save_dmp_checkpoint( if not isinstance(v, ShardedTensor) } class_metric_state_dict = { - "train": [m.state_dict() for m in metric_logger.class_metrics["train"]], - "eval": [m.state_dict() for m in metric_logger.class_metrics["eval"]], + "train": [ + _metric_blob_state_dict(m) + for m in metric_logger.class_metrics["train"] + ], + "eval": [ + _metric_blob_state_dict(m) + for m in metric_logger.class_metrics["eval"] + ], } regression_metric_state_dict = { "train": [ @@ -304,6 +402,13 @@ def save_dmp_checkpoint( # (pre-streaming-resume) still load as a normal restart. "train_ts": train_ts, "batch_idx_in_window": batch_idx_in_window, + # Immutable train:eval split + resume-determinism contract + # (train_split_percentage, split_salt, eval holdout window, + # batch_size, world_size). Validated on resume so a relaunch + # cannot silently change the split (which would desync the skip + # offset and/or train on held-out eval users). None for + # non-holdout / legacy runs. + "split_contract": split_contract, }, non_sparse_ckpt, ) @@ -314,6 +419,23 @@ def save_dmp_checkpoint( rng_path = f"{tmp_subdir}/rng_rank{rank}.pt" torch.save(_rng_state(device), rng_path) + # Per-rank cumulative metric state (lifetime-AUC buffers + cumulative-eval + # histograms/scalar sums). Written by EVERY rank (outside the rank-0 block) + # because this state is per-rank-local; restoring rank-0's copy to all ranks + # would lose (world_size-1)/world_size of the history. + if metric_logger is not None: + perrank_state = _collect_perrank_metric_state(metric_logger) + if perrank_state: + torch.save( + perrank_state, + f"{tmp_subdir}/{METRICBUF_FILE_FMT.format(rank=rank)}", + ) + logger.info( + "checkpoint save: cumulative metric state rank=%d samples=%s", + rank, + _perrank_sample_counts(metric_logger), + ) + torch.distributed.barrier() sparse_dict = {"sparse_dict": SparseState(model, sparse_tensor_keys)} torch.distributed.checkpoint.save( @@ -372,7 +494,7 @@ def load_nonsparse_checkpoint( metric_logger: Optional[MetricsLogger] = None, path: str = "", rank: int = 0, -) -> Tuple[Optional[int], int]: +) -> Tuple[Optional[int], int, Optional[Dict[str, Any]]]: """ Load non-sparse (dense) components from a checkpoint. @@ -381,12 +503,13 @@ def load_nonsparse_checkpoint( next to `non_sparse.ckpt`. Returns: - (train_ts, batch_idx_in_window) — the streaming resume hint stored at - save time. `(None, WINDOW_COMPLETE)` if not a streaming checkpoint or - no path supplied. + (train_ts, batch_idx_in_window, split_contract) — the streaming resume + hint and the saved train:eval split contract (None for legacy / non- + holdout checkpoints). `(None, WINDOW_COMPLETE, None)` if not a streaming + checkpoint or no path supplied. """ if path == "": - return None, WINDOW_COMPLETE + return None, WINDOW_COMPLETE, None non_sparse_ckpt = f"{path}/non_sparse.ckpt" # weights_only=False: these are our own trusted checkpoints, and they hold @@ -404,14 +527,69 @@ def load_nonsparse_checkpoint( metric_logger.global_step = non_sparse_state_dict["global_step"] class_metric_state_dict = non_sparse_state_dict["class_metrics"] regression_metric_state_dict = non_sparse_state_dict["reg_metrics"] - for i, m in enumerate(metric_logger.class_metrics["train"]): - m.load_state_dict(class_metric_state_dict["train"][i]) - for i, m in enumerate(metric_logger.class_metrics["eval"]): - m.load_state_dict(class_metric_state_dict["eval"][i]) - for i, m in enumerate(metric_logger.regression_metrics["train"]): - m.load_state_dict(regression_metric_state_dict["train"][i]) - for i, m in enumerate(metric_logger.regression_metrics["eval"]): - m.load_state_dict(regression_metric_state_dict["eval"][i]) + # Length-safe positional restore: if a checkpoint was written with a + # different metric set (e.g. tasks added/removed since), restore the + # overlap instead of crashing with an IndexError at run end. + def _restore_metric_list( + live: list, saved: Optional[list], label: str + ) -> None: + saved = saved or [] + if len(live) != len(saved): + logger.warning( + "metric count mismatch for %s: live=%d saved=%d; " + "restoring overlapping %d", + label, + len(live), + len(saved), + min(len(live), len(saved)), + ) + for i in range(min(len(live), len(saved))): + live[i].load_state_dict(saved[i]) + + _restore_metric_list( + metric_logger.class_metrics["train"], + class_metric_state_dict.get("train"), + "class/train", + ) + _restore_metric_list( + metric_logger.class_metrics["eval"], + class_metric_state_dict.get("eval"), + "class/eval", + ) + _restore_metric_list( + metric_logger.regression_metrics["train"], + regression_metric_state_dict.get("train"), + "reg/train", + ) + _restore_metric_list( + metric_logger.regression_metrics["eval"], + regression_metric_state_dict.get("eval"), + "reg/eval", + ) + + # Per-rank cumulative metric state restore. This runs AFTER the generic + # load above so it is authoritative: the shared blob carries no lifetime + # buffers (stripped at save) nor any eval_cum state, and each rank + # restores its OWN cumulative state here. Missing file = legacy/pre-fix + # checkpoint; cumulative metrics self-heal (lifetime AUC refills; the + # binned-AUC histograms / scalar sums restart from zero). + mb_path = f"{path}/{METRICBUF_FILE_FMT.format(rank=rank)}" + if os.path.exists(mb_path): + perrank_state = torch.load( + mb_path, map_location=device, weights_only=False + ) + _restore_perrank_metric_state(metric_logger, perrank_state) + logger.info( + "checkpoint load: cumulative metric state rank=%d samples=%s", + rank, + _perrank_sample_counts(metric_logger), + ) + else: + logger.info( + "checkpoint load: no per-rank cumulative metric state at %s " + "(legacy/pre-fix checkpoint); cumulative metrics will refill", + mb_path, + ) # Per-rank RNG restore. Missing file = bit-equal trajectory not requested at # save time; we silently continue (the test harness checks for both). @@ -426,7 +604,8 @@ def load_nonsparse_checkpoint( batch_idx_in_window = non_sparse_state_dict.get( "batch_idx_in_window", WINDOW_COMPLETE ) - return train_ts, batch_idx_in_window + split_contract = non_sparse_state_dict.get("split_contract") + return train_ts, batch_idx_in_window, split_contract @gin.configurable @@ -437,7 +616,7 @@ def load_dmp_checkpoint( device: torch.device, path: str = "", rank: int = 0, -) -> Tuple[Optional[int], int]: +) -> Tuple[Optional[int], int, Optional[Dict[str, Any]], bool]: """ Load a complete distributed model checkpoint (both sparse and dense components). @@ -447,12 +626,17 @@ def load_dmp_checkpoint( no load. Returns: - (train_ts, batch_idx_in_window) — streaming resume hint. Callers that - don't need it can ignore. + (train_ts, batch_idx_in_window, split_contract, cold_start) — streaming + resume hint plus the saved split contract, and `cold_start` which is True + iff there was nothing to load (no checkpoint resolved). `cold_start` + distinguishes a genuine fresh run (no weights loaded) from a resume that + merely lacks a split contract (e.g. a legacy/non-streaming checkpoint), + which the caller's split-contract guard must still reject. """ resolved = _resolve_latest_subdir(path) + cold_start = resolved == "" load_sparse_checkpoint(model=model, path=resolved) - return load_nonsparse_checkpoint( + train_ts, batch_idx_in_window, split_contract = load_nonsparse_checkpoint( model=model, optimizer=optimizer, metric_logger=metric_logger, @@ -460,3 +644,4 @@ def load_dmp_checkpoint( device=device, rank=rank, ) + return train_ts, batch_idx_in_window, split_contract, cold_start diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/datasets/synthetic_streaming.py b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/synthetic_streaming.py index 437e5ae8e..6e38fe334 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/datasets/synthetic_streaming.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/synthetic_streaming.py @@ -269,12 +269,15 @@ def get_timestamp_uih( ) -> List[int]: return [1] * size - def set_ts(self, ts: int) -> None: + def set_ts(self, ts: int, train_only: bool = False) -> None: """ Set the current timestamp and load associated request data. Args: ts: Timestamp index to set. + train_only: Accepted for API parity with the yambda dataset (which + supports a user-level train:eval holdout). This synthetic + dataset has no holdout, so the flag is ignored. """ logger.warning(f"Streaming dataset ts set to {ts}") if ts == self.ts: diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/datasets/yambda.py b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/yambda.py index bed1aafb8..ad24513f4 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/datasets/yambda.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/yambda.py @@ -57,6 +57,28 @@ def _load_npy_readonly(path: Union[str, Path]) -> np.ndarray: arr.flags.writeable = False return arr + +def _uid_unit_hash(uids: np.ndarray, salt: int) -> np.ndarray: + """Deterministic uniform-in-[0,1) hash of user ids (splitmix64 finalizer). + + Pure function of (uid, salt): the same uid always maps to the same value, + so the train/eval user split is identical across processes, ranks, and + crash/resume — the property the no-leakage holdout relies on. Vectorized + uint64 arithmetic wraps mod 2**64 (defined for unsigned), so we silence the + benign overflow warnings. + """ + GOLDEN = np.uint64(0x9E3779B97F4A7C15) + M1 = np.uint64(0xBF58476D1CE4E5B9) + M2 = np.uint64(0x94D049BB133111EB) + s30, s27, s31 = np.uint64(30), np.uint64(27), np.uint64(31) + with np.errstate(over="ignore"): + z = uids.astype(np.uint64) + GOLDEN + np.uint64(salt & 0xFFFFFFFFFFFFFFFF) + z = (z ^ (z >> s30)) * M1 + z = (z ^ (z >> s27)) * M2 + z = z ^ (z >> s31) + # Top 53 bits -> uniform [0, 1) double (same trick numpy uses for randoms). + return (z >> np.uint64(11)).astype(np.float64) * (1.0 / 9007199254740992.0) + # Yambda event-type encoding written by preprocess_public_data.py. LISTEN_TYPE = 0 LIKE_TYPE = 1 @@ -187,6 +209,8 @@ def __init__( is_inference: bool = False, streaming_window_seconds: int = 86400, streaming_sort_within_window: bool = False, + train_split_percentage: float = 1.0, + split_salt: int = 0, *args, **kwargs, ) -> None: @@ -201,6 +225,18 @@ def __init__( # is byte-for-byte unaffected. self._streaming_window_seconds: int = streaming_window_seconds self._streaming_sort_within_window: bool = streaming_sort_within_window + # User-level train:eval split. `train_split_percentage >= 1.0` means no + # holdout (legacy behavior: every anchor is trainable). Otherwise the + # top `1 - train_split_percentage` fraction of users (by a deterministic + # hash of `uid + split_salt`) are held out: NEVER trained, used only to + # build the fixed eval set. The split is a pure function of (uid, salt), + # so it is identical across crash/resume (no leakage on failover). + self._train_split_percentage: float = train_split_percentage + self._split_salt: int = split_salt + # Cache only the (small) fixed eval-holdout index list; the per-window + # train filter is computed on the fly to avoid a full-length mask. + self._eval_holdout_cache: Optional[np.ndarray] = None + self._eval_holdout_cache_key: Optional[Tuple[int, int]] = None self._active: Optional[np.ndarray] = None self.is_eval: bool = False self._anchor_ts: Optional[np.ndarray] = None @@ -509,16 +545,79 @@ def window_indices( logger.warning(f"window_indices({ts}): [{lo}, {hi}) -> {idx.size:,} anchors") return idx.astype(np.int64) - def set_ts(self, ts: int) -> None: + def _eval_anchor_mask(self, anchor_idx: np.ndarray) -> np.ndarray: + """Bool mask (aligned to ``anchor_idx``) marking held-out eval users. + + Computed on the fly for just this slice of anchors (a window is ~tens of + millions, not the full ~3B ``_positions``), so we never materialize a + full-length mask. ``uid``-hash >= ``train_split_percentage`` -> eval. + """ + uids = self.store.flat_uid[self._positions[anchor_idx]] + return _uid_unit_hash(uids, self._split_salt) >= self._train_split_percentage + + def train_window_indices(self, ts: int) -> np.ndarray: + """Global anchor indices for TRAIN in window ``ts``: ``window_indices`` + with held-out eval users removed. Identical across resume because both + ``window_indices`` and the uid hash are pure functions, so the per-rank + round-robin slice (and the mid-window skip offset) stay consistent.""" + idx = self.window_indices(ts) + if self._train_split_percentage >= 1.0: + return idx + kept = idx[~self._eval_anchor_mask(idx)] + logger.warning( + f"train_window_indices({ts}): {idx.size:,} -> {kept.size:,} anchors " + f"(holdout tsp={self._train_split_percentage}, salt={self._split_salt})" + ) + return kept + + def eval_holdout_indices(self, start_ts: int, num_windows: int = 1) -> np.ndarray: + """Fixed eval set: held-out users' anchors over windows + ``[start_ts, start_ts + num_windows)``. Computed once and cached, so the + SAME anchors are evaluated at every eval step (stable, comparable curve). + With no holdout (tsp>=1.0) this falls back to the full window(s).""" + key = (int(start_ts), int(num_windows)) + if self._eval_holdout_cache is not None and self._eval_holdout_cache_key == key: + return self._eval_holdout_cache + parts: List[np.ndarray] = [] + for ts in range(start_ts, start_ts + max(1, num_windows)): + idx = self.window_indices(ts) + if self._train_split_percentage < 1.0: + idx = idx[self._eval_anchor_mask(idx)] + parts.append(idx) + holdout = ( + np.concatenate(parts).astype(np.int64) + if parts + else np.empty(0, dtype=np.int64) + ) + logger.warning( + f"eval_holdout_indices(start_ts={start_ts}, num_windows={num_windows}): " + f"{holdout.size:,} held-out anchors (tsp={self._train_split_percentage})" + ) + self._eval_holdout_cache = holdout + self._eval_holdout_cache_key = key + return holdout + + def set_ts(self, ts: int, train_only: bool = False) -> None: """Restrict the active sample set to anchors in window ``ts`` (used by the per-window-DataLoader path, where ``iloc``/``get_item_count`` index through ``_active``). - Forward-only temporal slicing for streaming train/eval. History for any - anchor is still gathered causally (``scan_start:flat_pos``) and may span - earlier windows, so there is no feature leakage from future events. + ``train_only=True`` removes held-out eval users so the non-persistent + TRAIN loader never sees them (closes the leakage path). Forward-only + temporal slicing for streaming train/eval. History for any anchor is + still gathered causally (``scan_start:flat_pos``) and may span earlier + windows, so there is no feature leakage from future events. """ - self._active = self.window_indices(ts) + self._active = ( + self.train_window_indices(ts) if train_only else self.window_indices(ts) + ) + + def set_active_indices(self, indices: np.ndarray) -> None: + """Restrict the active sample set to an explicit array of global anchor + indices (into ``_positions``). Used by the non-persistent eval path to + iterate the fixed user-holdout set (which spans a window range, not a + single ``ts``).""" + self._active = np.asarray(indices, dtype=np.int64) def load_query_samples(self, sample_list) -> None: max_num_candidates = ( diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/tests/test_lifetime_auc_resume.py b/recommendation_v4/generative_recommenders/dlrm_v3/tests/test_lifetime_auc_resume.py new file mode 100644 index 000000000..3d5bbd1ee --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/tests/test_lifetime_auc_resume.py @@ -0,0 +1,146 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-strict + +""" +Round-trip correctness test for ``LifetimeAUCMetricComputation`` checkpoint +serialization. + +Background: torchrec's ``AUCMetricComputation`` registers its +PREDICTIONS/LABELS/WEIGHTS buffers with ``persistent=False``, so the default +``state_dict()`` returns them empty and a separate ``_num_samples`` counter is +dropped too. Without the overrides on ``LifetimeAUCMetricComputation`` every +checkpoint resume would silently restart the lifetime AUC from an empty buffer. + +These tests assert: + 1. update -> compute == A; state_dict -> load_state_dict on a fresh metric -> + compute == A (buffers survive the round trip). + 2. ``_num_samples`` round-trips exactly (required so the next update() does + not take the init-sentinel branch and desync windowed eviction). + 3. The shared-blob path (buffers stripped) leaves a fresh metric empty, so the + per-rank artifact is the sole authority for the trailing buffer. + +Runs in <1s on CPU. Skipped automatically if torchrec is unavailable. +""" + +import unittest + +import torch + +try: + from generative_recommenders.dlrm_v3.utils import LifetimeAUCMetricComputation + + _HAVE_DEPS = True +except Exception: # pragma: no cover - import guard for envs without torchrec + _HAVE_DEPS = False + + +def _make_metric(n_tasks: int = 1, window: int = 10_000_000): + return LifetimeAUCMetricComputation( + my_rank=0, + batch_size=128, + n_tasks=n_tasks, + window_size=window, + ) + + +def _feed(metric, preds, labels, weights) -> None: + metric.update( + predictions=preds, + labels=labels, + weights=weights, + ) + + +@unittest.skipUnless(_HAVE_DEPS, "torchrec / generative_recommenders not importable") +class LifetimeAUCResumeTest(unittest.TestCase): + def setUp(self) -> None: + torch.manual_seed(0) + self.n_tasks = 1 + self.n = 4096 + self.preds = torch.rand(self.n_tasks, self.n) + self.labels = (torch.rand(self.n_tasks, self.n) > 0.5).float() + self.weights = torch.ones(self.n_tasks, self.n) + + def _compute_value(self, metric) -> float: + reports = metric._compute() + return float(reports[0].value.flatten()[0].item()) + + def test_state_dict_round_trip_preserves_auc(self) -> None: + m = _make_metric(self.n_tasks) + _feed(m, self.preds, self.labels, self.weights) + auc_a = self._compute_value(m) + n_a = m.lifetime_sample_count() + self.assertEqual(n_a, self.n) + + sd = m.state_dict() + + fresh = _make_metric(self.n_tasks) + fresh.load_state_dict(sd) + auc_b = self._compute_value(fresh) + + self.assertEqual(fresh.lifetime_sample_count(), self.n) + self.assertAlmostEqual(auc_a, auc_b, places=6) + + def test_num_samples_round_trips(self) -> None: + m = _make_metric(self.n_tasks) + _feed(m, self.preds, self.labels, self.weights) + sd = m.state_dict() + fresh = _make_metric(self.n_tasks) + fresh.load_state_dict(sd) + self.assertEqual(fresh._num_samples, m._num_samples) + + def test_continued_update_after_resume_matches_uninterrupted(self) -> None: + # Splitting a stream and resuming in the middle must equal feeding it all + # at once (this is what fails when _num_samples is not restored). + half = self.n // 2 + p1, p2 = self.preds[:, :half], self.preds[:, half:] + l1, l2 = self.labels[:, :half], self.labels[:, half:] + w1, w2 = self.weights[:, :half], self.weights[:, half:] + + ref = _make_metric(self.n_tasks) + _feed(ref, p1, l1, w1) + _feed(ref, p2, l2, w2) + auc_ref = self._compute_value(ref) + + part = _make_metric(self.n_tasks) + _feed(part, p1, l1, w1) + resumed = _make_metric(self.n_tasks) + resumed.load_state_dict(part.state_dict()) + _feed(resumed, p2, l2, w2) + auc_resumed = self._compute_value(resumed) + + self.assertAlmostEqual(auc_ref, auc_resumed, places=6) + + def test_blob_state_dict_strips_buffers(self) -> None: + from generative_recommenders.dlrm_v3.checkpoint import ( + _metric_blob_state_dict, + ) + + m = _make_metric(self.n_tasks) + _feed(m, self.preds, self.labels, self.weights) + blob = _metric_blob_state_dict(m) + prefix = LifetimeAUCMetricComputation._LIFETIME_KEY_PREFIX + self.assertFalse(any(k.startswith(prefix) for k in blob.keys())) + + # A fresh metric loaded from the stripped blob must NOT have history — + # the per-rank artifact is the only source of the trailing buffer. + fresh = _make_metric(self.n_tasks) + fresh.load_state_dict(blob) + self.assertEqual(fresh.lifetime_sample_count(), 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index 19c23e5b6..123027fdd 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -45,11 +45,21 @@ DATA_PATH = @data/env_path() data/env_path.key = "DLRM_DATA_PATH" data/env_path.default = "/apps/chcai/dlrm_data" +# Shared train:eval split: fraction of USERS used for training; the remaining +# (1 - this) fraction are held out as a fixed eval set and NEVER trained. +# Bound to BOTH the static train-eval path (make_train_test_dataloaders, a +# positional split) and the streaming path (get_dataset, an explicit by-user +# hash split), so one value configures the holdout in either mode. +# 1.0 = no holdout (legacy streaming behavior). Override via $TRAIN_SPLIT_PERCENTAGE. +TRAIN_SPLIT_PERCENTAGE = @tsp/env_float() +tsp/env_float.key = "TRAIN_SPLIT_PERCENTAGE" +tsp/env_float.default = 0.90 + # dataloader configs make_train_test_dataloaders.batch_size = %batch_size make_train_test_dataloaders.eval_batch_size = 1024 make_train_test_dataloaders.dataset_type = %dataset -make_train_test_dataloaders.train_split_percentage = 0.90 +make_train_test_dataloaders.train_split_percentage = %TRAIN_SPLIT_PERCENTAGE make_train_test_dataloaders.new_path_prefix = %DATA_PATH make_train_test_dataloaders.num_workers = %num_workers make_train_test_dataloaders.prefetch_factor = %prefetch_factor @@ -99,6 +109,15 @@ msl/env_int.default = 2048 # to the dataset's available window count at runtime; override via $NUM_TRAIN_TS. get_dataset.streaming_window_seconds = 86400 get_dataset.streaming_sort_within_window = False +# User-level train:eval holdout for the streaming path. With tsp<1.0, the top +# (1 - tsp) fraction of users (by a deterministic hash of uid+split_salt) are +# held out as a FIXED eval set and never trained -> no temporal/user leakage, +# stable comparable eval curve, bounded eval time. split_salt lets you draw a +# different holdout without changing the ratio. Override salt via $SPLIT_SALT. +get_dataset.train_split_percentage = %TRAIN_SPLIT_PERCENTAGE +get_dataset.split_salt = @ssalt/env_int() +ssalt/env_int.key = "SPLIT_SALT" +ssalt/env_int.default = 0 make_streaming_dataloader.batch_size = %batch_size make_streaming_dataloader.num_workers = %num_workers make_streaming_dataloader.prefetch_factor = %prefetch_factor @@ -144,6 +163,16 @@ evn/env_int.default = 1 streaming_train_eval_loop.double_buffer = @db/env_int() db/env_int.key = "DOUBLE_BUFFER" db/env_int.default = 1 +# Fixed eval-holdout window range (held-out users' anchors over these windows +# form the eval set evaluated at EVERY eval step). EVAL_HOLDOUT_TS<0 (default) +# resolves at runtime to start_ts+num_train_ts (the window just past training), +# which is stable across resume. EVAL_HOLDOUT_NUM_WINDOWS widens the eval span. +streaming_train_eval_loop.eval_holdout_ts = @eht/env_int() +eht/env_int.key = "EVAL_HOLDOUT_TS" +eht/env_int.default = -1 +streaming_train_eval_loop.eval_holdout_num_windows = @ehnw/env_int() +ehnw/env_int.key = "EVAL_HOLDOUT_NUM_WINDOWS" +ehnw/env_int.default = 1 # num_train_batches / num_eval_batches unset => consume each full window. # Set them (e.g. via gin) to cap per-window steps for short experiments. @@ -185,6 +214,29 @@ tbp/env_path.key = "TENSORBOARD_LOG_PATH" tbp/env_path.default = "/apps/chcai/tb/yambda_5b/" MetricsLogger.world_size = 8 MetricsLogger.auc_threshold = 0.80275 +# Lifetime-AUC backend, selectable independently for the train cumulative AUC and +# the eval cumulative ("lifetime_*") AUC. Both default to "binned": +# "binned" = BinnedCumulativeAUC: exact-cumulative AUC from an O(bins) score +# histogram (additive all-reduce, memory independent of #samples/#windows). +# "capped" = LifetimeAUCMetricComputation: AUC over a trailing buffer of +# `lifetime_auc_window` samples/rank (legacy; per-rank buffer all-gathered). +# Override per-run via $TRAIN_LIFETIME_AUC_MODE / $EVAL_LIFETIME_AUC_MODE. +MetricsLogger.train_lifetime_auc_mode = @tlam/env_str() +tlam/env_str.key = "TRAIN_LIFETIME_AUC_MODE" +tlam/env_str.default = "binned" +MetricsLogger.eval_lifetime_auc_mode = @elam/env_str() +elam/env_str.key = "EVAL_LIFETIME_AUC_MODE" +elam/env_str.default = "binned" +# Score-histogram resolution for the "binned" backend. Higher = finer AUC +# resolution at O(bins) memory. Override via $CUMULATIVE_AUC_BINS. +MetricsLogger.cumulative_auc_bins = @cab/env_int() +cab/env_int.key = "CUMULATIVE_AUC_BINS" +cab/env_int.default = 100000 +# Trailing-buffer size (samples/rank) for the "capped" backend. Override via +# $LIFETIME_AUC_WINDOW. Ignored when the backend is "binned". +MetricsLogger.lifetime_auc_window = @law/env_int() +law/env_int.key = "LIFETIME_AUC_WINDOW" +law/env_int.default = 10000000 # Checkpointing disabled by default — a full DMP checkpoint is ~100s of GB and # the streaming loop always saves on the final window. save_dmp_checkpoint # no-ops on the empty path. Set $CKPT_PATH to a directory to re-enable; the diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py index 59520bd16..6ae88eba2 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py @@ -120,6 +120,9 @@ def _main_func( gpu_peak_flops = get_gpu_peak_flops( "bf16" if getattr(model_configs, "bf16_training", True) else "fp32" ) + # Streaming fixed-holdout eval uses the dual fresh/cumulative metric sets: + # window_* = fresh per-pass full-holdout, lifetime_* = cumulative across + # passes (AUC via O(bins) histogram). Other modes keep the legacy single set. metrics = MetricsLogger( multitask_configs=model_configs.multitask_configs, batch_size=train_dataloader.batch_size, @@ -129,17 +132,24 @@ def _main_func( num_flops_per_sample=num_flops_per_sample, gpu_peak_flops=gpu_peak_flops, model=model, + eval_cumulative=(mode == "streaming-train-eval"), + # Lifetime-AUC backend + bins/window come from gin (see yambda_5b.gin: + # MetricsLogger.{train,eval}_lifetime_auc_mode / cumulative_auc_bins / + # lifetime_auc_window), env-overridable. eval_cumulative stays explicit + # because it is runtime-mode dependent, not a config knob. ) # Capture streaming resume hint (None for cold start / non-streaming # checkpoints). For the streaming-train-eval mode, we forward this into # streaming_train_eval_loop so it can advance past the last completed # window OR re-enter the partial window and skip already-trained batches. - resume_train_ts, resume_batch_idx_in_window = load_dmp_checkpoint( - model=model, - optimizer=optimizer, - metric_logger=metrics, - device=device, - rank=rank, + resume_train_ts, resume_batch_idx_in_window, resume_split_contract, resume_cold_start = ( + load_dmp_checkpoint( + model=model, + optimizer=optimizer, + metric_logger=metrics, + device=device, + rank=rank, + ) ) # train loop @@ -190,6 +200,8 @@ def _main_func( embedding_table_configs=embedding_table_configs, resume_train_ts=resume_train_ts, resume_batch_idx_in_window=resume_batch_idx_in_window, + resume_split_contract=resume_split_contract, + resume_cold_start=resume_cold_start, ) except Exception as e: logger.info(traceback.format_exc()) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py index e437aca60..936ae229e 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -418,12 +418,23 @@ def make_optimizer_and_shard( @gin.configurable def make_streaming_dataloader( dataset: HammerToTorchDataset, - ts: int, - batch_size: int, - num_workers: int, - prefetch_factor: int, + ts: Optional[int] = None, + batch_size: int = 0, + num_workers: int = 0, + prefetch_factor: int = 0, + train_only: bool = False, + indices: Optional["np.ndarray"] = None, ) -> DataLoader: - dataset.dataset.set_ts(ts) # pyre-ignore [16] + # `indices` (explicit anchor index array) is used by the eval path to + # iterate the FIXED user-holdout set, which spans a window range rather than + # a single ts. Otherwise restrict to window `ts`; train_only=True drops + # held-out eval users so the non-persistent TRAIN loader never trains on + # them (no-leakage guarantee). + if indices is not None: + dataset.dataset.set_active_indices(indices) # pyre-ignore [16] + else: + assert ts is not None, "make_streaming_dataloader needs ts or indices" + dataset.dataset.set_ts(ts, train_only=train_only) # pyre-ignore [16] total_items = dataset.dataset.get_item_count() subset = torch.utils.data.Subset(dataset, range(total_items)) # shuffle=False keeps temporal order within the window: a non-shuffling @@ -550,13 +561,14 @@ def __init__( self._iters: List[Optional[object]] = [None] * n_buffers def _prepare(self, buf: int, ts: int, skip_samples: int = 0) -> None: - # window_indices() is the O(N) mask; numpy releases the GIL for it, so it - # overlaps the main thread's GPU dispatch. iter() then kicks off this - # pool's background prefetch. - # `skip_samples` is non-zero only for the very first window after a - # mid-window resume; subsequent windows always start at 0. + # train_window_indices() is the O(N) mask (+ uid-hash filter for the + # holdout); numpy releases the GIL for it, so it overlaps the main + # thread's GPU dispatch. iter() then kicks off this pool's background + # prefetch. This is a TRAIN-only loader, so held-out eval users are + # excluded here. `skip_samples` is non-zero only for the very first + # window after a mid-window resume; subsequent windows always start at 0. self._samplers[buf].set_window( - self._dataset.dataset.window_indices(ts), skip_samples=skip_samples + self._dataset.dataset.train_window_indices(ts), skip_samples=skip_samples ) self._iters[buf] = iter(self._dls[buf]) @@ -1087,6 +1099,58 @@ def select_in_window_checkpoint_reason( return None +def _validate_split_contract( + saved: Optional[Dict[str, Any]], + live: Dict[str, Any], + rank: int, +) -> None: + """Guarantee the train:eval split (and the inputs the resume skip-offset + depends on) are unchanged across a crash/resume. + + `saved` is the contract recovered from the checkpoint (None on cold start or + legacy pre-holdout checkpoints). Any mismatch is fatal: continuing would + either desync the mid-window skip (duplicate/skip batches) or reassign users + so that previously held-out eval users get trained (leakage). Set + ALLOW_SPLIT_MISMATCH=1 to override (e.g. intentionally resuming a legacy + checkpoint into a holdout run, accepting the risk). + """ + allow = os.environ.get("ALLOW_SPLIT_MISMATCH", "0") == "1" + if saved is None: + # Legacy / cold-start checkpoint with no recorded contract. Only a + # problem if this run actually holds users out (tsp < 1.0): we cannot + # prove the earlier run used the same split. + if live.get("train_split_percentage", 1.0) < 1.0 and not allow: + raise RuntimeError( + "Resuming a checkpoint with NO saved split contract into a " + f"user-holdout run (train_split_percentage=" + f"{live['train_split_percentage']}). The earlier run's split " + "cannot be verified, so held-out eval users may have been " + "trained. Set ALLOW_SPLIT_MISMATCH=1 to override." + ) + return + mismatches = { + k: (saved.get(k), live.get(k)) + for k in live + if saved.get(k) != live.get(k) + } + if mismatches: + msg = ( + "Split/resume contract mismatch between checkpoint and current run: " + + ", ".join( + f"{k}: checkpoint={s!r} current={c!r}" for k, (s, c) in mismatches.items() + ) + + ". Resuming would desync the skip offset and/or leak held-out " + "users into training." + ) + if allow: + if rank == 0: + logger.warning("%s ALLOW_SPLIT_MISMATCH=1 set — continuing anyway.", msg) + else: + raise RuntimeError(msg + " Set ALLOW_SPLIT_MISMATCH=1 to override.") + elif rank == 0: + logger.info("Split/resume contract verified against checkpoint: %s", live) + + @gin.configurable def streaming_train_eval_loop( rank: int, @@ -1107,9 +1171,22 @@ def streaming_train_eval_loop( eval_each_window: bool = True, eval_every_n_windows: int = 1, double_buffer: bool = False, + # --- fixed user-holdout eval set --- + # Window range the fixed eval set is drawn from. None -> default to + # original_end_ts (start_ts + num_train_ts), the window just past training. + eval_holdout_ts: Optional[int] = None, + eval_holdout_num_windows: int = 1, # --- resume / mid-window-exact-once knobs --- resume_train_ts: Optional[int] = None, resume_batch_idx_in_window: int = WINDOW_COMPLETE, + # Split contract recovered from the checkpoint (None on cold start or + # legacy checkpoints). Validated below against the live split so a resumed + # run cannot silently train a different user-split (would leak). + resume_split_contract: Optional[Dict[str, Any]] = None, + # True iff no checkpoint was loaded (genuine fresh run). Distinguishes a + # cold start (safe to establish a new split) from a resume that merely lacks + # a contract (legacy/non-streaming checkpoint), which the guard must reject. + resume_cold_start: bool = False, in_window_checkpoint_frequency: int = 0, # --- global step / wall-clock checkpoint cadences --- checkpoint_step_frequency: int = 0, @@ -1187,6 +1264,47 @@ def streaming_train_eval_loop( dataset=dataset, sampler=window_sampler ) + # The fixed user-holdout eval is yambda-specific (needs window_indices + + # the split API). Other streaming datasets (synthetic) keep the legacy + # per-window eval. Detect support once. + supports_holdout = hasattr(dataset.dataset, "eval_holdout_indices") + + # Fixed eval-holdout window range. Captured from the REQUESTED (start_ts, + # num_train_ts) BEFORE the resume block mutates them, so it is identical on + # cold start and on every resume (the supervisor relaunches with the same + # START_TS / NUM_TRAIN_TS). Defaults to the window just past training. + requested_end_ts = start_ts + num_train_ts + # None (Python default) or <0 (the env-binding default) both mean "use the + # window just past training", which is stable across resume. + eval_holdout_ts_resolved = ( + eval_holdout_ts + if (eval_holdout_ts is not None and eval_holdout_ts >= 0) + else requested_end_ts + ) + + # The split is an immutable run contract: a silent change across resume + # would both desync the mid-window skip offset AND turn held-out eval users + # into trained users (leakage). Build the live contract and validate the + # one recovered from the checkpoint against it; abort on any mismatch unless + # ALLOW_SPLIT_MISMATCH=1 is set (e.g. deliberately resuming a legacy run). + live_split_contract: Optional[Dict[str, Any]] = None + if supports_holdout: + live_split_contract = { + "train_split_percentage": dataset.dataset._train_split_percentage, # pyre-ignore[16] + "split_salt": dataset.dataset._split_salt, # pyre-ignore[16] + "eval_holdout_ts": eval_holdout_ts_resolved, + "eval_holdout_num_windows": eval_holdout_num_windows, + "batch_size": persistent_dl.batch_size if persistent_dl is not None else None, + "world_size": world_size, + } + # Only validate on an actual resume. On a genuine cold start there is no + # prior split to verify and establishing this run's split is always safe; + # validating there would wrongly reject every fresh holdout run. A resume + # that lacks a contract (legacy/non-streaming checkpoint) is NOT a cold + # start and is still validated (and rejected) below. + if not resume_cold_start: + _validate_split_contract(resume_split_contract, live_split_contract, rank) + # Apply resume hint: advance start_ts past the last completed window, or # re-enter the partial window with a per-rank skip on its first iter. # Shrink num_train_ts by the same amount so the resumed run finishes at @@ -1232,10 +1350,13 @@ def streaming_train_eval_loop( ) def _window_iter(ts: int, skip_samples: int = 0): + # TRAIN-only iterator: both branches exclude held-out eval users via + # train_window_indices / set_ts(train_only=True). (Eval uses the fixed + # holdout set, never this helper.) if persistent_loader: assert window_sampler is not None and persistent_dl is not None window_sampler.set_window( - dataset.dataset.window_indices(ts), # pyre-ignore [16] + dataset.dataset.train_window_indices(ts), # pyre-ignore [16] skip_samples=skip_samples, ) return iter(persistent_dl) @@ -1243,7 +1364,9 @@ def _window_iter(ts: int, skip_samples: int = 0): raise NotImplementedError( "skip_samples>0 requires persistent_loader=True" ) - return iter(make_streaming_dataloader(dataset=dataset, ts=ts)) + return iter( + make_streaming_dataloader(dataset=dataset, ts=ts, train_only=True) + ) # Windows are [start_ts, start_ts + num_train_ts); each step trains window T # then evals window T+1, so the last eval window is start_ts + num_train_ts, # which must be < num_windows(). Anchors require >= history_length prior @@ -1292,6 +1415,7 @@ def _save_mid_window(train_ts: int, batch_idx_in_window: int) -> None: train_ts=train_ts, batch_idx_in_window=batch_idx_in_window, device=device, + split_contract=live_split_contract, ) def _run_train_window( @@ -1418,7 +1542,20 @@ def _run_train_window( ) def _run_eval_window(eval_data_iterator, label: Optional[str] = None) -> None: + # DO NOT add a checkpoint trigger anywhere inside this function. The eval + # data iterator's position is not serializable, so a checkpoint taken + # mid-eval could not be resumed deterministically. `_maybe_checkpoint` + # only fires after a completed eval window or mid-train-window, so any + # restored state always sits on a completed-eval boundary -- which is + # also why the eval reset below is safe across resume. model.eval() + # Reset eval metrics so each pass reports a clean number over the FIXED + # holdout set. Without this, lifetime/window eval metrics would keep + # accumulating across eval steps (the old behavior, made worse now that + # every step sees the identical set), making the eval-AUC trajectory + # uninterpretable. With the reset, each eval point == AUC over the whole + # fixed holdout at that train step -> directly comparable across steps. + metric_logger.reset(mode="eval") eval_batch_idx = 0 first_wait: Optional[float] = None _t_enter = time.perf_counter() if (label and rank == 0) else None @@ -1486,6 +1623,7 @@ def _maybe_checkpoint(train_ts: int) -> None: train_ts=train_ts, batch_idx_in_window=WINDOW_COMPLETE, device=device, + split_contract=live_split_contract, ) last_ckpt_time[0] = time.time() @@ -1508,6 +1646,26 @@ def _should_eval(i: int) -> bool: return True return i % eval_every_n_windows == 0 or i == n_train - 1 + # Fixed eval set: held-out users' anchors over the resolved holdout window + # range, computed ONCE and reused at every eval step. Same anchors every + # step -> stable, comparable eval-AUC curve, and bounded eval time + # (~(1 - train_split_percentage) of a window). Cached inside the dataset so + # re-deriving it (e.g. on resume) returns the identical set. None for + # datasets without holdout support (synthetic) -> legacy per-window eval. + eval_global_indices: Optional["np.ndarray"] = None + if supports_holdout: + eval_global_indices = dataset.dataset.eval_holdout_indices( # pyre-ignore [16] + eval_holdout_ts_resolved, eval_holdout_num_windows + ) + if rank == 0: + logger.info( + "Fixed eval holdout: ts=[%d, %d) -> %d anchors (train_split_percentage=%s)", + eval_holdout_ts_resolved, + eval_holdout_ts_resolved + eval_holdout_num_windows, + len(eval_global_indices), + dataset.dataset._train_split_percentage, # pyre-ignore[16] + ) + if persistent_loader and double_buffer: # Double-buffered: next window prepared in the background during the # current window's compute. Eval (if enabled) uses its own pre-forked @@ -1540,11 +1698,10 @@ def _should_eval(i: int) -> bool: # thread holding an allocator/GIL-released lock. (Deferring this # first fork into the loop — as a sparse-eval cadence naively might — # hangs the run.) _should_eval(0) is always True when eval is enabled - # (0 % K == 0), so the first eval window is always train_ts_list[0]+1; - # arm it now so it prefetches during the i=0 train window. - eval_sampler.set_window( - dataset.dataset.window_indices(train_ts_list[0] + 1) # pyre-ignore [16] - ) + # (0 % K == 0). The eval set is the FIXED holdout (same every step), + # so we install it on the sampler ONCE here; later evals just call + # iter() again to replay the identical set (no set_window churn). + eval_sampler.set_window(eval_global_indices) eval_iter = iter(eval_dl) for i, (train_ts, train_data_iterator) in enumerate( # Only the FIRST window after a mid-window resume needs the skip @@ -1571,20 +1728,15 @@ def _should_eval(i: int) -> bool: if _should_eval(i): dataset.dataset.is_eval = True # pyre-ignore [16] assert eval_sampler is not None and eval_dl is not None - _run_eval_window(eval_iter, label=f"eval_ts={train_ts + 1}") - # Re-arm the (already-forked) eval pool for the NEXT window that - # will eval (i+1 in dense mode, i+K in sparse mode), so it warms - # up during the upcoming train window(s). iter() reuses the + _run_eval_window(eval_iter, label=f"eval_holdout@train_ts={train_ts}") + # Re-arm the (already-forked) eval pool for the NEXT eval. The + # holdout set is fixed, so the sampler window is unchanged; we + # only need a fresh iter() to replay it. iter() reuses the # persistent workers — no fork, safe alongside the bg thread. next_eval_i = next( (j for j in range(i + 1, n_train) if _should_eval(j)), None ) if next_eval_i is not None: - eval_sampler.set_window( - dataset.dataset.window_indices( # pyre-ignore [16] - train_ts_list[next_eval_i] + 1 - ) - ) eval_iter = iter(eval_dl) _maybe_checkpoint(train_ts) else: @@ -1603,49 +1755,37 @@ def _should_eval(i: int) -> bool: ) if _should_eval(i): dataset.dataset.is_eval = True # pyre-ignore [16] - _run_eval_window(_window_iter(train_ts + 1)) + if eval_global_indices is not None: + _run_eval_window( + iter( + make_streaming_dataloader( + dataset=dataset, indices=eval_global_indices + ) + ), + label=f"eval_holdout@train_ts={train_ts}", + ) + else: + # Legacy per-window eval (datasets without user holdout). + _run_eval_window( + iter(make_streaming_dataloader(dataset=dataset, ts=train_ts + 1)) + ) _maybe_checkpoint(train_ts) - eval_ts = num_train_ts - dataset.dataset.is_eval = True - model.eval() - eval_batch_idx: int = 0 - eval_dataloader = make_streaming_dataloader(dataset=dataset, ts=eval_ts) - eval_data_iterator = iter(eval_dataloader) - with torch.no_grad(): - while True: - try: - sample = next(eval_data_iterator) - except StopIteration: - break - sample.to(device) - ( - _, - _, - _, - mt_target_preds, - mt_target_labels, - mt_target_weights, - ) = model.forward( - sample.uih_features_kjt, - sample.candidates_features_kjt, - ) - metric_logger.update( - mode="eval", - predictions=mt_target_preds, - labels=mt_target_labels, - weights=mt_target_weights, - num_candidates=sample.candidates_features_kjt.lengths().view( - len(sample.candidates_features_kjt.keys()), -1 - )[0], - ) - eval_batch_idx += 1 - if output_trace: - assert profiler is not None - profiler.step() - if eval_batch_idx % metric_log_frequency == 0: - metric_logger.compute_and_log(mode="eval") - if num_eval_batches is not None and eval_batch_idx >= num_eval_batches: - break + # Final eval over the SAME fixed user-holdout set (consistent with the + # per-window evals above). Reuses _run_eval_window so metrics are reset and + # reported the same way. Falls back to the legacy final-window eval for + # datasets without user holdout. + dataset.dataset.is_eval = True # pyre-ignore [16] + if eval_global_indices is not None: + _run_eval_window( + iter(make_streaming_dataloader(dataset=dataset, indices=eval_global_indices)), + label="eval_holdout@final", + ) + else: + _run_eval_window( + iter(make_streaming_dataloader(dataset=dataset, ts=num_train_ts)), + label="eval@final", + ) + if rank == 0: for k, v in metric_logger.compute(mode="eval").items(): print(f"{k}: {v}") diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py index be94af667..a9c060324 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py @@ -22,7 +22,7 @@ import os import time from pathlib import Path -from typing import Callable, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional import gin import tensorboard # @manual=//tensorboard:lib # noqa: F401 - required implicit dep when using torch.utils.tensorboard @@ -57,7 +57,31 @@ class LifetimeAUCMetricComputation(AUCMetricComputation): - """AUC over all predictions seen so far (uncapped buffer); emits with the LIFETIME prefix.""" + """AUC over a 10M-sample (~5.5 eval-window) trailing buffer; emits with the + LIFETIME prefix. + + NOTE: despite the name, this is NOT an uncapped since-step-0 AUC. The parent + ``AUCMetricComputation`` evicts the prediction/label/weight buffers down to + ``window_size`` in ``update()``; we instantiate it with + ``window_size=10_000_000``, so "lifetime" is a ~10M-sample trailing window. + Raise ``window_size`` (accepting unbounded buffer growth) if true cumulative + AUC is ever required. + + Checkpoint correctness: torchrec registers the PREDICTIONS/LABELS/WEIGHTS + buffers with ``persistent=False`` (so the default ``state_dict()`` drops + them) and tracks a separate ``self._num_samples`` counter. Without the + overrides below, every checkpoint resume would silently restart this metric + from an empty buffer. We therefore serialize the buffers AND ``_num_samples`` + explicitly; restoring ``_num_samples`` is mandatory, since leaving it at 0 + makes the next ``update()`` take the init-sentinel branch and desync the + windowed eviction. These buffers are per-rank-local (cross-rank gather only + happens transiently at compute time), so the checkpoint layer MUST persist + and restore them per-rank — see ``checkpoint.py``. + """ + + # Prefix used for the explicitly-serialized non-persistent buffers so the + # keys can't collide with any persistent state the parent might register. + _LIFETIME_KEY_PREFIX: str = "_lifetime_" def _compute(self) -> List[MetricComputationReport]: from typing import cast as _cast @@ -76,6 +100,173 @@ def _compute(self) -> List[MetricComputationReport]: ) ] + def lifetime_sample_count(self) -> int: + """Current number of buffered samples (greppable for sanity logs).""" + return int(getattr(self, "_num_samples", 0)) + + def state_dict( + self, + destination: Optional[Dict[str, Any]] = None, + prefix: str = "", + keep_vars: bool = False, + ) -> Dict[str, Any]: + from torchrec.metrics.auc import LABELS, PREDICTIONS, WEIGHTS + + destination = super().state_dict( + destination=destination, prefix=prefix, keep_vars=keep_vars + ) + # The parent registers these buffers persistent=False, so they are absent + # from `destination`. Concatenate each buffer list to one (n_tasks, N) + # tensor and serialize it alongside the sample counter. + for attr in (PREDICTIONS, LABELS, WEIGHTS): + buf = getattr(self, attr) + if isinstance(buf, (list, tuple)) and len(buf) > 0: + flat = torch.cat([t for t in buf], dim=-1) + elif isinstance(buf, torch.Tensor): + flat = buf + else: + flat = torch.empty(0) + destination[prefix + self._LIFETIME_KEY_PREFIX + attr] = ( + flat.detach().cpu().clone() + ) + destination[prefix + self._LIFETIME_KEY_PREFIX + "num_samples"] = ( + torch.tensor(int(getattr(self, "_num_samples", 0)), dtype=torch.long) + ) + return destination + + def load_state_dict( + self, + state_dict: Dict[str, Any], + strict: bool = True, + ) -> Any: + from torchrec.metrics.auc import LABELS, PREDICTIONS, WEIGHTS + + # Copy so we can strip our custom keys before delegating to the parent + # (whose strict load would otherwise reject them as unexpected). + remaining = dict(state_dict) + saved_bufs: Dict[str, torch.Tensor] = {} + for attr in (PREDICTIONS, LABELS, WEIGHTS): + key = self._LIFETIME_KEY_PREFIX + attr + if key in remaining: + saved_bufs[attr] = remaining.pop(key) + num_key = self._LIFETIME_KEY_PREFIX + "num_samples" + saved_num = remaining.pop(num_key, None) + + result = super().load_state_dict(remaining, strict=strict) + + if saved_bufs: + # Device of the live (init-sentinel) buffers; keep restored buffers + # co-located so subsequent update()/compute() stay on-device. + existing = getattr(self, PREDICTIONS) + dev = ( + existing[0].device + if isinstance(existing, (list, tuple)) and len(existing) > 0 + else torch.device("cpu") + ) + for attr, val in saved_bufs.items(): + setattr(self, attr, [val.to(dev)]) + if saved_num is not None: + self._num_samples = int(saved_num.item()) + return result + + +# Sentinel "window size" used for the FRESH eval metrics so torchrec's windowed +# eviction never fires within a single eval pass (the per-pass reset bounds the +# buffer to exactly one full holdout pass). 1<<60 is far above any realistic +# per-rank sample count and avoids sys.maxsize overflow inside torchrec math. +UNBOUNDED_WINDOW: int = 1 << 60 + + +class BinnedCumulativeAUC(RecMetricComputation): + """Cumulative AUC via a fixed-resolution score histogram (LIFETIME prefix). + + Global AUC is a rank statistic, so it has no fixed-size additive sufficient + statistic the way NE/Accuracy do - exact cumulative AUC otherwise needs every + (score, label) pair retained and sorted (the buffer-based ``AUCMetricComputation`` + / ``LifetimeAUCMetricComputation``). Instead we keep two weighted histograms of + positive/negative mass per score bin. This gives an AUC exact up to bin width + with O(num_bins) memory that does NOT grow with sample count, and - because + histograms are additive - cross-rank sync is a cheap all-reduce (dist_reduce_fx + "sum") rather than all-gathering millions of predictions. The state is truly + cumulative across all eval passes (never evicted, never reset on eval). + + Predictions MUST be probabilities in [0, 1] (the same tensor feeds NE, which + requires probabilities; the model applies sigmoid in multitask_module). Values + are clamped into [0, 1] defensively. + """ + + def __init__(self, *args, num_bins: int = 100_000, **kwargs) -> None: + # window_size is irrelevant here (no windowed state); pass through. + super().__init__(*args, **kwargs) + self._num_bins: int = int(num_bins) + self._add_state( + "pos_hist", + torch.zeros((self._n_tasks, self._num_bins), dtype=torch.float64), + add_window_state=False, + dist_reduce_fx="sum", + persistent=True, + ) + self._add_state( + "neg_hist", + torch.zeros((self._n_tasks, self._num_bins), dtype=torch.float64), + add_window_state=False, + dist_reduce_fx="sum", + persistent=True, + ) + + def cumulative_sample_count(self) -> int: + """Total weighted samples in the histograms (greppable for sanity logs).""" + return int((self.pos_hist.sum() + self.neg_hist.sum()).item()) + + def update( + self, + *, + predictions: Optional[torch.Tensor], + labels: torch.Tensor, + weights: Optional[torch.Tensor], + **kwargs: Dict[str, Any], + ) -> None: + if predictions is None or weights is None: + raise ValueError( + "BinnedCumulativeAUC.update requires predictions and weights" + ) + preds = predictions.float().clamp_(0.0, 1.0) # (n_tasks, n_examples) + labels = labels.float() + weights = weights.float() + # Bin index per example; the top edge (p==1.0) folds into the last bin. + idx = (preds * self._num_bins).long().clamp_(0, self._num_bins - 1) + pos_w = (weights * labels).to(self.pos_hist.dtype) + neg_w = (weights * (1.0 - labels)).to(self.neg_hist.dtype) + self.pos_hist.scatter_add_(1, idx, pos_w) + self.neg_hist.scatter_add_(1, idx, neg_w) + + def _compute(self) -> List[MetricComputationReport]: + # By compute() time torchmetrics has all-reduced (summed) the histograms + # across ranks, so these are the global per-bin masses. + pos = self.pos_hist # (n_tasks, num_bins) + neg = self.neg_hist + total_pos = pos.sum(dim=1) + total_neg = neg.sum(dim=1) + # Lower bin index == lower score. A positive in bin b outranks every + # negative in bins < b (exclusive prefix sum), and ties in bin b score + # 0.5. AUC = sum_b pos_b * (neg_below_b + 0.5*neg_b) / (P * N). + neg_below = torch.cumsum(neg, dim=1) - neg + numerator = (pos * (neg_below + 0.5 * neg)).sum(dim=1) + denom = total_pos * total_neg + auc = torch.where( + denom > 0, + numerator / denom, + torch.full_like(numerator, 0.5), + ).to(torch.float32) + return [ + MetricComputationReport( + name=MetricName.AUC, + metric_prefix=MetricPrefix.LIFETIME, + value=auc, + ) + ] + + logging.basicConfig(level=logging.INFO) logger = logging.getLogger("utils") @@ -696,6 +887,11 @@ def __init__( num_flops_per_sample: float = 0.0, gpu_peak_flops: float = 0.0, model: Optional[torch.nn.Module] = None, + eval_cumulative: bool = False, + cumulative_auc_bins: int = 100_000, + train_lifetime_auc_mode: str = "binned", + eval_lifetime_auc_mode: str = "binned", + lifetime_auc_window: int = 10_000_000, ) -> None: # tflops/mfu reporting state (optional — when both num_flops_per_sample # and gpu_peak_flops are set, the train perf line gains tflops_algo/gpu, @@ -725,75 +921,95 @@ def __init__( ] self.task_names: List[str] = all_classification_tasks + all_regression_tasks - self.class_metrics: Dict[str, List[RecMetricComputation]] = { - "train": [], - "eval": [], - } + # Eval metric semantics: + # eval_cumulative=False (default, legacy / static / non-streaming eval): + # a single eval set with the configured window_size, including a + # lifetime AUC. Unchanged behavior. + # eval_cumulative=True (streaming fixed-holdout eval): a FRESH eval set + # (window_size=UNBOUNDED, reset each pass -> per-pass full-holdout + # "window_*") PLUS a CUMULATIVE set ("eval_cum", never reset -> + # "lifetime_*"). NE/Accuracy/GAUC are cumulative for free via their + # persistent scalar sums; AUC cumulative uses the selected backend. + # + # Lifetime-AUC backend is configurable independently for train and eval: + # "binned" (default): BinnedCumulativeAUC - exact-cumulative AUC via an + # O(num_bins) score histogram (additive all-reduce, no unbounded + # buffer, memory independent of #samples/#windows). + # "capped": LifetimeAUCMetricComputation - AUC over a trailing buffer of + # `lifetime_auc_window` samples/rank (the legacy approach; per-rank + # buffer all-gathered at compute). + self._eval_cumulative: bool = eval_cumulative + self._cumulative_auc_bins: int = int(cumulative_auc_bins) + self._train_lifetime_auc_mode: str = str(train_lifetime_auc_mode) + self._eval_lifetime_auc_mode: str = str(eval_lifetime_auc_mode) + self._lifetime_auc_window: int = int(lifetime_auc_window) + n_cls = len(all_classification_tasks) + n_reg = len(all_regression_tasks) + + def _make_lifetime_auc(mode: str) -> RecMetricComputation: + if mode == "binned": + # window_size=0: no torchrec windowed state; histograms only. + return BinnedCumulativeAUC( + my_rank=rank, batch_size=batch_size, n_tasks=n_cls, + window_size=0, num_bins=self._cumulative_auc_bins, + ).to(device) + if mode == "capped": + return LifetimeAUCMetricComputation( + my_rank=rank, batch_size=batch_size, n_tasks=n_cls, + window_size=self._lifetime_auc_window, + ).to(device) + raise ValueError( + f"lifetime_auc_mode must be 'binned' or 'capped', got {mode!r}" + ) + + def _make_class(ws: int, lifetime_mode: Optional[str]) -> List[RecMetricComputation]: + mets: List[RecMetricComputation] = [ + NEMetricComputation(my_rank=rank, batch_size=batch_size, n_tasks=n_cls, window_size=ws).to(device), + AccuracyMetricComputation(my_rank=rank, batch_size=batch_size, n_tasks=n_cls, window_size=ws).to(device), + GAUCMetricComputation(my_rank=rank, batch_size=batch_size, n_tasks=n_cls, window_size=ws).to(device), + AUCMetricComputation(my_rank=rank, batch_size=batch_size, n_tasks=n_cls, window_size=ws).to(device), + ] + if lifetime_mode is not None: + mets.append(_make_lifetime_auc(lifetime_mode)) + return mets + + def _make_class_cumulative() -> List[RecMetricComputation]: + # NE/Accuracy/GAUC: cumulative via persistent lifetime sums (window + # value ignored at compute). AUC: selected lifetime backend. + return [ + NEMetricComputation(my_rank=rank, batch_size=batch_size, n_tasks=n_cls, window_size=window_size).to(device), + AccuracyMetricComputation(my_rank=rank, batch_size=batch_size, n_tasks=n_cls, window_size=window_size).to(device), + GAUCMetricComputation(my_rank=rank, batch_size=batch_size, n_tasks=n_cls, window_size=window_size).to(device), + _make_lifetime_auc(self._eval_lifetime_auc_mode), + ] + + def _make_reg(ws: int) -> List[RecMetricComputation]: + return [ + MSEMetricComputation(my_rank=rank, batch_size=batch_size, n_tasks=n_reg, window_size=ws).to(device), + MAEMetricComputation(my_rank=rank, batch_size=batch_size, n_tasks=n_reg, window_size=ws).to(device), + ] + + self.class_metrics: Dict[str, List[RecMetricComputation]] = {"train": [], "eval": []} + self.regression_metrics: Dict[str, List[RecMetricComputation]] = {"train": [], "eval": []} + if eval_cumulative: + self.class_metrics["eval_cum"] = [] + self.regression_metrics["eval_cum"] = [] + if all_classification_tasks: - for mode in ["train", "eval"]: - self.class_metrics[mode].append( - NEMetricComputation( - my_rank=rank, - batch_size=batch_size, - n_tasks=len(all_classification_tasks), - window_size=window_size, - ).to(device) - ) - self.class_metrics[mode].append( - AccuracyMetricComputation( - my_rank=rank, - batch_size=batch_size, - n_tasks=len(all_classification_tasks), - window_size=window_size, - ).to(device) - ) - self.class_metrics[mode].append( - GAUCMetricComputation( - my_rank=rank, - batch_size=batch_size, - n_tasks=len(all_classification_tasks), - window_size=window_size, - ).to(device) - ) - self.class_metrics[mode].append( - AUCMetricComputation( - my_rank=rank, - batch_size=batch_size, - n_tasks=len(all_classification_tasks), - window_size=window_size, - ).to(device) - ) - self.class_metrics[mode].append( - LifetimeAUCMetricComputation( - my_rank=rank, - batch_size=batch_size, - n_tasks=len(all_classification_tasks), - window_size=10_000_000, - ).to(device) - ) + self.class_metrics["train"] = _make_class(window_size, lifetime_mode=self._train_lifetime_auc_mode) + if eval_cumulative: + self.class_metrics["eval"] = _make_class(UNBOUNDED_WINDOW, lifetime_mode=None) + self.class_metrics["eval_cum"] = _make_class_cumulative() + else: + self.class_metrics["eval"] = _make_class(window_size, lifetime_mode=self._eval_lifetime_auc_mode) - self.regression_metrics: Dict[str, List[RecMetricComputation]] = { - "train": [], - "eval": [], - } if all_regression_tasks: - for mode in ["train", "eval"]: - self.regression_metrics[mode].append( - MSEMetricComputation( - my_rank=rank, - batch_size=batch_size, - n_tasks=len(all_regression_tasks), - window_size=window_size, - ).to(device) - ) - self.regression_metrics[mode].append( - MAEMetricComputation( - my_rank=rank, - batch_size=batch_size, - n_tasks=len(all_regression_tasks), - window_size=window_size, - ).to(device) - ) + self.regression_metrics["train"] = _make_reg(window_size) + if eval_cumulative: + self.regression_metrics["eval"] = _make_reg(UNBOUNDED_WINDOW) + self.regression_metrics["eval_cum"] = _make_reg(window_size) + else: + self.regression_metrics["eval"] = _make_reg(window_size) self.global_step: Dict[str, int] = {"train": 0, "eval": 0} self.tb_logger: Optional[SummaryWriter] = None @@ -822,10 +1038,15 @@ def all_metrics(self) -> Dict[str, List[RecMetricComputation]]: Returns: Dictionary mapping mode ('train'/'eval') to list of metric computations. """ - return { + out = { "train": self.class_metrics["train"] + self.regression_metrics["train"], "eval": self.class_metrics["eval"] + self.regression_metrics["eval"], } + if "eval_cum" in self.class_metrics or "eval_cum" in self.regression_metrics: + out["eval_cum"] = self.class_metrics.get( + "eval_cum", [] + ) + self.regression_metrics.get("eval_cum", []) + return out def update( self, @@ -845,7 +1066,12 @@ def update( num_candidates: Number of candidates per sample (for GAUC). mode: Either 'train' or 'eval'. """ - for metric in self.all_metrics[mode]: + # On eval, update BOTH the fresh set and the never-reset cumulative set + # (if enabled) from the same batch. + update_targets = list(self.all_metrics[mode]) + if mode == "eval" and "eval_cum" in self.all_metrics: + update_targets = update_targets + self.all_metrics["eval_cum"] + for metric in update_targets: if isinstance(metric, GAUCMetricComputation): metric.update( predictions=predictions, @@ -880,13 +1106,41 @@ def compute(self, mode: str = "train") -> Dict[str, float]: """ all_computed_metrics = {} - for metric in self.all_metrics[mode]: - computed_metrics = metric.compute() - for computed in computed_metrics: - all_values = computed.value.cpu() - for i, task_name in enumerate(self.task_names): - key = f"metric/{str(computed.metric_prefix) + str(computed.name)}/{task_name}" - all_computed_metrics[key] = all_values[i] + if mode == "eval" and "eval_cum" in self.all_metrics: + # Dual-set eval: `window_*` (fresh per-pass) from the reset-each-pass + # set; `lifetime_*` (cumulative across passes) from the never-reset + # set. Filter each set to the matching prefix, and drop GAUC's + # auxiliary `*_num_samples` reports. Key names are unchanged + # (`window_auc`, `lifetime_ne`, ...) so dashboards keep working. + def _emit( + metrics: List[RecMetricComputation], keep_prefix: str + ) -> None: + for metric in metrics: + for computed in metric.compute(): + pfx = str(computed.metric_prefix) + name = str(computed.name) + if pfx != keep_prefix or name.endswith("num_samples"): + continue + all_values = computed.value.cpu() + for i, task_name in enumerate(self.task_names): + if i >= len(all_values): + break + all_computed_metrics[f"metric/{pfx}{name}/{task_name}"] = ( + all_values[i] + ) + + _emit(self.all_metrics["eval"], "window_") + _emit(self.all_metrics["eval_cum"], "lifetime_") + else: + for metric in self.all_metrics[mode]: + computed_metrics = metric.compute() + for computed in computed_metrics: + all_values = computed.value.cpu() + for i, task_name in enumerate(self.task_names): + if i >= len(all_values): + break + key = f"metric/{str(computed.metric_prefix) + str(computed.name)}/{task_name}" + all_computed_metrics[key] = all_values[i] logger.info( f"{mode} - Step {self.global_step[mode]} metrics: {all_computed_metrics}" @@ -1066,6 +1320,21 @@ def env_path(key: str = "", default: str = "") -> str: return os.environ.get(key, default) if key else default +@gin.configurable +def env_str(key: str = "", default: str = "") -> str: + """Resolve a string from os.environ[key], falling back to `default`. + + Companion to `env_int`/`env_float` for categorical/string overrides (e.g. a + metric backend selector). Example gin usage: + + MetricsLogger.train_lifetime_auc_mode = @tlam/env_str() + tlam/env_str.key = "TRAIN_LIFETIME_AUC_MODE" + tlam/env_str.default = "binned" + """ + raw = os.environ.get(key) if key else None + return raw if raw else default + + @gin.configurable def env_int(key: str = "", default: int = 0) -> int: """Resolve an int from os.environ[key], falling back to `default`. @@ -1155,6 +1424,8 @@ def get_dataset( history_length: Optional[int] = None, streaming_window_seconds: int = 86400, streaming_sort_within_window: bool = False, + train_split_percentage: float = 1.0, + split_salt: int = 0, ): """ Get dataset class and configuration by name. @@ -1285,6 +1556,11 @@ def get_dataset( # streaming-train-eval; ignored by the default train-eval path). "streaming_window_seconds": streaming_window_seconds, "streaming_sort_within_window": streaming_sort_within_window, + # User-level train:eval holdout for the streaming path. 1.0 = + # no holdout (legacy). <1.0 holds out (1 - tsp) of users as a + # fixed eval set; those users are never trained. + "train_split_percentage": train_split_percentage, + "split_salt": split_salt, }, ) if name == "sampled-streaming-100b": diff --git a/recommendation_v4/scripts/run_streaming_e2e.sh b/recommendation_v4/scripts/run_streaming_e2e.sh index 40dc5fe81..70ec8fc8f 100755 --- a/recommendation_v4/scripts/run_streaming_e2e.sh +++ b/recommendation_v4/scripts/run_streaming_e2e.sh @@ -114,7 +114,7 @@ NUM_TRAIN_TS=149 START_TS=150 EVAL_EVERY=5 CKPT_TIME_INTERVAL=7200 -KEEP_LAST_N=2 +KEEP_LAST_N=1 CKPT_PATH=/apps/chcai/ckpts/yambda_5b_e2e RUN_NAME=yambda_5b_e2e LOG=/apps/chcai/yambda_5b_e2e.log @@ -122,6 +122,14 @@ MAX_RELAUNCH=50 NUM_TRAIN_BATCHES=0 # 0 = full window (only capped for validation/tests) NUM_EVAL_BATCHES=0 # 0 = full holdout eval (only capped for validation) DIE_AT_STEP=-1 # >=0 = test-only failure injection +# Train:eval split (fraction of USERS trained; 1 - this held out as a FIXED, +# never-trained eval set). Passed on EVERY relaunch so the split stays an +# immutable run contract — a changed split would abort on resume (validated in +# the loop) to prevent skip-offset desync and held-out users leaking into train. +TRAIN_SPLIT_PERCENTAGE=0.90 +SPLIT_SALT=0 +EVAL_HOLDOUT_TS=-1 # <0 = window just past training (start_ts+num_train_ts) +EVAL_HOLDOUT_NUM_WINDOWS=1 IN_WINDOW_FREQ=0 # >0 = also save every N batches within a window ATTACH=0 # 1 = (re)attach to an already-running trainer without # killing it or truncating its log — used to restore @@ -377,6 +385,10 @@ launch() { NUM_TRAIN_BATCHES=$NUM_TRAIN_BATCHES \ NUM_EVAL_BATCHES=$NUM_EVAL_BATCHES \ DIE_AT_STEP=$DIE_AT_STEP \ + TRAIN_SPLIT_PERCENTAGE=$TRAIN_SPLIT_PERCENTAGE \ + SPLIT_SALT=$SPLIT_SALT \ + EVAL_HOLDOUT_TS=$EVAL_HOLDOUT_TS \ + EVAL_HOLDOUT_NUM_WINDOWS=$EVAL_HOLDOUT_NUM_WINDOWS \ METRIC_LOG_FREQ=50 \ RUN_NAME=$RUN_NAME \ TENSORBOARD_LOG_PATH=/apps/chcai/tb/$RUN_NAME/ \ From efc512621e2afecb225d518394b8b8ac3648ad3e Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Mon, 8 Jun 2026 22:40:58 -0500 Subject: [PATCH 040/127] dlrmv4: default yambda-5b to the 4k-no-truncation seq shape MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Set HISTORY_LENGTH=4086 / MAX_SEQ_LEN=4096 as the gin defaults (3*1362+9=4095 ≤ 4096, the no-overfill 4k analog of the prior 2039/2048 shape). Override via $HISTORY_LENGTH/$MAX_SEQ_LEN; use 2039/2048 to reuse the 2k single-task cache. Co-authored-by: Cursor --- .../dlrm_v3/train/gin/yambda_5b.gin | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index 123027fdd..e7eed30cb 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -77,29 +77,30 @@ get_dataset.new_path_prefix = %DATA_PATH # across 3 behaviour pools (listen+ / like / skip) at L//3 events each. # Per-sample sequence the model sees = # 3 × (L // 3) + 8 contextual + 1 candidate -# Choosing 2039 makes 3 × 679 + 9 = 2046, the largest value that fits -# get_hstu_configs.max_seq_len = 2048 with no dataset-side truncation. +# Choosing 4086 makes 3 × 1362 + 9 = 4095, the largest value that fits +# get_hstu_configs.max_seq_len = 4096 with no dataset-side truncation (the +# 4k analog of the previous 2039/2048 shape, where 3×679+9=2046 ≤ 2048). # Larger L overflows the budget; the dataset truncates UIH events to fit. # Note: like events are only 1.9% of the yambda corpus and max user lifetime # is ~28k events, so the like pool fills to ~105 events per anchor on -# average (not 679) — TRITON's jagged attention skips the unfilled slots, +# average (not 1362) — TRITON's jagged attention skips the unfilled slots, # so the under-fill costs sequence budget but not GPU compute. # Cache is keyed by L on disk under /hstu_cache_L/; -# switching L reuses an existing cache or builds a new one (~5 min). Override -# via $HISTORY_LENGTH (default 2039 keeps the existing single-task cache hot). +# switching L reuses an existing cache or builds a new one. Override via +# $HISTORY_LENGTH (default 4086 = the 4k-no-truncation shape; use 2039 with +# MAX_SEQ_LEN=2048 to reuse the previous 2k single-task cache). get_dataset.history_length = @hl/env_int() hl/env_int.key = "HISTORY_LENGTH" -hl/env_int.default = 2039 +hl/env_int.default = 4086 # Model-side attention budget. Dataset truncates UIH to fit this value if # `history_length + contextual + candidate` would overflow. Override via -# $MAX_SEQ_LEN (default 2048 preserves the production single-task shape). -# Pair MAX_SEQ_LEN=4096 with HISTORY_LENGTH=4086 for the 4k-no-truncation -# analog (3*1362+9=4095 ≤ 4096); pair with HISTORY_LENGTH=4096 to reuse the -# existing hstu_cache_L4096/ cache with ~8 events of trailing truncation. +# $MAX_SEQ_LEN (default 4096, the 4k-no-truncation shape paired with +# HISTORY_LENGTH=4086: 3*1362+9=4095 ≤ 4096). Set MAX_SEQ_LEN=2048 with +# HISTORY_LENGTH=2039 for the previous 2k production single-task shape. get_hstu_configs.max_seq_len = @msl/env_int() msl/env_int.key = "MAX_SEQ_LEN" -msl/env_int.default = 2048 +msl/env_int.default = 4096 # --- streaming (temporal-order) training ------------------------------------- # Only consumed under `--mode streaming-train-eval`; the default train-eval From 03362da5a8fec3fbab5b6dfd37232ff69965a010 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Mon, 8 Jun 2026 23:04:57 -0500 Subject: [PATCH 041/127] dlrmv4: reservation-aware sbatch failover in streaming supervisor Replace the salloc-based node failover with an sbatch hold job (--wrap "sleep infinity", bounded by --time), since interactive salloc on meta64 is capped at 240 min and can't hold a multi-day run. Add --reservation so a replacement node is re-acquired from the same SLURM reservation, plus --acquire-wait-max to tolerate brief queueing. Provisioning still runs via srun --overlap afterward. Co-authored-by: Cursor --- .../scripts/run_streaming_e2e.sh | 66 +++++++++++++------ 1 file changed, 47 insertions(+), 19 deletions(-) diff --git a/recommendation_v4/scripts/run_streaming_e2e.sh b/recommendation_v4/scripts/run_streaming_e2e.sh index 70ec8fc8f..59a48b703 100755 --- a/recommendation_v4/scripts/run_streaming_e2e.sh +++ b/recommendation_v4/scripts/run_streaming_e2e.sh @@ -49,12 +49,16 @@ # never false-trip this.) # # NODE FAILOVER (case 3, the --allow-failover path) -# ensure_ready -> acquire_node: `salloc --no-shell --exclusive` a fresh node on -# $PARTITION, wait for RUNNING, then provision_node runs $PROVISION_SCRIPT on -# it (docker pull + container create + dep install; ~15 min on a cold node). -# Allocations WE create are tracked and `scancel`ed (container removed first) -# on success via release_acquired; the user's original --jobid is never -# cancelled. Checkpoints on shared NFS make the resume seamless. +# ensure_ready -> acquire_node: submit an `sbatch` hold job (`--wrap "sleep +# infinity"`, bounded by --time=$ALLOC_TIME) for a fresh exclusive node on +# $PARTITION, optionally from --reservation $RESERVATION; wait for RUNNING, +# then provision_node runs $PROVISION_SCRIPT on it via `srun --jobid --overlap` +# (docker pull + container create + dep install; ~15 min on a cold node). +# sbatch (not salloc) because interactive salloc on some partitions (e.g. +# meta64) is capped at 240 min, which a multi-day hold would exceed. Jobs WE +# create are tracked and `scancel`ed (container removed first) on success via +# release_acquired; the user's original --jobid is never cancelled. +# Checkpoints on shared NFS make the resume seamless. # # CHECKPOINTS / DISK # The trainer saves atomically (write to .tmp, fsync, rename to ) and @@ -67,7 +71,8 @@ # ckpt: --ckpt-path --keep-last-n --ckpt-time-interval --in-window-freq # logging: --run-name --log # resilience: --max-relaunch --min-free-gib --stall-s -# failover: --partition --alloc-time --allow-failover --provision-script +# failover: --partition --reservation --alloc-time --allow-failover +# --provision-script --acquire-wait-max # validation: --num-train-batches --num-eval-batches (>0 caps batches/window # for fast tests; 0 = full window / full-holdout eval) # test-only: --die-at-step (>=0 injects a crash at that global step) @@ -85,11 +90,15 @@ # # EXAMPLE # nohup bash scripts/run_streaming_e2e.sh \ +# --jobid 12074 \ # --ckpt-path /apps/chcai/ckpts/yambda_5b_e2e \ # --run-name yambda_5b_e2e --log /apps/chcai/yambda_5b_e2e.log \ # --start-ts 150 --num-train-ts 149 --eval-every 10 \ -# --ckpt-time-interval 7200 --keep-last-n 2 --max-relaunch 50 \ +# --ckpt-time-interval 3600 --keep-last-n 1 --max-relaunch 100 \ +# --reservation NAN_issue_debug \ # > /apps/chcai/yambda_5b_e2e.supervisor.console.log 2>&1 & +# (--reservation makes node-death failover re-acquire from that reservation; +# omit it to fall back to the open $PARTITION pool.) # ============================================================================= set -uo pipefail @@ -143,9 +152,15 @@ CTRL_WAIT_MAX=3600 # max seconds to wait for an unreachable SLURM controlle # the container on it, and resume — checkpoints + code live on shared NFS # (/apps/chcai, /home/chcai), so any node in the partition can continue. PARTITION=meta64 -ALLOC_TIME=7-00:00:00 # SLURM --time for a failover allocation +RESERVATION="" # if set, failover acquires from this SLURM + # reservation (e.g. NAN_issue_debug) so a + # replacement node comes from the same pool. +ALLOC_TIME=7-00:00:00 # SLURM --time for a failover hold job ALLOW_FAILOVER=1 # 0 = never acquire a new node PROVISION_SCRIPT=/home/chcai/_provision_yambda_primus.sh +ACQUIRE_WAIT_MAX=1800 # max seconds to wait for a failover sbatch + # hold job to reach RUNNING (tolerates brief + # queueing before the node is granted). # Disk guard: require at least this many GiB free on the ckpt volume before a # (re)launch. One checkpoint is ~560 GB. A save writes a fresh .tmp BEFORE the @@ -182,9 +197,11 @@ while [[ $# -gt 0 ]]; do --min-free-gib) MIN_FREE_GIB="$2"; shift 2;; --stall-s) STALL_S="$2"; shift 2;; --partition) PARTITION="$2"; shift 2;; + --reservation) RESERVATION="$2"; shift 2;; --alloc-time) ALLOC_TIME="$2"; shift 2;; --allow-failover) ALLOW_FAILOVER="$2"; shift 2;; --provision-script) PROVISION_SCRIPT="$2"; shift 2;; + --acquire-wait-max) ACQUIRE_WAIT_MAX="$2"; shift 2;; *) echo "Unknown arg: $1"; exit 1;; esac done @@ -280,27 +297,37 @@ provision_node() { } # Acquire a fresh exclusive node on $PARTITION; sets global JOBID on success. +# Uses `sbatch` (not `salloc`): interactive salloc on some partitions (meta64) +# is capped at 240 min, which an $ALLOC_TIME multi-day hold exceeds. The batch +# job merely pins the node (`sleep infinity`, bounded by --time); the container +# is provisioned afterward by provision_node via `srun --jobid --overlap`. +# Honors --reservation so failover re-acquires from the SAME reservation pool. acquire_node() { if [[ "$ALLOW_FAILOVER" != "1" ]]; then sup "failover disabled (--allow-failover 0); cannot acquire a new node"; return 1 fi - sup "requesting a fresh node on partition=$PARTITION (exclusive, time=$ALLOC_TIME)" - local out jid - out=$(salloc --no-shell --partition="$PARTITION" --nodes=1 --exclusive \ - --time="$ALLOC_TIME" --job-name=e2e_failover 2>&1) - jid=$(echo "$out" | grep -oiE "Granted job allocation [0-9]+" | grep -oE "[0-9]+" | head -1) - if [[ -z "$jid" ]]; then - sup "FATAL: salloc did not grant a node: $out"; return 1 + local resv_arg="" + [[ -n "$RESERVATION" ]] && resv_arg="--reservation=$RESERVATION" + sup "requesting a fresh node via sbatch (partition=$PARTITION${RESERVATION:+ reservation=$RESERVATION}, exclusive, time=$ALLOC_TIME)" + local jid + jid=$(sbatch --parsable --partition="$PARTITION" $resv_arg --nodes=1 --exclusive \ + --time="$ALLOC_TIME" --job-name=e2e_failover \ + --output="${LOG%.log}.failover_hold.%j.log" \ + --wrap="echo \"[failover-hold] node=\$(hostname) jobid=\$SLURM_JOB_ID start=\$(date -Is)\"; sleep infinity" 2>&1) + # --parsable => "" or ";"; strip whitespace + cluster. + jid=$(echo "$jid" | tr -d ' ' | cut -d';' -f1) + if ! [[ "$jid" =~ ^[0-9]+$ ]]; then + sup "FATAL: sbatch did not return a jobid: $jid"; return 1 fi ACQUIRED_JOBIDS+=("$jid") - sup "granted new allocation jobid=$jid; waiting for RUNNING" + sup "submitted failover hold job jobid=$jid; waiting for RUNNING (max ${ACQUIRE_WAIT_MAX}s)" local waited=0 - while (( waited < 600 )); do + while (( waited < ACQUIRE_WAIT_MAX )); do [[ "$(squeue -h -j "$jid" -o '%T' 2>/dev/null | head -1)" == "RUNNING" ]] && break sleep 10; waited=$((waited + 10)) done if [[ "$(squeue -h -j "$jid" -o '%T' 2>/dev/null | head -1)" != "RUNNING" ]]; then - sup "FATAL: new allocation $jid never reached RUNNING (waited ${waited}s)"; return 1 + sup "FATAL: failover hold job $jid never reached RUNNING (waited ${waited}s)"; return 1 fi JOBID="$jid" sup "new node ready: jobid=$JOBID node=$(squeue -h -j "$JOBID" -o '%N' 2>/dev/null | head -1)" @@ -410,6 +437,7 @@ sup "jobid=$JOBID container=$CONTAINER repo=$REPO" sup "start_ts=$START_TS num_train_ts=$NUM_TRAIN_TS eval_every=$EVAL_EVERY" sup "ckpt_path=$CKPT_PATH keep_last_n=$KEEP_LAST_N ckpt_time_interval=${CKPT_TIME_INTERVAL}s in_window_freq=$IN_WINDOW_FREQ" sup "log=$LOG num_train_batches=$NUM_TRAIN_BATCHES die_at_step=$DIE_AT_STEP max_relaunch=$MAX_RELAUNCH" +sup "failover: allow=$ALLOW_FAILOVER partition=$PARTITION reservation=${RESERVATION:-} alloc_time=$ALLOC_TIME" cexec "mkdir -p '$CKPT_PATH' '/apps/chcai/tb/$RUN_NAME'" # Initialize this run's metrics log ONCE. launch_smoke_8gpu.sh appends (tee -a), From 888f701926b84ddaba2877276dc82c93e8c2c671 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Mon, 8 Jun 2026 23:11:18 -0500 Subject: [PATCH 042/127] dlrmv4: two-tier reservation-then-open-pool failover in streaming supervisor acquire_node now prefers the configured --reservation (tier 1, short RESV_WAIT_MAX wait since a free reservation node starts ~immediately), then falls back to the open partition pool (tier 2, ACQUIRE_WAIT_MAX). The pending reservation hold job is scancel'd before the fallback resubmit so we never end up holding two nodes. Factors the sbatch submit and RUNNING-wait into _submit_hold_job/_wait_running helpers; adds the --resv-wait-max knob. Co-authored-by: Cursor --- .../scripts/run_streaming_e2e.sh | 106 +++++++++++++----- 1 file changed, 76 insertions(+), 30 deletions(-) diff --git a/recommendation_v4/scripts/run_streaming_e2e.sh b/recommendation_v4/scripts/run_streaming_e2e.sh index 59a48b703..e8febc62c 100755 --- a/recommendation_v4/scripts/run_streaming_e2e.sh +++ b/recommendation_v4/scripts/run_streaming_e2e.sh @@ -158,9 +158,15 @@ RESERVATION="" # if set, failover acquires from this SLUR ALLOC_TIME=7-00:00:00 # SLURM --time for a failover hold job ALLOW_FAILOVER=1 # 0 = never acquire a new node PROVISION_SCRIPT=/home/chcai/_provision_yambda_primus.sh -ACQUIRE_WAIT_MAX=1800 # max seconds to wait for a failover sbatch - # hold job to reach RUNNING (tolerates brief - # queueing before the node is granted). +ACQUIRE_WAIT_MAX=1800 # max seconds to wait for the OPEN-POOL + # (tier-2) failover hold job to reach + # RUNNING (tolerates brief queueing). +RESV_WAIT_MAX=300 # max seconds to wait for a RESERVATION + # (tier-1) node before giving up on it and + # falling back to the open $PARTITION pool. + # Short, since a free reservation node + # starts ~immediately; a longer wait just + # means the reservation is currently full. # Disk guard: require at least this many GiB free on the ckpt volume before a # (re)launch. One checkpoint is ~560 GB. A save writes a fresh .tmp BEFORE the @@ -202,6 +208,7 @@ while [[ $# -gt 0 ]]; do --allow-failover) ALLOW_FAILOVER="$2"; shift 2;; --provision-script) PROVISION_SCRIPT="$2"; shift 2;; --acquire-wait-max) ACQUIRE_WAIT_MAX="$2"; shift 2;; + --resv-wait-max) RESV_WAIT_MAX="$2"; shift 2;; *) echo "Unknown arg: $1"; exit 1;; esac done @@ -296,42 +303,81 @@ provision_node() { container_up "$jid" } -# Acquire a fresh exclusive node on $PARTITION; sets global JOBID on success. -# Uses `sbatch` (not `salloc`): interactive salloc on some partitions (meta64) -# is capped at 240 min, which an $ALLOC_TIME multi-day hold exceeds. The batch -# job merely pins the node (`sleep infinity`, bounded by --time); the container -# is provisioned afterward by provision_node via `srun --jobid --overlap`. -# Honors --reservation so failover re-acquires from the SAME reservation pool. +# Submit an sbatch hold job that merely pins one exclusive node (`sleep +# infinity`, bounded by --time=$ALLOC_TIME); echoes the jobid. $1 = extra sbatch +# args (e.g. "--reservation=NAN_issue_debug" or ""). sbatch (not salloc) because +# interactive salloc on some partitions (meta64) is capped at 240 min, which an +# $ALLOC_TIME multi-day hold exceeds. The container is provisioned afterward by +# provision_node via `srun --jobid --overlap`. +_submit_hold_job() { + local extra="$1" out + out=$(sbatch --parsable --partition="$PARTITION" $extra --nodes=1 --exclusive \ + --time="$ALLOC_TIME" --job-name=e2e_failover \ + --output="${LOG%.log}.failover_hold.%j.log" \ + --wrap="echo \"[failover-hold] node=\$(hostname) jobid=\$SLURM_JOB_ID start=\$(date -Is)\"; sleep infinity" 2>&1) + # --parsable => "" or ";"; strip whitespace + cluster. + echo "$out" | tr -d ' ' | cut -d';' -f1 +} + +# Wait up to $2 seconds for job $1 to reach RUNNING. Returns 0 if RUNNING. +_wait_running() { + local jid="$1" max="$2" waited=0 st + while (( waited < max )); do + st=$(squeue -h -j "$jid" -o '%T' 2>/dev/null | head -1) + [[ "$st" == "RUNNING" ]] && return 0 + sleep 10; waited=$((waited + 10)) + done + return 1 +} + +# Acquire a fresh exclusive node and set global JOBID on success. Two-tier: +# tier 1 (preferred): the SLURM --reservation $RESERVATION, if configured. +# Waited on for only RESV_WAIT_MAX — a free reservation node starts almost +# immediately, so a longer wait means the reservation is currently full. +# tier 2 (fallback): the open $PARTITION pool (no reservation), waited on for +# ACQUIRE_WAIT_MAX. Used when no reservation is set, or the reservation had +# no node free within RESV_WAIT_MAX (the pending reservation job is +# cancelled before we resubmit so we never end up holding two nodes). acquire_node() { if [[ "$ALLOW_FAILOVER" != "1" ]]; then sup "failover disabled (--allow-failover 0); cannot acquire a new node"; return 1 fi - local resv_arg="" - [[ -n "$RESERVATION" ]] && resv_arg="--reservation=$RESERVATION" - sup "requesting a fresh node via sbatch (partition=$PARTITION${RESERVATION:+ reservation=$RESERVATION}, exclusive, time=$ALLOC_TIME)" local jid - jid=$(sbatch --parsable --partition="$PARTITION" $resv_arg --nodes=1 --exclusive \ - --time="$ALLOC_TIME" --job-name=e2e_failover \ - --output="${LOG%.log}.failover_hold.%j.log" \ - --wrap="echo \"[failover-hold] node=\$(hostname) jobid=\$SLURM_JOB_ID start=\$(date -Is)\"; sleep infinity" 2>&1) - # --parsable => "" or ";"; strip whitespace + cluster. - jid=$(echo "$jid" | tr -d ' ' | cut -d';' -f1) + + # --- tier 1: reservation (preferred) ------------------------------------- + if [[ -n "$RESERVATION" ]]; then + sup "failover tier-1: requesting a node from reservation=$RESERVATION (exclusive, time=$ALLOC_TIME)" + jid=$(_submit_hold_job "--reservation=$RESERVATION") + if [[ "$jid" =~ ^[0-9]+$ ]]; then + ACQUIRED_JOBIDS+=("$jid") # track for cleanup even if it never starts + sup "reservation hold job jobid=$jid submitted; waiting up to ${RESV_WAIT_MAX}s for RUNNING" + if _wait_running "$jid" "$RESV_WAIT_MAX"; then + JOBID="$jid" + sup "new node ready (reservation $RESERVATION): jobid=$JOBID node=$(squeue -h -j "$JOBID" -o '%N' 2>/dev/null | head -1)" + return 0 + fi + sup "reservation $RESERVATION has no free node within ${RESV_WAIT_MAX}s — cancelling pending $jid and falling back to open pool" + scancel "$jid" 2>/dev/null || true + else + sup "reservation sbatch did not return a jobid ($jid) — falling back to open pool" + fi + fi + + # --- tier 2: open partition pool (fallback) ------------------------------ + sup "failover tier-2: requesting a node from open partition=$PARTITION (exclusive, time=$ALLOC_TIME)" + jid=$(_submit_hold_job "") if ! [[ "$jid" =~ ^[0-9]+$ ]]; then - sup "FATAL: sbatch did not return a jobid: $jid"; return 1 + sup "FATAL: open-pool sbatch did not return a jobid: $jid"; return 1 fi ACQUIRED_JOBIDS+=("$jid") - sup "submitted failover hold job jobid=$jid; waiting for RUNNING (max ${ACQUIRE_WAIT_MAX}s)" - local waited=0 - while (( waited < ACQUIRE_WAIT_MAX )); do - [[ "$(squeue -h -j "$jid" -o '%T' 2>/dev/null | head -1)" == "RUNNING" ]] && break - sleep 10; waited=$((waited + 10)) - done - if [[ "$(squeue -h -j "$jid" -o '%T' 2>/dev/null | head -1)" != "RUNNING" ]]; then - sup "FATAL: failover hold job $jid never reached RUNNING (waited ${waited}s)"; return 1 + sup "open-pool hold job jobid=$jid submitted; waiting up to ${ACQUIRE_WAIT_MAX}s for RUNNING" + if _wait_running "$jid" "$ACQUIRE_WAIT_MAX"; then + JOBID="$jid" + sup "new node ready (open $PARTITION): jobid=$JOBID node=$(squeue -h -j "$JOBID" -o '%N' 2>/dev/null | head -1)" + return 0 fi - JOBID="$jid" - sup "new node ready: jobid=$JOBID node=$(squeue -h -j "$JOBID" -o '%N' 2>/dev/null | head -1)" - return 0 + sup "FATAL: open-pool hold job $jid never reached RUNNING (waited ${ACQUIRE_WAIT_MAX}s)" + return 1 } # Ensure $JOBID is a healthy allocation with the container up, failing over to a From 7c36891b3ea2d0e1fac625a20e69f5e28f349e25 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Tue, 9 Jun 2026 12:14:47 -0500 Subject: [PATCH 043/127] dlrmv4: cap failover at <=1 reservation node + fix trainer-alive self-match trainer_alive: `pgrep -f generative_recommenders` always matched the probe shell's own cmdline, so it could never report the trainer dead -- defeating the stall watchdog and making ATTACH mode falsely "adopt" a nonexistent trainer. Use the `set -f; pgrep -f [g]enerative_recommenders` self-match guard. Reservation cap: every node we acquire is an sbatch --job-name=e2e_failover hold, so reap_failover_holds() reaps strays by name at startup (catching holds leaked by a prior supervisor that died mid-failover) and before every acquire (no stacking). wait_for_original_recover() waits --orig-recover-wait (def 600s) for SLURM to requeue the lost ORIGINAL job and reuses it instead of grabbing a SECOND reservation node. Together these keep us at <=1 reservation node. Co-authored-by: Cursor --- .../scripts/run_streaming_e2e.sh | 88 ++++++++++++++++++- 1 file changed, 85 insertions(+), 3 deletions(-) diff --git a/recommendation_v4/scripts/run_streaming_e2e.sh b/recommendation_v4/scripts/run_streaming_e2e.sh index e8febc62c..5009ecbce 100755 --- a/recommendation_v4/scripts/run_streaming_e2e.sh +++ b/recommendation_v4/scripts/run_streaming_e2e.sh @@ -72,7 +72,11 @@ # logging: --run-name --log # resilience: --max-relaunch --min-free-gib --stall-s # failover: --partition --reservation --alloc-time --allow-failover -# --provision-script --acquire-wait-max +# --provision-script --acquire-wait-max --resv-wait-max +# --orig-recover-wait +# (failover holds <=1 reservation node: stray/leaked e2e_failover +# holds are reaped, and a lost ORIGINAL job is waited on for SLURM +# requeue and reused before a SEPARATE node is acquired.) # validation: --num-train-batches --num-eval-batches (>0 caps batches/window # for fast tests; 0 = full window / full-holdout eval) # test-only: --die-at-step (>=0 injects a crash at that global step) @@ -167,6 +171,13 @@ RESV_WAIT_MAX=300 # max seconds to wait for a RESERVATION # Short, since a free reservation node # starts ~immediately; a longer wait just # means the reservation is currently full. +ORIG_RECOVER_WAIT=600 # when the user's ORIGINAL reservation job + # is lost, wait this long for SLURM to + # auto-requeue it back to RUNNING before + # acquiring a SEPARATE node. Reusing the + # requeued original keeps us at <=1 + # reservation node and skips a redundant + # acquire (observed requeue latency ~2 min). # Disk guard: require at least this many GiB free on the ckpt volume before a # (re)launch. One checkpoint is ~560 GB. A save writes a fresh .tmp BEFORE the @@ -209,6 +220,7 @@ while [[ $# -gt 0 ]]; do --provision-script) PROVISION_SCRIPT="$2"; shift 2;; --acquire-wait-max) ACQUIRE_WAIT_MAX="$2"; shift 2;; --resv-wait-max) RESV_WAIT_MAX="$2"; shift 2;; + --orig-recover-wait) ORIG_RECOVER_WAIT="$2"; shift 2;; *) echo "Unknown arg: $1"; exit 1;; esac done @@ -342,6 +354,10 @@ acquire_node() { if [[ "$ALLOW_FAILOVER" != "1" ]]; then sup "failover disabled (--allow-failover 0); cannot acquire a new node"; return 1 fi + # Release any prior/leaked failover hold BEFORE grabbing a new one, so we + # never transiently pin two reservation nodes (e.g. a dead tier-1 hold + the + # replacement we are about to submit). + reap_failover_holds "" local jid # --- tier 1: reservation (preferred) ------------------------------------- @@ -395,6 +411,17 @@ ensure_ready() { sup "provisioning on $JOBID failed; will try a fresh node" else sup "current allocation $JOBID unavailable (job not RUNNING or node down/drained)" + # Prefer the SLURM-requeued original over acquiring a SEPARATE node, so we + # stay at <=1 reservation node. (No-op once we've already failed over off + # the original.) + if wait_for_original_recover; then + JOBID="$ORIGINAL_JOBID" + refresh_node >/dev/null + if container_up "$JOBID"; then sup "reusing recovered original jobid=$JOBID"; return 0; fi + sup "recovered original $JOBID up but container '$CONTAINER' not present — (re)provisioning" + provision_node "$JOBID" && return 0 + sup "provisioning recovered original $JOBID failed; will acquire a fresh node" + fi fi acquire_node || return 1 provision_node "$JOBID" || { sup "provisioning new node $JOBID failed"; return 1; } @@ -413,15 +440,65 @@ release_acquired() { done } +# Enforce "at most ONE reservation node held by this run at a time" and reap +# orphans. Every node WE acquire is an `sbatch --job-name=e2e_failover` hold, so +# all our holds are discoverable by name even across a supervisor restart — which +# is how a previous supervisor that died mid-failover (e.g. on a provisioning +# error) can leave a hold pinning a second reservation node. Cancels every +# e2e_failover hold owned by us EXCEPT $1 (the one to keep) and the user's +# ORIGINAL_JOBID (never ours to cancel). Containers are removed before the node +# is freed so they don't linger for the next tenant. +reap_failover_holds() { + local keep="${1:-}" me jid + me=$(id -un 2>/dev/null) + [[ -z "$me" ]] && return 0 + while read -r jid; do + [[ -z "$jid" ]] && continue + [[ "$jid" == "$keep" || "$jid" == "$ORIGINAL_JOBID" ]] && continue + sup "reaping stray failover hold $jid (enforcing <=1 reservation node held by this run)" + srun --jobid="$jid" --overlap docker rm -f "$CONTAINER" >/dev/null 2>&1 || true + scancel "$jid" 2>/dev/null || true + done < <(squeue -h -u "$me" -n e2e_failover -o '%i' 2>/dev/null) +} + +# When the user's ORIGINAL reservation job is lost, SLURM typically auto-requeues +# it back onto a (fresh) reservation node within a couple of minutes. Waiting for +# that and REUSING it — rather than immediately acquiring a SEPARATE node — is +# what keeps us at <=1 reservation node (the alternative is the original requeue +# AND a failover hold both pinning reservation nodes) and skips a redundant +# acquire+provision. Only meaningful while we are still on the original job. +wait_for_original_recover() { + [[ "$JOBID" != "$ORIGINAL_JOBID" ]] && return 1 + local waited=0 + while (( waited < ORIG_RECOVER_WAIT )); do + if alloc_healthy "$ORIGINAL_JOBID"; then + sup "original job $ORIGINAL_JOBID is RUNNING again (SLURM requeue) after ${waited}s — reusing it (no second node)" + return 0 + fi + sup "waiting for original job $ORIGINAL_JOBID to requeue before acquiring a separate node (${waited}s/${ORIG_RECOVER_WAIT}s)…" + sleep 15; waited=$((waited + 15)) + done + sup "original job $ORIGINAL_JOBID did not recover within ${ORIG_RECOVER_WAIT}s — acquiring a fresh node" + return 1 +} + # Returns 0 (true) if a trainer process is alive in the container. Uses SLURM # (srun) when the controller is up, else falls back to a direct SSH probe so a # control-plane outage can't make a live trainer look dead. trainer_alive() { local n + # `set -f; pgrep -f [g]enerative...` is the classic self-match guard: the + # probe shell's OWN cmdline contains the pattern, so a naive `pgrep -f + # generative_recommenders` ALWAYS matches itself and returns >=1 even when + # the trainer is dead — which would defeat the stall watchdog and make + # ATTACH mode falsely "adopt" a nonexistent trainer. The [g] char-class + # matches "generative" in real trainer cmdlines but NOT the literal + # "[g]enerative" in the probe's cmdline; `set -f` keeps the bracket from + # being glob-expanded (works under both bash -lc wrappers, no quotes). if controller_up; then - n=$(cexec "pgrep -f generative_recommenders | wc -l" | tr -d ' ') + n=$(cexec "set -f; pgrep -f [g]enerative_recommenders | wc -l" | tr -d ' ') else - n=$(dexec "pgrep -f generative_recommenders | wc -l" | tr -d ' ') + n=$(dexec "set -f; pgrep -f [g]enerative_recommenders | wc -l" | tr -d ' ') fi [[ "${n:-0}" -gt 0 ]] } @@ -485,6 +562,11 @@ sup "ckpt_path=$CKPT_PATH keep_last_n=$KEEP_LAST_N ckpt_time_interval=${CKPT_TIM sup "log=$LOG num_train_batches=$NUM_TRAIN_BATCHES die_at_step=$DIE_AT_STEP max_relaunch=$MAX_RELAUNCH" sup "failover: allow=$ALLOW_FAILOVER partition=$PARTITION reservation=${RESERVATION:-} alloc_time=$ALLOC_TIME" +# Reap any failover hold(s) leaked by a PREVIOUS supervisor that died mid-failover +# (e.g. exited on a provisioning error before release_acquired could run). Without +# this, such an orphan keeps pinning a second reservation node indefinitely. +reap_failover_holds "" + cexec "mkdir -p '$CKPT_PATH' '/apps/chcai/tb/$RUN_NAME'" # Initialize this run's metrics log ONCE. launch_smoke_8gpu.sh appends (tee -a), # so every relaunch attempt accumulates into this single file — the full-run From 7c9188f47de9b0f63d5d298029fa3626c46729e7 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Tue, 9 Jun 2026 13:06:59 -0500 Subject: [PATCH 044/127] dlrmv4: anchor sparse-eval cadence to absolute window ts (resume-invariant) _should_eval keyed the every-N-windows cadence off the per-call loop index `i`, so a mid-run resume (which rebases start_ts and restarts train_ts_list at the resume window) re-anchored the eval grid -- e.g. evals shifted from 150,160,170,... to 165,175,185,... after resuming at window 165. Capture the original start_ts as eval_anchor_ts BEFORE the resume block mutates start_ts, and gate eval on (train_ts_list[i] - eval_anchor_ts) % K == 0, so the eval grid is identical on cold start and every resume. Final-window eval preserved; the eval-pool fork is unconditional so dropping the i==0 eval on resume is safe. Co-authored-by: Cursor --- .../dlrm_v3/train/utils.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py index 936ae229e..8b06152a1 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -1274,6 +1274,13 @@ def streaming_train_eval_loop( # cold start and on every resume (the supervisor relaunches with the same # START_TS / NUM_TRAIN_TS). Defaults to the window just past training. requested_end_ts = start_ts + num_train_ts + # Eval-cadence anchor: the ORIGINAL requested start_ts, captured BEFORE the + # resume block rebases start_ts. `_should_eval` keys the every-N-windows + # cadence off the absolute window ts relative to THIS anchor, so the eval + # grid (e.g. 150,160,170,...) is identical on cold start and on every resume. + # (Keying off the per-call loop index instead would re-anchor the grid to + # whatever window a mid-run resume happens to restart from.) + eval_anchor_ts = start_ts # None (Python default) or <0 (the env-binding default) both mean "use the # window just past training", which is stable across resume. eval_holdout_ts_resolved = ( @@ -1637,14 +1644,19 @@ def _should_eval(i: int) -> bool: """Whether to run the full-holdout eval after training window index `i`. `eval_every_n_windows<=1` (default) preserves the per-window cadence. - For K>1 we eval on windows 0, K, 2K, ... and ALWAYS on the final window - so the trajectory ends with an eval point. Gated by `eval_each_window`. + For K>1 we eval when the ABSOLUTE window ts is on the grid anchored at + `eval_anchor_ts` (the original start_ts), i.e. ts in {anchor, anchor+K, + anchor+2K, ...}, and ALWAYS on the final window so the trajectory ends + with an eval point. Anchoring to the absolute ts (not the per-call loop + index `i`) keeps the eval grid (e.g. 150,160,170,...) stable across a + mid-run resume, which rebases start_ts/`train_ts_list` to the resume + window. Gated by `eval_each_window`. """ if not eval_each_window: return False if eval_every_n_windows <= 1: return True - return i % eval_every_n_windows == 0 or i == n_train - 1 + return (train_ts_list[i] - eval_anchor_ts) % eval_every_n_windows == 0 or i == n_train - 1 # Fixed eval set: held-out users' anchors over the resolved holdout window # range, computed ONCE and reused at every eval step. Same anchors every From dbb9e1d3b60de7212ee04a4c0a3ef7d7dfcfc7dd Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Wed, 10 Jun 2026 03:48:40 -0500 Subject: [PATCH 045/127] dlrmv4: multi-node (N>=1) training over RoCE RDMA via consolidated launch_slurm.sh Consolidate the SLURM orchestration, container/RDMA provisioning, and in-container trainer launch into a single self-dispatching scripts/launch_slurm.sh (phases: orchestrate -> provision -> worker) supporting N>=1 nodes; N=1 keeps the legacy single-node path byte-for-byte. Multi-node runs real RDMA over the 8 Broadcom bnxt_re RoCE HCAs. The key fix is an LD_PRELOAD/LD_LIBRARY_PATH overlay of the host's matched rdma-core (v61/v59): the container's stock v34 provider faults RCCL's deep create_qp (256 WRs) against the host kernel uapi -> "ibv_create_qp ... Bad address". The unversioned libibverbs.so symlink in the overlay is required so torch maps only the host lib. TCP bootstrap is pinned to the routable fenic0; RDMA data goes over bnxt_re (GID idx 3, TC 104). Python: derive the global rank (node_rank*gpus_per_node+local_rank), forward master_addr, pass local_world_size to the TorchRec planner and live world_size to metrics. Single-node behavior unchanged. Docs: add docs/multi_node_config.md (enablement details, lessons, and the cluster-specific knobs to change per fabric); README + perf_opt updated for the launch_slurm.sh rename. Co-authored-by: Cursor --- recommendation_v4/README.MD | 26 +- recommendation_v4/docs/multi_node_config.md | 230 +++++++++ recommendation_v4/docs/perf_opt.md | 2 +- .../dlrm_v3/train/train_ranker.py | 62 ++- .../dlrm_v3/train/utils.py | 16 +- recommendation_v4/scripts/launch_slurm.sh | 475 ++++++++++++++++++ .../scripts/launch_smoke_8gpu.sh | 65 --- .../scripts/run_streaming_e2e.sh | 4 +- 8 files changed, 797 insertions(+), 83 deletions(-) create mode 100644 recommendation_v4/docs/multi_node_config.md create mode 100755 recommendation_v4/scripts/launch_slurm.sh delete mode 100755 recommendation_v4/scripts/launch_smoke_8gpu.sh diff --git a/recommendation_v4/README.MD b/recommendation_v4/README.MD index 5c78f6627..dc1ced73b 100644 --- a/recommendation_v4/README.MD +++ b/recommendation_v4/README.MD @@ -4,19 +4,37 @@ This is a fork of [meta-recsys/generative-recommenders](https://github.com/meta- For the original repository and the underlying ICML'24 paper (*Actions Speak Louder than Words*), see the upstream README at the link above. This README focuses on what this fork adds: the Yambda data pipeline, the per-pool gather strategy, and how the data feeds into the HSTU `modules/` (dlrm_v3) path. -## 1. Quick start (8-GPU Yambda) +## 1. Quick start (Yambda, N×8-GPU) + +`scripts/launch_slurm.sh` is the single entry point for **N ≥ 1 nodes**. It +auto-detects its context: run inside the container it takes the single-node +worker path; submitted via `sbatch` it orchestrates the multi-node run +(provision + per-node launch). N=1 is byte-for-byte the legacy single-node path. + +**Single node (8-GPU), inside the container:** ```bash docker exec yambda_8gpu bash -c \ - 'cd /workspace/recommendation_v4 && bash scripts/launch_smoke_8gpu.sh' + 'cd /workspace/recommendation_v4 && bash scripts/launch_slurm.sh' ``` +**Multi-node (N×8-GPU) via SLURM:** + +```bash +sbatch --nodes=2 --partition=meta64 scripts/launch_slurm.sh +``` + +Multi-node uses real RDMA (RoCEv2). The fabric/NCCL setup and every +cluster-specific knob (interfaces, HCAs, GID/TC, RDMA overlay) are documented in +[docs/multi_node_config.md](docs/multi_node_config.md) — read it before running on +a different cluster. + Override the data path or run name without editing the gin: ```bash DLRM_DATA_PATH=/apps/chcai/dlrm_data \ RUN_NAME=my_experiment \ -bash scripts/launch_smoke_8gpu.sh +bash scripts/launch_slurm.sh ``` Data path resolves at runtime via `env_path` gin macros (see [yambda_5b.gin](generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin)). Traces and any per-run outputs land in `results//`. @@ -133,7 +151,7 @@ This means the model sees on average ~1,402 UIH events per sample, not the theor ## 5. Streaming (temporal-order) training -`scripts/launch_smoke_8gpu.sh` defaults to `--mode streaming-train-eval`, which +`scripts/launch_slurm.sh` defaults to `--mode streaming-train-eval`, which trains Yambda in strict wall-clock order instead of shuffling the whole corpus. The timeline is sliced into fixed-duration **windows** (default 1 day, `get_dataset.streaming_window_seconds = 86400`), and the loop walks them forward: diff --git a/recommendation_v4/docs/multi_node_config.md b/recommendation_v4/docs/multi_node_config.md new file mode 100644 index 000000000..52fdbbb69 --- /dev/null +++ b/recommendation_v4/docs/multi_node_config.md @@ -0,0 +1,230 @@ +# Multi-Node Training Enablement (yambda-5b, MI350X / Broadcom bnxt_re RoCE) + +How N-node (N×8-GPU) distributed training was brought up for the yambda-5b HSTU +ranker on the `meta64` cv350 cluster, the hard problems solved, and **exactly +which settings are cluster/fabric-specific** so this can be reused or re-tuned +when the underlying network changes. + +Companion to [`perf_opt.md`](./perf_opt.md) and [`training_recipe.md`](./training_recipe.md). +The single entry point is [`scripts/launch_slurm.sh`](../scripts/launch_slurm.sh); +the Python side is `generative_recommenders/dlrm_v3/train/{train_ranker,utils}.py`. + +--- + +## TL;DR + +- Multi-node works over **real RDMA** (RoCEv2 on 8× Broadcom bnxt_re HCAs). + 2-node = `world_size=16`, clean `rc=0`, ~7.7–8.0k `global_sps` (≈1.28× of + 1-node 6.2k; weak scaling, per-rank batch fixed). +- The one non-obvious blocker was a **userspace RDMA provider ABI mismatch** + inside the container, fixed with an `LD_PRELOAD`/`LD_LIBRARY_PATH` **overlay** + of the host's matched `rdma-core` (no container lib surgery). +- Everything is one script with three auto-detected phases + (`orchestrate` → `provision` → `worker`) plus small Python changes for global + ranks. All cluster-specific knobs are env-overridable and tagged + `[CLUSTER-SPECIFIC]` in the script. + +--- + +## Architecture: one script, three phases + +`launch_slurm.sh` self-dispatches by context (`LAUNCH_SLURM_PHASE`, else +auto-detected via `/.dockerenv`): + +| Phase | Runs on | Does | +|---|---|---| +| `orchestrate` | SLURM batch host | Resolve rendezvous (`MASTER_ADDR/PORT`), ensure container on every node (calls `provision`), then `docker exec` the `worker` phase on every node (one srun task per node). | +| `provision` | each compute node (host) | Ensure the `yambda_primus` container is up (baked image if present, else base image + pip), stage the host RDMA overlay on NFS. | +| `worker` | inside the container | Derive topology, set NCCL/RDMA env, apply the RDMA overlay, spawn this node's 8 GPU ranks via `train_ranker`. `NNODES==1` => legacy single-node path unchanged. | + +Why one script: multi-node enablement is then a single committable file. The +worker phase is also what the streaming-e2e supervisor invokes directly +(single-node, already inside the container), so the production path is unchanged. + +``` +sbatch --nodes=N launch_slurm.sh + │ (batch host: orchestrate) + ├─ srun: provision ──> docker container up + RDMA overlay staged (×N nodes) + └─ srun: docker exec launch_slurm.sh (worker) (×N nodes) + │ in container: topology + NCCL/RDMA env + LD overlay + └─ python train_ranker ──> 8 local ranks ──> RCCL rendezvous over RDMA +``` + +--- + +## The hard problems (lessons learned) + +### 1. RDMA provider ABI mismatch — the core blocker + +**Symptom:** multi-node RCCL died at init with +`ibv_create_qp ... Bad address`. + +**Root cause:** the container image (`rocm/primus:v26.3`) ships an **older** +userspace `rdma-core` (v34, `libbnxt_re-rdmav34.so`) than the **host kernel** +bnxt_re driver's uapi (host `rdma-core` v61 / `libbnxt_re-rdmav59.so`). The v34 +provider enumerates the HCAs and creates *shallow* QPs fine, but **faults when +creating a deep send queue** — RCCL uses `max_send_wr=256`. Verified with a +parameterized verbs probe: v34 `create_qp` is OK at depth ≤16 and faults at ≥64; +the host v59 provider works at **every** depth. So it is purely the **userspace +provider**, not the kernel or the fabric (a 2-node RoCEv2 RDMA-write test passes +on the stock stack, and bare-metal RCCL benchmarks run fine with the host libs). + +**Fix (no container surgery):** the `provision` phase stages the host's matched +`rdma-core` on shared NFS (`$OVERLAY`): + +``` +$OVERLAY/lib/libibverbs.so.1 # host libibverbs v61 +$OVERLAY/lib/libibverbs.so -> .so.1 # UNVERSIONED symlink (critical, see below) +$OVERLAY/lib/libnl-3.so.200, libnl-route-3.so.200 +$OVERLAY/lib/libibverbs/.so # incl. libbnxt_re-rdmav59.so +``` + +The `worker` phase makes RCCL load it at runtime: + +```bash +export LD_LIBRARY_PATH="$OVERLAY/lib:$OVERLAY/lib/libibverbs:$LD_LIBRARY_PATH" +export LD_PRELOAD="$OVERLAY/lib/libibverbs.so.1:$LD_PRELOAD" +``` + +We do **not** modify the container's system libs — only this process tree's +`LD_*`. Single-node and other users keep the stock stack. + +### 2. The UNVERSIONED `libibverbs.so` symlink is mandatory + +An earlier overlay attempt set `LD_LIBRARY_PATH` but still failed with +`Bad address`. Reason: at `import torch` the ROCm stack pulls in the +**unversioned** soname `libibverbs.so` (not `libibverbs.so.1`). If the overlay +only has `libibverbs.so.1`, that unversioned lookup misses the overlay, falls +through to the **container's** old lib, which then occupies the `libibverbs.so.1` +slot — so RCCL's later `dlopen("libibverbs.so.1")` binds the v34 stack and +`create_qp(256)` faults again. The overlay **must** expose +`libibverbs.so -> libibverbs.so.1`. With it (verified via `/proc//maps`), +the process maps **only** the host lib. `LD_PRELOAD` is belt-and-braces so the +host lib claims the soname slot first. + +### 3. Two network planes — pin TCP bootstrap, RDMA for data + +The container is `--network=host`, so RCCL sees **all** host interfaces and, left +to auto-detect, picks the wrong one. These nodes expose: +- `benic1p1..benic8p1` — per-GPU point-to-point RoCE links on `192.168.{1..8}.x/31`. + These are **not node-routable** for plain TCP; the very first bring-up **hung** + in `init_process_group` because RCCL tried the TCP bootstrap over a + non-routable `192.168.x` backend addr. +- `fenic0` — the routable front-end (`10.190.x`). + +So we split the planes explicitly: +- `NCCL_SOCKET_IFNAME=fenic0` → TCP bootstrap/rendezvous over the routable NIC. +- `NCCL_IB_HCA=bnxt_re0..7` → RDMA **data** over the 8 RoCE HCAs (the RoCEv2 + fabric *is* reachable rail-to-rail at the RDMA layer even though plain IP is not). + +### 4. Minimal proven bnxt_re NCCL config + +The minimal set proven on these nodes (matches cmcknigh's bare-metal RCCL +benchmarks): `NCCL_IB_GID_INDEX=3` (RoCEv2 IPv4 GID), `NCCL_IB_TC=104` (RoCE +lossless / PFC traffic class). **Do not** add the heavy +`QPS_PER_CONNECTION / ECE / DMABUF` block — that belongs to a different +(ionic AINIC) fabric and is counterproductive on bnxt_re. GPU-Direct RDMA +(`NCCL_NET_GDR_LEVEL`) is left **off**: it needs DMABUF/peermem, unavailable +in-container here, so RCCL stages through host memory (still real RDMA). + +### 5. Rendezvous must be resolved on the host + +The container image has **no SLURM client** (`scontrol` absent). So the +`orchestrate` phase resolves `MASTER_ADDR` (first host of the allocation) and a +deterministic `MASTER_PORT` (`20000 + job_id % 20000`, same on all nodes) **on +the host** and forwards them into the container via `docker exec -e`. + +### 6. Global rank derivation (Python) + +`mp.start_processes` hands out a node-local `local_rank` (0..7). Every downstream +consumer (data sharding, checkpoint I/O, metrics) needs the **global** rank: + +```python +rank = node_rank * gpus_per_node + local_rank # train_ranker._main_func +device = torch.device(f"cuda:{local_rank}") # CUDA device stays node-local +``` + +Also: `make_optimizer_and_shard(local_world_size=gpus_per_node)` so the TorchRec +planner respects the intra-node GPU count, and `MetricsLogger(world_size=...)` +gets the live world size (the gin default of 8 would mis-normalize multi-node). +`NNODES==1` makes `rank == local_rank` — identical to the old single-node path. + +### 7. `$0` is the staged `slurm_script`, not the repo path + +For an sbatch batch script, `$0` = +`/var/spool/slurmd/job/slurm_script` (node-local), so deriving the script / +repo path from `$0` gives a path that **doesn't exist on other nodes** (`bash +$SELF` → "No such file", and the worker's `cd $REPO` → exit 127). The +`orchestrate` phase instead resolves the real shared-NFS path from SLURM: + +```bash +SCRIPT_PATH=$(scontrol show job "$SLURM_JOB_ID" | grep -oP 'Command=\K\S+') +# fallbacks: $SLURM_SUBMIT_DIR/scripts/launch_slurm.sh, then $SELF +REPO=$(cd "$(dirname "$SCRIPT_PATH")/.." && pwd) +``` + +### 8. `srun ... bash -c "…"` host-vs-remote expansion + +Inside the double-quoted srun command string, **plain `$VAR` expands now on the +batch host** (values computed in orchestrate: `$MASTER_ADDR`, `$SCRIPT_PATH`, …) +while **`\$VAR` is deferred to each compute node** (`\$SLURM_NODEID`, +`\$(hostname)`) where the per-node SLURM env lives. Mixing these up sends every +rank the wrong node id. + +### 9. `memlock` ulimit for QP registration + +`docker run --ulimit memlock=-1:-1` is **required** — RDMA QP memory +registration needs unlimited locked memory. A container started with the default +8 MB memlock fails QP creation regardless of the overlay. + +### 10. Provisioning & the image-bake caveat + +Fresh nodes otherwise re-download a **6.1 GB** ROCm torch wheel + pip + build +torchrec-from-git every time. The script supports a pre-baked image +(`docker commit` → NFS tar → `docker load` offline). **Caveat:** the committed +image is **~127 GB** (ROCm base is huge), so the full-image NFS tar is impractical +(loading it can be slower than re-downloading 6 GB). For true download-avoidance +prefer a **local pip wheelhouse** (`pip install --no-index --find-links` from +~8 GB of NFS wheels) or a **local registry** (ships only the ~35 GB delta layer). +The bake hook is left in (`BAKE_IMAGE=1`) but defaults off; provisioning falls +back to base-image + pip. + +### Debunked theory (do not re-introduce) + +An earlier claim that the container's rdma-core was "too old → 0 devices / +Bad address" and needed an **in-place lib copy** was a red herring: the "0 +devices" came from a *broken in-place copy* of the host EL9 libs (mixing v34 +tooling that links `IBVERBS_PRIVATE_34` with host v61 libs breaks symbol-version +lookup). The stock container enumerates all 8 HCAs fine. The real issue is only +the deep-QP create path; the fix is the **LD overlay**, never in-place surgery. + +--- + +## Cluster-specific settings — change these when the fabric/hardware changes + +All are env-overridable and tagged `[CLUSTER-SPECIFIC]` in `launch_slurm.sh` +(`grep '\[CLUSTER-SPECIFIC\]' scripts/launch_slurm.sh`). + +| Setting | Default (meta64) | What it is | How to find the right value | +|---|---|---|---| +| `#SBATCH --partition` | `meta64` | scheduler partition | `sinfo` | +| bind mounts + default paths | `/home/chcai`, `/apps/chcai` | repo + scratch, **must be shared/NFS on all nodes** | `df -h`, cluster docs | +| `IMAGE` | `rocm/primus:v26.3` | base container (GPU arch + ROCm version) | vendor image registry | +| docker `--device` | `/dev/kfd /dev/dri` (AMD) | GPU passthrough | NVIDIA: `--gpus all` / nvidia runtime | +| `--ulimit memlock` | `-1` | locked mem for RDMA QP | keep `-1` for any RDMA fabric | +| `TORCH_IDX` / torch,vision,audio | `rocm7.2`, `2.12.0+rocm7.2` … | ROCm-version'd wheels | `download.pytorch.org/whl/` | +| `FBGEMM_WHL` | gfx950 wheel on NFS | GPU-arch fbgemm | build/stage per arch | +| `NCCL_SOCKET_IFNAME` | `fenic0` | **routable** host NIC for TCP bootstrap | `ip -br addr` (pick the routable one; NOT the per-GPU RDMA NICs) | +| `NCCL_IB_HCA` | `bnxt_re0..7` | RDMA HCA device names | `ibv_devices` (vendor: `mlx5_*`, `ionic_*`, …) | +| `NCCL_IB_GID_INDEX` | `3` | RoCEv2 IPv4 GID index | `show_gids` (v1/v2 & IPv4/IPv6 differ per port) | +| `NCCL_IB_TC` | `104` | RoCE lossless / PFC traffic class | fabric/switch admin | +| `RDMA_OVERLAY` (+ provider .so) | `/apps/chcai/rdma_host_el9_new` | host rdma-core overlay | only needed if container rdma-core < host kernel uapi; else set `RDMA_OVERLAY=` to disable. Stage the host's matching `/usr/lib64/libibverbs/.so` | + +**Different NIC vendor (e.g. Mellanox `mlx5`)** typically means: change +`NCCL_IB_HCA` names, re-check `NCCL_IB_GID_INDEX`/`NCCL_IB_TC`, and the RDMA +overlay is often **unnecessary** (Mellanox userspace in the image usually matches +the host) — set `RDMA_OVERLAY=` to skip it. + +**Emergency fallback:** `NCCL_NET_TRANSPORT=socket` disables IB and runs +allreduce over TCP (`fenic0`). Functional but ~100–200× slower; use only to +isolate a fabric problem. diff --git a/recommendation_v4/docs/perf_opt.md b/recommendation_v4/docs/perf_opt.md index 627ab74aa..7799848ad 100644 --- a/recommendation_v4/docs/perf_opt.md +++ b/recommendation_v4/docs/perf_opt.md @@ -67,7 +67,7 @@ rocm-smi -d 0 --showclocks # expect sclk ~2000+ MHz under load rocm-smi --setperflevel auto # restore boost ``` -`scripts/launch_smoke_8gpu.sh` now logs the perf level + a live `sclk` sample on +`scripts/launch_slurm.sh` (worker phase) now logs the perf level + a live `sclk` sample on every launch, auto-restores `auto` if it finds a `perf_determinism`/`manual`/`low` lock, and warns (to reset from the host) if it lacks permission inside the container. **Always sanity-check `sclk ≈ 2000+ MHz` before trusting a benchmark.** diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py index 6ae88eba2..55eece518 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py @@ -49,14 +49,27 @@ def _main_func( - rank: int, + local_rank: int, world_size: int, + node_rank: int, + gpus_per_node: int, + master_addr: str, master_port: int, gin_file: str, mode: str, ) -> None: - device = torch.device(f"cuda:{rank}") - logger.info(f"rank: {rank}, world_size: {world_size}, device: {device}") + # `local_rank` is the index handed out by mp.start_processes (0..gpus_per_node-1) + # and indexes this node's GPUs. The GLOBAL rank is what every downstream + # consumer wants (data sharding via StreamingWindowSampler, checkpoint I/O, + # metrics), so derive it once and pass it through as `rank`. Only the CUDA + # device must be node-local. Single-node (node_rank=0) → rank == local_rank, + # exactly as before. + rank = node_rank * gpus_per_node + local_rank + device = torch.device(f"cuda:{local_rank}") + logger.info( + f"rank: {rank} (node_rank={node_rank} local_rank={local_rank}), " + f"world_size: {world_size}, device: {device}" + ) # Phase 1: parse gin early with skip_unknown=True so env-bootstrap # bindings take effect BEFORE any module-level @gin.configurable # discovers itself. This is required because triton @triton.autotune @@ -89,6 +102,7 @@ def _main_func( setup( rank=rank, world_size=world_size, + master_addr=master_addr, master_port=master_port, device=device, ) @@ -101,7 +115,10 @@ def _main_func( model, model_configs, embedding_table_configs = make_model() model, optimizer = make_optimizer_and_shard( - model=model, device=device, world_size=world_size + model=model, + device=device, + world_size=world_size, + local_world_size=gpus_per_node, ) train_dataloader, test_dataloader = make_train_test_dataloaders( hstu_config=model_configs, @@ -129,6 +146,10 @@ def _main_func( window_size=2500, device=device, rank=rank, + # Pass the live world_size so metric normalization is correct at any + # node count; the gin's MetricsLogger.world_size default (=8) is only a + # single-node fallback and would mis-normalize a multi-node run. + world_size=world_size, num_flops_per_sample=num_flops_per_sample, gpu_peak_flops=gpu_peak_flops, model=model, @@ -171,6 +192,7 @@ def _main_func( window_size=1000, device=device, rank=rank, + world_size=world_size, ) eval_loop( rank=rank, @@ -236,14 +258,38 @@ def main() -> None: "train-eval", "streaming-train-eval", ], f"Unsupported mode: {args.mode}" - WORLD_SIZE = int(os.environ.get("WORLD_SIZE", 1)) - MASTER_PORT = str(get_free_port()) + # Distributed topology (single-node defaults reproduce the legacy behavior): + # GPUS_PER_NODE local procs to spawn on THIS node (default: all visible GPUs) + # NNODES/NODE_RANK multi-node fan-out, set by the SLURM launcher + # WORLD_SIZE global rank count = NNODES * GPUS_PER_NODE + # MASTER_ADDR/PORT rank-0 rendezvous; the port MUST match across nodes, so + # honor it from the env when set and only fall back to a + # random free port for the standalone single-node path. + GPUS_PER_NODE = int(os.environ.get("GPUS_PER_NODE", 0)) or torch.cuda.device_count() + NNODES = int(os.environ.get("NNODES", 1)) + NODE_RANK = int(os.environ.get("NODE_RANK", 0)) + WORLD_SIZE = NNODES * GPUS_PER_NODE + MASTER_ADDR = os.environ.get("MASTER_ADDR", "localhost") + MASTER_PORT = str(os.environ.get("MASTER_PORT") or get_free_port()) gin_path = f"{os.path.dirname(__file__)}/gin/{SUPPORTED_CONFIGS[args.dataset]}" + logger.info( + f"launching: nnodes={NNODES} node_rank={NODE_RANK} " + f"gpus_per_node={GPUS_PER_NODE} world_size={WORLD_SIZE} " + f"master={MASTER_ADDR}:{MASTER_PORT}" + ) mp.start_processes( _main_func, - args=(WORLD_SIZE, MASTER_PORT, gin_path, args.mode), - nprocs=WORLD_SIZE, + args=( + WORLD_SIZE, + NODE_RANK, + GPUS_PER_NODE, + MASTER_ADDR, + MASTER_PORT, + gin_path, + args.mode, + ), + nprocs=GPUS_PER_NODE, join=True, start_method="spawn", ) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py index 8b06152a1..c27f38761 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -77,9 +77,15 @@ def setup( - rank: int, world_size: int, master_port: int, device: torch.device + rank: int, + world_size: int, + master_port: int, + device: torch.device, + master_addr: str = "localhost", ) -> dist.ProcessGroup: - os.environ["MASTER_ADDR"] = "localhost" + # Default "localhost" keeps the single-node path unchanged; multi-node + # launches pass the rank-0 host so every node rendezvouses at the same addr. + os.environ["MASTER_ADDR"] = master_addr os.environ["MASTER_PORT"] = str(master_port) BACKEND = dist.Backend.NCCL @@ -339,6 +345,7 @@ def make_optimizer_and_shard( model: torch.nn.Module, device: torch.device, world_size: int, + local_world_size: Optional[int] = None, hbm_cap_gb: int = 260, ) -> Tuple[DistributedModelParallel, torch.optim.Optimizer]: dense_opt_cls, dense_opt_args, dense_opt_factory = ( @@ -357,9 +364,12 @@ def make_optimizer_and_shard( sparse_opt_cls, [param], sparse_opt_args ) sharders = get_default_sharders() + # local_world_size = GPUs per node so the planner respects the intra-node + # (xGMI/NVLink) vs inter-node hierarchy when placing shards. Defaults to + # world_size for the single-node case (no behavior change). planner = EmbeddingShardingPlanner( topology=Topology( - local_world_size=world_size, + local_world_size=local_world_size or world_size, world_size=world_size, compute_device="cuda", hbm_cap=hbm_cap_gb * 1024 * 1024 * 1024, diff --git a/recommendation_v4/scripts/launch_slurm.sh b/recommendation_v4/scripts/launch_slurm.sh new file mode 100755 index 000000000..0b5547497 --- /dev/null +++ b/recommendation_v4/scripts/launch_slurm.sh @@ -0,0 +1,475 @@ +#!/bin/bash +#SBATCH --job-name=yambda_slurm +#SBATCH --nodes=1 +#SBATCH --ntasks-per-node=1 +#SBATCH --exclusive +#SBATCH --partition=meta64 # [CLUSTER-SPECIFIC] partition name +#SBATCH --time=01:10:00 +#SBATCH --output=/apps/chcai/yambda_slurm.%j.out +# ============================================================================= +# launch_slurm.sh — single entry point for the yambda-5b trainer on N>=1 nodes. +# +# Consolidates what used to be three separate files so multi-node enablement is +# ONE committable script (plus the train_ranker.py / utils.py python changes): +# * sbatch_smoke_multinode.sh -> the `orchestrate` phase (host SLURM glue) +# * _provision_yambda_primus.sh -> the `provision` phase (container + RDMA) +# * launch_smoke_8gpu.sh -> the `worker` phase (in-container train) +# +# PHASES (auto-detected from context; force with LAUNCH_SLURM_PHASE=): +# orchestrate Runs on the SLURM batch host (no /.dockerenv). Resolves the +# rendezvous (MASTER_ADDR/PORT), ensures the container on every +# node (provision phase), then `docker exec`s the worker phase on +# every node, one task per node. +# provision Runs on a compute-node host. Ensures the `yambda_primus` +# container is up (loads the pre-baked image if present — no +# internet/pip — else builds from the base image) and stages the +# host RDMA userspace overlay on shared NFS. +# worker Runs INSIDE the container. Sets the distributed topology + +# NCCL/RDMA env and spawns this node's GPU ranks via train_ranker. +# N==1 transparently uses the legacy single-node path (localhost, +# node_rank 0), byte-for-byte as before, so the streaming-e2e +# supervisor's direct `bash scripts/launch_slurm.sh` is unchanged. +# +# USAGE +# Multi-node (N>=1): sbatch --nodes=2 scripts/launch_slurm.sh +# Single-node direct: bash scripts/launch_slurm.sh (already inside container; +# what run_streaming_e2e.sh invokes per relaunch) +# Perf pair: +# LOG=/apps/chcai/perf_1node.log NUM_TRAIN_BATCHES=200 NUM_EVAL_BATCHES=0 \ +# EVAL_EACH_WINDOW=0 METRIC_LOG_FREQ=20 \ +# sbatch --nodes=1 --job-name=y1 scripts/launch_slurm.sh +# LOG=/apps/chcai/perf_2node.log NUM_TRAIN_BATCHES=200 NUM_EVAL_BATCHES=0 \ +# EVAL_EACH_WINDOW=0 METRIC_LOG_FREQ=20 \ +# sbatch --nodes=2 --job-name=y2 scripts/launch_slurm.sh +# # then: bash scripts/compare_node_perf.sh /apps/chcai/perf_1node.log /apps/chcai/perf_2node.log +# +# ONE-TIME IMAGE BAKE (so fresh nodes skip the multi-GB torch download + pip): +# BAKE_IMAGE=1 LAUNCH_SLURM_PHASE=provision bash scripts/launch_slurm.sh +# (commits the deps-installed container to $BAKED_IMAGE and `docker save`s it to +# $BAKED_TAR on NFS; subsequent provisions `docker load` it offline.) +# +# ----------------------------------------------------------------------------- +# PORTABILITY — what to change for a DIFFERENT cluster / network / hardware. +# Every such knob is also tagged inline with "[CLUSTER-SPECIFIC]" (grep for it). +# All are env-overridable, so you can adapt without editing this file. +# +# A) SLURM / scheduler +# - #SBATCH --partition=meta64 : partition name. CHANGE per cluster. +# - #SBATCH --time / --exclusive : policy; adjust to taste. +# +# B) Filesystems (must be shared/NFS across ALL nodes — this script re-invokes +# itself and reads the overlay + data from these paths cluster-wide) +# - /home/chcai (repo + this script) and /apps/chcai (scratch: logs, overlay, +# baked tar, data, pip tarball). CHANGE both the bind mounts in the +# `docker run` (provision) and the default LOG/BAKED_TAR/OVERLAY/PIP_* paths. +# +# C) Container image / GPU software stack (tied to the GPU arch + ROCm version) +# - IMAGE=rocm/primus:v26.3 : base image. ROCm/AMD-specific. +# - docker run --device=/dev/kfd --device=/dev/dri --group-add video : AMD ROCm +# device passthrough. For NVIDIA this is --gpus all / nvidia runtime instead. +# - --ulimit memlock=-1 : REQUIRED for RDMA QP registration (do not drop). +# - TORCH_IDX (rocm7.2), torch/vision/audio ==*+rocm7.2, FBGEMM_WHL (a gfx950 +# wheel), torchrec pin : the whole deps set is arch/ROCm-version-specific. +# +# D) Network fabric — THE trickiest part; defaults are PROVEN on meta64 cv350 +# (Broadcom bnxt_re RoCEv2). On a different fabric these almost certainly change +# (see the worker-phase block for the full rationale): +# - NCCL_SOCKET_IFNAME=fenic0 : the ONE routable host NIC for TCP bootstrap. +# Find yours with `ip -br addr`; the per-GPU RDMA NICs are usually NOT +# routable for plain TCP, so auto-detect hangs init — you MUST pin this. +# - NCCL_IB_HCA=bnxt_re0..7 : the RDMA HCA device names. List with `ibv_devices`. +# Different NIC vendor (e.g. mlx5_*, ionic_*) => different names AND a +# different userspace provider, which changes the RDMA overlay below. +# - NCCL_IB_GID_INDEX=3 : RoCEv2 IPv4 GID index. Check `show_gids`; v1/v2 and +# IPv4/IPv6 live at different indices per port. +# - NCCL_IB_TC=104 : RoCE lossless (PFC) traffic class. Fabric/switch-specific. +# - RDMA overlay (provision phase): only needed when the CONTAINER's rdma-core +# is older than the HOST kernel driver's uapi (our bnxt_re v34-vs-v59 case). +# Different NIC/host => different /usr/lib64 provider .so to stage, or the +# overlay may be unnecessary entirely (set RDMA_OVERLAY= to disable). If RDMA +# can't be made to work, NCCL_NET_TRANSPORT=socket falls back to TCP. +# +# E) Not cluster-specific (auto-derived): GPUS_PER_NODE (torch.cuda.device_count), +# NNODES/NODE_RANK/MASTER_ADDR (from SLURM), WORLD_SIZE. +# ============================================================================= +set -uo pipefail + +# Absolute path to THIS script so the orchestrate phase can re-invoke it on every +# node (home is shared NFS, so the same path resolves cluster-wide). +SELF=$(cd "$(dirname "$0")" && pwd)/$(basename "$0") +REPO_ROOT=$(cd "$(dirname "$0")/.." && pwd) + +# ---- phase detection -------------------------------------------------------- +PHASE="${LAUNCH_SLURM_PHASE:-}" +if [ -z "$PHASE" ]; then + if [ -f /.dockerenv ]; then PHASE=worker; else PHASE=orchestrate; fi +fi + +# ---- shared config (env-overridable) ---------------------------------------- +CONTAINER=${CONTAINER:-yambda_primus} +REPO=${REPO:-$REPO_ROOT} # repo path inside the container +IMAGE=${IMAGE:-rocm/primus:v26.3} # [CLUSTER-SPECIFIC] ROCm/arch base image +BAKED_IMAGE=${BAKED_IMAGE:-yambda_primus_baked:latest} +BAKED_TAR=${BAKED_TAR:-/apps/chcai/yambda_primus_baked.tar} # [CLUSTER-SPECIFIC] shared-NFS path +USE_BAKED=${USE_BAKED:-1} +OVERLAY=${RDMA_OVERLAY:-/apps/chcai/rdma_host_el9_new} # [CLUSTER-SPECIFIC] shared-NFS RDMA overlay + +# ============================================================================= +# PHASE: orchestrate (SLURM batch host) +# ============================================================================= +orchestrate() { + # When run as the SLURM batch script, $0 is the node-local staged copy + # (/var/spool/slurmd/job/slurm_script), so $SELF / $REPO_ROOT are WRONG + # here (they don't exist on other nodes). Resolve the REAL shared-NFS script + # path + repo root from SLURM so we can re-invoke this script on every node and + # `cd` to the right repo inside the container. + SCRIPT_PATH=$(scontrol show job "${SLURM_JOB_ID:-0}" 2>/dev/null | grep -oP 'Command=\K\S+') + [ -f "${SCRIPT_PATH:-}" ] || SCRIPT_PATH="${SLURM_SUBMIT_DIR:-$REPO_ROOT}/scripts/launch_slurm.sh" + [ -f "$SCRIPT_PATH" ] || SCRIPT_PATH="$SELF" + REPO=$(cd "$(dirname "$SCRIPT_PATH")/.." && pwd) + + LOG=${LOG:-/apps/chcai/yambda_slurm.${SLURM_JOB_ID:-manual}.log} + + # Smoke defaults — override via env for a perf run (see header USAGE). + MODE=${MODE:-streaming-train-eval} + START_TS=${START_TS:-150} + NUM_TRAIN_TS=${NUM_TRAIN_TS:-1} + NUM_TRAIN_BATCHES=${NUM_TRAIN_BATCHES:-20} + NUM_EVAL_BATCHES=${NUM_EVAL_BATCHES:-10} + EVAL_EACH_WINDOW=${EVAL_EACH_WINDOW:-1} + EVAL_EVERY_N_WINDOWS=${EVAL_EVERY_N_WINDOWS:-1} + METRIC_LOG_FREQ=${METRIC_LOG_FREQ:-5} + FORCE_PROVISION=${FORCE_PROVISION:-0} + + : > "$LOG" + echo "[$(date)] launch_slurm/orchestrate: job=${SLURM_JOB_ID:-?} nodes=${SLURM_JOB_NODELIST:-?} nnodes=${SLURM_NNODES:-1}" | tee -a "$LOG" + echo "[$(date)] resolved SCRIPT_PATH=$SCRIPT_PATH REPO=$REPO" | tee -a "$LOG" + echo "[$(date)] config: MODE=$MODE START_TS=$START_TS NUM_TRAIN_TS=$NUM_TRAIN_TS NUM_TRAIN_BATCHES=$NUM_TRAIN_BATCHES NUM_EVAL_BATCHES=$NUM_EVAL_BATCHES METRIC_LOG_FREQ=$METRIC_LOG_FREQ" | tee -a "$LOG" + + # Rendezvous resolved on the HOST (the container image has no SLURM client). + MASTER_ADDR=$(scontrol show hostnames "$SLURM_JOB_NODELIST" 2>/dev/null | head -1) + MASTER_ADDR=${MASTER_ADDR:-localhost} + MASTER_PORT=$(( 20000 + ${SLURM_JOB_ID:-0} % 20000 )) + echo "[$(date)] rendezvous: MASTER_ADDR=$MASTER_ADDR MASTER_PORT=$MASTER_PORT" | tee -a "$LOG" + + # Optional NCCL/RCCL fabric overrides — forwarded into the container only when + # set at submit time (docker exec does NOT inherit the srun task env). The + # worker phase applies its own validated multi-node bnxt_re defaults when these + # are unset. Common: NCCL_NET_TRANSPORT=socket (TCP fallback), NCCL_DEBUG=INFO. + NCCL_ENV_ARGS="" + for v in NCCL_NET_TRANSPORT NCCL_DEBUG NCCL_SOCKET_IFNAME NCCL_IB_HCA NCCL_IB_GID_INDEX \ + NCCL_IB_TC NCCL_IB_TIMEOUT NCCL_IGNORE_CPU_AFFINITY RCCL_MSCCL_ENABLE NCCL_NET_GDR_LEVEL \ + NCCL_IB_PCI_RELAXED_ORDERING NCCL_IB_USE_INLINE NCCL_IB_QPS_PER_CONNECTION \ + NCCL_IB_ECE_ENABLE NCCL_DMABUF_ENABLE NCCL_GDRCOPY_ENABLE NCCL_GDR_FLUSH_DISABLE \ + NCCL_PXN_DISABLE NCCL_CHECKS_DISABLE NCCL_CROSS_NIC RDMA_OVERLAY; do + eval "val=\${$v:-}" + if [ -n "$val" ]; then NCCL_ENV_ARGS="$NCCL_ENV_ARGS -e $v=$val"; fi + done + + # TRICKY — variable expansion inside the `srun ... bash -c "..."` blocks below: + # the string is double-quoted, so PLAIN $VAR expands NOW on the batch host (e.g. + # $MASTER_ADDR, $CONTAINER, $SCRIPT_PATH — values computed above), while + # BACKSLASH-escaped \$VAR is passed through literally and expands LATER on each + # compute node inside the srun task (e.g. \$SLURM_NODEID, \$(hostname)) where the + # per-node SLURM_* env actually lives. Mixing these up sends every rank the + # wrong node id or breaks the docker exec — keep the \$ on per-node values. + + # --- step 1: ensure the container is up on every node ---------------------- + echo "[$(date)] ensuring container '$CONTAINER' on all nodes (force=$FORCE_PROVISION)" | tee -a "$LOG" + srun --ntasks-per-node=1 bash -c " + if [ \"$FORCE_PROVISION\" = \"1\" ] || ! docker exec $CONTAINER true >/dev/null 2>&1; then + echo \"[\$(hostname)] (re)provisioning container\" + LAUNCH_SLURM_PHASE=provision CONTAINER=$CONTAINER IMAGE=$IMAGE \ + BAKED_IMAGE=$BAKED_IMAGE BAKED_TAR=$BAKED_TAR USE_BAKED=$USE_BAKED \ + BAKE_IMAGE=${BAKE_IMAGE:-0} RDMA_OVERLAY=$OVERLAY REPO=$REPO bash $SCRIPT_PATH + else + echo \"[\$(hostname)] container already up\" + fi + " 2>&1 | tee -a "$LOG" + + # --- step 2: launch the worker (trainer) inside the container on every node - + echo "[$(date)] launching trainer (worker phase) on all nodes" | tee -a "$LOG" + srun --ntasks-per-node=1 bash -c " + docker exec \ + -e LAUNCH_SLURM_PHASE=worker \ + -e SLURM_NNODES=\$SLURM_NNODES \ + -e SLURM_NODEID=\$SLURM_NODEID \ + -e SLURM_PROCID=\$SLURM_PROCID \ + -e SLURM_JOB_NODELIST=\"\$SLURM_JOB_NODELIST\" \ + -e SLURM_JOB_ID=\$SLURM_JOB_ID \ + -e MASTER_ADDR=$MASTER_ADDR \ + -e MASTER_PORT=$MASTER_PORT \ + -e HSTU_HAMMER_KERNEL=${HSTU_HAMMER_KERNEL:-TRITON} \ + -e MODE=$MODE \ + -e START_TS=$START_TS \ + -e NUM_TRAIN_TS=$NUM_TRAIN_TS \ + -e EVAL_EACH_WINDOW=$EVAL_EACH_WINDOW \ + -e EVAL_EVERY_N_WINDOWS=$EVAL_EVERY_N_WINDOWS \ + -e NUM_TRAIN_BATCHES=$NUM_TRAIN_BATCHES \ + -e NUM_EVAL_BATCHES=$NUM_EVAL_BATCHES \ + -e METRIC_LOG_FREQ=$METRIC_LOG_FREQ \ + -e TRAIN_SPLIT_PERCENTAGE=${TRAIN_SPLIT_PERCENTAGE:-0.90} \ + -e SPLIT_SALT=${SPLIT_SALT:-0} \ + -e EVAL_HOLDOUT_TS=${EVAL_HOLDOUT_TS:--1} \ + -e EVAL_HOLDOUT_NUM_WINDOWS=${EVAL_HOLDOUT_NUM_WINDOWS:-1} \ + ${RUN_NAME:+-e RUN_NAME=$RUN_NAME} \ + ${TENSORBOARD_LOG_PATH:+-e TENSORBOARD_LOG_PATH=$TENSORBOARD_LOG_PATH} \ + ${CKPT_PATH:+-e CKPT_PATH=$CKPT_PATH} \ + -e LOG=$LOG \ + $NCCL_ENV_ARGS \ + $CONTAINER bash -lc 'cd $REPO && LAUNCH_SLURM_PHASE=worker bash scripts/launch_slurm.sh' + " 2>&1 | tee -a "$LOG" + rc=${PIPESTATUS[0]} + echo "[$(date)] launch_slurm/orchestrate finished rc=$rc" | tee -a "$LOG" + exit $rc +} + +# ============================================================================= +# PHASE: provision (compute-node host) +# ============================================================================= +provision() { + export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${PATH:-}" + DOCKER=$(command -v docker 2>/dev/null || true); DOCKER=${DOCKER:-/usr/bin/docker} + FBGEMM_WHL=${FBGEMM_WHL:-/apps/chcai/FBGEMM/fbgemm_gpu/dist/fbgemm_gpu_nightly_rocm-2026.6.2-cp312-cp312-linux_x86_64.whl} # [CLUSTER-SPECIFIC] gfx950/ROCm wheel + TORCH_IDX=${TORCH_IDX:-https://download.pytorch.org/whl/rocm7.2} # [CLUSTER-SPECIFIC] ROCm version index + echo "[provision] host=$(hostname) container=$CONTAINER docker=$DOCKER" + + # Resolve which image to run + whether deps must be installed. Prefer a pre-baked + # image (deps already installed) to skip the multi-GB torch download + pip / + # torchrec-from-git build on every fresh node: + # 1) baked image in this node's docker -> use it, skip deps + # 2) baked image tar on NFS -> docker load (local, no internet) + # 3) neither -> base image + pip (slow path, which + # can then be baked via BAKE_IMAGE=1) + NEED_DEPS=1 + RUN_IMAGE="$IMAGE" + if [ "$USE_BAKED" = "1" ]; then + if "$DOCKER" image inspect "$BAKED_IMAGE" >/dev/null 2>&1; then + echo "[provision] using baked image $BAKED_IMAGE (deps preinstalled, no download)" + RUN_IMAGE="$BAKED_IMAGE"; NEED_DEPS=0 + elif [ -f "$BAKED_TAR" ]; then + echo "[provision] loading baked image from $BAKED_TAR (local, no internet)..." + if "$DOCKER" load -i "$BAKED_TAR" >/dev/null 2>&1 && "$DOCKER" image inspect "$BAKED_IMAGE" >/dev/null 2>&1; then + RUN_IMAGE="$BAKED_IMAGE"; NEED_DEPS=0; echo "[provision] baked image loaded" + else + echo "[provision] WARNING: docker load failed; falling back to base-image + pip" + fi + fi + fi + if ! "$DOCKER" image inspect "$RUN_IMAGE" >/dev/null 2>&1; then + echo "[provision] pulling $RUN_IMAGE (this can take a while)..."; "$DOCKER" pull "$RUN_IMAGE" + fi + + echo "[provision] (re)starting container $CONTAINER from $RUN_IMAGE" + "$DOCKER" rm -f "$CONTAINER" >/dev/null 2>&1 || true + "$DOCKER" run -d --name "$CONTAINER" \ + --network=host --ipc=host --shm-size=64g \ + --device=/dev/kfd --device=/dev/dri --group-add video \ + `# [CLUSTER-SPECIFIC] AMD ROCm device passthrough; NVIDIA uses --gpus all / nvidia runtime` \ + --cap-add=SYS_PTRACE --cap-add=CAP_SYS_ADMIN --cap-add=IPC_LOCK \ + --ulimit memlock=-1:-1 --ulimit stack=67108864:67108864 \ + `# memlock=-1 is REQUIRED for RDMA QP memory registration — do not drop` \ + --security-opt seccomp=unconfined --privileged \ + -v /home/chcai:/home/chcai \ + -v /apps/chcai:/apps/chcai \ + `# [CLUSTER-SPECIFIC] shared-NFS bind mounts: repo + scratch (overlay/logs/data)` \ + -w "$REPO" \ + "$RUN_IMAGE" sleep infinity + + # --- RDMA userspace overlay for in-container RCCL (bnxt_re) ----------------- + # The image (rocm/primus, rdma-core 50/libbnxt_re-rdmav34) ships an OLDER RDMA + # userspace than the host kernel bnxt_re driver. The stock v34 provider faults + # RCCL's deep-queue create_qp (max_send_wr=256) against the newer kernel uapi + # -> "ibv_create_qp ... Bad address". Fix: stage the host's matched rdma-core + # (libibverbs v61 + libbnxt_re-rdmav59 + libnl) on NFS so the worker phase makes + # RCCL load it via LD_PRELOAD + LD_LIBRARY_PATH. The UNVERSIONED libibverbs.so + # symlink is essential (import torch pulls the unversioned soname; without it + # the lookup falls through to the container v34 lib and the fix regresses). + if [ "${FORCE_OVERLAY:-0}" != "1" ] && ls "$OVERLAY/lib/libibverbs/"libbnxt_re-rdmav*.so >/dev/null 2>&1 && [ -L "$OVERLAY/lib/libibverbs.so" ]; then + echo "[provision] host RDMA overlay already staged at $OVERLAY (shared NFS) — skipping" + else + echo "[provision] staging host RDMA userspace overlay -> $OVERLAY" + rm -rf "${OVERLAY}.tmp" 2>/dev/null + mkdir -p "${OVERLAY}.tmp/lib/libibverbs" "${OVERLAY}.tmp/libibverbs.d" + cp -L /usr/lib64/libibverbs.so.1 /usr/lib64/libnl-3.so.200 /usr/lib64/libnl-route-3.so.200 "${OVERLAY}.tmp/lib/" 2>/dev/null || true + ln -sf libibverbs.so.1 "${OVERLAY}.tmp/lib/libibverbs.so" + cp -L /usr/lib64/libibverbs/*.so "${OVERLAY}.tmp/lib/libibverbs/" 2>/dev/null || true + cp /etc/libibverbs.d/*.driver "${OVERLAY}.tmp/libibverbs.d/" 2>/dev/null || true + if ls "${OVERLAY}.tmp/lib/libibverbs/"libbnxt_re-rdmav*.so >/dev/null 2>&1; then + rm -rf "$OVERLAY" 2>/dev/null + mv "${OVERLAY}.tmp" "$OVERLAY" 2>/dev/null || { mkdir -p "$OVERLAY"; cp -a "${OVERLAY}.tmp/." "$OVERLAY/"; } + echo "[provision] host RDMA overlay staged: $(ls "$OVERLAY/lib/libibverbs" | wc -l) providers + libibverbs.so symlink" + else + echo "[provision] WARNING: host bnxt_re provider not found at /usr/lib64/libibverbs — multi-node RDMA will fail 'Bad address'; use NCCL_NET_TRANSPORT=socket" + fi + fi + + if [ "$NEED_DEPS" = "0" ]; then + echo "[provision] baked image — deps preinstalled; verifying imports only" + "$DOCKER" exec "$CONTAINER" bash -lc ' +python -c "import torch, fbgemm_gpu, torchrec, polars, xxhash, gin; print(\"imports OK,\", torch.__version__, torch.version.hip, torch.cuda.device_count(), \"gpus\")" +' || echo "[provision] WARNING: baked-image import smoke failed" + else + echo "[provision] installing recipe deps (base image, slow path)" + # Install misc deps FIRST, then pin the rocm torch stack + fbgemm + torchrec + # LAST with --no-deps so nothing pulls a CUDA torch over the rocm build. + "$DOCKER" exec "$CONTAINER" bash -lc ' +set -e +echo "=== native torch ==="; python -c "import torch;print(torch.__version__)" || true +echo "=== misc python deps ===" +pip install --no-cache-dir polars-u64-idx pyarrow pyyaml tqdm psutil numba xxhash gin-config \ + absl-py pandas tensorboard torchmetrics tensordict pyre-extensions iopath typing-inspect 2>&1 | tail -3 || true +echo "=== rocm torch stack (force, no-deps, LAST) ===" +pip install --force-reinstall --no-deps --index-url '"$TORCH_IDX"' \ + torch==2.12.0+rocm7.2 torchvision==0.27.0+rocm7.2 torchaudio==2.11.0+rocm7.2 +echo "=== fbgemm (local gfx950 wheel) ===" +pip install --force-reinstall --no-deps '"$FBGEMM_WHL"' +echo "=== torchrec v2026.06.01.00 (force, no-deps) ===" +pip install --force-reinstall --no-deps "git+https://github.com/pytorch/torchrec.git@v2026.06.01.00" +echo "=== import smoke ===" +python -c "import torch, fbgemm_gpu, torchrec, polars, xxhash, gin; print(\"imports OK,\", torch.__version__, torch.version.hip, torch.cuda.device_count(), \"gpus\")" +' + fi + + # --- one-time bake: snapshot the deps-installed container into a reusable image + # and save it to NFS so future nodes skip the download/pip path entirely. + if [ "${BAKE_IMAGE:-0}" = "1" ]; then + echo "[provision] baking: docker commit $CONTAINER -> $BAKED_IMAGE" + if "$DOCKER" commit "$CONTAINER" "$BAKED_IMAGE" >/dev/null; then + echo "[provision] saving $BAKED_IMAGE -> $BAKED_TAR (one-time, tens of GB)" + if "$DOCKER" save "$BAKED_IMAGE" -o "${BAKED_TAR}.tmp.$$" && mv -f "${BAKED_TAR}.tmp.$$" "$BAKED_TAR"; then + echo "[provision] bake done: $(ls -lh "$BAKED_TAR" 2>/dev/null | awk '{print $5}')" + else + echo "[provision] WARNING: docker save failed"; rm -f "${BAKED_TAR}.tmp.$$" 2>/dev/null + fi + else + echo "[provision] WARNING: docker commit failed" + fi + fi + echo "[provision] DONE" +} + +# ============================================================================= +# PHASE: worker (inside the container) +# ============================================================================= +worker() { + cd "$REPO_ROOT" + LOG=${LOG:-/apps/chcai/yambda_5b_8gpu.log} + # Append (not truncate): under the streaming-e2e supervisor a run may relaunch + # many times into the SAME $LOG; the supervisor initializes it once at run start. + echo "[$(date)] REPO_ROOT=$REPO_ROOT" | tee -a "$LOG" + + # polars-u64-idx (NOT stock polars) — yambda parquet's flat-explode overruns + # 32-bit row index. Reserved node has no outbound DNS, so install from a + # pre-staged tarball under /apps/chcai/. Override PIP_LOCAL_TGZ for other hosts. + PIP_LOCAL_TGZ=${PIP_LOCAL_TGZ:-/apps/chcai/pip_local_yambda.tgz} # [CLUSTER-SPECIFIC] shared-NFS path + PIP_LOCAL_DIR=${PIP_LOCAL_DIR:-/tmp/pip_local} + if [ ! -f "$PIP_LOCAL_DIR/lib/python3.12/site-packages/polars/__init__.py" ]; then + rm -rf "$PIP_LOCAL_DIR" + mkdir -p "$PIP_LOCAL_DIR" && tar xzf "$PIP_LOCAL_TGZ" -C "$(dirname "$PIP_LOCAL_DIR")" 2>&1 | tail -3 | tee -a "$LOG" + fi + + export PYTHONPATH="$PIP_LOCAL_DIR/lib/python3.12/site-packages:$REPO_ROOT:${PYTHONPATH:-}" + export HOME=${HOME:-/tmp} + echo "[$(date)] PYTHONPATH=$PYTHONPATH" | tee -a "$LOG" + python -c "import torch, fbgemm_gpu, torchrec, polars, xxhash, gin; print('imports OK,', torch.__version__, torch.cuda.device_count(),'gpus')" 2>&1 | tee -a "$LOG" + + export HIP_VISIBLE_DEVICES=${HIP_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7} + export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7} + + # --- distributed topology --------------------------------------------------- + GPUS_PER_NODE=$(python -c "import torch; print(torch.cuda.device_count())") + # Multi-node when launched one-task-per-node under SLURM (SLURM_NNODES>1); + # otherwise fall through to legacy single-node defaults (localhost, node_rank 0). + if [ "${SLURM_NNODES:-1}" -gt 1 ] && [ -n "${SLURM_JOB_NODELIST:-}" ]; then + NNODES=${SLURM_NNODES} + NODE_RANK=${SLURM_NODEID:-${SLURM_PROCID:-0}} + # PREFER a MASTER_ADDR/PORT forwarded from the orchestrate phase (resolved on + # the host, which has scontrol); the container image carries no SLURM client. + if [ -z "${MASTER_ADDR:-}" ]; then + MASTER_ADDR=$(scontrol show hostnames "$SLURM_JOB_NODELIST" 2>/dev/null | head -1) + MASTER_ADDR=${MASTER_ADDR:-localhost} + fi + MASTER_PORT=${MASTER_PORT:-$(( 20000 + ${SLURM_JOB_ID:-0} % 20000 ))} + else + NNODES=${NNODES:-1} + NODE_RANK=${NODE_RANK:-0} + MASTER_ADDR=${MASTER_ADDR:-localhost} + MASTER_PORT=${MASTER_PORT:-} # empty => train_ranker picks a free port + fi + export NNODES NODE_RANK GPUS_PER_NODE MASTER_ADDR MASTER_PORT + export WORLD_SIZE=$(( NNODES * GPUS_PER_NODE )) + echo "[$(date)] topology: nnodes=$NNODES node_rank=$NODE_RANK gpus_per_node=$GPUS_PER_NODE world_size=$WORLD_SIZE master=$MASTER_ADDR:${MASTER_PORT:-}" | tee -a "$LOG" + + # RCCL/NCCL cross-node knobs (multi-node only; single-node leaves auto-detect). + # The container is --network=host so RCCL sees ALL host interfaces; split the + # two planes explicitly: TCP bootstrap over the routable fenic0, RDMA data over + # the 8 Broadcom bnxt_re RoCE HCAs (the per-GPU benic* 192.168.x/31 links are + # NOT node-routable for TCP — auto-detect there hangs init). + if [ "$NNODES" -gt 1 ]; then + # [CLUSTER-SPECIFIC] routable host NIC for TCP bootstrap (find via `ip -br addr`). + export NCCL_SOCKET_IFNAME=${NCCL_SOCKET_IFNAME:-fenic0} + NCCL_NET_TRANSPORT=${NCCL_NET_TRANSPORT:-ib} + if [ "$NCCL_NET_TRANSPORT" = "socket" ]; then + export NCCL_IB_DISABLE=1 + echo "[$(date)] NCCL: IB disabled — allreduce over TCP (fenic0). Functional, not RDMA-fast." | tee -a "$LOG" + else + # bnxt_re userspace provider ABI overlay (REQUIRED for RCCL). The stock v34 + # provider faults RCCL's create_qp (256 WRs) against the host kernel uapi + # ("Bad address"); the host v61/v59 set staged by the provision phase works. + # The libibverbs.so (UNVERSIONED) symlink + LD_PRELOAD are both required so + # the torch process maps ONLY the host lib (see provision phase comment). + if [ -e "$OVERLAY/lib/libibverbs.so.1" ]; then + [ -e "$OVERLAY/lib/libibverbs.so" ] || ln -sf libibverbs.so.1 "$OVERLAY/lib/libibverbs.so" 2>/dev/null || true + export LD_LIBRARY_PATH="$OVERLAY/lib:$OVERLAY/lib/libibverbs:${LD_LIBRARY_PATH:-}" + export LD_PRELOAD="$OVERLAY/lib/libibverbs.so.1${LD_PRELOAD:+:$LD_PRELOAD}" + echo "[$(date)] NCCL: bnxt_re provider overlay -> $OVERLAY (host rdma-core v61/v59; symlink+LD_PRELOAD so RCCL binds the host lib for QP creation)" | tee -a "$LOG" + else + echo "[$(date)] WARNING: RDMA overlay $OVERLAY missing — RCCL QP creation will fail 'Bad address' on stock v34 provider; set RDMA_OVERLAY or use NCCL_NET_TRANSPORT=socket" | tee -a "$LOG" + fi + # MINIMAL bnxt_re set PROVEN on these meta64 cv350 nodes (cmcknigh RCCL + # benchmarks + confirmed e2e here). NCCL_IB_TC=104 (RoCE lossless PFC class) + # is required; do NOT add the ionic-AINIC QPS/ECE/DMABUF block. + # [CLUSTER-SPECIFIC] RDMA HCA names (`ibv_devices`); other vendors => mlx5_*/ionic_* + export NCCL_IB_HCA=${NCCL_IB_HCA:-bnxt_re0,bnxt_re1,bnxt_re2,bnxt_re3,bnxt_re4,bnxt_re5,bnxt_re6,bnxt_re7} + export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-3} # [CLUSTER-SPECIFIC] RoCEv2 IPv4 GID idx (`show_gids`) + export NCCL_IB_TC=${NCCL_IB_TC:-104} # [CLUSTER-SPECIFIC] RoCE lossless/PFC traffic class + export NCCL_IB_TIMEOUT=${NCCL_IB_TIMEOUT:-14} + export NCCL_IGNORE_CPU_AFFINITY=${NCCL_IGNORE_CPU_AFFINITY:-1} + export RCCL_MSCCL_ENABLE=${RCCL_MSCCL_ENABLE:-0} + # GPU-Direct RDMA needs DMABUF/peermem (neither in-container here) — leave + # GDR off so RCCL stages through host memory (still real RDMA over bnxt_re). + export NCCL_NET_GDR_LEVEL=${NCCL_NET_GDR_LEVEL:-0} + echo "[$(date)] NCCL: RDMA over bnxt_re (GID idx ${NCCL_IB_GID_INDEX}, TC ${NCCL_IB_TC}, GDR_LEVEL=${NCCL_NET_GDR_LEVEL}; meta64 bnxt_re config, validated)" | tee -a "$LOG" + fi + fi + export NCCL_DEBUG=${NCCL_DEBUG:-WARN} + export HSTU_HAMMER_KERNEL=${HSTU_HAMMER_KERNEL:-} + export PYTORCH_CUDA_ALLOC_CONF=${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True} + + # --- GPU clock sanity guard ------------------------------------------------- + # A leftover perf_determinism cap (half clock) silently slows every kernel ~1.9x. + # Log the perf level + a live sclk sample and try to restore boost (non-fatal). + if command -v rocm-smi >/dev/null 2>&1; then + echo "[$(date)] GPU perf-level check:" | tee -a "$LOG" + rocm-smi --showperflevel 2>/dev/null | grep -iE "GPU\[[0-9]+\]" | tee -a "$LOG" || true + if rocm-smi --showperflevel 2>/dev/null | grep -iqE "Performance Level: *(perf_determinism|manual|low)"; then + echo "[$(date)] WARNING: GPUs not in 'auto' perf level — attempting --setperflevel auto" | tee -a "$LOG" + rocm-smi --setperflevel auto 2>/dev/null | grep -iE "set to auto" | tee -a "$LOG" \ + || echo "[$(date)] WARNING: could not set perf level (no permission?). Run 'rocm-smi --setperflevel auto' on the HOST before benchmarking — clocks may be capped." | tee -a "$LOG" + fi + echo "[$(date)] sclk sample (GPU0):$(rocm-smi -d 0 --showclocks 2>/dev/null | grep -i 'sclk clock level' | sed -E 's/.*sclk clock level//')" | tee -a "$LOG" || true + fi + + echo "[$(date)] launching train_ranker with WORLD_SIZE=$WORLD_SIZE" | tee -a "$LOG" + python -m generative_recommenders.dlrm_v3.train.train_ranker \ + --dataset yambda-5b --mode "${MODE:-streaming-train-eval}" 2>&1 | tee -a "$LOG" +} + +# ---- dispatch --------------------------------------------------------------- +case "$PHASE" in + orchestrate) orchestrate ;; + provision) provision ;; + worker) worker ;; + *) echo "launch_slurm.sh: unknown LAUNCH_SLURM_PHASE='$PHASE'" >&2; exit 2 ;; +esac diff --git a/recommendation_v4/scripts/launch_smoke_8gpu.sh b/recommendation_v4/scripts/launch_smoke_8gpu.sh deleted file mode 100755 index eaa0aa19b..000000000 --- a/recommendation_v4/scripts/launch_smoke_8gpu.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/bash -# 8-GPU yambda-5b run. Resolves the package root from this script's location, -# so it works from any container mount point. Dataset path is in the gin file -# (generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin). -set -uo pipefail - -REPO_ROOT=$(cd "$(dirname "$0")/.." && pwd) -cd "$REPO_ROOT" - -LOG=${LOG:-/apps/chcai/yambda_5b_8gpu.log} -# Append (not truncate): under the streaming-e2e supervisor a run may relaunch -# many times into the SAME $LOG, and we want the full NE/AUC history preserved -# across attempts. The supervisor initializes ($LOG) once at run start. For a -# standalone invocation, set a fresh $LOG (or truncate it yourself) per run. -echo "[$(date)] REPO_ROOT=$REPO_ROOT" | tee -a "$LOG" - -# polars-u64-idx (NOT stock polars) — yambda parquet's flat-explode overruns -# 32-bit row index. Reserved node has no outbound DNS, so we install from a -# pre-staged tarball under /apps/chcai/. Override PIP_LOCAL_TGZ for other hosts. -PIP_LOCAL_TGZ=${PIP_LOCAL_TGZ:-/apps/chcai/pip_local_yambda.tgz} -PIP_LOCAL_DIR=${PIP_LOCAL_DIR:-/tmp/pip_local} -if [ ! -f "$PIP_LOCAL_DIR/lib/python3.12/site-packages/polars/__init__.py" ]; then - rm -rf "$PIP_LOCAL_DIR" - mkdir -p "$PIP_LOCAL_DIR" && tar xzf "$PIP_LOCAL_TGZ" -C "$(dirname "$PIP_LOCAL_DIR")" 2>&1 | tail -3 | tee -a "$LOG" -fi - -export PYTHONPATH="$PIP_LOCAL_DIR/lib/python3.12/site-packages:$REPO_ROOT:${PYTHONPATH:-}" -export HOME=${HOME:-/tmp} -echo "[$(date)] PYTHONPATH=$PYTHONPATH" | tee -a "$LOG" -python -c "import torch, fbgemm_gpu, torchrec, polars, xxhash, gin; print('imports OK,', torch.__version__, torch.cuda.device_count(),'gpus')" 2>&1 | tee -a "$LOG" - -export HIP_VISIBLE_DEVICES=${HIP_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7} -export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7} -export WORLD_SIZE=$(python -c "import torch; print(torch.cuda.device_count())") -# HSTU attention backend is selected in the gin (make_model.hammer_kernel), -# defaulting to TRITON — fused/flash-style, so it avoids the dense [B,H,N,N] -# score tensor the PYTORCH path materializes (~32 GiB at N=2048/bs=1024) and is -# both faster and far lighter on HBM. Only export HSTU_HAMMER_KERNEL=PYTORCH -# before launch for a one-off fallback (e.g. a ROCm Triton PassManager error). -export HSTU_HAMMER_KERNEL=${HSTU_HAMMER_KERNEL:-} -export PYTORCH_CUDA_ALLOC_CONF=${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True} - -# --- GPU clock sanity guard --------------------------------------------------- -# Leftover node state once pinned all 8 GPUs into `perf_determinism` at half -# clock (1093 vs 2200 MHz max). That uniformly slowed every Triton kernel ~1.9x -# and silently masked real perf changes for an entire debugging session. Always -# log the perf level + a live sclk sample so a capped run is obvious from the -# log, and try to restore boost. Fully non-fatal (rocm-smi may be absent or -# lack permission inside the container — in that case reset from the host). -if command -v rocm-smi >/dev/null 2>&1; then - echo "[$(date)] GPU perf-level check:" | tee -a "$LOG" - rocm-smi --showperflevel 2>/dev/null | grep -iE "GPU\[[0-9]+\]" | tee -a "$LOG" || true - if rocm-smi --showperflevel 2>/dev/null | grep -iqE "Performance Level: *(perf_determinism|manual|low)"; then - echo "[$(date)] WARNING: GPUs not in 'auto' perf level — attempting --setperflevel auto" | tee -a "$LOG" - rocm-smi --setperflevel auto 2>/dev/null | grep -iE "set to auto" | tee -a "$LOG" \ - || echo "[$(date)] WARNING: could not set perf level (no permission?). Run 'rocm-smi --setperflevel auto' on the HOST before benchmarking — clocks may be capped." | tee -a "$LOG" - fi - echo "[$(date)] sclk sample (GPU0):$(rocm-smi -d 0 --showclocks 2>/dev/null | grep -i 'sclk clock level' | sed -E 's/.*sclk clock level//')" | tee -a "$LOG" || true -fi -# ----------------------------------------------------------------------------- - -echo "[$(date)] launching train_ranker with WORLD_SIZE=$WORLD_SIZE" | tee -a "$LOG" - -python -m generative_recommenders.dlrm_v3.train.train_ranker \ - --dataset yambda-5b --mode "${MODE:-streaming-train-eval}" 2>&1 | tee -a "$LOG" diff --git a/recommendation_v4/scripts/run_streaming_e2e.sh b/recommendation_v4/scripts/run_streaming_e2e.sh index 5009ecbce..f97a0e483 100755 --- a/recommendation_v4/scripts/run_streaming_e2e.sh +++ b/recommendation_v4/scripts/run_streaming_e2e.sh @@ -543,7 +543,7 @@ launch() { RUN_NAME=$RUN_NAME \ TENSORBOARD_LOG_PATH=/apps/chcai/tb/$RUN_NAME/ \ LOG=$LOG \ - bash scripts/launch_smoke_8gpu.sh; + bash scripts/launch_slurm.sh; echo \"E2E_RUN_EXIT=\$? \$(date '+%F %T')\" >> $LOG " } @@ -568,7 +568,7 @@ sup "failover: allow=$ALLOW_FAILOVER partition=$PARTITION reservation=${RESERVAT reap_failover_holds "" cexec "mkdir -p '$CKPT_PATH' '/apps/chcai/tb/$RUN_NAME'" -# Initialize this run's metrics log ONCE. launch_smoke_8gpu.sh appends (tee -a), +# Initialize this run's metrics log ONCE. launch_slurm.sh (worker) appends (tee -a), # so every relaunch attempt accumulates into this single file — the full-run # NE/AUC history survives crashes and node failover instead of being truncated # on each relaunch. (Starting the supervisor = starting a fresh run.) In ATTACH From 15c4a00ba18dad6567bce6ac3eaf942a4e327193 Mon Sep 17 00:00:00 2001 From: suachong Date: Wed, 10 Jun 2026 14:10:32 -0500 Subject: [PATCH 046/127] local dlrmv4 changes: docker setup, run scripts, walkthrough docs, smoke log path Co-authored-by: Cursor --- recommendation_v4/Dockerfile | 80 +++ recommendation_v4/Dockerfile.nvidia | 92 +++ .../docs/v4_vs_v2_and_hstu_walkthrough.md | 534 ++++++++++++++++++ recommendation_v4/scripts/run_docker.sh | 59 ++ 4 files changed, 765 insertions(+) create mode 100644 recommendation_v4/Dockerfile create mode 100644 recommendation_v4/Dockerfile.nvidia create mode 100644 recommendation_v4/docs/v4_vs_v2_and_hstu_walkthrough.md create mode 100755 recommendation_v4/scripts/run_docker.sh diff --git a/recommendation_v4/Dockerfile b/recommendation_v4/Dockerfile new file mode 100644 index 000000000..112d605df --- /dev/null +++ b/recommendation_v4/Dockerfile @@ -0,0 +1,80 @@ +# MI350X path — implements docs/training_recipe.md §"MI350X". + +FROM rocm/primus:v26.3 + +ENV PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 + +WORKDIR /workspace/recommendation_v4 + +# torch / torchvision / torchaudio — training_recipe.md:38-40. +RUN pip install --upgrade --no-deps \ + --index-url https://download.pytorch.org/whl/rocm7.2 \ + torch==2.12.0+rocm7.2 \ + torchvision==0.27.0+rocm7.2 \ + torchaudio==2.11.0+rocm7.2 + +# torchrec — training_recipe.md:43. +RUN pip install --force-reinstall --no-deps \ + "git+https://github.com/pytorch/torchrec.git@v2026.06.01.00" + +# fbgemm_gpu — training_recipe.md:42. Build from FBGEMM commit 10b77573 for +# gfx950 against the replaced torch. ~30-60 min. +RUN apt-get update && apt-get install -y --no-install-recommends git build-essential && \ + rm -rf /var/lib/apt/lists/* && \ + git clone --recursive https://github.com/pytorch/FBGEMM.git /tmp/FBGEMM && \ + cd /tmp/FBGEMM && \ + git checkout 10b775730212923f65f7b78f79b6a01d80cf3c29 && \ + git submodule update --init --recursive && \ + cd fbgemm_gpu && \ + # Filter `fairscale` and the torch family from fbgemm's requirements.txt: + # fairscale pulls a CPU torch that would clobber the +rocm7.2 wheel installed + # above. fairscale is a distributed-training lib used by fbgemm tests, not + # by the build itself. + grep -v -E '^(fairscale|torch|torchvision|torchaudio)([<>=!]|$)' requirements.txt > /tmp/req.txt && \ + pip install -r /tmp/req.txt && \ + python setup.py -j 32 bdist_wheel \ + --build-target=default \ + --build-variant=rocm \ + -DHIP_ROOT_DIR=/opt/rocm \ + -DAMDGPU_TARGETS=gfx950 && \ + pip install --force-reinstall --no-deps dist/fbgemm_gpu_nightly_rocm*.whl && \ + cd / && rm -rf /tmp/FBGEMM + +# polars-u64-idx — training_recipe.md:44 (mandatory; yambda-5b > 4.29 B rows). +# Remaining packages — training_recipe.md:156-159 ("Additional Python deps") plus +# `datasets` + `huggingface_hub`, which the recipe does not list but +# preprocess_public_data.py:278 imports to download yambda from HuggingFace. +RUN pip install \ + polars-u64-idx==1.33.1 \ + gin-config \ + absl-py \ + datasets \ + huggingface_hub \ + pyre-extensions \ + iopath \ + typing-inspect \ + psutil \ + tqdm \ + pyyaml \ + lightning-utilities && \ + # torchmetrics and tensordict declare `torch` as a dep; without --no-deps + # pip pulls torch==2.12.0+cu130 from PyPI which clobbers the +rocm7.2 wheel + # we installed above (libtorch_hip.so disappears, fbgemm_gpu fails to load). + pip install --no-deps \ + torchmetrics==1.0.3 \ + tensordict + +# Smoke-test the 6 imports the launch script checks at +# scripts/launch_smoke_8gpu.sh:26. +RUN python -c "import torch, fbgemm_gpu, torchrec, polars, xxhash, gin; \ +print('torch', torch.__version__, '| hip', getattr(torch.version, 'hip', None))" + +COPY . /workspace/recommendation_v4 + +ENV PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \ + HSTU_HAMMER_KERNEL=TRITON \ + DLRM_DATA_PATH=/data/mlperf_dlrm_v4 + +CMD ["bash"] diff --git a/recommendation_v4/Dockerfile.nvidia b/recommendation_v4/Dockerfile.nvidia new file mode 100644 index 000000000..388ab8e5e --- /dev/null +++ b/recommendation_v4/Dockerfile.nvidia @@ -0,0 +1,92 @@ +# B200 path — implements docs/training_recipe.md §"B200". + +FROM nvcr.io/nvidia/pytorch:26.04-py3 + +ENV PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 + +WORKDIR /workspace/recommendation_v4 + +# torch / triton — training_recipe.md:137-138, 149-150. Native to the image +# and must NOT be reinstalled (CUPTI / sm_100 support depends on it). + +# torchrec — training_recipe.md:152. Nightly cu130 wheel, --no-deps. +RUN pip install --force-reinstall --no-deps \ + --index-url https://download.pytorch.org/whl/nightly/cu130 \ + torchrec==1.7.0.dev20260601+cu130 + +# fbgemm_gpu — training_recipe.md:151. Build from FBGEMM commit 10b77573 for +# sm_100 against the image's native torch. ~55 min (sm_100 TBE-forward via ptxas). +# NOTE: --nvml_lib_path diverges from training_recipe.md:151. The recipe points +# at /usr/lib/x86_64-linux-gnu/libnvidia-ml.so, which is mounted only at +# `docker run --gpus all` time. During `docker build` no GPU runtime is +# attached, so we link against the NVML stub that ships inside the CUDA SDK in +# the NGC image; the real driver-side libnvidia-ml.so is used at runtime. +RUN apt-get update && apt-get install -y --no-install-recommends git build-essential && \ + rm -rf /var/lib/apt/lists/* && \ + git clone --recursive https://github.com/pytorch/FBGEMM.git /tmp/FBGEMM && \ + cd /tmp/FBGEMM && \ + git checkout 10b775730212923f65f7b78f79b6a01d80cf3c29 && \ + git submodule update --init --recursive && \ + cd fbgemm_gpu && \ + # Filter `fairscale` and the torch family from fbgemm's requirements.txt: + # fairscale pulls a CPU torch that would clobber the image's native torch. + # fairscale is a distributed-training lib used by fbgemm tests, not by the + # build itself. + grep -v -E '^(fairscale|torch|torchvision|torchaudio)([<>=!]|$)' requirements.txt > /tmp/req.txt && \ + pip install -r /tmp/req.txt && \ + TORCH_CUDA_ARCH_LIST=10.0 python setup.py bdist_wheel \ + --build-target default \ + --build-variant cuda \ + --package_channel nightly \ + --nvml_lib_path /usr/local/cuda/lib64/stubs/libnvidia-ml.so && \ + pip install --force-reinstall --no-deps dist/fbgemm_gpu_nightly-*.whl && \ + cd / && rm -rf /tmp/FBGEMM + +# polars-u64-idx — training_recipe.md:153 (mandatory; yambda-5b > 4.29 B rows). +# Remaining packages — training_recipe.md:156-159 ("Additional Python deps") plus +# `datasets` + `huggingface_hub`, which the recipe does not list but +# preprocess_public_data.py:278 imports to download yambda from HuggingFace. +RUN pip install \ + polars-u64-idx==1.33.1 \ + gin-config \ + absl-py \ + datasets \ + huggingface_hub \ + pyre-extensions \ + iopath \ + typing-inspect \ + psutil \ + tqdm \ + pyyaml \ + lightning-utilities && \ + # torchmetrics and tensordict declare `torch` as a dep; without --no-deps + # pip resolves and reinstalls torch, clobbering the image's native NGC + # torch (which would break CUPTI + sm_100 support per training_recipe.md:199). + pip install --no-deps \ + torchmetrics==1.0.3 \ + tensordict + +# Smoke-test that packages are installed at the right versions. Cannot dlopen +# fbgemm_gpu / torchrec here because their SONAME deps (libnvidia-ml.so.1, etc.) +# only resolve when the container runs with `--gpus all` — which docker build +# can't do. The real 6-import check at scripts/launch_smoke_8gpu.sh:26 runs at +# `docker run` time when the driver is mounted in. +RUN python -c "import torch, polars, xxhash, gin; \ +print('torch', torch.__version__, '| cuda', getattr(torch.version, 'cuda', None)); \ +import importlib.metadata as m; \ +print('fbgemm_gpu installed:', m.version('fbgemm_gpu_nightly')); \ +print('torchrec installed: ', m.version('torchrec'))" + +COPY . /workspace/recommendation_v4 + +# B200 runtime env — training_recipe.md:184-195. +ENV PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \ + HSTU_HAMMER_KERNEL=TRITON \ + TORCH_CUDA_ARCH_LIST=10.0 \ + HBM_CAP_GB=150 \ + TRITON_CACHE_DIR=/workspace/recommendation_v4/.triton_cache \ + DLRM_DATA_PATH=/data/mlperf_dlrm_v4 + +CMD ["bash"] diff --git a/recommendation_v4/docs/v4_vs_v2_and_hstu_walkthrough.md b/recommendation_v4/docs/v4_vs_v2_and_hstu_walkthrough.md new file mode 100644 index 000000000..4ef46c069 --- /dev/null +++ b/recommendation_v4/docs/v4_vs_v2_and_hstu_walkthrough.md @@ -0,0 +1,534 @@ +# recommendation_v4 (HSTU + Yambda-5b) — reference + +A walkthrough of what the proposed `recommendation_v4` MLPerf-training benchmark +is, how it differs from `recommendation_v2`, what the HSTU model is composed of, +and how to download the dataset and run training as-is. + +All claims below are grounded in code/config paths inside this tree. Every +numeric constant cites a `file:line` source. Where doc and source disagree, the +source wins and the discrepancy is called out. + +--- + +## 0. Sources of truth + +The following files were read to assemble this document. If you change any of +them, audit this doc against the change. + +- `training/recommendation_v2/torchrec_dlrm/README.MD` (v2 reference) +- `training/recommendation_v4/README.MD` (v4 fork overview) +- `training/recommendation_v4/docs/training_recipe.md` (v4 stacks/configs) +- `training/recommendation_v4/generative_recommenders/modules/stu.py` (HSTU layer) +- `training/recommendation_v4/generative_recommenders/modules/dlrm_hstu.py` (top-level `DlrmHSTU` + config dataclass) +- `training/recommendation_v4/generative_recommenders/modules/hstu_transducer.py` (preprocessor → STU stack → postprocessor) +- `training/recommendation_v4/generative_recommenders/ops/pytorch/pt_hstu_attention.py` (reference HSTU attention math in plain PyTorch) +- `training/recommendation_v4/generative_recommenders/ops/hstu_attention.py` (kernel dispatcher: PYTORCH / TRITON / TRITON_CC) +- `training/recommendation_v4/generative_recommenders/dlrm_v3/configs.py` (per-dataset HSTU config + embedding tables) +- `training/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin` (run config) +- `training/recommendation_v4/generative_recommenders/dlrm_v3/preprocess_public_data.py` (Yambda HuggingFace downloader/preprocessor) +- `training/recommendation_v4/generative_recommenders/dlrm_v3/datasets/yambda.py` (dataset feeding HSTU) +- `training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py` (metrics logger / `auc_threshold` consumer) +- `training/recommendation_v4/scripts/launch_smoke_8gpu.sh` (run wrapper) + +--- + +## 1. `recommendation_v4` vs `recommendation_v2` + +v4 is **not** an evolution of v2 — it replaces a tabular CTR benchmark +(DLRMv2 + DCN on Criteo 1 TB) with a **sequential generative-recommender +benchmark** (HSTU on Yandex Yambda-5b). Codebase, dataset, task, loss +labeling, hyperparameters, and software stack are all different. They share +basically nothing except the "recommendation" label. + +### 1.1 Upstream codebase / repo origin + +| | v2 | v4 | +|---|---|---| +| Upstream repo | `pytorch/torchrec` examples (DLRM) | fork of `meta-recsys/generative-recommenders` (`README.MD:3`) | +| Layout | single dir: `torchrec_dlrm/` with `dlrm_main.py` | full repo tree: `generative_recommenders/`, `configs/`, `scripts/`, `main.py`, `setup.py`, gin-driven | +| Config style | argparse CLI flags | gin-config files under `generative_recommenders/dlrm_v3/train/gin/` (e.g. `yambda_5b.gin`) | + +### 1.2 Model architecture + +| | v2 | v4 | +|---|---|---| +| Model | **DLRM v2** — dense MLP + sparse embeddings + feature interaction (paper: Naumov et al. 1906.00091) | **HSTU** — Hierarchical Sequential Transducer Units (ICML'24 *Actions Speak Louder than Words*) (`README.MD:3`) | +| Interaction arch | DCN v2: `--interaction_type=dcn --dcn_num_layers=3 --dcn_low_rank_dim=512` (`recommendation_v2/torchrec_dlrm/README.MD:167-169`) | Transformer-style sequential self-attention over a User Interaction History (UIH) of length 2048, jagged-attention TRITON kernel (`README.MD:114, 132`; `training_recipe.md:57, 71`) | +| Embedding dim | 128 (`recommendation_v2/torchrec_dlrm/README.MD:157`) | 512 (`dlrm_v3/configs.py:33, 353`) | +| Pipeline | TorchRec model-parallel embeddings + data-parallel MLP, overlapped (`recommendation_v2/torchrec_dlrm/README.MD:3`) | TorchRec sharded embeddings + HSTU ranker; per-GPU HBM cap 260 GiB MI350X / 150 GiB B200 (`training_recipe.md:59, 176`) | + +### 1.3 Dataset + +| | v2 | v4 | +|---|---|---| +| Dataset | **Criteo 1 TB click logs** → multi-hot preprocessed variant (~3.8 TB materialized) (`recommendation_v2/torchrec_dlrm/README.MD:142-146`) | **Yambda-5b** (Yandex music, HuggingFace `yandex/yambda`, 5b variant) (`README.MD:3, 28`) | +| Domain | CTR prediction on tabular ads features (26 categorical + 13 dense) | Sequential music-recommendation events (listen / like / skip / dislike / unlike / undislike) per-user timelines | +| Size | `TOTAL_TRAINING_SAMPLES=4,195,197,692` rows (`recommendation_v2/torchrec_dlrm/README.MD:153`) | 4.76 B events, 1.00 M users, 9.39 M items; 3.23 B usable training anchors (`README.MD:62-69`) | +| Storage layout | numpy contiguous shuffled `.npy` (or preprocessed multi-hot bin) | parquet: `train_sessions.parquet` 47 GB, `test_events.parquet` 152 MB, etc. (`README.MD:40-52`) | +| Preprocessing | `process_Criteo_1TB_Click_Logs_dataset.sh` — 700 GB RAM, 1–2 days, then `materialize_synthetic_multihot_dataset.py` | `generative_recommenders.dlrm_v3.preprocess_public_data --dataset yambda-5b` — ~53 min end-to-end for 5b (`README.MD:54`) | +| Embedding cardinalities | `num_embeddings_per_feature` 26-vec, top entries 40 M (`recommendation_v2/torchrec_dlrm/README.MD:161`) | item 9.39 M, artist 1.29 M, album 3.37 M, uid 1.00 M, + 7 cross-features up to 100 M (`dlrm_v3/configs.py:40-48, 686-722`) | +| Required pre-processor pkg | none unusual | **`polars-u64-idx`** because yambda-5b exceeds polars' 32-bit row index (`training_recipe.md:44, 102-103`) | + +### 1.4 Task formulation / supervision + +| | v2 | v4 | +|---|---|---| +| Task | binary CTR (click / no-click) | sequential next-action ranking: given UIH, predict whether the candidate LISTEN event will be a "listen_plus" (`played_ratio ≥ 50%`) (`README.MD:103`) | +| Label | Criteo click label | `action_weight` bitmask on the candidate; supervision masked to `(supervision_bitmask & task_weight) > 0` with `task_weight = 1` (LP bit) → only `listen_plus` candidates supervise (`README.MD:103`) | +| Loss | BCE | BCE on `listen_plus` task | + +### 1.5 Target metric + +| | v2 | v4 | +|---|---|---| +| Target | **AUROC ≥ 0.80275** within 1 epoch on Criteo (`recommendation_v2/torchrec_dlrm/README.MD:173`) | `MetricsLogger.auc_threshold = 0.80275` (`yambda_5b.gin:107`). Same numeric value as v2 — likely inherited from the upstream DLRM-DCNv2 reporting convention rather than independently chosen for HSTU. Consumed in `dlrm_v3/utils.py:587-608` to log `time_to_auc_0.80275_sec` as soon as the `listen_plus` task's AUC crosses the threshold. Confirm with the proposing team whether this is the intended final benchmark target or a placeholder. | + +### 1.6 Training hyperparameters + +| | v2 (MLPerf example, 8 GPU) | v4 (`yambda_5b.gin`, 8 GPU) | +|---|---|---| +| Global batch | 65,536 (`recommendation_v2/torchrec_dlrm/README.MD:154`) | **8,192** (`batch_size=1024 × world_size=8`) (`yambda_5b.gin:1, 44`). Note `docs/training_recipe.md:65, 182` shows `32 × 8 = 256` — that doc has drifted; the gin file is the launch source of truth. | +| Epochs | 1 (`recommendation_v2/torchrec_dlrm/README.MD:163`) | 1 (`yambda_5b.gin:81`) | +| Dense optimizer | Adagrad, lr 0.005 (`recommendation_v2/torchrec_dlrm/README.MD:170-171`) | **Adam**, lr 1e-3, betas (0.95, 0.999), eps 1e-8 (`yambda_5b.gin:19-24`) | +| Sparse optimizer | (Adagrad on embeddings via TorchRec) | **RowWiseAdagrad**, lr 1e-3, betas (0.95, 0.999), eps 1e-8 (`yambda_5b.gin:27-32`) | +| Precision | fp32 (no bf16 flag in v2 example) | **bf16** mixed precision, gated on the TRITON HSTU kernel (`yambda_5b.gin:8`; `training_recipe.md:58, 109-111`) | +| Sequence length | n/a (non-sequential model) | `history_length=2039`, `max_seq_len=2048` (`yambda_5b.gin:74, 78`) | + +### 1.7 Software stack + +| | v2 | v4 (MI350X) | v4 (B200) | +|---|---|---|---| +| Container | none specified (bare AWS p4d, CUDA 11.0, NCCL 2.10.3) (`recommendation_v2/torchrec_dlrm/README.MD:37`) | `rocm/primus:v26.3` (`training_recipe.md:24`) | `nvcr.io/nvidia/pytorch:26.04-py3` (`training_recipe.md:132`) | +| GPU target | A100 40 GB | **MI350X** (`gfx950`, ROCm 7.2.1, 288 GiB HBM3e) | **B200** (`sm_100`, ~183 GiB HBM) | +| torch | TorchRec example era; CUDA 11.0 | `2.12.0+rocm7.2` (`training_recipe.md:38`) | `2.12.0a0` native NGC (CUDA 13.2) (`training_recipe.md:149`) | +| triton | not central | `3.6.0` (image native; required for HSTU TRITON backend) (`training_recipe.md:41`) | `3.6.0` (`training_recipe.md:150`) | +| fbgemm_gpu | TorchRec default | `fbgemm_gpu_nightly_rocm-2026.6.2` built from FBGEMM `10b77573` for `gfx950` (`training_recipe.md:42`) | same SHA, built for `sm_100` (`training_recipe.md:151`) | +| torchrec | (whatever TorchRec was current) | `1.7.0a0+bf55480` (`v2026.06.01.00`) (`training_recipe.md:43`) | `1.7.0.dev20260601+cu130` (`training_recipe.md:152`) | +| Launcher | `torchx … dist.ddp` | `scripts/launch_smoke_8gpu.sh` | `scripts/launch_smoke_8gpu.sh` | +| Key kernel | TorchRec EmbeddingBag + DCN | **HSTU TRITON jagged-attention** (`HSTU_HAMMER_KERNEL=TRITON`) (`training_recipe.md:71`) | same (`training_recipe.md:188`) | + +--- + +## 2. HSTU model walkthrough + +### 2.1 What HSTU is, in one paragraph + +**HSTU = Hierarchical Sequential Transducer Units**, from the Meta paper +*Actions Speak Louder than Words* (ICML'24). It is a **decoder-only Transformer +variant**, redesigned for *recommendation* sequences (very long, very ragged, +heavy on categorical features). The block looks like a standard transformer +block superficially — attention + MLP — but two things are different from +GPT/SASRec attention: + +1. **Pointwise SiLU instead of softmax** in the attention non-linearity (no + log-sum-exp normalization). +2. **Gated output**: an extra projected stream `U` multiplies the attention + output before the residual. + +Everything else (residual connections, layer-norm, multi-head, positional +encoding, causal masking, KV-cache) is conventional transformer. The "S" in +STU = "Sequential Transducer Unit" = one HSTU block. + +### 2.2 The composition: top-level model (DLRM-v3 / `DlrmHSTU`) + +The full thing in `dlrm_hstu.py` is a small pipeline. Top-down: + +``` +KeyedJaggedTensor of raw ids + │ + ▼ +[1] TorchRec EmbeddingCollection (≈150 G sparse params, sharded across GPUs) + │ emits per-feature jagged embedding lookups + ▼ +[2] ContextualPreprocessor (interleaves UIH + appends candidate, adds + positional / action / timestamp encodings) + │ output: jagged sequence of length L per user, dim = transducer_embedding_dim + ▼ +[3] HSTUTransducer ── STUStack of N HSTULayers (the "HSTU" attention blocks) + │ output: contextualized per-position embedding + ▼ +[4] DefaultMultitaskModule (linear → BCE on listen_plus bit) + │ + ▼ +Per-anchor logit → BCE loss +``` + +For yambda-5b the per-dataset overrides in `dlrm_v3/configs.py:78-90, 346-425` +give: + +| component | value | source | +|---|---|---| +| embedding tables | `item_id` 9.39 M × 512, `artist_id` 1.29 M × 512, `album_id` 3.37 M × 512, `uid` 1.00 M × 512, + 7 cross-features (e.g. `user_x_artist` 100 M × 512) | `dlrm_v3/configs.py:686-722` | +| embedding dim | 512 (`HSTU_EMBEDDING_DIM`) | `dlrm_v3/configs.py:33, 353` | +| HSTU layers | **5** (`hstu_attn_num_layers=5`) | `dlrm_v3/configs.py:82` | +| attention heads | 4 | `dlrm_v3/configs.py:79` | +| Q/K dim per head | 128 | `dlrm_v3/configs.py:81` | +| V/U (linear) dim per head | 128 | `dlrm_v3/configs.py:80` | +| transducer embedding dim | 512 | `dlrm_v3/configs.py:85, 354` | +| dropout | input 0.2, linear 0.1 | `dlrm_v3/configs.py:87-88` | +| max attention budget (model) | 8192 (yambda default; gin further caps to 2048 via `get_hstu_configs.max_seq_len = 2048` in `yambda_5b.gin:78`) | `dlrm_v3/configs.py:355` | +| task | `listen_plus`, BINARY_CLASSIFICATION, BCE | `dlrm_v3/configs.py:419-424` | + +**Sparse-side parameter count, by table** (just the explicit ones; cross-features +add 282 M more rows × 512 dim ≈ 144 G params, which dominate): + +``` +item_id : 9_390_624 × 512 ≈ 4.81 B +artist_id : 1_293_395 × 512 ≈ 662 M +album_id : 3_367_692 × 512 ≈ 1.72 B +uid : 1_000_001 × 512 ≈ 512 M +crosses : ~282 M × 512 ≈ 144.4 B ← dominant +``` + +This is overwhelmingly an embedding-bound model — the dense HSTU stack (5 +layers × ~1 M parameters each) is a rounding error next to the embedding +tables, which is why `make_optimizer_and_shard.hbm_cap_gb = 260` and why +TorchRec sharding is central. + +### 2.3 Inside one STU (HSTU) layer + +From `modules/stu.py:182-246, 292-355`. A single STU layer holds **four** +weight matrices, not the usual two (QKV + out): + +``` +_uvqk_weight : (E, (hidden_dim·2 + attn_dim·2) · num_heads) +_uvqk_beta : (...,) bias for the above +_input_norm : LayerNorm(E) +_output_weight : (hidden_dim · num_heads · 3, E) +_output_norm : LayerNorm +``` + +Forward pass on input `x` of shape `[L, E]` (jagged): + +#### 2.3.1 Fused U/V/Q/K projection + +``` +normed = LayerNorm(x) +[U | V | Q | K] = normed @ _uvqk_weight + _uvqk_beta # one GEMM, then split + # U, V ∈ R^{H·hidden_dim} + # Q, K ∈ R^{H·attn_dim} +``` + +Compared to a regular transformer, you get an **extra projected stream `U`**. +`U` will gate the attention output later. + +#### 2.3.2 HSTU attention (the core difference vs softmax attention) + +Reference math, exactly as written in +`ops/pytorch/pt_hstu_attention.py:151, 167, 179, 182`: + +```python +qk_attn = einsum("bhxa,bhya->bhxy", Q, K) * alpha # alpha = 1 / sqrt(attn_dim) +qk_attn = F.silu(qk_attn) / max_seq_len # ← pointwise SiLU, scalar divide +qk_attn = qk_attn * valid_attn_mask # mask (see 2.3.3) +attn = einsum("bhxd,bhdv->bhxv", qk_attn, V) +``` + +Contrast with a vanilla transformer: + +```python +qk = (Q @ K.T) / sqrt(d) +qk = softmax(qk + mask, dim=-1) # ← softmax normalises rows +attn = qk @ V +``` + +Two consequences of dropping softmax: + +- **No row-wise normalization** → the per-key contribution is decoupled across + positions. The paper argues this is *better* for recommendation, because a + 5-year-old "like" event shouldn't have its weight diluted just because the + user has a longer history (which softmax would do). +- **Numerically more delicate**: the recipe warns *"`pt_hstu_attention`'s QK + einsum backward overflows in bf16 at N > 1k and produces NaN at step 1; bf16 + is only safe with TRITON"* (`docs/training_recipe.md:109-111`). The TRITON + kernel handles bf16 accumulation carefully; the reference PyTorch path + doesn't. + +#### 2.3.3 Custom attention mask (`_get_valid_attn_mask`, `pt_hstu_attention.py:32-84`) + +HSTU supports four mask-combination knobs simultaneously: + +- **causal**: lower triangle only (standard). +- **target-aware** (`num_targets`): the last `num_targets` positions are the + candidate targets; their "row index" is clamped so all targets see the same + prefix (the user's UIH) but cannot peek at each other. +- **max_attn_len** (sliding window): each position attends only to the previous + `max_attn_len` events — bounds the receptive field for very long histories. +- **contextual_seq_len**: the first `contextual_seq_len` positions are + *contextual* tokens (uid + cross-features). They are allowed to attend to + everything (and everything attends back to them), regardless of causal order. + This is how `uid` / `user_x_artist` etc. get full visibility despite living + at the head of the sequence. + +#### 2.3.4 Output: gated MLP + +From `stu.py:336-354` → `hstu_compute_output`: + +``` +y = SwishLayerNorm(attn) # SiLU(x · sigmoid(x)) then LN +y = concat([y, U]) @ _output_weight # gating with the U stream +y = y · x + dropout # residual back to original x +``` + +The `U · y` gating is the second non-standard piece. It is reminiscent of +GLU / SwiGLU but applied to the *attention output*, not just an MLP. + +#### 2.3.5 Stack + +`STUStack` (`stu.py:426`) is just `nn.ModuleList` of N `STULayer`s applied +sequentially with the same jagged-tensor convention. No cross-layer fanciness. + +### 2.4 "Transformer-style sequential attention over a UIH" — what the inputs actually look like + +UIH = **User Interaction History**. For yambda, the input to one training +sample is one **anchor LISTEN event** plus that user's history. From +`README.MD:88-101` and `dlrm_v3/configs.py:399-418`: + +``` +sequence position: 0 .. 7 | 8 .. (L-2) | L-1 + ─────────┼─────────────────────────────┼────────── +content: contextual│ UIH (interleaved 3 pools) │ candidate + │ │ +features per position: uid, 7 cross-features (length-1 each) + item_id, artist_id, album_id, + action_weight (LP/LIKE/SKIP bitmask), + action_timestamp, dummy_watch_time + candidate's: + item_candidate_id, + item_candidate_artist_id, + item_candidate_album_id, + item_query_time, + item_action_weight, + item_dummy_watchtime +``` + +The HSTU stack runs causal attention over this `L = 2048` sequence. The label +is the candidate's `listen_plus` bit (1 if `played_ratio ≥ 50%`, else 0), and +BCE is taken on the logit emitted at position `L-1`. So "transformer-style +sequential attention over UIH" literally means: the user's last ~2 k actions +are tokens, the candidate song is the last token, and a 5-layer HSTU +transformer predicts whether that candidate will be a `listen_plus`. + +This is the conceptual jump from DLRMv2: + +| | DLRMv2 (Criteo, v2) | HSTU (Yambda, v4) | +|---|---|---| +| Input shape | flat: 26 categorical + 13 dense features per ad impression | sequence of ~2 k past events per user, each a structured tuple | +| Mixing op | DCN: cross-products of feature vectors, then MLP | self-attention across positions (SiLU-gated, multi-head, causal) | +| Temporal modelling | none (each ad impression is i.i.d.) | central — masks, timestamps, action types are first-class | +| Depth | 1-shot (interaction arch + over-arch MLP) | 5 stacked HSTU blocks | +| "Why is the candidate good?" | low-rank cross of user/ad embeddings | attention over user's relevant past songs/artists/albums | + +DLRMv2 is *wide-and-shallow* over tabular features. HSTU is *narrow-and-deep* +over a temporal sequence. Different paradigm. + +### 2.5 Jagged attention — what it is and why it's used + +A user's history length varies — yambda median is 2,695 events, max is 27,738 +(`README.MD:65`). For a single training step you have a batch of B users with +very different sequence lengths. Two ways to lay this out on the GPU: + +**Padded layout (standard transformer):** + +``` +input shape: [B, N_max, D] e.g. [1024, 2048, 512] +``` + +This wastes compute proportional to `(N_max − N_user) / N_max` per row. On +yambda the average fill is ~1402/2037 ≈ 69%, so ~30% of every kernel is +multiplying zeros. + +**Jagged layout (what HSTU uses):** + +``` +flat values : [L_total, D] L_total = Σ user_lengths (≤ B · N_max) +offsets : [B + 1] cumulative starts, so user i occupies + values[offsets[i] : offsets[i+1]] +``` + +`pt_hstu_attention.py:148, 183` shows the round-trip: + +- `torch.ops.fbgemm.jagged_to_padded_dense(...)` only when calling into a dense + einsum +- `torch.ops.fbgemm.dense_to_jagged(...)` on the way out + +That's the reference path. The **TRITON jagged-attention kernel** +(`ops/triton/triton_hstu_attention.py`, dispatched in `ops/hstu_attention.py:27, +71`) skips the padded intermediate entirely: each Triton program handles one +user's `[N_user, N_user]` attention block directly, so: + +- **No wasted FLOPs.** Empty positions never enter a GEMM. +- **No wasted memory.** No padded `[B, H, N_max, N_max]` attention scores buffer + — that buffer alone would be `1024 · 4 · 2048 · 2048 · 2 bytes ≈ 34 GB` per + step (at global batch 1024 × bf16). +- **Variable-length backward is correct without masking tricks.** The kernel + iterates `[offsets[i], offsets[i+1])` per program; the gradient never touches + non-existent positions. + +This is *the* enabling optimization for the under-filled `like` pool to be +cheap. The README notes (`README.MD:132`): *"With the TRITON jagged-attention +backend the GPU only does work for the actual events, so the under-fill costs +sequence budget but not GPU compute"*. With a padded kernel, the unused 31% of +every sequence would cost real FLOPs. + +Practically: jagged attention is a generic technique (it shows up in +FlashAttention's varlen variants too); HSTU's TRITON kernel is its +specialization with SiLU + gated output + the four-way mask. + +--- + +## 3. Yambda-5b — size, contents, download, run + +### 3.1 What's in it + +[`yandex/yambda`](https://huggingface.co/datasets/yandex/yambda) on HuggingFace. +From `dlrm_v3/preprocess_public_data.py:233-245` + `README.MD:56-81`: + +| field | value | +|---|---| +| Provider | Yandex Music recommendation logs | +| Sizes | yambda-50m, yambda-500m, **yambda-5b** (v4 uses 5b) | +| Events | 4.76 B interactions across 300 days | +| Users | 1.00 M unique | +| Items | 9.39 M songs (+ 1.29 M artists, 3.37 M albums) | +| Event types | `listen` / `like` / `dislike` / `unlike` / `undislike` (encoded as uint8 0–4) | +| Listen events also carry | `played_ratio` (used to derive the `listen_plus` label at 50% threshold) | +| Train / test split | Global Temporal Split: 300 days train, 30-min gap, 1 day test | + +### 3.2 On-disk footprint after preprocessing (`README.MD:39-52`) + +``` +/ +├── raw/5b/multi_event.parquet 50 GB (downloaded) +├── shared_metadata/ +│ ├── artist_item_mapping.parquet 60 MB +│ ├── album_item_mapping.parquet 76 MB +│ └── embeddings.parquet 18 GB (unused by HSTU training) +└── processed_5b/ + ├── train_sessions.parquet 47 GB ← main training input + ├── test_events.parquet 152 MB + ├── session_index.parquet 600 MB + ├── item_popularity.npy 75 MB + └── split_meta.json anchor + boundary stats +``` + +Plan for **~115 GB free disk** to do everything end-to-end (raw + shared + +processed). If you skip the unused `embeddings.parquet` (which the script +downloads anyway), you still need ~97 GB. + +### 3.3 Download + preprocess + +Both happen in one command. Download is via the `datasets` library +(HuggingFace), so you need internet and `pip install datasets`. From +`dlrm_v3/preprocess_public_data.py:276-317`: + +```bash +pip install datasets polars-u64-idx pyarrow xxhash gin-config absl-py pandas + +export DLRM_DATA_PATH=/your/big/disk/dlrm_data +mkdir -p "$DLRM_DATA_PATH" + +cd /home/suachong/training/recommendation_v4 +python3 -m generative_recommenders.dlrm_v3.preprocess_public_data \ + --dataset yambda-5b \ + --data-path "$DLRM_DATA_PATH" +``` + +Per `README.MD:54`: **~53 minutes end-to-end** for the 5b variant on a +reasonable box. For a quick smoke test substitute `--dataset yambda-50m` +(~2 min, ~1 GB on disk). + +Critical: **install `polars-u64-idx`, not stock `polars`.** yambda-5b has +>4.29 B rows and overflows polars' default 32-bit row index silently +(`training_recipe.md:102-103`, `scripts/launch_smoke_8gpu.sh:13-20`). + +### 3.4 Run training (8-GPU smoke) + +From `scripts/launch_smoke_8gpu.sh` and `README.MD:9-22`. + +**Inside the validated container** (recommended; everything's pre-staged): + +```bash +docker exec yambda_8gpu bash -c \ + 'cd /workspace/recommendation_v4 && bash scripts/launch_smoke_8gpu.sh' +``` + +Override data path / run name without editing the gin: + +```bash +DLRM_DATA_PATH=/your/big/disk/dlrm_data \ +RUN_NAME=my_experiment \ +bash scripts/launch_smoke_8gpu.sh +``` + +**From scratch on a bare host**, you need to assemble the stack per +`docs/training_recipe.md`. The hard requirements are: + +- **ROCm path**: `rocm/primus:v26.3`, torch `2.12.0+rocm7.2`, triton `3.6.0`, + fbgemm_gpu built from commit `10b77573` for `gfx950`, torchrec + `1.7.0a0+bf55480`. See `training_recipe.md:30-45`. +- **CUDA path**: `nvcr.io/nvidia/pytorch:26.04-py3`, native torch (do NOT + reinstall), fbgemm_gpu built from same commit for `sm_100`, torchrec + `1.7.0.dev20260601+cu130`. See `training_recipe.md:147-155`. + +In both cases the actual launch is just: + +```bash +python -m generative_recommenders.dlrm_v3.train.train_ranker \ + --dataset yambda-5b --mode train-eval +``` + +(plus `HSTU_HAMMER_KERNEL=TRITON` for CUDA; `=PYTORCH` is forced on ROCm in +the smoke script because the Triton kernel hits PassManager errors on some +shapes there — see `scripts/launch_smoke_8gpu.sh:31-33`. The PYTORCH fallback +gives ~190 ms/step baseline, not the ~52 ms primus-pinned number.) + +### 3.5 What you'll see + +Per `training_recipe.md:84-91`, on 8× MI350X in the optimal config: + +- ~52 ms/step at global batch 256 (per the doc; gin says 8,192 — see §1.6 note) +- ~4,970 samples/sec +- ~7.6 days for one epoch over 3.23 B training anchors + +If `auc_threshold = 0.80275` is the real benchmark target (still TBD), +`time_to_auc_0.80275_sec` will be logged as soon as the eval AUC on +`listen_plus` crosses it (`dlrm_v3/utils.py:587-608`). + +--- + +## 4. TL;DR + +- **HSTU ≈ decoder-only Transformer** with two tweaks: SiLU/N replaces softmax + in attention, and a `U`-gated output replaces the standard MLP block. +- **DLRMv3 (yambda) = TorchRec embeddings → contextual preprocessor → 5 + stacked HSTU layers → BCE head on `listen_plus`.** Sparse tables (≈150 G + params) dominate the model; the dense HSTU stack is tiny by comparison. +- **UIH = user interaction history.** Each sample is one anchor LISTEN event + plus that user's last ~2 k events (LISTEN_PLUS / LIKE / SKIP, interleaved + chronologically, gathered with a `L//3`-per-pool cap), and HSTU does causal + self-attention across them. +- **Jagged attention** packs variable-length per-user sequences as + `(flat_values, offsets)` instead of padding to `N_max`, so the Triton kernel + never spends FLOPs on empty positions — essential because the average + sequence is only 69% full on yambda. +- **Yambda-5b** is a 4.76 B-event / 1 M-user Yandex Music dataset on + HuggingFace (`yandex/yambda`); downloading + preprocessing takes ~53 min and + ~115 GB disk; run via + `python -m generative_recommenders.dlrm_v3.train.train_ranker --dataset yambda-5b --mode train-eval` + (or `scripts/launch_smoke_8gpu.sh`). + +--- + +## 5. Open questions to bring back to the proposing team + +1. **Target metric.** `yambda_5b.gin:107` reuses DLRMv2's `0.80275` AUC + threshold. Is this the intended final v4 benchmark target, or a placeholder + inherited from upstream? An HSTU model on a different dataset would + normally need its own threshold chosen from a reference run. +2. **Batch size canonicalization.** `yambda_5b.gin:1` = `1024` per rank + (global 8,192); `docs/training_recipe.md:65, 182` says `32` per rank + (global 256). Which is the submission config? +3. **Convergence reference runs.** No `reference_results.md`-style table exists + yet under `training/recommendation_v4`. Submission-quality v4 will need + reference epochs-to-target numbers per dataset variant. diff --git a/recommendation_v4/scripts/run_docker.sh b/recommendation_v4/scripts/run_docker.sh new file mode 100755 index 000000000..bb1d58fca --- /dev/null +++ b/recommendation_v4/scripts/run_docker.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# Launch a yambda_8gpu container from rocm/mlperf:dlrm_v3_mi355 with the repo +# and data directories bind-mounted at matching host/container paths. +# +# Usage: +# bash scripts/run_docker.sh # interactive shell +# bash scripts/run_docker.sh -- bash scripts/launch_smoke_8gpu.sh # one-shot +# +# Overrides (export before invoking): +# IMAGE docker image (default: rocm/mlperf:dlrm_v3_mi355) +# CONTAINER_NAME container name (default: yambda_8gpu) +# REPO_HOST host path to repo (default: this script's parent) +# DATA_HOST host path to dataset root (default: /data/mlperf_dlrm_v4) + +set -euo pipefail + +IMAGE=${IMAGE:-rocm/mlperf:dlrm_v3_mi355} +CONTAINER_NAME=${CONTAINER_NAME:-mlperf-recommendation-v4} +REPO_HOST=${REPO_HOST:-$(cd "$(dirname "$0")/.." && pwd)} +DATA_HOST=${DATA_HOST:-/data/mlperf_dlrm_v4} + +# Mount host paths at the same string inside the container so DLRM_DATA_PATH +# can be set from either side and resolve identically (env_path() in +# dlrm_v3/utils.py:641-653 does a literal os.environ.get). +REPO_CONT=/workspace/recommendation_v4 +DATA_CONT=${DATA_HOST} + +if [ ! -d "${DATA_HOST}" ]; then + echo "warning: ${DATA_HOST} does not exist on host. Run preprocess_public_data first or override DATA_HOST." >&2 +fi + +# If a container with this name is already running, exec into it instead of +# starting a new one. Matches the `docker exec yambda_8gpu ...` pattern in +# README.MD:9-12. +if docker ps --format '{{.Names}}' | grep -qx "${CONTAINER_NAME}"; then + echo "container ${CONTAINER_NAME} already running; exec'ing in" >&2 + exec docker exec -it "${CONTAINER_NAME}" "${@:-bash}" +fi + +# Remove a stopped container with the same name so --name doesn't collide. +if docker ps -a --format '{{.Names}}' | grep -qx "${CONTAINER_NAME}"; then + docker rm "${CONTAINER_NAME}" >/dev/null +fi + +exec docker run --rm -it \ + --name "${CONTAINER_NAME}" \ + --device=/dev/kfd --device=/dev/dri \ + --group-add video --group-add render \ + --cap-add=SYS_PTRACE --security-opt seccomp=unconfined \ + --ipc=host --network=host \ + --shm-size=64g --ulimit memlock=-1 --ulimit stack=67108864 \ + -v "${REPO_HOST}:${REPO_CONT}" \ + -v "${DATA_HOST}:${DATA_CONT}" \ + -e DLRM_DATA_PATH="${DATA_CONT}" \ + -e HSTU_HAMMER_KERNEL="${HSTU_HAMMER_KERNEL:-TRITON}" \ + -e RUN_NAME="${RUN_NAME:-default}" \ + -w "${REPO_CONT}" \ + "${IMAGE}" \ + "${@:-bash}" From d03fb2888002917b5b633da8f0c716f30a1cd8c2 Mon Sep 17 00:00:00 2001 From: suachong Date: Wed, 10 Jun 2026 21:26:12 +0000 Subject: [PATCH 047/127] dlrmv4: decouple multi-node launch from per-user paths for portable baseline Run the consolidated launch/streaming flow from any $HOME without editing another user's tree: - Derive REPO_MOUNT/DATA_MOUNT/SCRATCH from $HOME with env overrides; keep the shared read-only dataset path intact (no data duplication). - Per-user container name (yambda_$USER) to avoid collisions. - In-repo provisioning via launch_slurm.sh (drop external _provision script dep). - chmod log files world-writable so the container (nobody) can tee-append under NFS root_squash; fixes spurious pipefail rc=1. - Neutral in-repo TensorBoard default path. Validated: 2-node ROCm/RDMA smoke (world_size 16) completed rc=0, 20-batch train window with metrics logged. Co-authored-by: Cursor --- .../dlrm_v3/train/gin/yambda_5b.gin | 12 +++-- .../train/tests/streaming_resume_test.sh | 14 +++-- .../generative_recommenders/dlrm_v3/utils.py | 2 +- recommendation_v4/scripts/launch_slurm.sh | 53 ++++++++++++++----- .../scripts/run_streaming_e2e.sh | 44 +++++++++------ 5 files changed, 86 insertions(+), 39 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index e7eed30cb..5b5ec03e9 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -206,13 +206,15 @@ Profiler.active = 5 Profiler.trace_dir = @run_results_dir() # logger variables -# TensorBoard event dir. Default lives on shared NFS (not container-local /tmp, -# which is wiped on node failover) so the NE/AUC scalars survive relaunches and -# failover. Override per-run via $TENSORBOARD_LOG_PATH (the supervisor sets it -# to /apps/chcai/tb/$RUN_NAME/). +# TensorBoard event dir, driven by $TENSORBOARD_LOG_PATH. launch_slurm.sh (worker) +# always sets it to a writable NFS scratch path ($SCRATCH/tb/...) and the e2e +# supervisor pins a per-run path, so this literal default is only a last-resort +# fallback for a bare `train_ranker` invocation. It is RELATIVE on purpose (lands +# under the trainer's cwd = the repo, which is writable) so no user/site path is +# baked into the gin; set $TENSORBOARD_LOG_PATH for anything else. MetricsLogger.tensorboard_log_path = @tbp/env_path() tbp/env_path.key = "TENSORBOARD_LOG_PATH" -tbp/env_path.default = "/apps/chcai/tb/yambda_5b/" +tbp/env_path.default = "tb/yambda_5b/" MetricsLogger.world_size = 8 MetricsLogger.auc_threshold = 0.80275 # Lifetime-AUC backend, selectable independently for the train cumulative AUC and diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh b/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh index c3690e652..7c4bc8179 100755 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh @@ -17,7 +17,7 @@ # # Usage: # bash generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh --jobid -# [--container yambda_primus] +# [--container yambda_suachong] # [--num-train-batches 200] # [--die-at-step 350] # [--keep] # retain LOG_DIR + CKPT after run for inspection @@ -25,7 +25,7 @@ set -uo pipefail JOBID="" -CONTAINER="yambda_primus" +CONTAINER="yambda_${USER:-$(id -un)}" NUM_TRAIN_BATCHES=200 DIE_AT_STEP=350 IN_WINDOW_FREQ=50 @@ -40,9 +40,13 @@ KEEP=0 # correctness gates are the functional-invariant checks below (RNG restored, # resumed-at-correct-step, atomic/keep_last_n), not this number. ATOL=0.15 -CKPT_ROOT=/apps/chcai/ckpts_resume_test -LOG_DIR=/apps/chcai/streaming_resume_test -REPO=/home/chcai/training/recommendation_v4 +# Writable scratch ($HOME-derived) + repo root (this file is at +# /generative_recommenders/dlrm_v3/train/tests/, i.e. 4 levels deep). +# Both env-overridable; nothing is hardwired to a specific user/site. +SCRATCH=${SCRATCH:-$HOME/yambda_runs} +CKPT_ROOT=${CKPT_ROOT:-$SCRATCH/ckpts_resume_test} +LOG_DIR=${LOG_DIR:-$SCRATCH/streaming_resume_test} +REPO=${REPO:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)} while [[ $# -gt 0 ]]; do case $1 in diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py index a9c060324..1996323a8 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py @@ -807,7 +807,7 @@ class Profiler: All knobs are gin-tunable, e.g. in a gin file:: - Profiler.trace_dir = "/apps/chcai/dlrm_runs/exp42/trace" + Profiler.trace_dir = "/path/to/results/exp42/trace" Profiler.trace_steps = [500, 1000, 5000] Profiler.warmup = 5 Profiler.active = 10 diff --git a/recommendation_v4/scripts/launch_slurm.sh b/recommendation_v4/scripts/launch_slurm.sh index 0b5547497..098a69529 100755 --- a/recommendation_v4/scripts/launch_slurm.sh +++ b/recommendation_v4/scripts/launch_slurm.sh @@ -5,7 +5,10 @@ #SBATCH --exclusive #SBATCH --partition=meta64 # [CLUSTER-SPECIFIC] partition name #SBATCH --time=01:10:00 -#SBATCH --output=/apps/chcai/yambda_slurm.%j.out +#SBATCH --output=yambda_slurm.%j.out +# ^ relative to the submit dir (SLURM parses #SBATCH before any shell runs, so it +# cannot expand env vars). The real consolidated run log is $LOG (see below), +# which defaults under $SCRATCH; this file just captures the batch stdout. # ============================================================================= # launch_slurm.sh — single entry point for the yambda-5b trainer on N>=1 nodes. # @@ -59,9 +62,10 @@ # # B) Filesystems (must be shared/NFS across ALL nodes — this script re-invokes # itself and reads the overlay + data from these paths cluster-wide) -# - /home/chcai (repo + this script) and /apps/chcai (scratch: logs, overlay, -# baked tar, data, pip tarball). CHANGE both the bind mounts in the -# `docker run` (provision) and the default LOG/BAKED_TAR/OVERLAY/PIP_* paths. +# - REPO_MOUNT (repo + this script, e.g. /home/suachong) is bind-mounted rw; +# DATA_MOUNT (e.g. /apps/chcai) holds the read-only dataset + overlay + +# baked tar + pip tarball; SCRATCH (e.g. /home/suachong/yambda_runs) is the +# writable log/output root. Override any via env — nothing is user-hardwired. # # C) Container image / GPU software stack (tied to the GPU arch + ROCm version) # - IMAGE=rocm/primus:v26.3 : base image. ROCm/AMD-specific. @@ -106,13 +110,24 @@ if [ -z "$PHASE" ]; then fi # ---- shared config (env-overridable) ---------------------------------------- -CONTAINER=${CONTAINER:-yambda_primus} +CONTAINER=${CONTAINER:-yambda_${USER:-$(id -un)}} # per-user container name (do NOT reuse another user's container — its bind mounts differ) REPO=${REPO:-$REPO_ROOT} # repo path inside the container IMAGE=${IMAGE:-rocm/primus:v26.3} # [CLUSTER-SPECIFIC] ROCm/arch base image BAKED_IMAGE=${BAKED_IMAGE:-yambda_primus_baked:latest} -BAKED_TAR=${BAKED_TAR:-/apps/chcai/yambda_primus_baked.tar} # [CLUSTER-SPECIFIC] shared-NFS path +BAKED_TAR=${BAKED_TAR:-/apps/chcai/yambda_primus_baked.tar} # [CLUSTER-SPECIFIC] shared-NFS path (read-only build asset) USE_BAKED=${USE_BAKED:-1} -OVERLAY=${RDMA_OVERLAY:-/apps/chcai/rdma_host_el9_new} # [CLUSTER-SPECIFIC] shared-NFS RDMA overlay +OVERLAY=${RDMA_OVERLAY:-/apps/chcai/rdma_host_el9_new} # [CLUSTER-SPECIFIC] shared-NFS RDMA overlay (read-only, already staged) + +# Bind mounts + scratch — all on shared NFS, identical path on every node. +# REPO_MOUNT : NFS home root that contains THIS repo (bind-mounted rw). +# DATA_MOUNT : NFS root with the (shared, read-only) dataset + RDMA overlay + +# pip/fbgemm build assets. Kept as-is so the dataset is NOT +# duplicated. You only need read access here. +# SCRATCH : this run's WRITABLE output root (logs / tb / traces). +# All env-overridable, so nothing is hardwired to one user's home. +REPO_MOUNT=${REPO_MOUNT:-$HOME} # NFS home holding the repo (must contain $REPO); override if your repo lives elsewhere +DATA_MOUNT=${DATA_MOUNT:-/apps/chcai} # shared dataset + RDMA overlay + pip/fbgemm assets (read-only) +SCRATCH=${SCRATCH:-$HOME/yambda_runs} # writable output root (logs / tb / traces) # ============================================================================= # PHASE: orchestrate (SLURM batch host) @@ -128,7 +143,8 @@ orchestrate() { [ -f "$SCRIPT_PATH" ] || SCRIPT_PATH="$SELF" REPO=$(cd "$(dirname "$SCRIPT_PATH")/.." && pwd) - LOG=${LOG:-/apps/chcai/yambda_slurm.${SLURM_JOB_ID:-manual}.log} + mkdir -p "$SCRATCH" 2>/dev/null || true + LOG=${LOG:-$SCRATCH/yambda_slurm.${SLURM_JOB_ID:-manual}.log} # Smoke defaults — override via env for a perf run (see header USAGE). MODE=${MODE:-streaming-train-eval} @@ -142,6 +158,11 @@ orchestrate() { FORCE_PROVISION=${FORCE_PROVISION:-0} : > "$LOG" + # World-writable so the in-container worker (running as root, squashed to + # `nobody` over root-squashed NFS) can append via `tee -a $LOG`. Without this + # the worker's tee opens the file read-only-denied and exits non-zero, which + # pipefail turns into a spurious rc=1 even when training succeeds. + chmod 666 "$LOG" 2>/dev/null || true echo "[$(date)] launch_slurm/orchestrate: job=${SLURM_JOB_ID:-?} nodes=${SLURM_JOB_NODELIST:-?} nnodes=${SLURM_NNODES:-1}" | tee -a "$LOG" echo "[$(date)] resolved SCRIPT_PATH=$SCRIPT_PATH REPO=$REPO" | tee -a "$LOG" echo "[$(date)] config: MODE=$MODE START_TS=$START_TS NUM_TRAIN_TS=$NUM_TRAIN_TS NUM_TRAIN_BATCHES=$NUM_TRAIN_BATCHES NUM_EVAL_BATCHES=$NUM_EVAL_BATCHES METRIC_LOG_FREQ=$METRIC_LOG_FREQ" | tee -a "$LOG" @@ -181,7 +202,8 @@ orchestrate() { echo \"[\$(hostname)] (re)provisioning container\" LAUNCH_SLURM_PHASE=provision CONTAINER=$CONTAINER IMAGE=$IMAGE \ BAKED_IMAGE=$BAKED_IMAGE BAKED_TAR=$BAKED_TAR USE_BAKED=$USE_BAKED \ - BAKE_IMAGE=${BAKE_IMAGE:-0} RDMA_OVERLAY=$OVERLAY REPO=$REPO bash $SCRIPT_PATH + BAKE_IMAGE=${BAKE_IMAGE:-0} RDMA_OVERLAY=$OVERLAY REPO=$REPO \ + REPO_MOUNT=$REPO_MOUNT DATA_MOUNT=$DATA_MOUNT SCRATCH=$SCRATCH bash $SCRIPT_PATH else echo \"[\$(hostname)] container already up\" fi @@ -192,6 +214,7 @@ orchestrate() { srun --ntasks-per-node=1 bash -c " docker exec \ -e LAUNCH_SLURM_PHASE=worker \ + -e SCRATCH=$SCRATCH \ -e SLURM_NNODES=\$SLURM_NNODES \ -e SLURM_NODEID=\$SLURM_NODEID \ -e SLURM_PROCID=\$SLURM_PROCID \ @@ -270,9 +293,9 @@ provision() { --ulimit memlock=-1:-1 --ulimit stack=67108864:67108864 \ `# memlock=-1 is REQUIRED for RDMA QP memory registration — do not drop` \ --security-opt seccomp=unconfined --privileged \ - -v /home/chcai:/home/chcai \ - -v /apps/chcai:/apps/chcai \ - `# [CLUSTER-SPECIFIC] shared-NFS bind mounts: repo + scratch (overlay/logs/data)` \ + -v "$REPO_MOUNT:$REPO_MOUNT" \ + -v "$DATA_MOUNT:$DATA_MOUNT" \ + `# shared-NFS bind mounts: repo home (REPO_MOUNT, rw) + dataset/build assets (DATA_MOUNT)` \ -w "$REPO" \ "$RUN_IMAGE" sleep infinity @@ -354,7 +377,11 @@ python -c "import torch, fbgemm_gpu, torchrec, polars, xxhash, gin; print(\"impo # ============================================================================= worker() { cd "$REPO_ROOT" - LOG=${LOG:-/apps/chcai/yambda_5b_8gpu.log} + mkdir -p "$SCRATCH" 2>/dev/null || true + LOG=${LOG:-$SCRATCH/yambda_5b_8gpu.log} + # TensorBoard under the writable scratch root unless the caller (e.g. the e2e + # supervisor) pinned a per-run path. Keeps the gin default from ever being used. + export TENSORBOARD_LOG_PATH=${TENSORBOARD_LOG_PATH:-$SCRATCH/tb/yambda_5b} # Append (not truncate): under the streaming-e2e supervisor a run may relaunch # many times into the SAME $LOG; the supervisor initializes it once at run start. echo "[$(date)] REPO_ROOT=$REPO_ROOT" | tee -a "$LOG" diff --git a/recommendation_v4/scripts/run_streaming_e2e.sh b/recommendation_v4/scripts/run_streaming_e2e.sh index f97a0e483..6eb8ee319 100755 --- a/recommendation_v4/scripts/run_streaming_e2e.sh +++ b/recommendation_v4/scripts/run_streaming_e2e.sh @@ -26,8 +26,9 @@ # lived docker container ($CONTAINER) on the compute node held by a SLURM # allocation ($JOBID). All control flow is `srun --jobid --overlap # docker exec ...` into that container. The container bind-mounts shared NFS -# (/home/chcai = code, /apps/chcai = checkpoints+logs), which is what makes -# node failover possible: any node in $PARTITION sees the same code+state. +# (REPO_MOUNT = code, e.g. /home/suachong; DATA_MOUNT = dataset/build assets; +# SCRATCH = checkpoints+logs), which is what makes node failover possible: any +# node in $PARTITION sees the same code+state. # # MAIN LOOP (state machine, up to --max-relaunch attempts) # for each attempt: @@ -95,12 +96,12 @@ # EXAMPLE # nohup bash scripts/run_streaming_e2e.sh \ # --jobid 12074 \ -# --ckpt-path /apps/chcai/ckpts/yambda_5b_e2e \ -# --run-name yambda_5b_e2e --log /apps/chcai/yambda_5b_e2e.log \ +# --ckpt-path "$HOME/yambda_runs/ckpts/yambda_5b_e2e" \ +# --run-name yambda_5b_e2e --log "$HOME/yambda_runs/yambda_5b_e2e.log" \ # --start-ts 150 --num-train-ts 149 --eval-every 10 \ # --ckpt-time-interval 3600 --keep-last-n 1 --max-relaunch 100 \ # --reservation NAN_issue_debug \ -# > /apps/chcai/yambda_5b_e2e.supervisor.console.log 2>&1 & +# > "$HOME/yambda_runs/yambda_5b_e2e.supervisor.console.log" 2>&1 & # (--reservation makes node-death failover re-acquire from that reservation; # omit it to fall back to the open $PARTITION pool.) # ============================================================================= @@ -108,8 +109,12 @@ set -uo pipefail JOBID=11367 -CONTAINER=yambda_primus -REPO=/home/chcai/training/recommendation_v4 +CONTAINER=yambda_${USER:-$(id -un)} +# Repo (NFS path, identical inside the bind-mounted container) and the writable +# output root. Both derive from $HOME so nothing is hardwired to one user; override +# REPO/SCRATCH via env if your checkout or scratch lives elsewhere. +REPO=${REPO:-$HOME/training/recommendation_v4} +SCRATCH=${SCRATCH:-$HOME/yambda_runs} # Direct-SSH fallback so the supervisor can probe the node even while the SLURM # control plane is unreachable — a transient controller outage must NOT be @@ -128,9 +133,13 @@ START_TS=150 EVAL_EVERY=5 CKPT_TIME_INTERVAL=7200 KEEP_LAST_N=1 -CKPT_PATH=/apps/chcai/ckpts/yambda_5b_e2e +# NOTE: a full DMP checkpoint is ~560 GB, which a typical ~100 GB home quota +# cannot hold — point --ckpt-path at a large writable volume (and lower +# --min-free-gib accordingly) before relying on checkpointing. The $SCRATCH +# default below is fine for short/uncheckpointed runs. +CKPT_PATH=$SCRATCH/ckpts/yambda_5b_e2e RUN_NAME=yambda_5b_e2e -LOG=/apps/chcai/yambda_5b_e2e.log +LOG=$SCRATCH/yambda_5b_e2e.log MAX_RELAUNCH=50 NUM_TRAIN_BATCHES=0 # 0 = full window (only capped for validation/tests) NUM_EVAL_BATCHES=0 # 0 = full holdout eval (only capped for validation) @@ -154,14 +163,17 @@ CTRL_WAIT_MAX=3600 # max seconds to wait for an unreachable SLURM controlle # --- node failover ---------------------------------------------------------- # If the current allocation/node goes away, acquire a FRESH node, (re)provision # the container on it, and resume — checkpoints + code live on shared NFS -# (/apps/chcai, /home/chcai), so any node in the partition can continue. +# (SCRATCH + REPO_MOUNT), so any node in the partition can continue. PARTITION=meta64 RESERVATION="" # if set, failover acquires from this SLURM # reservation (e.g. NAN_issue_debug) so a # replacement node comes from the same pool. ALLOC_TIME=7-00:00:00 # SLURM --time for a failover hold job ALLOW_FAILOVER=1 # 0 = never acquire a new node -PROVISION_SCRIPT=/home/chcai/_provision_yambda_primus.sh +# Failover (re)provisioning reuses launch_slurm.sh's own `provision` phase +# (LAUNCH_SLURM_PHASE=provision, set in provision_node) — no dependency on any +# out-of-repo provisioning script. +PROVISION_SCRIPT="$REPO/scripts/launch_slurm.sh" ACQUIRE_WAIT_MAX=1800 # max seconds to wait for the OPEN-POOL # (tier-2) failover hold job to reach # RUNNING (tolerates brief queueing). @@ -311,7 +323,9 @@ provision_node() { local jid="$1" node node=$(squeue -h -j "$jid" -o '%N' 2>/dev/null | head -1) sup "provisioning container '$CONTAINER' on job $jid (node ${node:-?}) — cold node can take ~15 min" - srun --jobid="$jid" --overlap bash "$PROVISION_SCRIPT" >> "${LOG%.log}.provision.log" 2>&1 + srun --jobid="$jid" --overlap env LAUNCH_SLURM_PHASE=provision \ + CONTAINER="$CONTAINER" REPO="$REPO" bash "$PROVISION_SCRIPT" \ + >> "${LOG%.log}.provision.log" 2>&1 container_up "$jid" } @@ -541,7 +555,7 @@ launch() { EVAL_HOLDOUT_NUM_WINDOWS=$EVAL_HOLDOUT_NUM_WINDOWS \ METRIC_LOG_FREQ=50 \ RUN_NAME=$RUN_NAME \ - TENSORBOARD_LOG_PATH=/apps/chcai/tb/$RUN_NAME/ \ + TENSORBOARD_LOG_PATH=$SCRATCH/tb/$RUN_NAME/ \ LOG=$LOG \ bash scripts/launch_slurm.sh; echo \"E2E_RUN_EXIT=\$? \$(date '+%F %T')\" >> $LOG @@ -567,7 +581,7 @@ sup "failover: allow=$ALLOW_FAILOVER partition=$PARTITION reservation=${RESERVAT # this, such an orphan keeps pinning a second reservation node indefinitely. reap_failover_holds "" -cexec "mkdir -p '$CKPT_PATH' '/apps/chcai/tb/$RUN_NAME'" +cexec "mkdir -p '$CKPT_PATH' '$SCRATCH/tb/$RUN_NAME'" # Initialize this run's metrics log ONCE. launch_slurm.sh (worker) appends (tee -a), # so every relaunch attempt accumulates into this single file — the full-run # NE/AUC history survives crashes and node failover instead of being truncated @@ -579,7 +593,7 @@ else cexec ": > '$LOG'" sup "metrics log initialized (relaunch-append): $LOG" fi -sup "tensorboard (NFS): /apps/chcai/tb/$RUN_NAME/" +sup "tensorboard (NFS): $SCRATCH/tb/$RUN_NAME/" attempt=0 while (( attempt < MAX_RELAUNCH )); do From c55736eaf1f4ae94a19e28830ede898a72cc193e Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Wed, 10 Jun 2026 18:52:49 -0500 Subject: [PATCH 048/127] dlrmv4: deterministic in-window shuffle diversity dial + opt-in diagnostics Add STREAMING_SHUFFLE_FRACTION (0..1) as a config-invariant control over in-window embedding diversity, replacing the legacy block/buffer knobs. Full shuffle is the deterministic, seeded default (fraction=1.0). Add an opt-in unique-embedding diagnostic (DIAG_UNIQUE_EMB) and gate chrome-trace capture behind OUTPUT_TRACE; both default off to keep production runs overhead-free. Forward the new env knobs through launch_slurm.sh. Co-authored-by: Cursor --- .../dlrm_v3/datasets/yambda.py | 66 +++++++++++++++++-- .../dlrm_v3/train/gin/yambda_5b.gin | 48 ++++++++++++-- .../dlrm_v3/train/utils.py | 39 +++++++++++ .../generative_recommenders/dlrm_v3/utils.py | 7 ++ recommendation_v4/scripts/launch_slurm.sh | 6 ++ 5 files changed, 156 insertions(+), 10 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/datasets/yambda.py b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/yambda.py index ad24513f4..6627033bc 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/datasets/yambda.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/yambda.py @@ -209,6 +209,8 @@ def __init__( is_inference: bool = False, streaming_window_seconds: int = 86400, streaming_sort_within_window: bool = False, + streaming_shuffle_fraction: float = 0.0, + streaming_shuffle_seed: int = 0, train_split_percentage: float = 1.0, split_salt: int = 0, *args, @@ -225,6 +227,17 @@ def __init__( # is byte-for-byte unaffected. self._streaming_window_seconds: int = streaming_window_seconds self._streaming_sort_within_window: bool = streaming_sort_within_window + # In-window shuffle dial in [0, 1] to break user-major batching (default + # 0.0 = off, user-major order preserved for page-local mmap scans). Maps to + # a within-segment shuffle with K = round(fraction * per-window train-anchor + # count): 1.0 = full per-element shuffle (max user diversity per batch), + # intermediate = interpolation. Computed from the global anchor count BEFORE + # round-robin striding, so a given fraction yields the same diversity + # regardless of world_size / #nodes / batch_size (config-invariant). The + # permutation is a pure function of (seed, ts) so the per-rank round-robin + # slice + mid-window resume skip stay deterministic across restarts. + self._streaming_shuffle_fraction: float = streaming_shuffle_fraction + self._streaming_shuffle_seed: int = streaming_shuffle_seed # User-level train:eval split. `train_split_percentage >= 1.0` means no # holdout (legacy behavior: every anchor is trainable). Otherwise the # top `1 - train_split_percentage` fraction of users (by a deterministic @@ -555,20 +568,63 @@ def _eval_anchor_mask(self, anchor_idx: np.ndarray) -> np.ndarray: uids = self.store.flat_uid[self._positions[anchor_idx]] return _uid_unit_hash(uids, self._split_salt) >= self._train_split_percentage + def _shuffle_window(self, idx: np.ndarray, ts: int) -> np.ndarray: + """Optionally break user-major ordering within a train window. + + ``streaming_shuffle_fraction`` (0..1) is the single diversity dial. It + maps to a within-segment shuffle with ``K = round(fraction * N)`` where + ``N`` is this window's train-anchor count: + + - 0.0 -> off: return ``idx`` unchanged (user-major, page-local scans). + - 1.0 -> full per-element shuffle (max user diversity per batch). + - else -> permute WITHIN each contiguous size-K segment (segment order + preserved). A per-rank batch then draws across a bounded user-major + region, so diversity scales with the fraction while the concurrently + touched mmap working set stays within ~one K-segment (page locality). + + Because ``N`` is a property of the dataset/window (not the compute layout) + and the permutation is applied BEFORE the per-rank round-robin striding, a + given fraction yields the same diversity across world_size / #nodes / + batch_size (config-invariant). + + The permutation is a pure function of ``(seed, ts)`` via + ``np.random.default_rng(seed + ts)``, so every (re)run of this window + yields the IDENTICAL order. This keeps the per-rank round-robin slice and + the mid-window resume ``skip_samples`` offset consistent across restarts, + exactly like the unshuffled path. + """ + frac = self._streaming_shuffle_fraction + if idx.size <= 1 or not frac or frac <= 0.0: + return idx + rng = np.random.default_rng(self._streaming_shuffle_seed + ts) + if frac >= 1.0: + return idx[rng.permutation(idx.size)] + # Within-segment shuffle (K = round(fraction * N)): a single vectorized + # lexsort over per-element random keys, stable within each size-K segment + # so elements never cross a segment boundary (bounds the working set). O(N + # log K), run once per window in the background prep thread. + n = idx.size + k = max(1, int(round(frac * n))) + seg = np.arange(n, dtype=np.int64) // k + keys = rng.random(n) + order = np.lexsort((keys, seg)) + return idx[order] + def train_window_indices(self, ts: int) -> np.ndarray: """Global anchor indices for TRAIN in window ``ts``: ``window_indices`` - with held-out eval users removed. Identical across resume because both - ``window_indices`` and the uid hash are pure functions, so the per-rank - round-robin slice (and the mid-window skip offset) stay consistent.""" + with held-out eval users removed. Identical across resume because + ``window_indices``, the uid hash, and the (seed,ts)-keyed in-window + shuffle are all pure functions, so the per-rank round-robin slice (and + the mid-window skip offset) stay consistent.""" idx = self.window_indices(ts) if self._train_split_percentage >= 1.0: - return idx + return self._shuffle_window(idx, ts) kept = idx[~self._eval_anchor_mask(idx)] logger.warning( f"train_window_indices({ts}): {idx.size:,} -> {kept.size:,} anchors " f"(holdout tsp={self._train_split_percentage}, salt={self._split_salt})" ) - return kept + return self._shuffle_window(kept, ts) def eval_holdout_indices(self, start_ts: int, num_windows: int = 1) -> np.ndarray: """Fixed eval set: held-out users' anchors over windows diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index e7eed30cb..92c2404fc 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -1,6 +1,13 @@ batch_size = 1024 -num_workers = 4 -prefetch_factor = 8 +# Dataloader parallelism. Env-overridable so a perf sweep can probe whether the +# shuffle steady-state cost is CPU-gather latency (hidden by more workers) vs +# GPU-side embedding work (not). Defaults preserve prior behavior (4 / 8). +num_workers = @nw/env_int() +nw/env_int.key = "NUM_WORKERS" +nw/env_int.default = 4 +prefetch_factor = @pf/env_int() +pf/env_int.key = "PREFETCH_FACTOR" +pf/env_int.default = 8 dataset = "yambda-5b" # model parameters @@ -110,6 +117,28 @@ msl/env_int.default = 4096 # to the dataset's available window count at runtime; override via $NUM_TRAIN_TS. get_dataset.streaming_window_seconds = 86400 get_dataset.streaming_sort_within_window = False +# In-window shuffle to break user-major batching (consecutive sliding-window +# anchors otherwise come from the same few users -> few unique embedding reads). +# Diversity dial in [0,1] -- the AGREED, config-invariant benchmark knob. Maps to +# a within-segment shuffle with K = round(fraction * per-window train-anchor +# count): 0 = off (user-major, page-local mmap scans), 1 = full per-element +# shuffle (max diversity), intermediate = interpolation. Same fraction => same +# diversity regardless of world_size / #nodes / batch_size. +# +# Default 1.0 (full shuffle) so the standard/benchmark run is maximally diverse +# and, together with the fixed seed below, the in-window order is fully +# DETERMINISTIC and identical across runs/resumes (pure function of (seed, ts)). +# Override per-run via $STREAMING_SHUFFLE_FRACTION (e.g. 0.0 for user-major, +# 0.03 for the diversity/locality sweet spot). +streaming_shuffle_fraction = 1.0 +get_dataset.streaming_shuffle_fraction = @ssf/env_float() +ssf/env_float.key = "STREAMING_SHUFFLE_FRACTION" +ssf/env_float.default = %streaming_shuffle_fraction +# Fixed shuffle seed -> reproducible permutation. Exposed as a knob; override via +# $STREAMING_SHUFFLE_SEED only if you deliberately want a different draw. +get_dataset.streaming_shuffle_seed = @ssfseed/env_int() +ssfseed/env_int.key = "STREAMING_SHUFFLE_SEED" +ssfseed/env_int.default = 0 # User-level train:eval holdout for the streaming path. With tsp<1.0, the top # (1 - tsp) fraction of users (by a deterministic hash of uid+split_salt) are # held out as a FIXED eval set and never trained -> no temporal/user leakage, @@ -139,10 +168,19 @@ sts/env_int.default = 150 streaming_train_eval_loop.metric_log_frequency = @mlf/env_int() mlf/env_int.key = "METRIC_LOG_FREQ" mlf/env_int.default = 50 -# Trace on by default: reuses the shared Profiler.* bindings below (5-step +# Diagnostic: log per-batch unique/total embedding-id counts on logged steps +# (rank 0). Quantifies the user-major batching redundancy and the realized +# diversity from get_dataset.streaming_shuffle_fraction. Off; set $DIAG_UNIQUE_EMB=1. +streaming_train_eval_loop.streaming_diag_unique_emb = @due/env_int() +due/env_int.key = "DIAG_UNIQUE_EMB" +due/env_int.default = 0 +# Chrome trace capture: reuses the shared Profiler.* bindings below (5-step # window at step 52). The streaming step counter advances across train+eval -# batches, so step 52 lands in the first (train) window's compute. -streaming_train_eval_loop.output_trace = True +# batches, so step 52 lands in the first (train) window's compute. Off by +# default to avoid profiler overhead in production runs; set $OUTPUT_TRACE=1. +streaming_train_eval_loop.output_trace = @ot/env_int() +ot/env_int.key = "OUTPUT_TRACE" +ot/env_int.default = 0 # Reuse one DataLoader (persistent workers) across windows instead of respawning # per window. Skip eval to isolate window-reset cost. Override via env. streaming_train_eval_loop.persistent_loader = @pl/env_int() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py index c27f38761..076412d6e 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -688,6 +688,38 @@ def make_train_test_dataloaders( return train_dataloader, test_dataloader +def _log_unique_embedding_diag(sample, rank: int, step: int) -> None: + """Diagnostic: log per-batch unique-vs-total embedding-id counts. + + Quantifies the user-major batching concern — when consecutive sliding-window + anchors come from the same few users, a batch reads very few UNIQUE embedding + rows (low unique/total), so embedding lookups are highly redundant. In-window + shuffle should raise this ratio. Rank-0 only, gated by the caller (only fires + on logged steps), and fully non-fatal so it can never break training. + """ + if rank != 0: + return + try: + parts = [] + for tag, kjt in ( + ("uih", sample.uih_features_kjt), + ("cand", sample.candidates_features_kjt), + ): + for key in kjt.keys(): + if not key.endswith(("item_id", "artist_id", "album_id")): + continue + vals = kjt[key].values() + total = int(vals.numel()) + if total == 0: + continue + uniq = int(torch.unique(vals).numel()) + parts.append(f"{tag}.{key}={uniq}/{total} ({100.0 * uniq / total:.1f}%)") + if parts: + logger.info(f"emb-diag - Step {step}: unique/total " + " ".join(parts)) + except Exception as e: # diagnostic must never break training + logger.warning(f"emb-diag failed: {e}") + + @gin.configurable def train_loop( rank: int, @@ -702,6 +734,7 @@ def train_loop( metric_log_frequency: int = 1, checkpoint_frequency: int = 100, start_batch_idx: int = 0, + streaming_diag_unique_emb: bool = False, # lr_scheduler: to-do: Add a scheduler ) -> None: model.train() @@ -711,6 +744,8 @@ def train_loop( for epoch in range(num_epochs): dataloader.sampler.set_epoch(epoch) # pyre-ignore [16] for sample in dataloader: + if streaming_diag_unique_emb and batch_idx % metric_log_frequency == 0: + _log_unique_embedding_diag(sample, rank, batch_idx) optimizer.zero_grad() sample.to(device) ( @@ -1201,6 +1236,8 @@ def streaming_train_eval_loop( # --- global step / wall-clock checkpoint cadences --- checkpoint_step_frequency: int = 0, checkpoint_time_interval_s: float = 0.0, + # --- diagnostic: log per-batch unique/total embedding-id counts --- + streaming_diag_unique_emb: bool = False, # --- test-only failure injection knob --- die_at_step: int = -1, ) -> None: @@ -1457,6 +1494,8 @@ def _run_train_window( break if _t_next is not None and first_wait is None: first_wait = time.perf_counter() - _t_next + if streaming_diag_unique_emb and train_batch_idx % metric_log_frequency == 0: + _log_unique_embedding_diag(sample, rank, train_batch_idx) optimizer.zero_grad() sample.to(device) ( diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py index a9c060324..0abfaa86a 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py @@ -1424,6 +1424,8 @@ def get_dataset( history_length: Optional[int] = None, streaming_window_seconds: int = 86400, streaming_sort_within_window: bool = False, + streaming_shuffle_fraction: float = 0.0, + streaming_shuffle_seed: int = 0, train_split_percentage: float = 1.0, split_salt: int = 0, ): @@ -1556,6 +1558,11 @@ def get_dataset( # streaming-train-eval; ignored by the default train-eval path). "streaming_window_seconds": streaming_window_seconds, "streaming_sort_within_window": streaming_sort_within_window, + # In-window shuffle diversity dial in [0,1]: K=round(frac*N) within- + # segment shuffle. 0=off/user-major, 1=full. Config-invariant and + # deterministic by (seed, ts). + "streaming_shuffle_fraction": streaming_shuffle_fraction, + "streaming_shuffle_seed": streaming_shuffle_seed, # User-level train:eval holdout for the streaming path. 1.0 = # no holdout (legacy). <1.0 holds out (1 - tsp) of users as a # fixed eval set; those users are never trained. diff --git a/recommendation_v4/scripts/launch_slurm.sh b/recommendation_v4/scripts/launch_slurm.sh index 0b5547497..42f8780ed 100755 --- a/recommendation_v4/scripts/launch_slurm.sh +++ b/recommendation_v4/scripts/launch_slurm.sh @@ -208,6 +208,12 @@ orchestrate() { -e NUM_TRAIN_BATCHES=$NUM_TRAIN_BATCHES \ -e NUM_EVAL_BATCHES=$NUM_EVAL_BATCHES \ -e METRIC_LOG_FREQ=$METRIC_LOG_FREQ \ + ${STREAMING_SHUFFLE_FRACTION:+-e STREAMING_SHUFFLE_FRACTION=$STREAMING_SHUFFLE_FRACTION} \ + ${STREAMING_SHUFFLE_SEED:+-e STREAMING_SHUFFLE_SEED=$STREAMING_SHUFFLE_SEED} \ + ${NUM_WORKERS:+-e NUM_WORKERS=$NUM_WORKERS} \ + ${PREFETCH_FACTOR:+-e PREFETCH_FACTOR=$PREFETCH_FACTOR} \ + ${DIAG_UNIQUE_EMB:+-e DIAG_UNIQUE_EMB=$DIAG_UNIQUE_EMB} \ + ${OUTPUT_TRACE:+-e OUTPUT_TRACE=$OUTPUT_TRACE} \ -e TRAIN_SPLIT_PERCENTAGE=${TRAIN_SPLIT_PERCENTAGE:-0.90} \ -e SPLIT_SALT=${SPLIT_SALT:-0} \ -e EVAL_HOLDOUT_TS=${EVAL_HOLDOUT_TS:--1} \ From 851a354de5842e2163e9f1fcc5b11f8838f2b4a9 Mon Sep 17 00:00:00 2001 From: suachong Date: Thu, 11 Jun 2026 21:48:52 +0000 Subject: [PATCH 049/127] dlrmv4: MLPerf training compliance logging for streaming-train-eval Wire mlperf_logging (mllog) into the yambda-5b streaming-train-eval path: rank-0-gated MLPerfLogger facade emitting the full event lifecycle (cache_clear/init_start -> submission_info + hyperparameters -> init_stop/run_start -> per-window block_start/stop + eval_start/accuracy/stop -> run_stop), driven by a configurable convergence target. Key points: - AUC_THRESHOLD (gin, env-overridable; default 0.80275) doubles as the MLPerf convergence target: rank 0 decides on the global lifetime eval AUC and BROADCASTS the stop boolean so all ranks break in lockstep (avoids the ALLTOALL collective-timeout deadlock from a per-rank decision). - MLPerfLogger uses the explicit global rank passed by train_ranker (computed pre-dist-init), so only true rank 0 logs and the compliance file has exactly one event each. Per-job MLPERF_LOG_PATH avoids stale append accumulation. - Per-step train_loss POINT_IN_TIME event (global cross-rank mean) with samples_count + lr, plus a console/TensorBoard readout. - cumulative_train_samples counter (global, checkpointed) as the samples_count progress unit; lifecycle gated on cold start so e2e-supervisor resumes never emit orphaned run boundaries. Validated end-to-end at 1/2/4 nodes (8/16/32 GPUs): clean compliance log, single run_stop=success, passes the common/closed_common compliance checks. Co-authored-by: Cursor --- recommendation_v4/.gitignore | 8 + .../dlrm_v3/checkpoint.py | 8 + .../dlrm_v3/train/gin/yambda_5b.gin | 9 +- .../dlrm_v3/train/mlperf_logging_utils.py | 175 ++++++++++++++ .../dlrm_v3/train/train_ranker.py | 82 +++++++ .../dlrm_v3/train/utils.py | 225 ++++++++++++++++-- .../generative_recommenders/dlrm_v3/utils.py | 75 ++++++ recommendation_v4/requirements.txt | 1 + recommendation_v4/scripts/launch_slurm.sh | 16 +- 9 files changed, 576 insertions(+), 23 deletions(-) create mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py diff --git a/recommendation_v4/.gitignore b/recommendation_v4/.gitignore index 5edddc5b3..5a0329448 100644 --- a/recommendation_v4/.gitignore +++ b/recommendation_v4/.gitignore @@ -157,3 +157,11 @@ dmypy.json # Cython debug symbols cython_debug/ + +# SLURM batch stdout + local run artifacts (run logs are never committed; +# the real run logs / MLPerf compliance logs live under $SCRATCH, outside the +# repo). The trainer's #SBATCH --output lands in the submit dir as +# yambda_slurm..out. +yambda_slurm.*.out +yambda_slurm.*.log +compliance_checker.log diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/checkpoint.py b/recommendation_v4/generative_recommenders/dlrm_v3/checkpoint.py index 0ef223b23..031d19701 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/checkpoint.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/checkpoint.py @@ -397,6 +397,10 @@ def save_dmp_checkpoint( "class_metrics": class_metric_state_dict, "reg_metrics": regression_metric_state_dict, "global_step": metric_logger.global_step, + # MLPerf progress counter (global trained samples). Defaulted on + # load so pre-existing checkpoints restore as 0 and resume the + # count from there. + "cumulative_train_samples": metric_logger.cumulative_train_samples, "sparse_tensor_keys": sparse_tensor_keys, # Streaming resume fields. Defaulted on load so old checkpoints # (pre-streaming-resume) still load as a normal restart. @@ -525,6 +529,10 @@ def load_nonsparse_checkpoint( print("optimizer checkpoint successfully loaded") if metric_logger is not None: metric_logger.global_step = non_sparse_state_dict["global_step"] + # Defaulted for legacy checkpoints written before the counter existed. + metric_logger.cumulative_train_samples = non_sparse_state_dict.get( + "cumulative_train_samples", 0 + ) class_metric_state_dict = non_sparse_state_dict["class_metrics"] regression_metric_state_dict = non_sparse_state_dict["reg_metrics"] # Length-safe positional restore: if a checkpoint was written with a diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index 5b5ec03e9..ba0183543 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -216,7 +216,14 @@ MetricsLogger.tensorboard_log_path = @tbp/env_path() tbp/env_path.key = "TENSORBOARD_LOG_PATH" tbp/env_path.default = "tb/yambda_5b/" MetricsLogger.world_size = 8 -MetricsLogger.auc_threshold = 0.80275 +# Time-to-target AUC threshold. Doubles as the MLPerf convergence target: when +# the cumulative ("lifetime_") listen_plus eval AUC first reaches this value the +# streaming-train-eval run emits a SUCCESS RUN_STOP and terminates gracefully. +# Override via $AUC_THRESHOLD (e.g. 0.5 to smoke-test the early-stop path on a +# short run). MLPerf's DLRM-DCNv2 reference uses 0.80275. +MetricsLogger.auc_threshold = @at/env_float() +at/env_float.key = "AUC_THRESHOLD" +at/env_float.default = 0.80275 # Lifetime-AUC backend, selectable independently for the train cumulative AUC and # the eval cumulative ("lifetime_*") AUC. Both default to "binned": # "binned" = BinnedCumulativeAUC: exact-cumulative AUC from an O(bins) score diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py new file mode 100644 index 000000000..51e7971b5 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py @@ -0,0 +1,175 @@ +# Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe +"""MLPerf Training compliance logging for the DLRMv3 streaming-train-eval path. + +Thin, rank-0-gated wrapper around ``mlperf_logging.mllog`` so the streaming +loop can emit the MLPerf event stream (INIT/RUN/BLOCK/EVAL/RUN_STOP) without +every call site re-checking the rank or guarding against a missing dependency. + +Modeled on recommendation_v2/torchrec_dlrm's inline ``submission_info`` but +extended with rank-0 gating + optional distributed barriers (the NeMo / unet3d +``sync`` pattern), so a multi-rank run produces exactly one valid log. +""" + +import logging +import os +from typing import Any, Dict, Optional + +import gin +import torch + +logger: logging.Logger = logging.getLogger(__name__) + +try: + from mlperf_logging import mllog + from mlperf_logging.mllog import constants as mllog_constants + + _MLLOG_AVAILABLE = True +except Exception as e: # pragma: no cover - import-time guard + mllog = None # type: ignore[assignment] + mllog_constants = None # type: ignore[assignment] + _MLLOG_AVAILABLE = False + logger.warning( + "mlperf_logging not importable (%s); MLPerf logging disabled. " + "Install via `pip install git+https://github.com/mlcommons/logging.git`.", + e, + ) + + +def _rank() -> int: + if torch.distributed.is_available() and torch.distributed.is_initialized(): + return torch.distributed.get_rank() + return 0 + + +def _barrier() -> None: + if torch.distributed.is_available() and torch.distributed.is_initialized(): + torch.distributed.barrier() + + +class MLPerfLogger: + """Rank-0-gated facade over ``mllog``. + + All event methods no-op on non-zero ranks and when ``mlperf_logging`` is not + installed, so callers never need to guard. ``sync=True`` inserts an + all-rank ``dist.barrier()`` before the (rank-0-only) emission so the logged + timestamp reflects the slowest rank reaching the boundary -- required for + INIT_STOP/RUN_START/RUN_STOP per the MLPerf rules. + """ + + def __init__( + self, + rank: Optional[int] = None, + log_path: Optional[str] = None, + default_stack_offset: int = 2, + benchmark_name: str = "hstu", + submitter_name: str = "reference_implementation", + ): + self.enabled: bool = _MLLOG_AVAILABLE + # CRITICAL: use the EXPLICIT global rank passed by the caller, not a + # dist.get_rank() lookup. This logger is constructed BEFORE + # dist.init_process_group (so the init phase can be timed), at which + # point torch.distributed.get_rank() is unavailable and would return 0 + # for every process -> all 16 ranks would log everything. The caller + # (train_ranker) already knows the true global rank + # (node_rank * gpus_per_node + local_rank), so trust it. Fall back to a + # best-effort dist/zero lookup only when not provided. + self.rank: int = rank if rank is not None else _rank() + self.benchmark_name: str = benchmark_name + self.submitter_name: str = submitter_name + self._logger = None + if not self.enabled: + return + if log_path: + os.makedirs(os.path.dirname(log_path), exist_ok=True) + mllog.config(filename=log_path, default_stack_offset=default_stack_offset) + else: + mllog.config(default_stack_offset=default_stack_offset) + self._logger = mllog.get_mllogger() + + @property + def constants(self): # pyre-ignore[3] + return mllog_constants + + def event( + self, + key: str, + value: Any = None, + metadata: Optional[Dict[str, Any]] = None, + sync: bool = False, + ) -> None: + if sync: + _barrier() + if self.enabled and self.rank == 0: + self._logger.event(key=key, value=value, metadata=metadata or {}) + + def start( + self, + key: str, + value: Any = None, + metadata: Optional[Dict[str, Any]] = None, + sync: bool = False, + ) -> None: + if sync: + _barrier() + if self.enabled and self.rank == 0: + self._logger.start(key=key, value=value, metadata=metadata or {}) + + def end( + self, + key: str, + value: Any = None, + metadata: Optional[Dict[str, Any]] = None, + sync: bool = False, + ) -> None: + if sync: + _barrier() + if self.enabled and self.rank == 0: + self._logger.end(key=key, value=value, metadata=metadata or {}) + + def submission_info(self, benchmark_name: str, submitter_name: str) -> None: + """Emit the five SUBMISSION_* events required for a valid submission.""" + if not (self.enabled and self.rank == 0): + return + c = mllog_constants + self.event(key=c.SUBMISSION_BENCHMARK, value=benchmark_name) + self.event(key=c.SUBMISSION_ORG, value=submitter_name) + self.event(key=c.SUBMISSION_DIVISION, value=c.CLOSED) + self.event(key=c.SUBMISSION_STATUS, value=c.ONPREM) + self.event(key=c.SUBMISSION_PLATFORM, value=submitter_name) + + +@gin.configurable +def get_mlperf_logger( + rank: int = 0, + log_path: str = "", + benchmark_name: str = "hstu", + submitter_name: str = "reference_implementation", +) -> MLPerfLogger: + """Build a configured :class:`MLPerfLogger`. + + ``benchmark_name`` / ``submitter_name`` are gin-configurable (and the path is + env-overridable via ``$MLPERF_LOG_PATH``) so a submission can stamp its own + benchmark string without code changes. The log path defaults to + ``$MLPERF_LOG_PATH`` when set, else ``""`` (mllog logs to stdout). + """ + resolved_path = os.environ.get("MLPERF_LOG_PATH", log_path) + return MLPerfLogger( + rank=rank, + log_path=resolved_path, + benchmark_name=benchmark_name, + submitter_name=submitter_name, + ) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py index 55eece518..0168add5b 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py @@ -76,10 +76,33 @@ def _main_func( # decorators in generative_recommenders.ops.triton.* read env vars at # module import time, and the heavy imports below pull those in. from generative_recommenders.dlrm_v3.train._env_bootstrap import apply_env_bootstrap + from generative_recommenders.dlrm_v3.train.mlperf_logging_utils import ( + get_mlperf_logger, + ) gin.parse_config_file(gin_file, skip_unknown=True) apply_env_bootstrap() + # MLPerf compliance logging is only wired for the streaming-train-eval + # (yambda-5b) benchmark path. Build the rank-0-gated logger now. No-ops on + # non-zero ranks and when mlperf_logging is not installed. + mlperf_logger = ( + get_mlperf_logger(rank=rank) if mode == "streaming-train-eval" else None + ) + # Emit the init-start boundary before setup so the init phase is measured, + # but ONLY when this is guaranteed to be a cold start. CKPT_PATH unset means + # checkpoints are disabled (the default + submission config) -> always cold + # start. When CKPT_PATH is set (resumable / e2e-supervisor runs) we defer the + # decision to resume_cold_start below and skip the pre-setup markers, so a + # resume relaunch never emits an orphaned INIT_START/RUN_START. The whole + # downstream sequence (INIT_STOP/RUN_START/blocks/eval/RUN_STOP) is gated on + # this flag so the log is always balanced. + mlperf_init_logged = False + if mlperf_logger is not None and not os.environ.get("CKPT_PATH", ""): + mlperf_logger.event(key=mlperf_logger.constants.CACHE_CLEAR, value=True) + mlperf_logger.start(key=mlperf_logger.constants.INIT_START) + mlperf_init_logged = True + # Phase 2: heavy imports. Triton kernel modules evaluate their autotune # decorators here, using the env vars set above. from generative_recommenders.dlrm_v3.checkpoint import load_dmp_checkpoint @@ -173,6 +196,62 @@ def _main_func( ) ) + # MLPerf: submission info + hyperparameters, then the init/run boundary. + # Gated on (init markers emitted AND genuine cold start) so the e2e + # supervisor's resume relaunches don't restart the INIT/RUN markers + # mid-stream (which would invalidate the single-run log), and so the log is + # never left with an INIT_STOP that has no matching INIT_START. + mlperf_run_active = ( + mlperf_logger is not None and mlperf_init_logged and resume_cold_start + ) + if mlperf_run_active: + c = mlperf_logger.constants + mlperf_logger.submission_info( + benchmark_name=mlperf_logger.benchmark_name, + submitter_name=mlperf_logger.submitter_name, + ) + + def _gin_param(name: str, default: object) -> object: + try: + return gin.query_parameter(name) + except (ValueError, KeyError): + return default + + global_batch_size = world_size * int(train_dataloader.batch_size) + mlperf_logger.event(key=c.GLOBAL_BATCH_SIZE, value=global_batch_size) + mlperf_logger.event(key=c.GRADIENT_ACCUMULATION_STEPS, value=1) + # Seed is fixed in setup() (_SEED = 1); kept in sync here. + mlperf_logger.event(key=c.SEED, value=1) + # Dense (Adam) + sparse (RowWiseAdagrad) optimizer hyperparameters, + # read from the active gin bindings. + mlperf_logger.event( + key=c.OPT_NAME, + value=_gin_param( + "dense_optimizer_factory_and_class.optimizer_name", "Adam" + ), + ) + mlperf_logger.event( + key=c.OPT_BASE_LR, + value=_gin_param( + "dense_optimizer_factory_and_class.learning_rate", None + ), + ) + mlperf_logger.event( + key="opt_sparse_name", + value=_gin_param( + "sparse_optimizer_factory_and_class.optimizer_name", + "RowWiseAdagrad", + ), + ) + mlperf_logger.event( + key="opt_sparse_base_learning_rate", + value=_gin_param( + "sparse_optimizer_factory_and_class.learning_rate", None + ), + ) + mlperf_logger.end(key=c.INIT_STOP, sync=True) + mlperf_logger.start(key=c.RUN_START, sync=True) + # train loop try: if mode == "train": @@ -224,6 +303,9 @@ def _main_func( resume_batch_idx_in_window=resume_batch_idx_in_window, resume_split_contract=resume_split_contract, resume_cold_start=resume_cold_start, + # Only pass the logger when the run boundaries were emitted, so + # the loop never produces orphan block/eval events. + mlperf_logger=mlperf_logger if mlperf_run_active else None, ) except Exception as e: logger.info(traceback.format_exc()) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py index c27f38761..96f826f84 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -1203,6 +1203,10 @@ def streaming_train_eval_loop( checkpoint_time_interval_s: float = 0.0, # --- test-only failure injection knob --- die_at_step: int = -1, + # MLPerf compliance logger (rank-0-gated facade). None disables all MLPerf + # event emission; the loop is otherwise unchanged. Supplied by train_ranker + # for the streaming-train-eval benchmark path. + mlperf_logger: Optional[Any] = None, ) -> None: """Streaming train+eval loop with per-window (and optionally mid-window) checkpoints. @@ -1558,7 +1562,9 @@ def _run_train_window( f"[boundary] {label} train first-batch data-wait={first_wait * 1000:.1f}ms" ) - def _run_eval_window(eval_data_iterator, label: Optional[str] = None) -> None: + def _run_eval_window( + eval_data_iterator, label: Optional[str] = None + ) -> Dict[str, float]: # DO NOT add a checkpoint trigger anywhere inside this function. The eval # data iterator's position is not serializable, so a checkpoint taken # mid-eval could not be resumed deterministically. `_maybe_checkpoint` @@ -1614,7 +1620,8 @@ def _run_eval_window(eval_data_iterator, label: Optional[str] = None) -> None: metric_logger.compute_and_log(mode="eval") if num_eval_batches is not None and eval_batch_idx >= num_eval_batches: break - for k, v in metric_logger.compute(mode="eval").items(): + eval_metrics = metric_logger.compute(mode="eval") + for k, v in eval_metrics.items(): print(f"{k}: {v}") if label and rank == 0 and _t_enter is not None: _eval_total = time.perf_counter() - _t_enter @@ -1623,6 +1630,7 @@ def _run_eval_window(eval_data_iterator, label: Optional[str] = None) -> None: f"[boundary] {label} eval first-batch data-wait={_fw:.1f}ms " f"total_eval={_eval_total * 1000:.1f}ms batches={eval_batch_idx}" ) + return eval_metrics def _maybe_checkpoint(train_ts: int) -> None: if ( @@ -1688,6 +1696,152 @@ def _should_eval(i: int) -> bool: dataset.dataset._train_split_percentage, # pyre-ignore[16] ) + # --- MLPerf progress accounting + event helpers --------------------------- + # `total_train_samples` is the denominator for the MLPerf samples_count / + # epoch_num progress unit: the total GLOBAL trainable samples over the + # configured window range. Computed once up-front (one O(N) anchor mask per + # window -- the same call the loop makes per window) and logged as + # TRAIN_SAMPLES. Skipped entirely when no MLPerf logger is attached, since + # the masks are not free. `mlperf_run_stopped` guards single RUN_STOP. + total_train_samples = 0 + mlperf_run_stopped = [False] + if mlperf_logger is not None: + _twi = getattr(dataset.dataset, "train_window_indices", None) + _wi = getattr(dataset.dataset, "window_indices", None) + _idx_fn = _twi or _wi + if _idx_fn is not None: + for _ts in train_ts_list: + total_train_samples += int(_idx_fn(_ts).size) + if rank == 0: + logger.info( + "MLPerf: total_train_samples=%d over %d windows", + total_train_samples, + n_train, + ) + mlperf_logger.event( + key=mlperf_logger.constants.TRAIN_SAMPLES, value=total_train_samples + ) + if eval_global_indices is not None: + mlperf_logger.event( + key=mlperf_logger.constants.EVAL_SAMPLES, + value=int(eval_global_indices.size), + ) + # Let MetricsLogger.compute_and_log emit the per-step MLPerf `train_loss` + # event (rank-0 gated) at the metric-logging cadence, stamped with the + # current base LR. Read param_groups[0] defensively (KeyedOptimizer + # exposes it; guard against any optimizer that does not). + metric_logger.mlperf_logger = mlperf_logger + + def _current_lr() -> float: + return float(optimizer.param_groups[0]["lr"]) + + metric_logger.lr_getter = _current_lr + + def _mlperf_progress() -> Dict[str, Any]: + samples = metric_logger.cumulative_train_samples + epoch_num = (samples / total_train_samples) if total_train_samples > 0 else 0.0 + return { + mlperf_logger.constants.SAMPLES_COUNT: samples, + mlperf_logger.constants.EPOCH_NUM: epoch_num, + } + + def _lifetime_auc(metrics: Dict[str, float]) -> Optional[float]: + # Convergence metric: the cumulative ("lifetime_") listen_plus AUC. + # Key format is `metric/{prefix}{name}/{task}` (see MetricsLogger.compute), + # e.g. `metric/lifetime_auc/listen_plus`. Match the `lifetime_auc` short + # name; ignore GAUC. + for key, val in metrics.items(): + short = key.split("/")[-2] if "/" in key else key + if short == "lifetime_auc": + return float(val) + return None + + def _mlperf_block_start() -> None: + if mlperf_logger is not None: + mlperf_logger.start( + key=mlperf_logger.constants.BLOCK_START, metadata=_mlperf_progress() + ) + + def _mlperf_block_stop() -> None: + if mlperf_logger is not None: + mlperf_logger.end( + key=mlperf_logger.constants.BLOCK_STOP, metadata=_mlperf_progress() + ) + + def _mlperf_eval_start() -> None: + if mlperf_logger is not None: + mlperf_logger.start( + key=mlperf_logger.constants.EVAL_START, metadata=_mlperf_progress() + ) + + def _mlperf_run_stop(status: object) -> None: + # Emit RUN_STOP exactly once, with an all-rank barrier so the timestamp + # reflects the slowest rank (MLPerf requirement). + if mlperf_logger is None or mlperf_run_stopped[0]: + return + mlperf_logger.end( + key=mlperf_logger.constants.RUN_STOP, + metadata={mlperf_logger.constants.STATUS: status, **_mlperf_progress()}, + sync=True, + ) + mlperf_run_stopped[0] = True + + def _mlperf_eval_stop(eval_metrics: Dict[str, float]) -> bool: + # Emit EVAL_ACCURACY (lifetime listen_plus AUC) + EVAL_STOP, and drive an + # early SUCCESS RUN_STOP when the target threshold is reached. Returns + # True iff the run should stop now -- the SAME value on every rank. + # + # CRITICAL (deadlock avoidance): the cumulative lifetime AUC is produced + # by a reduce that is only valid on global rank 0, so a per-rank + # `lifetime >= thr` test diverges (only rank 0 sees the value) and the + # ranks that "stop" hit the RUN_STOP barrier while the rest march into + # the next window's embedding all-to-all -> NCCL collective-timeout hang + # (observed: 600s ALLTOALL_BASE watchdog abort). So rank 0 decides and + # BROADCASTS the boolean; all ranks then break (or continue) in lockstep. + if mlperf_logger is None: + return False + lifetime = _lifetime_auc(eval_metrics) + if lifetime is not None: + mlperf_logger.event( + key=mlperf_logger.constants.EVAL_ACCURACY, + value=lifetime, + metadata=_mlperf_progress(), + ) + mlperf_logger.end( + key=mlperf_logger.constants.EVAL_STOP, metadata=_mlperf_progress() + ) + thr = metric_logger.auc_threshold + decision = torch.zeros(1, device=device) + if ( + rank == 0 + and not mlperf_run_stopped[0] + and lifetime is not None + and thr is not None + and lifetime >= thr + ): + decision[0] = 1.0 + if torch.distributed.is_initialized(): + torch.distributed.broadcast(decision, src=0) + should_stop = bool(decision.item() > 0.5) + if should_stop: + # All ranks agree -> all reach the RUN_STOP barrier together. + _mlperf_run_stop(mlperf_logger.constants.SUCCESS) + return should_stop + + def _mlperf_finalize(final_metrics: Dict[str, float]) -> None: + # End-of-run RUN_STOP when the threshold was never crossed: SUCCESS iff + # the final lifetime AUC meets the target, else ABORTED. + if mlperf_logger is None or mlperf_run_stopped[0]: + return + lifetime = _lifetime_auc(final_metrics) + thr = metric_logger.auc_threshold + success = lifetime is not None and thr is not None and lifetime >= thr + _mlperf_run_stop( + mlperf_logger.constants.SUCCESS + if success + else mlperf_logger.constants.ABORTED + ) + if persistent_loader and double_buffer: # Double-buffered: next window prepared in the background during the # current window's compute. Eval (if enabled) uses its own pre-forked @@ -1741,16 +1895,23 @@ def _should_eval(i: int) -> bool: if i == 0 and resume_batch_idx_in_window > 0 else 0 ) + _mlperf_block_start() _run_train_window( train_data_iterator, train_ts=train_ts, start_batch_idx=start_batch, label=f"train_ts={train_ts}", ) + _mlperf_block_stop() + should_stop = False if _should_eval(i): dataset.dataset.is_eval = True # pyre-ignore [16] assert eval_sampler is not None and eval_dl is not None - _run_eval_window(eval_iter, label=f"eval_holdout@train_ts={train_ts}") + _mlperf_eval_start() + eval_metrics = _run_eval_window( + eval_iter, label=f"eval_holdout@train_ts={train_ts}" + ) + should_stop = _mlperf_eval_stop(eval_metrics) # Re-arm the (already-forked) eval pool for the NEXT eval. The # holdout set is fixed, so the sampler window is unchanged; we # only need a fresh iter() to replay it. iter() reuses the @@ -1761,6 +1922,9 @@ def _should_eval(i: int) -> bool: if next_eval_i is not None: eval_iter = iter(eval_dl) _maybe_checkpoint(train_ts) + if should_stop: + # MLPerf target reached: RUN_STOP already emitted; stop training. + break else: for i, train_ts in enumerate(train_ts_list): dataset.dataset.is_eval = False # pyre-ignore [16] @@ -1770,15 +1934,19 @@ def _should_eval(i: int) -> bool: if i == 0 and resume_batch_idx_in_window > 0 else 0 ) + _mlperf_block_start() _run_train_window( _window_iter(train_ts, skip_samples=skip), train_ts=train_ts, start_batch_idx=start_batch, ) + _mlperf_block_stop() + should_stop = False if _should_eval(i): dataset.dataset.is_eval = True # pyre-ignore [16] + _mlperf_eval_start() if eval_global_indices is not None: - _run_eval_window( + eval_metrics = _run_eval_window( iter( make_streaming_dataloader( dataset=dataset, indices=eval_global_indices @@ -1788,26 +1956,43 @@ def _should_eval(i: int) -> bool: ) else: # Legacy per-window eval (datasets without user holdout). - _run_eval_window( + eval_metrics = _run_eval_window( iter(make_streaming_dataloader(dataset=dataset, ts=train_ts + 1)) ) + should_stop = _mlperf_eval_stop(eval_metrics) _maybe_checkpoint(train_ts) + if should_stop: + # MLPerf target reached: RUN_STOP already emitted; stop training. + break # Final eval over the SAME fixed user-holdout set (consistent with the # per-window evals above). Reuses _run_eval_window so metrics are reset and # reported the same way. Falls back to the legacy final-window eval for - # datasets without user holdout. - dataset.dataset.is_eval = True # pyre-ignore [16] - if eval_global_indices is not None: - _run_eval_window( - iter(make_streaming_dataloader(dataset=dataset, indices=eval_global_indices)), - label="eval_holdout@final", - ) - else: - _run_eval_window( - iter(make_streaming_dataloader(dataset=dataset, ts=num_train_ts)), - label="eval@final", - ) - if rank == 0: - for k, v in metric_logger.compute(mode="eval").items(): - print(f"{k}: {v}") + # datasets without user holdout. Skipped if the MLPerf target was already + # reached mid-run (RUN_STOP already emitted, run is over). + if not mlperf_run_stopped[0]: + dataset.dataset.is_eval = True # pyre-ignore [16] + _mlperf_eval_start() + if eval_global_indices is not None: + final_metrics = _run_eval_window( + iter( + make_streaming_dataloader( + dataset=dataset, indices=eval_global_indices + ) + ), + label="eval_holdout@final", + ) + else: + final_metrics = _run_eval_window( + iter(make_streaming_dataloader(dataset=dataset, ts=num_train_ts)), + label="eval@final", + ) + # EVAL_ACCURACY/EVAL_STOP for the final pass (may emit SUCCESS RUN_STOP + # if the target was met exactly at the end). + _mlperf_eval_stop(final_metrics) + if rank == 0: + for k, v in final_metrics.items(): + print(f"{k}: {v}") + # End-of-run RUN_STOP: SUCCESS iff the final lifetime AUC met the target, + # else ABORTED. No-op if a SUCCESS RUN_STOP already fired above. + _mlperf_finalize(final_metrics) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py index 1996323a8..6a4f29508 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py @@ -1012,6 +1012,21 @@ def _make_reg(ws: int) -> List[RecMetricComputation]: self.regression_metrics["eval"] = _make_reg(window_size) self.global_step: Dict[str, int] = {"train": 0, "eval": 0} + # Monotonic, resume-safe count of GLOBAL trained samples (summed across + # ranks), used as the MLPerf `samples_count` progress unit. Distinct from + # the perf-only `_perf_total_samples` below (per-rank, not checkpointed): + # this one is persisted/restored alongside `global_step` so a resumed + # streaming run continues the convergence-progress count. + self.cumulative_train_samples: int = 0 + self._rank: int = int(rank) + # Optional MLPerf logger + learning-rate accessor, wired by the streaming + # loop (kept duck-typed -- expects `.event(key, value, metadata)` and + # `.constants` -- to avoid a train-module import cycle). When set, + # compute_and_log emits a POINT_IN_TIME `train_loss` event (rank-0 gated + # inside the logger) at the metric-logging cadence, mirroring the per-step + # MLPerf train_loss readout other benchmarks log. + self.mlperf_logger: Optional[Any] = None + self.lr_getter: Optional[Callable[[], float]] = None self.tb_logger: Optional[SummaryWriter] = None if tensorboard_log_path != "": self.tb_logger = SummaryWriter(log_dir=tensorboard_log_path, purge_step=0) @@ -1030,6 +1045,13 @@ def _make_reg(ws: int) -> List[RecMetricComputation]: 1, dtype=torch.long, device=device ) + @property + def auc_threshold(self) -> Optional[float]: + """Configured time-to-target AUC threshold (None if unset). Exposed so + the streaming loop can drive the MLPerf SUCCESS RUN_STOP off the same + target without reaching into the private attribute.""" + return self._auc_threshold + @property def all_metrics(self) -> Dict[str, List[RecMetricComputation]]: """ @@ -1093,6 +1115,12 @@ def update( self._perf_samples_counter.dtype ) self._perf_steps_in_window += 1 + # MLPerf progress counter: global trained samples this step. Local + # batch sample count (num_candidates is per-rank) scaled by world + # size approximates the global count without an extra collective; + # accumulated on CPU as a plain int so it serializes trivially into + # the checkpoint (see save/load_nonsparse_checkpoint). + self.cumulative_train_samples += int(num_candidates.numel()) * self._world_size def compute(self, mode: str = "train") -> Dict[str, float]: """ @@ -1183,6 +1211,53 @@ def compute_and_log( global_step=self.global_step[mode], ) + # Train-loss readout: surface a single GLOBAL (cross-rank mean) training + # loss on the regular console logger every `metric_log_frequency` batches, + # so progress is visible from step 0 instead of only at the first + # end-of-window eval. The per-loss-term breakdown already goes to + # TensorBoard above (losses/train_*); here we add the combined scalar. + # The all-reduce is a cheap 1-element collective run by EVERY rank at the + # same deterministic steps (this method is called in lockstep), so it + # cannot desync. Set METRIC_LOG_FREQ low (e.g. 1-5) to see it per step. + if mode == "train" and additional_logs is not None and "losses" in additional_logs: + loss_terms = additional_logs["losses"] + if loss_terms: + loss_t = torch.stack( + [v.detach().float().sum() for v in loss_terms.values()] + ).sum() + if torch.distributed.is_available() and torch.distributed.is_initialized(): + torch.distributed.all_reduce( + loss_t, op=torch.distributed.ReduceOp.SUM + ) + loss_t = loss_t / self._world_size + train_loss = float(loss_t) + self.tb_logger.add_scalar( + "train_loss", train_loss, global_step=self.global_step["train"] + ) + if self._rank == 0: + logger.info( + f"train - Step {self.global_step['train']} " + f"train_loss={train_loss:.5f}" + ) + # MLPerf POINT_IN_TIME train_loss (rank-0 gated in the logger). + # samples_count = cumulative GLOBAL trained samples (the same + # progress unit as block/eval events); lr = current base LR. + if self.mlperf_logger is not None: + c = self.mlperf_logger.constants + md: Dict[str, Any] = { + c.SAMPLES_COUNT: self.cumulative_train_samples, + } + if self.lr_getter is not None: + try: + md["lr"] = float(self.lr_getter()) + except Exception: + pass + self.mlperf_logger.event( + key=getattr(c, "TRAIN_LOSS", "train_loss"), + value=train_loss, + metadata=md, + ) + # Throughput metrics (train only). One GPU->CPU sync per call. if mode == "train" and self._perf_steps_in_window > 0: now = time.perf_counter() diff --git a/recommendation_v4/requirements.txt b/recommendation_v4/requirements.txt index 023c22332..a8637bf5e 100644 --- a/recommendation_v4/requirements.txt +++ b/recommendation_v4/requirements.txt @@ -5,3 +5,4 @@ gin_config>=0.5.0 pandas>=2.2.0 tensorboard>=2.19.0 pybind11 +git+https://github.com/mlcommons/logging.git diff --git a/recommendation_v4/scripts/launch_slurm.sh b/recommendation_v4/scripts/launch_slurm.sh index 098a69529..c100a6828 100755 --- a/recommendation_v4/scripts/launch_slurm.sh +++ b/recommendation_v4/scripts/launch_slurm.sh @@ -232,11 +232,13 @@ orchestrate() { -e NUM_EVAL_BATCHES=$NUM_EVAL_BATCHES \ -e METRIC_LOG_FREQ=$METRIC_LOG_FREQ \ -e TRAIN_SPLIT_PERCENTAGE=${TRAIN_SPLIT_PERCENTAGE:-0.90} \ + -e AUC_THRESHOLD=${AUC_THRESHOLD:-0.80275} \ -e SPLIT_SALT=${SPLIT_SALT:-0} \ -e EVAL_HOLDOUT_TS=${EVAL_HOLDOUT_TS:--1} \ -e EVAL_HOLDOUT_NUM_WINDOWS=${EVAL_HOLDOUT_NUM_WINDOWS:-1} \ ${RUN_NAME:+-e RUN_NAME=$RUN_NAME} \ ${TENSORBOARD_LOG_PATH:+-e TENSORBOARD_LOG_PATH=$TENSORBOARD_LOG_PATH} \ + ${MLPERF_LOG_PATH:+-e MLPERF_LOG_PATH=$MLPERF_LOG_PATH} \ ${CKPT_PATH:+-e CKPT_PATH=$CKPT_PATH} \ -e LOG=$LOG \ $NCCL_ENV_ARGS \ @@ -382,8 +384,18 @@ worker() { # TensorBoard under the writable scratch root unless the caller (e.g. the e2e # supervisor) pinned a per-run path. Keeps the gin default from ever being used. export TENSORBOARD_LOG_PATH=${TENSORBOARD_LOG_PATH:-$SCRATCH/tb/yambda_5b} - # Append (not truncate): under the streaming-e2e supervisor a run may relaunch - # many times into the SAME $LOG; the supervisor initializes it once at run start. + # MLPerf Training compliance log (streaming-train-eval path). Lands beside the + # other run outputs under scratch unless the caller pins it. Rank 0 writes it; + # check it post-run with: + # python -m mlperf_logging.compliance_checker --usage training \ + # --ruleset 5.0.0 "$MLPERF_LOG_PATH" + # Default to a PER-JOB filename so each standalone `sbatch` gets a clean + # compliance log: mllog opens the file in APPEND mode, so a fixed name would + # accumulate events across runs and fail the compliance_checker (duplicate + # INIT_START/RUN_START). The streaming-e2e supervisor pins MLPERF_LOG_PATH + # explicitly (and inits it once at run start), so its relaunch-into-same-file + # append semantics are preserved untouched. + export MLPERF_LOG_PATH=${MLPERF_LOG_PATH:-$SCRATCH/mlperf/yambda_5b_mlperf.${SLURM_JOB_ID:-manual}.log} echo "[$(date)] REPO_ROOT=$REPO_ROOT" | tee -a "$LOG" # polars-u64-idx (NOT stock polars) — yambda parquet's flat-explode overruns From 2fc3cb1c31dcb400e657302ea45ce655ff6c3889 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Fri, 12 Jun 2026 20:48:18 -0500 Subject: [PATCH 050/127] dlrmv4: min_history anchor-eligibility floor (decoupled from history_length) A LISTEN event qualifies as a train/eval anchor once the user has >= min_history prior events, decoupled from history_length (the gather/truncation cap) since jagged attention handles short UIH. Anchor positions/anchor_ts caches are keyed by (history_length, min_history) and built independently of the _READY-gated 150GB flat store, so changing the floor rebuilds only the cheap positions array. Default None preserves the legacy full-history behavior (which dropped ~60% of users). Co-authored-by: Cursor --- recommendation_v4/README.MD | 9 +- .../dlrm_v3/datasets/yambda.py | 87 ++++++++++++++++++- .../generative_recommenders/dlrm_v3/utils.py | 7 ++ 3 files changed, 97 insertions(+), 6 deletions(-) diff --git a/recommendation_v4/README.MD b/recommendation_v4/README.MD index dc1ced73b..acd59cc31 100644 --- a/recommendation_v4/README.MD +++ b/recommendation_v4/README.MD @@ -100,7 +100,7 @@ The `like` pool is roughly **30× rarer** than `lp` — important context for th ## 4. How data is fed to HSTU -For every training anchor (a LISTEN event with ≥ `history_length` prior events), the dataset builds a `(uih_kjt, candidate_kjt)` pair: +For every training anchor (a LISTEN event with ≥ `min_history` prior events — default `1`, i.e. ~all users; set `$MIN_HISTORY=4086` for the legacy "full `history_length` of context required" filter that dropped ~60% of users), the dataset builds a `(uih_kjt, candidate_kjt)` pair: ``` UIH (User Interaction History): @@ -186,8 +186,10 @@ The streaming path enforces **no future leakage** at two levels: Note this is a *temporal* split on the training stream — distinct from the preprocessing GTS split (§2) that carves off the final test day. Windows are indexed off the per-anchor target timestamp via a lazily-built, mmap'd -`anchor_ts_L{H}.npy` cache (built once on first use; the default non-streaming -path never touches it). +`anchor_ts_L{H}[_m{MIN_HISTORY}].npy` cache (built once on first use; the +default non-streaming path never touches it). The anchor `positions` and +`anchor_ts` arrays are keyed by `(history_length, min_history)` so different +floors don't collide and the expensive flat store is shared across them. ### 5.2 Knobs @@ -201,6 +203,7 @@ with env overrides: | `PERSISTENT_LOADER` | `streaming_train_eval_loop.persistent_loader` | 1 | reuse one worker pool across windows (no per-window respawn) | | `DOUBLE_BUFFER` | `streaming_train_eval_loop.double_buffer` | 1 | prepare the next window in a background thread during compute | | `EVAL_EACH_WINDOW` | `streaming_train_eval_loop.eval_each_window` | 1 | eval window T+1 after training window T | +| `MIN_HISTORY` | `get_dataset.min_history` | 1 | anchor-eligibility floor: min prior events for a LISTEN to be a sample (1 = ~all users; 4086 = legacy full-history filter) | | — | `streaming_train_eval_loop.num_train_batches` / `num_eval_batches` | unset | cap per-window steps (unset = consume full window) | ### 5.3 Hiding the window-reset overhead diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/datasets/yambda.py b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/yambda.py index 6627033bc..ed98a1013 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/datasets/yambda.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/yambda.py @@ -204,6 +204,7 @@ def __init__( metadata_dir: str, history_length: int = 2048, scan_window: int = 20000, + min_history: Optional[int] = None, cross_specs: Optional[Sequence[Tuple[str, Sequence[str], int, int]]] = None, cache_dir: Optional[str] = None, is_inference: bool = False, @@ -221,6 +222,15 @@ def __init__( self._metadata_dir: str = metadata_dir self._history_length: int = history_length self._scan_window: int = scan_window + # Minimum prior-event count for a LISTEN event to qualify as an anchor. + # Decoupled from history_length (which is only the gather/truncation cap): + # jagged attention handles short UIH, so we no longer require a full + # history_length of context to include a sample. Default None preserves + # the legacy "need a full history_length of prior events" behavior (which + # dropped ~60% of users); set small (e.g. 1) to include ~all users. + self._min_history: int = ( + history_length if min_history is None else int(min_history) + ) # Streaming/temporal-order state. Everything here is LAZY: nothing is # built or read until the first set_ts()/num_windows() call (only the # streaming-train-eval loop does that), so the default train-eval path @@ -274,15 +284,83 @@ def __init__( self._cache_dir = cache_dir self._ensure_cache_built(cache_dir, processed_dir, history_length) self.store: _FlatEventStore = _FlatEventStore.load_mmap(cache_dir) - # Mmap the positions file built alongside the flat columns. + # Anchor positions depend on min_history (the eligibility floor), not + # just history_length (the gather cap), so they live in a + # min_history-versioned file that shares the flat store. Built + # independently of the _READY sentinel so changing the floor rebuilds + # only this (cheap) array, not the whole 150 GB cache. + self._positions_name: str = self._positions_filename( + history_length, self._min_history + ) + self._ensure_positions_built( + cache_dir, self._positions_name, self._min_history + ) self._positions: np.ndarray = _load_npy_readonly( - os.path.join(cache_dir, f"positions_L{history_length}.npy") + os.path.join(cache_dir, self._positions_name) ) logger.info( f"Yambda dataset ready: {self.store.total_events:,} events, " f"{len(self._positions):,} training positions" ) + @staticmethod + def _positions_filename(history_length: int, min_history: int) -> str: + """Anchor-positions filename. Uses the legacy name when the floor equals + the gather cap (the historical "full history required" behavior) so + existing caches are reused as-is; otherwise a min_history-tagged name.""" + if min_history == history_length: + return f"positions_L{history_length}.npy" + return f"positions_L{history_length}_m{min_history}.npy" + + @staticmethod + def _ensure_positions_built( + cache_dir: str, positions_name: str, min_history: int + ) -> None: + """Build the anchor-positions array for ``min_history`` if absent. + + Anchors are LISTEN events whose user-local offset is >= ``min_history`` + (i.e. the user already has that many prior events). This is decoupled + from the _READY-gated flat-store build so a new floor only rebuilds this + (cheap, ~one int64 scan) array rather than the whole 150 GB cache. + Multi-rank safe via an exclusive lock + atomic rename; all ranks then + mmap the result read-only. + """ + import fcntl + + positions_path = os.path.join(cache_dir, positions_name) + if os.path.exists(positions_path): + return + lock_path = os.path.join(cache_dir, "_positions_lock") + with open(lock_path, "w") as lf: + logger.info(f"Acquiring positions build lock for {positions_path}...") + fcntl.flock(lf, fcntl.LOCK_EX) + try: + if os.path.exists(positions_path): + return + flat_uid = _load_npy_readonly( + os.path.join(cache_dir, "flat_uid.npy") + ) + event_types = _load_npy_readonly( + os.path.join(cache_dir, "flat_event_types.npy") + ) + user_start = _load_npy_readonly( + os.path.join(cache_dir, "user_start.npy") + ) + idx = np.arange(len(flat_uid), dtype=np.int64) + keep = (idx - user_start[flat_uid] >= min_history) & ( + event_types == LISTEN_TYPE + ) + positions = np.where(keep)[0].astype(np.int64) + tmp = positions_path + ".tmp.npy" + np.save(tmp, positions) + os.replace(tmp, positions_path) + logger.info( + f"Wrote {positions_name}: {len(positions):,} anchors " + f"(min_history={min_history})" + ) + finally: + fcntl.flock(lf, fcntl.LOCK_UN) + @staticmethod def _ensure_cache_built( cache_dir: str, processed_dir: str, history_length: int @@ -496,8 +574,11 @@ def _ensure_streaming_index(self) -> None: import fcntl assert self._cache_dir is not None + # Target-ts array is per-anchor, so it must track the same min_history + # versioning as the positions file it indexes into. anchor_path = os.path.join( - self._cache_dir, f"anchor_ts_L{self._history_length}.npy" + self._cache_dir, + self._positions_name.replace("positions_", "anchor_ts_", 1), ) if not os.path.exists(anchor_path): lock_path = os.path.join(self._cache_dir, "_anchor_ts_lock") diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py index 0abfaa86a..dcda51365 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py @@ -1422,6 +1422,7 @@ def get_dataset( name: str, new_path_prefix: str = "", history_length: Optional[int] = None, + min_history: Optional[int] = None, streaming_window_seconds: int = 86400, streaming_sort_within_window: bool = False, streaming_shuffle_fraction: float = 0.0, @@ -1553,6 +1554,12 @@ def get_dataset( # Override via `get_dataset.history_length = N` in gin. "history_length": history_length if history_length is not None else 4096, "scan_window": 20000, + # Anchor-eligibility floor: a LISTEN event qualifies once the + # user has >= min_history prior events. Decoupled from + # history_length (gather cap) since jagged attention handles + # short UIH. None = legacy (require a full history_length). + # Override via `get_dataset.min_history = N` / $MIN_HISTORY. + "min_history": min_history, "cross_specs": YAMBDA_5B_CROSS_SPECS, # Temporal-streaming knobs (only used under --mode # streaming-train-eval; ignored by the default train-eval path). From e5214b8b8ddfea6a3b5270a4aef385bfecd8b4af Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Fri, 12 Jun 2026 20:48:18 -0500 Subject: [PATCH 051/127] =?UTF-8?q?dlrmv4:=20gin=20defaults=20=E2=80=94=20?= =?UTF-8?q?min=5Fhistory=3D1,=20BATCH=5FSIZE=20env=20knob,=20default=20to?= =?UTF-8?q?=20no=20in-window=20shuffle?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bind get_dataset.min_history to $MIN_HISTORY (default 1 = ~all users). Make batch_size env-overridable via $BATCH_SIZE (default 1024). Change default streaming_shuffle_fraction 1.0 -> 0.0 (user-major, production streaming order); override per-run via $STREAMING_SHUFFLE_FRACTION (1.0 = full shuffle). Co-authored-by: Cursor --- .../dlrm_v3/train/gin/yambda_5b.gin | 31 ++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index 92c2404fc..3bfccc510 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -1,4 +1,9 @@ -batch_size = 1024 +# Per-rank batch size. Env-overridable so a denser per-sample shape can fit in +# HBM by lowering it without editing gin. Default 1024 preserves prior runs; +# streaming train+eval both read this macro (make_*streaming_dataloader.batch_size). +batch_size = @bs/env_int() +bs/env_int.key = "BATCH_SIZE" +bs/env_int.default = 1024 # Dataloader parallelism. Env-overridable so a perf sweep can probe whether the # shuffle steady-state cost is CPU-gather latency (hidden by more workers) vs # GPU-side embedding work (not). Defaults preserve prior behavior (4 / 8). @@ -100,6 +105,18 @@ get_dataset.history_length = @hl/env_int() hl/env_int.key = "HISTORY_LENGTH" hl/env_int.default = 4086 +# Anchor-eligibility floor: a LISTEN event becomes a trainable/eval anchor once +# the user has >= MIN_HISTORY prior events. Decoupled from history_length (which +# is only the gather/truncation cap) — jagged attention handles short UIH, so we +# no longer need a full history_length of context to include a sample. The legacy +# behavior (require a full history_length, which dropped ~60% of users) is +# MIN_HISTORY=4086; the default below (1) includes ~all users with any history. +# Positions/anchor-ts caches are keyed by (L, MIN_HISTORY) so floors don't +# collide. Override via $MIN_HISTORY. +get_dataset.min_history = @mh/env_int() +mh/env_int.key = "MIN_HISTORY" +mh/env_int.default = 1 + # Model-side attention budget. Dataset truncates UIH to fit this value if # `history_length + contextual + candidate` would overflow. Override via # $MAX_SEQ_LEN (default 4096, the 4k-no-truncation shape paired with @@ -125,12 +142,12 @@ get_dataset.streaming_sort_within_window = False # shuffle (max diversity), intermediate = interpolation. Same fraction => same # diversity regardless of world_size / #nodes / batch_size. # -# Default 1.0 (full shuffle) so the standard/benchmark run is maximally diverse -# and, together with the fixed seed below, the in-window order is fully -# DETERMINISTIC and identical across runs/resumes (pure function of (seed, ts)). -# Override per-run via $STREAMING_SHUFFLE_FRACTION (e.g. 0.0 for user-major, -# 0.03 for the diversity/locality sweet spot). -streaming_shuffle_fraction = 1.0 +# Default 0.0 (no shuffle -> user-major, page-local mmap scans) so the standard +# run matches the production streaming order. Together with the fixed seed below +# the in-window order is fully DETERMINISTIC and identical across runs/resumes. +# Override per-run via $STREAMING_SHUFFLE_FRACTION (e.g. 1.0 for a full +# per-element shuffle / max diversity, 0.03 for the diversity/locality sweet spot). +streaming_shuffle_fraction = 0.0 get_dataset.streaming_shuffle_fraction = @ssf/env_float() ssf/env_float.key = "STREAMING_SHUFFLE_FRACTION" ssf/env_float.default = %streaming_shuffle_fraction From 5fdf70c0e7e5817a81edb1e775b8e135d65fb4c9 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Fri, 12 Jun 2026 20:48:19 -0500 Subject: [PATCH 052/127] dlrmv4: launch_slurm container hygiene + readiness gating + env passthroughs Reap stale foreign GPU containers and restart our own to reclaim leaked HBM before launch; gate the worker exec on container State.Running + a probe with retry to fix container-restart races (which caused NCCL TCPStore 600s timeouts); APPEND_LOG=1 appends the metrics log on resume. Forward MIN_HISTORY/MAX_SEQ_LEN/HISTORY_LENGTH/BATCH_SIZE/CKPT_TIME_INTERVAL_S/DIAG_EMB_STEPS into the container. Drop the hardcoded --time; tidy header comments after launch_smoke_8gpu removal. Co-authored-by: Cursor --- recommendation_v4/scripts/launch_slurm.sh | 148 +++++++++++++++++++--- 1 file changed, 133 insertions(+), 15 deletions(-) diff --git a/recommendation_v4/scripts/launch_slurm.sh b/recommendation_v4/scripts/launch_slurm.sh index 42f8780ed..6b5ba7607 100755 --- a/recommendation_v4/scripts/launch_slurm.sh +++ b/recommendation_v4/scripts/launch_slurm.sh @@ -4,16 +4,15 @@ #SBATCH --ntasks-per-node=1 #SBATCH --exclusive #SBATCH --partition=meta64 # [CLUSTER-SPECIFIC] partition name -#SBATCH --time=01:10:00 #SBATCH --output=/apps/chcai/yambda_slurm.%j.out # ============================================================================= # launch_slurm.sh — single entry point for the yambda-5b trainer on N>=1 nodes. # -# Consolidates what used to be three separate files so multi-node enablement is +# Consolidates what used to be separate scripts so multi-node enablement is # ONE committable script (plus the train_ranker.py / utils.py python changes): -# * sbatch_smoke_multinode.sh -> the `orchestrate` phase (host SLURM glue) -# * _provision_yambda_primus.sh -> the `provision` phase (container + RDMA) -# * launch_smoke_8gpu.sh -> the `worker` phase (in-container train) +# * orchestrate phase (host SLURM glue) — formerly sbatch_smoke_multinode.sh +# * provision phase (container + RDMA) — formerly _provision_yambda_primus.sh +# * worker phase (in-container train) — now inlined below # # PHASES (auto-detected from context; force with LAUNCH_SLURM_PHASE=): # orchestrate Runs on the SLURM batch host (no /.dockerenv). Resolves the @@ -141,7 +140,15 @@ orchestrate() { METRIC_LOG_FREQ=${METRIC_LOG_FREQ:-5} FORCE_PROVISION=${FORCE_PROVISION:-0} - : > "$LOG" + # Truncate the metrics log on a FRESH run; APPEND on a supervised relaunch + # (APPEND_LOG=1) so the full-run NE/AUC history survives crash/node-failover + # resubmits instead of being wiped on every attempt (mirrors the single-node + # supervisor's init-once/append model). + if [ "${APPEND_LOG:-0}" = "1" ]; then + echo "[$(date)] === resume: appending to existing $LOG (APPEND_LOG=1) ===" >> "$LOG" + else + : > "$LOG" + fi echo "[$(date)] launch_slurm/orchestrate: job=${SLURM_JOB_ID:-?} nodes=${SLURM_JOB_NODELIST:-?} nnodes=${SLURM_NNODES:-1}" | tee -a "$LOG" echo "[$(date)] resolved SCRIPT_PATH=$SCRIPT_PATH REPO=$REPO" | tee -a "$LOG" echo "[$(date)] config: MODE=$MODE START_TS=$START_TS NUM_TRAIN_TS=$NUM_TRAIN_TS NUM_TRAIN_BATCHES=$NUM_TRAIN_BATCHES NUM_EVAL_BATCHES=$NUM_EVAL_BATCHES METRIC_LOG_FREQ=$METRIC_LOG_FREQ" | tee -a "$LOG" @@ -177,19 +184,78 @@ orchestrate() { # --- step 1: ensure the container is up on every node ---------------------- echo "[$(date)] ensuring container '$CONTAINER' on all nodes (force=$FORCE_PROVISION)" | tee -a "$LOG" srun --ntasks-per-node=1 bash -c " + # Reap stale/foreign GPU containers from prior jobs BEFORE (re)provisioning. + # The node is allocated --exclusive, so any GPU container other than + # '$CONTAINER' is an orphan left by a previous job (its container outlives the + # SLURM allocation). We remove every such container that has GPU access + # (/dev/kfd or /dev/dri) — running OR stopped, whether or not it currently + # pins VRAM ('docker ps -aq' includes stopped ones) — since idle orphans can + # still hold device handles or wake up; leaked HBM from these has caused both + # OOMs and RCCL collective hangs. We deliberately SKIP non-GPU containers + # (e.g. 'k8s-node-services-*' and other cluster system services) so we don't + # disrupt node infrastructure. docker teardown lets the driver reclaim HBM. + for _c in \$(docker ps -aq 2>/dev/null); do + _nm=\$(docker inspect -f '{{.Name}}' \"\$_c\" 2>/dev/null | sed 's#^/##') + [ \"\$_nm\" = \"$CONTAINER\" ] && continue + _dev=\$(docker inspect -f '{{range .HostConfig.Devices}}{{.PathOnHost}} {{end}}' \"\$_c\" 2>/dev/null) + case \"\$_dev\" in + *kfd*|*dri*) + echo \"[\$(hostname)] reaping stale GPU container \$_nm (\$_c)\" + docker rm -f \"\$_c\" >/dev/null 2>&1 || true ;; + *) + echo \"[\$(hostname)] keeping non-GPU/system container \$_nm (\$_c)\" ;; + esac + done if [ \"$FORCE_PROVISION\" = \"1\" ] || ! docker exec $CONTAINER true >/dev/null 2>&1; then echo \"[\$(hostname)] (re)provisioning container\" LAUNCH_SLURM_PHASE=provision CONTAINER=$CONTAINER IMAGE=$IMAGE \ BAKED_IMAGE=$BAKED_IMAGE BAKED_TAR=$BAKED_TAR USE_BAKED=$USE_BAKED \ BAKE_IMAGE=${BAKE_IMAGE:-0} RDMA_OVERLAY=$OVERLAY REPO=$REPO bash $SCRIPT_PATH else - echo \"[\$(hostname)] container already up\" + # Container persists across jobs; the reap above only removes FOREIGN GPU + # containers, so our own '$CONTAINER' can still pin HBM via stray trainer + # ranks left by a prior OOM/crash (this caused repeated 'CUDA out of memory' + # on relaunch onto the same node). Restart it to kill every exec'd proc and + # let the driver reclaim HBM — cheap (keeps the installed deps in the + # container fs; NFS RDMA overlay also persists), no full re-provision. + echo \"[\$(hostname)] container already up — restarting to free any leaked HBM before launch\" + docker restart $CONTAINER >/dev/null 2>&1 || true + # Readiness gate: a bare 'docker exec true' can pass while the runtime is + # still settling, so the SUBSEQUENT (heavier) worker exec races the restart + # and dies with 'container is not running' / OCI 'setns' errors (observed on + # c07-08 and e08-08 -> the peer never joins rendezvous -> master 600s + # TCPStore timeout). Require State.Running=true AND a successful probe, then + # a short settle, before considering the container ready. + for _w in \$(seq 1 30); do + [ \"\$(docker inspect -f '{{.State.Running}}' $CONTAINER 2>/dev/null)\" = \"true\" ] \ + && docker exec $CONTAINER true >/dev/null 2>&1 && break + sleep 2 + done + sleep 2 + echo \"[\$(hostname)] container restarted (HBM reclaimed; running=\$(docker inspect -f '{{.State.Running}}' $CONTAINER 2>/dev/null))\" fi " 2>&1 | tee -a "$LOG" # --- step 2: launch the worker (trainer) inside the container on every node - echo "[$(date)] launching trainer (worker phase) on all nodes" | tee -a "$LOG" srun --ntasks-per-node=1 bash -c " + # Pre-flight readiness gate (per node): step 1 ran in a SEPARATE srun, so the + # container can still be settling here. Wait for State.Running=true + a probe + # before the worker exec so we don't race a just-restarted container. + for _w in \$(seq 1 30); do + [ \"\$(docker inspect -f '{{.State.Running}}' $CONTAINER 2>/dev/null)\" = \"true\" ] \ + && docker exec $CONTAINER true >/dev/null 2>&1 && break + [ \$_w -eq 1 ] && echo \"[\$(hostname)] worker pre-flight: waiting for container to be ready...\" + sleep 2 + done + # Retry wrapper: docker exec startup failures (rc 125 daemon 'container is not + # running', 126/127 OCI/setns 'exec failed') mean the container wasn't ready, + # NOT that the trainer ran and failed. Restart + re-gate + retry a few times. + # Any OTHER rc (the trainer actually started and exited) is propagated so the + # supervisor's resume-from-checkpoint logic owns real failures. + _wattempt=0 + while : ; do + _wattempt=\$((_wattempt+1)) docker exec \ -e LAUNCH_SLURM_PHASE=worker \ -e SLURM_NNODES=\$SLURM_NNODES \ @@ -213,7 +279,16 @@ orchestrate() { ${NUM_WORKERS:+-e NUM_WORKERS=$NUM_WORKERS} \ ${PREFETCH_FACTOR:+-e PREFETCH_FACTOR=$PREFETCH_FACTOR} \ ${DIAG_UNIQUE_EMB:+-e DIAG_UNIQUE_EMB=$DIAG_UNIQUE_EMB} \ + ${DIAG_EMB_STEPS:+-e DIAG_EMB_STEPS=$DIAG_EMB_STEPS} \ ${OUTPUT_TRACE:+-e OUTPUT_TRACE=$OUTPUT_TRACE} \ + ${MIN_HISTORY:+-e MIN_HISTORY=$MIN_HISTORY} \ + ${MAX_SEQ_LEN:+-e MAX_SEQ_LEN=$MAX_SEQ_LEN} \ + ${HISTORY_LENGTH:+-e HISTORY_LENGTH=$HISTORY_LENGTH} \ + ${BATCH_SIZE:+-e BATCH_SIZE=$BATCH_SIZE} \ + ${CKPT_TIME_INTERVAL_S:+-e CKPT_TIME_INTERVAL_S=$CKPT_TIME_INTERVAL_S} \ + ${KEEP_LAST_N:+-e KEEP_LAST_N=$KEEP_LAST_N} \ + ${IN_WINDOW_CKPT_FREQ:+-e IN_WINDOW_CKPT_FREQ=$IN_WINDOW_CKPT_FREQ} \ + ${CKPT_STEP_FREQ:+-e CKPT_STEP_FREQ=$CKPT_STEP_FREQ} \ -e TRAIN_SPLIT_PERCENTAGE=${TRAIN_SPLIT_PERCENTAGE:-0.90} \ -e SPLIT_SALT=${SPLIT_SALT:-0} \ -e EVAL_HOLDOUT_TS=${EVAL_HOLDOUT_TS:--1} \ @@ -224,6 +299,20 @@ orchestrate() { -e LOG=$LOG \ $NCCL_ENV_ARGS \ $CONTAINER bash -lc 'cd $REPO && LAUNCH_SLURM_PHASE=worker bash scripts/launch_slurm.sh' + _wrc=\$? + if { [ \$_wrc -eq 125 ] || [ \$_wrc -eq 126 ] || [ \$_wrc -eq 127 ]; } && [ \$_wattempt -lt 5 ]; then + echo \"[\$(hostname)] worker exec failed to START (rc=\$_wrc, attempt \$_wattempt/5) — container not ready; restarting + retrying\" + docker restart $CONTAINER >/dev/null 2>&1 || true + for _w in \$(seq 1 30); do + [ \"\$(docker inspect -f '{{.State.Running}}' $CONTAINER 2>/dev/null)\" = \"true\" ] \ + && docker exec $CONTAINER true >/dev/null 2>&1 && break + sleep 2 + done + sleep 3 + continue + fi + exit \$_wrc + done " 2>&1 | tee -a "$LOG" rc=${PIPESTATUS[0]} echo "[$(date)] launch_slurm/orchestrate finished rc=$rc" | tee -a "$LOG" @@ -400,21 +489,29 @@ worker() { else NNODES=${NNODES:-1} NODE_RANK=${NODE_RANK:-0} - MASTER_ADDR=${MASTER_ADDR:-localhost} + # Single-node: all ranks live on THIS host, so rendezvous over loopback and + # do NOT use the SLURM hostname. On some nodes the hostname resolves to a + # non-routable per-GPU RoCE /31 (benic 192.168.x) address; using it makes the + # NCCL bootstrap fail with "No route to host". localhost is node-independent. + MASTER_ADDR=localhost MASTER_PORT=${MASTER_PORT:-} # empty => train_ranker picks a free port fi export NNODES NODE_RANK GPUS_PER_NODE MASTER_ADDR MASTER_PORT export WORLD_SIZE=$(( NNODES * GPUS_PER_NODE )) echo "[$(date)] topology: nnodes=$NNODES node_rank=$NODE_RANK gpus_per_node=$GPUS_PER_NODE world_size=$WORLD_SIZE master=$MASTER_ADDR:${MASTER_PORT:-}" | tee -a "$LOG" - # RCCL/NCCL cross-node knobs (multi-node only; single-node leaves auto-detect). - # The container is --network=host so RCCL sees ALL host interfaces; split the - # two planes explicitly: TCP bootstrap over the routable fenic0, RDMA data over - # the 8 Broadcom bnxt_re RoCE HCAs (the per-GPU benic* 192.168.x/31 links are - # NOT node-routable for TCP — auto-detect there hangs init). + # NCCL bootstrap NIC — pin for BOTH single- and multi-node. The container is + # --network=host so RCCL sees ALL host interfaces; if left to auto-detect, NCCL + # can pick a non-routable per-GPU RoCE /31 (benic* 192.168.x) link and fail + # bootstrap with "No route to host" (this is node-dependent: it happened to + # work on some nodes and not others, causing repetitive single-node init + # failures). Pinning the routable host NIC fixes it everywhere. + # [CLUSTER-SPECIFIC] routable host NIC for TCP bootstrap (find via `ip -br addr`). + export NCCL_SOCKET_IFNAME=${NCCL_SOCKET_IFNAME:-fenic0} + + # Multi-node additionally needs the RDMA data-plane (bnxt_re HCAs) configured; + # single-node uses intra-node P2P (XGMI/PCIe) so only the bootstrap NIC matters. if [ "$NNODES" -gt 1 ]; then - # [CLUSTER-SPECIFIC] routable host NIC for TCP bootstrap (find via `ip -br addr`). - export NCCL_SOCKET_IFNAME=${NCCL_SOCKET_IFNAME:-fenic0} NCCL_NET_TRANSPORT=${NCCL_NET_TRANSPORT:-ib} if [ "$NCCL_NET_TRANSPORT" = "socket" ]; then export NCCL_IB_DISABLE=1 @@ -467,6 +564,27 @@ worker() { echo "[$(date)] sclk sample (GPU0):$(rocm-smi -d 0 --showclocks 2>/dev/null | grep -i 'sclk clock level' | sed -E 's/.*sclk clock level//')" | tee -a "$LOG" || true fi + # --- stray-trainer / leaked-VRAM guard ------------------------------------- + # The trainer runs via `docker exec` into a long-lived container, so its procs + # live in the container PID namespace, NOT the SLURM job cgroup. If a prior job + # OOM'd/crashed, a rank can leak and keep holding ~half of every GPU's VRAM, + # which persists across jobs (container survives) and guarantees the next + # attempt OOMs. Before launching, reap any pre-existing trainer procs (there + # should be none at this point) and wait for VRAM to drain. [g]-guard avoids + # self-match. Non-fatal. + if pgrep -f '[g]enerative_recommenders' >/dev/null 2>&1; then + echo "[$(date)] WARNING: leaked trainer procs found pre-launch — killing." | tee -a "$LOG" + pkill -9 -f '[g]enerative_recommenders' 2>/dev/null || true + for _i in $(seq 1 15); do + pgrep -f '[g]enerative_recommenders' >/dev/null 2>&1 || break + sleep 2 + done + sleep 5 # let the driver release VRAM after process exit + if command -v rocm-smi >/dev/null 2>&1; then + echo "[$(date)] post-cleanup GPU0 used GiB:$(rocm-smi --showmeminfo vram 2>/dev/null | awk -F: '/Used/{printf " %.0f", $3/1073741824; exit}')" | tee -a "$LOG" + fi + fi + echo "[$(date)] launching train_ranker with WORLD_SIZE=$WORLD_SIZE" | tee -a "$LOG" python -m generative_recommenders.dlrm_v3.train.train_ranker \ --dataset yambda-5b --mode "${MODE:-streaming-train-eval}" 2>&1 | tee -a "$LOG" From 6182ab46dce96c3b52ccdcd60c429ebd078f5566 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Fri, 12 Jun 2026 20:48:19 -0500 Subject: [PATCH 053/127] dlrmv4: durable per-boundary eval metrics JSONL sink + aggregate emb-diag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Write one JSON line per eval boundary to .metrics.jsonl capturing the end-of-pass metrics over the fixed holdout (append-only, rank 0, survives restarts/resumes) — no interim/averaging ambiguity. Rework the unique-embedding diagnostic into an aggregate over DIAG_EMB_STEPS batches covering the cross-feature tables. Co-authored-by: Cursor --- .../dlrm_v3/train/utils.py | 178 ++++++++++++++++-- 1 file changed, 162 insertions(+), 16 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py index 076412d6e..6208214fb 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -688,34 +688,129 @@ def make_train_test_dataloaders( return train_dataloader, test_dataloader -def _log_unique_embedding_diag(sample, rank: int, step: int) -> None: - """Diagnostic: log per-batch unique-vs-total embedding-id counts. +# THROWAWAY DIAG state: per-embedding-table lookup stats accumulated across a +# fixed window of steps (DIAG_EMB_STEPS, default 100) so the reported numbers are +# averaged/aggregated rather than a single noisy batch. Rank-0 only. +_EMB_DIAG_ACC: Dict[str, Dict[str, Any]] = {} +_EMB_DIAG_NBATCH: int = 0 +# Cap (per-batch lookups) below which we also track a TRUE global-unique set +# across the whole window (cheap for contextual/cross tables, total == batch +# size). Sequential tables (item/artist/album) blow past this and only get the +# per-batch averages. +_EMB_DIAG_GLOBAL_CAP: int = 1 << 17 # 131072 + + +def _log_unique_embedding_diag( + sample, rank: int, step: int, max_steps: int = 100, log_every: int = 50 +) -> None: + """Diagnostic: aggregate per-embedding-table lookup stats over a step window. Quantifies the user-major batching concern — when consecutive sliding-window anchors come from the same few users, a batch reads very few UNIQUE embedding - rows (low unique/total), so embedding lookups are highly redundant. In-window - shuffle should raise this ratio. Rank-0 only, gated by the caller (only fires - on logged steps), and fully non-fatal so it can never break training. + rows (low unique/total), so embedding lookups are highly redundant. Covers the + base id tables AND the cross-feature tables (user_x_* / *_x_hour hashed + combos); the user_x_* tables should be the most redundant under shuffle OFF. + + Accumulates over ``max_steps`` batches and emits an aggregate summary (mean + per-batch unique%/hot%/top10%, plus a true global-unique over the whole + window for the small contextual/cross tables). Rank-0 only, non-fatal. """ if rank != 0: return + global _EMB_DIAG_NBATCH try: - parts = [] + from generative_recommenders.dlrm_v3.configs import YAMBDA_5B_CROSS_SPECS + + cross_caps = {name: n for (name, _k, n, _s) in YAMBDA_5B_CROSS_SPECS} + + def _table_of(key: str): + # cross tables match by exact name; resolve BEFORE substring fallbacks + # so e.g. 'user_x_artist' isn't misread as the artist_id table. + if key in cross_caps: + return key, cross_caps[key] + if key == "uid" or key.endswith("_uid") or key.endswith(".uid"): + return "uid", 0 + if "artist" in key: + return "artist_id", 0 + if "album" in key: + return "album_id", 0 + if "item" in key and key.endswith("id"): + return "item_id", 0 + return None, 0 + for tag, kjt in ( ("uih", sample.uih_features_kjt), ("cand", sample.candidates_features_kjt), ): for key in kjt.keys(): - if not key.endswith(("item_id", "artist_id", "album_id")): + table, cap = _table_of(key) + if table is None: continue vals = kjt[key].values() total = int(vals.numel()) if total == 0: continue - uniq = int(torch.unique(vals).numel()) - parts.append(f"{tag}.{key}={uniq}/{total} ({100.0 * uniq / total:.1f}%)") - if parts: - logger.info(f"emb-diag - Step {step}: unique/total " + " ".join(parts)) + u, counts = torch.unique(vals, return_counts=True) + uniq = int(u.numel()) + hot1 = int(counts.max().item()) + k = min(10, uniq) + topk = int(torch.topk(counts, k).values.sum().item()) + + slot = _EMB_DIAG_ACC.setdefault( + f"{tag}.{key}", + { + "table": table, + "cap": cap, + "n": 0, + "tot": 0, + "uniq": 0, + "upct": 0.0, + "upct_min": 100.0, + "upct_max": 0.0, + "hot1pct": 0.0, + "topkpct": 0.0, + "glob": None, # running global-unique id tensor (small tables) + }, + ) + upct = 100.0 * uniq / total + slot["n"] += 1 + slot["tot"] += total + slot["uniq"] += uniq + slot["upct"] += upct + slot["upct_min"] = min(slot["upct_min"], upct) + slot["upct_max"] = max(slot["upct_max"], upct) + slot["hot1pct"] += 100.0 * hot1 / total + slot["topkpct"] += 100.0 * topk / total + if total <= _EMB_DIAG_GLOBAL_CAP: + prev = slot["glob"] + merged = u if prev is None else torch.cat([prev, u]) + slot["glob"] = torch.unique(merged) + + _EMB_DIAG_NBATCH += 1 + n = _EMB_DIAG_NBATCH + if n % log_every == 0 or n >= max_steps: + lines = [f"emb-diag AGGREGATE over {n} batches (step<= {step}):"] + for name in sorted(_EMB_DIAG_ACC): + s = _EMB_DIAG_ACC[name] + c = max(1, s["n"]) + cap_s = f" cap={s['cap']/1e6:.0f}M" if s["cap"] else "" + glob_s = "" + if s["glob"] is not None: + g = int(s["glob"].numel()) + glob_s = ( + f" | global_uniq={g} over {s['tot']} seen " + f"({s['tot']/max(1,g):.1f}x reuse)" + ) + lines.append( + f" {name}[{s['table']}]{cap_s}: " + f"avg_tot={s['tot']/c:.0f} " + f"avg_uniq%={s['upct']/c:.1f} " + f"(min={s['upct_min']:.1f} max={s['upct_max']:.1f}) " + f"avg_hot1%={s['hot1pct']/c:.1f} " + f"avg_top10%={s['topkpct']/c:.1f}" + f"{glob_s}" + ) + logger.info("\n".join(lines)) except Exception as e: # diagnostic must never break training logger.warning(f"emb-diag failed: {e}") @@ -744,8 +839,16 @@ def train_loop( for epoch in range(num_epochs): dataloader.sampler.set_epoch(epoch) # pyre-ignore [16] for sample in dataloader: - if streaming_diag_unique_emb and batch_idx % metric_log_frequency == 0: - _log_unique_embedding_diag(sample, rank, batch_idx) + if streaming_diag_unique_emb and batch_idx < int( + os.environ.get("DIAG_EMB_STEPS", "100") + ): + _log_unique_embedding_diag( + sample, + rank, + batch_idx, + max_steps=int(os.environ.get("DIAG_EMB_STEPS", "100")), + log_every=metric_log_frequency, + ) optimizer.zero_grad() sample.to(device) ( @@ -1494,8 +1597,16 @@ def _run_train_window( break if _t_next is not None and first_wait is None: first_wait = time.perf_counter() - _t_next - if streaming_diag_unique_emb and train_batch_idx % metric_log_frequency == 0: - _log_unique_embedding_diag(sample, rank, train_batch_idx) + if streaming_diag_unique_emb and train_batch_idx < int( + os.environ.get("DIAG_EMB_STEPS", "100") + ): + _log_unique_embedding_diag( + sample, + rank, + train_batch_idx, + max_steps=int(os.environ.get("DIAG_EMB_STEPS", "100")), + log_every=metric_log_frequency, + ) optimizer.zero_grad() sample.to(device) ( @@ -1653,7 +1764,8 @@ def _run_eval_window(eval_data_iterator, label: Optional[str] = None) -> None: metric_logger.compute_and_log(mode="eval") if num_eval_batches is not None and eval_batch_idx >= num_eval_batches: break - for k, v in metric_logger.compute(mode="eval").items(): + _eval_metrics = metric_logger.compute(mode="eval") + for k, v in _eval_metrics.items(): print(f"{k}: {v}") if label and rank == 0 and _t_enter is not None: _eval_total = time.perf_counter() - _t_enter @@ -1662,6 +1774,40 @@ def _run_eval_window(eval_data_iterator, label: Optional[str] = None) -> None: f"[boundary] {label} eval first-batch data-wait={_fw:.1f}ms " f"total_eval={_eval_total * 1000:.1f}ms batches={eval_batch_idx}" ) + # Dedicated per-eval metrics sink. One JSON line per eval boundary + # capturing the END-OF-PASS metric over the FIXED holdout -- the single + # correct value for that eval point (no interim/averaging ambiguity). + # Rank 0 only; append-only so it survives restarts and the trajectory + # accumulates across resumes. Written next to the main run log + # (".metrics.jsonl"), falling back to cwd if LOG is unset. + import json + import re as _re + + _log = os.environ.get("LOG") + if _log: + _base = _log[:-4] if _log.endswith(".log") else _log + _metrics_path = f"{_base}.metrics.jsonl" + else: + _metrics_path = "streaming_eval_metrics.jsonl" + _ts_m = _re.search(r"train_ts=(\d+)", label) + _rec = { + "label": label, + "train_ts": int(_ts_m.group(1)) if _ts_m else None, + "global_step": int(metric_logger.global_step.get("train", -1)), + "eval_batches": eval_batch_idx, + "total_eval_ms": round(_eval_total * 1000, 1), + "wall_time": time.time(), + } + for _k, _v in _eval_metrics.items(): + try: + _rec[_k] = float(_v) + except (TypeError, ValueError): + pass + try: + with open(_metrics_path, "a") as _f: + _f.write(json.dumps(_rec) + "\n") + except OSError as _e: + logger.warning("failed to write metrics sink %s: %s", _metrics_path, _e) def _maybe_checkpoint(train_ts: int) -> None: if ( From 51e6e5f1dfd6aa182a4762ab71fbe629834af027 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Fri, 12 Jun 2026 20:53:11 -0500 Subject: [PATCH 054/127] dlrmv4: default MIN_HISTORY=0 (include cold-start first events) to match min0 runs All live min0 runs already export MIN_HISTORY=0; make the gin default match so future runs without an override include each user's cold-start first event (zero prior context) instead of requiring >=1 prior event. Co-authored-by: Cursor --- .../dlrm_v3/train/gin/yambda_5b.gin | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index 3bfccc510..e6922f594 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -110,12 +110,12 @@ hl/env_int.default = 4086 # is only the gather/truncation cap) — jagged attention handles short UIH, so we # no longer need a full history_length of context to include a sample. The legacy # behavior (require a full history_length, which dropped ~60% of users) is -# MIN_HISTORY=4086; the default below (1) includes ~all users with any history. -# Positions/anchor-ts caches are keyed by (L, MIN_HISTORY) so floors don't -# collide. Override via $MIN_HISTORY. +# MIN_HISTORY=4086; the default below (0) includes ~all users AND their +# cold-start first event (zero prior context). Positions/anchor-ts caches are +# keyed by (L, MIN_HISTORY) so floors don't collide. Override via $MIN_HISTORY. get_dataset.min_history = @mh/env_int() mh/env_int.key = "MIN_HISTORY" -mh/env_int.default = 1 +mh/env_int.default = 0 # Model-side attention budget. Dataset truncates UIH to fit this value if # `history_length + contextual + candidate` would overflow. Override via From 03a5be13268c5102354fa98183215a93210456cf Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Fri, 12 Jun 2026 23:46:39 -0500 Subject: [PATCH 055/127] dlrmv4: default NUM_TRAIN_TS=149 to sweep full ts=150..298 streaming range Matches the long e2e runs (start_ts=150 + 149 daily windows). EVAL_EACH_WINDOW and EVAL_EVERY_N_WINDOWS already default to 1. Clamped to the dataset's available window count at runtime; override via $NUM_TRAIN_TS. Co-authored-by: Cursor --- .../generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index e6922f594..ad268aec7 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -173,7 +173,10 @@ make_persistent_streaming_dataloader.num_workers = %num_workers make_persistent_streaming_dataloader.prefetch_factor = %prefetch_factor streaming_train_eval_loop.num_train_ts = @nts/env_int() nts/env_int.key = "NUM_TRAIN_TS" -nts/env_int.default = 30 +# 149 daily windows -> with start_ts=150 the run sweeps ts=150..298, the full +# dense range of the corpus (matches the long e2e runs). Clamped to the +# dataset's available window count at runtime. Override via $NUM_TRAIN_TS. +nts/env_int.default = 149 # Anchors need >= history_length prior events, so the first ~130 daily windows # are near-empty warm-up; start at a dense window. Override via $START_TS. streaming_train_eval_loop.start_ts = @sts/env_int() From 20f691776fff68b04ca1cb0ce0cc21039c88182c Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Fri, 12 Jun 2026 23:57:09 -0500 Subject: [PATCH 056/127] dlrmv4: consolidate eval cadence into single EVAL_EVERY_N_WINDOWS knob Drop the redundant EVAL_EACH_WINDOW on/off boolean and fold "disable eval" into the cadence int: 0 = eval off (train-only / resume test), 1 = every window, N>1 = every Nth window (anchored to the absolute ts grid, stable across resume). Updates streaming_train_eval_loop, the gin bindings, and the resume test harness accordingly. No behavior change for the live runs (EVAL_EVERY_N_WINDOWS=1). Co-authored-by: Cursor --- .../dlrm_v3/train/gin/yambda_5b.gin | 14 +++++----- .../train/tests/streaming_resume_test.sh | 8 +++--- .../dlrm_v3/train/utils.py | 26 ++++++++++--------- 3 files changed, 25 insertions(+), 23 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index ad268aec7..0b2c884bd 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -206,13 +206,13 @@ ot/env_int.default = 0 streaming_train_eval_loop.persistent_loader = @pl/env_int() pl/env_int.key = "PERSISTENT_LOADER" pl/env_int.default = 1 -streaming_train_eval_loop.eval_each_window = @ev/env_int() -ev/env_int.key = "EVAL_EACH_WINDOW" -ev/env_int.default = 1 -# Full-holdout eval cadence: run eval every Nth train window (and always on the -# final window) instead of every window. 1 (default) = eval every window (no -# behavior change). Set >1 (e.g. 5 via $EVAL_EVERY_N_WINDOWS) to amortize the -# cost of consuming the full next-day eval window over several train windows. +# Full-holdout eval cadence (single knob; replaces the old EVAL_EACH_WINDOW +# on/off switch). 0 = eval disabled (train-only, e.g. perf benchmarking or the +# resume test; the eval dataloader isn't even built). 1 (default) = eval after +# every window. N>1 (e.g. 5 via $EVAL_EVERY_N_WINDOWS) = eval every Nth window +# (and always the final one) to amortize the cost of consuming the full next-day +# eval window. The cadence is anchored to the absolute ts grid so eval points +# stay stable across a mid-run resume. streaming_train_eval_loop.eval_every_n_windows = @evn/env_int() evn/env_int.key = "EVAL_EVERY_N_WINDOWS" evn/env_int.default = 1 diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh b/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh index c3690e652..e14e557e8 100755 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh @@ -13,7 +13,7 @@ # # Driven entirely via env-driven gin knobs defined in yambda_5b.gin: # NUM_TRAIN_TS / NUM_TRAIN_BATCHES / IN_WINDOW_CKPT_FREQ / DIE_AT_STEP / -# CKPT_PATH / KEEP_LAST_N / EVAL_EACH_WINDOW +# CKPT_PATH / KEEP_LAST_N / EVAL_EVERY_N_WINDOWS # # Usage: # bash generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh --jobid @@ -127,7 +127,7 @@ run_phase() { clean_ckpt run_phase baseline \ "NUM_TRAIN_TS=1" \ - "EVAL_EACH_WINDOW=0" \ + "EVAL_EVERY_N_WINDOWS=0" \ "METRIC_LOG_FREQ=1" \ "NUM_TRAIN_BATCHES=$NUM_TRAIN_BATCHES" \ "DIE_AT_STEP=-1" @@ -140,7 +140,7 @@ cleanup_workers clean_ckpt run_phase interrupt \ "NUM_TRAIN_TS=1" \ - "EVAL_EACH_WINDOW=0" \ + "EVAL_EVERY_N_WINDOWS=0" \ "METRIC_LOG_FREQ=1" \ "NUM_TRAIN_BATCHES=$NUM_TRAIN_BATCHES" \ "IN_WINDOW_CKPT_FREQ=$IN_WINDOW_FREQ" \ @@ -158,7 +158,7 @@ echo "Saved checkpoints after interrupt: $SAVED" # === Phase 3: resume === run_phase resume \ "NUM_TRAIN_TS=1" \ - "EVAL_EACH_WINDOW=0" \ + "EVAL_EVERY_N_WINDOWS=0" \ "METRIC_LOG_FREQ=1" \ "NUM_TRAIN_BATCHES=$NUM_TRAIN_BATCHES" \ "IN_WINDOW_CKPT_FREQ=$IN_WINDOW_FREQ" \ diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py index 6208214fb..9ef251951 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -1316,7 +1316,6 @@ def streaming_train_eval_loop( checkpoint_frequency: int = 100, start_ts: int = 0, persistent_loader: bool = False, - eval_each_window: bool = True, eval_every_n_windows: int = 1, double_buffer: bool = False, # --- fixed user-holdout eval set --- @@ -1838,18 +1837,21 @@ def _maybe_checkpoint(train_ts: int) -> None: def _should_eval(i: int) -> bool: """Whether to run the full-holdout eval after training window index `i`. - `eval_every_n_windows<=1` (default) preserves the per-window cadence. - For K>1 we eval when the ABSOLUTE window ts is on the grid anchored at - `eval_anchor_ts` (the original start_ts), i.e. ts in {anchor, anchor+K, - anchor+2K, ...}, and ALWAYS on the final window so the trajectory ends - with an eval point. Anchoring to the absolute ts (not the per-call loop - index `i`) keeps the eval grid (e.g. 150,160,170,...) stable across a - mid-run resume, which rebases start_ts/`train_ts_list` to the resume - window. Gated by `eval_each_window`. + Single cadence knob `eval_every_n_windows`: + * <=0 -> eval disabled entirely (train-only; e.g. perf benchmarking or + the resume test). The eval dataloader is not even built. + * 1 (default) -> eval after every window. + * K>1 -> eval when the ABSOLUTE window ts is on the grid anchored at + `eval_anchor_ts` (the original start_ts), i.e. ts in {anchor, + anchor+K, anchor+2K, ...}, and ALWAYS on the final window so the + trajectory ends with an eval point. Anchoring to the absolute ts + (not the per-call loop index `i`) keeps the eval grid (e.g. + 150,160,170,...) stable across a mid-run resume, which rebases + start_ts/`train_ts_list` to the resume window. """ - if not eval_each_window: + if eval_every_n_windows <= 0: return False - if eval_every_n_windows <= 1: + if eval_every_n_windows == 1: return True return (train_ts_list[i] - eval_anchor_ts) % eval_every_n_windows == 0 or i == n_train - 1 @@ -1892,7 +1894,7 @@ def _should_eval(i: int) -> bool: # sample content depends only on the sampler window, not is_eval, so # prefetching during train is safe. eval_iter: Optional[Iterator] = None - if eval_each_window and len(train_ts_list) > 0: + if eval_every_n_windows > 0 and len(train_ts_list) > 0: eval_sampler = StreamingWindowSampler(rank, world_size) eval_dl = make_persistent_streaming_dataloader( dataset=dataset, sampler=eval_sampler From 9171e4f4871b59a94374fa171e1be3e5433b9fb0 Mon Sep 17 00:00:00 2001 From: suachong Date: Mon, 15 Jun 2026 06:45:10 +0000 Subject: [PATCH 057/127] dlrmv4: random per-run seed + graceful teardown + no double-logging - setup(): init the process group first, then draw a fresh random seed per run (rank 0 broadcasts so all ranks agree), export it to $SEED, and log it; pin $SEED to reproduce a run exactly. Data order/split are unaffected (still time-deterministic + $SPLIT_SALT-governed); the seed governs dense weight init. MLPerf SEED event now logs the actual chosen value, not hardcoded 1. - train_ranker(): move distributed teardown into a finally block so a clean finish also barriers + destroy_process_group()s in lockstep, silencing the noisy TCPStore "broken pipe" / "should dump" warnings at exit. Best-effort so teardown never masks a real error. - launch_slurm.sh: orchestrate sets WORKER_TEE=0 so the worker points its own file sink at /dev/null (stdout is already tee'd upstream), avoiding every log line being written twice. Direct single-node (e2e supervisor) keeps WORKER_TEE unset and still writes $LOG itself. Co-authored-by: Cursor --- .../dlrm_v3/train/train_ranker.py | 26 +++++++++-- .../dlrm_v3/train/utils.py | 46 ++++++++++++++----- recommendation_v4/scripts/launch_slurm.sh | 9 ++++ 3 files changed, 66 insertions(+), 15 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py index 0168add5b..d22d09a30 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py @@ -220,8 +220,9 @@ def _gin_param(name: str, default: object) -> object: global_batch_size = world_size * int(train_dataloader.batch_size) mlperf_logger.event(key=c.GLOBAL_BATCH_SIZE, value=global_batch_size) mlperf_logger.event(key=c.GRADIENT_ACCUMULATION_STEPS, value=1) - # Seed is fixed in setup() (_SEED = 1); kept in sync here. - mlperf_logger.event(key=c.SEED, value=1) + # Log the ACTUAL seed chosen in setup() (random per-run unless $SEED is + # pinned; setup() exports the chosen value to $SEED). + mlperf_logger.event(key=c.SEED, value=int(os.environ.get("SEED", "1"))) # Dense (Adam) + sparse (RowWiseAdagrad) optimizer hyperparameters, # read from the active gin bindings. mlperf_logger.event( @@ -309,8 +310,27 @@ def _gin_param(name: str, default: object) -> object: ) except Exception as e: logger.info(traceback.format_exc()) - cleanup() raise Exception(e) + finally: + # Graceful distributed teardown (runs on BOTH success and failure). + # Previously cleanup() ran only in the except branch, so a clean finish + # returned without destroying the process group: ranks that returned + # first let rank 0's TCPStore close while peers' ProcessGroupNCCL + # heartbeat-monitor threads were still polling it, emitting the noisy + # (but harmless) "Failed to check the 'should dump' flag on TCPStore / + # Broken pipe" warnings + C++ stack traces at exit. Barrier first so all + # ranks reach the end in lockstep, then destroy_process_group() stops + # each rank's monitor thread and closes NCCL/the store in order. Both + # steps are guarded/best-effort so teardown never masks a real error. + if torch.distributed.is_initialized(): + try: + torch.distributed.barrier() + except Exception: + logger.info("teardown barrier failed (non-fatal)") + try: + cleanup() + except Exception: + logger.info("teardown destroy_process_group failed (non-fatal)") def get_args(): # pyre-ignore [3] diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py index 96f826f84..79a8571b0 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -96,27 +96,49 @@ def setup( # leaving stale allocations and triggering OOMs on rank 0. torch.cuda.set_device(device) - # Seed all RNGs so weight init (make_model, called after setup) is - # reproducible across runs. Same seed on every rank → dense params are - # initialized identically across ranks; sharded embeddings are init'd from - # the meta device by DMP. Fixed seed makes pipeline-vs-non-pipeline an - # init-matched A/B (data order is already deterministic via the sampler). import random import numpy as np - _SEED = 1 - random.seed(_SEED) - np.random.seed(_SEED) - torch.manual_seed(_SEED) - torch.cuda.manual_seed_all(_SEED) - - # initialize the process group + # initialize the process group FIRST so ranks can agree on a shared seed + # (the seed MUST be identical on every rank, else dense weight init diverges + # across ranks and DDP/AllReduce trains garbage). if not dist.is_initialized(): dist.init_process_group( "nccl", rank=rank, world_size=world_size, device_id=device ) + # Seed selection. By default we draw a FRESH RANDOM seed every run so each + # launch explores a different dense weight init; pin $SEED to reproduce a + # specific run exactly. rank 0 draws the seed and broadcasts it so all ranks + # share one value; the chosen seed is exported to $SEED and logged so any run + # can be reproduced after the fact. NOTE (streaming-train-eval): the data + # ORDER and train/holdout split do NOT depend on this seed — order is + # time-deterministic (StreamingWindowSampler) and the split is governed by + # $SPLIT_SALT. The seed governs dense weight init + global-RNG stochastic ops. + env_seed = os.environ.get("SEED", "").strip() + if env_seed: + seed = int(env_seed) + else: + seed = int.from_bytes(os.urandom(4), "little") if rank == 0 else 0 + _seed_t = torch.tensor([seed], dtype=torch.int64, device=device) + dist.broadcast(_seed_t, src=0) + seed = int(_seed_t.item()) + os.environ["SEED"] = str(seed) + + # Seed all RNGs with the agreed value so weight init (make_model, called + # after setup) is identical across ranks; sharded embeddings are init'd from + # the meta device by DMP. + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + if rank == 0: + logger.info( + f"[seed] using seed={seed} " + f"({'pinned via $SEED' if env_seed else 'random per-run; set $SEED to reproduce'})" + ) + pg = dist.new_group( backend=BACKEND, timeout=timedelta(seconds=TIMEOUT), diff --git a/recommendation_v4/scripts/launch_slurm.sh b/recommendation_v4/scripts/launch_slurm.sh index c100a6828..bcb14442b 100755 --- a/recommendation_v4/scripts/launch_slurm.sh +++ b/recommendation_v4/scripts/launch_slurm.sh @@ -214,6 +214,7 @@ orchestrate() { srun --ntasks-per-node=1 bash -c " docker exec \ -e LAUNCH_SLURM_PHASE=worker \ + -e WORKER_TEE=0 \ -e SCRATCH=$SCRATCH \ -e SLURM_NNODES=\$SLURM_NNODES \ -e SLURM_NODEID=\$SLURM_NODEID \ @@ -381,6 +382,14 @@ worker() { cd "$REPO_ROOT" mkdir -p "$SCRATCH" 2>/dev/null || true LOG=${LOG:-$SCRATCH/yambda_5b_8gpu.log} + # Avoid double-logging. When launched by the orchestrate phase, our stdout is + # ALREADY captured into the real $LOG by orchestrate's `tee` (and, multi-node, + # funneled through one srun pipe). Re-`tee`ing $LOG here would write every line + # twice. Orchestrate sets WORKER_TEE=0 to point our own file sink at /dev/null: + # we still echo to stdout (captured upstream) but don't duplicate the file. + # Direct single-node invocation (the streaming-e2e supervisor) leaves + # WORKER_TEE unset, so the worker keeps writing $LOG itself. + [ "${WORKER_TEE:-1}" = "0" ] && LOG=/dev/null # TensorBoard under the writable scratch root unless the caller (e.g. the e2e # supervisor) pinned a per-run path. Keeps the gin default from ever being used. export TENSORBOARD_LOG_PATH=${TENSORBOARD_LOG_PATH:-$SCRATCH/tb/yambda_5b} From ded9c30e081ee5e42d19aed177f00f0202e80924 Mon Sep 17 00:00:00 2001 From: suachong Date: Mon, 15 Jun 2026 02:52:25 -0500 Subject: [PATCH 058/127] dlrmv4: fix single-node-without-slurm via run_docker.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit run_docker.sh: strip an optional leading `--` so the documented `run_docker.sh -- bash scripts/launch_slurm.sh` form works (it was forwarded verbatim to `docker run` and failed exec: "--" not found); wire run_docker.sh to the launch_slurm worker flow (CONTAINER_NAME, LOG/MODE/MAX_SEQ_LEN/HISTORY_LENGTH passthrough) and forward NCCL_SOCKET_IFNAME so the bootstrap NIC is host-overridable. launch_slurm.sh: default NCCL_SOCKET_IFNAME=lo for single-node (NNODES==1) instead of the meta64-only fenic0 — loopback is reachable by all local ranks on any host (data plane is intra-node XGMI/PCIe), so the single-node path now runs out-of-the-box on dev boxes with no fenic0. Multi-node keeps the fenic0 default; both stay overridable. Verified: 8-GPU single-node streaming-train-eval smoke runs clean (RCCL init over lo, train+eval, MLPerf run_stop) on a lone MI355 node. Co-authored-by: Cursor --- recommendation_v4/scripts/launch_slurm.sh | 28 ++++++++++++++++------- recommendation_v4/scripts/run_docker.sh | 24 ++++++++++++++++--- 2 files changed, 41 insertions(+), 11 deletions(-) diff --git a/recommendation_v4/scripts/launch_slurm.sh b/recommendation_v4/scripts/launch_slurm.sh index b035ac671..3ab70078a 100755 --- a/recommendation_v4/scripts/launch_slurm.sh +++ b/recommendation_v4/scripts/launch_slurm.sh @@ -549,14 +549,26 @@ worker() { export WORLD_SIZE=$(( NNODES * GPUS_PER_NODE )) echo "[$(date)] topology: nnodes=$NNODES node_rank=$NODE_RANK gpus_per_node=$GPUS_PER_NODE world_size=$WORLD_SIZE master=$MASTER_ADDR:${MASTER_PORT:-}" | tee -a "$LOG" - # NCCL bootstrap NIC — pin for BOTH single- and multi-node. The container is - # --network=host so RCCL sees ALL host interfaces; if left to auto-detect, NCCL - # can pick a non-routable per-GPU RoCE /31 (benic* 192.168.x) link and fail - # bootstrap with "No route to host" (this is node-dependent: it happened to - # work on some nodes and not others, causing repetitive single-node init - # failures). Pinning the routable host NIC fixes it everywhere. - # [CLUSTER-SPECIFIC] routable host NIC for TCP bootstrap (find via `ip -br addr`). - export NCCL_SOCKET_IFNAME=${NCCL_SOCKET_IFNAME:-fenic0} + # NCCL bootstrap NIC. The container is --network=host so RCCL sees ALL host + # interfaces; if left to auto-detect, NCCL can pick a non-routable per-GPU RoCE + # /31 (benic* 192.168.x) link and fail bootstrap with "No route to host" (this + # is node-dependent: it worked on some nodes and not others, causing repetitive + # single-node init failures). Pin it explicitly to avoid that. + # * Single-node (NNODES==1): all ranks are on THIS host, so only the bootstrap + # control-plane crosses the socket NIC (data plane is intra-node XGMI/PCIe, + # see below). Loopback is reachable by every local rank on ANY host and is + # node-independent — same rationale as MASTER_ADDR=localhost above — so it + # "just works" on dev boxes that have no fenic0 (e.g. a single MI355 node). + # * Multi-node (NNODES>1): needs a routable host NIC shared across nodes for + # the cross-node TCP rendezvous; default to the meta64 fenic0. + # Both remain ${NCCL_SOCKET_IFNAME:-...}-overridable for other fabrics. + # [CLUSTER-SPECIFIC] multi-node routable host NIC for TCP bootstrap (find via `ip -br addr`). + if [ "$NNODES" -gt 1 ]; then + export NCCL_SOCKET_IFNAME=${NCCL_SOCKET_IFNAME:-fenic0} + else + export NCCL_SOCKET_IFNAME=${NCCL_SOCKET_IFNAME:-lo} + fi + echo "[$(date)] NCCL_SOCKET_IFNAME=$NCCL_SOCKET_IFNAME (nnodes=$NNODES)" | tee -a "$LOG" # Multi-node additionally needs the RDMA data-plane (bnxt_re HCAs) configured; # single-node uses intra-node P2P (XGMI/PCIe) so only the bootstrap NIC matters. diff --git a/recommendation_v4/scripts/run_docker.sh b/recommendation_v4/scripts/run_docker.sh index bb1d58fca..864746550 100755 --- a/recommendation_v4/scripts/run_docker.sh +++ b/recommendation_v4/scripts/run_docker.sh @@ -3,14 +3,21 @@ # and data directories bind-mounted at matching host/container paths. # # Usage: -# bash scripts/run_docker.sh # interactive shell -# bash scripts/run_docker.sh -- bash scripts/launch_smoke_8gpu.sh # one-shot +# bash scripts/run_docker.sh # interactive shell +# bash scripts/run_docker.sh -- bash scripts/launch_slurm.sh # one-shot single-node train +# +# Inside the container /.dockerenv exists, so launch_slurm.sh auto-selects its +# SLURM-free `worker` phase (NNODES=1) — identical to the old launch_smoke_8gpu.sh. # # Overrides (export before invoking): # IMAGE docker image (default: rocm/mlperf:dlrm_v3_mi355) -# CONTAINER_NAME container name (default: yambda_8gpu) +# CONTAINER_NAME container name (default: mlperf-recommendation-v4) # REPO_HOST host path to repo (default: this script's parent) # DATA_HOST host path to dataset root (default: /data/mlperf_dlrm_v4) +# LOG in-container train log path (default: /workspace/recommendation_v4/mlperf_dlrm_v4.log) +# MODE launch_slurm.sh mode (default: launcher default = streaming-train-eval; set train-eval for classic) +# MAX_SEQ_LEN / HISTORY_LENGTH seq shape; set 2048 / 2039 for the previous 2k shape +# NCCL_SOCKET_IFNAME NCCL bootstrap NIC (default: launch_slurm picks lo single-node / fenic0 multi-node; override per host) set -euo pipefail @@ -29,6 +36,12 @@ if [ ! -d "${DATA_HOST}" ]; then echo "warning: ${DATA_HOST} does not exist on host. Run preprocess_public_data first or override DATA_HOST." >&2 fi +# Drop an optional `--` separating this script's invocation from the in-container +# command (the documented `run_docker.sh -- bash scripts/launch_slurm.sh` form). +# Without this, `--` is forwarded verbatim to `docker run` as the command and +# fails with: exec: "--": executable file not found. +if [ "${1:-}" = "--" ]; then shift; fi + # If a container with this name is already running, exec into it instead of # starting a new one. Matches the `docker exec yambda_8gpu ...` pattern in # README.MD:9-12. @@ -54,6 +67,11 @@ exec docker run --rm -it \ -e DLRM_DATA_PATH="${DATA_CONT}" \ -e HSTU_HAMMER_KERNEL="${HSTU_HAMMER_KERNEL:-TRITON}" \ -e RUN_NAME="${RUN_NAME:-default}" \ + -e LOG="${LOG:-/workspace/recommendation_v4/mlperf_dlrm_v4.log}" \ + ${MODE:+-e MODE="${MODE}"} \ + ${MAX_SEQ_LEN:+-e MAX_SEQ_LEN="${MAX_SEQ_LEN}"} \ + ${HISTORY_LENGTH:+-e HISTORY_LENGTH="${HISTORY_LENGTH}"} \ + ${NCCL_SOCKET_IFNAME:+-e NCCL_SOCKET_IFNAME="${NCCL_SOCKET_IFNAME}"} \ -w "${REPO_CONT}" \ "${IMAGE}" \ "${@:-bash}" From 380b1cabd6ce0e4d0067b956a67dd18746380f76 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Mon, 15 Jun 2026 23:11:25 -0500 Subject: [PATCH 059/127] dlrmv4: gin-configurable quantized (bf16/fp16) embedding all-to-all Quantize the bandwidth-bound embedding-shuffle all-to-all via TorchRec QCommsConfig on the sequence EmbeddingCollectionSharder. Exposed as two gin knobs on make_optimizer_and_shard (env-overridable): sparse_a2a_precision = fp32 (off, default) | bf16 | fp16 sparse_a2a_quantize_backward = 1 (default) | 0 (forward-only) Default fp32 keeps the path byte-for-byte identical to trunk. launch_slurm forwards the env overrides and reuses a stopped container instead of destructively re-provisioning. Co-authored-by: Cursor --- .../dlrm_v3/train/gin/yambda_5b.gin | 15 ++++ .../dlrm_v3/train/utils.py | 82 +++++++++++++++++++ recommendation_v4/scripts/launch_slurm.sh | 19 +++++ 3 files changed, 116 insertions(+) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index 0b2c884bd..2bbf0dabf 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -83,6 +83,21 @@ make_optimizer_and_shard.hbm_cap_gb = @env_int() env_int.key = "HBM_CAP_GB" env_int.default = 260 +# Sparse embedding all-to-all wire precision. The embedding shuffle is the +# dominant, bandwidth-bound (esp. multi-node) collective; quantizing it via +# TorchRec QCommsConfig halves (bf16/fp16) the wire volume. "fp32" = off +# (default; numerically untouched). Set "bf16" (or "fp16") to enable. +# Override via $DLRMV4_SPARSE_A2A_PRECISION. +make_optimizer_and_shard.sparse_a2a_precision = @saap/env_str() +saap/env_str.key = "DLRMV4_SPARSE_A2A_PRECISION" +saap/env_str.default = "fp32" +# Also quantize the backward (gradient) a2a. 1 = yes (default), 0 = forward-only +# (keeps gradients fp32 for a more conservative numerical profile). +# Override via $DLRMV4_BF16_SPARSE_BWD. +make_optimizer_and_shard.sparse_a2a_quantize_backward = @saab/env_int() +saab/env_int.key = "DLRMV4_BF16_SPARSE_BWD" +saab/env_int.default = 1 + get_dataset.name = %dataset get_dataset.new_path_prefix = %DATA_PATH # Total user-interaction-history (UIH) budget per sample, distributed evenly diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py index 9ef251951..fcb562b7d 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -340,6 +340,80 @@ def sparse_optimizer_factory_and_class( return optimizer_cls, kwargs, optimizer_factory +def _maybe_apply_qcomm_a2a( + sharders: List[Any], + device: torch.device, + precision: str = "fp32", + quantize_backward: bool = True, +) -> List[Any]: + """Optionally quantize the embedding all-to-all payload via TorchRec qcomm. + + The yambda-5b embedding shuffle is the dominant, bandwidth-bound (multi-node) + collective (~14.5 GB/rank fp32); BF16/FP16 forward(+backward) halves the wire + volume. Quant/dequant happen inside the comm op, transparent to the lookup + consumer. Ported from the DLRMv2 R2 lever, retargeted from + ``EmbeddingBagCollectionSharder`` to the sequence ``EmbeddingCollectionSharder`` + this model uses. + + Args (set via gin on ``make_optimizer_and_shard``, env-overridable): + precision: ``fp32`` (off, default) | ``bf16`` | ``fp16`` — the a2a wire dtype. + quantize_backward: also quantize the gradient a2a (default True). + """ + precision = (precision or "fp32").strip().lower() + rank0 = (not dist.is_initialized()) or dist.get_rank() == 0 + if precision in ("fp32", "", "off", "none"): + return sharders + if precision not in ("bf16", "fp16"): + if rank0: + logger.warning( + "DLRMV4 qcomm a2a: unknown precision %r (want fp32|bf16|fp16); " + "using fp32 a2a", + precision, + ) + return sharders + try: + from torchrec.distributed.embedding import EmbeddingCollectionSharder + from torchrec.distributed.fbgemm_qcomm_codec import ( + CommType, + get_qcomm_codecs_registry, + QCommsConfig, + ) + + fwd_prec = {"bf16": CommType.BF16, "fp16": CommType.FP16}[precision] + qcfg = QCommsConfig( + forward_precision=fwd_prec, + backward_precision=fwd_prec if quantize_backward else CommType.FP32, + ) + registry = get_qcomm_codecs_registry(qcfg, device=device) + new_sharders = [] + replaced = False + for s in sharders: + if type(s).__name__ == "EmbeddingCollectionSharder" and not replaced: + new_sharders.append( + EmbeddingCollectionSharder(qcomm_codecs_registry=registry) + ) + replaced = True + else: + new_sharders.append(s) + if rank0: + logger.info( + "DLRMV4 qcomm a2a ENABLED: forward=%s backward=%s " + "replaced_ec_sharder=%s", + fwd_prec.value, + fwd_prec.value if quantize_backward else "fp32", + replaced, + ) + return new_sharders + except Exception as e: # noqa: BLE001 — fall back to fp32 a2a on any failure + if rank0: + logger.warning( + "DLRMV4 qcomm a2a: failed to enable (%s: %s); using fp32 a2a", + type(e).__name__, + e, + ) + return sharders + + @gin.configurable def make_optimizer_and_shard( model: torch.nn.Module, @@ -347,6 +421,8 @@ def make_optimizer_and_shard( world_size: int, local_world_size: Optional[int] = None, hbm_cap_gb: int = 260, + sparse_a2a_precision: str = "fp32", + sparse_a2a_quantize_backward: bool = True, ) -> Tuple[DistributedModelParallel, torch.optim.Optimizer]: dense_opt_cls, dense_opt_args, dense_opt_factory = ( dense_optimizer_factory_and_class() @@ -364,6 +440,12 @@ def make_optimizer_and_shard( sparse_opt_cls, [param], sparse_opt_args ) sharders = get_default_sharders() + sharders = _maybe_apply_qcomm_a2a( + sharders, + device, + precision=sparse_a2a_precision, + quantize_backward=bool(sparse_a2a_quantize_backward), + ) # local_world_size = GPUs per node so the planner respects the intra-node # (xGMI/NVLink) vs inter-node hierarchy when placing shards. Defaults to # world_size for the single-node case (no behavior change). diff --git a/recommendation_v4/scripts/launch_slurm.sh b/recommendation_v4/scripts/launch_slurm.sh index 6b5ba7607..22beb886b 100755 --- a/recommendation_v4/scripts/launch_slurm.sh +++ b/recommendation_v4/scripts/launch_slurm.sh @@ -206,6 +206,11 @@ orchestrate() { echo \"[\$(hostname)] keeping non-GPU/system container \$_nm (\$_c)\" ;; esac done + # Reuse a STOPPED '$CONTAINER' (its installed deps persist in the container + # fs) instead of destructively re-provisioning from the base image + pip. + # Harmless no-op on a fresh node (no such container) -> falls through to + # provision below. Repo code is bind-mounted, so live edits are still picked up. + docker start $CONTAINER >/dev/null 2>&1 || true if [ \"$FORCE_PROVISION\" = \"1\" ] || ! docker exec $CONTAINER true >/dev/null 2>&1; then echo \"[\$(hostname)] (re)provisioning container\" LAUNCH_SLURM_PHASE=provision CONTAINER=$CONTAINER IMAGE=$IMAGE \ @@ -293,9 +298,13 @@ orchestrate() { -e SPLIT_SALT=${SPLIT_SALT:-0} \ -e EVAL_HOLDOUT_TS=${EVAL_HOLDOUT_TS:--1} \ -e EVAL_HOLDOUT_NUM_WINDOWS=${EVAL_HOLDOUT_NUM_WINDOWS:-1} \ + ${WORKER_CMD:+-e WORKER_CMD=\"$WORKER_CMD\"} \ ${RUN_NAME:+-e RUN_NAME=$RUN_NAME} \ ${TENSORBOARD_LOG_PATH:+-e TENSORBOARD_LOG_PATH=$TENSORBOARD_LOG_PATH} \ ${CKPT_PATH:+-e CKPT_PATH=$CKPT_PATH} \ + ${DLRMV4_BF16_SPARSE_A2A:+-e DLRMV4_BF16_SPARSE_A2A=$DLRMV4_BF16_SPARSE_A2A} \ + ${DLRMV4_SPARSE_A2A_PRECISION:+-e DLRMV4_SPARSE_A2A_PRECISION=$DLRMV4_SPARSE_A2A_PRECISION} \ + ${DLRMV4_BF16_SPARSE_BWD:+-e DLRMV4_BF16_SPARSE_BWD=$DLRMV4_BF16_SPARSE_BWD} \ -e LOG=$LOG \ $NCCL_ENV_ARGS \ $CONTAINER bash -lc 'cd $REPO && LAUNCH_SLURM_PHASE=worker bash scripts/launch_slurm.sh' @@ -585,6 +594,16 @@ worker() { fi fi + # WORKER_CMD override: run an arbitrary in-container command (e.g. an a2a/RCCL + # micro-benchmark) instead of the trainer, REUSING all the NCCL/RDMA/topology + # setup above so it exercises the exact transport the trainer uses. The + # supervisor never sets WORKER_CMD, so the training path is unchanged. + if [ -n "${WORKER_CMD:-}" ]; then + echo "[$(date)] WORKER_CMD override (WORLD_SIZE=$WORLD_SIZE): $WORKER_CMD" | tee -a "$LOG" + bash -lc "cd $REPO_ROOT && $WORKER_CMD" 2>&1 | tee -a "$LOG" + return + fi + echo "[$(date)] launching train_ranker with WORLD_SIZE=$WORLD_SIZE" | tee -a "$LOG" python -m generative_recommenders.dlrm_v3.train.train_ranker \ --dataset yambda-5b --mode "${MODE:-streaming-train-eval}" 2>&1 | tee -a "$LOG" From 3e776a8e0776f77cc1d2312a227db8183050ade5 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Mon, 15 Jun 2026 23:35:02 -0500 Subject: [PATCH 060/127] dlrmv4: split quantized a2a into independent fwd/bwd precision knobs Replace the single sparse-a2a precision knob with separate forward and backward precision settings ($SPARSE_A2A_FWD / $SPARSE_A2A_BWD, each fp32|bf16|fp16; both fp32 = off, identical to baseline). This enables the TorchRec golden_training recommended mix (fwd=fp16, bwd=bf16): fp16's mantissa suits bounded forward activations while bf16's wider exponent range avoids gradient overflow. 2-node A/B shows fp16/bf16 at perf parity with bf16/bf16 (both 2-byte wire), so it's a numerical-safety win at zero perf cost. Co-authored-by: Cursor --- .../dlrm_v3/train/gin/yambda_5b.gin | 25 +++---- .../dlrm_v3/train/utils.py | 70 +++++++++++-------- recommendation_v4/scripts/launch_slurm.sh | 5 +- 3 files changed, 55 insertions(+), 45 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index 2bbf0dabf..49a94e573 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -85,18 +85,19 @@ env_int.default = 260 # Sparse embedding all-to-all wire precision. The embedding shuffle is the # dominant, bandwidth-bound (esp. multi-node) collective; quantizing it via -# TorchRec QCommsConfig halves (bf16/fp16) the wire volume. "fp32" = off -# (default; numerically untouched). Set "bf16" (or "fp16") to enable. -# Override via $DLRMV4_SPARSE_A2A_PRECISION. -make_optimizer_and_shard.sparse_a2a_precision = @saap/env_str() -saap/env_str.key = "DLRMV4_SPARSE_A2A_PRECISION" -saap/env_str.default = "fp32" -# Also quantize the backward (gradient) a2a. 1 = yes (default), 0 = forward-only -# (keeps gradients fp32 for a more conservative numerical profile). -# Override via $DLRMV4_BF16_SPARSE_BWD. -make_optimizer_and_shard.sparse_a2a_quantize_backward = @saab/env_int() -saab/env_int.key = "DLRMV4_BF16_SPARSE_BWD" -saab/env_int.default = 1 +# TorchRec QCommsConfig halves (bf16/fp16, both 2 bytes) the wire volume. +# Forward and backward are set independently (each: "fp32" | "bf16" | "fp16"). +# Both "fp32" = off (default; numerically identical to baseline trunk). +# Per TorchRec golden_training, fwd=fp16 / bwd=bf16 is the recommended quantized +# mix: fp16's mantissa suits bounded forward activations, while bf16's wider +# exponent range avoids overflow on gradients. +# Override via $SPARSE_A2A_FWD / $SPARSE_A2A_BWD. +make_optimizer_and_shard.sparse_a2a_forward_precision = @saaf/env_str() +saaf/env_str.key = "SPARSE_A2A_FWD" +saaf/env_str.default = "fp32" +make_optimizer_and_shard.sparse_a2a_backward_precision = @saab/env_str() +saab/env_str.key = "SPARSE_A2A_BWD" +saab/env_str.default = "fp32" get_dataset.name = %dataset get_dataset.new_path_prefix = %DATA_PATH diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py index fcb562b7d..b9610e18f 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -343,33 +343,44 @@ def sparse_optimizer_factory_and_class( def _maybe_apply_qcomm_a2a( sharders: List[Any], device: torch.device, - precision: str = "fp32", - quantize_backward: bool = True, + forward_precision: str = "fp32", + backward_precision: str = "fp32", ) -> List[Any]: """Optionally quantize the embedding all-to-all payload via TorchRec qcomm. The yambda-5b embedding shuffle is the dominant, bandwidth-bound (multi-node) - collective (~14.5 GB/rank fp32); BF16/FP16 forward(+backward) halves the wire - volume. Quant/dequant happen inside the comm op, transparent to the lookup - consumer. Ported from the DLRMv2 R2 lever, retargeted from - ``EmbeddingBagCollectionSharder`` to the sequence ``EmbeddingCollectionSharder`` - this model uses. - - Args (set via gin on ``make_optimizer_and_shard``, env-overridable): - precision: ``fp32`` (off, default) | ``bf16`` | ``fp16`` — the a2a wire dtype. - quantize_backward: also quantize the gradient a2a (default True). + collective (~14.5 GB/rank fp32); a bf16/fp16 wire dtype halves it. Quant/ + dequant happen inside the comm op, transparent to the lookup consumer. Ported + from the DLRMv2 R2 lever, retargeted from ``EmbeddingBagCollectionSharder`` to + the sequence ``EmbeddingCollectionSharder`` this model uses. + + Forward and backward are configured independently because they have different + numerical needs (TorchRec golden_training/train_dlrm.py recommends + forward=fp16, backward=bf16): the forward carries bounded embedding + activations where fp16's extra mantissa helps, while gradients have a wider + range that can overflow fp16, so bf16 (fp32 exponent range) is safer there. + bf16 and fp16 are both 2 bytes, so the wire volume / perf is identical — the + choice is purely numerical. + + Args (set via gin on ``make_optimizer_and_shard``, env-overridable). Each is + one of ``fp32`` (that direction unquantized) | ``bf16`` | ``fp16``. If BOTH + are fp32 the sharders are returned untouched (identical to baseline trunk). """ - precision = (precision or "fp32").strip().lower() + _COMM = {"bf16": "BF16", "fp16": "FP16", "fp32": "FP32"} + fwd = (forward_precision or "fp32").strip().lower() + bwd = (backward_precision or "fp32").strip().lower() rank0 = (not dist.is_initialized()) or dist.get_rank() == 0 - if precision in ("fp32", "", "off", "none"): - return sharders - if precision not in ("bf16", "fp16"): - if rank0: - logger.warning( - "DLRMV4 qcomm a2a: unknown precision %r (want fp32|bf16|fp16); " - "using fp32 a2a", - precision, - ) + for name, p in (("forward", fwd), ("backward", bwd)): + if p not in _COMM: + if rank0: + logger.warning( + "DLRMV4 qcomm a2a: unknown %s precision %r (want " + "fp32|bf16|fp16); using fp32 a2a", + name, + p, + ) + return sharders + if fwd == "fp32" and bwd == "fp32": return sharders try: from torchrec.distributed.embedding import EmbeddingCollectionSharder @@ -379,10 +390,9 @@ def _maybe_apply_qcomm_a2a( QCommsConfig, ) - fwd_prec = {"bf16": CommType.BF16, "fp16": CommType.FP16}[precision] qcfg = QCommsConfig( - forward_precision=fwd_prec, - backward_precision=fwd_prec if quantize_backward else CommType.FP32, + forward_precision=getattr(CommType, _COMM[fwd]), + backward_precision=getattr(CommType, _COMM[bwd]), ) registry = get_qcomm_codecs_registry(qcfg, device=device) new_sharders = [] @@ -399,8 +409,8 @@ def _maybe_apply_qcomm_a2a( logger.info( "DLRMV4 qcomm a2a ENABLED: forward=%s backward=%s " "replaced_ec_sharder=%s", - fwd_prec.value, - fwd_prec.value if quantize_backward else "fp32", + fwd, + bwd, replaced, ) return new_sharders @@ -421,8 +431,8 @@ def make_optimizer_and_shard( world_size: int, local_world_size: Optional[int] = None, hbm_cap_gb: int = 260, - sparse_a2a_precision: str = "fp32", - sparse_a2a_quantize_backward: bool = True, + sparse_a2a_forward_precision: str = "fp32", + sparse_a2a_backward_precision: str = "fp32", ) -> Tuple[DistributedModelParallel, torch.optim.Optimizer]: dense_opt_cls, dense_opt_args, dense_opt_factory = ( dense_optimizer_factory_and_class() @@ -443,8 +453,8 @@ def make_optimizer_and_shard( sharders = _maybe_apply_qcomm_a2a( sharders, device, - precision=sparse_a2a_precision, - quantize_backward=bool(sparse_a2a_quantize_backward), + forward_precision=sparse_a2a_forward_precision, + backward_precision=sparse_a2a_backward_precision, ) # local_world_size = GPUs per node so the planner respects the intra-node # (xGMI/NVLink) vs inter-node hierarchy when placing shards. Defaults to diff --git a/recommendation_v4/scripts/launch_slurm.sh b/recommendation_v4/scripts/launch_slurm.sh index 22beb886b..fe82a1dda 100755 --- a/recommendation_v4/scripts/launch_slurm.sh +++ b/recommendation_v4/scripts/launch_slurm.sh @@ -302,9 +302,8 @@ orchestrate() { ${RUN_NAME:+-e RUN_NAME=$RUN_NAME} \ ${TENSORBOARD_LOG_PATH:+-e TENSORBOARD_LOG_PATH=$TENSORBOARD_LOG_PATH} \ ${CKPT_PATH:+-e CKPT_PATH=$CKPT_PATH} \ - ${DLRMV4_BF16_SPARSE_A2A:+-e DLRMV4_BF16_SPARSE_A2A=$DLRMV4_BF16_SPARSE_A2A} \ - ${DLRMV4_SPARSE_A2A_PRECISION:+-e DLRMV4_SPARSE_A2A_PRECISION=$DLRMV4_SPARSE_A2A_PRECISION} \ - ${DLRMV4_BF16_SPARSE_BWD:+-e DLRMV4_BF16_SPARSE_BWD=$DLRMV4_BF16_SPARSE_BWD} \ + ${SPARSE_A2A_FWD:+-e SPARSE_A2A_FWD=$SPARSE_A2A_FWD} \ + ${SPARSE_A2A_BWD:+-e SPARSE_A2A_BWD=$SPARSE_A2A_BWD} \ -e LOG=$LOG \ $NCCL_ENV_ARGS \ $CONTAINER bash -lc 'cd $REPO && LAUNCH_SLURM_PHASE=worker bash scripts/launch_slurm.sh' From 5f76993ad9800e4e774ae95f608d2d246b03b20a Mon Sep 17 00:00:00 2001 From: suachong Date: Tue, 16 Jun 2026 19:08:07 +0000 Subject: [PATCH 061/127] dlrmv4: enable GPUDirect RDMA by default in slurm worker Set NCCL_NET_GDR_LEVEL=5 and NCCL_DMABUF_ENABLE=1 by default so RCCL does true GPU<->NIC DMA over bnxt_re instead of host-memory staging. The brcmrdma host kernel ships the inbox peer-memory client, so GDR works with no container/host changes. Measured ~+22% throughput at 2 nodes (65.7%->79.8% weak-scaling efficiency). Overridable via NCCL_NET_GDR_LEVEL=0 and non-fatal (falls back to host staging if peermem is absent). Co-authored-by: Cursor --- recommendation_v4/scripts/launch_slurm.sh | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/recommendation_v4/scripts/launch_slurm.sh b/recommendation_v4/scripts/launch_slurm.sh index 3ab70078a..53982923d 100755 --- a/recommendation_v4/scripts/launch_slurm.sh +++ b/recommendation_v4/scripts/launch_slurm.sh @@ -601,10 +601,20 @@ worker() { export NCCL_IB_TIMEOUT=${NCCL_IB_TIMEOUT:-14} export NCCL_IGNORE_CPU_AFFINITY=${NCCL_IGNORE_CPU_AFFINITY:-1} export RCCL_MSCCL_ENABLE=${RCCL_MSCCL_ENABLE:-0} - # GPU-Direct RDMA needs DMABUF/peermem (neither in-container here) — leave - # GDR off so RCCL stages through host memory (still real RDMA over bnxt_re). - export NCCL_NET_GDR_LEVEL=${NCCL_NET_GDR_LEVEL:-0} - echo "[$(date)] NCCL: RDMA over bnxt_re (GID idx ${NCCL_IB_GID_INDEX}, TC ${NCCL_IB_TC}, GDR_LEVEL=${NCCL_NET_GDR_LEVEL}; meta64 bnxt_re config, validated)" | tee -a "$LOG" + # GPU-Direct RDMA: ENABLED by default. The brcmrdma host kernel ships the + # inbox peer-memory client (`ib_register_peer_memory_client` in + # /proc/kallsyms), so RCCL does true GPU<->NIC DMA over bnxt_re instead of + # bouncing through host memory. Measured ~+22% throughput at 2 nodes + # (65.7%->79.8% weak-scaling efficiency) vs the old host-staged path. + # GDR_LEVEL=5 (most permissive) is required so GDR is used even when the GPU + # and NIC cross the CPU root complex. NCCL_DMABUF_ENABLE=1 is a harmless + # no-op here (kernel lacks CONFIG_DMABUF_MOVE_NOTIFY/CONFIG_PCI_P2PDMA, so + # peermem carries it). Enabling is non-fatal: if peermem is ever absent RCCL + # just logs "GDR 0" and falls back to host staging. Override with + # NCCL_NET_GDR_LEVEL=0 to force the legacy host-staged path. + export NCCL_NET_GDR_LEVEL=${NCCL_NET_GDR_LEVEL:-5} + export NCCL_DMABUF_ENABLE=${NCCL_DMABUF_ENABLE:-1} + echo "[$(date)] NCCL: RDMA over bnxt_re (GID idx ${NCCL_IB_GID_INDEX}, TC ${NCCL_IB_TC}, GDR_LEVEL=${NCCL_NET_GDR_LEVEL}, DMABUF=${NCCL_DMABUF_ENABLE}; meta64 bnxt_re config, validated)" | tee -a "$LOG" fi fi export NCCL_DEBUG=${NCCL_DEBUG:-WARN} From a34facc3308fc5d4760f29676908c9a38fc84551 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Tue, 16 Jun 2026 17:51:12 -0500 Subject: [PATCH 062/127] dlrmv4: gin-configurable RNG seed ($SEED) + default TRAIN_SPLIT_PERCENTAGE=1.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a gin-configurable global seed so weight init (dense params + embedding tables) is reproducible run-to-run and runs can be init-matched A/Bs. The seed is bound via $SEED (seed_everything.seed = @seed/env_int(), default 1) and applied by a new seed_everything() called in train_ranker right before make_model() — after the full gin parse, so the binding resolves in the second parse where env_int is registered. Move the old hardcoded seed=1 out of setup() (too early to be gin-configurable). Forward $SEED through launch_slurm.sh. Flip the default TRAIN_SPLIT_PERCENTAGE 0.90 -> 1.0 (all users trained AND evaluated, matching the alleval/qa2a production runs) in both the gin default and the launch_slurm.sh fallback. Validated with two short 1-node runs: SEED=1 and SEED=2 each log their seed on all ranks, and tsp=1.0 is applied without exporting TRAIN_SPLIT_PERCENTAGE. Co-authored-by: Cursor --- .../dlrm_v3/train/gin/yambda_5b.gin | 14 ++++++- .../dlrm_v3/train/train_ranker.py | 6 +++ .../dlrm_v3/train/utils.py | 39 ++++++++++++------- recommendation_v4/scripts/launch_slurm.sh | 3 +- 4 files changed, 46 insertions(+), 16 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index 49a94e573..93f88920f 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -32,6 +32,16 @@ make_model.hammer_kernel = "TRITON" # pinned constants in ops/triton/_autotune_pinning.py call sites. apply_env_bootstrap.TRITON_FULL_AUTOTUNE = False +# Global RNG seed for reproducible weight init (dense params + embedding tables) +# and any seeded RNG consumers. Same seed on every rank => identical dense init; +# fixing it makes runs an init-matched A/B (data order is already deterministic +# via the sampler). seed_everything() is called right before make_model() in +# train_ranker (after the full gin parse), so this binding is resolved in the +# second parse where env_int is registered. Override per-run via $SEED. +seed_everything.seed = @seed/env_int() +seed/env_int.key = "SEED" +seed/env_int.default = 1 + # dense model optimizer dense_optimizer_factory_and_class.learning_rate = 0.001 dense_optimizer_factory_and_class.optimizer_name = "Adam" @@ -63,9 +73,11 @@ data/env_path.default = "/apps/chcai/dlrm_data" # positional split) and the streaming path (get_dataset, an explicit by-user # hash split), so one value configures the holdout in either mode. # 1.0 = no holdout (legacy streaming behavior). Override via $TRAIN_SPLIT_PERCENTAGE. +# Default 1.0: all users are trained AND evaluated (full-coverage eval), matching +# the alleval/qa2a production runs; set <1.0 (e.g. 0.90) for a clean held-out cohort. TRAIN_SPLIT_PERCENTAGE = @tsp/env_float() tsp/env_float.key = "TRAIN_SPLIT_PERCENTAGE" -tsp/env_float.default = 0.90 +tsp/env_float.default = 1.0 # dataloader configs make_train_test_dataloaders.batch_size = %batch_size diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py index 55eece518..dc2a9a8a9 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py @@ -89,6 +89,7 @@ def _main_func( make_model, make_optimizer_and_shard, make_train_test_dataloaders, + seed_everything, setup, streaming_train_eval_loop, train_eval_loop, @@ -113,6 +114,11 @@ def _main_func( # make_train_test_dataloaders, etc. gin.parse_config_file(gin_file) + # Seed all RNGs (gin-configurable $SEED) BEFORE make_model() so weight init + # is reproducible run-to-run. Must follow the full parse above so the binding + # is wired, and precede make_model() below. + seed_everything(rank=rank) + model, model_configs, embedding_table_configs = make_model() model, optimizer = make_optimizer_and_shard( model=model, diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py index b9610e18f..92b09f3f1 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -76,6 +76,27 @@ } +@gin.configurable +def seed_everything(seed: int = 1, rank: int = 0) -> None: + """Seed all RNGs so weight init (make_model) is reproducible across runs. + + Same seed on every rank => dense params are initialized identically across + ranks; sharded embeddings are init'd from the meta device by DMP. Fixing the + seed makes runs an init-matched A/B (data order is already deterministic via + the sampler). gin-configurable via $SEED (yambda_5b.gin: seed_everything.seed); + call this right before make_model(), after the full gin parse. + """ + import random + + import numpy as np + + logger.info(f"[rank {rank}] seeding all RNGs with SEED={seed}") + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + + def setup( rank: int, world_size: int, @@ -96,20 +117,10 @@ def setup( # leaving stale allocations and triggering OOMs on rank 0. torch.cuda.set_device(device) - # Seed all RNGs so weight init (make_model, called after setup) is - # reproducible across runs. Same seed on every rank → dense params are - # initialized identically across ranks; sharded embeddings are init'd from - # the meta device by DMP. Fixed seed makes pipeline-vs-non-pipeline an - # init-matched A/B (data order is already deterministic via the sampler). - import random - - import numpy as np - - _SEED = 1 - random.seed(_SEED) - np.random.seed(_SEED) - torch.manual_seed(_SEED) - torch.cuda.manual_seed_all(_SEED) + # NOTE: RNG seeding for reproducible weight init lives in seed_everything(), + # which train_ranker calls right before make_model() (after the full gin + # parse, so the gin-configurable $SEED is bound). Seeding here would be too + # early to be gin-configurable and is redundant with that call. # initialize the process group if not dist.is_initialized(): diff --git a/recommendation_v4/scripts/launch_slurm.sh b/recommendation_v4/scripts/launch_slurm.sh index fe82a1dda..f639215e7 100755 --- a/recommendation_v4/scripts/launch_slurm.sh +++ b/recommendation_v4/scripts/launch_slurm.sh @@ -287,6 +287,7 @@ orchestrate() { ${DIAG_EMB_STEPS:+-e DIAG_EMB_STEPS=$DIAG_EMB_STEPS} \ ${OUTPUT_TRACE:+-e OUTPUT_TRACE=$OUTPUT_TRACE} \ ${MIN_HISTORY:+-e MIN_HISTORY=$MIN_HISTORY} \ + ${SEED:+-e SEED=$SEED} \ ${MAX_SEQ_LEN:+-e MAX_SEQ_LEN=$MAX_SEQ_LEN} \ ${HISTORY_LENGTH:+-e HISTORY_LENGTH=$HISTORY_LENGTH} \ ${BATCH_SIZE:+-e BATCH_SIZE=$BATCH_SIZE} \ @@ -294,7 +295,7 @@ orchestrate() { ${KEEP_LAST_N:+-e KEEP_LAST_N=$KEEP_LAST_N} \ ${IN_WINDOW_CKPT_FREQ:+-e IN_WINDOW_CKPT_FREQ=$IN_WINDOW_CKPT_FREQ} \ ${CKPT_STEP_FREQ:+-e CKPT_STEP_FREQ=$CKPT_STEP_FREQ} \ - -e TRAIN_SPLIT_PERCENTAGE=${TRAIN_SPLIT_PERCENTAGE:-0.90} \ + -e TRAIN_SPLIT_PERCENTAGE=${TRAIN_SPLIT_PERCENTAGE:-1.0} \ -e SPLIT_SALT=${SPLIT_SALT:-0} \ -e EVAL_HOLDOUT_TS=${EVAL_HOLDOUT_TS:--1} \ -e EVAL_HOLDOUT_NUM_WINDOWS=${EVAL_HOLDOUT_NUM_WINDOWS:-1} \ From 40c696a9c4a6c55ea232d50a37052d3493554fde Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Wed, 17 Jun 2026 08:14:06 -0500 Subject: [PATCH 063/127] dlrmv4: env-configurable dense/sparse LR + optimizer LR logging Make the dense ($DENSE_LR) and sparse ($SPARSE_LR) optimizer learning rates overridable per-run via env (defaults unchanged at 0.001), and log the resolved LR at optimizer construction so runs are self-documenting. Forward both vars through launch_slurm.sh into the container. Co-authored-by: Cursor --- .../dlrm_v3/train/gin/yambda_5b.gin | 10 ++++++++-- .../generative_recommenders/dlrm_v3/train/utils.py | 8 ++++++++ recommendation_v4/scripts/launch_slurm.sh | 2 ++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index 93f88920f..61bdd70e4 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -43,7 +43,10 @@ seed/env_int.key = "SEED" seed/env_int.default = 1 # dense model optimizer -dense_optimizer_factory_and_class.learning_rate = 0.001 +# Learning rate is env-overridable via $DENSE_LR (default 0.001, unchanged). +dense_optimizer_factory_and_class.learning_rate = @dlr/env_float() +dlr/env_float.key = "DENSE_LR" +dlr/env_float.default = 0.001 dense_optimizer_factory_and_class.optimizer_name = "Adam" dense_optimizer_factory_and_class.momentum = 0 dense_optimizer_factory_and_class.weight_decay = 0 @@ -51,7 +54,10 @@ dense_optimizer_factory_and_class.eps = 1e-8 dense_optimizer_factory_and_class.betas = (0.95, 0.999) # sparse model optimizer -sparse_optimizer_factory_and_class.learning_rate = 0.001 +# Learning rate is env-overridable via $SPARSE_LR (default 0.001, unchanged). +sparse_optimizer_factory_and_class.learning_rate = @slr/env_float() +slr/env_float.key = "SPARSE_LR" +slr/env_float.default = 0.001 sparse_optimizer_factory_and_class.optimizer_name = "RowWiseAdagrad" sparse_optimizer_factory_and_class.momentum = 0 sparse_optimizer_factory_and_class.weight_decay = 0 diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py index 92b09f3f1..aaed8c2a0 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -308,6 +308,10 @@ def dense_optimizer_factory_and_class( optimizer_factory = lambda params: optimizer_cls(params, **kwargs) + logger.info( + f"[dense optimizer] {optimizer_name} learning_rate={learning_rate} " + f"(resolved from gin; override via $DENSE_LR)" + ) return optimizer_cls, kwargs, optimizer_factory @@ -348,6 +352,10 @@ def sparse_optimizer_factory_and_class( optimizer_factory = lambda params: optimizer_cls(params, **kwargs) + logger.info( + f"[sparse optimizer] {optimizer_name} learning_rate={learning_rate} " + f"(resolved from gin; override via $SPARSE_LR)" + ) return optimizer_cls, kwargs, optimizer_factory diff --git a/recommendation_v4/scripts/launch_slurm.sh b/recommendation_v4/scripts/launch_slurm.sh index f639215e7..c6b73cb8d 100755 --- a/recommendation_v4/scripts/launch_slurm.sh +++ b/recommendation_v4/scripts/launch_slurm.sh @@ -288,6 +288,8 @@ orchestrate() { ${OUTPUT_TRACE:+-e OUTPUT_TRACE=$OUTPUT_TRACE} \ ${MIN_HISTORY:+-e MIN_HISTORY=$MIN_HISTORY} \ ${SEED:+-e SEED=$SEED} \ + ${DENSE_LR:+-e DENSE_LR=$DENSE_LR} \ + ${SPARSE_LR:+-e SPARSE_LR=$SPARSE_LR} \ ${MAX_SEQ_LEN:+-e MAX_SEQ_LEN=$MAX_SEQ_LEN} \ ${HISTORY_LENGTH:+-e HISTORY_LENGTH=$HISTORY_LENGTH} \ ${BATCH_SIZE:+-e BATCH_SIZE=$BATCH_SIZE} \ From d646b2acd095936752216ced1706f7c548921401 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Wed, 17 Jun 2026 08:14:49 -0500 Subject: [PATCH 064/127] dlrmv4: env-configurable HSTU transformer depth ($HSTU_NUM_LAYERS) Make the HSTU attention layer count overridable per-run via $HSTU_NUM_LAYERS (default 5, unchanged), resolved in the full gin parse and forwarded through launch_slurm.sh. Changing depth alters model shape, so a run with a new depth must use a fresh CKPT_PATH (incompatible with existing checkpoints). Co-authored-by: Cursor --- .../dlrm_v3/train/gin/yambda_5b.gin | 10 ++++++++++ recommendation_v4/scripts/launch_slurm.sh | 1 + 2 files changed, 11 insertions(+) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index 61bdd70e4..3099cd67e 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -160,6 +160,16 @@ get_hstu_configs.max_seq_len = @msl/env_int() msl/env_int.key = "MAX_SEQ_LEN" msl/env_int.default = 4096 +# HSTU transformer depth (number of attention layers). Default 5 (unchanged). +# Override per-run via $HSTU_NUM_LAYERS. NOTE: changing depth changes the model +# shape, so a run with a new depth MUST use a FRESH CKPT_PATH (incompatible with +# 5-layer checkpoints). Resolved in the full gin parse (get_hstu_configs is not +# registered during the early skip_unknown parse), so the @env_int reference is +# skipped on the first pass — same safe path as the LR knobs. +get_hstu_configs.hstu_attn_num_layers = @nl/env_int() +nl/env_int.key = "HSTU_NUM_LAYERS" +nl/env_int.default = 5 + # --- streaming (temporal-order) training ------------------------------------- # Only consumed under `--mode streaming-train-eval`; the default train-eval # path above is unaffected. Trains time window T then evals window T+1, diff --git a/recommendation_v4/scripts/launch_slurm.sh b/recommendation_v4/scripts/launch_slurm.sh index c6b73cb8d..f20934285 100755 --- a/recommendation_v4/scripts/launch_slurm.sh +++ b/recommendation_v4/scripts/launch_slurm.sh @@ -290,6 +290,7 @@ orchestrate() { ${SEED:+-e SEED=$SEED} \ ${DENSE_LR:+-e DENSE_LR=$DENSE_LR} \ ${SPARSE_LR:+-e SPARSE_LR=$SPARSE_LR} \ + ${HSTU_NUM_LAYERS:+-e HSTU_NUM_LAYERS=$HSTU_NUM_LAYERS} \ ${MAX_SEQ_LEN:+-e MAX_SEQ_LEN=$MAX_SEQ_LEN} \ ${HISTORY_LENGTH:+-e HISTORY_LENGTH=$HISTORY_LENGTH} \ ${BATCH_SIZE:+-e BATCH_SIZE=$BATCH_SIZE} \ From 922aeec9db0eb013c79e4b261377e5d58d3ca568 Mon Sep 17 00:00:00 2001 From: suachong Date: Thu, 18 Jun 2026 19:07:38 +0000 Subject: [PATCH 065/127] dlrmv4: env-overridable dense/sparse LRs for sweeps + holdout default 1.0 Make dense (Adam) and sparse (RowWiseAdagrad) learning rates overridable via $DENSE_LR / $SPARSE_LR with gin defaults preserved at 0.001, so LR sweeps don't require editing gin. Resolve gin macro references in the MLPerf param logger so env-overridden LRs are logged as real numbers. Default TRAIN_SPLIT_PERCENTAGE to 1.0 (no holdout) and log the resolved LR overrides at orchestration time. Co-authored-by: Cursor --- .../dlrm_v3/train/gin/yambda_5b.gin | 18 +++++++++++++++--- .../dlrm_v3/train/train_ranker.py | 13 ++++++++++++- recommendation_v4/scripts/launch_slurm.sh | 5 ++++- 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index 8623fb21d..984fb20c3 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -33,7 +33,13 @@ make_model.hammer_kernel = "TRITON" apply_env_bootstrap.TRITON_FULL_AUTOTUNE = False # dense model optimizer -dense_optimizer_factory_and_class.learning_rate = 0.001 +# Learning rate is env-overridable (default 0.001 preserves prior runs) so an LR +# sweep can probe the dense Adam rate without editing gin. Scoped (`dlr/`) so +# this env_float binding doesn't collide with the other env_float call sites. +# Override via $DENSE_LR. +dense_optimizer_factory_and_class.learning_rate = @dlr/env_float() +dlr/env_float.key = "DENSE_LR" +dlr/env_float.default = 0.001 dense_optimizer_factory_and_class.optimizer_name = "Adam" dense_optimizer_factory_and_class.momentum = 0 dense_optimizer_factory_and_class.weight_decay = 0 @@ -41,7 +47,13 @@ dense_optimizer_factory_and_class.eps = 1e-8 dense_optimizer_factory_and_class.betas = (0.95, 0.999) # sparse model optimizer -sparse_optimizer_factory_and_class.learning_rate = 0.001 +# Learning rate is env-overridable (default 0.001 preserves prior runs) so an LR +# sweep can probe the sparse RowWiseAdagrad rate without editing gin. Scoped +# (`slr/`) so this env_float binding doesn't collide with the other env_float +# call sites. Override via $SPARSE_LR. +sparse_optimizer_factory_and_class.learning_rate = @slr/env_float() +slr/env_float.key = "SPARSE_LR" +slr/env_float.default = 0.001 sparse_optimizer_factory_and_class.optimizer_name = "RowWiseAdagrad" sparse_optimizer_factory_and_class.momentum = 0 sparse_optimizer_factory_and_class.weight_decay = 0 @@ -65,7 +77,7 @@ data/env_path.default = "/apps/chcai/dlrm_data" # 1.0 = no holdout (legacy streaming behavior). Override via $TRAIN_SPLIT_PERCENTAGE. TRAIN_SPLIT_PERCENTAGE = @tsp/env_float() tsp/env_float.key = "TRAIN_SPLIT_PERCENTAGE" -tsp/env_float.default = 0.90 +tsp/env_float.default = 1.0 # dataloader configs make_train_test_dataloaders.batch_size = %batch_size diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py index d22d09a30..6df73f351 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py @@ -213,9 +213,20 @@ def _main_func( def _gin_param(name: str, default: object) -> object: try: - return gin.query_parameter(name) + value = gin.query_parameter(name) except (ValueError, KeyError): return default + # When a binding is a gin macro/configurable reference (e.g. + # `@dlr/env_float()`), query_parameter returns the unevaluated + # reference object, which the MLPerf logger cannot encode. Resolve + # it to its actual value so env-overridden LRs are logged as real + # numbers. Plain literals pass through unchanged. + if hasattr(value, "scoped_configurable_fn"): + try: + return value.scoped_configurable_fn() + except Exception: + return default + return value global_batch_size = world_size * int(train_dataloader.batch_size) mlperf_logger.event(key=c.GLOBAL_BATCH_SIZE, value=global_batch_size) diff --git a/recommendation_v4/scripts/launch_slurm.sh b/recommendation_v4/scripts/launch_slurm.sh index 53982923d..a71370143 100755 --- a/recommendation_v4/scripts/launch_slurm.sh +++ b/recommendation_v4/scripts/launch_slurm.sh @@ -174,6 +174,7 @@ orchestrate() { echo "[$(date)] launch_slurm/orchestrate: job=${SLURM_JOB_ID:-?} nodes=${SLURM_JOB_NODELIST:-?} nnodes=${SLURM_NNODES:-1}" | tee -a "$LOG" echo "[$(date)] resolved SCRIPT_PATH=$SCRIPT_PATH REPO=$REPO" | tee -a "$LOG" echo "[$(date)] config: MODE=$MODE START_TS=$START_TS NUM_TRAIN_TS=$NUM_TRAIN_TS NUM_TRAIN_BATCHES=$NUM_TRAIN_BATCHES NUM_EVAL_BATCHES=$NUM_EVAL_BATCHES METRIC_LOG_FREQ=$METRIC_LOG_FREQ" | tee -a "$LOG" + echo "[$(date)] lr-override: DENSE_LR=${DENSE_LR:-} SPARSE_LR=${SPARSE_LR:-}" | tee -a "$LOG" # Rendezvous resolved on the HOST (the container image has no SLURM client). MASTER_ADDR=$(scontrol show hostnames "$SLURM_JOB_NODELIST" 2>/dev/null | head -1) @@ -310,11 +311,13 @@ orchestrate() { ${MAX_SEQ_LEN:+-e MAX_SEQ_LEN=$MAX_SEQ_LEN} \ ${HISTORY_LENGTH:+-e HISTORY_LENGTH=$HISTORY_LENGTH} \ ${BATCH_SIZE:+-e BATCH_SIZE=$BATCH_SIZE} \ + ${DENSE_LR:+-e DENSE_LR=$DENSE_LR} \ + ${SPARSE_LR:+-e SPARSE_LR=$SPARSE_LR} \ ${CKPT_TIME_INTERVAL_S:+-e CKPT_TIME_INTERVAL_S=$CKPT_TIME_INTERVAL_S} \ ${KEEP_LAST_N:+-e KEEP_LAST_N=$KEEP_LAST_N} \ ${IN_WINDOW_CKPT_FREQ:+-e IN_WINDOW_CKPT_FREQ=$IN_WINDOW_CKPT_FREQ} \ ${CKPT_STEP_FREQ:+-e CKPT_STEP_FREQ=$CKPT_STEP_FREQ} \ - -e TRAIN_SPLIT_PERCENTAGE=${TRAIN_SPLIT_PERCENTAGE:-0.90} \ + -e TRAIN_SPLIT_PERCENTAGE=${TRAIN_SPLIT_PERCENTAGE:-1.0} \ -e AUC_THRESHOLD=${AUC_THRESHOLD:-0.80275} \ -e SPLIT_SALT=${SPLIT_SALT:-0} \ -e EVAL_HOLDOUT_TS=${EVAL_HOLDOUT_TS:--1} \ From e0d6e46d9c4e7d1f9d8c49ffabf5da4b4d3b292e Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Thu, 18 Jun 2026 14:17:43 -0500 Subject: [PATCH 066/127] dlrmv4: disable TensorBoard by default (no-op writer) The shared-NFS tfevents writer was the only metrics sink whose I/O error was uncaught, and it repeatedly crashed trainers on transient /apps Errno 121 (Remote I/O) hiccups. Default TENSORBOARD_LOG_PATH is now empty, which installs a _NoOpSummaryWriter so the metrics path (compute + text-log + .metrics.jsonl sinks) runs unchanged and never crashes on TB file I/O. Nothing we consume reads TensorBoard. Re-enable per-run by setting $TENSORBOARD_LOG_PATH to a non-empty dir. Co-authored-by: Cursor --- .../dlrm_v3/train/gin/yambda_5b.gin | 13 ++++++---- .../generative_recommenders/dlrm_v3/utils.py | 24 +++++++++++++++++++ 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index 3099cd67e..09ebf296e 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -308,13 +308,16 @@ Profiler.active = 5 Profiler.trace_dir = @run_results_dir() # logger variables -# TensorBoard event dir. Default lives on shared NFS (not container-local /tmp, -# which is wiped on node failover) so the NE/AUC scalars survive relaunches and -# failover. Override per-run via $TENSORBOARD_LOG_PATH (the supervisor sets it -# to /apps/chcai/tb/$RUN_NAME/). +# TensorBoard event dir. DISABLED BY DEFAULT (empty path): the shared-NFS +# tfevents writer is the only metrics sink whose I/O error is uncaught, and it +# repeatedly crashed trainers on transient /apps `Errno 121` (Remote I/O) hiccups. +# Nothing we consume reads TensorBoard — eval-window AUCs come from the durable +# `.metrics.jsonl` sink (try/except-guarded) and the text run log. An empty +# path makes MetricsLogger install a no-op writer (see _NoOpSummaryWriter). +# Re-enable for a run by setting $TENSORBOARD_LOG_PATH to a non-empty dir. MetricsLogger.tensorboard_log_path = @tbp/env_path() tbp/env_path.key = "TENSORBOARD_LOG_PATH" -tbp/env_path.default = "/apps/chcai/tb/yambda_5b/" +tbp/env_path.default = "" MetricsLogger.world_size = 8 MetricsLogger.auc_threshold = 0.80275 # Lifetime-AUC backend, selectable independently for the train cumulative AUC and diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py index dcda51365..10c107bdc 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py @@ -857,6 +857,25 @@ def step(self) -> None: self._profiler.step() +class _NoOpSummaryWriter: + """Drop-in stand-in for SummaryWriter used when TensorBoard is disabled + (empty ``tensorboard_log_path``). All scalar writes become no-ops so the + metrics path (compute + text-log + ``.metrics.jsonl`` sinks) runs unchanged + and never crashes on TB file I/O. The shared ``/apps`` tfevents writer was a + crash source under transient filer ``Errno 121`` (Remote I/O) errors, and + nothing we consume reads TensorBoard — eval-window AUCs come from the JSONL + sink and the text log.""" + + def add_scalar(self, *args, **kwargs) -> None: + pass + + def flush(self) -> None: + pass + + def close(self) -> None: + pass + + @gin.configurable class MetricsLogger: """ @@ -1016,6 +1035,11 @@ def _make_reg(ws: int) -> List[RecMetricComputation]: if tensorboard_log_path != "": self.tb_logger = SummaryWriter(log_dir=tensorboard_log_path, purge_step=0) self.tb_logger.flush() + else: + # TB disabled: use a no-op writer so the existing call sites (and the + # `assert self.tb_logger is not None` in compute_and_log) keep working + # while no tfevents are written to the fragile shared filer. + self.tb_logger = _NoOpSummaryWriter() # Throughput / time-to-target tracking. Counters are train-only; eval # samples are not relevant for headline samples/sec numbers. From bc743b580be38b0bf651007d7d7c0e51b45cc936 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Thu, 18 Jun 2026 14:18:31 -0500 Subject: [PATCH 067/127] dlrmv4: default dense/sparse LR=1e-5 and HSTU depth=3 Change the gin defaults to the configuration validated by the recent power-user (min4086) ht299 runs: dense+sparse LR 0.001 -> 1e-5 and HSTU attention depth 5 -> 3 (reaches ~0.78-0.81 holdout-299 AUC at ~1.4x faster training than the 5-layer/5e-5 setup). Both remain env-overridable via $DENSE_LR / $SPARSE_LR / $HSTU_NUM_LAYERS. Co-authored-by: Cursor --- .../dlrm_v3/train/gin/yambda_5b.gin | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index 09ebf296e..f18c866a2 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -43,10 +43,10 @@ seed/env_int.key = "SEED" seed/env_int.default = 1 # dense model optimizer -# Learning rate is env-overridable via $DENSE_LR (default 0.001, unchanged). +# Learning rate is env-overridable via $DENSE_LR (default 1e-5). dense_optimizer_factory_and_class.learning_rate = @dlr/env_float() dlr/env_float.key = "DENSE_LR" -dlr/env_float.default = 0.001 +dlr/env_float.default = 0.00001 dense_optimizer_factory_and_class.optimizer_name = "Adam" dense_optimizer_factory_and_class.momentum = 0 dense_optimizer_factory_and_class.weight_decay = 0 @@ -54,10 +54,10 @@ dense_optimizer_factory_and_class.eps = 1e-8 dense_optimizer_factory_and_class.betas = (0.95, 0.999) # sparse model optimizer -# Learning rate is env-overridable via $SPARSE_LR (default 0.001, unchanged). +# Learning rate is env-overridable via $SPARSE_LR (default 1e-5). sparse_optimizer_factory_and_class.learning_rate = @slr/env_float() slr/env_float.key = "SPARSE_LR" -slr/env_float.default = 0.001 +slr/env_float.default = 0.00001 sparse_optimizer_factory_and_class.optimizer_name = "RowWiseAdagrad" sparse_optimizer_factory_and_class.momentum = 0 sparse_optimizer_factory_and_class.weight_decay = 0 @@ -160,15 +160,15 @@ get_hstu_configs.max_seq_len = @msl/env_int() msl/env_int.key = "MAX_SEQ_LEN" msl/env_int.default = 4096 -# HSTU transformer depth (number of attention layers). Default 5 (unchanged). +# HSTU transformer depth (number of attention layers). Default 3. # Override per-run via $HSTU_NUM_LAYERS. NOTE: changing depth changes the model # shape, so a run with a new depth MUST use a FRESH CKPT_PATH (incompatible with -# 5-layer checkpoints). Resolved in the full gin parse (get_hstu_configs is not -# registered during the early skip_unknown parse), so the @env_int reference is -# skipped on the first pass — same safe path as the LR knobs. +# checkpoints of a different depth). Resolved in the full gin parse +# (get_hstu_configs is not registered during the early skip_unknown parse), so +# the @env_int reference is skipped on the first pass — same safe path as the LR knobs. get_hstu_configs.hstu_attn_num_layers = @nl/env_int() nl/env_int.key = "HSTU_NUM_LAYERS" -nl/env_int.default = 5 +nl/env_int.default = 3 # --- streaming (temporal-order) training ------------------------------------- # Only consumed under `--mode streaming-train-eval`; the default train-eval From b4ce3ba40acb746cf71b0f4221de5838e72674f7 Mon Sep 17 00:00:00 2001 From: suachong Date: Thu, 18 Jun 2026 19:24:49 +0000 Subject: [PATCH 068/127] dlrmv4: report EVAL_ACCURACY as per-window AUC (configurable, default window) The MLPerf EVAL_ACCURACY event and the convergence decision (early SUCCESS RUN_STOP + end-of-run finalize) now use the per-pass full-holdout "window_auc" instead of the cumulative "lifetime_auc". Made selectable via a new gin knob streaming_train_eval_loop.eval_accuracy_auc_mode ($EVAL_ACCURACY_AUC_MODE), default "window"; set "lifetime" to restore prior behavior. Both AUCs are still computed and logged to TensorBoard. The rank-0-decides-then-broadcast deadlock guard is preserved. Forward $EVAL_ACCURACY_AUC_MODE through launch_slurm.sh. Co-authored-by: Cursor --- .../dlrm_v3/train/gin/yambda_5b.gin | 17 +++-- .../dlrm_v3/train/utils.py | 70 ++++++++++++------- recommendation_v4/scripts/launch_slurm.sh | 1 + 3 files changed, 60 insertions(+), 28 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index 6dc2358fc..6efa64244 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -328,13 +328,22 @@ tbp/env_path.key = "TENSORBOARD_LOG_PATH" tbp/env_path.default = "tb/yambda_5b/" MetricsLogger.world_size = 8 # Time-to-target AUC threshold. Doubles as the MLPerf convergence target: when -# the cumulative ("lifetime_") listen_plus eval AUC first reaches this value the -# streaming-train-eval run emits a SUCCESS RUN_STOP and terminates gracefully. -# Override via $AUC_THRESHOLD (e.g. 0.5 to smoke-test the early-stop path on a -# short run). MLPerf's DLRM-DCNv2 reference uses 0.80275. +# the selected listen_plus eval AUC (see eval_accuracy_auc_mode below; default +# the per-pass "window_" AUC) first reaches this value the streaming-train-eval +# run emits a SUCCESS RUN_STOP and terminates gracefully. Override via +# $AUC_THRESHOLD (e.g. 0.5 to smoke-test the early-stop path on a short run). +# MLPerf's DLRM-DCNv2 reference uses 0.80275. MetricsLogger.auc_threshold = @at/env_float() at/env_float.key = "AUC_THRESHOLD" at/env_float.default = 0.80275 +# Which eval AUC is reported as EVAL_ACCURACY and drives the convergence / +# SUCCESS RUN_STOP decision: "window" (per-pass full-holdout AUC, reset each eval +# pass; the default) or "lifetime" (cumulative across all eval passes). Both AUCs +# are still computed and logged to TensorBoard regardless; this only selects the +# one used for MLPerf EVAL_ACCURACY + early-stop. Override via $EVAL_ACCURACY_AUC_MODE. +streaming_train_eval_loop.eval_accuracy_auc_mode = @eaam/env_str() +eaam/env_str.key = "EVAL_ACCURACY_AUC_MODE" +eaam/env_str.default = "window" # Lifetime-AUC backend, selectable independently for the train cumulative AUC and # the eval cumulative ("lifetime_*") AUC. Both default to "binned": # "binned" = BinnedCumulativeAUC: exact-cumulative AUC from an O(bins) score diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py index ca6291ffc..d28f2ebec 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -1488,6 +1488,12 @@ def streaming_train_eval_loop( # event emission; the loop is otherwise unchanged. Supplied by train_ranker # for the streaming-train-eval benchmark path. mlperf_logger: Optional[Any] = None, + # Which eval AUC drives the reported EVAL_ACCURACY and the convergence + # decision (early SUCCESS RUN_STOP + end-of-run finalize): "window" = the + # per-pass full-holdout AUC (reset each eval pass; the default), or + # "lifetime" = the cumulative AUC across all eval passes. Override via + # $EVAL_ACCURACY_AUC_MODE. + eval_accuracy_auc_mode: str = "window", ) -> None: """Streaming train+eval loop with per-window (and optionally mid-window) checkpoints. @@ -2076,14 +2082,29 @@ def _mlperf_progress() -> Dict[str, Any]: mlperf_logger.constants.EPOCH_NUM: epoch_num, } - def _lifetime_auc(metrics: Dict[str, float]) -> Optional[float]: - # Convergence metric: the cumulative ("lifetime_") listen_plus AUC. - # Key format is `metric/{prefix}{name}/{task}` (see MetricsLogger.compute), - # e.g. `metric/lifetime_auc/listen_plus`. Match the `lifetime_auc` short - # name; ignore GAUC. + # Convergence/EVAL_ACCURACY metric short name, selected by + # eval_accuracy_auc_mode: "window_auc" (per-pass full-holdout AUC, default) + # or "lifetime_auc" (cumulative across eval passes). + _eval_auc_short = ( + "lifetime_auc" + if str(eval_accuracy_auc_mode).strip().lower() == "lifetime" + else "window_auc" + ) + if rank == 0 and mlperf_logger is not None: + logger.info( + f"[mlperf] EVAL_ACCURACY / convergence metric = {_eval_auc_short} " + f"(eval_accuracy_auc_mode={eval_accuracy_auc_mode!r})" + ) + + def _eval_target_auc(metrics: Dict[str, float]) -> Optional[float]: + # Convergence metric: the listen_plus eval AUC selected by + # eval_accuracy_auc_mode (window vs lifetime). Key format is + # `metric/{prefix}{name}/{task}` (see MetricsLogger.compute), e.g. + # `metric/window_auc/listen_plus`. Match the selected short name; + # ignore GAUC. for key, val in metrics.items(): short = key.split("/")[-2] if "/" in key else key - if short == "lifetime_auc": + if short == _eval_auc_short: return float(val) return None @@ -2118,24 +2139,25 @@ def _mlperf_run_stop(status: object) -> None: mlperf_run_stopped[0] = True def _mlperf_eval_stop(eval_metrics: Dict[str, float]) -> bool: - # Emit EVAL_ACCURACY (lifetime listen_plus AUC) + EVAL_STOP, and drive an - # early SUCCESS RUN_STOP when the target threshold is reached. Returns - # True iff the run should stop now -- the SAME value on every rank. + # Emit EVAL_ACCURACY (the selected eval listen_plus AUC) + EVAL_STOP, and + # drive an early SUCCESS RUN_STOP when the target threshold is reached. + # Returns True iff the run should stop now -- the SAME value on every rank. # - # CRITICAL (deadlock avoidance): the cumulative lifetime AUC is produced - # by a reduce that is only valid on global rank 0, so a per-rank - # `lifetime >= thr` test diverges (only rank 0 sees the value) and the - # ranks that "stop" hit the RUN_STOP barrier while the rest march into - # the next window's embedding all-to-all -> NCCL collective-timeout hang - # (observed: 600s ALLTOALL_BASE watchdog abort). So rank 0 decides and - # BROADCASTS the boolean; all ranks then break (or continue) in lockstep. + # CRITICAL (deadlock avoidance): the eval AUC is produced by a reduce + # that is only guaranteed valid on global rank 0, so a per-rank + # `eval_auc >= thr` test could diverge (only rank 0 sees the value) and + # the ranks that "stop" hit the RUN_STOP barrier while the rest march + # into the next window's embedding all-to-all -> NCCL collective-timeout + # hang (observed: 600s ALLTOALL_BASE watchdog abort). So rank 0 decides + # and BROADCASTS the boolean; all ranks then break (or continue) in + # lockstep. if mlperf_logger is None: return False - lifetime = _lifetime_auc(eval_metrics) - if lifetime is not None: + eval_auc = _eval_target_auc(eval_metrics) + if eval_auc is not None: mlperf_logger.event( key=mlperf_logger.constants.EVAL_ACCURACY, - value=lifetime, + value=eval_auc, metadata=_mlperf_progress(), ) mlperf_logger.end( @@ -2146,9 +2168,9 @@ def _mlperf_eval_stop(eval_metrics: Dict[str, float]) -> bool: if ( rank == 0 and not mlperf_run_stopped[0] - and lifetime is not None + and eval_auc is not None and thr is not None - and lifetime >= thr + and eval_auc >= thr ): decision[0] = 1.0 if torch.distributed.is_initialized(): @@ -2161,12 +2183,12 @@ def _mlperf_eval_stop(eval_metrics: Dict[str, float]) -> bool: def _mlperf_finalize(final_metrics: Dict[str, float]) -> None: # End-of-run RUN_STOP when the threshold was never crossed: SUCCESS iff - # the final lifetime AUC meets the target, else ABORTED. + # the final eval AUC meets the target, else ABORTED. if mlperf_logger is None or mlperf_run_stopped[0]: return - lifetime = _lifetime_auc(final_metrics) + eval_auc = _eval_target_auc(final_metrics) thr = metric_logger.auc_threshold - success = lifetime is not None and thr is not None and lifetime >= thr + success = eval_auc is not None and thr is not None and eval_auc >= thr _mlperf_run_stop( mlperf_logger.constants.SUCCESS if success diff --git a/recommendation_v4/scripts/launch_slurm.sh b/recommendation_v4/scripts/launch_slurm.sh index 045495b22..50b235dd2 100755 --- a/recommendation_v4/scripts/launch_slurm.sh +++ b/recommendation_v4/scripts/launch_slurm.sh @@ -326,6 +326,7 @@ orchestrate() { ${CKPT_STEP_FREQ:+-e CKPT_STEP_FREQ=$CKPT_STEP_FREQ} \ -e TRAIN_SPLIT_PERCENTAGE=${TRAIN_SPLIT_PERCENTAGE:-1.0} \ -e AUC_THRESHOLD=${AUC_THRESHOLD:-0.80275} \ + ${EVAL_ACCURACY_AUC_MODE:+-e EVAL_ACCURACY_AUC_MODE=$EVAL_ACCURACY_AUC_MODE} \ -e SPLIT_SALT=${SPLIT_SALT:-0} \ -e EVAL_HOLDOUT_TS=${EVAL_HOLDOUT_TS:--1} \ -e EVAL_HOLDOUT_NUM_WINDOWS=${EVAL_HOLDOUT_NUM_WINDOWS:-1} \ From c5469a4faf4a9212f609c2f6c5f13e7df2c88521 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Thu, 18 Jun 2026 16:52:38 -0500 Subject: [PATCH 069/127] dlrmv4: consolidate streaming e2e supervisor to one sbatch-wrapping script Replace the single-node docker-exec supervisor with the sbatch-job-level model (formerly run_streaming_e2e_multinode.sh), which handles 1..N nodes via launch_slurm.sh. Node replacement is now SLURM's job on resubmit, so the in-place node-acquisition/provision/exec-sentinel logic is dropped. Co-authored-by: Cursor --- .../scripts/run_streaming_e2e.sh | 836 +++++------------- 1 file changed, 221 insertions(+), 615 deletions(-) diff --git a/recommendation_v4/scripts/run_streaming_e2e.sh b/recommendation_v4/scripts/run_streaming_e2e.sh index f97a0e483..c913bac7b 100755 --- a/recommendation_v4/scripts/run_streaming_e2e.sh +++ b/recommendation_v4/scripts/run_streaming_e2e.sh @@ -1,689 +1,295 @@ #!/bin/bash # ============================================================================= -# run_streaming_e2e.sh — self-healing supervisor for the long-run yambda-5b -# streaming train+eval (NE/AUC over the full ~5B dataset) +# run_streaming_e2e.sh — self-healing supervisor for a yambda-5b streaming +# train+eval run (sbatch-job level). Works for 1..N nodes. # ============================================================================= # -# WHAT IT DOES -# Owns a multi-day "streaming-train-eval" run and keeps it alive unattended -# across the three failure modes that actually kill long runs: -# 1. trainer process crash / OOM / nonzero exit -# 2. silent death (the whole process group gets SIGKILLed — no exit code) -# 3. the SLURM node itself going away (down / drained / job ended) -# In every case it relaunches the trainer from the latest on-disk checkpoint -# (failing over to a brand-new node for case 3) until the run finishes. +# WHAT IT SUPERVISES +# The run is an `sbatch [--nodes=N] scripts/launch_slurm.sh` BATCH job. That +# batch script is fully self-contained: it runs orchestrate -> provision +# (container + RDMA) -> worker (in-container trainer) on EVERY node, so it +# handles single-node (--nodes=1) and multi-node (world_size=8N) identically. +# This supervisor wraps THAT job: it monitors it and, on crash / node-failure +# / hang, RESUBMITS it (which resumes from the latest checkpoint via load +# auto-latest), bounded by --max-relaunch. There is no docker-exec lifecycle +# or in-place node failover here — node replacement is SLURM's job on resubmit. # -# WHY A RELAUNCH "JUST WORKS" (resume model) -# The training stack already implements exact-once resume: on startup it picks -# the latest numeric checkpoint subdir under $CKPT_PATH, restores model + -# optimizer + per-rank RNG, and (for mid-window in-window saves) skips the -# batches already trained in the partially-done window. So relaunching with -# the SAME --ckpt-path transparently continues from where it died — no manual -# bookkeeping here beyond pointing every attempt at the same base dir. +# RESUME MODEL (why a resubmit "just works") +# The trainer checkpoints to $CKPT_PATH and on startup load_dmp_checkpoint +# auto-resolves to the highest-numbered subdir, restoring model+optimizer+RNG +# and skipping already-trained batches of a partial window. So resubmitting the +# SAME submit-script (same CKPT_PATH/LOG) continues from where it died. +# Resubmits set APPEND_LOG=1 so the metrics log is preserved across attempts. # -# WHERE IT RUNS / HOW IT DRIVES WORK -# This script runs on the SLURM HEAD node. The trainer runs inside a long- -# lived docker container ($CONTAINER) on the compute node held by a SLURM -# allocation ($JOBID). All control flow is `srun --jobid --overlap -# docker exec ...` into that container. The container bind-mounts shared NFS -# (/home/chcai = code, /apps/chcai = checkpoints+logs), which is what makes -# node failover possible: any node in $PARTITION sees the same code+state. +# WHAT IT DETECTS (poll every --poll-s) +# * job left the queue -> read sacct State/ExitCode: +# COMPLETED+0 => run finished (success, exit 0) +# CANCELLED => user intent (stop, exit 0 — NOT our place to resubmit) +# FAILED/NODE_FAIL/TIMEOUT/OUT_OF_MEMORY/BOOT_FAIL/PREEMPTED => relaunch +# * hang watchdog: job RUNNING but LOG frozen >= --stall-s AND no trainer +# process alive on ANY node (cross-node pgrep) => scancel + relaunch. +# * disk guard before each (re)submit: require --min-free-gib on the ckpt vol. # -# MAIN LOOP (state machine, up to --max-relaunch attempts) -# for each attempt: -# ensure_ready — guarantee a healthy allocation whose container is up, -# failing over to a freshly-provisioned node if not. -# disk_guard — sweep crash-orphaned *.tmp/*.old saves; abort if the -# ckpt volume has < --min-free-gib free. -# cleanup_workers— kill any stragglers from a previous attempt. -# launch — detached `docker exec -d` of the trainer; a trailing -# echo appends an `E2E_RUN_EXIT=` sentinel to the log -# when the trainer returns (clean OR crash). -# monitor loop (every --poll-s): -# * node watchdog — if $JOBID stops being healthy mid-run, break and -# let the next attempt fail over. -# * exit sentinel — E2E_RUN_EXIT=0 => success (done); nonzero => relaunch. -# * stall watchdog — if the log stops growing AND no trainer process is -# alive for --stall-s, treat as silent death=>relaunch. -# (Long blocking saves keep the process alive, so they -# never false-trip this.) +# WHERE IT RUNS +# On the SLURM head node (NFS-mounted /home/chcai code + /apps/chcai +# ckpts/logs are visible here for squeue/sacct/df and the cross-node pgrep). # -# NODE FAILOVER (case 3, the --allow-failover path) -# ensure_ready -> acquire_node: submit an `sbatch` hold job (`--wrap "sleep -# infinity"`, bounded by --time=$ALLOC_TIME) for a fresh exclusive node on -# $PARTITION, optionally from --reservation $RESERVATION; wait for RUNNING, -# then provision_node runs $PROVISION_SCRIPT on it via `srun --jobid --overlap` -# (docker pull + container create + dep install; ~15 min on a cold node). -# sbatch (not salloc) because interactive salloc on some partitions (e.g. -# meta64) is capped at 240 min, which a multi-day hold would exceed. Jobs WE -# create are tracked and `scancel`ed (container removed first) on success via -# release_acquired; the user's original --jobid is never cancelled. -# Checkpoints on shared NFS make the resume seamless. +# USAGE +# # Submit a fresh job from the launch script, then supervise it: +# nohup bash scripts/run_streaming_e2e.sh \ +# --submit-script /apps/chcai/yambda_5b_e2e//launch_1node.sh \ +# --log /apps/chcai/yambda_5b_e2e//.log \ +# --ckpt-path /apps/chcai/yambda_5b_e2e//ckpts \ +# --run-name \ +# > /apps/chcai/yambda_5b_e2e//.supervisor.console.log 2>&1 & # -# CHECKPOINTS / DISK -# The trainer saves atomically (write to .tmp, fsync, rename to ) and -# prunes to keep_last_n newest. One checkpoint is ~560 GB; a save blocks the -# step it fires on for ~83 s (measured, no NFS contention). Cadence is driven -# by --ckpt-time-interval (time-based) and optional --in-window-freq. +# # Adopt an already-submitted job instead of submitting a new one: +# nohup bash scripts/run_streaming_e2e.sh --jobid 13235 \ +# --submit-script .../launch_2node.sh --log .../run.log \ +# --ckpt-path .../ckpts --run-name > .../console.log 2>&1 & # -# ARGS (all optional; defaults target the full production run) -# run shape: --jobid --container --start-ts --num-train-ts --eval-every -# ckpt: --ckpt-path --keep-last-n --ckpt-time-interval --in-window-freq -# logging: --run-name --log -# resilience: --max-relaunch --min-free-gib --stall-s -# failover: --partition --reservation --alloc-time --allow-failover -# --provision-script --acquire-wait-max --resv-wait-max -# --orig-recover-wait -# (failover holds <=1 reservation node: stray/leaked e2e_failover -# holds are reaped, and a lost ORIGINAL job is waited on for SLURM -# requeue and reused before a SEPARATE node is acquired.) -# validation: --num-train-batches --num-eval-batches (>0 caps batches/window -# for fast tests; 0 = full window / full-holdout eval) -# test-only: --die-at-step (>=0 injects a crash at that global step) +# The node count, partition, and reservation all live in the --submit-script's +# sbatch line (launch_1node.sh / launch_2node.sh / ...), not here. # # EXIT CODES -# 0 run completed (E2E_RUN_EXIT=0 — all windows + final eval done) -# 1 exhausted --max-relaunch without completing -# 3 disk guard tripped (insufficient free space) -# 4 could not secure a healthy allocation (failover failed / disabled) -# -# OUTPUTS (next to --log) -# trainer stdout/stderr + E2E_RUN_EXIT sentinels -# .supervisor.log this supervisor's own timeline -# .provision.log node-provisioning output (failover only) -# -# EXAMPLE -# nohup bash scripts/run_streaming_e2e.sh \ -# --jobid 12074 \ -# --ckpt-path /apps/chcai/ckpts/yambda_5b_e2e \ -# --run-name yambda_5b_e2e --log /apps/chcai/yambda_5b_e2e.log \ -# --start-ts 150 --num-train-ts 149 --eval-every 10 \ -# --ckpt-time-interval 3600 --keep-last-n 1 --max-relaunch 100 \ -# --reservation NAN_issue_debug \ -# > /apps/chcai/yambda_5b_e2e.supervisor.console.log 2>&1 & -# (--reservation makes node-death failover re-acquire from that reservation; -# omit it to fall back to the open $PARTITION pool.) +# 0 run completed (COMPLETED+0) or user-cancelled +# 1 exhausted --max-relaunch without completion (or submit failed) +# 3 disk guard tripped # ============================================================================= - set -uo pipefail -JOBID=11367 +JOBID="" # adopt this job; empty => submit fresh +SUBMIT_SCRIPT="" +LOG="" +CKPT_PATH="" +RUN_NAME="yambda_5b_e2e" CONTAINER=yambda_primus -REPO=/home/chcai/training/recommendation_v4 - -# Direct-SSH fallback so the supervisor can probe the node even while the SLURM -# control plane is unreachable — a transient controller outage must NOT be -# mistaken for node death (which would needlessly tear down a healthy run). -SSH_OPTS="-o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=no" -LAST_NODE="" # last known node hostname for $JOBID (cached for direct probes) - -# Defaults are sized from measurement: ~560 GB/checkpoint, ~83 s/save (blocking, -# attributed to the step it fires on), ~650 ms/train step @ global batch 8192, -# ~1465 steps (~16 min) per full ~12M-anchor window, full-holdout eval -# ~6-7 min/window. A ~2h time-based checkpoint interval keeps save overhead ~1% -# while bounding crash-loss to ~2h of compute; eval every N windows -# (EVAL_EVERY_N_WINDOWS) amortizes the full-holdout eval cost. -NUM_TRAIN_TS=149 -START_TS=150 -EVAL_EVERY=5 -CKPT_TIME_INTERVAL=7200 -KEEP_LAST_N=1 -CKPT_PATH=/apps/chcai/ckpts/yambda_5b_e2e -RUN_NAME=yambda_5b_e2e -LOG=/apps/chcai/yambda_5b_e2e.log MAX_RELAUNCH=50 -NUM_TRAIN_BATCHES=0 # 0 = full window (only capped for validation/tests) -NUM_EVAL_BATCHES=0 # 0 = full holdout eval (only capped for validation) -DIE_AT_STEP=-1 # >=0 = test-only failure injection -# Train:eval split (fraction of USERS trained; 1 - this held out as a FIXED, -# never-trained eval set). Passed on EVERY relaunch so the split stays an -# immutable run contract — a changed split would abort on resume (validated in -# the loop) to prevent skip-offset desync and held-out users leaking into train. -TRAIN_SPLIT_PERCENTAGE=0.90 -SPLIT_SALT=0 -EVAL_HOLDOUT_TS=-1 # <0 = window just past training (start_ts+num_train_ts) -EVAL_HOLDOUT_NUM_WINDOWS=1 -IN_WINDOW_FREQ=0 # >0 = also save every N batches within a window -ATTACH=0 # 1 = (re)attach to an already-running trainer without - # killing it or truncating its log — used to restore - # supervision over a trainer that outlived a previous - # supervisor (e.g. one a control-plane outage killed). -CTRL_WAIT_MAX=3600 # max seconds to wait for an unreachable SLURM controller - # to recover before concluding failover is needed. - -# --- node failover ---------------------------------------------------------- -# If the current allocation/node goes away, acquire a FRESH node, (re)provision -# the container on it, and resume — checkpoints + code live on shared NFS -# (/apps/chcai, /home/chcai), so any node in the partition can continue. -PARTITION=meta64 -RESERVATION="" # if set, failover acquires from this SLURM - # reservation (e.g. NAN_issue_debug) so a - # replacement node comes from the same pool. -ALLOC_TIME=7-00:00:00 # SLURM --time for a failover hold job -ALLOW_FAILOVER=1 # 0 = never acquire a new node -PROVISION_SCRIPT=/home/chcai/_provision_yambda_primus.sh -ACQUIRE_WAIT_MAX=1800 # max seconds to wait for the OPEN-POOL - # (tier-2) failover hold job to reach - # RUNNING (tolerates brief queueing). -RESV_WAIT_MAX=300 # max seconds to wait for a RESERVATION - # (tier-1) node before giving up on it and - # falling back to the open $PARTITION pool. - # Short, since a free reservation node - # starts ~immediately; a longer wait just - # means the reservation is currently full. -ORIG_RECOVER_WAIT=600 # when the user's ORIGINAL reservation job - # is lost, wait this long for SLURM to - # auto-requeue it back to RUNNING before - # acquiring a SEPARATE node. Reusing the - # requeued original keeps us at <=1 - # reservation node and skips a redundant - # acquire (observed requeue latency ~2 min). - -# Disk guard: require at least this many GiB free on the ckpt volume before a -# (re)launch. One checkpoint is ~560 GB. A save writes a fresh .tmp BEFORE the -# old copy is pruned, so peak transient usage is (keep_last_n + 1) copies. With -# keep_last_n=1 that is ~1120 GB; require ~1200 GiB free at launch so the run -# never wedges mid-save on a near-full shared NFS volume. MIN_FREE_GIB=1200 -# Stall watchdog: if the log hasn't grown AND no trainer process is alive for -# this many seconds with no exit sentinel, treat it as a silent death. Comfortably -# exceeds one blocking checkpoint save (~83 s); and because a save keeps the -# trainer process alive, an in-progress save never trips the watchdog anyway. -STALL_S=1200 +STALL_S=2400 # 40 min: comfortably exceeds a full-holdout eval + # window + a blocking ckpt save; only trips when + # the log is frozen AND no trainer proc is alive. POLL_S=30 while [[ $# -gt 0 ]]; do case $1 in --jobid) JOBID="$2"; shift 2;; - --container) CONTAINER="$2"; shift 2;; - --num-train-ts) NUM_TRAIN_TS="$2"; shift 2;; - --start-ts) START_TS="$2"; shift 2;; - --eval-every) EVAL_EVERY="$2"; shift 2;; - --ckpt-time-interval) CKPT_TIME_INTERVAL="$2"; shift 2;; - --keep-last-n) KEEP_LAST_N="$2"; shift 2;; + --submit-script) SUBMIT_SCRIPT="$2"; shift 2;; + --log) LOG="$2"; shift 2;; --ckpt-path) CKPT_PATH="$2"; shift 2;; --run-name) RUN_NAME="$2"; shift 2;; - --log) LOG="$2"; shift 2;; + --container) CONTAINER="$2"; shift 2;; --max-relaunch) MAX_RELAUNCH="$2"; shift 2;; - --num-train-batches) NUM_TRAIN_BATCHES="$2"; shift 2;; - --num-eval-batches) NUM_EVAL_BATCHES="$2"; shift 2;; - --die-at-step) DIE_AT_STEP="$2"; shift 2;; - --in-window-freq) IN_WINDOW_FREQ="$2"; shift 2;; - --attach) ATTACH="$2"; shift 2;; - --ctrl-wait-max) CTRL_WAIT_MAX="$2"; shift 2;; --min-free-gib) MIN_FREE_GIB="$2"; shift 2;; --stall-s) STALL_S="$2"; shift 2;; - --partition) PARTITION="$2"; shift 2;; - --reservation) RESERVATION="$2"; shift 2;; - --alloc-time) ALLOC_TIME="$2"; shift 2;; - --allow-failover) ALLOW_FAILOVER="$2"; shift 2;; - --provision-script) PROVISION_SCRIPT="$2"; shift 2;; - --acquire-wait-max) ACQUIRE_WAIT_MAX="$2"; shift 2;; - --resv-wait-max) RESV_WAIT_MAX="$2"; shift 2;; - --orig-recover-wait) ORIG_RECOVER_WAIT="$2"; shift 2;; + --poll-s) POLL_S="$2"; shift 2;; *) echo "Unknown arg: $1"; exit 1;; esac done -ORIGINAL_JOBID="$JOBID" # never scancel the user's own hold allocation -ACQUIRED_JOBIDS=() # failover allocations WE created (released on success) +[[ -n "$SUBMIT_SCRIPT" && -f "$SUBMIT_SCRIPT" ]] || { echo "FATAL: --submit-script required and must exist"; exit 1; } +[[ -n "$LOG" ]] || { echo "FATAL: --log required"; exit 1; } SUP_LOG="${LOG%.log}.supervisor.log" - sup() { echo "[$(date '+%F %T')] [supervisor] $*" | tee -a "$SUP_LOG"; } -# Run a command inside the allocation's container, capturing its stdout. Wrapped -# in `timeout` so a hung control plane / NFS can never wedge the supervisor. -cexec() { timeout 90 srun --jobid="$JOBID" --overlap docker exec "$CONTAINER" bash -lc "$1" 2>/dev/null; } - -# Is the SLURM control plane reachable right now? -controller_up() { timeout 12 sinfo -h -o '%P' >/dev/null 2>&1; } - -# Refresh + echo the node hostname for $JOBID (cached in LAST_NODE for direct -# probes that must work even while the controller is down). -refresh_node() { - local n; n=$(timeout 12 squeue -h -j "$JOBID" -o '%N' 2>/dev/null | head -1) - [[ -n "$n" ]] && LAST_NODE="$n" - echo "$LAST_NODE" -} - -# Run a (simple) command in the container by SSHing the node DIRECTLY, bypassing -# SLURM — the only way to observe the trainer during a controller outage. Needs a -# previously-cached LAST_NODE. Keep "$1" free of embedded double quotes. -dexec() { - [[ -z "$LAST_NODE" ]] && return 1 - timeout 40 ssh $SSH_OPTS "$LAST_NODE" "docker exec $CONTAINER bash -lc '$1'" 2>/dev/null -} - -# Block (with backoff) until the controller is reachable again, up to -# CTRL_WAIT_MAX. A controller outage leaves RUNNING jobs running, so waiting it -# out is almost always preferable to abandoning a healthy node. -wait_for_controller() { - local waited=0 - controller_up && return 0 - while ! controller_up; do - if (( waited >= CTRL_WAIT_MAX )); then - sup "controller still unreachable after ${waited}s (max ${CTRL_WAIT_MAX}s) — proceeding." - return 1 - fi - sup "SLURM controller unreachable; waiting for recovery (${waited}s/${CTRL_WAIT_MAX}s)…" - sleep 30; waited=$((waited + 30)) - done - sup "SLURM controller reachable again after ${waited}s." - return 0 -} - -cleanup_workers() { - # The trainer spawns 8 rank processes + dataloader workers whose cmdlines - # don't all match `train_ranker`/`spawn_main`, so target them, then fall - # back to `pkill python` — safe because this container is dedicated to this - # training (only the trainer runs python here during a supervised run). - srun --jobid="$JOBID" --overlap docker exec "$CONTAINER" bash -lc \ - "pkill -9 -f train_ranker 2>/dev/null; pkill -9 -f multiprocessing 2>/dev/null; \ - sleep 2; pkill -9 python 2>/dev/null; sleep 3; true" 2>/dev/null || true -} - -# --- node-failover helpers --------------------------------------------------- - -# Healthy = the job is RUNNING and its node is not down/drained/failing. -alloc_healthy() { - local jid="$1" - [[ -z "$jid" ]] && return 1 - local st node nstate - st=$(squeue -h -j "$jid" -o '%T' 2>/dev/null | head -1) - [[ "$st" != "RUNNING" ]] && return 1 - node=$(squeue -h -j "$jid" -o '%N' 2>/dev/null | head -1) - [[ -z "$node" ]] && return 1 - nstate=$(sinfo -h -n "$node" -o '%t' 2>/dev/null | head -1) - case "$nstate" in - *down*|*drain*|*fail*|*unk*|*boot*|"") return 1;; - esac - return 0 -} - -# Can we actually exec in the training container on this allocation? -container_up() { - timeout 30 srun --jobid="$1" --overlap docker exec "$CONTAINER" true >/dev/null 2>&1 -} - -# (Re)create + dep-install the container on the given allocation's node. -provision_node() { - local jid="$1" node - node=$(squeue -h -j "$jid" -o '%N' 2>/dev/null | head -1) - sup "provisioning container '$CONTAINER' on job $jid (node ${node:-?}) — cold node can take ~15 min" - srun --jobid="$jid" --overlap bash "$PROVISION_SCRIPT" >> "${LOG%.log}.provision.log" 2>&1 - container_up "$jid" -} - -# Submit an sbatch hold job that merely pins one exclusive node (`sleep -# infinity`, bounded by --time=$ALLOC_TIME); echoes the jobid. $1 = extra sbatch -# args (e.g. "--reservation=NAN_issue_debug" or ""). sbatch (not salloc) because -# interactive salloc on some partitions (meta64) is capped at 240 min, which an -# $ALLOC_TIME multi-day hold exceeds. The container is provisioned afterward by -# provision_node via `srun --jobid --overlap`. -_submit_hold_job() { - local extra="$1" out - out=$(sbatch --parsable --partition="$PARTITION" $extra --nodes=1 --exclusive \ - --time="$ALLOC_TIME" --job-name=e2e_failover \ - --output="${LOG%.log}.failover_hold.%j.log" \ - --wrap="echo \"[failover-hold] node=\$(hostname) jobid=\$SLURM_JOB_ID start=\$(date -Is)\"; sleep infinity" 2>&1) - # --parsable => "" or ";"; strip whitespace + cluster. - echo "$out" | tr -d ' ' | cut -d';' -f1 -} - -# Wait up to $2 seconds for job $1 to reach RUNNING. Returns 0 if RUNNING. -_wait_running() { - local jid="$1" max="$2" waited=0 st - while (( waited < max )); do - st=$(squeue -h -j "$jid" -o '%T' 2>/dev/null | head -1) - [[ "$st" == "RUNNING" ]] && return 0 - sleep 10; waited=$((waited + 10)) +# Is the job in the queue right now (single read)? +job_in_queue() { [[ -n "$(squeue -h -j "$1" -o '%T' 2>/dev/null | head -1)" ]]; } +job_state() { squeue -h -j "$1" -o '%T' 2>/dev/null | head -1; } + +# Is the job still active? squeue/the SLURM control plane can transiently return +# empty during an NFS/controller blip even though the job is alive (this once +# killed all supervisors at once: empty squeue -> sacct said RUNNING -> a bogus +# "relaunch"). So a SINGLE empty read is not trusted: re-check a few times before +# believing the job is really gone. +job_active() { + job_in_queue "$1" && return 0 + local k + for k in 1 2 3; do + sleep 10 + job_in_queue "$1" && return 0 done return 1 } -# Acquire a fresh exclusive node and set global JOBID on success. Two-tier: -# tier 1 (preferred): the SLURM --reservation $RESERVATION, if configured. -# Waited on for only RESV_WAIT_MAX — a free reservation node starts almost -# immediately, so a longer wait means the reservation is currently full. -# tier 2 (fallback): the open $PARTITION pool (no reservation), waited on for -# ACQUIRE_WAIT_MAX. Used when no reservation is set, or the reservation had -# no node free within RESV_WAIT_MAX (the pending reservation job is -# cancelled before we resubmit so we never end up holding two nodes). -acquire_node() { - if [[ "$ALLOW_FAILOVER" != "1" ]]; then - sup "failover disabled (--allow-failover 0); cannot acquire a new node"; return 1 - fi - # Release any prior/leaked failover hold BEFORE grabbing a new one, so we - # never transiently pin two reservation nodes (e.g. a dead tier-1 hold + the - # replacement we are about to submit). - reap_failover_holds "" - local jid - - # --- tier 1: reservation (preferred) ------------------------------------- - if [[ -n "$RESERVATION" ]]; then - sup "failover tier-1: requesting a node from reservation=$RESERVATION (exclusive, time=$ALLOC_TIME)" - jid=$(_submit_hold_job "--reservation=$RESERVATION") - if [[ "$jid" =~ ^[0-9]+$ ]]; then - ACQUIRED_JOBIDS+=("$jid") # track for cleanup even if it never starts - sup "reservation hold job jobid=$jid submitted; waiting up to ${RESV_WAIT_MAX}s for RUNNING" - if _wait_running "$jid" "$RESV_WAIT_MAX"; then - JOBID="$jid" - sup "new node ready (reservation $RESERVATION): jobid=$JOBID node=$(squeue -h -j "$JOBID" -o '%N' 2>/dev/null | head -1)" - return 0 - fi - sup "reservation $RESERVATION has no free node within ${RESV_WAIT_MAX}s — cancelling pending $jid and falling back to open pool" - scancel "$jid" 2>/dev/null || true - else - sup "reservation sbatch did not return a jobid ($jid) — falling back to open pool" - fi - fi - - # --- tier 2: open partition pool (fallback) ------------------------------ - sup "failover tier-2: requesting a node from open partition=$PARTITION (exclusive, time=$ALLOC_TIME)" - jid=$(_submit_hold_job "") - if ! [[ "$jid" =~ ^[0-9]+$ ]]; then - sup "FATAL: open-pool sbatch did not return a jobid: $jid"; return 1 - fi - ACQUIRED_JOBIDS+=("$jid") - sup "open-pool hold job jobid=$jid submitted; waiting up to ${ACQUIRE_WAIT_MAX}s for RUNNING" - if _wait_running "$jid" "$ACQUIRE_WAIT_MAX"; then - JOBID="$jid" - sup "new node ready (open $PARTITION): jobid=$JOBID node=$(squeue -h -j "$JOBID" -o '%N' 2>/dev/null | head -1)" - return 0 - fi - sup "FATAL: open-pool hold job $jid never reached RUNNING (waited ${ACQUIRE_WAIT_MAX}s)" - return 1 -} +# Terminal State + ExitCode from accounting once the job has left the queue. +job_final() { sacct -j "$1" -X -n -o State,ExitCode 2>/dev/null | head -1 | tr -s ' '; } -# Ensure $JOBID is a healthy allocation with the container up, failing over to a -# fresh provisioned node if not. Resume is automatic: the latest checkpoint is -# on shared NFS, reachable from whatever node we end up on. -ensure_ready() { - # A controller outage leaves RUNNING jobs running; wait it out before deciding - # anything is wrong, so we never abandon a healthy node over a transient blip. - wait_for_controller || true - if alloc_healthy "$JOBID"; then - refresh_node >/dev/null - if container_up "$JOBID"; then return 0; fi - sup "alloc $JOBID healthy but container '$CONTAINER' not up — (re)provisioning" - provision_node "$JOBID" && return 0 - sup "provisioning on $JOBID failed; will try a fresh node" - else - sup "current allocation $JOBID unavailable (job not RUNNING or node down/drained)" - # Prefer the SLURM-requeued original over acquiring a SEPARATE node, so we - # stay at <=1 reservation node. (No-op once we've already failed over off - # the original.) - if wait_for_original_recover; then - JOBID="$ORIGINAL_JOBID" - refresh_node >/dev/null - if container_up "$JOBID"; then sup "reusing recovered original jobid=$JOBID"; return 0; fi - sup "recovered original $JOBID up but container '$CONTAINER' not present — (re)provisioning" - provision_node "$JOBID" && return 0 - sup "provisioning recovered original $JOBID failed; will acquire a fresh node" - fi - fi - acquire_node || return 1 - provision_node "$JOBID" || { sup "provisioning new node $JOBID failed"; return 1; } - sup "failover complete — now running on jobid=$JOBID" - return 0 -} - -release_acquired() { - local jid - for jid in "${ACQUIRED_JOBIDS[@]:-}"; do - [[ -n "$jid" && "$jid" != "$ORIGINAL_JOBID" ]] || continue - # docker is independent of SLURM, so remove the container before freeing - # the node, otherwise it lingers for the next tenant. - srun --jobid="$jid" --overlap docker rm -f "$CONTAINER" >/dev/null 2>&1 || true - scancel "$jid" 2>/dev/null && sup "released failover allocation $jid (container removed)" - done -} - -# Enforce "at most ONE reservation node held by this run at a time" and reap -# orphans. Every node WE acquire is an `sbatch --job-name=e2e_failover` hold, so -# all our holds are discoverable by name even across a supervisor restart — which -# is how a previous supervisor that died mid-failover (e.g. on a provisioning -# error) can leave a hold pinning a second reservation node. Cancels every -# e2e_failover hold owned by us EXCEPT $1 (the one to keep) and the user's -# ORIGINAL_JOBID (never ours to cancel). Containers are removed before the node -# is freed so they don't linger for the next tenant. -reap_failover_holds() { - local keep="${1:-}" me jid - me=$(id -un 2>/dev/null) - [[ -z "$me" ]] && return 0 - while read -r jid; do - [[ -z "$jid" ]] && continue - [[ "$jid" == "$keep" || "$jid" == "$ORIGINAL_JOBID" ]] && continue - sup "reaping stray failover hold $jid (enforcing <=1 reservation node held by this run)" - srun --jobid="$jid" --overlap docker rm -f "$CONTAINER" >/dev/null 2>&1 || true - scancel "$jid" 2>/dev/null || true - done < <(squeue -h -u "$me" -n e2e_failover -o '%i' 2>/dev/null) -} - -# When the user's ORIGINAL reservation job is lost, SLURM typically auto-requeues -# it back onto a (fresh) reservation node within a couple of minutes. Waiting for -# that and REUSING it — rather than immediately acquiring a SEPARATE node — is -# what keeps us at <=1 reservation node (the alternative is the original requeue -# AND a failover hold both pinning reservation nodes) and skips a redundant -# acquire+provision. Only meaningful while we are still on the original job. -wait_for_original_recover() { - [[ "$JOBID" != "$ORIGINAL_JOBID" ]] && return 1 - local waited=0 - while (( waited < ORIG_RECOVER_WAIT )); do - if alloc_healthy "$ORIGINAL_JOBID"; then - sup "original job $ORIGINAL_JOBID is RUNNING again (SLURM requeue) after ${waited}s — reusing it (no second node)" - return 0 - fi - sup "waiting for original job $ORIGINAL_JOBID to requeue before acquiring a separate node (${waited}s/${ORIG_RECOVER_WAIT}s)…" - sleep 15; waited=$((waited + 15)) - done - sup "original job $ORIGINAL_JOBID did not recover within ${ORIG_RECOVER_WAIT}s — acquiring a fresh node" - return 1 +# sacct/SLURM states that mean the job is STILL ALIVE (not terminal). If we see +# one of these after the monitor loop exits, squeue lied (transient) — resume +# monitoring instead of relaunching (which could spawn a DUPLICATE job). +is_active_state() { + case "$1" in + RUNNING|PENDING|CONFIGURING|COMPLETING|REQUEUED|RESIZING|SUSPENDED|REQUEUE_HOLD|REQUEUE_FED|SIGNALING|STAGE_OUT) return 0;; + *) return 1;; + esac } -# Returns 0 (true) if a trainer process is alive in the container. Uses SLURM -# (srun) when the controller is up, else falls back to a direct SSH probe so a -# control-plane outage can't make a live trainer look dead. +# Any trainer process alive on ANY node of the allocation? (cross-node pgrep via +# overlap srun into each node's container). [g]enerative self-match guard avoids +# pgrep matching its own command line. trainer_alive() { - local n - # `set -f; pgrep -f [g]enerative...` is the classic self-match guard: the - # probe shell's OWN cmdline contains the pattern, so a naive `pgrep -f - # generative_recommenders` ALWAYS matches itself and returns >=1 even when - # the trainer is dead — which would defeat the stall watchdog and make - # ATTACH mode falsely "adopt" a nonexistent trainer. The [g] char-class - # matches "generative" in real trainer cmdlines but NOT the literal - # "[g]enerative" in the probe's cmdline; `set -f` keeps the bracket from - # being glob-expanded (works under both bash -lc wrappers, no quotes). - if controller_up; then - n=$(cexec "set -f; pgrep -f [g]enerative_recommenders | wc -l" | tr -d ' ') - else - n=$(dexec "set -f; pgrep -f [g]enerative_recommenders | wc -l" | tr -d ' ') - fi + local jid="$1" n + n=$(timeout 70 srun --jobid="$jid" --overlap --ntasks-per-node=1 bash -c \ + "docker exec $CONTAINER bash -lc 'set -f; pgrep -f [g]enerative_recommenders | wc -l' 2>/dev/null" 2>/dev/null \ + | awk '{s+=$1} END{print s+0}') [[ "${n:-0}" -gt 0 ]] } +# Free GiB on the ckpt volume (NFS is mounted on this head node, so df locally). +disk_free_gib() { + df -BG --output=avail "$CKPT_PATH" 2>/dev/null | tail -1 | tr -dc '0-9' +} + disk_guard() { - # Sweep crash-orphaned partial saves, then check free space. - cexec "for d in '$CKPT_PATH'/*.tmp '$CKPT_PATH'/*.old; do [ -e \"\$d\" ] && rm -rf \"\$d\" && echo swept \"\$d\"; done; true" - local free_gib - free_gib=$(cexec "df -BG --output=avail '$CKPT_PATH' 2>/dev/null | tail -1 | tr -dc '0-9'") - free_gib=${free_gib:-0} - sup "disk guard: ${free_gib} GiB free on $CKPT_PATH (min ${MIN_FREE_GIB})" - if (( free_gib < MIN_FREE_GIB )); then - sup "FATAL: insufficient free space (${free_gib} < ${MIN_FREE_GIB} GiB). Aborting." + [[ -z "$CKPT_PATH" ]] && return 0 + local free; free=$(disk_free_gib); free=${free:-0} + sup "disk guard: ${free} GiB free on $CKPT_PATH (min ${MIN_FREE_GIB})" + if (( free < MIN_FREE_GIB )); then + sup "FATAL: insufficient free space (${free} < ${MIN_FREE_GIB} GiB). Aborting." return 1 fi return 0 } -launch() { - # Detached launch. The trailing echo appends a definitive exit sentinel to - # the log once the trainer returns (clean finish OR crash with nonzero rc). - srun --jobid="$JOBID" --overlap docker exec -d "$CONTAINER" bash -lc " - cd $REPO && - HSTU_HAMMER_KERNEL=TRITON \ - MODE=streaming-train-eval \ - START_TS=$START_TS \ - NUM_TRAIN_TS=$NUM_TRAIN_TS \ - EVAL_EACH_WINDOW=1 \ - EVAL_EVERY_N_WINDOWS=$EVAL_EVERY \ - CKPT_PATH=$CKPT_PATH \ - KEEP_LAST_N=$KEEP_LAST_N \ - CKPT_TIME_INTERVAL_S=$CKPT_TIME_INTERVAL \ - IN_WINDOW_CKPT_FREQ=$IN_WINDOW_FREQ \ - NUM_TRAIN_BATCHES=$NUM_TRAIN_BATCHES \ - NUM_EVAL_BATCHES=$NUM_EVAL_BATCHES \ - DIE_AT_STEP=$DIE_AT_STEP \ - TRAIN_SPLIT_PERCENTAGE=$TRAIN_SPLIT_PERCENTAGE \ - SPLIT_SALT=$SPLIT_SALT \ - EVAL_HOLDOUT_TS=$EVAL_HOLDOUT_TS \ - EVAL_HOLDOUT_NUM_WINDOWS=$EVAL_HOLDOUT_NUM_WINDOWS \ - METRIC_LOG_FREQ=50 \ - RUN_NAME=$RUN_NAME \ - TENSORBOARD_LOG_PATH=/apps/chcai/tb/$RUN_NAME/ \ - LOG=$LOG \ - bash scripts/launch_slurm.sh; - echo \"E2E_RUN_EXIT=\$? \$(date '+%F %T')\" >> $LOG - " +# Resubmit the run; resumes from latest checkpoint. APPEND_LOG=1 preserves the +# metrics log. Echoes the new jobid. +resubmit() { + local out newjid + out=$(APPEND_LOG=1 bash "$SUBMIT_SCRIPT" 2>&1) + newjid=$(echo "$out" | grep -oE 'Submitted batch job [0-9]+' | grep -oE '[0-9]+' | head -1) + echo "$out" | sed 's/^/ /' >> "$SUP_LOG" + echo "$newjid" } -# Returns the exit code from the most recent E2E_RUN_EXIT sentinel APPENDED -# since `since_marker` bytes, or empty if none yet. -last_exit_since() { - local since_line="$1" - cexec "tail -n +$since_line '$LOG' 2>/dev/null | grep -aoE 'E2E_RUN_EXIT=[0-9]+' | tail -1 | cut -d= -f2" +# Submit with retries+backoff. A transient NFS / control-plane error (e.g. +# "sbatch: error: ... I/O error writing script/environment to file") must NOT +# kill the supervisor — it leaves runs unsupervised / unlaunched. Echoes a jobid +# on success, or empty after all retries. +submit_retry() { + local cand sub_try + for sub_try in $(seq 1 12); do + cand=$(resubmit) + if [[ "$cand" =~ ^[0-9]+$ ]]; then echo "$cand"; return 0; fi + sup "submit attempt $sub_try/12 failed (transient sbatch/NFS error) — backing off." + sleep $(( sub_try < 5 ? 30 : 120 )) + done + return 1 } sup "=== streaming e2e supervisor start ===" -sup "jobid=$JOBID container=$CONTAINER repo=$REPO" -sup "start_ts=$START_TS num_train_ts=$NUM_TRAIN_TS eval_every=$EVAL_EVERY" -sup "ckpt_path=$CKPT_PATH keep_last_n=$KEEP_LAST_N ckpt_time_interval=${CKPT_TIME_INTERVAL}s in_window_freq=$IN_WINDOW_FREQ" -sup "log=$LOG num_train_batches=$NUM_TRAIN_BATCHES die_at_step=$DIE_AT_STEP max_relaunch=$MAX_RELAUNCH" -sup "failover: allow=$ALLOW_FAILOVER partition=$PARTITION reservation=${RESERVATION:-} alloc_time=$ALLOC_TIME" - -# Reap any failover hold(s) leaked by a PREVIOUS supervisor that died mid-failover -# (e.g. exited on a provisioning error before release_acquired could run). Without -# this, such an orphan keeps pinning a second reservation node indefinitely. -reap_failover_holds "" - -cexec "mkdir -p '$CKPT_PATH' '/apps/chcai/tb/$RUN_NAME'" -# Initialize this run's metrics log ONCE. launch_slurm.sh (worker) appends (tee -a), -# so every relaunch attempt accumulates into this single file — the full-run -# NE/AUC history survives crashes and node failover instead of being truncated -# on each relaunch. (Starting the supervisor = starting a fresh run.) In ATTACH -# mode we are adopting an already-running trainer, so we KEEP its existing log. -if [[ "$ATTACH" == "1" ]]; then - sup "ATTACH mode: adopting existing run — keeping metrics log intact: $LOG" -else - cexec ": > '$LOG'" - sup "metrics log initialized (relaunch-append): $LOG" -fi -sup "tensorboard (NFS): /apps/chcai/tb/$RUN_NAME/" +sup "run=$RUN_NAME submit=$SUBMIT_SCRIPT log=$LOG ckpt=$CKPT_PATH" +sup "max_relaunch=$MAX_RELAUNCH min_free_gib=$MIN_FREE_GIB stall_s=$STALL_S poll_s=$POLL_S" attempt=0 -while (( attempt < MAX_RELAUNCH )); do - attempt=$((attempt + 1)) - sup "--- attempt $attempt/$MAX_RELAUNCH ---" - - # Make sure we have a live, container-ready node (failover + provision if the - # current allocation/node has gone away). - if ! ensure_ready; then - sup "FATAL: could not secure a healthy allocation (failover failed)." - exit 4 - fi - refresh_node >/dev/null # cache LAST_NODE for direct probes during outages - - # ATTACH (first attempt only): if a trainer is already running for this run, - # adopt it in place — DON'T disk-guard (its sweep would delete an in-flight - # .tmp save), DON'T cleanup_workers (would kill it), DON'T launch. Just begin - # monitoring. Any subsequent relaunch is a normal launch from the checkpoint. - adopt=0 - if [[ "$ATTACH" == "1" ]] && trainer_alive; then - adopt=1; ATTACH=0 - sup "ATTACH mode: trainer already alive on ${LAST_NODE:-node} — monitoring in place (no relaunch/kill/sweep)." - fi - - if (( adopt )); then - # Mark current end of log so we only read sentinels produced from here on. - start_line=$(cexec "wc -l < '$LOG' 2>/dev/null" | tr -d ' '); start_line=${start_line:-0} - start_line=$((start_line + 1)) - sup "monitoring adopted run (reading sentinels from log line $start_line)" - else - if ! disk_guard; then exit 3; fi - cleanup_workers - # Mark current end of log so we only read sentinels produced by THIS attempt. - start_line=$(cexec "wc -l < '$LOG' 2>/dev/null" | tr -d ' '); start_line=${start_line:-0} - start_line=$((start_line + 1)) - sup "launching (reading sentinels from log line $start_line)" - launch - sleep 15 # let docker exec spin up the process - fi - - # Monitor loop. - last_size=0 - stall_accum=0 - hb=0 - while true; do - # Node/allocation watchdog: if the node we're on goes down/drains or the - # job ends, bail out of the monitor — the next attempt's ensure_ready - # will fail over to a fresh node and resume from the latest checkpoint. - hb=$((hb + 1)) - if (( hb % 4 == 0 )) && ! alloc_healthy "$JOBID"; then - if ! controller_up; then - # Control plane unreachable != node down. If the trainer is still - # alive on the node (direct SSH probe), this is a transient blip — - # keep monitoring rather than tearing down a healthy run. - if trainer_alive; then - sup "control plane unreachable but trainer still alive on ${LAST_NODE:-node} — transient; continuing to monitor." - else - sup "control plane unreachable AND trainer absent on ${LAST_NODE:-node} — relaunching with failover." - break +if [[ -z "$JOBID" ]]; then + if ! disk_guard; then exit 3; fi + sup "no --jobid given; submitting a fresh job" + JOBID=$(submit_retry) + [[ "$JOBID" =~ ^[0-9]+$ ]] || { sup "FATAL: could not submit after 12 retries — aborting."; exit 1; } +fi +attempt=1 +sup "supervising jobid=$JOBID (attempt $attempt/$MAX_RELAUNCH)" + +while (( attempt <= MAX_RELAUNCH )); do + # --- wait for the job to be schedulable / running --- + wait_pend=0 + while job_active "$JOBID" && [[ "$(job_state "$JOBID")" != "RUNNING" ]]; do + (( wait_pend % 10 == 0 )) && sup "job $JOBID state=$(job_state "$JOBID") — waiting to run…" + sleep "$POLL_S"; wait_pend=$((wait_pend + 1)) + done + [[ "$(job_state "$JOBID")" == "RUNNING" ]] && sup "job $JOBID RUNNING on $(squeue -h -j "$JOBID" -o '%N' 2>/dev/null | head -1)" + + # --- monitor loop --- + last_size=0; stall_accum=0; hb=0; self_cancelled=0 + while job_active "$JOBID"; do + st=$(job_state "$JOBID") + if [[ "$st" == "RUNNING" ]]; then + cur_size=$(stat -c %s "$LOG" 2>/dev/null || echo 0) + if [[ "$cur_size" == "$last_size" ]]; then + # frozen log: only count as a stall if no trainer proc is alive + # (a long eval / blocking save keeps the process up -> not a stall) + hb=$((hb + 1)) + if (( hb % 4 == 0 )); then + if trainer_alive "$JOBID"; then + stall_accum=0 + else + stall_accum=$((stall_accum + POLL_S * 4)) + sup "log frozen + no trainer alive (${stall_accum}s/${STALL_S}s)" + if (( stall_accum >= STALL_S )); then + sup "STALL: hung run — scancel $JOBID and relaunch." + self_cancelled=1 + scancel "$JOBID" 2>/dev/null || true + sleep 20 + break + fi + fi fi else - sup "allocation $JOBID lost mid-run (node down/job ended) — relaunching with failover." - break + stall_accum=0; last_size=$cur_size fi fi + sleep "$POLL_S" + done - rc=$(last_exit_since "$start_line") - if [[ -n "$rc" ]]; then - if [[ "$rc" == "0" ]]; then - sup "RUN COMPLETED CLEANLY (E2E_RUN_EXIT=0) on attempt $attempt." - cleanup_workers - final_ckpts=$(cexec "ls '$CKPT_PATH' 2>/dev/null | grep -E '^[0-9]+$' | tr '\n' ' '") - sup "final checkpoints retained: ${final_ckpts:-}" - release_acquired - sup "=== streaming e2e supervisor done (success) ===" + # --- job has left the queue (or we scancel'd it): decide --- + sleep 5 + final=$(job_final "$JOBID") + state=$(echo "$final" | awk '{print $1}') + code=$(echo "$final" | awk '{print $2}') + sup "job $JOBID ended: state='${state:-?}' exit='${code:-?}'" + + # The monitor loop only exits when squeue has been empty across several + # confirming reads. If accounting STILL reports an active state, the job is + # actually alive (squeue/control-plane blip) — resume monitoring rather than + # relaunching, which would create a duplicate job. + if is_active_state "$state"; then + sup "sacct reports still-active state '$state' — transient squeue blip; resuming monitoring (NOT relaunching)." + sleep "$POLL_S" + continue + fi + + case "$state" in + COMPLETED) + if [[ "$code" == "0:0" ]]; then + sup "RUN COMPLETED CLEANLY on attempt $attempt." + sup "=== supervisor done (success) ===" exit 0 fi - sup "trainer exited nonzero (E2E_RUN_EXIT=$rc). Will relaunch from latest checkpoint." - break - fi - - # Stall watchdog: track log growth; if frozen and no trainer alive, die. - cur_size=$(cexec "wc -c < '$LOG' 2>/dev/null" | tr -d ' '); cur_size=${cur_size:-0} - if [[ "$cur_size" == "$last_size" ]]; then - if trainer_alive; then - stall_accum=0 # alive but quiet (e.g. long save / eval) — ok + sup "COMPLETED but nonzero exit ($code) — relaunching." + ;; + CANCELLED*) + if (( self_cancelled )); then + sup "job CANCELLED by our own stall recovery — relaunching from latest checkpoint." else - stall_accum=$((stall_accum + POLL_S)) - if (( stall_accum >= STALL_S )); then - sup "STALL: log frozen ${stall_accum}s and no trainer alive — silent death. Relaunching." - break - fi + sup "job CANCELLED (user/admin intent) — NOT resubmitting. Stopping supervisor." + sup "=== supervisor done (cancelled) ===" + exit 0 fi - else - stall_accum=0 - last_size=$cur_size - fi - sleep "$POLL_S" - done + ;; + FAILED|NODE_FAIL|TIMEOUT|OUT_OF_MEMORY|BOOT_FAIL|PREEMPTED|"") + sup "failure state '${state:-unknown}' — will relaunch from latest checkpoint." + ;; + *) + sup "unrecognized terminal state '${state}' — relaunching to be safe." + ;; + esac - cleanup_workers - sleep $(( attempt < 5 ? 20 : 60 )) # small backoff + if (( attempt >= MAX_RELAUNCH )); then break; fi + if ! disk_guard; then exit 3; fi + sleep $(( attempt < 5 ? 20 : 60 )) # small backoff + # Resubmit with retries. A transient NFS / control-plane error (e.g. + # "sbatch: error: Batch job submission failed: I/O error writing + # script/environment to file") must NOT kill the supervisor — that once + # left a live run permanently unsupervised. Retry with backoff first. + JOBID=$(submit_retry) + if ! [[ "$JOBID" =~ ^[0-9]+$ ]]; then + sup "FATAL: resubmit failed after 12 retries — aborting."; exit 1 + fi + attempt=$((attempt + 1)) + sup "relaunched as jobid=$JOBID (attempt $attempt/$MAX_RELAUNCH)" done sup "FATAL: exhausted MAX_RELAUNCH=$MAX_RELAUNCH without completion." -sup "=== streaming e2e supervisor done (failure) ===" +sup "=== supervisor done (failure) ===" exit 1 From be892d8209e020b7141e4c0fcdffdc06bc60147a Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Fri, 19 Jun 2026 00:22:01 +0000 Subject: [PATCH 070/127] dlrmv4: add non-SLURM local launcher + self-healing supervisor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit launch_local.sh: single-host, scheduler-free analog of launch_slurm.sh's worker phase — same train_ranker entry point / yambda_5b.gin config, smoke or full run on a GPU host with no SLURM/docker/RDMA overlay. run_streaming_e2e_local.sh: local analog of run_streaming_e2e.sh. Backgrounds a per-run submit script (so wait $PID yields the trainer's real exit code), relaunches from the latest checkpoint on crash/nonzero-exit/hang, with a hang watchdog (frozen log + no trainer proc) and a pre-launch disk guard. cleanup_container waits for GPU HBM to actually drain before relaunching so an OOM/crash can't cascade into a dirty-GPU OOM loop. --- recommendation_v4/scripts/launch_local.sh | 154 ++++++++++++++ .../scripts/run_streaming_e2e_local.sh | 200 ++++++++++++++++++ 2 files changed, 354 insertions(+) create mode 100755 recommendation_v4/scripts/launch_local.sh create mode 100755 recommendation_v4/scripts/run_streaming_e2e_local.sh diff --git a/recommendation_v4/scripts/launch_local.sh b/recommendation_v4/scripts/launch_local.sh new file mode 100755 index 000000000..9a69a36e5 --- /dev/null +++ b/recommendation_v4/scripts/launch_local.sh @@ -0,0 +1,154 @@ +#!/bin/bash +# ============================================================================= +# launch_local.sh — single-host, NON-SLURM launcher for the yambda-5b trainer. +# +# This is the SLURM-free analog of scripts/launch_slurm.sh's `worker` phase: +# it sets the single-node distributed topology + sane env and invokes the SAME +# entry point (`train_ranker.py --dataset yambda-5b`) reading the SAME +# train/gin/yambda_5b.gin config. No scheduler, no docker, no RDMA overlay — +# everything runs directly on this host against an already-prepared dataset. +# +# Use it to: +# * Smoke-test the launch path on a single GPU box (SMOKE=1, the default — +# a few train/eval batches of one streaming window), or +# * Run the full gin-default workload (SMOKE=0 — consumes whole windows). +# +# PREREQUISITES +# 1) Data prepared (run once, CPU-only — no GPU needed): +# python generative_recommenders/dlrm_v3/preprocess_public_data.py \ +# --dataset yambda-5b --data-path "$DLRM_DATA_PATH" +# producing $DLRM_DATA_PATH/processed_5b/{train_sessions.parquet,...} +# and $DLRM_DATA_PATH/shared_metadata/{artist,album}_item_mapping.parquet +# 2) The train_recipe GPU stack importable by $PYTHON (see docs/training_recipe.md): +# torch (rocm or cuda build), fbgemm_gpu, torchrec, polars-u64-idx, +# gin-config, xxhash, pandas, tensorboard, ... +# This box must have visible GPUs (the trainer shards embeddings onto HBM). +# +# USAGE +# # smoke (default): one window, 20 train + 10 eval batches +# DLRM_DATA_PATH=/home/chcai/dlrm_data bash scripts/launch_local.sh +# +# # full gin-default run (whole windows; long) +# SMOKE=0 DLRM_DATA_PATH=/home/chcai/dlrm_data bash scripts/launch_local.sh +# +# # restrict to 2 GPUs, custom log, plain (non-streaming) train-eval +# GPUS_PER_NODE=2 MODE=train-eval LOG=/tmp/y.log bash scripts/launch_local.sh +# +# Every knob below is env-overridable; defaults reproduce launch_slurm.sh's +# single-node smoke path so a local run matches the known-good cluster path. +# ============================================================================= +set -uo pipefail + +REPO_ROOT=$(cd "$(dirname "$0")/.." && pwd) +cd "$REPO_ROOT" + +# ---- interpreter ------------------------------------------------------------ +# Default to the venv created for data prep if present, else system python3. +# Override with PYTHON=/path/to/python (e.g. the in-container recipe python). +DEFAULT_PY=/home/chcai/dlrmv4_venv/bin/python +PYTHON=${PYTHON:-$([ -x "$DEFAULT_PY" ] && echo "$DEFAULT_PY" || echo python3)} + +# ---- dataset / data path ---------------------------------------------------- +DATASET=${DATASET:-yambda-5b} +MODE=${MODE:-streaming-train-eval} +# Mirrors the yambda_5b.gin default ("/apps/chcai/dlrm_data"); point at wherever +# preprocess_public_data.py wrote processed_5b/ + shared_metadata/. +export DLRM_DATA_PATH=${DLRM_DATA_PATH:-/home/chcai/dlrm_data} + +LOG=${LOG:-$REPO_ROOT/yambda_local.$(date +%Y%m%d_%H%M%S).log} + +# ---- single-node distributed topology -------------------------------------- +# train_ranker reads these from the env (see train_ranker.main): it spawns +# GPUS_PER_NODE ranks via torch.multiprocessing on THIS host. localhost +# rendezvous; empty MASTER_PORT => train_ranker picks a free port. +export NNODES=${NNODES:-1} +export NODE_RANK=${NODE_RANK:-0} +export MASTER_ADDR=${MASTER_ADDR:-localhost} +export MASTER_PORT=${MASTER_PORT:-} +# GPUS_PER_NODE: 0/unset => train_ranker auto-detects torch.cuda.device_count(). +export GPUS_PER_NODE=${GPUS_PER_NODE:-0} + +# ---- runtime env (matches launch_slurm.sh worker defaults) ------------------ +export HSTU_HAMMER_KERNEL=${HSTU_HAMMER_KERNEL:-TRITON} +export PYTORCH_CUDA_ALLOC_CONF=${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True} +export PYTHONPATH="$REPO_ROOT:${PYTHONPATH:-}" +# Single-node RCCL bootstrap: all ranks rendezvous over localhost, so pin the +# loopback NIC. Left to auto-detect, RCCL can grab a non-routable per-GPU RoCE +# NIC and hang/"No route to host" at init (same failure launch_slurm.sh pins +# fenic0 to avoid). Override NCCL_SOCKET_IFNAME for a routable multi-host setup. +export NCCL_SOCKET_IFNAME=${NCCL_SOCKET_IFNAME:-lo} +export NCCL_DEBUG=${NCCL_DEBUG:-WARN} + +# ---- smoke caps ------------------------------------------------------------- +# SMOKE=1 (default): apply small per-window batch caps so a launch finishes in +# minutes (validates the path end-to-end). SMOKE=0: leave the gin defaults +# untouched (consume full windows — the real workload). +SMOKE=${SMOKE:-1} +if [ "$SMOKE" = "1" ]; then + export START_TS=${START_TS:-150} + export NUM_TRAIN_TS=${NUM_TRAIN_TS:-1} + export NUM_TRAIN_BATCHES=${NUM_TRAIN_BATCHES:-20} + export NUM_EVAL_BATCHES=${NUM_EVAL_BATCHES:-10} + export EVAL_EVERY_N_WINDOWS=${EVAL_EVERY_N_WINDOWS:-1} + export METRIC_LOG_FREQ=${METRIC_LOG_FREQ:-5} + # Smaller per-sample shape keeps the smoke run light; drop these to use the + # gin defaults (4086/4096). Reuse an existing hstu_cache_L/ if present. + export BATCH_SIZE=${BATCH_SIZE:-32} +fi + +mkdir -p "$(dirname "$LOG")" +{ + echo "[$(date)] launch_local: dataset=$DATASET mode=$MODE smoke=$SMOKE" + echo "[$(date)] PYTHON=$PYTHON" + echo "[$(date)] DLRM_DATA_PATH=$DLRM_DATA_PATH" + echo "[$(date)] topology: nnodes=$NNODES node_rank=$NODE_RANK gpus_per_node(req)=$GPUS_PER_NODE master=$MASTER_ADDR:${MASTER_PORT:-}" +} | tee -a "$LOG" + +# ---- preflight: data present? ---------------------------------------------- +SUFFIX=${DATASET#yambda-} +PROCESSED="$DLRM_DATA_PATH/processed_${SUFFIX}/train_sessions.parquet" +META="$DLRM_DATA_PATH/shared_metadata/artist_item_mapping.parquet" +if [ "$DATASET" = "yambda-5b" ] && { [ ! -f "$PROCESSED" ] || [ ! -f "$META" ]; }; then + echo "[$(date)] ERROR: prepared data not found." | tee -a "$LOG" + echo " expected: $PROCESSED" | tee -a "$LOG" + echo " and: $META" | tee -a "$LOG" + echo " run preprocessing first:" | tee -a "$LOG" + echo " $PYTHON generative_recommenders/dlrm_v3/preprocess_public_data.py --dataset $DATASET --data-path $DLRM_DATA_PATH" | tee -a "$LOG" + exit 1 +fi + +# ---- preflight: GPU stack importable + GPUs visible? ------------------------ +echo "[$(date)] preflight: checking torch / fbgemm_gpu / torchrec + GPU count" | tee -a "$LOG" +"$PYTHON" - <<'PY' 2>&1 | tee -a "$LOG" +import sys +missing = [] +for m in ("torch", "fbgemm_gpu", "torchrec", "polars", "gin", "xxhash"): + try: + __import__(m) + except Exception as e: + missing.append(f"{m} ({e.__class__.__name__})") +if missing: + print("PREFLIGHT FAIL: missing/broken imports: " + ", ".join(missing)) + print("Install the train_recipe GPU stack (see docs/training_recipe.md).") + sys.exit(3) +import torch +n = torch.cuda.device_count() +print(f"imports OK, torch {torch.__version__}, cuda/hip available={torch.cuda.is_available()}, {n} GPU(s)") +if n == 0: + print("PREFLIGHT FAIL: no GPUs visible — the HSTU trainer shards embeddings " + "onto GPU HBM and cannot run CPU-only. Launch on a GPU host.") + sys.exit(4) +PY +pf=${PIPESTATUS[0]} +if [ "$pf" -ne 0 ]; then + echo "[$(date)] preflight failed (rc=$pf) — not launching trainer." | tee -a "$LOG" + exit "$pf" +fi + +# ---- launch ----------------------------------------------------------------- +echo "[$(date)] launching train_ranker ($DATASET, mode=$MODE)" | tee -a "$LOG" +"$PYTHON" -m generative_recommenders.dlrm_v3.train.train_ranker \ + --dataset "$DATASET" --mode "$MODE" 2>&1 | tee -a "$LOG" +rc=${PIPESTATUS[0]} +echo "[$(date)] launch_local finished rc=$rc" | tee -a "$LOG" +exit "$rc" diff --git a/recommendation_v4/scripts/run_streaming_e2e_local.sh b/recommendation_v4/scripts/run_streaming_e2e_local.sh new file mode 100755 index 000000000..85a817c8c --- /dev/null +++ b/recommendation_v4/scripts/run_streaming_e2e_local.sh @@ -0,0 +1,200 @@ +#!/bin/bash +# ============================================================================= +# run_streaming_e2e_local.sh — self-healing supervisor for a SINGLE-HOST +# (NON-SLURM) yambda-5b streaming train+eval run. Local analog of +# scripts/run_streaming_e2e.sh (the SLURM/sbatch supervisor). +# +# WHAT IT SUPERVISES +# The "job" is one foreground run of --submit-script (default +# scripts/launch_e2e_local.sh), which `docker exec`s the trainer in the +# container. The supervisor runs that submit-script in the BACKGROUND so: +# * its host PID == liveness (kill -0 / hang watchdog), and +# * `wait $PID` == the trainer's EXIT CODE (success vs. failure). +# On crash / nonzero-exit / hang it RELAUNCHES the same submit-script (same +# $CKPT_PATH/$LOG → resumes from the latest checkpoint), bounded by +# --max-relaunch. This is the SLURM supervisor's sacct/squeue/scancel control +# plane re-expressed with a local process + `docker exec` lifecycle. +# +# WHAT IT DETECTS (poll every --poll-s) +# * submit-script process exits -> read its exit code: +# 0 => run finished cleanly (success) +# != 0 => crash/OOM/die_at_step(42)/etc. => relaunch from latest ckpt +# * hang watchdog: process alive but $LOG frozen >= --stall-s AND no trainer +# process alive in the container (pgrep via docker exec) => kill + pkill in +# container + relaunch. A long eval / blocking ckpt save keeps the trainer +# process up, so it is NOT counted as a stall. +# * disk guard before each (re)launch: require --min-free-gib on the ckpt vol. +# +# USAGE +# nohup bash scripts/run_streaming_e2e_local.sh \ +# --submit-script scripts/launch_e2e_local.sh \ +# --log /home/chcai/yambda_5b_e2e//.log \ +# --ckpt-path /home/chcai/yambda_5b_e2e//ckpts \ +# --run-name \ +# > /home/chcai/yambda_5b_e2e//.supervisor.console.log 2>&1 & +# +# Per-run hyperparameters live in the --submit-script's env defaults (or are +# exported before invoking this supervisor), not here. +# +# EXIT CODES +# 0 run completed cleanly +# 1 exhausted --max-relaunch without completion (or launch failed) +# 3 disk guard tripped +# ============================================================================= +set -uo pipefail + +SUBMIT_SCRIPT="scripts/launch_e2e_local.sh" +LOG="" +CKPT_PATH="" +RUN_NAME="yambda_5b_e2e_local" +CONTAINER=${CONTAINER:-yambda_local} +DOCKER=${DOCKER:-sudo docker} +MAX_RELAUNCH=50 +MIN_FREE_GIB=700 # one full DMP ckpt (~600 GB) + headroom for the atomic + # .tmp written beside the retained one during a save. +STALL_S=2400 # 40 min frozen-log + no-trainer-proc => hung. +POLL_S=30 + +while [[ $# -gt 0 ]]; do + case $1 in + --submit-script) SUBMIT_SCRIPT="$2"; shift 2;; + --log) LOG="$2"; shift 2;; + --ckpt-path) CKPT_PATH="$2"; shift 2;; + --run-name) RUN_NAME="$2"; shift 2;; + --container) CONTAINER="$2"; shift 2;; + --docker) DOCKER="$2"; shift 2;; + --max-relaunch) MAX_RELAUNCH="$2"; shift 2;; + --min-free-gib) MIN_FREE_GIB="$2"; shift 2;; + --stall-s) STALL_S="$2"; shift 2;; + --poll-s) POLL_S="$2"; shift 2;; + *) echo "Unknown arg: $1"; exit 1;; + esac +done + +[[ -n "$SUBMIT_SCRIPT" && -f "$SUBMIT_SCRIPT" ]] || { echo "FATAL: --submit-script required and must exist ($SUBMIT_SCRIPT)"; exit 1; } +[[ -n "$LOG" ]] || { echo "FATAL: --log required"; exit 1; } + +SUP_LOG="${LOG%.log}.supervisor.log" +# Create the log + ckpt dirs up front so the disk guard's df has a real path to +# stat (df on a nonexistent dir returns 0 avail -> false "disk full" abort). +mkdir -p "$(dirname "$SUP_LOG")" +[[ -n "$CKPT_PATH" ]] && mkdir -p "$CKPT_PATH" +sup() { echo "[$(date '+%F %T')] [supervisor] $*" | tee -a "$SUP_LOG"; } + +# Any trainer process alive in the container? [g]enerative self-match guard +# avoids pgrep matching its own command line. +trainer_alive() { + local n + n=$($DOCKER exec "$CONTAINER" bash -lc 'set -f; pgrep -f "[g]enerative_recommenders" | wc -l' 2>/dev/null | tr -dc '0-9') + [[ "${n:-0}" -gt 0 ]] +} + +# Hard-kill any trainer processes left in the container AND wait for GPU HBM to +# actually drain before returning. A rank stuck in a HIP/RCCL collective sits in +# uninterruptible D-state and keeps its multi-hundred-GB embedding shard resident +# for many seconds after SIGKILL; relaunching before that frees makes the next +# attempt OOM on dirty GPUs (an OOM-crash -> dirty-GPU -> OOM cascade). So kill, +# then poll rocm-smi until every GPU is <5 GB (or give up after ~120s). +cleanup_container() { + $DOCKER exec "$CONTAINER" bash -lc \ + 'pkill -9 -f generative_recommenders 2>/dev/null; pkill -9 -f spawn_main 2>/dev/null; pkill -9 -f resource_tracker 2>/dev/null; true' \ + 2>/dev/null || true + local k busy + for k in $(seq 1 24); do # up to ~120s + busy=$($DOCKER exec "$CONTAINER" bash -lc \ + "rocm-smi --showmeminfo vram 2>/dev/null | awk '/Used/{if (\$NF+0 > 5e9) c++} END{print c+0}'" \ + 2>/dev/null | tr -dc '0-9') + busy=${busy:-0} + [[ "$busy" == "0" ]] && return 0 + sup "waiting for GPU HBM to drain ($busy GPU(s) still >5GB)…" + $DOCKER exec "$CONTAINER" bash -lc 'pkill -9 -f spawn_main 2>/dev/null; true' 2>/dev/null || true + sleep 5 + done + sup "WARNING: GPUs still show residual HBM after 120s — launching anyway." + return 0 +} + +disk_free_gib() { df -BG --output=avail "$CKPT_PATH" 2>/dev/null | tail -1 | tr -dc '0-9'; } + +disk_guard() { + [[ -z "$CKPT_PATH" ]] && return 0 + local free; free=$(disk_free_gib); free=${free:-0} + sup "disk guard: ${free} GiB free on $CKPT_PATH (min ${MIN_FREE_GIB})" + if (( free < MIN_FREE_GIB )); then + sup "FATAL: insufficient free space (${free} < ${MIN_FREE_GIB} GiB). Aborting." + return 1 + fi + return 0 +} + +# Run the submit-script in the FOREGROUND (its exit status == the trainer's). +# APPEND_LOG=1 preserves the metrics log across relaunches. This is invoked as +# `launch & PID=$!` from the main loop so the backgrounded copy is a DIRECT child +# of this shell — otherwise `wait $PID` can't reap it and always returns 127, +# making every clean completion look like a failure (infinite relaunch loop). +launch() { + APPEND_LOG=1 CONTAINER="$CONTAINER" DOCKER="$DOCKER" \ + RUN_NAME="$RUN_NAME" LOG="$LOG" CKPT_PATH="$CKPT_PATH" \ + bash "$SUBMIT_SCRIPT" >>"$SUP_LOG" 2>&1 +} + +sup "=== streaming e2e LOCAL supervisor start ===" +sup "run=$RUN_NAME submit=$SUBMIT_SCRIPT log=$LOG ckpt=$CKPT_PATH container=$CONTAINER" +sup "max_relaunch=$MAX_RELAUNCH min_free_gib=$MIN_FREE_GIB stall_s=$STALL_S poll_s=$POLL_S" + +attempt=1 +while (( attempt <= MAX_RELAUNCH )); do + if ! disk_guard; then exit 3; fi + sup "launching attempt $attempt/$MAX_RELAUNCH" + cleanup_container # ensure no stragglers from a prior attempt + launch & PID=$! # direct child => wait $PID reaps the real rc + sup "submit-script running as host pid=$PID" + + # --- monitor loop --- + last_size=0; stall_accum=0; hb=0; hung=0 + while kill -0 "$PID" 2>/dev/null; do + cur_size=$(stat -c %s "$LOG" 2>/dev/null || echo 0) + if [[ "$cur_size" == "$last_size" ]]; then + hb=$((hb + 1)) + # Re-check liveness only every 4 polls (cheap docker exec amortized). + if (( hb % 4 == 0 )); then + if trainer_alive; then + stall_accum=0 + else + stall_accum=$((stall_accum + POLL_S * 4)) + sup "log frozen + no trainer alive (${stall_accum}s/${STALL_S}s)" + if (( stall_accum >= STALL_S )); then + sup "STALL: hung run — killing pid=$PID + container trainer procs, will relaunch." + hung=1 + kill -9 "$PID" 2>/dev/null || true + cleanup_container + break + fi + fi + fi + else + stall_accum=0; last_size=$cur_size + fi + sleep "$POLL_S" + done + + # --- the submit-script has exited (or we killed it): decide --- + wait "$PID" 2>/dev/null; rc=$? + if (( hung )); then + sup "attempt $attempt ended via STALL recovery (rc=$rc) — relaunching from latest checkpoint." + elif (( rc == 0 )); then + sup "RUN COMPLETED CLEANLY on attempt $attempt." + sup "=== supervisor done (success) ===" + exit 0 + else + sup "attempt $attempt exited rc=$rc (crash/OOM/die_at_step) — relaunching from latest checkpoint." + fi + + if (( attempt >= MAX_RELAUNCH )); then break; fi + sleep $(( attempt < 5 ? 20 : 60 )) # small backoff + attempt=$((attempt + 1)) +done + +sup "FATAL: exhausted MAX_RELAUNCH=$MAX_RELAUNCH without completion." +sup "=== supervisor done (failure) ===" +exit 1 From 7f73ac7b0dffe0c01c100ac7083fe4c26b0081c1 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Fri, 19 Jun 2026 01:42:02 -0500 Subject: [PATCH 071/127] dlrmv4: set default-PG timeout to TIMEOUT (1800s) to survive checkpoint skew The checkpoint DCP collectives run on the default process group created by init_process_group, which had no explicit timeout and thus used NCCL's stock 600s watchdog. The 560GB sparse-embedding checkpoint is written to shared NFS with a badly imbalanced sharding plan (per-rank shards ~37GB..~95GB), so the fastest rank can wait >600s in the post-write allgather/barrier for the slowest rank, tripping the watchdog and SIGABRTing an otherwise-healthy job (observed on 3 nodes, always rank 7). Pass timeout=timedelta(seconds=TIMEOUT) so the default PG tolerates the skew, matching the secondary new_group. Co-authored-by: Cursor --- .../generative_recommenders/dlrm_v3/train/utils.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py index aaed8c2a0..113841a3c 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -123,9 +123,20 @@ def setup( # early to be gin-configurable and is redundant with that call. # initialize the process group + # + # The default PG timeout must match TIMEOUT (not the 600s NCCL default): + # checkpoint saves go through DCP collectives on *this* default PG, and the + # 560GB sparse-embedding write is both slow on shared NFS and badly skewed + # across ranks (shards range ~37GB..~95GB), so the fastest rank can sit in + # the post-write allgather/barrier well past 600s waiting for the slowest + # rank. The stock 600s watchdog then SIGABRTs an otherwise-healthy job. if not dist.is_initialized(): dist.init_process_group( - "nccl", rank=rank, world_size=world_size, device_id=device + "nccl", + rank=rank, + world_size=world_size, + device_id=device, + timeout=timedelta(seconds=TIMEOUT), ) pg = dist.new_group( From 09b4204589fc5f13e8944256ec0d5bde1b5ac8dc Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Mon, 22 Jun 2026 00:57:40 -0500 Subject: [PATCH 072/127] dlrmv4: optional gradient clipping for the streaming path ($GRAD_CLIP_NORM) Add env-configurable global-norm gradient clipping to streaming_train_eval_loop, applied to dense params after backward() and before optimizer.step() (sparse tables use a fused optimizer and are unaffected, matching the non-streaming path's clip_grad_norm_). Wired via gin to $GRAD_CLIP_NORM and forwarded into the container by launch_slurm.sh. Default 0.0 = OFF, so existing streaming runs are unchanged. Eliminates the window-131 eval-AUC dip at LR=1e-5. Co-authored-by: Cursor --- .../dlrm_v3/train/gin/yambda_5b.gin | 7 +++++++ .../dlrm_v3/train/utils.py | 19 +++++++++++++++++++ recommendation_v4/scripts/launch_slurm.sh | 1 + 3 files changed, 27 insertions(+) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index f18c866a2..49870a6b4 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -64,6 +64,13 @@ sparse_optimizer_factory_and_class.weight_decay = 0 sparse_optimizer_factory_and_class.eps = 1e-8 sparse_optimizer_factory_and_class.betas = (0.95, 0.999) +# Gradient clipping for the STREAMING path (clips dense params; sparse tables +# use a fused optimizer and are unaffected). Env-overridable via $GRAD_CLIP_NORM. +# Default 0.0 = OFF so existing streaming runs are unchanged; set >0 to enable. +streaming_train_eval_loop.grad_clip_norm = @gcn/env_float() +gcn/env_float.key = "GRAD_CLIP_NORM" +gcn/env_float.default = 0.0 + # Data root: resolved at runtime from $DLRM_DATA_PATH if set, else the literal # below. Used by both make_train_test_dataloaders and get_dataset. # Scoped (`data/env_path`) so this binding doesn't collide with the RUN_NAME diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py index 113841a3c..c6c299998 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -1460,6 +1460,9 @@ def streaming_train_eval_loop( # --- global step / wall-clock checkpoint cadences --- checkpoint_step_frequency: int = 0, checkpoint_time_interval_s: float = 0.0, + # --- gradient clipping (streaming path, dense params). 0.0 = OFF, which + # preserves legacy streaming behavior. Wired to $GRAD_CLIP_NORM via gin. --- + grad_clip_norm: float = 0.0, # --- diagnostic: log per-batch unique/total embedding-id counts --- streaming_diag_unique_emb: bool = False, # --- test-only failure injection knob --- @@ -1627,6 +1630,13 @@ def streaming_train_eval_loop( original_end_ts, start_ts, ) + if rank == 0: + logger.info( + "[grad-clip] streaming path gradient clipping %s (max_norm=%.4g via $GRAD_CLIP_NORM)", + "ENABLED" if (grad_clip_norm and grad_clip_norm > 0) else "OFF", + grad_clip_norm, + ) + def _window_iter(ts: int, skip_samples: int = 0): # TRAIN-only iterator: both branches exclude held-out eval users via # train_window_indices / set_ts(train_only=True). (Eval uses the fixed @@ -1743,6 +1753,15 @@ def _run_train_window( ) # pyre-ignore sum(aux_losses.values()).backward() + # Gradient clipping for the streaming path. Clips dense params (the + # sparse embedding tables use a fused optimizer and are unaffected, + # same as the non-streaming path's clip_grad_norm_). OFF by default + # (grad_clip_norm=0.0 via $GRAD_CLIP_NORM) so legacy streaming runs + # are byte-for-byte unchanged; set >0 to enable. + if grad_clip_norm and grad_clip_norm > 0: + torch.nn.utils.clip_grad_norm_( + model.parameters(), max_norm=grad_clip_norm + ) optimizer.step() metric_logger.update( mode="train", diff --git a/recommendation_v4/scripts/launch_slurm.sh b/recommendation_v4/scripts/launch_slurm.sh index f20934285..9b0c1987d 100755 --- a/recommendation_v4/scripts/launch_slurm.sh +++ b/recommendation_v4/scripts/launch_slurm.sh @@ -290,6 +290,7 @@ orchestrate() { ${SEED:+-e SEED=$SEED} \ ${DENSE_LR:+-e DENSE_LR=$DENSE_LR} \ ${SPARSE_LR:+-e SPARSE_LR=$SPARSE_LR} \ + ${GRAD_CLIP_NORM:+-e GRAD_CLIP_NORM=$GRAD_CLIP_NORM} \ ${HSTU_NUM_LAYERS:+-e HSTU_NUM_LAYERS=$HSTU_NUM_LAYERS} \ ${MAX_SEQ_LEN:+-e MAX_SEQ_LEN=$MAX_SEQ_LEN} \ ${HISTORY_LENGTH:+-e HISTORY_LENGTH=$HISTORY_LENGTH} \ From 3cb9eb64a858abc386776365c2af784690ec7108 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Mon, 22 Jun 2026 15:09:57 -0500 Subject: [PATCH 073/127] dlrmv4: data-fraction eval cadence + lr1e-7/grad-clip-on defaults Add a data-percentage-based eval cadence for the streaming loop, an alternative to the per-window cadence. EVAL_EVERY_DATA_PCT>0 runs the full-holdout eval every fixed FRACTION of the run's total training data, so eval points are evenly spaced by data volume regardless of per-window sample counts. The fraction is converted once into a global train-step interval (round(pct * total_train_anchors / (batch_size*world_size))) over the original requested window range, and eval fires on global_step % interval -- mid-window and resume-stable, mirroring checkpoint_step_frequency. Each eval label carries @step= for plotting against data volume. - yambda: total_train_anchors(start_ts, num_ts) one-time O(N) count. - streaming_train_eval_loop: eval_every_data_pct param + interval calc; the two cadences are mutually exclusive (ValueError if both >0). - launch_slurm.sh: forward EVAL_EVERY_DATA_PCT into the container. Also flip the yambda-5b gin defaults to the validated config: dense and sparse LR 1e-5 -> 1e-7, and gradient clipping ON by default (GRAD_CLIP_NORM 0.0 -> 1.0). HSTU depth stays 3. Verified e2e on an open-pool node: interval computed correctly, two mid-window evals fired on the step grid + final eval (rc=0), and the both-enabled config raises the expected ValueError. Co-authored-by: Cursor --- .../dlrm_v3/datasets/yambda.py | 35 +++++ .../dlrm_v3/train/gin/yambda_5b.gin | 48 +++++- .../dlrm_v3/train/utils.py | 137 +++++++++++++++++- recommendation_v4/scripts/launch_slurm.sh | 1 + 4 files changed, 214 insertions(+), 7 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/datasets/yambda.py b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/yambda.py index ed98a1013..1ced622e5 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/datasets/yambda.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/yambda.py @@ -734,6 +734,41 @@ def eval_holdout_indices(self, start_ts: int, num_windows: int = 1) -> np.ndarra self._eval_holdout_cache_key = key return holdout + def total_train_anchors(self, start_ts: int, num_ts: int) -> int: + """Total TRAIN anchors across windows ``[start_ts, start_ts + num_ts)``. + + A single O(N) pass over the cached ``_anchor_ts`` array (NOT per-window + ``train_window_indices`` scans). Used to convert a "fraction of training + data" eval cadence into a global train-step interval. With a user holdout + (``train_split_percentage`` < 1.0) the held-out eval users are excluded + via the SAME uid hash as ``train_window_indices``, so the count matches + what is actually trained. + + NOTE: this is an UPPER BOUND on the realized train STEP count — the + per-window samplers truncate each window to a multiple of ``world_size`` + and drop the last partial per-rank batch (``drop_last=True``). The small + overcount is acceptable for a cadence knob (it only shifts the eval grid + by a fraction of a window). + """ + self._ensure_streaming_index() + assert self._anchor_ts is not None and self._t_min is not None + if num_ts <= 0: + return 0 + w = self._streaming_window_seconds + lo = self._t_min + start_ts * w + hi = self._t_min + (start_ts + num_ts) * w + in_range = (self._anchor_ts >= lo) & (self._anchor_ts < hi) + if self._train_split_percentage >= 1.0: + total = int(np.count_nonzero(in_range)) + else: + sel = np.where(in_range)[0] + total = int(np.count_nonzero(~self._eval_anchor_mask(sel))) + logger.warning( + f"total_train_anchors(start_ts={start_ts}, num_ts={num_ts}): " + f"{total:,} train anchors (tsp={self._train_split_percentage})" + ) + return total + def set_ts(self, ts: int, train_only: bool = False) -> None: """Restrict the active sample set to anchors in window ``ts`` (used by the per-window-DataLoader path, where ``iloc``/``get_item_count`` index diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index 49870a6b4..d14412196 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -43,10 +43,10 @@ seed/env_int.key = "SEED" seed/env_int.default = 1 # dense model optimizer -# Learning rate is env-overridable via $DENSE_LR (default 1e-5). +# Learning rate is env-overridable via $DENSE_LR (default 1e-7). dense_optimizer_factory_and_class.learning_rate = @dlr/env_float() dlr/env_float.key = "DENSE_LR" -dlr/env_float.default = 0.00001 +dlr/env_float.default = 0.0000001 dense_optimizer_factory_and_class.optimizer_name = "Adam" dense_optimizer_factory_and_class.momentum = 0 dense_optimizer_factory_and_class.weight_decay = 0 @@ -54,10 +54,10 @@ dense_optimizer_factory_and_class.eps = 1e-8 dense_optimizer_factory_and_class.betas = (0.95, 0.999) # sparse model optimizer -# Learning rate is env-overridable via $SPARSE_LR (default 1e-5). +# Learning rate is env-overridable via $SPARSE_LR (default 1e-7). sparse_optimizer_factory_and_class.learning_rate = @slr/env_float() slr/env_float.key = "SPARSE_LR" -slr/env_float.default = 0.00001 +slr/env_float.default = 0.0000001 sparse_optimizer_factory_and_class.optimizer_name = "RowWiseAdagrad" sparse_optimizer_factory_and_class.momentum = 0 sparse_optimizer_factory_and_class.weight_decay = 0 @@ -66,10 +66,10 @@ sparse_optimizer_factory_and_class.betas = (0.95, 0.999) # Gradient clipping for the STREAMING path (clips dense params; sparse tables # use a fused optimizer and are unaffected). Env-overridable via $GRAD_CLIP_NORM. -# Default 0.0 = OFF so existing streaming runs are unchanged; set >0 to enable. +# Default 1.0 = ON (max_norm=1.0); set 0.0 via $GRAD_CLIP_NORM to disable. streaming_train_eval_loop.grad_clip_norm = @gcn/env_float() gcn/env_float.key = "GRAD_CLIP_NORM" -gcn/env_float.default = 0.0 +gcn/env_float.default = 1.0 # Data root: resolved at runtime from $DLRM_DATA_PATH if set, else the literal # below. Used by both make_train_test_dataloaders and get_dataset. @@ -257,6 +257,13 @@ ot/env_int.default = 0 streaming_train_eval_loop.persistent_loader = @pl/env_int() pl/env_int.key = "PERSISTENT_LOADER" pl/env_int.default = 1 +# ---- Eval cadence: choose EXACTLY ONE of the two knobs below ---------------- +# The streaming loop can decide WHEN to run the full-holdout eval pass in one of +# two ways. They are MUTUALLY EXCLUSIVE — enabling the data-fraction cadence +# (EVAL_EVERY_DATA_PCT>0) requires EVAL_EVERY_N_WINDOWS=0; setting both >0 raises +# a ValueError at startup. The final end-of-run eval always runs in either mode. +# +# (1) PER-WINDOW cadence (EVAL_EVERY_N_WINDOWS, the default). # Full-holdout eval cadence (single knob; replaces the old EVAL_EACH_WINDOW # on/off switch). 0 = eval disabled (train-only, e.g. perf benchmarking or the # resume test; the eval dataloader isn't even built). 1 (default) = eval after @@ -264,9 +271,38 @@ pl/env_int.default = 1 # (and always the final one) to amortize the cost of consuming the full next-day # eval window. The cadence is anchored to the absolute ts grid so eval points # stay stable across a mid-run resume. +# NOTE: each daily window has a DIFFERENT number of training samples, so a +# per-window cadence produces eval points that are UNEVENLY spaced in terms of +# how much data was trained between them. Use the data-fraction cadence below if +# you want evenly-spaced-by-data eval points instead. streaming_train_eval_loop.eval_every_n_windows = @evn/env_int() evn/env_int.key = "EVAL_EVERY_N_WINDOWS" evn/env_int.default = 1 +# +# (2) DATA-FRACTION cadence (EVAL_EVERY_DATA_PCT). +# Run the full-holdout eval every time the run has trained this FRACTION of the +# run's TOTAL training data, so eval points are EVENLY spaced by data volume +# (compute), independent of how many samples each daily window happens to hold. +# This is the fix for the per-window cadence's uneven spacing noted above. +# value semantics (it is a fraction in (0, 1], NOT a percent number): +# 0.0 (default) = OFF -> fall back to the per-window EVAL_EVERY_N_WINDOWS. +# 0.01 = eval every 1% of the data -> ~100 eval points total. +# 0.05 = eval every 5% of the data -> ~20 eval points total. +# 0.10 = eval every 10% of the data -> ~10 eval points total. +# How "fraction of data" becomes an eval trigger: at startup the fraction is +# converted ONCE into a global train-step interval +# eval_interval_steps = round(pct * total_train_anchors / (batch_size * world_size)) +# where total_train_anchors is counted over the ORIGINAL requested window range +# [start_ts, start_ts+num_train_ts). Eval then fires whenever the monotonic +# train global_step crosses a multiple of that interval (it can fire MID-WINDOW, +# i.e. partway through a daily window, and across window boundaries). Because the +# interval is computed over the original range and global_step is +# checkpoint-restored, the eval grid is identical on a cold start and on every +# resume. Each eval record's label carries "@step=" so the +# trajectory can be plotted against data volume. Override via $EVAL_EVERY_DATA_PCT. +streaming_train_eval_loop.eval_every_data_pct = @edp/env_float() +edp/env_float.key = "EVAL_EVERY_DATA_PCT" +edp/env_float.default = 0.0 # Double-buffer windows: prepare the next window (index mask + first-batch # prefetch) in a background thread during the current window's compute, hiding # the per-window reset. Needs persistent_loader=1. Override via env. diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py index c6c299998..0e9e456e9 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -1439,6 +1439,11 @@ def streaming_train_eval_loop( start_ts: int = 0, persistent_loader: bool = False, eval_every_n_windows: int = 1, + # Data-fraction eval cadence (mutually exclusive with eval_every_n_windows). + # >0 = eval every this FRACTION of the run's total training data, converted + # once to a global train-step interval. 0.0 = OFF (use the per-window + # cadence). Wired to $EVAL_EVERY_DATA_PCT via gin. + eval_every_data_pct: float = 0.0, double_buffer: bool = False, # --- fixed user-holdout eval set --- # Window range the fixed eval set is drawn from. None -> default to @@ -1507,6 +1512,18 @@ def streaming_train_eval_loop( save fires. Used by the failure-injection test to crash at a deterministic boundary and then resume. """ + # Exactly one eval cadence may be active. eval_every_n_windows defaults to 1 + # (eval every window), so enabling the data-fraction cadence REQUIRES + # explicitly disabling the per-window one (EVAL_EVERY_N_WINDOWS=0). Fail fast + # on a contradictory config rather than silently picking one. + if (eval_every_data_pct and eval_every_data_pct > 0) and eval_every_n_windows > 0: + raise ValueError( + "Conflicting eval cadences: eval_every_data_pct=" + f"{eval_every_data_pct} (>0) AND eval_every_n_windows=" + f"{eval_every_n_windows} (>0). They are mutually exclusive. To use " + "the data-fraction cadence set EVAL_EVERY_N_WINDOWS=0; to use the " + "per-window cadence set EVAL_EVERY_DATA_PCT=0." + ) profiler = Profiler(rank) if output_trace else None # Normalize the per-window caps: <=0 (the env-binding default) means "no cap # = consume the full window". The eval-break check below is `is not None and @@ -1563,6 +1580,53 @@ def streaming_train_eval_loop( else requested_end_ts ) + # Data-fraction eval cadence: convert eval_every_data_pct into a global + # train-step interval ONCE, over the ORIGINAL requested window range + # [eval_anchor_ts, requested_end_ts). Keying the later trigger off + # `global_step % eval_interval_steps` (global_step is monotonic and + # checkpoint-restored) makes the eval grid identical on cold start and on + # every resume, exactly like checkpoint_step_frequency. 0 => disabled. + eval_interval_steps = 0 + if eval_every_data_pct and eval_every_data_pct > 0: + # Per-rank batch size: the persistent loader carries it directly; the + # per-window path uses the same gin %batch_size (env BATCH_SIZE, + # default 1024 — matches make_streaming_dataloader.batch_size). + bs = ( + persistent_dl.batch_size + if persistent_dl is not None + else int(os.environ.get("BATCH_SIZE", "1024")) + ) + if hasattr(dataset.dataset, "total_train_anchors"): + total_train_anchors = dataset.dataset.total_train_anchors( # pyre-ignore[16] + eval_anchor_ts, requested_end_ts - eval_anchor_ts + ) + total_train_steps = total_train_anchors // max(1, bs * world_size) + eval_interval_steps = max( + 1, round(eval_every_data_pct * total_train_steps) + ) + if rank == 0: + logger.info( + "[data-pct-eval] eval_every_data_pct=%.6g -> " + "eval_interval_steps=%d (total_train_anchors=%d bs=%d " + "world_size=%d total_train_steps=%d over windows [%d, %d))", + eval_every_data_pct, + eval_interval_steps, + total_train_anchors, + bs, + world_size, + total_train_steps, + eval_anchor_ts, + requested_end_ts, + ) + elif rank == 0: + logger.warning( + "[data-pct-eval] dataset %s has no total_train_anchors(); " + "data-fraction eval is DISABLED (no per-window eval either, " + "since EVAL_EVERY_N_WINDOWS must be 0 to reach here) — only the " + "final eval will run.", + type(dataset.dataset).__name__, + ) + # The split is an immutable run contract: a silent change across resume # would both desync the mid-window skip offset AND turn held-out eval users # into trained users (leakage). Build the live contract and validate the @@ -1711,6 +1775,7 @@ def _run_train_window( train_ts: int, start_batch_idx: int = 0, label: Optional[str] = None, + do_eval: Optional[Callable[[int, int], None]] = None, ) -> None: # `start_batch_idx` is set when we're re-entering a window that was # interrupted mid-way (in_window resume); the dataloader iterator was @@ -1817,6 +1882,30 @@ def _run_train_window( # Reset the wall-clock anchor on ANY save so the next time # trigger is measured from the most recent checkpoint. last_ckpt_time[0] = time.time() + # Data-fraction eval cadence: run the full-holdout eval whenever the + # monotonic global step crosses a multiple of eval_interval_steps + # (i.e. every eval_every_data_pct of the training data). Keyed off + # global_step (checkpoint-restored) so the eval grid is identical + # across resume. Mid-window-safe: eval sets model.eval(), so restore + # train mode + dataset.is_eval afterward. do_eval is None unless the + # data-pct cadence is enabled. + if ( + do_eval is not None + and eval_interval_steps > 0 + and gstep > 0 + and gstep % eval_interval_steps == 0 + ): + if rank == 0: + logger.info( + "[data-pct-eval] trigger eval train_ts=%d global_step=%d " + "(interval=%d)", + train_ts, + gstep, + eval_interval_steps, + ) + do_eval(train_ts, gstep) + model.train() + dataset.dataset.is_eval = False # pyre-ignore [16] # Test-only: deterministic crash for the failure-injection test. # Triggered AFTER the save above, so on resume we re-enter at # batch_idx_in_window=train_batch_idx and emit batches [K+1, end). @@ -2035,7 +2124,13 @@ def _should_eval(i: int) -> bool: # sample content depends only on the sampler window, not is_eval, so # prefetching during train is safe. eval_iter: Optional[Iterator] = None - if eval_every_n_windows > 0 and len(train_ts_list) > 0: + # Build/fork the eval pool when EITHER cadence needs it: the per-window + # cadence (eval_every_n_windows>0) or the data-fraction cadence + # (eval_interval_steps>0). Both are never simultaneously on (validated + # at entry), so this is "eval is enabled at all". + if (eval_every_n_windows > 0 or eval_interval_steps > 0) and len( + train_ts_list + ) > 0: eval_sampler = StreamingWindowSampler(rank, world_size) eval_dl = make_persistent_streaming_dataloader( dataset=dataset, sampler=eval_sampler @@ -2053,6 +2148,22 @@ def _should_eval(i: int) -> bool: # iter() again to replay the identical set (no set_window churn). eval_sampler.set_window(eval_global_indices) eval_iter = iter(eval_dl) + + # Data-fraction eval callback (double-buffer path). Fired mid-window by + # _run_train_window on the global-step cadence. Reuses the already-forked + # persistent eval pool: iter(eval_dl) here runs on the MAIN thread (a + # reset, not a fork — the only fork was the up-front iter() above), so it + # stays safe alongside the background window-prefetch thread. + def _do_eval_db(train_ts: int, gstep: int) -> None: + dataset.dataset.is_eval = True # pyre-ignore [16] + assert eval_dl is not None + _run_eval_window( + iter(eval_dl), + label=f"eval_holdout@train_ts={train_ts}@step={gstep}", + ) + + _db_do_eval = _do_eval_db if eval_interval_steps > 0 else None + for i, (train_ts, train_data_iterator) in enumerate( # Only the FIRST window after a mid-window resume needs the skip # (handed via prefetcher.stream's first_skip_samples). The skip is @@ -2074,6 +2185,7 @@ def _should_eval(i: int) -> bool: train_ts=train_ts, start_batch_idx=start_batch, label=f"train_ts={train_ts}", + do_eval=_db_do_eval, ) if _should_eval(i): dataset.dataset.is_eval = True # pyre-ignore [16] @@ -2090,6 +2202,28 @@ def _should_eval(i: int) -> bool: eval_iter = iter(eval_dl) _maybe_checkpoint(train_ts) else: + # Data-fraction eval callback (non-double-buffer path). Builds a fresh + # eval dataloader per call over the FIXED holdout set (or the legacy + # next-window eval when the dataset has no holdout support). + def _do_eval_nb(train_ts: int, gstep: int) -> None: + dataset.dataset.is_eval = True # pyre-ignore [16] + if eval_global_indices is not None: + _run_eval_window( + iter( + make_streaming_dataloader( + dataset=dataset, indices=eval_global_indices + ) + ), + label=f"eval_holdout@train_ts={train_ts}@step={gstep}", + ) + else: + _run_eval_window( + iter(make_streaming_dataloader(dataset=dataset, ts=train_ts + 1)), + label=f"eval@train_ts={train_ts}@step={gstep}", + ) + + _nb_do_eval = _do_eval_nb if eval_interval_steps > 0 else None + for i, train_ts in enumerate(train_ts_list): dataset.dataset.is_eval = False # pyre-ignore [16] skip = first_skip_samples if i == 0 else 0 @@ -2102,6 +2236,7 @@ def _should_eval(i: int) -> bool: _window_iter(train_ts, skip_samples=skip), train_ts=train_ts, start_batch_idx=start_batch, + do_eval=_nb_do_eval, ) if _should_eval(i): dataset.dataset.is_eval = True # pyre-ignore [16] diff --git a/recommendation_v4/scripts/launch_slurm.sh b/recommendation_v4/scripts/launch_slurm.sh index 9b0c1987d..579d4427a 100755 --- a/recommendation_v4/scripts/launch_slurm.sh +++ b/recommendation_v4/scripts/launch_slurm.sh @@ -276,6 +276,7 @@ orchestrate() { -e NUM_TRAIN_TS=$NUM_TRAIN_TS \ -e EVAL_EACH_WINDOW=$EVAL_EACH_WINDOW \ -e EVAL_EVERY_N_WINDOWS=$EVAL_EVERY_N_WINDOWS \ + ${EVAL_EVERY_DATA_PCT:+-e EVAL_EVERY_DATA_PCT=$EVAL_EVERY_DATA_PCT} \ -e NUM_TRAIN_BATCHES=$NUM_TRAIN_BATCHES \ -e NUM_EVAL_BATCHES=$NUM_EVAL_BATCHES \ -e METRIC_LOG_FREQ=$METRIC_LOG_FREQ \ From c2342c5d992680c8ae1b458e6470bf02b448e584 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Mon, 22 Jun 2026 20:18:39 +0000 Subject: [PATCH 074/127] dlrmv4: seed embedding init + reproducibility checksum ($SEED) Make embedding-table init a deterministic function of $SEED (dense init already was), so runs are a clean init-matched A/B when data order ($STREAMING_SHUFFLE_SEED) and the holdout split ($SPLIT_SALT) are fixed. - configs.py: attach a per-table seeded uniform init_fn (per-table seed = sha256($SEED, table_name)); meta-safe (skips the meta device DMP builds the unsharded module on). Init bounds mirror stock (+/-1/sqrt(N) or a table's explicit weight_init_min/max), so the distribution -- and thus model quality -- is unchanged; only determinism/seeding differs. - utils.py (make_optimizer_and_shard): re-seed torch/torch.cuda from $SEED right before DistributedModelParallel(...) so the fused FBGEMM TBE on-device embedding init is reproducible for a FIXED sharding plan (Tier 1). Dense params are already built in make_model, so untouched. - utils.py: log a post-DMP init checksum (per-table count/sum/sumsq + a one-line digest; sharded stats are all-reduced so the fingerprint covers the whole table regardless of shard layout). Gate via INIT_CHECKSUM (default on). Verified on idle nodes: same seed -> same digest across reruns and node types; different seed -> different digest. - yambda_5b.gin: document precisely what $SEED controls and what it does not (streaming data order, holdout split). Co-authored-by: Cursor --- .../dlrm_v3/configs.py | 80 ++++++++++++++++++- .../dlrm_v3/train/gin/yambda_5b.gin | 50 ++++++++++-- .../dlrm_v3/train/utils.py | 64 +++++++++++++++ 3 files changed, 187 insertions(+), 7 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/configs.py b/recommendation_v4/generative_recommenders/dlrm_v3/configs.py index 1b6ecf62f..387fb4900 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/configs.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/configs.py @@ -19,9 +19,13 @@ This module provides configuration functions for the HSTU model architecture and embedding table configurations. """ -from typing import Dict, Optional +import hashlib +import math +import os +from typing import Callable, Dict, Optional, Tuple import gin +import torch from generative_recommenders.modules.dlrm_hstu import DlrmHSTUConfig from generative_recommenders.modules.multitask_module import ( @@ -509,10 +513,63 @@ def get_hstu_configs( return hstu_config +def _stable_table_seed(init_seed: int, table_name: str) -> int: + """Deterministic 63-bit seed from (init_seed, table_name). + + Uses sha256 (not Python's salted built-in ``hash()``) so the per-table seed + is identical across processes/ranks/runs for a given ``$SEED`` + table name. + """ + digest = hashlib.sha256(f"{init_seed}:{table_name}".encode("utf-8")).digest() + return int.from_bytes(digest[:8], "big") & 0x7FFF_FFFF_FFFF_FFFF + + +def _uniform_init_bounds(cfg: EmbeddingConfig) -> Tuple[float, float]: + """Mirror TorchREC's default per-table init bounds. + + TorchREC falls back to ``uniform_(-1/sqrt(N), +1/sqrt(N))`` when a table does + not set ``weight_init_min/max``; honor any explicit bounds the config carries. + """ + bound = math.sqrt(1.0 / cfg.num_embeddings) + lo = -bound if cfg.weight_init_min is None else cfg.weight_init_min + hi = bound if cfg.weight_init_max is None else cfg.weight_init_max + return lo, hi + + +def _make_seeded_uniform_init( + table_seed: int, lo: float, hi: float +) -> Callable[[torch.Tensor], torch.Tensor]: + """Build a seeded in-place uniform initializer for one table's weight. + + TorchREC/FBGEMM calls ``init_fn`` with the (per-rank) local shard tensor on + its compute device, so we seed a generator on that same device. For a fixed + sharding plan (world size + plan unchanged) this makes embedding init + byte-reproducible run-to-run. + """ + + def _init(weight: torch.Tensor) -> torch.Tensor: + # TorchREC builds the unsharded EmbeddingCollection on the META device + # first (DMP materializes real storage on the compute device later). + # Meta tensors have no storage and torch.Generator(device="meta") is + # invalid ("META device type not an accelerator"), so skip them: the + # seeded init for the sharded/fused TBE path is provided by the RNG + # re-seed right before DMP in make_optimizer_and_shard. On a real + # device (eager/non-meta path) we still apply the per-table seeded fill. + if weight.device.type == "meta": + return weight + gen = torch.Generator(device=weight.device) + gen.manual_seed(table_seed) + with torch.no_grad(): + weight.uniform_(lo, hi, generator=gen) + return weight + + return _init + + @gin.configurable def get_embedding_table_config( dataset: str = "debug", embedding_dim: Optional[int] = None, + init_seed: Optional[int] = None, ) -> Dict[str, EmbeddingConfig]: """ Create and return embedding table configurations. @@ -527,10 +584,31 @@ def get_embedding_table_config( `HSTU_EMBEDDING_DIM`. Keep in sync with the matching gin override on `get_hstu_configs.hstu_embedding_table_dim` — the model and the tables must agree on dim or sharding will reject the plan. + init_seed: Base seed for the per-table seeded `init_fn` (Tier 1 + reproducible embedding init). When None, falls back to `$SEED` + (default 1), matching `seed_everything`. Each table draws from a + generator seeded by `sha256(init_seed, table_name)` so init is + reproducible run-to-run for a fixed sharding plan. Returns: Dict mapping table names to their EmbeddingConfig objects. """ + tables = _build_embedding_table_config(dataset=dataset, embedding_dim=embedding_dim) + + if init_seed is None: + init_seed = int(os.environ.get("SEED", "1")) + for name, cfg in tables.items(): + lo, hi = _uniform_init_bounds(cfg) + cfg.init_fn = _make_seeded_uniform_init( + _stable_table_seed(init_seed, name), lo, hi + ) + return tables + + +def _build_embedding_table_config( + dataset: str = "debug", + embedding_dim: Optional[int] = None, +) -> Dict[str, EmbeddingConfig]: DIM = embedding_dim if embedding_dim is not None else HSTU_EMBEDDING_DIM if "movielens" in dataset: assert dataset in [ diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index d14412196..227233101 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -32,12 +32,50 @@ make_model.hammer_kernel = "TRITON" # pinned constants in ops/triton/_autotune_pinning.py call sites. apply_env_bootstrap.TRITON_FULL_AUTOTUNE = False -# Global RNG seed for reproducible weight init (dense params + embedding tables) -# and any seeded RNG consumers. Same seed on every rank => identical dense init; -# fixing it makes runs an init-matched A/B (data order is already deterministic -# via the sampler). seed_everything() is called right before make_model() in -# train_ranker (after the full gin parse), so this binding is resolved in the -# second parse where env_int is registered. Override per-run via $SEED. +# ============================================================================= +# $SEED — global RNG seed for reproducible MODEL INITIALIZATION. +# +# WHAT IT CONTROLS (all weight init is a deterministic function of $SEED): +# 1. Dense parameters (HSTU transformer blocks, MLPs, action embeddings, the +# postprocessor). seed_everything() seeds python/numpy/torch/torch.cuda +# with $SEED right before make_model(). The SAME seed is set on every rank, +# so dense weights are initialized identically across ranks AND reproducibly +# run-to-run. +# 2. Sparse embedding tables (item_id, artist_id, album_id, uid, the cross +# tables). These are materialized on-device while DMP shards the model, NOT +# in make_model(), so seed_everything() alone does not pin them. Two +# mechanisms tie them to $SEED: +# (a) make_optimizer_and_shard() RE-SEEDS torch/torch.cuda from $SEED +# immediately before DistributedModelParallel(...), so the fused +# FBGEMM TBE init (which draws from the global RNG on-device) is +# reproducible. This is the path that actually applies here. +# (b) get_embedding_table_config() also attaches a per-table seeded +# init_fn (seed derived from sha256($SEED, table_name)) for the +# eager/non-meta code path. It no-ops on the meta device that DMP +# uses, so (a) is the effective guarantee for this setup. +# 3. Any other seeded RNG consumers (e.g. dropout's init-time draws). +# +# SCOPE ("Tier 1"): reproducible for a FIXED sharding plan — same GPU/world +# size AND same planner output. It is NOT invariant to changing the GPU count +# or sharding (the per-shard draw boundaries move). The init DISTRIBUTION is +# unchanged from stock (uniform +/-1/sqrt(num_embeddings), or a table's +# explicit weight_init_min/max), so $SEED affects determinism, not quality. +# +# VERIFY: INIT_CHECKSUM=1 (default) logs a per-table fingerprint + a one-line +# "[init-checksum] SEED=.. digest=.." right after DMP. Two builds with the same +# $SEED + plan print the same digest; different seeds differ. (INIT_CHECKSUM=0 +# to skip.) +# +# WHAT IT DOES *NOT* CONTROL (separate, independent knobs): +# - Streaming data order / shuffle permutation -> $STREAMING_SHUFFLE_SEED +# (get_dataset.streaming_shuffle_seed, below). +# - Train/eval holdout user split -> $SPLIT_SALT (below). +# So holding $STREAMING_SHUFFLE_SEED + $SPLIT_SALT fixed and varying ONLY $SEED +# isolates the effect of model initialization (a clean init-seed A/B / sweep). +# +# PARSE NOTE: seed_everything() runs right before make_model() in train_ranker +# (after the full gin parse), so this binding resolves in the second parse where +# env_int is registered. Override per-run via $SEED. seed_everything.seed = @seed/env_int() seed/env_int.key = "SEED" seed/env_int.default = 1 diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py index 0e9e456e9..85acd7711 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -504,6 +504,18 @@ def make_optimizer_and_shard( plan = planner.collective_plan(model, sharders, pg) + # Re-seed right before DMP materializes/inits the sharded embedding tables. + # The per-table seeded init_fn (configs.get_embedding_table_config) handles + # the eager path, but the fused FBGEMM TBE path inits weights on-device and + # may bypass init_fn, drawing from the global RNG instead. Re-seeding here + # (same value on every rank) makes embedding init reproducible run-to-run for + # a fixed sharding plan (Tier 1). Dense params are already initialized in + # make_model, so this does not perturb them. + _emb_seed = int(os.environ.get("SEED", "1")) + torch.manual_seed(_emb_seed) + torch.cuda.manual_seed_all(_emb_seed) + logger.info(f"[emb-init] re-seeded RNGs before DMP with SEED={_emb_seed}") + # Shard model model = DistributedModelParallel( module=model, @@ -511,6 +523,58 @@ def make_optimizer_and_shard( plan=plan, sharders=sharders, ) + + # --- startup init checksum (reproducibility probe) ------------------------- + # Right after DMP materializes real weights, log a cheap deterministic + # fingerprint of every parameter so two builds with the same $SEED + sharding + # plan can be diffed for byte-level init reproducibility. For sharded + # embeddings we all-reduce the per-shard (count, sum, sumsq) so the + # fingerprint covers the WHOLE table independent of how rows split across + # ranks; replicated dense params use rank 0's local copy. Stats are computed + # as in-place reductions with fp64 accumulation (no full-size temporaries — + # the embedding shards are tens of GB), so this stays light. Disable with + # INIT_CHECKSUM=0. + if os.environ.get("INIT_CHECKSUM", "1") == "1": + import hashlib + + _rank = dist.get_rank() if dist.is_initialized() else 0 + _fps: List[str] = [] + for _name, _p in sorted(model.named_parameters(), key=lambda kv: kv[0]): + _sharded = isinstance(_p, ShardedTensor) + if _sharded: + _shards = _p.local_shards() + _loc = _shards[0].tensor if _shards else None + else: + _loc = _p + if _loc is None or _loc.numel() == 0: + _cnt, _sm, _sq = 0.0, 0.0, 0.0 + else: + _det = _loc.detach() + _cnt = float(_det.numel()) + _sm = _det.sum(dtype=torch.float64).item() + _nrm = torch.linalg.vector_norm( + _det, ord=2, dtype=torch.float64 + ).item() + _sq = _nrm * _nrm + if _sharded and dist.is_initialized(): + _stat = torch.tensor( + [_cnt, _sm, _sq], dtype=torch.float64, device=device + ) + dist.all_reduce(_stat, op=dist.ReduceOp.SUM) + _cnt, _sm, _sq = _stat.tolist() + _fps.append(f"{_name}|{int(_cnt)}|{_sm:.6f}|{_sq:.6f}") + if _rank == 0: + logger.info( + f"[init-checksum] {'sharded' if _sharded else 'dense'} " + f"{_name} n={int(_cnt)} sum={_sm:.6f} sumsq={_sq:.6f}" + ) + if _rank == 0: + _digest = hashlib.sha256("\n".join(_fps).encode()).hexdigest()[:16] + logger.info( + f"[init-checksum] SEED={os.environ.get('SEED', '?')} " + f"params={len(_fps)} digest={_digest}" + ) + # Create keyed optimizer all_optimizers = [] all_params = {} From eef33047ed7715c33fd95d7ebd43d3e5b5c48509 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Mon, 22 Jun 2026 18:06:33 -0500 Subject: [PATCH 075/127] dlrmv4: default INIT_CHECKSUM off (fp64 shard copy OOMs the build) The startup reproducibility checksum computes per-shard sum/norm with dtype=float64, which materializes a full fp64 copy of each local embedding shard (>150 GiB for the big tables). After sharding leaves ~95 GiB of fp32 tables resident, that temporary leaves almost no HBM headroom and OOMs the build during make_optimizer_and_shard on any node with residual memory. Flip INIT_CHECKSUM to default 0 (opt-in) so normal launches and supervisor resubmits never run it. Correct the utils.py + yambda_5b.gin comments that claimed it had no full-size temporaries. Co-authored-by: Cursor --- .../dlrm_v3/train/gin/yambda_5b.gin | 12 ++++++---- .../dlrm_v3/train/utils.py | 24 +++++++++++-------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index 227233101..e838b346f 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -61,10 +61,14 @@ apply_env_bootstrap.TRITON_FULL_AUTOTUNE = False # unchanged from stock (uniform +/-1/sqrt(num_embeddings), or a table's # explicit weight_init_min/max), so $SEED affects determinism, not quality. # -# VERIFY: INIT_CHECKSUM=1 (default) logs a per-table fingerprint + a one-line -# "[init-checksum] SEED=.. digest=.." right after DMP. Two builds with the same -# $SEED + plan print the same digest; different seeds differ. (INIT_CHECKSUM=0 -# to skip.) +# VERIFY: INIT_CHECKSUM=1 (OFF by default) logs a per-table fingerprint + a +# one-line "[init-checksum] SEED=.. digest=.." right after DMP. Two builds with +# the same $SEED + plan print the same digest; different seeds differ. It is OFF +# by default because the fp64 per-shard reductions materialize a full fp64 copy +# of each local embedding shard (>150 GiB for the big tables), leaving almost no +# HBM headroom after sharding and OOMing the build on any node with residual +# memory. Enable only for an explicit reproducibility check (ideally a clean +# node / smaller batch). # # WHAT IT DOES *NOT* CONTROL (separate, independent knobs): # - Streaming data order / shuffle permutation -> $STREAMING_SHUFFLE_SEED diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py index 85acd7711..785320020 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -525,16 +525,20 @@ def make_optimizer_and_shard( ) # --- startup init checksum (reproducibility probe) ------------------------- - # Right after DMP materializes real weights, log a cheap deterministic - # fingerprint of every parameter so two builds with the same $SEED + sharding - # plan can be diffed for byte-level init reproducibility. For sharded - # embeddings we all-reduce the per-shard (count, sum, sumsq) so the - # fingerprint covers the WHOLE table independent of how rows split across - # ranks; replicated dense params use rank 0's local copy. Stats are computed - # as in-place reductions with fp64 accumulation (no full-size temporaries — - # the embedding shards are tens of GB), so this stays light. Disable with - # INIT_CHECKSUM=0. - if os.environ.get("INIT_CHECKSUM", "1") == "1": + # Right after DMP materializes real weights, log a deterministic fingerprint + # of every parameter so two builds with the same $SEED + sharding plan can be + # diffed for byte-level init reproducibility. For sharded embeddings we + # all-reduce the per-shard (count, sum, sumsq) so the fingerprint covers the + # WHOLE table independent of how rows split across ranks; replicated dense + # params use rank 0's local copy. + # OFF BY DEFAULT: the fp64 reductions below (.sum(dtype=float64) / + # vector_norm(dtype=float64)) materialize a full fp64 copy of each local + # embedding shard (~2x the fp32 shard, i.e. >150 GiB for the big tables), + # which leaves almost no HBM headroom after sharding and will OOM the build on + # any node with residual memory. Only enable for explicit reproducibility + # checks, ideally with a smaller batch / on a clean node. Enable with + # INIT_CHECKSUM=1. + if os.environ.get("INIT_CHECKSUM", "0") == "1": import hashlib _rank = dist.get_rank() if dist.is_initialized() else 0 From 38a9175a9ebd9d85c6af33b3f8776fab3f4eed4b Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Tue, 23 Jun 2026 02:59:51 +0000 Subject: [PATCH 076/127] dlrmv4: add last_n UIH history strategy ($HISTORY_STRATEGY) Adds a configurable Yambda UIH construction strategy alongside the existing per-pool interleaved scheme. "last_n" takes the last HISTORY_LENGTH events of any pool (listen+/like/skip) with no per-pool split, raising effective sequence length (~2.7k -> ~4.1k) and letting the like share fall to its natural rate. Default stays "interleaved" (no behavior change); strategy is resolved at sample-build time so it reuses the existing on-disk cache (no rebuild). Co-authored-by: Cursor --- .../dlrm_v3/datasets/yambda.py | 137 ++++++++++++++---- .../dlrm_v3/train/gin/yambda_5b.gin | 37 +++++ .../generative_recommenders/dlrm_v3/utils.py | 8 + 3 files changed, 156 insertions(+), 26 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/datasets/yambda.py b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/yambda.py index 1ced622e5..00b22cff9 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/datasets/yambda.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/yambda.py @@ -190,8 +190,12 @@ class DLRMv3YambdaDataset(DLRMv3RandomDataset): hstu_config: DlrmHSTUConfig (must come from `get_hstu_configs("yambda-5b")`). processed_dir: directory with `train_sessions.parquet` + `item_popularity.npy`. metadata_dir: directory with `{artist,album}_item_mapping.parquet`. - history_length: per-pool truncation cap (total interleaved ≤ 3 * this). + history_length: UIH cap. Under "interleaved" it is the per-pool cap + (total ≤ 3 * history_length // 3); under "last_n" it is the literal + total number of pooled events kept. scan_window: how far back to scan when filling each pool. + history_strategy: "interleaved" (equal per-pool L//3 cap, re-interleaved) + or "last_n" (last history_length pooled events, no per-pool split). cross_specs: list of (name, keys, num_embeddings, salt). Source of truth in `dlrm_v3/configs.py:YAMBDA_5B_CROSS_SPECS`. is_inference: passed through to base class. @@ -205,6 +209,7 @@ def __init__( history_length: int = 2048, scan_window: int = 20000, min_history: Optional[int] = None, + history_strategy: str = "interleaved", cross_specs: Optional[Sequence[Tuple[str, Sequence[str], int, int]]] = None, cache_dir: Optional[str] = None, is_inference: bool = False, @@ -222,6 +227,21 @@ def __init__( self._metadata_dir: str = metadata_dir self._history_length: int = history_length self._scan_window: int = scan_window + # UIH construction strategy: + # "interleaved" (default) — equal history_length//3 cap per behavior + # pool (listen+/like/skip), re-interleaved chronologically. Likes are + # ~1.9% of the corpus so the like pool over-fills relative to its + # natural frequency while the sequence under-fills overall. + # "last_n" — take the last history_length events of ANY pool type with + # no per-pool split. Fills the sequence to ~history_length (higher + # effective length) and lets the like share fall to its natural rate. + # Both exclude dislike/unlike/undislike (no action-weight bit). + if history_strategy not in ("interleaved", "last_n"): + raise ValueError( + f"history_strategy must be 'interleaved' or 'last_n', got " + f"{history_strategy!r}" + ) + self._history_strategy: str = history_strategy # Minimum prior-event count for a LISTEN event to qualify as an anchor. # Decoupled from history_length (which is only the gather/truncation cap): # jagged attention handles short UIH, so we no longer require a full @@ -814,6 +834,64 @@ def get_sample(self, idx: int) -> Tuple[KeyedJaggedTensor, KeyedJaggedTensor]: flat_pos = self.iloc(idx) return self._build_sample(flat_pos, max_num_candidates) + @staticmethod + def _empty_history() -> Tuple[ + np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray + ]: + empty = np.empty(0, dtype=np.int64) + return empty, empty, empty, empty, empty + + def _read_scan_window( + self, flat_pos: int, user_start: int + ) -> Optional[ + Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray] + ]: + """Read the causal scan window [scan_start, flat_pos) for an anchor. + Returns (item_ids, timestamps, is_lp, is_like, is_skip) views, or None + if the window is empty.""" + scan_start = max(int(user_start), int(flat_pos) - self._scan_window) + scan_end = int(flat_pos) + if scan_end <= scan_start: + return None + return ( + self.store.flat_item_ids[scan_start:scan_end], + self.store.flat_timestamps[scan_start:scan_end], + self.store.flat_is_listen_plus[scan_start:scan_end], + self.store.flat_is_like[scan_start:scan_end], + self.store.flat_is_skip[scan_start:scan_end], + ) + + def _materialize_history( + self, + keep_local: np.ndarray, + item_ids: np.ndarray, + timestamps: np.ndarray, + is_lp: np.ndarray, + is_like: np.ndarray, + is_skip: np.ndarray, + ) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + """Gather item/artist/album/ts + pool-bitmask `weight` for the kept + (chronologically-ordered) local indices.""" + items = item_ids[keep_local] + ts = timestamps[keep_local] + artists = self.item_to_artist[np.clip(items, 0, self.item_to_artist.shape[0] - 1)] + albums = self.item_to_album[np.clip(items, 0, self.item_to_album.shape[0] - 1)] + # Pool bitmask per kept event (LP/LIKE/SKIP are mutually exclusive in + # the source data, but OR is safe and forward-compatible). + weight = np.zeros(keep_local.shape[0], dtype=np.int64) + weight[is_lp[keep_local]] |= LP_BIT + weight[is_like[keep_local]] |= LIKE_BIT + weight[is_skip[keep_local]] |= SKIP_BIT + return items, artists, albums, ts, weight + + def _gather_history( + self, flat_pos: int, user_start: int + ) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + """Dispatch UIH construction to the configured strategy.""" + if self._history_strategy == "last_n": + return self._gather_last_n_history(flat_pos, user_start) + return self._gather_interleaved_history(flat_pos, user_start) + def _gather_interleaved_history( self, flat_pos: int, user_start: int ) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]: @@ -822,17 +900,10 @@ def _gather_interleaved_history( (LP_BIT/LIKE_BIT/SKIP_BIT). Per-pool cap = history_length // 3.""" L = self._history_length per_pool = max(1, L // 3) - scan_start = max(int(user_start), int(flat_pos) - self._scan_window) - scan_end = int(flat_pos) - if scan_end <= scan_start: - empty = np.empty(0, dtype=np.int64) - return empty, empty, empty, empty, empty - - item_ids = self.store.flat_item_ids[scan_start:scan_end] - timestamps = self.store.flat_timestamps[scan_start:scan_end] - is_lp = self.store.flat_is_listen_plus[scan_start:scan_end] - is_like = self.store.flat_is_like[scan_start:scan_end] - is_skip = self.store.flat_is_skip[scan_start:scan_end] + scan = self._read_scan_window(flat_pos, user_start) + if scan is None: + return self._empty_history() + item_ids, timestamps, is_lp, is_like, is_skip = scan # Local indices into the scan window — preserves chronological order # within each pool and lets us interleave by re-sorting. @@ -843,25 +914,39 @@ def _gather_interleaved_history( keep_local = np.concatenate([lp_idx, like_idx, skip_idx]) if keep_local.size == 0: - empty = np.empty(0, dtype=np.int64) - return empty, empty, empty, empty, empty + return self._empty_history() order = np.argsort(keep_local, kind="stable") keep_local = keep_local[order] - items = item_ids[keep_local] - ts = timestamps[keep_local] - artists = self.item_to_artist[np.clip(items, 0, self.item_to_artist.shape[0] - 1)] - albums = self.item_to_album[np.clip(items, 0, self.item_to_album.shape[0] - 1)] + return self._materialize_history( + keep_local, item_ids, timestamps, is_lp, is_like, is_skip + ) - # Pool bitmask per kept event (LP/LIKE/SKIP are mutually exclusive in - # the source data, but OR is safe and forward-compatible). - weight = np.zeros(keep_local.shape[0], dtype=np.int64) - weight[is_lp[keep_local]] |= LP_BIT - weight[is_like[keep_local]] |= LIKE_BIT - weight[is_skip[keep_local]] |= SKIP_BIT + def _gather_last_n_history( + self, flat_pos: int, user_start: int + ) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + """Build the UIH from the last `history_length` events of ANY pool type + (listen+/like/skip) with no per-pool split. Vs the interleaved strategy + this fills the sequence to ~history_length (higher effective length) and + lets the like share fall to its natural corpus rate (~1.9%). Events + outside the 3 pools (dislike/unlike/undislike) are excluded as before.""" + L = self._history_length + scan = self._read_scan_window(flat_pos, user_start) + if scan is None: + return self._empty_history() + item_ids, timestamps, is_lp, is_like, is_skip = scan + + member = is_lp | is_like | is_skip + # Last L pooled events, in chronological order (already position-sorted + # within the scan window, so no re-sort is needed). + keep_local = np.arange(item_ids.shape[0], dtype=np.int64)[member][-L:] + if keep_local.size == 0: + return self._empty_history() - return items, artists, albums, ts, weight + return self._materialize_history( + keep_local, item_ids, timestamps, is_lp, is_like, is_skip + ) def _build_sample( self, flat_pos: int, max_num_candidates: int @@ -869,7 +954,7 @@ def _build_sample( uid = int(self.store.flat_uid[flat_pos]) user_start = int(self.store.user_start[uid]) - items, artists, albums, ts, weight = self._gather_interleaved_history( + items, artists, albums, ts, weight = self._gather_history( flat_pos, user_start ) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index e838b346f..41a2b6234 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -188,6 +188,43 @@ get_dataset.history_length = @hl/env_int() hl/env_int.key = "HISTORY_LENGTH" hl/env_int.default = 4086 +# UIH construction strategy — how a user's prior events become the sequence the +# model attends over. Override via $HISTORY_STRATEGY. Both strategies scan the +# last `scan_window` (20000) events before the anchor and consider ONLY the 3 +# behavior pools (listen+ / like / skip); dislike/unlike/undislike are excluded +# (no action-weight bit), so the model's action_weights=[1,2,4] are unchanged. +# The two differ only in HOW they pick events out of that scan window: +# +# "interleaved" (default): +# - Budget = equal per-pool quota of HISTORY_LENGTH//3 (=1362) events; take +# the last 1362 of EACH pool independently, then merge + re-sort by time. +# - HISTORY_LENGTH is thus a PER-POOL cap (nominal total = 3 * L//3). +# - Consequence: likes are only ~1.9% of the corpus, so the like pool +# under-fills (~105 of its 1362 slots) and that budget is NOT reallocated +# — the sequence comes up short. Measured: ~2663 events effective, ~4.0% +# of them likes (i.e. likes OVER-represented vs their natural rate). +# - Use when you want guaranteed like exposure per sample (class-balanced UIH). +# +# "last_n": +# - Budget = a single pool-agnostic cap: take the last HISTORY_LENGTH events +# of ANY pool type, already chronological (no per-pool split, no re-sort). +# - HISTORY_LENGTH is thus the LITERAL TOTAL UIH cap (not a per-pool L//3). +# - Consequence: the sequence fills to ~HISTORY_LENGTH (the only limit is how +# many pooled events exist in the scan window), so effective length is +# higher and the like share falls to its natural rate. Measured: ~4085 +# events effective (~1.5x interleaved), ~1.2% of them likes. +# - Trade-off: more recent listen+/skip context (longer effective sequence) +# at the cost of fewer likes; ~1.4x eval/step compute (more non-skipped +# jagged-attention positions). Keep HISTORY_LENGTH=4086 to fill the 4k +# model budget. +# +# Both strategies are strategy-INDEPENDENT on disk: the gather runs at +# sample-build time, so switching reuses the SAME hstu_cache_L/ +# and positions cache — no rebuild needed. +get_dataset.history_strategy = @hs/env_str() +hs/env_str.key = "HISTORY_STRATEGY" +hs/env_str.default = "interleaved" + # Anchor-eligibility floor: a LISTEN event becomes a trainable/eval anchor once # the user has >= MIN_HISTORY prior events. Decoupled from history_length (which # is only the gather/truncation cap) — jagged attention handles short UIH, so we diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py index 10c107bdc..7a805c195 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py @@ -1447,6 +1447,7 @@ def get_dataset( new_path_prefix: str = "", history_length: Optional[int] = None, min_history: Optional[int] = None, + history_strategy: str = "interleaved", streaming_window_seconds: int = 86400, streaming_sort_within_window: bool = False, streaming_shuffle_fraction: float = 0.0, @@ -1584,6 +1585,13 @@ def get_dataset( # short UIH. None = legacy (require a full history_length). # Override via `get_dataset.min_history = N` / $MIN_HISTORY. "min_history": min_history, + # UIH construction: "interleaved" (per-pool L//3 cap) or + # "last_n" (last history_length pooled events, no per-pool + # split). Strategy-independent on disk — both reuse the same + # hstu_cache_L/ and positions file (the gather + # runs at sample-construction time), so switching needs no + # rebuild. Override via $HISTORY_STRATEGY. + "history_strategy": history_strategy, "cross_specs": YAMBDA_5B_CROSS_SPECS, # Temporal-streaming knobs (only used under --mode # streaming-train-eval; ignored by the default train-eval path). From f14485a5cbaca45f2c057c0f7c3a68d4d5b770dd Mon Sep 17 00:00:00 2001 From: suachong Date: Wed, 24 Jun 2026 00:49:37 +0000 Subject: [PATCH 077/127] =?UTF-8?q?dlrmv4:=20logging-freeze=20prep=20?= =?UTF-8?q?=E2=80=94=20MIN=5FHISTORY=3D4086=20default=20+=20HISTORY=5FSTRA?= =?UTF-8?q?TEGY=20passthrough?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - gin: MIN_HISTORY default 0 -> 4086 (power-users floor at the full history budget; maps to the existing positions_L4086.npy cache, no rebuild/no shared-dir write). AUC_THRESHOLD left unchanged (0.80275) pending finalization. - launch_slurm.sh: forward $HISTORY_STRATEGY through the worker docker exec -e block (was silently dropped, so the knob never reached the worker); fix the stale lr-override echo (gin default is 1e-7, not 0.001). - README: document MIN_HISTORY default as 4086. Co-authored-by: Cursor --- recommendation_v4/README.MD | 4 ++-- .../generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin | 5 ++++- recommendation_v4/scripts/launch_slurm.sh | 3 ++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/recommendation_v4/README.MD b/recommendation_v4/README.MD index acd59cc31..d09a1567f 100644 --- a/recommendation_v4/README.MD +++ b/recommendation_v4/README.MD @@ -100,7 +100,7 @@ The `like` pool is roughly **30× rarer** than `lp` — important context for th ## 4. How data is fed to HSTU -For every training anchor (a LISTEN event with ≥ `min_history` prior events — default `1`, i.e. ~all users; set `$MIN_HISTORY=4086` for the legacy "full `history_length` of context required" filter that dropped ~60% of users), the dataset builds a `(uih_kjt, candidate_kjt)` pair: +For every training anchor (a LISTEN event with ≥ `min_history` prior events — frozen default `4086`, the "full `history_length` of context required" power-users filter; set `$MIN_HISTORY=0` to include ~all users plus their cold-start first event), the dataset builds a `(uih_kjt, candidate_kjt)` pair: ``` UIH (User Interaction History): @@ -203,7 +203,7 @@ with env overrides: | `PERSISTENT_LOADER` | `streaming_train_eval_loop.persistent_loader` | 1 | reuse one worker pool across windows (no per-window respawn) | | `DOUBLE_BUFFER` | `streaming_train_eval_loop.double_buffer` | 1 | prepare the next window in a background thread during compute | | `EVAL_EACH_WINDOW` | `streaming_train_eval_loop.eval_each_window` | 1 | eval window T+1 after training window T | -| `MIN_HISTORY` | `get_dataset.min_history` | 1 | anchor-eligibility floor: min prior events for a LISTEN to be a sample (1 = ~all users; 4086 = legacy full-history filter) | +| `MIN_HISTORY` | `get_dataset.min_history` | 4086 | anchor-eligibility floor: min prior events for a LISTEN to be a sample (frozen default 4086 = full-history power-users filter; 0 = ~all users incl. cold-start) | | — | `streaming_train_eval_loop.num_train_batches` / `num_eval_batches` | unset | cap per-window steps (unset = consume full window) | ### 5.3 Hiding the window-reset overhead diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index b4a675de8..7b7959c2b 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -239,7 +239,10 @@ hs/env_str.default = "interleaved" # keyed by (L, MIN_HISTORY) so floors don't collide. Override via $MIN_HISTORY. get_dataset.min_history = @mh/env_int() mh/env_int.key = "MIN_HISTORY" -mh/env_int.default = 0 +# Freeze default: power-users-only floor at the full history budget. With +# HISTORY_LENGTH=4086 this maps to the existing positions_L4086.npy cache (no +# rebuild, no shared-dir write — see yambda.py _positions_filename legacy path). +mh/env_int.default = 4086 # Model-side attention budget. Dataset truncates UIH to fit this value if # `history_length + contextual + candidate` would overflow. Override via diff --git a/recommendation_v4/scripts/launch_slurm.sh b/recommendation_v4/scripts/launch_slurm.sh index 7db407624..d543aacba 100755 --- a/recommendation_v4/scripts/launch_slurm.sh +++ b/recommendation_v4/scripts/launch_slurm.sh @@ -174,7 +174,7 @@ orchestrate() { echo "[$(date)] launch_slurm/orchestrate: job=${SLURM_JOB_ID:-?} nodes=${SLURM_JOB_NODELIST:-?} nnodes=${SLURM_NNODES:-1}" | tee -a "$LOG" echo "[$(date)] resolved SCRIPT_PATH=$SCRIPT_PATH REPO=$REPO" | tee -a "$LOG" echo "[$(date)] config: MODE=$MODE START_TS=$START_TS NUM_TRAIN_TS=$NUM_TRAIN_TS NUM_TRAIN_BATCHES=$NUM_TRAIN_BATCHES NUM_EVAL_BATCHES=$NUM_EVAL_BATCHES METRIC_LOG_FREQ=$METRIC_LOG_FREQ" | tee -a "$LOG" - echo "[$(date)] lr-override: DENSE_LR=${DENSE_LR:-} SPARSE_LR=${SPARSE_LR:-}" | tee -a "$LOG" + echo "[$(date)] lr-override: DENSE_LR=${DENSE_LR:-} SPARSE_LR=${SPARSE_LR:-}" | tee -a "$LOG" # Rendezvous resolved on the HOST (the container image has no SLURM client). MASTER_ADDR=$(scontrol show hostnames "$SLURM_JOB_NODELIST" 2>/dev/null | head -1) @@ -314,6 +314,7 @@ orchestrate() { ${DIAG_EMB_STEPS:+-e DIAG_EMB_STEPS=$DIAG_EMB_STEPS} \ ${OUTPUT_TRACE:+-e OUTPUT_TRACE=$OUTPUT_TRACE} \ ${MIN_HISTORY:+-e MIN_HISTORY=$MIN_HISTORY} \ + ${HISTORY_STRATEGY:+-e HISTORY_STRATEGY=$HISTORY_STRATEGY} \ ${SEED:+-e SEED=$SEED} \ ${DENSE_LR:+-e DENSE_LR=$DENSE_LR} \ ${SPARSE_LR:+-e SPARSE_LR=$SPARSE_LR} \ From 53f517628bd8c0339da677980f5f3b9d543601a2 Mon Sep 17 00:00:00 2001 From: suachong Date: Wed, 24 Jun 2026 00:54:47 +0000 Subject: [PATCH 078/127] dlrmv4: untrack docs/v4_vs_v2_and_hstu_walkthrough.md Stop tracking the local walkthrough doc (kept on disk, no longer in the repo). Co-authored-by: Cursor --- .../docs/v4_vs_v2_and_hstu_walkthrough.md | 534 ------------------ 1 file changed, 534 deletions(-) delete mode 100644 recommendation_v4/docs/v4_vs_v2_and_hstu_walkthrough.md diff --git a/recommendation_v4/docs/v4_vs_v2_and_hstu_walkthrough.md b/recommendation_v4/docs/v4_vs_v2_and_hstu_walkthrough.md deleted file mode 100644 index 4ef46c069..000000000 --- a/recommendation_v4/docs/v4_vs_v2_and_hstu_walkthrough.md +++ /dev/null @@ -1,534 +0,0 @@ -# recommendation_v4 (HSTU + Yambda-5b) — reference - -A walkthrough of what the proposed `recommendation_v4` MLPerf-training benchmark -is, how it differs from `recommendation_v2`, what the HSTU model is composed of, -and how to download the dataset and run training as-is. - -All claims below are grounded in code/config paths inside this tree. Every -numeric constant cites a `file:line` source. Where doc and source disagree, the -source wins and the discrepancy is called out. - ---- - -## 0. Sources of truth - -The following files were read to assemble this document. If you change any of -them, audit this doc against the change. - -- `training/recommendation_v2/torchrec_dlrm/README.MD` (v2 reference) -- `training/recommendation_v4/README.MD` (v4 fork overview) -- `training/recommendation_v4/docs/training_recipe.md` (v4 stacks/configs) -- `training/recommendation_v4/generative_recommenders/modules/stu.py` (HSTU layer) -- `training/recommendation_v4/generative_recommenders/modules/dlrm_hstu.py` (top-level `DlrmHSTU` + config dataclass) -- `training/recommendation_v4/generative_recommenders/modules/hstu_transducer.py` (preprocessor → STU stack → postprocessor) -- `training/recommendation_v4/generative_recommenders/ops/pytorch/pt_hstu_attention.py` (reference HSTU attention math in plain PyTorch) -- `training/recommendation_v4/generative_recommenders/ops/hstu_attention.py` (kernel dispatcher: PYTORCH / TRITON / TRITON_CC) -- `training/recommendation_v4/generative_recommenders/dlrm_v3/configs.py` (per-dataset HSTU config + embedding tables) -- `training/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin` (run config) -- `training/recommendation_v4/generative_recommenders/dlrm_v3/preprocess_public_data.py` (Yambda HuggingFace downloader/preprocessor) -- `training/recommendation_v4/generative_recommenders/dlrm_v3/datasets/yambda.py` (dataset feeding HSTU) -- `training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py` (metrics logger / `auc_threshold` consumer) -- `training/recommendation_v4/scripts/launch_smoke_8gpu.sh` (run wrapper) - ---- - -## 1. `recommendation_v4` vs `recommendation_v2` - -v4 is **not** an evolution of v2 — it replaces a tabular CTR benchmark -(DLRMv2 + DCN on Criteo 1 TB) with a **sequential generative-recommender -benchmark** (HSTU on Yandex Yambda-5b). Codebase, dataset, task, loss -labeling, hyperparameters, and software stack are all different. They share -basically nothing except the "recommendation" label. - -### 1.1 Upstream codebase / repo origin - -| | v2 | v4 | -|---|---|---| -| Upstream repo | `pytorch/torchrec` examples (DLRM) | fork of `meta-recsys/generative-recommenders` (`README.MD:3`) | -| Layout | single dir: `torchrec_dlrm/` with `dlrm_main.py` | full repo tree: `generative_recommenders/`, `configs/`, `scripts/`, `main.py`, `setup.py`, gin-driven | -| Config style | argparse CLI flags | gin-config files under `generative_recommenders/dlrm_v3/train/gin/` (e.g. `yambda_5b.gin`) | - -### 1.2 Model architecture - -| | v2 | v4 | -|---|---|---| -| Model | **DLRM v2** — dense MLP + sparse embeddings + feature interaction (paper: Naumov et al. 1906.00091) | **HSTU** — Hierarchical Sequential Transducer Units (ICML'24 *Actions Speak Louder than Words*) (`README.MD:3`) | -| Interaction arch | DCN v2: `--interaction_type=dcn --dcn_num_layers=3 --dcn_low_rank_dim=512` (`recommendation_v2/torchrec_dlrm/README.MD:167-169`) | Transformer-style sequential self-attention over a User Interaction History (UIH) of length 2048, jagged-attention TRITON kernel (`README.MD:114, 132`; `training_recipe.md:57, 71`) | -| Embedding dim | 128 (`recommendation_v2/torchrec_dlrm/README.MD:157`) | 512 (`dlrm_v3/configs.py:33, 353`) | -| Pipeline | TorchRec model-parallel embeddings + data-parallel MLP, overlapped (`recommendation_v2/torchrec_dlrm/README.MD:3`) | TorchRec sharded embeddings + HSTU ranker; per-GPU HBM cap 260 GiB MI350X / 150 GiB B200 (`training_recipe.md:59, 176`) | - -### 1.3 Dataset - -| | v2 | v4 | -|---|---|---| -| Dataset | **Criteo 1 TB click logs** → multi-hot preprocessed variant (~3.8 TB materialized) (`recommendation_v2/torchrec_dlrm/README.MD:142-146`) | **Yambda-5b** (Yandex music, HuggingFace `yandex/yambda`, 5b variant) (`README.MD:3, 28`) | -| Domain | CTR prediction on tabular ads features (26 categorical + 13 dense) | Sequential music-recommendation events (listen / like / skip / dislike / unlike / undislike) per-user timelines | -| Size | `TOTAL_TRAINING_SAMPLES=4,195,197,692` rows (`recommendation_v2/torchrec_dlrm/README.MD:153`) | 4.76 B events, 1.00 M users, 9.39 M items; 3.23 B usable training anchors (`README.MD:62-69`) | -| Storage layout | numpy contiguous shuffled `.npy` (or preprocessed multi-hot bin) | parquet: `train_sessions.parquet` 47 GB, `test_events.parquet` 152 MB, etc. (`README.MD:40-52`) | -| Preprocessing | `process_Criteo_1TB_Click_Logs_dataset.sh` — 700 GB RAM, 1–2 days, then `materialize_synthetic_multihot_dataset.py` | `generative_recommenders.dlrm_v3.preprocess_public_data --dataset yambda-5b` — ~53 min end-to-end for 5b (`README.MD:54`) | -| Embedding cardinalities | `num_embeddings_per_feature` 26-vec, top entries 40 M (`recommendation_v2/torchrec_dlrm/README.MD:161`) | item 9.39 M, artist 1.29 M, album 3.37 M, uid 1.00 M, + 7 cross-features up to 100 M (`dlrm_v3/configs.py:40-48, 686-722`) | -| Required pre-processor pkg | none unusual | **`polars-u64-idx`** because yambda-5b exceeds polars' 32-bit row index (`training_recipe.md:44, 102-103`) | - -### 1.4 Task formulation / supervision - -| | v2 | v4 | -|---|---|---| -| Task | binary CTR (click / no-click) | sequential next-action ranking: given UIH, predict whether the candidate LISTEN event will be a "listen_plus" (`played_ratio ≥ 50%`) (`README.MD:103`) | -| Label | Criteo click label | `action_weight` bitmask on the candidate; supervision masked to `(supervision_bitmask & task_weight) > 0` with `task_weight = 1` (LP bit) → only `listen_plus` candidates supervise (`README.MD:103`) | -| Loss | BCE | BCE on `listen_plus` task | - -### 1.5 Target metric - -| | v2 | v4 | -|---|---|---| -| Target | **AUROC ≥ 0.80275** within 1 epoch on Criteo (`recommendation_v2/torchrec_dlrm/README.MD:173`) | `MetricsLogger.auc_threshold = 0.80275` (`yambda_5b.gin:107`). Same numeric value as v2 — likely inherited from the upstream DLRM-DCNv2 reporting convention rather than independently chosen for HSTU. Consumed in `dlrm_v3/utils.py:587-608` to log `time_to_auc_0.80275_sec` as soon as the `listen_plus` task's AUC crosses the threshold. Confirm with the proposing team whether this is the intended final benchmark target or a placeholder. | - -### 1.6 Training hyperparameters - -| | v2 (MLPerf example, 8 GPU) | v4 (`yambda_5b.gin`, 8 GPU) | -|---|---|---| -| Global batch | 65,536 (`recommendation_v2/torchrec_dlrm/README.MD:154`) | **8,192** (`batch_size=1024 × world_size=8`) (`yambda_5b.gin:1, 44`). Note `docs/training_recipe.md:65, 182` shows `32 × 8 = 256` — that doc has drifted; the gin file is the launch source of truth. | -| Epochs | 1 (`recommendation_v2/torchrec_dlrm/README.MD:163`) | 1 (`yambda_5b.gin:81`) | -| Dense optimizer | Adagrad, lr 0.005 (`recommendation_v2/torchrec_dlrm/README.MD:170-171`) | **Adam**, lr 1e-3, betas (0.95, 0.999), eps 1e-8 (`yambda_5b.gin:19-24`) | -| Sparse optimizer | (Adagrad on embeddings via TorchRec) | **RowWiseAdagrad**, lr 1e-3, betas (0.95, 0.999), eps 1e-8 (`yambda_5b.gin:27-32`) | -| Precision | fp32 (no bf16 flag in v2 example) | **bf16** mixed precision, gated on the TRITON HSTU kernel (`yambda_5b.gin:8`; `training_recipe.md:58, 109-111`) | -| Sequence length | n/a (non-sequential model) | `history_length=2039`, `max_seq_len=2048` (`yambda_5b.gin:74, 78`) | - -### 1.7 Software stack - -| | v2 | v4 (MI350X) | v4 (B200) | -|---|---|---|---| -| Container | none specified (bare AWS p4d, CUDA 11.0, NCCL 2.10.3) (`recommendation_v2/torchrec_dlrm/README.MD:37`) | `rocm/primus:v26.3` (`training_recipe.md:24`) | `nvcr.io/nvidia/pytorch:26.04-py3` (`training_recipe.md:132`) | -| GPU target | A100 40 GB | **MI350X** (`gfx950`, ROCm 7.2.1, 288 GiB HBM3e) | **B200** (`sm_100`, ~183 GiB HBM) | -| torch | TorchRec example era; CUDA 11.0 | `2.12.0+rocm7.2` (`training_recipe.md:38`) | `2.12.0a0` native NGC (CUDA 13.2) (`training_recipe.md:149`) | -| triton | not central | `3.6.0` (image native; required for HSTU TRITON backend) (`training_recipe.md:41`) | `3.6.0` (`training_recipe.md:150`) | -| fbgemm_gpu | TorchRec default | `fbgemm_gpu_nightly_rocm-2026.6.2` built from FBGEMM `10b77573` for `gfx950` (`training_recipe.md:42`) | same SHA, built for `sm_100` (`training_recipe.md:151`) | -| torchrec | (whatever TorchRec was current) | `1.7.0a0+bf55480` (`v2026.06.01.00`) (`training_recipe.md:43`) | `1.7.0.dev20260601+cu130` (`training_recipe.md:152`) | -| Launcher | `torchx … dist.ddp` | `scripts/launch_smoke_8gpu.sh` | `scripts/launch_smoke_8gpu.sh` | -| Key kernel | TorchRec EmbeddingBag + DCN | **HSTU TRITON jagged-attention** (`HSTU_HAMMER_KERNEL=TRITON`) (`training_recipe.md:71`) | same (`training_recipe.md:188`) | - ---- - -## 2. HSTU model walkthrough - -### 2.1 What HSTU is, in one paragraph - -**HSTU = Hierarchical Sequential Transducer Units**, from the Meta paper -*Actions Speak Louder than Words* (ICML'24). It is a **decoder-only Transformer -variant**, redesigned for *recommendation* sequences (very long, very ragged, -heavy on categorical features). The block looks like a standard transformer -block superficially — attention + MLP — but two things are different from -GPT/SASRec attention: - -1. **Pointwise SiLU instead of softmax** in the attention non-linearity (no - log-sum-exp normalization). -2. **Gated output**: an extra projected stream `U` multiplies the attention - output before the residual. - -Everything else (residual connections, layer-norm, multi-head, positional -encoding, causal masking, KV-cache) is conventional transformer. The "S" in -STU = "Sequential Transducer Unit" = one HSTU block. - -### 2.2 The composition: top-level model (DLRM-v3 / `DlrmHSTU`) - -The full thing in `dlrm_hstu.py` is a small pipeline. Top-down: - -``` -KeyedJaggedTensor of raw ids - │ - ▼ -[1] TorchRec EmbeddingCollection (≈150 G sparse params, sharded across GPUs) - │ emits per-feature jagged embedding lookups - ▼ -[2] ContextualPreprocessor (interleaves UIH + appends candidate, adds - positional / action / timestamp encodings) - │ output: jagged sequence of length L per user, dim = transducer_embedding_dim - ▼ -[3] HSTUTransducer ── STUStack of N HSTULayers (the "HSTU" attention blocks) - │ output: contextualized per-position embedding - ▼ -[4] DefaultMultitaskModule (linear → BCE on listen_plus bit) - │ - ▼ -Per-anchor logit → BCE loss -``` - -For yambda-5b the per-dataset overrides in `dlrm_v3/configs.py:78-90, 346-425` -give: - -| component | value | source | -|---|---|---| -| embedding tables | `item_id` 9.39 M × 512, `artist_id` 1.29 M × 512, `album_id` 3.37 M × 512, `uid` 1.00 M × 512, + 7 cross-features (e.g. `user_x_artist` 100 M × 512) | `dlrm_v3/configs.py:686-722` | -| embedding dim | 512 (`HSTU_EMBEDDING_DIM`) | `dlrm_v3/configs.py:33, 353` | -| HSTU layers | **5** (`hstu_attn_num_layers=5`) | `dlrm_v3/configs.py:82` | -| attention heads | 4 | `dlrm_v3/configs.py:79` | -| Q/K dim per head | 128 | `dlrm_v3/configs.py:81` | -| V/U (linear) dim per head | 128 | `dlrm_v3/configs.py:80` | -| transducer embedding dim | 512 | `dlrm_v3/configs.py:85, 354` | -| dropout | input 0.2, linear 0.1 | `dlrm_v3/configs.py:87-88` | -| max attention budget (model) | 8192 (yambda default; gin further caps to 2048 via `get_hstu_configs.max_seq_len = 2048` in `yambda_5b.gin:78`) | `dlrm_v3/configs.py:355` | -| task | `listen_plus`, BINARY_CLASSIFICATION, BCE | `dlrm_v3/configs.py:419-424` | - -**Sparse-side parameter count, by table** (just the explicit ones; cross-features -add 282 M more rows × 512 dim ≈ 144 G params, which dominate): - -``` -item_id : 9_390_624 × 512 ≈ 4.81 B -artist_id : 1_293_395 × 512 ≈ 662 M -album_id : 3_367_692 × 512 ≈ 1.72 B -uid : 1_000_001 × 512 ≈ 512 M -crosses : ~282 M × 512 ≈ 144.4 B ← dominant -``` - -This is overwhelmingly an embedding-bound model — the dense HSTU stack (5 -layers × ~1 M parameters each) is a rounding error next to the embedding -tables, which is why `make_optimizer_and_shard.hbm_cap_gb = 260` and why -TorchRec sharding is central. - -### 2.3 Inside one STU (HSTU) layer - -From `modules/stu.py:182-246, 292-355`. A single STU layer holds **four** -weight matrices, not the usual two (QKV + out): - -``` -_uvqk_weight : (E, (hidden_dim·2 + attn_dim·2) · num_heads) -_uvqk_beta : (...,) bias for the above -_input_norm : LayerNorm(E) -_output_weight : (hidden_dim · num_heads · 3, E) -_output_norm : LayerNorm -``` - -Forward pass on input `x` of shape `[L, E]` (jagged): - -#### 2.3.1 Fused U/V/Q/K projection - -``` -normed = LayerNorm(x) -[U | V | Q | K] = normed @ _uvqk_weight + _uvqk_beta # one GEMM, then split - # U, V ∈ R^{H·hidden_dim} - # Q, K ∈ R^{H·attn_dim} -``` - -Compared to a regular transformer, you get an **extra projected stream `U`**. -`U` will gate the attention output later. - -#### 2.3.2 HSTU attention (the core difference vs softmax attention) - -Reference math, exactly as written in -`ops/pytorch/pt_hstu_attention.py:151, 167, 179, 182`: - -```python -qk_attn = einsum("bhxa,bhya->bhxy", Q, K) * alpha # alpha = 1 / sqrt(attn_dim) -qk_attn = F.silu(qk_attn) / max_seq_len # ← pointwise SiLU, scalar divide -qk_attn = qk_attn * valid_attn_mask # mask (see 2.3.3) -attn = einsum("bhxd,bhdv->bhxv", qk_attn, V) -``` - -Contrast with a vanilla transformer: - -```python -qk = (Q @ K.T) / sqrt(d) -qk = softmax(qk + mask, dim=-1) # ← softmax normalises rows -attn = qk @ V -``` - -Two consequences of dropping softmax: - -- **No row-wise normalization** → the per-key contribution is decoupled across - positions. The paper argues this is *better* for recommendation, because a - 5-year-old "like" event shouldn't have its weight diluted just because the - user has a longer history (which softmax would do). -- **Numerically more delicate**: the recipe warns *"`pt_hstu_attention`'s QK - einsum backward overflows in bf16 at N > 1k and produces NaN at step 1; bf16 - is only safe with TRITON"* (`docs/training_recipe.md:109-111`). The TRITON - kernel handles bf16 accumulation carefully; the reference PyTorch path - doesn't. - -#### 2.3.3 Custom attention mask (`_get_valid_attn_mask`, `pt_hstu_attention.py:32-84`) - -HSTU supports four mask-combination knobs simultaneously: - -- **causal**: lower triangle only (standard). -- **target-aware** (`num_targets`): the last `num_targets` positions are the - candidate targets; their "row index" is clamped so all targets see the same - prefix (the user's UIH) but cannot peek at each other. -- **max_attn_len** (sliding window): each position attends only to the previous - `max_attn_len` events — bounds the receptive field for very long histories. -- **contextual_seq_len**: the first `contextual_seq_len` positions are - *contextual* tokens (uid + cross-features). They are allowed to attend to - everything (and everything attends back to them), regardless of causal order. - This is how `uid` / `user_x_artist` etc. get full visibility despite living - at the head of the sequence. - -#### 2.3.4 Output: gated MLP - -From `stu.py:336-354` → `hstu_compute_output`: - -``` -y = SwishLayerNorm(attn) # SiLU(x · sigmoid(x)) then LN -y = concat([y, U]) @ _output_weight # gating with the U stream -y = y · x + dropout # residual back to original x -``` - -The `U · y` gating is the second non-standard piece. It is reminiscent of -GLU / SwiGLU but applied to the *attention output*, not just an MLP. - -#### 2.3.5 Stack - -`STUStack` (`stu.py:426`) is just `nn.ModuleList` of N `STULayer`s applied -sequentially with the same jagged-tensor convention. No cross-layer fanciness. - -### 2.4 "Transformer-style sequential attention over a UIH" — what the inputs actually look like - -UIH = **User Interaction History**. For yambda, the input to one training -sample is one **anchor LISTEN event** plus that user's history. From -`README.MD:88-101` and `dlrm_v3/configs.py:399-418`: - -``` -sequence position: 0 .. 7 | 8 .. (L-2) | L-1 - ─────────┼─────────────────────────────┼────────── -content: contextual│ UIH (interleaved 3 pools) │ candidate - │ │ -features per position: uid, 7 cross-features (length-1 each) - item_id, artist_id, album_id, - action_weight (LP/LIKE/SKIP bitmask), - action_timestamp, dummy_watch_time - candidate's: - item_candidate_id, - item_candidate_artist_id, - item_candidate_album_id, - item_query_time, - item_action_weight, - item_dummy_watchtime -``` - -The HSTU stack runs causal attention over this `L = 2048` sequence. The label -is the candidate's `listen_plus` bit (1 if `played_ratio ≥ 50%`, else 0), and -BCE is taken on the logit emitted at position `L-1`. So "transformer-style -sequential attention over UIH" literally means: the user's last ~2 k actions -are tokens, the candidate song is the last token, and a 5-layer HSTU -transformer predicts whether that candidate will be a `listen_plus`. - -This is the conceptual jump from DLRMv2: - -| | DLRMv2 (Criteo, v2) | HSTU (Yambda, v4) | -|---|---|---| -| Input shape | flat: 26 categorical + 13 dense features per ad impression | sequence of ~2 k past events per user, each a structured tuple | -| Mixing op | DCN: cross-products of feature vectors, then MLP | self-attention across positions (SiLU-gated, multi-head, causal) | -| Temporal modelling | none (each ad impression is i.i.d.) | central — masks, timestamps, action types are first-class | -| Depth | 1-shot (interaction arch + over-arch MLP) | 5 stacked HSTU blocks | -| "Why is the candidate good?" | low-rank cross of user/ad embeddings | attention over user's relevant past songs/artists/albums | - -DLRMv2 is *wide-and-shallow* over tabular features. HSTU is *narrow-and-deep* -over a temporal sequence. Different paradigm. - -### 2.5 Jagged attention — what it is and why it's used - -A user's history length varies — yambda median is 2,695 events, max is 27,738 -(`README.MD:65`). For a single training step you have a batch of B users with -very different sequence lengths. Two ways to lay this out on the GPU: - -**Padded layout (standard transformer):** - -``` -input shape: [B, N_max, D] e.g. [1024, 2048, 512] -``` - -This wastes compute proportional to `(N_max − N_user) / N_max` per row. On -yambda the average fill is ~1402/2037 ≈ 69%, so ~30% of every kernel is -multiplying zeros. - -**Jagged layout (what HSTU uses):** - -``` -flat values : [L_total, D] L_total = Σ user_lengths (≤ B · N_max) -offsets : [B + 1] cumulative starts, so user i occupies - values[offsets[i] : offsets[i+1]] -``` - -`pt_hstu_attention.py:148, 183` shows the round-trip: - -- `torch.ops.fbgemm.jagged_to_padded_dense(...)` only when calling into a dense - einsum -- `torch.ops.fbgemm.dense_to_jagged(...)` on the way out - -That's the reference path. The **TRITON jagged-attention kernel** -(`ops/triton/triton_hstu_attention.py`, dispatched in `ops/hstu_attention.py:27, -71`) skips the padded intermediate entirely: each Triton program handles one -user's `[N_user, N_user]` attention block directly, so: - -- **No wasted FLOPs.** Empty positions never enter a GEMM. -- **No wasted memory.** No padded `[B, H, N_max, N_max]` attention scores buffer - — that buffer alone would be `1024 · 4 · 2048 · 2048 · 2 bytes ≈ 34 GB` per - step (at global batch 1024 × bf16). -- **Variable-length backward is correct without masking tricks.** The kernel - iterates `[offsets[i], offsets[i+1])` per program; the gradient never touches - non-existent positions. - -This is *the* enabling optimization for the under-filled `like` pool to be -cheap. The README notes (`README.MD:132`): *"With the TRITON jagged-attention -backend the GPU only does work for the actual events, so the under-fill costs -sequence budget but not GPU compute"*. With a padded kernel, the unused 31% of -every sequence would cost real FLOPs. - -Practically: jagged attention is a generic technique (it shows up in -FlashAttention's varlen variants too); HSTU's TRITON kernel is its -specialization with SiLU + gated output + the four-way mask. - ---- - -## 3. Yambda-5b — size, contents, download, run - -### 3.1 What's in it - -[`yandex/yambda`](https://huggingface.co/datasets/yandex/yambda) on HuggingFace. -From `dlrm_v3/preprocess_public_data.py:233-245` + `README.MD:56-81`: - -| field | value | -|---|---| -| Provider | Yandex Music recommendation logs | -| Sizes | yambda-50m, yambda-500m, **yambda-5b** (v4 uses 5b) | -| Events | 4.76 B interactions across 300 days | -| Users | 1.00 M unique | -| Items | 9.39 M songs (+ 1.29 M artists, 3.37 M albums) | -| Event types | `listen` / `like` / `dislike` / `unlike` / `undislike` (encoded as uint8 0–4) | -| Listen events also carry | `played_ratio` (used to derive the `listen_plus` label at 50% threshold) | -| Train / test split | Global Temporal Split: 300 days train, 30-min gap, 1 day test | - -### 3.2 On-disk footprint after preprocessing (`README.MD:39-52`) - -``` -/ -├── raw/5b/multi_event.parquet 50 GB (downloaded) -├── shared_metadata/ -│ ├── artist_item_mapping.parquet 60 MB -│ ├── album_item_mapping.parquet 76 MB -│ └── embeddings.parquet 18 GB (unused by HSTU training) -└── processed_5b/ - ├── train_sessions.parquet 47 GB ← main training input - ├── test_events.parquet 152 MB - ├── session_index.parquet 600 MB - ├── item_popularity.npy 75 MB - └── split_meta.json anchor + boundary stats -``` - -Plan for **~115 GB free disk** to do everything end-to-end (raw + shared + -processed). If you skip the unused `embeddings.parquet` (which the script -downloads anyway), you still need ~97 GB. - -### 3.3 Download + preprocess - -Both happen in one command. Download is via the `datasets` library -(HuggingFace), so you need internet and `pip install datasets`. From -`dlrm_v3/preprocess_public_data.py:276-317`: - -```bash -pip install datasets polars-u64-idx pyarrow xxhash gin-config absl-py pandas - -export DLRM_DATA_PATH=/your/big/disk/dlrm_data -mkdir -p "$DLRM_DATA_PATH" - -cd /home/suachong/training/recommendation_v4 -python3 -m generative_recommenders.dlrm_v3.preprocess_public_data \ - --dataset yambda-5b \ - --data-path "$DLRM_DATA_PATH" -``` - -Per `README.MD:54`: **~53 minutes end-to-end** for the 5b variant on a -reasonable box. For a quick smoke test substitute `--dataset yambda-50m` -(~2 min, ~1 GB on disk). - -Critical: **install `polars-u64-idx`, not stock `polars`.** yambda-5b has ->4.29 B rows and overflows polars' default 32-bit row index silently -(`training_recipe.md:102-103`, `scripts/launch_smoke_8gpu.sh:13-20`). - -### 3.4 Run training (8-GPU smoke) - -From `scripts/launch_smoke_8gpu.sh` and `README.MD:9-22`. - -**Inside the validated container** (recommended; everything's pre-staged): - -```bash -docker exec yambda_8gpu bash -c \ - 'cd /workspace/recommendation_v4 && bash scripts/launch_smoke_8gpu.sh' -``` - -Override data path / run name without editing the gin: - -```bash -DLRM_DATA_PATH=/your/big/disk/dlrm_data \ -RUN_NAME=my_experiment \ -bash scripts/launch_smoke_8gpu.sh -``` - -**From scratch on a bare host**, you need to assemble the stack per -`docs/training_recipe.md`. The hard requirements are: - -- **ROCm path**: `rocm/primus:v26.3`, torch `2.12.0+rocm7.2`, triton `3.6.0`, - fbgemm_gpu built from commit `10b77573` for `gfx950`, torchrec - `1.7.0a0+bf55480`. See `training_recipe.md:30-45`. -- **CUDA path**: `nvcr.io/nvidia/pytorch:26.04-py3`, native torch (do NOT - reinstall), fbgemm_gpu built from same commit for `sm_100`, torchrec - `1.7.0.dev20260601+cu130`. See `training_recipe.md:147-155`. - -In both cases the actual launch is just: - -```bash -python -m generative_recommenders.dlrm_v3.train.train_ranker \ - --dataset yambda-5b --mode train-eval -``` - -(plus `HSTU_HAMMER_KERNEL=TRITON` for CUDA; `=PYTORCH` is forced on ROCm in -the smoke script because the Triton kernel hits PassManager errors on some -shapes there — see `scripts/launch_smoke_8gpu.sh:31-33`. The PYTORCH fallback -gives ~190 ms/step baseline, not the ~52 ms primus-pinned number.) - -### 3.5 What you'll see - -Per `training_recipe.md:84-91`, on 8× MI350X in the optimal config: - -- ~52 ms/step at global batch 256 (per the doc; gin says 8,192 — see §1.6 note) -- ~4,970 samples/sec -- ~7.6 days for one epoch over 3.23 B training anchors - -If `auc_threshold = 0.80275` is the real benchmark target (still TBD), -`time_to_auc_0.80275_sec` will be logged as soon as the eval AUC on -`listen_plus` crosses it (`dlrm_v3/utils.py:587-608`). - ---- - -## 4. TL;DR - -- **HSTU ≈ decoder-only Transformer** with two tweaks: SiLU/N replaces softmax - in attention, and a `U`-gated output replaces the standard MLP block. -- **DLRMv3 (yambda) = TorchRec embeddings → contextual preprocessor → 5 - stacked HSTU layers → BCE head on `listen_plus`.** Sparse tables (≈150 G - params) dominate the model; the dense HSTU stack is tiny by comparison. -- **UIH = user interaction history.** Each sample is one anchor LISTEN event - plus that user's last ~2 k events (LISTEN_PLUS / LIKE / SKIP, interleaved - chronologically, gathered with a `L//3`-per-pool cap), and HSTU does causal - self-attention across them. -- **Jagged attention** packs variable-length per-user sequences as - `(flat_values, offsets)` instead of padding to `N_max`, so the Triton kernel - never spends FLOPs on empty positions — essential because the average - sequence is only 69% full on yambda. -- **Yambda-5b** is a 4.76 B-event / 1 M-user Yandex Music dataset on - HuggingFace (`yandex/yambda`); downloading + preprocessing takes ~53 min and - ~115 GB disk; run via - `python -m generative_recommenders.dlrm_v3.train.train_ranker --dataset yambda-5b --mode train-eval` - (or `scripts/launch_smoke_8gpu.sh`). - ---- - -## 5. Open questions to bring back to the proposing team - -1. **Target metric.** `yambda_5b.gin:107` reuses DLRMv2's `0.80275` AUC - threshold. Is this the intended final v4 benchmark target, or a placeholder - inherited from upstream? An HSTU model on a different dataset would - normally need its own threshold chosen from a reference run. -2. **Batch size canonicalization.** `yambda_5b.gin:1` = `1024` per rank - (global 8,192); `docs/training_recipe.md:65, 182` says `32` per rank - (global 256). Which is the submission config? -3. **Convergence reference runs.** No `reference_results.md`-style table exists - yet under `training/recommendation_v4`. Submission-quality v4 will need - reference epochs-to-target numbers per dataset variant. From 0f7a6ebb89ab1e4bd9595d3f8ede0bb69d17276e Mon Sep 17 00:00:00 2001 From: suachong Date: Wed, 24 Jun 2026 00:58:59 +0000 Subject: [PATCH 079/127] dlrmv4: scrub hardcoded username from reference comments Replace the example username in launch_slurm.sh / streaming_resume_test.sh comments with a generic placeholder. Runtime defaults already derive the container name + mounts from $USER, so only doc/example strings changed. Co-authored-by: Cursor --- .../dlrm_v3/train/tests/streaming_resume_test.sh | 2 +- recommendation_v4/scripts/launch_slurm.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh b/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh index a497959a0..c7bb5bab9 100755 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh @@ -17,7 +17,7 @@ # # Usage: # bash generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh --jobid -# [--container yambda_suachong] +# [--container yambda_] # [--num-train-batches 200] # [--die-at-step 350] # [--keep] # retain LOG_DIR + CKPT after run for inspection diff --git a/recommendation_v4/scripts/launch_slurm.sh b/recommendation_v4/scripts/launch_slurm.sh index d543aacba..f5e07ad0a 100755 --- a/recommendation_v4/scripts/launch_slurm.sh +++ b/recommendation_v4/scripts/launch_slurm.sh @@ -62,9 +62,9 @@ # # B) Filesystems (must be shared/NFS across ALL nodes — this script re-invokes # itself and reads the overlay + data from these paths cluster-wide) -# - REPO_MOUNT (repo + this script, e.g. /home/suachong) is bind-mounted rw; +# - REPO_MOUNT (repo + this script, e.g. /home/) is bind-mounted rw; # DATA_MOUNT (e.g. /apps/chcai) holds the read-only dataset + overlay + -# baked tar + pip tarball; SCRATCH (e.g. /home/suachong/yambda_runs) is the +# baked tar + pip tarball; SCRATCH (e.g. /home//yambda_runs) is the # writable log/output root. Override any via env — nothing is user-hardwired. # # C) Container image / GPU software stack (tied to the GPU arch + ROCm version) From 31a38facc5e8a5a586863586dbed5b5eea3a8732 Mon Sep 17 00:00:00 2001 From: suachong Date: Wed, 24 Jun 2026 01:06:18 +0000 Subject: [PATCH 080/127] =?UTF-8?q?dlrmv4:=20README=20=E2=80=94=20add=20fu?= =?UTF-8?q?ll=20single/multi-node=20reference=20run=20example?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Document a complete sbatch launch (run-shape + data-fraction eval cadence) for 1-node and 2-node, noting the launchers differ only in --nodes. Co-authored-by: Cursor --- recommendation_v4/README.MD | 45 +++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/recommendation_v4/README.MD b/recommendation_v4/README.MD index d09a1567f..ed8a9ace4 100644 --- a/recommendation_v4/README.MD +++ b/recommendation_v4/README.MD @@ -39,6 +39,51 @@ bash scripts/launch_slurm.sh Data path resolves at runtime via `env_path` gin macros (see [yambda_5b.gin](generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin)). Traces and any per-run outputs land in `results//`. +### 1.1 Full reference run (single + multi-node) + +For a complete benchmark run you typically wrap the `sbatch` call in a small +submit script that pins the run-shape + eval cadence (most other +hyperparameters already match the [yambda_5b.gin](generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin) +defaults, so only the knobs below need to be set). The single- and multi-node +launchers are identical except for `--nodes` (the trainer auto-derives +`NNODES`/`NODE_RANK`/`MASTER_ADDR`/`WORLD_SIZE` from SLURM): + +**Single node (1×8 GPU):** + +```bash +cd +MODE=streaming-train-eval \ + START_TS=0 NUM_TRAIN_TS=299 \ + TRAIN_SPLIT_PERCENTAGE=1.0 \ + EVAL_HOLDOUT_TS=299 EVAL_HOLDOUT_NUM_WINDOWS=1 \ + EVAL_EVERY_N_WINDOWS=0 EVAL_EVERY_DATA_PCT=0.005 \ + AUC_THRESHOLD=0.885 \ + BATCH_SIZE=1024 \ + METRIC_LOG_FREQ=20 \ + RUN_NAME=yc-full-epoch \ + LOG="$HOME/yambda_runs/yc-full-epoch/run.log" \ + sbatch --nodes=1 --time=4-00:00:00 --job-name=yambda-1node scripts/launch_slurm.sh +``` + +**Multi-node (2×8 GPU):** same env, only the `sbatch` line changes: + +```bash + ... same env knobs as above ... + RUN_NAME=yc-full-epoch-2node \ + LOG="$HOME/yambda_runs/yc-full-epoch-2node/run.log" \ + sbatch --nodes=2 --ntasks-per-node=1 --time=4-00:00:00 --job-name=yambda-2node scripts/launch_slurm.sh +``` + +Notes: +- `EVAL_EVERY_DATA_PCT=0.005` + `EVAL_EVERY_N_WINDOWS=0` selects the + data-fraction eval cadence (eval every 0.5% of the training stream — a fixed + number of samples between evals, independent of node count). The two eval + cadences are mutually exclusive. +- `AUC_THRESHOLD` is the convergence target: a SUCCESS `RUN_STOP` fires when the + per-pass `window_auc` first reaches it. +- Keep all run outputs (`LOG`, checkpoints, mllog, TensorBoard) under a writable + scratch path you own; the dataset mount is read-only. + ## 2. Data preparation ```bash From 3cd9f6d623694e803354ad0611282bd94cd83fee Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Wed, 24 Jun 2026 01:12:10 +0000 Subject: [PATCH 081/127] dlrmv4: exclude eval/checkpoint overhead from step_ms timing MetricsLogger now brackets eval and checkpoint phases with pause/resume perf timers so the reported step_ms reflects pure train-step latency. Adds wall_step_ms (inclusive), eval_ms, and ckpt_ms to the perf log line and TensorBoard scalars (appended for parser backward-compat). Checkpoint saves and eval windows are wrapped with categorized pause/resume calls. Co-authored-by: Cursor --- .../dlrm_v3/checkpoint.py | 18 +++++- .../dlrm_v3/train/utils.py | 10 ++++ .../generative_recommenders/dlrm_v3/utils.py | 59 ++++++++++++++++++- 3 files changed, 84 insertions(+), 3 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/checkpoint.py b/recommendation_v4/generative_recommenders/dlrm_v3/checkpoint.py index 0ef223b23..bd113959a 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/checkpoint.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/checkpoint.py @@ -25,6 +25,7 @@ import os import random import shutil +import time from datetime import datetime from typing import Any, Dict, Optional, Set, Tuple @@ -345,6 +346,15 @@ def save_dmp_checkpoint( """ if path == "": return + # Exclude checkpoint wall-time from the train step-time window so step_ms + # reports canonical compute latency; the duration is surfaced separately + # (window_ckpt_time_ms + the per-save log below). pause/resume are no-ops if + # metric_logger is None. Not wrapped in try/finally: a save that raises + # crashes the process (supervisor restarts fresh), so a dangling pause on + # the soon-dead logger is irrelevant. + _t_ckpt_start = time.perf_counter() + if metric_logger is not None: + metric_logger.pause_perf("ckpt") base_path = path # Atomic-save layout: write to .tmp, rename to final, prune older. tmp_subdir = f"{base_path}/{batch_idx}.tmp" @@ -460,8 +470,14 @@ def save_dmp_checkpoint( else: os.replace(tmp_subdir, final_subdir) _prune_old_checkpoints(base_path, keep_last_n, final_subdir) - logger.info("checkpoint successfully saved → %s", final_subdir) + logger.info( + "checkpoint successfully saved → %s (wall-time %.2fs)", + final_subdir, + time.perf_counter() - _t_ckpt_start, + ) torch.distributed.barrier() + if metric_logger is not None: + metric_logger.resume_perf("ckpt") @gin.configurable diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py index 785320020..d460c9d07 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -1102,6 +1102,8 @@ def eval_loop( # lr_scheduler: to-do: Add a scheduler ) -> None: model.eval() + # Exclude eval wall-time from the train step-time window (see _run_eval_window). + metric_logger.pause_perf("eval") batch_idx: int = 0 profiler = Profiler(rank) if output_trace else None metric_logger.reset(mode="eval") @@ -1139,6 +1141,7 @@ def eval_loop( metric_logger.compute_and_log(mode="eval") for k, v in metric_logger.compute(mode="eval").items(): print(f"{k}: {v}") + metric_logger.resume_perf("eval") class _PipelineModelWrapper(torch.nn.Module): @@ -2012,6 +2015,12 @@ def _run_eval_window(eval_data_iterator, label: Optional[str] = None) -> None: # only fires after a completed eval window or mid-train-window, so any # restored state always sits on a completed-eval boundary -- which is # also why the eval reset below is safe across resume. + # + # Exclude this eval pass's wall-time from the train step-time window so + # step_ms stays canonical even when eval coincides with a train interval; + # the duration is reported separately (window_eval_time_ms + total_eval + # below). Resumed unconditionally at the end of this function. + metric_logger.pause_perf("eval") model.eval() # Reset eval metrics so each pass reports a clean number over the FIXED # holdout set. Without this, lifetime/window eval metrics would keep @@ -2105,6 +2114,7 @@ def _run_eval_window(eval_data_iterator, label: Optional[str] = None) -> None: _f.write(json.dumps(_rec) + "\n") except OSError as _e: logger.warning("failed to write metrics sink %s: %s", _metrics_path, _e) + metric_logger.resume_perf("eval") def _maybe_checkpoint(train_ts: int) -> None: if ( diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py index 7a805c195..506790d0b 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py @@ -1053,6 +1053,32 @@ def _make_reg(ws: int) -> List[RecMetricComputation]: self._perf_samples_counter: torch.Tensor = torch.zeros( 1, dtype=torch.long, device=device ) + # Non-train wall-time to exclude from the train step-time window so + # `step_ms` reports the canonical per-step compute latency even when an + # interval coincides with eval or checkpointing. Categorized so the + # excluded time is also reportable (eval_ms / ckpt_ms) rather than just + # discarded. The trainer brackets eval/ckpt regions via + # pause_perf(cat)/resume_perf(cat); accumulators reset each train-perf log. + self._perf_excluded: Dict[str, float] = {"eval": 0.0, "ckpt": 0.0} + self._perf_pause: Dict[str, Optional[float]] = {} + + def pause_perf(self, category: str) -> None: + """Start excluding wall-time under `category` (e.g. "eval"/"ckpt") from + the train step-time window. Idempotent: a second pause without an + intervening resume is a no-op (keeps the earliest start).""" + if self._perf_pause.get(category) is None: + self._perf_pause[category] = time.perf_counter() + + def resume_perf(self, category: str) -> None: + """Stop excluding `category` and fold the elapsed interval into the + per-category accumulator. No-op if not currently paused.""" + t0 = self._perf_pause.get(category) + if t0 is not None: + self._perf_excluded[category] = ( + self._perf_excluded.get(category, 0.0) + + (time.perf_counter() - t0) + ) + self._perf_pause[category] = None @property def all_metrics(self) -> Dict[str, List[RecMetricComputation]]: @@ -1210,12 +1236,22 @@ def compute_and_log( # Throughput metrics (train only). One GPU->CPU sync per call. if mode == "train" and self._perf_steps_in_window > 0: now = time.perf_counter() - dt = max(now - self._perf_t_window, 1e-6) + wall_dt = max(now - self._perf_t_window, 1e-6) + # Subtract bracketed eval/checkpoint wall-time so step_ms / sps / + # MFU reflect canonical train-step compute, not eval+ckpt stalls + # that happened to land in this window. The excluded time is also + # surfaced separately below (eval_ms / ckpt_ms) rather than discarded. + eval_s = self._perf_excluded.get("eval", 0.0) + ckpt_s = self._perf_excluded.get("ckpt", 0.0) + dt = max(wall_dt - eval_s - ckpt_s, 1e-6) n_samples = int(self._perf_samples_counter.item()) self._perf_total_samples += n_samples local_sps = n_samples / dt global_sps = local_sps * self._world_size step_ms = dt * 1000.0 / self._perf_steps_in_window + wall_step_ms = wall_dt * 1000.0 / self._perf_steps_in_window + eval_ms = eval_s * 1000.0 + ckpt_ms = ckpt_s * 1000.0 elapsed = now - self._perf_t_start step = self.global_step["train"] self.tb_logger.add_scalar( @@ -1227,6 +1263,17 @@ def compute_and_log( self.tb_logger.add_scalar( "perf/train_step_time_ms", step_ms, global_step=step ) + # Inclusive (old-semantics) per-step wall time and the eval/ckpt + # breakdown that was excluded from step_ms above. + self.tb_logger.add_scalar( + "perf/train_wall_step_time_ms", wall_step_ms, global_step=step + ) + self.tb_logger.add_scalar( + "perf/window_eval_time_ms", eval_ms, global_step=step + ) + self.tb_logger.add_scalar( + "perf/window_ckpt_time_ms", ckpt_ms, global_step=step + ) self.tb_logger.add_scalar( "perf/train_total_samples", self._perf_total_samples, global_step=step ) @@ -1269,12 +1316,20 @@ def compute_and_log( logger.info( f"train - Step {step} perf: local_sps={local_sps:.1f} " f"global_sps={global_sps:.1f} step_ms={step_ms:.2f} " - f"elapsed_sec={elapsed:.1f} total_samples={self._perf_total_samples}" + f"elapsed_sec={elapsed:.1f} total_samples={self._perf_total_samples} " + f"wall_step_ms={wall_step_ms:.2f} eval_ms={eval_ms:.1f} " + f"ckpt_ms={ckpt_ms:.1f}" + tflops_str ) self._perf_t_window = now self._perf_steps_in_window = 0 self._perf_samples_counter.zero_() + # Reset the excluded-time accumulators for the next window. Any + # still-open pause (eval/ckpt straddling this log) is cleared so its + # remaining time is not double-counted; in practice perf logs fire + # only after a train step, never mid eval/ckpt. + self._perf_excluded = {"eval": 0.0, "ckpt": 0.0} + self._perf_pause = {} # Time-to-target: latch wall-clock once any task's AUC crosses threshold. # Matches MLPerf DLRM-DCNv2 reporting style (default upstream target 0.80275). From dc412ffd6d852d0ac91943f237801e5607e96095 Mon Sep 17 00:00:00 2001 From: suachong Date: Wed, 24 Jun 2026 01:13:14 +0000 Subject: [PATCH 082/127] =?UTF-8?q?dlrmv4:=20README=20=E2=80=94=20use=20AU?= =?UTF-8?q?C=5FTHRESHOLD=3D0.80275=20in=20example=20for=20gin-default=20co?= =?UTF-8?q?nsistency?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Cursor --- recommendation_v4/README.MD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recommendation_v4/README.MD b/recommendation_v4/README.MD index ed8a9ace4..c8df7b272 100644 --- a/recommendation_v4/README.MD +++ b/recommendation_v4/README.MD @@ -57,7 +57,7 @@ MODE=streaming-train-eval \ TRAIN_SPLIT_PERCENTAGE=1.0 \ EVAL_HOLDOUT_TS=299 EVAL_HOLDOUT_NUM_WINDOWS=1 \ EVAL_EVERY_N_WINDOWS=0 EVAL_EVERY_DATA_PCT=0.005 \ - AUC_THRESHOLD=0.885 \ + AUC_THRESHOLD=0.80275 \ BATCH_SIZE=1024 \ METRIC_LOG_FREQ=20 \ RUN_NAME=yc-full-epoch \ From fad177f2fa768de976851b36fef0f5e9385ae13e Mon Sep 17 00:00:00 2001 From: suachong Date: Wed, 24 Jun 2026 01:27:44 +0000 Subject: [PATCH 083/127] dlrmv4: make a bare sbatch reproduce the frozen reference run orchestrate() now defaults to the reference run-shape (START_TS=0, NUM_TRAIN_TS=299, full windows) + the data-fraction eval cadence (EVAL_EVERY_DATA_PCT=0.005, per-window off), so `sbatch scripts/launch_slurm.sh` needs no env knobs. SMOKE=1 restores the previous fast functional defaults (short window, capped batches, per-window eval). The two eval cadences are auto-deconflicted (explicit EVAL_EVERY_N_WINDOWS>0 disables data-pct). gin library defaults + the resume/local smoke paths are unchanged. README updated to the bare single/multi-node commands. Co-authored-by: Cursor --- recommendation_v4/README.MD | 80 ++++++++++------------- recommendation_v4/scripts/launch_slurm.sh | 46 ++++++++++--- 2 files changed, 69 insertions(+), 57 deletions(-) diff --git a/recommendation_v4/README.MD b/recommendation_v4/README.MD index c8df7b272..6476e7729 100644 --- a/recommendation_v4/README.MD +++ b/recommendation_v4/README.MD @@ -11,17 +11,27 @@ auto-detects its context: run inside the container it takes the single-node worker path; submitted via `sbatch` it orchestrates the multi-node run (provision + per-node launch). N=1 is byte-for-byte the legacy single-node path. -**Single node (8-GPU), inside the container:** +A bare submit reproduces the **frozen reference run** (full 299-window sweep + +data-fraction eval cadence) — all run-shape/cadence defaults are baked in, so no +env knobs are required: + +**Single node (8-GPU):** ```bash -docker exec yambda_8gpu bash -c \ - 'cd /workspace/recommendation_v4 && bash scripts/launch_slurm.sh' +sbatch --nodes=1 scripts/launch_slurm.sh ``` -**Multi-node (N×8-GPU) via SLURM:** +**Multi-node (N×8-GPU):** ```bash -sbatch --nodes=2 --partition=meta64 scripts/launch_slurm.sh +sbatch --nodes=2 scripts/launch_slurm.sh +``` + +For a fast functional check instead of a full run, prepend `SMOKE=1` (short +window, capped batches, per-window eval): + +```bash +SMOKE=1 sbatch --nodes=1 scripts/launch_slurm.sh ``` Multi-node uses real RDMA (RoCEv2). The fabric/NCCL setup and every @@ -39,50 +49,26 @@ bash scripts/launch_slurm.sh Data path resolves at runtime via `env_path` gin macros (see [yambda_5b.gin](generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin)). Traces and any per-run outputs land in `results//`. -### 1.1 Full reference run (single + multi-node) - -For a complete benchmark run you typically wrap the `sbatch` call in a small -submit script that pins the run-shape + eval cadence (most other -hyperparameters already match the [yambda_5b.gin](generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin) -defaults, so only the knobs below need to be set). The single- and multi-node -launchers are identical except for `--nodes` (the trainer auto-derives -`NNODES`/`NODE_RANK`/`MASTER_ADDR`/`WORLD_SIZE` from SLURM): - -**Single node (1×8 GPU):** +### 1.1 The frozen reference shape -```bash -cd -MODE=streaming-train-eval \ - START_TS=0 NUM_TRAIN_TS=299 \ - TRAIN_SPLIT_PERCENTAGE=1.0 \ - EVAL_HOLDOUT_TS=299 EVAL_HOLDOUT_NUM_WINDOWS=1 \ - EVAL_EVERY_N_WINDOWS=0 EVAL_EVERY_DATA_PCT=0.005 \ - AUC_THRESHOLD=0.80275 \ - BATCH_SIZE=1024 \ - METRIC_LOG_FREQ=20 \ - RUN_NAME=yc-full-epoch \ - LOG="$HOME/yambda_runs/yc-full-epoch/run.log" \ - sbatch --nodes=1 --time=4-00:00:00 --job-name=yambda-1node scripts/launch_slurm.sh -``` +The reference run-shape and eval cadence are the built-in defaults (set in the +orchestrate phase of `scripts/launch_slurm.sh`), so the bare `sbatch` commands +above ARE the reference run. The single- and multi-node launchers are identical +except for `--nodes`; the trainer auto-derives `NNODES`/`NODE_RANK`/`MASTER_ADDR`/ +`WORLD_SIZE` from SLURM. The baked-in shape: -**Multi-node (2×8 GPU):** same env, only the `sbatch` line changes: - -```bash - ... same env knobs as above ... - RUN_NAME=yc-full-epoch-2node \ - LOG="$HOME/yambda_runs/yc-full-epoch-2node/run.log" \ - sbatch --nodes=2 --ntasks-per-node=1 --time=4-00:00:00 --job-name=yambda-2node scripts/launch_slurm.sh -``` - -Notes: -- `EVAL_EVERY_DATA_PCT=0.005` + `EVAL_EVERY_N_WINDOWS=0` selects the - data-fraction eval cadence (eval every 0.5% of the training stream — a fixed - number of samples between evals, independent of node count). The two eval - cadences are mutually exclusive. -- `AUC_THRESHOLD` is the convergence target: a SUCCESS `RUN_STOP` fires when the - per-pass `window_auc` first reaches it. -- Keep all run outputs (`LOG`, checkpoints, mllog, TensorBoard) under a writable - scratch path you own; the dataset mount is read-only. +| knob | reference default | +|---|---| +| `START_TS` / `NUM_TRAIN_TS` | 0 / 299 (full sweep) | +| eval cadence | `EVAL_EVERY_DATA_PCT=0.005` (every 0.5% of the training stream — a fixed number of samples between evals, independent of node count), per-window cadence off | +| `NUM_TRAIN_BATCHES` / `NUM_EVAL_BATCHES` | 0 / 0 (consume full windows) | + +To customize, override any knob via env (e.g. `RUN_NAME=...`, `LOG=...`, +`AUC_THRESHOLD=...`). Selecting the per-window eval cadence +(`EVAL_EVERY_N_WINDOWS>0`) automatically disables the data-fraction one (they are +mutually exclusive). Keep all run outputs (`LOG`, checkpoints, mllog, +TensorBoard) under a writable scratch path you own — the dataset mount is +read-only. ## 2. Data preparation diff --git a/recommendation_v4/scripts/launch_slurm.sh b/recommendation_v4/scripts/launch_slurm.sh index f5e07ad0a..e81510434 100755 --- a/recommendation_v4/scripts/launch_slurm.sh +++ b/recommendation_v4/scripts/launch_slurm.sh @@ -34,9 +34,14 @@ # supervisor's direct `bash scripts/launch_slurm.sh` is unchanged. # # USAGE -# Multi-node (N>=1): sbatch --nodes=2 scripts/launch_slurm.sh +# Reference run (1 node): sbatch --nodes=1 scripts/launch_slurm.sh +# Reference run (N node): sbatch --nodes=N scripts/launch_slurm.sh +# ^ a bare submit reproduces the FROZEN REFERENCE shape (full 299-window +# sweep + data-fraction eval cadence). Prepend SMOKE=1 for a fast +# functional check (short window, capped batches). # Single-node direct: bash scripts/launch_slurm.sh (already inside container; -# what run_streaming_e2e.sh invokes per relaunch) +# what run_streaming_e2e.sh invokes per relaunch — uses the +# gin defaults, NOT the orchestrate reference shape) # Perf pair: # LOG=/apps/chcai/perf_1node.log NUM_TRAIN_BATCHES=200 NUM_EVAL_BATCHES=0 \ # EVAL_EACH_WINDOW=0 METRIC_LOG_FREQ=20 \ @@ -146,15 +151,36 @@ orchestrate() { mkdir -p "$SCRATCH" 2>/dev/null || true LOG=${LOG:-$SCRATCH/yambda_slurm.${SLURM_JOB_ID:-manual}.log} - # Smoke defaults — override via env for a perf run (see header USAGE). + # Run-shape defaults. By DEFAULT a bare `sbatch scripts/launch_slurm.sh` + # reproduces the FROZEN REFERENCE run: full 299-window sweep (START_TS=0) with + # the data-fraction eval cadence (eval every 0.5% of the training stream). Set + # SMOKE=1 for a fast functional check (short dense window, capped batches, + # per-window eval). Any individual knob below stays env-overridable. MODE=${MODE:-streaming-train-eval} - START_TS=${START_TS:-150} - NUM_TRAIN_TS=${NUM_TRAIN_TS:-1} - NUM_TRAIN_BATCHES=${NUM_TRAIN_BATCHES:-20} - NUM_EVAL_BATCHES=${NUM_EVAL_BATCHES:-10} + if [ "${SMOKE:-0}" = "1" ]; then + START_TS=${START_TS:-150} + NUM_TRAIN_TS=${NUM_TRAIN_TS:-1} + NUM_TRAIN_BATCHES=${NUM_TRAIN_BATCHES:-20} + NUM_EVAL_BATCHES=${NUM_EVAL_BATCHES:-10} + EVAL_EVERY_N_WINDOWS=${EVAL_EVERY_N_WINDOWS:-1} + METRIC_LOG_FREQ=${METRIC_LOG_FREQ:-5} + fi + START_TS=${START_TS:-0} + NUM_TRAIN_TS=${NUM_TRAIN_TS:-299} + NUM_TRAIN_BATCHES=${NUM_TRAIN_BATCHES:-0} + NUM_EVAL_BATCHES=${NUM_EVAL_BATCHES:-0} EVAL_EACH_WINDOW=${EVAL_EACH_WINDOW:-1} - EVAL_EVERY_N_WINDOWS=${EVAL_EVERY_N_WINDOWS:-1} - METRIC_LOG_FREQ=${METRIC_LOG_FREQ:-5} + METRIC_LOG_FREQ=${METRIC_LOG_FREQ:-20} + # Eval cadence — the two knobs are mutually exclusive (the worker raises if both + # are >0). Data-fraction is the reference default; if the caller explicitly + # selected the per-window cadence (EVAL_EVERY_N_WINDOWS>0) leave data-pct off, + # otherwise default to the reference 0.5%-of-data cadence (per-window disabled). + if [ "${EVAL_EVERY_N_WINDOWS:-0}" -gt 0 ] 2>/dev/null; then + EVAL_EVERY_DATA_PCT=${EVAL_EVERY_DATA_PCT:-0} + else + EVAL_EVERY_N_WINDOWS=0 + EVAL_EVERY_DATA_PCT=${EVAL_EVERY_DATA_PCT:-0.005} + fi FORCE_PROVISION=${FORCE_PROVISION:-0} # Truncate the metrics log on a FRESH run; APPEND on a supervised relaunch @@ -173,7 +199,7 @@ orchestrate() { chmod 666 "$LOG" 2>/dev/null || true echo "[$(date)] launch_slurm/orchestrate: job=${SLURM_JOB_ID:-?} nodes=${SLURM_JOB_NODELIST:-?} nnodes=${SLURM_NNODES:-1}" | tee -a "$LOG" echo "[$(date)] resolved SCRIPT_PATH=$SCRIPT_PATH REPO=$REPO" | tee -a "$LOG" - echo "[$(date)] config: MODE=$MODE START_TS=$START_TS NUM_TRAIN_TS=$NUM_TRAIN_TS NUM_TRAIN_BATCHES=$NUM_TRAIN_BATCHES NUM_EVAL_BATCHES=$NUM_EVAL_BATCHES METRIC_LOG_FREQ=$METRIC_LOG_FREQ" | tee -a "$LOG" + echo "[$(date)] config: MODE=$MODE START_TS=$START_TS NUM_TRAIN_TS=$NUM_TRAIN_TS NUM_TRAIN_BATCHES=$NUM_TRAIN_BATCHES NUM_EVAL_BATCHES=$NUM_EVAL_BATCHES METRIC_LOG_FREQ=$METRIC_LOG_FREQ SMOKE=${SMOKE:-0} EVAL_EVERY_N_WINDOWS=$EVAL_EVERY_N_WINDOWS EVAL_EVERY_DATA_PCT=$EVAL_EVERY_DATA_PCT" | tee -a "$LOG" echo "[$(date)] lr-override: DENSE_LR=${DENSE_LR:-} SPARSE_LR=${SPARSE_LR:-}" | tee -a "$LOG" # Rendezvous resolved on the HOST (the container image has no SLURM client). From b50fe5876fce2d90b3678ab95cd1a12f929f9f6d Mon Sep 17 00:00:00 2001 From: suachong Date: Wed, 24 Jun 2026 16:32:20 +0000 Subject: [PATCH 084/127] =?UTF-8?q?dlrmv4:=20address=20PR=20review=20?= =?UTF-8?q?=E2=80=94=20mlperf=5Flogging=20install/pin=20+=20logging-utils?= =?UTF-8?q?=20robustness?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Dockerfiles (AMD+NVIDIA): install mlperf_logging (--no-deps) so compliance logging is not silently disabled at runtime. - Pin mlcommons/logging to 6.0.0-rc6 in requirements.txt + both Dockerfiles for reproducibility. - mlperf_logging_utils: guard empty os.path.dirname(log_path); attach file handler on rank 0 only; get_mlperf_logger() returns None when the dep is unavailable so callers' `is not None` guards disable logging cleanly. - launch_slurm.sh: chmod 622 (not 666) on the job log — tee -a needs write only, avoids world-readable logs on shared NFS. Co-authored-by: Cursor --- recommendation_v4/Dockerfile | 6 +++++ recommendation_v4/Dockerfile.nvidia | 6 +++++ .../dlrm_v3/train/mlperf_logging_utils.py | 22 +++++++++++++++---- recommendation_v4/requirements.txt | 2 +- recommendation_v4/scripts/launch_slurm.sh | 10 +++++---- 5 files changed, 37 insertions(+), 9 deletions(-) diff --git a/recommendation_v4/Dockerfile b/recommendation_v4/Dockerfile index 112d605df..450a5ab55 100644 --- a/recommendation_v4/Dockerfile +++ b/recommendation_v4/Dockerfile @@ -66,6 +66,12 @@ RUN pip install \ torchmetrics==1.0.3 \ tensordict +# mlperf_logging — required by train/mlperf_logging_utils.py for MLPerf +# compliance logs. Pinned to the Training 6.0 tag for reproducibility; --no-deps +# so pip does not resolve requirements.txt's torch/fbgemm_gpu/torchrec pins and +# clobber the +rocm7.2 wheels above. +RUN pip install --no-deps "git+https://github.com/mlcommons/logging.git@6.0.0-rc6" + # Smoke-test the 6 imports the launch script checks at # scripts/launch_smoke_8gpu.sh:26. RUN python -c "import torch, fbgemm_gpu, torchrec, polars, xxhash, gin; \ diff --git a/recommendation_v4/Dockerfile.nvidia b/recommendation_v4/Dockerfile.nvidia index 388ab8e5e..a1a9a3319 100644 --- a/recommendation_v4/Dockerfile.nvidia +++ b/recommendation_v4/Dockerfile.nvidia @@ -68,6 +68,12 @@ RUN pip install \ torchmetrics==1.0.3 \ tensordict +# mlperf_logging — required by train/mlperf_logging_utils.py for MLPerf +# compliance logs. Pinned to the Training 6.0 tag for reproducibility; --no-deps +# so pip does not resolve requirements.txt's torch/fbgemm_gpu/torchrec pins and +# clobber the image's native NGC torch. +RUN pip install --no-deps "git+https://github.com/mlcommons/logging.git@6.0.0-rc6" + # Smoke-test that packages are installed at the right versions. Cannot dlopen # fbgemm_gpu / torchrec here because their SONAME deps (libnvidia-ml.so.1, etc.) # only resolve when the container runs with `--gpus all` — which docker build diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py index 51e7971b5..74eac4ede 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py @@ -93,8 +93,16 @@ def __init__( self._logger = None if not self.enabled: return - if log_path: - os.makedirs(os.path.dirname(log_path), exist_ok=True) + # Only rank 0 emits events, so only rank 0 needs the file handler: + # attaching it on every rank wastes file handles and risks contention on + # a shared log path. Non-zero ranks configure mllog without a filename + # (their event methods no-op anyway). + if log_path and self.rank == 0: + log_dir = os.path.dirname(log_path) + # dirname is "" when log_path has no directory component (e.g. + # MLPERF_LOG_PATH=mlperf.log); os.makedirs("") raises, so guard it. + if log_dir: + os.makedirs(log_dir, exist_ok=True) mllog.config(filename=log_path, default_stack_offset=default_stack_offset) else: mllog.config(default_stack_offset=default_stack_offset) @@ -158,14 +166,20 @@ def get_mlperf_logger( log_path: str = "", benchmark_name: str = "hstu", submitter_name: str = "reference_implementation", -) -> MLPerfLogger: - """Build a configured :class:`MLPerfLogger`. +) -> Optional[MLPerfLogger]: + """Build a configured :class:`MLPerfLogger`, or ``None`` if unavailable. ``benchmark_name`` / ``submitter_name`` are gin-configurable (and the path is env-overridable via ``$MLPERF_LOG_PATH``) so a submission can stamp its own benchmark string without code changes. The log path defaults to ``$MLPERF_LOG_PATH`` when set, else ``""`` (mllog logs to stdout). + + Returns ``None`` when ``mlperf_logging`` is not installed so callers' existing + ``mlperf_logger is not None`` guards cleanly disable logging -- otherwise they + would pass the guard and then hit ``logger.constants`` (which is ``None``). """ + if not _MLLOG_AVAILABLE: + return None resolved_path = os.environ.get("MLPERF_LOG_PATH", log_path) return MLPerfLogger( rank=rank, diff --git a/recommendation_v4/requirements.txt b/recommendation_v4/requirements.txt index a8637bf5e..d1aba1e95 100644 --- a/recommendation_v4/requirements.txt +++ b/recommendation_v4/requirements.txt @@ -5,4 +5,4 @@ gin_config>=0.5.0 pandas>=2.2.0 tensorboard>=2.19.0 pybind11 -git+https://github.com/mlcommons/logging.git +git+https://github.com/mlcommons/logging.git@6.0.0-rc6 diff --git a/recommendation_v4/scripts/launch_slurm.sh b/recommendation_v4/scripts/launch_slurm.sh index e81510434..829c5ce44 100755 --- a/recommendation_v4/scripts/launch_slurm.sh +++ b/recommendation_v4/scripts/launch_slurm.sh @@ -192,11 +192,13 @@ orchestrate() { else : > "$LOG" fi - # World-writable so the in-container worker (running as root, squashed to - # `nobody` over root-squashed NFS) can append via `tee -a $LOG`. Without this - # the worker's tee opens the file read-only-denied and exits non-zero, which + # Group/other write (but NOT read) so the in-container worker (running as root, + # squashed to `nobody` over root-squashed NFS) can append via `tee -a $LOG`. + # `tee -a` opens write-only, so 622 is sufficient -- avoid 666, which would let + # other users on the shared filesystem read (and tamper with) the job log. + # Without the write bit the worker's tee is denied and exits non-zero, which # pipefail turns into a spurious rc=1 even when training succeeds. - chmod 666 "$LOG" 2>/dev/null || true + chmod 622 "$LOG" 2>/dev/null || true echo "[$(date)] launch_slurm/orchestrate: job=${SLURM_JOB_ID:-?} nodes=${SLURM_JOB_NODELIST:-?} nnodes=${SLURM_NNODES:-1}" | tee -a "$LOG" echo "[$(date)] resolved SCRIPT_PATH=$SCRIPT_PATH REPO=$REPO" | tee -a "$LOG" echo "[$(date)] config: MODE=$MODE START_TS=$START_TS NUM_TRAIN_TS=$NUM_TRAIN_TS NUM_TRAIN_BATCHES=$NUM_TRAIN_BATCHES NUM_EVAL_BATCHES=$NUM_EVAL_BATCHES METRIC_LOG_FREQ=$METRIC_LOG_FREQ SMOKE=${SMOKE:-0} EVAL_EVERY_N_WINDOWS=$EVAL_EVERY_N_WINDOWS EVAL_EVERY_DATA_PCT=$EVAL_EVERY_DATA_PCT" | tee -a "$LOG" From 9d1dbf8a70945f2afcb8edb741529afa8e3c4ad7 Mon Sep 17 00:00:00 2001 From: suachong Date: Wed, 24 Jun 2026 17:19:00 +0000 Subject: [PATCH 085/127] dlrmv4: decorrelate per-rank runtime RNG for HSTU dropout Add decorrelate_runtime_rng(rank): after make_model + DMP init, re-seed torch/cuda with $SEED + rank so HSTU dropout draws different masks per data-parallel rank instead of the identical masks implied by the shared init seed. Runs strictly after init so replicated dense weights and sharded embeddings stay init-identical across ranks. Toggle via $DECORRELATE_DROPOUT (default 1; 0 = legacy identical-mask behavior). Offset is a pure fn of resolved $SEED + rank and per-rank RNG state is checkpointed, so reproducibility/resume are preserved. Co-authored-by: Cursor --- .../dlrm_v3/train/gin/yambda_5b.gin | 13 ++++++ .../dlrm_v3/train/train_ranker.py | 6 +++ .../dlrm_v3/train/utils.py | 42 +++++++++++++++++++ 3 files changed, 61 insertions(+) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index 7b7959c2b..864b45351 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -88,6 +88,19 @@ seed_everything.seed = @seed/env_int() seed/env_int.key = "SEED" seed/env_int.default = -1 +# $DECORRELATE_DROPOUT — per-rank decorrelation of RUNTIME stochastic ops. +# After init (make_model + DMP), re-seed torch/cuda with $SEED + rank so HSTU +# dropout (input_dropout=0.2, linear_dropout_rate=0.1) draws DIFFERENT masks on +# every data-parallel rank instead of the identical masks implied by the shared +# init seed — the standard data-parallel RNG track. Init (dense weights + +# sharded embeddings) is untouched: this runs strictly after it. 1 = on +# (default), 0 = legacy identical-mask-on-every-rank behavior. Reproducible: the +# offset is a pure fn of resolved $SEED + rank, and per-rank RNG state is +# checkpointed across resume. +decorrelate_runtime_rng.enabled = @drr/env_int() +drr/env_int.key = "DECORRELATE_DROPOUT" +drr/env_int.default = 1 + # dense model optimizer # Learning rate is env-overridable via $DENSE_LR (default 1e-7). dense_optimizer_factory_and_class.learning_rate = @dlr/env_float() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py index 7ee39c2ed..25711bd02 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py @@ -108,6 +108,7 @@ def _main_func( from generative_recommenders.dlrm_v3.checkpoint import load_dmp_checkpoint from generative_recommenders.dlrm_v3.train.utils import ( cleanup, + decorrelate_runtime_rng, eval_loop, make_model, make_optimizer_and_shard, @@ -149,6 +150,11 @@ def _main_func( world_size=world_size, local_world_size=gpus_per_node, ) + # Decorrelate forward-time stochasticity (HSTU dropout) per data-parallel + # rank. MUST run after make_model() + make_optimizer_and_shard() so the + # replicated dense weights and sharded embeddings stay init-identical across + # ranks; this only offsets the global RNG by rank so dropout masks differ. + decorrelate_runtime_rng(rank=rank) train_dataloader, test_dataloader = make_train_test_dataloaders( hstu_config=model_configs, embedding_table_configs=embedding_table_configs, diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py index d0c4a8538..1e537b32d 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -127,6 +127,48 @@ def seed_everything(seed: int = -1, rank: int = 0) -> None: torch.cuda.manual_seed_all(seed) +@gin.configurable +def decorrelate_runtime_rng(rank: int = 0, enabled: bool = True) -> None: + """Offset the global RNG by ``rank`` so RUNTIME stochastic ops draw + decorrelated draws per data-parallel rank. + + The only such op here is HSTU dropout (input_dropout=0.2, + linear_dropout_rate=0.1; see configs.get_hstu_configs). seed_everything() + sets an IDENTICAL seed on every rank — required so replicated dense weights + init identically — which also makes every rank draw the SAME dropout masks + in the forward. Gradients still differ (each rank sees different data), so + that is not incorrect, but identical masks waste the extra mask diversity + that decorrelated replicas give per global batch. This re-seeds torch/cuda + with $SEED + rank to recover it (the standard data-parallel RNG track, cf. + Megatron's tensor/data-parallel RNG separation). + + ORDERING IS LOAD-BEARING: call this AFTER everything that must be identical + across ranks — make_model() (dense weight init) AND make_optimizer_and_shard() + (the pre-DMP re-seed + sharded embedding init). It deliberately perturbs only + forward-time stochasticity, never init. + + Reproducibility is preserved: the offset is a pure function of the resolved + $SEED (exported by seed_everything) and rank, and per-rank RNG state is + snapshotted/restored on checkpoint resume (see checkpoint.py). Set + enabled=False (gin: decorrelate_runtime_rng.enabled) to restore the legacy + identical-mask-on-every-rank behavior. + """ + if not enabled: + logger.info( + f"[rank {rank}] decorrelate_runtime_rng disabled; dropout masks " + f"identical across ranks" + ) + return + base = int(os.environ.get("SEED", "1")) + offset_seed = base + int(rank) + torch.manual_seed(offset_seed) + torch.cuda.manual_seed_all(offset_seed) + logger.info( + f"[rank {rank}] decorrelated runtime RNG: SEED={base} + rank={rank} " + f"=> {offset_seed} (per-rank dropout masks)" + ) + + def setup( rank: int, world_size: int, From 784516aeb8947840bc0e6fea993f27e195bb381e Mon Sep 17 00:00:00 2001 From: suachong Date: Wed, 24 Jun 2026 18:41:14 +0000 Subject: [PATCH 086/127] dlrmv4: drop non-portable build/run helpers from the baseline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove the B200 Dockerfile.nvidia and the local scripts/run_docker.sh helper — neither is wired into launch_slurm.sh (which uses rocm/primus directly) and both are environment-specific. Gitignore them plus the local ad-hoc analysis artifacts so they are not re-added. Co-authored-by: Cursor --- recommendation_v4/.gitignore | 17 +++++ recommendation_v4/Dockerfile.nvidia | 98 ------------------------- recommendation_v4/scripts/run_docker.sh | 77 ------------------- 3 files changed, 17 insertions(+), 175 deletions(-) delete mode 100644 recommendation_v4/Dockerfile.nvidia delete mode 100755 recommendation_v4/scripts/run_docker.sh diff --git a/recommendation_v4/.gitignore b/recommendation_v4/.gitignore index 5a0329448..4bfbaa0ca 100644 --- a/recommendation_v4/.gitignore +++ b/recommendation_v4/.gitignore @@ -165,3 +165,20 @@ cython_debug/ yambda_slurm.*.out yambda_slurm.*.log compliance_checker.log + +# Local container build/run helpers — environment-specific, not committed. +/Dockerfile.nvidia +/scripts/run_docker.sh + +# Local dlrmv4 analysis artifacts (per-job plots/scripts/dumps); kept on disk +# for ad-hoc analysis but never committed. +/analyze_*.py +/dump_eval_*.py +/perday_*.py +/parse_lr_sweep.py +/gen_canvas.py +/plot_*.py +/*.png +/*.csv +/scripts/bench_collectives.py +/docs/v4_vs_v2_and_hstu_walkthrough.md diff --git a/recommendation_v4/Dockerfile.nvidia b/recommendation_v4/Dockerfile.nvidia deleted file mode 100644 index a1a9a3319..000000000 --- a/recommendation_v4/Dockerfile.nvidia +++ /dev/null @@ -1,98 +0,0 @@ -# B200 path — implements docs/training_recipe.md §"B200". - -FROM nvcr.io/nvidia/pytorch:26.04-py3 - -ENV PYTHONUNBUFFERED=1 \ - PIP_NO_CACHE_DIR=1 \ - PIP_DISABLE_PIP_VERSION_CHECK=1 - -WORKDIR /workspace/recommendation_v4 - -# torch / triton — training_recipe.md:137-138, 149-150. Native to the image -# and must NOT be reinstalled (CUPTI / sm_100 support depends on it). - -# torchrec — training_recipe.md:152. Nightly cu130 wheel, --no-deps. -RUN pip install --force-reinstall --no-deps \ - --index-url https://download.pytorch.org/whl/nightly/cu130 \ - torchrec==1.7.0.dev20260601+cu130 - -# fbgemm_gpu — training_recipe.md:151. Build from FBGEMM commit 10b77573 for -# sm_100 against the image's native torch. ~55 min (sm_100 TBE-forward via ptxas). -# NOTE: --nvml_lib_path diverges from training_recipe.md:151. The recipe points -# at /usr/lib/x86_64-linux-gnu/libnvidia-ml.so, which is mounted only at -# `docker run --gpus all` time. During `docker build` no GPU runtime is -# attached, so we link against the NVML stub that ships inside the CUDA SDK in -# the NGC image; the real driver-side libnvidia-ml.so is used at runtime. -RUN apt-get update && apt-get install -y --no-install-recommends git build-essential && \ - rm -rf /var/lib/apt/lists/* && \ - git clone --recursive https://github.com/pytorch/FBGEMM.git /tmp/FBGEMM && \ - cd /tmp/FBGEMM && \ - git checkout 10b775730212923f65f7b78f79b6a01d80cf3c29 && \ - git submodule update --init --recursive && \ - cd fbgemm_gpu && \ - # Filter `fairscale` and the torch family from fbgemm's requirements.txt: - # fairscale pulls a CPU torch that would clobber the image's native torch. - # fairscale is a distributed-training lib used by fbgemm tests, not by the - # build itself. - grep -v -E '^(fairscale|torch|torchvision|torchaudio)([<>=!]|$)' requirements.txt > /tmp/req.txt && \ - pip install -r /tmp/req.txt && \ - TORCH_CUDA_ARCH_LIST=10.0 python setup.py bdist_wheel \ - --build-target default \ - --build-variant cuda \ - --package_channel nightly \ - --nvml_lib_path /usr/local/cuda/lib64/stubs/libnvidia-ml.so && \ - pip install --force-reinstall --no-deps dist/fbgemm_gpu_nightly-*.whl && \ - cd / && rm -rf /tmp/FBGEMM - -# polars-u64-idx — training_recipe.md:153 (mandatory; yambda-5b > 4.29 B rows). -# Remaining packages — training_recipe.md:156-159 ("Additional Python deps") plus -# `datasets` + `huggingface_hub`, which the recipe does not list but -# preprocess_public_data.py:278 imports to download yambda from HuggingFace. -RUN pip install \ - polars-u64-idx==1.33.1 \ - gin-config \ - absl-py \ - datasets \ - huggingface_hub \ - pyre-extensions \ - iopath \ - typing-inspect \ - psutil \ - tqdm \ - pyyaml \ - lightning-utilities && \ - # torchmetrics and tensordict declare `torch` as a dep; without --no-deps - # pip resolves and reinstalls torch, clobbering the image's native NGC - # torch (which would break CUPTI + sm_100 support per training_recipe.md:199). - pip install --no-deps \ - torchmetrics==1.0.3 \ - tensordict - -# mlperf_logging — required by train/mlperf_logging_utils.py for MLPerf -# compliance logs. Pinned to the Training 6.0 tag for reproducibility; --no-deps -# so pip does not resolve requirements.txt's torch/fbgemm_gpu/torchrec pins and -# clobber the image's native NGC torch. -RUN pip install --no-deps "git+https://github.com/mlcommons/logging.git@6.0.0-rc6" - -# Smoke-test that packages are installed at the right versions. Cannot dlopen -# fbgemm_gpu / torchrec here because their SONAME deps (libnvidia-ml.so.1, etc.) -# only resolve when the container runs with `--gpus all` — which docker build -# can't do. The real 6-import check at scripts/launch_smoke_8gpu.sh:26 runs at -# `docker run` time when the driver is mounted in. -RUN python -c "import torch, polars, xxhash, gin; \ -print('torch', torch.__version__, '| cuda', getattr(torch.version, 'cuda', None)); \ -import importlib.metadata as m; \ -print('fbgemm_gpu installed:', m.version('fbgemm_gpu_nightly')); \ -print('torchrec installed: ', m.version('torchrec'))" - -COPY . /workspace/recommendation_v4 - -# B200 runtime env — training_recipe.md:184-195. -ENV PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \ - HSTU_HAMMER_KERNEL=TRITON \ - TORCH_CUDA_ARCH_LIST=10.0 \ - HBM_CAP_GB=150 \ - TRITON_CACHE_DIR=/workspace/recommendation_v4/.triton_cache \ - DLRM_DATA_PATH=/data/mlperf_dlrm_v4 - -CMD ["bash"] diff --git a/recommendation_v4/scripts/run_docker.sh b/recommendation_v4/scripts/run_docker.sh deleted file mode 100755 index 864746550..000000000 --- a/recommendation_v4/scripts/run_docker.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/bash -# Launch a yambda_8gpu container from rocm/mlperf:dlrm_v3_mi355 with the repo -# and data directories bind-mounted at matching host/container paths. -# -# Usage: -# bash scripts/run_docker.sh # interactive shell -# bash scripts/run_docker.sh -- bash scripts/launch_slurm.sh # one-shot single-node train -# -# Inside the container /.dockerenv exists, so launch_slurm.sh auto-selects its -# SLURM-free `worker` phase (NNODES=1) — identical to the old launch_smoke_8gpu.sh. -# -# Overrides (export before invoking): -# IMAGE docker image (default: rocm/mlperf:dlrm_v3_mi355) -# CONTAINER_NAME container name (default: mlperf-recommendation-v4) -# REPO_HOST host path to repo (default: this script's parent) -# DATA_HOST host path to dataset root (default: /data/mlperf_dlrm_v4) -# LOG in-container train log path (default: /workspace/recommendation_v4/mlperf_dlrm_v4.log) -# MODE launch_slurm.sh mode (default: launcher default = streaming-train-eval; set train-eval for classic) -# MAX_SEQ_LEN / HISTORY_LENGTH seq shape; set 2048 / 2039 for the previous 2k shape -# NCCL_SOCKET_IFNAME NCCL bootstrap NIC (default: launch_slurm picks lo single-node / fenic0 multi-node; override per host) - -set -euo pipefail - -IMAGE=${IMAGE:-rocm/mlperf:dlrm_v3_mi355} -CONTAINER_NAME=${CONTAINER_NAME:-mlperf-recommendation-v4} -REPO_HOST=${REPO_HOST:-$(cd "$(dirname "$0")/.." && pwd)} -DATA_HOST=${DATA_HOST:-/data/mlperf_dlrm_v4} - -# Mount host paths at the same string inside the container so DLRM_DATA_PATH -# can be set from either side and resolve identically (env_path() in -# dlrm_v3/utils.py:641-653 does a literal os.environ.get). -REPO_CONT=/workspace/recommendation_v4 -DATA_CONT=${DATA_HOST} - -if [ ! -d "${DATA_HOST}" ]; then - echo "warning: ${DATA_HOST} does not exist on host. Run preprocess_public_data first or override DATA_HOST." >&2 -fi - -# Drop an optional `--` separating this script's invocation from the in-container -# command (the documented `run_docker.sh -- bash scripts/launch_slurm.sh` form). -# Without this, `--` is forwarded verbatim to `docker run` as the command and -# fails with: exec: "--": executable file not found. -if [ "${1:-}" = "--" ]; then shift; fi - -# If a container with this name is already running, exec into it instead of -# starting a new one. Matches the `docker exec yambda_8gpu ...` pattern in -# README.MD:9-12. -if docker ps --format '{{.Names}}' | grep -qx "${CONTAINER_NAME}"; then - echo "container ${CONTAINER_NAME} already running; exec'ing in" >&2 - exec docker exec -it "${CONTAINER_NAME}" "${@:-bash}" -fi - -# Remove a stopped container with the same name so --name doesn't collide. -if docker ps -a --format '{{.Names}}' | grep -qx "${CONTAINER_NAME}"; then - docker rm "${CONTAINER_NAME}" >/dev/null -fi - -exec docker run --rm -it \ - --name "${CONTAINER_NAME}" \ - --device=/dev/kfd --device=/dev/dri \ - --group-add video --group-add render \ - --cap-add=SYS_PTRACE --security-opt seccomp=unconfined \ - --ipc=host --network=host \ - --shm-size=64g --ulimit memlock=-1 --ulimit stack=67108864 \ - -v "${REPO_HOST}:${REPO_CONT}" \ - -v "${DATA_HOST}:${DATA_CONT}" \ - -e DLRM_DATA_PATH="${DATA_CONT}" \ - -e HSTU_HAMMER_KERNEL="${HSTU_HAMMER_KERNEL:-TRITON}" \ - -e RUN_NAME="${RUN_NAME:-default}" \ - -e LOG="${LOG:-/workspace/recommendation_v4/mlperf_dlrm_v4.log}" \ - ${MODE:+-e MODE="${MODE}"} \ - ${MAX_SEQ_LEN:+-e MAX_SEQ_LEN="${MAX_SEQ_LEN}"} \ - ${HISTORY_LENGTH:+-e HISTORY_LENGTH="${HISTORY_LENGTH}"} \ - ${NCCL_SOCKET_IFNAME:+-e NCCL_SOCKET_IFNAME="${NCCL_SOCKET_IFNAME}"} \ - -w "${REPO_CONT}" \ - "${IMAGE}" \ - "${@:-bash}" From 3f36f61abde50a8e01de87b4ebd109f3f7b54d57 Mon Sep 17 00:00:00 2001 From: suachong Date: Wed, 24 Jun 2026 19:19:08 +0000 Subject: [PATCH 087/127] dlrmv4: trim PR comments, default DECORRELATE_DROPOUT off, extend eval metric - Collapse verbose comment blocks across gin/launch_slurm/resume-test to 1-2 lines; revert container name to yambda_primus and drop #SBATCH --time. - Default DECORRELATE_DROPOUT=0 (identical dropout masks across ranks). - Generalize EVAL_ACCURACY_AUC_MODE to any {window,lifetime}_{auc,gauc, accuracy,ne} metric, with NE handled as lower-is-better for early-stop. Co-authored-by: Cursor --- .../dlrm_v3/train/gin/yambda_5b.gin | 38 ++------ .../train/tests/streaming_resume_test.sh | 7 +- .../dlrm_v3/train/utils.py | 85 ++++++++--------- recommendation_v4/scripts/launch_slurm.sh | 91 +++---------------- 4 files changed, 59 insertions(+), 162 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index 864b45351..dd4dd4715 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -80,26 +80,16 @@ apply_env_bootstrap.TRITON_FULL_AUTOTUNE = False # PARSE NOTE: seed_everything() runs right before make_model() in train_ranker # (after the full gin parse), so this binding resolves in the second parse where # env_int is registered. Override per-run via $SEED. -# -# DEFAULT -1 = draw a FRESH RANDOM seed every run (rank 0 draws + broadcasts so -# all ranks agree; the chosen value is exported to $SEED and logged), so each -# launch explores a different init. Pin $SEED >= 0 to reproduce a specific run. +# Default -1 draws a fresh random seed each run; pin $SEED >= 0 to reproduce. seed_everything.seed = @seed/env_int() seed/env_int.key = "SEED" seed/env_int.default = -1 -# $DECORRELATE_DROPOUT — per-rank decorrelation of RUNTIME stochastic ops. -# After init (make_model + DMP), re-seed torch/cuda with $SEED + rank so HSTU -# dropout (input_dropout=0.2, linear_dropout_rate=0.1) draws DIFFERENT masks on -# every data-parallel rank instead of the identical masks implied by the shared -# init seed — the standard data-parallel RNG track. Init (dense weights + -# sharded embeddings) is untouched: this runs strictly after it. 1 = on -# (default), 0 = legacy identical-mask-on-every-rank behavior. Reproducible: the -# offset is a pure fn of resolved $SEED + rank, and per-rank RNG state is -# checkpointed across resume. +# $DECORRELATE_DROPOUT — re-seed torch/cuda with $SEED + rank after init so HSTU +# dropout masks differ per data-parallel rank. 1 = on, 0 = identical masks (default). decorrelate_runtime_rng.enabled = @drr/env_int() drr/env_int.key = "DECORRELATE_DROPOUT" -drr/env_int.default = 1 +drr/env_int.default = 0 # dense model optimizer # Learning rate is env-overridable via $DENSE_LR (default 1e-7). @@ -252,9 +242,6 @@ hs/env_str.default = "interleaved" # keyed by (L, MIN_HISTORY) so floors don't collide. Override via $MIN_HISTORY. get_dataset.min_history = @mh/env_int() mh/env_int.key = "MIN_HISTORY" -# Freeze default: power-users-only floor at the full history budget. With -# HISTORY_LENGTH=4086 this maps to the existing positions_L4086.npy cache (no -# rebuild, no shared-dir write — see yambda.py _positions_filename legacy path). mh/env_int.default = 4086 # Model-side attention budget. Dataset truncates UIH to fit this value if @@ -461,23 +448,16 @@ MetricsLogger.tensorboard_log_path = @tbp/env_path() tbp/env_path.key = "TENSORBOARD_LOG_PATH" tbp/env_path.default = "" MetricsLogger.world_size = 8 -# Time-to-target AUC threshold. Doubles as the MLPerf convergence target: when -# the selected listen_plus eval AUC (see eval_accuracy_auc_mode below; default -# the per-pass "window_" AUC) first reaches this value the streaming-train-eval -# run emits a SUCCESS RUN_STOP and terminates gracefully. Override via -# $AUC_THRESHOLD (e.g. 0.5 to smoke-test the early-stop path on a short run). -# MLPerf's DLRM-DCNv2 reference uses 0.80275. +# MLPerf convergence target: run stops when the selected eval AUC reaches it. +# Override via $AUC_THRESHOLD. MetricsLogger.auc_threshold = @at/env_float() at/env_float.key = "AUC_THRESHOLD" at/env_float.default = 0.80275 -# Which eval AUC is reported as EVAL_ACCURACY and drives the convergence / -# SUCCESS RUN_STOP decision: "window" (per-pass full-holdout AUC, reset each eval -# pass; the default) or "lifetime" (cumulative across all eval passes). Both AUCs -# are still computed and logged to TensorBoard regardless; this only selects the -# one used for MLPerf EVAL_ACCURACY + early-stop. Override via $EVAL_ACCURACY_AUC_MODE. +# Eval metric driving EVAL_ACCURACY + early-stop: "{window|lifetime}_{auc|gauc| +# accuracy|ne}" (default window_auc). Override via $EVAL_ACCURACY_AUC_MODE. streaming_train_eval_loop.eval_accuracy_auc_mode = @eaam/env_str() eaam/env_str.key = "EVAL_ACCURACY_AUC_MODE" -eaam/env_str.default = "window" +eaam/env_str.default = "window_auc" # Lifetime-AUC backend, selectable independently for the train cumulative AUC and # the eval cumulative ("lifetime_*") AUC. Both default to "binned": # "binned" = BinnedCumulativeAUC: exact-cumulative AUC from an O(bins) score diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh b/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh index c7bb5bab9..47c451696 100755 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh @@ -17,7 +17,7 @@ # # Usage: # bash generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh --jobid -# [--container yambda_] +# [--container yambda_primus] # [--num-train-batches 200] # [--die-at-step 350] # [--keep] # retain LOG_DIR + CKPT after run for inspection @@ -25,7 +25,7 @@ set -uo pipefail JOBID="" -CONTAINER="yambda_${USER:-$(id -un)}" +CONTAINER="yambda_primus" NUM_TRAIN_BATCHES=200 DIE_AT_STEP=350 IN_WINDOW_FREQ=50 @@ -40,9 +40,6 @@ KEEP=0 # correctness gates are the functional-invariant checks below (RNG restored, # resumed-at-correct-step, atomic/keep_last_n), not this number. ATOL=0.15 -# Writable scratch ($HOME-derived) + repo root (this file is at -# /generative_recommenders/dlrm_v3/train/tests/, i.e. 4 levels deep). -# Both env-overridable; nothing is hardwired to a specific user/site. SCRATCH=${SCRATCH:-$HOME/yambda_runs} CKPT_ROOT=${CKPT_ROOT:-$SCRATCH/ckpts_resume_test} LOG_DIR=${LOG_DIR:-$SCRATCH/streaming_resume_test} diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py index 1e537b32d..641e8e67c 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -1620,12 +1620,10 @@ def streaming_train_eval_loop( # event emission; the loop is otherwise unchanged. Supplied by train_ranker # for the streaming-train-eval benchmark path. mlperf_logger: Optional[Any] = None, - # Which eval AUC drives the reported EVAL_ACCURACY and the convergence - # decision (early SUCCESS RUN_STOP + end-of-run finalize): "window" = the - # per-pass full-holdout AUC (reset each eval pass; the default), or - # "lifetime" = the cumulative AUC across all eval passes. Override via - # $EVAL_ACCURACY_AUC_MODE. - eval_accuracy_auc_mode: str = "window", + # Which eval metric drives EVAL_ACCURACY + the convergence decision. Format + # "{window|lifetime}_{auc|gauc|accuracy|ne}" (bare "window"/"lifetime" => auc). + # Override via $EVAL_ACCURACY_AUC_MODE. + eval_accuracy_auc_mode: str = "window_auc", ) -> None: """Streaming train+eval loop with per-window (and optionally mid-window) checkpoints. @@ -2321,32 +2319,33 @@ def _mlperf_progress() -> Dict[str, Any]: mlperf_logger.constants.EPOCH_NUM: epoch_num, } - # Convergence/EVAL_ACCURACY metric short name, selected by - # eval_accuracy_auc_mode: "window_auc" (per-pass full-holdout AUC, default) - # or "lifetime_auc" (cumulative across eval passes). - _eval_auc_short = ( - "lifetime_auc" - if str(eval_accuracy_auc_mode).strip().lower() == "lifetime" - else "window_auc" - ) + # Convergence/EVAL_ACCURACY metric short name selected by + # eval_accuracy_auc_mode. Bare "window"/"lifetime" defaults the metric to auc. + _eval_metric_short = str(eval_accuracy_auc_mode).strip().lower() + if _eval_metric_short in ("window", "lifetime"): + _eval_metric_short = f"{_eval_metric_short}_auc" + # NE is lower-is-better; auc/gauc/accuracy are higher-is-better. + _eval_lower_is_better = _eval_metric_short.endswith("_ne") if rank == 0 and mlperf_logger is not None: logger.info( - f"[mlperf] EVAL_ACCURACY / convergence metric = {_eval_auc_short} " - f"(eval_accuracy_auc_mode={eval_accuracy_auc_mode!r})" + f"[mlperf] EVAL_ACCURACY / convergence metric = {_eval_metric_short} " + f"(stop when {'<=' if _eval_lower_is_better else '>='} threshold)" ) - def _eval_target_auc(metrics: Dict[str, float]) -> Optional[float]: - # Convergence metric: the listen_plus eval AUC selected by - # eval_accuracy_auc_mode (window vs lifetime). Key format is - # `metric/{prefix}{name}/{task}` (see MetricsLogger.compute), e.g. - # `metric/window_auc/listen_plus`. Match the selected short name; - # ignore GAUC. + def _eval_target_metric(metrics: Dict[str, float]) -> Optional[float]: + # listen_plus eval metric selected by eval_accuracy_auc_mode. Key format + # is `metric/{prefix}_{name}/{task}` (see MetricsLogger.compute). for key, val in metrics.items(): short = key.split("/")[-2] if "/" in key else key - if short == _eval_auc_short: + if short == _eval_metric_short: return float(val) return None + def _meets_target(value: Optional[float], thr: Optional[float]) -> bool: + if value is None or thr is None: + return False + return value <= thr if _eval_lower_is_better else value >= thr + def _mlperf_block_start() -> None: if mlperf_logger is not None: mlperf_logger.start( @@ -2378,25 +2377,21 @@ def _mlperf_run_stop(status: object) -> None: mlperf_run_stopped[0] = True def _mlperf_eval_stop(eval_metrics: Dict[str, float]) -> bool: - # Emit EVAL_ACCURACY (the selected eval listen_plus AUC) + EVAL_STOP, and - # drive an early SUCCESS RUN_STOP when the target threshold is reached. - # Returns True iff the run should stop now -- the SAME value on every rank. + # Emit EVAL_ACCURACY (the selected eval listen_plus metric) + EVAL_STOP, + # and drive an early SUCCESS RUN_STOP when the target is reached. Returns + # True iff the run should stop now -- the SAME value on every rank. # - # CRITICAL (deadlock avoidance): the eval AUC is produced by a reduce - # that is only guaranteed valid on global rank 0, so a per-rank - # `eval_auc >= thr` test could diverge (only rank 0 sees the value) and - # the ranks that "stop" hit the RUN_STOP barrier while the rest march - # into the next window's embedding all-to-all -> NCCL collective-timeout - # hang (observed: 600s ALLTOALL_BASE watchdog abort). So rank 0 decides - # and BROADCASTS the boolean; all ranks then break (or continue) in - # lockstep. + # CRITICAL (deadlock avoidance): the eval metric is only valid on global + # rank 0, so rank 0 decides and BROADCASTS the boolean; all ranks then + # break (or continue) in lockstep. A per-rank test could diverge and hang + # the next window's embedding all-to-all (600s ALLTOALL_BASE watchdog). if mlperf_logger is None: return False - eval_auc = _eval_target_auc(eval_metrics) - if eval_auc is not None: + eval_value = _eval_target_metric(eval_metrics) + if eval_value is not None: mlperf_logger.event( key=mlperf_logger.constants.EVAL_ACCURACY, - value=eval_auc, + value=eval_value, metadata=_mlperf_progress(), ) mlperf_logger.end( @@ -2404,13 +2399,7 @@ def _mlperf_eval_stop(eval_metrics: Dict[str, float]) -> bool: ) thr = metric_logger.auc_threshold decision = torch.zeros(1, device=device) - if ( - rank == 0 - and not mlperf_run_stopped[0] - and eval_auc is not None - and thr is not None - and eval_auc >= thr - ): + if rank == 0 and not mlperf_run_stopped[0] and _meets_target(eval_value, thr): decision[0] = 1.0 if torch.distributed.is_initialized(): torch.distributed.broadcast(decision, src=0) @@ -2421,13 +2410,11 @@ def _mlperf_eval_stop(eval_metrics: Dict[str, float]) -> bool: return should_stop def _mlperf_finalize(final_metrics: Dict[str, float]) -> None: - # End-of-run RUN_STOP when the threshold was never crossed: SUCCESS iff - # the final eval AUC meets the target, else ABORTED. + # End-of-run RUN_STOP when the target was never crossed: SUCCESS iff the + # final eval metric meets the target, else ABORTED. if mlperf_logger is None or mlperf_run_stopped[0]: return - eval_auc = _eval_target_auc(final_metrics) - thr = metric_logger.auc_threshold - success = eval_auc is not None and thr is not None and eval_auc >= thr + success = _meets_target(_eval_target_metric(final_metrics), metric_logger.auc_threshold) _mlperf_run_stop( mlperf_logger.constants.SUCCESS if success diff --git a/recommendation_v4/scripts/launch_slurm.sh b/recommendation_v4/scripts/launch_slurm.sh index 829c5ce44..5ab1e950a 100755 --- a/recommendation_v4/scripts/launch_slurm.sh +++ b/recommendation_v4/scripts/launch_slurm.sh @@ -4,11 +4,7 @@ #SBATCH --ntasks-per-node=1 #SBATCH --exclusive #SBATCH --partition=meta64 # [CLUSTER-SPECIFIC] partition name -#SBATCH --time=01:10:00 #SBATCH --output=yambda_slurm.%j.out -# ^ relative to the submit dir (SLURM parses #SBATCH before any shell runs, so it -# cannot expand env vars). The real consolidated run log is $LOG (see below), -# which defaults under $SCRATCH; this file just captures the batch stdout. # ============================================================================= # launch_slurm.sh — single entry point for the yambda-5b trainer on N>=1 nodes. # @@ -34,14 +30,9 @@ # supervisor's direct `bash scripts/launch_slurm.sh` is unchanged. # # USAGE -# Reference run (1 node): sbatch --nodes=1 scripts/launch_slurm.sh -# Reference run (N node): sbatch --nodes=N scripts/launch_slurm.sh -# ^ a bare submit reproduces the FROZEN REFERENCE shape (full 299-window -# sweep + data-fraction eval cadence). Prepend SMOKE=1 for a fast -# functional check (short window, capped batches). +# Multi-node (N>=1): sbatch --nodes=2 scripts/launch_slurm.sh # Single-node direct: bash scripts/launch_slurm.sh (already inside container; -# what run_streaming_e2e.sh invokes per relaunch — uses the -# gin defaults, NOT the orchestrate reference shape) +# what run_streaming_e2e.sh invokes per relaunch) # Perf pair: # LOG=/apps/chcai/perf_1node.log NUM_TRAIN_BATCHES=200 NUM_EVAL_BATCHES=0 \ # EVAL_EACH_WINDOW=0 METRIC_LOG_FREQ=20 \ @@ -115,7 +106,7 @@ if [ -z "$PHASE" ]; then fi # ---- shared config (env-overridable) ---------------------------------------- -CONTAINER=${CONTAINER:-yambda_${USER:-$(id -un)}} # per-user container name (do NOT reuse another user's container — its bind mounts differ) +CONTAINER=${CONTAINER:-yambda_primus} REPO=${REPO:-$REPO_ROOT} # repo path inside the container IMAGE=${IMAGE:-rocm/primus:v26.3} # [CLUSTER-SPECIFIC] ROCm/arch base image BAKED_IMAGE=${BAKED_IMAGE:-yambda_primus_baked:latest} @@ -123,13 +114,6 @@ BAKED_TAR=${BAKED_TAR:-/apps/chcai/yambda_primus_baked.tar} # [CLUSTER-SPECIFI USE_BAKED=${USE_BAKED:-1} OVERLAY=${RDMA_OVERLAY:-/apps/chcai/rdma_host_el9_new} # [CLUSTER-SPECIFIC] shared-NFS RDMA overlay (read-only, already staged) -# Bind mounts + scratch — all on shared NFS, identical path on every node. -# REPO_MOUNT : NFS home root that contains THIS repo (bind-mounted rw). -# DATA_MOUNT : NFS root with the (shared, read-only) dataset + RDMA overlay + -# pip/fbgemm build assets. Kept as-is so the dataset is NOT -# duplicated. You only need read access here. -# SCRATCH : this run's WRITABLE output root (logs / tb / traces). -# All env-overridable, so nothing is hardwired to one user's home. REPO_MOUNT=${REPO_MOUNT:-$HOME} # NFS home holding the repo (must contain $REPO); override if your repo lives elsewhere DATA_MOUNT=${DATA_MOUNT:-/apps/chcai} # shared dataset + RDMA overlay + pip/fbgemm assets (read-only) SCRATCH=${SCRATCH:-$HOME/yambda_runs} # writable output root (logs / tb / traces) @@ -151,11 +135,6 @@ orchestrate() { mkdir -p "$SCRATCH" 2>/dev/null || true LOG=${LOG:-$SCRATCH/yambda_slurm.${SLURM_JOB_ID:-manual}.log} - # Run-shape defaults. By DEFAULT a bare `sbatch scripts/launch_slurm.sh` - # reproduces the FROZEN REFERENCE run: full 299-window sweep (START_TS=0) with - # the data-fraction eval cadence (eval every 0.5% of the training stream). Set - # SMOKE=1 for a fast functional check (short dense window, capped batches, - # per-window eval). Any individual knob below stays env-overridable. MODE=${MODE:-streaming-train-eval} if [ "${SMOKE:-0}" = "1" ]; then START_TS=${START_TS:-150} @@ -171,10 +150,6 @@ orchestrate() { NUM_EVAL_BATCHES=${NUM_EVAL_BATCHES:-0} EVAL_EACH_WINDOW=${EVAL_EACH_WINDOW:-1} METRIC_LOG_FREQ=${METRIC_LOG_FREQ:-20} - # Eval cadence — the two knobs are mutually exclusive (the worker raises if both - # are >0). Data-fraction is the reference default; if the caller explicitly - # selected the per-window cadence (EVAL_EVERY_N_WINDOWS>0) leave data-pct off, - # otherwise default to the reference 0.5%-of-data cadence (per-window disabled). if [ "${EVAL_EVERY_N_WINDOWS:-0}" -gt 0 ] 2>/dev/null; then EVAL_EVERY_DATA_PCT=${EVAL_EVERY_DATA_PCT:-0} else @@ -192,12 +167,6 @@ orchestrate() { else : > "$LOG" fi - # Group/other write (but NOT read) so the in-container worker (running as root, - # squashed to `nobody` over root-squashed NFS) can append via `tee -a $LOG`. - # `tee -a` opens write-only, so 622 is sufficient -- avoid 666, which would let - # other users on the shared filesystem read (and tamper with) the job log. - # Without the write bit the worker's tee is denied and exits non-zero, which - # pipefail turns into a spurious rc=1 even when training succeeds. chmod 622 "$LOG" 2>/dev/null || true echo "[$(date)] launch_slurm/orchestrate: job=${SLURM_JOB_ID:-?} nodes=${SLURM_JOB_NODELIST:-?} nnodes=${SLURM_NNODES:-1}" | tee -a "$LOG" echo "[$(date)] resolved SCRIPT_PATH=$SCRIPT_PATH REPO=$REPO" | tee -a "$LOG" @@ -523,28 +492,12 @@ worker() { cd "$REPO_ROOT" mkdir -p "$SCRATCH" 2>/dev/null || true LOG=${LOG:-$SCRATCH/yambda_5b_8gpu.log} - # Avoid double-logging. When launched by the orchestrate phase, our stdout is - # ALREADY captured into the real $LOG by orchestrate's `tee` (and, multi-node, - # funneled through one srun pipe). Re-`tee`ing $LOG here would write every line - # twice. Orchestrate sets WORKER_TEE=0 to point our own file sink at /dev/null: - # we still echo to stdout (captured upstream) but don't duplicate the file. - # Direct single-node invocation (the streaming-e2e supervisor) leaves - # WORKER_TEE unset, so the worker keeps writing $LOG itself. + # WORKER_TEE=0 (set by orchestrate) sends our file sink to /dev/null to avoid + # double-logging, since orchestrate already tees stdout into the real $LOG. [ "${WORKER_TEE:-1}" = "0" ] && LOG=/dev/null - # TensorBoard under the writable scratch root unless the caller (e.g. the e2e - # supervisor) pinned a per-run path. Keeps the gin default from ever being used. export TENSORBOARD_LOG_PATH=${TENSORBOARD_LOG_PATH:-$SCRATCH/tb/yambda_5b} - # MLPerf Training compliance log (streaming-train-eval path). Lands beside the - # other run outputs under scratch unless the caller pins it. Rank 0 writes it; - # check it post-run with: - # python -m mlperf_logging.compliance_checker --usage training \ - # --ruleset 5.0.0 "$MLPERF_LOG_PATH" - # Default to a PER-JOB filename so each standalone `sbatch` gets a clean - # compliance log: mllog opens the file in APPEND mode, so a fixed name would - # accumulate events across runs and fail the compliance_checker (duplicate - # INIT_START/RUN_START). The streaming-e2e supervisor pins MLPERF_LOG_PATH - # explicitly (and inits it once at run start), so its relaunch-into-same-file - # append semantics are preserved untouched. + # MLPerf compliance log (rank 0 writes it). Per-job filename so each standalone + # sbatch gets a clean log; the e2e supervisor pins MLPERF_LOG_PATH itself. export MLPERF_LOG_PATH=${MLPERF_LOG_PATH:-$SCRATCH/mlperf/yambda_5b_mlperf.${SLURM_JOB_ID:-manual}.log} echo "[$(date)] REPO_ROOT=$REPO_ROOT" | tee -a "$LOG" @@ -594,20 +547,9 @@ worker() { export WORLD_SIZE=$(( NNODES * GPUS_PER_NODE )) echo "[$(date)] topology: nnodes=$NNODES node_rank=$NODE_RANK gpus_per_node=$GPUS_PER_NODE world_size=$WORLD_SIZE master=$MASTER_ADDR:${MASTER_PORT:-}" | tee -a "$LOG" - # NCCL bootstrap NIC. The container is --network=host so RCCL sees ALL host - # interfaces; if left to auto-detect, NCCL can pick a non-routable per-GPU RoCE - # /31 (benic* 192.168.x) link and fail bootstrap with "No route to host" (this - # is node-dependent: it worked on some nodes and not others, causing repetitive - # single-node init failures). Pin it explicitly to avoid that. - # * Single-node (NNODES==1): all ranks are on THIS host, so only the bootstrap - # control-plane crosses the socket NIC (data plane is intra-node XGMI/PCIe, - # see below). Loopback is reachable by every local rank on ANY host and is - # node-independent — same rationale as MASTER_ADDR=localhost above — so it - # "just works" on dev boxes that have no fenic0 (e.g. a single MI355 node). - # * Multi-node (NNODES>1): needs a routable host NIC shared across nodes for - # the cross-node TCP rendezvous; default to the meta64 fenic0. - # Both remain ${NCCL_SOCKET_IFNAME:-...}-overridable for other fabrics. - # [CLUSTER-SPECIFIC] multi-node routable host NIC for TCP bootstrap (find via `ip -br addr`). + # NCCL bootstrap NIC: loopback single-node, routable host NIC multi-node (pin + # to avoid auto-detect picking a non-routable per-GPU RoCE link). Override via + # $NCCL_SOCKET_IFNAME. [CLUSTER-SPECIFIC] multi-node fenic0 (find via `ip -br addr`). if [ "$NNODES" -gt 1 ]; then export NCCL_SOCKET_IFNAME=${NCCL_SOCKET_IFNAME:-fenic0} else @@ -646,17 +588,8 @@ worker() { export NCCL_IB_TIMEOUT=${NCCL_IB_TIMEOUT:-14} export NCCL_IGNORE_CPU_AFFINITY=${NCCL_IGNORE_CPU_AFFINITY:-1} export RCCL_MSCCL_ENABLE=${RCCL_MSCCL_ENABLE:-0} - # GPU-Direct RDMA: ENABLED by default. The brcmrdma host kernel ships the - # inbox peer-memory client (`ib_register_peer_memory_client` in - # /proc/kallsyms), so RCCL does true GPU<->NIC DMA over bnxt_re instead of - # bouncing through host memory. Measured ~+22% throughput at 2 nodes - # (65.7%->79.8% weak-scaling efficiency) vs the old host-staged path. - # GDR_LEVEL=5 (most permissive) is required so GDR is used even when the GPU - # and NIC cross the CPU root complex. NCCL_DMABUF_ENABLE=1 is a harmless - # no-op here (kernel lacks CONFIG_DMABUF_MOVE_NOTIFY/CONFIG_PCI_P2PDMA, so - # peermem carries it). Enabling is non-fatal: if peermem is ever absent RCCL - # just logs "GDR 0" and falls back to host staging. Override with - # NCCL_NET_GDR_LEVEL=0 to force the legacy host-staged path. + # GPU-Direct RDMA on by default (~+22% throughput at 2 nodes via peermem). + # Set NCCL_NET_GDR_LEVEL=0 to force the legacy host-staged path. export NCCL_NET_GDR_LEVEL=${NCCL_NET_GDR_LEVEL:-5} export NCCL_DMABUF_ENABLE=${NCCL_DMABUF_ENABLE:-1} echo "[$(date)] NCCL: RDMA over bnxt_re (GID idx ${NCCL_IB_GID_INDEX}, TC ${NCCL_IB_TC}, GDR_LEVEL=${NCCL_NET_GDR_LEVEL}, DMABUF=${NCCL_DMABUF_ENABLE}; meta64 bnxt_re config, validated)" | tee -a "$LOG" From a9f8efdec4769c5c97590e6d9dda9076842f0290 Mon Sep 17 00:00:00 2001 From: suachong Date: Wed, 24 Jun 2026 19:40:09 +0000 Subject: [PATCH 088/127] dlrmv4: trim verbose MLPerf-wiring comments to 1-2 lines Collapse the large explanatory comment/docstring blocks in the MLPerf logging wiring (logger, train_ranker boundaries, streaming loop hooks, MetricsLogger counters) to keep the PR reviewable. No logic changes. Co-authored-by: Cursor --- .../dlrm_v3/train/mlperf_logging_utils.py | 46 ++++------------- .../dlrm_v3/train/train_ranker.py | 51 +++++-------------- .../dlrm_v3/train/utils.py | 44 ++++------------ .../generative_recommenders/dlrm_v3/utils.py | 40 ++++----------- 4 files changed, 46 insertions(+), 135 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py index 74eac4ede..47d544ad7 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py @@ -15,13 +15,8 @@ # pyre-unsafe """MLPerf Training compliance logging for the DLRMv3 streaming-train-eval path. -Thin, rank-0-gated wrapper around ``mlperf_logging.mllog`` so the streaming -loop can emit the MLPerf event stream (INIT/RUN/BLOCK/EVAL/RUN_STOP) without -every call site re-checking the rank or guarding against a missing dependency. - -Modeled on recommendation_v2/torchrec_dlrm's inline ``submission_info`` but -extended with rank-0 gating + optional distributed barriers (the NeMo / unet3d -``sync`` pattern), so a multi-rank run produces exactly one valid log. +Rank-0-gated wrapper around ``mlperf_logging.mllog`` so the streaming loop emits +the MLPerf event stream without every call site re-checking rank or the dep. """ import logging @@ -63,11 +58,9 @@ def _barrier() -> None: class MLPerfLogger: """Rank-0-gated facade over ``mllog``. - All event methods no-op on non-zero ranks and when ``mlperf_logging`` is not - installed, so callers never need to guard. ``sync=True`` inserts an - all-rank ``dist.barrier()`` before the (rank-0-only) emission so the logged - timestamp reflects the slowest rank reaching the boundary -- required for - INIT_STOP/RUN_START/RUN_STOP per the MLPerf rules. + Event methods no-op on non-zero ranks and when mlperf_logging is absent. + ``sync=True`` barriers before emit so the timestamp reflects the slowest rank + (required for INIT_STOP/RUN_START/RUN_STOP). """ def __init__( @@ -79,29 +72,18 @@ def __init__( submitter_name: str = "reference_implementation", ): self.enabled: bool = _MLLOG_AVAILABLE - # CRITICAL: use the EXPLICIT global rank passed by the caller, not a - # dist.get_rank() lookup. This logger is constructed BEFORE - # dist.init_process_group (so the init phase can be timed), at which - # point torch.distributed.get_rank() is unavailable and would return 0 - # for every process -> all 16 ranks would log everything. The caller - # (train_ranker) already knows the true global rank - # (node_rank * gpus_per_node + local_rank), so trust it. Fall back to a - # best-effort dist/zero lookup only when not provided. + # Use the EXPLICIT caller rank: this is built before init_process_group, + # when dist.get_rank() would return 0 on every rank (all would log). self.rank: int = rank if rank is not None else _rank() self.benchmark_name: str = benchmark_name self.submitter_name: str = submitter_name self._logger = None if not self.enabled: return - # Only rank 0 emits events, so only rank 0 needs the file handler: - # attaching it on every rank wastes file handles and risks contention on - # a shared log path. Non-zero ranks configure mllog without a filename - # (their event methods no-op anyway). + # Only rank 0 emits, so only rank 0 needs the file handler. if log_path and self.rank == 0: log_dir = os.path.dirname(log_path) - # dirname is "" when log_path has no directory component (e.g. - # MLPERF_LOG_PATH=mlperf.log); os.makedirs("") raises, so guard it. - if log_dir: + if log_dir: # guard: os.makedirs("") raises for a bare filename os.makedirs(log_dir, exist_ok=True) mllog.config(filename=log_path, default_stack_offset=default_stack_offset) else: @@ -169,14 +151,8 @@ def get_mlperf_logger( ) -> Optional[MLPerfLogger]: """Build a configured :class:`MLPerfLogger`, or ``None`` if unavailable. - ``benchmark_name`` / ``submitter_name`` are gin-configurable (and the path is - env-overridable via ``$MLPERF_LOG_PATH``) so a submission can stamp its own - benchmark string without code changes. The log path defaults to - ``$MLPERF_LOG_PATH`` when set, else ``""`` (mllog logs to stdout). - - Returns ``None`` when ``mlperf_logging`` is not installed so callers' existing - ``mlperf_logger is not None`` guards cleanly disable logging -- otherwise they - would pass the guard and then hit ``logger.constants`` (which is ``None``). + Path defaults to ``$MLPERF_LOG_PATH``. Returns ``None`` (not a disabled + logger) so callers' ``is not None`` guards cleanly skip logging. """ if not _MLLOG_AVAILABLE: return None diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py index 25711bd02..30b081946 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py @@ -83,20 +83,12 @@ def _main_func( gin.parse_config_file(gin_file, skip_unknown=True) apply_env_bootstrap() - # MLPerf compliance logging is only wired for the streaming-train-eval - # (yambda-5b) benchmark path. Build the rank-0-gated logger now. No-ops on - # non-zero ranks and when mlperf_logging is not installed. + # Rank-0-gated MLPerf logger, only for the streaming-train-eval path. mlperf_logger = ( get_mlperf_logger(rank=rank) if mode == "streaming-train-eval" else None ) - # Emit the init-start boundary before setup so the init phase is measured, - # but ONLY when this is guaranteed to be a cold start. CKPT_PATH unset means - # checkpoints are disabled (the default + submission config) -> always cold - # start. When CKPT_PATH is set (resumable / e2e-supervisor runs) we defer the - # decision to resume_cold_start below and skip the pre-setup markers, so a - # resume relaunch never emits an orphaned INIT_START/RUN_START. The whole - # downstream sequence (INIT_STOP/RUN_START/blocks/eval/RUN_STOP) is gated on - # this flag so the log is always balanced. + # Emit INIT_START before setup only on a guaranteed cold start (CKPT_PATH + # unset); resume relaunches skip it so the log stays balanced. mlperf_init_logged = False if mlperf_logger is not None and not os.environ.get("CKPT_PATH", ""): mlperf_logger.event(key=mlperf_logger.constants.CACHE_CLEAR, value=True) @@ -208,11 +200,8 @@ def _main_func( ) ) - # MLPerf: submission info + hyperparameters, then the init/run boundary. - # Gated on (init markers emitted AND genuine cold start) so the e2e - # supervisor's resume relaunches don't restart the INIT/RUN markers - # mid-stream (which would invalidate the single-run log), and so the log is - # never left with an INIT_STOP that has no matching INIT_START. + # MLPerf submission info + hyperparameters + INIT_STOP/RUN_START, only on a + # genuine cold start so resume relaunches don't reopen the run markers. mlperf_run_active = ( mlperf_logger is not None and mlperf_init_logged and resume_cold_start ) @@ -228,11 +217,8 @@ def _gin_param(name: str, default: object) -> object: value = gin.query_parameter(name) except (ValueError, KeyError): return default - # When a binding is a gin macro/configurable reference (e.g. - # `@dlr/env_float()`), query_parameter returns the unevaluated - # reference object, which the MLPerf logger cannot encode. Resolve - # it to its actual value so env-overridden LRs are logged as real - # numbers. Plain literals pass through unchanged. + # Resolve gin macro refs (e.g. @dlr/env_float()) to real values so + # env-overridden LRs log as numbers, not unencodable objects. if hasattr(value, "scoped_configurable_fn"): try: return value.scoped_configurable_fn() @@ -243,11 +229,9 @@ def _gin_param(name: str, default: object) -> object: global_batch_size = world_size * int(train_dataloader.batch_size) mlperf_logger.event(key=c.GLOBAL_BATCH_SIZE, value=global_batch_size) mlperf_logger.event(key=c.GRADIENT_ACCUMULATION_STEPS, value=1) - # Log the ACTUAL seed chosen in setup() (random per-run unless $SEED is - # pinned; setup() exports the chosen value to $SEED). + # Actual seed chosen in setup() (exported to $SEED). mlperf_logger.event(key=c.SEED, value=int(os.environ.get("SEED", "1"))) - # Dense (Adam) + sparse (RowWiseAdagrad) optimizer hyperparameters, - # read from the active gin bindings. + # Dense (Adam) + sparse (RowWiseAdagrad) optimizer hyperparameters from gin. mlperf_logger.event( key=c.OPT_NAME, value=_gin_param( @@ -327,24 +311,17 @@ def _gin_param(name: str, default: object) -> object: resume_batch_idx_in_window=resume_batch_idx_in_window, resume_split_contract=resume_split_contract, resume_cold_start=resume_cold_start, - # Only pass the logger when the run boundaries were emitted, so - # the loop never produces orphan block/eval events. + # Only pass the logger when run boundaries were emitted, so the + # loop never produces orphan block/eval events. mlperf_logger=mlperf_logger if mlperf_run_active else None, ) except Exception as e: logger.info(traceback.format_exc()) raise Exception(e) finally: - # Graceful distributed teardown (runs on BOTH success and failure). - # Previously cleanup() ran only in the except branch, so a clean finish - # returned without destroying the process group: ranks that returned - # first let rank 0's TCPStore close while peers' ProcessGroupNCCL - # heartbeat-monitor threads were still polling it, emitting the noisy - # (but harmless) "Failed to check the 'should dump' flag on TCPStore / - # Broken pipe" warnings + C++ stack traces at exit. Barrier first so all - # ranks reach the end in lockstep, then destroy_process_group() stops - # each rank's monitor thread and closes NCCL/the store in order. Both - # steps are guarded/best-effort so teardown never masks a real error. + # Graceful distributed teardown on both success and failure: barrier so + # all ranks finish in lockstep, then destroy the process group (best- + # effort) to avoid noisy TCPStore/NCCL shutdown warnings at exit. if torch.distributed.is_initialized(): try: torch.distributed.barrier() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py index 641e8e67c..f3f66e443 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -1616,9 +1616,7 @@ def streaming_train_eval_loop( streaming_diag_unique_emb: bool = False, # --- test-only failure injection knob --- die_at_step: int = -1, - # MLPerf compliance logger (rank-0-gated facade). None disables all MLPerf - # event emission; the loop is otherwise unchanged. Supplied by train_ranker - # for the streaming-train-eval benchmark path. + # MLPerf logger (rank-0-gated); None disables all MLPerf event emission. mlperf_logger: Optional[Any] = None, # Which eval metric drives EVAL_ACCURACY + the convergence decision. Format # "{window|lifetime}_{auc|gauc|accuracy|ne}" (bare "window"/"lifetime" => auc). @@ -2198,9 +2196,7 @@ def _run_eval_window( except OSError as _e: logger.warning("failed to write metrics sink %s: %s", _metrics_path, _e) metric_logger.resume_perf("eval") - # Return metrics to the streaming loop so the MLPerf EVAL_STOP / - # EVAL_ACCURACY hooks (and the stop-on-target check) can consume them. - # Unconditional (outside the rank-0 logging block) so every rank returns. + # Return metrics (on every rank) so the MLPerf eval hooks can consume them. return _eval_metrics def _maybe_checkpoint(train_ts: int) -> None: @@ -2271,12 +2267,8 @@ def _should_eval(i: int) -> bool: ) # --- MLPerf progress accounting + event helpers --------------------------- - # `total_train_samples` is the denominator for the MLPerf samples_count / - # epoch_num progress unit: the total GLOBAL trainable samples over the - # configured window range. Computed once up-front (one O(N) anchor mask per - # window -- the same call the loop makes per window) and logged as - # TRAIN_SAMPLES. Skipped entirely when no MLPerf logger is attached, since - # the masks are not free. `mlperf_run_stopped` guards single RUN_STOP. + # total_train_samples = epoch_num denominator (global trainable samples over + # the window range), computed once and logged as TRAIN_SAMPLES. total_train_samples = 0 mlperf_run_stopped = [False] if mlperf_logger is not None: @@ -2300,10 +2292,7 @@ def _should_eval(i: int) -> bool: key=mlperf_logger.constants.EVAL_SAMPLES, value=int(eval_global_indices.size), ) - # Let MetricsLogger.compute_and_log emit the per-step MLPerf `train_loss` - # event (rank-0 gated) at the metric-logging cadence, stamped with the - # current base LR. Read param_groups[0] defensively (KeyedOptimizer - # exposes it; guard against any optimizer that does not). + # Wire the logger + LR getter so compute_and_log emits the train_loss event. metric_logger.mlperf_logger = mlperf_logger def _current_lr() -> float: @@ -2377,14 +2366,9 @@ def _mlperf_run_stop(status: object) -> None: mlperf_run_stopped[0] = True def _mlperf_eval_stop(eval_metrics: Dict[str, float]) -> bool: - # Emit EVAL_ACCURACY (the selected eval listen_plus metric) + EVAL_STOP, - # and drive an early SUCCESS RUN_STOP when the target is reached. Returns - # True iff the run should stop now -- the SAME value on every rank. - # - # CRITICAL (deadlock avoidance): the eval metric is only valid on global - # rank 0, so rank 0 decides and BROADCASTS the boolean; all ranks then - # break (or continue) in lockstep. A per-rank test could diverge and hang - # the next window's embedding all-to-all (600s ALLTOALL_BASE watchdog). + # Emit EVAL_ACCURACY + EVAL_STOP, early SUCCESS RUN_STOP on target. + # Rank 0 decides + broadcasts the stop bool so all ranks break in lockstep + # (a per-rank test could diverge and hang the next all-to-all). if mlperf_logger is None: return False eval_value = _eval_target_metric(eval_metrics) @@ -2590,11 +2574,8 @@ def _do_eval_nb(train_ts: int, gstep: int) -> None: # MLPerf target reached: RUN_STOP already emitted; stop training. break - # Final eval over the SAME fixed user-holdout set (consistent with the - # per-window evals above). Reuses _run_eval_window so metrics are reset and - # reported the same way. Falls back to the legacy final-window eval for - # datasets without user holdout. Skipped if the MLPerf target was already - # reached mid-run (RUN_STOP already emitted, run is over). + # Final eval over the fixed user-holdout set (legacy final-window eval + # otherwise). Skipped if the MLPerf target already stopped the run mid-run. if not mlperf_run_stopped[0]: dataset.dataset.is_eval = True # pyre-ignore [16] _mlperf_eval_start() @@ -2612,12 +2593,9 @@ def _do_eval_nb(train_ts: int, gstep: int) -> None: iter(make_streaming_dataloader(dataset=dataset, ts=num_train_ts)), label="eval@final", ) - # EVAL_ACCURACY/EVAL_STOP for the final pass (may emit SUCCESS RUN_STOP - # if the target was met exactly at the end). _mlperf_eval_stop(final_metrics) if rank == 0: for k, v in final_metrics.items(): print(f"{k}: {v}") - # End-of-run RUN_STOP: SUCCESS iff the final lifetime AUC met the target, - # else ABORTED. No-op if a SUCCESS RUN_STOP already fired above. + # End-of-run RUN_STOP: SUCCESS if final metric met target, else ABORTED. _mlperf_finalize(final_metrics) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py index 390c9bfbb..879c93b97 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py @@ -1031,19 +1031,12 @@ def _make_reg(ws: int) -> List[RecMetricComputation]: self.regression_metrics["eval"] = _make_reg(window_size) self.global_step: Dict[str, int] = {"train": 0, "eval": 0} - # Monotonic, resume-safe count of GLOBAL trained samples (summed across - # ranks), used as the MLPerf `samples_count` progress unit. Distinct from - # the perf-only `_perf_total_samples` below (per-rank, not checkpointed): - # this one is persisted/restored alongside `global_step` so a resumed - # streaming run continues the convergence-progress count. + # MLPerf `samples_count` progress unit: global trained samples, persisted + # alongside global_step so a resumed run continues the count. self.cumulative_train_samples: int = 0 self._rank: int = int(rank) - # Optional MLPerf logger + learning-rate accessor, wired by the streaming - # loop (kept duck-typed -- expects `.event(key, value, metadata)` and - # `.constants` -- to avoid a train-module import cycle). When set, - # compute_and_log emits a POINT_IN_TIME `train_loss` event (rank-0 gated - # inside the logger) at the metric-logging cadence, mirroring the per-step - # MLPerf train_loss readout other benchmarks log. + # Optional MLPerf logger + LR accessor wired by the streaming loop (duck- + # typed to avoid a train-module import cycle); drives the train_loss event. self.mlperf_logger: Optional[Any] = None self.lr_getter: Optional[Callable[[], float]] = None self.tb_logger: Optional[SummaryWriter] = None @@ -1097,9 +1090,7 @@ def resume_perf(self, category: str) -> None: @property def auc_threshold(self) -> Optional[float]: - """Configured time-to-target AUC threshold (None if unset). Exposed so - the streaming loop can drive the MLPerf SUCCESS RUN_STOP off the same - target without reaching into the private attribute.""" + """Configured time-to-target AUC threshold (None if unset).""" return self._auc_threshold @property @@ -1165,11 +1156,8 @@ def update( self._perf_samples_counter.dtype ) self._perf_steps_in_window += 1 - # MLPerf progress counter: global trained samples this step. Local - # batch sample count (num_candidates is per-rank) scaled by world - # size approximates the global count without an extra collective; - # accumulated on CPU as a plain int so it serializes trivially into - # the checkpoint (see save/load_nonsparse_checkpoint). + # MLPerf progress counter: per-rank sample count scaled by world size + # approximates global trained samples without an extra collective. self.cumulative_train_samples += int(num_candidates.numel()) * self._world_size def compute(self, mode: str = "train") -> Dict[str, float]: @@ -1261,14 +1249,9 @@ def compute_and_log( global_step=self.global_step[mode], ) - # Train-loss readout: surface a single GLOBAL (cross-rank mean) training - # loss on the regular console logger every `metric_log_frequency` batches, - # so progress is visible from step 0 instead of only at the first - # end-of-window eval. The per-loss-term breakdown already goes to - # TensorBoard above (losses/train_*); here we add the combined scalar. - # The all-reduce is a cheap 1-element collective run by EVERY rank at the - # same deterministic steps (this method is called in lockstep), so it - # cannot desync. Set METRIC_LOG_FREQ low (e.g. 1-5) to see it per step. + # Global (cross-rank mean) train loss to console/TB + the MLPerf + # train_loss event, at the metric-logging cadence. The 1-element + # all-reduce runs on every rank in lockstep, so it cannot desync. if mode == "train" and additional_logs is not None and "losses" in additional_logs: loss_terms = additional_logs["losses"] if loss_terms: @@ -1289,9 +1272,6 @@ def compute_and_log( f"train - Step {self.global_step['train']} " f"train_loss={train_loss:.5f}" ) - # MLPerf POINT_IN_TIME train_loss (rank-0 gated in the logger). - # samples_count = cumulative GLOBAL trained samples (the same - # progress unit as block/eval events); lr = current base LR. if self.mlperf_logger is not None: c = self.mlperf_logger.constants md: Dict[str, Any] = { From 024e64b56e7e46376ace1d706115eb08f98e5f12 Mon Sep 17 00:00:00 2001 From: suachong Date: Wed, 24 Jun 2026 19:56:36 +0000 Subject: [PATCH 089/127] dlrmv4: trim seed_everything / decorrelate_runtime_rng docstrings Co-authored-by: Cursor --- .../dlrm_v3/train/utils.py | 54 +++++-------------- 1 file changed, 13 insertions(+), 41 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py index f3f66e443..bf5fd5512 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -78,24 +78,12 @@ @gin.configurable def seed_everything(seed: int = -1, rank: int = 0) -> None: - """Seed all RNGs so weight init (make_model) is reproducible across runs. - - Same seed on every rank => dense params are initialized identically across - ranks; sharded embeddings are init'd from the meta device by DMP. Fixing the - seed makes runs an init-matched A/B (data order is already deterministic via - the sampler). gin-configurable via $SEED (yambda_5b.gin: seed_everything.seed); - call this right before make_model(), AFTER setup() (the process group must be - initialized for the cross-rank broadcast below) and after the full gin parse. - - Default seed < 0 (the gin default, i.e. $SEED unset) => draw a FRESH RANDOM - seed every run so each launch explores a different dense weight init. rank 0 - draws the seed and broadcasts it so all ranks share one value; the chosen - seed is exported to $SEED and logged so any run can be reproduced after the - fact by re-pinning $SEED. seed >= 0 (i.e. $SEED pinned) reproduces a specific - run exactly. NOTE (streaming-train-eval): data ORDER and the train/holdout - split do NOT depend on this seed — order is time-deterministic - (StreamingWindowSampler) and the split is governed by $SPLIT_SALT. This seed - governs dense weight init + global-RNG stochastic ops. + """Seed all RNGs (same value on every rank) for reproducible dense weight init. + + Call right before make_model(), after setup() (process group needed for the + broadcast) and the gin parse. seed < 0 ($SEED unset) draws a fresh random seed + per run (rank 0 broadcasts; exported to $SEED); seed >= 0 reproduces a run. + Data order/split are independent of this seed (StreamingWindowSampler/$SPLIT_SALT). """ import random @@ -129,29 +117,13 @@ def seed_everything(seed: int = -1, rank: int = 0) -> None: @gin.configurable def decorrelate_runtime_rng(rank: int = 0, enabled: bool = True) -> None: - """Offset the global RNG by ``rank`` so RUNTIME stochastic ops draw - decorrelated draws per data-parallel rank. - - The only such op here is HSTU dropout (input_dropout=0.2, - linear_dropout_rate=0.1; see configs.get_hstu_configs). seed_everything() - sets an IDENTICAL seed on every rank — required so replicated dense weights - init identically — which also makes every rank draw the SAME dropout masks - in the forward. Gradients still differ (each rank sees different data), so - that is not incorrect, but identical masks waste the extra mask diversity - that decorrelated replicas give per global batch. This re-seeds torch/cuda - with $SEED + rank to recover it (the standard data-parallel RNG track, cf. - Megatron's tensor/data-parallel RNG separation). - - ORDERING IS LOAD-BEARING: call this AFTER everything that must be identical - across ranks — make_model() (dense weight init) AND make_optimizer_and_shard() - (the pre-DMP re-seed + sharded embedding init). It deliberately perturbs only - forward-time stochasticity, never init. - - Reproducibility is preserved: the offset is a pure function of the resolved - $SEED (exported by seed_everything) and rank, and per-rank RNG state is - snapshotted/restored on checkpoint resume (see checkpoint.py). Set - enabled=False (gin: decorrelate_runtime_rng.enabled) to restore the legacy - identical-mask-on-every-rank behavior. + """Re-seed torch/cuda with $SEED + rank so HSTU dropout draws different masks + per data-parallel rank (seed_everything's identical seed would draw the same). + + MUST run after make_model() + make_optimizer_and_shard() so init stays + identical across ranks; it perturbs only forward-time stochasticity. + Reproducible (pure fn of $SEED + rank; RNG state checkpointed). enabled=False + keeps the legacy identical-mask behavior. """ if not enabled: logger.info( From f63e5d4c5cd3f0d3b85c0c8f28e88c04b208ddc6 Mon Sep 17 00:00:00 2001 From: suachong Date: Wed, 24 Jun 2026 19:56:37 +0000 Subject: [PATCH 090/127] dlrmv4: drop .gitignore changes from the PR Revert recommendation_v4/.gitignore to base. Local run artifacts and ad-hoc analysis files are kept out of the repo via .git/info/exclude (local, uncommitted) instead, removing one file from the PR diff. Co-authored-by: Cursor --- recommendation_v4/.gitignore | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/recommendation_v4/.gitignore b/recommendation_v4/.gitignore index 4bfbaa0ca..5edddc5b3 100644 --- a/recommendation_v4/.gitignore +++ b/recommendation_v4/.gitignore @@ -157,28 +157,3 @@ dmypy.json # Cython debug symbols cython_debug/ - -# SLURM batch stdout + local run artifacts (run logs are never committed; -# the real run logs / MLPerf compliance logs live under $SCRATCH, outside the -# repo). The trainer's #SBATCH --output lands in the submit dir as -# yambda_slurm..out. -yambda_slurm.*.out -yambda_slurm.*.log -compliance_checker.log - -# Local container build/run helpers — environment-specific, not committed. -/Dockerfile.nvidia -/scripts/run_docker.sh - -# Local dlrmv4 analysis artifacts (per-job plots/scripts/dumps); kept on disk -# for ad-hoc analysis but never committed. -/analyze_*.py -/dump_eval_*.py -/perday_*.py -/parse_lr_sweep.py -/gen_canvas.py -/plot_*.py -/*.png -/*.csv -/scripts/bench_collectives.py -/docs/v4_vs_v2_and_hstu_walkthrough.md From 02f2d2b10db6f2ecbd5893f04821f541789d7e7e Mon Sep 17 00:00:00 2001 From: suachong Date: Wed, 24 Jun 2026 22:11:21 +0000 Subject: [PATCH 091/127] dlrmv4: centralize MLPerf emission + fix submission identity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the MLPerf event stream into mlperf_logging_utils.py: a MLPerfRunTracker state machine owns the block/eval/run markers, progress metadata, and the convergence decision (replacing ~145 lines of closures in streaming_train_eval_loop), and MLPerfLogger.log_run_start emits submission info + hyperparameters + INIT_STOP/RUN_START (collapsing the inline block in train_ranker). Convergence/EVAL_ACCURACY is fixed to per-window AUC: drop the eval_accuracy_auc_mode knob (gin + loop param + launch_slurm passthrough). Submission identity: SUBMISSION_ORG defaults to AMD, SUBMISSION_PLATFORM to MI355X (was the org name — a bug), both overridable via $MLPERF_SUBMISSION_PLATFORM. Co-authored-by: Cursor --- .../dlrm_v3/train/gin/yambda_5b.gin | 15 +- .../dlrm_v3/train/mlperf_logging_utils.py | 225 ++++++++++++++- .../dlrm_v3/train/train_ranker.py | 58 +--- .../dlrm_v3/train/utils.py | 260 ++++++++---------- .../generative_recommenders/dlrm_v3/utils.py | 79 ++++-- recommendation_v4/scripts/launch_slurm.sh | 8 +- 6 files changed, 416 insertions(+), 229 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index dd4dd4715..8a3e5b595 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -325,6 +325,14 @@ sts/env_int.default = 150 streaming_train_eval_loop.metric_log_frequency = @mlf/env_int() mlf/env_int.key = "METRIC_LOG_FREQ" mlf/env_int.default = 50 +# MLPerf train_loss event cadence (global train steps), INDEPENDENT of +# METRIC_LOG_FREQ above. 0 (default) = fall back to METRIC_LOG_FREQ, preserving +# the prior coupled behavior. Set $MLPERF_TRAIN_LOSS_LOG_FREQ>0 to log the MLPerf +# train_loss event at a different rate than the console/TB metrics. Disable the +# whole MLPerf stream with $MLPERF_LOGGING=0. +streaming_train_eval_loop.mlperf_train_loss_log_frequency = @mltlf/env_int() +mltlf/env_int.key = "MLPERF_TRAIN_LOSS_LOG_FREQ" +mltlf/env_int.default = 0 # Diagnostic: log per-batch unique/total embedding-id counts on logged steps # (rank 0). Quantifies the user-major batching redundancy and the realized # diversity from get_dataset.streaming_shuffle_fraction. Off; set $DIAG_UNIQUE_EMB=1. @@ -453,11 +461,8 @@ MetricsLogger.world_size = 8 MetricsLogger.auc_threshold = @at/env_float() at/env_float.key = "AUC_THRESHOLD" at/env_float.default = 0.80275 -# Eval metric driving EVAL_ACCURACY + early-stop: "{window|lifetime}_{auc|gauc| -# accuracy|ne}" (default window_auc). Override via $EVAL_ACCURACY_AUC_MODE. -streaming_train_eval_loop.eval_accuracy_auc_mode = @eaam/env_str() -eaam/env_str.key = "EVAL_ACCURACY_AUC_MODE" -eaam/env_str.default = "window_auc" +# EVAL_ACCURACY + early-stop are driven by per-window AUC (window_auc) vs the +# threshold above. # Lifetime-AUC backend, selectable independently for the train cumulative AUC and # the eval cumulative ("lifetime_*") AUC. Both default to "binned": # "binned" = BinnedCumulativeAUC: exact-cumulative AUC from an O(bins) score diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py index 47d544ad7..e190f0325 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py @@ -69,7 +69,8 @@ def __init__( log_path: Optional[str] = None, default_stack_offset: int = 2, benchmark_name: str = "hstu", - submitter_name: str = "reference_implementation", + submitter_name: str = "AMD", + submission_platform: str = "MI355X", ): self.enabled: bool = _MLLOG_AVAILABLE # Use the EXPLICIT caller rank: this is built before init_process_group, @@ -77,6 +78,7 @@ def __init__( self.rank: int = rank if rank is not None else _rank() self.benchmark_name: str = benchmark_name self.submitter_name: str = submitter_name + self.submission_platform: str = submission_platform self._logger = None if not self.enabled: return @@ -130,16 +132,211 @@ def end( if self.enabled and self.rank == 0: self._logger.end(key=key, value=value, metadata=metadata or {}) - def submission_info(self, benchmark_name: str, submitter_name: str) -> None: + def submission_info(self) -> None: """Emit the five SUBMISSION_* events required for a valid submission.""" if not (self.enabled and self.rank == 0): return c = mllog_constants - self.event(key=c.SUBMISSION_BENCHMARK, value=benchmark_name) - self.event(key=c.SUBMISSION_ORG, value=submitter_name) + self.event(key=c.SUBMISSION_BENCHMARK, value=self.benchmark_name) + self.event(key=c.SUBMISSION_ORG, value=self.submitter_name) self.event(key=c.SUBMISSION_DIVISION, value=c.CLOSED) self.event(key=c.SUBMISSION_STATUS, value=c.ONPREM) - self.event(key=c.SUBMISSION_PLATFORM, value=submitter_name) + self.event(key=c.SUBMISSION_PLATFORM, value=self.submission_platform) + + def log_run_start( + self, + global_batch_size: int, + seed: int, + gradient_accumulation_steps: int = 1, + ) -> None: + """Emit submission info + core hyperparameters, then INIT_STOP + RUN_START. + + Optimizer names/LRs are read from gin (dense Adam + sparse RowWiseAdagrad), + resolving env-macro refs to concrete values. Call once on a genuine cold + start, after the model is built. INIT_STOP/RUN_START barrier so the + timestamp reflects the slowest rank, so ALL ranks must call this together + (non-rank-0 / disabled calls no-op the emit but still hit the barrier). + """ + c = self.constants + self.submission_info() + self.event(key=c.GLOBAL_BATCH_SIZE, value=int(global_batch_size)) + self.event( + key=c.GRADIENT_ACCUMULATION_STEPS, value=int(gradient_accumulation_steps) + ) + self.event(key=c.SEED, value=int(seed)) + self.event( + key=c.OPT_NAME, + value=_gin_param("dense_optimizer_factory_and_class.optimizer_name", "Adam"), + ) + self.event( + key=c.OPT_BASE_LR, + value=_gin_param("dense_optimizer_factory_and_class.learning_rate", None), + ) + self.event( + key="opt_sparse_name", + value=_gin_param( + "sparse_optimizer_factory_and_class.optimizer_name", "RowWiseAdagrad" + ), + ) + self.event( + key="opt_sparse_base_learning_rate", + value=_gin_param( + "sparse_optimizer_factory_and_class.learning_rate", None + ), + ) + self.end(key=c.INIT_STOP, sync=True) + self.start(key=c.RUN_START, sync=True) + + +def _gin_param(name: str, default: Any) -> Any: + """Read a gin-bound parameter, resolving env-macro refs to concrete values. + + Returns ``default`` if the parameter is unbound or a macro ref cannot be + resolved (so env-overridden LRs log as numbers, not unencodable objects). + """ + try: + value = gin.query_parameter(name) + except (ValueError, KeyError): + return default + if hasattr(value, "scoped_configurable_fn"): + try: + return value.scoped_configurable_fn() + except Exception: + return default + return value + + +class MLPerfRunTracker: + """Centralized MLPerf run-boundary state machine for the streaming loop. + + Owns the block/eval/run markers, the SAMPLES_COUNT/EPOCH_NUM progress + metadata, and the convergence decision (per-window AUC vs the configured + ``auc_threshold``). Every method no-ops when ``logger`` is None, so the + streaming loop can call them unconditionally. The convergence metric is + fixed to per-window AUC (higher-is-better). + """ + + # MetricsLogger.compute key short name for per-window AUC. + _EVAL_METRIC_SHORT = "window_auc" + + def __init__( + self, + logger: Optional[MLPerfLogger], + metric_logger: Any, + total_train_samples: int, + rank: int, + device: Any, + ): + self.logger = logger + self.metric_logger = metric_logger + self.total_train_samples = int(total_train_samples) + self.rank = int(rank) + self.device = device + self.run_stopped: bool = False + # Idempotency flag so the boundary helpers and the outer loop can both + # call start/stop without risking a double BLOCK_START/STOP. + self._block_open: bool = False + + @property + def enabled(self) -> bool: + return self.logger is not None + + def _progress(self) -> Dict[str, Any]: + c = self.logger.constants + samples = self.metric_logger.cumulative_train_samples + epoch = ( + samples / self.total_train_samples if self.total_train_samples > 0 else 0.0 + ) + return {c.SAMPLES_COUNT: samples, c.EPOCH_NUM: epoch} + + def log_dataset_sizes(self, eval_samples: Optional[int] = None) -> None: + if not self.enabled: + return + c = self.logger.constants + self.logger.event(key=c.TRAIN_SAMPLES, value=self.total_train_samples) + if eval_samples is not None: + self.logger.event(key=c.EVAL_SAMPLES, value=int(eval_samples)) + + def block_start(self) -> None: + if self.enabled and not self._block_open: + self.logger.start( + key=self.logger.constants.BLOCK_START, metadata=self._progress() + ) + self._block_open = True + + def block_stop(self) -> None: + if self.enabled and self._block_open: + self.logger.end( + key=self.logger.constants.BLOCK_STOP, metadata=self._progress() + ) + self._block_open = False + + def eval_start(self) -> None: + if self.enabled: + self.logger.start( + key=self.logger.constants.EVAL_START, metadata=self._progress() + ) + + def _target_metric(self, metrics: Dict[str, float]) -> Optional[float]: + # Key format `metric/{prefix}_{name}/{task}` (see MetricsLogger.compute); + # match the per-window AUC short name. + for key, val in metrics.items(): + short = key.split("/")[-2] if "/" in key else key + if short == self._EVAL_METRIC_SHORT: + return float(val) + return None + + def _meets_target(self, value: Optional[float]) -> bool: + thr = self.metric_logger.auc_threshold + if value is None or thr is None: + return False + return value >= thr + + def run_stop(self, status: object) -> None: + # Emit RUN_STOP exactly once, with an all-rank barrier so the timestamp + # reflects the slowest rank (MLPerf requirement). + if not self.enabled or self.run_stopped: + return + c = self.logger.constants + self.logger.end( + key=c.RUN_STOP, + metadata={c.STATUS: status, **self._progress()}, + sync=True, + ) + self.run_stopped = True + + def eval_stop(self, eval_metrics: Dict[str, float]) -> bool: + # Emit EVAL_ACCURACY + EVAL_STOP, early SUCCESS RUN_STOP on target. + # Rank 0 decides + broadcasts the stop bool so all ranks break in lockstep + # (a per-rank test could diverge and hang the next all-to-all). + if not self.enabled: + return False + c = self.logger.constants + eval_value = self._target_metric(eval_metrics) + if eval_value is not None: + self.logger.event( + key=c.EVAL_ACCURACY, value=eval_value, metadata=self._progress() + ) + self.logger.end(key=c.EVAL_STOP, metadata=self._progress()) + decision = torch.zeros(1, device=self.device) + if self.rank == 0 and not self.run_stopped and self._meets_target(eval_value): + decision[0] = 1.0 + if torch.distributed.is_available() and torch.distributed.is_initialized(): + torch.distributed.broadcast(decision, src=0) + should_stop = bool(decision.item() > 0.5) + if should_stop: + # All ranks agree -> all reach the RUN_STOP barrier together. + self.run_stop(c.SUCCESS) + return should_stop + + def finalize(self, final_metrics: Dict[str, float]) -> None: + # End-of-run RUN_STOP when the target was never crossed: SUCCESS iff the + # final eval metric meets the target, else ABORTED. + if not self.enabled or self.run_stopped: + return + c = self.logger.constants + success = self._meets_target(self._target_metric(final_metrics)) + self.run_stop(c.SUCCESS if success else c.ABORTED) @gin.configurable @@ -147,19 +344,35 @@ def get_mlperf_logger( rank: int = 0, log_path: str = "", benchmark_name: str = "hstu", - submitter_name: str = "reference_implementation", + submitter_name: str = "AMD", + submission_platform: str = "MI355X", ) -> Optional[MLPerfLogger]: """Build a configured :class:`MLPerfLogger`, or ``None`` if unavailable. Path defaults to ``$MLPERF_LOG_PATH``. Returns ``None`` (not a disabled logger) so callers' ``is not None`` guards cleanly skip logging. + + Disable knob: set ``$MLPERF_LOGGING=0`` (or false/no/off) to turn the whole + MLPerf event stream off — returns ``None`` on EVERY rank, so the train loop's + ``is not None`` guards skip emission AND the cross-rank train-loss all-reduce + in lockstep. Default (unset / "1") = enabled, preserving prior behavior. """ if not _MLLOG_AVAILABLE: return None + if os.environ.get("MLPERF_LOGGING", "1").strip().lower() in ( + "0", "false", "no", "off", + ): + logger.info("MLPerf logging disabled via $MLPERF_LOGGING=0") + return None resolved_path = os.environ.get("MLPERF_LOG_PATH", log_path) + # SUBMISSION_PLATFORM defaults to "MI355X"; override per-submitter via env. + resolved_platform = os.environ.get( + "MLPERF_SUBMISSION_PLATFORM", submission_platform + ) return MLPerfLogger( rank=rank, log_path=resolved_path, benchmark_name=benchmark_name, submitter_name=submitter_name, + submission_platform=resolved_platform, ) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py index 30b081946..50d359ef6 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py @@ -206,60 +206,14 @@ def _main_func( mlperf_logger is not None and mlperf_init_logged and resume_cold_start ) if mlperf_run_active: - c = mlperf_logger.constants - mlperf_logger.submission_info( - benchmark_name=mlperf_logger.benchmark_name, - submitter_name=mlperf_logger.submitter_name, + # Submission info + hyperparameters + INIT_STOP/RUN_START, all emitted by + # the logger (optimizer names/LRs read from gin internally). Seed is the + # value setup() resolved and exported to $SEED. + mlperf_logger.log_run_start( + global_batch_size=world_size * int(train_dataloader.batch_size), + seed=int(os.environ.get("SEED", "1")), ) - def _gin_param(name: str, default: object) -> object: - try: - value = gin.query_parameter(name) - except (ValueError, KeyError): - return default - # Resolve gin macro refs (e.g. @dlr/env_float()) to real values so - # env-overridden LRs log as numbers, not unencodable objects. - if hasattr(value, "scoped_configurable_fn"): - try: - return value.scoped_configurable_fn() - except Exception: - return default - return value - - global_batch_size = world_size * int(train_dataloader.batch_size) - mlperf_logger.event(key=c.GLOBAL_BATCH_SIZE, value=global_batch_size) - mlperf_logger.event(key=c.GRADIENT_ACCUMULATION_STEPS, value=1) - # Actual seed chosen in setup() (exported to $SEED). - mlperf_logger.event(key=c.SEED, value=int(os.environ.get("SEED", "1"))) - # Dense (Adam) + sparse (RowWiseAdagrad) optimizer hyperparameters from gin. - mlperf_logger.event( - key=c.OPT_NAME, - value=_gin_param( - "dense_optimizer_factory_and_class.optimizer_name", "Adam" - ), - ) - mlperf_logger.event( - key=c.OPT_BASE_LR, - value=_gin_param( - "dense_optimizer_factory_and_class.learning_rate", None - ), - ) - mlperf_logger.event( - key="opt_sparse_name", - value=_gin_param( - "sparse_optimizer_factory_and_class.optimizer_name", - "RowWiseAdagrad", - ), - ) - mlperf_logger.event( - key="opt_sparse_base_learning_rate", - value=_gin_param( - "sparse_optimizer_factory_and_class.learning_rate", None - ), - ) - mlperf_logger.end(key=c.INIT_STOP, sync=True) - mlperf_logger.start(key=c.RUN_START, sync=True) - # train loop try: if mode == "train": diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py index bf5fd5512..526a6f89d 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -1551,6 +1551,11 @@ def streaming_train_eval_loop( num_eval_batches: Optional[int] = None, output_trace: bool = False, metric_log_frequency: int = 1, + # MLPerf `train_loss` event cadence, in global train steps, INDEPENDENT of + # metric_log_frequency (the console/TB cadence). 0 = fall back to + # metric_log_frequency (preserves prior coupled behavior). Wired to + # $MLPERF_TRAIN_LOSS_LOG_FREQ via gin. + mlperf_train_loss_log_frequency: int = 0, checkpoint_frequency: int = 100, start_ts: int = 0, persistent_loader: bool = False, @@ -1590,10 +1595,6 @@ def streaming_train_eval_loop( die_at_step: int = -1, # MLPerf logger (rank-0-gated); None disables all MLPerf event emission. mlperf_logger: Optional[Any] = None, - # Which eval metric drives EVAL_ACCURACY + the convergence decision. Format - # "{window|lifetime}_{auc|gauc|accuracy|ne}" (bare "window"/"lifetime" => auc). - # Override via $EVAL_ACCURACY_AUC_MODE. - eval_accuracy_auc_mode: str = "window_auc", ) -> None: """Streaming train+eval loop with per-window (and optionally mid-window) checkpoints. @@ -1646,6 +1647,14 @@ def streaming_train_eval_loop( "the data-fraction cadence set EVAL_EVERY_N_WINDOWS=0; to use the " "per-window cadence set EVAL_EVERY_DATA_PCT=0." ) + # MLPerf train_loss cadence: independent of metric_log_frequency. 0 (the + # env-binding default) falls back to metric_log_frequency so unset behavior + # matches the prior coupled implementation. + mlperf_loss_every = ( + mlperf_train_loss_log_frequency + if mlperf_train_loss_log_frequency and mlperf_train_loss_log_frequency > 0 + else metric_log_frequency + ) profiler = Profiler(rank) if output_trace else None # Normalize the per-window caps: <=0 (the env-binding default) means "no cap # = consume the full window". The eval-break check below is `is not None and @@ -1959,6 +1968,11 @@ def _run_train_window( len(sample.candidates_features_kjt.keys()), -1 )[0], ) + # MLPerf train_loss event on its own cadence (decoupled from the + # console/TB metric cadence below). Called every step; the cross-rank + # all-reduce only fires on the cadence, gated by the rank-identical + # global_step inside the method, so it stays in lockstep. + metric_logger.maybe_log_mlperf_train_loss(aux_losses, every=mlperf_loss_every) if train_batch_idx % metric_log_frequency == 0: metric_logger.compute_and_log( mode="train", @@ -2028,6 +2042,12 @@ def _run_train_window( do_eval(train_ts, gstep) model.train() dataset.dataset.is_eval = False # pyre-ignore [16] + # Data-fraction eval may hit the MLPerf target and emit RUN_STOP + # (via _do_eval_*). Stop the window immediately so we don't train + # past the convergence point; the outer window loop checks the + # same flag and breaks too. + if mlt.run_stopped: + break # Test-only: deterministic crash for the failure-injection test. # Triggered AFTER the save above, so on resume we re-enter at # batch_idx_in_window=train_batch_idx and emit batches [K+1, end). @@ -2238,15 +2258,14 @@ def _should_eval(i: int) -> bool: dataset.dataset._train_split_percentage, # pyre-ignore[16] ) - # --- MLPerf progress accounting + event helpers --------------------------- + # --- MLPerf run tracking -------------------------------------------------- # total_train_samples = epoch_num denominator (global trainable samples over # the window range), computed once and logged as TRAIN_SAMPLES. total_train_samples = 0 - mlperf_run_stopped = [False] if mlperf_logger is not None: - _twi = getattr(dataset.dataset, "train_window_indices", None) - _wi = getattr(dataset.dataset, "window_indices", None) - _idx_fn = _twi or _wi + _idx_fn = getattr( + dataset.dataset, "train_window_indices", None + ) or getattr(dataset.dataset, "window_indices", None) if _idx_fn is not None: for _ts in train_ts_list: total_train_samples += int(_idx_fn(_ts).size) @@ -2256,15 +2275,7 @@ def _should_eval(i: int) -> bool: total_train_samples, n_train, ) - mlperf_logger.event( - key=mlperf_logger.constants.TRAIN_SAMPLES, value=total_train_samples - ) - if eval_global_indices is not None: - mlperf_logger.event( - key=mlperf_logger.constants.EVAL_SAMPLES, - value=int(eval_global_indices.size), - ) - # Wire the logger + LR getter so compute_and_log emits the train_loss event. + # Wire the logger + LR getter so MetricsLogger.compute emits train_loss. metric_logger.mlperf_logger = mlperf_logger def _current_lr() -> float: @@ -2272,110 +2283,26 @@ def _current_lr() -> float: metric_logger.lr_getter = _current_lr - def _mlperf_progress() -> Dict[str, Any]: - samples = metric_logger.cumulative_train_samples - epoch_num = (samples / total_train_samples) if total_train_samples > 0 else 0.0 - return { - mlperf_logger.constants.SAMPLES_COUNT: samples, - mlperf_logger.constants.EPOCH_NUM: epoch_num, - } - - # Convergence/EVAL_ACCURACY metric short name selected by - # eval_accuracy_auc_mode. Bare "window"/"lifetime" defaults the metric to auc. - _eval_metric_short = str(eval_accuracy_auc_mode).strip().lower() - if _eval_metric_short in ("window", "lifetime"): - _eval_metric_short = f"{_eval_metric_short}_auc" - # NE is lower-is-better; auc/gauc/accuracy are higher-is-better. - _eval_lower_is_better = _eval_metric_short.endswith("_ne") - if rank == 0 and mlperf_logger is not None: - logger.info( - f"[mlperf] EVAL_ACCURACY / convergence metric = {_eval_metric_short} " - f"(stop when {'<=' if _eval_lower_is_better else '>='} threshold)" - ) - - def _eval_target_metric(metrics: Dict[str, float]) -> Optional[float]: - # listen_plus eval metric selected by eval_accuracy_auc_mode. Key format - # is `metric/{prefix}_{name}/{task}` (see MetricsLogger.compute). - for key, val in metrics.items(): - short = key.split("/")[-2] if "/" in key else key - if short == _eval_metric_short: - return float(val) - return None - - def _meets_target(value: Optional[float], thr: Optional[float]) -> bool: - if value is None or thr is None: - return False - return value <= thr if _eval_lower_is_better else value >= thr - - def _mlperf_block_start() -> None: - if mlperf_logger is not None: - mlperf_logger.start( - key=mlperf_logger.constants.BLOCK_START, metadata=_mlperf_progress() - ) - - def _mlperf_block_stop() -> None: - if mlperf_logger is not None: - mlperf_logger.end( - key=mlperf_logger.constants.BLOCK_STOP, metadata=_mlperf_progress() - ) - - def _mlperf_eval_start() -> None: - if mlperf_logger is not None: - mlperf_logger.start( - key=mlperf_logger.constants.EVAL_START, metadata=_mlperf_progress() - ) - - def _mlperf_run_stop(status: object) -> None: - # Emit RUN_STOP exactly once, with an all-rank barrier so the timestamp - # reflects the slowest rank (MLPerf requirement). - if mlperf_logger is None or mlperf_run_stopped[0]: - return - mlperf_logger.end( - key=mlperf_logger.constants.RUN_STOP, - metadata={mlperf_logger.constants.STATUS: status, **_mlperf_progress()}, - sync=True, - ) - mlperf_run_stopped[0] = True + # Centralized MLPerf run-boundary state machine: owns block/eval/run markers, + # SAMPLES_COUNT/EPOCH_NUM progress metadata, and the per-window-AUC vs + # auc_threshold convergence decision. Every method no-ops when mlperf_logger + # is None, so the loop below calls them unconditionally. + from generative_recommenders.dlrm_v3.train.mlperf_logging_utils import ( + MLPerfRunTracker, + ) - def _mlperf_eval_stop(eval_metrics: Dict[str, float]) -> bool: - # Emit EVAL_ACCURACY + EVAL_STOP, early SUCCESS RUN_STOP on target. - # Rank 0 decides + broadcasts the stop bool so all ranks break in lockstep - # (a per-rank test could diverge and hang the next all-to-all). - if mlperf_logger is None: - return False - eval_value = _eval_target_metric(eval_metrics) - if eval_value is not None: - mlperf_logger.event( - key=mlperf_logger.constants.EVAL_ACCURACY, - value=eval_value, - metadata=_mlperf_progress(), - ) - mlperf_logger.end( - key=mlperf_logger.constants.EVAL_STOP, metadata=_mlperf_progress() - ) - thr = metric_logger.auc_threshold - decision = torch.zeros(1, device=device) - if rank == 0 and not mlperf_run_stopped[0] and _meets_target(eval_value, thr): - decision[0] = 1.0 - if torch.distributed.is_initialized(): - torch.distributed.broadcast(decision, src=0) - should_stop = bool(decision.item() > 0.5) - if should_stop: - # All ranks agree -> all reach the RUN_STOP barrier together. - _mlperf_run_stop(mlperf_logger.constants.SUCCESS) - return should_stop - - def _mlperf_finalize(final_metrics: Dict[str, float]) -> None: - # End-of-run RUN_STOP when the target was never crossed: SUCCESS iff the - # final eval metric meets the target, else ABORTED. - if mlperf_logger is None or mlperf_run_stopped[0]: - return - success = _meets_target(_eval_target_metric(final_metrics), metric_logger.auc_threshold) - _mlperf_run_stop( - mlperf_logger.constants.SUCCESS - if success - else mlperf_logger.constants.ABORTED - ) + mlt = MLPerfRunTracker( + logger=mlperf_logger, + metric_logger=metric_logger, + total_train_samples=total_train_samples, + rank=rank, + device=device, + ) + mlt.log_dataset_sizes( + eval_samples=eval_global_indices.size + if eval_global_indices is not None + else None + ) if persistent_loader and double_buffer: # Double-buffered: next window prepared in the background during the @@ -2427,15 +2354,37 @@ def _mlperf_finalize(final_metrics: Dict[str, float]) -> None: # reset, not a fork — the only fork was the up-front iter() above), so it # stays safe alongside the background window-prefetch thread. def _do_eval_db(train_ts: int, gstep: int) -> None: + # Data-fraction eval boundary: this closes the current MLPerf block, + # runs the holdout eval with full EVAL_START/EVAL_STOP + EVAL_ACCURACY + # + convergence, then opens the next block. The block thus brackets + # exactly one eval_interval_steps of training (MLPerf block == work + # between two evals), instead of one timestamp window. dataset.dataset.is_eval = True # pyre-ignore [16] assert eval_dl is not None - _run_eval_window( + mlt.block_stop() + mlt.eval_start() + eval_metrics = _run_eval_window( iter(eval_dl), label=f"eval_holdout@train_ts={train_ts}@step={gstep}", ) + # Emits RUN_STOP (sets mlt.run_stopped) if the target is met; + # _run_train_window / the window loop break on that flag. + mlt.eval_stop(eval_metrics) + if not mlt.run_stopped: + mlt.block_start() _db_do_eval = _do_eval_db if eval_interval_steps > 0 else None + # Block placement depends on the eval cadence. Per-window cadence + # (eval_every_n_windows>0): one block per timestamp window. Otherwise + # (data-fraction cadence, or no eval): a single block spans the whole + # run, split at each data-fraction eval boundary by _do_eval_db. Open + # the first block here for the latter; the boundary helper + the + # post-loop stop handle the rest. + _per_window_blocks = eval_every_n_windows > 0 + if not _per_window_blocks: + mlt.block_start() + for i, (train_ts, train_data_iterator) in enumerate( # Only the FIRST window after a mid-window resume needs the skip # (handed via prefetcher.stream's first_skip_samples). The skip is @@ -2452,7 +2401,8 @@ def _do_eval_db(train_ts: int, gstep: int) -> None: if i == 0 and resume_batch_idx_in_window > 0 else 0 ) - _mlperf_block_start() + if _per_window_blocks: + mlt.block_start() _run_train_window( train_data_iterator, train_ts=train_ts, @@ -2460,16 +2410,17 @@ def _do_eval_db(train_ts: int, gstep: int) -> None: label=f"train_ts={train_ts}", do_eval=_db_do_eval, ) - _mlperf_block_stop() + if _per_window_blocks: + mlt.block_stop() should_stop = False if _should_eval(i): dataset.dataset.is_eval = True # pyre-ignore [16] assert eval_sampler is not None and eval_dl is not None - _mlperf_eval_start() + mlt.eval_start() eval_metrics = _run_eval_window( eval_iter, label=f"eval_holdout@train_ts={train_ts}" ) - should_stop = _mlperf_eval_stop(eval_metrics) + should_stop = mlt.eval_stop(eval_metrics) # Re-arm the (already-forked) eval pool for the NEXT eval. The # holdout set is fixed, so the sampler window is unchanged; we # only need a fresh iter() to replay it. iter() reuses the @@ -2480,17 +2431,31 @@ def _do_eval_db(train_ts: int, gstep: int) -> None: if next_eval_i is not None: eval_iter = iter(eval_dl) _maybe_checkpoint(train_ts) - if should_stop: + # should_stop: per-window convergence. mlt.run_stopped: + # data-fraction convergence (RUN_STOP fired mid-window by _do_eval_db). + if should_stop or mlt.run_stopped: # MLPerf target reached: RUN_STOP already emitted; stop training. break + + # Close the run-spanning block for the data-fraction / no-eval case. + # Idempotent: a no-op if the last eval boundary already closed it (i.e. + # convergence stopped the run) or if per-window blocks were used. + if not _per_window_blocks: + mlt.block_stop() else: # Data-fraction eval callback (non-double-buffer path). Builds a fresh # eval dataloader per call over the FIXED holdout set (or the legacy # next-window eval when the dataset has no holdout support). def _do_eval_nb(train_ts: int, gstep: int) -> None: + # Data-fraction eval boundary (non-double-buffer path). See _do_eval_db: + # close the current MLPerf block, run the eval with full markers + + # convergence, then open the next block so a block brackets one + # eval_interval_steps of training rather than a timestamp window. dataset.dataset.is_eval = True # pyre-ignore [16] + mlt.block_stop() + mlt.eval_start() if eval_global_indices is not None: - _run_eval_window( + eval_metrics = _run_eval_window( iter( make_streaming_dataloader( dataset=dataset, indices=eval_global_indices @@ -2499,13 +2464,23 @@ def _do_eval_nb(train_ts: int, gstep: int) -> None: label=f"eval_holdout@train_ts={train_ts}@step={gstep}", ) else: - _run_eval_window( + eval_metrics = _run_eval_window( iter(make_streaming_dataloader(dataset=dataset, ts=train_ts + 1)), label=f"eval@train_ts={train_ts}@step={gstep}", ) + mlt.eval_stop(eval_metrics) + if not mlt.run_stopped: + mlt.block_start() _nb_do_eval = _do_eval_nb if eval_interval_steps > 0 else None + # See the double-buffer branch: per-window blocks for the per-window + # cadence, else a single run-spanning block split at data-fraction eval + # boundaries by _do_eval_nb. + _per_window_blocks = eval_every_n_windows > 0 + if not _per_window_blocks: + mlt.block_start() + for i, train_ts in enumerate(train_ts_list): dataset.dataset.is_eval = False # pyre-ignore [16] skip = first_skip_samples if i == 0 else 0 @@ -2514,18 +2489,20 @@ def _do_eval_nb(train_ts: int, gstep: int) -> None: if i == 0 and resume_batch_idx_in_window > 0 else 0 ) - _mlperf_block_start() + if _per_window_blocks: + mlt.block_start() _run_train_window( _window_iter(train_ts, skip_samples=skip), train_ts=train_ts, start_batch_idx=start_batch, do_eval=_nb_do_eval, ) - _mlperf_block_stop() + if _per_window_blocks: + mlt.block_stop() should_stop = False if _should_eval(i): dataset.dataset.is_eval = True # pyre-ignore [16] - _mlperf_eval_start() + mlt.eval_start() if eval_global_indices is not None: eval_metrics = _run_eval_window( iter( @@ -2540,17 +2517,24 @@ def _do_eval_nb(train_ts: int, gstep: int) -> None: eval_metrics = _run_eval_window( iter(make_streaming_dataloader(dataset=dataset, ts=train_ts + 1)) ) - should_stop = _mlperf_eval_stop(eval_metrics) + should_stop = mlt.eval_stop(eval_metrics) _maybe_checkpoint(train_ts) - if should_stop: + # should_stop: per-window convergence. mlt.run_stopped: + # data-fraction convergence (RUN_STOP fired mid-window by _do_eval_nb). + if should_stop or mlt.run_stopped: # MLPerf target reached: RUN_STOP already emitted; stop training. break + # Close the run-spanning block for the data-fraction / no-eval case + # (idempotent; no-op under per-window blocks or after a convergence stop). + if not _per_window_blocks: + mlt.block_stop() + # Final eval over the fixed user-holdout set (legacy final-window eval # otherwise). Skipped if the MLPerf target already stopped the run mid-run. - if not mlperf_run_stopped[0]: + if not mlt.run_stopped: dataset.dataset.is_eval = True # pyre-ignore [16] - _mlperf_eval_start() + mlt.eval_start() if eval_global_indices is not None: final_metrics = _run_eval_window( iter( @@ -2565,9 +2549,9 @@ def _do_eval_nb(train_ts: int, gstep: int) -> None: iter(make_streaming_dataloader(dataset=dataset, ts=num_train_ts)), label="eval@final", ) - _mlperf_eval_stop(final_metrics) + mlt.eval_stop(final_metrics) if rank == 0: for k, v in final_metrics.items(): print(f"{k}: {v}") # End-of-run RUN_STOP: SUCCESS if final metric met target, else ABORTED. - _mlperf_finalize(final_metrics) + mlt.finalize(final_metrics) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py index 879c93b97..281a37b5d 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py @@ -1213,6 +1213,52 @@ def _emit( ) return all_computed_metrics + def _global_mean_loss(self, loss_terms: Dict[str, torch.Tensor]) -> float: + """Cross-rank mean of the summed per-task losses. + + The 1-element all-reduce MUST run on every rank in lockstep; callers gate + it on a rank-identical counter (global_step / a deterministic frequency) + so it cannot desync. + """ + loss_t = torch.stack( + [v.detach().float().sum() for v in loss_terms.values()] + ).sum() + if torch.distributed.is_available() and torch.distributed.is_initialized(): + torch.distributed.all_reduce(loss_t, op=torch.distributed.ReduceOp.SUM) + loss_t = loss_t / self._world_size + return float(loss_t) + + def maybe_log_mlperf_train_loss( + self, aux_losses: Dict[str, torch.Tensor], every: int + ) -> None: + """Emit the MLPerf ``train_loss`` event on its OWN cadence. + + Decoupled from the console/TB metric cadence (``compute_and_log``): call + this every train step with the just-computed ``aux_losses`` and the + desired interval ``every`` (in global train steps). The cross-rank loss + all-reduce only fires on the cadence, gated by ``global_step["train"]`` + which is incremented identically on all ranks in ``update()`` — so the + collective stays in lockstep. No-op when MLPerf logging is disabled + (``mlperf_logger is None`` on every rank) or ``every <= 0``. + """ + if self.mlperf_logger is None or every <= 0 or not aux_losses: + return + if self.global_step["train"] % every != 0: + return + train_loss = self._global_mean_loss(aux_losses) + c = self.mlperf_logger.constants + md: Dict[str, Any] = {c.SAMPLES_COUNT: self.cumulative_train_samples} + if self.lr_getter is not None: + try: + md["lr"] = float(self.lr_getter()) + except Exception: + pass + self.mlperf_logger.event( + key=getattr(c, "TRAIN_LOSS", "train_loss"), + value=train_loss, + metadata=md, + ) + def compute_and_log( self, mode: str = "train", @@ -1249,21 +1295,15 @@ def compute_and_log( global_step=self.global_step[mode], ) - # Global (cross-rank mean) train loss to console/TB + the MLPerf - # train_loss event, at the metric-logging cadence. The 1-element - # all-reduce runs on every rank in lockstep, so it cannot desync. + # Global (cross-rank mean) train loss to console/TB at the metric-logging + # cadence. The 1-element all-reduce runs on every rank in lockstep, so it + # cannot desync. The MLPerf `train_loss` EVENT is emitted separately via + # ``maybe_log_mlperf_train_loss`` so its cadence can be tuned independently + # of this console/TB cadence (see that method). if mode == "train" and additional_logs is not None and "losses" in additional_logs: loss_terms = additional_logs["losses"] if loss_terms: - loss_t = torch.stack( - [v.detach().float().sum() for v in loss_terms.values()] - ).sum() - if torch.distributed.is_available() and torch.distributed.is_initialized(): - torch.distributed.all_reduce( - loss_t, op=torch.distributed.ReduceOp.SUM - ) - loss_t = loss_t / self._world_size - train_loss = float(loss_t) + train_loss = self._global_mean_loss(loss_terms) self.tb_logger.add_scalar( "train_loss", train_loss, global_step=self.global_step["train"] ) @@ -1272,21 +1312,6 @@ def compute_and_log( f"train - Step {self.global_step['train']} " f"train_loss={train_loss:.5f}" ) - if self.mlperf_logger is not None: - c = self.mlperf_logger.constants - md: Dict[str, Any] = { - c.SAMPLES_COUNT: self.cumulative_train_samples, - } - if self.lr_getter is not None: - try: - md["lr"] = float(self.lr_getter()) - except Exception: - pass - self.mlperf_logger.event( - key=getattr(c, "TRAIN_LOSS", "train_loss"), - value=train_loss, - metadata=md, - ) # Throughput metrics (train only). One GPU->CPU sync per call. if mode == "train" and self._perf_steps_in_window > 0: diff --git a/recommendation_v4/scripts/launch_slurm.sh b/recommendation_v4/scripts/launch_slurm.sh index 5ab1e950a..9d8e90921 100755 --- a/recommendation_v4/scripts/launch_slurm.sh +++ b/recommendation_v4/scripts/launch_slurm.sh @@ -303,6 +303,8 @@ orchestrate() { -e NUM_TRAIN_BATCHES=$NUM_TRAIN_BATCHES \ -e NUM_EVAL_BATCHES=$NUM_EVAL_BATCHES \ -e METRIC_LOG_FREQ=$METRIC_LOG_FREQ \ + ${MLPERF_LOGGING:+-e MLPERF_LOGGING=$MLPERF_LOGGING} \ + ${MLPERF_TRAIN_LOSS_LOG_FREQ:+-e MLPERF_TRAIN_LOSS_LOG_FREQ=$MLPERF_TRAIN_LOSS_LOG_FREQ} \ ${STREAMING_SHUFFLE_FRACTION:+-e STREAMING_SHUFFLE_FRACTION=$STREAMING_SHUFFLE_FRACTION} \ ${STREAMING_SHUFFLE_SEED:+-e STREAMING_SHUFFLE_SEED=$STREAMING_SHUFFLE_SEED} \ ${NUM_WORKERS:+-e NUM_WORKERS=$NUM_WORKERS} \ @@ -326,7 +328,11 @@ orchestrate() { ${CKPT_STEP_FREQ:+-e CKPT_STEP_FREQ=$CKPT_STEP_FREQ} \ -e TRAIN_SPLIT_PERCENTAGE=${TRAIN_SPLIT_PERCENTAGE:-1.0} \ -e AUC_THRESHOLD=${AUC_THRESHOLD:-0.80275} \ - ${EVAL_ACCURACY_AUC_MODE:+-e EVAL_ACCURACY_AUC_MODE=$EVAL_ACCURACY_AUC_MODE} \ + ${MLPERF_SUBMISSION_PLATFORM:+-e MLPERF_SUBMISSION_PLATFORM=$MLPERF_SUBMISSION_PLATFORM} \ + ${TRAIN_LIFETIME_AUC_MODE:+-e TRAIN_LIFETIME_AUC_MODE=$TRAIN_LIFETIME_AUC_MODE} \ + ${EVAL_LIFETIME_AUC_MODE:+-e EVAL_LIFETIME_AUC_MODE=$EVAL_LIFETIME_AUC_MODE} \ + ${CUMULATIVE_AUC_BINS:+-e CUMULATIVE_AUC_BINS=$CUMULATIVE_AUC_BINS} \ + ${LIFETIME_AUC_WINDOW:+-e LIFETIME_AUC_WINDOW=$LIFETIME_AUC_WINDOW} \ -e SPLIT_SALT=${SPLIT_SALT:-0} \ -e EVAL_HOLDOUT_TS=${EVAL_HOLDOUT_TS:--1} \ -e EVAL_HOLDOUT_NUM_WINDOWS=${EVAL_HOLDOUT_NUM_WINDOWS:-1} \ From ae99293c463e376477d4ac4604e63646df449dd9 Mon Sep 17 00:00:00 2001 From: suachong Date: Wed, 24 Jun 2026 22:34:29 +0000 Subject: [PATCH 092/127] dlrmv4: hardcode lifetime-AUC backend to binned, drop the override The lifetime cumulative AUC always uses the exact binned backend now. Remove the TRAIN_LIFETIME_AUC_MODE / EVAL_LIFETIME_AUC_MODE env overrides (and the capped-only LIFETIME_AUC_WINDOW knob, now dead) from gin and launch_slurm. Co-authored-by: Cursor --- .../dlrm_v3/train/gin/yambda_5b.gin | 25 ++++++------------- .../dlrm_v3/train/train_ranker.py | 8 +++--- .../generative_recommenders/dlrm_v3/utils.py | 8 +++--- recommendation_v4/scripts/launch_slurm.sh | 3 --- 4 files changed, 15 insertions(+), 29 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index 8a3e5b595..3a938da36 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -463,29 +463,18 @@ at/env_float.key = "AUC_THRESHOLD" at/env_float.default = 0.80275 # EVAL_ACCURACY + early-stop are driven by per-window AUC (window_auc) vs the # threshold above. -# Lifetime-AUC backend, selectable independently for the train cumulative AUC and -# the eval cumulative ("lifetime_*") AUC. Both default to "binned": -# "binned" = BinnedCumulativeAUC: exact-cumulative AUC from an O(bins) score -# histogram (additive all-reduce, memory independent of #samples/#windows). -# "capped" = LifetimeAUCMetricComputation: AUC over a trailing buffer of -# `lifetime_auc_window` samples/rank (legacy; per-rank buffer all-gathered). -# Override per-run via $TRAIN_LIFETIME_AUC_MODE / $EVAL_LIFETIME_AUC_MODE. -MetricsLogger.train_lifetime_auc_mode = @tlam/env_str() -tlam/env_str.key = "TRAIN_LIFETIME_AUC_MODE" -tlam/env_str.default = "binned" -MetricsLogger.eval_lifetime_auc_mode = @elam/env_str() -elam/env_str.key = "EVAL_LIFETIME_AUC_MODE" -elam/env_str.default = "binned" +# Lifetime-AUC backend for the train + eval cumulative ("lifetime_*") AUC. +# Hardcoded to "binned" = BinnedCumulativeAUC: exact-cumulative AUC from an +# O(bins) score histogram (additive all-reduce, memory independent of +# #samples/#windows). The legacy "capped" trailing-buffer backend is no longer +# selectable. +MetricsLogger.train_lifetime_auc_mode = "binned" +MetricsLogger.eval_lifetime_auc_mode = "binned" # Score-histogram resolution for the "binned" backend. Higher = finer AUC # resolution at O(bins) memory. Override via $CUMULATIVE_AUC_BINS. MetricsLogger.cumulative_auc_bins = @cab/env_int() cab/env_int.key = "CUMULATIVE_AUC_BINS" cab/env_int.default = 100000 -# Trailing-buffer size (samples/rank) for the "capped" backend. Override via -# $LIFETIME_AUC_WINDOW. Ignored when the backend is "binned". -MetricsLogger.lifetime_auc_window = @law/env_int() -law/env_int.key = "LIFETIME_AUC_WINDOW" -law/env_int.default = 10000000 # Checkpointing disabled by default — a full DMP checkpoint is ~100s of GB and # the streaming loop always saves on the final window. save_dmp_checkpoint # no-ops on the empty path. Set $CKPT_PATH to a directory to re-enable; the diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py index 50d359ef6..81428a603 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py @@ -181,10 +181,10 @@ def _main_func( gpu_peak_flops=gpu_peak_flops, model=model, eval_cumulative=(mode == "streaming-train-eval"), - # Lifetime-AUC backend + bins/window come from gin (see yambda_5b.gin: - # MetricsLogger.{train,eval}_lifetime_auc_mode / cumulative_auc_bins / - # lifetime_auc_window), env-overridable. eval_cumulative stays explicit - # because it is runtime-mode dependent, not a config knob. + # Lifetime-AUC backend ("binned") + bins come from gin (see yambda_5b.gin: + # MetricsLogger.{train,eval}_lifetime_auc_mode / cumulative_auc_bins). + # eval_cumulative stays explicit because it is runtime-mode dependent, + # not a config knob. ) # Capture streaming resume hint (None for cold start / non-streaming # checkpoints). For the streaming-train-eval mode, we forward this into diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py index 281a37b5d..7c11b4b5a 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py @@ -1484,11 +1484,11 @@ def env_str(key: str = "", default: str = "") -> str: """Resolve a string from os.environ[key], falling back to `default`. Companion to `env_int`/`env_float` for categorical/string overrides (e.g. a - metric backend selector). Example gin usage: + strategy selector). Example gin usage: - MetricsLogger.train_lifetime_auc_mode = @tlam/env_str() - tlam/env_str.key = "TRAIN_LIFETIME_AUC_MODE" - tlam/env_str.default = "binned" + get_dataset.history_strategy = @hs/env_str() + hs/env_str.key = "HISTORY_STRATEGY" + hs/env_str.default = "interleaved" """ raw = os.environ.get(key) if key else None return raw if raw else default diff --git a/recommendation_v4/scripts/launch_slurm.sh b/recommendation_v4/scripts/launch_slurm.sh index 9d8e90921..dc03962a9 100755 --- a/recommendation_v4/scripts/launch_slurm.sh +++ b/recommendation_v4/scripts/launch_slurm.sh @@ -329,10 +329,7 @@ orchestrate() { -e TRAIN_SPLIT_PERCENTAGE=${TRAIN_SPLIT_PERCENTAGE:-1.0} \ -e AUC_THRESHOLD=${AUC_THRESHOLD:-0.80275} \ ${MLPERF_SUBMISSION_PLATFORM:+-e MLPERF_SUBMISSION_PLATFORM=$MLPERF_SUBMISSION_PLATFORM} \ - ${TRAIN_LIFETIME_AUC_MODE:+-e TRAIN_LIFETIME_AUC_MODE=$TRAIN_LIFETIME_AUC_MODE} \ - ${EVAL_LIFETIME_AUC_MODE:+-e EVAL_LIFETIME_AUC_MODE=$EVAL_LIFETIME_AUC_MODE} \ ${CUMULATIVE_AUC_BINS:+-e CUMULATIVE_AUC_BINS=$CUMULATIVE_AUC_BINS} \ - ${LIFETIME_AUC_WINDOW:+-e LIFETIME_AUC_WINDOW=$LIFETIME_AUC_WINDOW} \ -e SPLIT_SALT=${SPLIT_SALT:-0} \ -e EVAL_HOLDOUT_TS=${EVAL_HOLDOUT_TS:--1} \ -e EVAL_HOLDOUT_NUM_WINDOWS=${EVAL_HOLDOUT_NUM_WINDOWS:-1} \ From da748838616bdc3f68fcfd8f677720eaf0505eff Mon Sep 17 00:00:00 2001 From: suachong Date: Wed, 24 Jun 2026 22:38:12 +0000 Subject: [PATCH 093/127] Revert "dlrmv4: hardcode lifetime-AUC backend to binned, drop the override" This reverts commit 900d4f1311b93e281587e224ba9c3065a0f68cf1. --- .../dlrm_v3/train/gin/yambda_5b.gin | 25 +++++++++++++------ .../dlrm_v3/train/train_ranker.py | 8 +++--- .../generative_recommenders/dlrm_v3/utils.py | 8 +++--- recommendation_v4/scripts/launch_slurm.sh | 3 +++ 4 files changed, 29 insertions(+), 15 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index 3a938da36..8a3e5b595 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -463,18 +463,29 @@ at/env_float.key = "AUC_THRESHOLD" at/env_float.default = 0.80275 # EVAL_ACCURACY + early-stop are driven by per-window AUC (window_auc) vs the # threshold above. -# Lifetime-AUC backend for the train + eval cumulative ("lifetime_*") AUC. -# Hardcoded to "binned" = BinnedCumulativeAUC: exact-cumulative AUC from an -# O(bins) score histogram (additive all-reduce, memory independent of -# #samples/#windows). The legacy "capped" trailing-buffer backend is no longer -# selectable. -MetricsLogger.train_lifetime_auc_mode = "binned" -MetricsLogger.eval_lifetime_auc_mode = "binned" +# Lifetime-AUC backend, selectable independently for the train cumulative AUC and +# the eval cumulative ("lifetime_*") AUC. Both default to "binned": +# "binned" = BinnedCumulativeAUC: exact-cumulative AUC from an O(bins) score +# histogram (additive all-reduce, memory independent of #samples/#windows). +# "capped" = LifetimeAUCMetricComputation: AUC over a trailing buffer of +# `lifetime_auc_window` samples/rank (legacy; per-rank buffer all-gathered). +# Override per-run via $TRAIN_LIFETIME_AUC_MODE / $EVAL_LIFETIME_AUC_MODE. +MetricsLogger.train_lifetime_auc_mode = @tlam/env_str() +tlam/env_str.key = "TRAIN_LIFETIME_AUC_MODE" +tlam/env_str.default = "binned" +MetricsLogger.eval_lifetime_auc_mode = @elam/env_str() +elam/env_str.key = "EVAL_LIFETIME_AUC_MODE" +elam/env_str.default = "binned" # Score-histogram resolution for the "binned" backend. Higher = finer AUC # resolution at O(bins) memory. Override via $CUMULATIVE_AUC_BINS. MetricsLogger.cumulative_auc_bins = @cab/env_int() cab/env_int.key = "CUMULATIVE_AUC_BINS" cab/env_int.default = 100000 +# Trailing-buffer size (samples/rank) for the "capped" backend. Override via +# $LIFETIME_AUC_WINDOW. Ignored when the backend is "binned". +MetricsLogger.lifetime_auc_window = @law/env_int() +law/env_int.key = "LIFETIME_AUC_WINDOW" +law/env_int.default = 10000000 # Checkpointing disabled by default — a full DMP checkpoint is ~100s of GB and # the streaming loop always saves on the final window. save_dmp_checkpoint # no-ops on the empty path. Set $CKPT_PATH to a directory to re-enable; the diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py index 81428a603..50d359ef6 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py @@ -181,10 +181,10 @@ def _main_func( gpu_peak_flops=gpu_peak_flops, model=model, eval_cumulative=(mode == "streaming-train-eval"), - # Lifetime-AUC backend ("binned") + bins come from gin (see yambda_5b.gin: - # MetricsLogger.{train,eval}_lifetime_auc_mode / cumulative_auc_bins). - # eval_cumulative stays explicit because it is runtime-mode dependent, - # not a config knob. + # Lifetime-AUC backend + bins/window come from gin (see yambda_5b.gin: + # MetricsLogger.{train,eval}_lifetime_auc_mode / cumulative_auc_bins / + # lifetime_auc_window), env-overridable. eval_cumulative stays explicit + # because it is runtime-mode dependent, not a config knob. ) # Capture streaming resume hint (None for cold start / non-streaming # checkpoints). For the streaming-train-eval mode, we forward this into diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py index 7c11b4b5a..281a37b5d 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py @@ -1484,11 +1484,11 @@ def env_str(key: str = "", default: str = "") -> str: """Resolve a string from os.environ[key], falling back to `default`. Companion to `env_int`/`env_float` for categorical/string overrides (e.g. a - strategy selector). Example gin usage: + metric backend selector). Example gin usage: - get_dataset.history_strategy = @hs/env_str() - hs/env_str.key = "HISTORY_STRATEGY" - hs/env_str.default = "interleaved" + MetricsLogger.train_lifetime_auc_mode = @tlam/env_str() + tlam/env_str.key = "TRAIN_LIFETIME_AUC_MODE" + tlam/env_str.default = "binned" """ raw = os.environ.get(key) if key else None return raw if raw else default diff --git a/recommendation_v4/scripts/launch_slurm.sh b/recommendation_v4/scripts/launch_slurm.sh index dc03962a9..9d8e90921 100755 --- a/recommendation_v4/scripts/launch_slurm.sh +++ b/recommendation_v4/scripts/launch_slurm.sh @@ -329,7 +329,10 @@ orchestrate() { -e TRAIN_SPLIT_PERCENTAGE=${TRAIN_SPLIT_PERCENTAGE:-1.0} \ -e AUC_THRESHOLD=${AUC_THRESHOLD:-0.80275} \ ${MLPERF_SUBMISSION_PLATFORM:+-e MLPERF_SUBMISSION_PLATFORM=$MLPERF_SUBMISSION_PLATFORM} \ + ${TRAIN_LIFETIME_AUC_MODE:+-e TRAIN_LIFETIME_AUC_MODE=$TRAIN_LIFETIME_AUC_MODE} \ + ${EVAL_LIFETIME_AUC_MODE:+-e EVAL_LIFETIME_AUC_MODE=$EVAL_LIFETIME_AUC_MODE} \ ${CUMULATIVE_AUC_BINS:+-e CUMULATIVE_AUC_BINS=$CUMULATIVE_AUC_BINS} \ + ${LIFETIME_AUC_WINDOW:+-e LIFETIME_AUC_WINDOW=$LIFETIME_AUC_WINDOW} \ -e SPLIT_SALT=${SPLIT_SALT:-0} \ -e EVAL_HOLDOUT_TS=${EVAL_HOLDOUT_TS:--1} \ -e EVAL_HOLDOUT_NUM_WINDOWS=${EVAL_HOLDOUT_NUM_WINDOWS:-1} \ From 1ada43f01d479315eb13edc61e3be9d14aa87de5 Mon Sep 17 00:00:00 2001 From: suachong Date: Wed, 24 Jun 2026 22:47:37 +0000 Subject: [PATCH 094/127] dlrmv4: slim launch_slurm.sh to MLPerf wiring + path portability Reduce the upstream launch_slurm.sh diff to just what MLPerf logging needs: SCRATCH/REPO_MOUNT/DATA_MOUNT path portability (so outputs/log land off the hardcoded /home/chcai,/apps/chcai) and the MLPerf env wiring (MLPERF_LOG_PATH, AUC_THRESHOLD, MLPERF_LOGGING, MLPERF_SUBMISSION_PLATFORM, MLPERF_TRAIN_LOSS_LOG_FREQ). Reverted the unrelated baseline changes (NCCL GDR/IFNAME defaults, SMOKE/frozen run-shape, chmod/WORKER_TEE, HISTORY_STRATEGY, lifetime-AUC passthroughs) to Chris' base. Preserve the full kitchen-sink launcher as launch_slurm_suachong.sh for personal multi-node use (self-reinvoke paths repointed to itself). Co-authored-by: Cursor --- recommendation_v4/scripts/launch_slurm.sh | 74 +- .../scripts/launch_slurm_suachong.sh | 664 ++++++++++++++++++ 2 files changed, 689 insertions(+), 49 deletions(-) create mode 100755 recommendation_v4/scripts/launch_slurm_suachong.sh diff --git a/recommendation_v4/scripts/launch_slurm.sh b/recommendation_v4/scripts/launch_slurm.sh index 9d8e90921..fb71bdcb7 100755 --- a/recommendation_v4/scripts/launch_slurm.sh +++ b/recommendation_v4/scripts/launch_slurm.sh @@ -4,7 +4,7 @@ #SBATCH --ntasks-per-node=1 #SBATCH --exclusive #SBATCH --partition=meta64 # [CLUSTER-SPECIFIC] partition name -#SBATCH --output=yambda_slurm.%j.out +#SBATCH --output=/apps/chcai/yambda_slurm.%j.out # ============================================================================= # launch_slurm.sh — single entry point for the yambda-5b trainer on N>=1 nodes. # @@ -110,9 +110,9 @@ CONTAINER=${CONTAINER:-yambda_primus} REPO=${REPO:-$REPO_ROOT} # repo path inside the container IMAGE=${IMAGE:-rocm/primus:v26.3} # [CLUSTER-SPECIFIC] ROCm/arch base image BAKED_IMAGE=${BAKED_IMAGE:-yambda_primus_baked:latest} -BAKED_TAR=${BAKED_TAR:-/apps/chcai/yambda_primus_baked.tar} # [CLUSTER-SPECIFIC] shared-NFS path (read-only build asset) +BAKED_TAR=${BAKED_TAR:-/apps/chcai/yambda_primus_baked.tar} # [CLUSTER-SPECIFIC] shared-NFS path USE_BAKED=${USE_BAKED:-1} -OVERLAY=${RDMA_OVERLAY:-/apps/chcai/rdma_host_el9_new} # [CLUSTER-SPECIFIC] shared-NFS RDMA overlay (read-only, already staged) +OVERLAY=${RDMA_OVERLAY:-/apps/chcai/rdma_host_el9_new} # [CLUSTER-SPECIFIC] shared-NFS RDMA overlay REPO_MOUNT=${REPO_MOUNT:-$HOME} # NFS home holding the repo (must contain $REPO); override if your repo lives elsewhere DATA_MOUNT=${DATA_MOUNT:-/apps/chcai} # shared dataset + RDMA overlay + pip/fbgemm assets (read-only) @@ -135,27 +135,15 @@ orchestrate() { mkdir -p "$SCRATCH" 2>/dev/null || true LOG=${LOG:-$SCRATCH/yambda_slurm.${SLURM_JOB_ID:-manual}.log} + # Smoke defaults — override via env for a perf run (see header USAGE). MODE=${MODE:-streaming-train-eval} - if [ "${SMOKE:-0}" = "1" ]; then - START_TS=${START_TS:-150} - NUM_TRAIN_TS=${NUM_TRAIN_TS:-1} - NUM_TRAIN_BATCHES=${NUM_TRAIN_BATCHES:-20} - NUM_EVAL_BATCHES=${NUM_EVAL_BATCHES:-10} - EVAL_EVERY_N_WINDOWS=${EVAL_EVERY_N_WINDOWS:-1} - METRIC_LOG_FREQ=${METRIC_LOG_FREQ:-5} - fi - START_TS=${START_TS:-0} - NUM_TRAIN_TS=${NUM_TRAIN_TS:-299} - NUM_TRAIN_BATCHES=${NUM_TRAIN_BATCHES:-0} - NUM_EVAL_BATCHES=${NUM_EVAL_BATCHES:-0} + START_TS=${START_TS:-150} + NUM_TRAIN_TS=${NUM_TRAIN_TS:-1} + NUM_TRAIN_BATCHES=${NUM_TRAIN_BATCHES:-20} + NUM_EVAL_BATCHES=${NUM_EVAL_BATCHES:-10} EVAL_EACH_WINDOW=${EVAL_EACH_WINDOW:-1} - METRIC_LOG_FREQ=${METRIC_LOG_FREQ:-20} - if [ "${EVAL_EVERY_N_WINDOWS:-0}" -gt 0 ] 2>/dev/null; then - EVAL_EVERY_DATA_PCT=${EVAL_EVERY_DATA_PCT:-0} - else - EVAL_EVERY_N_WINDOWS=0 - EVAL_EVERY_DATA_PCT=${EVAL_EVERY_DATA_PCT:-0.005} - fi + EVAL_EVERY_N_WINDOWS=${EVAL_EVERY_N_WINDOWS:-1} + METRIC_LOG_FREQ=${METRIC_LOG_FREQ:-5} FORCE_PROVISION=${FORCE_PROVISION:-0} # Truncate the metrics log on a FRESH run; APPEND on a supervised relaunch @@ -167,11 +155,9 @@ orchestrate() { else : > "$LOG" fi - chmod 622 "$LOG" 2>/dev/null || true echo "[$(date)] launch_slurm/orchestrate: job=${SLURM_JOB_ID:-?} nodes=${SLURM_JOB_NODELIST:-?} nnodes=${SLURM_NNODES:-1}" | tee -a "$LOG" echo "[$(date)] resolved SCRIPT_PATH=$SCRIPT_PATH REPO=$REPO" | tee -a "$LOG" - echo "[$(date)] config: MODE=$MODE START_TS=$START_TS NUM_TRAIN_TS=$NUM_TRAIN_TS NUM_TRAIN_BATCHES=$NUM_TRAIN_BATCHES NUM_EVAL_BATCHES=$NUM_EVAL_BATCHES METRIC_LOG_FREQ=$METRIC_LOG_FREQ SMOKE=${SMOKE:-0} EVAL_EVERY_N_WINDOWS=$EVAL_EVERY_N_WINDOWS EVAL_EVERY_DATA_PCT=$EVAL_EVERY_DATA_PCT" | tee -a "$LOG" - echo "[$(date)] lr-override: DENSE_LR=${DENSE_LR:-} SPARSE_LR=${SPARSE_LR:-}" | tee -a "$LOG" + echo "[$(date)] config: MODE=$MODE START_TS=$START_TS NUM_TRAIN_TS=$NUM_TRAIN_TS NUM_TRAIN_BATCHES=$NUM_TRAIN_BATCHES NUM_EVAL_BATCHES=$NUM_EVAL_BATCHES METRIC_LOG_FREQ=$METRIC_LOG_FREQ" | tee -a "$LOG" # Rendezvous resolved on the HOST (the container image has no SLURM client). MASTER_ADDR=$(scontrol show hostnames "$SLURM_JOB_NODELIST" 2>/dev/null | head -1) @@ -284,7 +270,6 @@ orchestrate() { _wattempt=\$((_wattempt+1)) docker exec \ -e LAUNCH_SLURM_PHASE=worker \ - -e WORKER_TEE=0 \ -e SCRATCH=$SCRATCH \ -e SLURM_NNODES=\$SLURM_NNODES \ -e SLURM_NODEID=\$SLURM_NODEID \ @@ -313,7 +298,6 @@ orchestrate() { ${DIAG_EMB_STEPS:+-e DIAG_EMB_STEPS=$DIAG_EMB_STEPS} \ ${OUTPUT_TRACE:+-e OUTPUT_TRACE=$OUTPUT_TRACE} \ ${MIN_HISTORY:+-e MIN_HISTORY=$MIN_HISTORY} \ - ${HISTORY_STRATEGY:+-e HISTORY_STRATEGY=$HISTORY_STRATEGY} \ ${SEED:+-e SEED=$SEED} \ ${DENSE_LR:+-e DENSE_LR=$DENSE_LR} \ ${SPARSE_LR:+-e SPARSE_LR=$SPARSE_LR} \ @@ -329,10 +313,6 @@ orchestrate() { -e TRAIN_SPLIT_PERCENTAGE=${TRAIN_SPLIT_PERCENTAGE:-1.0} \ -e AUC_THRESHOLD=${AUC_THRESHOLD:-0.80275} \ ${MLPERF_SUBMISSION_PLATFORM:+-e MLPERF_SUBMISSION_PLATFORM=$MLPERF_SUBMISSION_PLATFORM} \ - ${TRAIN_LIFETIME_AUC_MODE:+-e TRAIN_LIFETIME_AUC_MODE=$TRAIN_LIFETIME_AUC_MODE} \ - ${EVAL_LIFETIME_AUC_MODE:+-e EVAL_LIFETIME_AUC_MODE=$EVAL_LIFETIME_AUC_MODE} \ - ${CUMULATIVE_AUC_BINS:+-e CUMULATIVE_AUC_BINS=$CUMULATIVE_AUC_BINS} \ - ${LIFETIME_AUC_WINDOW:+-e LIFETIME_AUC_WINDOW=$LIFETIME_AUC_WINDOW} \ -e SPLIT_SALT=${SPLIT_SALT:-0} \ -e EVAL_HOLDOUT_TS=${EVAL_HOLDOUT_TS:--1} \ -e EVAL_HOLDOUT_NUM_WINDOWS=${EVAL_HOLDOUT_NUM_WINDOWS:-1} \ @@ -498,10 +478,8 @@ worker() { cd "$REPO_ROOT" mkdir -p "$SCRATCH" 2>/dev/null || true LOG=${LOG:-$SCRATCH/yambda_5b_8gpu.log} - # WORKER_TEE=0 (set by orchestrate) sends our file sink to /dev/null to avoid - # double-logging, since orchestrate already tees stdout into the real $LOG. - [ "${WORKER_TEE:-1}" = "0" ] && LOG=/dev/null - export TENSORBOARD_LOG_PATH=${TENSORBOARD_LOG_PATH:-$SCRATCH/tb/yambda_5b} + # Append (not truncate): under the streaming-e2e supervisor a run may relaunch + # many times into the SAME $LOG; the supervisor initializes it once at run start. # MLPerf compliance log (rank 0 writes it). Per-job filename so each standalone # sbatch gets a clean log; the e2e supervisor pins MLPERF_LOG_PATH itself. export MLPERF_LOG_PATH=${MLPERF_LOG_PATH:-$SCRATCH/mlperf/yambda_5b_mlperf.${SLURM_JOB_ID:-manual}.log} @@ -553,15 +531,14 @@ worker() { export WORLD_SIZE=$(( NNODES * GPUS_PER_NODE )) echo "[$(date)] topology: nnodes=$NNODES node_rank=$NODE_RANK gpus_per_node=$GPUS_PER_NODE world_size=$WORLD_SIZE master=$MASTER_ADDR:${MASTER_PORT:-}" | tee -a "$LOG" - # NCCL bootstrap NIC: loopback single-node, routable host NIC multi-node (pin - # to avoid auto-detect picking a non-routable per-GPU RoCE link). Override via - # $NCCL_SOCKET_IFNAME. [CLUSTER-SPECIFIC] multi-node fenic0 (find via `ip -br addr`). - if [ "$NNODES" -gt 1 ]; then - export NCCL_SOCKET_IFNAME=${NCCL_SOCKET_IFNAME:-fenic0} - else - export NCCL_SOCKET_IFNAME=${NCCL_SOCKET_IFNAME:-lo} - fi - echo "[$(date)] NCCL_SOCKET_IFNAME=$NCCL_SOCKET_IFNAME (nnodes=$NNODES)" | tee -a "$LOG" + # NCCL bootstrap NIC — pin for BOTH single- and multi-node. The container is + # --network=host so RCCL sees ALL host interfaces; if left to auto-detect, NCCL + # can pick a non-routable per-GPU RoCE /31 (benic* 192.168.x) link and fail + # bootstrap with "No route to host" (this is node-dependent: it happened to + # work on some nodes and not others, causing repetitive single-node init + # failures). Pinning the routable host NIC fixes it everywhere. + # [CLUSTER-SPECIFIC] routable host NIC for TCP bootstrap (find via `ip -br addr`). + export NCCL_SOCKET_IFNAME=${NCCL_SOCKET_IFNAME:-fenic0} # Multi-node additionally needs the RDMA data-plane (bnxt_re HCAs) configured; # single-node uses intra-node P2P (XGMI/PCIe) so only the bootstrap NIC matters. @@ -594,11 +571,10 @@ worker() { export NCCL_IB_TIMEOUT=${NCCL_IB_TIMEOUT:-14} export NCCL_IGNORE_CPU_AFFINITY=${NCCL_IGNORE_CPU_AFFINITY:-1} export RCCL_MSCCL_ENABLE=${RCCL_MSCCL_ENABLE:-0} - # GPU-Direct RDMA on by default (~+22% throughput at 2 nodes via peermem). - # Set NCCL_NET_GDR_LEVEL=0 to force the legacy host-staged path. - export NCCL_NET_GDR_LEVEL=${NCCL_NET_GDR_LEVEL:-5} - export NCCL_DMABUF_ENABLE=${NCCL_DMABUF_ENABLE:-1} - echo "[$(date)] NCCL: RDMA over bnxt_re (GID idx ${NCCL_IB_GID_INDEX}, TC ${NCCL_IB_TC}, GDR_LEVEL=${NCCL_NET_GDR_LEVEL}, DMABUF=${NCCL_DMABUF_ENABLE}; meta64 bnxt_re config, validated)" | tee -a "$LOG" + # GPU-Direct RDMA needs DMABUF/peermem (neither in-container here) — leave + # GDR off so RCCL stages through host memory (still real RDMA over bnxt_re). + export NCCL_NET_GDR_LEVEL=${NCCL_NET_GDR_LEVEL:-0} + echo "[$(date)] NCCL: RDMA over bnxt_re (GID idx ${NCCL_IB_GID_INDEX}, TC ${NCCL_IB_TC}, GDR_LEVEL=${NCCL_NET_GDR_LEVEL}; meta64 bnxt_re config, validated)" | tee -a "$LOG" fi fi export NCCL_DEBUG=${NCCL_DEBUG:-WARN} diff --git a/recommendation_v4/scripts/launch_slurm_suachong.sh b/recommendation_v4/scripts/launch_slurm_suachong.sh new file mode 100755 index 000000000..579076d47 --- /dev/null +++ b/recommendation_v4/scripts/launch_slurm_suachong.sh @@ -0,0 +1,664 @@ +#!/bin/bash +#SBATCH --job-name=yambda_slurm +#SBATCH --nodes=1 +#SBATCH --ntasks-per-node=1 +#SBATCH --exclusive +#SBATCH --partition=meta64 # [CLUSTER-SPECIFIC] partition name +#SBATCH --output=yambda_slurm.%j.out +# ============================================================================= +# launch_slurm.sh — single entry point for the yambda-5b trainer on N>=1 nodes. +# +# Consolidates what used to be separate scripts so multi-node enablement is +# ONE committable script (plus the train_ranker.py / utils.py python changes): +# * orchestrate phase (host SLURM glue) — formerly sbatch_smoke_multinode.sh +# * provision phase (container + RDMA) — formerly _provision_yambda_primus.sh +# * worker phase (in-container train) — now inlined below +# +# PHASES (auto-detected from context; force with LAUNCH_SLURM_PHASE=): +# orchestrate Runs on the SLURM batch host (no /.dockerenv). Resolves the +# rendezvous (MASTER_ADDR/PORT), ensures the container on every +# node (provision phase), then `docker exec`s the worker phase on +# every node, one task per node. +# provision Runs on a compute-node host. Ensures the `yambda_primus` +# container is up (loads the pre-baked image if present — no +# internet/pip — else builds from the base image) and stages the +# host RDMA userspace overlay on shared NFS. +# worker Runs INSIDE the container. Sets the distributed topology + +# NCCL/RDMA env and spawns this node's GPU ranks via train_ranker. +# N==1 transparently uses the legacy single-node path (localhost, +# node_rank 0), byte-for-byte as before, so the streaming-e2e +# supervisor's direct `bash scripts/launch_slurm.sh` is unchanged. +# +# USAGE +# Multi-node (N>=1): sbatch --nodes=2 scripts/launch_slurm.sh +# Single-node direct: bash scripts/launch_slurm.sh (already inside container; +# what run_streaming_e2e.sh invokes per relaunch) +# Perf pair: +# LOG=/apps/chcai/perf_1node.log NUM_TRAIN_BATCHES=200 NUM_EVAL_BATCHES=0 \ +# EVAL_EACH_WINDOW=0 METRIC_LOG_FREQ=20 \ +# sbatch --nodes=1 --job-name=y1 scripts/launch_slurm.sh +# LOG=/apps/chcai/perf_2node.log NUM_TRAIN_BATCHES=200 NUM_EVAL_BATCHES=0 \ +# EVAL_EACH_WINDOW=0 METRIC_LOG_FREQ=20 \ +# sbatch --nodes=2 --job-name=y2 scripts/launch_slurm.sh +# # then: bash scripts/compare_node_perf.sh /apps/chcai/perf_1node.log /apps/chcai/perf_2node.log +# +# ONE-TIME IMAGE BAKE (so fresh nodes skip the multi-GB torch download + pip): +# BAKE_IMAGE=1 LAUNCH_SLURM_PHASE=provision bash scripts/launch_slurm.sh +# (commits the deps-installed container to $BAKED_IMAGE and `docker save`s it to +# $BAKED_TAR on NFS; subsequent provisions `docker load` it offline.) +# +# ----------------------------------------------------------------------------- +# PORTABILITY — what to change for a DIFFERENT cluster / network / hardware. +# Every such knob is also tagged inline with "[CLUSTER-SPECIFIC]" (grep for it). +# All are env-overridable, so you can adapt without editing this file. +# +# A) SLURM / scheduler +# - #SBATCH --partition=meta64 : partition name. CHANGE per cluster. +# - #SBATCH --time / --exclusive : policy; adjust to taste. +# +# B) Filesystems (must be shared/NFS across ALL nodes — this script re-invokes +# itself and reads the overlay + data from these paths cluster-wide) +# - REPO_MOUNT (repo + this script, e.g. /home/) is bind-mounted rw; +# DATA_MOUNT (e.g. /apps/chcai) holds the read-only dataset + overlay + +# baked tar + pip tarball; SCRATCH (e.g. /home//yambda_runs) is the +# writable log/output root. Override any via env — nothing is user-hardwired. +# +# C) Container image / GPU software stack (tied to the GPU arch + ROCm version) +# - IMAGE=rocm/primus:v26.3 : base image. ROCm/AMD-specific. +# - docker run --device=/dev/kfd --device=/dev/dri --group-add video : AMD ROCm +# device passthrough. For NVIDIA this is --gpus all / nvidia runtime instead. +# - --ulimit memlock=-1 : REQUIRED for RDMA QP registration (do not drop). +# - TORCH_IDX (rocm7.2), torch/vision/audio ==*+rocm7.2, FBGEMM_WHL (a gfx950 +# wheel), torchrec pin : the whole deps set is arch/ROCm-version-specific. +# +# D) Network fabric — THE trickiest part; defaults are PROVEN on meta64 cv350 +# (Broadcom bnxt_re RoCEv2). On a different fabric these almost certainly change +# (see the worker-phase block for the full rationale): +# - NCCL_SOCKET_IFNAME=fenic0 : the ONE routable host NIC for TCP bootstrap. +# Find yours with `ip -br addr`; the per-GPU RDMA NICs are usually NOT +# routable for plain TCP, so auto-detect hangs init — you MUST pin this. +# - NCCL_IB_HCA=bnxt_re0..7 : the RDMA HCA device names. List with `ibv_devices`. +# Different NIC vendor (e.g. mlx5_*, ionic_*) => different names AND a +# different userspace provider, which changes the RDMA overlay below. +# - NCCL_IB_GID_INDEX=3 : RoCEv2 IPv4 GID index. Check `show_gids`; v1/v2 and +# IPv4/IPv6 live at different indices per port. +# - NCCL_IB_TC=104 : RoCE lossless (PFC) traffic class. Fabric/switch-specific. +# - RDMA overlay (provision phase): only needed when the CONTAINER's rdma-core +# is older than the HOST kernel driver's uapi (our bnxt_re v34-vs-v59 case). +# Different NIC/host => different /usr/lib64 provider .so to stage, or the +# overlay may be unnecessary entirely (set RDMA_OVERLAY= to disable). If RDMA +# can't be made to work, NCCL_NET_TRANSPORT=socket falls back to TCP. +# +# E) Not cluster-specific (auto-derived): GPUS_PER_NODE (torch.cuda.device_count), +# NNODES/NODE_RANK/MASTER_ADDR (from SLURM), WORLD_SIZE. +# ============================================================================= +set -uo pipefail + +# Absolute path to THIS script so the orchestrate phase can re-invoke it on every +# node (home is shared NFS, so the same path resolves cluster-wide). +SELF=$(cd "$(dirname "$0")" && pwd)/$(basename "$0") +REPO_ROOT=$(cd "$(dirname "$0")/.." && pwd) + +# ---- phase detection -------------------------------------------------------- +PHASE="${LAUNCH_SLURM_PHASE:-}" +if [ -z "$PHASE" ]; then + if [ -f /.dockerenv ]; then PHASE=worker; else PHASE=orchestrate; fi +fi + +# ---- shared config (env-overridable) ---------------------------------------- +CONTAINER=${CONTAINER:-yambda_primus} +REPO=${REPO:-$REPO_ROOT} # repo path inside the container +IMAGE=${IMAGE:-rocm/primus:v26.3} # [CLUSTER-SPECIFIC] ROCm/arch base image +BAKED_IMAGE=${BAKED_IMAGE:-yambda_primus_baked:latest} +BAKED_TAR=${BAKED_TAR:-/apps/chcai/yambda_primus_baked.tar} # [CLUSTER-SPECIFIC] shared-NFS path (read-only build asset) +USE_BAKED=${USE_BAKED:-1} +OVERLAY=${RDMA_OVERLAY:-/apps/chcai/rdma_host_el9_new} # [CLUSTER-SPECIFIC] shared-NFS RDMA overlay (read-only, already staged) + +REPO_MOUNT=${REPO_MOUNT:-$HOME} # NFS home holding the repo (must contain $REPO); override if your repo lives elsewhere +DATA_MOUNT=${DATA_MOUNT:-/apps/chcai} # shared dataset + RDMA overlay + pip/fbgemm assets (read-only) +SCRATCH=${SCRATCH:-$HOME/yambda_runs} # writable output root (logs / tb / traces) + +# ============================================================================= +# PHASE: orchestrate (SLURM batch host) +# ============================================================================= +orchestrate() { + # When run as the SLURM batch script, $0 is the node-local staged copy + # (/var/spool/slurmd/job/slurm_script), so $SELF / $REPO_ROOT are WRONG + # here (they don't exist on other nodes). Resolve the REAL shared-NFS script + # path + repo root from SLURM so we can re-invoke this script on every node and + # `cd` to the right repo inside the container. + SCRIPT_PATH=$(scontrol show job "${SLURM_JOB_ID:-0}" 2>/dev/null | grep -oP 'Command=\K\S+') + [ -f "${SCRIPT_PATH:-}" ] || SCRIPT_PATH="${SLURM_SUBMIT_DIR:-$REPO_ROOT}/scripts/launch_slurm_suachong.sh" + [ -f "$SCRIPT_PATH" ] || SCRIPT_PATH="$SELF" + REPO=$(cd "$(dirname "$SCRIPT_PATH")/.." && pwd) + + mkdir -p "$SCRATCH" 2>/dev/null || true + LOG=${LOG:-$SCRATCH/yambda_slurm.${SLURM_JOB_ID:-manual}.log} + + MODE=${MODE:-streaming-train-eval} + if [ "${SMOKE:-0}" = "1" ]; then + START_TS=${START_TS:-150} + NUM_TRAIN_TS=${NUM_TRAIN_TS:-1} + NUM_TRAIN_BATCHES=${NUM_TRAIN_BATCHES:-20} + NUM_EVAL_BATCHES=${NUM_EVAL_BATCHES:-10} + EVAL_EVERY_N_WINDOWS=${EVAL_EVERY_N_WINDOWS:-1} + METRIC_LOG_FREQ=${METRIC_LOG_FREQ:-5} + fi + START_TS=${START_TS:-0} + NUM_TRAIN_TS=${NUM_TRAIN_TS:-299} + NUM_TRAIN_BATCHES=${NUM_TRAIN_BATCHES:-0} + NUM_EVAL_BATCHES=${NUM_EVAL_BATCHES:-0} + EVAL_EACH_WINDOW=${EVAL_EACH_WINDOW:-1} + METRIC_LOG_FREQ=${METRIC_LOG_FREQ:-20} + if [ "${EVAL_EVERY_N_WINDOWS:-0}" -gt 0 ] 2>/dev/null; then + EVAL_EVERY_DATA_PCT=${EVAL_EVERY_DATA_PCT:-0} + else + EVAL_EVERY_N_WINDOWS=0 + EVAL_EVERY_DATA_PCT=${EVAL_EVERY_DATA_PCT:-0.005} + fi + FORCE_PROVISION=${FORCE_PROVISION:-0} + + # Truncate the metrics log on a FRESH run; APPEND on a supervised relaunch + # (APPEND_LOG=1) so the full-run NE/AUC history survives crash/node-failover + # resubmits instead of being wiped on every attempt (mirrors the single-node + # supervisor's init-once/append model). + if [ "${APPEND_LOG:-0}" = "1" ]; then + echo "[$(date)] === resume: appending to existing $LOG (APPEND_LOG=1) ===" >> "$LOG" + else + : > "$LOG" + fi + chmod 622 "$LOG" 2>/dev/null || true + echo "[$(date)] launch_slurm/orchestrate: job=${SLURM_JOB_ID:-?} nodes=${SLURM_JOB_NODELIST:-?} nnodes=${SLURM_NNODES:-1}" | tee -a "$LOG" + echo "[$(date)] resolved SCRIPT_PATH=$SCRIPT_PATH REPO=$REPO" | tee -a "$LOG" + echo "[$(date)] config: MODE=$MODE START_TS=$START_TS NUM_TRAIN_TS=$NUM_TRAIN_TS NUM_TRAIN_BATCHES=$NUM_TRAIN_BATCHES NUM_EVAL_BATCHES=$NUM_EVAL_BATCHES METRIC_LOG_FREQ=$METRIC_LOG_FREQ SMOKE=${SMOKE:-0} EVAL_EVERY_N_WINDOWS=$EVAL_EVERY_N_WINDOWS EVAL_EVERY_DATA_PCT=$EVAL_EVERY_DATA_PCT" | tee -a "$LOG" + echo "[$(date)] lr-override: DENSE_LR=${DENSE_LR:-} SPARSE_LR=${SPARSE_LR:-}" | tee -a "$LOG" + + # Rendezvous resolved on the HOST (the container image has no SLURM client). + MASTER_ADDR=$(scontrol show hostnames "$SLURM_JOB_NODELIST" 2>/dev/null | head -1) + MASTER_ADDR=${MASTER_ADDR:-localhost} + MASTER_PORT=$(( 20000 + ${SLURM_JOB_ID:-0} % 20000 )) + echo "[$(date)] rendezvous: MASTER_ADDR=$MASTER_ADDR MASTER_PORT=$MASTER_PORT" | tee -a "$LOG" + + # Optional NCCL/RCCL fabric overrides — forwarded into the container only when + # set at submit time (docker exec does NOT inherit the srun task env). The + # worker phase applies its own validated multi-node bnxt_re defaults when these + # are unset. Common: NCCL_NET_TRANSPORT=socket (TCP fallback), NCCL_DEBUG=INFO. + NCCL_ENV_ARGS="" + for v in NCCL_NET_TRANSPORT NCCL_DEBUG NCCL_SOCKET_IFNAME NCCL_IB_HCA NCCL_IB_GID_INDEX \ + NCCL_IB_TC NCCL_IB_TIMEOUT NCCL_IGNORE_CPU_AFFINITY RCCL_MSCCL_ENABLE NCCL_NET_GDR_LEVEL \ + NCCL_IB_PCI_RELAXED_ORDERING NCCL_IB_USE_INLINE NCCL_IB_QPS_PER_CONNECTION \ + NCCL_IB_ECE_ENABLE NCCL_DMABUF_ENABLE NCCL_GDRCOPY_ENABLE NCCL_GDR_FLUSH_DISABLE \ + NCCL_PXN_DISABLE NCCL_CHECKS_DISABLE NCCL_CROSS_NIC RDMA_OVERLAY; do + eval "val=\${$v:-}" + if [ -n "$val" ]; then NCCL_ENV_ARGS="$NCCL_ENV_ARGS -e $v=$val"; fi + done + + # TRICKY — variable expansion inside the `srun ... bash -c "..."` blocks below: + # the string is double-quoted, so PLAIN $VAR expands NOW on the batch host (e.g. + # $MASTER_ADDR, $CONTAINER, $SCRIPT_PATH — values computed above), while + # BACKSLASH-escaped \$VAR is passed through literally and expands LATER on each + # compute node inside the srun task (e.g. \$SLURM_NODEID, \$(hostname)) where the + # per-node SLURM_* env actually lives. Mixing these up sends every rank the + # wrong node id or breaks the docker exec — keep the \$ on per-node values. + + # --- step 1: ensure the container is up on every node ---------------------- + echo "[$(date)] ensuring container '$CONTAINER' on all nodes (force=$FORCE_PROVISION)" | tee -a "$LOG" + srun --ntasks-per-node=1 bash -c " + # Reap stale/foreign GPU containers from prior jobs BEFORE (re)provisioning. + # The node is allocated --exclusive, so any GPU container other than + # '$CONTAINER' is an orphan left by a previous job (its container outlives the + # SLURM allocation). We remove every such container that has GPU access + # (/dev/kfd or /dev/dri) — running OR stopped, whether or not it currently + # pins VRAM ('docker ps -aq' includes stopped ones) — since idle orphans can + # still hold device handles or wake up; leaked HBM from these has caused both + # OOMs and RCCL collective hangs. We deliberately SKIP non-GPU containers + # (e.g. 'k8s-node-services-*' and other cluster system services) so we don't + # disrupt node infrastructure. docker teardown lets the driver reclaim HBM. + for _c in \$(docker ps -aq 2>/dev/null); do + _nm=\$(docker inspect -f '{{.Name}}' \"\$_c\" 2>/dev/null | sed 's#^/##') + [ \"\$_nm\" = \"$CONTAINER\" ] && continue + _dev=\$(docker inspect -f '{{range .HostConfig.Devices}}{{.PathOnHost}} {{end}}' \"\$_c\" 2>/dev/null) + case \"\$_dev\" in + *kfd*|*dri*) + echo \"[\$(hostname)] reaping stale GPU container \$_nm (\$_c)\" + docker rm -f \"\$_c\" >/dev/null 2>&1 || true ;; + *) + echo \"[\$(hostname)] keeping non-GPU/system container \$_nm (\$_c)\" ;; + esac + done + # Reuse a STOPPED '$CONTAINER' (its installed deps persist in the container + # fs) instead of destructively re-provisioning from the base image + pip. + # Harmless no-op on a fresh node (no such container) -> falls through to + # provision below. Repo code is bind-mounted, so live edits are still picked up. + docker start $CONTAINER >/dev/null 2>&1 || true + if [ \"$FORCE_PROVISION\" = \"1\" ] || ! docker exec $CONTAINER true >/dev/null 2>&1; then + echo \"[\$(hostname)] (re)provisioning container\" + LAUNCH_SLURM_PHASE=provision CONTAINER=$CONTAINER IMAGE=$IMAGE \ + BAKED_IMAGE=$BAKED_IMAGE BAKED_TAR=$BAKED_TAR USE_BAKED=$USE_BAKED \ + BAKE_IMAGE=${BAKE_IMAGE:-0} RDMA_OVERLAY=$OVERLAY REPO=$REPO \ + REPO_MOUNT=$REPO_MOUNT DATA_MOUNT=$DATA_MOUNT SCRATCH=$SCRATCH bash $SCRIPT_PATH + else + # Container persists across jobs; the reap above only removes FOREIGN GPU + # containers, so our own '$CONTAINER' can still pin HBM via stray trainer + # ranks left by a prior OOM/crash (this caused repeated 'CUDA out of memory' + # on relaunch onto the same node). Restart it to kill every exec'd proc and + # let the driver reclaim HBM — cheap (keeps the installed deps in the + # container fs; NFS RDMA overlay also persists), no full re-provision. + echo \"[\$(hostname)] container already up — restarting to free any leaked HBM before launch\" + docker restart $CONTAINER >/dev/null 2>&1 || true + # Readiness gate: a bare 'docker exec true' can pass while the runtime is + # still settling, so the SUBSEQUENT (heavier) worker exec races the restart + # and dies with 'container is not running' / OCI 'setns' errors (observed on + # c07-08 and e08-08 -> the peer never joins rendezvous -> master 600s + # TCPStore timeout). Require State.Running=true AND a successful probe, then + # a short settle, before considering the container ready. + for _w in \$(seq 1 30); do + [ \"\$(docker inspect -f '{{.State.Running}}' $CONTAINER 2>/dev/null)\" = \"true\" ] \ + && docker exec $CONTAINER true >/dev/null 2>&1 && break + sleep 2 + done + sleep 2 + echo \"[\$(hostname)] container restarted (HBM reclaimed; running=\$(docker inspect -f '{{.State.Running}}' $CONTAINER 2>/dev/null))\" + fi + " 2>&1 | tee -a "$LOG" + + # --- step 2: launch the worker (trainer) inside the container on every node - + echo "[$(date)] launching trainer (worker phase) on all nodes" | tee -a "$LOG" + srun --ntasks-per-node=1 bash -c " + # Pre-flight readiness gate (per node): step 1 ran in a SEPARATE srun, so the + # container can still be settling here. Wait for State.Running=true + a probe + # before the worker exec so we don't race a just-restarted container. + for _w in \$(seq 1 30); do + [ \"\$(docker inspect -f '{{.State.Running}}' $CONTAINER 2>/dev/null)\" = \"true\" ] \ + && docker exec $CONTAINER true >/dev/null 2>&1 && break + [ \$_w -eq 1 ] && echo \"[\$(hostname)] worker pre-flight: waiting for container to be ready...\" + sleep 2 + done + # Retry wrapper: docker exec startup failures (rc 125 daemon 'container is not + # running', 126/127 OCI/setns 'exec failed') mean the container wasn't ready, + # NOT that the trainer ran and failed. Restart + re-gate + retry a few times. + # Any OTHER rc (the trainer actually started and exited) is propagated so the + # supervisor's resume-from-checkpoint logic owns real failures. + _wattempt=0 + while : ; do + _wattempt=\$((_wattempt+1)) + docker exec \ + -e LAUNCH_SLURM_PHASE=worker \ + -e WORKER_TEE=0 \ + -e SCRATCH=$SCRATCH \ + -e SLURM_NNODES=\$SLURM_NNODES \ + -e SLURM_NODEID=\$SLURM_NODEID \ + -e SLURM_PROCID=\$SLURM_PROCID \ + -e SLURM_JOB_NODELIST=\"\$SLURM_JOB_NODELIST\" \ + -e SLURM_JOB_ID=\$SLURM_JOB_ID \ + -e MASTER_ADDR=$MASTER_ADDR \ + -e MASTER_PORT=$MASTER_PORT \ + -e HSTU_HAMMER_KERNEL=${HSTU_HAMMER_KERNEL:-TRITON} \ + -e MODE=$MODE \ + -e START_TS=$START_TS \ + -e NUM_TRAIN_TS=$NUM_TRAIN_TS \ + -e EVAL_EACH_WINDOW=$EVAL_EACH_WINDOW \ + -e EVAL_EVERY_N_WINDOWS=$EVAL_EVERY_N_WINDOWS \ + ${EVAL_EVERY_DATA_PCT:+-e EVAL_EVERY_DATA_PCT=$EVAL_EVERY_DATA_PCT} \ + -e NUM_TRAIN_BATCHES=$NUM_TRAIN_BATCHES \ + -e NUM_EVAL_BATCHES=$NUM_EVAL_BATCHES \ + -e METRIC_LOG_FREQ=$METRIC_LOG_FREQ \ + ${MLPERF_LOGGING:+-e MLPERF_LOGGING=$MLPERF_LOGGING} \ + ${MLPERF_TRAIN_LOSS_LOG_FREQ:+-e MLPERF_TRAIN_LOSS_LOG_FREQ=$MLPERF_TRAIN_LOSS_LOG_FREQ} \ + ${STREAMING_SHUFFLE_FRACTION:+-e STREAMING_SHUFFLE_FRACTION=$STREAMING_SHUFFLE_FRACTION} \ + ${STREAMING_SHUFFLE_SEED:+-e STREAMING_SHUFFLE_SEED=$STREAMING_SHUFFLE_SEED} \ + ${NUM_WORKERS:+-e NUM_WORKERS=$NUM_WORKERS} \ + ${PREFETCH_FACTOR:+-e PREFETCH_FACTOR=$PREFETCH_FACTOR} \ + ${DIAG_UNIQUE_EMB:+-e DIAG_UNIQUE_EMB=$DIAG_UNIQUE_EMB} \ + ${DIAG_EMB_STEPS:+-e DIAG_EMB_STEPS=$DIAG_EMB_STEPS} \ + ${OUTPUT_TRACE:+-e OUTPUT_TRACE=$OUTPUT_TRACE} \ + ${MIN_HISTORY:+-e MIN_HISTORY=$MIN_HISTORY} \ + ${HISTORY_STRATEGY:+-e HISTORY_STRATEGY=$HISTORY_STRATEGY} \ + ${SEED:+-e SEED=$SEED} \ + ${DENSE_LR:+-e DENSE_LR=$DENSE_LR} \ + ${SPARSE_LR:+-e SPARSE_LR=$SPARSE_LR} \ + ${GRAD_CLIP_NORM:+-e GRAD_CLIP_NORM=$GRAD_CLIP_NORM} \ + ${HSTU_NUM_LAYERS:+-e HSTU_NUM_LAYERS=$HSTU_NUM_LAYERS} \ + ${MAX_SEQ_LEN:+-e MAX_SEQ_LEN=$MAX_SEQ_LEN} \ + ${HISTORY_LENGTH:+-e HISTORY_LENGTH=$HISTORY_LENGTH} \ + ${BATCH_SIZE:+-e BATCH_SIZE=$BATCH_SIZE} \ + ${CKPT_TIME_INTERVAL_S:+-e CKPT_TIME_INTERVAL_S=$CKPT_TIME_INTERVAL_S} \ + ${KEEP_LAST_N:+-e KEEP_LAST_N=$KEEP_LAST_N} \ + ${IN_WINDOW_CKPT_FREQ:+-e IN_WINDOW_CKPT_FREQ=$IN_WINDOW_CKPT_FREQ} \ + ${CKPT_STEP_FREQ:+-e CKPT_STEP_FREQ=$CKPT_STEP_FREQ} \ + -e TRAIN_SPLIT_PERCENTAGE=${TRAIN_SPLIT_PERCENTAGE:-1.0} \ + -e AUC_THRESHOLD=${AUC_THRESHOLD:-0.80275} \ + ${MLPERF_SUBMISSION_PLATFORM:+-e MLPERF_SUBMISSION_PLATFORM=$MLPERF_SUBMISSION_PLATFORM} \ + ${TRAIN_LIFETIME_AUC_MODE:+-e TRAIN_LIFETIME_AUC_MODE=$TRAIN_LIFETIME_AUC_MODE} \ + ${EVAL_LIFETIME_AUC_MODE:+-e EVAL_LIFETIME_AUC_MODE=$EVAL_LIFETIME_AUC_MODE} \ + ${CUMULATIVE_AUC_BINS:+-e CUMULATIVE_AUC_BINS=$CUMULATIVE_AUC_BINS} \ + ${LIFETIME_AUC_WINDOW:+-e LIFETIME_AUC_WINDOW=$LIFETIME_AUC_WINDOW} \ + -e SPLIT_SALT=${SPLIT_SALT:-0} \ + -e EVAL_HOLDOUT_TS=${EVAL_HOLDOUT_TS:--1} \ + -e EVAL_HOLDOUT_NUM_WINDOWS=${EVAL_HOLDOUT_NUM_WINDOWS:-1} \ + ${WORKER_CMD:+-e WORKER_CMD=\"$WORKER_CMD\"} \ + ${RUN_NAME:+-e RUN_NAME=$RUN_NAME} \ + ${TENSORBOARD_LOG_PATH:+-e TENSORBOARD_LOG_PATH=$TENSORBOARD_LOG_PATH} \ + ${MLPERF_LOG_PATH:+-e MLPERF_LOG_PATH=$MLPERF_LOG_PATH} \ + ${CKPT_PATH:+-e CKPT_PATH=$CKPT_PATH} \ + ${SPARSE_A2A_FWD:+-e SPARSE_A2A_FWD=$SPARSE_A2A_FWD} \ + ${SPARSE_A2A_BWD:+-e SPARSE_A2A_BWD=$SPARSE_A2A_BWD} \ + -e LOG=$LOG \ + $NCCL_ENV_ARGS \ + $CONTAINER bash -lc 'cd $REPO && LAUNCH_SLURM_PHASE=worker bash scripts/launch_slurm_suachong.sh' + _wrc=\$? + if { [ \$_wrc -eq 125 ] || [ \$_wrc -eq 126 ] || [ \$_wrc -eq 127 ]; } && [ \$_wattempt -lt 5 ]; then + echo \"[\$(hostname)] worker exec failed to START (rc=\$_wrc, attempt \$_wattempt/5) — container not ready; restarting + retrying\" + docker restart $CONTAINER >/dev/null 2>&1 || true + for _w in \$(seq 1 30); do + [ \"\$(docker inspect -f '{{.State.Running}}' $CONTAINER 2>/dev/null)\" = \"true\" ] \ + && docker exec $CONTAINER true >/dev/null 2>&1 && break + sleep 2 + done + sleep 3 + continue + fi + exit \$_wrc + done + " 2>&1 | tee -a "$LOG" + rc=${PIPESTATUS[0]} + echo "[$(date)] launch_slurm/orchestrate finished rc=$rc" | tee -a "$LOG" + exit $rc +} + +# ============================================================================= +# PHASE: provision (compute-node host) +# ============================================================================= +provision() { + export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${PATH:-}" + DOCKER=$(command -v docker 2>/dev/null || true); DOCKER=${DOCKER:-/usr/bin/docker} + FBGEMM_WHL=${FBGEMM_WHL:-/apps/chcai/FBGEMM/fbgemm_gpu/dist/fbgemm_gpu_nightly_rocm-2026.6.2-cp312-cp312-linux_x86_64.whl} # [CLUSTER-SPECIFIC] gfx950/ROCm wheel + TORCH_IDX=${TORCH_IDX:-https://download.pytorch.org/whl/rocm7.2} # [CLUSTER-SPECIFIC] ROCm version index + echo "[provision] host=$(hostname) container=$CONTAINER docker=$DOCKER" + + # Resolve which image to run + whether deps must be installed. Prefer a pre-baked + # image (deps already installed) to skip the multi-GB torch download + pip / + # torchrec-from-git build on every fresh node: + # 1) baked image in this node's docker -> use it, skip deps + # 2) baked image tar on NFS -> docker load (local, no internet) + # 3) neither -> base image + pip (slow path, which + # can then be baked via BAKE_IMAGE=1) + NEED_DEPS=1 + RUN_IMAGE="$IMAGE" + if [ "$USE_BAKED" = "1" ]; then + if "$DOCKER" image inspect "$BAKED_IMAGE" >/dev/null 2>&1; then + echo "[provision] using baked image $BAKED_IMAGE (deps preinstalled, no download)" + RUN_IMAGE="$BAKED_IMAGE"; NEED_DEPS=0 + elif [ -f "$BAKED_TAR" ]; then + echo "[provision] loading baked image from $BAKED_TAR (local, no internet)..." + if "$DOCKER" load -i "$BAKED_TAR" >/dev/null 2>&1 && "$DOCKER" image inspect "$BAKED_IMAGE" >/dev/null 2>&1; then + RUN_IMAGE="$BAKED_IMAGE"; NEED_DEPS=0; echo "[provision] baked image loaded" + else + echo "[provision] WARNING: docker load failed; falling back to base-image + pip" + fi + fi + fi + if ! "$DOCKER" image inspect "$RUN_IMAGE" >/dev/null 2>&1; then + echo "[provision] pulling $RUN_IMAGE (this can take a while)..."; "$DOCKER" pull "$RUN_IMAGE" + fi + + echo "[provision] (re)starting container $CONTAINER from $RUN_IMAGE" + "$DOCKER" rm -f "$CONTAINER" >/dev/null 2>&1 || true + "$DOCKER" run -d --name "$CONTAINER" \ + --network=host --ipc=host --shm-size=64g \ + --device=/dev/kfd --device=/dev/dri --group-add video \ + `# [CLUSTER-SPECIFIC] AMD ROCm device passthrough; NVIDIA uses --gpus all / nvidia runtime` \ + --cap-add=SYS_PTRACE --cap-add=CAP_SYS_ADMIN --cap-add=IPC_LOCK \ + --ulimit memlock=-1:-1 --ulimit stack=67108864:67108864 \ + `# memlock=-1 is REQUIRED for RDMA QP memory registration — do not drop` \ + --security-opt seccomp=unconfined --privileged \ + -v "$REPO_MOUNT:$REPO_MOUNT" \ + -v "$DATA_MOUNT:$DATA_MOUNT" \ + `# shared-NFS bind mounts: repo home (REPO_MOUNT, rw) + dataset/build assets (DATA_MOUNT)` \ + -w "$REPO" \ + "$RUN_IMAGE" sleep infinity + + # --- RDMA userspace overlay for in-container RCCL (bnxt_re) ----------------- + # The image (rocm/primus, rdma-core 50/libbnxt_re-rdmav34) ships an OLDER RDMA + # userspace than the host kernel bnxt_re driver. The stock v34 provider faults + # RCCL's deep-queue create_qp (max_send_wr=256) against the newer kernel uapi + # -> "ibv_create_qp ... Bad address". Fix: stage the host's matched rdma-core + # (libibverbs v61 + libbnxt_re-rdmav59 + libnl) on NFS so the worker phase makes + # RCCL load it via LD_PRELOAD + LD_LIBRARY_PATH. The UNVERSIONED libibverbs.so + # symlink is essential (import torch pulls the unversioned soname; without it + # the lookup falls through to the container v34 lib and the fix regresses). + if [ "${FORCE_OVERLAY:-0}" != "1" ] && ls "$OVERLAY/lib/libibverbs/"libbnxt_re-rdmav*.so >/dev/null 2>&1 && [ -L "$OVERLAY/lib/libibverbs.so" ]; then + echo "[provision] host RDMA overlay already staged at $OVERLAY (shared NFS) — skipping" + else + echo "[provision] staging host RDMA userspace overlay -> $OVERLAY" + rm -rf "${OVERLAY}.tmp" 2>/dev/null + mkdir -p "${OVERLAY}.tmp/lib/libibverbs" "${OVERLAY}.tmp/libibverbs.d" + cp -L /usr/lib64/libibverbs.so.1 /usr/lib64/libnl-3.so.200 /usr/lib64/libnl-route-3.so.200 "${OVERLAY}.tmp/lib/" 2>/dev/null || true + ln -sf libibverbs.so.1 "${OVERLAY}.tmp/lib/libibverbs.so" + cp -L /usr/lib64/libibverbs/*.so "${OVERLAY}.tmp/lib/libibverbs/" 2>/dev/null || true + cp /etc/libibverbs.d/*.driver "${OVERLAY}.tmp/libibverbs.d/" 2>/dev/null || true + if ls "${OVERLAY}.tmp/lib/libibverbs/"libbnxt_re-rdmav*.so >/dev/null 2>&1; then + rm -rf "$OVERLAY" 2>/dev/null + mv "${OVERLAY}.tmp" "$OVERLAY" 2>/dev/null || { mkdir -p "$OVERLAY"; cp -a "${OVERLAY}.tmp/." "$OVERLAY/"; } + echo "[provision] host RDMA overlay staged: $(ls "$OVERLAY/lib/libibverbs" | wc -l) providers + libibverbs.so symlink" + else + echo "[provision] WARNING: host bnxt_re provider not found at /usr/lib64/libibverbs — multi-node RDMA will fail 'Bad address'; use NCCL_NET_TRANSPORT=socket" + fi + fi + + if [ "$NEED_DEPS" = "0" ]; then + echo "[provision] baked image — deps preinstalled; verifying imports only" + "$DOCKER" exec "$CONTAINER" bash -lc ' +python -c "import torch, fbgemm_gpu, torchrec, polars, xxhash, gin; print(\"imports OK,\", torch.__version__, torch.version.hip, torch.cuda.device_count(), \"gpus\")" +' || echo "[provision] WARNING: baked-image import smoke failed" + else + echo "[provision] installing recipe deps (base image, slow path)" + # Install misc deps FIRST, then pin the rocm torch stack + fbgemm + torchrec + # LAST with --no-deps so nothing pulls a CUDA torch over the rocm build. + "$DOCKER" exec "$CONTAINER" bash -lc ' +set -e +echo "=== native torch ==="; python -c "import torch;print(torch.__version__)" || true +echo "=== misc python deps ===" +pip install --no-cache-dir polars-u64-idx pyarrow pyyaml tqdm psutil numba xxhash gin-config \ + absl-py pandas tensorboard torchmetrics tensordict pyre-extensions iopath typing-inspect 2>&1 | tail -3 || true +echo "=== rocm torch stack (force, no-deps, LAST) ===" +pip install --force-reinstall --no-deps --index-url '"$TORCH_IDX"' \ + torch==2.12.0+rocm7.2 torchvision==0.27.0+rocm7.2 torchaudio==2.11.0+rocm7.2 +echo "=== fbgemm (local gfx950 wheel) ===" +pip install --force-reinstall --no-deps '"$FBGEMM_WHL"' +echo "=== torchrec v2026.06.01.00 (force, no-deps) ===" +pip install --force-reinstall --no-deps "git+https://github.com/pytorch/torchrec.git@v2026.06.01.00" +echo "=== import smoke ===" +python -c "import torch, fbgemm_gpu, torchrec, polars, xxhash, gin; print(\"imports OK,\", torch.__version__, torch.version.hip, torch.cuda.device_count(), \"gpus\")" +' + fi + + # --- one-time bake: snapshot the deps-installed container into a reusable image + # and save it to NFS so future nodes skip the download/pip path entirely. + if [ "${BAKE_IMAGE:-0}" = "1" ]; then + echo "[provision] baking: docker commit $CONTAINER -> $BAKED_IMAGE" + if "$DOCKER" commit "$CONTAINER" "$BAKED_IMAGE" >/dev/null; then + echo "[provision] saving $BAKED_IMAGE -> $BAKED_TAR (one-time, tens of GB)" + if "$DOCKER" save "$BAKED_IMAGE" -o "${BAKED_TAR}.tmp.$$" && mv -f "${BAKED_TAR}.tmp.$$" "$BAKED_TAR"; then + echo "[provision] bake done: $(ls -lh "$BAKED_TAR" 2>/dev/null | awk '{print $5}')" + else + echo "[provision] WARNING: docker save failed"; rm -f "${BAKED_TAR}.tmp.$$" 2>/dev/null + fi + else + echo "[provision] WARNING: docker commit failed" + fi + fi + echo "[provision] DONE" +} + +# ============================================================================= +# PHASE: worker (inside the container) +# ============================================================================= +worker() { + cd "$REPO_ROOT" + mkdir -p "$SCRATCH" 2>/dev/null || true + LOG=${LOG:-$SCRATCH/yambda_5b_8gpu.log} + # WORKER_TEE=0 (set by orchestrate) sends our file sink to /dev/null to avoid + # double-logging, since orchestrate already tees stdout into the real $LOG. + [ "${WORKER_TEE:-1}" = "0" ] && LOG=/dev/null + export TENSORBOARD_LOG_PATH=${TENSORBOARD_LOG_PATH:-$SCRATCH/tb/yambda_5b} + # MLPerf compliance log (rank 0 writes it). Per-job filename so each standalone + # sbatch gets a clean log; the e2e supervisor pins MLPERF_LOG_PATH itself. + export MLPERF_LOG_PATH=${MLPERF_LOG_PATH:-$SCRATCH/mlperf/yambda_5b_mlperf.${SLURM_JOB_ID:-manual}.log} + echo "[$(date)] REPO_ROOT=$REPO_ROOT" | tee -a "$LOG" + + # polars-u64-idx (NOT stock polars) — yambda parquet's flat-explode overruns + # 32-bit row index. Reserved node has no outbound DNS, so install from a + # pre-staged tarball under /apps/chcai/. Override PIP_LOCAL_TGZ for other hosts. + PIP_LOCAL_TGZ=${PIP_LOCAL_TGZ:-/apps/chcai/pip_local_yambda.tgz} # [CLUSTER-SPECIFIC] shared-NFS path + PIP_LOCAL_DIR=${PIP_LOCAL_DIR:-/tmp/pip_local} + if [ ! -f "$PIP_LOCAL_DIR/lib/python3.12/site-packages/polars/__init__.py" ]; then + rm -rf "$PIP_LOCAL_DIR" + mkdir -p "$PIP_LOCAL_DIR" && tar xzf "$PIP_LOCAL_TGZ" -C "$(dirname "$PIP_LOCAL_DIR")" 2>&1 | tail -3 | tee -a "$LOG" + fi + + export PYTHONPATH="$PIP_LOCAL_DIR/lib/python3.12/site-packages:$REPO_ROOT:${PYTHONPATH:-}" + export HOME=${HOME:-/tmp} + echo "[$(date)] PYTHONPATH=$PYTHONPATH" | tee -a "$LOG" + python -c "import torch, fbgemm_gpu, torchrec, polars, xxhash, gin; print('imports OK,', torch.__version__, torch.cuda.device_count(),'gpus')" 2>&1 | tee -a "$LOG" + + export HIP_VISIBLE_DEVICES=${HIP_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7} + export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7} + + # --- distributed topology --------------------------------------------------- + GPUS_PER_NODE=$(python -c "import torch; print(torch.cuda.device_count())") + # Multi-node when launched one-task-per-node under SLURM (SLURM_NNODES>1); + # otherwise fall through to legacy single-node defaults (localhost, node_rank 0). + if [ "${SLURM_NNODES:-1}" -gt 1 ] && [ -n "${SLURM_JOB_NODELIST:-}" ]; then + NNODES=${SLURM_NNODES} + NODE_RANK=${SLURM_NODEID:-${SLURM_PROCID:-0}} + # PREFER a MASTER_ADDR/PORT forwarded from the orchestrate phase (resolved on + # the host, which has scontrol); the container image carries no SLURM client. + if [ -z "${MASTER_ADDR:-}" ]; then + MASTER_ADDR=$(scontrol show hostnames "$SLURM_JOB_NODELIST" 2>/dev/null | head -1) + MASTER_ADDR=${MASTER_ADDR:-localhost} + fi + MASTER_PORT=${MASTER_PORT:-$(( 20000 + ${SLURM_JOB_ID:-0} % 20000 ))} + else + NNODES=${NNODES:-1} + NODE_RANK=${NODE_RANK:-0} + # Single-node: all ranks live on THIS host, so rendezvous over loopback and + # do NOT use the SLURM hostname. On some nodes the hostname resolves to a + # non-routable per-GPU RoCE /31 (benic 192.168.x) address; using it makes the + # NCCL bootstrap fail with "No route to host". localhost is node-independent. + MASTER_ADDR=localhost + MASTER_PORT=${MASTER_PORT:-} # empty => train_ranker picks a free port + fi + export NNODES NODE_RANK GPUS_PER_NODE MASTER_ADDR MASTER_PORT + export WORLD_SIZE=$(( NNODES * GPUS_PER_NODE )) + echo "[$(date)] topology: nnodes=$NNODES node_rank=$NODE_RANK gpus_per_node=$GPUS_PER_NODE world_size=$WORLD_SIZE master=$MASTER_ADDR:${MASTER_PORT:-}" | tee -a "$LOG" + + # NCCL bootstrap NIC: loopback single-node, routable host NIC multi-node (pin + # to avoid auto-detect picking a non-routable per-GPU RoCE link). Override via + # $NCCL_SOCKET_IFNAME. [CLUSTER-SPECIFIC] multi-node fenic0 (find via `ip -br addr`). + if [ "$NNODES" -gt 1 ]; then + export NCCL_SOCKET_IFNAME=${NCCL_SOCKET_IFNAME:-fenic0} + else + export NCCL_SOCKET_IFNAME=${NCCL_SOCKET_IFNAME:-lo} + fi + echo "[$(date)] NCCL_SOCKET_IFNAME=$NCCL_SOCKET_IFNAME (nnodes=$NNODES)" | tee -a "$LOG" + + # Multi-node additionally needs the RDMA data-plane (bnxt_re HCAs) configured; + # single-node uses intra-node P2P (XGMI/PCIe) so only the bootstrap NIC matters. + if [ "$NNODES" -gt 1 ]; then + NCCL_NET_TRANSPORT=${NCCL_NET_TRANSPORT:-ib} + if [ "$NCCL_NET_TRANSPORT" = "socket" ]; then + export NCCL_IB_DISABLE=1 + echo "[$(date)] NCCL: IB disabled — allreduce over TCP (fenic0). Functional, not RDMA-fast." | tee -a "$LOG" + else + # bnxt_re userspace provider ABI overlay (REQUIRED for RCCL). The stock v34 + # provider faults RCCL's create_qp (256 WRs) against the host kernel uapi + # ("Bad address"); the host v61/v59 set staged by the provision phase works. + # The libibverbs.so (UNVERSIONED) symlink + LD_PRELOAD are both required so + # the torch process maps ONLY the host lib (see provision phase comment). + if [ -e "$OVERLAY/lib/libibverbs.so.1" ]; then + [ -e "$OVERLAY/lib/libibverbs.so" ] || ln -sf libibverbs.so.1 "$OVERLAY/lib/libibverbs.so" 2>/dev/null || true + export LD_LIBRARY_PATH="$OVERLAY/lib:$OVERLAY/lib/libibverbs:${LD_LIBRARY_PATH:-}" + export LD_PRELOAD="$OVERLAY/lib/libibverbs.so.1${LD_PRELOAD:+:$LD_PRELOAD}" + echo "[$(date)] NCCL: bnxt_re provider overlay -> $OVERLAY (host rdma-core v61/v59; symlink+LD_PRELOAD so RCCL binds the host lib for QP creation)" | tee -a "$LOG" + else + echo "[$(date)] WARNING: RDMA overlay $OVERLAY missing — RCCL QP creation will fail 'Bad address' on stock v34 provider; set RDMA_OVERLAY or use NCCL_NET_TRANSPORT=socket" | tee -a "$LOG" + fi + # MINIMAL bnxt_re set PROVEN on these meta64 cv350 nodes (cmcknigh RCCL + # benchmarks + confirmed e2e here). NCCL_IB_TC=104 (RoCE lossless PFC class) + # is required; do NOT add the ionic-AINIC QPS/ECE/DMABUF block. + # [CLUSTER-SPECIFIC] RDMA HCA names (`ibv_devices`); other vendors => mlx5_*/ionic_* + export NCCL_IB_HCA=${NCCL_IB_HCA:-bnxt_re0,bnxt_re1,bnxt_re2,bnxt_re3,bnxt_re4,bnxt_re5,bnxt_re6,bnxt_re7} + export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-3} # [CLUSTER-SPECIFIC] RoCEv2 IPv4 GID idx (`show_gids`) + export NCCL_IB_TC=${NCCL_IB_TC:-104} # [CLUSTER-SPECIFIC] RoCE lossless/PFC traffic class + export NCCL_IB_TIMEOUT=${NCCL_IB_TIMEOUT:-14} + export NCCL_IGNORE_CPU_AFFINITY=${NCCL_IGNORE_CPU_AFFINITY:-1} + export RCCL_MSCCL_ENABLE=${RCCL_MSCCL_ENABLE:-0} + # GPU-Direct RDMA on by default (~+22% throughput at 2 nodes via peermem). + # Set NCCL_NET_GDR_LEVEL=0 to force the legacy host-staged path. + export NCCL_NET_GDR_LEVEL=${NCCL_NET_GDR_LEVEL:-5} + export NCCL_DMABUF_ENABLE=${NCCL_DMABUF_ENABLE:-1} + echo "[$(date)] NCCL: RDMA over bnxt_re (GID idx ${NCCL_IB_GID_INDEX}, TC ${NCCL_IB_TC}, GDR_LEVEL=${NCCL_NET_GDR_LEVEL}, DMABUF=${NCCL_DMABUF_ENABLE}; meta64 bnxt_re config, validated)" | tee -a "$LOG" + fi + fi + export NCCL_DEBUG=${NCCL_DEBUG:-WARN} + export HSTU_HAMMER_KERNEL=${HSTU_HAMMER_KERNEL:-} + export PYTORCH_CUDA_ALLOC_CONF=${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True} + + # --- GPU clock sanity guard ------------------------------------------------- + # A leftover perf_determinism cap (half clock) silently slows every kernel ~1.9x. + # Log the perf level + a live sclk sample and try to restore boost (non-fatal). + if command -v rocm-smi >/dev/null 2>&1; then + echo "[$(date)] GPU perf-level check:" | tee -a "$LOG" + rocm-smi --showperflevel 2>/dev/null | grep -iE "GPU\[[0-9]+\]" | tee -a "$LOG" || true + if rocm-smi --showperflevel 2>/dev/null | grep -iqE "Performance Level: *(perf_determinism|manual|low)"; then + echo "[$(date)] WARNING: GPUs not in 'auto' perf level — attempting --setperflevel auto" | tee -a "$LOG" + rocm-smi --setperflevel auto 2>/dev/null | grep -iE "set to auto" | tee -a "$LOG" \ + || echo "[$(date)] WARNING: could not set perf level (no permission?). Run 'rocm-smi --setperflevel auto' on the HOST before benchmarking — clocks may be capped." | tee -a "$LOG" + fi + echo "[$(date)] sclk sample (GPU0):$(rocm-smi -d 0 --showclocks 2>/dev/null | grep -i 'sclk clock level' | sed -E 's/.*sclk clock level//')" | tee -a "$LOG" || true + fi + + # --- stray-trainer / leaked-VRAM guard ------------------------------------- + # The trainer runs via `docker exec` into a long-lived container, so its procs + # live in the container PID namespace, NOT the SLURM job cgroup. If a prior job + # OOM'd/crashed, a rank can leak and keep holding ~half of every GPU's VRAM, + # which persists across jobs (container survives) and guarantees the next + # attempt OOMs. Before launching, reap any pre-existing trainer procs (there + # should be none at this point) and wait for VRAM to drain. [g]-guard avoids + # self-match. Non-fatal. + if pgrep -f '[g]enerative_recommenders' >/dev/null 2>&1; then + echo "[$(date)] WARNING: leaked trainer procs found pre-launch — killing." | tee -a "$LOG" + pkill -9 -f '[g]enerative_recommenders' 2>/dev/null || true + for _i in $(seq 1 15); do + pgrep -f '[g]enerative_recommenders' >/dev/null 2>&1 || break + sleep 2 + done + sleep 5 # let the driver release VRAM after process exit + if command -v rocm-smi >/dev/null 2>&1; then + echo "[$(date)] post-cleanup GPU0 used GiB:$(rocm-smi --showmeminfo vram 2>/dev/null | awk -F: '/Used/{printf " %.0f", $3/1073741824; exit}')" | tee -a "$LOG" + fi + fi + + # WORKER_CMD override: run an arbitrary in-container command (e.g. an a2a/RCCL + # micro-benchmark) instead of the trainer, REUSING all the NCCL/RDMA/topology + # setup above so it exercises the exact transport the trainer uses. The + # supervisor never sets WORKER_CMD, so the training path is unchanged. + if [ -n "${WORKER_CMD:-}" ]; then + echo "[$(date)] WORKER_CMD override (WORLD_SIZE=$WORLD_SIZE): $WORKER_CMD" | tee -a "$LOG" + bash -lc "cd $REPO_ROOT && $WORKER_CMD" 2>&1 | tee -a "$LOG" + return + fi + + echo "[$(date)] launching train_ranker with WORLD_SIZE=$WORLD_SIZE" | tee -a "$LOG" + python -m generative_recommenders.dlrm_v3.train.train_ranker \ + --dataset yambda-5b --mode "${MODE:-streaming-train-eval}" 2>&1 | tee -a "$LOG" +} + +# ---- dispatch --------------------------------------------------------------- +case "$PHASE" in + orchestrate) orchestrate ;; + provision) provision ;; + worker) worker ;; + *) echo "launch_slurm.sh: unknown LAUNCH_SLURM_PHASE='$PHASE'" >&2; exit 2 ;; +esac From 1d257133ceaf48d68147c3d98ab7dadd20180507 Mon Sep 17 00:00:00 2001 From: suachong Date: Wed, 24 Jun 2026 22:49:07 +0000 Subject: [PATCH 095/127] dlrmv4: drop launch_slurm_suachong.sh from the PR (keep local only) Co-authored-by: Cursor --- .../scripts/launch_slurm_suachong.sh | 664 ------------------ 1 file changed, 664 deletions(-) delete mode 100755 recommendation_v4/scripts/launch_slurm_suachong.sh diff --git a/recommendation_v4/scripts/launch_slurm_suachong.sh b/recommendation_v4/scripts/launch_slurm_suachong.sh deleted file mode 100755 index 579076d47..000000000 --- a/recommendation_v4/scripts/launch_slurm_suachong.sh +++ /dev/null @@ -1,664 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=yambda_slurm -#SBATCH --nodes=1 -#SBATCH --ntasks-per-node=1 -#SBATCH --exclusive -#SBATCH --partition=meta64 # [CLUSTER-SPECIFIC] partition name -#SBATCH --output=yambda_slurm.%j.out -# ============================================================================= -# launch_slurm.sh — single entry point for the yambda-5b trainer on N>=1 nodes. -# -# Consolidates what used to be separate scripts so multi-node enablement is -# ONE committable script (plus the train_ranker.py / utils.py python changes): -# * orchestrate phase (host SLURM glue) — formerly sbatch_smoke_multinode.sh -# * provision phase (container + RDMA) — formerly _provision_yambda_primus.sh -# * worker phase (in-container train) — now inlined below -# -# PHASES (auto-detected from context; force with LAUNCH_SLURM_PHASE=): -# orchestrate Runs on the SLURM batch host (no /.dockerenv). Resolves the -# rendezvous (MASTER_ADDR/PORT), ensures the container on every -# node (provision phase), then `docker exec`s the worker phase on -# every node, one task per node. -# provision Runs on a compute-node host. Ensures the `yambda_primus` -# container is up (loads the pre-baked image if present — no -# internet/pip — else builds from the base image) and stages the -# host RDMA userspace overlay on shared NFS. -# worker Runs INSIDE the container. Sets the distributed topology + -# NCCL/RDMA env and spawns this node's GPU ranks via train_ranker. -# N==1 transparently uses the legacy single-node path (localhost, -# node_rank 0), byte-for-byte as before, so the streaming-e2e -# supervisor's direct `bash scripts/launch_slurm.sh` is unchanged. -# -# USAGE -# Multi-node (N>=1): sbatch --nodes=2 scripts/launch_slurm.sh -# Single-node direct: bash scripts/launch_slurm.sh (already inside container; -# what run_streaming_e2e.sh invokes per relaunch) -# Perf pair: -# LOG=/apps/chcai/perf_1node.log NUM_TRAIN_BATCHES=200 NUM_EVAL_BATCHES=0 \ -# EVAL_EACH_WINDOW=0 METRIC_LOG_FREQ=20 \ -# sbatch --nodes=1 --job-name=y1 scripts/launch_slurm.sh -# LOG=/apps/chcai/perf_2node.log NUM_TRAIN_BATCHES=200 NUM_EVAL_BATCHES=0 \ -# EVAL_EACH_WINDOW=0 METRIC_LOG_FREQ=20 \ -# sbatch --nodes=2 --job-name=y2 scripts/launch_slurm.sh -# # then: bash scripts/compare_node_perf.sh /apps/chcai/perf_1node.log /apps/chcai/perf_2node.log -# -# ONE-TIME IMAGE BAKE (so fresh nodes skip the multi-GB torch download + pip): -# BAKE_IMAGE=1 LAUNCH_SLURM_PHASE=provision bash scripts/launch_slurm.sh -# (commits the deps-installed container to $BAKED_IMAGE and `docker save`s it to -# $BAKED_TAR on NFS; subsequent provisions `docker load` it offline.) -# -# ----------------------------------------------------------------------------- -# PORTABILITY — what to change for a DIFFERENT cluster / network / hardware. -# Every such knob is also tagged inline with "[CLUSTER-SPECIFIC]" (grep for it). -# All are env-overridable, so you can adapt without editing this file. -# -# A) SLURM / scheduler -# - #SBATCH --partition=meta64 : partition name. CHANGE per cluster. -# - #SBATCH --time / --exclusive : policy; adjust to taste. -# -# B) Filesystems (must be shared/NFS across ALL nodes — this script re-invokes -# itself and reads the overlay + data from these paths cluster-wide) -# - REPO_MOUNT (repo + this script, e.g. /home/) is bind-mounted rw; -# DATA_MOUNT (e.g. /apps/chcai) holds the read-only dataset + overlay + -# baked tar + pip tarball; SCRATCH (e.g. /home//yambda_runs) is the -# writable log/output root. Override any via env — nothing is user-hardwired. -# -# C) Container image / GPU software stack (tied to the GPU arch + ROCm version) -# - IMAGE=rocm/primus:v26.3 : base image. ROCm/AMD-specific. -# - docker run --device=/dev/kfd --device=/dev/dri --group-add video : AMD ROCm -# device passthrough. For NVIDIA this is --gpus all / nvidia runtime instead. -# - --ulimit memlock=-1 : REQUIRED for RDMA QP registration (do not drop). -# - TORCH_IDX (rocm7.2), torch/vision/audio ==*+rocm7.2, FBGEMM_WHL (a gfx950 -# wheel), torchrec pin : the whole deps set is arch/ROCm-version-specific. -# -# D) Network fabric — THE trickiest part; defaults are PROVEN on meta64 cv350 -# (Broadcom bnxt_re RoCEv2). On a different fabric these almost certainly change -# (see the worker-phase block for the full rationale): -# - NCCL_SOCKET_IFNAME=fenic0 : the ONE routable host NIC for TCP bootstrap. -# Find yours with `ip -br addr`; the per-GPU RDMA NICs are usually NOT -# routable for plain TCP, so auto-detect hangs init — you MUST pin this. -# - NCCL_IB_HCA=bnxt_re0..7 : the RDMA HCA device names. List with `ibv_devices`. -# Different NIC vendor (e.g. mlx5_*, ionic_*) => different names AND a -# different userspace provider, which changes the RDMA overlay below. -# - NCCL_IB_GID_INDEX=3 : RoCEv2 IPv4 GID index. Check `show_gids`; v1/v2 and -# IPv4/IPv6 live at different indices per port. -# - NCCL_IB_TC=104 : RoCE lossless (PFC) traffic class. Fabric/switch-specific. -# - RDMA overlay (provision phase): only needed when the CONTAINER's rdma-core -# is older than the HOST kernel driver's uapi (our bnxt_re v34-vs-v59 case). -# Different NIC/host => different /usr/lib64 provider .so to stage, or the -# overlay may be unnecessary entirely (set RDMA_OVERLAY= to disable). If RDMA -# can't be made to work, NCCL_NET_TRANSPORT=socket falls back to TCP. -# -# E) Not cluster-specific (auto-derived): GPUS_PER_NODE (torch.cuda.device_count), -# NNODES/NODE_RANK/MASTER_ADDR (from SLURM), WORLD_SIZE. -# ============================================================================= -set -uo pipefail - -# Absolute path to THIS script so the orchestrate phase can re-invoke it on every -# node (home is shared NFS, so the same path resolves cluster-wide). -SELF=$(cd "$(dirname "$0")" && pwd)/$(basename "$0") -REPO_ROOT=$(cd "$(dirname "$0")/.." && pwd) - -# ---- phase detection -------------------------------------------------------- -PHASE="${LAUNCH_SLURM_PHASE:-}" -if [ -z "$PHASE" ]; then - if [ -f /.dockerenv ]; then PHASE=worker; else PHASE=orchestrate; fi -fi - -# ---- shared config (env-overridable) ---------------------------------------- -CONTAINER=${CONTAINER:-yambda_primus} -REPO=${REPO:-$REPO_ROOT} # repo path inside the container -IMAGE=${IMAGE:-rocm/primus:v26.3} # [CLUSTER-SPECIFIC] ROCm/arch base image -BAKED_IMAGE=${BAKED_IMAGE:-yambda_primus_baked:latest} -BAKED_TAR=${BAKED_TAR:-/apps/chcai/yambda_primus_baked.tar} # [CLUSTER-SPECIFIC] shared-NFS path (read-only build asset) -USE_BAKED=${USE_BAKED:-1} -OVERLAY=${RDMA_OVERLAY:-/apps/chcai/rdma_host_el9_new} # [CLUSTER-SPECIFIC] shared-NFS RDMA overlay (read-only, already staged) - -REPO_MOUNT=${REPO_MOUNT:-$HOME} # NFS home holding the repo (must contain $REPO); override if your repo lives elsewhere -DATA_MOUNT=${DATA_MOUNT:-/apps/chcai} # shared dataset + RDMA overlay + pip/fbgemm assets (read-only) -SCRATCH=${SCRATCH:-$HOME/yambda_runs} # writable output root (logs / tb / traces) - -# ============================================================================= -# PHASE: orchestrate (SLURM batch host) -# ============================================================================= -orchestrate() { - # When run as the SLURM batch script, $0 is the node-local staged copy - # (/var/spool/slurmd/job/slurm_script), so $SELF / $REPO_ROOT are WRONG - # here (they don't exist on other nodes). Resolve the REAL shared-NFS script - # path + repo root from SLURM so we can re-invoke this script on every node and - # `cd` to the right repo inside the container. - SCRIPT_PATH=$(scontrol show job "${SLURM_JOB_ID:-0}" 2>/dev/null | grep -oP 'Command=\K\S+') - [ -f "${SCRIPT_PATH:-}" ] || SCRIPT_PATH="${SLURM_SUBMIT_DIR:-$REPO_ROOT}/scripts/launch_slurm_suachong.sh" - [ -f "$SCRIPT_PATH" ] || SCRIPT_PATH="$SELF" - REPO=$(cd "$(dirname "$SCRIPT_PATH")/.." && pwd) - - mkdir -p "$SCRATCH" 2>/dev/null || true - LOG=${LOG:-$SCRATCH/yambda_slurm.${SLURM_JOB_ID:-manual}.log} - - MODE=${MODE:-streaming-train-eval} - if [ "${SMOKE:-0}" = "1" ]; then - START_TS=${START_TS:-150} - NUM_TRAIN_TS=${NUM_TRAIN_TS:-1} - NUM_TRAIN_BATCHES=${NUM_TRAIN_BATCHES:-20} - NUM_EVAL_BATCHES=${NUM_EVAL_BATCHES:-10} - EVAL_EVERY_N_WINDOWS=${EVAL_EVERY_N_WINDOWS:-1} - METRIC_LOG_FREQ=${METRIC_LOG_FREQ:-5} - fi - START_TS=${START_TS:-0} - NUM_TRAIN_TS=${NUM_TRAIN_TS:-299} - NUM_TRAIN_BATCHES=${NUM_TRAIN_BATCHES:-0} - NUM_EVAL_BATCHES=${NUM_EVAL_BATCHES:-0} - EVAL_EACH_WINDOW=${EVAL_EACH_WINDOW:-1} - METRIC_LOG_FREQ=${METRIC_LOG_FREQ:-20} - if [ "${EVAL_EVERY_N_WINDOWS:-0}" -gt 0 ] 2>/dev/null; then - EVAL_EVERY_DATA_PCT=${EVAL_EVERY_DATA_PCT:-0} - else - EVAL_EVERY_N_WINDOWS=0 - EVAL_EVERY_DATA_PCT=${EVAL_EVERY_DATA_PCT:-0.005} - fi - FORCE_PROVISION=${FORCE_PROVISION:-0} - - # Truncate the metrics log on a FRESH run; APPEND on a supervised relaunch - # (APPEND_LOG=1) so the full-run NE/AUC history survives crash/node-failover - # resubmits instead of being wiped on every attempt (mirrors the single-node - # supervisor's init-once/append model). - if [ "${APPEND_LOG:-0}" = "1" ]; then - echo "[$(date)] === resume: appending to existing $LOG (APPEND_LOG=1) ===" >> "$LOG" - else - : > "$LOG" - fi - chmod 622 "$LOG" 2>/dev/null || true - echo "[$(date)] launch_slurm/orchestrate: job=${SLURM_JOB_ID:-?} nodes=${SLURM_JOB_NODELIST:-?} nnodes=${SLURM_NNODES:-1}" | tee -a "$LOG" - echo "[$(date)] resolved SCRIPT_PATH=$SCRIPT_PATH REPO=$REPO" | tee -a "$LOG" - echo "[$(date)] config: MODE=$MODE START_TS=$START_TS NUM_TRAIN_TS=$NUM_TRAIN_TS NUM_TRAIN_BATCHES=$NUM_TRAIN_BATCHES NUM_EVAL_BATCHES=$NUM_EVAL_BATCHES METRIC_LOG_FREQ=$METRIC_LOG_FREQ SMOKE=${SMOKE:-0} EVAL_EVERY_N_WINDOWS=$EVAL_EVERY_N_WINDOWS EVAL_EVERY_DATA_PCT=$EVAL_EVERY_DATA_PCT" | tee -a "$LOG" - echo "[$(date)] lr-override: DENSE_LR=${DENSE_LR:-} SPARSE_LR=${SPARSE_LR:-}" | tee -a "$LOG" - - # Rendezvous resolved on the HOST (the container image has no SLURM client). - MASTER_ADDR=$(scontrol show hostnames "$SLURM_JOB_NODELIST" 2>/dev/null | head -1) - MASTER_ADDR=${MASTER_ADDR:-localhost} - MASTER_PORT=$(( 20000 + ${SLURM_JOB_ID:-0} % 20000 )) - echo "[$(date)] rendezvous: MASTER_ADDR=$MASTER_ADDR MASTER_PORT=$MASTER_PORT" | tee -a "$LOG" - - # Optional NCCL/RCCL fabric overrides — forwarded into the container only when - # set at submit time (docker exec does NOT inherit the srun task env). The - # worker phase applies its own validated multi-node bnxt_re defaults when these - # are unset. Common: NCCL_NET_TRANSPORT=socket (TCP fallback), NCCL_DEBUG=INFO. - NCCL_ENV_ARGS="" - for v in NCCL_NET_TRANSPORT NCCL_DEBUG NCCL_SOCKET_IFNAME NCCL_IB_HCA NCCL_IB_GID_INDEX \ - NCCL_IB_TC NCCL_IB_TIMEOUT NCCL_IGNORE_CPU_AFFINITY RCCL_MSCCL_ENABLE NCCL_NET_GDR_LEVEL \ - NCCL_IB_PCI_RELAXED_ORDERING NCCL_IB_USE_INLINE NCCL_IB_QPS_PER_CONNECTION \ - NCCL_IB_ECE_ENABLE NCCL_DMABUF_ENABLE NCCL_GDRCOPY_ENABLE NCCL_GDR_FLUSH_DISABLE \ - NCCL_PXN_DISABLE NCCL_CHECKS_DISABLE NCCL_CROSS_NIC RDMA_OVERLAY; do - eval "val=\${$v:-}" - if [ -n "$val" ]; then NCCL_ENV_ARGS="$NCCL_ENV_ARGS -e $v=$val"; fi - done - - # TRICKY — variable expansion inside the `srun ... bash -c "..."` blocks below: - # the string is double-quoted, so PLAIN $VAR expands NOW on the batch host (e.g. - # $MASTER_ADDR, $CONTAINER, $SCRIPT_PATH — values computed above), while - # BACKSLASH-escaped \$VAR is passed through literally and expands LATER on each - # compute node inside the srun task (e.g. \$SLURM_NODEID, \$(hostname)) where the - # per-node SLURM_* env actually lives. Mixing these up sends every rank the - # wrong node id or breaks the docker exec — keep the \$ on per-node values. - - # --- step 1: ensure the container is up on every node ---------------------- - echo "[$(date)] ensuring container '$CONTAINER' on all nodes (force=$FORCE_PROVISION)" | tee -a "$LOG" - srun --ntasks-per-node=1 bash -c " - # Reap stale/foreign GPU containers from prior jobs BEFORE (re)provisioning. - # The node is allocated --exclusive, so any GPU container other than - # '$CONTAINER' is an orphan left by a previous job (its container outlives the - # SLURM allocation). We remove every such container that has GPU access - # (/dev/kfd or /dev/dri) — running OR stopped, whether or not it currently - # pins VRAM ('docker ps -aq' includes stopped ones) — since idle orphans can - # still hold device handles or wake up; leaked HBM from these has caused both - # OOMs and RCCL collective hangs. We deliberately SKIP non-GPU containers - # (e.g. 'k8s-node-services-*' and other cluster system services) so we don't - # disrupt node infrastructure. docker teardown lets the driver reclaim HBM. - for _c in \$(docker ps -aq 2>/dev/null); do - _nm=\$(docker inspect -f '{{.Name}}' \"\$_c\" 2>/dev/null | sed 's#^/##') - [ \"\$_nm\" = \"$CONTAINER\" ] && continue - _dev=\$(docker inspect -f '{{range .HostConfig.Devices}}{{.PathOnHost}} {{end}}' \"\$_c\" 2>/dev/null) - case \"\$_dev\" in - *kfd*|*dri*) - echo \"[\$(hostname)] reaping stale GPU container \$_nm (\$_c)\" - docker rm -f \"\$_c\" >/dev/null 2>&1 || true ;; - *) - echo \"[\$(hostname)] keeping non-GPU/system container \$_nm (\$_c)\" ;; - esac - done - # Reuse a STOPPED '$CONTAINER' (its installed deps persist in the container - # fs) instead of destructively re-provisioning from the base image + pip. - # Harmless no-op on a fresh node (no such container) -> falls through to - # provision below. Repo code is bind-mounted, so live edits are still picked up. - docker start $CONTAINER >/dev/null 2>&1 || true - if [ \"$FORCE_PROVISION\" = \"1\" ] || ! docker exec $CONTAINER true >/dev/null 2>&1; then - echo \"[\$(hostname)] (re)provisioning container\" - LAUNCH_SLURM_PHASE=provision CONTAINER=$CONTAINER IMAGE=$IMAGE \ - BAKED_IMAGE=$BAKED_IMAGE BAKED_TAR=$BAKED_TAR USE_BAKED=$USE_BAKED \ - BAKE_IMAGE=${BAKE_IMAGE:-0} RDMA_OVERLAY=$OVERLAY REPO=$REPO \ - REPO_MOUNT=$REPO_MOUNT DATA_MOUNT=$DATA_MOUNT SCRATCH=$SCRATCH bash $SCRIPT_PATH - else - # Container persists across jobs; the reap above only removes FOREIGN GPU - # containers, so our own '$CONTAINER' can still pin HBM via stray trainer - # ranks left by a prior OOM/crash (this caused repeated 'CUDA out of memory' - # on relaunch onto the same node). Restart it to kill every exec'd proc and - # let the driver reclaim HBM — cheap (keeps the installed deps in the - # container fs; NFS RDMA overlay also persists), no full re-provision. - echo \"[\$(hostname)] container already up — restarting to free any leaked HBM before launch\" - docker restart $CONTAINER >/dev/null 2>&1 || true - # Readiness gate: a bare 'docker exec true' can pass while the runtime is - # still settling, so the SUBSEQUENT (heavier) worker exec races the restart - # and dies with 'container is not running' / OCI 'setns' errors (observed on - # c07-08 and e08-08 -> the peer never joins rendezvous -> master 600s - # TCPStore timeout). Require State.Running=true AND a successful probe, then - # a short settle, before considering the container ready. - for _w in \$(seq 1 30); do - [ \"\$(docker inspect -f '{{.State.Running}}' $CONTAINER 2>/dev/null)\" = \"true\" ] \ - && docker exec $CONTAINER true >/dev/null 2>&1 && break - sleep 2 - done - sleep 2 - echo \"[\$(hostname)] container restarted (HBM reclaimed; running=\$(docker inspect -f '{{.State.Running}}' $CONTAINER 2>/dev/null))\" - fi - " 2>&1 | tee -a "$LOG" - - # --- step 2: launch the worker (trainer) inside the container on every node - - echo "[$(date)] launching trainer (worker phase) on all nodes" | tee -a "$LOG" - srun --ntasks-per-node=1 bash -c " - # Pre-flight readiness gate (per node): step 1 ran in a SEPARATE srun, so the - # container can still be settling here. Wait for State.Running=true + a probe - # before the worker exec so we don't race a just-restarted container. - for _w in \$(seq 1 30); do - [ \"\$(docker inspect -f '{{.State.Running}}' $CONTAINER 2>/dev/null)\" = \"true\" ] \ - && docker exec $CONTAINER true >/dev/null 2>&1 && break - [ \$_w -eq 1 ] && echo \"[\$(hostname)] worker pre-flight: waiting for container to be ready...\" - sleep 2 - done - # Retry wrapper: docker exec startup failures (rc 125 daemon 'container is not - # running', 126/127 OCI/setns 'exec failed') mean the container wasn't ready, - # NOT that the trainer ran and failed. Restart + re-gate + retry a few times. - # Any OTHER rc (the trainer actually started and exited) is propagated so the - # supervisor's resume-from-checkpoint logic owns real failures. - _wattempt=0 - while : ; do - _wattempt=\$((_wattempt+1)) - docker exec \ - -e LAUNCH_SLURM_PHASE=worker \ - -e WORKER_TEE=0 \ - -e SCRATCH=$SCRATCH \ - -e SLURM_NNODES=\$SLURM_NNODES \ - -e SLURM_NODEID=\$SLURM_NODEID \ - -e SLURM_PROCID=\$SLURM_PROCID \ - -e SLURM_JOB_NODELIST=\"\$SLURM_JOB_NODELIST\" \ - -e SLURM_JOB_ID=\$SLURM_JOB_ID \ - -e MASTER_ADDR=$MASTER_ADDR \ - -e MASTER_PORT=$MASTER_PORT \ - -e HSTU_HAMMER_KERNEL=${HSTU_HAMMER_KERNEL:-TRITON} \ - -e MODE=$MODE \ - -e START_TS=$START_TS \ - -e NUM_TRAIN_TS=$NUM_TRAIN_TS \ - -e EVAL_EACH_WINDOW=$EVAL_EACH_WINDOW \ - -e EVAL_EVERY_N_WINDOWS=$EVAL_EVERY_N_WINDOWS \ - ${EVAL_EVERY_DATA_PCT:+-e EVAL_EVERY_DATA_PCT=$EVAL_EVERY_DATA_PCT} \ - -e NUM_TRAIN_BATCHES=$NUM_TRAIN_BATCHES \ - -e NUM_EVAL_BATCHES=$NUM_EVAL_BATCHES \ - -e METRIC_LOG_FREQ=$METRIC_LOG_FREQ \ - ${MLPERF_LOGGING:+-e MLPERF_LOGGING=$MLPERF_LOGGING} \ - ${MLPERF_TRAIN_LOSS_LOG_FREQ:+-e MLPERF_TRAIN_LOSS_LOG_FREQ=$MLPERF_TRAIN_LOSS_LOG_FREQ} \ - ${STREAMING_SHUFFLE_FRACTION:+-e STREAMING_SHUFFLE_FRACTION=$STREAMING_SHUFFLE_FRACTION} \ - ${STREAMING_SHUFFLE_SEED:+-e STREAMING_SHUFFLE_SEED=$STREAMING_SHUFFLE_SEED} \ - ${NUM_WORKERS:+-e NUM_WORKERS=$NUM_WORKERS} \ - ${PREFETCH_FACTOR:+-e PREFETCH_FACTOR=$PREFETCH_FACTOR} \ - ${DIAG_UNIQUE_EMB:+-e DIAG_UNIQUE_EMB=$DIAG_UNIQUE_EMB} \ - ${DIAG_EMB_STEPS:+-e DIAG_EMB_STEPS=$DIAG_EMB_STEPS} \ - ${OUTPUT_TRACE:+-e OUTPUT_TRACE=$OUTPUT_TRACE} \ - ${MIN_HISTORY:+-e MIN_HISTORY=$MIN_HISTORY} \ - ${HISTORY_STRATEGY:+-e HISTORY_STRATEGY=$HISTORY_STRATEGY} \ - ${SEED:+-e SEED=$SEED} \ - ${DENSE_LR:+-e DENSE_LR=$DENSE_LR} \ - ${SPARSE_LR:+-e SPARSE_LR=$SPARSE_LR} \ - ${GRAD_CLIP_NORM:+-e GRAD_CLIP_NORM=$GRAD_CLIP_NORM} \ - ${HSTU_NUM_LAYERS:+-e HSTU_NUM_LAYERS=$HSTU_NUM_LAYERS} \ - ${MAX_SEQ_LEN:+-e MAX_SEQ_LEN=$MAX_SEQ_LEN} \ - ${HISTORY_LENGTH:+-e HISTORY_LENGTH=$HISTORY_LENGTH} \ - ${BATCH_SIZE:+-e BATCH_SIZE=$BATCH_SIZE} \ - ${CKPT_TIME_INTERVAL_S:+-e CKPT_TIME_INTERVAL_S=$CKPT_TIME_INTERVAL_S} \ - ${KEEP_LAST_N:+-e KEEP_LAST_N=$KEEP_LAST_N} \ - ${IN_WINDOW_CKPT_FREQ:+-e IN_WINDOW_CKPT_FREQ=$IN_WINDOW_CKPT_FREQ} \ - ${CKPT_STEP_FREQ:+-e CKPT_STEP_FREQ=$CKPT_STEP_FREQ} \ - -e TRAIN_SPLIT_PERCENTAGE=${TRAIN_SPLIT_PERCENTAGE:-1.0} \ - -e AUC_THRESHOLD=${AUC_THRESHOLD:-0.80275} \ - ${MLPERF_SUBMISSION_PLATFORM:+-e MLPERF_SUBMISSION_PLATFORM=$MLPERF_SUBMISSION_PLATFORM} \ - ${TRAIN_LIFETIME_AUC_MODE:+-e TRAIN_LIFETIME_AUC_MODE=$TRAIN_LIFETIME_AUC_MODE} \ - ${EVAL_LIFETIME_AUC_MODE:+-e EVAL_LIFETIME_AUC_MODE=$EVAL_LIFETIME_AUC_MODE} \ - ${CUMULATIVE_AUC_BINS:+-e CUMULATIVE_AUC_BINS=$CUMULATIVE_AUC_BINS} \ - ${LIFETIME_AUC_WINDOW:+-e LIFETIME_AUC_WINDOW=$LIFETIME_AUC_WINDOW} \ - -e SPLIT_SALT=${SPLIT_SALT:-0} \ - -e EVAL_HOLDOUT_TS=${EVAL_HOLDOUT_TS:--1} \ - -e EVAL_HOLDOUT_NUM_WINDOWS=${EVAL_HOLDOUT_NUM_WINDOWS:-1} \ - ${WORKER_CMD:+-e WORKER_CMD=\"$WORKER_CMD\"} \ - ${RUN_NAME:+-e RUN_NAME=$RUN_NAME} \ - ${TENSORBOARD_LOG_PATH:+-e TENSORBOARD_LOG_PATH=$TENSORBOARD_LOG_PATH} \ - ${MLPERF_LOG_PATH:+-e MLPERF_LOG_PATH=$MLPERF_LOG_PATH} \ - ${CKPT_PATH:+-e CKPT_PATH=$CKPT_PATH} \ - ${SPARSE_A2A_FWD:+-e SPARSE_A2A_FWD=$SPARSE_A2A_FWD} \ - ${SPARSE_A2A_BWD:+-e SPARSE_A2A_BWD=$SPARSE_A2A_BWD} \ - -e LOG=$LOG \ - $NCCL_ENV_ARGS \ - $CONTAINER bash -lc 'cd $REPO && LAUNCH_SLURM_PHASE=worker bash scripts/launch_slurm_suachong.sh' - _wrc=\$? - if { [ \$_wrc -eq 125 ] || [ \$_wrc -eq 126 ] || [ \$_wrc -eq 127 ]; } && [ \$_wattempt -lt 5 ]; then - echo \"[\$(hostname)] worker exec failed to START (rc=\$_wrc, attempt \$_wattempt/5) — container not ready; restarting + retrying\" - docker restart $CONTAINER >/dev/null 2>&1 || true - for _w in \$(seq 1 30); do - [ \"\$(docker inspect -f '{{.State.Running}}' $CONTAINER 2>/dev/null)\" = \"true\" ] \ - && docker exec $CONTAINER true >/dev/null 2>&1 && break - sleep 2 - done - sleep 3 - continue - fi - exit \$_wrc - done - " 2>&1 | tee -a "$LOG" - rc=${PIPESTATUS[0]} - echo "[$(date)] launch_slurm/orchestrate finished rc=$rc" | tee -a "$LOG" - exit $rc -} - -# ============================================================================= -# PHASE: provision (compute-node host) -# ============================================================================= -provision() { - export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${PATH:-}" - DOCKER=$(command -v docker 2>/dev/null || true); DOCKER=${DOCKER:-/usr/bin/docker} - FBGEMM_WHL=${FBGEMM_WHL:-/apps/chcai/FBGEMM/fbgemm_gpu/dist/fbgemm_gpu_nightly_rocm-2026.6.2-cp312-cp312-linux_x86_64.whl} # [CLUSTER-SPECIFIC] gfx950/ROCm wheel - TORCH_IDX=${TORCH_IDX:-https://download.pytorch.org/whl/rocm7.2} # [CLUSTER-SPECIFIC] ROCm version index - echo "[provision] host=$(hostname) container=$CONTAINER docker=$DOCKER" - - # Resolve which image to run + whether deps must be installed. Prefer a pre-baked - # image (deps already installed) to skip the multi-GB torch download + pip / - # torchrec-from-git build on every fresh node: - # 1) baked image in this node's docker -> use it, skip deps - # 2) baked image tar on NFS -> docker load (local, no internet) - # 3) neither -> base image + pip (slow path, which - # can then be baked via BAKE_IMAGE=1) - NEED_DEPS=1 - RUN_IMAGE="$IMAGE" - if [ "$USE_BAKED" = "1" ]; then - if "$DOCKER" image inspect "$BAKED_IMAGE" >/dev/null 2>&1; then - echo "[provision] using baked image $BAKED_IMAGE (deps preinstalled, no download)" - RUN_IMAGE="$BAKED_IMAGE"; NEED_DEPS=0 - elif [ -f "$BAKED_TAR" ]; then - echo "[provision] loading baked image from $BAKED_TAR (local, no internet)..." - if "$DOCKER" load -i "$BAKED_TAR" >/dev/null 2>&1 && "$DOCKER" image inspect "$BAKED_IMAGE" >/dev/null 2>&1; then - RUN_IMAGE="$BAKED_IMAGE"; NEED_DEPS=0; echo "[provision] baked image loaded" - else - echo "[provision] WARNING: docker load failed; falling back to base-image + pip" - fi - fi - fi - if ! "$DOCKER" image inspect "$RUN_IMAGE" >/dev/null 2>&1; then - echo "[provision] pulling $RUN_IMAGE (this can take a while)..."; "$DOCKER" pull "$RUN_IMAGE" - fi - - echo "[provision] (re)starting container $CONTAINER from $RUN_IMAGE" - "$DOCKER" rm -f "$CONTAINER" >/dev/null 2>&1 || true - "$DOCKER" run -d --name "$CONTAINER" \ - --network=host --ipc=host --shm-size=64g \ - --device=/dev/kfd --device=/dev/dri --group-add video \ - `# [CLUSTER-SPECIFIC] AMD ROCm device passthrough; NVIDIA uses --gpus all / nvidia runtime` \ - --cap-add=SYS_PTRACE --cap-add=CAP_SYS_ADMIN --cap-add=IPC_LOCK \ - --ulimit memlock=-1:-1 --ulimit stack=67108864:67108864 \ - `# memlock=-1 is REQUIRED for RDMA QP memory registration — do not drop` \ - --security-opt seccomp=unconfined --privileged \ - -v "$REPO_MOUNT:$REPO_MOUNT" \ - -v "$DATA_MOUNT:$DATA_MOUNT" \ - `# shared-NFS bind mounts: repo home (REPO_MOUNT, rw) + dataset/build assets (DATA_MOUNT)` \ - -w "$REPO" \ - "$RUN_IMAGE" sleep infinity - - # --- RDMA userspace overlay for in-container RCCL (bnxt_re) ----------------- - # The image (rocm/primus, rdma-core 50/libbnxt_re-rdmav34) ships an OLDER RDMA - # userspace than the host kernel bnxt_re driver. The stock v34 provider faults - # RCCL's deep-queue create_qp (max_send_wr=256) against the newer kernel uapi - # -> "ibv_create_qp ... Bad address". Fix: stage the host's matched rdma-core - # (libibverbs v61 + libbnxt_re-rdmav59 + libnl) on NFS so the worker phase makes - # RCCL load it via LD_PRELOAD + LD_LIBRARY_PATH. The UNVERSIONED libibverbs.so - # symlink is essential (import torch pulls the unversioned soname; without it - # the lookup falls through to the container v34 lib and the fix regresses). - if [ "${FORCE_OVERLAY:-0}" != "1" ] && ls "$OVERLAY/lib/libibverbs/"libbnxt_re-rdmav*.so >/dev/null 2>&1 && [ -L "$OVERLAY/lib/libibverbs.so" ]; then - echo "[provision] host RDMA overlay already staged at $OVERLAY (shared NFS) — skipping" - else - echo "[provision] staging host RDMA userspace overlay -> $OVERLAY" - rm -rf "${OVERLAY}.tmp" 2>/dev/null - mkdir -p "${OVERLAY}.tmp/lib/libibverbs" "${OVERLAY}.tmp/libibverbs.d" - cp -L /usr/lib64/libibverbs.so.1 /usr/lib64/libnl-3.so.200 /usr/lib64/libnl-route-3.so.200 "${OVERLAY}.tmp/lib/" 2>/dev/null || true - ln -sf libibverbs.so.1 "${OVERLAY}.tmp/lib/libibverbs.so" - cp -L /usr/lib64/libibverbs/*.so "${OVERLAY}.tmp/lib/libibverbs/" 2>/dev/null || true - cp /etc/libibverbs.d/*.driver "${OVERLAY}.tmp/libibverbs.d/" 2>/dev/null || true - if ls "${OVERLAY}.tmp/lib/libibverbs/"libbnxt_re-rdmav*.so >/dev/null 2>&1; then - rm -rf "$OVERLAY" 2>/dev/null - mv "${OVERLAY}.tmp" "$OVERLAY" 2>/dev/null || { mkdir -p "$OVERLAY"; cp -a "${OVERLAY}.tmp/." "$OVERLAY/"; } - echo "[provision] host RDMA overlay staged: $(ls "$OVERLAY/lib/libibverbs" | wc -l) providers + libibverbs.so symlink" - else - echo "[provision] WARNING: host bnxt_re provider not found at /usr/lib64/libibverbs — multi-node RDMA will fail 'Bad address'; use NCCL_NET_TRANSPORT=socket" - fi - fi - - if [ "$NEED_DEPS" = "0" ]; then - echo "[provision] baked image — deps preinstalled; verifying imports only" - "$DOCKER" exec "$CONTAINER" bash -lc ' -python -c "import torch, fbgemm_gpu, torchrec, polars, xxhash, gin; print(\"imports OK,\", torch.__version__, torch.version.hip, torch.cuda.device_count(), \"gpus\")" -' || echo "[provision] WARNING: baked-image import smoke failed" - else - echo "[provision] installing recipe deps (base image, slow path)" - # Install misc deps FIRST, then pin the rocm torch stack + fbgemm + torchrec - # LAST with --no-deps so nothing pulls a CUDA torch over the rocm build. - "$DOCKER" exec "$CONTAINER" bash -lc ' -set -e -echo "=== native torch ==="; python -c "import torch;print(torch.__version__)" || true -echo "=== misc python deps ===" -pip install --no-cache-dir polars-u64-idx pyarrow pyyaml tqdm psutil numba xxhash gin-config \ - absl-py pandas tensorboard torchmetrics tensordict pyre-extensions iopath typing-inspect 2>&1 | tail -3 || true -echo "=== rocm torch stack (force, no-deps, LAST) ===" -pip install --force-reinstall --no-deps --index-url '"$TORCH_IDX"' \ - torch==2.12.0+rocm7.2 torchvision==0.27.0+rocm7.2 torchaudio==2.11.0+rocm7.2 -echo "=== fbgemm (local gfx950 wheel) ===" -pip install --force-reinstall --no-deps '"$FBGEMM_WHL"' -echo "=== torchrec v2026.06.01.00 (force, no-deps) ===" -pip install --force-reinstall --no-deps "git+https://github.com/pytorch/torchrec.git@v2026.06.01.00" -echo "=== import smoke ===" -python -c "import torch, fbgemm_gpu, torchrec, polars, xxhash, gin; print(\"imports OK,\", torch.__version__, torch.version.hip, torch.cuda.device_count(), \"gpus\")" -' - fi - - # --- one-time bake: snapshot the deps-installed container into a reusable image - # and save it to NFS so future nodes skip the download/pip path entirely. - if [ "${BAKE_IMAGE:-0}" = "1" ]; then - echo "[provision] baking: docker commit $CONTAINER -> $BAKED_IMAGE" - if "$DOCKER" commit "$CONTAINER" "$BAKED_IMAGE" >/dev/null; then - echo "[provision] saving $BAKED_IMAGE -> $BAKED_TAR (one-time, tens of GB)" - if "$DOCKER" save "$BAKED_IMAGE" -o "${BAKED_TAR}.tmp.$$" && mv -f "${BAKED_TAR}.tmp.$$" "$BAKED_TAR"; then - echo "[provision] bake done: $(ls -lh "$BAKED_TAR" 2>/dev/null | awk '{print $5}')" - else - echo "[provision] WARNING: docker save failed"; rm -f "${BAKED_TAR}.tmp.$$" 2>/dev/null - fi - else - echo "[provision] WARNING: docker commit failed" - fi - fi - echo "[provision] DONE" -} - -# ============================================================================= -# PHASE: worker (inside the container) -# ============================================================================= -worker() { - cd "$REPO_ROOT" - mkdir -p "$SCRATCH" 2>/dev/null || true - LOG=${LOG:-$SCRATCH/yambda_5b_8gpu.log} - # WORKER_TEE=0 (set by orchestrate) sends our file sink to /dev/null to avoid - # double-logging, since orchestrate already tees stdout into the real $LOG. - [ "${WORKER_TEE:-1}" = "0" ] && LOG=/dev/null - export TENSORBOARD_LOG_PATH=${TENSORBOARD_LOG_PATH:-$SCRATCH/tb/yambda_5b} - # MLPerf compliance log (rank 0 writes it). Per-job filename so each standalone - # sbatch gets a clean log; the e2e supervisor pins MLPERF_LOG_PATH itself. - export MLPERF_LOG_PATH=${MLPERF_LOG_PATH:-$SCRATCH/mlperf/yambda_5b_mlperf.${SLURM_JOB_ID:-manual}.log} - echo "[$(date)] REPO_ROOT=$REPO_ROOT" | tee -a "$LOG" - - # polars-u64-idx (NOT stock polars) — yambda parquet's flat-explode overruns - # 32-bit row index. Reserved node has no outbound DNS, so install from a - # pre-staged tarball under /apps/chcai/. Override PIP_LOCAL_TGZ for other hosts. - PIP_LOCAL_TGZ=${PIP_LOCAL_TGZ:-/apps/chcai/pip_local_yambda.tgz} # [CLUSTER-SPECIFIC] shared-NFS path - PIP_LOCAL_DIR=${PIP_LOCAL_DIR:-/tmp/pip_local} - if [ ! -f "$PIP_LOCAL_DIR/lib/python3.12/site-packages/polars/__init__.py" ]; then - rm -rf "$PIP_LOCAL_DIR" - mkdir -p "$PIP_LOCAL_DIR" && tar xzf "$PIP_LOCAL_TGZ" -C "$(dirname "$PIP_LOCAL_DIR")" 2>&1 | tail -3 | tee -a "$LOG" - fi - - export PYTHONPATH="$PIP_LOCAL_DIR/lib/python3.12/site-packages:$REPO_ROOT:${PYTHONPATH:-}" - export HOME=${HOME:-/tmp} - echo "[$(date)] PYTHONPATH=$PYTHONPATH" | tee -a "$LOG" - python -c "import torch, fbgemm_gpu, torchrec, polars, xxhash, gin; print('imports OK,', torch.__version__, torch.cuda.device_count(),'gpus')" 2>&1 | tee -a "$LOG" - - export HIP_VISIBLE_DEVICES=${HIP_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7} - export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7} - - # --- distributed topology --------------------------------------------------- - GPUS_PER_NODE=$(python -c "import torch; print(torch.cuda.device_count())") - # Multi-node when launched one-task-per-node under SLURM (SLURM_NNODES>1); - # otherwise fall through to legacy single-node defaults (localhost, node_rank 0). - if [ "${SLURM_NNODES:-1}" -gt 1 ] && [ -n "${SLURM_JOB_NODELIST:-}" ]; then - NNODES=${SLURM_NNODES} - NODE_RANK=${SLURM_NODEID:-${SLURM_PROCID:-0}} - # PREFER a MASTER_ADDR/PORT forwarded from the orchestrate phase (resolved on - # the host, which has scontrol); the container image carries no SLURM client. - if [ -z "${MASTER_ADDR:-}" ]; then - MASTER_ADDR=$(scontrol show hostnames "$SLURM_JOB_NODELIST" 2>/dev/null | head -1) - MASTER_ADDR=${MASTER_ADDR:-localhost} - fi - MASTER_PORT=${MASTER_PORT:-$(( 20000 + ${SLURM_JOB_ID:-0} % 20000 ))} - else - NNODES=${NNODES:-1} - NODE_RANK=${NODE_RANK:-0} - # Single-node: all ranks live on THIS host, so rendezvous over loopback and - # do NOT use the SLURM hostname. On some nodes the hostname resolves to a - # non-routable per-GPU RoCE /31 (benic 192.168.x) address; using it makes the - # NCCL bootstrap fail with "No route to host". localhost is node-independent. - MASTER_ADDR=localhost - MASTER_PORT=${MASTER_PORT:-} # empty => train_ranker picks a free port - fi - export NNODES NODE_RANK GPUS_PER_NODE MASTER_ADDR MASTER_PORT - export WORLD_SIZE=$(( NNODES * GPUS_PER_NODE )) - echo "[$(date)] topology: nnodes=$NNODES node_rank=$NODE_RANK gpus_per_node=$GPUS_PER_NODE world_size=$WORLD_SIZE master=$MASTER_ADDR:${MASTER_PORT:-}" | tee -a "$LOG" - - # NCCL bootstrap NIC: loopback single-node, routable host NIC multi-node (pin - # to avoid auto-detect picking a non-routable per-GPU RoCE link). Override via - # $NCCL_SOCKET_IFNAME. [CLUSTER-SPECIFIC] multi-node fenic0 (find via `ip -br addr`). - if [ "$NNODES" -gt 1 ]; then - export NCCL_SOCKET_IFNAME=${NCCL_SOCKET_IFNAME:-fenic0} - else - export NCCL_SOCKET_IFNAME=${NCCL_SOCKET_IFNAME:-lo} - fi - echo "[$(date)] NCCL_SOCKET_IFNAME=$NCCL_SOCKET_IFNAME (nnodes=$NNODES)" | tee -a "$LOG" - - # Multi-node additionally needs the RDMA data-plane (bnxt_re HCAs) configured; - # single-node uses intra-node P2P (XGMI/PCIe) so only the bootstrap NIC matters. - if [ "$NNODES" -gt 1 ]; then - NCCL_NET_TRANSPORT=${NCCL_NET_TRANSPORT:-ib} - if [ "$NCCL_NET_TRANSPORT" = "socket" ]; then - export NCCL_IB_DISABLE=1 - echo "[$(date)] NCCL: IB disabled — allreduce over TCP (fenic0). Functional, not RDMA-fast." | tee -a "$LOG" - else - # bnxt_re userspace provider ABI overlay (REQUIRED for RCCL). The stock v34 - # provider faults RCCL's create_qp (256 WRs) against the host kernel uapi - # ("Bad address"); the host v61/v59 set staged by the provision phase works. - # The libibverbs.so (UNVERSIONED) symlink + LD_PRELOAD are both required so - # the torch process maps ONLY the host lib (see provision phase comment). - if [ -e "$OVERLAY/lib/libibverbs.so.1" ]; then - [ -e "$OVERLAY/lib/libibverbs.so" ] || ln -sf libibverbs.so.1 "$OVERLAY/lib/libibverbs.so" 2>/dev/null || true - export LD_LIBRARY_PATH="$OVERLAY/lib:$OVERLAY/lib/libibverbs:${LD_LIBRARY_PATH:-}" - export LD_PRELOAD="$OVERLAY/lib/libibverbs.so.1${LD_PRELOAD:+:$LD_PRELOAD}" - echo "[$(date)] NCCL: bnxt_re provider overlay -> $OVERLAY (host rdma-core v61/v59; symlink+LD_PRELOAD so RCCL binds the host lib for QP creation)" | tee -a "$LOG" - else - echo "[$(date)] WARNING: RDMA overlay $OVERLAY missing — RCCL QP creation will fail 'Bad address' on stock v34 provider; set RDMA_OVERLAY or use NCCL_NET_TRANSPORT=socket" | tee -a "$LOG" - fi - # MINIMAL bnxt_re set PROVEN on these meta64 cv350 nodes (cmcknigh RCCL - # benchmarks + confirmed e2e here). NCCL_IB_TC=104 (RoCE lossless PFC class) - # is required; do NOT add the ionic-AINIC QPS/ECE/DMABUF block. - # [CLUSTER-SPECIFIC] RDMA HCA names (`ibv_devices`); other vendors => mlx5_*/ionic_* - export NCCL_IB_HCA=${NCCL_IB_HCA:-bnxt_re0,bnxt_re1,bnxt_re2,bnxt_re3,bnxt_re4,bnxt_re5,bnxt_re6,bnxt_re7} - export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-3} # [CLUSTER-SPECIFIC] RoCEv2 IPv4 GID idx (`show_gids`) - export NCCL_IB_TC=${NCCL_IB_TC:-104} # [CLUSTER-SPECIFIC] RoCE lossless/PFC traffic class - export NCCL_IB_TIMEOUT=${NCCL_IB_TIMEOUT:-14} - export NCCL_IGNORE_CPU_AFFINITY=${NCCL_IGNORE_CPU_AFFINITY:-1} - export RCCL_MSCCL_ENABLE=${RCCL_MSCCL_ENABLE:-0} - # GPU-Direct RDMA on by default (~+22% throughput at 2 nodes via peermem). - # Set NCCL_NET_GDR_LEVEL=0 to force the legacy host-staged path. - export NCCL_NET_GDR_LEVEL=${NCCL_NET_GDR_LEVEL:-5} - export NCCL_DMABUF_ENABLE=${NCCL_DMABUF_ENABLE:-1} - echo "[$(date)] NCCL: RDMA over bnxt_re (GID idx ${NCCL_IB_GID_INDEX}, TC ${NCCL_IB_TC}, GDR_LEVEL=${NCCL_NET_GDR_LEVEL}, DMABUF=${NCCL_DMABUF_ENABLE}; meta64 bnxt_re config, validated)" | tee -a "$LOG" - fi - fi - export NCCL_DEBUG=${NCCL_DEBUG:-WARN} - export HSTU_HAMMER_KERNEL=${HSTU_HAMMER_KERNEL:-} - export PYTORCH_CUDA_ALLOC_CONF=${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True} - - # --- GPU clock sanity guard ------------------------------------------------- - # A leftover perf_determinism cap (half clock) silently slows every kernel ~1.9x. - # Log the perf level + a live sclk sample and try to restore boost (non-fatal). - if command -v rocm-smi >/dev/null 2>&1; then - echo "[$(date)] GPU perf-level check:" | tee -a "$LOG" - rocm-smi --showperflevel 2>/dev/null | grep -iE "GPU\[[0-9]+\]" | tee -a "$LOG" || true - if rocm-smi --showperflevel 2>/dev/null | grep -iqE "Performance Level: *(perf_determinism|manual|low)"; then - echo "[$(date)] WARNING: GPUs not in 'auto' perf level — attempting --setperflevel auto" | tee -a "$LOG" - rocm-smi --setperflevel auto 2>/dev/null | grep -iE "set to auto" | tee -a "$LOG" \ - || echo "[$(date)] WARNING: could not set perf level (no permission?). Run 'rocm-smi --setperflevel auto' on the HOST before benchmarking — clocks may be capped." | tee -a "$LOG" - fi - echo "[$(date)] sclk sample (GPU0):$(rocm-smi -d 0 --showclocks 2>/dev/null | grep -i 'sclk clock level' | sed -E 's/.*sclk clock level//')" | tee -a "$LOG" || true - fi - - # --- stray-trainer / leaked-VRAM guard ------------------------------------- - # The trainer runs via `docker exec` into a long-lived container, so its procs - # live in the container PID namespace, NOT the SLURM job cgroup. If a prior job - # OOM'd/crashed, a rank can leak and keep holding ~half of every GPU's VRAM, - # which persists across jobs (container survives) and guarantees the next - # attempt OOMs. Before launching, reap any pre-existing trainer procs (there - # should be none at this point) and wait for VRAM to drain. [g]-guard avoids - # self-match. Non-fatal. - if pgrep -f '[g]enerative_recommenders' >/dev/null 2>&1; then - echo "[$(date)] WARNING: leaked trainer procs found pre-launch — killing." | tee -a "$LOG" - pkill -9 -f '[g]enerative_recommenders' 2>/dev/null || true - for _i in $(seq 1 15); do - pgrep -f '[g]enerative_recommenders' >/dev/null 2>&1 || break - sleep 2 - done - sleep 5 # let the driver release VRAM after process exit - if command -v rocm-smi >/dev/null 2>&1; then - echo "[$(date)] post-cleanup GPU0 used GiB:$(rocm-smi --showmeminfo vram 2>/dev/null | awk -F: '/Used/{printf " %.0f", $3/1073741824; exit}')" | tee -a "$LOG" - fi - fi - - # WORKER_CMD override: run an arbitrary in-container command (e.g. an a2a/RCCL - # micro-benchmark) instead of the trainer, REUSING all the NCCL/RDMA/topology - # setup above so it exercises the exact transport the trainer uses. The - # supervisor never sets WORKER_CMD, so the training path is unchanged. - if [ -n "${WORKER_CMD:-}" ]; then - echo "[$(date)] WORKER_CMD override (WORLD_SIZE=$WORLD_SIZE): $WORKER_CMD" | tee -a "$LOG" - bash -lc "cd $REPO_ROOT && $WORKER_CMD" 2>&1 | tee -a "$LOG" - return - fi - - echo "[$(date)] launching train_ranker with WORLD_SIZE=$WORLD_SIZE" | tee -a "$LOG" - python -m generative_recommenders.dlrm_v3.train.train_ranker \ - --dataset yambda-5b --mode "${MODE:-streaming-train-eval}" 2>&1 | tee -a "$LOG" -} - -# ---- dispatch --------------------------------------------------------------- -case "$PHASE" in - orchestrate) orchestrate ;; - provision) provision ;; - worker) worker ;; - *) echo "launch_slurm.sh: unknown LAUNCH_SLURM_PHASE='$PHASE'" >&2; exit 2 ;; -esac From 1dae61d687e366b467b1dec38d2db76ac6500242 Mon Sep 17 00:00:00 2001 From: suachong Date: Wed, 24 Jun 2026 22:57:52 +0000 Subject: [PATCH 096/127] dlrmv4: revert streaming_resume_test.sh to base (out of MLPerf PR scope) Co-authored-by: Cursor --- .../dlrm_v3/train/tests/streaming_resume_test.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh b/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh index 47c451696..e14e557e8 100755 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh @@ -40,10 +40,9 @@ KEEP=0 # correctness gates are the functional-invariant checks below (RNG restored, # resumed-at-correct-step, atomic/keep_last_n), not this number. ATOL=0.15 -SCRATCH=${SCRATCH:-$HOME/yambda_runs} -CKPT_ROOT=${CKPT_ROOT:-$SCRATCH/ckpts_resume_test} -LOG_DIR=${LOG_DIR:-$SCRATCH/streaming_resume_test} -REPO=${REPO:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)} +CKPT_ROOT=/apps/chcai/ckpts_resume_test +LOG_DIR=/apps/chcai/streaming_resume_test +REPO=/home/chcai/training/recommendation_v4 while [[ $# -gt 0 ]]; do case $1 in From 7ec6fcc565c7f589f7ccf2e3e7559782dde41a24 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Thu, 25 Jun 2026 00:48:23 +0000 Subject: [PATCH 097/127] dlrmv4: make MLPerf run markers resume-aware via checkpoint state Persist mlperf_run_started in the checkpoint so a resume relaunch continues the SAME MLPerf run instead of re-emitting INIT_START/RUN_START (compliance requires EXACTLY_ONE). Cold-vs-resume is detected from the on-disk checkpoint before setup(); the log is truncated on a cold start and appended on a resume so the single run's event stream accumulates into one file. Legacy/cold checkpoints default the flag to False. Co-authored-by: Cursor --- .../dlrm_v3/checkpoint.py | 9 +++++ .../dlrm_v3/train/mlperf_logging_utils.py | 35 ++++++++++++++++ .../dlrm_v3/train/train_ranker.py | 40 +++++++++++++------ .../generative_recommenders/dlrm_v3/utils.py | 6 +++ 4 files changed, 77 insertions(+), 13 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/checkpoint.py b/recommendation_v4/generative_recommenders/dlrm_v3/checkpoint.py index 1d7f7f391..46cc10e2e 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/checkpoint.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/checkpoint.py @@ -411,6 +411,9 @@ def save_dmp_checkpoint( # load so pre-existing checkpoints restore as 0 and resume the # count from there. "cumulative_train_samples": metric_logger.cumulative_train_samples, + # MLPerf run-marker state: lets a resume relaunch continue the + # SAME run's event stream without re-emitting INIT_START/RUN_START. + "mlperf_run_started": metric_logger.mlperf_run_started, "sparse_tensor_keys": sparse_tensor_keys, # Streaming resume fields. Defaulted on load so old checkpoints # (pre-streaming-resume) still load as a normal restart. @@ -549,6 +552,12 @@ def load_nonsparse_checkpoint( metric_logger.cumulative_train_samples = non_sparse_state_dict.get( "cumulative_train_samples", 0 ) + # Defaulted False for legacy/cold checkpoints: a resume that loads a + # checkpoint where the run was already open continues without re-emitting + # the run markers. + metric_logger.mlperf_run_started = non_sparse_state_dict.get( + "mlperf_run_started", False + ) class_metric_state_dict = non_sparse_state_dict["class_metrics"] regression_metric_state_dict = non_sparse_state_dict["reg_metrics"] # Length-safe positional restore: if a checkpoint was written with a diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py index e190f0325..8a716f87b 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py @@ -71,6 +71,7 @@ def __init__( benchmark_name: str = "hstu", submitter_name: str = "AMD", submission_platform: str = "MI355X", + fresh: bool = True, ): self.enabled: bool = _MLLOG_AVAILABLE # Use the EXPLICIT caller rank: this is built before init_process_group, @@ -87,6 +88,14 @@ def __init__( log_dir = os.path.dirname(log_path) if log_dir: # guard: os.makedirs("") raises for a bare filename os.makedirs(log_dir, exist_ok=True) + # mllog's FileHandler APPENDS (mode "a"), which is what a resume needs + # so the single run's event stream accumulates across relaunches into + # one file. On a genuine cold start, truncate first so a re-used run + # dir / a previous crashed-cold-start's orphaned stream can't leave a + # second run_start in the file (the compliance checker requires + # EXACTLY_ONE). Resume (fresh=False) appends to continue the stream. + if fresh: + open(log_path, "w").close() mllog.config(filename=log_path, default_stack_offset=default_stack_offset) else: mllog.config(default_stack_offset=default_stack_offset) @@ -339,6 +348,30 @@ def finalize(self, final_metrics: Dict[str, float]) -> None: self.run_stop(c.SUCCESS if success else c.ABORTED) +def mlperf_checkpoint_present(ckpt_path: str) -> bool: + """True iff ``ckpt_path`` resolves to an existing checkpoint (i.e. a resume). + + A dependency-light mirror of ``checkpoint._resolve_latest_subdir`` so + ``train_ranker`` can decide cold-start vs resume BEFORE the heavy checkpoint + import + ``setup()``. This gates the one-time INIT_START/RUN_START markers: + emit them on a genuine cold start only, and never re-emit on a resume + relaunch (the MLPerf run spans the resume). Matches the loader's resolution: + empty path or a base dir with no numeric subdirs => cold start. + """ + if not ckpt_path: + return False + base = ckpt_path.rstrip("/") + # A leaf save (numeric basename) is a resume iff that dir actually exists. + if os.path.basename(base).isdigit(): + return os.path.isdir(base) + if not os.path.isdir(base): + return False + for name in os.listdir(base): + if name.isdigit() and os.path.isdir(os.path.join(base, name)): + return True + return False + + @gin.configurable def get_mlperf_logger( rank: int = 0, @@ -346,6 +379,7 @@ def get_mlperf_logger( benchmark_name: str = "hstu", submitter_name: str = "AMD", submission_platform: str = "MI355X", + fresh: bool = True, ) -> Optional[MLPerfLogger]: """Build a configured :class:`MLPerfLogger`, or ``None`` if unavailable. @@ -375,4 +409,5 @@ def get_mlperf_logger( benchmark_name=benchmark_name, submitter_name=submitter_name, submission_platform=resolved_platform, + fresh=fresh, ) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py index 50d359ef6..d5797697b 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py @@ -78,22 +78,31 @@ def _main_func( from generative_recommenders.dlrm_v3.train._env_bootstrap import apply_env_bootstrap from generative_recommenders.dlrm_v3.train.mlperf_logging_utils import ( get_mlperf_logger, + mlperf_checkpoint_present, ) gin.parse_config_file(gin_file, skip_unknown=True) apply_env_bootstrap() - # Rank-0-gated MLPerf logger, only for the streaming-train-eval path. + # Cold-start vs resume, decided from the on-disk checkpoint BEFORE setup so + # the one-time INIT/RUN markers fire on a genuine cold start only and are NOT + # re-emitted on a resume relaunch — the MLPerf run (run_start..run_stop) spans + # the resume as a single coherent event stream in one appended log file. + mlperf_resume = mlperf_checkpoint_present(os.environ.get("CKPT_PATH", "")) + # Rank-0-gated MLPerf logger, only for the streaming-train-eval path. `fresh` + # truncates the log on cold start (one run_start per file) but appends on a + # resume so the pre-crash events are preserved and continued. mlperf_logger = ( - get_mlperf_logger(rank=rank) if mode == "streaming-train-eval" else None + get_mlperf_logger(rank=rank, fresh=not mlperf_resume) + if mode == "streaming-train-eval" + else None ) - # Emit INIT_START before setup only on a guaranteed cold start (CKPT_PATH - # unset); resume relaunches skip it so the log stays balanced. - mlperf_init_logged = False - if mlperf_logger is not None and not os.environ.get("CKPT_PATH", ""): + # INIT_START fires before setup on a cold start only (resume continues the + # already-open run, whose markers were emitted by the original process). + mlperf_cold_start = mlperf_logger is not None and not mlperf_resume + if mlperf_cold_start: mlperf_logger.event(key=mlperf_logger.constants.CACHE_CLEAR, value=True) mlperf_logger.start(key=mlperf_logger.constants.INIT_START) - mlperf_init_logged = True # Phase 2: heavy imports. Triton kernel modules evaluate their autotune # decorators here, using the env vars set above. @@ -200,12 +209,13 @@ def _main_func( ) ) - # MLPerf submission info + hyperparameters + INIT_STOP/RUN_START, only on a - # genuine cold start so resume relaunches don't reopen the run markers. - mlperf_run_active = ( - mlperf_logger is not None and mlperf_init_logged and resume_cold_start - ) - if mlperf_run_active: + # MLPerf run markers: open the run exactly once. On a cold start emit + # submission info + hyperparameters + INIT_STOP/RUN_START and mark the run as + # started (persisted in the checkpoint via metrics.mlperf_run_started). On a + # resume, load_dmp_checkpoint restored mlperf_run_started=True, so we skip the + # markers and just continue the stream. `metrics.mlperf_run_started` guards a + # double-emit even if cold/resume detection and the checkpoint ever disagree. + if mlperf_cold_start and not metrics.mlperf_run_started: # Submission info + hyperparameters + INIT_STOP/RUN_START, all emitted by # the logger (optimizer names/LRs read from gin internally). Seed is the # value setup() resolved and exported to $SEED. @@ -213,6 +223,10 @@ def _main_func( global_batch_size=world_size * int(train_dataloader.batch_size), seed=int(os.environ.get("SEED", "1")), ) + metrics.mlperf_run_started = True + # Pass the logger to the loop whenever MLPerf logging is enabled, so block / + # eval / train_loss / run_stop events emit on BOTH a cold start and a resume. + mlperf_run_active = mlperf_logger is not None and metrics.mlperf_run_started # train loop try: diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py index 281a37b5d..ed456bde6 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py @@ -1034,6 +1034,12 @@ def _make_reg(ws: int) -> List[RecMetricComputation]: # MLPerf `samples_count` progress unit: global trained samples, persisted # alongside global_step so a resumed run continues the count. self.cumulative_train_samples: int = 0 + # Whether the MLPerf run markers (RUN_START etc.) were already emitted for + # this logical run. Checkpointed so a resume relaunch knows the run is + # already open and does NOT re-emit INIT_START/RUN_START (the compliance + # checker requires EXACTLY_ONE); the resumed process continues the same + # event stream and emits the single RUN_STOP at convergence/end. + self.mlperf_run_started: bool = False self._rank: int = int(rank) # Optional MLPerf logger + LR accessor wired by the streaming loop (duck- # typed to avoid a train-module import cycle); drives the train_loss event. From 5c2b9405c7e6b52edcec50935794b120bba90f5c Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Thu, 25 Jun 2026 01:04:45 +0000 Subject: [PATCH 098/127] dlrmv4: reproducible-by-default config (seed=1, AUC_THRESHOLD=1.0) Default SEED back to 1 for a fixed, reproducible weight init out of the box ($SEED=-1 still draws a fresh random seed per run). Default AUC_THRESHOLD to 1.0 (unreachable) in both the gin binding and the launch_slurm.sh fallback so a streaming-train-eval run trains through all windows by default instead of early-stopping; set $AUC_THRESHOLD=0.80275 for the MLPerf convergence target. Co-authored-by: Cursor --- .../dlrm_v3/train/gin/yambda_5b.gin | 10 ++++++---- recommendation_v4/scripts/launch_slurm.sh | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index 8a3e5b595..dd645f06a 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -80,10 +80,11 @@ apply_env_bootstrap.TRITON_FULL_AUTOTUNE = False # PARSE NOTE: seed_everything() runs right before make_model() in train_ranker # (after the full gin parse), so this binding resolves in the second parse where # env_int is registered. Override per-run via $SEED. -# Default -1 draws a fresh random seed each run; pin $SEED >= 0 to reproduce. +# Default 1 gives a fixed, reproducible seed each run; override $SEED to vary it +# (set $SEED = -1 to draw a fresh random seed per run). seed_everything.seed = @seed/env_int() seed/env_int.key = "SEED" -seed/env_int.default = -1 +seed/env_int.default = 1 # $DECORRELATE_DROPOUT — re-seed torch/cuda with $SEED + rank after init so HSTU # dropout masks differ per data-parallel rank. 1 = on, 0 = identical masks (default). @@ -457,10 +458,11 @@ tbp/env_path.key = "TENSORBOARD_LOG_PATH" tbp/env_path.default = "" MetricsLogger.world_size = 8 # MLPerf convergence target: run stops when the selected eval AUC reaches it. -# Override via $AUC_THRESHOLD. +# Default 1.0 is unreachable, so the run trains through all windows (no early +# stop) out of the box; set $AUC_THRESHOLD=0.80275 for the MLPerf target. MetricsLogger.auc_threshold = @at/env_float() at/env_float.key = "AUC_THRESHOLD" -at/env_float.default = 0.80275 +at/env_float.default = 1.0 # EVAL_ACCURACY + early-stop are driven by per-window AUC (window_auc) vs the # threshold above. # Lifetime-AUC backend, selectable independently for the train cumulative AUC and diff --git a/recommendation_v4/scripts/launch_slurm.sh b/recommendation_v4/scripts/launch_slurm.sh index fb71bdcb7..74d7ca154 100755 --- a/recommendation_v4/scripts/launch_slurm.sh +++ b/recommendation_v4/scripts/launch_slurm.sh @@ -311,7 +311,7 @@ orchestrate() { ${IN_WINDOW_CKPT_FREQ:+-e IN_WINDOW_CKPT_FREQ=$IN_WINDOW_CKPT_FREQ} \ ${CKPT_STEP_FREQ:+-e CKPT_STEP_FREQ=$CKPT_STEP_FREQ} \ -e TRAIN_SPLIT_PERCENTAGE=${TRAIN_SPLIT_PERCENTAGE:-1.0} \ - -e AUC_THRESHOLD=${AUC_THRESHOLD:-0.80275} \ + -e AUC_THRESHOLD=${AUC_THRESHOLD:-1.0} \ ${MLPERF_SUBMISSION_PLATFORM:+-e MLPERF_SUBMISSION_PLATFORM=$MLPERF_SUBMISSION_PLATFORM} \ -e SPLIT_SALT=${SPLIT_SALT:-0} \ -e EVAL_HOLDOUT_TS=${EVAL_HOLDOUT_TS:--1} \ From ff55513f805b8d00b38933e7bd0ee48d9d8c93ca Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Thu, 25 Jun 2026 01:44:54 +0000 Subject: [PATCH 099/127] dlrmv4: re-enable GPUDirect RDMA by default in slurm worker Restore NCCL_NET_GDR_LEVEL=5 + NCCL_DMABUF_ENABLE=1 defaults so RCCL does true GPU<->NIC DMA over bnxt_re instead of host-memory staging (~+22% throughput at 2 nodes; 65.7%->79.8% weak-scaling efficiency). The brcmrdma host kernel ships the inbox peer-memory client, so GDR works with no container/host changes; non-fatal fallback to host staging if peermem is absent. Override with NCCL_NET_GDR_LEVEL=0. Co-authored-by: Cursor --- recommendation_v4/scripts/launch_slurm.sh | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/recommendation_v4/scripts/launch_slurm.sh b/recommendation_v4/scripts/launch_slurm.sh index 74d7ca154..a1b6334e3 100755 --- a/recommendation_v4/scripts/launch_slurm.sh +++ b/recommendation_v4/scripts/launch_slurm.sh @@ -571,10 +571,20 @@ worker() { export NCCL_IB_TIMEOUT=${NCCL_IB_TIMEOUT:-14} export NCCL_IGNORE_CPU_AFFINITY=${NCCL_IGNORE_CPU_AFFINITY:-1} export RCCL_MSCCL_ENABLE=${RCCL_MSCCL_ENABLE:-0} - # GPU-Direct RDMA needs DMABUF/peermem (neither in-container here) — leave - # GDR off so RCCL stages through host memory (still real RDMA over bnxt_re). - export NCCL_NET_GDR_LEVEL=${NCCL_NET_GDR_LEVEL:-0} - echo "[$(date)] NCCL: RDMA over bnxt_re (GID idx ${NCCL_IB_GID_INDEX}, TC ${NCCL_IB_TC}, GDR_LEVEL=${NCCL_NET_GDR_LEVEL}; meta64 bnxt_re config, validated)" | tee -a "$LOG" + # GPU-Direct RDMA: ENABLED by default. The brcmrdma host kernel ships the + # inbox peer-memory client (`ib_register_peer_memory_client` in + # /proc/kallsyms), so RCCL does true GPU<->NIC DMA over bnxt_re instead of + # bouncing through host memory. Measured ~+22% throughput at 2 nodes + # (65.7%->79.8% weak-scaling efficiency) vs the old host-staged path. + # GDR_LEVEL=5 (most permissive) is required so GDR is used even when the GPU + # and NIC cross the CPU root complex. NCCL_DMABUF_ENABLE=1 is a harmless + # no-op here (kernel lacks CONFIG_DMABUF_MOVE_NOTIFY/CONFIG_PCI_P2PDMA, so + # peermem carries it). Enabling is non-fatal: if peermem is ever absent RCCL + # just logs "GDR 0" and falls back to host staging. Override with + # NCCL_NET_GDR_LEVEL=0 to force the legacy host-staged path. + export NCCL_NET_GDR_LEVEL=${NCCL_NET_GDR_LEVEL:-5} + export NCCL_DMABUF_ENABLE=${NCCL_DMABUF_ENABLE:-1} + echo "[$(date)] NCCL: RDMA over bnxt_re (GID idx ${NCCL_IB_GID_INDEX}, TC ${NCCL_IB_TC}, GDR_LEVEL=${NCCL_NET_GDR_LEVEL}, DMABUF=${NCCL_DMABUF_ENABLE}; meta64 bnxt_re config, validated)" | tee -a "$LOG" fi fi export NCCL_DEBUG=${NCCL_DEBUG:-WARN} From 7b5e8690500e1f3b5ad88eccafdc23a0fadc6823 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Thu, 25 Jun 2026 01:52:05 +0000 Subject: [PATCH 100/127] =?UTF-8?q?dlrmv4:=20README=20=E2=80=94=20match=20?= =?UTF-8?q?launcher's=20actual=20smoke-shaped=20defaults?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The slimmed launch_slurm.sh has smoke-shaped run defaults (START_TS=150, NUM_TRAIN_TS=1, NUM_TRAIN_BATCHES=20, per-window eval) and no SMOKE=1 toggle, so a bare submit is a short functional run — not the 299-window reference. Document the bare submit as the smoke run and give the explicit env-override command for the full reference sweep; drop the unimplemented SMOKE=1 instructions. Co-authored-by: Cursor --- recommendation_v4/README.MD | 57 ++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/recommendation_v4/README.MD b/recommendation_v4/README.MD index 6476e7729..b472cb0a8 100644 --- a/recommendation_v4/README.MD +++ b/recommendation_v4/README.MD @@ -11,9 +11,10 @@ auto-detects its context: run inside the container it takes the single-node worker path; submitted via `sbatch` it orchestrates the multi-node run (provision + per-node launch). N=1 is byte-for-byte the legacy single-node path. -A bare submit reproduces the **frozen reference run** (full 299-window sweep + -data-fraction eval cadence) — all run-shape/cadence defaults are baked in, so no -env knobs are required: +A bare submit runs a **short functional smoke run** (a single capped window with +per-window eval) — the built-in defaults are intentionally small so a bare +submit validates the full provision + launch + train/eval path quickly without +consuming a whole window: **Single node (8-GPU):** @@ -27,11 +28,14 @@ sbatch --nodes=1 scripts/launch_slurm.sh sbatch --nodes=2 scripts/launch_slurm.sh ``` -For a fast functional check instead of a full run, prepend `SMOKE=1` (short -window, capped batches, per-window eval): +To run the **full reference sweep** instead, set the run-shape/cadence knobs +explicitly (full 299-window sweep + data-fraction eval cadence): ```bash -SMOKE=1 sbatch --nodes=1 scripts/launch_slurm.sh +START_TS=0 NUM_TRAIN_TS=299 \ +NUM_TRAIN_BATCHES=0 NUM_EVAL_BATCHES=0 \ +EVAL_EVERY_N_WINDOWS=0 EVAL_EVERY_DATA_PCT=0.005 \ +sbatch --nodes=1 scripts/launch_slurm.sh ``` Multi-node uses real RDMA (RoCEv2). The fabric/NCCL setup and every @@ -49,26 +53,27 @@ bash scripts/launch_slurm.sh Data path resolves at runtime via `env_path` gin macros (see [yambda_5b.gin](generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin)). Traces and any per-run outputs land in `results//`. -### 1.1 The frozen reference shape - -The reference run-shape and eval cadence are the built-in defaults (set in the -orchestrate phase of `scripts/launch_slurm.sh`), so the bare `sbatch` commands -above ARE the reference run. The single- and multi-node launchers are identical -except for `--nodes`; the trainer auto-derives `NNODES`/`NODE_RANK`/`MASTER_ADDR`/ -`WORLD_SIZE` from SLURM. The baked-in shape: - -| knob | reference default | -|---|---| -| `START_TS` / `NUM_TRAIN_TS` | 0 / 299 (full sweep) | -| eval cadence | `EVAL_EVERY_DATA_PCT=0.005` (every 0.5% of the training stream — a fixed number of samples between evals, independent of node count), per-window cadence off | -| `NUM_TRAIN_BATCHES` / `NUM_EVAL_BATCHES` | 0 / 0 (consume full windows) | - -To customize, override any knob via env (e.g. `RUN_NAME=...`, `LOG=...`, -`AUC_THRESHOLD=...`). Selecting the per-window eval cadence -(`EVAL_EVERY_N_WINDOWS>0`) automatically disables the data-fraction one (they are -mutually exclusive). Keep all run outputs (`LOG`, checkpoints, mllog, -TensorBoard) under a writable scratch path you own — the dataset mount is -read-only. +### 1.1 Run shape: smoke default vs. full reference + +The run-shape and eval cadence come from env-overridable defaults set in the +orchestrate phase of `scripts/launch_slurm.sh`. The built-in defaults are +**smoke-shaped** (small, for a fast functional check); the **full reference +sweep** is the same launcher with the run-shape knobs overridden. The +single- and multi-node launchers are identical except for `--nodes`; the trainer +auto-derives `NNODES`/`NODE_RANK`/`MASTER_ADDR`/`WORLD_SIZE` from SLURM. + +| knob | smoke default (bare submit) | full reference (override) | +|---|---|---| +| `START_TS` / `NUM_TRAIN_TS` | 150 / 1 (one window) | 0 / 299 (full sweep) | +| `NUM_TRAIN_BATCHES` / `NUM_EVAL_BATCHES` | 20 / 10 (capped) | 0 / 0 (consume full windows) | +| eval cadence | `EVAL_EVERY_N_WINDOWS=1` (per-window) | `EVAL_EVERY_N_WINDOWS=0` + `EVAL_EVERY_DATA_PCT=0.005` (every 0.5% of the training stream — a fixed number of samples between evals, independent of node count) | + +Override any knob via env (e.g. `RUN_NAME=...`, `LOG=...`, `AUC_THRESHOLD=...`); +see the full-reference command in §1 above. The per-window and data-fraction eval +cadences are mutually exclusive — selecting one (`EVAL_EVERY_N_WINDOWS>0` or +`EVAL_EVERY_DATA_PCT>0`) requires the other be 0. Keep all run outputs (`LOG`, +checkpoints, mllog, TensorBoard) under a writable scratch path you own — the +dataset mount is read-only. ## 2. Data preparation From 881d92518d81474e7c37db90d864f08006ed7510 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Thu, 25 Jun 2026 02:37:19 +0000 Subject: [PATCH 101/127] recommendation_v4: add MLPerf reference scripts, structure, and docs Bring the HSTU/yambda-5b benchmark in line with MLPerf Training reference conventions: - add download_dataset.sh / verify_dataset.sh / run_and_time.sh wrappers - add md5sums checksum file (placeholder hashes) for dataset verification - restructure README.MD to the MLPerf spec (summaries, model+paper, hyperparameter table with tuning rules, quality target, eval frequency, steps-to-run) - freeze requirements.txt to the exact Dockerfile/training_recipe versions - add blank RCP placeholder (rcp/README.md) to be filled once convergence runs are generated Co-authored-by: Cursor --- recommendation_v4/README.MD | 472 +++++++++++------- recommendation_v4/download_dataset.sh | 39 ++ .../md5sums_yambda_5b_processed.txt | 22 + recommendation_v4/rcp/README.md | 28 ++ recommendation_v4/requirements.txt | 51 +- recommendation_v4/run_and_time.sh | 65 +++ recommendation_v4/verify_dataset.sh | 64 +++ 7 files changed, 548 insertions(+), 193 deletions(-) create mode 100755 recommendation_v4/download_dataset.sh create mode 100644 recommendation_v4/md5sums_yambda_5b_processed.txt create mode 100644 recommendation_v4/rcp/README.md create mode 100755 recommendation_v4/run_and_time.sh create mode 100755 recommendation_v4/verify_dataset.sh diff --git a/recommendation_v4/README.MD b/recommendation_v4/README.MD index b472cb0a8..e078bcf0a 100644 --- a/recommendation_v4/README.MD +++ b/recommendation_v4/README.MD @@ -1,115 +1,128 @@ -# Recommendation v4 — HSTU + Yambda-5b +# Recommendation v4 — HSTU sequential recommendation (Yambda-5b) + +MLPerf Training reference benchmark. This is a fork of +[meta-recsys/generative-recommenders](https://github.com/meta-recsys/generative-recommenders) +extended to train an HSTU (Hierarchical Sequential Transduction Units) ranking +model on the [Yambda-5b](https://huggingface.co/datasets/yandex/yambda) +music-recommendation dataset, sized as an MLPerf-style training benchmark inside +the `mlcommons/training` tree. + +## 1. Summary + +This benchmark trains a model that predicts what a person will listen to next. +Given the history of songs a user has played, liked, or skipped, the model +learns to rank which song the user is most likely to genuinely listen to (rather +than skip) next. This is the same kind of "what should we recommend next?" +problem that powers music and video streaming feeds. The model is trained on a +large public dataset of anonymized music-listening events and is scored on how +well it predicts future listens it has never seen. + +## 2. Benchmark overview (technical) + +The model is a **sequential recommender**: instead of treating each interaction +independently (as classic click-through-rate models like DLRM-DCNv2 do), it +consumes a user's chronologically ordered interaction history as a sequence and +applies a Transformer-style attention stack (HSTU) over it. Each training +example is one "anchor" listen event together with that user's prior history +(user interaction history, or UIH) and a set of contextual/cross features. The +supervised target is a binary `listen_plus` label (a real listen: played for at +least 50% of the track) versus a skip. + +Training is **streaming / temporal-order**: the timeline is sliced into +fixed-duration windows and the model trains on window `T` then evaluates on the +strictly-future window `T+1`, so every reported metric is genuine +next-period generalization with no future leakage. The quality metric is +**AUC** on the held-out future window, and the convergence target is +**AUC >= 0.80275** (matching the DLRM-DCNv2-style target). + +The reference runs on 8 GPUs (validated on AMD Instinct MI350X / MI355X and +NVIDIA B200; see [docs/training_recipe.md](docs/training_recipe.md)) and scales +to multi-node via SLURM. + +## 3. Directions — steps to run + +The benchmark follows the standard MLPerf reference script flow: -This is a fork of [meta-recsys/generative-recommenders](https://github.com/meta-recsys/generative-recommenders) extended to train HSTU (Hierarchical Sequential Transducer Units) on the [Yambda-5b](https://huggingface.co/datasets/yandex/yambda) music-recommendation dataset, sized as an MLPerf-style training benchmark inside the `mlcommons/training` tree. - -For the original repository and the underlying ICML'24 paper (*Actions Speak Louder than Words*), see the upstream README at the link above. This README focuses on what this fork adds: the Yambda data pipeline, the per-pool gather strategy, and how the data feeds into the HSTU `modules/` (dlrm_v3) path. - -## 1. Quick start (Yambda, N×8-GPU) - -`scripts/launch_slurm.sh` is the single entry point for **N ≥ 1 nodes**. It -auto-detects its context: run inside the container it takes the single-node -worker path; submitted via `sbatch` it orchestrates the multi-node run -(provision + per-node launch). N=1 is byte-for-byte the legacy single-node path. +```bash +# 0. build/enter the container (canonical frozen environment) +docker build -t recommendation_v4 . +docker run --rm -it --device=/dev/kfd --device=/dev/dri \ + -v /path/to/dlrm_data:/data/mlperf_dlrm_v4 recommendation_v4 -A bare submit runs a **short functional smoke run** (a single capped window with -per-window eval) — the built-in defaults are intentionally small so a bare -submit validates the full provision + launch + train/eval path quickly without -consuming a whole window: +# 1. download + preprocess the dataset +DLRM_DATA_PATH=/data/mlperf_dlrm_v4 ./download_dataset.sh -**Single node (8-GPU):** +# 2. verify the preprocessed dataset +DLRM_DATA_PATH=/data/mlperf_dlrm_v4 ./verify_dataset.sh -```bash -sbatch --nodes=1 scripts/launch_slurm.sh +# 3. run the benchmark to the quality target and report wall-clock time +DLRM_DATA_PATH=/data/mlperf_dlrm_v4 ./run_and_time.sh ``` -**Multi-node (N×8-GPU):** +- [`download_dataset.sh`](download_dataset.sh) wraps the preprocessing pipeline + in `generative_recommenders.dlrm_v3.preprocess_public_data` (HuggingFace + download + temporal split + session segmentation + item-popularity counts). +- [`verify_dataset.sh`](verify_dataset.sh) checks the preprocessed files against + [`md5sums_yambda_5b_processed.txt`](md5sums_yambda_5b_processed.txt) (falls + back to a layout check until the canonical checksums are pinned). +- [`run_and_time.sh`](run_and_time.sh) runs the full-reference streaming + train+eval sweep on a single 8-GPU host with `AUC_THRESHOLD=0.80275` and MLPerf + compliance logging, printing the elapsed time of the timed region. -```bash -sbatch --nodes=2 scripts/launch_slurm.sh -``` +### 3.1 Multi-node (SLURM) -To run the **full reference sweep** instead, set the run-shape/cadence knobs -explicitly (full 299-window sweep + data-fraction eval cadence): +For N >= 1 nodes use [`scripts/launch_slurm.sh`](scripts/launch_slurm.sh), which +provisions the container on each node and launches the same trainer. A bare +submit runs a small functional smoke run; set the run-shape knobs for the full +sweep: ```bash +# smoke (fast functional check) +sbatch --nodes=1 scripts/launch_slurm.sh + +# full reference sweep START_TS=0 NUM_TRAIN_TS=299 \ NUM_TRAIN_BATCHES=0 NUM_EVAL_BATCHES=0 \ EVAL_EVERY_N_WINDOWS=0 EVAL_EVERY_DATA_PCT=0.005 \ -sbatch --nodes=1 scripts/launch_slurm.sh -``` - -Multi-node uses real RDMA (RoCEv2). The fabric/NCCL setup and every -cluster-specific knob (interfaces, HCAs, GID/TC, RDMA overlay) are documented in -[docs/multi_node_config.md](docs/multi_node_config.md) — read it before running on -a different cluster. - -Override the data path or run name without editing the gin: - -```bash -DLRM_DATA_PATH=/apps/chcai/dlrm_data \ -RUN_NAME=my_experiment \ -bash scripts/launch_slurm.sh +AUC_THRESHOLD=0.80275 \ +sbatch --nodes=2 scripts/launch_slurm.sh ``` -Data path resolves at runtime via `env_path` gin macros (see [yambda_5b.gin](generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin)). Traces and any per-run outputs land in `results//`. - -### 1.1 Run shape: smoke default vs. full reference +Multi-node uses real RDMA (RoCEv2); the fabric/NCCL setup is documented in +[docs/multi_node_config.md](docs/multi_node_config.md). Keep all run outputs +(log, checkpoints, mllog, TensorBoard) under a writable scratch path you own — +the dataset mount is read-only. -The run-shape and eval cadence come from env-overridable defaults set in the -orchestrate phase of `scripts/launch_slurm.sh`. The built-in defaults are -**smoke-shaped** (small, for a fast functional check); the **full reference -sweep** is the same launcher with the run-shape knobs overridden. The -single- and multi-node launchers are identical except for `--nodes`; the trainer -auto-derives `NNODES`/`NODE_RANK`/`MASTER_ADDR`/`WORLD_SIZE` from SLURM. +## 4. Model -| knob | smoke default (bare submit) | full reference (override) | -|---|---|---| -| `START_TS` / `NUM_TRAIN_TS` | 150 / 1 (one window) | 0 / 299 (full sweep) | -| `NUM_TRAIN_BATCHES` / `NUM_EVAL_BATCHES` | 20 / 10 (capped) | 0 / 0 (consume full windows) | -| eval cadence | `EVAL_EVERY_N_WINDOWS=1` (per-window) | `EVAL_EVERY_N_WINDOWS=0` + `EVAL_EVERY_DATA_PCT=0.005` (every 0.5% of the training stream — a fixed number of samples between evals, independent of node count) | +The model is **HSTU** (Hierarchical Sequential Transduction Units), the +generative-recommender architecture from Meta's ICML'24 paper *Actions Speak +Louder than Words: Trillion-Parameter Sequential Transducers for Generative +Recommendations* ([arXiv:2402.17152](https://arxiv.org/abs/2402.17152)). -Override any knob via env (e.g. `RUN_NAME=...`, `LOG=...`, `AUC_THRESHOLD=...`); -see the full-reference command in §1 above. The per-window and data-fraction eval -cadences are mutually exclusive — selecting one (`EVAL_EVERY_N_WINDOWS>0` or -`EVAL_EVERY_DATA_PCT>0`) requires the other be 0. Keep all run outputs (`LOG`, -checkpoints, mllog, TensorBoard) under a writable scratch path you own — the -dataset mount is read-only. +HSTU replaces the feature-interaction stack of a classic DLRM with a stack of +pointwise-attention "transducer" layers operating over the user's interaction +sequence. In this benchmark (the `dlrm_v3` path): -## 2. Data preparation +- **Embeddings**: sparse tables for `item_id`, `artist_id`, `album_id`, `uid`, + and 7 cross-feature hashes (e.g. `user_x_artist`, `item_x_hour`), sharded + across GPUs with TorchRec `DistributedModelParallel`. +- **Sequence model**: an HSTU attention stack (`HSTU_NUM_LAYERS`, default 3) + over the interleaved UIH, computed with a fused jagged-attention Triton kernel + in bf16. +- **Supervision**: a single `listen_plus` binary task. The candidate event's + `action_weight` carries the supervision bit, and BCE loss is masked to + listen_plus candidates. -```bash -python3 -m generative_recommenders.dlrm_v3.preprocess_public_data \ - --dataset yambda-5b --data-path /apps/chcai/dlrm_data -``` +See `generative_recommenders/dlrm_v3/configs.py` +(`get_hstu_configs`, `get_embedding_table_config`) for the exact architecture +and table specs, and the upstream README for the original modeling code. -This downloads the 5b variant of [yandex/yambda](https://huggingface.co/datasets/yandex/yambda) from HuggingFace, then: +## 5. Dataset -1. **Encodes** the raw `event_type` string column into a uint8 lookup (listen=0, like=1, dislike=2, unlike=3, undislike=4). -2. **Splits** events temporally — 300 train days, 30-min gap, 1 test day — by `Global Temporal Split` (GTS). -3. **Segments** per-user event timelines into sessions on a 30-min inactivity gap. -4. **Computes** per-item popularity for downstream metric weighting. -5. **Writes** the layout `DLRMv3YambdaDataset` expects: - -``` -/ -├── raw/5b/multi_event.parquet 50 GB (downloaded) -├── shared_metadata/ -│ ├── artist_item_mapping.parquet 60 MB -│ ├── album_item_mapping.parquet 76 MB -│ └── embeddings.parquet 18 GB (unused by HSTU training) -└── processed_5b/ - ├── train_sessions.parquet 47 GB ← main training input - ├── test_events.parquet 152 MB - ├── session_index.parquet 600 MB - ├── item_popularity.npy 75 MB - └── split_meta.json anchor + boundary stats -``` - -For smaller variants (yambda-50m / yambda-500m) substitute the dataset name. The preprocessor takes ~2 min for 50m and ~53 min for 5b end-to-end. - -## 3. Yambda dataset statistics - -Numbers from the 5b variant, after preprocessing: +[Yambda-5b](https://huggingface.co/datasets/yandex/yambda) is a public +anonymized music-recommendation dataset from Yandex. The `5b` variant is used +for the reference. Statistics after preprocessing: | | | |---|---| @@ -120,23 +133,61 @@ Numbers from the 5b variant, after preprocessing: | Mean events per user | 4,763 | | Train events (300d) | 4.76 B | | Test events (1d) | 22.4 M | -| Training positions (≥2039 prior events filter) | **3.23 B** | | Item catalog size | 9.39 M | -### 3.1 Per-event-type distribution (across the full 4.76 B corpus) +### 5.1 Per-event-type distribution (across the full 4.76 B corpus) | Pool | Definition | Count | Share | |---|---|---|---| -| **listen_plus (lp)** | `is_listen AND played_ratio ≥ 50%` | 2.92 B | **61.3%** | +| **listen_plus (lp)** | `is_listen AND played_ratio >= 50%` | 2.92 B | **61.3%** | | **skip** | `is_listen AND played_ratio < 50%` | 1.71 B | **35.9%** | | **like** | explicit thumbs-up action | 89 M | **1.9%** | | other | dislike / unlike / undislike | 47 M | 1.0% | -The `like` pool is roughly **30× rarer** than `lp` — important context for the gather strategy below. +The `like` pool is roughly **30x rarer** than `lp` — important context for the +gather strategy in §6. -## 4. How data is fed to HSTU +### 5.2 Preprocessing & download -For every training anchor (a LISTEN event with ≥ `min_history` prior events — frozen default `4086`, the "full `history_length` of context required" power-users filter; set `$MIN_HISTORY=0` to include ~all users plus their cold-start first event), the dataset builds a `(uih_kjt, candidate_kjt)` pair: +`./download_dataset.sh` (which calls +`python3 -m generative_recommenders.dlrm_v3.preprocess_public_data --dataset +yambda-5b --data-path `) downloads the 5b variant from HuggingFace, then: + +1. **Encodes** the raw `event_type` string into a uint8 lookup (listen=0, + like=1, dislike=2, unlike=3, undislike=4). +2. **Splits** events temporally — 300 train days, 30-min gap, 1 test day — by + Global Temporal Split (GTS). +3. **Segments** per-user event timelines into sessions on a 30-min inactivity + gap. +4. **Computes** per-item popularity for downstream metric weighting. +5. **Writes** the layout `DLRMv3YambdaDataset` expects: + +``` +/ +├── raw/5b/multi_event.parquet 50 GB (downloaded) +├── shared_metadata/ +│ ├── artist_item_mapping.parquet 60 MB +│ ├── album_item_mapping.parquet 76 MB +│ └── embeddings.parquet 18 GB (unused by HSTU training) +└── processed_5b/ + ├── train_sessions.parquet 47 GB ← main training input + ├── test_events.parquet 152 MB + ├── session_index.parquet 600 MB + ├── item_popularity.npy 75 MB + └── split_meta.json anchor + boundary stats +``` + +For smaller variants (`yambda-50m` / `yambda-500m`) substitute the dataset name +(`DATASET=yambda-50m ./download_dataset.sh`). Preprocessing takes ~2 min for 50m +and ~53 min for 5b end-to-end. + +Integrity is verified with `./verify_dataset.sh` against +[`md5sums_yambda_5b_processed.txt`](md5sums_yambda_5b_processed.txt). + +## 6. How data is fed to HSTU + +For every training anchor (a LISTEN event with >= `min_history` prior events), +the dataset builds a `(uih_kjt, candidate_kjt)` pair: ``` UIH (User Interaction History): @@ -154,42 +205,121 @@ CANDIDATE (the LISTEN event at the anchor): item_dummy_watchtime ``` -The candidate's `action_weight` is **the supervision label**: HSTU's `_get_supervision_labels_and_weights` masks BCE training to `(supervision_bitmask & task_weight) > 0`, with `task_weight = 1` (LP bit) for the single `listen_plus` task — so only listen_plus candidates supervise. +The candidate's `action_weight` is **the supervision label**: HSTU's +`_get_supervision_labels_and_weights` masks BCE training to +`(supervision_bitmask & task_weight) > 0`, with `task_weight = 1` (LP bit) for +the single `listen_plus` task — so only listen_plus candidates supervise. -### 4.1 Per-pool gather (the cap = L // 3 strategy) +### 6.1 Per-pool gather (the cap = L // 3 strategy) -The UIH is built by `DLRMv3YambdaDataset._gather_interleaved_history`. For each anchor, it: +The UIH is built by `DLRMv3YambdaDataset._gather_interleaved_history`. For each +anchor it: -1. Scans the most recent `scan_window` (default 20,000) events of any type before the anchor, **clipped to user_start** so users with shorter history get a smaller window. -2. From those, takes **the last `L // 3` events** from each of the three pools (lp, like, skip) independently. -3. Concatenates the three streams and **re-sorts chronologically** to produce an interleaved sequence. -4. Tags each event's pool identity into `action_weight` via OR'd bitmask (LP=1, LIKE=2, SKIP=4). +1. Scans the most recent `scan_window` (default 20,000) events of any type + before the anchor, **clipped to user_start**. +2. From those, takes **the last `L // 3` events** from each of the three pools + (lp, like, skip) independently. +3. Concatenates and **re-sorts chronologically** to produce an interleaved + sequence. +4. Tags each event's pool identity into `action_weight` via OR'd bitmask + (LP=1, LIKE=2, SKIP=4). -With `L = 2039` and `max_seq_len = 2048`: -- Per-pool cap = `L // 3 = 679` -- Maximum total UIH = `3 × 679 = 2037` events -- Plus `8 contextual + 1 candidate = 9` overhead → 2046 ≤ 2048 model budget (no truncation) +With `history_length = 4086` and `max_seq_len = 4096`: per-pool cap = `4086 // +3 = 1362`, and `3 × 1362 + 8 contextual + 1 candidate = 4095 <= 4096` (no +truncation). Because the `like` pool is rare (1.9%) it under-fills (~105 events +per anchor on average); the Triton jagged-attention backend skips unfilled +slots, so the under-fill costs sequence budget but not GPU compute. -### 4.2 Effective per-anchor fill on real data +## 7. Optimizer -Because the `like` pool is rare (1.9% of events) and the average user has only ~4,763 lifetime events: +Two optimizers, configured in +[`yambda_5b.gin`](generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin): -| Pool | per-pool cap (L//3) | actual avg fill per anchor | fill rate | +| component | optimizer | gin binding | key settings | |---|---|---|---| -| lp | 679 | ~673 | **99%** | -| like | 679 | ~105 | **15%** (data-bounded, not cap-bounded) | -| skip | 679 | ~624 | 92% | -| **total UIH** | 2037 max | **~1402** | 69% | - -The `like` cap of 679 is unreachable for yambda data — at the 1.9% global like rate, filling 679 likes would require a user to have ~36k prior events, but the **longest user in the dataset has only 27,738 events total** (and the median user has 2,695). So under-fill on `like` is fundamental to the data. - -This means the model sees on average ~1,402 UIH events per sample, not the theoretical 2,037. With the TRITON jagged-attention backend the GPU only does work for the actual events, so the under-fill costs **sequence budget but not GPU compute** — no wasted attention work, just less context per sample than the budget suggests. - -## 5. Streaming (temporal-order) training - -`scripts/launch_slurm.sh` defaults to `--mode streaming-train-eval`, which -trains Yambda in strict wall-clock order instead of shuffling the whole corpus. -The timeline is sliced into fixed-duration **windows** (default 1 day, +| Dense params (HSTU blocks, MLPs) | **Adam** | `dense_optimizer_factory_and_class.*` | lr `DENSE_LR`, betas (0.95, 0.999), eps 1e-8, weight_decay 0 | +| Sparse embedding tables | **RowWiseAdagrad** (fused FBGEMM TBE) | `sparse_optimizer_factory_and_class.*` | lr `SPARSE_LR`, eps 1e-8, weight_decay 0 | + +Gradient clipping (`GRAD_CLIP_NORM`, default `max_norm=1.0`) is applied to the +dense parameters on the streaming path; the fused sparse optimizer is +unaffected. Training is bf16 mixed precision (`make_model.bf16_training=True`). + +## 8. Hyperparameters + +All tunable hyperparameters live in +[`yambda_5b.gin`](generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin) +(the config-file source of truth) and are **overridable via environment +variables** (the env value takes precedence over the gin default, per MLPerf +CONTRIBUTING rule 4d). The gin macros (`@env_int`, `@env_float`, `@env_str`) +enforce the correct type for each parameter. + +| hyperparameter | env var | gin binding | type | default | tuning rule | +|---|---|---|---|---|---| +| Per-rank batch size | `BATCH_SIZE` | `batch_size` | int | 1024 | positive integer (global batch = `BATCH_SIZE × world_size`) | +| Dense learning rate | `DENSE_LR` | `dense_optimizer_factory_and_class.learning_rate` | float | 1e-7 | positive float | +| Sparse learning rate | `SPARSE_LR` | `sparse_optimizer_factory_and_class.learning_rate` | float | 1e-7 | positive float | +| Grad clip max-norm | `GRAD_CLIP_NORM` | `streaming_train_eval_loop.grad_clip_norm` | float | 1.0 | float >= 0 (0 disables) | +| RNG seed | `SEED` | `seed_everything.seed` | int | 1 | any integer (-1 = random per run) | +| HSTU attention layers | `HSTU_NUM_LAYERS` | `get_hstu_configs.hstu_attn_num_layers` | int | 3 | positive integer | +| UIH history length | `HISTORY_LENGTH` | `get_dataset.history_length` | int | 4086 | positive integer (per-pool cap = L//3) | +| Max sequence length | `MAX_SEQ_LEN` | `get_hstu_configs.max_seq_len` | int | 4096 | positive integer (>= `history_length + 9`) | +| History strategy | `HISTORY_STRATEGY` | `get_dataset.history_strategy` | str | `interleaved` | one of `interleaved` \| `last_n` | +| Min history (anchor floor) | `MIN_HISTORY` | `get_dataset.min_history` | int | 4086 | integer >= 0 | +| Train user split | `TRAIN_SPLIT_PERCENTAGE` | `*.train_split_percentage` | float | 1.0 | float in (0, 1] | +| Streaming shuffle fraction | `STREAMING_SHUFFLE_FRACTION` | `get_dataset.streaming_shuffle_fraction` | float | 0.0 | float in [0, 1] | +| Streaming shuffle seed | `STREAMING_SHUFFLE_SEED` | `get_dataset.streaming_shuffle_seed` | int | 0 | any integer | +| Split salt | `SPLIT_SALT` | `get_dataset.split_salt` | int | 0 | any integer | +| Start window | `START_TS` | `streaming_train_eval_loop.start_ts` | int | 150 | integer >= 0 | +| Number of train windows | `NUM_TRAIN_TS` | `streaming_train_eval_loop.num_train_ts` | int | 149 | positive integer (clamped to available) | +| Sparse A2A fwd precision | `SPARSE_A2A_FWD` | `make_optimizer_and_shard.sparse_a2a_forward_precision` | str | `fp32` | one of `fp32` \| `bf16` \| `fp16` | +| Sparse A2A bwd precision | `SPARSE_A2A_BWD` | `make_optimizer_and_shard.sparse_a2a_backward_precision` | str | `fp32` | one of `fp32` \| `bf16` \| `fp16` | + +Non-tunable / fixed reference values (optimizer betas (0.95, 0.999), eps 1e-8, +weight_decay 0, bf16 training, streaming window = 86400 s) are pinned in the gin +file. Submitters tuning hyperparameters must follow the allowed values above and +the +[MLPerf training rules](https://github.com/mlcommons/training_policies/blob/master/training_rules.adoc#hyperparameters). + +## 9. Quality target & evaluation + +- **Metric**: AUC on the held-out future evaluation window (`window_auc` for the + `listen_plus` task), computed by `MetricsLogger` in + `generative_recommenders/dlrm_v3/utils.py`. +- **Target**: **eval AUC >= 0.80275**. Set via `AUC_THRESHOLD=0.80275` + (`MetricsLogger.auc_threshold`); the run logs `RUN_STOP` with `SUCCESS` and + stops once the target is reached. The gin default of `1.0` is unreachable + (trains all windows with no early stop) and is overridden by the reference + scripts. +- **Evaluation frequency**: the full-reference run uses + `EVAL_EVERY_DATA_PCT=0.005` — evaluate every 0.5% of the training stream + (~200 evenly-data-spaced eval points), independent of node count. The + alternative per-window cadence (`EVAL_EVERY_N_WINDOWS`) is mutually exclusive. +- **Evaluation set**: a fixed held-out future window (`eval_holdout_ts`, default + `start_ts + num_train_ts`); with `TRAIN_SPLIT_PERCENTAGE < 1.0` the held-out + users' anchors over that window form the eval set. The temporal one-window + lead guarantees no future leakage (see §11). + +Evaluation is always one window ahead of training, so reported AUC is genuine +next-period generalization. + +## 10. Reference Convergence Points (RCP) + +*Placeholder — to be generated.* + +RCPs have **not yet been generated** for this benchmark. Per the MLPerf +[CONTRIBUTING guidance](https://github.com/mlcommons/training_policies/blob/master/CONTRIBUTING.md), +RCPs must be generated for at least 3 reasonable batch sizes using at least 2N +seeds (N = number of submission runs), in FP32 or BF16, with the exact precision +recorded in the RCP JSON. The convergence curves (steps/samples to reach +AUC >= 0.80275) will be added under [`rcp/`](rcp/) once the convergence runs are +complete. This section is intentionally left blank for now. + +## 11. Streaming (temporal-order) training + +`scripts/launch_slurm.sh` and `run_and_time.sh` default to +`--mode streaming-train-eval`, which trains Yambda in strict wall-clock order +instead of shuffling the whole corpus. The timeline is sliced into +fixed-duration **windows** (default 1 day, `get_dataset.streaming_window_seconds = 86400`), and the loop walks them forward: ``` @@ -203,76 +333,48 @@ window T: train window T+1: eval (then train) window T+2: eval (t i.e. for each step it **trains window T, then evaluates window T+1** before advancing — always predicting the immediate future from the past. -### 5.1 Temporal guarantee +### 11.1 Temporal guarantee The streaming path enforces **no future leakage** at two levels: -1. **Across windows** — a window is the set of anchors whose *target/candidate* +1. **Across windows** — a window is the set of anchors whose target/candidate timestamp falls in `[t_min + T·W, t_min + (T+1)·W)`. Training only ever sees - windows `≤ T`; the evaluation window `T+1` is strictly in the future of every - training anchor it is scored against. Eval always leads train by exactly one - window, so reported eval NE/AUC is genuine next-period generalization, never - an in-sample measurement. -2. **Within an anchor** — history is still gathered **causally**: the UIH scan - is `scan_start:flat_pos` (events strictly before the anchor), so even though a - long user history may reach back across earlier windows, no event at or after - the anchor's timestamp can enter its features. Forward-time windowing and - causal history are independent guarantees, and both hold simultaneously. - -Note this is a *temporal* split on the training stream — distinct from the -preprocessing GTS split (§2) that carves off the final test day. Windows are + windows `<= T`; the evaluation window `T+1` is strictly in the future of every + training anchor it is scored against. +2. **Within an anchor** — history is gathered **causally**: the UIH scan is + `scan_start:flat_pos` (events strictly before the anchor), so no event at or + after the anchor's timestamp can enter its features. + +This is a *temporal* split on the training stream — distinct from the +preprocessing GTS split (§5) that carves off the final test day. Windows are indexed off the per-anchor target timestamp via a lazily-built, mmap'd -`anchor_ts_L{H}[_m{MIN_HISTORY}].npy` cache (built once on first use; the -default non-streaming path never touches it). The anchor `positions` and -`anchor_ts` arrays are keyed by `(history_length, min_history)` so different -floors don't collide and the expensive flat store is shared across them. +`anchor_ts` cache keyed by `(history_length, min_history)`. -### 5.2 Knobs +### 11.2 Streaming knobs -All configurable via gin ([yambda_5b.gin](generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin)) -with env overrides: +All configurable via +[`yambda_5b.gin`](generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin) with +env overrides: | env | gin | default | meaning | |---|---|---|---| -| `START_TS` | `streaming_train_eval_loop.start_ts` | 150 | first window (early windows are near-empty warm-up; start dense) | -| `NUM_TRAIN_TS` | `streaming_train_eval_loop.num_train_ts` | 30 | number of train windows (clamped to available) | -| `PERSISTENT_LOADER` | `streaming_train_eval_loop.persistent_loader` | 1 | reuse one worker pool across windows (no per-window respawn) | -| `DOUBLE_BUFFER` | `streaming_train_eval_loop.double_buffer` | 1 | prepare the next window in a background thread during compute | -| `EVAL_EACH_WINDOW` | `streaming_train_eval_loop.eval_each_window` | 1 | eval window T+1 after training window T | -| `MIN_HISTORY` | `get_dataset.min_history` | 4086 | anchor-eligibility floor: min prior events for a LISTEN to be a sample (frozen default 4086 = full-history power-users filter; 0 = ~all users incl. cold-start) | -| — | `streaming_train_eval_loop.num_train_batches` / `num_eval_batches` | unset | cap per-window steps (unset = consume full window) | - -### 5.3 Hiding the window-reset overhead - -Advancing to a new window has a fixed cost — selecting the window's anchor -indices and warming the dataloader's first batch — that, done naively, stalls -training at every window boundary. Three layers drive it to ~0: - -1. **Persistent loader** (`persistent_loader=1`). The naive path recreates a - `DataLoader` per window, re-forking workers and paying first-batch warmup - each time (~11 s/window). Instead we build **one** `DataLoader` backed by a - stateful `StreamingWindowSampler` whose index set is swapped per window - (`set_window`), so workers fork once and persist. This removes the respawn - but still pays the index-mask + first-batch stall (~3.6 s/window). -2. **Double buffering** (`double_buffer=1`). Two pre-forked worker pools - ping-pong: while the current window trains on pool A, the *next* window's - index mask (`window_indices`, a GIL-releasing NumPy `np.where`) and - first-batch prefetch are prepared on pool B in a **background thread**, so - that work overlaps GPU compute. The boundary train batch then arrives warm — - measured train first-batch data-wait drops to **~1–3 ms**. Pools are forked - up front on the main thread (never inside the background thread), so a forking - worker can never race a thread holding a lock. -3. **Eval prefetch one window ahead.** With `eval_each_window=1` the eval window - (`T+1`) is prepared *before* training window `T` runs, so the idle eval pool - prefetches its first batches concurrently with train compute. This hides the - eval-side first-batch stall (**~0.55 s → ~2 ms**). It is safe because a - sample's content depends only on the sampler's window indices, not on any - train/eval flag. - -Net effect: steady-state throughput matches the non-streaming baseline and the -per-window reset is effectively free; the only remaining one-time cost is the -process cold start (CUDA-graph capture + the first lazy `anchor_ts` mmap). - -## 6. License +| `START_TS` | `streaming_train_eval_loop.start_ts` | 150 | first window (early windows are near-empty warm-up) | +| `NUM_TRAIN_TS` | `streaming_train_eval_loop.num_train_ts` | 149 | number of train windows (clamped to available) | +| `PERSISTENT_LOADER` | `streaming_train_eval_loop.persistent_loader` | 1 | reuse one worker pool across windows | +| `DOUBLE_BUFFER` | `streaming_train_eval_loop.double_buffer` | 1 | prepare the next window in a background thread | +| `EVAL_EVERY_N_WINDOWS` | `streaming_train_eval_loop.eval_every_n_windows` | 1 | eval cadence by window count (0 to use data-pct) | +| `EVAL_EVERY_DATA_PCT` | `streaming_train_eval_loop.eval_every_data_pct` | 0.0 | eval cadence by fraction of train data (full ref: 0.005) | +| `MIN_HISTORY` | `get_dataset.min_history` | 4086 | anchor-eligibility floor (0 = ~all users incl. cold-start) | + +### 11.3 Checkpointing & resume + +The streaming loop is resume-aware: set `CKPT_PATH` to enable DMP checkpoint +save/load (auto-resolves to the highest-numbered subdir), with retention via +`KEEP_LAST_N` and cadences `IN_WINDOW_CKPT_FREQ` / `CKPT_STEP_FREQ` / +`CKPT_TIME_INTERVAL_S`. The MLPerf run state (run-started flag, global sample +count) is persisted across resume so compliance logging is continuous. See +`generative_recommenders/dlrm_v3/checkpoint.py`. + +## 12. License Apache 2.0 (inherited from upstream). diff --git a/recommendation_v4/download_dataset.sh b/recommendation_v4/download_dataset.sh new file mode 100755 index 000000000..d02f382dc --- /dev/null +++ b/recommendation_v4/download_dataset.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# MLPerf Training reference script: download + preprocess the dataset. +# +# Downloads the Yambda dataset from HuggingFace (yandex/yambda) and runs the +# preprocessing pipeline (event-type encoding, temporal GTS split, session +# segmentation, item-popularity counts) into the on-disk layout that +# DLRMv3YambdaDataset consumes. This is a thin wrapper over +# generative_recommenders.dlrm_v3.preprocess_public_data +# so the full reference data pipeline lives in one place. +# +# Usage: +# DLRM_DATA_PATH=/path/to/dlrm_data ./download_dataset.sh +# DATASET=yambda-50m DLRM_DATA_PATH=/path/to/dlrm_data ./download_dataset.sh +# +# Env: +# DATASET dataset variant (default: yambda-5b). One of +# kuairand-1k | kuairand-27k | yambda-50m | yambda-500m | yambda-5b +# DLRM_DATA_PATH destination data root (required). +set -euo pipefail + +DATASET="${DATASET:-yambda-5b}" +: "${DLRM_DATA_PATH:?Set DLRM_DATA_PATH to the destination data root}" + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "${REPO_ROOT}" + +echo "[download_dataset] dataset=${DATASET} data-path=${DLRM_DATA_PATH}" +mkdir -p "${DLRM_DATA_PATH}" + +python3 -m generative_recommenders.dlrm_v3.preprocess_public_data \ + --dataset "${DATASET}" \ + --data-path "${DLRM_DATA_PATH}" + +echo "[download_dataset] done. Preprocessed layout under ${DLRM_DATA_PATH}:" +echo " raw/5b/multi_event.parquet" +echo " shared_metadata/{artist,album}_item_mapping.parquet, embeddings.parquet" +echo " processed_5b/{train_sessions,test_events,session_index}.parquet" +echo " processed_5b/item_popularity.npy, processed_5b/split_meta.json" +echo "[download_dataset] verify integrity with: ./verify_dataset.sh" diff --git a/recommendation_v4/md5sums_yambda_5b_processed.txt b/recommendation_v4/md5sums_yambda_5b_processed.txt new file mode 100644 index 000000000..82998cca3 --- /dev/null +++ b/recommendation_v4/md5sums_yambda_5b_processed.txt @@ -0,0 +1,22 @@ +# MD5 checksums for the preprocessed yambda-5b dataset (processed_5b/). +# +# Format: standard `md5sum` output -> " ". +# Paths are relative to ${DLRM_DATA_PATH}/processed_5b/. +# +# These hashes are PLACEHOLDERS (TODO). They must be generated from a canonical +# preprocessing run before this benchmark is submitted, e.g.: +# +# cd "${DLRM_DATA_PATH}/processed_5b" +# md5sum train_sessions.parquet test_events.parquet session_index.parquet \ +# item_popularity.npy split_meta.json \ +# > /md5sums_yambda_5b_processed.txt +# +# Until then `verify_dataset.sh` falls back to an existence/layout check and +# warns that checksums are not yet pinned. +# +# TODO(rcp/data): replace the lines below with real md5 hashes. +TODO_GENERATE_HASH train_sessions.parquet +TODO_GENERATE_HASH test_events.parquet +TODO_GENERATE_HASH session_index.parquet +TODO_GENERATE_HASH item_popularity.npy +TODO_GENERATE_HASH split_meta.json diff --git a/recommendation_v4/rcp/README.md b/recommendation_v4/rcp/README.md new file mode 100644 index 000000000..02a977862 --- /dev/null +++ b/recommendation_v4/rcp/README.md @@ -0,0 +1,28 @@ +# Reference Convergence Points (RCP) + +**Status: placeholder — RCPs not yet generated. Intentionally left blank.** + +This directory will hold the Reference Convergence Points for the +recommendation_v4 (HSTU / yambda-5b) benchmark once convergence runs are +complete. + +Per the MLPerf Training +[CONTRIBUTING guidance](https://github.com/mlcommons/training_policies/blob/master/CONTRIBUTING.md) +("Some things to note while generating reference convergence points"): + +- Use FP32 or BF16 precision and record the exact precision used in the RCP JSON. +- Generate RCPs for at least **3 reasonable batch sizes**. +- Run RCPs with an eval frequency **higher** than the chosen benchmark eval + frequency (more data points for picking the target accuracy). +- Run at least **2N seeds**, where N = number of submission runs. + +The convergence target for this benchmark is **eval AUC >= 0.80275** (see +[../README.MD](../README.MD) §9). The RCP JSON files and convergence-curve plots +(samples-to-converge vs. batch size / seed) will be committed here. + +## TODO + +- [ ] Run >= 2N-seed convergence sweeps at >= 3 batch sizes. +- [ ] Record precision (FP32/BF16) per the rules. +- [ ] Add `rcp_.json` files in the mlperf_logging RCP format. +- [ ] Add convergence-curve plots and the chosen target-accuracy justification. diff --git a/recommendation_v4/requirements.txt b/recommendation_v4/requirements.txt index d1aba1e95..852aa149b 100644 --- a/recommendation_v4/requirements.txt +++ b/recommendation_v4/requirements.txt @@ -1,8 +1,43 @@ -torch>=2.6.0 -fbgemm_gpu>=1.1.0 -torchrec>=1.1.0 -gin_config>=0.5.0 -pandas>=2.2.0 -tensorboard>=2.19.0 -pybind11 -git+https://github.com/mlcommons/logging.git@6.0.0-rc6 +# Frozen dependency versions for the recommendation_v4 (HSTU / yambda-5b) MLPerf +# reference. The CANONICAL, fully-reproducible environment is the Dockerfile +# (built on rocm/primus:v26.3); see docs/training_recipe.md for the per-platform +# (MI350X / B200) install commands and rationale. The pins below mirror that +# stack. torch / torchvision / torchaudio / fbgemm_gpu / torchrec are +# accelerator-specific and MUST be installed with --no-deps from the matching +# index (see Dockerfile) so pip does not clobber the +rocm wheels. + +# --- accelerator stack (install via Dockerfile; --no-deps, matching index) --- +# torch==2.12.0+rocm7.2 # --index-url https://download.pytorch.org/whl/rocm7.2 +# torchvision==0.27.0+rocm7.2 +# torchaudio==2.11.0+rocm7.2 +# fbgemm_gpu # built from FBGEMM commit 10b775730212923f65f7b78f79b6a01d80cf3c29 for gfx950 +torch==2.12.0 +fbgemm_gpu==1.7.0 +torchrec @ git+https://github.com/pytorch/torchrec.git@v2026.06.01.00 + +# --- data / config / logging ------------------------------------------------- +polars-u64-idx==1.33.1 +gin-config==0.5.0 +absl-py==2.1.0 +pandas==2.2.3 +pyarrow==17.0.0 +numpy==1.26.4 +xxhash==3.5.0 +datasets==3.2.0 +huggingface_hub==0.27.0 + +# --- metrics / training utils ------------------------------------------------ +torchmetrics==1.0.3 +tensordict==0.6.2 +tensorboard==2.19.0 +pyre-extensions==0.0.32 +iopath==0.1.10 +typing-inspect==0.9.0 +psutil==6.1.0 +tqdm==4.67.1 +pyyaml==6.0.2 +pybind11==2.13.6 +lightning-utilities==0.11.9 + +# --- MLPerf compliance logging (pinned to the Training 6.0 tag) -------------- +mlperf-logging @ git+https://github.com/mlcommons/logging.git@6.0.0-rc6 diff --git a/recommendation_v4/run_and_time.sh b/recommendation_v4/run_and_time.sh new file mode 100755 index 000000000..a0207e71c --- /dev/null +++ b/recommendation_v4/run_and_time.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# MLPerf Training reference script: run the benchmark and report wall-clock time. +# +# Runs the full-reference HSTU / yambda-5b streaming train+eval sweep to the +# MLPerf quality target (eval AUC >= 0.80275) and prints the elapsed time of the +# timed region. This is the canonical single-host (8-GPU) entry point; for +# multi-node SLURM launches use scripts/launch_slurm.sh (which calls into the +# same trainer). +# +# Usage: +# DLRM_DATA_PATH=/path/to/dlrm_data ./run_and_time.sh +# +# Env (run shape / cadence -- defaults are the FULL reference sweep): +# DLRM_DATA_PATH data root (required). +# SEED RNG seed (default 1). +# START_TS / NUM_TRAIN_TS window range (default 0 / 299 = full sweep). +# EVAL_EVERY_DATA_PCT eval cadence as a fraction of train data (default 0.005). +# AUC_THRESHOLD convergence target (default 0.80275). +# GPUS_PER_NODE GPUs on this host (default 8). +# RUN_NAME results dir name under results/ (default reference_run). +set -euo pipefail + +: "${DLRM_DATA_PATH:?Set DLRM_DATA_PATH to the data root}" + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "${REPO_ROOT}" + +# ---- Reference run shape (full sweep to the quality target) ----------------- +export SEED="${SEED:-1}" +export START_TS="${START_TS:-0}" +export NUM_TRAIN_TS="${NUM_TRAIN_TS:-299}" +export NUM_TRAIN_BATCHES="${NUM_TRAIN_BATCHES:-0}" +export NUM_EVAL_BATCHES="${NUM_EVAL_BATCHES:-0}" +export EVAL_EVERY_N_WINDOWS="${EVAL_EVERY_N_WINDOWS:-0}" +export EVAL_EVERY_DATA_PCT="${EVAL_EVERY_DATA_PCT:-0.005}" +export AUC_THRESHOLD="${AUC_THRESHOLD:-0.80275}" +export RUN_NAME="${RUN_NAME:-reference_run}" + +# ---- Single-host distributed topology (override for multi-node) ------------- +export GPUS_PER_NODE="${GPUS_PER_NODE:-8}" +export NNODES="${NNODES:-1}" +export NODE_RANK="${NODE_RANK:-0}" +export WORLD_SIZE="${WORLD_SIZE:-$((NNODES * GPUS_PER_NODE))}" +export MASTER_ADDR="${MASTER_ADDR:-127.0.0.1}" +export MASTER_PORT="${MASTER_PORT:-29500}" + +# ---- MLPerf compliance logging ---------------------------------------------- +export MLPERF_LOGGING="${MLPERF_LOGGING:-1}" +export MLPERF_LOG_PATH="${MLPERF_LOG_PATH:-${REPO_ROOT}/results/${RUN_NAME}/mlperf/yambda_5b_mlperf.log}" +export MLPERF_SUBMISSION_PLATFORM="${MLPERF_SUBMISSION_PLATFORM:-MI355X}" +mkdir -p "$(dirname "${MLPERF_LOG_PATH}")" + +# ---- Timed region ----------------------------------------------------------- +# Pull the start timestamp into a clear region per the MLPerf run_and_time.sh idiom. +start=$(date +%s) +echo "STARTING TIMING RUN AT $(date -u '+%Y-%m-%d %r')" + +python -m generative_recommenders.dlrm_v3.train.train_ranker \ + --dataset yambda-5b \ + --mode streaming-train-eval + +end=$(date +%s) +result=$(( end - start )) +echo "ENDING TIMING RUN AT $(date -u '+%Y-%m-%d %r')" +echo "RESULT,recommendation_v4_hstu_yambda_5b,${SEED},${result},$(whoami),$(date -u '+%Y-%m-%d %r')" diff --git a/recommendation_v4/verify_dataset.sh b/recommendation_v4/verify_dataset.sh new file mode 100755 index 000000000..839ccb91f --- /dev/null +++ b/recommendation_v4/verify_dataset.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# MLPerf Training reference script: verify the preprocessed dataset. +# +# Checks the integrity of the preprocessed dataset under +# ${DLRM_DATA_PATH}/${PROCESSED_SUBDIR} +# against md5sums_yambda_5b_processed.txt (standard `md5sum -c` format). +# +# If the checksum file still contains placeholder hashes (TODO_GENERATE_HASH), +# the script falls back to an existence/layout check and warns that the +# canonical checksums have not been pinned yet. +# +# Usage: +# DLRM_DATA_PATH=/path/to/dlrm_data ./verify_dataset.sh +# +# Env: +# DLRM_DATA_PATH data root (required). +# PROCESSED_SUBDIR processed subdir under the data root (default: processed_5b). +set -euo pipefail + +: "${DLRM_DATA_PATH:?Set DLRM_DATA_PATH to the data root}" +PROCESSED_SUBDIR="${PROCESSED_SUBDIR:-processed_5b}" + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CHECKSUM_FILE="${REPO_ROOT}/md5sums_yambda_5b_processed.txt" +PROCESSED_DIR="${DLRM_DATA_PATH}/${PROCESSED_SUBDIR}" + +echo "[verify_dataset] processed dir: ${PROCESSED_DIR}" + +if [[ ! -d "${PROCESSED_DIR}" ]]; then + echo "[verify_dataset] ERROR: ${PROCESSED_DIR} does not exist. Run ./download_dataset.sh first." >&2 + exit 1 +fi + +EXPECTED_FILES=( + train_sessions.parquet + test_events.parquet + session_index.parquet + item_popularity.npy + split_meta.json +) + +# Detect whether the checksum file has real (32 hex char) hashes or placeholders. +if grep -qiE '^[0-9a-f]{32}[[:space:]]' "${CHECKSUM_FILE}"; then + echo "[verify_dataset] checking md5 checksums from ${CHECKSUM_FILE}" + (cd "${PROCESSED_DIR}" && md5sum -c "${CHECKSUM_FILE}") + echo "[verify_dataset] OK: all checksums match." +else + echo "[verify_dataset] WARNING: ${CHECKSUM_FILE} contains placeholder hashes;" >&2 + echo "[verify_dataset] falling back to existence/layout check only." >&2 + missing=0 + for f in "${EXPECTED_FILES[@]}"; do + if [[ -s "${PROCESSED_DIR}/${f}" ]]; then + echo " OK ${f}" + else + echo " MISS ${f}" >&2 + missing=1 + fi + done + if [[ "${missing}" -ne 0 ]]; then + echo "[verify_dataset] ERROR: one or more expected files are missing/empty." >&2 + exit 1 + fi + echo "[verify_dataset] layout OK (checksums NOT yet pinned -- see TODO in ${CHECKSUM_FILE})." +fi From 4cf4d859f9e7c4e268f7c0969817ef43c30e391d Mon Sep 17 00:00:00 2001 From: Chris Cai Date: Thu, 25 Jun 2026 18:53:45 +0000 Subject: [PATCH 102/127] recommendation_v4: prune inference/AOT/CUDA-cpp/research + non-yambda configs Remove subtrees unused by the yambda-5b TRITON training path: - dlrm_v3/inference/ (incl thirdparty/loadgen) - ops/cpp/ (CUTLASS CUDA kernels) and ops/triton_aot/ (AOT-inference kernels) - generative_recommenders/research/ and its entrypoints (main.py, run_fractal_expansion.py, repo-root preprocess_public_data.py) - configs/{ml-1m,ml-20m,ml-3b,amzn-books} and non-yambda train gins (keep yambda_5b.gin + debug.gin) - ops/benchmarks/hstu_attention_bench.py (dangling ops.cpp import) The TRITON (default) and PYTORCH kernel paths are unaffected: aot_* calls are gated behind HammerKernel.TRITON_INFERENCE and ops/triton/* has no cpp/aot deps. Validated by import smoke + a streaming-train-eval e2e smoke (rc=0). Co-authored-by: Cursor --- .../hstu-sampled-softmax-n512-final.gin | 49 - .../hstu-sampled-softmax-n512-large-final.gin | 49 - .../sasrec-sampled-softmax-n512-final.gin | 50 - .../ml-1m/hstu-sampled-softmax-n128-final.gin | 45 - .../hstu-sampled-softmax-n128-large-final.gin | 45 - .../sasrec-sampled-softmax-n128-final.gin | 44 - .../hstu-sampled-softmax-n128-final.gin | 45 - .../hstu-sampled-softmax-n128-large-final.gin | 45 - .../sasrec-sampled-softmax-n128-final.gin | 44 - ...tu-sampled-softmax-n96-seqlen500-final.gin | 42 - ...pled-softmax-n96-seqlen500-large-final.gin | 42 - ...ec-sampled-softmax-n96-seqlen500-final.gin | 42 - .../dlrm_v3/inference/README.md | 88 - .../dlrm_v3/inference/accuracy.py | 86 - .../dlrm_v3/inference/cpp/hstu_runner.cpp | 215 -- .../dlrm_v3/inference/data_producer.py | 227 -- .../dlrm_v3/inference/dense_predict_module.py | 96 - .../dlrm_v3/inference/end_to_end_test.py | 795 ----- .../dlrm_v3/inference/gin/debug.gin | 13 - .../dlrm_v3/inference/gin/kuairand_1k.gin | 14 - .../dlrm_v3/inference/gin/movielens_13b.gin | 16 - .../dlrm_v3/inference/gin/streaming_100b.gin | 15 - .../dlrm_v3/inference/gin/streaming_400m.gin | 15 - .../dlrm_v3/inference/inference_modules.py | 253 -- .../dlrm_v3/inference/main.py | 805 ----- .../dlrm_v3/inference/mlperf.conf | 98 - .../dlrm_v3/inference/model_family.py | 705 ---- .../inference/sparse_predict_module.py | 106 - .../dlrm_v3/inference/tests/inference_test.py | 39 - .../inference/tests/test_scripted_parity.py | 236 -- .../thirdparty/loadgen/.clang-format | 2 - .../thirdparty/loadgen/CMakeLists.txt | 113 - .../inference/thirdparty/loadgen/MANIFEST.in | 2 - .../inference/thirdparty/loadgen/README.md | 223 -- .../thirdparty/loadgen/README_BUILD.md | 47 - .../thirdparty/loadgen/README_FAQ.md | 78 - .../inference/thirdparty/loadgen/VERSION.txt | 1 - .../thirdparty/loadgen/benchmark/.gitignore | 2 - .../thirdparty/loadgen/benchmark/README.md | 10 - .../thirdparty/loadgen/benchmark/repro.cpp | 296 -- .../thirdparty/loadgen/benchmark/run.sh | 21 - .../thirdparty/loadgen/benchmark/run_debug.sh | 21 - .../thirdparty/loadgen/bindings/c_api.cc | 176 - .../thirdparty/loadgen/bindings/c_api.h | 95 - .../thirdparty/loadgen/bindings/python_api.cc | 484 --- .../thirdparty/loadgen/demos/lon/README.md | 67 - .../loadgen/demos/lon/py_demo_server_lon.py | 191 - .../demos/lon/sut_over_network_demo.py | 88 - .../loadgen/demos/py_demo_multi_stream.py | 86 - .../loadgen/demos/py_demo_offline.py | 81 - .../loadgen/demos/py_demo_server.py | 74 - .../loadgen/demos/py_demo_single_stream.py | 84 - .../token_metrics/py_demo_multi_stream.py | 142 - .../demos/token_metrics/py_demo_offline.py | 130 - .../token_metrics/py_demo_offline_inferred.py | 130 - .../demos/token_metrics/py_demo_server.py | 132 - .../token_metrics/py_demo_server_inferred.py | 125 - .../token_metrics/py_demo_single_stream.py | 129 - .../loadgen/diagram_network_submission.png | Bin 51192 -> 0 bytes .../thirdparty/loadgen/diagram_submission.png | Bin 36510 -> 0 bytes .../thirdparty/loadgen/docs/src/BUILD.gn | 33 - .../thirdparty/loadgen/docs/src/README.md | 34 - .../thirdparty/loadgen/docs/src/doxygen.cfg | 2495 ------------- .../loadgen/docs/src/doxygen_footer.html | 26 - .../loadgen/docs/src/doxygen_header.html | 49 - .../docs/src/doxygen_html_generator.py | 37 - .../loadgen/docs/src/doxygen_layout.xml | 211 -- .../loadgen/docs/src/doxygen_stylesheet.css | 1629 --------- .../docs/src/loadgen_integration_diagram.dia | Bin 1943 -> 0 bytes .../loadgen/docs/src/mlperf_icon.png | Bin 4632 -> 0 bytes .../docs/src/mlperf_logo_horizontal_color.svg | 55 - .../thirdparty/loadgen/early_stopping.cc | 117 - .../thirdparty/loadgen/early_stopping.h | 27 - .../loadgen/generated/version_generated.cc | 98 - .../loadgen/issue_query_controller.cc | 552 --- .../loadgen/issue_query_controller.h | 215 -- .../inference/thirdparty/loadgen/loadgen.cc | 1345 ------- .../inference/thirdparty/loadgen/loadgen.h | 103 - .../loadgen/loadgen_integration_diagram.svg | 85 - .../inference/thirdparty/loadgen/logging.cc | 1301 ------- .../inference/thirdparty/loadgen/logging.h | 816 ----- .../inference/thirdparty/loadgen/mlperf.conf | 164 - .../thirdparty/loadgen/mlperf_conf.h | 167 - .../thirdparty/loadgen/pyproject.toml | 7 - .../loadgen/query_dispatch_library.h | 42 - .../thirdparty/loadgen/query_sample.h | 91 - .../thirdparty/loadgen/query_sample_library.h | 75 - .../thirdparty/loadgen/requirements.txt | 1 - .../inference/thirdparty/loadgen/results.cc | 856 ----- .../inference/thirdparty/loadgen/results.h | 128 - .../inference/thirdparty/loadgen/setup.py | 136 - .../thirdparty/loadgen/system_under_test.h | 67 - .../thirdparty/loadgen/test_settings.h | 329 -- .../loadgen/test_settings_internal.cc | 800 ----- .../loadgen/test_settings_internal.h | 182 - .../thirdparty/loadgen/tests/BUILD.gn | 25 - .../thirdparty/loadgen/tests/README.md | 42 - .../thirdparty/loadgen/tests/basic.cc | 314 -- .../thirdparty/loadgen/tests/loadgen_test.h | 198 -- .../loadgen/tests/loadgen_test_main.cc | 33 - .../loadgen/tests/perftests_null_sut.cc | 230 -- .../loadgen/tests/perftests_null_sut.py | 61 - .../loadgen/tools/mlperf-trace.ipynb | 441 --- .../inference/thirdparty/loadgen/utils.cc | 124 - .../inference/thirdparty/loadgen/utils.h | 70 - .../inference/thirdparty/loadgen/version.cc | 85 - .../inference/thirdparty/loadgen/version.h | 39 - .../thirdparty/loadgen/version_generator.py | 141 - .../dlrm_v3/inference/ts_types.py | 70 - .../dlrm_v3/inference/user.conf | 5 - .../dlrm_v3/train/gin/kuairand_1k.gin | 41 - .../dlrm_v3/train/gin/movielens_13b.gin | 41 - .../dlrm_v3/train/gin/movielens_18b.gin | 56 - .../dlrm_v3/train/gin/movielens_1m.gin | 38 - .../dlrm_v3/train/gin/movielens_20m.gin | 56 - .../dlrm_v3/train/gin/streaming_100b.gin | 52 - .../dlrm_v3/train/gin/streaming_200b.gin | 63 - .../dlrm_v3/train/gin/streaming_400m.gin | 61 - .../ops/benchmarks/hstu_attention_bench.py | 406 --- .../concat_1d_jagged_jagged_bench.py | 125 - .../benchmarks/jagged_transpose_1d_bench.py | 117 - .../replace_last_n_with_jagged_bench.py | 150 - .../split_1d_jagged_jagged_bench.py | 116 - .../generative_recommenders/ops/cpp/common.h | 60 - .../ops/cpp/complete_cumsum.cpp | 44 - .../ops/cpp/complete_cumsum.cu | 51 - .../ops/cpp/concat_1d_jagged_jagged.cpp | 111 - .../ops/cpp/concat_1d_jagged_jagged.cu | 130 - .../ops/cpp/cpp_ops.cpp | 207 -- .../ops/cpp/cuda_hstu_attention.py | 193 - .../cpp/cuda_hstu_preprocess_and_attention.py | 668 ---- .../ops/cpp/expand_1d_jagged_to_dense.cpp | 97 - .../ops/cpp/expand_1d_jagged_to_dense.cu | 103 - .../hstu_attention/copy_sm90_bulk_reduce.h | 66 - .../ops/cpp/hstu_attention/epilogue_bwd.h | 481 --- .../ops/cpp/hstu_attention/epilogue_fwd.h | 550 --- .../ops/cpp/hstu_attention/flash.h | 157 - .../ops/cpp/hstu_attention/flash_api.cpp | 322 -- .../ops/cpp/hstu_attention/flash_api_cpu.cpp | 256 -- .../hstu_attention/flash_bwd_kernel_sm90.h | 402 --- .../flash_bwd_launch_template.h | 492 --- .../flash_bwd_postprocess_kernel.h | 348 -- .../flash_bwd_preprocess_kernel.h | 349 -- .../ops/cpp/hstu_attention/flash_common.cpp | 1165 ------ .../ops/cpp/hstu_attention/flash_common.h | 149 - .../cpp/hstu_attention/flash_common_cpu.cpp | 172 - .../ops/cpp/hstu_attention/flash_common_cpu.h | 114 - .../hstu_attention/flash_fwd_kernel_sm90.h | 511 --- .../flash_fwd_launch_template.h | 376 -- .../cpp/hstu_attention/generate_kernels.py | 236 -- ...lash_bwd_hdim128_bf16_softmaxfalse_sm90.cu | 33 - ...flash_bwd_hdim128_bf16_softmaxtrue_sm90.cu | 33 - ...lash_bwd_hdim128_fp16_softmaxfalse_sm90.cu | 33 - ...flash_bwd_hdim128_fp16_softmaxtrue_sm90.cu | 33 - ...lash_bwd_hdim192_bf16_softmaxfalse_sm90.cu | 33 - ...flash_bwd_hdim192_bf16_softmaxtrue_sm90.cu | 33 - ...lash_bwd_hdim192_fp16_softmaxfalse_sm90.cu | 33 - ...flash_bwd_hdim192_fp16_softmaxtrue_sm90.cu | 33 - ...lash_bwd_hdim256_bf16_softmaxfalse_sm90.cu | 33 - ...flash_bwd_hdim256_bf16_softmaxtrue_sm90.cu | 33 - ...lash_bwd_hdim256_fp16_softmaxfalse_sm90.cu | 33 - ...flash_bwd_hdim256_fp16_softmaxtrue_sm90.cu | 33 - ...flash_bwd_hdim64_bf16_softmaxfalse_sm90.cu | 33 - .../flash_bwd_hdim64_bf16_softmaxtrue_sm90.cu | 33 - ...flash_bwd_hdim64_fp16_softmaxfalse_sm90.cu | 33 - .../flash_bwd_hdim64_fp16_softmaxtrue_sm90.cu | 33 - ...flash_bwd_hdim96_bf16_softmaxfalse_sm90.cu | 33 - .../flash_bwd_hdim96_bf16_softmaxtrue_sm90.cu | 33 - ...flash_bwd_hdim96_fp16_softmaxfalse_sm90.cu | 33 - .../flash_bwd_hdim96_fp16_softmaxtrue_sm90.cu | 33 - ...lash_fwd_hdim128_bf16_softmaxfalse_sm90.cu | 33 - ...flash_fwd_hdim128_bf16_softmaxtrue_sm90.cu | 33 - ...lash_fwd_hdim128_e4m3_softmaxfalse_sm90.cu | 33 - ...flash_fwd_hdim128_e4m3_softmaxtrue_sm90.cu | 33 - ...lash_fwd_hdim128_fp16_softmaxfalse_sm90.cu | 33 - ...flash_fwd_hdim128_fp16_softmaxtrue_sm90.cu | 33 - ...lash_fwd_hdim192_bf16_softmaxfalse_sm90.cu | 33 - ...flash_fwd_hdim192_bf16_softmaxtrue_sm90.cu | 33 - ...lash_fwd_hdim192_e4m3_softmaxfalse_sm90.cu | 33 - ...flash_fwd_hdim192_e4m3_softmaxtrue_sm90.cu | 33 - ...lash_fwd_hdim192_fp16_softmaxfalse_sm90.cu | 33 - ...flash_fwd_hdim192_fp16_softmaxtrue_sm90.cu | 33 - ...lash_fwd_hdim256_bf16_softmaxfalse_sm90.cu | 33 - ...flash_fwd_hdim256_bf16_softmaxtrue_sm90.cu | 33 - ...lash_fwd_hdim256_e4m3_softmaxfalse_sm90.cu | 33 - ...flash_fwd_hdim256_e4m3_softmaxtrue_sm90.cu | 33 - ...lash_fwd_hdim256_fp16_softmaxfalse_sm90.cu | 33 - ...flash_fwd_hdim256_fp16_softmaxtrue_sm90.cu | 33 - ...flash_fwd_hdim64_bf16_softmaxfalse_sm90.cu | 33 - .../flash_fwd_hdim64_bf16_softmaxtrue_sm90.cu | 33 - ...flash_fwd_hdim64_e4m3_softmaxfalse_sm90.cu | 33 - .../flash_fwd_hdim64_e4m3_softmaxtrue_sm90.cu | 33 - ...flash_fwd_hdim64_fp16_softmaxfalse_sm90.cu | 33 - .../flash_fwd_hdim64_fp16_softmaxtrue_sm90.cu | 33 - ...flash_fwd_hdim96_bf16_softmaxfalse_sm90.cu | 33 - .../flash_fwd_hdim96_bf16_softmaxtrue_sm90.cu | 33 - ...flash_fwd_hdim96_e4m3_softmaxfalse_sm90.cu | 33 - .../flash_fwd_hdim96_e4m3_softmaxtrue_sm90.cu | 33 - ...flash_fwd_hdim96_fp16_softmaxfalse_sm90.cu | 33 - .../flash_fwd_hdim96_fp16_softmaxtrue_sm90.cu | 33 - .../mainloop_bwd_sm90_tma_gmma_ws.h | 3166 ----------------- .../mainloop_fwd_sm90_tma_gmma_ws.h | 2180 ------------ .../ops/cpp/hstu_attention/mask.h | 396 --- .../ops/cpp/hstu_attention/named_barrier.h | 101 - .../ops/cpp/hstu_attention/seqlen.h | 134 - .../hstu_attention/sm90_pipeline_no_cluster.h | 150 - .../ops/cpp/hstu_attention/softmax.h | 256 -- .../ops/cpp/hstu_attention/static_switch.h | 135 - .../ops/cpp/hstu_attention/tile_scheduler.h | 616 ---- .../ops/cpp/hstu_attention/tile_size.h | 220 -- .../ops/cpp/hstu_attention/utils.h | 789 ---- .../ops/cpp/hstu_attention/version.txt | 1 - .../ops/cpp/jagged_transpose_1d.cpp | 130 - .../ops/cpp/jagged_transpose_1d.cu | 127 - .../ops/cpp/replace_last_n_with_jagged.cpp | 139 - .../ops/cpp/replace_last_n_with_jagged.cu | 156 - .../generative_recommenders/ops/cpp/setup.py | 487 --- .../ops/cpp/sort_kv_pairs_cuda.cpp | 40 - .../sort_kv_pairs_cuda_kernels_template.cu | 82 - .../cpp/sort_kv_pairs_cuda_kernels_template.h | 15 - .../ops/cpp/split_1d_jagged_jagged.cpp | 136 - .../ops/cpp/split_1d_jagged_jagged.cu | 147 - .../cpp/tests/concat_1d_jagged_jagged_test.py | 135 - .../ops/cpp/tests/hstu_mha_cpu_test.py | 39 - .../ops/cpp/tests/jagged_transpose_1d_test.py | 132 - .../tests/replace_last_n_with_jagged_test.py | 105 - .../cpp/tests/split_1d_jagged_jagged_test.py | 100 - .../ops/triton_aot/README.md | 54 - .../ops/triton_aot/compile/arg_descriptor.py | 146 - .../ops/triton_aot/compile/codegen.py | 780 ---- .../ops/triton_aot/compile/compile_state.py | 409 --- .../ops/triton_aot/compile/pipeline.py | 300 -- .../ops/triton_aot/compile/spec_processing.py | 593 --- .../ops/triton_aot/compile/stable_types.py | 35 - .../triton_aot/compile/triton_aot_compile.py | 149 - .../ops/triton_aot/compile/utils.py | 47 - .../ops/triton_aot/preprocess.py | 76 - .../ops/triton_aot/shared/compat.py | 91 - .../ops/triton_aot/shared/spec_conversion.py | 389 -- .../ops/triton_aot/shared/types.py | 58 - .../triton_aot/templates/embedded_cubins.cpp | 7 - .../ops/triton_aot/templates/kernel.cpp | 104 - .../ops/triton_aot/templates/kernel.h | 36 - .../triton_aot/templates/template_utils.py | 96 - .../ops/triton_aot/templates/torch_op.cpp | 22 - .../ops/triton_aot/transform/import_utils.py | 89 - .../transform/kernel_wrapper_codegen.py | 500 --- .../triton_aot/transform/replace_kernels.py | 137 - .../triton_aot/transform/transform_kernels.py | 29 - .../ops/triton_aot/triton_addmm.py | 347 -- .../ops/triton_aot/triton_concat_2d_jagged.py | 183 - .../triton_group_norm_mul_dropout.py | 124 - .../ops/triton_aot/triton_layer_norm.py | 119 - .../triton_layer_norm_mul_dropout.py | 162 - .../ops/triton_aot/triton_position.py | 176 - .../triton_ragged_hstu_attention.py | 366 -- .../ops/triton_aot/triton_rms_norm.py | 114 - .../ops/triton_aot/triton_split_2d_jagged.py | 138 - .../ops/triton_aot/types.py | 181 - .../research/data/dataset.py | 248 -- .../research/data/eval.py | 263 -- .../research/data/item_features.py | 29 - .../research/data/preprocessor.py | 474 --- .../research/data/reco_dataset.py | 176 - .../research/indexing/candidate_index.py | 179 - .../research/indexing/utils.py | 43 - .../research/modeling/initialization.py | 35 - .../sequential/autoregressive_losses.py | 477 --- .../modeling/sequential/embedding_modules.py | 108 - .../modeling/sequential/encoder_utils.py | 150 - .../research/modeling/sequential/features.py | 94 - .../research/modeling/sequential/hstu.py | 808 ----- .../input_features_preprocessors.py | 259 -- .../sequential/losses/sampled_softmax.py | 193 - .../sequential/output_postprocessors.py | 82 - .../research/modeling/sequential/sasrec.py | 316 -- .../research/modeling/sequential/utils.py | 129 - .../research/modeling/similarity_module.py | 68 - .../research/modeling/similarity_utils.py | 222 -- .../rails/indexing/candidate_index.py | 41 - .../research/rails/indexing/mips_top_k.py | 80 - .../research/rails/indexing/mol_top_k.py | 132 - .../similarities/dot_product_similarity_fn.py | 68 - .../research/rails/similarities/layers.py | 82 - .../research/rails/similarities/module.py | 55 - .../rails/similarities/mol/embeddings_fn.py | 52 - .../similarities/mol/item_embeddings_fn.py | 99 - .../similarities/mol/query_embeddings_fn.py | 164 - .../rails/similarities/mol/similarity_fn.py | 388 -- .../research/trainer/data_loader.py | 57 - .../research/trainer/train.py | 532 --- recommendation_v4/main.py | 82 - recommendation_v4/preprocess_public_data.py | 32 - recommendation_v4/run_fractal_expansion.py | 588 --- 294 files changed, 55303 deletions(-) delete mode 100644 recommendation_v4/configs/amzn-books/hstu-sampled-softmax-n512-final.gin delete mode 100644 recommendation_v4/configs/amzn-books/hstu-sampled-softmax-n512-large-final.gin delete mode 100644 recommendation_v4/configs/amzn-books/sasrec-sampled-softmax-n512-final.gin delete mode 100644 recommendation_v4/configs/ml-1m/hstu-sampled-softmax-n128-final.gin delete mode 100644 recommendation_v4/configs/ml-1m/hstu-sampled-softmax-n128-large-final.gin delete mode 100644 recommendation_v4/configs/ml-1m/sasrec-sampled-softmax-n128-final.gin delete mode 100644 recommendation_v4/configs/ml-20m/hstu-sampled-softmax-n128-final.gin delete mode 100644 recommendation_v4/configs/ml-20m/hstu-sampled-softmax-n128-large-final.gin delete mode 100644 recommendation_v4/configs/ml-20m/sasrec-sampled-softmax-n128-final.gin delete mode 100644 recommendation_v4/configs/ml-3b/hstu-sampled-softmax-n96-seqlen500-final.gin delete mode 100644 recommendation_v4/configs/ml-3b/hstu-sampled-softmax-n96-seqlen500-large-final.gin delete mode 100644 recommendation_v4/configs/ml-3b/sasrec-sampled-softmax-n96-seqlen500-final.gin delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/README.md delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/accuracy.py delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/cpp/hstu_runner.cpp delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/data_producer.py delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/dense_predict_module.py delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/end_to_end_test.py delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/gin/debug.gin delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/gin/kuairand_1k.gin delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/gin/movielens_13b.gin delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/gin/streaming_100b.gin delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/gin/streaming_400m.gin delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/inference_modules.py delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/main.py delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/mlperf.conf delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/model_family.py delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/sparse_predict_module.py delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/tests/inference_test.py delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/tests/test_scripted_parity.py delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/.clang-format delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/CMakeLists.txt delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/MANIFEST.in delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/README.md delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/README_BUILD.md delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/README_FAQ.md delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/VERSION.txt delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/benchmark/.gitignore delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/benchmark/README.md delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/benchmark/repro.cpp delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/benchmark/run.sh delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/benchmark/run_debug.sh delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/bindings/c_api.cc delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/bindings/c_api.h delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/bindings/python_api.cc delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/lon/README.md delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/lon/py_demo_server_lon.py delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/lon/sut_over_network_demo.py delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/py_demo_multi_stream.py delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/py_demo_offline.py delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/py_demo_server.py delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/py_demo_single_stream.py delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_multi_stream.py delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_offline.py delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_offline_inferred.py delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_server.py delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_server_inferred.py delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_single_stream.py delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/diagram_network_submission.png delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/diagram_submission.png delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/BUILD.gn delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/README.md delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/doxygen.cfg delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/doxygen_footer.html delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/doxygen_header.html delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/doxygen_html_generator.py delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/doxygen_layout.xml delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/doxygen_stylesheet.css delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/loadgen_integration_diagram.dia delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/mlperf_icon.png delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/mlperf_logo_horizontal_color.svg delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/early_stopping.cc delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/early_stopping.h delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/generated/version_generated.cc delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/issue_query_controller.cc delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/issue_query_controller.h delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/loadgen.cc delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/loadgen.h delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/loadgen_integration_diagram.svg delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/logging.cc delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/logging.h delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/mlperf.conf delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/mlperf_conf.h delete mode 100755 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/pyproject.toml delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/query_dispatch_library.h delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/query_sample.h delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/query_sample_library.h delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/requirements.txt delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/results.cc delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/results.h delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/setup.py delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/system_under_test.h delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/test_settings.h delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/test_settings_internal.cc delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/test_settings_internal.h delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/BUILD.gn delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/README.md delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/basic.cc delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/loadgen_test.h delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/loadgen_test_main.cc delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/perftests_null_sut.cc delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/perftests_null_sut.py delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tools/mlperf-trace.ipynb delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/utils.cc delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/utils.h delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/version.cc delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/version.h delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/version_generator.py delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/ts_types.py delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/inference/user.conf delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/train/gin/kuairand_1k.gin delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/train/gin/movielens_13b.gin delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/train/gin/movielens_18b.gin delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/train/gin/movielens_1m.gin delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/train/gin/movielens_20m.gin delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/train/gin/streaming_100b.gin delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/train/gin/streaming_200b.gin delete mode 100644 recommendation_v4/generative_recommenders/dlrm_v3/train/gin/streaming_400m.gin delete mode 100644 recommendation_v4/generative_recommenders/ops/benchmarks/hstu_attention_bench.py delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/benchmarks/concat_1d_jagged_jagged_bench.py delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/benchmarks/jagged_transpose_1d_bench.py delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/benchmarks/replace_last_n_with_jagged_bench.py delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/benchmarks/split_1d_jagged_jagged_bench.py delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/common.h delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/complete_cumsum.cpp delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/complete_cumsum.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/concat_1d_jagged_jagged.cpp delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/concat_1d_jagged_jagged.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/cpp_ops.cpp delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/cuda_hstu_attention.py delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/cuda_hstu_preprocess_and_attention.py delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/expand_1d_jagged_to_dense.cpp delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/expand_1d_jagged_to_dense.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/copy_sm90_bulk_reduce.h delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/epilogue_bwd.h delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/epilogue_fwd.h delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash.h delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_api.cpp delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_api_cpu.cpp delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_bwd_kernel_sm90.h delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_bwd_launch_template.h delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_bwd_postprocess_kernel.h delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_bwd_preprocess_kernel.h delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_common.cpp delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_common.h delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_common_cpu.cpp delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_common_cpu.h delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_fwd_kernel_sm90.h delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_fwd_launch_template.h delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/generate_kernels.py delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim128_bf16_softmaxfalse_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim128_bf16_softmaxtrue_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim128_fp16_softmaxfalse_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim128_fp16_softmaxtrue_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim192_bf16_softmaxfalse_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim192_bf16_softmaxtrue_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim192_fp16_softmaxfalse_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim192_fp16_softmaxtrue_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim256_bf16_softmaxfalse_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim256_bf16_softmaxtrue_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim256_fp16_softmaxfalse_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim256_fp16_softmaxtrue_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim64_bf16_softmaxfalse_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim64_bf16_softmaxtrue_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim64_fp16_softmaxfalse_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim64_fp16_softmaxtrue_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim96_bf16_softmaxfalse_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim96_bf16_softmaxtrue_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim96_fp16_softmaxfalse_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim96_fp16_softmaxtrue_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_bf16_softmaxfalse_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_bf16_softmaxtrue_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_e4m3_softmaxfalse_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_e4m3_softmaxtrue_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_fp16_softmaxfalse_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_fp16_softmaxtrue_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_bf16_softmaxfalse_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_bf16_softmaxtrue_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_e4m3_softmaxfalse_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_e4m3_softmaxtrue_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_fp16_softmaxfalse_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_fp16_softmaxtrue_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_bf16_softmaxfalse_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_bf16_softmaxtrue_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_e4m3_softmaxfalse_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_e4m3_softmaxtrue_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_fp16_softmaxfalse_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_fp16_softmaxtrue_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_bf16_softmaxfalse_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_bf16_softmaxtrue_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_e4m3_softmaxfalse_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_e4m3_softmaxtrue_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_fp16_softmaxfalse_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_fp16_softmaxtrue_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_bf16_softmaxfalse_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_bf16_softmaxtrue_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_e4m3_softmaxfalse_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_e4m3_softmaxtrue_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_fp16_softmaxfalse_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_fp16_softmaxtrue_sm90.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/mainloop_bwd_sm90_tma_gmma_ws.h delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/mainloop_fwd_sm90_tma_gmma_ws.h delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/mask.h delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/named_barrier.h delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/seqlen.h delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/sm90_pipeline_no_cluster.h delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/softmax.h delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/static_switch.h delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/tile_scheduler.h delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/tile_size.h delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/utils.h delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/version.txt delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/jagged_transpose_1d.cpp delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/jagged_transpose_1d.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/replace_last_n_with_jagged.cpp delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/replace_last_n_with_jagged.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/setup.py delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/sort_kv_pairs_cuda.cpp delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/sort_kv_pairs_cuda_kernels_template.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/sort_kv_pairs_cuda_kernels_template.h delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/split_1d_jagged_jagged.cpp delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/split_1d_jagged_jagged.cu delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/tests/concat_1d_jagged_jagged_test.py delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/tests/hstu_mha_cpu_test.py delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/tests/jagged_transpose_1d_test.py delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/tests/replace_last_n_with_jagged_test.py delete mode 100644 recommendation_v4/generative_recommenders/ops/cpp/tests/split_1d_jagged_jagged_test.py delete mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/README.md delete mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/compile/arg_descriptor.py delete mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/compile/codegen.py delete mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/compile/compile_state.py delete mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/compile/pipeline.py delete mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/compile/spec_processing.py delete mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/compile/stable_types.py delete mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/compile/triton_aot_compile.py delete mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/compile/utils.py delete mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/preprocess.py delete mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/shared/compat.py delete mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/shared/spec_conversion.py delete mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/shared/types.py delete mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/templates/embedded_cubins.cpp delete mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/templates/kernel.cpp delete mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/templates/kernel.h delete mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/templates/template_utils.py delete mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/templates/torch_op.cpp delete mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/transform/import_utils.py delete mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/transform/kernel_wrapper_codegen.py delete mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/transform/replace_kernels.py delete mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/transform/transform_kernels.py delete mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/triton_addmm.py delete mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/triton_concat_2d_jagged.py delete mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/triton_group_norm_mul_dropout.py delete mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/triton_layer_norm.py delete mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/triton_layer_norm_mul_dropout.py delete mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/triton_position.py delete mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/triton_ragged_hstu_attention.py delete mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/triton_rms_norm.py delete mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/triton_split_2d_jagged.py delete mode 100644 recommendation_v4/generative_recommenders/ops/triton_aot/types.py delete mode 100644 recommendation_v4/generative_recommenders/research/data/dataset.py delete mode 100644 recommendation_v4/generative_recommenders/research/data/eval.py delete mode 100644 recommendation_v4/generative_recommenders/research/data/item_features.py delete mode 100644 recommendation_v4/generative_recommenders/research/data/preprocessor.py delete mode 100644 recommendation_v4/generative_recommenders/research/data/reco_dataset.py delete mode 100644 recommendation_v4/generative_recommenders/research/indexing/candidate_index.py delete mode 100644 recommendation_v4/generative_recommenders/research/indexing/utils.py delete mode 100644 recommendation_v4/generative_recommenders/research/modeling/initialization.py delete mode 100644 recommendation_v4/generative_recommenders/research/modeling/sequential/autoregressive_losses.py delete mode 100644 recommendation_v4/generative_recommenders/research/modeling/sequential/embedding_modules.py delete mode 100644 recommendation_v4/generative_recommenders/research/modeling/sequential/encoder_utils.py delete mode 100644 recommendation_v4/generative_recommenders/research/modeling/sequential/features.py delete mode 100644 recommendation_v4/generative_recommenders/research/modeling/sequential/hstu.py delete mode 100644 recommendation_v4/generative_recommenders/research/modeling/sequential/input_features_preprocessors.py delete mode 100644 recommendation_v4/generative_recommenders/research/modeling/sequential/losses/sampled_softmax.py delete mode 100644 recommendation_v4/generative_recommenders/research/modeling/sequential/output_postprocessors.py delete mode 100644 recommendation_v4/generative_recommenders/research/modeling/sequential/sasrec.py delete mode 100644 recommendation_v4/generative_recommenders/research/modeling/sequential/utils.py delete mode 100644 recommendation_v4/generative_recommenders/research/modeling/similarity_module.py delete mode 100644 recommendation_v4/generative_recommenders/research/modeling/similarity_utils.py delete mode 100644 recommendation_v4/generative_recommenders/research/rails/indexing/candidate_index.py delete mode 100644 recommendation_v4/generative_recommenders/research/rails/indexing/mips_top_k.py delete mode 100644 recommendation_v4/generative_recommenders/research/rails/indexing/mol_top_k.py delete mode 100644 recommendation_v4/generative_recommenders/research/rails/similarities/dot_product_similarity_fn.py delete mode 100644 recommendation_v4/generative_recommenders/research/rails/similarities/layers.py delete mode 100644 recommendation_v4/generative_recommenders/research/rails/similarities/module.py delete mode 100644 recommendation_v4/generative_recommenders/research/rails/similarities/mol/embeddings_fn.py delete mode 100644 recommendation_v4/generative_recommenders/research/rails/similarities/mol/item_embeddings_fn.py delete mode 100644 recommendation_v4/generative_recommenders/research/rails/similarities/mol/query_embeddings_fn.py delete mode 100644 recommendation_v4/generative_recommenders/research/rails/similarities/mol/similarity_fn.py delete mode 100644 recommendation_v4/generative_recommenders/research/trainer/data_loader.py delete mode 100644 recommendation_v4/generative_recommenders/research/trainer/train.py delete mode 100644 recommendation_v4/main.py delete mode 100644 recommendation_v4/preprocess_public_data.py delete mode 100644 recommendation_v4/run_fractal_expansion.py diff --git a/recommendation_v4/configs/amzn-books/hstu-sampled-softmax-n512-final.gin b/recommendation_v4/configs/amzn-books/hstu-sampled-softmax-n512-final.gin deleted file mode 100644 index 8fb8b258c..000000000 --- a/recommendation_v4/configs/amzn-books/hstu-sampled-softmax-n512-final.gin +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Frozen config, validated on 04/12/2024. -# Based on HSTU results (w/ identical configurations as a SotA Transformer baseline) in -# Actions Speak Louder than Words: Trillion-Parameter Sequential Transducers for Generative Recommendations (https://arxiv.org/abs/2402.17152). -# -# Run this as: -# mkdir -p logs/amzn-books-l50/ -# CUDA_VISIBLE_DEVICES=1 python3 main.py --gin_config_file=configs/amzn-books/hstu-sampled-softmax-n512-final.gin --master_port=12346 2>&1 | tee logs/amzn-books-l50/hstu-sampled-softmax-n512-final.log - -train_fn.dataset_name = "amzn-books" -train_fn.max_sequence_length = 50 -train_fn.local_batch_size = 128 -train_fn.eval_batch_size = 128 - -train_fn.main_module = "HSTU" -train_fn.dropout_rate = 0.5 -train_fn.user_embedding_norm = "l2_norm" -train_fn.item_embedding_dim = 64 - -hstu_encoder.num_blocks = 4 -hstu_encoder.num_heads = 4 -hstu_encoder.dv = 16 -hstu_encoder.dqk = 16 -hstu_encoder.linear_dropout_rate = 0.5 - -train_fn.eval_interval = 4000 -train_fn.num_epochs = 201 -train_fn.learning_rate = 1e-3 -train_fn.weight_decay = 0 -train_fn.num_warmup_steps = 0 - -train_fn.interaction_module_type = "DotProduct" -train_fn.top_k_method = "MIPSBruteForceTopK" - -train_fn.loss_module = "SampledSoftmaxLoss" -train_fn.num_negatives = 512 - -train_fn.sampling_strategy = "local" -train_fn.temperature = 0.05 -train_fn.item_l2_norm = True -train_fn.l2_norm_eps = 1e-6 - -train_fn.enable_tf32 = True -train_fn.full_eval_every_n = 5 -train_fn.partial_eval_num_iters = 64 - -create_data_loader.prefetch_factor = 1024 -create_data_loader.num_workers = 8 diff --git a/recommendation_v4/configs/amzn-books/hstu-sampled-softmax-n512-large-final.gin b/recommendation_v4/configs/amzn-books/hstu-sampled-softmax-n512-large-final.gin deleted file mode 100644 index 097d4cbc7..000000000 --- a/recommendation_v4/configs/amzn-books/hstu-sampled-softmax-n512-large-final.gin +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Frozen config, validated on 04/12/2024. -# Based on HSTU-large results in -# Actions Speak Louder than Words: Trillion-Parameter Sequential Transducers for Generative Recommendations (https://arxiv.org/abs/2402.17152). -# -# Run this as: -# mkdir -p logs/amzn-books-l50/ -# CUDA_VISIBLE_DEVICES=1 python3 main.py --gin_config_file=configs/amzn-books/hstu-sampled-softmax-n512-large-final.gin --master_port=12346 2>&1 | tee logs/amzn-books-l50/hstu-sampled-softmax-n512-large-final2.log - -train_fn.dataset_name = "amzn-books" -train_fn.max_sequence_length = 50 -train_fn.local_batch_size = 128 -train_fn.eval_batch_size = 128 - -train_fn.main_module = "HSTU" -train_fn.dropout_rate = 0.5 -train_fn.user_embedding_norm = "l2_norm" -train_fn.item_embedding_dim = 64 - -hstu_encoder.num_blocks = 16 -hstu_encoder.num_heads = 8 -hstu_encoder.dv = 8 -hstu_encoder.dqk = 8 -hstu_encoder.linear_dropout_rate = 0.5 - -train_fn.eval_interval = 4000 -train_fn.num_epochs = 201 -train_fn.learning_rate = 1e-3 -train_fn.weight_decay = 0 -train_fn.num_warmup_steps = 0 - -train_fn.interaction_module_type = "DotProduct" -train_fn.top_k_method = "MIPSBruteForceTopK" - -train_fn.loss_module = "SampledSoftmaxLoss" -train_fn.num_negatives = 512 - -train_fn.sampling_strategy = "local" -train_fn.temperature = 0.05 -train_fn.item_l2_norm = True -train_fn.l2_norm_eps = 1e-6 - -train_fn.enable_tf32 = True -train_fn.full_eval_every_n = 5 -train_fn.partial_eval_num_iters = 64 - -create_data_loader.prefetch_factor = 1024 -create_data_loader.num_workers = 8 diff --git a/recommendation_v4/configs/amzn-books/sasrec-sampled-softmax-n512-final.gin b/recommendation_v4/configs/amzn-books/sasrec-sampled-softmax-n512-final.gin deleted file mode 100644 index bc899c9fb..000000000 --- a/recommendation_v4/configs/amzn-books/sasrec-sampled-softmax-n512-final.gin +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Frozen config, validated on 04/12/2024. -# Based on baseline settings in Revisiting Neural Retrieval on Accelerators (https://arxiv.org/abs/2306.04039, KDD'23). -# -# Run this as: -# mkdir -p logs/amzn-books-l50/ -# CUDA_VISIBLE_DEVICES=1 python3 main.py --gin_config_file=configs/amzn-books/sasrec-sampled-softmax-n512-final.gin --master_port=12346 2>&1 | tee logs/amzn-books-l50/sasrec-sampled-softmax-n512-final.log - -train_fn.dataset_name = "amzn-books" -train_fn.max_sequence_length = 50 -train_fn.local_batch_size = 128 -train_fn.eval_batch_size = 128 - -train_fn.main_module = "SASRec" -train_fn.dropout_rate = 0.5 -train_fn.user_embedding_norm = "l2_norm" -train_fn.item_embedding_dim = 64 - -sasrec_encoder.num_blocks = 4 -sasrec_encoder.num_heads = 4 -sasrec_encoder.ffn_dropout_rate = 0.5 -sasrec_encoder.ffn_hidden_dim = 64 -sasrec_encoder.ffn_activation_fn = "relu" - -train_fn.eval_interval = 4000 -train_fn.num_epochs = 201 -train_fn.learning_rate = 1e-3 -train_fn.weight_decay = 0 -train_fn.num_warmup_steps = 0 - -train_fn.save_ckpt_every_n = 10 - -train_fn.interaction_module_type = "DotProduct" -train_fn.top_k_method = "MIPSBruteForceTopK" - -train_fn.loss_module = "SampledSoftmaxLoss" -train_fn.num_negatives = 512 - -train_fn.sampling_strategy = "local" -train_fn.temperature = 0.05 -train_fn.item_l2_norm = True -train_fn.l2_norm_eps = 1e-6 - -train_fn.enable_tf32 = True -train_fn.full_eval_every_n = 5 -train_fn.partial_eval_num_iters = 64 - -create_data_loader.prefetch_factor = 1024 -create_data_loader.num_workers = 8 diff --git a/recommendation_v4/configs/ml-1m/hstu-sampled-softmax-n128-final.gin b/recommendation_v4/configs/ml-1m/hstu-sampled-softmax-n128-final.gin deleted file mode 100644 index 841b1c80a..000000000 --- a/recommendation_v4/configs/ml-1m/hstu-sampled-softmax-n128-final.gin +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Frozen config, validated on 04/11/2024. -# Based on HSTU results (w/ identical configurations as a SotA Transformer baseline) in -# Actions Speak Louder than Words: Trillion-Parameter Sequential Transducers for Generative Recommendations (https://arxiv.org/abs/2402.17152). -# -# Run this as: -# mkdir -p logs/ml-1m-l200/ -# CUDA_VISIBLE_DEVICES=0 python3 main.py --gin_config_file=configs/ml-1m/hstu-sampled-softmax-n128-final.gin --master_port=12345 2>&1 | tee logs/ml-1m-l200/hstu-sampled-softmax-n128-final.log - -train_fn.dataset_name = "ml-1m" -train_fn.max_sequence_length = 200 -train_fn.local_batch_size = 128 - -train_fn.main_module = "HSTU" -train_fn.dropout_rate = 0.2 -train_fn.user_embedding_norm = "l2_norm" -train_fn.num_epochs = 101 -train_fn.item_embedding_dim = 50 - -hstu_encoder.num_blocks = 2 -hstu_encoder.num_heads = 1 -hstu_encoder.dqk = 50 -hstu_encoder.dv = 50 -hstu_encoder.linear_dropout_rate = 0.2 - -train_fn.learning_rate = 1e-3 -train_fn.weight_decay = 0 -train_fn.num_warmup_steps = 0 - -train_fn.interaction_module_type = "DotProduct" -train_fn.top_k_method = "MIPSBruteForceTopK" - -train_fn.loss_module = "SampledSoftmaxLoss" -train_fn.num_negatives = 128 - -train_fn.sampling_strategy = "local" -train_fn.temperature = 0.05 -train_fn.item_l2_norm = True -train_fn.l2_norm_eps = 1e-6 - -train_fn.enable_tf32 = True - -create_data_loader.prefetch_factor = 128 -create_data_loader.num_workers = 8 diff --git a/recommendation_v4/configs/ml-1m/hstu-sampled-softmax-n128-large-final.gin b/recommendation_v4/configs/ml-1m/hstu-sampled-softmax-n128-large-final.gin deleted file mode 100644 index 7ffc7ef64..000000000 --- a/recommendation_v4/configs/ml-1m/hstu-sampled-softmax-n128-large-final.gin +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Frozen config, validated on 04/11/2024. -# Based on HSTU-large results in -# Actions Speak Louder than Words: Trillion-Parameter Sequential Transducers for Generative Recommendations (https://arxiv.org/abs/2402.17152). -# -# Run this as: -# mkdir -p logs/ml-1m-l200/ -# CUDA_VISIBLE_DEVICES=1 python3 main.py --gin_config_file=configs/ml-1m/hstu-sampled-softmax-n128-large-final.gin --master_port=12346 2>&1 | tee logs/ml-1m-l200/hstu-sampled-softmax-n128-large-final.log - -train_fn.dataset_name = "ml-1m" -train_fn.max_sequence_length = 200 -train_fn.local_batch_size = 128 - -train_fn.main_module = "HSTU" -train_fn.dropout_rate = 0.2 -train_fn.user_embedding_norm = "l2_norm" -train_fn.num_epochs = 101 -train_fn.item_embedding_dim = 50 - -hstu_encoder.num_blocks = 8 -hstu_encoder.num_heads = 2 -hstu_encoder.dqk = 25 -hstu_encoder.dv = 25 -hstu_encoder.linear_dropout_rate = 0.2 - -train_fn.learning_rate = 1e-3 -train_fn.weight_decay = 0 -train_fn.num_warmup_steps = 0 - -train_fn.interaction_module_type = "DotProduct" -train_fn.top_k_method = "MIPSBruteForceTopK" - -train_fn.loss_module = "SampledSoftmaxLoss" -train_fn.num_negatives = 128 - -train_fn.sampling_strategy = "local" -train_fn.temperature = 0.05 -train_fn.item_l2_norm = True -train_fn.l2_norm_eps = 1e-6 - -train_fn.enable_tf32 = True - -create_data_loader.prefetch_factor = 128 -create_data_loader.num_workers = 8 diff --git a/recommendation_v4/configs/ml-1m/sasrec-sampled-softmax-n128-final.gin b/recommendation_v4/configs/ml-1m/sasrec-sampled-softmax-n128-final.gin deleted file mode 100644 index ead7bb21c..000000000 --- a/recommendation_v4/configs/ml-1m/sasrec-sampled-softmax-n128-final.gin +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Frozen config, validated on 04/11/2024. -# Based on baseline settings in Revisiting Neural Retrieval on Accelerators (https://arxiv.org/abs/2306.04039, KDD'23). -# -# Run this as: -# mkdir -p logs/ml-1m-l200/ -# CUDA_VISIBLE_DEVICES=0 python3 main.py --gin_config_file=configs/ml-1m/sasrec-sampled-softmax-n128-final.gin --master_port=12345 2>&1 | tee logs/ml-1m-l200/sasrec-sampled-softmax-n128-final.log - -train_fn.dataset_name = "ml-1m" -train_fn.max_sequence_length = 200 -train_fn.local_batch_size = 128 - -train_fn.main_module = "SASRec" -train_fn.dropout_rate = 0.2 -train_fn.user_embedding_norm = "l2_norm" -train_fn.num_epochs = 101 -train_fn.item_embedding_dim = 50 - -sasrec_encoder.num_blocks = 2 -sasrec_encoder.num_heads = 1 -sasrec_encoder.ffn_dropout_rate = 0.2 -sasrec_encoder.ffn_hidden_dim = 50 -sasrec_encoder.ffn_activation_fn = "relu" - -train_fn.learning_rate = 1e-3 -train_fn.weight_decay = 0 -train_fn.num_warmup_steps = 0 - -train_fn.top_k_method = "MIPSBruteForceTopK" -train_fn.interaction_module_type = "DotProduct" - -train_fn.loss_module = "SampledSoftmaxLoss" -train_fn.num_negatives = 128 - -train_fn.sampling_strategy = "local" -train_fn.temperature = 0.05 -train_fn.item_l2_norm = True -train_fn.l2_norm_eps = 1e-6 - -train_fn.enable_tf32 = True - -create_data_loader.prefetch_factor = 128 -create_data_loader.num_workers = 8 diff --git a/recommendation_v4/configs/ml-20m/hstu-sampled-softmax-n128-final.gin b/recommendation_v4/configs/ml-20m/hstu-sampled-softmax-n128-final.gin deleted file mode 100644 index 5823ad5b6..000000000 --- a/recommendation_v4/configs/ml-20m/hstu-sampled-softmax-n128-final.gin +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Frozen config, validated on 04/12/2024. -# Based on HSTU results (w/ identical configurations as a SotA Transformer baseline) in -# Actions Speak Louder than Words: Trillion-Parameter Sequential Transducers for Generative Recommendations (https://arxiv.org/abs/2402.17152). -# -# Run this as: -# mkdir -p logs/ml-20m-l200/ -# CUDA_VISIBLE_DEVICES=0 python3 main.py --gin_config_file=configs/ml-20m/hstu-sampled-softmax-n128-final.gin --master_port=12345 2>&1 | tee logs/ml-20m-l200/hstu-sampled-softmax-n128-final.log - -train_fn.dataset_name = "ml-20m" -train_fn.max_sequence_length = 200 -train_fn.local_batch_size = 128 - -train_fn.main_module = "HSTU" -train_fn.dropout_rate = 0.2 -train_fn.user_embedding_norm = "l2_norm" -train_fn.num_epochs = 101 -train_fn.item_embedding_dim = 256 - -hstu_encoder.num_blocks = 4 -hstu_encoder.num_heads = 4 -hstu_encoder.dv = 64 -hstu_encoder.dqk = 64 -hstu_encoder.linear_dropout_rate = 0.2 - -train_fn.learning_rate = 1e-3 -train_fn.weight_decay = 0 -train_fn.num_warmup_steps = 0 - -train_fn.interaction_module_type = "DotProduct" -train_fn.top_k_method = "MIPSBruteForceTopK" - -train_fn.loss_module = "SampledSoftmaxLoss" -train_fn.num_negatives = 128 - -train_fn.sampling_strategy = "local" -train_fn.temperature = 0.05 -train_fn.item_l2_norm = True -train_fn.l2_norm_eps = 1e-6 - -train_fn.enable_tf32 = True - -create_data_loader.prefetch_factor = 128 -create_data_loader.num_workers = 8 diff --git a/recommendation_v4/configs/ml-20m/hstu-sampled-softmax-n128-large-final.gin b/recommendation_v4/configs/ml-20m/hstu-sampled-softmax-n128-large-final.gin deleted file mode 100644 index 0199afa24..000000000 --- a/recommendation_v4/configs/ml-20m/hstu-sampled-softmax-n128-large-final.gin +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Frozen config, validated on 04/12/2024. -# Based on HSTU-large results in -# Actions Speak Louder than Words: Trillion-Parameter Sequential Transducers for Generative Recommendations (https://arxiv.org/abs/2402.17152). -# -# Run this as: -# mkdir -p logs/ml-20m-l200/ -# CUDA_VISIBLE_DEVICES=0 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True python3 main.py --gin_config_file=configs/ml-20m/hstu-sampled-softmax-n128-large-final.gin --master_port=12345 2>&1 | tee logs/ml-20m-l200/hstu-sampled-softmax-n128-large-final.log - -train_fn.dataset_name = "ml-20m" -train_fn.max_sequence_length = 200 -train_fn.local_batch_size = 128 - -train_fn.main_module = "HSTU" -train_fn.dropout_rate = 0.2 -train_fn.user_embedding_norm = "l2_norm" -train_fn.num_epochs = 101 -train_fn.item_embedding_dim = 256 - -hstu_encoder.num_blocks = 16 -hstu_encoder.num_heads = 8 -hstu_encoder.dv = 32 -hstu_encoder.dqk = 32 -hstu_encoder.linear_dropout_rate = 0.2 - -train_fn.learning_rate = 1e-3 -train_fn.weight_decay = 0 -train_fn.num_warmup_steps = 0 - -train_fn.interaction_module_type = "DotProduct" -train_fn.top_k_method = "MIPSBruteForceTopK" - -train_fn.loss_module = "SampledSoftmaxLoss" -train_fn.num_negatives = 128 - -train_fn.sampling_strategy = "local" -train_fn.temperature = 0.05 -train_fn.item_l2_norm = True -train_fn.l2_norm_eps = 1e-6 - -train_fn.enable_tf32 = True - -create_data_loader.prefetch_factor = 128 -create_data_loader.num_workers = 8 diff --git a/recommendation_v4/configs/ml-20m/sasrec-sampled-softmax-n128-final.gin b/recommendation_v4/configs/ml-20m/sasrec-sampled-softmax-n128-final.gin deleted file mode 100644 index 3c666f802..000000000 --- a/recommendation_v4/configs/ml-20m/sasrec-sampled-softmax-n128-final.gin +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Frozen config, validated on 04/12/2024. -# Based on baseline settings in Revisiting Neural Retrieval on Accelerators (https://arxiv.org/abs/2306.04039, KDD'23). -# -# Run this as: -# mkdir -p logs/ml-20m-l200/ -# CUDA_VISIBLE_DEVICES=0 python3 main.py --gin_config_file=configs/ml-20m/sasrec-sampled-softmax-n128-final.gin --master_port=12345 2>&1 | tee logs/ml-20m-l200/sasrec-sampled-softmax-n128-final.log - -train_fn.dataset_name = "ml-20m" -train_fn.max_sequence_length = 200 -train_fn.local_batch_size = 128 - -train_fn.main_module = "SASRec" -train_fn.dropout_rate = 0.2 -train_fn.user_embedding_norm = "l2_norm" -train_fn.num_epochs = 101 -train_fn.item_embedding_dim = 256 - -sasrec_encoder.num_blocks = 4 -sasrec_encoder.num_heads = 4 -sasrec_encoder.ffn_dropout_rate = 0.2 -sasrec_encoder.ffn_hidden_dim = 256 -sasrec_encoder.ffn_activation_fn = "relu" - -train_fn.learning_rate = 1e-3 -train_fn.weight_decay = 0 -train_fn.num_warmup_steps = 0 - -train_fn.top_k_method = "MIPSBruteForceTopK" -train_fn.interaction_module_type = "DotProduct" - -train_fn.loss_module = "SampledSoftmaxLoss" -train_fn.num_negatives = 128 - -train_fn.sampling_strategy = "local" -train_fn.temperature = 0.05 -train_fn.item_l2_norm = True -train_fn.l2_norm_eps = 1e-6 - -train_fn.enable_tf32 = True - -create_data_loader.prefetch_factor = 128 -create_data_loader.num_workers = 8 diff --git a/recommendation_v4/configs/ml-3b/hstu-sampled-softmax-n96-seqlen500-final.gin b/recommendation_v4/configs/ml-3b/hstu-sampled-softmax-n96-seqlen500-final.gin deleted file mode 100644 index ac7a85350..000000000 --- a/recommendation_v4/configs/ml-3b/hstu-sampled-softmax-n96-seqlen500-final.gin +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Run this as: -# mkdir -p logs/ml-3b-l500/ -# CUDA_VISIBLE_DEVICES=0 python3 main.py --gin_config_file=configs/ml-3b/hstu-sampled-softmax-n96-seqlen500-final.gin --master_port=12345 2>&1 | tee logs/ml-3b-l500/hstu-sampled-softmax-n96-seqlen500-final.log - -train_fn.dataset_name = "ml-3b" -train_fn.max_sequence_length = 500 -train_fn.local_batch_size = 96 -train_fn.eval_batch_size = 96 - -train_fn.main_module = "HSTU" -train_fn.dropout_rate = 0.2 -train_fn.user_embedding_norm = "l2_norm" -train_fn.num_epochs = 100 -train_fn.item_embedding_dim = 256 - -hstu_encoder.num_blocks = 4 -hstu_encoder.num_heads = 4 -hstu_encoder.dv = 64 -hstu_encoder.dqk = 64 -hstu_encoder.linear_dropout_rate = 0.2 - -train_fn.learning_rate = 1e-3 -train_fn.weight_decay = 0 -train_fn.num_warmup_steps = 0 - -train_fn.interaction_module_type = "DotProduct" -train_fn.top_k_method = "MIPSBruteForceTopK" - -train_fn.loss_module = "SampledSoftmaxLoss" -train_fn.num_negatives = 128 - -train_fn.sampling_strategy = "local" -train_fn.temperature = 0.05 -train_fn.item_l2_norm = True -train_fn.l2_norm_eps = 1e-6 - -train_fn.enable_tf32 = True - -create_data_loader.prefetch_factor = 128 -create_data_loader.num_workers = 8 diff --git a/recommendation_v4/configs/ml-3b/hstu-sampled-softmax-n96-seqlen500-large-final.gin b/recommendation_v4/configs/ml-3b/hstu-sampled-softmax-n96-seqlen500-large-final.gin deleted file mode 100644 index a30ad3657..000000000 --- a/recommendation_v4/configs/ml-3b/hstu-sampled-softmax-n96-seqlen500-large-final.gin +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Run this as: -# mkdir -p logs/ml-3b-l500/ -# CUDA_VISIBLE_DEVICES=0 python3 main.py --gin_config_file=configs/ml-3b/hstu-sampled-softmax-n96-seqlen500-large-final.gin --master_port=12345 2>&1 | tee logs/ml-3b-l500/hstu-sampled-softmax-n96-seqlen500-large-final.log - -train_fn.dataset_name = "ml-3b" -train_fn.max_sequence_length = 500 -train_fn.local_batch_size = 96 -train_fn.eval_batch_size = 96 - -train_fn.main_module = "HSTU" -train_fn.dropout_rate = 0.2 -train_fn.user_embedding_norm = "l2_norm" -train_fn.num_epochs = 100 -train_fn.item_embedding_dim = 256 - -hstu_encoder.num_blocks = 16 -hstu_encoder.num_heads = 8 -hstu_encoder.dv = 32 -hstu_encoder.dqk = 32 -hstu_encoder.linear_dropout_rate = 0.2 - -train_fn.learning_rate = 1e-3 -train_fn.weight_decay = 0 -train_fn.num_warmup_steps = 0 - -train_fn.interaction_module_type = "DotProduct" -train_fn.top_k_method = "MIPSBruteForceTopK" - -train_fn.loss_module = "SampledSoftmaxLoss" -train_fn.num_negatives = 128 - -train_fn.sampling_strategy = "local" -train_fn.temperature = 0.05 -train_fn.item_l2_norm = True -train_fn.l2_norm_eps = 1e-6 - -train_fn.enable_tf32 = True - -create_data_loader.prefetch_factor = 128 -create_data_loader.num_workers = 8 diff --git a/recommendation_v4/configs/ml-3b/sasrec-sampled-softmax-n96-seqlen500-final.gin b/recommendation_v4/configs/ml-3b/sasrec-sampled-softmax-n96-seqlen500-final.gin deleted file mode 100644 index 034c478b4..000000000 --- a/recommendation_v4/configs/ml-3b/sasrec-sampled-softmax-n96-seqlen500-final.gin +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Run this as: -# mkdir -p logs/ml-3b-l500/ -# CUDA_VISIBLE_DEVICES=0 python3 main.py --gin_config_file=configs/ml-3b/sasrec-sampled-softmax-n96-seqlen500-final.gin --master_port=12345 2>&1 | tee logs/ml-3b-l500/sasrec-sampled-softmax-n96-seqlen500-final.log - -train_fn.dataset_name = "ml-3b" -train_fn.max_sequence_length = 500 -train_fn.local_batch_size = 96 -train_fn.eval_batch_size = 96 - -train_fn.main_module = "SASRec" -train_fn.dropout_rate = 0.2 -train_fn.user_embedding_norm = "l2_norm" -train_fn.num_epochs = 100 -train_fn.item_embedding_dim = 256 - -sasrec_encoder.num_blocks = 4 -sasrec_encoder.num_heads = 4 -sasrec_encoder.ffn_dropout_rate = 0.2 -sasrec_encoder.ffn_hidden_dim = 256 -sasrec_encoder.ffn_activation_fn = "relu" - -train_fn.learning_rate = 1e-3 -train_fn.weight_decay = 0 -train_fn.num_warmup_steps = 0 - -train_fn.top_k_method = "MIPSBruteForceTopK" -train_fn.interaction_module_type = "DotProduct" - -train_fn.loss_module = "SampledSoftmaxLoss" -train_fn.num_negatives = 128 - -train_fn.sampling_strategy = "local" -train_fn.temperature = 0.05 -train_fn.item_l2_norm = True -train_fn.l2_norm_eps = 1e-6 - -train_fn.enable_tf32 = True - -create_data_loader.prefetch_factor = 128 -create_data_loader.num_workers = 8 diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/README.md b/recommendation_v4/generative_recommenders/dlrm_v3/inference/README.md deleted file mode 100644 index ef1c9686d..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/README.md +++ /dev/null @@ -1,88 +0,0 @@ -# MLPerf Inference reference implementation for DLRMv3 - -## Install dependencies - -The reference implementation has been tested on a single host, with x86_64 CPUs -and 8 NVIDIA H100/B200 GPUs. Dependencies can be installed below, - -``` -cd generative_recommenders/ -pip install -e . -``` - -## Build loadgen - -``` -cd generative_recommenders/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/ -CFLAGS="-std=c++14 -O3" python -m pip install . -``` - -## Dataset download - -DLRMv3 uses a synthetic dataset specifically designed to match the model and -system characteristics of large-scale sequential recommendation (large item set -and long average sequence length for each request). To generate the dataset used -for both training and inference, run - -``` -cd generative_recommenders/dlrm_v3/ -python streaming_synthetic_data.py -``` - -The generated dataset has 2TB size, and contains 5 million users interacting -with a billion items over 100 timestamps. - -Only 1% of the dataset is used in the inference benchmark. The sampled DLRMv3 -dataset and trained checkpoint are available at -https://inference.mlcommons-storage.org/. - -Script to download the sampled dataset used in inference benchmark: - -``` -bash <(curl -s https://raw.githubusercontent.com/mlcommons/r2-downloader/refs/heads/main/mlc-r2-downloader.sh) https://inference.mlcommons-storage.org/metadata/dlrm-v3-dataset.uri -``` - -Script to download the 1TB trained checkpoint: - -``` -bash <(curl -s https://raw.githubusercontent.com/mlcommons/r2-downloader/refs/heads/main/mlc-r2-downloader.sh) https://inference.mlcommons-storage.org/metadata/dlrm-v3-checkpoint.uri -``` - -## Inference benchmark - -``` -cd generative_recommenders/generative_recommenders/dlrm_v3/inference/ -WORLD_SIZE=8 python main.py --dataset sampled-streaming-100b -``` - -The config file is listed in `dlrm_v3/inference/gin/streaming_100b.gin`. -`WORLD_SIZE` is the number of GPUs used in the inference benchmark. - -To load checkpoint from training, modify `run.model_path` inside the inference -gin config file. (We will relase the checkpoint soon.) - -To achieve the best performance, tune `run.target_qps` and `run.batch_size` in -the config file. - -## Accuracy test - -Set `run.compute_eval` will run the accuracy test and dump prediction outputs in -`mlperf_log_accuracy.json`. To check the accuracy, run - -``` -python accuracy.py --path path/to/mlperf_log_accuracy.json -``` - -We use normalized entropy (NE), accuracy, and AUC as the metrics to evaluate the model quality. For accepted submissions, all three metrics (NE, Accuracy, AUC) must be within 99% of the reference implementation values. The accuracy for the reference implementation evaluated on 34,996 requests across 10 inference timestamps are listed below: - -``` -NE: 86.687% -Accuracy: 69.651% -AUC: 78.663% -``` - -## Run unit tests - -``` -python tests/inference_test.py -``` diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/accuracy.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/accuracy.py deleted file mode 100644 index 19242f7bd..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/accuracy.py +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-strict -""" -Tool to calculate accuracy for loadgen accuracy output found in mlperf_log_accuracy.json -""" - -import argparse -import json -import logging - -import numpy as np -import torch -from generative_recommenders.dlrm_v3.configs import get_hstu_configs -from generative_recommenders.dlrm_v3.utils import MetricsLogger - -logger: logging.Logger = logging.getLogger("main") - - -def get_args() -> argparse.Namespace: - """Parse commandline.""" - parser = argparse.ArgumentParser() - parser.add_argument( - "--path", - required=True, - help="path to mlperf_log_accuracy.json", - ) - args = parser.parse_args() - return args - - -def main() -> None: - """ - Main function to calculate accuracy metrics from loadgen output. - - Reads the mlperf_log_accuracy.json file, parses the results, and computes - accuracy metrics using the MetricsLogger. Each result entry contains - predictions, labels, and weights packed as float32 numpy arrays. - """ - args = get_args() - logger.warning("Parsing loadgen accuracy log...") - with open(args.path, "r") as f: - results = json.load(f) - hstu_config = get_hstu_configs(dataset="sampled-streaming-100b") - metrics = MetricsLogger( - multitask_configs=hstu_config.multitask_configs, - batch_size=1, - window_size=3000, - device=torch.device("cpu"), - rank=0, - ) - logger.warning(f"results have {len(results)} entries") - for result in results: - data = np.frombuffer(bytes.fromhex(result["data"]), np.float32) - num_candidates = data[-1].astype(int) - assert len(data) == 1 + num_candidates * 3 - mt_target_preds = torch.from_numpy(data[0:num_candidates]) - mt_target_labels = torch.from_numpy(data[num_candidates : num_candidates * 2]) - mt_target_weights = torch.from_numpy( - data[num_candidates * 2 : num_candidates * 3] - ) - num_candidates = torch.tensor([num_candidates]) - metrics.update( - predictions=mt_target_preds.view(1, -1), - labels=mt_target_labels.view(1, -1), - weights=mt_target_weights.view(1, -1), - num_candidates=num_candidates, - ) - for k, v in metrics.compute().items(): - logger.warning(f"{k}: {v}") - - -if __name__ == "__main__": - main() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/cpp/hstu_runner.cpp b/recommendation_v4/generative_recommenders/dlrm_v3/inference/cpp/hstu_runner.cpp deleted file mode 100644 index d4d0d4082..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/cpp/hstu_runner.cpp +++ /dev/null @@ -1,215 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. -// -// End-to-end runner for the HSTU torch.jit / torch.package artifacts produced -// by generative_recommenders/dlrm_v3/inference/packager.py and exercised by -// :end_to_end_test. -// -// CLI: -// hstu_runner [--aott_library ...] -// -// -// Where: -// sparse.pt ScriptModule whose forward(uih, candidates) returns -// Tuple[Dict[str,Tensor], Dict[str,Tensor], -// Dict[str,Tensor], Tensor, Tensor] -// dense.pt ScriptModule (cuda:0, bf16) whose forward(...) returns -// Tuple[Tensor, Optional[Tensor], Optional[Tensor]] -// inputs.pt ScriptModule whose forward() returns -// Tuple[KeyedJaggedTensor, KeyedJaggedTensor] -// output.pt torch::pickle_save destination for the predictions tensor; -// readable from Python as ``torch.load(output.pt)``. - -#include - -#include -#include -#include -#include -#include -#include - -#include -#include - -namespace { - -struct RunnerArgs { - std::vector aottLibraryPaths; - std::string sparsePath; - std::string densePath; - std::string inputsPath; - std::string outputPath; -}; - -RunnerArgs parseArgs(int argc, char** argv) { - RunnerArgs args; - std::vector positional; - for (int i = 1; i < argc; ++i) { - const std::string arg{argv[i]}; - if (arg == "--aott_library") { - if (++i >= argc) { - throw std::runtime_error("--aott_library requires a path"); - } - args.aottLibraryPaths.emplace_back(argv[i]); - } else { - positional.push_back(arg); - } - } - - if (positional.size() != 4) { - throw std::runtime_error( - "Usage: hstu_runner [--aott_library ...] " - " "); - } - args.sparsePath = positional[0]; - args.densePath = positional[1]; - args.inputsPath = positional[2]; - args.outputPath = positional[3]; - return args; -} - -void loadAottLibraries( - const std::vector& libraryPaths, - const std::function& log) { - for (const auto& path : libraryPaths) { - log("[runner] loading AOT-T library " + path); - void* handle = dlopen(path.c_str(), RTLD_GLOBAL | RTLD_NOW); - if (handle == nullptr) { - throw std::runtime_error( - "failed to dlopen AOT-T library " + path + ": " + dlerror()); - } - } -} - -torch::jit::Module loadModule(const std::string& path) { - // @patternlint-disable-next-line no-torch-low-level-api - auto m = torch::jit::load(path); - m.eval(); - return m; -} - -// Walk a Dict and replace every value with .to(device) (and -// optionally .to(bfloat16)). C++ analog of move_sparse_output_to_device. -void moveDictToDevice( - c10::impl::GenericDict& d, - const torch::Device& device, - bool toBfloat16) { - for (auto& kv : d) { - auto t = kv.value().toTensor().to(device); - if (toBfloat16) { - t = t.to(torch::kBFloat16); - } - d.insert_or_assign(kv.key(), t); - } -} - -void writePickle(const torch::Tensor& t, const std::string& path) { - // torch::pickle_save returns a byte buffer in the same wire format as - // ``torch.save(tensor, ...)``, so the Python side can read it with - // ``torch.load(path)``. - const auto data = torch::jit::pickle_save(c10::IValue(t)); - std::ofstream out(path, std::ios::binary); - if (!out) { - throw std::runtime_error("failed to open output: " + path); - } - out.write(data.data(), static_cast(data.size())); -} - -} // namespace - -int main(int argc, char** argv) { - RunnerArgs args; - try { - args = parseArgs(argc, argv); - } catch (const std::exception& e) { - std::cerr << e.what() << '\n'; - return 1; - } - - // Log to a file next to the output so we can inspect even if - // buck2 swallows stderr. - const std::string logPath = args.outputPath + ".log"; - std::ofstream logFile(logPath); - auto log = [&](const std::string& msg) { - logFile << msg << std::endl; - logFile.flush(); - std::cerr << msg << std::endl; - }; - - try { - log("[runner] step 0: loading AOT-T libraries"); - loadAottLibraries(args.aottLibraryPaths, log); - log("[runner] step 0 done: loaded " + - std::to_string(args.aottLibraryPaths.size()) + " AOT-T libraries"); - - log("[runner] step 1: loading sparse module from " + args.sparsePath); - auto sparse = loadModule(args.sparsePath); - - log("[runner] step 2: loading dense module from " + args.densePath); - auto dense = loadModule(args.densePath); - - log("[runner] step 3: loading inputs module from " + args.inputsPath); - auto inputs = loadModule(args.inputsPath); - - log("[runner] step 4: running inputs.forward()"); - auto inputsTuple = inputs.forward({}).toTuple(); - auto uihLengths = inputsTuple->elements()[0]; - auto uihValues = inputsTuple->elements()[1]; - auto candidatesLengths = inputsTuple->elements()[2]; - auto candidatesValues = inputsTuple->elements()[3]; - log("[runner] step 4 done: got 4 input tensors"); - - log("[runner] step 5: running sparse.forward()"); - std::vector sparseInputs{ - uihLengths, uihValues, candidatesLengths, candidatesValues}; - auto sparseOut = sparse.forward(sparseInputs).toTuple(); - log("[runner] step 5 done: sparse forward returned " + - std::to_string(sparseOut->elements().size()) + " elements"); - - log("[runner] step 6: unpacking sparse output dicts"); - auto seqEmbValues = sparseOut->elements()[0].toGenericDict(); - auto seqEmbLengths = sparseOut->elements()[1].toGenericDict(); - auto payloadFeatures = sparseOut->elements()[2].toGenericDict(); - auto uihSeqLengths = sparseOut->elements()[3].toTensor(); - auto numCandidates = sparseOut->elements()[4].toTensor(); - log("[runner] step 6 done: unpacked dicts"); - - log("[runner] step 7: moving dicts to cuda:0"); - const auto device = torch::Device(torch::kCUDA, 0); - moveDictToDevice(seqEmbValues, device, /*toBfloat16=*/true); - log("[runner] step 7a: seqEmbValues moved"); - moveDictToDevice(seqEmbLengths, device, /*toBfloat16=*/false); - log("[runner] step 7b: seqEmbLengths moved"); - moveDictToDevice(payloadFeatures, device, /*toBfloat16=*/false); - log("[runner] step 7c: payloadFeatures moved"); - uihSeqLengths = uihSeqLengths.to(device); - numCandidates = numCandidates.to(device); - log("[runner] step 7 done: all on cuda:0"); - - log("[runner] step 8: running dense.forward()"); - std::vector denseInputs{ - seqEmbValues, - seqEmbLengths, - payloadFeatures, - uihSeqLengths, - numCandidates, - }; - auto denseOut = dense.forward(denseInputs); - log("[runner] step 8 done: dense forward returned"); - - auto preds = denseOut.toTensor().detach().cpu(); - log("[runner] step 9: preds on cpu"); - - std::cout << "preds shape: " << preds.sizes() << '\n'; - std::cout << "preds sum: " - << preds.to(torch::kFloat32).sum().item() << '\n'; - - writePickle(preds, args.outputPath); - std::cout << "wrote " << args.outputPath << '\n'; - log("[runner] step 10: done, wrote output"); - return 0; - } catch (const std::exception& e) { - log(std::string("hstu_runner FAILED: ") + e.what()); - return 1; - } -} diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/data_producer.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/data_producer.py deleted file mode 100644 index 6a8db77c8..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/data_producer.py +++ /dev/null @@ -1,227 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-strict -""" -Data producer module for DLRMv3 inference. - -This module provides classes for producing and managing query data during inference, -supporting both single-threaded and multi-threaded data production modes. -""" - -import logging -import threading -import time -from queue import Queue -from typing import List, Optional, Tuple, Union - -import torch -from generative_recommenders.dlrm_v3.datasets.dataset import Dataset, Samples - -logging.basicConfig(level=logging.INFO) -logger: logging.Logger = logging.getLogger("data_producer") - - -class QueryItem: - """ - Container for a query item to be processed by the inference thread pool. - - Attributes: - query_ids: List of unique identifiers for the queries in this batch. - samples: The sample data containing features for the queries. - start: Time when the query was first received. - dt_queue: Time spent in the queue before processing. - dt_batching: Time spent on batching the data. - """ - - def __init__( - self, - query_ids: List[int], - samples: Samples, - start: float, - dt_queue: float, - dt_batching: float, - ) -> None: - self.query_ids = query_ids - self.samples = samples - self.start: float = start - self.dt_queue: float = dt_queue - self.dt_batching: float = dt_batching - - -class SingleThreadDataProducer: - """ - Single-threaded data producer for synchronous query processing. - - This producer processes queries on the main thread without any parallelism, - suitable for debugging or low-throughput scenarios. - - Args: - ds: The dataset to fetch samples from. - run_one_item: Callback function to process a single QueryItem. - """ - - def __init__(self, ds: Dataset, run_one_item) -> None: # pyre-ignore [2] - self.ds = ds - self.run_one_item = run_one_item # pyre-ignore [4] - - def enqueue( - self, query_ids: List[int], content_ids: List[int], t0: float, dt_queue: float - ) -> None: - """ - Enqueue queries for immediate synchronous processing. - - Args: - query_ids: List of unique query identifiers. - content_ids: List of content/sample identifiers to fetch. - t0: Timestamp when the query batch was created. - dt_queue: Time spent waiting in the queue. - """ - with torch.profiler.record_function("data batching"): - t0_batching: float = time.time() - samples: Union[Samples, List[Samples]] = self.ds.get_samples(content_ids) - dt_batching: float = time.time() - t0_batching - if isinstance(samples, Samples): - query = QueryItem( - query_ids=query_ids, - samples=samples, - start=t0, - dt_queue=dt_queue, - dt_batching=dt_batching, - ) - self.run_one_item(query) - else: - start_idx = 0 - for sample in samples: - batch_size: int = sample.batch_size() - query = QueryItem( - query_ids=query_ids[start_idx : start_idx + batch_size], - samples=sample, - start=t0, - dt_queue=dt_queue, - dt_batching=dt_batching, - ) - start_idx += batch_size - self.run_one_item(query) - - def finish(self) -> None: - """Finalize the producer. No-op for single-threaded mode.""" - pass - - -class MultiThreadDataProducer: - """ - Multi-threaded data producer for parallel query processing. - - Uses a thread pool to fetch and batch data in parallel with model inference, - improving throughput for high-load scenarios. - - Args: - ds: The dataset to fetch samples from. - threads: Number of worker threads to use. - run_one_item: Callback function to process a single QueryItem. - """ - - def __init__( - self, - ds: Dataset, - threads: int, - run_one_item, # pyre-ignore [2] - ) -> None: - queue_size_multiplier = 4 - self.ds = ds - self.threads = threads - self.run_one_item = run_one_item # pyre-ignore [4] - self.tasks: Queue[Optional[Tuple[List[int], List[int], float, float]]] = Queue( - maxsize=threads * queue_size_multiplier - ) - self.workers: List[threading.Thread] = [] - for _ in range(self.threads): - worker = threading.Thread(target=self.handle_tasks, args=(self.tasks,)) - worker.daemon = True - self.workers.append(worker) - worker.start() - - def handle_tasks( - self, tasks_queue: Queue[Optional[Tuple[List[int], List[int], float, float]]] - ) -> None: - """ - Worker thread main loop to process tasks from the queue. - - Each worker maintains its own CUDA stream for parallel execution. - - Args: - tasks_queue: Queue containing task tuples or None for termination. - """ - stream = torch.cuda.Stream() - while True: - query_and_content_ids = tasks_queue.get() - if query_and_content_ids is None: - tasks_queue.task_done() - break - query_ids, content_ids, t0, dt_queue = query_and_content_ids - t0_batching: float = time.time() - samples: Union[Samples, List[Samples]] = self.ds.get_samples(content_ids) - dt_batching: float = time.time() - t0_batching - if isinstance(samples, Samples): - qitem = QueryItem( - query_ids=query_ids, - samples=samples, - start=t0, - dt_queue=dt_queue, - dt_batching=dt_batching, - ) - with torch.inference_mode(), torch.cuda.stream(stream): - self.run_one_item(qitem) - else: - start_idx = 0 - for sample in samples: - batch_size: int = sample.batch_size() - qitem = QueryItem( - query_ids=query_ids[start_idx : start_idx + batch_size], - samples=sample, - start=t0, - dt_queue=dt_queue, - dt_batching=dt_batching, - ) - start_idx += batch_size - with torch.inference_mode(), torch.cuda.stream(stream): - self.run_one_item(qitem) - tasks_queue.task_done() - - def enqueue( - self, query_ids: List[int], content_ids: List[int], t0: float, dt_queue: float - ) -> None: - """ - Enqueue queries for asynchronous processing by worker threads. - - Args: - query_ids: List of unique query identifiers. - content_ids: List of content/sample identifiers to fetch. - t0: Timestamp when the query batch was created. - dt_queue: Time spent waiting in the queue. - """ - with torch.profiler.record_function("data batching"): - self.tasks.put((query_ids, content_ids, t0, dt_queue)) - - def finish(self) -> None: - """ - Signal all worker threads to terminate and wait for completion. - - Sends None to each worker to trigger graceful shutdown. - """ - for _ in self.workers: - self.tasks.put(None) - for worker in self.workers: - worker.join() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/dense_predict_module.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/dense_predict_module.py deleted file mode 100644 index add2781bc..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/dense_predict_module.py +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-strict - -""" -TorchScript-friendly wrapper for the HSTU dense path (GPU transformer). - -``HSTUDenseScriptModule`` accepts the *flattened* sparse-output dicts produced -by :class:`HSTUSparseScriptModule`, reconstructs ``Dict[str, -SequenceEmbedding]`` for the existing :meth:`DlrmHSTU.main_forward` and -returns a 3-tuple of ``(preds, labels, weights)`` -- the only fields the -predictor actually consumes. -""" - -from typing import Dict - -import torch -from generative_recommenders.dlrm_v3.inference.inference_modules import get_hstu_model -from generative_recommenders.dlrm_v3.inference.ts_types import ( - SeqEmbLengths, - SeqEmbValues, - unflatten_seq_embeddings, -) -from generative_recommenders.modules.dlrm_hstu import DlrmHSTU, DlrmHSTUConfig -from torchrec.modules.embedding_configs import EmbeddingConfig - - -class HSTUDenseScriptModule(torch.nn.Module): - """Script-friendly dense module. - - The wrapper owns a dense-only :class:`DlrmHSTU` (no - ``_embedding_collection``) and delegates to ``main_forward`` after - reconstructing the ``SequenceEmbedding`` NamedTuple form. - """ - - def __init__( - self, - hstu_config: DlrmHSTUConfig, - table_config: Dict[str, EmbeddingConfig], - ) -> None: - super().__init__() - self._hstu_model: DlrmHSTU = get_hstu_model( - table_config=table_config, - hstu_config=hstu_config, - table_device="cpu", - is_dense=True, - ) - - def forward( - self, - seq_emb_values: SeqEmbValues, - seq_emb_lengths: SeqEmbLengths, - payload_features: Dict[str, torch.Tensor], - uih_seq_lengths: torch.Tensor, - num_candidates: torch.Tensor, - ) -> torch.Tensor: - # TorchScript supports ``int(tensor.item())`` on a 0-d tensor. - max_uih_len: int = int(uih_seq_lengths.max().item()) - max_num_candidates: int = int(num_candidates.max().item()) - - seq_embeddings = unflatten_seq_embeddings(seq_emb_values, seq_emb_lengths) - - ( - _, - _, - _, - mt_target_preds, - _mt_target_labels, - _mt_target_weights, - ) = self._hstu_model.main_forward( - seq_embeddings=seq_embeddings, - payload_features=payload_features, - max_uih_len=max_uih_len, - uih_seq_lengths=uih_seq_lengths, - max_num_candidates=max_num_candidates, - num_candidates=num_candidates, - ) - assert mt_target_preds is not None - # Return just the predictions tensor; labels/weights are unused by - # the predictor at inference time and would force ``Optional[Tensor]`` - # in the return type, which torch.jit.trace rejects ("Only tensors, - # lists, tuples of tensors, or dictionary of tensors can be output - # from traced functions"). - return mt_target_preds diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/end_to_end_test.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/end_to_end_test.py deleted file mode 100644 index f1b956d9c..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/end_to_end_test.py +++ /dev/null @@ -1,795 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-strict - -""" -End-to-end smoke test for the HSTU TorchScript + C++ deployment pipeline. - -What this binary does, in order: - -1. Build a synthetic batch (uih_kjt, candidates_kjt) via :func:`get_random_data`. -2. Build the eager :class:`HSTUSparseScriptModule` and - :class:`HSTUDenseScriptModule`. -3. Run them eagerly to obtain the reference ``preds_eager``. -4. ``torch.jit.script`` + save: - - ``sparse.pt`` (CPU) - - ``dense.pt`` (cuda:0, bf16) - - ``inputs.pt`` (an :class:`InputsBundle` ScriptModule whose - ``forward()`` returns ``Tuple[KeyedJaggedTensor, KeyedJaggedTensor]``) -5. Run the C++ runner - ``hstu_runner [--aott_library ...] ``. -6. ``torch.load`` the runner's output and compare against ``preds_eager`` - with :func:`torch.testing.assert_close` (loose tolerance because the - scripted path may use either the PyTorch fallback trace or AOT-T-loaded - Triton inference kernels). - -Usage (manual override of the runner path): - - buck2 run @mode/opt //generative_recommenders/dlrm_v3/inference:end_to_end_test \\ - -- --cpp_runner /path/to/hstu_runner - -By default the binary locates the runner via ``libfb.py.parutil`` -- it ships -inside the par as a resource (see BUCK). -""" - -import argparse -import logging -import os -import shutil -import sys -import tempfile -from typing import Any, Dict, List, Tuple - -import torch -from generative_recommenders.dlrm_v3.configs import ( - get_embedding_table_config, - get_hstu_configs, -) -from generative_recommenders.dlrm_v3.datasets.dataset import get_random_data -from generative_recommenders.dlrm_v3.inference.dense_predict_module import ( - HSTUDenseScriptModule, -) -from generative_recommenders.dlrm_v3.inference.sparse_predict_module import ( - HSTUSparseScriptModule, -) -from generative_recommenders.dlrm_v3.inference.ts_types import ( - SeqEmbLengths, - SeqEmbValues, - unflatten_seq_embeddings, -) -from generative_recommenders.modules.dlrm_hstu import DlrmHSTUConfig -from security.frameworks.python.exec.subprocess import TrustedSubprocessWithList -from torchrec.modules.embedding_configs import EmbeddingConfig -from torchrec.sparse.jagged_tensor import KeyedJaggedTensor - - -logger: logging.Logger = logging.getLogger(__name__) - - -_DEFAULT_DATASET = "kuairand-1k" - - -class InputsBundle(torch.nn.Module): - """Scripted holder for the test inputs. - - Returns the constituent tensors of the two KJTs as a 4-tuple - ``(uih_lengths, uih_values, candidates_lengths, candidates_values)`` so - the traced sparse module can rebuild the KJTs inside its forward (KJT - instances themselves are not traceable inputs). - """ - - def __init__( - self, - uih_kjt: KeyedJaggedTensor, - candidates_kjt: KeyedJaggedTensor, - ) -> None: - super().__init__() - self.register_buffer("uih_lengths", uih_kjt.lengths()) - self.register_buffer("uih_values", uih_kjt.values()) - self.register_buffer("candidates_lengths", candidates_kjt.lengths()) - self.register_buffer("candidates_values", candidates_kjt.values()) - - def forward( - self, - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: - return ( - self.uih_lengths, - self.uih_values, - self.candidates_lengths, - self.candidates_values, - ) - - -class _SparseTraceShim(torch.nn.Module): - """Adapter that takes raw tensors and rebuilds the KJTs inside forward. - - ``torch.jit.trace`` does not accept ``KeyedJaggedTensor`` (or any - non-Tensor / non-collection-of-Tensor type) as a top-level forward - input, so we make the traced boundary tensor-only and bake the - ``List[str]`` of feature keys in as Python constants captured by the - closure / module attribute. - """ - - def __init__( - self, - sparse_module: HSTUSparseScriptModule, - uih_keys: List[str], - candidates_keys: List[str], - ) -> None: - super().__init__() - self._sparse_module: HSTUSparseScriptModule = sparse_module - self._uih_keys: List[str] = uih_keys - self._candidates_keys: List[str] = candidates_keys - - def forward( - self, - uih_lengths: torch.Tensor, - uih_values: torch.Tensor, - candidates_lengths: torch.Tensor, - candidates_values: torch.Tensor, - ) -> Tuple[ - SeqEmbValues, - SeqEmbLengths, - Dict[str, torch.Tensor], - torch.Tensor, - torch.Tensor, - ]: - uih_kjt = KeyedJaggedTensor( - keys=self._uih_keys, - lengths=uih_lengths, - values=uih_values, - ) - candidates_kjt = KeyedJaggedTensor( - keys=self._candidates_keys, - lengths=candidates_lengths, - values=candidates_values, - ) - return self._sparse_module( - uih_features=uih_kjt, candidates_features=candidates_kjt - ) - - -class _DenseAottTraceShim(torch.nn.Module): - """FX-traceable dense adapter for the representative AOT-T shape.""" - - def __init__( - self, - dense_module: HSTUDenseScriptModule, - max_uih_len: int, - max_num_candidates: int, - total_uih_len: int, - total_targets: int, - ) -> None: - super().__init__() - self._dense_module: HSTUDenseScriptModule = dense_module - self._max_uih_len: int = max_uih_len - self._max_num_candidates: int = max_num_candidates - self._total_uih_len: int = total_uih_len - self._total_targets: int = total_targets - - def forward( - self, - seq_emb_values: SeqEmbValues, - seq_emb_lengths: SeqEmbLengths, - payload_features: Dict[str, torch.Tensor], - uih_seq_lengths: torch.Tensor, - num_candidates: torch.Tensor, - ) -> torch.Tensor: - seq_embeddings = unflatten_seq_embeddings(seq_emb_values, seq_emb_lengths) - - ( - _, - _, - _, - mt_target_preds, - _mt_target_labels, - _mt_target_weights, - ) = self._dense_module._hstu_model.main_forward( - seq_embeddings=seq_embeddings, - payload_features=payload_features, - max_uih_len=self._max_uih_len, - uih_seq_lengths=uih_seq_lengths, - max_num_candidates=self._max_num_candidates, - num_candidates=num_candidates, - total_uih_len=self._total_uih_len, - total_targets=self._total_targets, - ) - assert mt_target_preds is not None - return mt_target_preds - - -def _dense_aott_concrete_args( - dense_inputs: Tuple[ - Dict[str, torch.Tensor], - Dict[str, torch.Tensor], - Dict[str, torch.Tensor], - torch.Tensor, - torch.Tensor, - ], -) -> Dict[str, Any]: - from torch.fx._symbolic_trace import PH - - seq_emb_values, seq_emb_lengths, payload_features, _, _ = dense_inputs - return { - "seq_emb_values": {key: PH for key in seq_emb_values}, - "seq_emb_lengths": {key: PH for key in seq_emb_lengths}, - "payload_features": {key: PH for key in payload_features}, - } - - -def _find_cpp_runner() -> str: - """Locate the bundled hstu_runner binary. - - Tries ``importlib.resources`` (the canonical fbcode resource resolver, - works whether the binary is in a par or unpacked), and falls back to - looking next to ``sys.argv[0]``. - """ - try: - from importlib.resources import files - - path = files("generative_recommenders.dlrm_v3.inference.cpp").joinpath( - "hstu_runner" - ) - if path.is_file(): - return str(path) - except Exception as exc: - logger.debug("importlib.resources lookup failed: %s", exc) - - candidate = os.path.join( - os.path.dirname(os.path.abspath(sys.argv[0])), "hstu_runner" - ) - if os.path.exists(candidate): - return candidate - - raise RuntimeError( - "Could not find hstu_runner binary. " - "Pass --cpp_runner= or build the cpp_binary target first." - ) - - -def _eager_run( - sparse_module: HSTUSparseScriptModule, - dense_module: HSTUDenseScriptModule, - uih_kjt: KeyedJaggedTensor, - candidates_kjt: KeyedJaggedTensor, - device: torch.device, -) -> torch.Tensor: - """Reference path: sparse → device-move + bf16 → dense, all in Python.""" - with torch.no_grad(): - seq_emb_values, seq_emb_lengths, payload, uih_lens, num_cands = sparse_module( - uih_features=uih_kjt, candidates_features=candidates_kjt - ) - seq_emb_values = { - k: v.to(device).to(torch.bfloat16) for k, v in seq_emb_values.items() - } - seq_emb_lengths = {k: v.to(device) for k, v in seq_emb_lengths.items()} - payload = {k: v.to(device) for k, v in payload.items()} - uih_lens = uih_lens.to(device) - num_cands = num_cands.to(device) - preds = dense_module( - seq_emb_values, seq_emb_lengths, payload, uih_lens, num_cands - ) - return preds.detach().to(torch.float32).cpu() - - -def _find_aott_libraries() -> List[str]: - from generative_recommenders.ops.triton_aot.compile.compile_state import ( - get_aott_compile_path, - ) - - compile_path = get_aott_compile_path() - libraries: List[str] = [] - for root, _, files in os.walk(compile_path): - for filename in files: - if filename.endswith(".so"): - libraries.append(os.path.join(root, filename)) - return sorted(libraries) - - -def _copy_aott_libraries_to_workdir( - library_paths: List[str], workdir: str -) -> List[str]: - copied: List[str] = [] - for index, path in enumerate(library_paths): - dst = os.path.join(workdir, f"aott_{index}_{os.path.basename(path)}") - shutil.copy2(path, dst) - copied.append(dst) - return copied - - -def _load_aott_libraries_for_python(library_paths: List[str]) -> None: - for library_path in library_paths: - logger.info("Python roundtrip: loading AOT-T library %s", library_path) - torch.ops.load_library(library_path) - - -def _save_aott_dense_module( - dense_module: HSTUDenseScriptModule, - dense_inputs: Tuple[ - Dict[str, torch.Tensor], - Dict[str, torch.Tensor], - Dict[str, torch.Tensor], - torch.Tensor, - torch.Tensor, - ], - dense_path: str, - workdir: str, - atol: float, - rtol: float, -) -> List[str]: - """Lower the dense module with AOT-T and save a TorchScript artifact. - - This follows the AOT-T example flow: - - 1. FX trace the module. - 2. Unwrap outer `aot_triton_kernel_wrapper_*` nodes. - 3. Run representative CUDA inputs under `TritonAOTCompile`. - 4. `transform_kernels` to replace wrappers with `torch.ops.triton_aot.*`. - 5. Script and save the transformed dense module. - - The full HSTU dense wrapper has historically needed tracing rather than FX, - so failures here are reported with context and the default path remains the - D102 traced TorchScript fallback. - """ - from generative_recommenders.ops.triton_aot.compile.triton_aot_compile import ( - TritonAOTCompile, - ) - from generative_recommenders.ops.triton_aot.preprocess import ( - unwrap_aott_wrapper_nodes, - ) - from generative_recommenders.ops.triton_aot.transform.transform_kernels import ( - transform_kernels, - ) - from tgif.fx.tgif_tracer import TGIFTracer - - max_uih_len = int(dense_inputs[3].max().item()) - max_num_candidates = int(dense_inputs[4].max().item()) - total_uih_len = int(dense_inputs[3].sum().item()) - total_targets = int(dense_inputs[4].sum().item()) - trace_shim = _DenseAottTraceShim( - dense_module=dense_module, - max_uih_len=max_uih_len, - max_num_candidates=max_num_candidates, - total_uih_len=total_uih_len, - total_targets=total_targets, - ).eval() - - logger.info( - "AOT-T dense: FX tracing representative shape " - "(max_uih_len=%d, max_num_candidates=%d, " - "total_uih_len=%d, total_targets=%d)...", - max_uih_len, - max_num_candidates, - total_uih_len, - total_targets, - ) - try: - fx_dense = TGIFTracer().symbolic_trace( - trace_shim, - concrete_args=_dense_aott_concrete_args(dense_inputs), - ) - lowered_dense = unwrap_aott_wrapper_nodes(fx_dense, TGIFTracer()) - except Exception as exc: - raise RuntimeError( - "AOT-T dense lowering requires an FX-traceable dense entry point. " - "Use --dense_backend=torchscript to fall back to the D102 traced " - "TorchScript path." - ) from exc - - logger.info("AOT-T dense: compiling Triton kernels from sample inputs...") - with torch.no_grad(): - with TritonAOTCompile(): - ref_output = lowered_dense(*dense_inputs) - - original_code = lowered_dense.code - lowered_dense = transform_kernels(lowered_dense) - if lowered_dense.code == original_code: - logger.warning( - "AOT-T dense: transform_kernels did not change the FX graph. " - "This usually means no aot_triton_kernel_wrapper_* nodes were " - "present in the dense path." - ) - - libraries = _find_aott_libraries() - if not libraries: - raise RuntimeError( - "AOT-T dense lowering produced no .so files. Ensure the dense path " - "uses HammerKernel.TRITON_INFERENCE branches backed by triton_aot ops." - ) - - with torch.no_grad(): - lowered_output = lowered_dense(*dense_inputs) - torch.testing.assert_close(ref_output, lowered_output, atol=atol, rtol=rtol) - - logger.info("AOT-T dense: tracing transformed module...") - torch.jit.trace( - lowered_dense, - example_inputs=dense_inputs, - strict=False, - check_trace=False, - ).save(dense_path) - copied_libraries = _copy_aott_libraries_to_workdir(libraries, workdir) - logger.info("AOT-T dense: copied %d library file(s)", len(copied_libraries)) - return copied_libraries - - -def _build_synthetic_inputs( - hstu_config: DlrmHSTUConfig, - table_config: Dict[str, EmbeddingConfig], - uih_max_seq_len: int, -) -> Tuple[KeyedJaggedTensor, KeyedJaggedTensor]: - contextual: List[str] = list(hstu_config.contextual_feature_to_max_length.keys()) - # The kuairand-1k dataset has tiny embedding tables for some contextual - # features (e.g. user_active_degree has num_embeddings=8). Clamp the - # random value range so every index stays in range for every table. - min_rows = min(t.num_embeddings for t in table_config.values()) - value_bound = max(2, min_rows) - logger.info( - "synthetic value_bound=%d (min table rows=%d across %d tables)", - value_bound, - min_rows, - len(table_config), - ) - return get_random_data( - contexual_features=contextual, - hstu_uih_keys=hstu_config.hstu_uih_feature_names, - hstu_candidates_keys=hstu_config.hstu_candidate_feature_names, - uih_max_seq_len=uih_max_seq_len, - max_num_candidates=hstu_config.max_num_candidates_inference, - value_bound=value_bound, - ) - - -def _parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--cpp_runner", - type=str, - default=None, - help="Path to the hstu_runner binary; default: bundled resource.", - ) - parser.add_argument( - "--dataset", - type=str, - default=_DEFAULT_DATASET, - help="Dataset key for HSTU/embedding configs.", - ) - parser.add_argument( - "--device", type=str, default="cuda:0", help="Dense-module device." - ) - parser.add_argument( - "--uih_max_seq_len", - type=int, - default=128, - help="Max UIH length for the synthetic batch.", - ) - parser.add_argument("--seed", type=int, default=0) - parser.add_argument("--atol", type=float, default=1e-2) - parser.add_argument("--rtol", type=float, default=1e-2) - parser.add_argument( - "--dense_backend", - choices=("torchscript", "aott"), - default="torchscript", - help="Dense artifact backend. aott lowers TRITON_INFERENCE wrappers and passes compiled libraries to the C++ runner.", - ) - parser.add_argument( - "--aott_library", - action="append", - default=[], - help="Additional prebuilt AOT-T shared library to dlopen before loading dense.pt. May be repeated.", - ) - parser.add_argument( - "--keep_workdir", - action="store_true", - help="Do not delete the temp dir holding the saved artifacts.", - ) - return parser.parse_args() - - -def main() -> None: # noqa: C901 - logging.basicConfig(level=logging.INFO, format="[e2e] %(message)s", force=True) - logger.setLevel(logging.DEBUG) - args = _parse_args() - - if not torch.cuda.is_available(): - logger.error("CUDA is required; aborting.") - sys.exit(2) - - runner_path = args.cpp_runner or _find_cpp_runner() - logger.info("Using C++ runner: %s", runner_path) - - torch.manual_seed(args.seed) - device = torch.device(args.device) - torch.cuda.set_device(device) - - hstu_config = get_hstu_configs(args.dataset) - table_config = get_embedding_table_config(args.dataset) - - uih_kjt, candidates_kjt = _build_synthetic_inputs( - hstu_config, table_config, args.uih_max_seq_len - ) - - sparse_module = HSTUSparseScriptModule( - table_config=table_config, - hstu_config=hstu_config, - use_no_copy_embedding_collection=True, - ).eval() - dense_module = ( - HSTUDenseScriptModule(hstu_config=hstu_config, table_config=table_config) - .to(torch.bfloat16) - .to(device) - .eval() - ) - - from generative_recommenders.common import HammerKernel - - dense_kernel = ( - HammerKernel.TRITON_INFERENCE - if args.dense_backend == "aott" - else HammerKernel.PYTORCH - ) - sparse_module._sparse._hstu_model.set_hammer_kernel(HammerKernel.PYTORCH) - dense_module._hstu_model.set_hammer_kernel(dense_kernel) - - # Diagnostic: walk every HammerModule submodule and print its effective - # kernel selection, so any submodule that didn't pick up the override - # surfaces immediately. Triton/Triton-CC selections will fail at trace - # time, so this print is critical for triaging the next iteration if - # tracing fails. - from generative_recommenders.common import HammerModule as _HM - - for name, m in list(sparse_module.named_modules()) + list( - dense_module.named_modules() - ): - if isinstance(m, _HM): - logger.info( - "kernel-pin %-60s -> %s (is_inference=%s, use_triton_cc=%s)", - name or "", - m.hammer_kernel().value, - m._is_inference, - m._use_triton_cc, - ) - - # === 1. Eager reference === - logger.info("Running eager reference...") - preds_eager = _eager_run( - sparse_module, dense_module, uih_kjt, candidates_kjt, device - ) - logger.info( - "preds_eager shape=%s sum=%.6f", - tuple(preds_eager.shape), - preds_eager.sum().item(), - ) - - # === 2. Trace/lower + save === - # The default path keeps D102's trace-based TorchScript artifact. The - # AOT-T path follows ModelStore's compile/transform flow and saves a - # scripted FX module whose Triton kernels dispatch through torch.ops. - workdir = tempfile.mkdtemp(prefix="hstu_e2e_") - sparse_path = os.path.join(workdir, "sparse.pt") - dense_path = os.path.join(workdir, "dense.pt") - inputs_path = os.path.join(workdir, "inputs.pt") - cpp_out_path = os.path.join(workdir, "preds_cpp.pt") - eager_out_path = os.path.join(workdir, "preds_eager.pt") - aott_library_paths: List[str] = list(args.aott_library) - python_roundtrip_aott_library_paths: List[str] = list(args.aott_library) - logger.info("workdir: %s", workdir) - - # Re-run sparse eagerly to capture an example output that can drive the - # dense trace. - with torch.no_grad(): - sparse_out = sparse_module( - uih_features=uih_kjt, candidates_features=candidates_kjt - ) - seq_emb_values = { - k: v.to(device).to(torch.bfloat16) for k, v in sparse_out[0].items() - } - seq_emb_lengths = {k: v.to(device) for k, v in sparse_out[1].items()} - payload = {k: v.to(device) for k, v in sparse_out[2].items()} - uih_lens = sparse_out[3].to(device) - num_cands = sparse_out[4].to(device) - - logger.info("Tracing sparse module via raw-tensor shim (CPU)...") - sparse_shim = _SparseTraceShim( - sparse_module=sparse_module, - uih_keys=list(uih_kjt.keys()), - candidates_keys=list(candidates_kjt.keys()), - ) - traced_sparse = torch.jit.trace( - sparse_shim, - example_inputs=( - uih_kjt.lengths(), - uih_kjt.values(), - candidates_kjt.lengths(), - candidates_kjt.values(), - ), - strict=False, - check_trace=False, - ) - traced_sparse.save(sparse_path) - - dense_inputs = ( - seq_emb_values, - seq_emb_lengths, - payload, - uih_lens, - num_cands, - ) - if args.dense_backend == "aott": - logger.info("Lowering dense module with AOT-T...") - generated_aott_library_paths = _save_aott_dense_module( - dense_module, - dense_inputs, - dense_path, - workdir, - args.atol, - args.rtol, - ) - aott_library_paths.extend(generated_aott_library_paths) - else: - logger.info("Tracing dense module (cuda:0, bf16)...") - traced_dense = torch.jit.trace( - dense_module, - example_inputs=dense_inputs, - strict=False, - check_trace=False, - ) - traced_dense.save(dense_path) - - logger.info("Scripting + saving inputs bundle...") - torch.jit.script(InputsBundle(uih_kjt, candidates_kjt)).save(inputs_path) - torch.save(preds_eager, eager_out_path) - - # === 2.5. Python-side roundtrip verification === - # Load the saved traced artifacts back in Python and verify they produce - # the same results as the eager run. This proves the artifacts are correct - # independently of the C++ runner. - logger.info("Python roundtrip: loading traced artifacts back...") - if python_roundtrip_aott_library_paths: - _load_aott_libraries_for_python(python_roundtrip_aott_library_paths) - rt_inputs = torch.jit.load(inputs_path) - rt_sparse = torch.jit.load(sparse_path) - rt_dense = torch.jit.load(dense_path) - - with torch.no_grad(): - rt_uih_l, rt_uih_v, rt_cand_l, rt_cand_v = rt_inputs() - logger.info( - " rt inputs: uih_l=%s uih_v=%s cand_l=%s cand_v=%s", - rt_uih_l.shape, - rt_uih_v.shape, - rt_cand_l.shape, - rt_cand_v.shape, - ) - - rt_sparse_out = rt_sparse(rt_uih_l, rt_uih_v, rt_cand_l, rt_cand_v) - - for i, elem in enumerate(rt_sparse_out): - if isinstance(elem, dict): - for k, v in elem.items(): - has_nan = torch.isnan(v).any().item() - has_inf = torch.isinf(v).any().item() - logger.info( - " sparse_out[%d][%s] shape=%s dtype=%s nan=%s inf=%s", - i, - k, - tuple(v.shape), - v.dtype, - has_nan, - has_inf, - ) - elif isinstance(elem, torch.Tensor): - logger.info( - " sparse_out[%d] shape=%s dtype=%s nan=%s inf=%s", - i, - tuple(elem.shape), - elem.dtype, - torch.isnan(elem).any().item(), - torch.isinf(elem).any().item(), - ) - - rt_sev = { - k: v.to(device).to(torch.bfloat16) for k, v in rt_sparse_out[0].items() - } - rt_sel = {k: v.to(device) for k, v in rt_sparse_out[1].items()} - rt_pay = {k: v.to(device) for k, v in rt_sparse_out[2].items()} - rt_uih = rt_sparse_out[3].to(device) - rt_nc = rt_sparse_out[4].to(device) - - preds_rt = rt_dense(rt_sev, rt_sel, rt_pay, rt_uih, rt_nc) - - preds_rt_cpu = preds_rt.detach().to(torch.float32).cpu() - logger.info( - "preds_roundtrip shape=%s sum=%.6f nan=%s inf=%s", - tuple(preds_rt_cpu.shape), - preds_rt_cpu.sum().item(), - torch.isnan(preds_rt_cpu).any().item(), - torch.isinf(preds_rt_cpu).any().item(), - ) - - try: - torch.testing.assert_close( - preds_eager, preds_rt_cpu, atol=args.atol, rtol=args.rtol - ) - except AssertionError as e: - logger.error("PYTHON ROUNDTRIP PARITY FAILED: %s", e) - if not args.keep_workdir: - logger.info("(workdir kept for inspection: %s)", workdir) - sys.exit(1) - logger.info("PYTHON ROUNDTRIP PASSED (atol=%g rtol=%g)", args.atol, args.rtol) - - # === 3. Invoke C++ runner === - runner_args: List[str] = [] - for library_path in aott_library_paths: - runner_args.extend(["--aott_library", library_path]) - runner_args.extend([sparse_path, dense_path, inputs_path, cpp_out_path]) - - logger.info("Running C++: %s %s", runner_path, " ".join(runner_args)) - # pyre-fixme[6]: TrustedSubprocessWithList requires Literal[str] but this - # runner is resolved from a built resource or explicit test argument. - result = TrustedSubprocessWithList.run( - executable=runner_path, - cmd_args=runner_args, - capture_output=True, - text=True, - check=False, - ) - if result.stdout: - logger.info("--- runner stdout ---\n%s", result.stdout.rstrip()) - if result.stderr: - logger.info("--- runner stderr ---\n%s", result.stderr.rstrip()) - if result.returncode != 0: - if result.returncode == -11: - logger.warning( - "C++ runner SIGSEGV (exit -11). This is a known issue with " - "torch-cpp-cuda static initialization on some machines. " - "Python roundtrip verification passed above. " - "Artifacts in: %s", - workdir, - ) - args.keep_workdir = True - else: - logger.error("C++ runner exited with code %d", result.returncode) - if not args.keep_workdir: - shutil.rmtree(workdir, ignore_errors=True) - sys.exit(result.returncode) - - # === 4. Compare === - if not os.path.exists(cpp_out_path): - logger.error("C++ runner did not produce %s", cpp_out_path) - sys.exit(1) - preds_cpp = torch.load(cpp_out_path, weights_only=False).to(torch.float32).cpu() - logger.info( - "preds_cpp shape=%s sum=%.6f", - tuple(preds_cpp.shape), - preds_cpp.sum().item(), - ) - - try: - torch.testing.assert_close( - preds_eager, preds_cpp, atol=args.atol, rtol=args.rtol - ) - except AssertionError as e: - logger.error("PARITY FAILED: %s", e) - if not args.keep_workdir: - logger.info("(workdir kept for inspection: %s)", workdir) - sys.exit(1) - - logger.info("PASSED: eager and C++ agree (atol=%g rtol=%g)", args.atol, args.rtol) - if not args.keep_workdir: - shutil.rmtree(workdir, ignore_errors=True) - - -if __name__ == "__main__": - main() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/gin/debug.gin b/recommendation_v4/generative_recommenders/dlrm_v3/inference/gin/debug.gin deleted file mode 100644 index e2025dee0..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/gin/debug.gin +++ /dev/null @@ -1,13 +0,0 @@ -run.model_path = "" -run.scenario_name = "Server" -run.batchsize = 16 -run.output_trace = False -run.data_producer_threads = 4 -run.compute_eval = False -run.find_peak_performance = False -run.train_split_percentage = 0.75 - -# below will override mlperf rules compliant settings - don't use for official submission -run.target_qps = 2000 -run.num_queries = 10000 -run.numpy_rand_seed = 123 diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/gin/kuairand_1k.gin b/recommendation_v4/generative_recommenders/dlrm_v3/inference/gin/kuairand_1k.gin deleted file mode 100644 index a770aa014..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/gin/kuairand_1k.gin +++ /dev/null @@ -1,14 +0,0 @@ -# run.model_path = "/home/linjianma/ckpts/kuairand_1k/2025_01_12_17_56_43/" -run.scenario_name = "Server" -run.batchsize = 16 -run.output_trace = False -run.data_producer_threads = 4 -run.compute_eval = False -run.find_peak_performance = False -run.train_split_percentage = 0.75 - -# below will override mlperf rules compliant settings - don't use for official submission -run.target_qps = 2000 -run.num_queries = 10000 -run.numpy_rand_seed = 123 -run.dataset_path_prefix = "/home/linjianma" diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/gin/movielens_13b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/inference/gin/movielens_13b.gin deleted file mode 100644 index 3121ac0e7..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/gin/movielens_13b.gin +++ /dev/null @@ -1,16 +0,0 @@ -run.model_path = "" -run.scenario_name = "Server" -run.batchsize = 5 -run.output_trace = False -run.data_producer_threads = 8 -run.compute_eval = False -run.find_peak_performance = False -run.train_split_percentage = 0.75 -run.sparse_quant = False - -# below will override mlperf rules compliant settings - don't use for official submission -run.target_qps = 5000 -run.num_queries = 30000 -run.numpy_rand_seed = 123 -run.dataset_path_prefix = "/home/linjianma" -run.dataset_percentage = 0.0625 diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/gin/streaming_100b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/inference/gin/streaming_100b.gin deleted file mode 100644 index 0655734c2..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/gin/streaming_100b.gin +++ /dev/null @@ -1,15 +0,0 @@ -# run.model_path = "/home/linjianma/ckpts/streaming_100b/89/" -run.scenario_name = "Server" -run.batchsize = 10 -run.output_trace = False -run.data_producer_threads = 16 -run.compute_eval = False -run.find_peak_performance = False -run.sparse_quant = False -run.numpy_rand_seed = 123 -run.dataset_path_prefix = "/home/linjianma" -run.dataset_percentage = 0.001 -run.warmup_ratio = 0.3 -run.num_queries = 20000 -# Needs to be tuned for different implementations to balance latency and throughput -run.target_qps = 1000 diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/gin/streaming_400m.gin b/recommendation_v4/generative_recommenders/dlrm_v3/inference/gin/streaming_400m.gin deleted file mode 100644 index eed13e0ff..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/gin/streaming_400m.gin +++ /dev/null @@ -1,15 +0,0 @@ -run.model_path = "" -run.scenario_name = "Server" -run.batchsize = 5 -run.output_trace = False -run.data_producer_threads = 8 -run.compute_eval = False -run.find_peak_performance = False -run.train_split_percentage = 0.75 -run.sparse_quant = False - -# below will override mlperf rules compliant settings - don't use for official submission -run.target_qps = 5000 -run.numpy_rand_seed = 123 -run.dataset_path_prefix = "/home/linjianma" -run.dataset_percentage = 0.00625 diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/inference_modules.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/inference_modules.py deleted file mode 100644 index cb567df63..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/inference_modules.py +++ /dev/null @@ -1,253 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-unsafe -""" -Inference modules for DLRMv3. - -This module provides inference-specific components for the HSTU model, -including sparse inference modules and utilities for moving tensors between devices. -""" - -from typing import Dict, Optional, Tuple - -import torch -import torchrec -from generative_recommenders.modules.dlrm_hstu import ( - DlrmHSTU, - DlrmHSTUConfig, - SequenceEmbedding, -) -from torchrec.modules.embedding_modules import ( - EmbeddingBagCollection, - EmbeddingCollection, -) -from torchrec.sparse.jagged_tensor import JaggedTensor, KeyedJaggedTensor -from torchrec.sparse.tensor_dict import maybe_td_to_kjt - - -IS_INFERENCE: bool = True - - -class _NoCopyEmbeddingCollection(torchrec.EmbeddingCollection): - """ - EmbeddingCollection variant that skips the dtype-cast copy in - ``EmbeddingCollection.forward`` and clamps indices into the hash-size - range. This is the script-mode replacement for the - ``functools.partial`` monkey-patch in - :func:`generative_recommenders.dlrm_v3.inference.model_family.ec_patched_forward_wo_embedding_copy`. - - The body mirrors that helper exactly so that the eager and scripted paths - produce the same embeddings. - """ - - def forward( - self, - features: KeyedJaggedTensor, - ) -> Dict[str, JaggedTensor]: - features = maybe_td_to_kjt(features, None) - feature_embeddings: Dict[str, JaggedTensor] = {} - jt_dict: Dict[str, JaggedTensor] = features.to_dict() - # Inline HASH_SIZE_1B - 1 as a literal so TorchScript can see it; the - # imported module-level constant is treated as an opaque "closed-over - # global" by jit.script and would fail with - # "python value of type 'int' cannot be used as a value". - max_index: int = 999_999_999 # HASH_SIZE_1B - 1 - for i, emb_module in enumerate(self.embeddings.values()): - feature_names = self._feature_names[i] - embedding_names = self._embedding_names_by_table[i] - for j, embedding_name in enumerate(embedding_names): - feature_name = feature_names[j] - f = jt_dict[feature_name] - indices = torch.clamp(f.values(), min=0, max=max_index) - lookup = emb_module(input=indices) - feature_embeddings[embedding_name] = JaggedTensor( - values=lookup, - lengths=f.lengths(), - weights=f.values() if self._need_indices else None, - ) - return feature_embeddings - - -def set_is_inference(is_inference: bool = False) -> None: - """ - Set the global inference mode flag. - - Args: - is_inference: If True, model operates in inference mode (no labels/weights). - If False, model operates in training/eval mode with labels. - """ - global IS_INFERENCE - IS_INFERENCE = is_inference - - -def get_hstu_model( - table_config, - hstu_config: DlrmHSTUConfig, - table_device: str = "meta", - max_hash_size: Optional[int] = None, - is_dense: bool = False, -) -> DlrmHSTU: - """ - Create and initialize an HSTU model for inference. - - Args: - table_config: Dictionary of embedding table configurations. - hstu_config: HSTU model configuration object. - table_device: Device to place embedding tables on ('meta', 'cpu', or 'cuda'). - max_hash_size: Optional maximum hash size to cap embedding table sizes. - is_dense: If True, creates model for dense-only operations. - - Returns: - Initialized DlrmHSTU model in eval mode. - """ - if max_hash_size is not None: - for t in table_config.values(): - t.num_embeddings = ( - max_hash_size if t.num_embeddings > max_hash_size else t.num_embeddings - ) - model = DlrmHSTU( - hstu_configs=hstu_config, - embedding_tables=table_config, - is_inference=IS_INFERENCE, - is_dense=is_dense, - ) - model.eval() - model.recursive_setattr("_use_triton_cc", False) - for _, module in model.named_modules(): - if isinstance(module, EmbeddingBagCollection) or isinstance( - module, EmbeddingCollection - ): - module.to_empty(device=table_device) - # to_empty leaves parameters uninitialized; fill with small random - # values so downstream bf16 ops don't produce NaN from - # uninitialized memory. - for p in module.parameters(): - if not p.is_meta: - torch.nn.init.uniform_(p, -0.01, 0.01) - return model - - -class HSTUSparseInferenceModule(torch.nn.Module): - """ - Module for sparse (embedding) inference operations. - - Handles embedding lookups and preprocessing for the HSTU model, - running on CPU to handle large embedding tables. - - Args: - table_config: Dictionary of embedding table configurations. - hstu_config: HSTU model configuration object. - """ - - def __init__( - self, - table_config, - hstu_config: DlrmHSTUConfig, - ) -> None: - super().__init__() - self._hstu_model: DlrmHSTU = get_hstu_model( - table_config, - hstu_config, - table_device="cpu", - ) - - def forward( - self, - uih_features: KeyedJaggedTensor, - candidates_features: KeyedJaggedTensor, - ) -> Tuple[ - Dict[str, SequenceEmbedding], - Dict[str, torch.Tensor], - int, - torch.Tensor, - int, - torch.Tensor, - ]: - """ - Run sparse preprocessing and embedding lookups. - - Args: - uih_features: User interaction history features as KeyedJaggedTensor. - candidates_features: Candidate item features as KeyedJaggedTensor. - - Returns: - Tuple containing: - - seq_embeddings: Dictionary of sequence embeddings per feature. - - payload_features: Dictionary of payload feature tensors. - - max_uih_len: Maximum user interaction history length. - - uih_seq_lengths: Tensor of UIH sequence lengths per batch item. - - max_num_candidates: Maximum number of candidates. - - num_candidates: Tensor of candidate counts per batch item. - """ - ( - seq_embeddings, - payload_features, - max_uih_len, - uih_seq_lengths, - max_num_candidates, - num_candidates, - ) = self._hstu_model.preprocess( - uih_features=uih_features, - candidates_features=candidates_features, - ) - return ( - seq_embeddings, - payload_features, - max_uih_len, - uih_seq_lengths, - max_num_candidates, - num_candidates, - ) - - -def move_sparse_output_to_device( - seq_embeddings: Dict[str, SequenceEmbedding], - payload_features: Dict[str, torch.Tensor], - uih_seq_lengths: torch.Tensor, - num_candidates: torch.Tensor, - device: torch.device, -) -> Tuple[ - Dict[str, SequenceEmbedding], - Dict[str, torch.Tensor], - torch.Tensor, - torch.Tensor, -]: - """ - Move sparse module outputs from CPU to the target device (typically GPU). - - Converts embeddings to bfloat16 for efficient GPU computation. - - Args: - seq_embeddings: Dictionary of sequence embeddings to move. - payload_features: Dictionary of payload features to move. - uih_seq_lengths: UIH sequence lengths tensor to move. - num_candidates: Number of candidates tensor to move. - device: Target device (e.g., torch.device('cuda:0')). - - Returns: - Tuple of moved tensors on the target device. - """ - num_candidates = num_candidates.to(device) - uih_seq_lengths = uih_seq_lengths.to(device) - seq_embeddings = { - k: SequenceEmbedding( - lengths=seq_embeddings[k].lengths.to(device), - embedding=seq_embeddings[k].embedding.to(device).to(torch.bfloat16), - ) - for k in seq_embeddings.keys() - } - for k, v in payload_features.items(): - payload_features[k] = v.to(device) - return seq_embeddings, payload_features, uih_seq_lengths, num_candidates diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/main.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/main.py deleted file mode 100644 index 00e334119..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/main.py +++ /dev/null @@ -1,805 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-strict -""" -mlperf dlrm_v3 inference benchmarking tool. -""" - -import argparse -import array -import logging -import random -import threading - -logging.basicConfig(level=logging.INFO) -import os -import sys -import time -from typing import Any, Dict, List, Optional, Union - -import gin - -# pyre-ignore [21] -import mlperf_loadgen as lg # @manual -import numpy as np -import torch -from generative_recommenders.common import set_dev_mode, set_verbose_level -from generative_recommenders.dlrm_v3.configs import ( - get_embedding_table_config, - get_hstu_configs, -) -from generative_recommenders.dlrm_v3.datasets.dataset import Dataset, Samples -from generative_recommenders.dlrm_v3.datasets.synthetic_streaming import ( - DLRMv3SyntheticStreamingDataset, -) -from generative_recommenders.dlrm_v3.inference.data_producer import ( - MultiThreadDataProducer, - QueryItem, - SingleThreadDataProducer, -) -from generative_recommenders.dlrm_v3.inference.inference_modules import set_is_inference -from generative_recommenders.dlrm_v3.inference.model_family import HSTUModelFamily -from generative_recommenders.dlrm_v3.utils import ( - get_dataset, - profiler_or_nullcontext, - SUPPORTED_DATASETS, -) - - -logger: logging.Logger = logging.getLogger("main") - -torch.multiprocessing.set_start_method("spawn", force=True) - -USER_CONF = f"{os.path.dirname(__file__)}/user.conf" - -SUPPORTED_CONFIGS = { - "debug": "debug.gin", - "kuairand-1k": "kuairand_1k.gin", - "movielens-13b": "movielens_13b.gin", - "streaming-400m": "streaming_400m.gin", - "sampled-streaming-100b": "streaming_100b.gin", -} - - -SCENARIO_MAP = { # pyre-ignore [5] - "Server": lg.TestScenario.Server, - "Offline": lg.TestScenario.Offline, -} - - -def get_args(): # pyre-ignore [3] - """Parse commandline.""" - parser = argparse.ArgumentParser() - parser.add_argument( - "--dataset", default="debug", choices=SUPPORTED_DATASETS, help="dataset" - ) - args, unknown_args = parser.parse_known_args() - logger.warning(f"unknown_args: {unknown_args}") - return args - - -class Runner: - """ - Orchestrates inference benchmark execution. - - Manages data production, model inference, and result collection for - MLPerf LoadGen-based benchmarking. - - Args: - model: The HSTU model family instance for making predictions. - ds: Dataset to fetch samples from. - num_queries: Total number of queries to process. - data_producer_threads: Number of threads for data loading (default: 1). - batchsize: Batch size for inference (default: 128). - compute_eval: Whether to compute evaluation metrics (default: False). - """ - - def __init__( - self, - model: HSTUModelFamily, - ds: Dataset, - num_queries: int, - data_producer_threads: int = 1, - batchsize: int = 128, - compute_eval: bool = False, - ) -> None: - self.model = model - if data_producer_threads == 1: - self.data_producer: Union[ - MultiThreadDataProducer, SingleThreadDataProducer - ] = SingleThreadDataProducer(ds, self.run_one_item) - else: - self.data_producer = MultiThreadDataProducer( - ds, data_producer_threads, self.run_one_item - ) - self.batchsize = batchsize - self.compute_eval = compute_eval - self.reset_states(num_queries=num_queries) - - def reset_states(self, num_queries: int) -> None: - """ - Reset all internal state for a new benchmark run. - - Args: - num_queries: Number of queries expected in this run. - """ - self.result_timing: List[Dict[str, float]] = [] - self.result_batches: List[int] = [] - self.current_query_ids: List[int] = [] - self.current_content_ids: List[int] = [] - self.current_t0: List[float] = [] - self.num_queries: int = num_queries - self.processed_queries: int = 0 - - def run_one_item(self, qitem: QueryItem) -> None: - """ - Process a single query item through model inference. - - Runs prediction, records timing metrics, and sends results back to LoadGen. - - Args: - qitem: Query item containing batch of samples to process. - """ - try: - t0_prediction: float = time.time() - prediction_output = self.model.predict(qitem.samples) - dt_prediction: float = time.time() - t0_prediction - assert prediction_output is not None - ( - mt_target_preds, - mt_target_labels, - mt_target_weights, - dt_sparse, - dt_dense, - ) = prediction_output - if self.compute_eval: - assert mt_target_labels is not None - assert mt_target_weights is not None - self.result_timing.append( - { - "total": time.time() - qitem.start, - "prediction": dt_prediction, - "queue": qitem.dt_queue, - "batching": qitem.dt_batching, - "sparse": dt_sparse, - "dense": dt_dense, - } - ) - self.result_batches.append(len(qitem.query_ids)) - except Exception as ex: # pylint: disable=broad-except - logger.error("thread: failed, %s", ex) - finally: - candidate_size = mt_target_preds.size(1) // len(qitem.query_ids) - if not self.compute_eval: - for i, query_id in enumerate(qitem.query_ids): - query_mt_target_preds = ( - mt_target_preds[ # pyre-ignore [61] - 0, - candidate_size * i : candidate_size * (i + 1), - ] - .view(-1) - .float() - .numpy() - ) - response_array = array.array("B", query_mt_target_preds.tobytes()) - bi = response_array.buffer_info() - # since we send buffer to loadgen, needs `response_array` in memory during send - lg.QuerySamplesComplete( - [lg.QuerySampleResponse(query_id, bi[0], bi[1])] - ) - else: - for i, query_id in enumerate(qitem.query_ids): - query_mt_target_preds = ( - mt_target_preds[ # pyre-ignore [61] - 0, candidate_size * i : candidate_size * (i + 1) - ] - .view(-1) - .float() - .numpy() - ) - query_mt_target_labels = ( - mt_target_labels[ # pyre-ignore [16,61] - 0, candidate_size * i : candidate_size * (i + 1) - ] - .view(-1) - .float() - .numpy() - ) - query_mt_target_weights = ( - mt_target_weights[ # pyre-ignore [61] - 0, candidate_size * i : candidate_size * (i + 1) - ] - .view(-1) - .float() - .numpy() - ) - np_array = np.concatenate( - [ - query_mt_target_preds, - query_mt_target_labels, - query_mt_target_weights, - np.array([candidate_size]).astype(np.float32), - ] - ) - response_array = array.array("B", np_array.tobytes()) - bi = response_array.buffer_info() - # since we send buffer to loadgen, needs `response_array` in memory during send - lg.QuerySamplesComplete( - [lg.QuerySampleResponse(query_id, bi[0], bi[1])] - ) - - def enqueue(self, query_samples, t0: float) -> None: # pyre-ignore [2] - """ - Enqueue query samples for batch processing. - - Collects samples until batch size is reached, then dispatches to data producer. - - Args: - query_samples: List of LoadGen query sample objects. - t0: Timestamp when this batch started. - """ - self.current_query_ids.extend([q.id for q in query_samples]) - self.current_content_ids.extend([q.index for q in query_samples]) - self.current_t0.append(t0) - self.processed_queries += len(query_samples) - t0: float = min(self.current_t0) - dt_queue: float = max(self.current_t0) - min(self.current_t0) - if ( - self.processed_queries >= self.num_queries - or len(self.current_query_ids) >= self.batchsize - ): - for i in range(len(self.current_query_ids) // self.batchsize): - self.data_producer.enqueue( - query_ids=self.current_query_ids[ - i * self.batchsize : (i + 1) * self.batchsize - ], - content_ids=self.current_content_ids[ - i * self.batchsize : (i + 1) * self.batchsize - ], - t0=t0, - dt_queue=dt_queue, - ) - remaining_s: int = len(self.current_query_ids) % self.batchsize - if remaining_s > 0: - self.data_producer.enqueue( - query_ids=self.current_query_ids[-remaining_s:], - content_ids=self.current_content_ids[-remaining_s:], - t0=t0, - dt_queue=dt_queue, - ) - self.current_query_ids = [] - self.current_content_ids = [] - self.current_t0 = [] - - def finish(self) -> None: - """Signal data producer to finish and wait for completion.""" - self.data_producer.finish() - - -def add_results( - final_results: Dict[str, Any], - result_timing: List[Dict[str, float]], - result_batches: List[int], -) -> None: - """ - Aggregate and log benchmark results. - - Computes percentile statistics and QPS metrics from timing data. - - Args: - final_results: Dictionary to populate with aggregated results. - result_timing: List of timing dictionaries for each batch. - result_batches: List of batch sizes processed. - """ - percentiles: list[float] = [50.0, 80.0, 90.0, 95.0, 99.0, 99.9] - buckets_dict: Dict[str, List[float]] = {} - buckets_str_dict: Dict[str, str] = {} - total_timing: list[float] = [result["total"] for result in result_timing] - for key in ["total", "prediction", "queue", "batching", "sparse", "dense"]: - timing: list[float] = [result[key] for result in result_timing] - buckets: List[float] = np.percentile(timing, percentiles).tolist() - buckets_str: str = ",".join( - ["| {}:{:.4f}| ".format(p, b) for p, b in zip(percentiles, buckets)] - ) - buckets_dict[key] = buckets - buckets_str_dict[key] = buckets_str - total_batches = sum(result_batches) - - final_results["good"] = len(total_timing) - final_results["avg_time"] = np.mean(total_timing) - final_results["percentiles"] = { - str(k): v for k, v in zip(percentiles, buckets_dict["total"]) - } - final_results["qps"] = total_batches / final_results["took"] - final_results["count"] = total_batches - - for i, timing in enumerate(result_timing): - logger.warning(f"timing of {i}: {timing}") - - logger.warning( - "{} qps={:.2f}, avg_query_time={:.4f}, time={:.3f}, queries={}, tiles={}".format( - final_results["scenario"], - final_results["qps"], - final_results["avg_time"], - final_results["took"], - len(result_timing), - buckets_str_dict["total"], - ) - ) - for key in ["prediction", "queue", "batching", "sparse", "dense"]: - logger.warning(f"{key}: {buckets_str_dict[key]}") - - -def get_num_queries( - input_size: Optional[int], - one_pass_size: int, - scenario_name: str, - offline_target_qps: int, - target_duration: float, -) -> int: - """ - Determine the number of queries to run based on scenario and settings. - - Args: - input_size: User-specified query count (None to use defaults). - one_pass_size: Size of one complete pass through the dataset. - scenario_name: MLPerf scenario name ('Server' or 'Offline'). - offline_target_qps: Target QPS for offline scenario. - target_duration: Target duration in milliseconds. - - Returns: - Number of queries to execute in the benchmark run. - """ - if scenario_name == "Offline": - # consistent with https://github.com/mlcommons/inference/blob/8999c4d686f6e4a180da14597c97063fce7c9f33/loadgen/test_settings_internal.cc#L147 - return int(1.1 * target_duration / 1000 * offline_target_qps) - else: - if input_size is None: - return one_pass_size - return input_size - - -class StreamingQuerySampler: - """ - Sampler for streaming dataset - The execution order is determined by `StreamingQuerySampler.run_order`, not by the QSL or input query ID. - This ensures that queries are executed according to their timestamp constraints. - """ - - def __init__( - self, - ds: DLRMv3SyntheticStreamingDataset, - dataset_percentage: float, - scenario_name: str, - offline_target_qps: int, - target_duration: float, - input_queries: Optional[int] = None, - compute_eval: bool = False, - ) -> None: - self.ds: DLRMv3SyntheticStreamingDataset = ds - self.ds.is_inference = True - self.inference_ts: int = self.ds.total_ts - self.ds.train_ts - self.start_ts: int = self.ds.train_ts - self.dataset_percentage: float = dataset_percentage - self.num_unique_requests: List[int] = self.get_num_unique_requests( - warmup_ratio=1.0 - ) - self.num_unique_requests_cumsum: List[int] = np.cumsum( - self.num_unique_requests - ).tolist() - self.total_requests: int = sum(self.num_unique_requests) - self.run_order: List[List[int]] = self.build_random_exec_order() - self.ts_idx: int = 0 - self.ts_processed_cnt: int = 0 - self.last_loaded: float = -1.0 - num_queries: int = get_num_queries( - input_size=input_queries, - one_pass_size=self.total_requests, - scenario_name=scenario_name, - offline_target_qps=offline_target_qps, - target_duration=target_duration, - ) - logger.warning( - f"StreamingQuerySampler constructred to handle {num_queries} queries" - ) - self.num_repeats: int = ( - max(1, num_queries // self.total_requests) if not compute_eval else 1 - ) - self.remaining_queries: int = ( - num_queries % self.total_requests if not compute_eval else 0 - ) - self._lock = threading.Lock() - - def get_num_unique_requests(self, warmup_ratio: float) -> List[int]: - """ - Calculate number of unique requests per timestamp. - - Args: - warmup_ratio: Fraction of users to include in warmup. - - Returns: - List of request counts per timestamp. - """ - num_unique_requests = [ - int( - self.ds.ts_to_users_cumsum[t][-1] - * self.dataset_percentage - * warmup_ratio - ) - for t in range(self.start_ts, self.start_ts + self.inference_ts) - ] - return num_unique_requests - - def build_random_exec_order(self) -> List[List[int]]: - """ - Build randomized execution order for each timestamp. - - Returns: - List of shuffled index lists, one per timestamp. - """ - order = [] - for req_size in self.num_unique_requests: - within_ts_order = list(range(req_size)) - random.shuffle(within_ts_order) - order.append(within_ts_order) - return order - - def init_sut(self) -> None: - """Initialize System Under Test state for a new benchmark run.""" - self.ts_idx = 0 - self.ts_processed_cnt = 0 - self.ds.set_ts(self.start_ts) - - def load_query_samples(self, query_ids: List[Optional[int]]) -> None: - """ - Load query samples into memory for the benchmark. - - Args: - query_ids: List of query identifiers to load. - """ - length = len(query_ids) - ts_idx: int = 0 - while self.num_unique_requests_cumsum[ts_idx] < length: - ts_idx += 1 - for i in range(0, ts_idx): - self.ds.set_ts(i + self.start_ts) - self.ds.load_query_samples(self.run_order[i]) - self.ds.set_ts(ts_idx + self.start_ts) - delta_length = ( - length - if ts_idx == 0 - else length - self.num_unique_requests_cumsum[ts_idx - 1] - ) - self.ds.load_query_samples(self.run_order[ts_idx][:delta_length]) - self.init_sut() - self.last_loaded = time.time() - - def unload_query_samples(self, sample_list: List[int]) -> None: - """ - Unload query samples from memory. - - Args: - sample_list: List of sample identifiers to unload. - """ - self.ds.unload_query_samples(sample_list) - - def get_samples(self, id_list: List[int]) -> List[Samples]: - """ - Get samples for a batch of queries, handling timestamp boundaries. - - Args: - id_list: List of query identifiers. - - Returns: - List of Samples objects, potentially spanning multiple timestamps. - """ - batch_size: int = len(id_list) - with self._lock: - curr_ts_idx: int = self.ts_idx - curr_ts_unique_requests: int = self.num_unique_requests[curr_ts_idx] - curr_ts_queries: int = curr_ts_unique_requests * self.num_repeats - if curr_ts_idx == self.inference_ts - 1: - curr_ts_queries += self.remaining_queries - begin_query_idx: int = self.ts_processed_cnt - end_query_idx: int = min(begin_query_idx + batch_size, curr_ts_queries) - begin_request_idx: int = begin_query_idx % curr_ts_unique_requests - end_request_idx: int = end_query_idx % curr_ts_unique_requests - if begin_query_idx + batch_size >= curr_ts_queries: - self.ts_idx += 1 - self.ts_processed_cnt = begin_query_idx + batch_size - curr_ts_queries - else: - self.ts_processed_cnt = begin_query_idx + batch_size - # requests of current ts - outputs: List[Samples] = [] - if end_request_idx > begin_request_idx: - output: Samples = self.ds.get_samples_with_ts( - self.run_order[curr_ts_idx][begin_request_idx:end_request_idx], - curr_ts_idx + self.start_ts, - ) - outputs.append(output) - else: - if begin_request_idx < curr_ts_unique_requests: - output: Samples = self.ds.get_samples_with_ts( - self.run_order[curr_ts_idx][begin_request_idx:], - curr_ts_idx + self.start_ts, - ) - outputs.append(output) - if end_request_idx > 0: - output = self.ds.get_samples_with_ts( - self.run_order[curr_ts_idx][0:end_request_idx], - curr_ts_idx + self.start_ts, - ) - outputs.append(output) - # requests of next ts - if begin_query_idx + batch_size > curr_ts_queries: - output: Samples = self.ds.get_samples_with_ts( - self.run_order[curr_ts_idx + 1][ - : begin_query_idx + batch_size - curr_ts_queries - ], - curr_ts_idx + 1 + self.start_ts, - ) - outputs.append(output) - return outputs - - def get_item_count(self) -> int: - """ - Get total number of items in the dataset. - - Returns: - Total request count across all timestamps. - """ - return self.total_requests - - -@gin.configurable -def run( - dataset: str = "sampled-streaming-100b", - model_path: str = "", - scenario_name: str = "Server", - batchsize: int = 16, - output_trace: bool = False, - data_producer_threads: int = 4, - compute_eval: bool = False, - find_peak_performance: bool = False, - dataset_path_prefix: str = "", - train_split_percentage: float = 0.75, - warmup_ratio: float = 0.1, - target_qps: Optional[int] = None, - num_queries: Optional[int] = None, - numpy_rand_seed: int = 123, - sparse_quant: bool = False, - dataset_percentage: float = 1.0, -) -> None: - """ - Execute the MLPerf DLRMv3 inference benchmark. - - Sets up the model, dataset, and LoadGen infrastructure, then runs - warmup and official benchmark phases. - - Args: - dataset: Dataset identifier to use. - model_path: Path to model checkpoint directory. - scenario_name: MLPerf scenario ('Server' or 'Offline'). - batchsize: Batch size for inference. - output_trace: Whether to output profiling traces. - data_producer_threads: Number of data loading threads. - compute_eval: Whether to compute accuracy metrics. - find_peak_performance: Whether to run peak performance finding mode. - dataset_path_prefix: Prefix path for dataset files. - warmup_ratio: Fraction of data to use for warmup. - target_qps: Target queries per second. - num_queries: Number of queries to run (None for automatic). - numpy_rand_seed: Random seed for reproducibility. - sparse_quant: Whether to quantize sparse embeddings. - dataset_percentage: Fraction of dataset to use. - """ - set_dev_mode(False) - if scenario_name not in SCENARIO_MAP: - raise NotImplementedError("valid scanarios:" + str(list(SCENARIO_MAP.keys()))) - scenario = SCENARIO_MAP[scenario_name] - np.random.seed(numpy_rand_seed) - random.seed(numpy_rand_seed) - - hstu_config = get_hstu_configs(dataset) - hstu_config.max_num_candidates = hstu_config.max_num_candidates_inference - table_config = get_embedding_table_config(dataset) - set_is_inference(is_inference=not compute_eval) - - user_conf = os.path.abspath(USER_CONF) - if not os.path.exists(user_conf): - logger.error("{} not found".format(user_conf)) - sys.exit(1) - - settings = lg.TestSettings() - settings.FromConfig(user_conf, model_path, scenario_name) - settings.scenario = scenario - settings.mode = lg.TestMode.PerformanceOnly - if compute_eval: - settings.mode = lg.TestMode.AccuracyOnly - if find_peak_performance: - settings.mode = lg.TestMode.FindPeakPerformance - if target_qps: - settings.server_target_qps = float(target_qps) - settings.offline_expected_qps = float(target_qps) - - model_family = HSTUModelFamily( - hstu_config=hstu_config, - table_config=table_config, - sparse_quant=sparse_quant, - output_trace=output_trace, - compute_eval=compute_eval, - ) - is_streaming: bool = "streaming" in dataset - dataset, kwargs = get_dataset(dataset, dataset_path_prefix) - - ds: Dataset = dataset( - hstu_config=hstu_config, - embedding_config=table_config, - is_inference=not compute_eval, - **kwargs, - ) - if is_streaming: - ds = StreamingQuerySampler( # pyre-ignore - ds=ds, # pyre-ignore [6] - dataset_percentage=dataset_percentage, - input_queries=num_queries, - compute_eval=compute_eval, - scenario_name=scenario_name, - offline_target_qps=settings.offline_expected_qps, - target_duration=settings.min_duration_ms, - ) - model_family.load(model_path) - - # warmup - for autotune_bs in range(batchsize, 0, -1): - logger.warning(f"Autotune for batch size {autotune_bs}") - warmup_ids = list(range(autotune_bs)) - ds.load_query_samples(warmup_ids) - for _ in range(4 * int(os.environ.get("WORLD_SIZE", 1))): - if is_streaming: - ds.init_sut() # pyre-ignore [16] - sample: Union[Samples, List[Samples]] = ds.get_samples(warmup_ids) - if isinstance(sample, Samples): - model_family.predict(sample) - else: - for s in sample: - model_family.predict(s) - ds.unload_query_samples(None) - for h in logger.handlers: - h.flush() - logger.info("Model forward warmup done") - - count = int( - ds.get_item_count() * dataset_percentage - if not is_streaming - else ds.get_item_count() - ) - train_size: int = round(train_split_percentage * count) if not is_streaming else 0 - if compute_eval: - count = count - train_size - - runner: Runner = Runner( - model_family, - ds, - data_producer_threads=data_producer_threads, - batchsize=batchsize, - compute_eval=compute_eval, - num_queries=count, - ) - - def issue_queries(query_samples) -> None: # pyre-ignore [2] - if compute_eval: - for sample in query_samples: - sample.index = sample.index + train_size - runner.enqueue(query_samples, time.time()) - - def load_query_samples(query_ids: List[int]) -> None: - if compute_eval: - query_ids = [q + train_size for q in query_ids] - ds.load_query_samples(query_ids) - - def flush_queries() -> None: - pass - - if scenario == lg.TestScenario.Server: - # inference benchmark warmup - if is_streaming: - ds.init_sut() - warmup_count: int = sum( - ds.get_num_unique_requests( # pyre-ignore [16] - warmup_ratio=warmup_ratio - ) - ) - else: - warmup_count: int = int(count * warmup_ratio) - runner.reset_states(num_queries=warmup_count) - final_results = { - "runtime": model_family.name(), - "version": model_family.version(), - "time": int(time.time()), - "scenario": str(scenario), - } - settings.min_query_count = warmup_count - settings.max_query_count = warmup_count - sut = lg.ConstructSUT(issue_queries, flush_queries) - qsl = lg.ConstructQSL( - warmup_count, - warmup_count, - load_query_samples, - ds.unload_query_samples, - ) - with profiler_or_nullcontext(enabled=output_trace, with_stack=False): - logger.info(f"starting warmup {scenario} with {warmup_count} queries") - lg.StartTest(sut, qsl, settings) - lg.DestroyQSL(qsl) - lg.DestroySUT(sut) - - # official run - if is_streaming: - ds.init_sut() - final_results = { - "runtime": model_family.name(), - "version": model_family.version(), - "time": int(time.time()), - "scenario": str(scenario), - } - query_size: int = get_num_queries( - input_size=num_queries, - one_pass_size=count, - scenario_name=scenario_name, - offline_target_qps=settings.offline_expected_qps, - target_duration=settings.min_duration_ms, - ) - settings.min_query_count = query_size - settings.max_query_count = query_size - runner.reset_states(num_queries=query_size if not compute_eval else count) - sut = lg.ConstructSUT(issue_queries, flush_queries) - qsl = lg.ConstructQSL( - count, - count, - load_query_samples, - ds.unload_query_samples, - ) - with profiler_or_nullcontext(enabled=output_trace, with_stack=False): - logger.info( - f"starting {scenario} with {query_size} queries and {query_size // count} repeats" - ) - lg.StartTest(sut, qsl, settings) - runner.finish() - final_results["took"] = time.time() - ds.last_loaded - lg.DestroyQSL(qsl) - lg.DestroySUT(sut) - - add_results( - final_results, - runner.result_timing, - runner.result_batches, - ) - # If multiple subprocesses are running the model send a signal to stop them - if int(os.environ.get("WORLD_SIZE", 1)) > 1: - model_family.predict(None) - - -def main() -> None: - set_verbose_level(1) - args = get_args() - logger.info(args) - gin_path = f"{os.path.dirname(__file__)}/gin/{SUPPORTED_CONFIGS[args.dataset]}" - gin.parse_config_file(gin_path) - run(dataset=args.dataset) - - -if __name__ == "__main__": - main() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/mlperf.conf b/recommendation_v4/generative_recommenders/dlrm_v3/inference/mlperf.conf deleted file mode 100644 index a2b4f6fff..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/mlperf.conf +++ /dev/null @@ -1,98 +0,0 @@ -# The format of this config file is 'key = value'. -# The key has the format 'model.scenario.key'. Value is mostly int64_t. -# Model maybe '*' as wildcard. In that case the value applies to all models. -# All times are in milli seconds - -# Set performance_sample_count for each model. -# User can optionally set this to higher values in user.conf. -resnet50.*.performance_sample_count_override = 1024 -ssd-mobilenet.*.performance_sample_count_override = 256 -retinanet.*.performance_sample_count_override = 64 -bert.*.performance_sample_count_override = 10833 -dlrm.*.performance_sample_count_override = 204800 -dlrm-v2.*.performance_sample_count_override = 204800 -rnnt.*.performance_sample_count_override = 2513 -gptj.*.performance_sample_count_override = 13368 -llama2-70b.*.performance_sample_count_override = 24576 -stable-diffusion-xl.*.performance_sample_count_override = 5000 -# set to 0 to let entire sample set to be performance sample -3d-unet.*.performance_sample_count_override = 0 - -# Set seeds. The seeds will be distributed two weeks before the submission. -*.*.qsl_rng_seed = 3066443479025735752 -*.*.sample_index_rng_seed = 10688027786191513374 -*.*.schedule_rng_seed = 14962580496156340209 -# Set seeds for TEST_05. The seeds will be distributed two weeks before the submission. -*.*.test05_qsl_rng_seed = 16799458546791641818 -*.*.test05_sample_index_rng_seed = 5453809927556429288 -*.*.test05_schedule_rng_seed = 5435552105434836064 - - -*.SingleStream.target_latency_percentile = 90 -*.SingleStream.min_duration = 600000 - -*.MultiStream.target_latency_percentile = 99 -*.MultiStream.samples_per_query = 8 -*.MultiStream.min_duration = 600000 -*.MultiStream.min_query_count = 662 -retinanet.MultiStream.target_latency = 528 - -# 3D-UNet uses equal issue mode because it has non-uniform inputs -3d-unet.*.sample_concatenate_permutation = 1 - -# LLM benchmarks have non-uniform inputs and outputs, and use equal issue mode for all latency scenario -gptj.*.sample_concatenate_permutation = 1 -llama2-70b.*.sample_concatenate_permutation = 1 -mixtral-8x7b.*.sample_concatenate_permutation = 1 - -*.Server.target_latency = 10 -*.Server.target_latency_percentile = 99 -*.Server.target_duration = 0 -*.Server.min_duration = 600000 -resnet50.Server.target_latency = 15 -retinanet.Server.target_latency = 100 -bert.Server.target_latency = 130 -dlrm.Server.target_latency = 60 -dlrm-v2.Server.target_latency = 60 -rnnt.Server.target_latency = 1000 -gptj.Server.target_latency = 20000 -stable-diffusion-xl.Server.target_latency = 20000 -# Llama2-70b benchmarks measures token latencies -llama2-70b.*.use_token_latencies = 1 -mixtral-8x7b.*.use_token_latencies = 1 -# gptj benchmark infers token latencies -gptj.*.infer_token_latencies = 1 -gptj.*.token_latency_scaling_factor = 69 -# Only ttft and tpot are tracked for the llama2-70b & mixtral-8x7B benchmark therefore target_latency = 0 -llama2-70b.Server.target_latency = 0 -llama2-70b.Server.ttft_latency = 2000 -llama2-70b.Server.tpot_latency = 200 - -mixtral-8x7b.Server.target_latency = 0 -mixtral-8x7b.Server.ttft_latency = 2000 -mixtral-8x7b.Server.tpot_latency = 200 - -*.Offline.target_latency_percentile = 90 -*.Offline.min_duration = 600000 - -# In Offline scenario, we always have one query. But LoadGen maps this to -# min_sample_count internally in Offline scenario. If the dataset size is larger -# than 24576 we limit the min_query_count to 24576 and otherwise we use -# the dataset size as the limit - -resnet50.Offline.min_query_count = 24576 -retinanet.Offline.min_query_count = 24576 -dlrm-v2.Offline.min_query_count = 24576 -bert.Offline.min_query_count = 10833 -gptj.Offline.min_query_count = 13368 -rnnt.Offline.min_query_count = 2513 -3d-unet.Offline.min_query_count = 43 -stable-diffusion-xl.Offline.min_query_count = 5000 -llama2-70b.Offline.min_query_count = 24576 -mixtral-8x7b.Offline.min_query_count = 15000 - -# These fields should be defined and overridden by user.conf. -*.SingleStream.target_latency = 10 -*.MultiStream.target_latency = 80 -*.Server.target_qps = 1.0 -*.Offline.target_qps = 1.0 diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/model_family.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/model_family.py deleted file mode 100644 index 1c8bcd237..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/model_family.py +++ /dev/null @@ -1,705 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-strict -""" -model_family for dlrm_v3. -""" - -import copy -import functools -import logging -import os -import time -import uuid -from threading import Event -from typing import Dict, List, Optional, Tuple, Union - -import torch -import torch.multiprocessing as mp -import torchrec -from generative_recommenders.dlrm_v3.checkpoint import ( - load_nonsparse_checkpoint, - load_sparse_checkpoint, -) -from generative_recommenders.dlrm_v3.configs import HASH_SIZE_1B -from generative_recommenders.dlrm_v3.datasets.dataset import Samples -from generative_recommenders.dlrm_v3.inference.inference_modules import ( - get_hstu_model, - HSTUSparseInferenceModule, - move_sparse_output_to_device, - set_is_inference, -) -from generative_recommenders.dlrm_v3.utils import Profiler -from generative_recommenders.modules.dlrm_hstu import DlrmHSTUConfig, SequenceEmbedding -from pyre_extensions import none_throws -from torch import quantization as quant -from torchrec.distributed.quant_embedding import QuantEmbeddingCollection -from torchrec.modules.embedding_configs import EmbeddingConfig, QuantConfig -from torchrec.sparse.jagged_tensor import JaggedTensor, KeyedJaggedTensor -from torchrec.sparse.tensor_dict import maybe_td_to_kjt -from torchrec.test_utils import get_free_port - -logger: logging.Logger = logging.getLogger(__name__) - - -class HSTUModelFamily: - """ - High-level interface for the HSTU model family. - - Manages both sparse (embedding) and dense (transformer) components of the - HSTU model, supporting distributed inference across multiple GPUs. - - Args: - hstu_config: Configuration object for the HSTU model. - table_config: Dictionary of embedding table configurations. - output_trace: Whether to enable profiling trace output. - sparse_quant: Whether to quantize sparse embeddings. - compute_eval: Whether to compute evaluation metrics (includes labels). - """ - - def __init__( - self, - hstu_config: DlrmHSTUConfig, - table_config: Dict[str, EmbeddingConfig], - output_trace: bool = False, - sparse_quant: bool = False, - compute_eval: bool = False, - ) -> None: - self.hstu_config = hstu_config - self.table_config = table_config - self.sparse: ModelFamilySparseDist = ModelFamilySparseDist( - hstu_config=hstu_config, - table_config=table_config, - quant=sparse_quant, - ) - - assert torch.cuda.is_available(), "CUDA is required for this benchmark." - ngpus = torch.cuda.device_count() - self.world_size = int(os.environ.get("WORLD_SIZE", str(ngpus))) - logger.warning(f"Using {self.world_size} GPU(s)...") - dense_model_family_clazz = ( - ModelFamilyDenseDist - if self.world_size > 1 - else ModelFamilyDenseSingleWorker - ) - self.dense: Union[ModelFamilyDenseDist, ModelFamilyDenseSingleWorker] = ( - dense_model_family_clazz( - hstu_config=hstu_config, - table_config=table_config, - output_trace=output_trace, - compute_eval=compute_eval, - ) - ) - - def version(self) -> str: - """Return the PyTorch version string.""" - return torch.__version__ - - def name(self) -> str: - """Return the model family name identifier.""" - return "model-family-hstu" - - def load(self, model_path: str) -> None: - """ - Load model checkpoints from disk. - - Args: - model_path: Base path to the model checkpoint directory. - """ - self.sparse.load(model_path=model_path) - self.dense.load(model_path=model_path) - - def predict( - self, samples: Optional[Samples] - ) -> Optional[ - Tuple[ - torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor], float, float - ] - ]: - """ - Run inference on a batch of samples. - - Processes samples through sparse embeddings, then dense forward pass. - - Args: - samples: Input samples containing features. If None, signals shutdown. - - Returns: - Tuple of (predictions, labels, weights, sparse_time, dense_time) or None. - """ - with torch.no_grad(): - if samples is None: - self.dense.predict(None, None, 0, None, 0, None) - return None - ( - seq_embeddings, - payload_features, - max_uih_len, - uih_seq_lengths, - max_num_candidates, - num_candidates, - dt_sparse, - ) = self.sparse.predict(samples) - out = self.dense.predict( - seq_embeddings, - payload_features, - max_uih_len, - uih_seq_lengths, - max_num_candidates, - num_candidates, - ) - ( # pyre-ignore [23] - mt_target_preds, - mt_target_labels, - mt_target_weights, - dt_dense, - ) = out - return ( - mt_target_preds, - mt_target_labels, - mt_target_weights, - dt_sparse, - dt_dense, - ) - - -def ec_patched_forward_wo_embedding_copy( - ec_module: torchrec.EmbeddingCollection, - features: KeyedJaggedTensor, # can also take TensorDict as input -) -> Dict[str, JaggedTensor]: - """ - Run the EmbeddingBagCollection forward pass. This method takes in a `KeyedJaggedTensor` - and returns a `Dict[str, JaggedTensor]`, which is the result of the individual embeddings for each feature. - - Args: - features (KeyedJaggedTensor): KJT of form [F X B X L]. - - Returns: - Dict[str, JaggedTensor] - """ - features = maybe_td_to_kjt(features, None) - feature_embeddings: Dict[str, JaggedTensor] = {} - jt_dict: Dict[str, JaggedTensor] = features.to_dict() - for i, emb_module in enumerate(ec_module.embeddings.values()): - feature_names = ec_module._feature_names[i] - embedding_names = ec_module._embedding_names_by_table[i] - for j, embedding_name in enumerate(embedding_names): - feature_name = feature_names[j] - f = jt_dict[feature_name] - indices = torch.clamp(f.values(), min=0, max=HASH_SIZE_1B - 1) - lookup = emb_module( - input=indices - ) # remove the dtype cast at https://github.com/meta-pytorch/torchrec/blob/0a2cebd5472a7edc5072b3c912ad8aaa4179b9d9/torchrec/modules/embedding_modules.py#L486 - feature_embeddings[embedding_name] = JaggedTensor( - values=lookup, - lengths=f.lengths(), - weights=f.values() if ec_module._need_indices else None, - ) - return feature_embeddings - - -class ModelFamilySparseDist: - """ - Sparse Arch module manager. - - Handles loading and inference of sparse embedding lookups, optionally - with quantization for memory efficiency. - - Args: - hstu_config: HSTU model configuration. - table_config: Embedding table configurations. - quant: Whether to apply dynamic quantization to embeddings. - """ - - def __init__( - self, - hstu_config: DlrmHSTUConfig, - table_config: Dict[str, EmbeddingConfig], - quant: bool = False, - ) -> None: - super(ModelFamilySparseDist, self).__init__() - self.hstu_config = hstu_config - self.table_config = table_config - self.module: Optional[torch.nn.Module] = None - self.quant: bool = quant - - def load(self, model_path: str) -> None: - """ - Load sparse model checkpoint and optionally apply quantization. - - Args: - model_path: Path to the model checkpoint directory. - """ - logger.warning(f"Loading sparse module from {model_path}") - - sparse_arch: HSTUSparseInferenceModule = HSTUSparseInferenceModule( - table_config=self.table_config, - hstu_config=self.hstu_config, - ) - load_sparse_checkpoint(model=sparse_arch._hstu_model, path=model_path) - sparse_arch.eval() - if self.quant: - self.module = quant.quantize_dynamic( - sparse_arch, - qconfig_spec={ - torchrec.EmbeddingCollection: QuantConfig( - activation=quant.PlaceholderObserver.with_args( - dtype=torch.float - ), - weight=quant.PlaceholderObserver.with_args(dtype=torch.int8), - ), - }, - mapping={ - torchrec.EmbeddingCollection: QuantEmbeddingCollection, - }, - inplace=False, - ) - else: - sparse_arch._hstu_model._embedding_collection.forward = ( # pyre-ignore[8] - functools.partial( - ec_patched_forward_wo_embedding_copy, - sparse_arch._hstu_model._embedding_collection, - ) - ) - self.module = sparse_arch - logger.warning(f"sparse module is {self.module}") - - def predict( - self, samples: Samples - ) -> Tuple[ - Dict[str, SequenceEmbedding], - Dict[str, torch.Tensor], - int, - torch.Tensor, - int, - torch.Tensor, - float, - ]: - """ - Run sparse forward pass (embedding lookups). - - Args: - samples: Input samples with feature tensors. - - Returns: - Tuple of (seq_embeddings, payload_features, max_uih_len, uih_seq_lengths, - max_num_candidates, num_candidates, elapsed_time). - """ - with torch.profiler.record_function("sparse forward"): - module: torch.nn.Module = none_throws(self.module) - assert self.module is not None - uih_features = samples.uih_features_kjt - candidates_features = samples.candidates_features_kjt - t0: float = time.time() - ( - seq_embeddings, - payload_features, - max_uih_len, - uih_seq_lengths, - max_num_candidates, - num_candidates, - ) = module( - uih_features=uih_features, - candidates_features=candidates_features, - ) - dt_sparse: float = time.time() - t0 - return ( - seq_embeddings, - payload_features, - max_uih_len, - uih_seq_lengths, - max_num_candidates, - num_candidates, - dt_sparse, - ) - - -class ModelFamilyDenseDist: - """ - Distributed dense module manager for multi-GPU inference. - - Spawns worker processes for each GPU to run dense forward passes in parallel, - with samples distributed via inter-process queues. - - Args: - hstu_config: HSTU model configuration. - table_config: Embedding table configurations. - output_trace: Whether to enable profiling traces. - compute_eval: Whether to compute evaluation metrics. - """ - - def __init__( - self, - hstu_config: DlrmHSTUConfig, - table_config: Dict[str, EmbeddingConfig], - output_trace: bool = False, - compute_eval: bool = False, - ) -> None: - super(ModelFamilyDenseDist, self).__init__() - self.hstu_config = hstu_config - self.table_config = table_config - self.output_trace = output_trace - self.compute_eval = compute_eval - - ngpus = torch.cuda.device_count() - self.world_size = int(os.environ.get("WORLD_SIZE", str(ngpus))) - self.rank = 0 - os.environ["MASTER_ADDR"] = "localhost" - os.environ["MASTER_PORT"] = str(get_free_port()) - self.dist_backend = "nccl" - - ctx = mp.get_context("spawn") - self.samples_q: List[mp.Queue] = [ctx.Queue() for _ in range(self.world_size)] - self.result_q: List[mp.Queue] = [ctx.Queue() for _ in range(self.world_size)] - - def load(self, model_path: str) -> None: - """ - Load dense model and spawn worker processes for distributed inference. - - Args: - model_path: Path to the model checkpoint directory. - """ - logger.warning(f"Loading dense module from {model_path}") - - ctx = mp.get_context("spawn") - processes = [] - for rank in range(self.world_size): - p = ctx.Process( - target=self.distributed_setup, - args=( - rank, - self.world_size, - model_path, - ), - ) - p.start() - processes.append(p) - - def distributed_setup(self, rank: int, world_size: int, model_path: str) -> None: - """ - Initialize and run a dense worker process. - - Each worker loads the model, processes samples from its queue, and - returns results. - - Args: - rank: Process rank (GPU index). - world_size: Total number of worker processes. - model_path: Path to model checkpoint. - """ - nprocs_per_rank = 16 - start_core: int = nprocs_per_rank * rank - cores: set[int] = set([start_core + i for i in range(nprocs_per_rank)]) - os.sched_setaffinity(0, cores) - set_is_inference(is_inference=not self.compute_eval) - model = get_hstu_model( - table_config=self.table_config, - hstu_config=self.hstu_config, - table_device="cpu", - max_hash_size=100, - is_dense=True, - ).to(torch.bfloat16) - model.set_training_dtype(torch.bfloat16) - device = torch.device(f"cuda:{rank}") - torch.cuda.set_device(f"cuda:{rank}") - load_nonsparse_checkpoint( - model=model, device=device, optimizer=None, path=model_path - ) - model = model.to(device) - model.eval() - profiler = Profiler(rank) if self.output_trace else None - - with torch.no_grad(): - while True: - item = self.samples_q[rank].get() - # If -1 is received terminate all subprocesses - if item == -1: - break - if self.output_trace: - assert profiler is not None - profiler.step() - with torch.profiler.record_function("get_item_from_queue"): - # Copy here to release data in the producer to avoid invalid cuda caching allocator release. - item = copy.deepcopy(item) - ( - id, - seq_embeddings, - payload_features, - max_uih_len, - uih_seq_lengths, - max_num_candidates, - num_candidates, - ) = item - assert seq_embeddings is not None - with torch.profiler.record_function("dense forward"): - ( - _, - _, - _, - mt_target_preds, - mt_target_labels, - mt_target_weights, - ) = model.main_forward( - seq_embeddings=seq_embeddings, - payload_features=payload_features, - max_uih_len=max_uih_len, - uih_seq_lengths=uih_seq_lengths, - max_num_candidates=max_num_candidates, - num_candidates=num_candidates, - ) - # mt_target_preds = torch.empty(1, 2048 * 20).to(device="cpu") - # mt_target_labels = None - # mt_target_weights = None - assert mt_target_preds is not None - mt_target_preds = mt_target_preds.detach().to(device="cpu") - if mt_target_labels is not None: - mt_target_labels = mt_target_labels.detach().to(device="cpu") - if mt_target_weights is not None: - mt_target_weights = mt_target_weights.detach().to(device="cpu") - self.result_q[rank].put( - (id, mt_target_preds, mt_target_labels, mt_target_weights) - ) - - def capture_output( - self, id: uuid.UUID, rank: int - ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor]]: - """ - Retrieve inference results from a worker process. - - Args: - id: Unique identifier for the request. - rank: Worker rank to retrieve from. - - Returns: - Tuple of (predictions, labels, weights). - """ - while True: - recv_id, preds, labels, weights = self.result_q[rank].get() - assert recv_id == id - return preds, labels, weights - - def get_rank(self) -> int: - """ - Get the next worker rank for load balancing. - - Returns: - Rank index, cycling through available workers. - """ - rank = self.rank - self.rank = (self.rank + 1) % self.world_size - return rank - - def predict( - self, - seq_embeddings: Optional[Dict[str, SequenceEmbedding]], - payload_features: Optional[Dict[str, torch.Tensor]], - max_uih_len: int, - uih_seq_lengths: Optional[torch.Tensor], - max_num_candidates: int, - num_candidates: Optional[torch.Tensor], - ) -> Optional[ - Tuple[torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor], float] - ]: - """ - Run distributed dense forward pass. - - Dispatches work to a worker process and collects results. - - Args: - seq_embeddings: Sequence embeddings from sparse module. - payload_features: Additional feature tensors. - max_uih_len: Maximum UIH sequence length. - uih_seq_lengths: Per-sample UIH lengths. - max_num_candidates: Maximum candidates per sample. - num_candidates: Per-sample candidate counts. - - Returns: - Tuple of (predictions, labels, weights, elapsed_time) or None if shutdown. - """ - id = uuid.uuid4() - # If none is received terminate all subprocesses - if seq_embeddings is None: - for rank in range(self.world_size): - self.samples_q[rank].put(-1) - return None - rank = self.get_rank() - device = torch.device(f"cuda:{rank}") - assert ( - payload_features is not None - and num_candidates is not None - and uih_seq_lengths is not None - ) - t0: float = time.time() - seq_embeddings, payload_features, uih_seq_lengths, num_candidates = ( - move_sparse_output_to_device( - seq_embeddings=seq_embeddings, - payload_features=payload_features, - uih_seq_lengths=uih_seq_lengths, - num_candidates=num_candidates, - device=device, - ) - ) - self.samples_q[rank].put( - ( - id, - seq_embeddings, - payload_features, - max_uih_len, - uih_seq_lengths, - max_num_candidates, - num_candidates, - ) - ) - (mt_target_preds, mt_target_labels, mt_target_weights) = self.capture_output( - id, rank - ) - dt_dense = time.time() - t0 - return ( - mt_target_preds, - mt_target_labels, - mt_target_weights, - dt_dense, - ) - - -class ModelFamilyDenseSingleWorker: - """ - Single-worker dense module manager for single-GPU inference. - - Simpler alternative to ModelFamilyDenseDist for single-GPU setups. - - Args: - hstu_config: HSTU model configuration. - table_config: Embedding table configurations. - output_trace: Whether to enable profiling traces. - compute_eval: Whether to compute evaluation metrics. - """ - - def __init__( - self, - hstu_config: DlrmHSTUConfig, - table_config: Dict[str, EmbeddingConfig], - output_trace: bool = False, - compute_eval: bool = False, - ) -> None: - self.model: Optional[torch.nn.Module] = None - self.hstu_config = hstu_config - self.table_config = table_config - self.output_trace = output_trace - self.device: torch.device = torch.device("cuda:0") - torch.cuda.set_device(self.device) - self.profiler: Optional[Profiler] = ( - Profiler(rank=0) if self.output_trace else None - ) - - def load(self, model_path: str) -> None: - """ - Load dense model for single-GPU inference. - - Args: - model_path: Path to the model checkpoint directory. - """ - logger.warning(f"Loading dense module from {model_path}") - self.model = ( - get_hstu_model( - table_config=self.table_config, - hstu_config=self.hstu_config, - table_device="cpu", - is_dense=True, - ) - .to(self.device) - .to(torch.bfloat16) - ) - self.model.set_training_dtype(torch.bfloat16) - load_nonsparse_checkpoint( - model=self.model, device=self.device, optimizer=None, path=model_path - ) - assert self.model is not None - self.model.eval() - - def predict( - self, - seq_embeddings: Optional[Dict[str, SequenceEmbedding]], - payload_features: Optional[Dict[str, torch.Tensor]], - max_uih_len: int, - uih_seq_lengths: Optional[torch.Tensor], - max_num_candidates: int, - num_candidates: Optional[torch.Tensor], - ) -> Optional[ - Tuple[ - torch.Tensor, - Optional[torch.Tensor], - Optional[torch.Tensor], - float, - ] - ]: - """ - Run dense forward pass on single GPU. - - Args: - seq_embeddings: Sequence embeddings from sparse module. - payload_features: Additional feature tensors. - max_uih_len: Maximum UIH sequence length. - uih_seq_lengths: Per-sample UIH lengths. - max_num_candidates: Maximum candidates per sample. - num_candidates: Per-sample candidate counts. - - Returns: - Tuple of (predictions, labels, weights, elapsed_time). - """ - if self.output_trace: - assert self.profiler is not None - self.profiler.step() - assert ( - payload_features is not None - and uih_seq_lengths is not None - and num_candidates is not None - and seq_embeddings is not None - ) - t0: float = time.time() - with torch.profiler.record_function("dense forward"): - seq_embeddings, payload_features, uih_seq_lengths, num_candidates = ( - move_sparse_output_to_device( - seq_embeddings=seq_embeddings, - payload_features=payload_features, - uih_seq_lengths=uih_seq_lengths, - num_candidates=num_candidates, - device=self.device, - ) - ) - assert self.model is not None - ( - _, - _, - _, - mt_target_preds, - mt_target_labels, - mt_target_weights, - ) = self.model.main_forward( # pyre-ignore [29] - seq_embeddings=seq_embeddings, - payload_features=payload_features, - max_uih_len=max_uih_len, - uih_seq_lengths=uih_seq_lengths, - max_num_candidates=max_num_candidates, - num_candidates=num_candidates, - ) - assert mt_target_preds is not None - mt_target_preds = mt_target_preds.detach().to(device="cpu") - if mt_target_labels is not None: - mt_target_labels = mt_target_labels.detach().to(device="cpu") - if mt_target_weights is not None: - mt_target_weights = mt_target_weights.detach().to(device="cpu") - dt_dense: float = time.time() - t0 - return mt_target_preds, mt_target_labels, mt_target_weights, dt_dense diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/sparse_predict_module.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/sparse_predict_module.py deleted file mode 100644 index e3ec10415..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/sparse_predict_module.py +++ /dev/null @@ -1,106 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-strict - -""" -TorchScript-friendly wrapper for the HSTU sparse path (CPU embedding lookup). - -``HSTUSparseScriptModule`` wraps :class:`HSTUSparseInferenceModule` and -flattens the ``Dict[str, SequenceEmbedding]`` output into the parallel -value/length dicts defined in :mod:`ts_types` so the boundary is composed -entirely of TorchScript-supported types. -""" - -from typing import Dict, Tuple - -import torch -from generative_recommenders.dlrm_v3.inference.inference_modules import ( - _NoCopyEmbeddingCollection, - HSTUSparseInferenceModule, -) -from generative_recommenders.dlrm_v3.inference.ts_types import ( - flatten_seq_embeddings, - SeqEmbLengths, - SeqEmbValues, -) -from generative_recommenders.modules.dlrm_hstu import DlrmHSTUConfig -from torchrec.modules.embedding_configs import EmbeddingConfig -from torchrec.sparse.jagged_tensor import KeyedJaggedTensor - - -class HSTUSparseScriptModule(torch.nn.Module): - """Script-friendly sparse module. - - ``forward`` returns 5 tensors / dicts (no Python ``int`` scalars): - - 1. ``seq_emb_values`` ``Dict[str, Tensor]`` -- jagged embedding values. - 2. ``seq_emb_lengths`` ``Dict[str, Tensor]`` -- per-feature lengths. - 3. ``payload_features`` ``Dict[str, Tensor]`` -- side features. - 4. ``uih_seq_lengths`` ``Tensor[B]`` -- UIH lengths. - 5. ``num_candidates`` ``Tensor[B]`` -- candidate counts. - - The dense module (or the C++ glue) recovers the ``int`` ``max_uih_len`` / - ``max_num_candidates`` values from these tensors via ``.max().item()``. - """ - - def __init__( - self, - table_config: Dict[str, EmbeddingConfig], - hstu_config: DlrmHSTUConfig, - use_no_copy_embedding_collection: bool = True, - ) -> None: - super().__init__() - self._sparse: HSTUSparseInferenceModule = HSTUSparseInferenceModule( - table_config=table_config, - hstu_config=hstu_config, - ) - if use_no_copy_embedding_collection: - # Re-class the existing EmbeddingCollection so TorchScript picks up - # the no-copy ``forward`` override (matches the eager-only - # ``ec_patched_forward_wo_embedding_copy`` monkey-patch). - self._sparse._hstu_model._embedding_collection.__class__ = ( - _NoCopyEmbeddingCollection - ) - - def forward( - self, - uih_features: KeyedJaggedTensor, - candidates_features: KeyedJaggedTensor, - ) -> Tuple[ - SeqEmbValues, - SeqEmbLengths, - Dict[str, torch.Tensor], - torch.Tensor, - torch.Tensor, - ]: - ( - seq_embeddings, - payload_features, - _max_uih_len, - uih_seq_lengths, - _max_num_candidates, - num_candidates, - ) = self._sparse( - uih_features=uih_features, - candidates_features=candidates_features, - ) - seq_emb_values, seq_emb_lengths = flatten_seq_embeddings(seq_embeddings) - return ( - seq_emb_values, - seq_emb_lengths, - payload_features, - uih_seq_lengths, - num_candidates, - ) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/tests/inference_test.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/tests/inference_test.py deleted file mode 100644 index 948f10618..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/tests/inference_test.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import unittest - -from generative_recommenders.common import gpu_unavailable -from generative_recommenders.dlrm_v3.inference.main import main -from hypothesis import given, settings, strategies as st, Verbosity - - -class DLRMV3InferenceTest(unittest.TestCase): - @unittest.skipIf(*gpu_unavailable) - @given( - world_size=st.sampled_from([1]), - ) - @settings( - verbosity=Verbosity.verbose, - max_examples=1, - deadline=None, - ) - def test_e2e(self, world_size: int) -> None: - os.environ["WORLD_SIZE"] = str(world_size) - main() - - -if __name__ == "__main__": - unittest.main() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/tests/test_scripted_parity.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/tests/test_scripted_parity.py deleted file mode 100644 index 34d0388ea..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/tests/test_scripted_parity.py +++ /dev/null @@ -1,236 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-strict - -""" -Numerical parity test: eager HSTU vs traced (sparse + dense) on a synthetic -batch. - -The production deployment path (see ``end_to_end_test.py``) uses -``torch.jit.trace``, not ``torch.jit.script``, for the HSTU sparse/dense -wrappers. Tracing records the actual tensor ops executed during a forward -pass and ignores source-level dispatch logic (HammerKernel enum, -``is_fx_tracing()``, ``torch.autocast``, IntEnum branches) that scripting -cannot compile. This unit test mirrors that path. - -Tolerances are deliberately loose because the traced path replaces the -Triton fused kernels with PyTorch fallbacks and skips ``torch.autocast`` in -the user-forward block; both can perturb low-order bits in bf16. -""" - -import unittest -from typing import Dict, List, Tuple - -import torch -from generative_recommenders.common import gpu_unavailable, HammerKernel -from generative_recommenders.dlrm_v3.configs import ( - get_embedding_table_config, - get_hstu_configs, -) -from generative_recommenders.dlrm_v3.datasets.dataset import get_random_data -from generative_recommenders.dlrm_v3.inference.dense_predict_module import ( - HSTUDenseScriptModule, -) -from generative_recommenders.dlrm_v3.inference.sparse_predict_module import ( - HSTUSparseScriptModule, -) -from generative_recommenders.dlrm_v3.inference.ts_types import ( - SeqEmbLengths, - SeqEmbValues, -) -from torchrec.sparse.jagged_tensor import KeyedJaggedTensor - - -_DATASET = "kuairand-1k" - - -def _move_dense_inputs( - seq_emb_values: Dict[str, torch.Tensor], - seq_emb_lengths: Dict[str, torch.Tensor], - payload_features: Dict[str, torch.Tensor], - uih_seq_lengths: torch.Tensor, - num_candidates: torch.Tensor, - device: torch.device, -) -> Tuple[ - Dict[str, torch.Tensor], - Dict[str, torch.Tensor], - Dict[str, torch.Tensor], - torch.Tensor, - torch.Tensor, -]: - """C++-side ``move_sparse_output_to_device`` analog for the test.""" - return ( - {k: v.to(device).to(torch.bfloat16) for k, v in seq_emb_values.items()}, - {k: v.to(device) for k, v in seq_emb_lengths.items()}, - {k: v.to(device) for k, v in payload_features.items()}, - uih_seq_lengths.to(device), - num_candidates.to(device), - ) - - -class _SparseTraceShim(torch.nn.Module): - """Adapter that takes raw tensors and rebuilds the KJTs inside forward. - - ``torch.jit.trace`` does not accept ``KeyedJaggedTensor`` (or any - non-Tensor / non-collection-of-Tensor type) as a top-level forward - input, so we make the traced boundary tensor-only and bake the - ``List[str]`` of feature keys in as module attributes. - """ - - def __init__( - self, - sparse_module: HSTUSparseScriptModule, - uih_keys: List[str], - candidates_keys: List[str], - ) -> None: - super().__init__() - self._sparse_module: HSTUSparseScriptModule = sparse_module - self._uih_keys: List[str] = uih_keys - self._candidates_keys: List[str] = candidates_keys - - def forward( - self, - uih_lengths: torch.Tensor, - uih_values: torch.Tensor, - candidates_lengths: torch.Tensor, - candidates_values: torch.Tensor, - ) -> Tuple[ - SeqEmbValues, - SeqEmbLengths, - Dict[str, torch.Tensor], - torch.Tensor, - torch.Tensor, - ]: - uih_kjt = KeyedJaggedTensor( - keys=self._uih_keys, - lengths=uih_lengths, - values=uih_values, - ) - candidates_kjt = KeyedJaggedTensor( - keys=self._candidates_keys, - lengths=candidates_lengths, - values=candidates_values, - ) - return self._sparse_module( - uih_features=uih_kjt, candidates_features=candidates_kjt - ) - - -class HSTUScriptedParityTest(unittest.TestCase): - @unittest.skipIf(*gpu_unavailable) - def test_scripted_matches_eager(self) -> None: - torch.manual_seed(0) - device = torch.device("cuda:0") - torch.cuda.set_device(device) - - hstu_config = get_hstu_configs(_DATASET) - table_config = get_embedding_table_config(_DATASET) - - # Some embedding tables in kuairand-1k are tiny (e.g. - # user_active_degree has num_embeddings=8). Clamp the random value - # range so every index stays in range for every table; otherwise the - # default value_bound=1000 triggers an out-of-range embedding lookup. - min_rows = min(t.num_embeddings for t in table_config.values()) - value_bound = max(2, min_rows) - - uih_kjt, candidates_kjt = get_random_data( - contexual_features=list( - hstu_config.contextual_feature_to_max_length.keys() - ), - hstu_uih_keys=hstu_config.hstu_uih_feature_names, - hstu_candidates_keys=hstu_config.hstu_candidate_feature_names, - uih_max_seq_len=128, - max_num_candidates=hstu_config.max_num_candidates_inference, - value_bound=value_bound, - ) - - sparse_module = HSTUSparseScriptModule( - table_config=table_config, - hstu_config=hstu_config, - use_no_copy_embedding_collection=True, - ).eval() - dense_module = ( - HSTUDenseScriptModule( - hstu_config=hstu_config, - table_config=table_config, - ) - .to(torch.bfloat16) - .to(device) - .eval() - ) - - # Pin the HammerKernel to PyTorch on both wrappers. The Triton - # kernels use Python-level dispatch (autotune, constexpr arguments) - # that interacts badly with torch.jit.trace's recording pass. The - # eager reference run uses the same setting so the comparison is - # apples-to-apples. - sparse_module._sparse._hstu_model.set_hammer_kernel(HammerKernel.PYTORCH) - dense_module._hstu_model.set_hammer_kernel(HammerKernel.PYTORCH) - - # === Eager reference path === - with torch.no_grad(): - sparse_out_e = sparse_module( - uih_features=uih_kjt, candidates_features=candidates_kjt - ) - dense_inputs_e = _move_dense_inputs(*sparse_out_e, device=device) - preds_eager = dense_module(*dense_inputs_e) - - # === Traced path === - # Sparse is traced via a raw-tensor shim because KJT is not a valid - # traced input. Dense is traced directly with the eager sparse - # output as the example. - sparse_shim = _SparseTraceShim( - sparse_module=sparse_module, - uih_keys=list(uih_kjt.keys()), - candidates_keys=list(candidates_kjt.keys()), - ) - traced_sparse = torch.jit.trace( - sparse_shim, - example_inputs=( - uih_kjt.lengths(), - uih_kjt.values(), - candidates_kjt.lengths(), - candidates_kjt.values(), - ), - strict=False, - check_trace=False, - ) - traced_dense = torch.jit.trace( - dense_module, - example_inputs=tuple(dense_inputs_e), - strict=False, - check_trace=False, - ) - - with torch.no_grad(): - sparse_out_t = traced_sparse( - uih_kjt.lengths(), - uih_kjt.values(), - candidates_kjt.lengths(), - candidates_kjt.values(), - ) - dense_inputs_t = _move_dense_inputs(*sparse_out_t, device=device) - preds_traced = traced_dense(*dense_inputs_t) - - torch.testing.assert_close( - preds_eager.float(), - preds_traced.float(), - atol=1e-2, - rtol=1e-2, - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/.clang-format b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/.clang-format deleted file mode 100644 index f08c9c2c8..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/.clang-format +++ /dev/null @@ -1,2 +0,0 @@ -BasedOnStyle: Google -Standard: Cpp11 diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/CMakeLists.txt b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/CMakeLists.txt deleted file mode 100644 index 4fec0e44f..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/CMakeLists.txt +++ /dev/null @@ -1,113 +0,0 @@ -cmake_minimum_required(VERSION 3.12) - -project(mlperf_loadgen) - -# Read the version file -file(READ "${CMAKE_SOURCE_DIR}/VERSION.txt" VERSION_CONTENTS) - -# Extract the major, minor, and patch versions from the VERSION file (assuming "MAJOR.MINOR.PATCH" format) -string(REGEX MATCH "^([0-9]+)\\.([0-9]+)\\.([0-9]+)" VERSION_MATCH ${VERSION_CONTENTS}) - -# Set the variables for the major, minor, and patch versions -set(mlperf_loadgen_VERSION_MAJOR "${CMAKE_MATCH_1}") -set(mlperf_loadgen_VERSION_MINOR "${CMAKE_MATCH_2}") -set(mlperf_loadgen_VERSION_PATCH "${CMAKE_MATCH_3}") - -# Check if the version format was parsed correctly -if(NOT DEFINED mlperf_loadgen_VERSION_MAJOR OR NOT DEFINED mlperf_loadgen_VERSION_MINOR OR NOT DEFINED mlperf_loadgen_VERSION_PATCH) - message(FATAL_ERROR "Version format in VERSION.txt is incorrect. Expected format: MAJOR.MINOR.PATCH") -endif() - -# Print out the version -message("mlperf_loadgen v${mlperf_loadgen_VERSION_MAJOR}.${mlperf_loadgen_VERSION_MINOR}.${mlperf_loadgen_VERSION_PATCH}") - -# Set build options. NB: CXX_STANDARD is supported since CMake 3.1. -if (NOT MSVC) -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -W -Wall") -endif() -# Extra build options can be specified by setting the MLPERF_LOADGEN_CXX_FLAGS variable -if (MLPERF_LOADGEN_CXX_FLAGS) -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${MLPERF_LOADGEN_CXX_FLAGS}") -endif() -message(STATUS "Using C++ compiler flags: ${CMAKE_CXX_FLAGS}") -set(CMAKE_CXX_STANDARD "14") -message(STATUS "Using C++ standard: ${CMAKE_CXX_STANDARD}") -message(STATUS "Using static linker flags: ${CMAKE_STATIC_LINKER_FLAGS}") -message(STATUS "Using shared linker flags: ${CMAKE_SHARED_LINKER_FLAGS}") - -# Output directory for libraries. -set(LIBRARY_OUTPUT_PATH ${CMAKE_BINARY_DIR}) -message(STATUS "Using output path: ${LIBRARY_OUTPUT_PATH}") - -# Detect Python to use for generating source file with version info. -# NB: PythonInterp has been deprecated since CMake 3.12 -# but it works with earlier versions of CMake. -find_package(PythonInterp) -message(STATUS "Using Python interpreter: ${PYTHON_EXECUTABLE}") - -# Specify the source and destination files -set(CONF_FILE "mlperf.conf") -set(HEADER_FILE "mlperf_conf.h") - -# Read the content of the configuration file -file(READ ${CONF_FILE} CONF_CONTENTS) - -# Escape all double quotes and backslashes -string(REPLACE "\\" "\\\\" CONF_CONTENTS "${CONF_CONTENTS}") -string(REPLACE "\"" "\\\"" CONF_CONTENTS "${CONF_CONTENTS}") - -# Handle new lines -string(REPLACE "\n" "\\n\"\n\"" CONF_CONTENTS "${CONF_CONTENTS}") - -# Wrap the content in a C++ string declaration -set(FORMATTED_CONTENT "const char* mlperf_conf =\n\"${CONF_CONTENTS}\";\n") - -# Write the formatted content to the header file -file(WRITE ${HEADER_FILE} "${FORMATTED_CONTENT}") - -message(STATUS "Output config: ${CMAKE_BINARY_DIR}/mlperf_conf.h") - -# Generate source file with version info. -execute_process(COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/version_generator.py ${CMAKE_BINARY_DIR}/version_generated.cc ${CMAKE_CURRENT_SOURCE_DIR}) - -# Add source files. -set(SOURCE - ${CMAKE_CURRENT_SOURCE_DIR}/bindings/c_api.h - ${CMAKE_CURRENT_SOURCE_DIR}/bindings/c_api.cc - ${CMAKE_CURRENT_SOURCE_DIR}/early_stopping.cc - ${CMAKE_CURRENT_SOURCE_DIR}/issue_query_controller.cc - ${CMAKE_CURRENT_SOURCE_DIR}/loadgen.cc - ${CMAKE_CURRENT_SOURCE_DIR}/logging.cc - ${CMAKE_CURRENT_SOURCE_DIR}/logging.h - ${CMAKE_CURRENT_SOURCE_DIR}/test_settings_internal.cc - ${CMAKE_CURRENT_SOURCE_DIR}/test_settings_internal.h - ${CMAKE_CURRENT_SOURCE_DIR}/utils.cc - ${CMAKE_CURRENT_SOURCE_DIR}/utils.h - ${CMAKE_CURRENT_SOURCE_DIR}/results.h - ${CMAKE_CURRENT_SOURCE_DIR}/results.cc - ${CMAKE_CURRENT_SOURCE_DIR}/version.cc - ${CMAKE_CURRENT_SOURCE_DIR}/version.h - ${CMAKE_CURRENT_SOURCE_DIR}/mlperf_conf.h - ${CMAKE_CURRENT_SOURCE_DIR}/VERSION.txt - ${CMAKE_BINARY_DIR}/version_generated.cc -) - -include_directories(${CMAKE_CURRENT_SOURCE_DIR}) - -add_library(mlperf_loadgen STATIC ${SOURCE}) -target_link_libraries(mlperf_loadgen) - -if(WIN32) -set (LIBS "") -else() -set (LIBS pthread) -endif() - -add_executable(benchmark benchmark/repro.cpp) -target_link_libraries(benchmark PUBLIC mlperf_loadgen ${LIBS}) - -# Install library and headers. -install(TARGETS mlperf_loadgen - DESTINATION ${CMAKE_INSTALL_PREFIX}/lib) -install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/ - DESTINATION ${CMAKE_INSTALL_PREFIX}/include FILES_MATCHING PATTERN "*.h") diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/MANIFEST.in b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/MANIFEST.in deleted file mode 100644 index 152b53111..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/MANIFEST.in +++ /dev/null @@ -1,2 +0,0 @@ -include VERSION.txt -include mlperf.conf diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/README.md b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/README.md deleted file mode 100644 index 212c8a53c..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/README.md +++ /dev/null @@ -1,223 +0,0 @@ -# Overview {#mainpage} - -## Introduction - -* The LoadGen is a *reusable* module that *efficiently* and *fairly* measures - the performance of inference systems. -* It generates traffic for scenarios as formulated by a diverse set of experts - in the [MLCommons working group](https://mlcommons.org/). -* The scenarios emulate the workloads seen in mobile devices, - autonomous vehicles, robotics, and cloud-based setups. -* Although the LoadGen is not model or dataset aware, its strength is in its - reusability with logic that is. - -## Integration Example and Flow -The following is an diagram of how the LoadGen can be integrated into an -inference system, resembling how some of the MLPerf reference models are -implemented. -
- -
    -
  1. Benchmark knows the model, dataset, and preprocessing.
    2. -
  2. Benchmark hands dataset sample IDs to LoadGen.
    3. -
  3. LoadGen starts generating queries of sample IDs.
    4. -
  4. Benchmark creates requests to backend.
    5. -
  5. Result is post processed and forwarded to LoadGen.
    6. -
  6. LoadGen outputs logs for analysis.
    -
-
- -## Useful Links -* [FAQ](README_FAQ.md) -* [LoadGen Build Instructions](README_BUILD.md) -* [LoadGen API](loadgen.h) -* [Test Settings](test_settings.h) - - A good description of available scenarios, modes, and knobs. -* [MLPerf Inference Code](https://github.com/mlcommons/inference) - - Includes source for the LoadGen and reference models that use the LoadGen. -* [MLPerf Inference Rules](https://github.com/mlcommons/inference_policies) - - Any mismatch with this is a bug in the LoadGen. - -## Scope of the LoadGen's Responsibilities - -### In Scope -* **Provide a reusable** C++ library with python bindings. -* **Implement** the traffic patterns of the MLPerf Inference scenarios and - modes. -* **Record** all traffic generated and received for later analysis and - verification. -* **Summarize** the results and whether performance constraints were met. -* **Target high-performance** systems with efficient multi-thread friendly - logging utilities. -* **Generate trust** via a shared, well-tested, and community-hardened - code base. - -### Out of Scope -The LoadGen is: -* **NOT** aware of the ML model it is running against. -* **NOT** aware of the data formats of the model's inputs and outputs. -* **NOT** aware of how to score the accuracy of a model's outputs. -* **NOT** aware of MLPerf rules regarding scenario-specific constraints. - -Limitting the scope of the LoadGen in this way keeps it reusable across -different models and datasets without modification. Using composition and -dependency injection, the user can define their own model, datasets, and -metrics. - -Additionally, not hardcoding MLPerf-specific test constraints, like test -duration and performance targets, allows users to use the LoadGen unmodified -for custom testing and continuous integration purposes. - -## Submission Considerations - -### Upstream all local modifications -* As a rule, no local modifications to the LoadGen's C++ library are allowed -for submission. -* Please upstream early and often to keep the playing field level. - -### Choose your TestSettings carefully! -* Since the LoadGen is oblivious to the model, it can't enforce the MLPerf -requirements for submission. *e.g.:* target percentiles and latencies. -* For verification, the values in TestSettings are logged. -* To help make sure your settings are spec compliant, use -TestSettings::FromConfig in conjunction with the relevant config file provided -with the reference models. - -## Responsibilities of a LoadGen User - -### Implement the Interfaces -* Implement the SystemUnderTest and QuerySampleLibrary interfaces and pass - them to the StartTest function. -* Call QuerySampleComplete for every sample received by - SystemUnderTest::IssueQuery. - -### Assess Accuracy -* Process the *mlperf_log_accuracy.json* output by the LoadGen to determine - the accuracy of your system. -* For the official models, Python scripts will be provided by the MLPerf model - owners for you to do this automatically. - -For templates of how to do the above in detail, refer to code for the demos, -tests, and reference models. - - -## LoadGen over the Network - -For reference, on a high level a submission looks like this: - -
- -
- -The LoadGen implementation is common to all submissions, while the QSL (“Query Sample Library”) and SUT (“System Under Test”) are implemented by submitters. QSL is responsible for loading the data and includes untimed preprocessing. - -A submission over the network introduces a new component “QDL” (query dispatch library) that is added to the system as presented in the following diagram: - -
- -
- -QDL is a proxy for a load-balancer, that dispatches queries to SUT over a physical network, receives the responses and passes them back to LoadGen. It is implemented by the submitter. The interface of the QDL is the same as the API to SUT. - -In scenarios using QDL, data may be compressed in QSL at the choice of the submitter in order to reduce network transmission time. Decompression is part of the timed processing in SUT. A set of approved standard compression schemes will be specified for each benchmark; additional compression schemes must be approved in advance by the Working Group. - -All communication between LoadGen/QSL and SUT is via QDL, and all communication between QDL and SUT must pass over a physical network. - -QDL implements the protocol to transmit queries over the network and receive responses. It also implements decompression of any response returned by the SUT, where compression of responses is allowed. Performing any part of the timed preprocessing or inference in QDL is specifically disallowed. Currently no batching is allowed in QDL, although this may be revisited in future. - -The MLperf over the Network will run in Server mode and Offline mode. All LoadGen modes are expected to work as is with insignificant changes. These include running the test in performance mode, accuracy mode, find peak performance mode and compliance mode. The same applies for power measurements. - -### QDL details -The Query Dispatch Library is implemented by the submitter and interfaces with LoadGen using the same SUT API. All MLPerf Inference SUTs implement the `mlperf::SystemUnderTest` class which is defined in system_under_test.h. The QDL implements `mlperf::QueryDispatchLibrary` class which inherits the `mlperf::SystemUnderTest` class and has the same API and support all existing `mlperf::SystemUnderTest` methods. It has a separate header file query_dispatch_library.h. Using sut with `mlperf::SystemUnderTest` class in LoadGen StartTest is natively upcasting `mlperf::QueryDispatchLibrary` class. - -#### QDL Query issue and response over the network - -The QDL gets the queries from the LoadGen through -```CPP -void IssueQuery(const std::vector& samples) -``` - -The QDL dispatches the queries to the SUT over the physical media. The exact method and implementation for it are submitter specific and would not be specified at MLCommons. Submitter implementation includes all methods required to serialize the query, load balance, drive it to the Operating system and network interface card and send to the SUT. - -The QDL receives the query responses over the network from the SUT. The exact method and implementation for it are submitter specific and would not be specified at MLCommons. The submitter implementation includes all methods required to receive the network data from the Network Interface card, go through the Operating system, deserialize the query response, and provide it back to the LoadGen through query completion by: - -```CPP -struct QuerySampleResponse { - ResponseId id; - uintptr_t data; - size_t size; -}; -void QuerySamplesComplete(QuerySampleResponse* responses, - size_t response_count); - -``` - -#### QDL Additional Methods - -In addition to that the QDL needs to implement the following methods that are provided by the SUT interface to the LoadGen: -```CPP -const std::string& Name(); -``` -The `Name` function returns a known string for over the Network SUTs to identify it as over the network benchmark. -```CPP -void FlushQueries(); -``` - -It is not specified here how the QDL would query and configure the SUT to execute the above methods. The QDL responds to the LoadGen after receiving its own response from the SUT. - -### Example - -Refer to [LON demo](demos/lon) for a reference example illustrating usage of Loadgen over the network. - -## Find Peak Performance Mode - -The Find Peak Performance mode can be used to find the optimal queries per second (QPS) for the server scenario. - -### Setup - -You can setup loadgen to run this mode by setting the `mode` variable in the `test_settings` used to run the test. Using the Python API: - -```python -settings = mlperf_loadgen.TestSettings() -settings.server_target_qps = 100 -settings.scenario = mlperf_loadgen.TestScenario.Server -settings.mode = mlperf_loadgen.TestMode.FindPeakPerformance -... - -mlperf_loadgen.StartTest(sut, qsl, settings) -``` - -Using the C/C++ API: -```CPP -mlperf::TestSettings settings; -setting.server_target_qps = 100; -settings.scenario = mlperf::TestScenario::Server; -settings.mode = mlperf::TestMode::FindPeakPerformance; -mlperf::LogSettings log_settings; -/* -Construct QSL and SUT -*/ -mlperf::StartTest(&sut, &qsl, settings, log_settings); -``` - -**Note:** Make sure you are setting the TestScenario to server and you are providing an initial target QPS. - -### Description - -The Find Peak Performance mode works by finding a lower and upper boundary for the optimal QPS. Then performing a binary search between the lower and upper bound to find the optimal QPS. - -#### Finding lower and upper boundary - -LoadGen begins by running performance mode at the specified target QPS. If the test passes, this value is used as the lower bound; otherwise, an error is raised. The algorithm then guesses the upper bound as twice the target QPS. - -Then LoadGen will run performance mode using the upper bound guess. If the test is successful, both the lower bound and upper bound will be doubled. This repeats until the upper bound guess fails the test. - -``` -[initial_target_qps, 2*initial_target_qps] -> [2*initial_target_qps, 4*initial_target_qps] -> [4*initial_target_qps, 8*initial_target_qps]... -``` - -Finally, the final lower bound and upper bound are set to their current values. This process assures that the lower bound passes the performance mode, but the upper bound doesn’t. - -#### Binary Search - -Once the lower and upper bounds are set, binary search can be performed over the range `[lower, upper]`` to find the optimal QPS. If a given QPS fails in performance mode, the optimal value lies below it; if it passes, the optimal is higher. diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/README_BUILD.md b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/README_BUILD.md deleted file mode 100644 index 499cc360a..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/README_BUILD.md +++ /dev/null @@ -1,47 +0,0 @@ -# Building the LoadGen {#ReadmeBuild} - -## Prerequisites - - sudo apt-get install libglib2.0-dev python-pip python3-pip - pip2 install absl-py numpy - pip3 install absl-py numpy - -## Quick Start -### Installation - Python - - pip install absl-py numpy - git clone --recurse-submodules https://github.com/mlcommons/inference.git mlperf_inference - cd mlperf_inference/loadgen - CFLAGS="-std=c++14 -O3" python -m pip install . - -This will fetch the loadgen source, build and install the loadgen as a python module, and run a simple end-to-end demo. - -Alternatively, we provide wheels for several python versions and operating system that can be installed using pip directly. - - pip install mlperf-loadgen - -**NOTE:** Take into account that we only update the published wheels after an official release, they may not include the latest changes. - -### Testing your Installation -The following command will run a simple end-to-end demo: - - python mlperf_inference/loadgen/demos/py_demo_single_stream.py - -A summary of the test results can be found in the *"mlperf_log_summary.txt"* logfile. - -For a timeline visualization of what happened during the test, open the *"mlperf_log_trace.json"* file in Chrome: -* Type “chrome://tracing” in the address bar, then drag-n-drop the json. -* This may be useful for SUT performance tuning and understanding + debugging the loadgen. - -### Installation - C++ -To build the loadgen as a C++ library, rather than a python module: - - git clone https://github.com/mlcommons/inference.git mlperf_inference - cd mlperf_inference - mkdir loadgen/build/ && cd loadgen/build/ - cmake .. && cmake --build . - cp libmlperf_loadgen.a .. - -## Quick start: Loadgen Over the Network - -Refer to [LON demo](demos/lon/README.md) for a basic example. diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/README_FAQ.md b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/README_FAQ.md deleted file mode 100644 index ab4e0c75d..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/README_FAQ.md +++ /dev/null @@ -1,78 +0,0 @@ -# LoadGen FAQ {#ReadmeFAQ} - -## Q: The LoadGen does not match the MLPerf specification. Who is right? -**A:** -The MLPerf spec is *always* right. -Please file a LoadGen bug so it may be resolved. - -## Q: How can I file a bug? -**A:** -On GitHub: https://github.com/mlcommons/inference/issues/new - -## Q: Can I make local modifications to the LoadGen for submission? -**A:** -No. To keep the playing field level, please upstream any local -modificiations you need to make. Ideally upstream such changes behind a runtime -flag or via an abstract interface the client can implement. This will help -with testability. - -## Q: Where can I find the results of a test? -**A:** -By default, the loadgen will output an *mlperf_log_summary.txt* file -that summarizes the target metrics and constraints of the test, along with -other stats about the run. - -*Note:* LogSettings also has a flag to forward the results to stdout and -there's an outstanding TODO to make this more programmable. - -## Q: The reference implementation for \<*some_model*\> prints out results of its own. Are those for submission? -**A:** -They are not. The LoadGen results are the ground truth for submission -results since they will work even for systems that forgo the python bindings. -If you notice a bug in the LoadGen's results, please file a bug or submit a -patch. - -## Q: I'm getting linker errors for LoadgenVersion definitions. Where is *version_generated.cc*? -**A:** -If you have a custom build setup, make sure you run the *version_generator.py* -script, which will create the cc file you are looking for. The official build -files that come with the LoadGen do this for you out of the box. - -## Q: What is this *version_generator.py* script? -**A:** -The LoadGen records git stats (if available) and the SHA1 of all its -source files (always) at build time for verification purposes. This is easy -to circumvent, but try your best to run *version_generator.py* correctly; -ideally integrated with your build system if you have a custom build. -The intention is more to help with debugging efforts and detect accidental -version missmatches than to detect bad actors. - -## Q: How do I view the *mlperf_log_trace.json* file? -**A:** -This file uses the [Trace Event Format] -(https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/edit) -to record a timeline of all the threads involved. -You can view the file by typing [chrome://tracing](chrome://tracing) into -Chrome's address bar and dragging the json file there. -This file zips well and you can drag the zip file directly into -[chrome://tracing](chrome://tracing) too. -Please include zipped traces (and the other logs) when filing bug reports. - -## Q: Why is the code littered with so many lambdas? My eyes hurt. -**A:** -Lambdas are a convenient and efficient way to ship arbitrary data + deferred -logic over to the logging thread without much boilerplate. -Much of the loadgen is built on top of the logging utilities. -Thus the lambdas. (Sorry about the eyes.) - -## Q: What C++ version does the LoadGen target? -**A:** -It currently targets and requires C++14. It should compile with recent -versions of clang, gcc, and msvc. - -## Q: What dependencies does the LoadGen code have? -**A:** -The C++ code has no external dependencies. The loadgen itself, logging -utilities, and unit test utilities are built solely on the C++ Standard Library. -The python bindings, however, do require -[pybind11](https://github.com/pybind/pybind11). diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/VERSION.txt b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/VERSION.txt deleted file mode 100644 index ac14c3dfa..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/VERSION.txt +++ /dev/null @@ -1 +0,0 @@ -5.1.1 diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/benchmark/.gitignore b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/benchmark/.gitignore deleted file mode 100644 index e792c8e55..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/benchmark/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -loadgen_build -build \ No newline at end of file diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/benchmark/README.md b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/benchmark/README.md deleted file mode 100644 index 24e872983..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/benchmark/README.md +++ /dev/null @@ -1,10 +0,0 @@ -Note: please install jemalloc first. See: http://jemalloc.net/ -Command: bash run.sh <0=Basic,1=Queue> - -Experiments: -- On Intel(R) Xeon(R) CPU E5-1650 v4 @ 3.60GHz -- Basic SUT : 500-600k i/s -- Basic SUT + jemalloc: 800-900k i/s (`bash run.sh 800000 0`) -- Queued SUT (2 complete threads) + jemalloc: 1.2-1.3M i/s (`bash run.sh 1200000 1 2 2048`) -- Queued SUT (2 complete threads) + jemalloc + server_coalesce_queries: 1.4-1.5M is/ (`bash run.sh 1400000 1 2 512 1`) -- Basic SUT + jemalloc + server_coalesce_queries + 4 IssueQueryThreads: 2.4-2.5M is/ (`bash run.sh 2400000 0 2 512 1 4`) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/benchmark/repro.cpp b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/benchmark/repro.cpp deleted file mode 100644 index 44ff53efa..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/benchmark/repro.cpp +++ /dev/null @@ -1,296 +0,0 @@ -/* - * Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include -#include -#include -#include -#include -#include - -#include "loadgen.h" -#include "query_sample_library.h" -#include "system_under_test.h" -#include "test_settings.h" - -class QSL : public mlperf::QuerySampleLibrary { - public: - ~QSL() override{}; - const std::string& Name() override { return mName; } - size_t TotalSampleCount() override { return 1000000; } - size_t PerformanceSampleCount() override { return TotalSampleCount(); } - void LoadSamplesToRam(const std::vector&) override { - } - void UnloadSamplesFromRam( - const std::vector&) override {} - - private: - std::string mName{"Dummy QSL"}; -}; - -class BasicSUT : public mlperf::SystemUnderTest { - public: - BasicSUT() { - // Start with some large value so that we don't reallocate memory. - initResponse(10000); - } - ~BasicSUT() override {} - const std::string& Name() override { return mName; } - void IssueQuery(const std::vector& samples) override { - size_t n = samples.size(); - if (n > mResponses.size()) { - std::cerr << "Warning: reallocating response buffer in BasicSUT. Maybe " - "you should initResponse with larger value!?" - << std::endl; - initResponse(samples.size()); - } - for (size_t i = 0; i < n; i++) { - mResponses[i].id = samples[i].id; - } - mlperf::QuerySamplesComplete(mResponses.data(), n); - } - void FlushQueries() override {} - - private: - void initResponse(int size) { - mResponses.resize(size, - {0, reinterpret_cast(&mBuf), sizeof(int)}); - } - int mBuf{0}; - std::string mName{"BasicSUT"}; - std::vector mResponses; -}; - -class QueueSUT : public mlperf::SystemUnderTest { - public: - QueueSUT(int numCompleteThreads, int maxSize) { - // Each thread handle at most maxSize at a time. - std::cout << "QueueSUT: maxSize = " << maxSize << std::endl; - initResponse(numCompleteThreads, maxSize); - // Launch complete threads - for (int i = 0; i < numCompleteThreads; i++) { - mThreads.emplace_back(&QueueSUT::CompleteThread, this, i); - } - } - ~QueueSUT() override { - { - std::unique_lock lck(mMtx); - mDone = true; - mCondVar.notify_all(); - } - for (auto& thread : mThreads) { - thread.join(); - } - } - const std::string& Name() override { return mName; } - void IssueQuery(const std::vector& samples) override { - std::unique_lock lck(mMtx); - for (const auto& sample : samples) { - mIdQueue.push_back(sample.id); - } - // Let some worker thread to consume tasks - mCondVar.notify_one(); - } - void FlushQueries() override {} - - private: - void CompleteThread(int threadIdx) { - auto& responses = mResponses[threadIdx]; - size_t maxSize{responses.size()}; - size_t actualSize{0}; - while (true) { - { - std::unique_lock lck(mMtx); - mCondVar.wait(lck, [&]() { return !mIdQueue.empty() || mDone; }); - - if (mDone) { - break; - } - - actualSize = std::min(maxSize, mIdQueue.size()); - for (size_t i = 0; i < actualSize; i++) { - responses[i].id = mIdQueue.front(); - mIdQueue.pop_front(); - } - mCondVar.notify_one(); - } - mlperf::QuerySamplesComplete(responses.data(), actualSize); - } - } - void initResponse(int numCompleteThreads, int size) { - mResponses.resize(numCompleteThreads); - for (auto& responses : mResponses) { - responses.resize(size, - {0, reinterpret_cast(&mBuf), sizeof(int)}); - } - } - int mBuf{0}; - std::string mName{"QueueSUT"}; - std::vector> mResponses; - std::vector mThreads; - std::deque mIdQueue; - std::mutex mMtx; - std::condition_variable mCondVar; - bool mDone{false}; -}; - -class MultiBasicSUT : public mlperf::SystemUnderTest { - public: - MultiBasicSUT(int numThreads) - : mNumThreads(numThreads), mResponses(numThreads) { - // Start with some large value so that we don't reallocate memory. - initResponse(10000); - for (int i = 0; i < mNumThreads; ++i) { - mThreads.emplace_back(&MultiBasicSUT::startIssueThread, this, i); - } - } - ~MultiBasicSUT() override { - for (auto& thread : mThreads) { - thread.join(); - } - } - const std::string& Name() override { return mName; } - void IssueQuery(const std::vector& samples) override { - int thread_idx = mThreadMap[std::this_thread::get_id()]; - size_t n = samples.size(); - auto& reponses = mResponses[thread_idx]; - if (n > reponses.size()) { - std::cout - << "Warning: reallocating response buffer in MultiBasicSUT. Maybe " - "you should initResponse with larger value!?" - << std::endl; - initResponse(samples.size()); - } - for (size_t i = 0; i < n; i++) { - reponses[i].id = samples[i].id; - } - mlperf::QuerySamplesComplete(reponses.data(), n); - } - void FlushQueries() override {} - - private: - void initResponse(int size) { - for (auto& responses : mResponses) { - responses.resize(size, - {0, reinterpret_cast(&mBuf), sizeof(int)}); - } - } - void startIssueThread(int thread_idx) { - { - std::lock_guard lock(mMtx); - mThreadMap[std::this_thread::get_id()] = thread_idx; - } - mlperf::RegisterIssueQueryThread(); - } - int mBuf{0}; - int mNumThreads{0}; - std::string mName{"MultiBasicSUT"}; - std::vector> mResponses; - std::mutex mMtx; - std::vector mThreads; - std::map mThreadMap; -}; - -int main(int argc, char** argv) { - assert(argc >= 2 && "Need to pass in at least one argument: target_qps"); - int target_qps = std::stoi(argv[1]); - std::cout << "target_qps = " << target_qps << std::endl; - - bool useQueue{false}; - int numCompleteThreads{4}; - int maxSize{1}; - bool server_coalesce_queries{false}; - int num_issue_threads{0}; - if (argc >= 3) { - useQueue = std::stoi(argv[2]) != 0; - } - if (argc >= 4) { - numCompleteThreads = std::stoi(argv[3]); - } - if (argc >= 5) { - maxSize = std::stoi(argv[4]); - } - if (argc >= 6) { - server_coalesce_queries = std::stoi(argv[5]) != 0; - } - if (argc >= 7) { - num_issue_threads = std::stoi(argv[6]); - } - - QSL qsl; - std::unique_ptr sut; - - // Configure the test settings - mlperf::TestSettings testSettings; - testSettings.scenario = mlperf::TestScenario::Server; - testSettings.mode = mlperf::TestMode::PerformanceOnly; - testSettings.server_target_qps = target_qps; - testSettings.server_target_latency_ns = 10000000; // 10ms - testSettings.server_target_latency_percentile = 0.99; - testSettings.min_duration_ms = 60000; - testSettings.min_query_count = 270000; - testSettings.server_coalesce_queries = server_coalesce_queries; - std::cout << "testSettings.server_coalesce_queries = " - << (server_coalesce_queries ? "True" : "False") << std::endl; - testSettings.server_num_issue_query_threads = num_issue_threads; - std::cout << "num_issue_threads = " << num_issue_threads << std::endl; - - // Configure the logging settings - mlperf::LogSettings logSettings; - logSettings.log_output.outdir = "build"; - logSettings.log_output.prefix = "mlperf_log_"; - logSettings.log_output.suffix = ""; - logSettings.log_output.prefix_with_datetime = false; - logSettings.log_output.copy_detail_to_stdout = false; - logSettings.log_output.copy_summary_to_stdout = true; - logSettings.log_mode = mlperf::LoggingMode::AsyncPoll; - logSettings.log_mode_async_poll_interval_ms = 1000; - logSettings.enable_trace = false; - - // Choose SUT - if (num_issue_threads == 0) { - if (useQueue) { - std::cout << "Using QueueSUT with " << numCompleteThreads - << " complete threads" << std::endl; - sut.reset(new QueueSUT(numCompleteThreads, maxSize)); - } else { - std::cout << "Using BasicSUT" << std::endl; - sut.reset(new BasicSUT()); - } - } else { - if (useQueue) { - std::cout << "Using MultiQueueSUT with " << numCompleteThreads - << " complete threads" << std::endl; - std::cerr << "!!!! MultiQueueSUT is NOT implemented yet !!!!" - << std::endl; - return 1; - // sut.reset(new MultiQueueSUT(num_issue_threads, numCompleteThreads, - // maxSize)); - } else { - std::cout << "Using MultiBasicSUT" << std::endl; - sut.reset(new MultiBasicSUT(num_issue_threads)); - } - } - - // Start test - std::cout << "Start test..." << std::endl; - mlperf::StartTest(sut.get(), &qsl, testSettings, logSettings); - std::cout << "Test done. Clean up SUT..." << std::endl; - sut.reset(); - std::cout << "Done!" << std::endl; - return 0; -} diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/benchmark/run.sh b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/benchmark/run.sh deleted file mode 100644 index 62559c1a8..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/benchmark/run.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/bash -# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -echo "Building loadgen..." -if [ ! -e loadgen_build ]; then mkdir loadgen_build; fi; -cd loadgen_build && cmake ../.. && make -j && cd .. -echo "Building test program..." -if [ ! -e build ]; then mkdir build; fi; -g++ --std=c++11 -O3 -I.. -o build/repro.exe repro.cpp -Lloadgen_build -lmlperf_loadgen -lpthread && \ -LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.2 build/repro.exe $1 $2 $3 $4 $5 $6 diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/benchmark/run_debug.sh b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/benchmark/run_debug.sh deleted file mode 100644 index ba63727c8..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/benchmark/run_debug.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/bash -# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -echo "Building loadgen in Debug mode..." -if [ ! -e loadgen_build ]; then mkdir loadgen_build; fi; -cd loadgen_build && cmake -DCMAKE_BUILD_TYPE=Debug ../.. && make -j && cd .. -echo "Building test program in Debug mode..." -if [ ! -e build ]; then mkdir build; fi; -g++ --std=c++11 -O0 -g -I.. -o build/repro.exe repro.cpp -Lloadgen_build -lmlperf_loadgen -lpthread && \ -gdb --args build/repro.exe $1 $2 $3 $4 $5 $6 diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/bindings/c_api.cc b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/bindings/c_api.cc deleted file mode 100644 index 0248a1c16..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/bindings/c_api.cc +++ /dev/null @@ -1,176 +0,0 @@ -/* Copyright 2019 The MLPerf Authors. All Rights Reserved. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "c_api.h" - -#include - -#include "../loadgen.h" -#include "../query_sample.h" -#include "../query_sample_library.h" -#include "../system_under_test.h" -#include "../test_settings.h" - -namespace mlperf { -namespace c { -namespace { - -// Forwards SystemUnderTest calls to relevant callbacks. -class SystemUnderTestTrampoline : public SystemUnderTest { - public: - SystemUnderTestTrampoline(ClientData client_data, std::string name, - IssueQueryCallback issue_cb, - FlushQueriesCallback flush_queries_cb) - : client_data_(client_data), - name_(std::move(name)), - issue_cb_(issue_cb), - flush_queries_cb_(flush_queries_cb) {} - ~SystemUnderTestTrampoline() override = default; - - const std::string& Name() override { return name_; } - - void IssueQuery(const std::vector& samples) override { - (*issue_cb_)(client_data_, samples.data(), samples.size()); - } - - void FlushQueries() override { (*flush_queries_cb_)(); } - - private: - ClientData client_data_; - std::string name_; - IssueQueryCallback issue_cb_; - FlushQueriesCallback flush_queries_cb_; -}; - -} // namespace - -void* ConstructSUT(ClientData client_data, const char* name, size_t name_length, - IssueQueryCallback issue_cb, - FlushQueriesCallback flush_queries_cb) { - SystemUnderTestTrampoline* sut = new SystemUnderTestTrampoline( - client_data, std::string(name, name_length), issue_cb, flush_queries_cb); - return reinterpret_cast(sut); -} - -void DestroySUT(void* sut) { - SystemUnderTestTrampoline* sut_cast = - reinterpret_cast(sut); - delete sut_cast; -} - -namespace { - -// Forwards QuerySampleLibrary calls to relevant callbacks. -class QuerySampleLibraryTrampoline : public QuerySampleLibrary { - public: - QuerySampleLibraryTrampoline( - ClientData client_data, std::string name, size_t total_sample_count, - size_t performance_sample_count, - LoadSamplesToRamCallback load_samples_to_ram_cb, - UnloadSamplesFromRamCallback unload_samples_from_ram_cb) - : client_data_(client_data), - name_(std::move(name)), - total_sample_count_(total_sample_count), - performance_sample_count_(performance_sample_count), - load_samples_to_ram_cb_(load_samples_to_ram_cb), - unload_samples_from_ram_cb_(unload_samples_from_ram_cb) {} - ~QuerySampleLibraryTrampoline() override = default; - - const std::string& Name() override { return name_; } - size_t TotalSampleCount() override { return total_sample_count_; } - size_t PerformanceSampleCount() override { return performance_sample_count_; } - - void LoadSamplesToRam(const std::vector& samples) override { - (*load_samples_to_ram_cb_)(client_data_, samples.data(), samples.size()); - } - void UnloadSamplesFromRam( - const std::vector& samples) override { - (*unload_samples_from_ram_cb_)(client_data_, samples.data(), - samples.size()); - } - - private: - ClientData client_data_; - std::string name_; - size_t total_sample_count_; - size_t performance_sample_count_; - LoadSamplesToRamCallback load_samples_to_ram_cb_; - UnloadSamplesFromRamCallback unload_samples_from_ram_cb_; -}; - -} // namespace - -void* ConstructQSL(ClientData client_data, const char* name, size_t name_length, - size_t total_sample_count, size_t performance_sample_count, - LoadSamplesToRamCallback load_samples_to_ram_cb, - UnloadSamplesFromRamCallback unload_samples_from_ram_cb) { - QuerySampleLibraryTrampoline* qsl = new QuerySampleLibraryTrampoline( - client_data, std::string(name, name_length), total_sample_count, - performance_sample_count, load_samples_to_ram_cb, - unload_samples_from_ram_cb); - return reinterpret_cast(qsl); -} - -void DestroyQSL(void* qsl) { - QuerySampleLibraryTrampoline* qsl_cast = - reinterpret_cast(qsl); - delete qsl_cast; -} - -// mlperf::c::StartTest just forwards to mlperf::StartTest after doing the -// proper cast. -void StartTest(void* sut, void* qsl, const TestSettings& settings, - const std::string& audit_config_filename = "audit.config") { - SystemUnderTestTrampoline* sut_cast = - reinterpret_cast(sut); - QuerySampleLibraryTrampoline* qsl_cast = - reinterpret_cast(qsl); - LogSettings default_log_settings; - mlperf::StartTest(sut_cast, qsl_cast, settings, default_log_settings, - audit_config_filename); -} - -void QuerySamplesComplete(QuerySampleResponse* responses, - size_t response_count) { - mlperf::QuerySamplesComplete(responses, response_count); -} - -void QuerySamplesCompleteResponseCb(QuerySampleResponse* responses, - size_t response_count, - ResponseCallback response_cb, - ClientData client_data) { - mlperf::QuerySamplesComplete( - responses, response_count, - [client_data, response_cb](QuerySampleResponse* response) { - response_cb(client_data, response); - }); -} - -void FirstTokenComplete(QuerySampleResponse* responses, size_t response_count) { - mlperf::FirstTokenComplete(responses, response_count); -} - -void FirstTokenCompleteResponseCb(QuerySampleResponse* responses, - size_t response_count, - ResponseCallback response_cb, - ClientData client_data) { - mlperf::FirstTokenComplete( - responses, response_count, - [client_data, response_cb](QuerySampleResponse* response) { - response_cb(client_data, response); - }); -} - -void RegisterIssueQueryThread() { mlperf::RegisterIssueQueryThread(); } - -} // namespace c -} // namespace mlperf diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/bindings/c_api.h b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/bindings/c_api.h deleted file mode 100644 index 0ee44fb71..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/bindings/c_api.h +++ /dev/null @@ -1,95 +0,0 @@ -/* Copyright 2019 The MLPerf Authors. All Rights Reserved. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -/// \file -/// \brief A C API wrapping the C++ loadgen. Not tested. Needs work. -/// \details The C API allows a C or Python client to easily create -/// a SystemUnderTest without having to expose the SystemUnderTest class -/// directly. -/// ConstructSUT works with a bunch of function poitners instead that are -/// called from an underlying trampoline class. - -#ifndef SYSTEM_UNDER_TEST_C_API_H_ -#define SYSTEM_UNDER_TEST_C_API_H_ - -#include -#include - -#include "../query_sample.h" -#include "../test_settings.h" - -namespace mlperf { - -namespace c { - -/// \brief Optional opaque client data that creators of SUTs and QSLs can have -/// the loadgen pass back to their callback invocations. -/// Helps avoids global variables. -typedef uintptr_t ClientData; - -typedef void (*IssueQueryCallback)(ClientData, const QuerySample*, size_t); -typedef void (*FlushQueriesCallback)(); -typedef void (*ResponseCallback)(ClientData, QuerySampleResponse*); - -/// \brief SUT calls this function to report query result back to loadgen -void QuerySamplesComplete(QuerySampleResponse* responses, - size_t response_count); - -void QuerySamplesCompleteResponseCb(QuerySampleResponse* responses, - size_t response_count, - ResponseCallback response_cb, - ClientData client_data); - -void FirstTokenComplete(QuerySampleResponse* responses, size_t response_count); - -void FirstTokenCompleteResponseCb(QuerySampleResponse* responses, - size_t response_count, - ResponseCallback response_cb, - ClientData client_data); - -/// \brief Create an opaque SUT pointer based on C callbacks. -void* ConstructSUT(ClientData client_data, const char* name, size_t name_length, - IssueQueryCallback issue_cb, - FlushQueriesCallback flush_queries_cb); -/// \brief Destroys the SUT created by ConstructSUT. -void DestroySUT(void* sut); - -typedef void (*LoadSamplesToRamCallback)(ClientData, const QuerySampleIndex*, - size_t); -typedef void (*UnloadSamplesFromRamCallback)(ClientData, - const QuerySampleIndex*, size_t); - -/// \brief Create an opaque QSL pointer based on C callbacks. -void* ConstructQSL(ClientData client_data, const char* name, size_t name_length, - size_t total_sample_count, size_t performance_sample_count, - LoadSamplesToRamCallback load_samples_to_ram_cb, - UnloadSamplesFromRamCallback unload_samples_from_ram_cb); -/// \brief Destroys the QSL created by ConsructQSL. -void DestroyQSL(void* qsl); - -/// \brief Run tests on a SUT created by ConstructSUT(). -/// \details This is the C entry point. See mlperf::StartTest for the C++ entry -/// point. -void StartTest(void* sut, void* qsl, const TestSettings& settings, - const std::string& audit_config_filename); - -/// -/// \brief Register a thread for query issuing in Server scenario. -/// \details This is the C entry point. See mlperf::RegisterIssueQueryThread for -/// the C++ entry point. -/// -void RegisterIssueQueryThread(); - -} // namespace c -} // namespace mlperf - -#endif // SYSTEM_UNDER_TEST_C_API_H_ diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/bindings/python_api.cc b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/bindings/python_api.cc deleted file mode 100644 index 96396dab9..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/bindings/python_api.cc +++ /dev/null @@ -1,484 +0,0 @@ -/* Copyright 2019 The MLPerf Authors. All Rights Reserved. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -/// \file -/// \brief Python bindings for the loadgen using pybind11. - -#ifndef PYTHON_BINDINGS_H -#define PYTHON_BINDINGS_H - -#include - -#include "../loadgen.h" -#include "../query_dispatch_library.h" -#include "../query_sample.h" -#include "../query_sample_library.h" -#include "../system_under_test.h" -#include "../test_settings.h" -#include "pybind11/functional.h" -#include "pybind11/pybind11.h" -#include "pybind11/stl.h" -#include "pybind11/stl_bind.h" - -namespace mlperf { - -namespace { - -using IssueQueryCallback = std::function)>; -using FastIssueQueriesCallback = - std::function, std::vector)>; -using FlushQueriesCallback = std::function; -using NameCallback = std::function; - -// Forwards SystemUnderTest calls to relevant callbacks. -class SystemUnderTestTrampoline : public SystemUnderTest { - public: - SystemUnderTestTrampoline(std::string name, IssueQueryCallback issue_cb, - FlushQueriesCallback flush_queries_cb) - : name_(std::move(name)), - issue_cb_(issue_cb), - flush_queries_cb_(flush_queries_cb) {} - ~SystemUnderTestTrampoline() override = default; - - const std::string& Name() override { return name_; } - - void IssueQuery(const std::vector& samples) override { - pybind11::gil_scoped_acquire gil_acquirer; - issue_cb_(samples); - } - - void FlushQueries() override { flush_queries_cb_(); } - - protected: - std::string name_; - IssueQueryCallback issue_cb_; - FlushQueriesCallback flush_queries_cb_; -}; - -class FastSystemUnderTestTrampoline : public SystemUnderTestTrampoline { - public: - FastSystemUnderTestTrampoline(std::string name, - FastIssueQueriesCallback fast_issue_cb, - FlushQueriesCallback flush_queries_cb) - : SystemUnderTestTrampoline(name, nullptr, flush_queries_cb), - fast_issue_cb_(fast_issue_cb) {} - ~FastSystemUnderTestTrampoline() override = default; - - void IssueQuery(const std::vector& samples) override { - pybind11::gil_scoped_acquire gil_acquirer; - std::vector responseIds; - std::vector querySampleIndices; - for (auto& s : samples) { - responseIds.push_back(s.id); - querySampleIndices.push_back(s.index); - } - fast_issue_cb_(responseIds, querySampleIndices); - } - - private: - FastIssueQueriesCallback fast_issue_cb_; -}; - -using LoadSamplesToRamCallback = - std::function)>; -using UnloadSamplesFromRamCallback = - std::function)>; - -// Forwards QuerySampleLibrary calls to relevant callbacks. -class QuerySampleLibraryTrampoline : public QuerySampleLibrary { - public: - QuerySampleLibraryTrampoline( - std::string name, size_t total_sample_count, - size_t performance_sample_count, - LoadSamplesToRamCallback load_samples_to_ram_cb, - UnloadSamplesFromRamCallback unload_samples_from_ram_cb) - : name_(std::move(name)), - total_sample_count_(total_sample_count), - performance_sample_count_(performance_sample_count), - load_samples_to_ram_cb_(load_samples_to_ram_cb), - unload_samples_from_ram_cb_(unload_samples_from_ram_cb) {} - ~QuerySampleLibraryTrampoline() override = default; - - const std::string& Name() override { return name_; } - size_t TotalSampleCount() { return total_sample_count_; } - size_t PerformanceSampleCount() { return performance_sample_count_; } - - void LoadSamplesToRam(const std::vector& samples) override { - pybind11::gil_scoped_acquire gil_acquirer; - load_samples_to_ram_cb_(samples); - } - void UnloadSamplesFromRam( - const std::vector& samples) override { - pybind11::gil_scoped_acquire gil_acquirer; - unload_samples_from_ram_cb_(samples); - } - - private: - std::string name_; - size_t total_sample_count_; - size_t performance_sample_count_; - LoadSamplesToRamCallback load_samples_to_ram_cb_; - UnloadSamplesFromRamCallback unload_samples_from_ram_cb_; -}; - -// A QDL that allows defining callbacks for -// IssueQuery, FlushQueries, and Name methods. -class QueryDispatchLibraryTrampoline : public QueryDispatchLibrary { - public: - QueryDispatchLibraryTrampoline(IssueQueryCallback issue_query_callback, - FlushQueriesCallback flush_queries_callback, - NameCallback name_callback) - : issue_query_callback_(issue_query_callback), - flush_queries_callback_(flush_queries_callback), - name_callback_(name_callback) {} - - // Returns the name of the SUT. Name shall be returned over the network - // TODO: other bindings should also be fixed eventually to be used over the - // network - const std::string& Name() override { - static std::string name; // HACK: avoid returning a reference to temporary. - pybind11::gil_scoped_acquire gil_acquirer; - name = name_callback_(); // name_callback_() shall returned name over the - // network. - return name; - } - - void IssueQuery(const std::vector& samples) override { - pybind11::gil_scoped_acquire gil_acquirer; - issue_query_callback_(samples); - } - - void FlushQueries() override { flush_queries_callback_(); } - - protected: - IssueQueryCallback issue_query_callback_; - FlushQueriesCallback flush_queries_callback_; - NameCallback name_callback_; -}; - -} // namespace - -/// \brief Python bindings. -namespace py { - -uintptr_t ConstructSUT(IssueQueryCallback issue_cb, - FlushQueriesCallback flush_queries_cb) { - SystemUnderTestTrampoline* sut = - new SystemUnderTestTrampoline("PySUT", issue_cb, flush_queries_cb); - return reinterpret_cast(sut); -} - -void DestroySUT(uintptr_t sut) { - SystemUnderTestTrampoline* sut_cast = - reinterpret_cast(sut); - delete sut_cast; -} - -uintptr_t ConstructFastSUT(FastIssueQueriesCallback fast_issue_cb, - FlushQueriesCallback flush_queries_cb) { - FastSystemUnderTestTrampoline* sut = new FastSystemUnderTestTrampoline( - "PyFastSUT", fast_issue_cb, flush_queries_cb); - return reinterpret_cast(sut); -} - -void DestroyFastSUT(uintptr_t sut) { - FastSystemUnderTestTrampoline* sut_cast = - reinterpret_cast(sut); - delete sut_cast; -} - -uintptr_t ConstructQSL( - size_t total_sample_count, size_t performance_sample_count, - LoadSamplesToRamCallback load_samples_to_ram_cb, - UnloadSamplesFromRamCallback unload_samples_from_ram_cb) { - QuerySampleLibraryTrampoline* qsl = new QuerySampleLibraryTrampoline( - "PyQSL", total_sample_count, performance_sample_count, - load_samples_to_ram_cb, unload_samples_from_ram_cb); - return reinterpret_cast(qsl); -} - -void DestroyQSL(uintptr_t qsl) { - QuerySampleLibraryTrampoline* qsl_cast = - reinterpret_cast(qsl); - delete qsl_cast; -} - -uintptr_t ConstructQDL(IssueQueryCallback issue_cb, - FlushQueriesCallback flush_queries_cb, - NameCallback name_callback) { - QueryDispatchLibraryTrampoline* qdl = new QueryDispatchLibraryTrampoline( - issue_cb, flush_queries_cb, name_callback); - return reinterpret_cast(qdl); -} - -void DestroyQDL(uintptr_t qdl) { - QueryDispatchLibraryTrampoline* qdl_cast = - reinterpret_cast(qdl); - delete qdl_cast; -} - -void StartTest(uintptr_t sut, uintptr_t qsl, mlperf::TestSettings test_settings, - const std::string& audit_config_filename) { - pybind11::gil_scoped_release gil_releaser; - SystemUnderTestTrampoline* sut_cast = - reinterpret_cast(sut); - QuerySampleLibraryTrampoline* qsl_cast = - reinterpret_cast(qsl); - LogSettings default_log_settings; - mlperf::StartTest(sut_cast, qsl_cast, test_settings, default_log_settings, - audit_config_filename); -} - -void StartTestWithLogSettings(uintptr_t sut, uintptr_t qsl, - mlperf::TestSettings test_settings, - mlperf::LogSettings log_settings, - const std::string& audit_config_filename) { - pybind11::gil_scoped_release gil_releaser; - SystemUnderTestTrampoline* sut_cast = - reinterpret_cast(sut); - QuerySampleLibraryTrampoline* qsl_cast = - reinterpret_cast(qsl); - mlperf::StartTest(sut_cast, qsl_cast, test_settings, log_settings, - audit_config_filename); -} - -using ResponseCallback = std::function; - -/// TODO: Get rid of copies. -void QuerySamplesComplete(std::vector responses, - ResponseCallback response_cb = {}) { - pybind11::gil_scoped_release gil_releaser; - mlperf::QuerySamplesComplete(responses.data(), responses.size(), response_cb); -} - -void FirstTokenComplete(std::vector responses, - ResponseCallback response_cb = {}) { - pybind11::gil_scoped_release gil_releaser; - mlperf::FirstTokenComplete(responses.data(), responses.size(), response_cb); -} - -PYBIND11_MODULE(mlperf_loadgen, m) { - m.doc() = "MLPerf Inference load generator."; - - pybind11::enum_(m, "TestScenario") - .value("SingleStream", TestScenario::SingleStream) - .value("MultiStream", TestScenario::MultiStream) - .value("Server", TestScenario::Server) - .value("Offline", TestScenario::Offline); - - pybind11::enum_(m, "TestMode") - .value("SubmissionRun", TestMode::SubmissionRun) - .value("AccuracyOnly", TestMode::AccuracyOnly) - .value("PerformanceOnly", TestMode::PerformanceOnly) - .value("FindPeakPerformance", TestMode::FindPeakPerformance); - - pybind11::class_(m, "TestSettings") - .def(pybind11::init<>()) - .def_readwrite("scenario", &TestSettings::scenario) - .def_readwrite("mode", &TestSettings::mode) - .def_readwrite("single_stream_expected_latency_ns", - &TestSettings::single_stream_expected_latency_ns) - .def_readwrite("single_stream_target_latency_percentile", - &TestSettings::single_stream_target_latency_percentile) - .def_readwrite("multi_stream_expected_latency_ns", - &TestSettings::multi_stream_expected_latency_ns) - .def_readwrite("multi_stream_target_latency_percentile", - &TestSettings::multi_stream_target_latency_percentile) - .def_readwrite("multi_stream_samples_per_query", - &TestSettings::multi_stream_samples_per_query) - .def_readwrite("server_target_qps", &TestSettings::server_target_qps) - .def_readwrite("server_target_latency_ns", - &TestSettings::server_target_latency_ns) - .def_readwrite("server_target_latency_percentile", - &TestSettings::server_target_latency_percentile) - .def_readwrite("server_coalesce_queries", - &TestSettings::server_coalesce_queries) - .def_readwrite("server_find_peak_qps_decimals_of_precision", - &TestSettings::server_find_peak_qps_decimals_of_precision) - .def_readwrite("server_find_peak_qps_boundary_step_size", - &TestSettings::server_find_peak_qps_boundary_step_size) - .def_readwrite("server_max_async_queries", - &TestSettings::server_max_async_queries) - .def_readwrite("server_num_issue_query_threads", - &TestSettings::server_num_issue_query_threads) - .def_readwrite("offline_expected_qps", - &TestSettings::offline_expected_qps) - .def_readwrite("min_duration_ms", &TestSettings::min_duration_ms) - .def_readwrite("max_duration_ms", &TestSettings::max_duration_ms) - .def_readwrite("min_query_count", &TestSettings::min_query_count) - .def_readwrite("max_query_count", &TestSettings::max_query_count) - .def_readwrite("qsl_rng_seed", &TestSettings::qsl_rng_seed) - .def_readwrite("sample_index_rng_seed", - &TestSettings::sample_index_rng_seed) - .def_readwrite("schedule_rng_seed", &TestSettings::schedule_rng_seed) - .def_readwrite("accuracy_log_rng_seed", - &TestSettings::accuracy_log_rng_seed) - .def_readwrite("accuracy_log_probability", - &TestSettings::accuracy_log_probability) - .def_readwrite("print_timestamps", &TestSettings::print_timestamps) - .def_readwrite("performance_issue_unique", - &TestSettings::performance_issue_unique) - .def_readwrite("performance_issue_same", - &TestSettings::performance_issue_same) - .def_readwrite("performance_issue_same_index", - &TestSettings::performance_issue_same_index) - .def_readwrite("performance_sample_count_override", - &TestSettings::performance_sample_count_override) - .def_readwrite("test05", &TestSettings::test05) - .def_readwrite("test05_qsl_rng_seed", &TestSettings::test05_qsl_rng_seed) - .def_readwrite("test05_sample_index_rng_seed", - &TestSettings::test05_sample_index_rng_seed) - .def_readwrite("test05_schedule_rng_seed", - &TestSettings::test05_schedule_rng_seed) - .def_readwrite("use_token_latencies", &TestSettings::use_token_latencies) - .def_readwrite("ttft_latency", &TestSettings::server_ttft_latency) - .def_readwrite("tpot_latency", &TestSettings::server_tpot_latency) - .def_readwrite("infer_token_latencies", - &TestSettings::infer_token_latencies) - .def_readwrite("token_latency_scaling_factor", - &TestSettings::token_latency_scaling_factor) - .def("FromConfig", &TestSettings::FromConfig, pybind11::arg("path"), - pybind11::arg("model"), pybind11::arg("scenario"), - pybind11::arg("conf_type") = 1, - "This function configures settings from the given user " - "configuration file, model, and scenario. The conf_type flag " - "should be set to 1 for loading user.conf or else only the default " - "mlperf_conf file " - "will be loaded by the loadgen."); - - pybind11::enum_(m, "LoggingMode") - .value("AsyncPoll", LoggingMode::AsyncPoll) - .value("EndOfTestOnly", LoggingMode::EndOfTestOnly) - .value("Synchronous", LoggingMode::Synchronous); - - pybind11::class_(m, "LogOutputSettings") - .def(pybind11::init<>()) - .def_readwrite("outdir", &LogOutputSettings::outdir) - .def_readwrite("prefix", &LogOutputSettings::prefix) - .def_readwrite("suffix", &LogOutputSettings::suffix) - .def_readwrite("prefix_with_datetime", - &LogOutputSettings::prefix_with_datetime) - .def_readwrite("copy_detail_to_stdout", - &LogOutputSettings::copy_detail_to_stdout) - .def_readwrite("copy_summary_to_stdout", - &LogOutputSettings::copy_summary_to_stdout); - - pybind11::class_(m, "LogSettings") - .def(pybind11::init<>()) - .def_readwrite("log_output", &LogSettings::log_output) - .def_readwrite("log_mode", &LogSettings::log_mode) - .def_readwrite("log_mode_async_poll_interval_ms", - &LogSettings::log_mode_async_poll_interval_ms) - .def_readwrite("enable_trace", &LogSettings::enable_trace); - - pybind11::class_(m, "QuerySample") - .def(pybind11::init<>()) - .def(pybind11::init()) - .def_readwrite("id", &QuerySample::id) - .def_readwrite("index", &QuerySample::index) - .def(pybind11::pickle( - [](const QuerySample& qs) { // __getstate__ - /*Return a tuple that fully encodes state of object*/ - return pybind11::make_tuple(qs.id, qs.index); - }, - [](pybind11::tuple t) { // __setstate__ - if (t.size() != 2) - throw std::runtime_error("Invalid state for QuerySample"); - /* Create a new C++ instance*/ - QuerySample q; - q.id = t[0].cast(); - q.index = t[1].cast(); - return q; - })); - - pybind11::class_(m, "QuerySampleResponse") - .def(pybind11::init<>()) - .def(pybind11::init()) - .def(pybind11::init()) - .def_readwrite("id", &QuerySampleResponse::id) - .def_readwrite("data", &QuerySampleResponse::data) - .def_readwrite("size", &QuerySampleResponse::size) - .def_readwrite("n_tokens", &QuerySampleResponse::n_tokens) - .def(pybind11::pickle( - [](const QuerySampleResponse& qsr) { // __getstate__ - /* Return a tuple that fully encodes state of object*/ - return pybind11::make_tuple(qsr.id, qsr.data, qsr.size); - }, - [](pybind11::tuple t) { // __setstate__ - if ((t.size() != 3) || (t.size() != 4)) - throw std::runtime_error("Invalid state for QuerySampleResponse"); - /* Create a new C++ instance*/ - QuerySampleResponse q; - q.id = t[0].cast(); - q.data = t[1].cast(); - q.size = t[2].cast(); - if (t.size() == 4) { - q.n_tokens = t[3].cast(); - } else { - q.n_tokens = 0; - } - return q; - })); - - // TODO: Use PYBIND11_MAKE_OPAQUE for the following vector types. - pybind11::bind_vector>(m, "VectorQuerySample"); - pybind11::bind_vector>( - m, "VectorQuerySampleResponse"); - - m.def("ConstructSUT", &py::ConstructSUT, "Construct the system under test."); - m.def("DestroySUT", &py::DestroySUT, - "Destroy the object created by ConstructSUT."); - - m.def("ConstructFastSUT", &py::ConstructFastSUT, - "Construct the system under test, fast issue query"); - m.def("DestroyFastSUT", &py::DestroyFastSUT, - "Destroy the object created by ConstructFastSUT."); - - m.def("ConstructQSL", &py::ConstructQSL, - "Construct the query sample library."); - m.def("DestroyQSL", &py::DestroyQSL, - "Destroy the object created by ConstructQSL."); - - m.def("ConstructQDL", &py::ConstructQDL, - "Construct the query sample library, communicating with the SUT over " - "the network."); - m.def("DestroyQDL", &py::DestroyQDL, - "Destroy the object created by ConstructQDL."); - - m.def("StartTest", &py::StartTest, - "Run tests on a SUT created by ConstructSUT() with the provided QSL. " - "Uses default log settings.", - pybind11::arg("sut"), pybind11::arg("qsl"), - pybind11::arg("test_settings"), - pybind11::arg("audit_config_filename") = "audit.config"); - m.def("StartTestWithLogSettings", &py::StartTestWithLogSettings, - "Run tests on a SUT created by ConstructSUT() with the provided QSL. " - "Accepts custom log settings.", - pybind11::arg("sut"), pybind11::arg("qsl"), - pybind11::arg("test_settings"), pybind11::arg("log_settings"), - pybind11::arg("audit_config_filename") = "audit.config"); - m.def("QuerySamplesComplete", &py::QuerySamplesComplete, - "Called by the SUT to indicate that samples from some combination of" - "IssueQuery calls have finished.", - pybind11::arg("responses"), - pybind11::arg("response_cb") = ResponseCallback{}); - m.def("FirstTokenComplete", &py::FirstTokenComplete, - "Called by the SUT to indicate that tokens from some combination of" - "IssueQuery calls have finished.", - pybind11::arg("responses"), - pybind11::arg("response_cb") = ResponseCallback{}); -} - -} // namespace py -} // namespace mlperf - -#endif // PYTHON_BINDINGS_H diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/lon/README.md b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/lon/README.md deleted file mode 100644 index f46e22a65..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/lon/README.md +++ /dev/null @@ -1,67 +0,0 @@ -# Demo - -## Loadgen Over the Network - -### Overview - - -This folder provides a demo implementation for LoadGen over the network.\ -Two sides are implemented: - -1. The SUT side which is implemented in [sut_over_network_demo.py](sut_over_network_demo.py). Each Node should run it for multiple Nodes operation. -2. The LoadGen node running the LoadGen, QSL and QDL instances, implemented in [py_demo_server_lon.py](py_demo_server_lon.py) - -The demo SUT is implemented with a Flask server. the LON node implements a Flask client for network operation. - -The test runs in MLPerf Server mode. the SUT is not implementing a benchmark but contains dummy interface to preprocessing, postprocessing and model calling functions. - -### Setup - -Install python packages: - -```sh -pip install absl-py numpy wheel flask requests -``` - -Clone: - -```sh -git clone --recurse-submodules https://github.com/mlcommons/inference.git mlperf_inference -``` - -Build: - -```sh -cd mlperf_inference/loadgen -CFLAGS="-std=c++14 -O3" python setup.py bdist_wheel -cd ..; pip install --force-reinstall loadgen/dist/`ls -r loadgen/dist/ | head -n1` ; cd - -``` - -### Run the demo (single machine) - -Start the demo SUT server (run this at a separate terminal): - -```sh -python demos/lon/sut_over_network_demo.py --port 8000 -``` - -Start the test: - -```sh -python demos/lon/py_demo_server_lon.py --sut_server http://localhost:8000 -``` - -### Run the demo (over the network) - -To run over a network - simply run the demo SUT over on a different machine. For multiple Nodes run the demo SUT on each machine specifying the node number.\ - -```sh -python demos/lon/sut_over_network_demo.py --port 8000 --node N1 -``` - -Then, when running the client, replace `localhost` with the correct IP. - - -```sh -python demos/lon/py_demo_server_lon.py --sut_server IP1:8000,IP2:8000,IP3:8000 -``` diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/lon/py_demo_server_lon.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/lon/py_demo_server_lon.py deleted file mode 100644 index 1248215db..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/lon/py_demo_server_lon.py +++ /dev/null @@ -1,191 +0,0 @@ -# Copyright 2019 The MLPerf Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -""" -Python demo showing how to use the MLPerf Inference LoadGen over the Network bindings. -This programs runs in the LON Node side. -It runs the demo in MLPerf server mode over the network. -It communicates over the network with a Network SUT node, -which is running the Network SUT demo based on a flask server, implemented in SUT_over_network.py -""" - -import threading -import requests -import array -import time - -from absl import app -from absl import flags -import mlperf_loadgen - -FLAGS = flags.FLAGS - -flags.DEFINE_list( - "sut_server", "http://localhost:8000", "Address of the server(s) under test." -) - - -class QSL: - """Demo QuerySampleLibrary with dummy features.""" - - def __init__(self, total_sample_count, performance_sample_count): - self.eval_features = { - i: f"what_is_my_dummy_feature_{i}?" for i in range(total_sample_count) - } - self.qsl = mlperf_loadgen.ConstructQSL( - total_sample_count, - performance_sample_count, - self.load_samples_to_ram, - self.unload_samples_from_ram, - ) - - def get_features(self, sample_id): - """Returns the feature for a given sample id.""" - return self.eval_features[sample_id] - - def load_samples_to_ram(self, query_samples): - """Loads the features for the given query samples into RAM.""" - # Current implementation is not using this functionality. - del query_samples - return - - def unload_samples_from_ram(self, query_samples): - """Unloads the features for the given query samples from RAM.""" - # Current implementation is not using this functionality. - del query_samples - return - - def __del__(self): - mlperf_loadgen.DestroyQSL(self.qsl) - - -class QDL: - """QDL acting as a proxy to the SUT. - This QDL communicates with the SUT via HTTP. - It uses two endpoints to communicate with the SUT: - - /predict/ : Send a query to the SUT and get a response. - - /getname/ : Get the name of the SUT. Send a getname to the SUT and get a response. - """ - - def __init__(self, qsl: QSL, sut_server_addr: list): - """ - Constructor for the QDL. - Args: - qsl: The QSL to use. - sut_server_addr: A list of addresses of the SUT. - """ - self.qsl = qsl - - # Construct QDL from the python binding - self.qdl = mlperf_loadgen.ConstructQDL( - self.issue_query, self.flush_queries, self.client_get_name - ) - self.sut_server_addr = sut_server_addr - self.num_nodes = len(sut_server_addr) - - # For round robin between the SUTs: - self.next_sut_id = 0 - self.lock = threading.Lock() - - def issue_query(self, query_samples): - """Process the query to send to the SUT""" - threading.Thread( - target=self.process_query_async, - args=[query_samples]).start() - - def flush_queries(self): - """Flush the queries. Dummy implementation.""" - pass - - def process_query_async(self, query_samples): - """ - This function is called by the Loadgen in a separate thread. - It is responsible for - 1. Creating a query for the SUT, by reading the features from the QSL. - 2. Sending the query to the SUT. - 3. Waiting for the response from the SUT. - 4. Deserializing the response. - 5. Calling mlperf_loadgen.QuerySamplesComplete(query_samples, response) - Args: - query_samples: A list of QuerySample objects. - """ - responses = [] - for s in query_samples: - # Overall process: - # QDL builds a real-world query and sends to SUT --> SUT processes --> SUT sends back to QDL - # Read features from the QSL - features = self.qsl.get_features(s.index) - - time.sleep(0.001) # Ensure a maximal rate of queries to the SUT - - # Send the query to SUT in round robin - # Wait for a response - sut_result = self.client_predict(features, s.index) - response_array = array.array("B", sut_result.encode("utf-8")) - bi = response_array.buffer_info() - responses.append( - mlperf_loadgen.QuerySampleResponse( - s.id, bi[0], bi[1])) - mlperf_loadgen.QuerySamplesComplete(responses) - - def get_sut_id_round_robin(self): - """Get the SUT id in round robin.""" - with self.lock: - res = self.next_sut_id - self.next_sut_id = (self.next_sut_id + 1) % self.num_nodes - return res - - def client_predict(self, query, id): - """Serialize the query, send it to the SUT in round robin, and return the deserialized response.""" - url = "{}/predict/".format( - self.sut_server_addr[self.get_sut_id_round_robin()]) - response = requests.post(url, json={"query": query, id: id}) - return response.json()["result"] - - def client_get_name(self): - """Get the name of the SUT from ALL the SUTS.""" - if len(self.sut_server_addr) == 1: - return requests.post( - f"{self.sut_server_addr[0]}/getname/").json()["name"] - - sut_names = [ - requests.post(f"{addr}/getname/").json()["name"] - for addr in self.sut_server_addr - ] - return "Multi-node SUT: " + ", ".join(sut_names) - - def __del__(self): - mlperf_loadgen.DestroyQDL(self.qdl) - - -def main(argv): - del argv - settings = mlperf_loadgen.TestSettings() - settings.scenario = mlperf_loadgen.TestScenario.Server - settings.mode = mlperf_loadgen.TestMode.PerformanceOnly - settings.server_target_qps = 100 - settings.server_target_latency_ns = 100000000 - settings.min_query_count = 100 - settings.min_duration_ms = 10000 - - # QDL and QSL - qsl = QSL(1024, 128) - qdl = QDL(qsl, sut_server_addr=FLAGS.sut_server) - - mlperf_loadgen.StartTest(qdl.qdl, qsl.qsl, settings) - - -if __name__ == "__main__": - app.run(main) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/lon/sut_over_network_demo.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/lon/sut_over_network_demo.py deleted file mode 100644 index 55e5e038d..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/lon/sut_over_network_demo.py +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright 2019 The MLPerf Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - - -""" -Python demo showing how to use the MLPerf Inference load generator bindings over the network. -This part of the demo runs the "demo SUT" which is connected over the network to the LON node. -A corresponding "demo LON node" with the demo test is implemented in py_demo_server_lon.py. - -The SUT is implemented using a Flask server, with dummy implementation of the inference processing. -Two endpoints are exposed: -- /predict/ : Receives a query (e.g., a text) runs inference, and returns a prediction. -- /getname/ : Get the name of the SUT. - -The current implementation is a dummy implementation, which does not use -a real DNN model, batching, or pre/postprocessing code, -but rather just returns subset of the input query as a response, -Yet, it illustrates the basic structure of a SUT server. -""" - -import argparse -from flask import Flask, request, jsonify - - -app = Flask(__name__) - - -node = "" - - -def preprocess(query): - """[SUT Node] A dummy preprocess.""" - # Here may come for example batching, tokenization, resizing, - # normalization, etc. - response = query - return response - - -def dnn_model(query): - """[SUT Node] A dummy DNN model.""" - # Here may come for example a call to a dnn model such as resnet, bert, - # etc. - response = query - return response - - -def postprocess(query): - """[SUT Node] A dummy postprocess.""" - # Here may come for example a postprocessing call, e.g., NMS, - # detokenization, etc. - response = query - return response - - -@app.route("/predict/", methods=["POST"]) -def predict(): - """Receives a query (e.g., a text) runs inference, and returns a prediction.""" - query = request.get_json(force=True)["query"] - result = postprocess(dnn_model(preprocess(query))) - return jsonify(result=result) - - -@app.route("/getname/", methods=["POST", "GET"]) -def getname(): - """Returns the name of the SUT.""" - return jsonify(name=f"Demo SUT (Network SUT) node" + - (" " + node) if node else "") - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--port", type=int, default=8000) - parser.add_argument("--node", type=str, default="") - args = parser.parse_args() - node = args.node - app.run(debug=False, port=args.port) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/py_demo_multi_stream.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/py_demo_multi_stream.py deleted file mode 100644 index f6082cad6..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/py_demo_multi_stream.py +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright 2019 The MLPerf Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -"""Python demo showing how to use the MLPerf Inference load generator bindings. -""" - -from __future__ import print_function - -import threading -import time - -from absl import app -import mlperf_loadgen - -from datetime import datetime - -# Global var -NUM_AGENTS = 8 -LOOPBACK_LATENCY_S = 0.001 - - -def load_samples_to_ram(query_samples): - del query_samples - return - - -def unload_samples_from_ram(query_samples): - del query_samples - return - - -# Processes queries in NUM_AGENTS slices that complete at different times. -def process_query_async(query_samples, i_slice): - time.sleep(LOOPBACK_LATENCY_S * (i_slice + 1)) - responses = [] - samples_to_complete = query_samples[i_slice: len( - query_samples): NUM_AGENTS] - for j, s in enumerate(samples_to_complete): - responses.append(mlperf_loadgen.QuerySampleResponse(s.id, 0, 0)) - mlperf_loadgen.QuerySamplesComplete(responses) - - -def issue_query(query_samples): - for i in range(8): - threading.Thread( - target=process_query_async, args=( - query_samples, i)).start() - - -def flush_queries(): - pass - - -def main(argv): - del argv - settings = mlperf_loadgen.TestSettings() - settings.scenario = mlperf_loadgen.TestScenario.MultiStream - settings.mode = mlperf_loadgen.TestMode.PerformanceOnly - settings.multi_stream_expected_latency_ns = 8000000 - settings.multi_stream_samples_per_query = 8 - settings.min_query_count = 100 - settings.min_duration_ms = 10000 - - sut = mlperf_loadgen.ConstructSUT(issue_query, flush_queries) - qsl = mlperf_loadgen.ConstructQSL( - 1024, 128, load_samples_to_ram, unload_samples_from_ram - ) - mlperf_loadgen.StartTest(sut, qsl, settings) - mlperf_loadgen.DestroyQSL(qsl) - mlperf_loadgen.DestroySUT(sut) - - -if __name__ == "__main__": - app.run(main) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/py_demo_offline.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/py_demo_offline.py deleted file mode 100644 index 909585edc..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/py_demo_offline.py +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright 2019 The MLPerf Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -"""Python demo showing how to use the MLPerf Inference load generator bindings. -""" - -from __future__ import print_function - -import threading -import time - -from absl import app -import mlperf_loadgen - - -def load_samples_to_ram(query_samples): - del query_samples - return - - -def unload_samples_from_ram(query_samples): - del query_samples - return - - -# Processes queries in 3 slices that complete at different times. -def process_query_async(query_samples, i_slice): - time.sleep(3 * (i_slice + 1)) - responses = [] - samples_to_complete = query_samples[i_slice: len(query_samples): 3] - for s in samples_to_complete: - responses.append(mlperf_loadgen.QuerySampleResponse(s.id, 0, 0)) - mlperf_loadgen.QuerySamplesComplete(responses) - - -def issue_query(query_samples): - threading.Thread( - target=process_query_async, args=( - query_samples, 0)).start() - threading.Thread( - target=process_query_async, args=( - query_samples, 1)).start() - threading.Thread( - target=process_query_async, args=( - query_samples, 2)).start() - - -def flush_queries(): - pass - - -def main(argv): - del argv - settings = mlperf_loadgen.TestSettings() - settings.scenario = mlperf_loadgen.TestScenario.Offline - settings.mode = mlperf_loadgen.TestMode.PerformanceOnly - settings.offline_expected_qps = 1000 - - sut = mlperf_loadgen.ConstructSUT(issue_query, flush_queries) - qsl = mlperf_loadgen.ConstructQSL( - 1024, 128, load_samples_to_ram, unload_samples_from_ram - ) - mlperf_loadgen.StartTest(sut, qsl, settings) - mlperf_loadgen.DestroyQSL(qsl) - mlperf_loadgen.DestroySUT(sut) - - -if __name__ == "__main__": - app.run(main) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/py_demo_server.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/py_demo_server.py deleted file mode 100644 index 8b6f2b826..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/py_demo_server.py +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright 2019 The MLPerf Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -"""Python demo showing how to use the MLPerf Inference load generator bindings. -""" - -from __future__ import print_function - -import threading -import time - -from absl import app -import mlperf_loadgen - - -def load_samples_to_ram(query_samples): - del query_samples - return - - -def unload_samples_from_ram(query_samples): - del query_samples - return - - -def process_query_async(query_samples): - time.sleep(0.001) - responses = [] - for s in query_samples: - responses.append(mlperf_loadgen.QuerySampleResponse(s.id, 0, 0)) - mlperf_loadgen.QuerySamplesComplete(responses) - - -def issue_query(query_samples): - threading.Thread(target=process_query_async, args=[query_samples]).start() - - -def flush_queries(): - pass - - -def main(argv): - del argv - settings = mlperf_loadgen.TestSettings() - settings.scenario = mlperf_loadgen.TestScenario.Server - settings.mode = mlperf_loadgen.TestMode.PerformanceOnly - settings.server_target_qps = 100 - settings.server_target_latency_ns = 100000000 - settings.min_query_count = 100 - settings.min_duration_ms = 10000 - - sut = mlperf_loadgen.ConstructSUT(issue_query, flush_queries) - qsl = mlperf_loadgen.ConstructQSL( - 1024, 128, load_samples_to_ram, unload_samples_from_ram - ) - mlperf_loadgen.StartTest(sut, qsl, settings) - mlperf_loadgen.DestroyQSL(qsl) - mlperf_loadgen.DestroySUT(sut) - - -if __name__ == "__main__": - app.run(main) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/py_demo_single_stream.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/py_demo_single_stream.py deleted file mode 100644 index 8806271bd..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/py_demo_single_stream.py +++ /dev/null @@ -1,84 +0,0 @@ -# Copyright 2019 The MLPerf Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -"""Python demo showing how to use the MLPerf Inference load generator bindings. -""" - -from __future__ import print_function - -import array -import threading -import time - -from absl import app -import mlperf_loadgen - - -def load_samples_to_ram(query_samples): - del query_samples - return - - -def unload_samples_from_ram(query_samples): - del query_samples - return - - -def process_query_async(query_samples): - """Processes the list of queries.""" - time.sleep(0.001) - responses = [] - response_array = array.array( - "f", [0, 1, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128, 254, 255] - ) - response_info = response_array.buffer_info() - response_data = response_info[0] - response_size = response_info[1] * response_array.itemsize - for s in query_samples: - responses.append( - mlperf_loadgen.QuerySampleResponse( - s.id, response_data, response_size) - ) - mlperf_loadgen.QuerySamplesComplete(responses) - - -def issue_query(query_samples): - threading.Thread(target=process_query_async, args=[query_samples]).start() - - -def flush_queries(): - pass - - -def main(argv): - del argv - settings = mlperf_loadgen.TestSettings() - settings.scenario = mlperf_loadgen.TestScenario.SingleStream - settings.mode = mlperf_loadgen.TestMode.PerformanceOnly - settings.single_stream_expected_latency_ns = 1000000 - settings.min_query_count = 100 - settings.min_duration_ms = 10000 - - sut = mlperf_loadgen.ConstructSUT(issue_query, flush_queries) - qsl = mlperf_loadgen.ConstructQSL( - 1024, 128, load_samples_to_ram, unload_samples_from_ram - ) - mlperf_loadgen.StartTest(sut, qsl, settings) - mlperf_loadgen.DestroyQSL(qsl) - mlperf_loadgen.DestroySUT(sut) - - -if __name__ == "__main__": - app.run(main) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_multi_stream.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_multi_stream.py deleted file mode 100644 index e4b083853..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_multi_stream.py +++ /dev/null @@ -1,142 +0,0 @@ -# Copyright 2019 The MLPerf Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -"""Python demo showing how to use the MLPerf Inference load generator bindings. -""" - -from __future__ import print_function - -import argparse -import threading -import time -import numpy as np -import array - -import mlperf_loadgen - -from datetime import datetime - -# Global var -NUM_AGENTS = 8 -LOOPBACK_LATENCY_S = 0.001 - - -def f(x, y): - return 4 + 3 * x * y + x**3 + y**2 - - -def create_responses(n, m, mod=4): - r = [] - for i in range(n): - r.append([f(i, j) for j in range(m + (i % mod))]) - return r - - -responses = create_responses(1024, 20) - - -def load_samples_to_ram(query_samples): - del query_samples - return - - -def unload_samples_from_ram(query_samples): - del query_samples - return - - -# Processes queries in NUM_AGENTS slices that complete at different times. -def process_query_async(query_samples, i_slice): - time.sleep(LOOPBACK_LATENCY_S * (i_slice + 1)) - query_responses = [] - samples_to_complete = query_samples[i_slice: len( - query_samples): NUM_AGENTS] - for j, s in enumerate(samples_to_complete): - response_array = np.array(responses[s.index], np.int32) - token = response_array[0] - time.sleep(0.0002) - response_token = array.array("B", token.tobytes()) - response_token_info = response_token.buffer_info() - response_token_data = response_token_info[0] - response_token_size = response_token_info[1] * response_token.itemsize - mlperf_loadgen.FirstTokenComplete( - [ - mlperf_loadgen.QuerySampleResponse( - s.id, response_token_data, response_token_size - ) - ] - ) - time.sleep(0.02) - n_tokens = len(response_array) - response_array = array.array("B", response_array.tobytes()) - response_info = response_array.buffer_info() - response_data = response_info[0] - response_size = response_info[1] * response_array.itemsize - query_responses.append( - mlperf_loadgen.QuerySampleResponse( - s.id, response_data, response_size, n_tokens - ) - ) - mlperf_loadgen.QuerySamplesComplete(query_responses) - - -def issue_query(query_samples): - for i in range(8): - threading.Thread( - target=process_query_async, args=( - query_samples, i)).start() - - -def flush_queries(): - pass - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument( - "--mode", choices=["performance", "accuracy"], default="performance" - ) - parser.add_argument("--expected-latency", type=int, default=8000000) - parser.add_argument("--samples-per-query", type=int, default=8) - parser.add_argument("--min-query-count", type=int, default=100) - parser.add_argument("--min-duration-ms", type=int, default=30000) - return parser.parse_args() - - -def main(): - args = get_args() - settings = mlperf_loadgen.TestSettings() - settings.scenario = mlperf_loadgen.TestScenario.MultiStream - if args.mode == "performance": - settings.mode = mlperf_loadgen.TestMode.PerformanceOnly - else: - settings.mode = mlperf_loadgen.TestMode.AccuracyOnly - settings.multi_stream_expected_latency_ns = args.expected_latency - settings.multi_stream_samples_per_query = args.samples_per_query - settings.min_query_count = args.min_query_count - settings.min_duration_ms = args.min_duration_ms - settings.use_token_latencies = True - - sut = mlperf_loadgen.ConstructSUT(issue_query, flush_queries) - qsl = mlperf_loadgen.ConstructQSL( - 1024, 128, load_samples_to_ram, unload_samples_from_ram - ) - mlperf_loadgen.StartTest(sut, qsl, settings) - mlperf_loadgen.DestroyQSL(qsl) - mlperf_loadgen.DestroySUT(sut) - - -if __name__ == "__main__": - main() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_offline.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_offline.py deleted file mode 100644 index 2e190cdd5..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_offline.py +++ /dev/null @@ -1,130 +0,0 @@ -# Copyright 2019 The MLPerf Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -"""Python demo showing how to use the MLPerf Inference load generator bindings. -""" - -from __future__ import print_function - -import argparse -import threading -import time -import numpy as np -import array - -import mlperf_loadgen - - -def f(x, y): - return 4 + 3 * x * y + x**3 + y**2 - - -def create_responses(n, m, mod=4): - r = [] - for i in range(n): - r.append([f(i, j) for j in range(m + (i % mod))]) - return r - - -responses = create_responses(1024, 20) - - -def load_samples_to_ram(query_samples): - del query_samples - return - - -def unload_samples_from_ram(query_samples): - del query_samples - return - - -# Processes queries in 3 slices that complete at different times. -def process_query_async(query_samples, i_slice): - time.sleep(3 * (i_slice + 1)) - query_responses = [] - samples_to_complete = query_samples[i_slice: len(query_samples): 3] - for s in samples_to_complete: - response_array = np.array(responses[s.index], np.int32) - token = response_array[0] - time.sleep(0.0002) - response_token = array.array("B", token.tobytes()) - response_token_info = response_token.buffer_info() - response_token_data = response_token_info[0] - response_token_size = response_token_info[1] * response_token.itemsize - # mlperf_loadgen.FirstTokenComplete([mlperf_loadgen.QuerySampleResponse(s.id, response_token_data, response_token_size)]) - time.sleep(0.02) - n_tokens = len(response_array) - response_array = array.array("B", response_array.tobytes()) - response_info = response_array.buffer_info() - response_data = response_info[0] - response_size = response_info[1] * response_array.itemsize - query_responses.append( - mlperf_loadgen.QuerySampleResponse( - s.id, response_data, response_size, n_tokens - ) - ) - mlperf_loadgen.QuerySamplesComplete(query_responses) - - -def issue_query(query_samples): - threading.Thread( - target=process_query_async, args=( - query_samples, 0)).start() - threading.Thread( - target=process_query_async, args=( - query_samples, 1)).start() - threading.Thread( - target=process_query_async, args=( - query_samples, 2)).start() - - -def flush_queries(): - pass - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument( - "--mode", choices=["performance", "accuracy"], default="performance" - ) - parser.add_argument("--expected-qps", type=int, default=1000) - parser.add_argument("--min-duration-ms", type=int, default=30000) - return parser.parse_args() - - -def main(): - args = get_args() - settings = mlperf_loadgen.TestSettings() - settings.scenario = mlperf_loadgen.TestScenario.Offline - if args.mode == "performance": - settings.mode = mlperf_loadgen.TestMode.PerformanceOnly - else: - settings.mode = mlperf_loadgen.TestMode.AccuracyOnly - settings.offline_expected_qps = args.expected_qps - settings.min_duration_ms = args.min_duration_ms - settings.use_token_latencies = True - - sut = mlperf_loadgen.ConstructSUT(issue_query, flush_queries) - qsl = mlperf_loadgen.ConstructQSL( - 1024, 128, load_samples_to_ram, unload_samples_from_ram - ) - mlperf_loadgen.StartTest(sut, qsl, settings) - mlperf_loadgen.DestroyQSL(qsl) - mlperf_loadgen.DestroySUT(sut) - - -if __name__ == "__main__": - main() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_offline_inferred.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_offline_inferred.py deleted file mode 100644 index 9325b8410..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_offline_inferred.py +++ /dev/null @@ -1,130 +0,0 @@ -# Copyright 2019 The MLPerf Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -"""Python demo showing how to use the MLPerf Inference load generator bindings. -""" - -from __future__ import print_function - -import argparse -import threading -import time -import numpy as np -import array - -import mlperf_loadgen - - -def f(x, y): - return 4 + 3 * x * y + x**3 + y**2 - - -def create_responses(n, m, mod=4): - r = [] - for i in range(n): - r.append([f(i, j) for j in range(m + (i % mod))]) - return r - - -responses = create_responses(1024, 20, mod=3) - - -def load_samples_to_ram(query_samples): - del query_samples - return - - -def unload_samples_from_ram(query_samples): - del query_samples - return - - -# Processes queries in 3 slices that complete at different times. -def process_query_async(query_samples, i_slice): - time.sleep(3 * (i_slice + 1)) - query_responses = [] - samples_to_complete = query_samples[i_slice: len(query_samples): 3] - for s in samples_to_complete: - response_array = np.array(responses[s.index], np.int32) - token = response_array[0] - time.sleep(0.0002) - response_token = array.array("B", token.tobytes()) - response_token_info = response_token.buffer_info() - response_token_data = response_token_info[0] - response_token_size = response_token_info[1] * response_token.itemsize - # mlperf_loadgen.FirstTokenComplete([mlperf_loadgen.QuerySampleResponse(s.id, response_token_data, response_token_size)]) - time.sleep(0.02) - n_tokens = len(response_array) - response_array = array.array("B", response_array.tobytes()) - response_info = response_array.buffer_info() - response_data = response_info[0] - response_size = response_info[1] * response_array.itemsize - query_responses.append( - mlperf_loadgen.QuerySampleResponse( - s.id, response_data, response_size) - ) - mlperf_loadgen.QuerySamplesComplete(query_responses) - - -def issue_query(query_samples): - threading.Thread( - target=process_query_async, args=( - query_samples, 0)).start() - threading.Thread( - target=process_query_async, args=( - query_samples, 1)).start() - threading.Thread( - target=process_query_async, args=( - query_samples, 2)).start() - - -def flush_queries(): - pass - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument( - "--mode", choices=["performance", "accuracy"], default="performance" - ) - parser.add_argument("--expected-qps", type=int, default=1000) - parser.add_argument("--min-duration-ms", type=int, default=30000) - return parser.parse_args() - - -def main(): - args = get_args() - settings = mlperf_loadgen.TestSettings() - settings.scenario = mlperf_loadgen.TestScenario.Offline - if args.mode == "performance": - settings.mode = mlperf_loadgen.TestMode.PerformanceOnly - else: - settings.mode = mlperf_loadgen.TestMode.AccuracyOnly - settings.offline_expected_qps = args.expected_qps - settings.min_duration_ms = args.min_duration_ms - settings.infer_token_latencies = 1 - settings.token_latency_scaling_factor = 21 - - sut = mlperf_loadgen.ConstructSUT(issue_query, flush_queries) - qsl = mlperf_loadgen.ConstructQSL( - 1024, 128, load_samples_to_ram, unload_samples_from_ram - ) - mlperf_loadgen.StartTest(sut, qsl, settings) - mlperf_loadgen.DestroyQSL(qsl) - mlperf_loadgen.DestroySUT(sut) - - -if __name__ == "__main__": - main() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_server.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_server.py deleted file mode 100644 index b564543cd..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_server.py +++ /dev/null @@ -1,132 +0,0 @@ -# Copyright 2019 The MLPerf Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -"""Python demo showing how to use the MLPerf Inference load generator bindings. -""" - -from __future__ import print_function - -import argparse -import array -import threading -import time -import numpy as np - -from absl import app -import mlperf_loadgen - - -def f(x, y): - return 4 + 3 * x * y + x**3 + y**2 - - -def create_responses(n, m, mod=4): - r = [] - for i in range(n): - r.append([f(i, j) for j in range(m + (i % mod))]) - return r - - -responses = create_responses(1024, 20) - - -def load_samples_to_ram(query_samples): - del query_samples - return - - -def unload_samples_from_ram(query_samples): - del query_samples - return - - -def process_query_async(query_samples): - """Processes the list of queries.""" - query_responses = [] - for s in query_samples: - response_array = np.array(responses[s.index], np.int32) - token = response_array[0] - time.sleep(0.0002) - response_token = array.array("B", token.tobytes()) - response_token_info = response_token.buffer_info() - response_token_data = response_token_info[0] - response_token_size = response_token_info[1] * response_token.itemsize - mlperf_loadgen.FirstTokenComplete( - [ - mlperf_loadgen.QuerySampleResponse( - s.id, response_token_data, response_token_size - ) - ] - ) - time.sleep(0.02) - n_tokens = len(response_array) - response_array = array.array("B", response_array.tobytes()) - response_info = response_array.buffer_info() - response_data = response_info[0] - response_size = response_info[1] * response_array.itemsize - # print(f"Reported size python: {n_tokens}") - query_responses.append( - mlperf_loadgen.QuerySampleResponse( - s.id, response_data, response_size, n_tokens - ) - ) - mlperf_loadgen.QuerySamplesComplete(query_responses) - - -def issue_query(query_samples): - threading.Thread(target=process_query_async, args=[query_samples]).start() - - -def flush_queries(): - pass - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument( - "--mode", choices=["performance", "accuracy"], default="performance" - ) - parser.add_argument("--target-qps", type=int, default=100) - parser.add_argument("--target-latency-ns", type=int, default=100000000) - parser.add_argument("--min-query-count", type=int, default=100) - parser.add_argument("--min-duration-ms", type=int, default=30000) - return parser.parse_args() - - -def main(): - args = get_args() - settings = mlperf_loadgen.TestSettings() - settings.scenario = mlperf_loadgen.TestScenario.Server - if args.mode == "performance": - settings.mode = mlperf_loadgen.TestMode.PerformanceOnly - else: - settings.mode = mlperf_loadgen.TestMode.AccuracyOnly - settings.server_target_qps = args.target_qps - settings.server_target_latency_ns = args.target_latency_ns - settings.min_query_count = args.min_query_count - settings.min_duration_ms = args.min_duration_ms - settings.use_token_latencies = True - - sut = mlperf_loadgen.ConstructSUT(issue_query, flush_queries) - qsl = mlperf_loadgen.ConstructQSL( - 1024, 128, load_samples_to_ram, unload_samples_from_ram - ) - mlperf_loadgen.StartTest(sut, qsl, settings) - mlperf_loadgen.DestroyQSL(qsl) - mlperf_loadgen.DestroySUT(sut) - - -if __name__ == "__main__": - main() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_server_inferred.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_server_inferred.py deleted file mode 100644 index 76461a75d..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_server_inferred.py +++ /dev/null @@ -1,125 +0,0 @@ -# Copyright 2019 The MLPerf Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -"""Python demo showing how to use the MLPerf Inference load generator bindings. -""" - -from __future__ import print_function - -import argparse -import array -import threading -import time -import numpy as np - -from absl import app -import mlperf_loadgen - - -def f(x, y): - return 4 + 3 * x * y + x**3 + y**2 - - -def create_responses(n, m, mod=4): - r = [] - for i in range(n): - r.append([f(i, j) for j in range(m + (i % mod))]) - return r - - -responses = create_responses(1024, 20, mod=3) - - -def load_samples_to_ram(query_samples): - del query_samples - return - - -def unload_samples_from_ram(query_samples): - del query_samples - return - - -def process_query_async(query_samples): - """Processes the list of queries.""" - query_responses = [] - for s in query_samples: - response_array = np.array(responses[s.index], np.int32) - token = response_array[0] - time.sleep(0.0002) - response_token = array.array("B", token.tobytes()) - response_token_info = response_token.buffer_info() - response_token_data = response_token_info[0] - response_token_size = response_token_info[1] * response_token.itemsize - time.sleep(0.02) - n_tokens = len(response_array) - response_array = array.array("B", response_array.tobytes()) - response_info = response_array.buffer_info() - response_data = response_info[0] - response_size = response_info[1] * response_array.itemsize - # print(f"Reported size python: {n_tokens}") - query_responses.append( - mlperf_loadgen.QuerySampleResponse( - s.id, response_data, response_size) - ) - mlperf_loadgen.QuerySamplesComplete(query_responses) - - -def issue_query(query_samples): - threading.Thread(target=process_query_async, args=[query_samples]).start() - - -def flush_queries(): - pass - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument( - "--mode", choices=["performance", "accuracy"], default="performance" - ) - parser.add_argument("--target-qps", type=int, default=100) - parser.add_argument("--target-latency-ns", type=int, default=100000000) - parser.add_argument("--min-query-count", type=int, default=100) - parser.add_argument("--min-duration-ms", type=int, default=30000) - return parser.parse_args() - - -def main(): - args = get_args() - settings = mlperf_loadgen.TestSettings() - settings.scenario = mlperf_loadgen.TestScenario.Server - if args.mode == "performance": - settings.mode = mlperf_loadgen.TestMode.PerformanceOnly - else: - settings.mode = mlperf_loadgen.TestMode.AccuracyOnly - settings.server_target_qps = args.target_qps - settings.server_target_latency_ns = args.target_latency_ns - settings.min_query_count = args.min_query_count - settings.min_duration_ms = args.min_duration_ms - settings.infer_token_latencies = 1 - settings.token_latency_scaling_factor = 21 - - sut = mlperf_loadgen.ConstructSUT(issue_query, flush_queries) - qsl = mlperf_loadgen.ConstructQSL( - 1024, 128, load_samples_to_ram, unload_samples_from_ram - ) - mlperf_loadgen.StartTest(sut, qsl, settings) - mlperf_loadgen.DestroyQSL(qsl) - mlperf_loadgen.DestroySUT(sut) - - -if __name__ == "__main__": - main() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_single_stream.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_single_stream.py deleted file mode 100644 index ca8d84591..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/demos/token_metrics/py_demo_single_stream.py +++ /dev/null @@ -1,129 +0,0 @@ -# Copyright 2019 The MLPerf Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -"""Python demo showing how to use the MLPerf Inference load generator bindings. -""" - -from __future__ import print_function - -import argparse -import array -import threading -import time -import numpy as np - -from absl import app -import mlperf_loadgen - - -def f(x, y): - return 4 + 3 * x * y + x**3 + y**2 - - -def create_responses(n, m, mod=4): - r = [] - for i in range(n): - r.append([f(i, j) for j in range(m + (i % mod))]) - return r - - -responses = create_responses(1024, 20) - - -def load_samples_to_ram(query_samples): - del query_samples - return - - -def unload_samples_from_ram(query_samples): - del query_samples - return - - -def process_query_async(query_samples): - """Processes the list of queries.""" - query_responses = [] - for s in query_samples: - response_array = np.array(responses[s.index], np.int32) - time.sleep(0.0002) - token = response_array[:1] - response_token = array.array("B", token.tobytes()) - response_token_info = response_token.buffer_info() - response_token_data = response_token_info[0] - response_token_size = response_token_info[1] * response_token.itemsize - mlperf_loadgen.FirstTokenComplete( - [ - mlperf_loadgen.QuerySampleResponse( - s.id, response_token_data, response_token_size - ) - ] - ) - time.sleep(0.02) - n_tokens = len(response_array) - response_array = array.array("B", response_array.tobytes()) - response_info = response_array.buffer_info() - response_data = response_info[0] - response_size = response_info[1] * response_array.itemsize - query_responses.append( - mlperf_loadgen.QuerySampleResponse( - s.id, response_data, response_size, n_tokens - ) - ) - mlperf_loadgen.QuerySamplesComplete(query_responses) - - -def issue_query(query_samples): - threading.Thread(target=process_query_async, args=[query_samples]).start() - - -def flush_queries(): - pass - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument( - "--mode", choices=["performance", "accuracy"], default="performance" - ) - parser.add_argument("--expected-latency", type=int, default=2050000) - parser.add_argument("--min-query-count", type=int, default=100) - parser.add_argument("--min-duration-ms", type=int, default=30000) - return parser.parse_args() - - -def main(): - args = get_args() - settings = mlperf_loadgen.TestSettings() - settings.scenario = mlperf_loadgen.TestScenario.SingleStream - if args.mode == "performance": - settings.mode = mlperf_loadgen.TestMode.PerformanceOnly - else: - settings.mode = mlperf_loadgen.TestMode.AccuracyOnly - settings.single_stream_expected_latency_ns = args.expected_latency - settings.min_query_count = args.min_query_count - settings.min_duration_ms = args.min_duration_ms - settings.use_token_latencies = True - - sut = mlperf_loadgen.ConstructSUT(issue_query, flush_queries) - qsl = mlperf_loadgen.ConstructQSL( - 1024, 128, load_samples_to_ram, unload_samples_from_ram - ) - mlperf_loadgen.StartTest(sut, qsl, settings) - mlperf_loadgen.DestroyQSL(qsl) - mlperf_loadgen.DestroySUT(sut) - - -if __name__ == "__main__": - main() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/diagram_network_submission.png b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/diagram_network_submission.png deleted file mode 100644 index 35663b97fe3ad1453c431cbf5c6155ab37c78851..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 51192 zcmeFYby$?o_cu(f4<-E_5A++{qsD}Ue|ri+%t2|nK}2FIdkSU8wpoeyhlh&h=GA|PgzM$3j+hQ z3IhWx34n(#sqKi1!oVQ5wUL#DE6d74;I2-VHue@67)p_^b@36}{Z!fdNeKyKSb0hW z&8**|Q-CjUFt?0Z*@95yW-xelK{R27fn0Td8B0_}MIhG$)Al;(SqlQLIU#QVl3#nO zBt7Z3=Cg6VK9VC~b*s(qwoi=l#_we$8|e=$h|x<(Y&^iEsOX)5vIZuWb1t@0V8>cN zTX}uGHHJmWU1L)t=7+qO9y-o9ZogZjNQ@vWn1O16!pS@ zDV0D-#;z|~U`Ur8NLWA8^PzJQxzN2Cww_w%7UDqK*MqMuv2$HWw=+VU?E4B12~2hQ zjM(s`zY{D=XD1AXBgh)H{cl0I6@gPEGpdlVLyVjuC*j(=o*a>((KSm_GYi9ze5waY z%18AAx1-;FJN1tylU`N@#Xd+;=y`Y)F4**1j9n$D6zCZ(U}j4BDXd23hh%>0#Y9`oRl4$N@u_A(mHGNjVtP1~6xLg|t9oo9dugg4%Ay`a$ixM_ukc_e zBx~6Peo&Z+K!VjOi(pO&e#Fd| zoyH}eH6`IiEY3bWti|$g#KTH7^kkh@SWv70X*P_SB`@ZX0xUS-G$;JiK(Rm{Ej>4V zW+G8GER!@}n(;R`Qzkp(St@b1bnGV6d_&ctag$dSXU|=;ztZ*d&fquJuN5R$6H7(R zm5Euy$XO$oys;$wkT$_Qv zm)^fH_tz(y!*lgsW=3L|hYty33t^4apTm+WH?{ozgS;@PbV7m zL-lg#)#|SOi^4*}Hh81+Cl)3YY3$aWo2NTj7p|=WwEZM^3tB#RQJIBT>FnaXBC+e( z#-tF76Evo-E+76GopX z^RQoIv2NrPp;U~E$v z7A5Rtj1@G3?s$s&MV`8`~wFf#V_fov|`8soN}^8RkOEiYc0K zBo-Mq6@e(!7*NzSBAt06f^|9U0urPICXel6wNb)Lij5!>;V04yOkpldxYzfjkoJ;B zBsf!ggn6&eVU_t?Xp@X&K3GNGLz7}0k0J6Zm+LX%Q#p?O>CI;!0R|C|WFL+_aOZgu zo-NIjJNcF5gncF&Q;osQcs*TUJ*2uC!NoAxQ8-0k4tm)8zUoCe+3l!qHAy+$bs&X{ z=tx)A5@B3dvNNM3WljiBx6oqaq3CarKV?9~`JxfvaWG$d;-carP>Wz1WrHGziI{LK z@h&}{Q_Vto%CRKMd}R0TY))4gVjf|8#O}!Qi{lB!Y-D_7GaGq-!m1J>q&um;5KD`| zM#6??OL&X0G(wF`)0^;X&kKbn3Fra_Y5Bxs5+Q`d;eU zo%D}9QTX&Y!7~vL8V~AE8QyQNQWXlO$AWFPK4Q)AzYyGG+r#{z5mK>~-~M&cZsZd8vooPLWRK69b==SLLr- z&b0O?F9`yLY%>h!T0!?PQL7sr7EAx+|YyBc!qP)$>=AyOC#-XANg> zcX`BeNwP^`;VR*(aU}(51${2bW{ni>{gcYO&(_?(XKW9-K60;c@!#xlj(o-Ws?^g< z@SCmqRo`n?eMNtlQ}$ z7ptdNQaOIQm2a?&QjLxp&Kqh~zNmcU^l+beKXzWo&)sj?H}xv>l>H*?V&v4}=w@+a zO?T0AY+ob5>h9&8#O}lwD)BJRb)j1*_D~0a573b;o57G`o??lH zPat0;2}ncPz<+eV7uC7)%+0$88J{7s{tP7efZ0}dyudo#+Qxka;l7(*UDi7vT^&c( zJKh_NP~Y92x^}5^(O&UqP^708s&JkAexZ3-yR1*KNsvr&#@3`tto$+?Db)W+IeR>* zJVTthLG;p3#d_d^g1m+ij=e?O85G=9CP zrz1V_zI!Y|^h3=9zI z>S$U+>gzPW7X$jH>8mbmtH>A6?HM>^+w+W2|Is{Ni}e z?djH=)(Le(pNhEx-$Bs<(aSG!QJYa=r#d^j<#!)t>X*DNl-6!zd+5^aeOn`2w+7Z1 zhd=47)ulLG)<6GvI#Czl;PL#)^R?N?S)$KUKNFh@PC0UN-h0ySam`+Or6F~De_i}6 ze&1YHb`IXxGYpyNv-@(UKIP+OwAm8&Vq?bPn&x_tBoO-uo5z3T;CIqI=Sk-~52c?mU$VVT{1jzh$X+M%DZZe+p(DN9KH90ce_S+B;H{X(R6wtfv|mJi zJ)E~xb~}&UrHI$R|9MU#!~1Ba)TMYvT`^12=l<`ZejS~i!&*$Mw|-`euns@|Xo;A-+Nq)JRdY$`_qaw^u!Y$Tquj4d*XHO`}aqf%A zl4URV?T3xe$BwNn+Q{O&UrjzJpLgewN^hye`X7LgnAnI^WDmCsP+%u{z{VicsF%1@aqRp0N$NfHOHso z<4-52ZExQ>-COu|HMmllI$uAJzkjrF+RC#huseIxz9GBQn-V{IRqV&&_x{5EuHjI1 zJk1CD5*wSmB1H@dYBQFtt-(On5@Ecc#c)T4<;LjF+txjymgULX4KDi`}dd8{OK9fpjytgPFp!rYOJVUr6o~ntOIXOHsF?TYxc!YFt{=)}D94U& z`41mC3pX=Y8)tVLCr8L1z9yzl9_|t>EPn+3*YBTxS|DxyTau&Ozr#WgkmpYg58oqR zp8w*FCKdlvDhjtjTG;E!**KtehL$1uL{M1#Z~OmG&3{Y$AC!pyrhF{G|34}JN6r6E z`P|LIRo2M?Ez(``zdzT%#Q(GMUqW%7KO_Gin)pY}e@oGNmLwGC`LAy#Nf?Zr{E7aI z^fq!D&(JB_$o~8=|JVlfpY%`OeF|F7_XGn&8beu5<{1)mx7pL(pf~ZCexEUhG7tw4 zD5A)lOwO*~K9FQFow^2{uCb5bJB+cK|3Yz>qi5G)ke%VN;FnD^y8_X#qi35{(W!A| zW53U9MUJM0q+_rg|EMI<7sY752gh+9X)j&J(wOq`)D8IA*z$gR+mC<5hKwgJkaZCt zj1?yx#hkE&lWNMkzATFMw-Ah_ZY)VgkEz&3=8U67j>%D#LjQz%2L1Ae?@Xe22vnYNZu0d{gIDm zhO_SWM--Yngq_S8OSRv`Esxyx56tbcF#VKO{=;J>f52Q#R{ihLqP>Ng| z0T+AtaOkVQeMB=k7Wq56K-PpLfN&7|*Q&$Wd;>SKPVJ(;i^COm9v%%y?Gk(>TjF7CY^>|{IH!%RZ4w!)Lf>kCvc*t3 zUuT}r%`a?W-ig!GQ!RacJLJdDpJz{jIx;drAZqH?b0ORD6936P4pFSFV6=*oVji!0 ztPjylw+6H}U9P{id|jzmt6({jX*0k;PDg@!lpp`&#}C)tDLzgvu9P2Tx~g7Ym(H7p zIeq}fSF*ApOI22*j7y{}W+ZT4ov5fN^Q)6Be!B^-3^C6NGCg(mIIl9lC(9hdST=v9 z5xXC#&HJ4jc&#QX#xzItqnf)ilrCfS7vJ#=Zj;MctH2WJ|RoYb6}}6{9Spu z+OMW#^$a1W`L*YnEV{Cb*Jpdl4zqOxW(K;t96$;@!T=N@b4_Y0)is@vV>z8CfDwj5EiZZD0swp=+cwqeq(Od9$Oi|*~zI2}wmE2{O`I~}J>jmDS91ENoz^pck5vM0ae zIXr*^UX4pw~^NxH6$KF>4yxX8M)jaAvt+g6WC7`(P zu5F*GXoT2d5eb@HU$<;btz*XC4(lt)VSA0llD*BIJkEfDU$5eUgjJHbBx{ZlQY$`$ zQRFmAQMq@DarD*YiY+T|vTrs7*cW>|km7y)SM&?Dmfh!zQRE!sitFY|6}g60kq=}l z(hR+oZ!dH1rpXbKXVvljpC%3M7^k`-DH;t%Xs046I0|cOw6guLY*=5@Io8Wi@fg~t zhYNm98155h6><$0B4kJ^FoY!-Qo%cal6?B43074wisGT! zzaYQrPOqs@#-y%z$9)hIlNf&(mlj_goyId?8dR%+>C5;1%ph)YPq9;;PI zl0;b1uuhvHU=_!y(3g0oDQ*rfjnw3D$SsHGkB20!idT^0P*o^SlbNn~H$4W4 zOiF9@4GNCq6C~OC4|0D-}pKM=qN&^7P>t75S zU-g$&w%BfbR|Fk=@z_zjgzbJLqH(@CoiuVV+*Z6@S zx<(EjLHGQ2w=@IE_Cln(r0+E6PjH>^-jVbyK1Ch{CV`RK)P>kyiE9ncOE3XMeQGsH z|KoAeUoShq(8+ZnWfQ=73B>iryL0~6dtnS7+W72X@tDcuQ7NZ|1>fW0SKJ=FOuTFf z@vB)7rb+zEzPeC)8*Kh%u&cA=(N~RCv!=^z$>dKeVr(Ko${Mmi?d!@lCCxvz+#3~cC~KGqbP>uJU=DSnk&n^~=}i8Zw{Fygq=xPXG>N-pams8ISeuF5$&983NXYtJ9q|e5s2dPWK*+ zcf;U6j)hlFr{`e;%Mt9!$5<|3rg-+QgAO8|^!(!Z-+Y#!o@5bB^^>(8Iw#+FXJ$O- zj!q&&euS+0BC0qvT=tqR6SA`*c`ou+AV}HHHp!BvU0s8{^}T~a2C}1II8eYX-n+Sw3GJPg`Pb7_jiH!In3XHn3wJH_ za8>%9w{-ORGM6%jh=Eaw^Gv-SM6<4;*E_u$M0Eniz;^51b?eho1D9^fqL&26OCotg zSnV>wP==9)or*>^Of!e?w*j{s0WYq{HLepVm>Ic#55E<9OO4gA=#h(p@Q%nXxk&7> zN-;_+F>QE@%Yq(lKdAI-F>_8yJ#4{ z%5s?WGF_~%_ocL?N|SHGncw-(^K9RQZ3+!>lN(vdJ}E;<02Aq(NhIu3)J^Nhzvdlw zs9`@P{&rQ^CxUoQOWC34jD^S=fXf#IFLtl0D2zSn%HAW(BOa zuEdF{-8xGYQUKC*w>z zn>Z2Z(UQ;=faV*Ac+=t%EFH+x6-YgigLN24AzW~fh`)k8y(utHxf;lU9my=lHe}aS zjOma3!aNk_EA-PnxnI_ZnfD++izCPP+k6NvBTwP5Z%7-MCjxIY{sswr!jDE@7|o87 zbAERLC~DSUY#TRm#&j0MOWIa0cAn;3ba4#nn=&#p`N_S4HPaUd^ii+NCXf=B(iH$Y z%(9Kkd39tY4-_QV+2rW^q8nD(Z~i(vzSfsmveo48O`37u@f&>b;xYm8MukeLKt#o3 zjqa`{i!g4tD{luP@Iu3>gJGSGCi`Sv!5(oPOZ$P|3aD3oEGS<1Zy<1hN|niV70g^F zb@N?HFqC?x$PZ=+B;K(@#z5IB6!o-tRy#!c?oKNMegYoddZ$Q5#lC_sLp=e+A`Mib z`0cs8t*0NHGQvSHjgO>!n)rsDP65~YcISVV-q=k5H_Uk9dj7n;vd<&yS7*VmMVB$F z5zGy4{krsCIjQyrDO}=qiW(g3s^Ia%-2+iDb{qwHND9PKTsq>-xi9`#8Db!}S+^(j zPT#5dM-QtZ$vPI@iU=ty%9$M}q$9koEwO>Z8I`+ySQJkxagtzySGv4p0$$?#M$L=Q zqeQq*7=q)~e)MOm#$aU7(X;L(?JN_mqI`kuD=6Qb->V$MizbKi%k2P#y-O&n}<#dm#2wDv$3HQIYVn6S5-$s?rw>XBYtD#(X0%q_vYl8!kv3 z+k8c|+!REajZFw9CGPGIvjd6e34-_sh~(v!0Ns=el2NpzB)G&hC@whwMQCUT@zKh3 z9`A}OKre3!Q5g(n=50~~&|}G#l6!YzhZt*4$lzTC`a;zK)DLfG7jEmwaBjLnWSPHv zw)7H(pj{IAO|PBI;djaGuQ@l1p$t&+&ha+jrM zikeQd?gI>-Dv0@zPXvBJxLbPs-67MR$trRC5yNWN^+ks`Owy2swMlL}ye)$?L0nj- zid6g{kmkkV{6Qbgbfv--+XG2n#3x@tR?6*a@)V|2gGHhxap#kSHHT< z81?O$$c50sLhdf&@2qi9#Ps1)34$#sFI*EXz?T1Mnb9X8Gm_6qI9#-pX>NsRHH>)i zYFt}`COhCVEx;y>pvDxZYF)+XcA4k4W0e)ve*t0Y3Qrtkx@^O{43+^A)52o=&ZkHH z&#*3+El$t3!v97AUhv1zHPRy}C+O>Fz^$9MFVsjpZ61Ijul$O3@AqKq?`Se`st_X> zh3bIm<;hjwGP5?(0Pr+08GVzIAkaS1mm7Ov80q~xR2#{+)Ean~_P7NPb0@%)Ai1Z3uGwA}xbhpHG+uny8 zL(uSzT73F~JR-5Wi=tb=@ax%rYo(K4%uD@%z-oke|c9u%8~tbBsHhTD|J6 z6qCT+r38K50E>9i7fvScN|Df6!KfkHuOz{vEp! z9F%wBe>rGGiwUy?B9LcHr=c(?@p6bXqF0u4850*PDuR?lioV16zmYhGdEg_VRoVWh^SEIumI_?RkFvov%cjk0w z3FfOzCmOV-qqythpbAA7TZQ zK%w9Vzn3uchTy(X9uN}_o=L(@M8`XSkboTx()tkho#HGCsF!KwfVQwF08JV7k7NOU zO|Id>daWVS!qkI|SSAjku^)kSHF?IeN?h5HK6M;BapsN;^I4hszIc`%H!v|)2=6A( z&Ha8HF3k(5=M_oAiB8*buheZd8Q7dC=n#k0{AR?n#ms~J4BKj(WQ6FsSJ}gsCvZ7wDjI2_WSs>FX87 zy-b)T7|K1lV?le)XbdYT=_wm;xgki*PJ8YI+AvQjBoW04ebt0r?on<46EzPFWy4F+ zoRR{aZRFPLeGkM?st9ZYkiuS~qTo><$+#>g>Zf{9EbHPa-*bNV=S-af>2_pM_=jp2 z*$L!JLfdU`tQ^8W+JFQMarVYECY<2jKnk3JQ{YN>MKQxw%m!vqo}4U#lpIEulnL7w zS_O(jLF_on`~-g8`fHcLsJ6B+=P6?#7!~pvWz|u8uN^1Ystg7;>fuXSWQNg8T~Cj4 zX(vd^7@HrP6egf#vtk5SBIR5(F`H=|I{j zf;E+hiq9SA(0d)yP!J%HE~i8C0U&UUoJtC<9cG>;5P%ISih6Zm3>QZdLghhRI1xfU zv?K!wE90bOn!@T|B%>&z>B(6n7zXL(qkqJvb%~PeNHMZZuI%Mi36lgZMJ{C};`W_> z#P?d^26muO^^92Tv%qep0Ba_!5Wtoc+f|v-%|M0KhwzmXvVGjp$WIzf^^6^`CllN7 z8mJe*k{3YyE}j~%|LC!IML?CE zkhsT^1AjJh7f5dwdl>YYQYm}|1i)z$;Xn!x;m6>v>p9t_nv8s9TnRrsW_`LIh+x*= z?hN4|0K^1SNB+)4Pf8lQVj0P+VBgFT4)skM`-8EP_y+WD?C}v_#p3r_T7!g9qwf^x zqg^%{&lDj>Oe#@A5+|HW4AZF5;2q9K^91O?A)S|1g@_#+bFn&i3L*s{e3(7@F_bbU zU%I`w(gqz|fN-L$@V)Y%g^(*Y;5lJg#BJbW=7A3B6W;RD52W!@L`CmtzWT^ivy7Ct zNi1()7SEGz=@gJ@kbMRfZVCCFkQ2<0K2>o=K2CUiU3KtLIPFaH4OFtCS2bj_KKtw< zhQG+DofQ**X3-arVh#oz(=A+tkx|s(V_{Q+1mt3lx@(P%C~x-5#v~n_C6-j1jIV;f z#orMvP4K>op`0gR!^{ieu=7d~zQJ*FE`XEPgo%QAoP&lsYq1kE{Bbl=zobh6>D}@^ zlTVN^J0$@$=s%+?yv@ z#drdBZ&K?^J$!d}h+AHY-N`EXctAXt) z!Wr0XHFw`PUTE|}j?z37#S2KMIC<;kUd>i;J3wu|`&5!Q@AkHjn!S;U?oZ%zua)m< zKch@4kg+hLH>|_dFG{dz@h0~-GmrQ)ajq2JRhrg2D7)DHA84fwAJ#^&i`~9XeeW?A z5X{KB#7oWzMU5bm2@F|uq`KrINHwsJShy7GSmslx(Rhmz58AhUvujGO`;4mT%QSD# zxmOFMjlL?sshWTR;;SG#hCuqIhuMJkFZbjWB8H>UfJz}X`V(yl&MVLAPXruPDaApu zay+0)(Vw8V$k7+~Oymf6U~G8_K=gr!oxb58sHXW(bX<9;el?%O)+S1w2jiavJPcz4 zPILn(Yn~|FXP-tTr2XxV#-HxE%_Y}OG7q8;p!(@iRQ-@HY*y^}Q(pcF0L>AYi+``_ z4;JP92T{si`PKXv$APTVQ~4JhQu0Ex)$#{4|AQ+Pa{L*9 z$gg?3e{d_tKe!bWZ2upyYU>Z?;|{ujmyN^J5FWA@C)0|VyWQIuz*x#UQTe^Izjte`hBjv}quVAb z4D4dO_nXg2L<>l4otj<$!SI4)WQ~v4hmXc&NtMy)k@!Iyc8%}lk&In*R8|`vrD5gV zpWXzW{5g#B>zVP{{sZr|RiTa79&xYOv*-WS{$HYv^7!gsySpi8bx*d&bVYjm&y1LK zFHQcLmXtxn{R0C@b4@ROo~|1T2c7bX{!o0smgTYYZhO(h#>pZ1QN+-n1xXmI6C)@l zCdRRO(G3mOs=aw4KT+NcyZl z^RI;$La~4^EW@}Anqj~0u5`_(2iBB-l5tc?hFN_Ni#=K-4dA+dvY(>twNePNlVj4o z__u=;K@=aXRu@ZOWa#l3FeZMl8x@moyKq#=gib#6M7L!_PaZCBUUZ~zcv$dvSZu`Tb;IMfS=j%N(#H3=wvotNF;Dd`yOpZ^Cq_D!spgjEa;&dS6}QYb8XoP~g`LF)oFtxi zHlq$>Pg~1!e#V+$E9lW=*aT@y1u6G`H+9QZJt5(}^pNwoEuOCz4U*Sr{m5E15%)^u z$2T?({ShUzW1IfLkwlvnRSWyuP3uCf-OY@;TGx{k7m?Zd2VQz2{Gy&HP#jkRnTQwX zM6xQy=YleA@Ct=!onSHDs=J+A>w`WW*84`)ob-GzRC+-)hBsfvG*nb@-puM3-VPDi z*xK=3Dy*%m9&Wnu^K(dXt~IB_)s3lT@1Ob2kWFn()M~lCmwB%~(cj)f%h^ZAsjF_l zmfZXL?A!bsVARlC7J!&pD88O4t*M>8w z&n1t4B+wL0snO`O5OJsrSxnYC-ermc9zV`WGB^zn)m8c0$H)H}8P!~*&LfAOLW&Rf z0K0|WMru(vDuzCB3_9-wRV4ZC-VK3`sA8W*F@zQ_h}3Au?c7si8YGM|A3^o=8#=8n zy?h}$v${Hce|pkcZoXpXQSWHB);mERh0?XBB^Rd5<4xuE>{p3eqTWqI58jy1Z#(zc z)SrL&-Wkwn#|ENX4`5mTws8DJ&D+aciO|HhtnP}a> z-tgLHuG&Hc9}3j!0^L&P&>YBua~{`Ci60wkG@*F#UB!9w^rW7XwFSY#X9#+tgXMg_k4an5a{}89=&D4PMvBgY%WL*!g=BW-qVBNfX zqMh*ZD&h30j&yp9mT5oV*;()%p0~J9T<^7OmwEecWKwZ)AU2<UN9YLkQnb%UhSEU)q|PCu2Y&NPY= zA7>8P*PdubQ4iV@=_R}z^Z_~eSXu3QKQByxOlsj9Vpkrr&mAmz&v5LJglfRBs7gY| z9;2q-;!jMTexz1K#FVP1n2GcCUIn!5Z(M6IK|eB?kBBaWOIt5}0qafQy}Ad6NL6N< z)!Rr+Q*X5zj1LA#Nbs89mlLl>17pL7Bh!^lSc^Bhwqq^4Epu{;4;|dZCrEo9KPld; z*P%7f%bL3B{4G*k-?N(cv|`I)KJ?72a{LE7!xg^dN%8Fo!Su6QMcYyO;^=SRv7`GL zIyK%_?_=PxDR;9Pz-yNGJ4b9!*d~+%!GieAg zNW77wdIp8oe>W$1(C@q#SzG*0ulC){=tF+Tso%RV^^-GrT{G{egXjocWbxOeQ<+_E zz3-7}DJJ{>(jp(!95pGg@T#VAzc6o3GHnFKf?2i4u_H;Dld?{4Ppj>@L8f&BapmTA ziOotz5Y{Av`&2OjJYtpanulQ?A7hh*-$hb5%uxBs2E~jpl=?9}?B2L$s%eF!uh1@w z{hTqMC-=kqSj+Ii6(%67etqqHl;e{Afu8 zcGi1t3MU`%)t0>Md04mjy)bWJTf5hO_?~?`*1I93jJp}Ab@^*(m~!GWtUd{rg&#L* z-TWcy5Fiji>+v(rynw1(F0aZ365>8E6&a;kH=L%BC9r6{0S#On^nLjC$#N~Uo!sK- zI`V9m7G4nIY?+Y)6O(lMQv)_pQb}4SNi#msuF~3b?Bj|7+ z7jSr4Iwr*L!>k<@y89_$yw?{>uW9TdEnGgzXeVmp2ELpb$B;9uupp^^L`qi;$VC3bB z1+NiCG; z1XD_LZ#+VaaDZtVE(zt2*eHL%SrH^xM43v9KBcX{O*aToI@>x8cW>U68?Dke9Lb~SO!73(y~)9=ZZ|`esX<{{IBD~Q z{9u`EU4xj>)B+z$t6|geCw#ndu<`c?@E!|X3Ulu1GL;9=#;bLTC)rT>I_r<`O@BBd z7I>95i6UX;y~C0@YMmZ+I$l-ZYSwt&A(X7J=f9wv>6%I0Ycum&2A#|_@2*bx#Cq!p zO2l_xJ%D$MS*S(+Xhtrp2|gkGut&nKx%xr#=)L$C4U)|(9tBdQfD!K#g>6K^2{o0* zq!+(tuWjb3XZQ$Dohh%v5Zy!qM3M0L)0KJ1aLxzO_zWw%`#j^ful2tHk0Y)~1t$i| z@P+h!+&I*#eN2PTlI_3xdGFH;8#?d{mN1W7;z6=sq|%rqdvLp2nq+v7R)5kLXCr3X zF5Djyi6Fj&@chCg{viLRy!|0gzi%!MR>)%eO2l&YVV`3si$*NgRcoPp-I>IW*dm(%HMcc5*92_dvl2fqNSZHOIwd^=vw($y~ z?DY9Qt+9-$P13gdCeB*ykyzg3@^o$$RVnOE9u6va5N;9i+~>ml1ACsc?8-go8^H4` zP`THLi0gYpryeysy6#B)9b`@?{MxcL&`lZ9 zGNpBxd{*jdWpnlZvBLJJzR1@xH3~Ah2h8$2tq&e0DfN_52h~g$7sj%LUL3wS zmY+KZ5!G5ud^mAeLnN*z*3cW!r+x<`$5&{7-(Mc|`i*A72STwH9M!jVy`{b~VO3$S zN-p=wT7;!t5~>UnVKCV{fe&LY=&>rE4r<<;ZOIPa1@Fy~eAeVt~dkI8WU zwKmmsD#3_MQAn@d@ND3P=wr25-^HjrT*)Ek9<}g!@p{56r^Pm~z$iRF<~qEH=aY9k z@u=PBqEfT2y&o|e;_GV`ZZXozDQy@9>T%6YP6cGguV?mM+1b|-S+wxk*B9wlCte>1 zI>kE@+8S4GMM{tH=`G`O%-z6R_qn2b3D@z}=|43@O?G*mSd|YDOn=9#(UyAmthxT( z113D0Fx}VlbYRVwA&Izp;o>2Q5 z0zXmngNgZ^^cYmy%t4>^nWfI7YB|*S-_>b+Q0HfEAZzfNEMb=_C>c@;q$6Ekxtb(( zOuYZ8qxxlOjwnCBz|f8?U)3n?yU(^qShcmas7rdR66{P9 zHU4di)JcF}9+S+m^Nrs;q9IAC@b=i%k*n@o5= z;nlR2(p0p9Gfj@gHrIe$=E^|MC!ue}wC2t9V8_#0_#9^nD)`4Ru-$gdd;Xv+eKJZP ztS|p9asWn$q}hMA$E!j2OGsJ?*_-;#8$nD> z8NTd;m$nc-jr3wxk8B{GMO;fwKml3zu-hcma%FF(dB#ZmWlsnjT^&*yj-0By2aB(q z^p~x$*k)LYq`(YxZi~8Xe!3yVxMT0&lr-|yG?s+dzra-i>_V0Y>3SM8rChM#kNj*~ z?Bz`C+u(h+`1B&-%a%SrB}difS7)q7Ijrb+8^8E>#+&f(edf~a?ztD}nwKl58K|Fv z0k=oEqo6uFvHX@@Q0)lleXe6NTQb*9iZ`-+`D6Mia&Yg;GAzYNprou`H5mnwyY{eQ z?Ns$I0_sPcZ!Jm}5W@_%i8=O;{GsP{cD5qEUrL9n?030Sx_qV3 z1*gGzp!5XOw29qgpuj42P@eIWzq8T@BhZ5#hr!ovfV3pGpx*Fe8E|=x!gb}fGy1|W zeZH&>otG&_3HgAo>J8y37*g!Qsz`^KM2sETQaXzP!4TIFFTs7^hGhEVf%svMfr?Ji zCv`I>QTD<8)ZfangG{V*<)Rg0Uca_2sZy6yO62O+Z15_w_q3G?<`_Xv(WaQn{6t~u z+0B)rhLO%c6)it#VZAji3{1lgTH;=5H|65Z8SoGnPU8i4g}0;lYh%y1`=vJ*aWOMB zlX~d9T^1VG+gCVc0K0Z#?^h0L$N_Hmc)+Gza44-{59pPCns?(DMjXy2KV;*FS?(W~ zT@6E4J%^O1K+Rns-B+B6{d!VEv8w4Nh!UznF+IiQJv7S(r$+>=ihOah_a$SH>!otQ z`;Ra6jVdNrhNlfupQeVFPN$N4%~kZQAFIJ|`5NmnmAhQsvyV;`N3UM>rG z&*wQjzS||v3%}#n;MSM`EJueT#viO(b`Gp7V+fU45Pvnv$ab*%B*^MmSFK_@7r|Uh z7}|F4HuZsclgPHE1Oe{I@b9PxBvdPUWs5#s3)y?MBPZW(?@_Jf^l1y9;BlZLVC?4I zw2X`tgjS9O8qm#0QbHt*kR3@BzGgWP5uQZU)j^A6QsDZWtDNCv4iZ%y_4?_8h-dH_ z6RuJT`XO`z9qN8QOE`$hf0~1jLA{2 zu%Vb)1}Aca03A+IDPVV5+8MsLmX+b7l_^b=wI*?;$tw(FCZ7;Hv_5nNe3HSLNvW%y z6X&gx&VU)0t5#3b%hTuBW5?BP8CG(b8>AaSJ`XjstPg8vP_0gtqAwwyO{!&o(xxu! z73&AYc%AJ2`Yf0^NhJV61ZWTOWiAiO2!ai@@;i-Tc(9%X2F4+Q#`z~2nH&>~H)Mnr z6MTM-lcGeOl%<-OEo|Cj;SW=c+k>E37bfLj?@tQ!9wG=DlBPPEI%>PGqj<`#$hK!= ztxDt+tPgbdgpNTggs|dK;Nl0j%4Z!Dn_XVwB^;L4zjSOSq$qXylRJ8mKjdv%7e>6|mJU6IVXr}JV4b{vd zUS?h%P#ly(I}nMG$_V!DWqGoLhel`F>MsFqY7fS`?H$5_gd2A*`(l=w1?JXt7V;Oo zExGj_3YHxY5O1Ig;x!v*<)3k#U_2C=76r>uvl# zdT3{Pi!wpZmzjusA5&f~jKz*vXjIQ3lG9k}h)|^#PD$6L4iIb^yA{3X_f<0EH8k^` zMJ-Q88-ny@F0&jLGR1xrXbSf>Mo|YUDx<&i7qXi!6(bRwY%hgF*B4K+&*JTy1-{J7 z0nVGNpAokONu>yE@*@u&-dz(Qm0W<{>pI#8ti0d`2vIu%39$KiaM1Lxnzi%F)DWJN6n`f|2XNO=@ z9lMX})aHh~+_sP${(A5@TSPBlLQ3$Nb;#p0QHgjnun+109B+(4PWu=e-uxvS541N? zau{T#Yz^9vxyCPueFbnZrBB*)@7#U??Ywb!b@WZb z1&ns5x!IyO>yOI~@7clTK5x2Is_?{Bm7=32xJJh-X_{ zK{5y?b0&l}6Df#?25To)nosngqmNL&I)93QgM5M&m+8o~3`2oHC8gN63t%4=tM@5v z`k;+g&?LyE2eQNiW^1QlKE$1ayZ|tT+;3UTl6W-ACW(F#lR1yL=;2B-(S-x1+ZB z*9AEs<`aKMZH#6Mx#%nqQH*N`h}i)z%r_lhT#=v76L8Ub{hf_T)Y&toIQeq0Ix3+h z5MZ}yjd8JupM5x;Zu94M7M0=E=I=5tQ1`juhIStF!w&Ab+}-vL(JoU92Eu5DZY-}V zM=}5xG|!u%OT@$RR|j9vFamLT9-=`{ z$ZB*=fk6$RQ?k-o!jXvPHT21CqHjAbqmJtXGQ<$}&=c&yPkF$OLZcJXWtN}x9pU8= z_bTI?2ig>lrgjf#YzwzRRKF{r|<@Uk1e$bZww$m`rejCAe#Fhu{R4 z-~k4=!GZ;M3-0a~g1bX-cXx;2?iT!R-uL^eZr%Iy-cxl>o%179X3wo^x|R=!erDO1}Y%6i)X!DK=IcDe3HclD8$xKOfQaaMlEBMyy(G z8c)6~UJKU-k@oK17Jw8&SE?SSUBm0=yMAo0luXex&JMz#Mq7SFBIRr$2F+Zfbw(P) z+zNZ6F@@lw!rv)~I-@fc{1(2!PziAU(BIKo{trX>tdd&bjQ_z;@@*fl6hCOHRzUl0 zmmvzcS6pkhcMrg}Wzep(RXtJ_!gaxF#eBlijS+zGdU&3?4y6DnKbD9NBqYQqUspY4 z4$)pBCeG@%B2wMKKofcnf6Ij=p1kv(jFYTYR`4+7{D)!lC%%C$yaWCMs6K4`4L&L| zQ-CD~8vFn^G{RQ*Y_L_V*Xt?zSVO){tDK0u7wORI6ZFggm1JG=erZW0?_6ggG!RDe z0!h5X@)E%CZ|>vy6gruK^UM>mFWWyy{(x3utbLd5>>zPhnTuMe4_1WRDPc&Q$BO?) z0_Ecgb~^=Qg|}eNQ)1w7p{i>`Rue?CR>wMHaIv9%=vd2z3$7))QO03nGJ9~Sp;7Ni z1wJ5fz1KktVUo`z?sY}oWwwzLq+0kb+CnTD5cOWwQ*w#QLNY-AQMpbX^@|J1Zt(-h zf(V0GiPZ-@aBnQ*s-|fDVH4ADI|eC=68Y$E#e5>1RHzh|PkTiH!pCTLp}lvM?Y9(u z-HB+bNb-AnUMOu%y}irpLVC^w6q{&D#%c5Hqp${HxXrR*urXTj*%)$E``!vT;4l4g zmM9dbXa|U--98k|&3K)&&V$YXC~@E?rJK!9W0D93;h>Mr&-+^Nm^pur-gsA!svEEt>@1?E0S{ku`b%B7ib!OX9LDi z%Y<=@vGEE^wFyqC8yMcOKdHv$3FoxDw;wK3uUlH-!_G(->T*u21A>tg-WASaz#5fE zW>Y7>J&oBM`1b0JCqG6&#t1sLcJWed=IV!UsK^k8_j{sQ#_{Wwd|%Pyp|bUf8j+%X zq$$|mfxld(y1GJRz`$^1t6}BPas>$~t~RUT03nP0)AGNx>f3pA7b$*13{pvNcykQM zpx@1WklBOlbQP2s&+M#59)c93KaqKee|(Q|Swq77f!5aMvU*fV(s(BB-HudcyX;fx z;`6iNjQF&lE<9!Q0jlDJn5PL+Xc{xM&U}2z>Q@l>5T4`>V~NAB-U*- z>g^K+^K25$az3Aw-`5E|l*m{#5kDWdlpO44Aq*3eN+_+I-b8lvEHIrVshA`8^uwu( zqP})0gU?9%EjC~`;vs^w>_qVRpJ+b8%*rWfK)fufC=8>*hnM%9h5Y_ijGve799|y{ zO+0WWp#56s{xGOh%hy082J(p#A4TcS$+J`nQxyJG%(|!+ir>?551Pxihfj|32-rsP z**3%34GQ2qcIB7*&ayx0YzFB@`y>yu-rF}wX_--hiyMkI-`~vVHpx#qPMAX4yNEyp zh4;D0K`L?nfR?~Eh)S0(<{k$&`izDn4r@4oWB7@A$GQKL7X~3o5!T4OUq1^LB(#V} zh1&2^{LnKGF*)BbfXj!P6^951KZGU3d6sN|4@YTOyK~z4k}7*t@V>+SON0o|>TtOc z9tH(38fvsBbD;ClRA}QE@+sU;eK|B93ishfEO}&~ACf2Bzn{7ler_>0I}1xVgFi}1 zJ}QAOO%m}wf7p4aUIyTh>oF+X&tNpeh{Gr(6Yq-mu@c4f^nR8sw^69A`}fpk$7v*3 zADMAg#>-$bcD!)=h@lECqT`q&9z}P8$m7O~vWPqNAjXQ63Ej*~v#yXeVT88aEcSy< zW5!ug*iKwD#I@+!9-ZXM?<{4isDZVt^ajG@_(e_&D59SO#GV%JOWGleSKQTc!AqDw z`HHb<-9rBC!y~MBpk4fhg`$2W6__k<=->_v6SyIdzF+x8;VeDXsP4-GQWop-WZ{#e z-||4=z#E+K&?)+(pm9YU`37X@fpm&!LH{-9$*}1S7;CW)(NjBfy{}<9_z}|Zi4TeR z;I`!92mjocRp+dJ2}qED!HBPVvp-Sqn=x=2Gq@8^>k;qU=ki@TMAR@+YvgyxVTZ;g z6)1o7!r|sKg&$78KcKM~zWEYcgL%``<2Fda^n>V7XZYQ{R4;y{4H;M~_o+Mf9X7K6 zOsu^?!Q{>Wy2Jgny^P%)w=XG+M_QS$SwXzL3nC&`Ijpa*hl9IU3nhd7nxjzT9cBul z%taU82x50Z*7K}m1~<--o*!Biz9C|6ct>aUu;eGQul>!xGNHSbr7$JTXP+KBIh=pe znd^o*oP92&zPb9qQFQfBb%j%y!kdEJEhH?0|1toi&+66n2!q{u=*JOyMYIuo<7b?# zh9+o;m0|CTy_WZ+uS1~HH0STyf2W}e45RFUVTo)5qZ2g zUu_;|oTfn)b_I&^ydc9b*ty|wejgoG;}PZMs9QLTD&W5^ z@^FzI5`UtyqTr;z#0_Qh6DkNyebdu-_bH4_g(?a`8nhjB(MxN5?1xN;)g-IGhw^o= zN7%=;IyUcX%coWsN@dyg&3F#e04!A&)M!e8Z!a|8xv$J$tTN`jj0BCd4Fk3tX~RZ> zg80|gdBuTZNeDs+%Egy;yUVJ~pdYAf*%S(B7N57jl+3oeVtrDX1t|_Yd!nJtO>+8( z9zkMwEB>8#N^Z21XTP9f4k8`7E%i(9#i3EG!a#l+S}%O(?+dGJXA^EX4{g?~|B4WY zvh{aujU%E0J^k$C+_-@GH+}6OfuAT(CL#<^q%x>)19soQ(s%gT9a3D?g;SSaByuHfbO+nW|f&D0Nqc0m$!AA7q9aC6kw`sent91q~ zXF_w^e9>{OtDg9Z-{qV80tRiH&9;X)Vm5bkGj z_FJxi64_n~lR5Lo=?zRWihiwvrMW}H%EHHBQb>b_qR(!MOXyd1e4qraS8&2%K?;}0 z``;uRpo`QTb!1rv1@@_NmOLUNXc`Fl#G|Ln6;coMz{_8|O&h8f3doLCQ61B<70b(R6Yt~AR4HcsNlnC;J zLz`|-*JrM2d+4xTav}Yo1E+eg=PiQf{Y`@1oG$U8tM`3aYVVsSdZjni&ZBY>P z6C6lgEmMbaz`nEdKM&*j)J|wv@&yH7I4FsE7&j=h0coHId#mP~%TDB|uB6{xyF(PA zCL1J>Jg2Y4`)i!{?Z7OXqrhkVr0X<6pMPEa1CJd+LT@@&Rn#jt_SgNMmoKZ|>;i+i>`C0ri&VT~$|W6@6G6ZH)3 z&R0KFm0HDNg&_MItCr1%Hm8MJIO~0SH%kJE%eRK)@-=wx{`IF81(OAI3%`g6n#S@# zrv9$C<+aC2(y;stwwxAsxjjlu&v|+G6IK}+mJiB|g2ut z>v!369MaO*zBqhV1al_I;{2&aAy?KQ8sCDsUuTYoz$UH%#nuiqm9M{hHwotLd0z%I zu{=qZXGJ)=Tpo=%D`XStsi!!vb>i-k^-CDLX(A{h%;hMOdoIrqJrr-8=&-(j0P^Mg zsf*bzzWy4vAtC1i+k3yX{kN2VN4Y7=6r6Eb`bf)30tv@oHg-{;`ZfcXX{N&ohMgoo zJ698guAe_YewqFE@YC*7R@-_;j@(HovsV;$c_bR`rQcW-0j!q0Tje6gZ5t+p{l-i} zZ{i2?OEzSJ_qtN#sB7K0cLUl%6d5eskI~~OG&eub*zO9(wW~a;MdJF3>ebyG7$LOr z8lTepdZx|PcZ$14V;$%ZG1PvClrT;~sLjp8{by|xM!duPd-=*=TOn6tJ^3g~HvZ6UFupASPw>P4eGsRWTV z+nWs|gg~!N!LI!072VjO_0cE`(|AK~$dM*kvDobFk7cUiXWo*>Mhv$+Z=axO{&_7U z{8}d9wG6S~f6$vko&S0ADyaYGNyPt2r6E9xaMUOGFKhbb^Cvp$;g~AL&1|ijw}wYZ3dN8ZlJTbGaRgy38b#$Rrc@>FN6%LJ4Y8o%I@H-4{$qox6*IOX-(pCA7NcE%O~5cCH9|8D>1vs5^M!TkS21!Ln#PAALU>*dwe z$~-(gU%iixESA!(Qwbchv#}-gK9i#DxVgDCIfU*W7#SOfjNNV#8&Owm5oeReN2UI0 zXizUIE;dRP5HFSqwir$0R?^X71PD~(o12?Sj1qEZa>^u2PzpwT{lVa+bM?ENY}cPtw$@gu3WLv6&JkCzX)O zD^&mXpi6RqA#0TOk~!=3ZBOY0=B9BCp$VS~(_D;#c@Dn&=DC3+@-ha717;SXTYc0(~C7^tIqA+sZwlN{yQwbwf?9}h?PoO}Xg zrHA5^E-2}Ehb9l?)A?vPRN!*!b}9&p`N@tMMp(%N<*DKmrNqSijc9gub~O5Jn73~l z34MmX{GHyPsynH58`D{+X;5lpz3RhHEQu?W*7FzZ!Ll*6w2W@do~bf-%bh0X-^iHX zek0I7$K!*-+CHG+(a*X3$1IlbZcBMGPmtZ?IJ|!7XU|Yh!5>lVG6@IFbbD9HQ;2r> z(L$yA&CxQIo6~U=f|$SU1EG_oe_3cW0CdY-JLY)-z`k!ml{w*d35;#}o8l;hCy9*# zm{_!J2Pyc_!J6ciZVKKB{z|JTifuURiX4K34(t#Q9Voule1)RqvgzSoOtRh1P-loy zmHV{@4PEB4Y|#B;c9p{rEcYL;NP_b68IDSgLN!)=HrE<@?YeIX3^Q#;L-Y;VOa9Lt z=#n6?hK5FblpJEl9KLlcO@8X=;B@jOz5QpS17ut_M=dTILn)$@V28Jq2!ep^S^tIX zFWKaP^0zSm`Imki!++Sqin*MPRqKZ5YRe-%H`wsRwWejmw3q4!HzNh;o1CoTEmq)W zb03T}*u#=C6V25%WD@D40Xha1rP(}pN>4Lp? zXQbuf24MkWGFI2WUPOD{%Q-NAPI*x&@1Ya>nYIyO2<5R!SCAqfKXRZ*g)YZ_Lv&?p z*~%Q6eHHvJ2X8EiMFbHEsY!d67}%i*{gU3j1w=y$&{-v{G&zMrjm;-}g_FyZ149!^ zqBeq!R=SbV15?NYbFVF814bhUpIe$JxTuF_-4o4g^tT7&>u#2ggK$jf9A_W4{X7+={Gvl9k-g^) z`uuo6%5Aribn;e{blV8eW?VGyD@tbMxM=RihCwp7U7iF1tf{+zDTkaM@rMR|$8^kj zW_*QHISSD5E!>y{!N~!`h5|0O&A;Cg1Y{jw|9+va0RPPVD$DCbIc55X-WOP*=Jm+= z*spngP*6st)&NA!Bx*F$XYW_>kR%}hI2?MPztqZ8fL&w9V;s^0c=R~u;ZbFufYJ4} zo2=geCtQhaf!>nf!hW7p$)On8u{H4-Q?utm|VfDL;#@Ne7gZpfe$*rd|Ga>tIDlXBAUHhwTZrK zO}{H^6Us=Rpvg38WtzP!n+t~S5saPPWl+350(igPvP>IH&&){Wu6CmsjjYue*NO}HG@wb zi>5F>F0Na5=akSVN67oG6<9?H3I)ha#xlZ3Zu#AEOg8|W?5TY#u}SM#CY{(OQf_v( zNL6)}(R4wyz*?-qH?db*Ly`cEFks#e#mWnq2^Q;UYSLLArziPEe`O5Zu3NH-U8pv3Y8n;%3>xrm zFVk+?I`7Mjl1=3ts0@JJ-#dz%$nNX^*(0-^$1C76ao1|HtGa(+kytm2#ddXJYxins z)axvgUkSiiO-9q8dpe7QKww~=2r*v;3O?5$Kt+~YdYKk8$EyCvxq*IsZX;&Ws-K<6 zI_k;INopqRNl8vFCHk%M>Vm2hh12hKpXAF)2^0 zyydJ*kAWCOlx%EiPfyQBqi4?`Aea*0pglI1d*dz&;^0urXlflVq9U=im0ZnQR{^0- zD2=(GnSyo)MaKZ4iCngu=Z00*w{o?jr~?DNt^>1L;gcI=$vj3?!zmn^A)(aH7!p;; zWkWVFAb<`N3$$Dh&{ST}@|c39 z!p&<|$1C_`9+wzO`?$nJd>0!pXDI@}yaaa%&}c+lbzkN6@>tGsl#9UK3@nwmQPj=^ zGK8_3?0ju_zCVQahT$ee``7OPJ3vRU@SgkJNsFyQcpc043T=;LSyqQ$i`2gU(+st$ z$c={!Zl9J&F^$%yBx+9AVP~1#vC&b*i$%I{l$Na_0rF8Pd6@B*yyrPbe zwfWSJjfEyk+kz3?g*i11OD9M-;reUI+YB9(TR3fk* zLdW?4hU^|px9Ba^S;k5bxrl!MerR+*+)yWIk#D>Z!9=jR0MIWywyPe1IV+s=+$}pW z0eZ|NjRLsgx2Yw`0X>av=2v@)F{!Dk7ap(SyQH!ivX6Yx#;2_w_i+&#sRtx;3TT_2Kf8|hXQ-z^;ilc@mKiu?AYDiX7k_QOTqx3mGq*tME@u16$zL(O=y@e z|A}7(;-23DE|LH3?`wsOfJf*^`0FG7|NQ=gP&-0NQcB(#jrtcT|;r@1eC37{~&8{9yt_Jo3*w-(742 zN4=EE7JfXgHr_yI&j;&YRBO|fcj?hIM%zXPmrd*qyY-k5V!Hy{Lsgf5>w!jxv&S)6 zAwym+e^A>4?(~T`re_u?L$Mg5#yoe}ZrbO*om|XbUN%kcI?Y(W#x9qsb%z( zQlL%Ql^IUxA9EY^!DB1eyPen0__mr&8Q%@~3dXBygC73npHAsoYA*tg0O!x-6^4gY zE?4VtluVjizy^*^CXrEU|HKL~z(jn4ZKsWWyH3)mJ8$6ITKXP!61M)+xi3zj!?(ZK zd4V<4_-%&qiUZ(6J3|hy%m@__gB~x|DrIFQyPdYECD2Qi_#c92=S*|wfEdavyi@zb zQUG%l#0x|=Y)ZreB*YaI#3b0KRsV5njs3@|m9q8fD!6;o;GF0W_ypM9flJYU|0XD2 z1H=d5`g@w_0d@GjZU?vv`y#2h0UISHAjG(81RS}sP%?o60-o#xuVLyJFt+E6NI(;& z|6bj6=h;#bKz%W&QK3%iKoeqJy8y%>l33Ojs4vFdTdR5KHMSE^;MEuSXc`0T-V(On zZf$5|0ZmMt0e6D`x9oDPoh{%`**`dl86A~#SwW!2Zsj-(>zUEX|0ZC_*p-U#KrTV#`|k`u5Pn|U;FLk4i(Lg9>+Zk`gDQa&3(o@ zeNU(K%r)&@dpQ>3&G-lAW#nO7l#Oho4io6GR}W%9Jmk88e%J)ih9T zXA7M!m{5YSe~})d8gTX?vyn;AUc?8|A`4_mof?qB|u zT^oTwnX9D5s`w&ZI{FVWbu=wuF?+`LVhh2{+PwObT0Wi8us0%9kl}a@&}Mm<{;bkc z6}b%4xNM#_{D_%>{aET;v0XlBTzovk9l$s}Z?kD+px*F7_ESQoArd8%V4G%-z7DZ< zK_WkIJyLlK7I_!)z{t?lL!MACRzy9wK%-vR)?SNW2bQi}CJzmV#QP=wzT0>240A0e zKhWT?Q9akxcmN<7{wp&Xz2&Ioc3D|jM>bVL9+D;}i})guhij_x`qQ!fk<8RYrp+rV z`b<`pol#v1<@~Xj;9MhC4eOd!zTcyx@69YNIarrf8>DWI7bs^`;I`v1&4A5f7B|c9SUq+ulG%+?Ff_78mEoy(5d0sE z)=ttd9@!#?d_0;)%n$cVS=t(myZaTDAP^|Dn;^5`8rzf?O|zK`w$RG;*0M?W2VAt_ ze#nfY=PAETxixSx+sA8(P}0yK6)`in5DsZLCoEL7F3fHj=_Dk--s!sB$U8Q*=A!h@ z&vVLfk-5ZXVA)s~CuTFY;PN~FlVtDaXf@S%`w2aR1@%207Z=yuL3fim_mV{hG0{w# z3lj9E*&i4a?2nor67PxQ>ermbZcbNd^5xTGq@=aDmTmI^e^tGv+x~M0Ldh-e_~N30 z_*B;0hcFW6@gR^8MkTy(*~o50#CtO{Gft*Vw9P-~VyteXpkxlGjFI!sbIcMSG=uWx ztBA*gEaJgcZQii00wimcZ2Au;sdA;*B8F5bK>#umk?yF@`-Y24L`&W8Vb zp1ev-s`A{~S3MmGn~4K8f*Da~4dLnetmCQ5;S#oGKofdUlU!xL{qZ6?^)w2%dS@{_ z%>SXYTK)iU%$R#&{?nAnNOa`dPm-XQYanA=5W@hUm6N=Dy#l6wg?nI8=lBF(^_gT$qpiKK#p2BcJYpm%15?IHM#-6iol)jf zcIueqa&6`hs+ZG{Oap$zP9FsbTmRrFtGwl9@AXALljm>J=SVyZ*MgGz(?8qlKAqkhPjV9 zC+GvcPL*)pXp+(4A5mQ9=0%RqcK5R4(=lQJH6p#%F9Ifb81H@%uhiaoY8$q-{5 zWj-JDN=PB;u?rHGjQSlEmvFWuZch?o5PrOcK&gnLZng4Q=HfiX;i`~z`4CcER(lt* zC8xL2SH4S4U6B!F|B~hio}KIrpZRp!`qaqxIQ0TJ_x{R?m9nf?uA`KC9dfvwH{{a? zZ$?CVY5Xw`i6P3dyMSh^7YIJAdui;QV`etUtLom7h8}Ku^dySQ;&o)Z?H+S&-3jth zWq-?Zzmd_6{a9SiT3&kJPfA_k+|GEhicrGKZ2ZeC$hC{tdVtCEk)JZ>X{Uurvr@e5 zuD`Ud_hTFE$1>yeu!448>ozL+r;|D3EsuZqsqBM4YUxWL#;_1RpY(CJy@W{5&Yjy5 z?n??kJQ0tQ@-Gp*b(VGa3f)nQ&Hf*4$97K>#ia8};@1cGuan&LVCpl<<@ET`_n*0e z2z2KzmW3v_CV$lm1v_?eufz1rny6qS!E2I~H*@ORY~&4wmjMnJIjeuE@gfhyTfPBD zQp+sSst=^S4%V$?s845Y?3cx|0*<%4(0Gh_=OFYTv175 zoUBZ#*!+c`P8r7E^-*|zdVuVBZoqr}fnY=%m$z1*IJg}5>~bvCO1A!+u}(G_YYvgG z+w(L!nrh`wF)Bd!Db?gW&^Tt1#Y>^iREi9CiEzT6ZNI;07ha)YnVBQ1J@kAUkF;kf zl@Yj0W*RK2t?UP90hnj+S$|;ounjO?WPxdGc04Bdl;qDjnkK-&HbeU~pD)04$x$x)gD~`CD7*aJDukVycL;&rv*_meRcRTGf3=)fQKBEu z%xpNNQ>wI3DPIz3)Mco%d>hlLNj4Bfk^2z$hUJX8ig{deV~@$FG7ueOvk|WC#cMmP ztbWD*K^x1J`bv_j0vIjD|M$_d=&NJ6Xws$YD9GiOmV!QaQYK^wC}Sr*D9o|Mj` z=rkx=|G56!`gkokcT_}obfMSL+H`^}YtGQ|QQunEMU8W#P6bZ!bC}++T%Y zNg2z{0Oq3$mVuXeSL|ok-ye|ahjeCw-spG9yWg*Rnv@HFqVN}0rzs?n25W-oW9ILqqA*iIEW{}QN`?8hPnPL>Fq*=w*a9EFXQ?Wz-(a| zkH5_&_)-}?1Rmkj`h_$-xAG&E!3s{{w~iEdFAwFNLCfk^?sn`Jz{aAB_gmdO5go(N z{p{-IWHoC&7EiO;_)Dt9zY6X+WabQZX1PqyH_6dtsFmj*z!8dR?Z^QFFLtOUyrRPm zCK=A(qzaT4?TKfBX}>)x?>S2bUKu-jQqB%Qxjis8f`Bc5^LQEhz;w;n=VVeMRH zn7j~#_t77J`up+_k$0q~xzQKGVeZg%fv4$Y2Rihi|2KP60cOiIFr&TzSh?vp{^dR_pU|TV<5wAEMCMGqM4)lK9H~SPv+N zv}ZoMa&!eE5q7Fj{yyxOikgn){$=$pV;n+H58mK#obSV}Gd#;1yz>L_EFpo6%p3)| zPI92#4pZC`aOApCa}cb)mB-P8b1;V+kffl~S3nomD)j3zev~~eYl4EZb5T;hUHHB{ za&f!nNGI$Q4SuXkv@PXF?*_IYm47vVQ6R`cj-?&TGt!aY&%mi_mP_Z+KgGw7loIi1F{Z0CmG*3P z(I;*^5m#o1(pUXCHrMlr1g?9X!iL|vT{zv(7ckuWg;%qney!GDD4)s17@AzspYS)y z_*d$pckA`U6xKfccB;XPh6v{tN@drKQjqZ)M4xTHLYF^fuwBBNuV*0Wbj6mRqx#lA z)gl!hX+$9Wy)xe_c4wI94Zdu><-)H43Qg!(gk@dy{b-Y*d8){OIFNrq+W~awCYtuT z-U^Q`(QzGeh8r3uRT4+jcRo|lTrZ#YKbMueU%pJBZD=%QDE!SugIE|k1fp-;KA$?V zvnuAJN2W3ubWrVMQb+LmI1qPQWdfCT*mMNekTX}k)pfo>=)g22#yXVGe5_ghdl1~S z_7@V4hT;E1_jB(QV!r{!$REYqxM*Yd$UI#6seH>Q#<2RC9EzT7TZqu^f^D2$@CEb- z(WuOzhp$z{j!%n4nAO~=OI3%G$MkzH&d7f@AhtcS7eHIz%hM0+aV(T%32>rc#)XY} zo}1rzYi01gw0@a+c#Popy)@z&Uriw7;ToD2YC821j7J<1?uM_1hRO5`Bcs08Yx$Q& z{8BGQW%V$>QYRR*OMWr_{kFX^QvVUy4>+?kx@S6qdr?MZ`!Oq|{4*h(>H_8R?RsAp zGGNQ0Vx;>UM+;Qsl?1ZQ@ajL02p^uvoqsg3`~HZdAEy`3kZiBUn+)cO`Ud zf$;5`5t57l`fMP-_Wti?G`h~h?=iW@27!yf+S&#r?4cMVx)Drr-(dv$Uk?9W_bYZo zxMe204I>|g!z+yM#%sA2yYkiHQWf|n7)jy+JNzqWmX|Feg*R{8N!k-L9(?(MT^w=f z)q+0-kTu~>hO~mY&E@y7W_cTi+Yj$mVhVl_>04P0Gm?w zdc9~|>vOPp0YO*f-B{O~&T=jiS@^a~Wzil0=C*10FU-0E)mR>v z$Ha-rjROt!oqY8{y>6F}oqYz^j@m_@fCGr%7J#d*!%?H|Y0MY9zP%(*@Z%<-$ms3x zbwT2IoGZfBI@;c7ew#A6nj?Qq5)QE|iqlHD29h;eddd2`e$^T>bd*eAt6G0CIX&{s zF8RsqX*v~>ccECMkVxz29f$R!h5cOI8u@#L8aO=VN>`+gMJEm)A;nbqE4!$yy^n@~ zs7%|NCX{)Q;?lky4){(6M z2MZC>jlnmY(J&kF?w?26;^KN~_2`rC-_4!Ap0z&|a|GFsH~LE=s9Y>4hx3W*wc4jf zw6lIT-t-6J9arW|I`_xqdX5SpEM2T`&TCAge@d`f-l|H%2Q8Cccj#U6DsF-29!8u`=y&vC$qjoW0M6(HxB3JT=N)t%*ReL@F2=(&Mq5cs55MLqCN);JHWvtH`4~KOl_Cl1{lJ4nif*+<#$GC4fHP$!jteSsrRE#5trD1g{~sMmXlI46l4X-U2C3|mghnCtQkYdF&!zSR`rp+M z{tz@@7h8>V5|bj}#MeRh=x}{N?M5!gl0ocHqV$topbfIHef)maTOq#i*UzrVCsE0$S@4bwJ2`9E zuS={VDzlBo{hc&_WbW+Fv-OQfRiu5JWUmR~bsT#%Vpu}aFM&|$GKZu}u`2W|rw~~; z-)_ogJ~GrM$6F0WbMugfus>B}w%2|R;ZN=hH!CPIpBmC-XCCDxuENFj?4AFT)xeif zeSKL9-L~b9lg>@5;r#p=C!}X1r7U+JAuP#bW~dQ&EZy{|4|ndTi$Sk^o_ofYBkx zt91WC9pfP!_con+@-GT9A97209peqkEd#zw|J%#lr7JMtqx(w_?k2`}wl$zP7PUNG zKp!~na5{{y6&C{6al&J&cAJya_|c1xrfUpKy3Tv_)~frrs(^P6z2S%Z@^|RLH>oRA zDfM07S^4LeA^CWbL@?;{i+ZE{qA@HuqKM#0S(RP?ku?7 z-JL!4Gl;CB`(4iVwy+S;|Np1L|KDY(<^@(|3=jMmtI-M_;@~{ z*}Gi>>!@9u$EE8nPHPBos6+*aYY+VQPaWq!=B^&b_4l2$k?cNVMS6NYCVIB%!rg0g zJ~6~g173LsK4y{7iH&Fw(6O-zpt{yA`B zf|-wxKwMnBbEDG$0$|eCF;lFdui!Lcb93|gdTU)A931$T7PrIuUFo(%4fo>Kf38#3 z=<|rQ3WeV~dt?%Yjf|;c=33n7+=Zr4!rL1iNGk^FE%p(Ci|)FY>*HvdHhc~3m+Q!4 zt|!lF@C`MPy7c9RN`hbgfAf_dr#mGP_QcnZ4V!-W0s|vW68y`%3LF!x)85P3%Q;V# zg3LL@xVmRD*Yt2IFt+p8+9@*S2~e? zvY*V$=27f5!nT~OIir%lI>pl_Ff+5F68L*WsjhmIXQ(%V(EK$YWH>$-V{ToDm=IW0 z2pl4*x7zG-^Vb(u!_u;J#de5+(`KWavqymv#$37eyVmP(5JiMQsyJ%qzsN{HWN270 z2oT8gv?=g~=r3k4g`U4nEi5ev_)Z@^%G-ZiK3x3&;6m-$kf%*ZaZyniv1Pmr(6@%x z*lnPFT!f>wwl#sfEzEkgNFx3cyWLS-O@(Vdpj+Dm_*G&+e`OJlmK&ykJLA9iILvz9 zgdSHDmW!Eq)|w!Uin{IMSwJYZ!rp~lD#fY`=H)Fr^!GsSM^ljivW$l5=U=<#D^U%f<<|G;$*6dtB=bNg!x_it5o1-{9g~RKCCBF2}WVi*Yo{CGwd9vq07~Aqk?J3JN{_2eoWW81x|cqxs4% zS>je8Y3WTrdE{p*HEoCZbh}Y*E4)gl)76b{K7imq?y)D+gUF&@Rq*iWxPFZjY_V8l zW;J;B6N$0HZj679Ni%pjFaf2Z96~-NT;IiNXN1s9T;hRAC+pL#2raHDB>Lw);y34?4ZoYaLZYT6{Awx)(Tn-F6ktovIFFwBLBDD=WU(at(9h*XG)dqT0702%By zZZ6IN&OxnNG0xdyH7169YdVP<#U_jjPEfHi&>l}lJAll#Tx<2{rN+mdH;m-p@w}aD z{g*cAe@UXmXwKp7JEy8$491hSj z2Q+5~IuP0TGcY*EW6cb1ao)9*+O-V87|{U=rqz%=(<|Ao5w3s|#bCHXq zbbCslR4t%WAfOeb_TWlV$#w(gt71TN=7E~@@6C^u=QhTBdxot_1`xv$P{>GPF$+byo##Uv4>k63s7)v3t@qzIa}yS66W^5gxjdft ztrY*TYS_lQ4LGhNkX5--Y7j9aO!Ioaa~x*rn?GCcDz#o~HJedkd^IB@WbLWR0BCLh zKrWfJrzN;SowrDd5y4`qZsawgtS=mD-Ro4N`kFMJ=g2A3&e(LsdV4c{fA6{}0n9dU z_&DG8K4LG6!&x8;Eealczw6DB`hSP0pscKJG@ZBUwz0M^OuHIK=3$%eS}VF>5hwLjz^V!b%m58fC|+L zg@<~>e@JLQTBtrgX*zY>Y`dNX6f;Fi{|g+?4+O}sh?xq}Ul{O`?X(>-ceRS31l_fM zR&@9bU^Y5q*YCff)Uymxd#!{57)Z(qMRhzt^wW;J>L^45D2a6~ceieLwiP z1H#{)Vv0A22bU=EQSyqB2mu58CVeqNB*wdz+`@qpME2t|48r_6-<29C@RddLH%&kz z!d^APPrW^lLhqe@V*owKg_HzlD3qFZ>OBI)=w4PeR0s|D$(KKsU9hAuhs;+85Wt3m z@(h1*0FO(v@;!iwq9G;(nx%F3UsDIE{u2>;Bl>F-crBg?4|sl=cHy(%y(FNI(ttJ2 ziWmxBz^F+3ENmPkgbqy4kiF}RaG;Pt!gndKC()0&4CCyjLyt z@!5ThxBsqkxFnc1z#Cj*pcOA8LaEna9~y??V}KUr{jqDzl0zB+q8Y8y1PoUp(6_>% zZ=^neEMX!-01aMD0E$`O1*|rL>FrU?-N~X10U;3Z=7YY*daHQHLVv^i0bRr@GFdXw z0%pW|i4IUo8mKzn`*(SYEbxLB--^MepzaAD)#^&MXxx=!|B4< zaALt#AoZx)ZwHT$Jb)1Ph-UwZlJtSA;Bk`YPF9*M0gbH#GP=5L$E%e9CeHD0J8`K% zG250;I(b$Vcv(v^)ni~*=3tnXd2_I{<1EY-sT5lPNvS!3C=1P2_XZSP)^4lDeX3Mu zec|uAMQB7oE6xHlL88A5;MvvMZ;qEtc%4pa8tir#mF4-j1)Po-xARa>|7^Zb<#Vy^ z2~!V9hy*0)PCpX^jEXqO4AfggDri1eR_nCt(r2Wn^>ztJchCgOX}_oN3j02~nvx%z zuPzD{0)+Y#n*$cTK;gfI7Sy!0)A{eVVi}QmFNfJy>^Ex6rb>XN%ki+Zei29UDlesdMuQ_Qu;E4ZxHJ{9pay%%^U$p5XG?7W5kM^#G zbtid{ZU@-_ykVet$zPS;Hp;B>3wncRKmz1y4)fVR)Rs{m>40!JozL{RhI=3O@gS70 zZP$QV(il@<0E|&hl#n4Q&vu;ha)?M3)5*1g;ouLz9pTuCKn)qo;9qeHzm4_gruToy z1eg3^obwhV5d=4*ResCBk9oK~wFYuXWTf&y3m*ZMJC2cItOW9g6C2m8MKU>{95J)| z-;01$KRKCKkVP?pbb_2kr*ji^zDp1~#5Jvjy0!dS72D|o3K49HXP zl59uZwn9+_Ux;~IgwFW!t(~RWjaI(eMWaiPhZ~2n&jkFgHExr9^68P*7!{wxH=Y?` z=*mtxY|;0AfU1p0s+vw3hh%Z96gImtX3Z-u$x9Q*#XMwpOF%mAY z^I$KKO2ZmRlm4Sz6qn67%&m$ql*?Jl2#Cp&(xlX*to(2yg8*6ADuW3X@`OKA8Z^mp z>E%B?d%PU02U5k)Pe>43LUpB5!Sum$HiHy;J?FLJw4es^c1$kkMOZE-yQk2ZfFSGz zwMi-%ncvE3tOyVr_tNMPq1U}^$Db;dx4^g~W;$ZQpzpy0Rv_Y6Mwql z5}4;QFyEpK5FlR&O7%m8B!N<-F4fzHK&iC!n-4%K5}?$3P-e9ir5?9k&@ehsiW(?o z^34wqqk_y?-SC4?J776McBV*ypO2}4O@j?k5n-QzJYY;Owon$xfJgK|MaKGw ztue1_g}FD`|IBAg;HvU}x1l_7zxRpL ze^LP3ipkPsI7*nIu&=zYOZDrzF5yN(`~)<%4yR7~|7!0$qnc{kb|FSc1cVT20@90s z(xeEQfKsKYC`DQj5R@Y2p$LTDJA%@s2^M-sN(e=|6qQ~=6-263frK->&-y|UI*bx zk3Z5G695&V)Q*FLpT@v*ly9A@s)mX@=znI+O(zBl#`k>@y9PI)F!%`4f;d|65Z(vb zxkVs8s-*nh1Pn_Bj13slArQfX&aa3&Yo-oiY>1RnK)m;c(9qDF+D@4`z-oHXtir5DzHDHiYEczz^K5r`S5 zri%S}x`>TC7!^0w;`66|{SM~T@mO}z%LE^QRb&K0;kDCOKd<#l_S-#87%vF`g$s?o zcp=t^!%bhY|Dl@(tCS*-sQ|{}lYrBWgPB0!1}$U=-W3j(4z`zea0LyxBRe%>0&E{hyTvOV7Sk7$VDq8S7~0t~*RNtJbkArJ$ywn8 ztgLYeaiyRn|GXOHb%YwjOPFKn*4>8zU9KZ0V5j&nKV(f2ucLot%fB}G?$E{v0*2Z} zQf~69xJ7%^_r8DE1mXsCks+(RAdNKBMYasj>F0LI) zfk8pE!C2|FEh7szAbE2~DrMb2?dBmDhG#XrN-XI{wyPaXVDtCMP!ZCgg`1&%WMK-YE3iA{K8;i;Fa*~VAE*NNb{*XICHt+M6%Qx5FzX#DUPv%7 zK*Gmd12{5}ne-SnhAVV}Yq-2*LM;;f76&d%s6&NLEy*o*{SQwxPFqoK1pTcF@&R-v z)IJ_6Vzd9uk$a#R+_V;9l%bCA_M+ZQrJHvB{7xFN3H3111n7a+nW~ekZ8~s+Fw-wv z$-J0F*Yg_>({6fI7sXCTJ}~p`JQlv)JrVh+-~uEQmtEUUah`4S=h5Ebe#wPzMwaiO zh03Fqx*DJp9^EP4@!4h)DFwp$HbxN-LSbF|$DET&h9oipF&dg(Y0sUfZeJeik0p!e z28M8vQ;j$NVNHh?HlSveeQ0kS#Mw_a45`G*PsMDYOB>kVS4D7_KArb0NZT&nMdM0L z+}EU}uWmB6-@yy5Y!-T3@L?8rXD64VEB6WLp*`g!{ME)ePuA<6-Y?|u+A=855YM`- zeNbJmxth`I&_I$)6Z0`v9?KjufE(yN{zwo4OG$55{;@TD6rpyG7z`B|BR(8t`<1;Q zB*fsKf4VTAe0xa`H_&(=TptVmt)VZqLhm>3;*KjL^iloHe(FG%YpUg_+wVxG~(ZdyPP#y?w@_vwd3k(xP(4q_~ z!c&Sb(7?ZB7a`a9k^A)mXxIe>mPkGfi~m_u6UggT4BSA1<5fPGoq1&41|sA=FpBi`)1UX(g24nnd$%3H%nImy-4o_I4wP)JL_0xEBY+t;CHRkN30ht6?)!M6+OFD4W4gwD+Ow6h$;_H{xVS17J-%L^xgyBA zGF(*8?&XbU_5Aul+PT+59{bx++GF|haFLgL(jw+?d`Q^7o%P#lAYWt-)_oKn7&@*m z`lmWW!rOALlkrNjb(7~RImlTuGQ3w{U`3}#XUdLAV{4l)9e&oaC#CP7pUBjVE-cEP zy}N@={Hbh+(Mn|SvWbBp~og&udxw~RvQuo)cvZBu&@DxJ-cnTw7jOMoZ z!&N%PXzYT&SWG#uCEoGht8K%vS}+aX$QCB!2`%94_=?iWC zzaD{MtmfMgEv?C+w(ZHMEdmL}=qUW%#Aia?4f>41t5lCZc|YZq6&=kv>soy?Rh!qS zel^0@MCH}B$`=*ZV`uhBXqktybCu%K(kouhyf;j8o;Kx0GS#h$;ZA;Nv91ySZ0CqL zRUscr+zyKP&~h+bpp&Vlq~rSSrKZ2L$(02yV7W~GVYwzBkAU?o?!ZmzP=Jd*g|KEUYc{VNZ_` z277pk)ymt*PRG*Pdp>nkUeeS(K!Y*F5&P4YP@0hQv*Tb`Dpg{2tUk7$6zoy9%=>?C zibZ@|of`EM<&nlWF~y3mKgYi~95>UNNVirRFWDT5;jon!^+gf+aDvy;m7}$79oBEo zg!-S-GE*Iub9GV$-9ngXNulRG-HSarK0fx5jC~brMm6ID328|`{}68z6aDzS^!{Oo zYxMA)wFZJ)eMTkq)KOMDgu`_yheL;f@Y^x9`E7TUm8b`eev`bDGPG{YYqI_|hpCCP z*zNxP1Oose${^rOlK$aP9E$wb#wsSRbbGk}_{+;~<>ubKO5hh=L`&iEf@KPUpIL?m zD_IMUfVn!t`=HOd&)|I~*sqjX?-B-8_bdBM>STL*ur!*G*K2+N!`goN4@*?UAm)2A zG8gQ+H|L`BX>mh4F^3`S3?+?Q&krwEDlX4m6#wDRHI&!>Pg|4R&kT0P`VpcrAbk1@ zlXB|I{K1m@FV{3Td@gyzVQT!LwB(};;$A&wSk0AiEx2`w6^m~!RzI_=iVc=N*n~wy>NZ#FNb-npf{j=br zFh(`+TJwzhj)=?cft0zbH-@&=KHMIBU|#uAv~J|%i63%x!JNmgQ~lj2yv)`Lc& zs0ya&-G59|&59&I&m=yRzQoH6p`7=6EtDuriGQ|8-ZrX{}|ff?cQ&rV)s zEDapg`eJEPpA!`|6f$X{>{h)!zihpvTJ5^c?_RML!#=T!ClLp8-)dJ=;eNZyU-8^Q zj&yZS8cdUXc5DPMX-V-unkKCYm$)yg-=0_b@nL`RXHY_QyH@J)2QiiY+ags@S+JLd z7@ab0b#6L0x&E|to8KK73)-5$qffms@nZYO*!K^*rzzNz-sE^WA-I7M_d_*qkoV>! zG!|At+AZE5Uzp9diHH;V`2E}buV1tmV_(j3HErjHuJ^^PpGmt;tXjUzuBzOBu|epO zo3HkcBOxJ&A$nr9D#!|xPn%nY2*J;Co)8#lRd^kT(xK1^ozfasM|gbrSnzyGwzZ)>uZVes2hjZz9j zKl?%#Yu=knu>(%A%>A57$v<#1-%LFYohFb|Om2HGGDYhJywbJ~uL(F8=!v-Jxb5x| zV&ANae^}u3VB|qpST|bjlbTNE*VWU!hhIovrY|X**Kya$6RpaH7G7Ym!sLDMCKW+f zT^J9G{>~U*mulFa?M2_5zS$;|%4n(BuAlR+BUA3UQfGr0(fEr>vSH@?mlP}7X?OiZ z@gf)6>BX^Nq|Qlz*TRJoG*>P+6;EK@-j50Uf$5%wj+tH5n9O02%1ply$6~gdEqdR4}@}8vWIt2MGU3UgHpofm-Eju4S#WE z8U37O3Vg2Vpd_?jCH8g1^BV{5{cm(~qJG+Y9VDS+ixccCDw;gZc}=b5nzOH#Jf0nk zc&>RAy}Rs`h!se^m40op&_P`e}+KX86=hJikxEsdqef5%SKRY<0iZj`xruSV_knfuZsjhwsX+PtO zcnbu_qXrj#=S4Ox>Is)G;M^+*<|~r8Rdqwta^H=MMmXfWL27Z~7#jP|4mPYD8s+Zw zk#>kCgV-l|(r|;zLiD9yAD~Y3AbZ!Th{F&NN;4{GjOJBAeq7?Ka*e!_j_&Z;pQ~`_ zx?xBwBUBNT!J5+5LdV~>G9T++`?U27heOosiRaF`K}^aJ>1Vw`YL1@u9a`}ZuN0

K;A2e}UxagM1m(RY!FHun5{%{hzHyl^CGGxv{k;Ce=Jgy$#I*2_ z6{63sxvSPY!EY-U!xzc+`Ja?m`kbK{yEj;1{8Cz}#G$Xgn%^^Fr+E32ltUp(_z_Ll zeG)eSGc7m%k?PDj!wecUPSh@&GI!NC(v?pQe!Je8{4AGAZ!i4Pg1pV*9yA zW%t$7e0`Y<*kzPbCf>;?Y(y%Rb;I=u7O|1}j4#-R?(X+0=d-3&TyKtszI+$5d0%^# z8RZPpduNwD+Nv0kGFAK|WvZQov|pPo2B6ePajeOc`OM(XYZh9G*sT_g{G(&qkqrXEYMoNB?t`<#AEqoqUUT5>`f|Z#`y=&}r@}#^cpGL`Y{xXkQ*4TrI%AOus z!v`%@*H<}(R~f7H{}roX|LKn$$a_W`KQ2XqJa3HaAMN>{EcE}cL>gK8fp+u_ad&T5 zqomxNN7dR+ap*R8spTNVYwv?X8>fE28b3< zH>)ofxvi#>uQl-CJwd`%&L&gdUYpsWc3+0_4IaI@4yAoo~EX^s6!8tVfU1=@r)VuWHUz>4CHL z3JMDI^V?NoFIo|ZeOA&dmGKN#zIp(XF%19UQve{3N5LCm86c5-rt$h}6@YCYfuru& zU;3~&zgIJR@Gu2r)e~hMdsT?mP(uRF+cc zy8ofM&st?eMFicSiE$5P1E&j`)Fga18qb_}Ruz=UcPUnAl`?7`qe=F{+p4b00R#*9Z6Qs zvOldqq~MW#04NwPcK8VJwvwRaCzj*c|BVu!10^|KJZ$^|`eB@{OpeaK1=PmSKJZZwae z<3XTWt-whX8>pfRBX_-17DK#4sNE)4HQ4F$q3 z00^s4@J(NrEZ|<8f_mxv$v_DZ$H#m$iV!{Z>6b;^anFIb^ga@h{RQZrPY=$}`=1BJ z0M7gKssTh#v&r^5_W&b!E_=m?_R^;V!+VhdP$rguMCpNP4jBPZbXd-39MI}BE}*1- z!{r*c(*TCDbLAHF9KhoA=v0@ilIcsq;pT2(3X8}%zlS(l;b&0T=*|5}i=(}@3gKe# z=A7t@5AW^u7=TecrZcE@%)WWJoZG&%^Yfc7O@Z76`wvwQ6rrh{V(#n4fW$nq`;saJ zggIirfQkqywOe(nbKpge*0~m3F{%50kS=Zi$a(SogMWk9A_k@S3~?Z?83WRmqU>z0 z4|S)ov{IHj1pGMsL04dYd#i)vdHEaBIKLP4hjs{mu(C#Tlq2l`u&p8?N^Xl#S>pvg zjClX?qcb@D$STR?b+^*Lia#v~h(*Ahd97N~$&UbH5<33}peq2<4}fm)iBkpNZg?t7 zhx?tj^DP*;Jns8v|Gx6ZsfJ-txEx3UKtY_ZfG=W*7Xp`o&q6%-7Hb->by~{*CioTq zzB2Z7+U`Wy0TvEasm2FV3+1=x{=V|X>FDI@TPB>A5~d&o<{U^rPFn^R&Z*$wQX_ZT z-wi)S+3zDh1pIyFCGhR{GQ${n*3*Uqbp90nzr}-(l=)*m|M|tSO%Dh82G7Y);sScx zx=iHR>Fn9#FDA(){+DdXKYk<$Q0YThxA*2N$6}D(-CZ}pP3*UB*a3;adPkEpsg!+3 zl?&8$?&sby%|%6Ux>GEY0m7gN;Ej;FM8Vyu1=k{H;>iOF+%SYUUYeH1d-D=TqY0O~1?aIejzi5Q(&LGA->eX+t>vA+ttHnxVC834#wd z`JR-I*4qqn9}!N7W?sbI5qm(JRa^Bi3g`h&H=no}imn8bw+<`w40-pf8otkXS^ZB$_ICmxL__*6gUn<`^J`%yPfxCCh}0n} zn-R~ULL3L)EQHU#-q^Ot&txTyd#S`$^l{*+dJeZ+^nC2aQpQr>H2h_BN$x=Yb3)DS z39vddtso>XR3lAv(>5Wgyk~zFJtkusO8^H0HntNaRm@%kunJ8mIe9Bmg?;p zJ2Vm9fbQcr!sl)5>(Or1)|Z;urtS1O-y;I6Z~MM|1BWa21>@_dds43H($!ajn)onW z9^!uD*Z43%U)4}g(c((S=(!o2@D$}^Elxp&KIrIo*A#V%Cs5I!3>Z3fx=KM;(I?yJ zDHKpteai4!|J1%KIv#6wlw-1IUPg)D(eSFA0Gd2~nV2-(hxxT!9lM3p^F;c}29vEJ zm&7CGb2}X&>^rRiFWYrfPy^p+S=Syj&8;%!ZL&wQF30x2Rd`+HJMwe~-A}nb+E0k0 zs>e3r99&R_h5kg%wQ113)fOG+j}Awg95mO?bD$l7ul)T5#qO85k#&6T-|&{>Jy_&ztrnLNaI5-Pfa-~`s)VPu#=-*^_n`z-{jKx#(D*@U8CIi%1H!QAs} z`|DFj6+jol{r>aXwqneTcDq7cb|wycREJm3;vwU5&YOpM-^OaD+saJUA(}v~xlMZ{lwMYA#T$?P!-M zLuzv<_QC;CpBvB#f5NTO2};_mt+`328)o#Rl|g|wq*fYye& z;A^e+tr-E2P1+=d+hn|@o3!bezkYWKOc+{cEGnJhEzmDy8>$ZG87J5dOOf873s61MFonNs`;-55)GVGSsu3v-8l>(U-Oz(n*1~S93n#g zia0BF?c9@5IL9it!6E5sBngf~wKkjT9eh3t<(UsayH&hQXfkhaPxfxwTuSA(n2{I7 z7e5X^dC`nNIoUNkS%h4V^hlnwIu3I(#%!LIwvZp`8QtU%fJX&7(NA9X1r&Aqmvptw z4AD?YNIUZ%7HPX=3l*bP8H;x6Q=#mykt3!-1^ig7h2Ok6;PV|A7Lsr~eEaN2M_)wm z__MI^wEP|_@fj-1V=taJ=Ma9}h-IT3<*1>ZIG7!^Xhr24fgeu1m(~K&kw&u9%Sid# z)29q^O+fo_q939GUaQICG^|CY^4&X&o9vK-G+R-5IBr`~6^<0B^KB?Zc`wAPrh6a+ zVRO(kjAT(jgk>WCX|sjOgzRK$!)GagiHNABqYXeSwq)8PMVgMLHslaIM%<2cYK>SJ zKWjBzTjx34X0T{&6s=-*kr{DCzZj)R?FIW9g>2@TK}LPbclnu@B$urW(<^$ksJCR! z%5oMmx0aKR*Y@>(EXnP>{Y7R1D5{M)xBcT8l5xHPoUB9(O5bm2NJvqViCc<=5Tvn>xC=aI;8@F0`{hH(;o9Q0vKVvLdTkWVm?aLWmz<$Eu$R< z$F!4ktR0XsYH8F~8&cZ3kIkB+I-Xoy`@kv3uP6$6f;@Q2YKGdxpIHZE9HH|R882Y5 zFK`2PeUCVnNle)m-EF+N(;RW9?Z+#@`XKRL*{e^H^$pmAvN9~&{7&lwA;{0QR^QPM z0qO=UTj*#ll`LdR!0Iq#`5l@Ks(xvWR8@DolAe{;-=-d*qgEj zRNQXliDaMA&q!rFK468WQ*AC~`r?du6|(leY4;?F)>vm^hf&9yw`;=45L;_JDq+#+ z`6ZMTUH@vJ4fGs)5 zCAXJ|@`iN3S|Cy!5oHTeVPRxpa^n<=3Y1%oC2NN0IdZacmhvd!d{^K4+F7hRD(#f9 zaJ8sNm*01|Jp+YE7tXNBI%csl9V5^OoOG+L+|){8p6aZ-2=PPjA1T(x7=a3|tO`-? zu7l%P>uy52!Zk*<)fr337U#D4?@7ko8NP4Fy|n=IHD6>o%hJ4KEhDAGqyPPavgI4z$mWKNYO$ADb}6oxpG-1=eh6E zLOL;Hk3RD^z^;rm6}=!2>T-R(cb>Zxvp#Qm{HnX;JqiUjAf=ha(iNI0{yh< z@{d8N3BRp5x{!0Bi+uqIY>a-zd&c5QyY z7V9`8ix;ST7^DT!gNo|onZH4!@LCjj76I!9s6*!1fk-4?VXp;knNJVL8_%oJixpqt zLZw&nh*Z7+TG~3lz|sN6HOXkD0Uj}DH?7jgyR0nK{!B;7sXQ~J8A`O{o>Tt%*-|Gz zflXtu0phCPASFZv&yVCnE@-lfD+?m*imeW)qpCY6VTGFF%|ZBX670f$y%UAbz@Dy? zrLm@=xJvu&8D6oMR|e7CL4MqomX1)*!zZcd@?mEbS(GohP?o6g z0=7Z4y2+KZ?}k(5F2Gkcv%j7f;M!DC=(K_FjJ6)o(K5F(hLA4u`&bws#1BOo8?~sy zDCchFcjMBd#I%n)ze}`ql?5o=ixpw^$FS{Ca!3dml%nS`HxgZP{Acd9lQdPGzqW3L z;LRx?zZg_z`h+}d7c5FRvUs0}jba+kTjlxHg6eKmu;9S8qhii@w`?en@?w-XozF&z zYvJi=^U2`GzDG5_zxveX72~s1ulv9Lz!AV|T@78~CFi;O(vWTd_B(xS+valL?|D}L z#Ghyh!Qxl(h~qDW1aOl~yFII#{G3 z$HWa^f{D&8{Bq_mbEq|L>5;BCPw5mmhpeq;tYtS1ZYvs3t07^Yf4|GY_jsP89tHadC05dq=a?M_m(J)0g=NLEA8+1N zWJ9#7S|(fG4Lie{LX{$|8)Phh%V0+9$pwNefLxa&70vgBo(nwI?>b-wgH=4d-3tJl&cnp;y5QrAQmRK_d=fT;d2h^ zWB7U0l{paq<&3z<{Q1JPNJ$^`9%ON({>kXnd6AGQbOt;3&nfJTisb9kk$y3DU$X)e`#JR zdn=FQ!uc3VHJ>A&%em1^OMW{G)(0Eal9^iQoxJ5&**hYu!TVP?;4YZUFqtty6m7ytuRkvOU@%=5}J0}~bPQQ>KEuj!_ zGt+L=4=wA2&paML%_|5ZF8Rok&$R#59u5=pk+qPK7Q-D>U49;j=JFRu-BzO$62-9T zgk6Ft=1&@wA{b-T(R`ZdgK#9ITsY<(a7fdCk}lhhThGfLjQA2qH20heOVKH{Ok%6^ z7dcz>x>4x90W@%SUbVN{llU>PHehLunCyIZaydTzOg$KYUA&R)PaTCHF($V_OE5yH zn&Zj(5{<`yTR`|n0_flYaPmw3T%(EQvD9lECtDm-J8ax|o%@)jixiLVP&k3?vcAG` zTtHVz2_c3P@cS&&jSy1`%DVGJkw2i+@k>bI`tkB1*EbUpIxgLa=P)Mj7J)g9A5wso z=m^Eav%bs*nv8}LqjEpkO6(}7!6>=a@>SXgA7JynyF6#qfAr!e5284a62dc&7X)IE zEy_kPCtVg!0XoZ9qu`x2IW2G+mWwz`Hcj%cZr(f^dFQ!9*u%Qajz62#R?~?x+8n%# z5F6@wz$+l2sejVpd)#3yAn~P)7X6qbMKr1bYiy>BF_b2Z;%bYyPOg$ujny+;tBwJ= z?6ca@z7+Yn&ttNRnYJvzl^VQPh_jXp&kDwG63s}9W|z21p>@80l6;+-lPX?7`UY(3 z!h1h{Z{OkZXYx0%i=Sa)65DH9ultpH1!Ji*X{}7r?kZU>8h|pyc+Wq&P(X&DPRAF-t4E$1tLMo~xkB$F&d{KO^M4W8ZZL>S6G<{-7AVvIyFdC=+yMqEAu}dhwmBb9~IhSs@$J zFUygsj_92C{+fRt-e#<8`Sf~Bdv-*uDU$FIdYZmP9HvH z`W7`TEZrR0dp%G96VDZI=(sn1k=(++`!?;(g>q#(mG{Is%0vD*wI`LD$VJ2<ciA;w5rpKwfi^G z-Eb)Jqw!IeHSfd~KCxm;e=GJ-t;sTGw4?0D+I9giHtl;LPYRpvE6dPQJgTXeV2z;{ zYsk`q$|WnP!kNh_n>qbnK6RXZb5xN0*kUDwtPhRI|5{7Yk%Ddx#{52AVpp?dRPnHN zkyuqh_$^BsQaBfkfm>xJ2kh4ygXtO>+aC3~ zmt;LeRT1ZzK~>93mT!Q|HivKokvl0m&fMm2qxp(Tcw850eX&Jc9Y3R^cW1bzN*;Q) zdG;dv7*%s75zDHX*Ai=+Gb!!ATwdF(!^CY2LdndY2g$D^t-p(foJpZ^li2MDheS0F zaK)fn)sbSPE=LhDhvSGaaYiKu^Vw8pH}O|63u#~O+x$ZZN-YziYA#ws1lq9JnNB#e zp#j^C?@jdo)OzRSmkorS4+AxWaFa*d-t_T|A=9Z=q_dQZ&eGXja_{a`NPaZeTX!8C z3V~FN#0TBwX4KSidZHz*ny^eS(0WT$Y z43nDQgbcEg=c4UY?tYr`n-4&&dK+T`1T_H+uV4U+^b_} zn!`Q_UY$eoUT+03+KD@pogqVeB@I(MJO(NS5wzZ!4E%h7%euqT5PBcNM?iQdJIqC8 zJ{5+XQ)Do2K?RW?8KC-DT)cS%k?#=#kT}|6=v{~uM1=z-X0gAI+rK$-{+p9yHg@&x zCI7Ue$xg93*0rywTS0N0FPBtxUmVBSd_@HaS0eP=3$u11O^@1IhQ;Lv`O+}%pKDe#R)gAe&u6bfTS zxE*rowlT2@-(Wezm0xu5{Sq~Tw>_L=>vB#&Z9H7kRD!Q9jfn9Vo&9B&%u+LQa$Yq5 zk1Wn36-8a^)4la@g(SY(n&foEta15>S;yk2bK%pINHUkXK1=0kHXiNMFM2Vnfz}X&#@}{OFj!g8j|@ZJ{5$>_{Z*j8cZhq zq11`v5<9$nD}cM1it8;J!k2UF1bQ=oo&43_G0V*qsyVexr=zR5gE^knTpFdQnSR;Y z5XAujGWzr}@E%_BdS-cDBh74t0sLhcvKrwHFNQvE6U8AE;Q3LZ!Q_>JFa#%yJn`Fk z@;JGj*toJmbU<^tGn8B1Ua^*G()*rMUUaCA`n_*v=han+7?sKeGA(=pEG5@cj&Mo-^Y zdaat)?A|mW{&BaDO(2Xp`ofNyP-=+$w|h`UlHy@jn=1k$$Q>L;$99pR22Sy&^|Yjb z(nD#tvasbaAA!w5lwvQXRGfH5*j>6wxES<70Fql3tCV0DEL|2z_b1}2FJa3 zzR6qzVtkwu+CEG2qHL>F9Co|jbS&j^K~yM5fS`DyR+FqO?810(D!m77HKvn|f-X^F zSM$X*>`e_i+h?Lg&BF#-qL61~d4kx^M7@gM8WeS9Ipew&yrln86xVu^PUdt2AAk(u zCD*Bl-7*K;`CD=MtHnDTe3)13%?hgmbT1JH3Vq+-&;~%Iy5Zf&7W}ew>GQ1Cqfqvh zuM~<5;0!{14=J4;iHDbx>LO^3E0u3T=*TlT@AyJ523*=5i&Y3wdoi1vjzY+-=+-1+ zr}Yl;6sw9N@V?dl{dDthDF0YI8?7;_BI_#laLZi7dwWkea}T3tgzAahr==yJZ+&J) z`1A{0HO0{qP~J0IDu|_4k`v0O>4(ppd^?Qem`t}FB6( z2=>GpG_w`Yf?b|r3W9*}%J#TFe@bKd2Y~It485}#i%)s0AV4$z&rSc2cErI6RY^#_ Vh{?rhaPSA2y0W%X@hwci{{qr(cGlY(vd^2MGx z9eoHG*Al2+nID5%W+GFWS3(ih|@!;FkZ+!}e%4 zkL|M-x5o)O+~+{!D0-|Zcxp3a>bL}Ci^9UMJc?>B;9YYN6yA4j4bYc1G}ysee}gu+ zG{5+fYwV@#`sDH09*t>6z47AxoA-hrIX1SW7t!I`l9-YR;Uu+p*@_4h@IjP4O`62i zJr4MSMo~U2u7$V)81wYK)iiDB?@2!meX>Ewal_ir2y1cd&p$(XtH)(V|4Q;V%DN;Z zaVSC`r&%lL86UCi{Ve9ZGIjVFT=uYwU^TQiTWEM}%LdER+B7te@O84{c>~Y$*umaKO;f<7(5A>UG9PJJLSiuv zkchFH&l__!u*sy!*+*ldboY3wW!A_e*WO7L#h?mCVUtO==u@L9tIs)pQjOBP10GZr zIHoEEr?OC7mYiNmLNa?a&)RWN1z!if$yZ2Ehv!0kTLRQ@NADrUZti!-%hTr1j{zU1 z(~uR5n&Pw*^qj%F$wVeou?#1N_mvzfUsy}uy)_D>RgFcZV#Q{Zdwmf4W!(*FBcjnf z4e_n+1a^XCDydisni)1};V^|iJ2n%;H_$yjBF))gqo8auQBhSN+e=>@)Ka{PQCYb9j|iVqij}Ke_*CYikxpcXkO#kQV?8) zXYPLmzgjwhKB7QEj?--m4}cGrq=&0o^Q8&RhgV0w%8sZVEaK@WC1)egN&AMb=S3B&E;%0^Wc6&vAF3Ds7V31AzrtTrA zFg!GThchN`dSzOk#9`aDb*`Ik<-s;sD^UC}zpcK9&@!Sz_wc1RrbE~M3p|l{K69eV zd{ohvCJ1_28I8+X=-DQa@|VP;n~pfar(yfC7Hp^4851gaHQ*4|_)aJb%9g@HGOz;DCD1}k$pw^?AF~VCZS_OoU=AXh` zzbJR|a-t0*G320^!Z~!lsz7u^YVH(w#hwghbcN4)LDXUGO0gJLQiRv?Qf!rCR~{P- z9>*4b8<|cs9Z9zyeoGyqK#UvLLuaq>Dmg9^M~EBE;C%{BNg_r+e*x(oiO`2E$x)i4 zey2^EiLe$a@jPNBSuYK|$yXFn_c^S*s46l{d2_qkHOMBB?9wdbuRS>oBOsC-IWxa7 zFB#@zUc8~OG~Z6=*$%C&)Mup_>MEEeFU4o+`(9yKit{|CSBY6l_V6ChO?b5D%NlBY z&qr5Eaf0knj-LXn&1b@o_(250kvFSm$h;r8I+IrASFtrw=D_w~T%eEz(;DX*@1pV- zu#XIFl2i?YUw3P|+%U~3Jv)Ol?J*NS-a=GDR4YC1K;otXD)rCgh5~p^6nimyjy=IW z)Z&kmMTKQHh1-RMWvOM1x-7aZdJFm9Z;qnpd1`~I2F%$#1_K^3Ie# zmfHUqn8+$HC@?7VRk%pDsV6iTl8gEAbLw-xPO+EFL7{Frzlnc}cd2*VwdRr7{7y~w zL87OlXP4)#Fq&80LDr=2o4(ytPyR3RW|AvX5-KN?4Eyu@?w%f=L!LK#b$i#ld%M27 z3xeQ#~Ckhr)EJz(cTQ>DxrrW2HEU}Kiv8wAmY?wnUr2P1E*ix5Mjmv*H zZiH@RYkOgMVfcZ~ku8}un(f+XvA#h6Uf;%`tkSXau#&r))PZm4d+&Zc-`sqkpSdb>s`Q3#1GF-n`A!&Bfk2-ZO_$*W%ZW*O`YLA~~24Okjjk zgmV11{IvXjw~v<1c%1_?iig@;p1(8phuzsd%iMx?yIiBZnZ1jBeEBBcS>5-4qBD{Y za=Y>z@mN~(^GO-~#+4r}&lm5vvAx>sQR3OWt?Gg0lYOwf`aIwLOM10(ZX=ax$)j|K zc8qXr%yijQv)r(p-G$`@{UmN#AkZ^#Js|Zy>x$tv{C4!p>HKMRXG?F@XZ%Dh*cNIG z6?-0&Ux+7+UFN!XdoK$oU~VKSPtC(g=jvcCpXiG%@a^@kMujz}1jzQ3Csm9kqBia7 zbq}8msk`8&*g}F><`Q{ym>f7Yye@1H%n;^;?2qk?1EDa*TgF=>;o`{?O2#H3XyiU; z>;rdiXnXkef)X;swzcv3Uemmjp3Jw4u(S8v(Dyt{uPo_Xj;V~t>6`5Pps#wkKl|WT z=cct0L?KU3Bv9tQ`1@Alta{xDZx`hw-Zg!TGPb%puY@2l+UmhcyjIqxW2WyVghfW@eLKz;0+lHeig7qGzJuOQY%oiRzP~AhrmhZRa zh|FlDM4wr+>QB$Ga3m&E>{1kzo!QULYOJ~1^lKWNq8^g3l-p9ymB-Q=Q$M8z8V(xy zJ@;uy=spTousRqlxR-Bb>S60e>MiP%)VSZgKGKmbp)99tZgR%(P}%$3KCNo%U$&SZ zFeE%EeAg5oy&D~VrF)=P3ayc9So5=1*m{oZB};P*Xpd^&8{A$UsWp02m*RBSpi^@- zT^H%(rNghYwGg#{Rxh!X)RKS2l%4(EhxCYb;m$V=q}O+RyHxbOwWQ>R_{6|8bh_W6 z=~{Kx-`8xnE!=Qt-syqlVF>d*!Z`v*(CF=5Z|ApAwuk`Cspq#(L%rRX-Cw;FmSUSA zeini9(uUFxNnG-Vq)%j6(Eam+GRBL-!F)gYG^%`ZBaq`N=-1h@jiSd*)FEDi5o7(L zScc#Ee6d^6ysG>caev0g;Q?LUgR|-vwwa@MUqsxOcLJUe%h9}u#5gFQM=t8SolKp) z(pot9jIUc4XT_VIW4UzMgt7agK-$3Xw~o-pGv&!Ne}p>(1l+O|5fFZdxpZ|E z9H<%%&X5$&6BM2ktGA4Zgs8D>hJtBMAYA5%Xg`MkPaJVB z;+*5-ZWV?qRv!TdB7$Wwav%bNiwYbyAN;9-Fz0x?z4__|Ch?&Y+~=a#CpoAzT_WAn zaEMLL)ctTPQRhMTD)B2fk~kqqw{1|OFDi+#n$9~A*yAJGT3^vdMFs9PERPKL@&zdz zA}sd;mc(C>{U4l;u{V9z8 zw7s^zr@qP?VM`Y$b_*+)x7O?+C)Yo8;6y>fu&k4{rv)|0$}dgFbM~P7hsb}?k+Jr$bhmf)w0Ci){zKQ| zt&5kZ7%lCej{fueXPnj``~U9A+2h}7!6wM@r-g%yos;7~v|*^Ce`vp%>U8y|HRbsuy&Vrae{T}DgNKf^)K@O zZ2T8dl;h9L{|6`jVe{Ww7@x&aMLGVnX5y&2_Mi;dGLqZNsA%248VpfN0VKa2rIvb$OPz$56jC1S!L@p}4yUg^yR{^Xb?`QA z2g0+q!t-=;yeA~GZJxrYOq+;I4<98-%h`oP{1MFG#gM+F_7XQlh8hQeOrN_Yqwr6G zm;hFaL*?40{@bSTGI`t`^Dkl|ax5yWBo{{PfeoYchn%9F5YoS3a)_`}J}_1&5=QY4 zE&7@)gnz;E;bA2WnqG35e+PkUIQpmKKVTXmlDPBVuq`oZ{}yGaQP_6=As*H7O_CbL zNjs7YDe0e~CxTNw{=t$l*QQPc^S|y%5B+!C3cw+u|C-@zYy}`0!P{Y}f5#1*9{K;@ z>7l|SVNp6_k_)=t^4JWg+Z@hTj@TD{nZ6ZMhSH9wepw-KFR4<#yDC4dvSlKI>c- zPUH`#ljYVCWscU#!^OtxxmpK3n(DpZMFJn0)SP?c$n;V+A}JOD$AWZLD{X-z$4kwb zM+^07D>!pKQFxiZ)g)RoI4!y?-je;~+#JUZ`5=jFn5_zadblBC2r~}=DOcN!ykk`P z?5XMhhU9zjP%7IEEZwg=NcMz%*XGsXO`QyROlGlV>Bc%0^IXqBO6aUb=T-NiGIF9l$$8Tgv2rz+s? ze7mxb(t$GIv=eBva_zJ-Z5(j+lc~0kqwV(XML%6a%2U zsy1oa0p|nT!O)xFOpd@kHtu}R0phD!bOG!t)wi;U}cX9>YAck?rC)1`*B-$8bfgGI;r_7&4jA(h5AWiYdpeOCu(!l31&AUCZ*yw?lYvQxyHG)WF ztbQDLCI3-O;iW1&Yzdf%#y;vUoFT^%EZ6n$8UOI0DZc;Wz-LssB($OxVD`99l)3WI zy24^nlIh%3+;}x@lqt}%hA}enRaxXCifDe21i>2HmEK9tQuvC4#FMAmOzk~vMaLy- zgXfE#rrgl$e$1Qx=vJGVzp7a{yf=~92OSWqzFdFzvU>Rh|C0D5W#qhq_qC0%MxCrvD?fg?YKGur(0*1iic*{0lIdf0% z!SQP*Vbhxjbk|wno;ju}{J}yCPW+b4lk+=<& z08F6Yoo|gJ8TwN@_doqdsZnLT(dH)S_rshN%#mo17Z7??n zv!z$_SX<%~jaO~V{gjc!v4HoHuPJ<3n}b?#*wW-$n>zzu-4Asqn zL`%P7=KUC6&{swNo0Z_;3z%%K0FzJs%naHQSuqHr<+;)oQpClPBh)mlb2PX+{$5h{ zt_}Lo-ro3lJ|q%h(pR!_UoJktAm8nkQb1=Z(4@dQB=#4Y&&sQV0Gp0sBP32TpW((FaJ?k9R zf!DZ-i7$TwLlk%uLyCi-<#9^*kY7|*AIt%)Gnd&PG}`M#%b{(0_0RS(nsR`BnAq^v zD8pynGFhMnZk8?IyQu_EA~6-Li+~H-PbD378mFJTdU@^(sURO+bFt<*U#*Ee9r;n} zT8&eN?wHKjj@2woQ=OB2dwWcyFgG!{#D| zS)m1wp*nPT`XKcEz}iMbA3?f6Fopb9yHF6T%?_Gj6smhL*pd6>@MGqpmVM^Z%*c!K zv*or6g&B#}GtXPt)p4MuL6I=7TW)I7r5a>dqm!%Efh)o~{rx?&Yif(@dRG`)%-ci? zra2h}B}N8B)nB)>2~v)%!8audVZmc!H@st_7jJ0Dyc}lMi#C&zdk~ZeLo~6KbiQ4? z17uKTbs@r28r94kt(Oyd4VPvj2@=mBMeiK_h3JH?U*Tt`+%>Yyv=TIePZzo004hg6 zj2o*0)P8+Ot3+;81Xb!OtW7^RWmjoO6p%m7Fx>(m^1ypGFq6MJ)nqX>0ty)?0i`GY zhb~!i)MT-P(&|AE$PP+&Bz&#Ubz1OU+DJE+gz^5mTi`=|u>XME*KbLZ&BQTW)Lxe6 z=({{y<%uDU>y%c^Z=&9>nU{+1^D@d%M>5*_I_Xd#^lS z*9#=a_seH5grPGuQ}YQ9jHlQ;94ET@Qzj2}_#%%9V;wLi)nB-qp?3?M1!UwdewZ6^Y$4kW*tqguJ3dOtMcl_M92?%8ZCX zjafrdA)R}I5mAu_&EwoJfJtUsDEgNeKGnoIx-vP5hM|l9!ouJM@>FX|Ymnw}O19f> z_9pV6&KmRH$;2$(W4Oz0IJfHHRI5T!a=oAURk0eu+uz<4)jLzxEx;Ouq`abUaTqk9 z(F^3b4+73(4r#295)4mqE4Oi52J|EO#BZdmptlK7eTD4B#282I^b$R>O<-@(z6g3v zLS*dR3jO*TgA>-UP%YVcN_ zN&06CXr_5tryn6|w>poaim~~o^(4@mG9B;QYed$aN)Rhfg(`OH;#9V!Jn+atSz-XO zm?VY{lj^nuX;4(g{kXPwU({-LHU{5azQZcndXwhwg+|7jt8z5!q4;vFb;w> zr(R8%@A+Sgv~7q)cNJpeFah&dnQ=B{e0TH3H6-=S0=@P)4i+*5t-_6CBi3cA@ClRi z0+NKjT8WA(K4}Ylbrg+E1M;b^Xk@WEdnvi?e9~+wqg!>kFLZfrn$2&1z5;z(`MQGs z5|!jm`)8!!7c_7Te9X!7@d-Pmw8=UyXdN#b5V)nd-)`$mdmmZBtmmY0nZo~J6$PvC zl+)ZEJ>F2gzqhl58n5S#oH%D;`dS2be~f!MA|*0$(%wg|P6DCi7$HOi4~_Of+IB(J1Sk}G$w6%Lk%0AGi31mprhgNOcYJN}<%T93VCbJZDJJi28#ytq&9w0VYgi9*>oQoCs4Pa&!jvkw# z3AFR+X%vh8II$sxLRUG<97qX=wex4*$|AiHr9taVL3xR<$F4_3PFObrVOk%WL2-Ml zysUk9EWG^m{*@@=C_=GJXhZNRQg8*pYMC6dILPW-U_tD7Er(*anq2Q0YF>C_9Wm*? zhl&X@dTs_9K5z2nFL~mN4vTm={8!O#)YyXb}AUe1otOUDXmoWh2v zlB82Uz-LNQj))V$S#_e{j!zpFN^Ee29JkX&xB5$+9$!%QD@EYcSo>{EJAg(dl_xAx zkeM}?dyMv_(#Nl=Qm;!rDRE1ha7n}DqS_voPT0Y68g!~&73*Oy*KEp_pW08++ST`S zvzU)zx^O$Q(;gGIC)Q!gchFQm7CzHDm~RZzq6^&Fp(VR`|27Nx(?s~E>o}6R$4fP+ zr^ceIAcR&H$hlofP4gs)o5K6{H-A?DE8h$>WK6eHxe+{5CWKB?5_oE=O;hO#PFcB4 zOXJ?*YOaKu?tK2SetmyrjHH9U3F>I7=QS$!b7_20tqt7obppDrMo)K1RCh_5X>Fgo zIg6Ta^U)k%+7>4EnejYbzf9ir0O z9?`@E(iL*9#(MknLA$Bw==Xw2Edr*WKJp-smLjvePiD+76R2Zv{>)Ly40eGAH)7NN zEX71*-YakA56rzy4ab##?v2{4QPF6-8*yQ$wD&z7m*^@ymW4x3l+O=@ST%pS<}qp4 zTTHCrEOup^7-t$Zf*#t_xD~)NL4@Y0@yVxvgwo}KpL^+gP<+5PVw{|%c(0UAkgo0?g{pA=XgYTosp9~+ z=$P{&IKYH=ZsfajWb4DWYoIu3Zt=%@V3N8dtJfg*3-xd(wA6`$3IJXht&~kiK5!UZ6XPGI#_Zl+a*`Ibmxk)5o__C$rHp(! z6Ld7ABr*T=^>B&l(yCm=?pQDQN-st}j9yI5?OYu@&aDPzQGe~*N(-sr=|rpv)EDkP zHZn{vNxB&n@4nTDDsuiD^t=vyC&(Q5>mRZ#l^Ti8*H0#|+^$GPW828?CqaE65dPRc%XLEPLqx?J9*DC}I$amk-bQ4T}tPpVQ24jMk0-vA~uvbnX& zI-rzICW*TNZpI$@sQEPaC!XxPVT(Mn+kj^F%|dEE(~XnI(G&bqY?nN4Xf zt*T8%ZFCNL!Nd*+|4$vU6jAmci*ke79{~58rY}G6Ce?U)o$u5lGtsJV@CjUI7Detl z*y)PZinp_DV{-HG&(ae3mmTx8gAL;b9;^Bazj&XC(kc2!nnA%+?5@l^vb(G|hM6na;0^1W*svf>+?Lg?^xz`#4E)o?uva-QmTs5c8P-?)MT&4Bjp z56$gHk=?TaiM^6oBToc$&|%qev&`$TIJ`uda5;TOvCz6fK5v0QyT(>}1(PhtaZH0pH2 zo`)^`=YL-v8L+bm;{7yEFQ3JVh4HH*n)zL{X+v}#PU=4DmAXx-eOT8=w(X*49bkR{ z&b-m`d1^Pf_noa58uQZ`71RyIXwGt@^0W!`?g+?y{u00zO@^{d7Spa`my(*eCW&=m zM@jnAMgq3gcnaR&eBMMDz|k4d1IiL6V?VZI6RJh-gt!(2&+r%_|6yua8WuP3UBv{Z zV>7B_m_Zy)NJBzIkMz{@FVW`xVlLwTPLC??BouDV>_g3?h0G5knp0`sA@xGJ(EB%S z{qrD}+(x8aFB08a_J_b@Tf1Vkz1Vs(zLPX*G>_ZAe1QNEEW16%U2{e#xlml<)t zYl&%(WkJi!EoFD4Aa!K+Og{H_qdb~A0ZXQNt!YK#&hH1Z9|v%0-}&R}bRKXxWKV2- zq2`DVg%={swJ$oJd3rv925HQT3ZlmC&<@YB<7bP}JV-}kkWd?J9nkP9>6qb^6kuDH z`WxZD#h}n#FQlf>GJkVw9M{aKvWpkSyik2iFwI2`?0LySl7P>SyVk0?Do;iTtc?I+~5uCbyZQ9GAoQT9mA`MNXNOtWoZI-yUl<=PA;ePhEKt3%4MfK{5QMn7^ z05>vp=%R2WpOdvgL8qK9`;_FGVFU@2%Vg5hfM25c$&>PwnQtlYzqLd7@Kn^Z+oG}Q zb88udg@e^#qPekc9xRuke%iuo|M0v*MhX4^l412TMGKS3P*%lhT31xBYQZeO{g5%3)gkyyU8_!y9axW&G}HUc zq#VG!#j9MEmo4-&=C2TBdbL6jzQ%BZ=#k&g3~E)n^lZ2~$JhqKi1%=a{5GY_0!b4v z>DP;~%yta58DI#_~eGzwlMHDjQQE4;cc=B=#1zb8eVq zBj+}nt;B`CFo$##VWK1XyYOAw+c%lzMfyf24#2H(f%Z*F9S=l#b^C<-B^3ytuLtI7 z`YvInjlC~mXNKG_FR>%?qV+Pzu?acecRiSAaai8z$}&3k?{C#s%vy8O@}^b>a^$Kh zB19$#z<3c#v88OG8QTJv%Q?18xHFzeR+ZBa(6XXdd=I~^qcN&{YQa_S`Lm`$wQ;m#6>)n?p)L{RtWCtP{`To0&T}dtYF* z?1MhQ$w^rK2gRm>YGl|%;i@m^zO3$M#I3L3J%%lKjhqOmna#=amQmVN&?QQ$OMUgl z8N~Y2d0`@|d+M7I*2LW?Kf9^lGIl4y&Gh4);*4zv2}eM#CB03yvx`H*u}{fU&9)@q6i z@_G0MOZ=zIG7xgy9LKAKE-P!a5{)*K%zLkvE)9I9@(yu)Tt4qGBx_M%X1}&wvV5*; zpe!$>o$fjY{5trID7X@^!%vCd{M?k#w=QlEO=-w$zfO5wd9M9V0efwiY3n;j3lPlU zfM3Z4oIF^h;)q|L3a-jE54#2E4z~EYRisSqEO(|ve-w_zd+wF2{1_z$u!=?wh=>LF z737xp&Z!v(~PRgz;blP zJJSC7m}B8Ly5QH z?#=g~XZ|%w@Ss*3W?oBxSS2oaB((Zv(%6jccktuSqiBSWcAhEd58_i;?=E}i z3f67~fjg^nQkW%{)4K9kJGEolC@7`t?jt8bpCZ$CdNLh}3Y70?Su;F0YER|BAuodOBh0(D=M-VLU&#aNL>P#m~Y^U#1bM3%9K4q(Ihh5S&s<4yCksT-}2 zB_CEz_s0pjr-t^ow%PG%$Y|i-jOa9CUfPG)ZF!U%x)EF*i|Jx&L85}(^66DfS4iw- z^*egSjvW%)1hF_YAGl{MsC$MYf0-Rktp~Z&;XQfEbfk~8-`kH#Z#axednH96!(ojX zp0`$Pb@=Kp3}@5f{Z)yKba)B55`co=obIY(4H_dqOjrn2m$Gep7w0P97+jWWp%yR8 zHyZ7hn#x$^mXsLwE4-|%SoWpDNBv5j^;O&y)g_+kgZZNjZpxQ+AsS$hBu&*fif1`N zldre{?%amU>?&Pw>gC7<&{n(_(li6q*r#i02{R3zmu4@$$M;Ar+(ilYElu@GK80!|iOiI(N zeEagWuD>~%ef{e&i;%-udMpXoUU}1N#pG7M*g3%wEU!I}FI1v(a!FjxP`eA))!BD- z%y~4H%cf$kE5~Mms#G{7ZpmmQA2nbvN%fuFI~B+_4Lbf97kUztB}V7P#A=fD;YQ$p{U}Js!`3z@AGubeRqRPOb|C!#w6t*pGtGC)KlFEj=D{!J^$(C12ga-SWehj_GcQD2 z9NbmXa9wWvS5Lgi=huEDqIBoK{i;U5fx0N4dHSy2x<%YeoP@*iyHDv@Z|T^53FP`w zaQTsR_T(PA9qRi7x^E80<^7#}Kk%wZoVK>k7-puO^;;ID_p$u3-0BY#`2zc*rQv!` zOu_Mve>{tIk5xDSx(p*=8qrYk|2KzT7IyAbcSio(&Hwlw+vFtSKhkG+rsC1^2Ez`P zZtB0@Mj7*OIEmH=SXY>JFs!?wa!t@z)fvy(PX3e|hCklXKRt#@bu_B<36Y1MZRh+U zQR9a_d63kCLi-;ds%sU_>G`X2S&!9~OrKXqs*&R@!hH{`EhS@Ptp5^;>KKsAY2#f` zD+iG|Dgwz^HGR$g@qTtx{Bd2fa@x}U3o0E4bAhVUWnD4-GXiupm|?UV-h)5+Uyto1 zOcBE1>oN%bzXbjd<3SN??$KOz(Og@g|AwuHtA53(++Tk?u`hD0{bv+BMqaHVRW^wS zUpN@#510${?mX?-QTK=O9QkZ;j!=5R^w?^mBYhZ_Qu+)lKC4}+nud%ry9A~ z6S5gzzP@~WssgD3U#}uy@lrKCNsrKBiB7(;>*X)vvk13R{3j|vFjg}Cegt9@>~W`~ z#}7R*k(e|lk$7m#ssekjwBGPhfuLprER=!Y8_jz^*jyLbKd}}a8vWg_-SHj0OADW^ zjiz~(*_(5GZ~TL58t>;F!dtPJCRUl1?QWI>)h?cIo@A?!qdtxNur-@zg>XS-T!;Me z{sajSUm=&yH5EStfYv5kXq%0Ef4~Fe%Shc42Ve8e5yhke4l#nc;VPf) zeyq*lZB(cqy9+4^vUX^KILKn5@a^R=gu69^-_4!J;_#N!O0IA%4Lr>Sm7~kmaAZ7* zSr8Byb2j{)MVNYn=t0PLQ8ePn*!$AvOI#t)&1J5ZuD7|^pmpn~&BQFqh_x-`_i(1L z2QS7&gQc(iNtw;lH5+0??ScTm$o1jt!F>>a!yTuPf}=CB*v_zUNd}+wyxYPcnH(m3 z9#P9RQRpzGznd%<+<@!9d02MYXZp9Cvkw@=s1T= zjvt_J7N@3O(|02i`cynjYEF!CnVhup>ELOax?c)*90Z8%Uwz*`4Q>yyP_a|ak&U(1 zHM%aZZh1w*=;;M|4jx# zYb-~78_W-k_y3G2rFkl-Yq9!oO!Q}nqMR5Q4t35l{C z{?5SzfKs(NbxIK%M~*rApc5qlp)5CpsuRDl5*~LhWp9eG`I0_018@0h&4WZZT94|R z04?rwP{B1?_Hu);Xbn@sey(q!xC^V-=l*AM5&rqOH&djc@s2*RFM^ znKg(Kdaou#1T6O;bj0FC^>J<j~kdj+xtUU{d@#NFtxZc%UGHAo za?70AaYzK$l&|Ii;3?W~(dewip|lLgX=WYbeVylbWpVOs$F#q@w9p(pcRw}K2N7+G z(0$k`+IbB@=+jsTstqoTn5Ox2zo@@5{_gK?UK-7GF6ff&pK|&Ohsv}`JF{uDZ zCuvv1{E9c!Si7pN^HIC5mvU^|WI>5=8KSUR5$V5H79|BNk{2r?!m==bldV}jUN3K_ zu^!P%8%6y&gV*`30jMoF=-_@kginp{cRr*nabvW#S$TSAn>eCvH%(TAq07mma2OEC zr8|_J#x2x4z3fp?fjvIG+jmcqsZ4QPBxw~h@I6I8H*icVUMW!Znr1@7CR@B!h?ux6 z=FD-aHGj@p^lYWz3=b7z`{v8kn=iayu|vc$3$s{(3wR_|Ev6ay4}h85Ly6fTdOmMO zxoRdg7n|@!+c)B^i3_6*K1SwEG`%|pUepvy`pKhvNgB40VU*-AxDBMghTNCtO3_3u zfilRBHl5Y0z)_8avusN4&qBV2idGdnNJDH%zupZeTTJTU86BdbR3|GoCx+hF9SPI7 zcPxnlQSq#OdwD#Hh8+w0(!pEI?wfkJrIM(*PGti_K6+~$tdk7BO@xWHyD^2$Xwj!6B04~A zH2|Hmihtcl;0QfS;?gt*xu@K2|c~y6Z!$T|EfQ>dnur}KjT zHWAQuEF%PJ@Oy_J&1TrCurF($L{ID_S*m*PE97yyCxD;uTnn4Di_H@Ck%as{Zg@b2 zc*akk`(ijznb?J`kuzS`#gyo2&UkyThEs+e(%eu?`7n=cEd>rCk!XDy z3AIK)@OF%9UG=iScZ_Nzl(Sq#atP0Bv@<243LR)#gK`U+KZC3|xOqn;97cs{QP2!*D7e{gZXhwusK+N9LVzQ5RjuO1YWD*8b(OMlt@V-&jFw; zoaP!1=@ySz3baT4w7*!x*le5K4SyUt_uutpwM z$3b^LK#KA<8*_nNZi7p(wsWUAsUoIm#TU2ix}t~)Vj`~!Z#Ju#v>jOz^=K#HhK?u^ zYM-F4vu`f;Z_iZcNwqc2@cRyBZbqf>bxo|Fk-Tgm!edL0Qe?y#k)cq}qEn@Av|m`! zsB)#7F_z>IuW2Ol(qn>%D%E;$SGG`by`D}W$0jDn@_Yj_j8cqk)-Pk;U=0mc`Lv0l z71+7|NYgAD@84nc1)m%%&Pz(_f_)UzK&yhMi;PxJ5r9moapr7Z!YKzLJI+o28bdZ? zz_X2iN9)%R;5B8r<*THf4CK{{MK0*q0#j|ZncrvS71SU57S9n`@P)t3o2&G(#A{j92wLT_E z^GgJ{+{booob!xfkQ~~Hn*!sVu$=rBo6E6tu3qG$(6poNu)p>%_jpV38TzCT+YJTD zU~hT=UrMFTQ26#UpIYi=@ZexiVWLgMgar+4(v%b^Ww}hh0>5Jh zDW%R--sXs__D9S7`jd{>EMBUgy5*s%>Py)0i-jiCFe zTmJ9~;J%OCJTEXyypkx$@`z5|G(Wha=9a6#M{kU^70xK6;0%cpx?eT zVWNcO7r!8BkM5BvOj4anPi7Pi$r}Y7Z+2sRB-gaQ-8&t28Qr{MKV}&eZ+RI_aQCUZ zTj(MAJTHA$n%B>^jP4eUd0;u`HBb@3`SH(pr)Yxb?%lZn1b3wPS|f$hkUv;V{~i z(?2F?7W*9hZmu=VbzpO`1Fv!f^JMAKk3VwXW9WqCoR2bmi44KVppM~?_$Cn+G&DNK z*4|F6vAM2~0yHGxZg-f<|4Gaf6v*vj{yYJJ*qWV%mS2;+p(R1SjUv)ZRi`yLQ?hs& z{3^G0`1JivDs_!_>V_{@`@|}BD9PYgzBmS>Y2M9Z=#*=7~{l~5k7;3#2 z4q?W?7v-oYXlJ@uaT74cs|!ZTq`wOSaQ6&#=VTE~pUfkhR90liCMI&`dg~V>$N8LZ z#*`=t_Uf~qEAM1$RqPtvS`M;=(HDX!O;ZNuc((cUBHk95qvp!a*LX-SbTz-n*?Or) ziKUh_k+1GsBYy3%><)`e_4OEI5B$DXLdP)Oe9oPFHE|u550Fqg*GCjGBYkb;Ot5Y5Y;Bg$~Hv51hlM&)?MN0UdMU*YQy|kjH~X4;PFKP_>&Xs%K%nm> zMI~x0-JDUXv8gD4@=!}E<+@jfd)VPAl;#ot6`a~bTiV&2;%cXR&cFt{Q zkxR4J`_;jJEENBU6Yc)I$F69kL79`Ei~egIs3theZsxx~v~6TOyj^o*Su42MW{p&z zTX#>GZNr|vRky;02fuNgHxeam5`2`!BJh=99r_==%=6hQOKx>63b;!s!4L0RTUf_ftc)iyRG zlN%l7b}=K{>6T#{KHC2@Gv0pZ1fiP};PHG*k&ZV_1>}h!u^_qwTUaEha@oc(`RtOD zW|d1kBy*oY5?Ylm!fzbfmK#o!C))Vp)gKz^bG;Op{zwh|Z}Z_HSsMm$N`gJW z3gA-^rB~j}_~v(NGD246fb#~$$|TYDMw?E=pIy>6rM74Z+S z(0@udGhR!w?jv>aNNctT$aoE{59K{0cYZ=#IcVljgdjq9V}>#fqymsJ3={-ipN^*z zKGlW!fj`WnO*1yNmRlb{9K2!T?mQ39HV-C^O@l@bFn=nsqFiBazPl!SOJo4o%4sDgATY7H080@mg@vQz z7AG)s{Lq36AKTDBA$sRb0UMQst{#%>YQp6hy`IqcUT}?t&17=5>TZXBC&JdDY*z2- zQVSN_?P6rM9j<^-#iceM7Mn;o!J9TxUk`pfq&quKyy01^q&1{W^{wDawg6S$Ag!Hu z6}_6{R8JJ)i0DHlld1832nV)6EGxKP;rtNl-e5amEW02dIWYt;UBy{k-ys{29{j+G zCNzw=8QdiU@&wDgWRJdcTBJo7lmu=hu|@Iq=G(-k>n-DMjL^^mu3dhdz22Em2zr1$ zJZO^WYv`nkUskf>lWy6qwQuBQMt_hvKQ6>hzA#InV(5dH7bWyGd6P0VI*V1|3!OM# zvMxu>P(;mf)h|Cz$_0FkLmcsMW=Te5y>+YH#qRd1%y90kN2|);)!)4lz0sdDlClH7 z?oZ{el{XK$0I{bxpF#n#hH0%$YT{l#C?tlnU0%o-0|A!<<>aUcqX2*Ml<(T--cr$9 zc2WS82ij{O_UDn{Xn%uCgj@|mU{s`1fAZI|Q+I$Ycw3z2TfK>^<=Hqf{8XY{{oM1D zHPFZeXbWk-K17SDdbc8Q>hl2L{ItBp$b0f4ti-M5Y|#Og@W{3#jqQ+;wFoK#?8S9_ zWM{#j<3c`k>I}~_C0(nS36uXe!k>FqW$H6JB}-FpHlU3{SWGSOjDT?(xX%NWB7omFGU|oA zabVpE*G|W@KkdbCjH=HsB($;tDJo7|HEerQoOZw2)y>j+d`@95zM-V{8U7*+kIL zY-HdNAAhd~YJ9KozAEm%{W;8e{2SM(iKaAc%zdQ&jnkJz;>YmEBumsmN?}+IfjWv1|xcpKO@HXAk_N8Q~Xx? z3L4f8dNwoz!X;iK?BRF}AwTFCv|J|Gu|>aF4-|YuNgN3?0KpyX7{93(Ls*y^)eUXk z_Ag-`NJZ+SzW1;V`X_8-!LElV3Mgav_K?W^lfzxj86OwPj}e2O5O&d&bxJ$wvM|CYv7Z|I))-# z6mM2k9;@;96=aJqh6WN67AVZgc)hTw9;D(qtpd$#NG!O7y$3CXQPZbfUm2K_`K#t% zD`0JLb03r>vMDlLz-p@FfzJ&R>aQ091GJh(qo29p{gG)^e5x{r)G(np;32Ly1uzNP z`6#awVEVrq#-@!)^cI^SS_liEjur`UQ^)i-hZc~PZb_Yv5GA?<5z(;J_?k2Rh z5#hOf{$BD#sF9H}NGlZRO%Ma?HLrI>+V=)haM;IkBxG*HLPMUDtJ_MFQb zlalCLPx^yMGnZ%jLJVCn2)t9mS|W+-C*s4!;VKrMY@~MhBWjbhE2~s1% zMMG`Qxs95(T9M|gHMd0;-v7lDFntPLS9C%kq=L=9Bv5;Lo)=3A?TNw5D)o{t|5?MD zHqBVE?2Jp=?9)$y3dt|`gUkcvF81>M{a+dycy${QB0k+MpJP~{y1Xsw7OR+ zQNw!^r0X#K?ufopRGVS3Qdr=pOZdBB|Y^-pB zB3((D$KETLvFF(40wPJz;Je*Xg}`PtPhP8#nd%5Dxe@%kJW-r=3{TsG0dD4tpuV$- zblrBhE$dvxtzH))P&r+nrJ z_A?}W(R#0l6W%N0TOwva&9W@hDtx?*KghxXxF)P$`1w}hZ(?eLA;6QmblP6-KDB*cg21@UBbaPlcHjtDq@1mW(I^kl?o+|F`y(5QZcr=#0 z2B;7HME+Lb)NT%<0EmdDDV-zu=}^|D>~U(sPCS(J0HB7p4{GCag`v*(VXekdB?a!( z-Mw~)H&L1KnnMO_IMWEe22GXkZh7;r$M&Eb6u>S>VsBM0$n*lbV$-aIJv;6yVwoR^ z0n81$gLdcvSdfk4lnui*EjC-C1Am1AV6zVf1QU3T2;3IeQ;pDei-}kzi5C%VZS^-@ zRZfkW`t{Th#T`e|Ma*)1_h0Lpp3@~(o3c+mH3fk08V}>H^WN%{P9UQN)DZ7CaYfzW zf3JnH&T2exJ95i;P5>thq3_88mHFvqI1xb%A0gH@Vgx-BbXUn~gvwasc`oe0@0R$z z)_lsJ9)B@Vi5ezQCu?*`%d08lVO`Zp6F==;xF21G#teipx)}i3fx*C>yK7L*DXPS( zxWo=n;ae)!Zr>KwVKs+;O(N>o0b<&H-UaX>tz5vt5M{$a?%^?4R(uxrdf`DAR``cRAB$OyMl6iqwJK}ma9UY1-;Lw0oMaSV6V-e~sa7B_>Sk$cDkO2KkNU>dF518>z}65o1p zZ%JHIm^cH@1khOr|Pl^l_w07_nJ&mLZ{gNKSP4iV4Aus>B+^XTLa>f&r;$uKX& z@MC?O>fiIokS0%x3o`P97Nv)D_Wr5Z?5I)F^*8W{1>w65QW>kB*8C=zd%$z_kf`2g z0bou4{X5zA(C#n_xH9<&>VGsj1JH!0KQw#VUhu{AO48cd-gh#;+Wm0J#Jmbdx`?hP zipxDYj!UnWI5TkvRCuZh=+J7~Lg+6%)>Eu(>BD(@phE|_I-i>VvZd4?<3tbg0`Xq> z`e=ZDW;4LYbA(PC#lzu02p8){ev9KHkfcC@g{R_STa2Q9i41Wot$R!I`)>mwuPF5?E! z7`JnV9&a;hoeog)2B+L{{%LP)q1dGvoBmEQ{t=4EiFX%-Wvj7_U*|TQVPKH$aZq4{ zi24C4(9bQQ!+qY9CAvb>ae~)Ca3oRLVz*bwm)K#aDr9NHXDbSIR7ZW-UPT1njTbgl zeDrTT`P7c>8-$8uK+m9h{S>(V2cYiX!KigShWD5tat0Ge{7U1_o;#WBXuXklQ+Bxs zL(`NbfYWe4DB;eI`Kd|5yeJS;f3J3~>ocGRksI?W3GVg}mI|6-G%9;+K@*7neE!tC zRall-(`3$x*XF%6=_cHFRTrz?!kbsW&ExrbC*ax5VI5=ZPaOzJN_Ken^cxgb!MC}T zUND=qhP0zx!Gl>oqEKh903TM=qi!n}ZnlAIBH*A$0p)&xrWxd+-kO~Pw|{??d=HB{ z%4*2unISMs3&H}45KE-&7b;Y=!x4*tIH@qAYN9#(hh@-Pwa}%)Aq~>TIKO{YEqg(d zf@ZpcD!(7-6OM(I9qqiM67B(nArJavEVvqRzYcE*ql+aBB7NC1-Vw2Y{rHKI$(9u4 zrWENpoZz+Y)pILK+@`AQ%|=R~iSqR!mgh<@_0NjC?xV)K2l{$Ji-gbmSmNg_h+9kayMV{CMb5zCg&f=y+`O@>)Kmr zO8eW>F}+#3+{5<^YYLo5E=n=9H?<|3Oj#2^*maEY{U~wMrcXSzU>cZ@qc96HAe%y$a!2w1BQ5XL zN4Xk>lkGZKH^l)z;Wj@RQ@GXmv^KL zfkPZNg||IDEbpqS=WAY>XS|9&KYn%$M=|!s)|0{1phml4(${o{`G5iB{H9FT7k#|w zTdOCYE!M@>E=Ze=k)Gp>UPH0jvu*y;(}j9bz5UpfLk}u?<2JfT0CX_pJwo77<8>9i zL!IuPdp3Fz2$;Pc%L3T3>I)cFjd_w~BMRs9gO z<#k5f0h^7)q}QE`H$m);vkvCN$`@5P`+bTBH`jcwv)={YTm-tZsF1 zT!6?K|8>u^;<%^h@ZztJsQPvnMX6njlND6^u2Xt*41}I`YN8TL&GS2FJa^V3o6&CQ z(au|Xt+`#&LFuhnJ~IMT5{%1_TEy7*F~UPB$B*h;l<-5ulA>p_wH^q~ePVYAp# zAhEdldJXpNfd?2R_4vB30&|`rU~ZYXnTQxJ&8h*)pb`p-G3_M1)7F~P8MGpj?2xqn zS=4e$O-K+FEfR>br#~-$^UUy1=5Lb9bj9l{wSnzfm~|p~xHM((!G)SQ1hFMID8$>AtV_VoA$+-d)=IrZ7IX%xX#!w0;@_78JflWdv+65f6JUlquP z{%n;z4Jj?(L4V0gwl`cxU|9E|}JgMxW)F zbGc~{}prg-r_J5??bIz)68etBWqIF0FT(stYKwS6>}pOTI?6~o;R z;j*ljC@Oq%<2zRhe+EkI=f<|X522%dB}|1XBoG&IGv{@-{M5M*=b((|MSkMUl%u!v z*PoF~xEec<_*zp0MelTPWS{V=(Z(*seTT50e7nce5u@K8yE*+yLiKy}z+e33Bp2Y_ zzF{d~W!m=s^6P}BI6=kr-NqBTxsU~KL-r6~Dd_iBF8$f964;f-4w{iRSi~*oGfT!z z>xD~7!-elPP4Q`bXfx;<_m^Y%`*V{PB|_V+0FRX(%Xuo^y2-dlu|eVkF@SdISYJ-W zjBwPTVM)lHSh7F#bcLN58Wc%MzHBy7AJGY5@fNI+80yWZrD7K8zrLyvC^3(j?i`AC zqacilzAChnLk=Roxvu4`EwNtAsXiEEb0DI$s@G#J$|0{NgML7 z=2t_;7Q%Q48jme`IIJY!ReHpAU?&`TQva;Q^ZKLMJ*atP275${%8_d^Q=sN(f@&oP zXV**{pT-Hsk#nV7nDUtFn%b`7#R6B*nRjIw@1>RiTlddNJHvj^Q)J3a@y40G=3JzH zNPiF!rh0dsx9&q8@;@=H*Nl`EeYIfE{2Kt{TyGTY4|^HGrwH21fThl9r@a=TL}d&F zB2$nT9hdpkfBda$d$Y1;zI3thEZEj+Dr4zZM|~```K;4X)g*n}Fi)b;7vK^j^05Gf zi{HvyLEMGWZf6^(($ z)6>oLj{EtAeIp0|V7%R6jNJF9oH!gQSEPvE{5!3jc=VleBDPExhKCeA`kJss`Cwr3 z9L1n7sZZ-EcSq-D5|2rW$LY(?2xpMk6W=EZCB_Qo9OG|GgP9A)iQ09`^kKp->9pcc z#=xkbiQeIV#tV{pq$x00HF!WNWMBg1<2%tG=^bSDW0qoApyf<+sT+2)7?!!iv6Tr{ zKTnHN+&ZWc!+*akK=z-z>Xd2Nu}RMoWVF*Aoz;;Vhwf!BQ)2ocS=}B4hZ6sXcuFW<@*0Qi64z*q-wBwy*YAyPkM5r0F}1n%nGv zXIXWiM)RG){iEV+8+Myt(V=(-QtSckEF7ebR#h^>XolClS!ah~UmZ#AE+6KkX3} ze1T7*)ZM@IVv$V{m~zacKYaB+gVKCNHj~g+-oO(H-`uVIRJ)u!oM$DQ zYFW6@UGRp&YSDKZqk2qY1zGY<;FzS$1g}>&P*sMTm7S@QO1S4;u|%~OI_`fvcGlm& zOfH+|cmv2;!}^qUbm)zZjq5Cz=&#TIuOKf)wdyf>cpaxq3XQhVP`Ip~2ty zA_R1d#AG0a6!C*a@#pdT0C$GuH!h9>0T6@=?LH+D1ZH?gK0v+y_W`x}BW&#S`G?`$ z8(10d=%}cXKYwZv5Rk$TH|*^%(a_OJW@Z%7aBy&>28=aAyYYqe%5)Hb46J0Nq}Uyy zp`j>VsX#&qG|j^%1wm<8z{$x;Lh_<&7AK$0aAk9W2-d;Al%_>^|IiT3jr({c>@U9$ zH`^;e`OQ^2N~)@oOG+gCS-w{ti)v|Qf(d@sC1Sr);y_sBfX!q7ySqCnPtUgb#f35d zKp0~uCz-|M>ZI*$z{vM@bxArE14BdQ-27fB995P~RJ$WldHL#wQl0ry9XxKQZ+R-O z7j3p}iOI=QVq(EKmG+5PkMTet{0sYM2{>(%`g+r;Q6aLr$?Gqq?8K5aciSGi)ua>h ziM_phiZxE9y9axc4i1Wf*<&)Hf|MkiU(>R<998*woFsTWv(Pa4hEg{U%yMibeX_0A zT2-!;DfFT#N`o9M!|j)fSAEQyW+J5yj{RH3OxS)pUmf{R85!{c7p;F&$0^J>Z2!`4 z;kT0&H{+`F((tdC)iHN*X&leOwUwb$-o{~rutgGr((>5)nzk@6b3Cv)9hz97NfO6x zx~T4|rI>Iqh8`neB4%;DKf&T?L9C9Cb@T}mFL!M(&+cTn;%OT*EhU~FM}sTA%c*5t z!8G>#I%{@qfqpH^nwg267RS7Fg=HcoiAGJzWFL4ZbK!skH*~zl&D>^pSvM1LGj269 zGLevb5~!%FL>9PICG{A9L8V?N5(unN@E%4GZ>}h1ftyG#YV8PU_4v^gJx)q3cJ95Y zprO;Q7$2`lDt0iodHZW|mQ)w-S$!pAYV_}64ejV;;sh>-lu(bN&=dyx4tA-1gNx@` zceyY;M2gODZsv&4=p<|8^VmFGkC|eVze>BcaW&riGD5ulB~AqPdbC z$8#4d&Szt%e}DU}4Ko)t&g^()q@iD1U}{92HgJ4MBgJ=5F@IgFCJruU7G|jNttXg& z@XZ~D!jU!7YBW^^tn0X@xPh##i*VD7g&4xa7a)uj8N`CccaM{#XT9zc@dR8X$i+e; zsXA-ymZP}#cWVY16KUjt@Oz7CLdzc2+6hRvE8Inw<9BDCy1_%_Mkg;kPCh@h>gpUx z?bp3xT3nqQ$1d}8qz%vP&AI2yv>pi8YK0doH0X|wjs{o9cA}o+U_p$Ka^fbn7GbIB zE1HudJ@WTsK~?OMQmN%de+P;JQ|c*0PS}efF#$uQ4JhWJf?qzKnDZ;y4vvmemsZ=x zZq3+hWa<);m5&+qC7*H2$6j2<$(qWIydtkBwH{NqIBm6)mKaKyOV(D6%sjDM#b##W z#oS+t15id`Iq}m(!4uEtohqBed4lOyT6m}^79>fD>BSnYmT5(Ug0z~rFiTLl$a8}v zgrQooO(!$syf1uzoJ<_=9xo6a5n@jt5|LP_QY97VcuIy#sk(a^8EXG*e_7Q%d=`qT z8VFMcX3&&UjSLDI4MmGP$yIjmc)u^18M32esi5*KcwqbG-q=XS=;ULJL@9E_z@O<@ z`sVYv;1%?%lzSfrk|h|U>wVcelUr(yqSn~59@tbM%)57K5EYc3otq<}qpNgyxH86> zrG!10WJ?R054%>{tA2g_uGsDnHFUsMQhHgb`y%q_K|ez7{CPypc8D{}Qd((hXrT2E zH&xil953V$q3~A-!`ibx%oZ&AwR_v41n1VY>}3J2h#K=MDTPvuo8OyA2co^5Pb@22 z=>SmN3Qp2|(4aPNzHPwZ$9Fys=KSU206@B&^||h`wRdo^Y|m&M4s3}Ui|-F z_WuEu3>D(z;KaT9L*Nj6jlaCKFDDp7#Oo*xX zQ6Fw~SP4IGlVdSB*h&}`91B}jkd9|bd&9=vIb#Nb_JP~weSG#G|zQU@BRd?z;?=GNi<5{M4C7qOmLydX%%W!+Y$jH=GLU{P+k) zsd`~UW7F95bm(|S4r!s~;;5lKAH`UN6ff4>c_bmtVWOeZsUxLYnUoi=Qb)wsw==h@ znZA4d@d>(UD0U@R*eb-H@rd4r53z0CLBNJF8Nc3iWdWA1_b3kn%3 zDk?J*6LEfizQfC-pOdTe@X_E`b3RwiR-RQ-77}d&pTGK4s(X8QE2mAkIgC88&HK~G zC%8r%PZ!c^_o5@iqjA5VA*r-#6`7gq$>@EmM8w70oDICZWGXQ_96HOjo%2+wyEuv9 z?3u0(qs5K(Mw8;>;{o>e_Vr47`^R4(Nm;w};9oKXzX>0s%q}dD-Eu4PEX@P5%KF13 z%@sR+5_b3LR05AjsmXv|IvaA%vz2&WN00SVUW0OqY-75_-j5abg9rI$7?WGMvMn|o|he%zz zJW^3I5El<&ZE758ah!0tl!}RrmX%N(g``r~DIoug+Vi#H8}T1)PXWqH^Xbm%&fHz> zRq5Z<>lEja7!cW#jxmykxmY4VejpPC!-;~Drs(-ei|U4sllJ2?kP)o}Th#-!bV{m% zhNfm2xU87kwQo;>Rrl}i-u$7}>ZGxs34WlWz~C_Rs^h@Z!4 zY9i{$tQd+vfzhnkDrRAa@nH8RKGDVy4Et63%lalB^>TwwZk*EG&#Ex95*H2kGeB+H`omF3jq# zfE65~JpwYpqn75Xrn4_8Cc^c`DLibB-WSLO&z`W|{oPUU&}|tnfi{cHmWJ3jVWnL{ zPY@HT|98|H+1I}0X*p%Ju_@onU{+SvW2B#9%oHY*mDVF&VSbSaF~ggJIT+x+cZ#lu zf@-Cvv)T_uZ{gPysyA0r&q-A>tAw{2vWBvqp}pIjXJvY=R_YUWn}pNMfqF9Z@g)N& z?p5$Cg774d5K?c~CS zfT@m#?^3EsZMbnTBHclP+RlZi-i9&#VqvH2FX4$PNiA|3FWMR^WS%y|ZC_5)kV`lx+ys^4mUou*l{^rfAT|v?mULKuKFjAmpCe`Ez~KE8 z;P>xG?x?2Ob+WlxjoANt#s80Vl|cr?B_vF_ykvTj36(j7p#lH_BqU_i)QZR?6uxay zZxpOKa1*ZMN!2oD{HWv%yAjB*cZuU2MR@5USTHHM2y@yOR55nX6JTRn5vvz9i><$8 zN_q;jPf|%@qeCHO?j80{94&dUTro(0QW=aRH^#j>(YI*)af6Be-V(8C3QF0TKG9B4f9FF{POb-B(%KR$1^x|4Gec|q)RAn zpqYvo!cUf9_B3s)5fHUG7$Qp!&B7ugSRMw4D26k+9C`O6JjKPtSYs3t%RuSHSt<&At36pVNne;Aqv%QO)h=2QgJ}Od!gSwRifxKOPom zmUK`;iIRJdv{`ce>&Nj|?)ZKFB+FrT?xero%Gb9Pq80}%nzvYlhso6rf3G9Ibq@Wr zjL>O8>~EX7CrfJuj(b<4M{0_=I5_Og@S@Vn%+lXu>96-GaM8k}!q}vEvhw`=fsBR#Qu0710y?SBLe(BRpVgaHhOTZlBdPzQGH^JUQXI< zr=LtpItg#={n=yLO3AF)4?29BTwE~rML%f-;>@blOQ)66d&xuA6bXcei3u*s!orG2_cgLZ7&KR-mDh3a)tNCGF0CAMsdJJ?iv%pH(vF?WN{q8gNHB z-lhplMQ7n#2VB1vU@yogLS#+5U&oV@voGdYRdoE{Yi6~klb!wTq?$gN9eJeTmWTor zc;wTTx#}uPM6OckS-Ab8vC+g-4>7V8aNah-L-UDs5|Y5Fab2&0Mq&oVwXN3up62go z_}1DBWLvnKLN}E(+V7n}2dsm!pmTcnx3!ItG$lk1hR&$``35H{N-h0S|GI)Ay5^|1 zv#$AftC)zL{?Wkr=G^deG^~t7QO!|Hj$?6T;QYEMB}@ZUGEB*m!dR70H%{qiREI^E zykNN`T}n+DN=HKCI9p(!KlB+|oB@*|oFs=Z)?%Wqwra4QR*ycind!^cF>#xiFLc>( zMw0;L0L_ww$vyzTRyMj@6O3v3zdc4|mhb{h1Xv)WyR;!)s)jzIctWaV2 z0~#zl_-1F;0Y53qcDu!W+3TPah-u;J=Kw~^X%mNnkG(Qf+S*k{D#Vt`ul4ng&?b&I zlX^D1UWZB#mtC*AEnY_fzu$(@>wI?Dspz$hx%K825Fd6r37@;(=)O$4jyA%$47VbF zZ!lCN*FLkps#o*<;o$pMx(OG(J)Wt88e1DofU^{h)MPIz3et!$k{$5Kr?6aHCrNvB zl6s1KT|I;&J#EXyWjByd?BYKpM~_KgLf*Ju(eKKTE-gc2(>HW5<$cQYLoY z8_`zzzMm&yl9fY^2I|nF$%He~EiNd7A*WX}F)#BYocLUm-th`6vNBQ@?m#Z|jx8Jf zmMp#nv+)d#2!}!uIP0;y5F`6n#RNjQiEg{qx8TZ?pVY3;B|E zmyDLW!j)1;dwm-YMIJ|@7=onx#zk^Ns_sKeg>!?Jw=Igq3RP(-i*%+v==kYH+us(U z==gm*L!FF|ESR|J%G?e0q0BvX%<|4Iefwu7${e@cPO!NtM9f}{#wctAJjteNIb>9Y?qLx2(OtE{@i<>pQQNL#ci*`kzPY*B( zDkq_DQ8~}*JW7c1MOn?5Qs3^^-$F4kDj7|j#&i5j^!BTkCYNjbVD_dd2up<(GkG6( z>lN*%u-H7mm<=&d{UBjQ`6`nxPoT{tV3eh2_)dS7U;hAMR?c~zZCtVwHJAmR1SHr|g>uR0X0x}v! zIxzTwuRw~+Gl#l#kBK!etABOS(9;8WwTNv~tcQ{sH+$@290kM0t1Y+UY6;YrYN&bg5dO7lM^kZUo$Ugn@`RylhPS8+>hBy}OD++( zEFQUbA`rPR5AwDrY)n`4*9tBu19UG}`SQq||PO*@&*3*j0hKTdO zDf>JUzL~U}627EIH?!Nskx8eV3{y_DKiiV|B_RI1I=}F#beyE)rXFR-=Zq}U%SLV( zK_y{LJ>(;E;|~~)_PI>MTi`mTbyj_7dwcCNI^)aW(?@e_z{7&bQL4yR2YQ&u==om!sEh zlc=>9;uxH6QfPIb%Da-8@$LO-w?6T)>C+>U%Z?j1loXSBBuYCchPLoF|8Z?#OPDT# z-2d|n{8rWJ(x`HydU2!*X9d)07w~(?HN4dgO*UP?w(MYtfGC@T7lCuxLZy0qSo_rY{^;0=gZM4&* zvGl9j^N`5I=Xp+|mVB*bn2`lu8$6t?+30&2=UB@!_^#fRJBIX_mz!^WsI-;nA0D9N z^UnL+kg#HUr4csv)qRDl|1M@TWdXH@H`bfLIedc)__YjEj7KvPzbYO~Z(<>Txi=Oa zaF{CcB(ptYuJ=LMcS^}@HCjmU5$xmwI<9!Qo^!uD+I@;#ed}Kogx)_Aw#B20mkhM! z@3v_+6wq~p;=?Xt6^la3&tDl`Pap1mnMlef?)lArN2w*Lq&jn84=kvvjfOm}vsunC zQe~B#*w--Pp6DwoUP>kyy=)o>xU*C&31P%bsvTwo$OGHL2F~!_{ptz363~%L^k>P? zxoImNl;yNtq%rN0g}xtcs}V*tPOQ=}CAYy}+&JMPa0$jzhhIssX+5X>e!VJX`O7+)} zu@w8fvG+x2iw0pFKdAeKa|L3s?dZuy9pPuN@f#$6o*uh&(KNZz1P4fSL{llVssn8F z?Up%{s{73$!cndyHIc?1yei^aX!wSY!F%LsCJN4~E>%@M<4jphsW@AFp-4ml$aY{b zkoI>8q$JqHvV`sr!idyKZ=d}+j91l8>>$TkgnqKPD$k?QpT`Bh`>I3p^9DHF(U zwQRFbG!4&wV(bLjp?%Q3;fnB3+v>>M|q1d0xJt#HCT zI`&EHGz5qY64mh!PhME;bbFHq>qfU6Yi0WkxqmJwhdtx$(r9i3q8GONN%XcBs@%a6Nx%>?vJm_!hj%x-BQ5t6>NwiS@^3( z(~}ltEO!p=w_FKwfc&qB;vX^YOT5aUA^dA*Py@#eF5jS&WSdV&=*=6dj;QCOmn`rL z*H~*~z8XBv+`lR8-3D0s{$+ARQm~z6W6dmvVh*JDpZ59N@s^L@e(g{_NGY0~FlKt# zcmc;z{6Ah05fzB!RZ)w|RxxSryaD6yWq9v8F-&=t44s-FReX7BL+VBZSj6|J_~YPp zCceQ@d(yamKV6^{Du&P_0nY$Mn;Fakt97BT+P;9evcNJJkq1VN(QCmV#K%Fz$2f{? zriY&PIoHY%(7uJ=>fc6iq*t{rB4HgSlL>>Bi=4#TMU$2!Dpo=_A~fQ_xvpK{jM0nJ! zFo#OKlq+jpcog&B4*A3)_0z)ze-3pqS6+Y_eUkye91>lJhGz4L%ejRkCh90Sr0lod zfjAU2?YLM2B<1$^uE$cQCf?VZ>pD5lWLRiW_Ssfo&SAulCEC`>#ggZdi@K+AiH*Kl%pKbol4N{ zZ3H+vl=D|zgX4qMm|{6W=bx*}CHXRXGIud?%~Tk%5JdPVKC&-y$^=|yzy|-4&X=_} z^F`b3$5E0TTmxXY9u&QpI_*+s9YqUF8u2g$Eq>SYh_4xTvyi(CPcS2w0Sy@~>Yqfd zJ-q@H`NQ8}aNyr0_Rl$x;iuFNUFXACPI}ly@3tb#SKz!2r(rk3$L#=;AudGSpi1NE ze35qX$O;q*>`DC4UoqYze}I$=c6{pj9~nkBF1-6vSx|rCc}np8Jp^+E^~rUyb=P~o zn0)DX0ws)0>AvKKeF>vu9bFiK`reO#>FeEAk$ZcW-1eM>WCQLF$J|8 z9F!|V2f5B3&9>gixNs6+bqIC$F%vJhl0~$dPl2lI^?@KC_qriLt*%bS#SF#J?Yy_u zo~G1)eDg;TiWZ?6imjWB!2^O!(mLQSb~;ls{2eO0PM9V?_%v9cBTLcscE-RO{}+x0 z<^%fZN1kj}N6252U(vVi7X2W7>)yN#KRZ!TkPivj9Tpmp-VUS7zvZbiBc+6IEQ09S=DKKq2!8d}=J?|^?#_jiTJd6oDvi)zmc;n6-HF)y9)|JiX4PU3$zF3EdE&i#SaHp*CNlcTT#kw zvY*7n@zT#Sl`-=ba3d~Sj(6-#|8g?@GO3p&IIaYBFnEr|;B{>)D?T}wne86>Dh_lc z-g;U6&ko8I#Y$@eS@TGh%bQ3VXFtU49uXK}=L+GVqdDWc;dALHs^(`n^+w?+0ISMN zp`bvLmZ{1p7z;nW5kwG}aBxp7qv`+>_BoSPn0Nazp-uK^t}lF z1KtlyatlOp_v`L+R;OW;$TBg~@Si6%St_^!$NjvtdfK;Sg5-#NXs9*J1FZY%m-ZG` zE62VuN(Jui{K=I!`uiB0KB<=GZVbf|OhqUXMn+AfQZXfBT6b`4yki7u;Pv;?)M@TF zns||XI$no#`4M`T@rfv8!$YyC^?$=tA zl4p|LonIT1Ow=l&$9yXE7a9$JuNv`{!{L&f8?CsnD(Cm>%jK&R^Irj(#i&3*3pe}b zUYMS(k37$fhbuY6Wb;Pe>{`kCWmL)ogYX|5EkcmIjC3$kV~2jP8$`i{Ie18i@(0{n zN#BsMEi4bL^104&y=&#}T`P__1nsavkLBFjA8;=1swzx?ISgw4K$%j_XkMth}VSTf1U+V|#MDq?!F6)uOu~O9SV3@h! zL6ogc3EanAY$dkjdOtUm2n?RmKCfu}Rn#IxawpOyW(=pt8e+=vtLb9P*3*8{q!;@$ z*xs8cu48O`h~9=SiEDphsm@QL!Sr0qXOQ2QGb^f@gt`@qC~~}oK07~rn3IK)FypAT zBTPz3kxuJ`bvaux?p&h?aQmjAK~{YZrgoS}m{n0XXEJY2`f6tJ)of^lw6Ypr{dVIh zgCr-F8`K@27+W%f3mZHQwbZMdpKDz&>P^+FVTb9^B3%lw`1Oo?B9~gJPw(P7rovWB zw2P$}5Kk!Pa(HWiguEUV8q2ug-v84Y#$}wH0aiQnj5N3Sd!tJr4yHjH@-7Y-t^mWp z?EU~&USxb`PWCL7D|3F&6_L+9!*J5#V!^>ZCMMdX<4cTveIQsG;dGB{U=c6bk_`r< zLhqwLT^!04qMYh9+%Pfw{J`*M(%?zl#qR(g@cGZPG`%X&0u(_K?6Ea)vQ#wSh#V4Z z65?hM09Li#7=M2V^nDmUHBJL9HOEbCF~zcZb?hh<0Y+Q{kswrDF|9L;;C?f!&U29t zMn^>DvcT5O7EYQ%J%O$}c)~t--UCbKc zwi7EfSAf?xZtI7%t3<6f1wV}1&a1In>7gh*z+V?$Bb^aghn&O$M;DI$B)&baTaHMg z+?-*;85g}pk;oAdd^+Ye5pdH7jE09!+k|(Y;rE<|GOD^k=z%BdeWCp%iUD1;Nurkl z!^)wO<#LHqxBpwFk~aKVR44q|B%AB4Jmj4Y+7AwZ(Ln*X!6d;a z0~of)l^UrU=CDZq_lkFg{GX*Oa3k~EC0}(1y>}qV)i0%~vAR4ohKW(7t?=%n1m~;b zjW;p0orB?X?Y4r7iqk5;-r=_srQNRBD_=EMD?b`A;1)uco5g7l^m(k$kyet#cG_~a>Q3A;GMfimm4~U~=MBs#Zuq2T58>Bq zy14hA=-{B3`%{){LG4ySq`XY%p#VAH4<-~q1V92X8VCovACFKJmXsXQr2P8(ac^T# z6&{#oJM0@b`Zk9(?E4FC1xyOv1OGCv!`?9_L=L!FdxY|hKOu1!L!-_pte}8Y-Zf37 zOX;=JqUs0MBKdm*6}}AgXX~HNH2}p9>`Ea*5^>P};TQ`tf?L_pAD#IVz-?_X%@{KJ z103CEEVJ;xlw;UWU}lNlk4e-2q`OhU(s+s{68{pKdj!Ee3559vng66Y!P0)hqX=RD zu$_sp!F=Taf9@Q~f2ASB!P0gTX~h5b)S*6t>4J>k4p{z^-ULfSB2lsYM~2oT5(L73 zI`$&}PZ|#_-A!y9{NFMVz{}_n%$oX7+7C>XhUY98kNtNU0si1+V3%B{{qLgxPk7UI z;;@x@c|a*CDWikQEJZCXyhx9$|K6Ql3duKJT?Wtl^Q6{RZZ40TlARSX88Ziu+hn@$ zq-<;|$@JRPR8&-QwmZ)MzHLD$JQBOTQA{q+y9%vNFAeZ7lyCm+Tm>{dU3BSo`|j-Q zL=$$y|7&%Au)pac+`#||M$^gvQ+G}35?pl)W0V0iVcCbg8wNqhU6dOi2pK} zN5an2%&hkBhH35ti=Q)Ydg1+R!oGaUKLtLh*l}Rv*~?2BY5np_P1yWbFg^Pf}D)qzb4P@P7d!Kz)G# diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/BUILD.gn b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/BUILD.gn deleted file mode 100644 index 865bc4d3b..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/BUILD.gn +++ /dev/null @@ -1,33 +0,0 @@ -generated_doxygen_out_dir = - get_path_info(".", "gen_dir") + "/.." - -loadgen_doxygen_sources = [ - "doxygen.cfg", - "doxygen_footer.html", - "doxygen_header.html", - "doxygen_layout.xml", - "doxygen_stylesheet.css", - "loadgen-integration_diagram.dia", - "mlperf_icon.png", - "mlperf_logo_horizontal_color.svg", - "README.md" -] - -source_set("loadgen_doxygen_sources") { - sources = loadgen_doxygen_sources -} - -source_set("doxygen_html_generator_script") { - sources = [ "doxygen_html_generator.py" ] -} - -action("generate_doxygen_html") { - script = "doxygen_html_generator.py" - args = [ rebase_path(generated_doxygen_out_dir, root_build_dir), - rebase_path("../..") ] - outputs = [ generated_doxygen_out_dir ] - deps = [ ":loadgen_doxygen_sources", - ":doxygen_html_generator_script", - "../..:mlperf_loadgen_sources_no_gen", - "../..:docs" ] -} diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/README.md b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/README.md deleted file mode 100644 index d5cf5fe18..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/README.md +++ /dev/null @@ -1,34 +0,0 @@ -# Generating the HTML docs {#ReadmeHtmlDocs} - -This document is generated from inline docstrings in the source and -various markdown files checked into the git repository. If you've -checked out the code, you can generate this documentation. - -*Prerequisite:* You must have [doxygen](http://www.doxygen.nl) installed -on your system: - -## With gn / ninja - -If you are using the gn build flow, you may run: - - ninja -C out/Release generate_doxygen_html - -* This will output the documentation to out/Release/gen/loadgen/docs/gen and -avoid poluting the source directory. - -## Manually - -Alternatively, you can manually run: - - python docs/src/doxygen_html_generator.py - -* If is omitted, it will default to ".". -* If is also omitted, it will default to "./docs/gen". - -## Hosting - -A version of this doc is currently hosted online at -https://mlperf.github.io/inference/loadgen/index.html - -To update the hosted version, submit a PR to the -[mlperf.github.io](https://github.com/mlperf/mlperf.github.io) repository. diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/doxygen.cfg b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/doxygen.cfg deleted file mode 100644 index fc05853d1..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/doxygen.cfg +++ /dev/null @@ -1,2495 +0,0 @@ -# Doxyfile 1.8.13 - -# This file describes the settings to be used by the documentation system -# doxygen (www.doxygen.org) for a project. -# -# All text after a double hash (##) is considered a comment and is placed in -# front of the TAG it is preceding. -# -# All text after a single hash (#) is considered a comment and will be ignored. -# The format is: -# TAG = value [value, ...] -# For lists, items can also be appended using: -# TAG += value [value, ...] -# Values that contain spaces should be placed between quotes (\" \"). - -#--------------------------------------------------------------------------- -# Project related configuration options -#--------------------------------------------------------------------------- - -# This tag specifies the encoding used for all characters in the config file -# that follow. The default is UTF-8 which is also the encoding used for all text -# before the first occurrence of this tag. Doxygen uses libiconv (or the iconv -# built into libc) for the transcoding. See http://www.gnu.org/software/libiconv -# for the list of possible encodings. -# The default value is: UTF-8. - -DOXYFILE_ENCODING = UTF-8 - -# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by -# double-quotes, unless you are using Doxywizard) that should identify the -# project for which the documentation is generated. This name is used in the -# title of most generated pages and in a few other places. -# The default value is: My Project. - -PROJECT_NAME = "LoadGen Guide" - -# The PROJECT_NUMBER tag can be used to enter a project or revision number. This -# could be handy for archiving the generated documentation or if some version -# control system is used. - -PROJECT_NUMBER = - -# Using the PROJECT_BRIEF tag one can provide an optional one line description -# for a project that appears at the top of each page and should give viewer a -# quick idea about the purpose of the project. Keep the description short. - -PROJECT_BRIEF = - -# With the PROJECT_LOGO tag one can specify a logo or an icon that is included -# in the documentation. The maximum height of the logo should not exceed 55 -# pixels and the maximum width should not exceed 200 pixels. Doxygen will copy -# the logo to the output directory. - -PROJECT_LOGO = $(MLPERF_LOADGEN_SRC_PATH)/docs/src/mlperf_logo_horizontal_color.svg - -# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path -# into which the generated documentation will be written. If a relative path is -# entered, it will be relative to the location where doxygen was started. If -# left blank the current directory will be used. - -OUTPUT_DIRECTORY = $(MLPERF_DOXYGEN_OUT_PATH) - -# If the CREATE_SUBDIRS tag is set to YES then doxygen will create 4096 sub- -# directories (in 2 levels) under the output directory of each output format and -# will distribute the generated files over these directories. Enabling this -# option can be useful when feeding doxygen a huge amount of source files, where -# putting all generated files in the same directory would otherwise causes -# performance problems for the file system. -# The default value is: NO. - -CREATE_SUBDIRS = NO - -# If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII -# characters to appear in the names of generated files. If set to NO, non-ASCII -# characters will be escaped, for example _xE3_x81_x84 will be used for Unicode -# U+3044. -# The default value is: NO. - -ALLOW_UNICODE_NAMES = NO - -# The OUTPUT_LANGUAGE tag is used to specify the language in which all -# documentation generated by doxygen is written. Doxygen will use this -# information to generate all constant output in the proper language. -# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Catalan, Chinese, -# Chinese-Traditional, Croatian, Czech, Danish, Dutch, English (United States), -# Esperanto, Farsi (Persian), Finnish, French, German, Greek, Hungarian, -# Indonesian, Italian, Japanese, Japanese-en (Japanese with English messages), -# Korean, Korean-en (Korean with English messages), Latvian, Lithuanian, -# Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, Romanian, Russian, -# Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, Swedish, Turkish, -# Ukrainian and Vietnamese. -# The default value is: English. - -OUTPUT_LANGUAGE = English - -# If the BRIEF_MEMBER_DESC tag is set to YES, doxygen will include brief member -# descriptions after the members that are listed in the file and class -# documentation (similar to Javadoc). Set to NO to disable this. -# The default value is: YES. - -BRIEF_MEMBER_DESC = YES - -# If the REPEAT_BRIEF tag is set to YES, doxygen will prepend the brief -# description of a member or function before the detailed description -# -# Note: If both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the -# brief descriptions will be completely suppressed. -# The default value is: YES. - -REPEAT_BRIEF = YES - -# This tag implements a quasi-intelligent brief description abbreviator that is -# used to form the text in various listings. Each string in this list, if found -# as the leading text of the brief description, will be stripped from the text -# and the result, after processing the whole list, is used as the annotated -# text. Otherwise, the brief description is used as-is. If left blank, the -# following values are used ($name is automatically replaced with the name of -# the entity):The $name class, The $name widget, The $name file, is, provides, -# specifies, contains, represents, a, an and the. - -ABBREVIATE_BRIEF = "The $name class" \ - "The $name widget" \ - "The $name file" \ - is \ - provides \ - specifies \ - contains \ - represents \ - a \ - an \ - the - -# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then -# doxygen will generate a detailed section even if there is only a brief -# description. -# The default value is: NO. - -ALWAYS_DETAILED_SEC = YES - -# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all -# inherited members of a class in the documentation of that class as if those -# members were ordinary class members. Constructors, destructors and assignment -# operators of the base classes will not be shown. -# The default value is: NO. - -INLINE_INHERITED_MEMB = NO - -# If the FULL_PATH_NAMES tag is set to YES, doxygen will prepend the full path -# before files name in the file list and in the header files. If set to NO the -# shortest path that makes the file name unique will be used -# The default value is: YES. - -FULL_PATH_NAMES = YES - -# The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path. -# Stripping is only done if one of the specified strings matches the left-hand -# part of the path. The tag can be used to show relative paths in the file list. -# If left blank the directory from which doxygen is run is used as the path to -# strip. -# -# Note that you can specify absolute paths here, but also relative paths, which -# will be relative from the directory where doxygen is started. -# This tag requires that the tag FULL_PATH_NAMES is set to YES. - -STRIP_FROM_PATH = - -# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the -# path mentioned in the documentation of a class, which tells the reader which -# header file to include in order to use a class. If left blank only the name of -# the header file containing the class definition is used. Otherwise one should -# specify the list of include paths that are normally passed to the compiler -# using the -I flag. - -STRIP_FROM_INC_PATH = - -# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but -# less readable) file names. This can be useful is your file systems doesn't -# support long names like on DOS, Mac, or CD-ROM. -# The default value is: NO. - -SHORT_NAMES = NO - -# If the JAVADOC_AUTOBRIEF tag is set to YES then doxygen will interpret the -# first line (until the first dot) of a Javadoc-style comment as the brief -# description. If set to NO, the Javadoc-style will behave just like regular Qt- -# style comments (thus requiring an explicit @brief command for a brief -# description.) -# The default value is: NO. - -JAVADOC_AUTOBRIEF = NO - -# If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first -# line (until the first dot) of a Qt-style comment as the brief description. If -# set to NO, the Qt-style will behave just like regular Qt-style comments (thus -# requiring an explicit \brief command for a brief description.) -# The default value is: NO. - -QT_AUTOBRIEF = NO - -# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make doxygen treat a -# multi-line C++ special comment block (i.e. a block of //! or /// comments) as -# a brief description. This used to be the default behavior. The new default is -# to treat a multi-line C++ comment block as a detailed description. Set this -# tag to YES if you prefer the old behavior instead. -# -# Note that setting this tag to YES also means that rational rose comments are -# not recognized any more. -# The default value is: NO. - -MULTILINE_CPP_IS_BRIEF = NO - -# If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the -# documentation from any documented member that it re-implements. -# The default value is: YES. - -INHERIT_DOCS = YES - -# If the SEPARATE_MEMBER_PAGES tag is set to YES then doxygen will produce a new -# page for each member. If set to NO, the documentation of a member will be part -# of the file/class/namespace that contains it. -# The default value is: NO. - -SEPARATE_MEMBER_PAGES = NO - -# The TAB_SIZE tag can be used to set the number of spaces in a tab. Doxygen -# uses this value to replace tabs by spaces in code fragments. -# Minimum value: 1, maximum value: 16, default value: 4. - -TAB_SIZE = 4 - -# This tag can be used to specify a number of aliases that act as commands in -# the documentation. An alias has the form: -# name=value -# For example adding -# "sideeffect=@par Side Effects:\n" -# will allow you to put the command \sideeffect (or @sideeffect) in the -# documentation, which will result in a user-defined paragraph with heading -# "Side Effects:". You can put \n's in the value part of an alias to insert -# newlines. - -ALIASES = - -# This tag can be used to specify a number of word-keyword mappings (TCL only). -# A mapping has the form "name=value". For example adding "class=itcl::class" -# will allow you to use the command class in the itcl::class meaning. - -TCL_SUBST = - -# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources -# only. Doxygen will then generate output that is more tailored for C. For -# instance, some of the names that are used will be different. The list of all -# members will be omitted, etc. -# The default value is: NO. - -OPTIMIZE_OUTPUT_FOR_C = NO - -# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java or -# Python sources only. Doxygen will then generate output that is more tailored -# for that language. For instance, namespaces will be presented as packages, -# qualified scopes will look different, etc. -# The default value is: NO. - -OPTIMIZE_OUTPUT_JAVA = NO - -# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran -# sources. Doxygen will then generate output that is tailored for Fortran. -# The default value is: NO. - -OPTIMIZE_FOR_FORTRAN = NO - -# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL -# sources. Doxygen will then generate output that is tailored for VHDL. -# The default value is: NO. - -OPTIMIZE_OUTPUT_VHDL = NO - -# Doxygen selects the parser to use depending on the extension of the files it -# parses. With this tag you can assign which parser to use for a given -# extension. Doxygen has a built-in mapping, but you can override or extend it -# using this tag. The format is ext=language, where ext is a file extension, and -# language is one of the parsers supported by doxygen: IDL, Java, Javascript, -# C#, C, C++, D, PHP, Objective-C, Python, Fortran (fixed format Fortran: -# FortranFixed, free formatted Fortran: FortranFree, unknown formatted Fortran: -# Fortran. In the later case the parser tries to guess whether the code is fixed -# or free formatted code, this is the default for Fortran type files), VHDL. For -# instance to make doxygen treat .inc files as Fortran files (default is PHP), -# and .f files as C (default is Fortran), use: inc=Fortran f=C. -# -# Note: For files without extension you can use no_extension as a placeholder. -# -# Note that for custom extensions you also need to set FILE_PATTERNS otherwise -# the files are not read by doxygen. - -EXTENSION_MAPPING = - -# If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments -# according to the Markdown format, which allows for more readable -# documentation. See http://daringfireball.net/projects/markdown/ for details. -# The output of markdown processing is further processed by doxygen, so you can -# mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in -# case of backward compatibilities issues. -# The default value is: YES. - -MARKDOWN_SUPPORT = YES - -# When the TOC_INCLUDE_HEADINGS tag is set to a non-zero value, all headings up -# to that level are automatically included in the table of contents, even if -# they do not have an id attribute. -# Note: This feature currently applies only to Markdown headings. -# Minimum value: 0, maximum value: 99, default value: 0. -# This tag requires that the tag MARKDOWN_SUPPORT is set to YES. - -TOC_INCLUDE_HEADINGS = 1 - -# When enabled doxygen tries to link words that correspond to documented -# classes, or namespaces to their corresponding documentation. Such a link can -# be prevented in individual cases by putting a % sign in front of the word or -# globally by setting AUTOLINK_SUPPORT to NO. -# The default value is: YES. - -AUTOLINK_SUPPORT = YES - -# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want -# to include (a tag file for) the STL sources as input, then you should set this -# tag to YES in order to let doxygen match functions declarations and -# definitions whose arguments contain STL classes (e.g. func(std::string); -# versus func(std::string) {}). This also make the inheritance and collaboration -# diagrams that involve STL classes more complete and accurate. -# The default value is: NO. - -BUILTIN_STL_SUPPORT = NO - -# If you use Microsoft's C++/CLI language, you should set this option to YES to -# enable parsing support. -# The default value is: NO. - -CPP_CLI_SUPPORT = NO - -# Set the SIP_SUPPORT tag to YES if your project consists of sip (see: -# http://www.riverbankcomputing.co.uk/software/sip/intro) sources only. Doxygen -# will parse them like normal C++ but will assume all classes use public instead -# of private inheritance when no explicit protection keyword is present. -# The default value is: NO. - -SIP_SUPPORT = NO - -# For Microsoft's IDL there are propget and propput attributes to indicate -# getter and setter methods for a property. Setting this option to YES will make -# doxygen to replace the get and set methods by a property in the documentation. -# This will only work if the methods are indeed getting or setting a simple -# type. If this is not the case, or you want to show the methods anyway, you -# should set this option to NO. -# The default value is: YES. - -IDL_PROPERTY_SUPPORT = YES - -# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC -# tag is set to YES then doxygen will reuse the documentation of the first -# member in the group (if any) for the other members of the group. By default -# all members of a group must be documented explicitly. -# The default value is: NO. - -DISTRIBUTE_GROUP_DOC = NO - -# If one adds a struct or class to a group and this option is enabled, then also -# any nested class or struct is added to the same group. By default this option -# is disabled and one has to add nested compounds explicitly via \ingroup. -# The default value is: NO. - -GROUP_NESTED_COMPOUNDS = NO - -# Set the SUBGROUPING tag to YES to allow class member groups of the same type -# (for instance a group of public functions) to be put as a subgroup of that -# type (e.g. under the Public Functions section). Set it to NO to prevent -# subgrouping. Alternatively, this can be done per class using the -# \nosubgrouping command. -# The default value is: YES. - -SUBGROUPING = YES - -# When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and unions -# are shown inside the group in which they are included (e.g. using \ingroup) -# instead of on a separate page (for HTML and Man pages) or section (for LaTeX -# and RTF). -# -# Note that this feature does not work in combination with -# SEPARATE_MEMBER_PAGES. -# The default value is: NO. - -INLINE_GROUPED_CLASSES = NO - -# When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and unions -# with only public data fields or simple typedef fields will be shown inline in -# the documentation of the scope in which they are defined (i.e. file, -# namespace, or group documentation), provided this scope is documented. If set -# to NO, structs, classes, and unions are shown on a separate page (for HTML and -# Man pages) or section (for LaTeX and RTF). -# The default value is: NO. - -INLINE_SIMPLE_STRUCTS = NO - -# When TYPEDEF_HIDES_STRUCT tag is enabled, a typedef of a struct, union, or -# enum is documented as struct, union, or enum with the name of the typedef. So -# typedef struct TypeS {} TypeT, will appear in the documentation as a struct -# with name TypeT. When disabled the typedef will appear as a member of a file, -# namespace, or class. And the struct will be named TypeS. This can typically be -# useful for C code in case the coding convention dictates that all compound -# types are typedef'ed and only the typedef is referenced, never the tag name. -# The default value is: NO. - -TYPEDEF_HIDES_STRUCT = NO - -# The size of the symbol lookup cache can be set using LOOKUP_CACHE_SIZE. This -# cache is used to resolve symbols given their name and scope. Since this can be -# an expensive process and often the same symbol appears multiple times in the -# code, doxygen keeps a cache of pre-resolved symbols. If the cache is too small -# doxygen will become slower. If the cache is too large, memory is wasted. The -# cache size is given by this formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range -# is 0..9, the default is 0, corresponding to a cache size of 2^16=65536 -# symbols. At the end of a run doxygen will report the cache usage and suggest -# the optimal cache size from a speed point of view. -# Minimum value: 0, maximum value: 9, default value: 0. - -LOOKUP_CACHE_SIZE = 0 - -#--------------------------------------------------------------------------- -# Build related configuration options -#--------------------------------------------------------------------------- - -# If the EXTRACT_ALL tag is set to YES, doxygen will assume all entities in -# documentation are documented, even if no documentation was available. Private -# class members and static file members will be hidden unless the -# EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES. -# Note: This will also disable the warnings about undocumented members that are -# normally produced when WARNINGS is set to YES. -# The default value is: NO. - -EXTRACT_ALL = NO - -# If the EXTRACT_PRIVATE tag is set to YES, all private members of a class will -# be included in the documentation. -# The default value is: NO. - -EXTRACT_PRIVATE = YES - -# If the EXTRACT_PACKAGE tag is set to YES, all members with package or internal -# scope will be included in the documentation. -# The default value is: NO. - -EXTRACT_PACKAGE = YES - -# If the EXTRACT_STATIC tag is set to YES, all static members of a file will be -# included in the documentation. -# The default value is: NO. - -EXTRACT_STATIC = YES - -# If the EXTRACT_LOCAL_CLASSES tag is set to YES, classes (and structs) defined -# locally in source files will be included in the documentation. If set to NO, -# only classes defined in header files are included. Does not have any effect -# for Java sources. -# The default value is: YES. - -EXTRACT_LOCAL_CLASSES = YES - -# This flag is only useful for Objective-C code. If set to YES, local methods, -# which are defined in the implementation section but not in the interface are -# included in the documentation. If set to NO, only methods in the interface are -# included. -# The default value is: NO. - -EXTRACT_LOCAL_METHODS = NO - -# If this flag is set to YES, the members of anonymous namespaces will be -# extracted and appear in the documentation as a namespace called -# 'anonymous_namespace{file}', where file will be replaced with the base name of -# the file that contains the anonymous namespace. By default anonymous namespace -# are hidden. -# The default value is: NO. - -EXTRACT_ANON_NSPACES = NO - -# If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all -# undocumented members inside documented classes or files. If set to NO these -# members will be included in the various overviews, but no documentation -# section is generated. This option has no effect if EXTRACT_ALL is enabled. -# The default value is: NO. - -HIDE_UNDOC_MEMBERS = NO - -# If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all -# undocumented classes that are normally visible in the class hierarchy. If set -# to NO, these classes will be included in the various overviews. This option -# has no effect if EXTRACT_ALL is enabled. -# The default value is: NO. - -HIDE_UNDOC_CLASSES = NO - -# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend -# (class|struct|union) declarations. If set to NO, these declarations will be -# included in the documentation. -# The default value is: NO. - -HIDE_FRIEND_COMPOUNDS = NO - -# If the HIDE_IN_BODY_DOCS tag is set to YES, doxygen will hide any -# documentation blocks found inside the body of a function. If set to NO, these -# blocks will be appended to the function's detailed documentation block. -# The default value is: NO. - -HIDE_IN_BODY_DOCS = NO - -# The INTERNAL_DOCS tag determines if documentation that is typed after a -# \internal command is included. If the tag is set to NO then the documentation -# will be excluded. Set it to YES to include the internal documentation. -# The default value is: NO. - -INTERNAL_DOCS = NO - -# If the CASE_SENSE_NAMES tag is set to NO then doxygen will only generate file -# names in lower-case letters. If set to YES, upper-case letters are also -# allowed. This is useful if you have classes or files whose names only differ -# in case and if your file system supports case sensitive file names. Windows -# and Mac users are advised to set this option to NO. -# The default value is: system dependent. - -CASE_SENSE_NAMES = YES - -# If the HIDE_SCOPE_NAMES tag is set to NO then doxygen will show members with -# their full class and namespace scopes in the documentation. If set to YES, the -# scope will be hidden. -# The default value is: NO. - -HIDE_SCOPE_NAMES = NO - -# If the HIDE_COMPOUND_REFERENCE tag is set to NO (default) then doxygen will -# append additional text to a page's title, such as Class Reference. If set to -# YES the compound reference will be hidden. -# The default value is: NO. - -HIDE_COMPOUND_REFERENCE= NO - -# If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of -# the files that are included by a file in the documentation of that file. -# The default value is: YES. - -SHOW_INCLUDE_FILES = YES - -# If the SHOW_GROUPED_MEMB_INC tag is set to YES then Doxygen will add for each -# grouped member an include statement to the documentation, telling the reader -# which file to include in order to use the member. -# The default value is: NO. - -SHOW_GROUPED_MEMB_INC = NO - -# If the FORCE_LOCAL_INCLUDES tag is set to YES then doxygen will list include -# files with double quotes in the documentation rather than with sharp brackets. -# The default value is: NO. - -FORCE_LOCAL_INCLUDES = NO - -# If the INLINE_INFO tag is set to YES then a tag [inline] is inserted in the -# documentation for inline members. -# The default value is: YES. - -INLINE_INFO = YES - -# If the SORT_MEMBER_DOCS tag is set to YES then doxygen will sort the -# (detailed) documentation of file and class members alphabetically by member -# name. If set to NO, the members will appear in declaration order. -# The default value is: YES. - -SORT_MEMBER_DOCS = YES - -# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the brief -# descriptions of file, namespace and class members alphabetically by member -# name. If set to NO, the members will appear in declaration order. Note that -# this will also influence the order of the classes in the class list. -# The default value is: NO. - -SORT_BRIEF_DOCS = NO - -# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen will sort the -# (brief and detailed) documentation of class members so that constructors and -# destructors are listed first. If set to NO the constructors will appear in the -# respective orders defined by SORT_BRIEF_DOCS and SORT_MEMBER_DOCS. -# Note: If SORT_BRIEF_DOCS is set to NO this option is ignored for sorting brief -# member documentation. -# Note: If SORT_MEMBER_DOCS is set to NO this option is ignored for sorting -# detailed member documentation. -# The default value is: NO. - -SORT_MEMBERS_CTORS_1ST = NO - -# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the hierarchy -# of group names into alphabetical order. If set to NO the group names will -# appear in their defined order. -# The default value is: NO. - -SORT_GROUP_NAMES = NO - -# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by -# fully-qualified names, including namespaces. If set to NO, the class list will -# be sorted only by class name, not including the namespace part. -# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. -# Note: This option applies only to the class list, not to the alphabetical -# list. -# The default value is: NO. - -SORT_BY_SCOPE_NAME = NO - -# If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to do proper -# type resolution of all parameters of a function it will reject a match between -# the prototype and the implementation of a member function even if there is -# only one candidate or it is obvious which candidate to choose by doing a -# simple string match. By disabling STRICT_PROTO_MATCHING doxygen will still -# accept a match between prototype and implementation in such cases. -# The default value is: NO. - -STRICT_PROTO_MATCHING = NO - -# The GENERATE_TODOLIST tag can be used to enable (YES) or disable (NO) the todo -# list. This list is created by putting \todo commands in the documentation. -# The default value is: YES. - -GENERATE_TODOLIST = YES - -# The GENERATE_TESTLIST tag can be used to enable (YES) or disable (NO) the test -# list. This list is created by putting \test commands in the documentation. -# The default value is: YES. - -GENERATE_TESTLIST = YES - -# The GENERATE_BUGLIST tag can be used to enable (YES) or disable (NO) the bug -# list. This list is created by putting \bug commands in the documentation. -# The default value is: YES. - -GENERATE_BUGLIST = YES - -# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or disable (NO) -# the deprecated list. This list is created by putting \deprecated commands in -# the documentation. -# The default value is: YES. - -GENERATE_DEPRECATEDLIST= YES - -# The ENABLED_SECTIONS tag can be used to enable conditional documentation -# sections, marked by \if ... \endif and \cond -# ... \endcond blocks. - -ENABLED_SECTIONS = - -# The MAX_INITIALIZER_LINES tag determines the maximum number of lines that the -# initial value of a variable or macro / define can have for it to appear in the -# documentation. If the initializer consists of more lines than specified here -# it will be hidden. Use a value of 0 to hide initializers completely. The -# appearance of the value of individual variables and macros / defines can be -# controlled using \showinitializer or \hideinitializer command in the -# documentation regardless of this setting. -# Minimum value: 0, maximum value: 10000, default value: 30. - -MAX_INITIALIZER_LINES = 30 - -# Set the SHOW_USED_FILES tag to NO to disable the list of files generated at -# the bottom of the documentation of classes and structs. If set to YES, the -# list will mention the files that were used to generate the documentation. -# The default value is: YES. - -SHOW_USED_FILES = YES - -# Set the SHOW_FILES tag to NO to disable the generation of the Files page. This -# will remove the Files entry from the Quick Index and from the Folder Tree View -# (if specified). -# The default value is: YES. - -SHOW_FILES = YES - -# Set the SHOW_NAMESPACES tag to NO to disable the generation of the Namespaces -# page. This will remove the Namespaces entry from the Quick Index and from the -# Folder Tree View (if specified). -# The default value is: YES. - -SHOW_NAMESPACES = YES - -# The FILE_VERSION_FILTER tag can be used to specify a program or script that -# doxygen should invoke to get the current version for each file (typically from -# the version control system). Doxygen will invoke the program by executing (via -# popen()) the command command input-file, where command is the value of the -# FILE_VERSION_FILTER tag, and input-file is the name of an input file provided -# by doxygen. Whatever the program writes to standard output is used as the file -# version. For an example see the documentation. - -FILE_VERSION_FILTER = - -# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed -# by doxygen. The layout file controls the global structure of the generated -# output files in an output format independent way. To create the layout file -# that represents doxygen's defaults, run doxygen with the -l option. You can -# optionally specify a file name after the option, if omitted DoxygenLayout.xml -# will be used as the name of the layout file. -# -# Note that if you run doxygen from a directory containing a file called -# DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE -# tag is left empty. - -LAYOUT_FILE = $(MLPERF_LOADGEN_SRC_PATH)/docs/src/doxygen_layout.xml - -# The CITE_BIB_FILES tag can be used to specify one or more bib files containing -# the reference definitions. This must be a list of .bib files. The .bib -# extension is automatically appended if omitted. This requires the bibtex tool -# to be installed. See also http://en.wikipedia.org/wiki/BibTeX for more info. -# For LaTeX the style of the bibliography can be controlled using -# LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the -# search path. See also \cite for info how to create references. - -CITE_BIB_FILES = - -#--------------------------------------------------------------------------- -# Configuration options related to warning and progress messages -#--------------------------------------------------------------------------- - -# The QUIET tag can be used to turn on/off the messages that are generated to -# standard output by doxygen. If QUIET is set to YES this implies that the -# messages are off. -# The default value is: NO. - -QUIET = NO - -# The WARNINGS tag can be used to turn on/off the warning messages that are -# generated to standard error (stderr) by doxygen. If WARNINGS is set to YES -# this implies that the warnings are on. -# -# Tip: Turn warnings on while writing the documentation. -# The default value is: YES. - -WARNINGS = YES - -# If the WARN_IF_UNDOCUMENTED tag is set to YES then doxygen will generate -# warnings for undocumented members. If EXTRACT_ALL is set to YES then this flag -# will automatically be disabled. -# The default value is: YES. - -WARN_IF_UNDOCUMENTED = NO - -# If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for -# potential errors in the documentation, such as not documenting some parameters -# in a documented function, or documenting parameters that don't exist or using -# markup commands wrongly. -# The default value is: YES. - -WARN_IF_DOC_ERROR = YES - -# This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that -# are documented, but have no documentation for their parameters or return -# value. If set to NO, doxygen will only warn about wrong or incomplete -# parameter documentation, but not about the absence of documentation. -# The default value is: NO. - -WARN_NO_PARAMDOC = NO - -# If the WARN_AS_ERROR tag is set to YES then doxygen will immediately stop when -# a warning is encountered. -# The default value is: NO. - -WARN_AS_ERROR = NO - -# The WARN_FORMAT tag determines the format of the warning messages that doxygen -# can produce. The string should contain the $file, $line, and $text tags, which -# will be replaced by the file and line number from which the warning originated -# and the warning text. Optionally the format may contain $version, which will -# be replaced by the version of the file (if it could be obtained via -# FILE_VERSION_FILTER) -# The default value is: $file:$line: $text. - -WARN_FORMAT = "$file:$line: $text" - -# The WARN_LOGFILE tag can be used to specify a file to which warning and error -# messages should be written. If left blank the output is written to standard -# error (stderr). - -WARN_LOGFILE = - -#--------------------------------------------------------------------------- -# Configuration options related to the input files -#--------------------------------------------------------------------------- - -# The INPUT tag is used to specify the files and/or directories that contain -# documented source files. You may enter file names like myfile.cpp or -# directories like /usr/src/myproject. Separate the files or directories with -# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING -# Note: If this tag is empty the current directory is searched. - -INPUT = $(MLPERF_LOADGEN_SRC_PATH) - -# This tag can be used to specify the character encoding of the source files -# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses -# libiconv (or the iconv built into libc) for the transcoding. See the libiconv -# documentation (see: http://www.gnu.org/software/libiconv) for the list of -# possible encodings. -# The default value is: UTF-8. - -INPUT_ENCODING = UTF-8 - -# If the value of the INPUT tag contains directories, you can use the -# FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and -# *.h) to filter out the source-files in the directories. -# -# Note that for custom extensions or not directly supported extensions you also -# need to set EXTENSION_MAPPING for the extension otherwise the files are not -# read by doxygen. -# -# If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp, -# *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, -# *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc, -# *.m, *.markdown, *.md, *.mm, *.dox, *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, -# *.f, *.for, *.tcl, *.vhd, *.vhdl, *.ucf and *.qsf. - -FILE_PATTERNS = *.c \ - *.cc \ - *.cxx \ - *.cpp \ - *.c++ \ - *.java \ - *.ii \ - *.ixx \ - *.ipp \ - *.i++ \ - *.inl \ - *.idl \ - *.ddl \ - *.odl \ - *.h \ - *.hh \ - *.hxx \ - *.hpp \ - *.h++ \ - *.cs \ - *.d \ - *.php \ - *.php4 \ - *.php5 \ - *.phtml \ - *.inc \ - *.m \ - *.markdown \ - *.md \ - *.mm \ - *.dox \ - *.py \ - *.pyw \ - *.f90 \ - *.f95 \ - *.f03 \ - *.f08 \ - *.f \ - *.for \ - *.tcl \ - *.vhd \ - *.vhdl \ - *.ucf \ - *.qsf - -# The RECURSIVE tag can be used to specify whether or not subdirectories should -# be searched for input files as well. -# The default value is: NO. - -RECURSIVE = YES - -# The EXCLUDE tag can be used to specify files and/or directories that should be -# excluded from the INPUT source files. This way you can easily exclude a -# subdirectory from a directory tree whose root is specified with the INPUT tag. -# -# Note that relative paths are relative to the directory from which doxygen is -# run. - -EXCLUDE = depot_tools - -# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or -# directories that are symbolic links (a Unix file system feature) are excluded -# from the input. -# The default value is: NO. - -EXCLUDE_SYMLINKS = NO - -# If the value of the INPUT tag contains directories, you can use the -# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude -# certain files from those directories. -# -# Note that the wildcards are matched against the file with absolute path, so to -# exclude all test directories for example use the pattern */test/* - -EXCLUDE_PATTERNS = - -# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names -# (namespaces, classes, functions, etc.) that should be excluded from the -# output. The symbol name can be a fully qualified name, a word, or if the -# wildcard * is used, a substring. Examples: ANamespace, AClass, -# AClass::ANamespace, ANamespace::*Test -# -# Note that the wildcards are matched against the file with absolute path, so to -# exclude all test directories use the pattern */test/* - -EXCLUDE_SYMBOLS = - -# The EXAMPLE_PATH tag can be used to specify one or more files or directories -# that contain example code fragments that are included (see the \include -# command). - -EXAMPLE_PATH = - -# If the value of the EXAMPLE_PATH tag contains directories, you can use the -# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and -# *.h) to filter out the source-files in the directories. If left blank all -# files are included. - -EXAMPLE_PATTERNS = * - -# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be -# searched for input files to be used with the \include or \dontinclude commands -# irrespective of the value of the RECURSIVE tag. -# The default value is: NO. - -EXAMPLE_RECURSIVE = NO - -# The IMAGE_PATH tag can be used to specify one or more files or directories -# that contain images that are to be included in the documentation (see the -# \image command). - -IMAGE_PATH = $(MLPERF_LOADGEN_SRC_PATH)/docs/src - -# The INPUT_FILTER tag can be used to specify a program that doxygen should -# invoke to filter for each input file. Doxygen will invoke the filter program -# by executing (via popen()) the command: -# -# -# -# where is the value of the INPUT_FILTER tag, and is the -# name of an input file. Doxygen will then use the output that the filter -# program writes to standard output. If FILTER_PATTERNS is specified, this tag -# will be ignored. -# -# Note that the filter must not add or remove lines; it is applied before the -# code is scanned, but not when the output code is generated. If lines are added -# or removed, the anchors will not be placed correctly. -# -# Note that for custom extensions or not directly supported extensions you also -# need to set EXTENSION_MAPPING for the extension otherwise the files are not -# properly processed by doxygen. - -INPUT_FILTER = - -# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern -# basis. Doxygen will compare the file name with each pattern and apply the -# filter if there is a match. The filters are a list of the form: pattern=filter -# (like *.cpp=my_cpp_filter). See INPUT_FILTER for further information on how -# filters are used. If the FILTER_PATTERNS tag is empty or if none of the -# patterns match the file name, INPUT_FILTER is applied. -# -# Note that for custom extensions or not directly supported extensions you also -# need to set EXTENSION_MAPPING for the extension otherwise the files are not -# properly processed by doxygen. - -FILTER_PATTERNS = - -# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using -# INPUT_FILTER) will also be used to filter the input files that are used for -# producing the source files to browse (i.e. when SOURCE_BROWSER is set to YES). -# The default value is: NO. - -FILTER_SOURCE_FILES = NO - -# The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file -# pattern. A pattern will override the setting for FILTER_PATTERN (if any) and -# it is also possible to disable source filtering for a specific pattern using -# *.ext= (so without naming a filter). -# This tag requires that the tag FILTER_SOURCE_FILES is set to YES. - -FILTER_SOURCE_PATTERNS = - -# If the USE_MDFILE_AS_MAINPAGE tag refers to the name of a markdown file that -# is part of the input, its contents will be placed on the main page -# (index.html). This can be useful if you have a project on for instance GitHub -# and want to reuse the introduction page also for the doxygen output. - -USE_MDFILE_AS_MAINPAGE = - -#--------------------------------------------------------------------------- -# Configuration options related to source browsing -#--------------------------------------------------------------------------- - -# If the SOURCE_BROWSER tag is set to YES then a list of source files will be -# generated. Documented entities will be cross-referenced with these sources. -# -# Note: To get rid of all source code in the generated output, make sure that -# also VERBATIM_HEADERS is set to NO. -# The default value is: NO. - -SOURCE_BROWSER = YES - -# Setting the INLINE_SOURCES tag to YES will include the body of functions, -# classes and enums directly into the documentation. -# The default value is: NO. - -INLINE_SOURCES = NO - -# Setting the STRIP_CODE_COMMENTS tag to YES will instruct doxygen to hide any -# special comment blocks from generated source code fragments. Normal C, C++ and -# Fortran comments will always remain visible. -# The default value is: YES. - -STRIP_CODE_COMMENTS = YES - -# If the REFERENCED_BY_RELATION tag is set to YES then for each documented -# function all documented functions referencing it will be listed. -# The default value is: NO. - -REFERENCED_BY_RELATION = NO - -# If the REFERENCES_RELATION tag is set to YES then for each documented function -# all documented entities called/used by that function will be listed. -# The default value is: NO. - -REFERENCES_RELATION = NO - -# If the REFERENCES_LINK_SOURCE tag is set to YES and SOURCE_BROWSER tag is set -# to YES then the hyperlinks from functions in REFERENCES_RELATION and -# REFERENCED_BY_RELATION lists will link to the source code. Otherwise they will -# link to the documentation. -# The default value is: YES. - -REFERENCES_LINK_SOURCE = YES - -# If SOURCE_TOOLTIPS is enabled (the default) then hovering a hyperlink in the -# source code will show a tooltip with additional information such as prototype, -# brief description and links to the definition and documentation. Since this -# will make the HTML file larger and loading of large files a bit slower, you -# can opt to disable this feature. -# The default value is: YES. -# This tag requires that the tag SOURCE_BROWSER is set to YES. - -SOURCE_TOOLTIPS = YES - -# If the USE_HTAGS tag is set to YES then the references to source code will -# point to the HTML generated by the htags(1) tool instead of doxygen built-in -# source browser. The htags tool is part of GNU's global source tagging system -# (see http://www.gnu.org/software/global/global.html). You will need version -# 4.8.6 or higher. -# -# To use it do the following: -# - Install the latest version of global -# - Enable SOURCE_BROWSER and USE_HTAGS in the config file -# - Make sure the INPUT points to the root of the source tree -# - Run doxygen as normal -# -# Doxygen will invoke htags (and that will in turn invoke gtags), so these -# tools must be available from the command line (i.e. in the search path). -# -# The result: instead of the source browser generated by doxygen, the links to -# source code will now point to the output of htags. -# The default value is: NO. -# This tag requires that the tag SOURCE_BROWSER is set to YES. - -USE_HTAGS = NO - -# If the VERBATIM_HEADERS tag is set the YES then doxygen will generate a -# verbatim copy of the header file for each class for which an include is -# specified. Set to NO to disable this. -# See also: Section \class. -# The default value is: YES. - -VERBATIM_HEADERS = YES - -# If the CLANG_ASSISTED_PARSING tag is set to YES then doxygen will use the -# clang parser (see: http://clang.llvm.org/) for more accurate parsing at the -# cost of reduced performance. This can be particularly helpful with template -# rich C++ code for which doxygen's built-in parser lacks the necessary type -# information. -# Note: The availability of this option depends on whether or not doxygen was -# generated with the -Duse-libclang=ON option for CMake. -# The default value is: NO. - -CLANG_ASSISTED_PARSING = YES - -# If clang assisted parsing is enabled you can provide the compiler with command -# line options that you would normally use when invoking the compiler. Note that -# the include paths will already be set by doxygen for the files and directories -# specified with INPUT and INCLUDE_PATH. -# This tag requires that the tag CLANG_ASSISTED_PARSING is set to YES. - -CLANG_OPTIONS = -I ../third_party/pybind/include --std=c++14 - -#--------------------------------------------------------------------------- -# Configuration options related to the alphabetical class index -#--------------------------------------------------------------------------- - -# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index of all -# compounds will be generated. Enable this if the project contains a lot o= -# classes, structs, unions or interfaces. -# The default value is: YES. - -ALPHABETICAL_INDEX = YES - -# The COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns in -# which the alphabetical index list will be split. -# Minimum value: 1, maximum value: 20, default value: 5. -# This tag requires that the tag ALPHABETICAL_INDEX is set to YES. - -COLS_IN_ALPHA_INDEX = 5 - -# In case all classes in a project start with a common prefix, all classes will -# be put under the same header in the alphabetical index. The IGNORE_PREFIX tag -# can be used to specify a prefix (or a list of prefixes) that should be ignored -# while generating the index headers. -# This tag requires that the tag ALPHABETICAL_INDEX is set to YES. - -IGNORE_PREFIX = - -#--------------------------------------------------------------------------- -# Configuration options related to the HTML output -#--------------------------------------------------------------------------- - -# If the GENERATE_HTML tag is set to YES, doxygen will generate HTML output -# The default value is: YES. - -GENERATE_HTML = YES - -# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a -# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of -# it. -# The default directory is: html. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_OUTPUT = html - -# The HTML_FILE_EXTENSION tag can be used to specify the file extension for each -# generated HTML page (for example: .htm, .php, .asp). -# The default value is: .html. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_FILE_EXTENSION = .html - -# The HTML_HEADER tag can be used to specify a user-defined HTML header file for -# each generated HTML page. If the tag is left blank doxygen will generate a -# standard header. -# -# To get valid HTML the header file that includes any scripts and style sheets -# that doxygen needs, which is dependent on the configuration options used (e.g. -# the setting GENERATE_TREEVIEW). It is highly recommended to start with a -# default header using -# doxygen -w html new_header.html new_footer.html new_stylesheet.css -# YourConfigFile -# and then modify the file new_header.html. See also section "Doxygen usage" -# for information on how to generate the default header that doxygen normally -# uses. -# Note: The header is subject to change so you typically have to regenerate the -# default header when upgrading to a newer version of doxygen. For a description -# of the possible markers and block names see the documentation. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_HEADER = $(MLPERF_LOADGEN_SRC_PATH)/docs/src/doxygen_header.html - -# The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each -# generated HTML page. If the tag is left blank doxygen will generate a standard -# footer. See HTML_HEADER for more information on how to generate a default -# footer and what special commands can be used inside the footer. See also -# section "Doxygen usage" for information on how to generate the default footer -# that doxygen normally uses. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_FOOTER = $(MLPERF_LOADGEN_SRC_PATH)/docs/src/doxygen_footer.html - -# The HTML_STYLESHEET tag can be used to specify a user-defined cascading style -# sheet that is used by each HTML page. It can be used to fine-tune the look of -# the HTML output. If left blank doxygen will generate a default style sheet. -# See also section "Doxygen usage" for information on how to generate the style -# sheet that doxygen normally uses. -# Note: It is recommended to use HTML_EXTRA_STYLESHEET instead of this tag, as -# it is more robust and this tag (HTML_STYLESHEET) will in the future become -# obsolete. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_STYLESHEET = - -# The HTML_EXTRA_STYLESHEET tag can be used to specify additional user-defined -# cascading style sheets that are included after the standard style sheets -# created by doxygen. Using this option one can overrule certain style aspects. -# This is preferred over using HTML_STYLESHEET since it does not replace the -# standard style sheet and is therefore more robust against future updates. -# Doxygen will copy the style sheet files to the output directory. -# Note: The order of the extra style sheet files is of importance (e.g. the last -# style sheet in the list overrules the setting of the previous ones in the -# list). For an example see the documentation. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_EXTRA_STYLESHEET = $(MLPERF_LOADGEN_SRC_PATH)/docs/src/doxygen_stylesheet.css - -# The HTML_EXTRA_FILES tag can be used to specify one or more extra images or -# other source files which should be copied to the HTML output directory. Note -# that these files will be copied to the base HTML output directory. Use the -# $relpath^ marker in the HTML_HEADER and/or HTML_FOOTER files to load these -# files. In the HTML_STYLESHEET file, use the file name only. Also note that the -# files will be copied as-is; there are no commands or markers available. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_EXTRA_FILES = $(MLPERF_LOADGEN_SRC_PATH)/docs/src/mlperf_icon.png \ - $(MLPERF_LOADGEN_SRC_PATH)/loadgen_integration_diagram.svg - -# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen -# will adjust the colors in the style sheet and background images according to -# this color. Hue is specified as an angle on a colorwheel, see -# http://en.wikipedia.org/wiki/Hue for more information. For instance the value -# 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300 -# purple, and 360 is red again. -# Minimum value: 0, maximum value: 359, default value: 220. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_COLORSTYLE_HUE = 220 - -# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors -# in the HTML output. For a value of 0 the output will use grayscales only. A -# value of 255 will produce the most vivid colors. -# Minimum value: 0, maximum value: 255, default value: 100. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_COLORSTYLE_SAT = 127 - -# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to the -# luminance component of the colors in the HTML output. Values below 100 -# gradually make the output lighter, whereas values above 100 make the output -# darker. The value divided by 100 is the actual gamma applied, so 80 represents -# a gamma of 0.8, The value 220 represents a gamma of 2.2, and 100 does not -# change the gamma. -# Minimum value: 40, maximum value: 240, default value: 80. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_COLORSTYLE_GAMMA = 80 - -# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML -# page will contain the date and time when the page was generated. Setting this -# to YES can help to show when doxygen was last run and thus if the -# documentation is up to date. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_TIMESTAMP = NO - -# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML -# documentation will contain sections that can be hidden and shown after the -# page has loaded. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_DYNAMIC_SECTIONS = YES - -# With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries -# shown in the various tree structured indices initially; the user can expand -# and collapse entries dynamically later on. Doxygen will expand the tree to -# such a level that at most the specified number of entries are visible (unless -# a fully collapsed tree already exceeds this amount). So setting the number of -# entries 1 will produce a full collapsed tree by default. 0 is a special value -# representing an infinite number of entries and will result in a full expanded -# tree by default. -# Minimum value: 0, maximum value: 9999, default value: 100. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_INDEX_NUM_ENTRIES = 50 - -# If the GENERATE_DOCSET tag is set to YES, additional index files will be -# generated that can be used as input for Apple's Xcode 3 integrated development -# environment (see: http://developer.apple.com/tools/xcode/), introduced with -# OSX 10.5 (Leopard). To create a documentation set, doxygen will generate a -# Makefile in the HTML output directory. Running make will produce the docset in -# that directory and running make install will install the docset in -# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at -# startup. See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html -# for more information. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -GENERATE_DOCSET = NO - -# This tag determines the name of the docset feed. A documentation feed provides -# an umbrella under which multiple documentation sets from a single provider -# (such as a company or product suite) can be grouped. -# The default value is: Doxygen generated docs. -# This tag requires that the tag GENERATE_DOCSET is set to YES. - -DOCSET_FEEDNAME = "Doxygen generated docs" - -# This tag specifies a string that should uniquely identify the documentation -# set bundle. This should be a reverse domain-name style string, e.g. -# com.mycompany.MyDocSet. Doxygen will append .docset to the name. -# The default value is: org.doxygen.Project. -# This tag requires that the tag GENERATE_DOCSET is set to YES. - -DOCSET_BUNDLE_ID = org.doxygen.Project - -# The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify -# the documentation publisher. This should be a reverse domain-name style -# string, e.g. com.mycompany.MyDocSet.documentation. -# The default value is: org.doxygen.Publisher. -# This tag requires that the tag GENERATE_DOCSET is set to YES. - -DOCSET_PUBLISHER_ID = org.doxygen.Publisher - -# The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher. -# The default value is: Publisher. -# This tag requires that the tag GENERATE_DOCSET is set to YES. - -DOCSET_PUBLISHER_NAME = Publisher - -# If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three -# additional HTML index files: index.hhp, index.hhc, and index.hhk. The -# index.hhp is a project file that can be read by Microsoft's HTML Help Workshop -# (see: http://www.microsoft.com/en-us/download/details.aspx?id=21138) on -# Windows. -# -# The HTML Help Workshop contains a compiler that can convert all HTML output -# generated by doxygen into a single compiled HTML file (.chm). Compiled HTML -# files are now used as the Windows 98 help format, and will replace the old -# Windows help format (.hlp) on all Windows platforms in the future. Compressed -# HTML files also contain an index, a table of contents, and you can search for -# words in the documentation. The HTML workshop also contains a viewer for -# compressed HTML files. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -GENERATE_HTMLHELP = NO - -# The CHM_FILE tag can be used to specify the file name of the resulting .chm -# file. You can add a path in front of the file if the result should not be -# written to the html output directory. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -CHM_FILE = - -# The HHC_LOCATION tag can be used to specify the location (absolute path -# including file name) of the HTML help compiler (hhc.exe). If non-empty, -# doxygen will try to run the HTML help compiler on the generated index.hhp. -# The file has to be specified with full path. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -HHC_LOCATION = - -# The GENERATE_CHI flag controls if a separate .chi index file is generated -# (YES) or that it should be included in the master .chm file (NO). -# The default value is: NO. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -GENERATE_CHI = NO - -# The CHM_INDEX_ENCODING is used to encode HtmlHelp index (hhk), content (hhc) -# and project file content. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -CHM_INDEX_ENCODING = - -# The BINARY_TOC flag controls whether a binary table of contents is generated -# (YES) or a normal table of contents (NO) in the .chm file. Furthermore it -# enables the Previous and Next buttons. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -BINARY_TOC = NO - -# The TOC_EXPAND flag can be set to YES to add extra items for group members to -# the table of contents of the HTML help documentation and to the tree view. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -TOC_EXPAND = NO - -# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and -# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that -# can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help -# (.qch) of the generated HTML documentation. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -GENERATE_QHP = NO - -# If the QHG_LOCATION tag is specified, the QCH_FILE tag can be used to specify -# the file name of the resulting .qch file. The path specified is relative to -# the HTML output folder. -# This tag requires that the tag GENERATE_QHP is set to YES. - -QCH_FILE = - -# The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help -# Project output. For more information please see Qt Help Project / Namespace -# (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#namespace). -# The default value is: org.doxygen.Project. -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHP_NAMESPACE = org.doxygen.Project - -# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt -# Help Project output. For more information please see Qt Help Project / Virtual -# Folders (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#virtual- -# folders). -# The default value is: doc. -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHP_VIRTUAL_FOLDER = doc - -# If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom -# filter to add. For more information please see Qt Help Project / Custom -# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom- -# filters). -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHP_CUST_FILTER_NAME = - -# The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the -# custom filter to add. For more information please see Qt Help Project / Custom -# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom- -# filters). -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHP_CUST_FILTER_ATTRS = - -# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this -# project's filter section matches. Qt Help Project / Filter Attributes (see: -# http://qt-project.org/doc/qt-4.8/qthelpproject.html#filter-attributes). -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHP_SECT_FILTER_ATTRS = - -# The QHG_LOCATION tag can be used to specify the location of Qt's -# qhelpgenerator. If non-empty doxygen will try to run qhelpgenerator on the -# generated .qhp file. -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHG_LOCATION = - -# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files will be -# generated, together with the HTML files, they form an Eclipse help plugin. To -# install this plugin and make it available under the help contents menu in -# Eclipse, the contents of the directory containing the HTML and XML files needs -# to be copied into the plugins directory of eclipse. The name of the directory -# within the plugins directory should be the same as the ECLIPSE_DOC_ID value. -# After copying Eclipse needs to be restarted before the help appears. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -GENERATE_ECLIPSEHELP = NO - -# A unique identifier for the Eclipse help plugin. When installing the plugin -# the directory name containing the HTML and XML files should also have this -# name. Each documentation set should have its own identifier. -# The default value is: org.doxygen.Project. -# This tag requires that the tag GENERATE_ECLIPSEHELP is set to YES. - -ECLIPSE_DOC_ID = org.doxygen.Project - -# If you want full control over the layout of the generated HTML pages it might -# be necessary to disable the index and replace it with your own. The -# DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) at top -# of each HTML page. A value of NO enables the index and the value YES disables -# it. Since the tabs in the index contain the same information as the navigation -# tree, you can set this option to YES if you also set GENERATE_TREEVIEW to YES. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -DISABLE_INDEX = NO - -# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index -# structure should be generated to display hierarchical information. If the tag -# value is set to YES, a side panel will be generated containing a tree-like -# index structure (just like the one that is generated for HTML Help). For this -# to work a browser that supports JavaScript, DHTML, CSS and frames is required -# (i.e. any modern browser). Windows users are probably better off using the -# HTML help feature. Via custom style sheets (see HTML_EXTRA_STYLESHEET) one can -# further fine-tune the look of the index. As an example, the default style -# sheet generated by doxygen has an example that shows how to put an image at -# the root of the tree instead of the PROJECT_NAME. Since the tree basically has -# the same information as the tab index, you could consider setting -# DISABLE_INDEX to YES when enabling this option. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -GENERATE_TREEVIEW = YES - -# The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that -# doxygen will group on one line in the generated HTML documentation. -# -# Note that a value of 0 will completely suppress the enum values from appearing -# in the overview section. -# Minimum value: 0, maximum value: 20, default value: 4. -# This tag requires that the tag GENERATE_HTML is set to YES. - -ENUM_VALUES_PER_LINE = 4 - -# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be used -# to set the initial width (in pixels) of the frame in which the tree is shown. -# Minimum value: 0, maximum value: 1500, default value: 250. -# This tag requires that the tag GENERATE_HTML is set to YES. - -TREEVIEW_WIDTH = 250 - -# If the EXT_LINKS_IN_WINDOW option is set to YES, doxygen will open links to -# external symbols imported via tag files in a separate window. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -EXT_LINKS_IN_WINDOW = NO - -# Use this tag to change the font size of LaTeX formulas included as images in -# the HTML documentation. When you change the font size after a successful -# doxygen run you need to manually remove any form_*.png images from the HTML -# output directory to force them to be regenerated. -# Minimum value: 8, maximum value: 50, default value: 10. -# This tag requires that the tag GENERATE_HTML is set to YES. - -FORMULA_FONTSIZE = 10 - -# Use the FORMULA_TRANPARENT tag to determine whether or not the images -# generated for formulas are transparent PNGs. Transparent PNGs are not -# supported properly for IE 6.0, but are supported on all modern browsers. -# -# Note that when changing this option you need to delete any form_*.png files in -# the HTML output directory before the changes have effect. -# The default value is: YES. -# This tag requires that the tag GENERATE_HTML is set to YES. - -FORMULA_TRANSPARENT = YES - -# Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see -# http://www.mathjax.org) which uses client side Javascript for the rendering -# instead of using pre-rendered bitmaps. Use this if you do not have LaTeX -# installed or if you want to formulas look prettier in the HTML output. When -# enabled you may also need to install MathJax separately and configure the path -# to it using the MATHJAX_RELPATH option. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -USE_MATHJAX = NO - -# When MathJax is enabled you can set the default output format to be used for -# the MathJax output. See the MathJax site (see: -# http://docs.mathjax.org/en/latest/output.html) for more details. -# Possible values are: HTML-CSS (which is slower, but has the best -# compatibility), NativeMML (i.e. MathML) and SVG. -# The default value is: HTML-CSS. -# This tag requires that the tag USE_MATHJAX is set to YES. - -MATHJAX_FORMAT = HTML-CSS - -# When MathJax is enabled you need to specify the location relative to the HTML -# output directory using the MATHJAX_RELPATH option. The destination directory -# should contain the MathJax.js script. For instance, if the mathjax directory -# is located at the same level as the HTML output directory, then -# MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax -# Content Delivery Network so you can quickly see the result without installing -# MathJax. However, it is strongly recommended to install a local copy of -# MathJax from http://www.mathjax.org before deployment. -# The default value is: http://cdn.mathjax.org/mathjax/latest. -# This tag requires that the tag USE_MATHJAX is set to YES. - -MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest - -# The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax -# extension names that should be enabled during MathJax rendering. For example -# MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols -# This tag requires that the tag USE_MATHJAX is set to YES. - -MATHJAX_EXTENSIONS = - -# The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces -# of code that will be used on startup of the MathJax code. See the MathJax site -# (see: http://docs.mathjax.org/en/latest/output.html) for more details. For an -# example see the documentation. -# This tag requires that the tag USE_MATHJAX is set to YES. - -MATHJAX_CODEFILE = - -# When the SEARCHENGINE tag is enabled doxygen will generate a search box for -# the HTML output. The underlying search engine uses javascript and DHTML and -# should work on any modern browser. Note that when using HTML help -# (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets (GENERATE_DOCSET) -# there is already a search function so this one should typically be disabled. -# For large projects the javascript based search engine can be slow, then -# enabling SERVER_BASED_SEARCH may provide a better solution. It is possible to -# search using the keyboard; to jump to the search box use + S -# (what the is depends on the OS and browser, but it is typically -# , /

- - -
-$generatedby   -doxygen - $doxygenversion -
- - - diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/doxygen_header.html b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/doxygen_header.html deleted file mode 100644 index 91d214b95..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/doxygen_header.html +++ /dev/null @@ -1,49 +0,0 @@ - - - - - - - - - -LoadGen: $title -$title - - - -$treeview -$search -$mathjax - -$extrastylesheet - - -
- - -
- - MLPerf - - -
-
$projectname -  $projectnumber -
-
$projectbrief
-
- - - -
$projectbrief
- - - - -
$searchbox
- - -
- - diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/doxygen_html_generator.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/doxygen_html_generator.py deleted file mode 100644 index 4065d7bd0..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/doxygen_html_generator.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2019 The MLPerf Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -# \file -# \brief A script that sets the environment variables expected by doxygen.cfg. -# \details This can be run manually without any arguments, but also allows a -# build system to customize the output directory. - -import os -import sys - - -def generate_doxygen_html(doxygen_out_dir, loadgen_root): - os.environ["MLPERF_LOADGEN_SRC_PATH"] = loadgen_root - os.environ["MLPERF_DOXYGEN_OUT_PATH"] = doxygen_out_dir - os.popen("doxygen " + loadgen_root + "/docs/src/doxygen.cfg") - - -def main(argv): - doxygen_out_dir = "./docs/gen" if len(argv) < 2 else argv[1] - loadgen_root = "." if len(argv) < 3 else argv[2] - generate_doxygen_html(doxygen_out_dir, loadgen_root) - - -main(sys.argv) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/doxygen_layout.xml b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/doxygen_layout.xml deleted file mode 100644 index 1fc5a9cb4..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/doxygen_layout.xml +++ /dev/null @@ -1,211 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/doxygen_stylesheet.css b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/doxygen_stylesheet.css deleted file mode 100644 index 3bd61261c..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/doxygen_stylesheet.css +++ /dev/null @@ -1,1629 +0,0 @@ -/* The standard CSS for doxygen 1.8.13 */ - -body, table, div, p, dl { - font: 400 14px/22px Roboto,sans-serif; -} - -p.reference, p.definition { - font: 400 14px/22px Roboto,sans-serif; -} - -/* @group Heading Levels */ - -h1.groupheader { - font-size: 150%; -} - -.title { - font: 400 14px/28px Roboto,sans-serif; - font-size: 175%; - font-weight: bold; - margin: 10px 2px; - color: #135384; -} - -h2.groupheader { - border-bottom: 1px solid #879ECB; - color: #354C7B; - font-size: 150%; - font-weight: normal; - margin-top: 1.75em; - padding-top: 8px; - padding-bottom: 4px; - width: 100%; -} - -h3.groupheader { - font-size: 100%; -} - -h1, h2, h3, h4, h5, h6 { - -webkit-transition: text-shadow 0.5s linear; - -moz-transition: text-shadow 0.5s linear; - -ms-transition: text-shadow 0.5s linear; - -o-transition: text-shadow 0.5s linear; - transition: text-shadow 0.5s linear; - margin-right: 15px; - color: #135384; - -} - -h1.glow, h2.glow, h3.glow, h4.glow, h5.glow, h6.glow { - text-shadow: 0 0 15px cyan; -} - -dt { - font-weight: bold; -} - -div.multicol { - -moz-column-gap: 1em; - -webkit-column-gap: 1em; - -moz-column-count: 3; - -webkit-column-count: 3; -} - -p.startli, p.startdd { - margin-top: 2px; -} - -p.starttd { - margin-top: 0px; -} - -p.endli { - margin-bottom: 0px; -} - -p.enddd { - margin-bottom: 4px; -} - -p.endtd { - margin-bottom: 2px; -} - -/* @end */ - -caption { - font-weight: bold; -} - -span.legend { - font-size: 70%; - text-align: center; -} - -h3.version { - font-size: 90%; - text-align: center; -} - -div.qindex, div.navtab{ - background-color: #EBEFF6; - border: 1px solid #A3B4D7; - text-align: center; -} - -div.qindex, div.navpath { - width: 100%; - line-height: 140%; -} - -div.navtab { - margin-right: 15px; -} - -/* @group Link Styling */ - -a { - color: #3D578C; - font-weight: normal; - text-decoration: none; -} - -.contents a:visited { - color: #4665A2; -} - -a:hover { - text-decoration: underline; -} - -a.qindex { - font-weight: bold; -} - -a.qindexHL { - font-weight: bold; - background-color: #9CAFD4; - color: #ffffff; - border: 1px double #869DCA; -} - -.contents a.qindexHL:visited { - color: #ffffff; -} - -a.el { - font-weight: bold; -} - -a.elRef { -} - -a.code, a.code:visited, a.line, a.line:visited { - color: #4665A2; -} - -a.codeRef, a.codeRef:visited, a.lineRef, a.lineRef:visited { - color: #4665A2; -} - -/* @end */ - -dl.el { - margin-left: -1cm; -} - -pre.fragment { - border: 1px solid #C4CFE5; - background-color: #FBFCFD; - padding: 4px 6px; - margin: 4px 8px 4px 2px; - overflow: auto; - word-wrap: break-word; - font-size: 9pt; - line-height: 125%; - font-family: monospace, fixed; - font-size: 105%; -} - -div.fragment { - padding: 0px; - margin: 4px 8px 4px 2px; - background-color: #FBFCFD; - border: 1px solid #C4CFE5; -} - -div.line { - font-family: monospace, fixed; - font-size: 13px; - min-height: 13px; - line-height: 1.0; - text-wrap: unrestricted; - white-space: -moz-pre-wrap; /* Moz */ - white-space: -pre-wrap; /* Opera 4-6 */ - white-space: -o-pre-wrap; /* Opera 7 */ - white-space: pre-wrap; /* CSS3 */ - word-wrap: break-word; /* IE 5.5+ */ - text-indent: -53px; - padding-left: 53px; - padding-bottom: 0px; - margin: 0px; - -webkit-transition-property: background-color, box-shadow; - -webkit-transition-duration: 0.5s; - -moz-transition-property: background-color, box-shadow; - -moz-transition-duration: 0.5s; - -ms-transition-property: background-color, box-shadow; - -ms-transition-duration: 0.5s; - -o-transition-property: background-color, box-shadow; - -o-transition-duration: 0.5s; - transition-property: background-color, box-shadow; - transition-duration: 0.5s; -} - -div.line:after { - content:"\000A"; - white-space: pre; -} - -div.line.glow { - background-color: cyan; - box-shadow: 0 0 10px cyan; -} - - -span.lineno { - padding-right: 4px; - text-align: right; - border-right: 2px solid #0F0; - background-color: #E8E8E8; - white-space: pre; -} -span.lineno a { - background-color: #D8D8D8; -} - -span.lineno a:hover { - background-color: #C8C8C8; -} - -.lineno { - -webkit-touch-callout: none; - -webkit-user-select: none; - -khtml-user-select: none; - -moz-user-select: none; - -ms-user-select: none; - user-select: none; -} - -div.ah, span.ah { - background-color: black; - font-weight: bold; - color: #ffffff; - margin-bottom: 3px; - margin-top: 3px; - padding: 0.2em; - border: solid thin #333; - border-radius: 0.5em; - -webkit-border-radius: .5em; - -moz-border-radius: .5em; - box-shadow: 2px 2px 3px #999; - -webkit-box-shadow: 2px 2px 3px #999; - -moz-box-shadow: rgba(0, 0, 0, 0.15) 2px 2px 2px; - background-image: -webkit-gradient(linear, left top, left bottom, from(#eee), to(#000),color-stop(0.3, #444)); - background-image: -moz-linear-gradient(center top, #eee 0%, #444 40%, #000 110%); -} - -div.classindex ul { - list-style: none; - padding-left: 0; -} - -div.classindex span.ai { - display: inline-block; -} - -div.groupHeader { - margin-left: 16px; - margin-top: 12px; - font-weight: bold; -} - -div.groupText { - margin-left: 16px; - font-style: italic; -} - -body { - background-color: white; - color: black; - margin: 0; -} - -div.contents { - margin-top: 10px; - margin-left: 12px; - margin-right: 8px; -} - -td.indexkey { - background-color: #EBEFF6; - font-weight: bold; - border: 1px solid #C4CFE5; - margin: 2px 0px 2px 0; - padding: 2px 10px; - white-space: nowrap; - vertical-align: top; -} - -td.indexvalue { - background-color: #EBEFF6; - border: 1px solid #C4CFE5; - padding: 2px 10px; - margin: 2px 0px; -} - -tr.memlist { - background-color: #EEF1F7; -} - -p.formulaDsp { - text-align: center; -} - -img.formulaDsp { - -} - -img.formulaInl { - vertical-align: middle; -} - -div.center { - text-align: center; - margin-top: 0px; - margin-bottom: 0px; - padding: 0px; -} - -div.center img { - border: 0px; -} - -address.footer { - text-align: right; - padding-right: 12px; -} - -img.footer { - border: 0px; - vertical-align: middle; -} - -/* @group Code Colorization */ - -span.keyword { - color: #008000 -} - -span.keywordtype { - color: #604020 -} - -span.keywordflow { - color: #e08000 -} - -span.comment { - color: #800000 -} - -span.preprocessor { - color: #806020 -} - -span.stringliteral { - color: #002080 -} - -span.charliteral { - color: #008080 -} - -span.vhdldigit { - color: #ff00ff -} - -span.vhdlchar { - color: #000000 -} - -span.vhdlkeyword { - color: #700070 -} - -span.vhdllogic { - color: #ff0000 -} - -blockquote { - background-color: #F7F8FB; - border-left: 2px solid #9CAFD4; - margin: 0 24px 0 4px; - padding: 0 12px 0 16px; -} - -/* @end */ - -/* -.search { - color: #003399; - font-weight: bold; -} - -form.search { - margin-bottom: 0px; - margin-top: 0px; -} - -input.search { - font-size: 75%; - color: #000080; - font-weight: normal; - background-color: #e8eef2; -} -*/ - -td.tiny { - font-size: 75%; -} - -.dirtab { - padding: 4px; - border-collapse: collapse; - border: 1px solid #A3B4D7; -} - -th.dirtab { - background: #EBEFF6; - font-weight: bold; -} - -hr { - height: 0px; - border: none; - border-top: 1px solid #4A6AAA; -} - -hr.footer { - height: 1px; -} - -/* @group Member Descriptions */ - -table.memberdecls { - border-spacing: 0px; - padding: 0px; -} - -.memberdecls td, .fieldtable tr { - -webkit-transition-property: background-color, box-shadow; - -webkit-transition-duration: 0.5s; - -moz-transition-property: background-color, box-shadow; - -moz-transition-duration: 0.5s; - -ms-transition-property: background-color, box-shadow; - -ms-transition-duration: 0.5s; - -o-transition-property: background-color, box-shadow; - -o-transition-duration: 0.5s; - transition-property: background-color, box-shadow; - transition-duration: 0.5s; -} - -.memberdecls td.glow, .fieldtable tr.glow { - background-color: cyan; - box-shadow: 0 0 15px cyan; -} - -.mdescLeft, .mdescRight, -.memItemLeft, .memItemRight, -.memTemplItemLeft, .memTemplItemRight, .memTemplParams { - background-color: #F9FAFC; - border: none; - margin: 4px; - padding: 1px 0 0 8px; -} - -.mdescLeft, .mdescRight { - padding: 0px 8px 4px 8px; - color: #555; -} - -.memSeparator { - border-bottom: 1px solid #DEE4F0; - line-height: 1px; - margin: 0px; - padding: 0px; -} - -.memItemLeft, .memTemplItemLeft { - white-space: nowrap; -} - -.memItemRight { - width: 100%; -} - -.memTemplParams { - color: #4665A2; - white-space: nowrap; - font-size: 80%; -} - -/* @end */ - -/* @group Member Details */ - -/* Styles for detailed member documentation */ - -.memtitle { - padding: 8px; - border-top: 1px solid #A8B8D9; - border-left: 1px solid #A8B8D9; - border-right: 1px solid #A8B8D9; - border-top-right-radius: 4px; - border-top-left-radius: 4px; - margin-bottom: -1px; - background-image: url('nav_f.png'); - background-repeat: repeat-x; - background-color: #E2E8F2; - line-height: 1.25; - font-weight: 300; - float:left; -} - -.permalink -{ - font-size: 65%; - display: inline-block; - vertical-align: middle; -} - -.memtemplate { - font-size: 80%; - color: #4665A2; - font-weight: normal; - margin-left: 9px; -} - -.memnav { - background-color: #EBEFF6; - border: 1px solid #A3B4D7; - text-align: center; - margin: 2px; - margin-right: 15px; - padding: 2px; -} - -.mempage { - width: 100%; -} - -.memitem { - padding: 0; - margin-bottom: 10px; - margin-right: 5px; - -webkit-transition: box-shadow 0.5s linear; - -moz-transition: box-shadow 0.5s linear; - -ms-transition: box-shadow 0.5s linear; - -o-transition: box-shadow 0.5s linear; - transition: box-shadow 0.5s linear; - display: table !important; - width: 100%; -} - -.memitem.glow { - box-shadow: 0 0 15px cyan; -} - -.memname { - font-weight: 400; - margin-left: 6px; -} - -.memname td { - vertical-align: bottom; -} - -.memproto, dl.reflist dt { - border-top: 1px solid #A8B8D9; - border-left: 1px solid #A8B8D9; - border-right: 1px solid #A8B8D9; - padding: 6px 0px 6px 0px; - color: #253555; - font-weight: bold; - text-shadow: 0px 1px 1px rgba(255, 255, 255, 0.9); - background-color: #DFE5F1; - /* opera specific markup */ - box-shadow: 5px 5px 5px rgba(0, 0, 0, 0.15); - border-top-right-radius: 4px; - /* firefox specific markup */ - -moz-box-shadow: rgba(0, 0, 0, 0.15) 5px 5px 5px; - -moz-border-radius-topright: 4px; - /* webkit specific markup */ - -webkit-box-shadow: 5px 5px 5px rgba(0, 0, 0, 0.15); - -webkit-border-top-right-radius: 4px; - -} - -.overload { - font-family: "courier new",courier,monospace; - font-size: 65%; -} - -.memdoc, dl.reflist dd { - border-bottom: 1px solid #A8B8D9; - border-left: 1px solid #A8B8D9; - border-right: 1px solid #A8B8D9; - padding: 6px 10px 2px 10px; - background-color: #FBFCFD; - border-top-width: 0; - background-image:url('nav_g.png'); - background-repeat:repeat-x; - background-color: #FFFFFF; - /* opera specific markup */ - border-bottom-left-radius: 4px; - border-bottom-right-radius: 4px; - box-shadow: 5px 5px 5px rgba(0, 0, 0, 0.15); - /* firefox specific markup */ - -moz-border-radius-bottomleft: 4px; - -moz-border-radius-bottomright: 4px; - -moz-box-shadow: rgba(0, 0, 0, 0.15) 5px 5px 5px; - /* webkit specific markup */ - -webkit-border-bottom-left-radius: 4px; - -webkit-border-bottom-right-radius: 4px; - -webkit-box-shadow: 5px 5px 5px rgba(0, 0, 0, 0.15); -} - -dl.reflist dt { - padding: 5px; -} - -dl.reflist dd { - margin: 0px 0px 10px 0px; - padding: 5px; -} - -.paramkey { - text-align: right; -} - -.paramtype { - white-space: nowrap; -} - -.paramname { - color: #602020; - white-space: nowrap; -} -.paramname em { - font-style: normal; -} -.paramname code { - line-height: 14px; -} - -.params, .retval, .exception, .tparams { - margin-left: 0px; - padding-left: 0px; -} - -.params .paramname, .retval .paramname { - font-weight: bold; - vertical-align: top; -} - -.params .paramtype { - font-style: italic; - vertical-align: top; -} - -.params .paramdir { - font-family: "courier new",courier,monospace; - vertical-align: top; -} - -table.mlabels { - border-spacing: 0px; -} - -td.mlabels-left { - width: 100%; - padding: 0px; -} - -td.mlabels-right { - vertical-align: bottom; - padding: 0px; - white-space: nowrap; -} - -span.mlabels { - margin-left: 8px; -} - -span.mlabel { - background-color: #728DC1; - border-top:1px solid #5373B4; - border-left:1px solid #5373B4; - border-right:1px solid #C4CFE5; - border-bottom:1px solid #C4CFE5; - text-shadow: none; - color: white; - margin-right: 4px; - padding: 2px 3px; - border-radius: 3px; - font-size: 7pt; - white-space: nowrap; - vertical-align: middle; -} - - - -/* @end */ - -/* these are for tree view inside a (index) page */ - -div.directory { - margin: 10px 0px; - border-top: 1px solid #9CAFD4; - border-bottom: 1px solid #9CAFD4; - width: 100%; -} - -.directory table { - border-collapse:collapse; -} - -.directory td { - margin: 0px; - padding: 0px; - vertical-align: top; -} - -.directory td.entry { - white-space: nowrap; - padding-right: 6px; - padding-top: 3px; -} - -.directory td.entry a { - outline:none; -} - -.directory td.entry a img { - border: none; -} - -.directory td.desc { - width: 100%; - padding-left: 6px; - padding-right: 6px; - padding-top: 3px; - border-left: 1px solid rgba(0,0,0,0.05); -} - -.directory tr.even { - padding-left: 6px; - background-color: #F7F8FB; -} - -.directory img { - vertical-align: -30%; -} - -.directory .levels { - white-space: nowrap; - width: 100%; - text-align: right; - font-size: 9pt; -} - -.directory .levels span { - cursor: pointer; - padding-left: 2px; - padding-right: 2px; - color: #3D578C; -} - -.arrow { - color: #9CAFD4; - -webkit-user-select: none; - -khtml-user-select: none; - -moz-user-select: none; - -ms-user-select: none; - user-select: none; - cursor: pointer; - font-size: 80%; - display: inline-block; - width: 16px; - height: 22px; -} - -.icon { - font-family: Arial, Helvetica; - font-weight: bold; - font-size: 12px; - height: 14px; - width: 16px; - display: inline-block; - background-color: #728DC1; - color: white; - text-align: center; - border-radius: 4px; - margin-left: 2px; - margin-right: 2px; -} - -.icona { - width: 24px; - height: 22px; - display: inline-block; -} - -.iconfopen { - width: 24px; - height: 18px; - margin-bottom: 4px; - background-image:url('folderopen.png'); - background-position: 0px -4px; - background-repeat: repeat-y; - vertical-align:top; - display: inline-block; -} - -.iconfclosed { - width: 24px; - height: 18px; - margin-bottom: 4px; - background-image:url('folderclosed.png'); - background-position: 0px -4px; - background-repeat: repeat-y; - vertical-align:top; - display: inline-block; -} - -.icondoc { - width: 24px; - height: 18px; - margin-bottom: 4px; - background-image:url('doc.png'); - background-position: 0px -4px; - background-repeat: repeat-y; - vertical-align:top; - display: inline-block; -} - -table.directory { - font: 400 14px Roboto,sans-serif; -} - -/* @end */ - -div.dynheader { - margin-top: 8px; - -webkit-touch-callout: none; - -webkit-user-select: none; - -khtml-user-select: none; - -moz-user-select: none; - -ms-user-select: none; - user-select: none; -} - -address { - font-style: normal; - color: #2A3D61; -} - -table.doxtable caption { - caption-side: top; -} - -table.doxtable { - border-collapse:collapse; - margin-top: 4px; - margin-bottom: 4px; -} - -table.doxtable td, table.doxtable th { - border: 1px solid #2D4068; - padding: 3px 7px 2px; -} - -table.doxtable th { - background-color: #374F7F; - color: #FFFFFF; - font-size: 110%; - padding-bottom: 4px; - padding-top: 5px; -} - -table.fieldtable { - /*width: 100%;*/ - margin-bottom: 10px; - border: 1px solid #A8B8D9; - border-spacing: 0px; - -moz-border-radius: 4px; - -webkit-border-radius: 4px; - border-radius: 4px; - -moz-box-shadow: rgba(0, 0, 0, 0.15) 2px 2px 2px; - -webkit-box-shadow: 2px 2px 2px rgba(0, 0, 0, 0.15); - box-shadow: 2px 2px 2px rgba(0, 0, 0, 0.15); -} - -.fieldtable td, .fieldtable th { - padding: 3px 7px 2px; -} - -.fieldtable td.fieldtype, .fieldtable td.fieldname { - white-space: nowrap; - border-right: 1px solid #A8B8D9; - border-bottom: 1px solid #A8B8D9; - vertical-align: top; -} - -.fieldtable td.fieldname { - padding-top: 3px; -} - -.fieldtable td.fielddoc { - border-bottom: 1px solid #A8B8D9; - /*width: 100%;*/ -} - -.fieldtable td.fielddoc p:first-child { - margin-top: 0px; -} - -.fieldtable td.fielddoc p:last-child { - margin-bottom: 2px; -} - -.fieldtable tr:last-child td { - border-bottom: none; -} - -.fieldtable th { - background-image:url('nav_f.png'); - background-repeat:repeat-x; - background-color: #E2E8F2; - font-size: 90%; - color: #253555; - padding-bottom: 4px; - padding-top: 5px; - text-align:left; - font-weight: 400; - -moz-border-radius-topleft: 4px; - -moz-border-radius-topright: 4px; - -webkit-border-top-left-radius: 4px; - -webkit-border-top-right-radius: 4px; - border-top-left-radius: 4px; - border-top-right-radius: 4px; - border-bottom: 1px solid #A8B8D9; -} - - -.tabsearch { - top: 0px; - left: 10px; - height: 36px; - background-image: url('tab_b.png'); - z-index: 101; - overflow: hidden; - font-size: 13px; -} - -.navpath ul { - display: flex; - flex-flow: row wrap; - justify-content: flex-start; - align-items: center; - font-size: 11px; - background-image:none; - background-repeat:repeat-x; - background-position: 0 -5px; - height:auto; - line-height:30px; - color:#8AA0CC; - border:solid 1px #C2CDE4; - overflow:hidden; - margin:0px; - padding:0px; -} - -.navpath li -{ - list-style-type:none; - float:left; - padding-left:10px; - padding-right:15px; - background-image:url('bc_s.png'); - background-repeat:no-repeat; - background-position:right; - color:#364D7C; -} - -.navpath li.navelem a -{ - height:32px; - display:block; - text-decoration: none; - outline: none; - color: #283A5D; - font-family: 'Lucida Grande',Geneva,Helvetica,Arial,sans-serif; - text-shadow: 0px 1px 1px rgba(255, 255, 255, 0.9); - text-decoration: none; -} - -.navpath li.navelem a:hover -{ - color:#6884BD; -} - -.navpath li.footer -{ - display: flex; - flex-flow: row wrap; - justify-content: flex-start; - align-items: center; - flex-grow: 1; - list-style-type:none; - float:none; - padding-left:10px; - padding-right:15px; - background-image:none; - background-repeat:no-repeat; - background-position:right; - color:#364D7C; - font-size: 8pt; -} - -div.summary -{ - float: right; - font-size: 8pt; - padding-right: 5px; - width: 50%; - text-align: right; -} - -div.summary a -{ - white-space: nowrap; -} - -table.classindex -{ - margin: 10px; - white-space: nowrap; - margin-left: 3%; - margin-right: 3%; - width: 94%; - border: 0; - border-spacing: 0; - padding: 0; -} - -div.ingroups -{ - font-size: 8pt; - width: 50%; - text-align: left; -} - -div.ingroups a -{ - white-space: nowrap; -} - -div.header -{ - background-image:url('nav_h.png'); - background-repeat:repeat-x; - background-color: #F9FAFC; - margin: 0px; - border-bottom: 1px solid #C4CFE5; -} - -div.headertitle -{ - padding: 5px 5px 5px 10px; - color: #135384; -} - -dl -{ - padding: 0 0 0 10px; -} - -/* dl.note, dl.warning, dl.attention, dl.pre, dl.post, dl.invariant, dl.deprecated, dl.todo, dl.test, dl.bug */ -dl.section -{ - margin-left: 0px; - padding-left: 0px; -} - -dl.note -{ - margin-left:-7px; - padding-left: 3px; - border-left:4px solid; - border-color: #D0C000; -} - -dl.warning, dl.attention -{ - margin-left:-7px; - padding-left: 3px; - border-left:4px solid; - border-color: #FF0000; -} - -dl.pre, dl.post, dl.invariant -{ - margin-left:-7px; - padding-left: 3px; - border-left:4px solid; - border-color: #00D000; -} - -dl.deprecated -{ - margin-left:-7px; - padding-left: 3px; - border-left:4px solid; - border-color: #505050; -} - -dl.todo -{ - margin-left:-7px; - padding-left: 3px; - border-left:4px solid; - border-color: #00C0E0; -} - -dl.test -{ - margin-left:-7px; - padding-left: 3px; - border-left:4px solid; - border-color: #3030E0; -} - -dl.bug -{ - margin-left:-7px; - padding-left: 3px; - border-left:4px solid; - border-color: #C08050; -} - -dl.section dd { - margin-bottom: 6px; -} - - -#projectlogo -{ - text-align: center; - vertical-align: bottom; - border-collapse: separate; -} - -#projectlogo img -{ - border: 0px none; -} - -#projectalign -{ - vertical-align: middle; -} - -#projectname -{ - font: 200% Tahoma, Arial,sans-serif; - margin: 0px; - padding: 2px 0px; -} - -#projectbrief -{ - font: 120% Tahoma, Arial,sans-serif; - margin: 0px; - padding: 0px; -} - -#projectnumber -{ - font: 50% Tahoma, Arial,sans-serif; - margin: 0px; - padding: 0px; -} - -#top { - border-bottom: 1px solid #5373B4; -} - -#titlearea -{ - flex-grow: 1; - padding: 0px; - margin: 0px; - width: auto; - border-bottom: none; -} - -#main-nav { -} - -#main-menu { - display: flex; - flex-flow: row wrap; - justify-content: flex-start; - align-items: center; - background-image: none; - min-width: 770px; -} - -.ui-resizable-e { - height: 100%; - background-repeat: repeat-y; -} - -.image -{ - text-align: center; -} - -.dotgraph -{ - text-align: center; -} - -.mscgraph -{ - text-align: center; -} - -.plantumlgraph -{ - text-align: center; -} - -.diagraph -{ - text-align: center; -} - -.caption -{ - font-weight: bold; -} - -div.zoom -{ - border: 1px solid #90A5CE; -} - -dl.citelist { - margin-bottom:50px; -} - -dl.citelist dt { - color:#334975; - float:left; - font-weight:bold; - margin-right:10px; - padding:5px; -} - -dl.citelist dd { - margin:2px 0; - padding:5px 0; -} - -div.toc { - padding: 14px 25px; - background-color: #F4F6FA; - border: 1px solid #D8DFEE; - border-radius: 7px 7px 7px 7px; - float: right; - height: auto; - margin: 0 8px 10px 10px; - width: 200px; -} - -div.toc li { - background: url("bdwn.png") no-repeat scroll 0 5px transparent; - font: 10px/1.2 Verdana,DejaVu Sans,Geneva,sans-serif; - margin-top: 5px; - padding-left: 10px; - padding-top: 2px; -} - -div.toc h3 { - font: bold 12px/1.2 Arial,FreeSans,sans-serif; - color: #4665A2; - border-bottom: 0 none; - margin: 0; -} - -div.toc ul { - list-style: none outside none; - border: medium none; - padding: 0px; -} - -div.toc li.level1 { - margin-left: 0px; -} - -div.toc li.level2 { - margin-left: 15px; -} - -div.toc li.level3 { - margin-left: 30px; -} - -div.toc li.level4 { - margin-left: 45px; -} - -.inherit_header { - font-weight: bold; - color: gray; - cursor: pointer; - -webkit-touch-callout: none; - -webkit-user-select: none; - -khtml-user-select: none; - -moz-user-select: none; - -ms-user-select: none; - user-select: none; -} - -.inherit_header td { - padding: 6px 0px 2px 5px; -} - -.inherit { - display: none; -} - -tr.heading h2 { - margin-top: 12px; - margin-bottom: 4px; -} - -/* tooltip related style info */ - -.ttc { - position: absolute; - display: none; -} - -#powerTip { - cursor: default; - white-space: nowrap; - background-color: white; - border: 1px solid gray; - border-radius: 4px 4px 4px 4px; - box-shadow: 1px 1px 7px gray; - display: none; - font-size: smaller; - max-width: 80%; - opacity: 0.9; - padding: 1ex 1em 1em; - position: absolute; - z-index: 2147483647; -} - -#powerTip div.ttdoc { - color: grey; - font-style: italic; -} - -#powerTip div.ttname a { - font-weight: bold; -} - -#powerTip div.ttname { - font-weight: bold; -} - -#powerTip div.ttdeci { - color: #006318; -} - -#powerTip div { - margin: 0px; - padding: 0px; - font: 12px/16px Roboto,sans-serif; -} - -#powerTip:before, #powerTip:after { - content: ""; - position: absolute; - margin: 0px; -} - -#powerTip.n:after, #powerTip.n:before, -#powerTip.s:after, #powerTip.s:before, -#powerTip.w:after, #powerTip.w:before, -#powerTip.e:after, #powerTip.e:before, -#powerTip.ne:after, #powerTip.ne:before, -#powerTip.se:after, #powerTip.se:before, -#powerTip.nw:after, #powerTip.nw:before, -#powerTip.sw:after, #powerTip.sw:before { - border: solid transparent; - content: " "; - height: 0; - width: 0; - position: absolute; -} - -#powerTip.n:after, #powerTip.s:after, -#powerTip.w:after, #powerTip.e:after, -#powerTip.nw:after, #powerTip.ne:after, -#powerTip.sw:after, #powerTip.se:after { - border-color: rgba(255, 255, 255, 0); -} - -#powerTip.n:before, #powerTip.s:before, -#powerTip.w:before, #powerTip.e:before, -#powerTip.nw:before, #powerTip.ne:before, -#powerTip.sw:before, #powerTip.se:before { - border-color: rgba(128, 128, 128, 0); -} - -#powerTip.n:after, #powerTip.n:before, -#powerTip.ne:after, #powerTip.ne:before, -#powerTip.nw:after, #powerTip.nw:before { - top: 100%; -} - -#powerTip.n:after, #powerTip.ne:after, #powerTip.nw:after { - border-top-color: #ffffff; - border-width: 10px; - margin: 0px -10px; -} -#powerTip.n:before { - border-top-color: #808080; - border-width: 11px; - margin: 0px -11px; -} -#powerTip.n:after, #powerTip.n:before { - left: 50%; -} - -#powerTip.nw:after, #powerTip.nw:before { - right: 14px; -} - -#powerTip.ne:after, #powerTip.ne:before { - left: 14px; -} - -#powerTip.s:after, #powerTip.s:before, -#powerTip.se:after, #powerTip.se:before, -#powerTip.sw:after, #powerTip.sw:before { - bottom: 100%; -} - -#powerTip.s:after, #powerTip.se:after, #powerTip.sw:after { - border-bottom-color: #ffffff; - border-width: 10px; - margin: 0px -10px; -} - -#powerTip.s:before, #powerTip.se:before, #powerTip.sw:before { - border-bottom-color: #808080; - border-width: 11px; - margin: 0px -11px; -} - -#powerTip.s:after, #powerTip.s:before { - left: 50%; -} - -#powerTip.sw:after, #powerTip.sw:before { - right: 14px; -} - -#powerTip.se:after, #powerTip.se:before { - left: 14px; -} - -#powerTip.e:after, #powerTip.e:before { - left: 100%; -} -#powerTip.e:after { - border-left-color: #ffffff; - border-width: 10px; - top: 50%; - margin-top: -10px; -} -#powerTip.e:before { - border-left-color: #808080; - border-width: 11px; - top: 50%; - margin-top: -11px; -} - -#powerTip.w:after, #powerTip.w:before { - right: 100%; -} -#powerTip.w:after { - border-right-color: #ffffff; - border-width: 10px; - top: 50%; - margin-top: -10px; -} -#powerTip.w:before { - border-right-color: #808080; - border-width: 11px; - top: 50%; - margin-top: -11px; -} - -@media print -{ - #top { display: none; } - #side-nav { display: none; } - #nav-path { display: none; } - body { overflow:visible; } - h1, h2, h3, h4, h5, h6 { page-break-after: avoid; } - .summary { display: none; } - .memitem { page-break-inside: avoid; } - #doc-content - { - margin-left:0 !important; - height:auto !important; - width:auto !important; - overflow:inherit; - display:inline; - } -} - -/* @group Markdown */ - -/* -table.markdownTable { - border-collapse:collapse; - margin-top: 4px; - margin-bottom: 4px; -} - -table.markdownTable td, table.markdownTable th { - border: 1px solid #2D4068; - padding: 3px 7px 2px; -} - -table.markdownTableHead tr { -} - -table.markdownTableBodyLeft td, table.markdownTable th { - border: 1px solid #2D4068; - padding: 3px 7px 2px; -} - -th.markdownTableHeadLeft th.markdownTableHeadRight th.markdownTableHeadCenter th.markdownTableHeadNone { - background-color: #374F7F; - color: #FFFFFF; - font-size: 110%; - padding-bottom: 4px; - padding-top: 5px; -} - -th.markdownTableHeadLeft { - text-align: left -} - -th.markdownTableHeadRight { - text-align: right -} - -th.markdownTableHeadCenter { - text-align: center -} -*/ - -table.markdownTable { - border-collapse:collapse; - margin-top: 4px; - margin-bottom: 4px; -} - -table.markdownTable td, table.markdownTable th { - border: 1px solid #2D4068; - padding: 3px 7px 2px; -} - -table.markdownTable tr { -} - -th.markdownTableHeadLeft, th.markdownTableHeadRight, th.markdownTableHeadCenter, th.markdownTableHeadNone { - background-color: #374F7F; - color: #FFFFFF; - font-size: 110%; - padding-bottom: 4px; - padding-top: 5px; -} - -th.markdownTableHeadLeft, td.markdownTableBodyLeft { - text-align: left -} - -th.markdownTableHeadRight, td.markdownTableBodyRight { - text-align: right -} - -th.markdownTableHeadCenter, td.markdownTableBodyCenter { - text-align: center -} - - -/* @end */ diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/loadgen_integration_diagram.dia b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/loadgen_integration_diagram.dia deleted file mode 100644 index 569089f243e4584e12134caf36d078248cb50af1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1943 zcmV;I2Wa>oiwFP!000021MOW~kJ>mCexF|v(Z^*%?A+VVRH|0H`_NS@ZTA@&;(=^K z%qE@-FZ1`E8Yu$2geet%`O_5|KoELebsi z%|r2;%nC!ZlBE;Yw`heI2}a-AjT`Lc``wIhaZ)wB*^)G5P3Uf0Ytmwe|2=9`v`Sf{ zcy<5g6Q&d=Z}M&x2M_j|I@{`qZcwrcrY`CE+X92`!J@1ncod# zzB&Ukoj9D{bH?S?T7@X^u#N35LbQ4e1Du|j%;h#MmexhH*}3apZP)YC1Yx?3(C+jQ zs*PG~p_qn@!%&I?u}m4G?JXs@6`By|h%ElWOF9sHw)t9<=huvA2scX-$810>6usRN z2G?|$tN7Zfm>;S{shL+c$#7Ei^y48u)e5#LdZHEmM@NSc8_v`I-O+r{zq(=`{}z$w zE)yKktB{8}`&cl=fv|WiT-zl}Lq(^^}07u#{+OG`2 zDW)xX(g}WQi&RNWoBZ)x`@4$)=r0D2Xfd!4V&DlE14oI05`#uD_>y4lOG1`7O-Z7pK)e*VN(y930qiXW zmQXE#=65kLffO;YM5=+K`=yr$l{}1E9!N@+KFHGt=<2qT2)Pnr_LT@=PJzgD0dk2j zxJ1BAB7DIT(S1BWI$`jXB8XOmk2qegHz@h0#6Ywd0G9=ZN(v-Ofi*z30EA}?#vxm< z#b*lwrN9*^1(uQm;ZndZ6OIEV1X8jBwTB>83hW`W0`X}9ILZrvDIqZcYAZp>L37zH zLBa1hR&pRAFJRzTYA~1@{Ka@RKuN&F;8Pgm1~qVctAQ~!1&V>swE*gl>A`oI46ty) zbU;T&SdKxc7{HduXN6dUXQ`jxEeq2nl>pA{;VK0 z%OC`>HUkJ@zRyhlGlHa zx7dUg$xC}Dd~F*-#!n+!+qM)hNxT2FVJ<3S8YeLgQUfaVQOUd%Ghf}#?{ibj+iH2c zKJOI&NamjbD{X-zsl)&pLp;z$B-=qt=XZzb4s^_S(AN#Xb3+!2xu!RoTfVhVWR)rg z6ds0Ua|P-U0#gy8Fe2;-_u7wJU0lO8Olcz=qP$ov=Vi5xob(RGjhynNR2Vnhs5K1T ztNO-~+~?!T3O?WvYd~2tBzg^KnpLmmYe3oJG#JUInWGkgWG(_#C!>lzM!5(C`&VAu zJQQ_xp_^uz4@dWn>~{XlpQ9FoR4UXw{}dOElZ$#6TT7s#BPLQrq=@L`+C7LLNhg22mH|rR`$qOSNHg%6&&$TR;UiFrG=x`O|EO*#A<)W$@v?*Ve7+H dUb*O!zt4VH`2ME!%ft6K{{zrB$(5Fa002n3teyY> diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/mlperf_icon.png b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/docs/src/mlperf_icon.png deleted file mode 100644 index 95321896d3e467b923909c3654a4260346df5b9f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4632 zcmb_gXHXMNyG2n1{1g!clqO9CBp9Uz2+B(h0@6F7BQRAaZ%{ulwU0f8Ck0`^@Z~bDr7Rvwt?}v4Qp_)*Gx03=Efabu^4m z>+!#Xndx*kspxh;E!TrIErU#8Zb6}r0j>-WU0_bGH+6j+-Cd1c9bFLq-L5JO4D484 zjfbZ2@zs3&08VYLr}rz)yNSCgFmgQeeHw9Q1eeE<(8)3h7k6`{og2>kvKC*(FKO^J z5eV$fjChu2;2`1j#U2~VGnny)Xm)IQvn=Xf#l6~A8TAK>pG-{`EH+$Q!`>Q8h%vkq z6rDWF5O?A4cmQYkJN~~7kgq0V{Ag5f`8x#@9Wx!a1SkQI%6R2@OxAA2Vq7bSB2YpbCM3ct@U0s2<-Lx1%cM>r zwzF{wFH}(Yx}S{v)uUm5#9$ZXMUlI5f;lo0Tt}hWMgM@!;u4VO6+CD^V}JOla{PpX zwtMN%ELZ8yvLK3y_CZ1Mq8jX$wO>9fYz*a|&=U~pAT!MNR7NXwpXU;@VAWjMJFl;u zZgEB}y1V3IwpP_S?E_{}g=C*@p6=dW8BKXt5_0qFCW`32_A=9!RcRfH^_5v8v^_HH z6vnCDp!d@sz)?jsi^)qBDn`P35;UO}ZWR<=WsEUExnU9_G^X4ym6;+U@Tq-eYo%Ir*L*Cs;M!{r^~um8-8JDSIRo*AwoK6t1Vi(M%Ozy zBCY{!XrUFtH8)1_m1`6%sIirN;1oQ!ZTJMXVJ;k$eJ*Rc^OBh+uk<@e5q;gredU zu@fU{oHe`eOw{cbgxuZjF^pQ0+j#9FCowRQgP%C+d0Rbs9;iQacd*Uf$-h$Vl{2r@ zcbZ#{lQetm=SQV-O)3RFd}f`D)7uwH;JaEbCX~&GKh{@f@`y=hZ#}*{5*9w;R4YCz z_IT=GPL!{=z^Ys?%aDSb)(&qs8gx@Rs((?N`g7adRCvmd8!N!;@6`w>!W?Nam1=~e zwjYUwN2R<@z3TCU_bz8{bV_&t=hzz){?UIB4XuRsBcX7<*Hpu(TCp)FzVRq3?iz`a zi1J_=0_P{0=OGowqL1&*?b=D*r#saFS+-`vB_gUHP{{zM{UxNk_JlnP`PnL{uy?Ih5N7)PST#s)OaJ(n zeC_JXP1;ILY)2+nYUYlLa9AUI+smA=K}=e~vWL3O><4blSZlNb3AbA93!7x>RLxx+ z@EUiV9jZY`HN@v&5sgda*1j$@;ten{IOWPvXY9z{?<6`y>_@bNv>;{BTA$8wv8yVG zsEHa{ps&tAKZ>X`!>oX9YN0$0_9oF_^IiPj?|_+&Lf|(41=CD(yi^4 zZP_sp%min08u!cIQK!|gq%3yQv~WIQf_pCsWb=wx{%(@*R*fBa2!JX!~o9gBOw(P@8bM;c45q;pU& zyzy>t@0KgISap7uY9^-KwfQHc4TwV0^7{K$P;XDhO!wPTL&to>BG$|bCwUb^**%way6-jA@9mHwhlASDol51mA~L2Rg6yf4`zTDJE3Mbl*IPAP2Rr*~gbH)f1**^toGpVIi5 z+!L`Af3>(Pp&;7OTh99}aJ_?lG#|YyMzU59X?(@zwJfG}RFs8mfZ(V2*UCrWqu|B# z)^9d?KQ1qzHw4E<^!$v?_l0S}Tem2l1#*e~Tw5T$rJXTdDcG=3_`qGc;2ntfd7`GV zP)36t5cFm|+9ZW;uz0t90v-ckqfcl(i*oQ>W|UYCnp7x&)sC8RIE^A@-L zL%rY;exe>M6)NO3jX%~Yx&OwkCRzJEKij#{M%rF~qlP~2n?pCGWE-u7UMs50!T!c= z(g3KJzRR_bk<2ky3z5;53NHbWB)h=~?L7G$=a}T}D*SXZfiri~bZ4lYM|$QF87fYw z|GZc)FXdCE^xZfe<|8?19oHtw^s=8{VJC!ZX=c_(-|lv@GKbXX$h=Iu(pn=Rt9VZX z*K}^5#iOtgBqH22J10yyOKQ~Iu&|26S2Xl-Lo?T?HG`S#?P_0;9ohF-w6N5EWF(&M zT;e6$b>PQkAO7z)W_W828wsL3J!4E$d}3`P!%iw`Tl3}5x~P77p5uxpY0G=QB0b6* znH4p|3nj(~Zsln8=9V=RZeOUUQdm>7*&+WP%Iyf$@v0#bYv_8pdNayUSvq{m)YU*z z5Ip9&oOQ{?to3XJa~eU7m{Jz?>|On=xgIJMSJ*Om$C zRp`2-5k8!fI=KXB=Wa$+nxFjVeyZO-U+wMCov;0Mx+v^6WOu-CE+T z23%sqZSSP3Zz^dWyi<=R(Re){%5~K(JkFAcoKJRjB|%9(0%a{%7OaTQg=fW6n4vQH zS>mfsv+jrXve@Gp-Sj;7a%-Zb8p^KV9Q-K%!(I{{)~Ir(&J$!9L`?yBE{ies>^9Wd z2KHT#vE?nFHC}0#5~=`aP%xcNY@NrtWSGh?q7>JBgrqaSnX(c^wZthc$DeE5-Ke8D zBwk@2dE;}x>Nu4CZaC-AmCUNdVMO(aV#1V{m5P1DQ7+% ztVB6}^0wGm6dJ!zoHHQizP2=2s_0jmV<~Q2-~51%T1v-20{aki$?ZY_5;4DPqM*Y9 zt0=)4NKL&hk5t+BB4in+q_N_GX{ico@iGSADlJocjYqt=Yj3;nM-gtRUK40YqQtz2 zOV2r+!1Y{I0CkFDm37F5^PhF$JHu*5O-g8cYndQx zLGxbkb<055Rb}p?F-cNDT*HqEt*kGu+>mlzJcWn-=%8XfdqOFqSo1t(gUjTA3hS9- zJw^rWuFNJ>wYc#w9@N(Oxr-HBWB|1I-QvNyL>ox9Y0n9X6Oc~IQIcA=vv#B+TsMcc!% znDl&;FjyVGND3qNNc878JAHrdRfXPdF}6Xfy&ib9+-!DJBdYnv~^k>f3=bK+-uQw!8)~jT@z9|dj3WiR3LJTjpYK_1B;V8>q z{+Yt_y@X|sUw|lpltZmJh;x4$KQ;Xg}D=}RWz`iS@@GRQG=^d8E z8@v#NQJ}lZ)#T&coew~O<9MBM&sqIA|EuIAm4G3?%li|A|cTI7STR z9vwx5{9LBvly87=ZRWx)S8Lka)t_(HFYH35xPA2?;uQB?&HdWIFx6H&GP#L%9Y9$% zh?60m#qK`O4*Im5CkW`OB(2DXula*s9BE|!R;M+wU*FG z^%k=-em^e{xYq`YT@~5!>(n3YgvRDi)Yn1ky!QbRxj2 z78AZZTw~?#dDhiTFx@zc8Fnz>U0;8tTjOM_;=W;Z8N!B{IMJD>=bl - - - - - - - - - - - - - - - - - - - - diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/early_stopping.cc b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/early_stopping.cc deleted file mode 100644 index 41f74b803..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/early_stopping.cc +++ /dev/null @@ -1,117 +0,0 @@ -#include "early_stopping.h" - -#include -#include -#include -#include -#include -#include -#include - -namespace mlperf { - -namespace loadgen { - -double lbeta(int64_t x, int64_t y) { - return std::lgamma(x) + std::lgamma(y) - std::lgamma(x + y); -} - -// The Gaussian Hypergeometric function specialized for a = 1. -// Based on http://dlmf.nist.gov/15.2.E1. -// Converges if c > 0 and (b <= 0 or x < 1). -// TODO(ckstanton): http://dlmf.nist.gov/15.2.E1 says there are transformations -// to replace x with with a value less than 0.5, for faster convergence. -// Presently, this function can take up to 200,000 iterations to converge. -double hypergeometric_2F1_A1(int64_t b, int64_t c, double x) { - // TODO(ckstanton): Is there a more principled way to pick kTolerance? - constexpr double kTolerance = 1.0 / (1LL << 33); - double term = 1.0; - double result = 1.0; - for (int64_t i = 0; std::abs(term) > kTolerance; ++i) { - term *= (b + i) * x / (c + i); - result += term; - } - return result; -} - -// BetaRegularized[x, a, b] = -// Beta[x, a, b]/Beta[a, b] = -// x^a/a Hypergeometric2F1[a, 1-b, 1+a, x]/Beta[a, b] = -// (http://dlmf.nist.gov/15.8.E1.) -// x^a/a (1-x)^(b-1) Hypergeometric2F1[1, 1-b, 1+a, x/(x-1)]/Beta[a, b] -double beta_regularized(double x, int64_t a, int64_t b) { - return std::exp(a * std::log(x) + (b - 1) * std::log(1 - x) - lbeta(a, b)) / - a * hypergeometric_2F1_A1(1 - b, 1 + a, x / (x - 1)); -} - -// Compute the odds of t or fewer overlatency queries in h + t total queries. -// The binomial distribution is the discrete probability distribution for -// independent boolean experiments. The CDF of the binomial distribution is: -// BetaRegularized[q, n - k, 1 + k] where 1 - q is the probability of an event -// per experiment, n is the total number of experiments, and k is the number of -// events. An even in our case is an overlatency query, so q = p - d, n = h + t, -// and k = t. -// Sum[Binomial[h + t, x] (p - d)^(h + t - x) (1 - p + d)^x, {x, 0, t}] = -// BetaRegularized[p - d, h, 1 + t] -double odds(int64_t h, int64_t t, double p, double d) { - return beta_regularized(p - d, h, 1 + t); -} - -// Binary search to find the minimum value h such that: -// odds(h, t, p, d) <= 1 - c on the range [min_h, max_h] given t, p, d, and c. -int64_t find_min_passing(int64_t min_h, int64_t max_h, int64_t t, double p, - double d, double c) { - int64_t count = max_h - min_h; - while (count > 0) { - int64_t step = count / 2; - int64_t h = min_h + step; - double prob = odds(h, t, p, d); - if (prob < 1 - c) { - count = step; - } else { - min_h = h + 1; - count -= step + 1; - } - } - return min_h; -} - -int64_t MinPassingQueriesFinder::operator()(int64_t t, double p, double d, - double c) { - // Given t, p, d, and c, return the minimum h such that odds(h, t, p, d) <= 1 - // - c - - auto &cache = caches_[std::make_tuple(p, d, c)]; - auto it = cache.lower_bound(t); - if (it != cache.end() && it->first == t) { - return it->second; - } - - int64_t x0 = -1; - int64_t y0 = 0; - int64_t x1 = 0; - int64_t y1 = std::ceil(std::log(1 - c) / std::log(p - d)); - - if (it != cache.begin()) { - --it; - x1 = it->first; - y1 = it->second; - } - - if (it != cache.begin()) { - --it; - x0 = it->first; - y0 = it->second; - } - - double min_slope = (p - d) / (1 - p + d); - double max_slope = (y1 - y0) * (x1 - x0); - int64_t min_h = (t - x1) * min_slope + y1; - int64_t max_h = (t - x1) * max_slope + y1 + 1; - int64_t h = find_min_passing(min_h, max_h, t, p, d, c); - cache[t] = h; - return h; -} - -} // namespace loadgen -} // namespace mlperf diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/early_stopping.h b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/early_stopping.h deleted file mode 100644 index 49b7a901e..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/early_stopping.h +++ /dev/null @@ -1,27 +0,0 @@ -#ifndef MLPERF_LOADGEN_EARLYSTOPPING_H_ -#define MLPERF_LOADGEN_EARLYSTOPPING_H_ - -#include -#include - -namespace mlperf { -namespace loadgen { - -class MinPassingQueriesFinder { - public: - int64_t operator()(int64_t t, double p, double d, double c); - - private: - // Memoize prior computations results and use them to bound the binary search - // range for subsequent computations. - - // TODO: Is there something more efficient to use besides std::map for - // caches_? - std::map, std::map> - caches_; -}; - -} // namespace loadgen -} // namespace mlperf - -#endif // MLPERF_LOADGEN_EARLYSTOPPING_H_ diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/generated/version_generated.cc b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/generated/version_generated.cc deleted file mode 100644 index 75fdc9519..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/generated/version_generated.cc +++ /dev/null @@ -1,98 +0,0 @@ -// DO NOT EDIT: Autogenerated by version_generator.py. - -#include - -namespace mlperf { - -const std::string& LoadgenVersion() { - static const std::string str = "4.1"; - return str; -} - -const std::string& LoadgenBuildDateLocal() { - static const std::string str = "2024-10-18T23:12:51.002440"; - return str; -} - -const std::string& LoadgenBuildDateUtc() { - static const std::string str = "2024-10-19T06:12:51.002446"; - return str; -} - -const std::string& LoadgenGitRevision() { - static const std::string str = "f5c8f17583"; - return str; -} - -const std::string& LoadgenGitCommitDate() { - static const std::string str = "2024-10-08T18:30:16+01:00"; - return str; -} - -const std::string& LoadgenGitStatus() { - static const std::string str = R"LGVG_RSLD()LGVG_RSLD"; - return str; -} - -const std::string& LoadgenGitLog() { - static const std::string str = R"LGVG_RSLD(f5c8f1758374aeaba26b2e84d31690111cfdf054 Fix bug: Loadgen ignoring token latency targets in user conf (#1874) -976bb1ad9c7946be79507f3ff67955c27426af52 Set correct remote repo (#1871) -41fa8aadd1ba0ecc97f6a519d8b42b04278e5f24 Add format files github action (#1682) -518b454fd8647bfbd23a074e875e87353f33393e Tflite tpu (#1449) -e0fdec1c7a75c98cfc194f13d62ac4388d419c8a Fix link in GettingStarted.ipynb (#1512) -92bd8198d15411d7fb7d7c27f8904bc5a0bcfe7a Fix warning in the submission checker (#1808) -224cfbf5c0e82cae6d48620025b7e1258ae3666a Fix typo in reference datatype (#1851) -3ef1249b7f50a250c02c568342e0aea6638fc5a7 Fix docs (#1853) -a0874c100c54cbc54fb743ac8bf9fb5fadc64135 Update build_wheels.yml (#1758) -6eff09986e337ccf03f675c9f244d8ee93644e16 Extend the final report generation script to output a json file of results (#1825) -54f3f93a73cc8ca5e3319ad87fb325e510574f56 Add binding for server_num_issue_query_threads parameter (#1862) -c4d0b3ea98e6fe7252e50cb573f0d523da7979df Update docs: SCC24, fix broken redirect (#1843) -7d2f0c41e5cd79c9178702867392e38f57953338 Update DLRM readme (#1811) -cf5fddc5d0746bf3820eb0ab7294bbf709d788ab Enable systems to be marked as power only (#1850) -81c2de69de4af90410cd1ba000fc5bd731bf6dee Documentation updates (#1821) -73b02798219c794a735a7f2ddabbc3df9173352d Fix error with generate_final_report.py when the input CSV file is empty (#1827))LGVG_RSLD"; - return str; -} - -const std::string& LoadgenSha1OfFiles() { - static const std::string str = R"LGVG_RSLD(012aad77e5206c89d50718c46c119d1f3cb056b2 /.clang-format -e173f4513f3c5dac1f0bea1473bb0a058e23f190 /=42 -d5274ff0b56e8d3cdb273174628a4461fca6f02a /CMakeLists.txt -20a55bb946c2c0bbb564ced2af1e48efd096b3a8 /README.md -5f6c6a784e9cd6995db47f9b9f70b1769909c9d8 /README_BUILD.md -01f9ae9887f50bc030dc6107e740f40c43ca388f /README_FAQ.md -32181da9e161c285f8fe46ddaa49e6cba2f9f918 /bindings/c_api.cc -91f58bd79b83b278f3240174a9af747fc38aff74 /bindings/c_api.h -ea4c89decad19eaf3217bfa2fb757d3b83a561d6 /bindings/python_api.cc -53dba8ad4272190ceb6335c12fd25e53dc02a8cb /diagram_network_submission.png -84c2f79309b237cef652aef6a187ba8e875a3952 /diagram_submission.png -0cd7b546a389deac73f7955cd39255ed76557d62 /early_stopping.cc -158fcae6a5f47e82150d6416fa1f7bcef37e77fe /early_stopping.h -126e952d00f4ea9efd12405fb209aa3ed585e4b2 /issue_query_controller.cc -923d9d5cdf598e3ec33d7a1110a31f7e11527ec7 /issue_query_controller.h -6650091ba7a918f343b06eb7a5aa540eae87275f /loadgen.cc -e00fdc6dbc85a8c9a8485dbcbfe2944f81251c4e /loadgen.h -47f748307536f80cfc606947b440dd732afc2637 /loadgen_integration_diagram.svg -197efc96d178e5d33a750d07fa7b2966417506ea /logging.cc -ddb961df7bcc145bcd7cce8c21f7cf075350dcbe /logging.h -ca17720f9c8246e821331946d893e830fc88f8bd /pyproject.toml -13ad6d842200cb161d6927eb74a3fafd79c46c75 /query_dispatch_library.h -e9187c8612bbdc972305b789feb6e15c26e96cfe /query_sample.h -8323a2225be1dff31f08ecc86b76eb3de06568bc /query_sample_library.h -a5ff7e77caa6e9e22ada90f0de0c865c987bf167 /requirements.txt -34e2d2a44324cb07c884f92146ecbb8ef9d704e2 /results.cc -d82500c326c2de83db411f1146882aa4692b419c /results.h -13c49b028b22749b5f3c44f3d9bb489e8c0574e9 /setup.py -18d4809589dae33317d88d9beeb5491a6e1ccdec /system_under_test.h -c15c3e150030089a8d634bd2ad6d4b644002e613 /test_settings.h -e21febd60f9b5bedd1fc81bb990f09c34b32043c /test_settings_internal.cc -f1d5335b53ca610c30e0edc5d07999a27b5b4b9a /test_settings_internal.h -3df8fdabf6eaea4697cf25d1dcb89cae88e36efd /utils.cc -40775e32d619ea6356826ae5ea4174c7911f6894 /utils.h -cbec2a5f98f9786c8c3d8b06b3d12df0b6550fa0 /version.cc -9d574baa64424e9c708fcfedd3dbb0b518a65fcc /version.h -eea9b9cb1a06cd1abe1bbdaee82f9af31527fedb /version_generator.py)LGVG_RSLD"; - return str; -} - -} // namespace mlperf diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/issue_query_controller.cc b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/issue_query_controller.cc deleted file mode 100644 index c1abea9d1..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/issue_query_controller.cc +++ /dev/null @@ -1,552 +0,0 @@ -/* Copyright 2019 The MLPerf Authors. All Rights Reserved. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -/// \file -/// \brief Implements IssueQueryController and other helper classes for -/// query issuing. - -#include "issue_query_controller.h" - -#include - -namespace mlperf { - -void RegisterIssueQueryThread() { - loadgen::IssueQueryController::GetInstance().RegisterThread(); -} - -/// \brief Loadgen implementation details. -namespace loadgen { - -QueryMetadata::QueryMetadata( - const std::vector& query_sample_indices, - std::chrono::nanoseconds scheduled_delta, - ResponseDelegate* response_delegate, SequenceGen* sequence_gen) - : scheduled_delta(scheduled_delta), - response_delegate(response_delegate), - sequence_id(sequence_gen->NextQueryId()), - wait_count_(query_sample_indices.size()) { - samples_.reserve(query_sample_indices.size()); - for (QuerySampleIndex qsi : query_sample_indices) { - samples_.push_back({this, sequence_gen->NextSampleId(), qsi, - sequence_gen->NextAccLogRng()}); - } - query_to_send.reserve(query_sample_indices.size()); - for (auto& s : samples_) { - query_to_send.push_back({reinterpret_cast(&s), s.sample_index}); - } -} - -QueryMetadata::QueryMetadata(QueryMetadata&& src) - : query_to_send(std::move(src.query_to_send)), - scheduled_delta(src.scheduled_delta), - response_delegate(src.response_delegate), - sequence_id(src.sequence_id), - wait_count_(src.samples_.size()), - samples_(std::move(src.samples_)) { - // The move constructor should only be called while generating a - // vector of QueryMetadata, before it's been used. - // Assert that wait_count_ is in its initial state. - assert(src.wait_count_.load() == samples_.size()); - // Update the "parent" of each sample to be this query; the old query - // address will no longer be valid. - // TODO: Only set up the sample parenting once after all the queries have - // been created, rather than re-parenting on move here. - for (size_t i = 0; i < samples_.size(); i++) { - SampleMetadata* s = &samples_[i]; - s->query_metadata = this; - query_to_send[i].id = reinterpret_cast(s); - } -} - -void QueryMetadata::NotifyOneSampleCompleted(PerfClock::time_point timestamp) { - size_t old_count = wait_count_.fetch_sub(1, std::memory_order_relaxed); - if (old_count == 1) { - all_samples_done_time = timestamp; - all_samples_done_.set_value(); - response_delegate->QueryComplete(); - } -} - -void QueryMetadata::WaitForAllSamplesCompleted() { - all_samples_done_.get_future().wait(); -} - -PerfClock::time_point QueryMetadata::WaitForAllSamplesCompletedWithTimestamp() { - all_samples_done_.get_future().wait(); - return all_samples_done_time; -} - -// When server_coalesce_queries is set to true in Server scenario, we -// sometimes coalesce multiple queries into one query. This is done by moving -// the other query's sample into current query, while maintaining their -// original scheduled_time. -void QueryMetadata::CoalesceQueries(QueryMetadata* queries, size_t first, - size_t last, size_t stride) { - // Copy sample data over to current query, boldly assuming that each query - // only has one sample. - query_to_send.reserve((last - first) / stride + - 2); // Extra one for the current query. - for (size_t i = first; i <= last; i += stride) { - auto& q = queries[i]; - auto& s = q.samples_[0]; - query_to_send.push_back({reinterpret_cast(&s), s.sample_index}); - q.scheduled_time = scheduled_time + q.scheduled_delta - scheduled_delta; - q.issued_start_time = issued_start_time; - } -} - -void QueryMetadata::Decoalesce() { query_to_send.resize(1); } - -/// \brief A base template that should never be used since each scenario has -/// its own specialization. -template -struct QueryScheduler { - static_assert(scenario != scenario, "Unhandled TestScenario"); -}; - -/// \brief Schedules queries for issuance in the single stream scenario. -template <> -struct QueryScheduler { - QueryScheduler(const TestSettingsInternal& /*settings*/, - const PerfClock::time_point) {} - - PerfClock::time_point Wait(QueryMetadata* next_query) { - auto tracer = MakeScopedTracer([](AsyncTrace& trace) { trace("Waiting"); }); - if (prev_query != nullptr) { - prev_query->WaitForAllSamplesCompleted(); - } - prev_query = next_query; - - auto now = PerfClock::now(); - next_query->scheduled_time = now; - next_query->issued_start_time = now; - return now; - } - - QueryMetadata* prev_query = nullptr; -}; - -/// \brief Schedules queries for issuance in the multi stream scenario. -template <> -struct QueryScheduler { - QueryScheduler(const TestSettingsInternal& /*settings*/, - const PerfClock::time_point) {} - - PerfClock::time_point Wait(QueryMetadata* next_query) { - auto tracer = MakeScopedTracer([](AsyncTrace& trace) { trace("Waiting"); }); - if (prev_query != nullptr) { - prev_query->WaitForAllSamplesCompleted(); - } - prev_query = next_query; - - auto now = PerfClock::now(); - next_query->scheduled_time = now; - next_query->issued_start_time = now; - return now; - } - - QueryMetadata* prev_query = nullptr; -}; - -/// \brief Schedules queries for issuance in the server scenario. -template <> -struct QueryScheduler { - QueryScheduler(const TestSettingsInternal& /*settings*/, - const PerfClock::time_point start) - : start(start) {} - - PerfClock::time_point Wait(QueryMetadata* next_query) { - auto tracer = - MakeScopedTracer([](AsyncTrace& trace) { trace("Scheduling"); }); - - auto scheduled_time = start + next_query->scheduled_delta; - next_query->scheduled_time = scheduled_time; - - auto now = PerfClock::now(); - if (now < scheduled_time) { - std::this_thread::sleep_until(scheduled_time); - now = PerfClock::now(); - } - next_query->issued_start_time = now; - return now; - } - - const PerfClock::time_point start; -}; - -/// \brief Schedules queries for issuance in the offline scenario. -template <> -struct QueryScheduler { - QueryScheduler(const TestSettingsInternal& /*settings*/, - const PerfClock::time_point start) - : start(start) {} - - PerfClock::time_point Wait(QueryMetadata* next_query) { - next_query->scheduled_time = start; - auto now = PerfClock::now(); - next_query->issued_start_time = now; - return now; - } - - const PerfClock::time_point start; -}; - -IssueQueryController& IssueQueryController::GetInstance() { - // The singleton. - static IssueQueryController instance; - return instance; -} - -void IssueQueryController::RegisterThread() { - // Push this thread to thread queue. - auto thread_id = std::this_thread::get_id(); - size_t thread_idx{0}; - { - std::lock_guard lock(mtx); - thread_idx = thread_ids.size(); - thread_ids.emplace_back(thread_id); - } - - LogDetail([thread_id, thread_idx](AsyncDetail& detail) { -#if USE_NEW_LOGGING_FORMAT - std::stringstream ss; - ss << "Registered IssueQueryThread[" << thread_idx - << "]. thread ID : " << std::hash()(thread_id); - MLPERF_LOG(detail, "generic_message", ss.str()); -#else - detail("Registered IssueQueryThread[" + std::to_string(thread_idx) + - "]. thread ID : ", - std::to_string(std::hash()(thread_id))); -#endif - }); - - // Start test. - while (true) { - // Wait until the main thread signals a start or the end. - { - std::unique_lock lock(mtx); - cond_var.wait(lock, [this]() { return issuing || end_test; }); - // The test has ended. - if (end_test) { - break; - } - } - - // Start issuing queries. - if (thread_idx <= num_threads) { - IssueQueriesInternal(num_threads, thread_idx); - { - std::lock_guard lock(mtx); - thread_complete[thread_idx] = true; - } - cond_var.notify_all(); - } - - // Wait until all issue threads complete. - { - std::unique_lock lock(mtx); - cond_var.wait(lock, [this]() { return !issuing; }); - } - } -} - -void IssueQueryController::SetNumThreads(size_t n) { - // Try waiting for IssueQueryThreads() to registered themselves. - std::unique_lock lock(mtx); - const std::chrono::seconds timeout(10); - num_threads = n; - cond_var.wait_for(lock, timeout, - [this]() { return thread_ids.size() >= num_threads; }); - // If the number of registered threads do not match the settings, report an - // error. - if (num_threads != thread_ids.size()) { - LogDetail([this](AsyncDetail& detail) { -#if USE_NEW_LOGGING_FORMAT - std::stringstream ss; - ss << "Mismatch between settings and number of registered " - << "IssueQueryThreads! settings.server_num_issue_query_threads = " - << num_threads << " but " << thread_ids.size() - << " threads registered."; - MLPERF_LOG_ERROR(detail, "error_runtime", ss.str()); -#else - detail.Error( - "Mismatch between settings and number of registered ", - "IssueQueryThreads! settings.server_num_issue_query_threads = ", - num_threads, " but ", thread_ids.size(), " threads registered."); -#endif - }); - } -} - -template -void IssueQueryController::StartIssueQueries(IssueQueryState* s) { - // Get the state. - state = s; - state->start_for_power = std::chrono::system_clock::now(); - state->start_time = PerfClock::now(); - - if (scenario != TestScenario::Server || num_threads == 0) { - // Usually, we just use the same thread to issue queries. - IssueQueriesInternal(1, 0); - } else { - // If server_num_issue_query_threads is non-zero, issue queries on the - // registered threads. - // Tell all threads to start issuing queries. - { - std::unique_lock lock(mtx); - issuing = true; - thread_complete.assign(num_threads, false); - } - cond_var.notify_all(); - // Wait until all issue threads complete. - { - std::unique_lock lock(mtx); - cond_var.wait(lock, [this]() { - return std::all_of(thread_complete.begin(), thread_complete.end(), - [](bool in) { return in; }); - }); - issuing = false; - } - cond_var.notify_all(); - } -} - -template void IssueQueryController::StartIssueQueries< - TestScenario::MultiStream>(IssueQueryState* s); -template void IssueQueryController::StartIssueQueries( - IssueQueryState* s); -template void IssueQueryController::StartIssueQueries( - IssueQueryState* s); -template void IssueQueryController::StartIssueQueries< - TestScenario::SingleStream>(IssueQueryState* s); - -void IssueQueryController::EndThreads() { - // Tell all the issue threads to end. - { - std::lock_guard lock(mtx); - end_test = true; - } - cond_var.notify_all(); -} - -template -void IssueQueryController::IssueQueriesInternal(size_t query_stride, - size_t thread_idx) { - // Get all the needed information. - auto sut = state->sut; - auto& queries = *state->queries; - auto& response_logger = *state->response_delegate; - - // Some book-keeping about the number of queries issued. - size_t queries_issued = 0; - size_t queries_issued_per_iter = 0; - size_t queries_count = queries.size(); - - // Calculate the min/max queries per issue thread. - const auto& settings = *state->settings; - const size_t min_query_count = settings.min_query_count; - const size_t min_query_count_for_thread = - (thread_idx < (min_query_count % query_stride)) - ? (min_query_count / query_stride + 1) - : (min_query_count / query_stride); - const size_t max_query_count = settings.max_query_count; - const size_t max_query_count_for_thread = - (thread_idx < (max_query_count % query_stride)) - ? (max_query_count / query_stride + 1) - : (max_query_count / query_stride); - - // Create query scheduler. - const auto start = state->start_time; - QueryScheduler query_scheduler(settings, start); - auto last_now = start; - - // We can never run out of generated queries in the server scenario, - // since the duration depends on the scheduled query time and not - // the actual issue time. - bool ran_out_of_generated_queries = scenario != TestScenario::Server; - // This is equal to the sum of numbers of samples issued. - size_t expected_latencies = 0; - - for (size_t queries_idx = thread_idx; queries_idx < queries_count; - queries_idx += query_stride) { - queries_issued_per_iter = 0; - auto& query = queries[queries_idx]; - auto tracer1 = - MakeScopedTracer([](AsyncTrace& trace) { trace("SampleLoop"); }); - last_now = query_scheduler.Wait(&query); - - // If in Server scenario and server_coalesce_queries is enabled, multiple - // queries are coalesed into one big query if the current time has already - // passed the scheduled time of multiple queries. - if (scenario == TestScenario::Server && - settings.requested.server_coalesce_queries) { - auto current_query_idx = queries_idx; - for (; queries_idx + query_stride < queries_count; - queries_idx += query_stride) { - auto next_scheduled_time = - start + queries[queries_idx + query_stride].scheduled_delta; - // If current time hasn't reached the next query's scheduled time yet, - // don't include next query. - if (last_now < next_scheduled_time) { - break; - } - queries_issued_per_iter++; - } - if (queries_idx > current_query_idx) { - // Coalesced all the pass due queries. - query.CoalesceQueries(queries.data(), current_query_idx + query_stride, - queries_idx, query_stride); - } - } - - // Issue the query to the SUT. - { - auto tracer3 = - MakeScopedTracer([](AsyncTrace& trace) { trace("IssueQuery"); }); - sut->IssueQuery(query.query_to_send); - } - - // Increment the counter. - expected_latencies += query.query_to_send.size(); - queries_issued_per_iter++; - queries_issued += queries_issued_per_iter; - - if (scenario == TestScenario::Server && - settings.requested.server_coalesce_queries) { - // Set the query back to its clean state. - query.Decoalesce(); - } - - if (state->mode == TestMode::AccuracyOnly) { - // TODO: Rate limit in accuracy mode so accuracy mode works even - // if the expected/target performance is way off. - continue; - } - - auto duration = (last_now - start); - if (scenario == TestScenario::Server) { - if (settings.max_async_queries != 0) { - // Checks if there are too many outstanding queries. - size_t queries_issued_total{0}; - if (multi_thread) { - // To check actual number of async queries in multi-thread case, - // we would have to combine the number of queries_issued from all - // issue threads. - { - std::lock_guard lock(state->mtx); - state->queries_issued += queries_issued_per_iter; - queries_issued_total = state->queries_issued; - } - } else { - queries_issued_total = queries_issued; - } - size_t queries_outstanding = - queries_issued_total - - response_logger.queries_completed.load(std::memory_order_relaxed); - if (queries_outstanding > settings.max_async_queries) { - LogDetail([thread_idx, queries_issued_total, - queries_outstanding](AsyncDetail& detail) { -#if USE_NEW_LOGGING_FORMAT - std::stringstream ss; - ss << "IssueQueryThread " << thread_idx - << " Ending early: Too many outstanding queries." << " issued " - << queries_issued_total << " outstanding " - << queries_outstanding; - MLPERF_LOG_ERROR(detail, "error_runtime", ss.str()); -#else - detail.Error("IssueQueryThread ", std::to_string(thread_idx), - " Ending early: Too many outstanding queries.", - "issued", std::to_string(queries_issued_total), - "outstanding", std::to_string(queries_outstanding)); -#endif - }); - break; - } - } - } else { - // Checks if we end normally. - if (queries_issued >= min_query_count_for_thread && - duration >= settings.target_duration) { - LogDetail([thread_idx](AsyncDetail& detail) { -#if USE_NEW_LOGGING_FORMAT - MLPERF_LOG( - detail, "generic_message", - "Ending naturally: Minimum query count and test duration met."); -#else - detail( - " Ending naturally: Minimum query count and test duration met."); -#endif - }); - ran_out_of_generated_queries = false; - break; - } - } - - // Checks if we have exceeded max_query_count for this thread. - if (settings.max_query_count != 0 && - queries_issued >= max_query_count_for_thread) { - LogDetail([thread_idx, queries_issued](AsyncDetail& detail) { -#if USE_NEW_LOGGING_FORMAT - std::stringstream ss; - ss << "IssueQueryThread " << thread_idx - << " Ending early: Max query count reached." << " query_count " - << queries_issued; - MLPERF_LOG_ERROR(detail, "error_runtime", ss.str()); -#else - detail.Error("IssueQueryThread ", std::to_string(thread_idx), - " Ending early: Max query count reached.", "query_count", - std::to_string(queries_issued)); -#endif - }); - ran_out_of_generated_queries = false; - break; - } - - // Checks if we have exceeded max_duration. - if (settings.max_duration.count() != 0 && - duration > settings.max_duration) { - LogDetail([thread_idx, duration](AsyncDetail& detail) { -#if USE_NEW_LOGGING_FORMAT - std::stringstream ss; - ss << "IssueQueryThread " << thread_idx - << " Ending early: Max test duration reached." << " duration_ns " - << duration.count(); - MLPERF_LOG_ERROR(detail, "error_runtime", ss.str()); -#else - detail.Error("IssueQueryThread ", std::to_string(thread_idx), - " Ending early: Max test duration reached.", "duration_ns", - std::to_string(duration.count())); -#endif - }); - ran_out_of_generated_queries = false; - break; - } - } - - // Combine the issuing statistics from multiple issue threads. - { - std::lock_guard lock(state->mtx); - state->ran_out_of_generated_queries |= ran_out_of_generated_queries; - // In Server scenario and when max_async_queries != 0, we would have set - // state->queries_issued when we check max_async_queries in the loop. - if (!(scenario == TestScenario::Server && settings.max_async_queries != 0 && - multi_thread)) { - state->queries_issued += queries_issued; - } - state->expected_latencies += expected_latencies; - } -} - -} // namespace loadgen - -} // namespace mlperf diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/issue_query_controller.h b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/issue_query_controller.h deleted file mode 100644 index 5668c574e..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/issue_query_controller.h +++ /dev/null @@ -1,215 +0,0 @@ -/* Copyright 2019 The MLPerf Authors. All Rights Reserved. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -/// \file -/// \brief Declare IssueQueryController and other helper classes for -/// query issuing. - -#ifndef MLPERF_LOADGEN_ISSUE_QUERY_CONTROLLER_H_ -#define MLPERF_LOADGEN_ISSUE_QUERY_CONTROLLER_H_ - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "loadgen.h" -#include "logging.h" -#include "query_sample.h" -#include "system_under_test.h" -#include "test_settings_internal.h" -#include "utils.h" - -namespace mlperf { - -namespace loadgen { - -struct SampleMetadata; -class QueryMetadata; - -/// \brief Every query and sample within a call to StartTest gets a unique -/// sequence id for easy cross reference, and a random number which is used to -/// determine accuracy logging when it is enabled. -struct SequenceGen { - uint64_t NextQueryId() { return query_id++; } - uint64_t NextSampleId() { return sample_id++; } - uint64_t CurrentSampleId() { return sample_id; } - double NextAccLogRng() { return accuracy_log_dist(accuracy_log_rng); } - void InitAccLogRng(uint64_t accuracy_log_rng_seed) { - accuracy_log_rng = std::mt19937(accuracy_log_rng_seed); - } - - private: - uint64_t query_id = 0; - uint64_t sample_id = 0; - std::mt19937 accuracy_log_rng; - std::uniform_real_distribution accuracy_log_dist = - std::uniform_real_distribution(0, 1); -}; - -/// \brief An interface for a particular scenario + mode to implement for -/// extended hanlding of sample completion. -struct ResponseDelegate { - virtual ~ResponseDelegate() = default; - virtual void SampleComplete(SampleMetadata*, QuerySampleResponse*, - PerfClock::time_point, - const ResponseCallback&) = 0; - virtual void TokenComplete(SampleMetadata*, QuerySampleResponse*, - PerfClock::time_point, - const ResponseCallback&) = 0; - virtual void QueryComplete() = 0; - std::atomic queries_completed{0}; -}; - -/// \brief Used by the loadgen to coordinate response data and completion. -struct SampleMetadata { - QueryMetadata* query_metadata; - uint64_t sequence_id; - QuerySampleIndex sample_index; - double accuracy_log_val; -}; - -/// \brief Maintains data and timing info for a query and all its samples. -class QueryMetadata { - public: - QueryMetadata(const std::vector& query_sample_indices, - std::chrono::nanoseconds scheduled_delta, - ResponseDelegate* response_delegate, SequenceGen* sequence_gen); - QueryMetadata(QueryMetadata&& src); - - void NotifyOneSampleCompleted(PerfClock::time_point timestamp); - - void WaitForAllSamplesCompleted(); - - PerfClock::time_point WaitForAllSamplesCompletedWithTimestamp(); - - /// \brief Coalesce multiple queries into one query. - /// When server_coalesce_queries is set to true in Server scenario, we - /// sometimes coalesce multiple queries into one query. This is done by moving - /// the other query's sample into current query, while maintaining their - /// original scheduled_time. - void CoalesceQueries(QueryMetadata* queries, size_t first, size_t last, - size_t stride); - - /// \brief Set a coalesced query back to its original state. - void Decoalesce(); - - public: - std::vector query_to_send; - const std::chrono::nanoseconds scheduled_delta; - ResponseDelegate* const response_delegate; - const uint64_t sequence_id; - - // Performance information. - - size_t scheduled_intervals = 0; // Number of intervals between queries, as - // actually scheduled during the run. - // For the MultiStream scenario only. - PerfClock::time_point scheduled_time; - PerfClock::time_point issued_start_time; - PerfClock::time_point all_samples_done_time; - - private: - std::atomic wait_count_; - std::promise all_samples_done_; - std::vector samples_; -}; - -/// \brief A state object for communications between the controller and its -/// caller. -struct IssueQueryState { - // Information from caller to controller. - SystemUnderTest* sut; - std::vector* queries; - ResponseDelegate* response_delegate; - const TestSettingsInternal* settings; - TestMode mode; - // Information from controller to caller. - std::chrono::system_clock::time_point start_for_power; - PerfClock::time_point start_time; - bool ran_out_of_generated_queries; - size_t queries_issued; - size_t expected_latencies; - // The lock to modify this state (in multi-thread case). - std::mutex mtx; -}; - -/// \brief Controls the query issuing part. -/// This controller handles both the cases if the user registers or does not -/// register IssueQueryThreads. It is implemented as a singleton, and is NOT -/// thread-safe (i.e. users should not call StartTest() on multiple threads). -/// It is thread-safe with regard to IssueQueryThreads. -class IssueQueryController { - public: - /// \brief Get the controller instance singleton. - static IssueQueryController& GetInstance(); - - /// \brief Don't allow copy. This is a singleton. - IssueQueryController(IssueQueryController const&) = delete; - void operator=(IssueQueryController const&) = delete; - - /// \brief Register an IssueQueryThread. - /// It is blocking until the entire test ends. - void RegisterThread(); - - /// \brief Set number of IssueQueryThreads and wait for thread registration. - /// If for any reason the number of registered threads do not match the - /// specified number, it prints out an error. - void SetNumThreads(size_t n); - - /// \brief Kick off the query issuing. - /// The query issuing will be done on the current thread if there is no - /// registered IssueQueryThreads or if it is not in Server scenario. - template - void StartIssueQueries(IssueQueryState* s); - - /// \brief Notify the IssueQueryThreads to end. - void EndThreads(); - - private: - /// \brief Hide constructor. This is a singleton. - IssueQueryController() {} - - /// \brief The internal helper which actually issues queries. - /// This should be called by the thread(s) which issues queries. - template - void IssueQueriesInternal(size_t query_stride, size_t thread_idx); - - /// \brief The issue query state. - IssueQueryState* state; - /// \brief Locks for communications across IssueQueryThreads and the main - /// thread. - std::mutex mtx; - std::condition_variable cond_var; - /// \brief Thread ids of the registered IssueQueryThreads. - std::vector thread_ids; - size_t num_threads{0}; - /// \brief Whether the threads should be actively issuing queries. - bool issuing{false}; - /// \brief Flags for each IssueQueryThread to mark that it is done. - std::vector thread_complete; - /// \brief Whether the threads can end now. - bool end_test{false}; -}; - -} // namespace loadgen - -} // namespace mlperf - -#endif // MLPERF_LOADGEN_ISSUE_QUERY_CONTROLLER_H_ diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/loadgen.cc b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/loadgen.cc deleted file mode 100644 index 42b2140de..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/loadgen.cc +++ /dev/null @@ -1,1345 +0,0 @@ -/* Copyright 2019 The MLPerf Authors. All Rights Reserved. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "loadgen.h" - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "early_stopping.h" -#include "issue_query_controller.h" -#include "logging.h" -#include "query_sample.h" -#include "query_sample_library.h" -#include "results.h" -#include "system_under_test.h" -#include "test_settings.h" -#include "test_settings_internal.h" -#include "utils.h" -#include "version.h" - -namespace mlperf { - -/// \brief Loadgen implementation details. -namespace loadgen { - -/// \brief A random set of samples in the QSL that should fit in RAM when -/// loaded together. -struct LoadableSampleSet { - std::vector set; - const size_t sample_distribution_end; // Excludes padding in MultiStream. -}; - -/// \brief Generates nanoseconds from a start time to multiple end times. -/// TODO: This isn't very useful anymore. Remove it. -struct DurationGeneratorNs { - const PerfClock::time_point start; - int64_t delta(PerfClock::time_point end) const { - return std::chrono::duration_cast(end - start) - .count(); - } -}; - -/// \brief ResponseDelegate implementation templated by scenario and mode. -template -struct ResponseDelegateDetailed : public ResponseDelegate { - double accuracy_log_offset = 0.0f; - double accuracy_log_prob = 0.0f; - - void SampleComplete(SampleMetadata* sample, QuerySampleResponse* response, - PerfClock::time_point complete_begin_time, - const ResponseCallback& response_cb) override { - // Using a raw pointer here should help us hit the std::function - // small buffer optimization code path when we aren't copying data. - // For some reason, using std::unique_ptr wasn't moving - // into the lambda; even with C++14. - std::vector* sample_data_copy = nullptr; - double accuracy_log_val = - sample->accuracy_log_val + accuracy_log_offset < 1.0 - ? sample->accuracy_log_val + accuracy_log_offset - : sample->accuracy_log_val + accuracy_log_offset - 1.0; - if (mode == TestMode::AccuracyOnly || - accuracy_log_val <= accuracy_log_prob) { - // if a response_cb callback is provided, data only needs to reside on the - // host *after* calling it note that the callback is blocking and will - // likely involve a memcpy from accelerator to host - if (response_cb) { - response_cb(response); - } - // TODO: Verify accuracy with the data copied here. - uint8_t* src_begin = reinterpret_cast(response->data); - uint8_t* src_end = src_begin + response->size; - sample_data_copy = new std::vector(src_begin, src_end); - } - int64_t n_tokens = response->n_tokens; - Log([sample, complete_begin_time, sample_data_copy, - n_tokens](AsyncLog& log) { - QueryMetadata* query = sample->query_metadata; - DurationGeneratorNs sched{query->scheduled_time}; - if (scenario == TestScenario::Server) { - // Trace the server scenario as a stacked graph via counter events. - DurationGeneratorNs issued{query->issued_start_time}; - log.TraceCounterEvent("Latency", query->scheduled_time, "issue_delay", - sched.delta(query->issued_start_time), - "issue_to_done", - issued.delta(complete_begin_time)); - } - - // While visualizing overlapping samples in offline mode is not - // practical, sample completion is still recorded for auditing purposes. - log.TraceSample("Sample", sample->sequence_id, query->scheduled_time, - complete_begin_time, "sample_seq", sample->sequence_id, - "query_seq", query->sequence_id, "sample_idx", - sample->sample_index, "issue_start_ns", - sched.delta(query->issued_start_time), "complete_ns", - sched.delta(complete_begin_time)); - - if (sample_data_copy) { - log.LogAccuracy(sample->sequence_id, sample->sample_index, - LogBinaryAsHexString{sample_data_copy}, n_tokens); - delete sample_data_copy; - } - - // Record the latency at the end, since it will unblock the issuing - // thread and potentially destroy the metadata being used above. - QuerySampleLatency latency = sched.delta(complete_begin_time); - log.RecordSampleCompletion(sample->sequence_id, complete_begin_time, - latency, n_tokens); - }); - } - - void TokenComplete(SampleMetadata* sample, QuerySampleResponse* response, - PerfClock::time_point complete_begin_time, - const ResponseCallback& response_cb) override { - // Using a raw pointer here should help us hit the std::function - // small buffer optimization code path when we aren't copying data. - // For some reason, using std::unique_ptr wasn't moving - // into the lambda; even with C++14. - std::vector* token_data_copy = nullptr; - double accuracy_log_val = - sample->accuracy_log_val + accuracy_log_offset < 1.0 - ? sample->accuracy_log_val + accuracy_log_offset - : sample->accuracy_log_val + accuracy_log_offset - 1.0; - if (mode == TestMode::AccuracyOnly || - accuracy_log_val <= accuracy_log_prob) { - uint8_t* src_begin = reinterpret_cast(response->data); - uint8_t* src_end = src_begin + response->size; - token_data_copy = new std::vector(src_begin, src_end); - } - Log([sample, complete_begin_time, token_data_copy](AsyncLog& log) { - QueryMetadata* query = sample->query_metadata; - DurationGeneratorNs sched{query->scheduled_time}; - if (scenario == TestScenario::Server) { - DurationGeneratorNs issued{query->issued_start_time}; - log.TraceCounterEvent( - "Token_Latency", query->scheduled_time, "issue_delay", - sched.delta(query->issued_start_time), "issue_to_done", - issued.delta(complete_begin_time)); - } else { - log.TraceSample("Token", sample->sequence_id, query->scheduled_time, - complete_begin_time, "sample_seq", sample->sequence_id, - "query_seq", query->sequence_id, "sample_idx", - sample->sample_index, "issue_start_ns", - sched.delta(query->issued_start_time), "complete_ns", - sched.delta(complete_begin_time)); - } - if (token_data_copy) { - log.CacheToken(sample->sequence_id, - LogBinaryAsHexString{token_data_copy}); - } - QuerySampleLatency latency = sched.delta(complete_begin_time); - log.RecordTokenCompletion(sample->sequence_id, complete_begin_time, - latency); - }); - } - - void QueryComplete() override { - // We only need to track outstanding queries in the server scenario to - // detect when the SUT has fallen too far behind. - if (scenario == TestScenario::Server) { - queries_completed.fetch_add(1, std::memory_order_relaxed); - } - } -}; - -/// \brief Selects the query timestamps for all scenarios except Server. -template -auto ScheduleDistribution(double qps) { - return [period = std::chrono::duration_cast( - std::chrono::duration(1.0 / qps))](auto& /*gen*/) { - return period; - }; -} - -/// \brief Selects the query timestamps for the Server scenario. -template <> -auto ScheduleDistribution(double qps) { - // Poisson arrival process corresponds to exponentially distributed - // interarrival times. - return [dist = std::exponential_distribution<>(qps)](auto& gen) mutable { - return std::chrono::duration_cast( - std::chrono::duration(dist(gen))); - }; -} - -/// \brief Selects samples for the accuracy mode. -template -auto SampleDistribution(size_t sample_count, size_t stride, std::mt19937* rng) { - std::vector indices; - for (size_t i = 0; i < sample_count; i += stride) { - indices.push_back(i); - } - std::shuffle(indices.begin(), indices.end(), *rng); - return [indices = std::move(indices), i = size_t(0)](auto& /*gen*/) mutable { - return indices.at(i++); - }; -} - -/// \brief Selects samples for the performance mode. -template <> -auto SampleDistribution(size_t sample_count, - size_t /*stride*/, - std::mt19937* /*rng*/) { - return [dist = std::uniform_int_distribution<>(0, sample_count - 1)]( - auto& gen) mutable { return dist(gen); }; -} - -/// \brief Sample across the dataset, and ensure coverage of each of the -/// samples. -// Useful for non-uniform dataset (e.g. Llama2, GPTJ, 3d-unet) -auto SampleDistributionEqualIssue(size_t sample_count, size_t set_size, - std::mt19937* rng) { - std::vector indices; - std::vector shuffle_indices(set_size); - std::iota(shuffle_indices.begin(), shuffle_indices.end(), 0); - for (size_t j = 0; j < sample_count; j += set_size) { - std::shuffle(shuffle_indices.begin(), shuffle_indices.end(), *rng); - indices.insert(indices.end(), shuffle_indices.begin(), - shuffle_indices.end()); - } - return [indices = std::move(indices), i = size_t(0)](auto& /*gen*/) mutable { - return indices.at((i++) % indices.size()); - }; -} - -/// \brief Generates queries for the requested settings, templated by -/// scenario and mode. -/// \todo Make GenerateQueries faster. -/// QueryMetadata is expensive to move; either reserve queries in advance -/// so the queries vector doesn't need to grow. And/or parent samples to their -/// queries only after all queries have been generated. -/// \todo For the server scenario only, scale the query timeline at the end so -/// the QPS as scheduled is equal to the QPS as requested. -template -std::vector GenerateQueries( - const TestSettingsInternal& settings, - const LoadableSampleSet& loaded_sample_set, SequenceGen* sequence_gen, - ResponseDelegate* response_delegate) { - auto tracer = - MakeScopedTracer([](AsyncTrace& trace) { trace("GenerateQueries"); }); - - auto& loaded_samples = loaded_sample_set.set; - - // Generate 2x more samples than we think we'll need given the expected - // QPS in case the SUT is faster than expected. - // We should exit before issuing all queries. - // Does not apply to the server scenario since the duration only - // depends on the ideal scheduled time, not the actual issue time. - const int duration_multiplier = scenario == TestScenario::Server ? 1 : 2; - std::chrono::microseconds gen_duration = - duration_multiplier * settings.target_duration; - size_t min_queries = settings.min_query_count; - - size_t samples_per_query = settings.samples_per_query; - if (mode == TestMode::AccuracyOnly && scenario == TestScenario::Offline) { - samples_per_query = loaded_sample_set.sample_distribution_end; - } - - // We should not exit early in accuracy mode. - if (mode == TestMode::AccuracyOnly || settings.performance_issue_unique) { - gen_duration = std::chrono::microseconds(0); - // Integer truncation here is intentional. - // For MultiStream, loaded samples is properly padded. - // For Offline, we create a 'remainder' query at the end of this function. - min_queries = loaded_samples.size() / samples_per_query; - } - - std::vector queries; - - // Using the std::mt19937 pseudo-random number generator ensures a modicum of - // cross platform reproducibility for trace generation. - std::mt19937 sample_rng(settings.sample_index_rng_seed); - std::mt19937 schedule_rng(settings.schedule_rng_seed); - - constexpr bool kIsMultiStream = scenario == TestScenario::MultiStream; - const size_t sample_stride = kIsMultiStream ? samples_per_query : 1; - - auto sample_distribution = SampleDistribution( - loaded_sample_set.sample_distribution_end, sample_stride, &sample_rng); - // Use the unique sample distribution same as in AccuracyMode to - // to choose samples when either flag performance_issue_unique - // or performance_issue_same is set. - auto sample_distribution_unique = SampleDistribution( - loaded_sample_set.sample_distribution_end, sample_stride, &sample_rng); - - auto sample_distribution_equal_issue = SampleDistributionEqualIssue( - min_queries, loaded_samples.size(), &sample_rng); - - auto schedule_distribution = - ScheduleDistribution(settings.target_qps); - - // When sample_concatenate_permutation is turned on, pad to a multiple of the - // complete dataset to ensure fairness. - auto enable_equal_issue = settings.sample_concatenate_permutation; - if (mode != TestMode::AccuracyOnly && enable_equal_issue) { - if (scenario == TestScenario::Offline && - samples_per_query % loaded_samples.size() != 0) { - // In offline mode, we pad samples_per_query - size_t pad_size = - (loaded_samples.size() - samples_per_query % loaded_samples.size()); - samples_per_query += pad_size; - } else if ((scenario != TestScenario::Offline) && - (min_queries % loaded_samples.size() != 0)) { - // In Server, SingleStream, MultiStream mode, the min_queries should be - // padded - size_t pad_size = - (loaded_samples.size() - min_queries % loaded_samples.size()); - min_queries += pad_size; - } - } - - std::vector samples(samples_per_query); - std::chrono::nanoseconds timestamp(0); - std::chrono::nanoseconds prev_timestamp(0); - // Choose a single sample to repeat when in performance_issue_same mode - QuerySampleIndex same_sample = settings.performance_issue_same_index; - - while (prev_timestamp < gen_duration || queries.size() < min_queries) { - if (kIsMultiStream) { - QuerySampleIndex sample_i = settings.performance_issue_unique - ? sample_distribution_unique(sample_rng) - : settings.performance_issue_same - ? same_sample - : sample_distribution(sample_rng); - for (auto& s : samples) { - // Select contiguous samples in the MultiStream scenario. - // This will not overflow, since GenerateLoadableSets adds padding at - // the end of the loadable sets in the MultiStream scenario. - // The padding allows the starting samples to be the same for each - // query with respect to samples_per_query. - s = loaded_samples[sample_i++]; - } - } else if (scenario == TestScenario::Offline) { - // For the Offline + Performance scenario, we also want to support - // contiguous samples. In this scenario the query can be much larger than - // what fits into memory. We simply repeat loaded_samples N times, plus a - // remainder to ensure we fill up samples. Note that this eliminates - // randomization. - size_t num_loaded_samples = loaded_samples.size(); - size_t num_full_repeats = samples_per_query / num_loaded_samples; - uint64_t remainder = samples_per_query % (num_loaded_samples); - if (settings.performance_issue_same) { - std::fill(samples.begin(), samples.begin() + samples_per_query, - loaded_samples[same_sample]); - } else { - for (size_t i = 0; i < num_full_repeats; ++i) { - std::copy(loaded_samples.begin(), loaded_samples.end(), - samples.begin() + i * num_loaded_samples); - - if (settings.sample_concatenate_permutation) { - std::shuffle(samples.begin() + i * num_loaded_samples, - samples.begin() + (i + 1) * num_loaded_samples, - sample_rng); - } - } - - std::copy(loaded_samples.begin(), loaded_samples.begin() + remainder, - samples.begin() + num_full_repeats * num_loaded_samples); - - if (settings.sample_concatenate_permutation) { - assert(remainder == 0); - } - } - } else { - for (auto& s : samples) { - s = loaded_samples[settings.performance_issue_unique - ? sample_distribution_unique(sample_rng) - : settings.performance_issue_same ? same_sample - : enable_equal_issue - ? sample_distribution_equal_issue(sample_rng) - : sample_distribution(sample_rng)]; - } - } - queries.emplace_back(samples, timestamp, response_delegate, sequence_gen); - prev_timestamp = timestamp; - timestamp += schedule_distribution(schedule_rng); - // In equal_issue mode, the min_queries will be bumped up by a multiple of - // the dataset size if the test time has not met the threshold. - if (enable_equal_issue && (queries.size() >= min_queries) && - (prev_timestamp < gen_duration) && - (scenario != TestScenario::Offline)) { - min_queries += loaded_samples.size(); - } - } - - // See if we need to create a "remainder" query for offline+accuracy to - // ensure we issue all samples in loaded_samples. Offline doesn't pad - // loaded_samples like MultiStream does. - if (scenario == TestScenario::Offline && mode == TestMode::AccuracyOnly) { - size_t remaining_samples = loaded_samples.size() % samples_per_query; - if (remaining_samples != 0) { - samples.resize(remaining_samples); - for (auto& s : samples) { - s = loaded_samples[sample_distribution(sample_rng)]; - } - queries.emplace_back(samples, timestamp, response_delegate, sequence_gen); - } - } - - LogDetail([count = queries.size(), spq = samples_per_query, - duration = timestamp.count()](AsyncDetail& detail) { -#if USE_NEW_LOGGING_FORMAT - MLPERF_LOG(detail, "generated_query_count", count); - MLPERF_LOG(detail, "generated_samples_per_query", spq); - MLPERF_LOG(detail, "generated_query_duration", duration); -#else - detail("GeneratedQueries: ", "queries", count, "samples per query", spq, - "duration", duration); -#endif - }); - - return queries; -} - -/// \brief Issues a series of pre-generated queries. -// TODO: Templates for scenario and mode are overused, given the loadgen -// no longer generates queries on the fly. Should we reduce the -// use of templates? -template -PerformanceResult IssueQueries(SystemUnderTest* sut, - const TestSettingsInternal& settings, - const LoadableSampleSet& loaded_sample_set, - SequenceGen* sequence_gen) { - // Create reponse handler. - ResponseDelegateDetailed response_logger; - std::uniform_real_distribution accuracy_log_offset_dist = - std::uniform_real_distribution(0.0, 1.0); - std::mt19937 accuracy_log_offset_rng(settings.accuracy_log_rng_seed); - response_logger.accuracy_log_offset = - accuracy_log_offset_dist(accuracy_log_offset_rng); - response_logger.accuracy_log_prob = settings.accuracy_log_probability; - - // Generate queries. - auto sequence_id_start = sequence_gen->CurrentSampleId(); - std::vector queries = GenerateQueries( - settings, loaded_sample_set, sequence_gen, &response_logger); - - // Calculated expected number of queries - uint64_t expected_queries = - settings.target_qps * settings.min_duration.count() / 1000; - uint64_t minimum_queries = - settings.min_query_count * settings.samples_per_query; - - if (scenario != TestScenario::Offline) { - expected_queries *= settings.samples_per_query; - } else { - minimum_queries = settings.min_sample_count; - } - - expected_queries = - expected_queries < minimum_queries ? minimum_queries : expected_queries; - - if (settings.accuracy_log_sampling_target > 0) { - response_logger.accuracy_log_prob = - (double)settings.accuracy_log_sampling_target / expected_queries; - } - auto sequence_id_end = sequence_gen->CurrentSampleId(); - size_t max_latencies_to_record = sequence_id_end - sequence_id_start; - - // Initialize logger for latency recording. - GlobalLogger().RestartLatencyRecording(sequence_id_start, - max_latencies_to_record); - - // Create and initialize an IssueQueryState. - IssueQueryState state{ - sut, &queries, &response_logger, &settings, mode, {}, {}, false, 0, - 0, {}}; - auto& controller = IssueQueryController::GetInstance(); - - // Set number of IssueQueryThreads and wait for the threads to register. - controller.SetNumThreads(settings.requested.server_num_issue_query_threads); - - // Start issuing the queries. - controller.StartIssueQueries(&state); - - // Gather query issuing statistics. - const auto start_for_power = state.start_for_power; - const auto start = state.start_time; - const auto ran_out_of_generated_queries = state.ran_out_of_generated_queries; - const auto queries_issued = state.queries_issued; - const auto expected_latencies = state.expected_latencies; - - // Let the SUT know it should not expect any more queries. - sut->FlushQueries(); - - if (mode == TestMode::PerformanceOnly && ran_out_of_generated_queries) { - LogDetail([](AsyncDetail& detail) { -#if USE_NEW_LOGGING_FORMAT - MLPERF_LOG_ERROR( - detail, "error_runtime", - "Ending early: Ran out of generated queries to issue before the " - "minimum query count and test duration were reached. " - "Please update the relevant expected latency or target qps in the " - "TestSettings so they are more accurate."); -#else - detail.Error( - "Ending early: Ran out of generated queries to issue before the " - "minimum query count and test duration were reached."); - detail( - "Please update the relevant expected latency or target qps in the " - "TestSettings so they are more accurate."); -#endif - }); - } - - // Wait for tail queries to complete and collect all the latencies. - // We have to keep the synchronization primitives alive until the SUT - // is done with them. - auto& final_query = queries[queries_issued - 1]; - std::vector sample_latencies( - GlobalLogger().GetLatenciesBlocking(expected_latencies)); - - std::vector first_token_latencies( - GlobalLogger().GetTokenLatencies(expected_latencies)); - - std::vector time_per_output_token_arr( - GlobalLogger().GetTimePerOutputToken(expected_latencies)); - - std::vector tokens_per_sample( - GlobalLogger().GetTokensPerSample(expected_latencies)); - - // Log contention counters after every test as a sanity check. - GlobalLogger().LogContentionAndAllocations(); - - // This properly accounts for the fact that the max completion time may not - // belong to the final query. It also excludes any time spent postprocessing - // in the loadgen itself after final completion, which may be significant - // in the offline scenario. - PerfClock::time_point max_completion_time = - GlobalLogger().GetMaxCompletionTime(); - auto sut_active_duration = max_completion_time - start; - LogDetail([start_for_power, sut_active_duration](AsyncDetail& detail) { - auto end_for_power = - start_for_power + - std::chrono::duration_cast( - sut_active_duration); -#if USE_NEW_LOGGING_FORMAT - MLPERF_LOG_INTERVAL_START(detail, "power_begin", - DateTimeStringForPower(start_for_power)); - MLPERF_LOG_INTERVAL_END(detail, "power_end", - DateTimeStringForPower(end_for_power)); -#else - detail("POWER_BEGIN: ", "mode", ToString(mode), "time", - DateTimeStringForPower(start_for_power)); - detail("POWER_END: ", "mode", ToString(mode), "time", - DateTimeStringForPower(end_for_power)); -#endif - }); - - double max_latency = - QuerySampleLatencyToSeconds(GlobalLogger().GetMaxLatencySoFar()); - double final_query_scheduled_time = - DurationToSeconds(final_query.scheduled_delta); - double final_query_issued_time = - DurationToSeconds(final_query.issued_start_time - start); - double final_query_all_samples_done_time = - DurationToSeconds(final_query.all_samples_done_time - start); - - std::vector query_latencies; - if (scenario == TestScenario::MultiStream) { - query_latencies.resize(queries_issued); - for (size_t i = 0; i < queries_issued; i++) { - query_latencies[i] = DurationGeneratorNs{queries[i].scheduled_time}.delta( - queries[i].all_samples_done_time); - } - } - - return PerformanceResult{ - std::move(sample_latencies), - std::move(query_latencies), - queries_issued, - max_latency, - final_query_scheduled_time, - final_query_issued_time, - final_query_all_samples_done_time, - TokenPerformanceResults{first_token_latencies, time_per_output_token_arr, - tokens_per_sample}}; -} - -void LoadSamplesToRam(QuerySampleLibrary* qsl, - const std::vector& samples) { - LogDetail([&samples](AsyncDetail& detail) { -#if USE_NEW_LOGGING_FORMAT - MLPERF_LOG(detail, "loaded_qsl_set", samples); -#else - std::string set("\"["); - for (auto i : samples) { - set += std::to_string(i) + ","; - } - set.resize(set.size() - 1); - set += "]\""; - detail("Loading QSL : ", "set", set); -#endif - }); - qsl->LoadSamplesToRam(samples); -} - -/// \brief Generates random sets of samples in the QSL that we can load into -/// RAM at the same time. -std::vector GenerateLoadableSets( - QuerySampleLibrary* qsl, const TestSettingsInternal& settings) { - auto tracer = MakeScopedTracer( - [](AsyncTrace& trace) { trace("GenerateLoadableSets"); }); - - std::vector result; - std::mt19937 qsl_rng(settings.qsl_rng_seed); - - // Generate indices for all available samples in the QSL. - const size_t qsl_total_count = qsl->TotalSampleCount(); - std::vector samples(qsl_total_count); - for (size_t i = 0; i < qsl_total_count; i++) { - samples[i] = static_cast(i); - } - - // Randomize the order of the samples. - std::shuffle(samples.begin(), samples.end(), qsl_rng); - - // Partition the samples into loadable sets. - const size_t set_size = settings.performance_sample_count; - const size_t set_padding = (settings.scenario == TestScenario::MultiStream) - ? settings.samples_per_query - 1 - : 0; - std::vector loadable_set; - loadable_set.reserve(set_size + set_padding); - - for (auto s : samples) { - loadable_set.push_back(s); - if (loadable_set.size() == set_size) { - result.push_back({std::move(loadable_set), set_size}); - loadable_set.clear(); - loadable_set.reserve(set_size + set_padding); - } - } - - if (!loadable_set.empty()) { - // Copy the size since it will become invalid after the move. - size_t loadable_set_size = loadable_set.size(); - result.push_back({std::move(loadable_set), loadable_set_size}); - } - - // Add padding for the multi stream scenario. Padding allows the - // starting sample to be the same for all SUTs, independent of the value - // of samples_per_query, while enabling samples in a query to be contiguous. - for (auto& loadable_set : result) { - auto& set = loadable_set.set; - for (size_t i = 0; i < set_padding; i++) { - // It's not clear in the spec if the STL deallocates the old container - // before assigning, which would invalidate the source before the - // assignment happens. Even though we should have reserved enough - // elements above, copy the source first anyway since we are just moving - // integers around. - QuerySampleIndex p = set[i]; - set.push_back(p); - } - } - - return result; -} - -/// \brief Opens and owns handles to all of the log files. -struct LogOutputs { - LogOutputs(const LogOutputSettings& output_settings, - const std::string& test_date_time) { - std::string prefix = output_settings.outdir; - prefix += "/" + output_settings.prefix; - if (output_settings.prefix_with_datetime) { - prefix += test_date_time + "_"; - } - const std::string& suffix = output_settings.suffix; - - summary_out.open(prefix + "summary" + suffix + ".txt"); - detail_out.open(prefix + "detail" + suffix + ".txt"); - accuracy_out.open(prefix + "accuracy" + suffix + ".json"); - trace_out.open(prefix + "trace" + suffix + ".json"); - } - - bool CheckOutputs() { - bool all_ofstreams_good = true; - if (!summary_out.good()) { - all_ofstreams_good = false; - std::cerr << "LoadGen: Failed to open summary file."; - } - if (!detail_out.good()) { - all_ofstreams_good = false; - std::cerr << "LoadGen: Failed to open detailed log file."; - } - if (!accuracy_out.good()) { - all_ofstreams_good = false; - std::cerr << "LoadGen: Failed to open accuracy log file."; - } - if (!trace_out.good()) { - all_ofstreams_good = false; - std::cerr << "LoadGen: Failed to open trace file."; - } - return all_ofstreams_good; - } - - std::ofstream summary_out; - std::ofstream detail_out; - std::ofstream accuracy_out; - std::ofstream trace_out; -}; - -/// \brief Find boundaries of performance settings by widening bounds -/// exponentially. -/// \details To find an upper bound of performance, widen an -/// upper bound exponentially until finding a bound that can't satisfy -/// performance constraints. i.e. [1, 2) -> [2, 4) -> [4, 8) -> ... -template -std::pair FindBoundaries( - SystemUnderTest* sut, QuerySampleLibrary* qsl, SequenceGen* sequence_gen, - PerformanceSummary l_perf_summary) { - // Get upper bound - TestSettingsInternal u_settings = l_perf_summary.settings; - find_peak_performance::WidenPerformanceField(&u_settings); - - LogDetail( - [l_field = find_peak_performance::ToStringPerformanceField( - l_perf_summary.settings), - u_field = find_peak_performance::ToStringPerformanceField( - u_settings)](AsyncDetail& detail) { -#if USE_NEW_LOGGING_FORMAT - MLPERF_LOG(detail, "generic_message", - "FindBoundaries: Checking fields [" + l_field + ", " + - u_field + ")"); -#else - detail("FindBoundaries: Checking fields [" + l_field + ", " + u_field + - ")"); -#endif - }); - - std::vector loadable_sets( - loadgen::GenerateLoadableSets(qsl, u_settings)); - const LoadableSampleSet& performance_set = loadable_sets.front(); - LoadSamplesToRam(qsl, performance_set.set); - - PerformanceResult u_pr(IssueQueries( - sut, u_settings, performance_set, sequence_gen)); - PerformanceSummary u_perf_summary{sut->Name(), u_settings, std::move(u_pr)}; - - qsl->UnloadSamplesFromRam(performance_set.set); - - std::string tmp; - if (!u_perf_summary.PerfConstraintsMet(&tmp)) { - return std::make_pair(l_perf_summary, u_perf_summary); - } else { - return FindBoundaries(sut, qsl, sequence_gen, u_perf_summary); - } -} - -/// \brief Find peak performance by binary search. -/// \details The found lower & upper bounds by the function 'FindBoundaries' are -/// used as initial bounds of binary search -template -PerformanceSummary FindPeakPerformanceBinarySearch( - SystemUnderTest* sut, QuerySampleLibrary* qsl, SequenceGen* sequence_gen, - const LoadableSampleSet& performance_set, PerformanceSummary l_perf_summary, - PerformanceSummary u_perf_summary) { - if (find_peak_performance::IsFinished(l_perf_summary.settings, - u_perf_summary.settings)) { - return l_perf_summary; - } - - const TestSettingsInternal m_settings = - find_peak_performance::MidOfBoundaries(l_perf_summary.settings, - u_perf_summary.settings); - - LogDetail([l_field = - find_peak_performance::ToStringPerformanceField( - l_perf_summary.settings), - u_field = - find_peak_performance::ToStringPerformanceField( - u_perf_summary.settings), - m_field = - find_peak_performance::ToStringPerformanceField( - m_settings)](AsyncDetail& detail) { -#if USE_NEW_LOGGING_FORMAT - MLPERF_LOG( - detail, "generic_message", - "FindPeakPerformanceBinarySearch: Testing the mid value of bounds [" + - l_field + ", " + u_field + "): " + m_field); -#else - detail( - "FindPeakPerformanceBinarySearch: Testing the mid value of bounds [" + - l_field + ", " + u_field + "): " + m_field); -#endif - }); - - PerformanceResult m_pr(IssueQueries( - sut, m_settings, performance_set, sequence_gen)); - PerformanceSummary m_perf_summary{sut->Name(), m_settings, std::move(m_pr)}; - - std::string tmp; - if (m_perf_summary.PerfConstraintsMet(&tmp)) { - return FindPeakPerformanceBinarySearch( - sut, qsl, sequence_gen, performance_set, m_perf_summary, - u_perf_summary); - } else { - return FindPeakPerformanceBinarySearch( - sut, qsl, sequence_gen, performance_set, l_perf_summary, - m_perf_summary); - } -} - -/// \brief Runs the performance mode, templated by scenario. -template -void RunPerformanceMode(SystemUnderTest* sut, QuerySampleLibrary* qsl, - const TestSettingsInternal& settings, - SequenceGen* sequence_gen) { - LogDetail([](AsyncDetail& detail) { -#if USE_NEW_LOGGING_FORMAT - MLPERF_LOG(detail, "generic_message", "Starting performance mode"); -#else - detail("Starting performance mode:"); -#endif - }); - - // Use first loadable set as the performance set. - std::vector loadable_sets( - loadgen::GenerateLoadableSets(qsl, settings)); - const LoadableSampleSet& performance_set = loadable_sets.front(); - LoadSamplesToRam(qsl, performance_set.set); - - // Start PerfClock/system_clock timers for measuring performance interval - // for comparison vs external timer. - auto pc_start_ts = PerfClock::now(); - auto sc_start_ts = std::chrono::system_clock::now(); - if (settings.print_timestamps) { - std::cout << "Loadgen :: Perf mode start. system_clock Timestamp = " - << std::chrono::system_clock::to_time_t(sc_start_ts) << "\n" - << std::flush; - } - - PerformanceResult pr(IssueQueries( - sut, settings, performance_set, sequence_gen)); - - // Measure PerfClock/system_clock timer durations for comparison vs - // external timer. - auto pc_stop_ts = PerfClock::now(); - auto sc_stop_ts = std::chrono::system_clock::now(); - auto pc_duration = std::chrono::duration_cast( - pc_stop_ts - pc_start_ts) - .count(); - auto sc_duration = std::chrono::duration_cast( - sc_stop_ts - sc_start_ts) - .count(); - float pc_sc_ratio = static_cast(pc_duration) / sc_duration; - if (settings.print_timestamps) { - std::cout << "Loadgen :: Perf mode stop. systme_clock Timestamp = " - << std::chrono::system_clock::to_time_t(sc_stop_ts) << "\n" - << std::flush; - std::cout << "Loadgen :: PerfClock Perf duration = " << pc_duration - << "ms\n" - << std::flush; - std::cout << "Loadgen :: system_clock Perf duration = " << sc_duration - << "ms\n" - << std::flush; - std::cout << "Loadgen :: PerfClock/system_clock ratio = " << std::fixed - << std::setprecision(4) << pc_sc_ratio << "\n" - << std::flush; - } - - if (pc_sc_ratio > 1.01 || pc_sc_ratio < 0.99) { - LogDetail([pc_sc_ratio](AsyncDetail& detail) { -#if USE_NEW_LOGGING_FORMAT - std::stringstream ss; - ss << "PerfClock and system_clock differ by more than 1%! " - << " pc_sc_ratio: " << pc_sc_ratio; - MLPERF_LOG_ERROR(detail, "error_runtime", ss.str()); -#else - detail.Error("PerfClock and system_clock differ by more than 1\%! ", - "pc_sc_ratio", pc_sc_ratio); -#endif - }); - } else if (pc_sc_ratio > 1.001 || pc_sc_ratio < 0.999) { - LogDetail([pc_sc_ratio](AsyncDetail& detail) { -#if USE_NEW_LOGGING_FORMAT - std::stringstream ss; - ss << "PerfClock and system_clock differ by more than 0.1%! " - << " pc_sc_ratio: " << pc_sc_ratio; - MLPERF_LOG_WARNING(detail, "warning_generic_message", ss.str()); -#else - detail.Warning("PerfClock and system_clock differ by more than 0.1\%. ", - "pc_sc_ratio", pc_sc_ratio); -#endif - }); - } - - PerformanceSummary perf_summary{sut->Name(), settings, std::move(pr)}; - LogSummary([perf_summary](AsyncSummary& summary) mutable { - perf_summary.LogSummary(summary); - }); - // Create a copy to prevent thread hazard between LogSummary and LogDetail. - PerformanceSummary perf_summary_detail{perf_summary}; - LogDetail([perf_summary_detail](AsyncDetail& detail) mutable { - perf_summary_detail.LogDetail(detail); - }); - - qsl->UnloadSamplesFromRam(performance_set.set); -} - -/// \brief Runs the binary search mode, templated by scenario. -/// \details 1. Check whether lower bound from user satisfies the performance -/// constraints, 2. Find an upper bound using the function 'FindBoundaries' -/// based on the lower bound, 3. Find peak performance settings using the -/// function 'FindPeakPerformanceBinarySearch'. note: Since we can't find a -/// lower bound programmatically because of the monotonicity issue of Server -/// scenario, rely on user's settings. After resolving this issue, we can -/// make the function 'FindBoundaries' find a lower bound as well from some -/// random initial settings. -template -void FindPeakPerformanceMode(SystemUnderTest* sut, QuerySampleLibrary* qsl, - const TestSettingsInternal& base_settings, - SequenceGen* sequence_gen) { - LogDetail([](AsyncDetail& detail) { -#if USE_NEW_LOGGING_FORMAT - MLPERF_LOG(detail, "generic_message", "Starting FindPeakPerformance mode"); -#else - detail("Starting FindPeakPerformance mode:"); -#endif - }); - - if (scenario != TestScenario::Server) { - LogDetail([unsupported_scenario = ToString(scenario)](AsyncDetail& detail) { -#if USE_NEW_LOGGING_FORMAT - MLPERF_LOG_ERROR(detail, "error_invalid_config", - find_peak_performance::kNotSupportedMsg); -#else - detail.Error(find_peak_performance::kNotSupportedMsg); -#endif - }); - return; - } - - LogDetail( - [base_field = find_peak_performance::ToStringPerformanceField( - base_settings)](AsyncDetail& detail) { -#if USE_NEW_LOGGING_FORMAT - MLPERF_LOG( - detail, "generic_message", - "FindPeakPerformance: Check validity of the base settings field: " + - base_field); -#else - detail( - "FindPeakPerformance: Check validity of the base settings field: " + - base_field); -#endif - }); - - // 1. Check whether the lower bound came from user satisfy performance - // constraints or not. - std::vector base_loadable_sets( - loadgen::GenerateLoadableSets(qsl, base_settings)); - const LoadableSampleSet& base_performance_set = base_loadable_sets.front(); - LoadSamplesToRam(qsl, base_performance_set.set); - - PerformanceResult base_pr(IssueQueries( - sut, base_settings, base_performance_set, sequence_gen)); - PerformanceSummary base_perf_summary{sut->Name(), base_settings, - std::move(base_pr)}; - - // We can also use all_constraints_met to check performance constraints, - // but to reduce searching time, leave it up to whether the settings satisfy - // min duration & min queries or not to users. - std::string msg; - if (!base_perf_summary.PerfConstraintsMet(&msg)) { - LogDetail([msg](AsyncDetail& detail) { -#if USE_NEW_LOGGING_FORMAT - std::stringstream ss; - ss << "FindPeakPerformance: Initial lower bound does not satisfy " - << "performance constraints, msg: " << msg; - MLPERF_LOG_ERROR(detail, "error_runtime", ss.str()); -#else - detail.Error( - "FindPeakPerformance: Initial lower bound does not satisfy " - "performance constraints, msg: " + - msg); -#endif - }); - - PerformanceSummary perf_summary{sut->Name(), base_settings, - std::move(base_perf_summary.pr)}; - LogSummary([perf_summary](AsyncSummary& summary) mutable { - perf_summary.LogSummary(summary); - }); - // Create a copy to prevent thread hazard between LogSummary and LogDetail. - PerformanceSummary perf_summary_detail{perf_summary}; - LogDetail([perf_summary_detail](AsyncDetail& detail) mutable { - perf_summary_detail.LogDetail(detail); - }); - - qsl->UnloadSamplesFromRam(base_performance_set.set); - - return; - } - - // Clear loaded samples. - qsl->UnloadSamplesFromRam(base_performance_set.set); - - // 2. Find an upper bound based on the lower bound. - std::pair boundaries = - FindBoundaries(sut, qsl, sequence_gen, base_perf_summary); - PerformanceSummary l_perf_summary = boundaries.first; - PerformanceSummary u_perf_summary = boundaries.second; - - LogDetail( - [l_field = find_peak_performance::ToStringPerformanceField( - l_perf_summary.settings), - u_field = find_peak_performance::ToStringPerformanceField( - u_perf_summary.settings)](AsyncDetail& detail) { -#if USE_NEW_LOGGING_FORMAT - MLPERF_LOG(detail, "generic_message", - "FindPeakPerformance: Found boundaries: [" + l_field + ", " + - u_field + ")"); -#else - detail("FindPeakPerformance: Found boundaries: [" + l_field + ", " + - u_field + ")"); -#endif - }); - - // Reuse performance_set, u_perf_summary has the largest 'samples_per_query'. - std::vector loadable_sets( - loadgen::GenerateLoadableSets(qsl, u_perf_summary.settings)); - const LoadableSampleSet& performance_set = loadable_sets.front(); - LoadSamplesToRam(qsl, performance_set.set); - - // 3. Find peak performance settings using the found boundaries - PerformanceSummary perf_summary = FindPeakPerformanceBinarySearch( - sut, qsl, sequence_gen, performance_set, l_perf_summary, u_perf_summary); - - // Print-out the peak performance test setting. - LogDetail([field = find_peak_performance::ToStringPerformanceField( - perf_summary.settings)](AsyncDetail& detail) { -#if USE_NEW_LOGGING_FORMAT - MLPERF_LOG(detail, "generic_message", - "FindPeakPerformance: Found peak performance field: " + field); -#else - detail("FindPeakPerformance: Found peak performance field: " + field); -#endif - }); - - LogSummary([perf_summary](AsyncSummary& summary) mutable { - perf_summary.LogSummary(summary); - }); - // Create a copy to prevent thread hazard between LogSummary and LogDetail. - PerformanceSummary perf_summary_detail{perf_summary}; - LogDetail([perf_summary_detail](AsyncDetail& detail) mutable { - perf_summary_detail.LogDetail(detail); - }); - - qsl->UnloadSamplesFromRam(performance_set.set); -} - -/// \brief Runs the accuracy mode, templated by scenario. -template -void RunAccuracyMode(SystemUnderTest* sut, QuerySampleLibrary* qsl, - const TestSettingsInternal& settings, - SequenceGen* sequence_gen) { - LogDetail([](AsyncDetail& detail) { -#if USE_NEW_LOGGING_FORMAT - MLPERF_LOG(detail, "generic_message", "Starting accuracy mode"); -#else - detail("Starting accuracy mode:"); -#endif - }); - - std::vector loadable_sets( - loadgen::GenerateLoadableSets(qsl, settings)); - - for (auto& loadable_set : loadable_sets) { - { - auto tracer = MakeScopedTracer( - [count = loadable_set.set.size()](AsyncTrace& trace) { - trace("LoadSamples", "count", count); - }); - LoadSamplesToRam(qsl, loadable_set.set); - } - - PerformanceResult pr(IssueQueries( - sut, settings, loadable_set, sequence_gen)); - - { - auto tracer = MakeScopedTracer( - [count = loadable_set.set.size()](AsyncTrace& trace) { - trace("UnloadSampes", "count", count); - }); - qsl->UnloadSamplesFromRam(loadable_set.set); - } - } -} - -/// \brief Routes runtime scenario requests to the corresponding instances -/// of its templated mode functions. -struct RunFunctions { - using Signature = void(SystemUnderTest* sut, QuerySampleLibrary* qsl, - const TestSettingsInternal& settings, - SequenceGen* sequence_gen); - - template - static RunFunctions GetCompileTime() { - return {(RunAccuracyMode), - (RunPerformanceMode), - (FindPeakPerformanceMode)}; - } - - static RunFunctions Get(TestScenario run_time_scenario) { - switch (run_time_scenario) { - case TestScenario::SingleStream: - return GetCompileTime(); - case TestScenario::MultiStream: - return GetCompileTime(); - case TestScenario::Server: - return GetCompileTime(); - case TestScenario::Offline: - return GetCompileTime(); - } - // We should not reach this point. - assert(false); - return GetCompileTime(); - } - - Signature& accuracy; - Signature& performance; - Signature& find_peak_performance; -}; - -} // namespace loadgen - -void StartTest(SystemUnderTest* sut, QuerySampleLibrary* qsl, - const TestSettings& requested_settings, - const LogSettings& log_settings, - const std::string audit_config_filename) { - GlobalLogger().StartIOThread(); - - const std::string test_date_time = CurrentDateTimeISO8601(); - - loadgen::LogOutputs log_outputs(log_settings.log_output, test_date_time); - if (!log_outputs.CheckOutputs()) { - return; - } - - GlobalLogger().StartLogging(&log_outputs.summary_out, &log_outputs.detail_out, - &log_outputs.accuracy_out, - log_settings.log_output.copy_detail_to_stdout, - log_settings.log_output.copy_summary_to_stdout); - - GlobalLogger().SetUseTokens(requested_settings.use_token_latencies); - bool needs_first_token = - (requested_settings.scenario != TestScenario::Offline); - GlobalLogger().SetNeedsFirstToken(needs_first_token); - - if (log_settings.enable_trace) { - GlobalLogger().StartNewTrace(&log_outputs.trace_out, PerfClock::now()); - } - - // measure sut->Name() response time - PerfClock::time_point pre_get_sut_name_ts = PerfClock::now(); - const std::string& sut_name = sut->Name(); - PerfClock::time_point post_get_sut_name_ts = PerfClock::now(); - - auto get_sut_name_duration_ns = - std::chrono::duration_cast( - post_get_sut_name_ts - pre_get_sut_name_ts) - .count(); - - LogLoadgenVersion(); - LogDetail([sut, qsl, test_date_time, &sut_name, - &get_sut_name_duration_ns](AsyncDetail& detail) { -#if USE_NEW_LOGGING_FORMAT - MLPERF_LOG(detail, "test_datetime", test_date_time); - MLPERF_LOG(detail, "sut_name", sut_name); - MLPERF_LOG(detail, "get_sut_name_duration_ns", get_sut_name_duration_ns); - MLPERF_LOG(detail, "qsl_name", qsl->Name()); - MLPERF_LOG(detail, "qsl_reported_total_count", qsl->TotalSampleCount()); - MLPERF_LOG(detail, "qsl_reported_performance_count", - qsl->PerformanceSampleCount()); -#else - detail("Date + time of test: ", test_date_time); - detail("System Under Test (SUT) name: ", sut_name); - detail("Get SUT name time [ns]: ", get_sut_name_duration_ns); - detail("Query Sample Library (QSL) name: ", qsl->Name()); - detail("QSL total size: ", qsl->TotalSampleCount()); - detail("QSL performance size*: ", qsl->PerformanceSampleCount()); - detail("*TestSettings (performance_sample_count_override) can override"); - detail("*Refer to Effective Settings for actual value"); -#endif - }); - - TestSettings test_settings = requested_settings; - // Look for Audit Config file to override TestSettings during audit - if (FileExists(audit_config_filename)) { - LogDetail([](AsyncDetail& detail) { -#if USE_NEW_LOGGING_FORMAT - MLPERF_LOG_WARNING(detail, "warning_generic_message", - "Found Audit Config file (audit.config)." - " Overriding TestSettings from audit.config file."); -#else - detail( - "Found Audit Config file (audit.config)." - " Overriding TestSettings from audit.config file."); -#endif - }); - std::string audit_scenario = loadgen::ToString(test_settings.scenario); - // Remove Spaces from the string - RemoveValue(&audit_scenario, ' '); - const std::string generic_model = "*"; - test_settings.FromConfig(audit_config_filename, generic_model, - audit_scenario, 2); - } - if (test_settings.test05) { - // If the configuration indicates we are running test05, - // random seeds - LogDetail([](AsyncDetail& detail) { -#if USE_NEW_LOGGING_FORMAT - MLPERF_LOG_WARNING(detail, "warning_generic_message", - "Test05 flag detected" - " Overriding random seeds"); -#else - detail( - "Test05 flag detected" - " Overriding random seeds"); -#endif - }); - test_settings.mode = TestMode::PerformanceOnly; - test_settings.qsl_rng_seed = requested_settings.test05_qsl_rng_seed; - test_settings.sample_index_rng_seed = - requested_settings.test05_sample_index_rng_seed; - test_settings.schedule_rng_seed = - requested_settings.test05_schedule_rng_seed; - } - - loadgen::TestSettingsInternal sanitized_settings( - test_settings, qsl->PerformanceSampleCount()); - sanitized_settings.LogAllSettings(); - - auto run_funcs = loadgen::RunFunctions::Get(sanitized_settings.scenario); - - loadgen::SequenceGen sequence_gen; - switch (sanitized_settings.mode) { - case TestMode::SubmissionRun: - run_funcs.accuracy(sut, qsl, sanitized_settings, &sequence_gen); - run_funcs.performance(sut, qsl, sanitized_settings, &sequence_gen); - break; - case TestMode::AccuracyOnly: - run_funcs.accuracy(sut, qsl, sanitized_settings, &sequence_gen); - break; - case TestMode::PerformanceOnly: - run_funcs.performance(sut, qsl, sanitized_settings, &sequence_gen); - break; - case TestMode::FindPeakPerformance: - run_funcs.find_peak_performance(sut, qsl, sanitized_settings, - &sequence_gen); - break; - } - - loadgen::IssueQueryController::GetInstance().EndThreads(); - - // Stop tracing after logging so all logs are captured in the trace. - GlobalLogger().StopLogging(); - GlobalLogger().StopTracing(); - GlobalLogger().StopIOThread(); -} - -void AbortTest() { - loadgen::IssueQueryController::GetInstance().EndThreads(); - GlobalLogger().StopLogging(); - GlobalLogger().StopTracing(); - GlobalLogger().StopIOThread(); -} - -void QuerySamplesComplete(QuerySampleResponse* responses, size_t response_count, - const ResponseCallback& response_cb) { - PerfClock::time_point timestamp = PerfClock::now(); - - auto tracer = MakeScopedTracer( - [](AsyncTrace& trace) { trace("QuerySamplesComplete"); }); - - const QuerySampleResponse* end = responses + response_count; - - // Notify first to unblock loadgen production ASAP. - for (QuerySampleResponse* response = responses; response < end; response++) { - loadgen::SampleMetadata* sample = - reinterpret_cast(response->id); - loadgen::QueryMetadata* query = sample->query_metadata; - query->NotifyOneSampleCompleted(timestamp); - } - - // Log samples. - for (QuerySampleResponse* response = responses; response < end; response++) { - loadgen::SampleMetadata* sample = - reinterpret_cast(response->id); - loadgen::QueryMetadata* query = sample->query_metadata; - query->response_delegate->SampleComplete(sample, response, timestamp, - response_cb); - } - // PerfClock::time_point end_timestamp = PerfClock::now(); - // mlperf::samples_overhead_acum += (end_timestamp - timestamp).count(); -} - -void FirstTokenComplete(QuerySampleResponse* responses, size_t response_count, - const ResponseCallback& response_cb) { - PerfClock::time_point timestamp = PerfClock::now(); - - auto tracer = - MakeScopedTracer([](AsyncTrace& trace) { trace("FirstTokenComplete"); }); - - const QuerySampleResponse* end = responses + response_count; - - // Log samples. - for (QuerySampleResponse* response = responses; response < end; response++) { - loadgen::SampleMetadata* sample = - reinterpret_cast(response->id); - loadgen::QueryMetadata* query = sample->query_metadata; - query->response_delegate->TokenComplete(sample, response, timestamp, - response_cb); - } - // PerfClock::time_point end_timestamp = PerfClock::now(); - // mlperf::tokens_overhead_acum += (end_timestamp - timestamp).count(); -} - -} // namespace mlperf diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/loadgen.h b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/loadgen.h deleted file mode 100644 index 84e02656c..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/loadgen.h +++ /dev/null @@ -1,103 +0,0 @@ -/* Copyright 2019 The MLPerf Authors. All Rights Reserved. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -/// \file -/// \brief Provides the entry points for a SUT to start a test and respond -/// to issued queries. - -#ifndef MLPERF_LOADGEN_LOADGEN_H_ -#define MLPERF_LOADGEN_LOADGEN_H_ - -#include -#include -#include -#include - -/// \brief Contains the loadgen API. -namespace mlperf { - -struct QuerySampleResponse; -class QuerySampleLibrary; -class SystemUnderTest; -struct TestSettings; -struct LogSettings; - -using ResponseCallback = std::function; - -/// \addtogroup LoadgenAPI Loadgen API -/// @{ - -/// -/// \brief SUT calls this to notify loadgen of completed samples. -/// \details -/// * The samples may be from any combination of queries or partial queries as -/// issued by \link mlperf::SystemUnderTest::IssueQuery -/// -/// SystemUnderTest::IssueQuery \endlink. -/// * The SUT is responsible for owning and allocating the reponse data. The -/// loadgen will copy the response data if needed (e.g. for accuracy mode). -/// + If no response callback is provided, the response data must remain valid -/// for the entire duration of this call. -/// + The response callback is untimed; it is called for each response in -/// responses after the loadgen records the completion time and before the -/// loadgen copies the response data. The response callback enables the -/// loadgen to simulate response data being stored in accelerator DRAM. -/// After the response callback is called, response data must reside on the -/// host so that the loadgen can copy it. Submitters must seek prior -/// approval to use this feature of loadgen (refer to -/// https://github.com/mlcommons/inference_policies/blob/master/inference_rules.adoc#5-load-generator). -/// * All calls to QuerySampleComplete are thread-safe and wait-free bounded. -/// + Any number of threads can call QuerySampleComplete simultaneously. -/// + Regardless of where any other thread stalls, the current thread will -/// finish QuerySampleComplete in a bounded number of cycles. -/// + Note: If a callback is provided, the SUT must ensure that the callback -/// is also thread-safe and wait-free bounded for the above to hold. -void QuerySamplesComplete(QuerySampleResponse* responses, size_t response_count, - const ResponseCallback& response_cb = {}); - -void FirstTokenComplete(QuerySampleResponse* responses, size_t response_count, - const ResponseCallback& response_cb = {}); - -/// -/// \brief Starts the test against SUT with the specified settings. -/// \details This is the C++ entry point. See mlperf::c::StartTest for the -/// C entry point. -/// -void StartTest(SystemUnderTest* sut, QuerySampleLibrary* qsl, - const TestSettings& requested_settings, - const LogSettings& log_settings, - const std::string audit_config_filename = "audit.config"); - -/// -/// \brief Aborts the running test. -/// \details This function will stop issueing new samples to the SUT. StartTest -/// will return after the current inference finishes. Since StartTest is a -/// blocking function, this function can only be called in another thread. -void AbortTest(); - -/// -/// \brief Register a thread for query issuing in Server scenario. -/// \details If a thread registers itself, the thread(s) is used to call SUT's -/// IssueQuery(). This function is blocking until the entire test is done. The -/// number of registered threads must match server_num_issue_query_threads in -/// TestSettings. This function only has effect in Server scenario. -/// This is the C++ entry point. See mlperf::c::RegisterIssueQueryThread for the -/// C entry point. -/// -void RegisterIssueQueryThread(); -// inline long long samples_overhead_acum; -// inline long long tokens_overhead_acum; -/// @} - -} // namespace mlperf - -#endif // MLPERF_LOADGEN_LOADGEN_H_ diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/loadgen_integration_diagram.svg b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/loadgen_integration_diagram.svg deleted file mode 100644 index 17dd1b481..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/loadgen_integration_diagram.svg +++ /dev/null @@ -1,85 +0,0 @@ - - - - - - - - -Model + Dataset - - - -Pre Processor - - - -Post Processor - - - -Benchmark - - - -Backend - - - -LoadGen - - - - - - - - - - - - - - - - - - - - - - - -1 - - - -2 - - -3 - - -5 - - -4 - - - -LoadGen Logs - - - - - -6 - - - - - - - \ No newline at end of file diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/logging.cc b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/logging.cc deleted file mode 100644 index 807c1954a..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/logging.cc +++ /dev/null @@ -1,1301 +0,0 @@ -/* Copyright 2019 The MLPerf Authors. All Rights Reserved. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -/// \file -/// \brief Implements a logging system with a central IO thread that handles -/// all stringification and IO. -/// \details Log-producing threads only submit lambdas to be executed on the -/// IO thread. -/// All producers and consumers use lock-free operations that guarantee -/// forward progress independent of a) other stalled threads and b) where -/// those threads are stalled. -/// Each thread uses a double-buffering scheme to queue its logs. One buffer -/// is always reserved for writes and the other is reserved for reads. -/// A producing thread sends requests to the IOThread to swap the buffers -/// and the IOThread does the actual read/write swap after it has finished -/// reading the buffer it was working on. - -#include "logging.h" - -#include -#include -#include -#include -#include -#include -#include - -#if defined(_WIN32) || defined(WIN32) || defined(_WIN64) || defined(WIN64) -#define WIN32_LEAN_AND_MEAN -#define NOMINMAX -#include -#include -#define MLPERF_GET_PID() _getpid() -#else -#include -#define MLPERF_GET_PID() getpid() -#endif - -// Use system-level TID for tracing. This enables correlation with other -// performance tools that are not aware of C++ std::thread::id. -#if defined(__linux__) -#include -#define MLPERF_GET_TID() syscall(SYS_gettid) -#elif defined(_WIN32) || defined(WIN32) || defined(_WIN64) || defined(WIN64) -#define MLPERF_GET_TID() GetCurrentThreadId() -#elif defined(__APPLE__) -#define MLPERF_GET_TID() \ - std::hash{}(std::this_thread::get_id()) -#else -// TODO: std::this_thread::id is a class but MLPERF_GET_TID() assigned to -// uint64_t -#define MLPERF_GET_TID() std::this_thread::get_id() -#endif - -#include "utils.h" - -namespace mlperf { -namespace logging { - -namespace { - -uintptr_t SwapRequestSlotIsWritableValue(size_t id) { - // LSB of 1 indicates that this isn't a pointer. - // MSBs encode the id to detect collisions when a slot in - // |thread_swap_request_slots_| is reused for a different id and the request - // for the previous id is very slow. - return (id << 1) | 0x1; -} - -bool SwapRequestSlotIsReadable(uintptr_t value) { - // Valid pointers will not have their lsb set. - return (value & 0x1) != 0x1; -} - -constexpr size_t kMaxThreadsToLog = 1024; -constexpr std::chrono::milliseconds kLogPollPeriod(10); - -/// \brief How many log entries to pre-allocate per thread to help avoid -/// runtime allocation. -constexpr size_t kTlsLogReservedEntryCount = 1024; - -constexpr auto kInvalidLatency = std::numeric_limits::min(); -constexpr auto nTokenInvalid = std::numeric_limits::min(); - -} // namespace - -const std::string& ArgValueTransform(const bool& value) { - static const std::string v_true("true"); - static const std::string v_false("false"); - return value ? v_true : v_false; -} - -char Bin2Hex(uint8_t four_bits) { - char number = '0' + four_bits; - char letter = ('A' - 10) + four_bits; - return four_bits < 10 ? number : letter; -} - -const std::string ArgValueTransform(const LogBinaryAsHexString& value) { - if (value.data == nullptr) { - return "\"\""; - } - std::string hex; - hex.reserve(value.data->size() + 2); - hex.push_back('"'); - for (auto b : *value.data) { - hex.push_back(Bin2Hex(b >> 4)); - hex.push_back(Bin2Hex(b & 0x0F)); - } - hex.push_back('"'); - return hex; -} - -#if USE_NEW_LOGGING_FORMAT -const std::string ArgValueTransform(const std::string& value) { - return std::string("\"") + value + std::string("\""); -} - -const std::string ArgValueTransform(const char* value) { - return std::string("\"") + std::string(value) + std::string("\""); -} - -const std::string ArgValueTransform(const std::vector& value) { - std::string s("["); - for (auto i : value) { - s += std::to_string(i) + ","; - } - s.resize(s.size() - 1); - s += "]"; - return s; -} - -const std::string ArgValueTransform( - const std::map& value) { - std::string s("{"); - for (const auto& i : value) { - s += "\""; - s += i.first; - s += "\":\""; - s += i.second; - s += "\","; - } - s.resize(s.size() - 1); - s += "}"; - return s; -} - -const std::string ArgValueTransform(const float value) { - if (value == std::numeric_limits::infinity()) { - return "Infinity"; - } else if (value == -std::numeric_limits::infinity()) { - return "-Infinity"; - } else if (std::isnan(value)) { - return "NaN"; - } - return std::to_string(value); -} - -const std::string ArgValueTransform(const double value) { - if (value == std::numeric_limits::infinity()) { - return "Infinity"; - } else if (value == -std::numeric_limits::infinity()) { - return "-Infinity"; - } else if (std::isnan(value)) { - return "NaN"; - } - return std::to_string(value); -} -#endif - -ChromeTracer::ChromeTracer(std::ostream* out, PerfClock::time_point origin) - : out_(out), origin_(origin) { - WriteTraceEventHeader(); -} - -ChromeTracer::~ChromeTracer() { - WriteTraceEventFooter(); - out_->flush(); -} - -void ChromeTracer::WriteTraceEventHeader() { - // Times and durations are converted from nanoseconds to microseconds, use - // 3 decimal digits to preserve precision. - *out_ << std::fixed << std::setprecision(3) << "{\"traceEvents\":[\n"; -} - -void ChromeTracer::WriteTraceEventFooter() { - *out_ << "{\"name\":\"LastTrace\"}\n" - << "],\n" - << "\"displayTimeUnit\":\"ns\",\n" - << "\"otherData\":{\n" - << "\"ts\":" << Micros(origin_.time_since_epoch()).count() << ",\n" - << "\"version\":\"MLPerf LoadGen v1.0\"\n" - << "}\n" - << "}\n"; -} - -void AsyncLog::SetCurrentPidTid(uint64_t pid, uint64_t tid) { - current_pid_ = pid; - current_tid_ = tid; -} - -void AsyncLog::SetLogFiles(std::ostream* summary, std::ostream* detail, - std::ostream* accuracy, bool copy_detail_to_stdout, - bool copy_summary_to_stdout, - PerfClock::time_point log_origin) { - std::unique_lock lock(log_mutex_); - if (summary_out_ != &std::cerr) { - std::string warning_summary; - if (log_warning_count_ == 0) { - warning_summary = "\nNo warnings encountered during test.\n"; - } else if (log_warning_count_ == 1) { - warning_summary = "\n1 warning encountered. See detailed log.\n"; - } else if (log_warning_count_ != 0) { - warning_summary = "\n" + std::to_string(log_warning_count_) + - " warnings encountered. See detailed log.\n"; - } - - std::string error_summary; - if (log_error_count_ == 0) { - error_summary = "\nNo errors encountered during test.\n"; - } else if (log_error_count_ == 1) { - error_summary = "\n1 ERROR encountered. See detailed log.\n"; - } else if (log_error_count_ != 0) { - error_summary = "\n" + std::to_string(log_error_count_) + - " ERRORS encountered. See detailed log.\n"; - } - - *summary_out_ << warning_summary << error_summary; - if (copy_summary_to_stdout_) { - std::cout << warning_summary << error_summary; - } - } - if (summary_out_) { - summary_out_->flush(); - } - if (detail_out_) { - detail_out_->flush(); - } - if (accuracy_out_ != &std::cerr) { - WriteAccuracyFooterLocked(); - accuracy_out_->flush(); - } - summary_out_ = summary; - detail_out_ = detail; - accuracy_out_ = accuracy; - if (accuracy_out_ != &std::cerr) { - WriteAccuracyHeaderLocked(); - } - copy_detail_to_stdout_ = copy_detail_to_stdout; - copy_summary_to_stdout_ = copy_summary_to_stdout; - log_origin_ = log_origin; - log_error_count_ = 0; - log_warning_count_ = 0; -} - -void AsyncLog::StartNewTrace(std::ostream* trace_out, - PerfClock::time_point origin) { - std::unique_lock lock(trace_mutex_); - if (trace_out) { - tracer_ = std::make_unique(trace_out, origin); - } else { - tracer_.reset(); - } -} - -void AsyncLog::StopTrace() { - std::unique_lock lock(trace_mutex_); - tracer_.reset(); -} - -void AsyncLog::LogAccuracy(uint64_t seq_id, const QuerySampleIndex qsl_idx, - const LogBinaryAsHexString& response, - int64_t n_tokens = 0) { - std::unique_lock lock(log_mutex_); - if (!accuracy_out_) { - return; - } - *accuracy_out_ << (accuracy_needs_comma_ ? ",\n{ " : "\n{ "); - if (!use_tokens_) { - LogArgs(accuracy_out_, "seq_id", seq_id, "qsl_idx", qsl_idx, "data", - response); - } else if (!needs_first_token_) { - LogArgs(accuracy_out_, "seq_id", seq_id, "qsl_idx", qsl_idx, "data", - response, "token_count", n_tokens); - } else { - const size_t i = seq_id - latencies_first_sample_sequence_id_; - LogArgs(accuracy_out_, "seq_id", seq_id, "qsl_idx", qsl_idx, "data", - response, "token_data", token_records_[i], "token_count", n_tokens); - } - - *accuracy_out_ << " }"; - accuracy_needs_comma_ = true; -} - -void AsyncLog::CacheToken(uint64_t seq_id, - const LogBinaryAsHexString& response) { - std::unique_lock lock(token_record_mutex_); - const size_t i = seq_id - latencies_first_sample_sequence_id_; - if (token_records_.size() <= i) { - token_records_.resize(i + 1); - } - token_records_[i] = response; -} - -void AsyncLog::Flush() { - { - std::unique_lock lock(log_mutex_); - if (summary_out_) { - summary_out_->flush(); - } - if (detail_out_) { - detail_out_->flush(); - } - if (accuracy_out_) { - accuracy_out_->flush(); - } - } - - { - std::unique_lock lock(trace_mutex_); - if (tracer_) { - tracer_->Flush(); - } - } -} - -void AsyncLog::WriteAccuracyHeaderLocked() { - *accuracy_out_ << "["; - accuracy_needs_comma_ = false; -} - -void AsyncLog::WriteAccuracyFooterLocked() { *accuracy_out_ << "\n]\n"; } - -void AsyncLog::RestartLatencyRecording(uint64_t first_sample_sequence_id, - size_t latencies_to_reserve) { - std::unique_lock lock(latencies_mutex_); - assert(latencies_.empty()); - assert(latencies_recorded_ == latencies_expected_); - latencies_recorded_ = 0; - latencies_expected_ = 0; - max_latency_ = 0; - max_completion_timstamp_ = PerfClock::now(); - latencies_first_sample_sequence_id_ = first_sample_sequence_id; - latencies_.reserve(latencies_to_reserve); - token_latencies_.reserve(latencies_to_reserve); - tokens_per_sample_.reserve(latencies_to_reserve); - time_per_output_token_.reserve(latencies_to_reserve); -} - -void AsyncLog::RecordSampleCompletion(uint64_t sample_sequence_id, - PerfClock::time_point completion_time, - QuerySampleLatency latency, - int64_t n_tokens = 0) { - std::unique_lock lock(latencies_mutex_); - - max_latency_ = std::max(max_latency_, latency); - - max_completion_timstamp_ = - std::max(max_completion_timstamp_, completion_time); - - if (sample_sequence_id < latencies_first_sample_sequence_id_) { - // Call LogErrorSync here since this kind of error could result in a - // segfault in the near future. -#if USE_NEW_LOGGING_FORMAT - std::stringstream ss; - ss << "Received completion for an old sample." - << " Min expected id: " << latencies_first_sample_sequence_id_ - << " Actual id: " << sample_sequence_id; - MLPERF_LOG_ERROR_SYNC(GlobalLogger(), "error_runtime", ss.str()); -#else - GlobalLogger().LogErrorSync( - "Received completion for an old sample.", "Min expected id", - latencies_first_sample_sequence_id_, "Actual id", sample_sequence_id); -#endif - return; - } - - const size_t i = sample_sequence_id - latencies_first_sample_sequence_id_; - - if (latencies_.size() <= i) { - // TODO: Reserve in advance. - latencies_.resize(i + 1, kInvalidLatency); - } else if (latencies_[i] != kInvalidLatency) { - // Call LogErrorSync here since this kind of error could result in a - // segfault in the near future. -#if USE_NEW_LOGGING_FORMAT - MLPERF_LOG_ERROR_SYNC(GlobalLogger(), "error_runtime", - "Attempted to complete a sample twice."); -#else - GlobalLogger().LogErrorSync("Attempted to complete a sample twice."); -#endif - - // Return without recording the latency again to avoid potentially - // ending the test before the SUT is actually done, which could result - // in a segfault. - // If the SUT recorded the wrong sample, the test will hang and see - // the error above. - return; - } - - if (use_tokens_) { - if (needs_first_token_ && (token_latencies_.size() <= i)) { - MLPERF_LOG_ERROR_SYNC(GlobalLogger(), "error_runtime", - "Attempted to record a sample latency before it's " - "first token latency"); - } else if (needs_first_token_ && (token_latencies_[i] == kInvalidLatency)) { - MLPERF_LOG_ERROR_SYNC(GlobalLogger(), "error_runtime", - "Attempted to record a sample latency before it's " - "first token latency"); - } - - if (tokens_per_sample_.size() <= i) { - // TODO: Reserve in advance. - tokens_per_sample_.resize(i + 1, nTokenInvalid); - } else if (tokens_per_sample_[i] != nTokenInvalid) { - // Call LogErrorSync here since this kind of error could result in a - // segfault in the near future. -#if USE_NEW_LOGGING_FORMAT - MLPERF_LOG_ERROR_SYNC(GlobalLogger(), "error_runtime", - "Attempted to complete a sample twice."); -#else - GlobalLogger().LogErrorSync("Attempted to complete a sample twice."); -#endif - - // Return without recording the latency again to avoid potentially - // ending the test before the SUT is actually done, which could result - // in a segfault. - // If the SUT recorded the wrong sample, the test will hang and see - // the error above. - return; - } - if (n_tokens == 0) { - MLPERF_LOG_ERROR_SYNC(GlobalLogger(), "error_runtime", - "n_tokens argument missing or attempted to record " - "0 as number of tokens"); - } else if (n_tokens < 0) { - MLPERF_LOG_ERROR_SYNC(GlobalLogger(), "error_runtime", - "Attempted to record a negative number of tokens"); - n_tokens = 0; - } else if (n_tokens == 1) { - MLPERF_LOG_ERROR_SYNC(GlobalLogger(), "error_runtime", - "Number of tokens need to be greater than 1"); - n_tokens = 0; - } - if (time_per_output_token_.size() <= i) { - time_per_output_token_.resize(i + 1, kInvalidLatency); - } else if (time_per_output_token_[i] != kInvalidLatency) { - // Call LogErrorSync here since this kind of error could result in a - // segfault in the near future. -#if USE_NEW_LOGGING_FORMAT - MLPERF_LOG_ERROR_SYNC(GlobalLogger(), "error_runtime", - "Attempted to complete a sample twice."); -#else - GlobalLogger().LogErrorSync("Attempted to complete a sample twice."); -#endif - - // Return without recording the latency again to avoid potentially - // ending the test before the SUT is actually done, which could result - // in a segfault. - // If the SUT recorded the wrong sample, the test will hang and see - // the error above. - return; - } - tokens_per_sample_[i] = n_tokens; - time_per_output_token_[i] = - (latency - token_latencies_[i]) / (n_tokens - 1); - } - latencies_[i] = latency; - latencies_recorded_++; - if (AllLatenciesRecorded()) { - all_latencies_recorded_.notify_all(); - } -} - -void AsyncLog::RecordTokenCompletion(uint64_t sample_sequence_id, - PerfClock::time_point completion_time, - QuerySampleLatency latency) { - std::unique_lock lock(token_latencies_mutex_); - // std::unique_lock lock(latencies_mutex_); - // max_latency_ = std::max(max_latency_, latency); - - // max_completion_timstamp_ = - // std::max(max_completion_timstamp_, completion_time); - - if (sample_sequence_id < latencies_first_sample_sequence_id_) { - // Call LogErrorSync here since this kind of error could result in a - // segfault in the near future. -#if USE_NEW_LOGGING_FORMAT - std::stringstream ss; - ss << "Received completion for an old sample." - << " Min expected id: " << latencies_first_sample_sequence_id_ - << " Actual id: " << sample_sequence_id; - MLPERF_LOG_ERROR_SYNC(GlobalLogger(), "error_runtime", ss.str()); -#else - GlobalLogger().LogErrorSync( - "Received completion for an old sample.", "Min expected id", - latencies_first_sample_sequence_id_, "Actual id", sample_sequence_id); -#endif - return; - } - - const size_t i = sample_sequence_id - latencies_first_sample_sequence_id_; - - if (latencies_.size() > i) { - if (latencies_[i] != kInvalidLatency) { -#if USE_NEW_LOGGING_FORMAT - MLPERF_LOG_ERROR_SYNC( - GlobalLogger(), "error_runtime", - "Attempted to record token latency after sample was completed"); -#else - GlobalLogger().LogErrorSync( - "Attempted to record token latency after sample was completed"); -#endif - - // Return without recording the latency again to avoid potentially - // ending the test before the SUT is actually done, which could result - // in a segfault. - // If the SUT recorded the wrong sample, the test will hang and see - // the error above. - return; - } - } - if (token_latencies_.size() <= i) { - // TODO: Reserve in advance. - token_latencies_.resize(i + 1, kInvalidLatency); - } else if (token_latencies_[i] != kInvalidLatency) { - // Call LogErrorSync here since this kind of error could result in a - // segfault in the near future. -#if USE_NEW_LOGGING_FORMAT - MLPERF_LOG_ERROR_SYNC(GlobalLogger(), "error_runtime", - "Attempted to complete a sample twice."); -#else - GlobalLogger().LogErrorSync("Attempted to complete a sample twice."); -#endif - - // Return without recording the latency again to avoid potentially - // ending the test before the SUT is actually done, which could result - // in a segfault. - // If the SUT recorded the wrong sample, the test will hang and see - // the error above. - return; - } - token_latencies_[i] = latency; -} - -std::vector AsyncLog::GetLatenciesBlocking( - size_t expected_count) { - std::vector latencies; - { - std::unique_lock lock(latencies_mutex_); - latencies_expected_ = expected_count; - all_latencies_recorded_.wait(lock, [&] { return AllLatenciesRecorded(); }); - latencies.swap(latencies_); - } - - if (latencies.size() != expected_count) { - // Call LogErrorSync here since this kind of error could result in a - // segfault in the near future. -#if USE_NEW_LOGGING_FORMAT - std::stringstream ss; - ss << "Received SequenceId that was too large." - << " expected_size: " << expected_count - << " actual_size: " << latencies.size(); - MLPERF_LOG_ERROR_SYNC(GlobalLogger(), "error_runtime", ss.str()); -#else - GlobalLogger().LogErrorSync("Received SequenceId that was too large.", - "expected_size", expected_count, "actual_size", - latencies.size()); -#endif - } - - size_t invalid_latency_count = 0; - for (auto l : latencies) { - if (l == kInvalidLatency) { - invalid_latency_count++; - } - } - if (invalid_latency_count != 0) { - // Call LogErrorSync here since this kind of error could result in a - // segfault in the near future. -#if USE_NEW_LOGGING_FORMAT - std::stringstream ss; - ss << "Encountered incomplete samples at the end of a series of queries." - << " count: " << invalid_latency_count; - MLPERF_LOG_ERROR_SYNC(GlobalLogger(), "error_runtime", ss.str()); -#else - GlobalLogger().LogErrorSync( - "Encountered incomplete samples at the end of a series of queries.", - "count", invalid_latency_count); -#endif - } - - return latencies; -} - -std::vector AsyncLog::GetTokenLatencies( - size_t expected_count) { - std::vector token_latencies; - token_latencies.swap(token_latencies_); - return token_latencies; -} - -std::vector AsyncLog::GetTimePerOutputToken( - size_t expected_count) { - std::vector tpot_latencies; - tpot_latencies.swap(time_per_output_token_); - return tpot_latencies; -} - -std::vector AsyncLog::GetTokensPerSample(size_t expected_count) { - std::vector tokens_per_sample; - tokens_per_sample.swap(tokens_per_sample_); - return tokens_per_sample; -} - -PerfClock::time_point AsyncLog::GetMaxCompletionTime() { - return max_completion_timstamp_; -} - -QuerySampleLatency AsyncLog::GetMaxLatencySoFar() { - std::unique_lock lock(latencies_mutex_); - return max_latency_; -} - -void AsyncLog::SetUseTokens(bool use_tokens) { use_tokens_ = use_tokens; } - -void AsyncLog::SetNeedsFirstToken(bool needs_first_token) { - needs_first_token_ = needs_first_token; -} - -/// \brief Records a single thread using thread-local storage and submits -/// entries to the central Logger. -/// -/// \details This setup allows for each log entry to be added: -/// * With forward-progress guarantees. (i.e.: no locking or blocking -/// operations even if other threads have stalled.) -/// * Without expensive syscalls or I/O operations, which are deferred to -/// the central Logger. -class TlsLogger { - public: - TlsLogger(std::function forced_detatch); - ~TlsLogger(); - void ForcedDetatchFromThread() { forced_detatch_(); } - - void Log(AsyncLogEntry&& entry); - void SwapBuffers(); - - std::vector* StartReadingEntries(); - void FinishReadingEntries(); - bool ReadBufferHasBeenConsumed(); - size_t MaxEntryVectorSize() { return max_entry_size_; } - - uint64_t Pid() const { return pid_; } - uint64_t Tid() const { return tid_; } - - void RequestSwapBuffersSlotRetried() { - swap_buffers_slot_retry_count_.fetch_add(1, std::memory_order_relaxed); - } - - size_t ReportLogCasFailCount() { - size_t c = log_cas_fail_count_.load(std::memory_order_relaxed); - log_cas_fail_count_.fetch_sub(c, std::memory_order_relaxed); - return c; - } - - size_t ReportSwapBuffersSlotRetryCount() { - size_t c = swap_buffers_slot_retry_count_.load(std::memory_order_relaxed); - swap_buffers_slot_retry_count_.fetch_sub(c, std::memory_order_relaxed); - return c; - } - - void TraceCounters(); - - private: - using EntryVector = std::vector; - enum class EntryState { Unlocked, ReadLock, WriteLock }; - - // Accessed by producer only. - size_t i_read_ = 0; - - // Accessed by producer and consumer atomically. - EntryVector entries_[2]; - std::atomic entry_states_[2]{{EntryState::ReadLock}, - {EntryState::Unlocked}}; - std::atomic i_write_{1}; - - std::atomic log_cas_fail_count_{0}; - std::atomic swap_buffers_slot_retry_count_{0}; - - // Accessed by consumer only. - size_t unread_swaps_ = 0; - size_t i_write_prev_ = 0; - uint64_t pid_; - uint64_t tid_; - size_t max_entry_size_ = kTlsLogReservedEntryCount; - - std::function forced_detatch_; -}; - -Logger::Logger(std::chrono::duration poll_period, - size_t max_threads_to_log) - : poll_period_(poll_period), - max_threads_to_log_(max_threads_to_log), - thread_swap_request_slots_(max_threads_to_log * 2) { - const size_t kSlotCount = max_threads_to_log * 2; - for (size_t i = 0; i < kSlotCount; i++) { - std::atomic_init(&thread_swap_request_slots_[i], - SwapRequestSlotIsWritableValue(i)); - } -} - -Logger::~Logger() { - // TlsLoggers might outlive this Logger when loaded as a python module. - // Forcefully make all currently registered TlsLoggers orphans. - std::unique_lock lock(tls_loggers_registerd_mutex_); - TlsLogger* tls_logger_prev = nullptr; - (void)tls_logger_prev; // Avoid unused error in release builds. - while (!tls_loggers_registerd_.empty()) { - TlsLogger* tls_logger = *tls_loggers_registerd_.begin(); - // Otherwise, this is an infinite loop. - assert(tls_logger != tls_logger_prev); - tls_loggers_registerd_mutex_.unlock(); - tls_logger->ForcedDetatchFromThread(); - tls_loggers_registerd_mutex_.lock(); - tls_logger_prev = tls_logger; - } -} - -void Logger::RequestSwapBuffers(TlsLogger* tls_logger) { - auto tls_logger_as_uint = reinterpret_cast(tls_logger); - assert(SwapRequestSlotIsReadable(tls_logger_as_uint)); - size_t id, slot; - uintptr_t slot_is_writeable_value; - // The compare_exchange below should almost always succeed. - // The compare_exchange may fail if a recycled slot is still actively used - // by another thread, so we retry with subsequent slots here if needed. - // Since the slot count is 2x the expected number of threads to log, - // the CAS should only fail at most 50% of the time when all logging threads - // happen to be descheduled between the fetch_add and CAS below, which is - // very unlikely. - id = swap_request_id_.fetch_add(1, std::memory_order_relaxed); - slot = id % thread_swap_request_slots_.size(); - slot_is_writeable_value = SwapRequestSlotIsWritableValue(id); - while (!thread_swap_request_slots_[slot].compare_exchange_strong( - slot_is_writeable_value, tls_logger_as_uint, std::memory_order_release)) { - id = swap_request_id_.fetch_add(1, std::memory_order_relaxed); - slot = id % thread_swap_request_slots_.size(); - slot_is_writeable_value = SwapRequestSlotIsWritableValue(id); - tls_logger->RequestSwapBuffersSlotRetried(); - } -} - -void Logger::RegisterTlsLogger(TlsLogger* tls_logger) { - std::unique_lock lock(tls_loggers_registerd_mutex_); - if (tls_loggers_registerd_.size() >= max_threads_to_log_) { -#if USE_NEW_LOGGING_FORMAT - MLPERF_LOG_ERROR_SYNC((*this), "error_runtime", - "Warning: More TLS loggers registerd than can be " - "active simultaneously."); -#else - LogErrorSync( - "Warning: More TLS loggers registerd than can " - "be active simultaneously.\n"); -#endif - } - tls_loggers_registerd_.insert(tls_logger); -} - -// This moves ownership of the tls_logger data to Logger so the -// exiting thread can exit immediately, even if all the logs of the -// exiting thread haven't been processed. -void Logger::UnRegisterTlsLogger(std::unique_ptr tls_logger) { - OrphanContainer::iterator orphan; - { - std::unique_lock lock(tls_logger_orphans_mutex_); - tls_logger_orphans_.emplace_front(std::move(tls_logger)); - orphan = tls_logger_orphans_.begin(); - } - - // Only remove the TlsLogger from the registry after adding to orphans so - // CollectTlsLoggerStats doesn't have any gaps in coverage. - { - std::unique_lock lock(tls_loggers_registerd_mutex_); - tls_loggers_registerd_.erase(orphan->get()); - } - - // This will flush the logs of |tls_logger| and mark it for destruction. - // Deferring destruction via orphans_to_destroy helps avoid use-after-frees - // when the IOThread calls FinishReadingEntries. - (*orphan)->Log([this, orphan](AsyncLog&) { - CollectTlsLoggerStats(orphan->get()); - orphans_to_destroy_.push_back(orphan); - }); -} - -void Logger::CollectTlsLoggerStats(TlsLogger* tls_logger) { - tls_total_log_cas_fail_count_ += tls_logger->ReportLogCasFailCount(); - tls_total_swap_buffers_slot_retry_count_ += - tls_logger->ReportSwapBuffersSlotRetryCount(); - - size_t max_entry_vector_size = tls_logger->MaxEntryVectorSize(); - if (max_entry_vector_size > kTlsLogReservedEntryCount) { -#if USE_NEW_LOGGING_FORMAT - std::stringstream msg; - msg << "Logging allocation detected:" << " tid: " << tls_logger->Tid() - << " reserved_entries: " << kTlsLogReservedEntryCount - << " max_entries: " << max_entry_vector_size; - MLPERF_LOG_WARNING((*this), "warning_generic_message", msg.str()); -#else - async_logger_.FlagWarning(); - async_logger_.LogDetail("Logging allocation detected: ", "tid", - tls_logger->Tid(), "reserved_entries", - kTlsLogReservedEntryCount, "max_entries", - max_entry_vector_size); -#endif - } -} - -void Logger::StartIOThread() { - { - std::unique_lock lock(io_thread_mutex_); - keep_io_thread_alive_ = true; - } - io_thread_ = std::thread(&Logger::IOThread, this); -} - -void Logger::StopIOThread() { - { - std::unique_lock lock(io_thread_mutex_); - keep_io_thread_alive_ = false; - io_thread_cv_.notify_all(); - } - io_thread_.join(); -} - -void Logger::StartLogging(std::ostream* summary, std::ostream* detail, - std::ostream* accuracy, bool copy_detail_to_stdout, - bool copy_summary_to_stdout) { - async_logger_.SetLogFiles(summary, detail, accuracy, copy_detail_to_stdout, - copy_summary_to_stdout, PerfClock::now()); -} - -void Logger::StopLogging() { - if (std::this_thread::get_id() == io_thread_.get_id()) { -#if USE_NEW_LOGGING_FORMAT - MLPERF_LOG_ERROR_SYNC((*this), "error_runtime", - "StopLogging() not supported from IO thread."); -#else - LogErrorSync("StopLogging() not supported from IO thread."); -#endif - return; - } - - // Flush logs from this thread. - std::promise io_thread_flushed_this_thread; - Log([&](AsyncLog&) { io_thread_flushed_this_thread.set_value(); }); - io_thread_flushed_this_thread.get_future().wait(); - async_logger_.SetLogFiles(&std::cerr, &std::cerr, &std::cerr, false, false, - PerfClock::now()); -} - -void Logger::StartNewTrace(std::ostream* trace_out, - PerfClock::time_point origin) { - async_logger_.StartNewTrace(trace_out, origin); -} - -void Logger::StopTracing() { - // Flush traces from this thread. - std::promise io_thread_flushed_this_thread; - Log([&](AsyncLog&) { io_thread_flushed_this_thread.set_value(); }); - io_thread_flushed_this_thread.get_future().wait(); - async_logger_.StopTrace(); -} - -void Logger::LogContentionAndAllocations() { - LogDetail([&](AsyncDetail& detail) { - { - std::unique_lock lock(tls_loggers_registerd_mutex_); - for (auto tls_logger : tls_loggers_registerd_) { - CollectTlsLoggerStats(tls_logger); - } - } - - { - std::unique_lock lock(tls_logger_orphans_mutex_); - for (auto& orphan : tls_logger_orphans_) { - CollectTlsLoggerStats(orphan.get()); - } - } - -#if USE_NEW_LOGGING_FORMAT - MLPERF_LOG(detail, "logger_swap_request_slots_retry_count", - swap_request_slots_retry_count_); - MLPERF_LOG(detail, "logger_swap_request_slots_retry_retry_count", - swap_request_slots_retry_retry_count_); - MLPERF_LOG(detail, "logger_swap_request_slots_retry_reencounter_count", - swap_request_slots_retry_reencounter_count_); - MLPERF_LOG(detail, "logger_start_reading_entries_retry_count", - start_reading_entries_retry_count_); - MLPERF_LOG(detail, "logger_tls_total_log_cas_fail_count", - tls_total_log_cas_fail_count_); - MLPERF_LOG(detail, "logger_tls_total_swap_buffers_slot_retry_count", - tls_total_swap_buffers_slot_retry_count_); -#else - detail("Log Contention Counters:"); - detail(std::to_string(swap_request_slots_retry_count_) + - " : swap_request_slots_retry_count"); - detail(std::to_string(swap_request_slots_retry_retry_count_) + - " : swap_request_slots_retry_retry_count"); - detail(std::to_string(swap_request_slots_retry_reencounter_count_) + - " : swap_request_slots_retry_reencounter_count"); - detail(std::to_string(start_reading_entries_retry_count_) + - " : start_reading_entries_retry_count"); - detail(std::to_string(tls_total_log_cas_fail_count_) + - " : tls_total_log_cas_fail_count"); - detail(std::to_string(tls_total_swap_buffers_slot_retry_count_) + - " : tls_total_swap_buffers_slot_retry_count"); -#endif - - swap_request_slots_retry_count_ = 0; - swap_request_slots_retry_retry_count_ = 0; - swap_request_slots_retry_reencounter_count_ = 0; - start_reading_entries_retry_count_ = 0; - tls_total_log_cas_fail_count_ = 0; - tls_total_swap_buffers_slot_retry_count_ = 0; - }); -} - -void Logger::RestartLatencyRecording(uint64_t first_sample_sequence_id, - size_t latencies_to_reserve) { - async_logger_.RestartLatencyRecording(first_sample_sequence_id, - latencies_to_reserve); -} - -std::vector Logger::GetLatenciesBlocking( - size_t expected_count) { - return async_logger_.GetLatenciesBlocking(expected_count); -} -std::vector Logger::GetTokenLatencies( - size_t expected_count) { - return async_logger_.GetTokenLatencies(expected_count); -} -std::vector Logger::GetTimePerOutputToken( - size_t expected_count) { - return async_logger_.GetTimePerOutputToken(expected_count); -} -std::vector Logger::GetTokensPerSample( - size_t expected_count) { - return async_logger_.GetTokensPerSample(expected_count); -} - -PerfClock::time_point Logger::GetMaxCompletionTime() { - return async_logger_.GetMaxCompletionTime(); -} - -QuerySampleLatency Logger::GetMaxLatencySoFar() { - return async_logger_.GetMaxLatencySoFar(); -} - -void Logger::SetUseTokens(bool use_tokens) { - async_logger_.SetUseTokens(use_tokens); -} - -void Logger::SetNeedsFirstToken(bool needs_first_token) { - async_logger_.SetNeedsFirstToken(needs_first_token); -} - -TlsLogger* Logger::GetTlsLoggerThatRequestedSwap(size_t slot, size_t next_id) { - uintptr_t slot_value = thread_swap_request_slots_[slot].load(); - if (SwapRequestSlotIsReadable(slot_value)) { - // TODO: Convert this block to a simple write once we are confidient - // that we don't need to check for success. - bool success = thread_swap_request_slots_[slot].compare_exchange_strong( - slot_value, SwapRequestSlotIsWritableValue(next_id)); - if (!success) { -#if USE_NEW_LOGGING_FORMAT - MLPERF_LOG_WARNING((*this), "warning_generic_message", "CAS failed."); -#else - LogErrorSync("CAS failed.", "line", __LINE__); -#endif - assert(success); - } - return reinterpret_cast(slot_value); - } - return nullptr; -} - -void Logger::GatherRetrySwapRequests(std::vector* threads_to_swap) { - if (swap_request_slots_to_retry_.empty()) { - return; - } - - std::vector retry_slots; - retry_slots.swap(swap_request_slots_to_retry_); - for (auto& slot_retry : retry_slots) { - TlsLogger* tls_logger = - GetTlsLoggerThatRequestedSwap(slot_retry.slot, slot_retry.next_id); - if (tls_logger) { - threads_to_swap->push_back(tls_logger); - } else { - swap_request_slots_to_retry_.push_back(slot_retry); - swap_request_slots_retry_retry_count_++; - } - } -} - -void Logger::GatherNewSwapRequests(std::vector* threads_to_swap) { - auto swap_request_end = swap_request_id_.load(std::memory_order_acquire); - for (; swap_request_id_read_ < swap_request_end; swap_request_id_read_++) { - size_t slot = swap_request_id_read_ % thread_swap_request_slots_.size(); - size_t next_id = swap_request_id_read_ + thread_swap_request_slots_.size(); - TlsLogger* tls_logger = GetTlsLoggerThatRequestedSwap(slot, next_id); - if (tls_logger) { - threads_to_swap->push_back(tls_logger); - } else { - swap_request_slots_retry_count_++; - // A thread is in the middle of its call to RequestSwapBuffers. - // Retry later once it's done. - auto it = std::find_if(swap_request_slots_to_retry_.begin(), - swap_request_slots_to_retry_.end(), - [=](SlotRetry& s) { return s.slot == slot; }); - if (it == swap_request_slots_to_retry_.end()) { - // This is the first time we are retrying the slot. - swap_request_slots_to_retry_.push_back({slot, next_id}); - } else { - // Whoa. We've been retrying this slot since the last time it was - // encountered. Just update the next_id. - it->next_id = next_id; - swap_request_slots_retry_reencounter_count_++; - } - } - } -} - -void Logger::IOThread() { - while (keep_io_thread_alive_) { - auto tracer1 = - MakeScopedTracer([](AsyncTrace& trace) { trace("IOThreadLoop"); }); - { - auto tracer2 = MakeScopedTracer([](AsyncTrace& trace) { trace("Wait"); }); - std::unique_lock lock(io_thread_mutex_); - io_thread_cv_.wait_for(lock, poll_period_, - [&] { return !keep_io_thread_alive_; }); - } - - { - auto tracer3 = - MakeScopedTracer([](AsyncTrace& trace) { trace("Gather"); }); - std::vector threads_to_swap; - threads_to_swap.swap(threads_to_swap_deferred_); - GatherRetrySwapRequests(&threads_to_swap); - GatherNewSwapRequests(&threads_to_swap); - for (TlsLogger* thread : threads_to_swap) { - if (thread->ReadBufferHasBeenConsumed()) { - thread->SwapBuffers(); - // After swapping a thread, it's ready to be read. - threads_to_read_.push_back(thread); - } else { - // Don't swap buffers again until we've finish reading the - // previous swap. - threads_to_swap_deferred_.push_back(thread); - } - } - } - - { - auto tracer4 = - MakeScopedTracer([](AsyncTrace& trace) { trace("Process"); }); - // Read from the threads we are confident have activity. - for (std::vector::iterator thread = threads_to_read_.begin(); - thread != threads_to_read_.end(); thread++) { - auto tracer5 = - MakeScopedTracer([tid = (*thread)->Tid()](AsyncTrace& trace) { - trace("Thread", "tid", tid); - }); - std::vector* entries = (*thread)->StartReadingEntries(); - if (!entries) { - start_reading_entries_retry_count_++; - continue; - } - - async_logger_.SetCurrentPidTid((*thread)->Pid(), (*thread)->Tid()); - for (auto& entry : *entries) { - // Execute the entry to perform the serialization and I/O. - entry(async_logger_); - } - (*thread)->FinishReadingEntries(); - // Mark for removal by the call to RemoveValue below. - *thread = nullptr; - } - - // Only remove threads where reading succeeded so we retry the failed - // threads the next time around. - RemoveValue(&threads_to_read_, nullptr); - } - - // Explicitly flush every time we wake up. The goal being minimization - // of large implicit flushes which could affect tail latency measurements, - // especially at percentiles closer to 100%. - /// \todo Determine if explicitly flushing logs every wake up is better - /// than relying on implicit flushing. - { - auto tracer6 = - MakeScopedTracer([](AsyncTrace& trace) { trace("FlushAll"); }); - async_logger_.Flush(); - } - - if (!orphans_to_destroy_.empty()) { - auto tracer7 = MakeScopedTracer( - [](AsyncTrace& trace) { trace("Abandoning Orphans"); }); - std::unique_lock lock(tls_logger_orphans_mutex_); - for (auto orphan : orphans_to_destroy_) { - tls_logger_orphans_.erase(orphan); - } - orphans_to_destroy_.clear(); - } - } -} - -TlsLogger::TlsLogger(std::function forced_detatch) - : pid_(MLPERF_GET_PID()), - tid_(MLPERF_GET_TID()), - forced_detatch_(std::move(forced_detatch)) { - for (auto& entry : entries_) { - entry.reserve(kTlsLogReservedEntryCount); - } -} - -TlsLogger::~TlsLogger() {} - -// Log always makes forward progress since it can unconditionally obtain a -// "lock" on at least one of the buffers for writing. -// Notificiation is also lock free. -void TlsLogger::Log(AsyncLogEntry&& entry) { - size_t cas_fail_count = 0; - auto unlocked = EntryState::Unlocked; - size_t i_write = i_write_.load(std::memory_order_relaxed); - while (!entry_states_[i_write].compare_exchange_strong( - unlocked, EntryState::WriteLock, std::memory_order_acquire, - std::memory_order_relaxed)) { - unlocked = EntryState::Unlocked; - i_write ^= 1; - // We may need to try 3 times, since there could be a race with a - // previous SwapBuffers request and we use memory_order_relaxed when - // loading i_write_ above. - cas_fail_count++; - if (cas_fail_count >= 3) { -#if USE_NEW_LOGGING_FORMAT - MLPERF_LOG_WARNING(GlobalLogger(), "warning_generic_message", - "CAS failed."); -#else - GlobalLogger().LogErrorSync("CAS failed.", "times", cas_fail_count, - "line", __LINE__); -#endif - } - log_cas_fail_count_.fetch_add(1, std::memory_order_relaxed); - } - entries_[i_write].emplace_back(std::forward(entry)); - - // TODO: Convert this block to a simple write once we are confidient - // that we don't need to check for success. - auto write_lock = EntryState::WriteLock; - bool success = entry_states_[i_write].compare_exchange_strong( - write_lock, EntryState::Unlocked, std::memory_order_release); - if (!success) { -#if USE_NEW_LOGGING_FORMAT - MLPERF_LOG_WARNING(GlobalLogger(), "warning_generic_message", - "CAS failed."); -#else - GlobalLogger().LogErrorSync("CAS failed.", "line", __LINE__); -#endif - assert(success); - } - - bool write_buffer_swapped = i_write_prev_ != i_write; - if (write_buffer_swapped) { - GlobalLogger().RequestSwapBuffers(this); - i_write_prev_ = i_write; - } -} - -void TlsLogger::SwapBuffers() { - // TODO: Convert this block to a simple write once we are confidient - // that we don't need to check for success. - auto read_lock = EntryState::ReadLock; - bool success = entry_states_[i_read_].compare_exchange_strong( - read_lock, EntryState::Unlocked, std::memory_order_release); - if (!success) { -#if USE_NEW_LOGGING_FORMAT - MLPERF_LOG_WARNING(GlobalLogger(), "warning_generic_message", - "CAS failed."); -#else - GlobalLogger().LogErrorSync("CAS failed.", "line", __LINE__); -#endif - assert(success); - } - - i_write_.store(i_read_, std::memory_order_relaxed); - i_read_ ^= 1; - unread_swaps_++; -} - -// Returns nullptr if read lock fails. -std::vector* TlsLogger::StartReadingEntries() { - auto unlocked = EntryState::Unlocked; - if (entry_states_[i_read_].compare_exchange_strong( - unlocked, EntryState::ReadLock, std::memory_order_acquire, - std::memory_order_relaxed)) { - return &entries_[i_read_]; - } - return nullptr; -} - -void TlsLogger::FinishReadingEntries() { - // Detect first logging allocation and track max allocated size. - size_t new_size = entries_[i_read_].size(); - if (new_size > max_entry_size_) { - if (max_entry_size_ == kTlsLogReservedEntryCount) { - Log([ts = PerfClock::now()](AsyncLog& log) { - log.TraceAsyncInstant("FirstAllocation", 0, ts); - }); - } - max_entry_size_ = new_size; - } - - entries_[i_read_].clear(); - unread_swaps_--; -} - -bool TlsLogger::ReadBufferHasBeenConsumed() { return unread_swaps_ == 0; } - -void TlsLogger::TraceCounters() { - auto tracer = MakeScopedTracer( - [lcfc = log_cas_fail_count_.load(std::memory_order_relaxed), - sbsrc = swap_buffers_slot_retry_count_.load(std::memory_order_relaxed)]( - AsyncTrace& trace) { - trace("TlsLogger:ContentionCounters", "log_cas_fail_count", lcfc, - "swap_buffers_slot_retry_count", sbsrc); - }); -} - -Logger& GlobalLogger() { - static Logger g_logger(kLogPollPeriod, kMaxThreadsToLog); - return g_logger; -} - -/// \brief Moves ownership of the TlsLogger to Logger on thread exit -/// so no round-trip synchronization with the IO thread is required. -struct TlsLoggerWrapper { - TlsLoggerWrapper(std::function forced_detatch) - : tls_logger(std::make_unique(std::move(forced_detatch))) { - GlobalLogger().RegisterTlsLogger(tls_logger.get()); - } - ~TlsLoggerWrapper() { - tls_logger->TraceCounters(); - GlobalLogger().UnRegisterTlsLogger(std::move(tls_logger)); - } - std::unique_ptr tls_logger; -}; - -TlsLoggerWrapper* InitializeMyTlsLoggerWrapper() { - thread_local std::unique_ptr tls_logger_wrapper; - // forced_detatch lets the global Logger forcefully detatch TlsLoggers - // from the thread in the Logger's destructor, which may run before - // thread-local variables are destroyed when the loadgen is used as a python - // module and dynamically unloaded. - // Note: We capture a pointer to the tls_logger_wrapper since variables of - // the thread-local storage class aren't actually captured. C++ spec says - // only variables of the automatic storage class are captured. - /// \todo There is a race where the same TlsLoggerWrapper might be - /// destroyed both naturally and via forced_detatch. Destruction of - /// the TlsLoggerWrapper should be locked. - auto forced_detatch = [tls_logger_wrapper = &tls_logger_wrapper]() { - tls_logger_wrapper->reset(); - }; - tls_logger_wrapper = std::make_unique(forced_detatch); - return tls_logger_wrapper.get(); -} - -TlsLogger* InitializeMyTlsLogger() { - thread_local TlsLoggerWrapper* wrapper = InitializeMyTlsLoggerWrapper(); - return wrapper->tls_logger.get(); -} - -void Log(AsyncLogEntry&& entry) { - thread_local TlsLogger* const tls_logger = InitializeMyTlsLogger(); - tls_logger->Log(std::forward(entry)); -} - -} // namespace logging -} // namespace mlperf diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/logging.h b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/logging.h deleted file mode 100644 index 8f1a398e9..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/logging.h +++ /dev/null @@ -1,816 +0,0 @@ -/* Copyright 2019 The MLPerf Authors. All Rights Reserved. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -/// \file -/// \brief Internal logging implementation details. - -#ifndef MLPERF_LOADGEN_LOGGING_H_ -#define MLPERF_LOADGEN_LOGGING_H_ - -#define USE_NEW_LOGGING_FORMAT 1 -#define MLPERF_LOG(logger, key, value) \ - logger.Log((key), (value), __FILE__, __LINE__) -#define MLPERF_LOG_ERROR(logger, key, value) \ - logger.LogError((key), (value), __FILE__, __LINE__) -#define MLPERF_LOG_ERROR_SYNC(logger, key, value) \ - logger.LogErrorSync((key), (value), __FILE__, __LINE__) -#define MLPERF_LOG_WARNING(logger, key, value) \ - logger.LogWarning((key), (value), __FILE__, __LINE__) -#define MLPERF_LOG_INTERVAL_START(logger, key, value) \ - logger.LogIntervalStart((key), (value), __FILE__, __LINE__) -#define MLPERF_LOG_INTERVAL_END(logger, key, value) \ - logger.LogIntervalEnd((key), (value), __FILE__, __LINE__) - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "query_sample.h" - -namespace mlperf { - -/// \brief Wait-free logging utilities that defer stringification -/// and syscalls to a worker thread. -namespace logging { - -class AsyncLog; -class Logger; -class TlsLogger; -struct TlsLoggerWrapper; - -/// \todo Verify lambas are not allocating when bounded to a std::function. -using AsyncLogEntry = std::function; -using PerfClock = std::chrono::high_resolution_clock; - -/// \brief Logs the raw bytes as a hexadecimal ascii string. -struct LogBinaryAsHexString { - std::vector* data; -}; - -/// \brief By default, print out the value directly. -template -const T& ArgValueTransform(const T& value) { - return value; -} - -/// \brief Print out True/False. -const std::string& ArgValueTransform(const bool& value); -/// \brief Print out binary day as hex string. -const std::string ArgValueTransform(const LogBinaryAsHexString& value); -#if USE_NEW_LOGGING_FORMAT -/// \brief Print out a string in JSON format (with quotes). -const std::string ArgValueTransform(const std::string& value); -const std::string ArgValueTransform(const char* value); -/// \brief Prints a list of int in JSON format. -const std::string ArgValueTransform(const std::vector& value); -/// \brief Prints a dict in JSON format. -const std::string ArgValueTransform( - const std::map& value); -#endif - -/// \brief Helper to print out values without quotes when value is a string. -template -const T& ArgValueTransformWithoutQuote(const T& value) { - return ArgValueTransform(value); -} -inline const std::string ArgValueTransformWithoutQuote( - const LogBinaryAsHexString& value) { - return ArgValueTransform(value); -} -/// \brief Helper to print out a string without the quotes. -inline const std::string ArgValueTransformWithoutQuote( - const std::string& value) { - return value; -} - -/// \brief Outputs a trace that can be uploaded to chrome://tracing for -/// visualization. -/// \details Trace event format definition: -/// https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/edit?usp=sharing -class ChromeTracer { - public: - ChromeTracer(std::ostream* trace_out, PerfClock::time_point origin); - ~ChromeTracer(); - - template - void AddCompleteEvent(const std::string& name, uint64_t pid, uint64_t tid, - PerfClock::time_point start, PerfClock::time_point end, - const Args... args) { - *out_ << "{\"name\":\"" << name << "\"," << "\"ph\":\"X\"," - << "\"pid\":" << pid << "," << "\"tid\":" << tid << "," - << "\"ts\":" << Micros(start - origin_).count() << "," - << "\"dur\":" << Micros(end - start).count() << "," << "\"args\":{"; - AddArgs(args...); - *out_ << "}},\n"; - } - - template - void AddAsyncBeginEvent(const std::string& name, uint64_t pid, uint64_t id, - PerfClock::time_point time, const Args... args) { - *out_ << "{\"name\":\"" << name << "\"," << "\"cat\":\"default\"," - << "\"ph\":\"b\"," << "\"pid\":" << pid << "," << "\"id\":" << id - << "," << "\"ts\":" << Micros(time - origin_).count() << "," - << "\"args\":{"; - AddArgs(args...); - *out_ << "}},\n"; - } - - template - void AddAsyncInstantEvent(const std::string& name, uint64_t pid, uint64_t id, - PerfClock::time_point time, const Args... args) { - *out_ << "{\"name\":\"" << name << "\"," << "\"cat\":\"default\"," - << "\"ph\":\"n\"," << "\"pid\":" << pid << "," << "\"id\":" << id - << "," << "\"ts\":" << Micros(time - origin_).count() << "," - << "\"args\":{"; - AddArgs(args...); - *out_ << "}},\n"; - } - - template - void AddAsyncEndEvent(const std::string& name, uint64_t pid, uint64_t id, - PerfClock::time_point time) { - *out_ << "{\"name\":\"" << name << "\"," << "\"cat\":\"default\"," - << "\"ph\":\"e\", " << "\"pid\":" << pid << "," << "\"id\":" << id - << "," << "\"ts\":" << Micros(time - origin_).count() << "},\n"; - } - - template - void AddCounterEvent(const std::string& name, uint64_t pid, - PerfClock::time_point time, const Args... args) { - *out_ << "{\"name\":\"" << name << "\"," << "\"ph\": \"C\"," - << "\"pid\":" << pid << "," - << "\"ts\":" << Micros(time - origin_).count() << "," - << "\"args\":{ "; - AddArgs(args...); - *out_ << "}},\n"; - } - - void Flush() { out_->flush(); } - - private: - using Micros = std::chrono::duration; - - void WriteTraceEventHeader(); - void WriteTraceEventFooter(); - - void AddArgs() {} - - template - void AddArgs(const std::string& arg_name, const T& arg_value) { - *out_ << "\"" << arg_name << "\":" << ArgValueTransform(arg_value); - } - - template - void AddArgs(const std::string& arg_name, const T& arg_value, - const Args... args) { - *out_ << "\"" << arg_name << "\":" << ArgValueTransform(arg_value) << ","; - AddArgs(args...); - } - - std::ostream* out_; - PerfClock::time_point origin_; -}; - -/// \brief The proxy all logging lambdas ultimately use to write any log type. -/// \details Passed as an argument to the log lambda on the -/// recording thread to serialize the data captured by the lambda and -/// forward it to the output stream. -/// \todo Make summary_out_, detail_out_, accuracy_out_, and trace_out_ -/// instances of a new LogOutput interface that the client may override. -class AsyncLog { - public: - void SetLogFiles(std::ostream* summary, std::ostream* detail, - std::ostream* accuracy, bool copy_detail_to_stdout, - bool copy_summary_to_stdout, - PerfClock::time_point log_origin); - void StartNewTrace(std::ostream* trace_out, PerfClock::time_point origin); - void StopTrace(); - void Flush(); - - void SetCurrentPidTid(uint64_t pid, uint64_t tid); - - void LogAccuracy(uint64_t seq_id, const QuerySampleIndex qsl_idx, - const LogBinaryAsHexString& response, int64_t n_tokens); - void CacheToken(uint64_t seq_id, const LogBinaryAsHexString& response); - - template - void LogSummary(const std::string& message, const Args... args); - - void SetLogDetailTime(PerfClock::time_point time) { log_detail_time_ = time; } - - void FlagError() { - std::unique_lock lock(log_mutex_); - log_error_count_++; - error_flagged_ = true; - } - - void FlagWarning() { - std::unique_lock lock(log_mutex_); - log_warning_count_++; - warning_flagged_ = true; - } - -#if USE_NEW_LOGGING_FORMAT - template - void LogDetail(const std::string& key, const T& value, - const std::string file_name, const unsigned int line_no); -#else - template - void LogDetail(const std::string& message, const Args... args); -#endif - - template - void Trace(const std::string& trace_name, PerfClock::time_point start, - PerfClock::time_point end, const Args... args) { - std::unique_lock lock(trace_mutex_); - if (tracer_) { - tracer_->AddCompleteEvent(trace_name, current_pid_, current_tid_, start, - end, args...); - } - } - - template - void TraceAsyncInstant(const std::string& trace_name, uint64_t id, - PerfClock::time_point instant_time, - const Args... args) { - std::unique_lock lock(trace_mutex_); - if (tracer_) { - tracer_->AddAsyncInstantEvent(trace_name, current_pid_, id, instant_time, - args...); - } - } - - void SetScopedTraceTimes(PerfClock::time_point start, - PerfClock::time_point end) { - scoped_start_ = start; - scoped_end_ = end; - } - - template - void ScopedTrace(const std::string& trace_name, const Args... args) { - std::unique_lock lock(trace_mutex_); - if (tracer_) { - tracer_->AddCompleteEvent(trace_name, current_pid_, current_tid_, - scoped_start_, scoped_end_, args...); - } - } - - template - void TraceSample(const std::string& trace_name, uint64_t id, - PerfClock::time_point start, PerfClock::time_point end, - const Args... args) { - std::unique_lock lock(trace_mutex_); - if (tracer_) { - tracer_->AddAsyncBeginEvent(trace_name, current_pid_, id, start, args...); - tracer_->AddAsyncEndEvent(trace_name, current_pid_, id, end); - } - } - - template - void TraceCounterEvent(const std::string& trace_name, - PerfClock::time_point time, const Args... args) { - std::unique_lock lock(trace_mutex_); - if (tracer_) { - tracer_->AddCounterEvent(trace_name, current_pid_, time, args...); - } - } - - void RestartLatencyRecording(uint64_t first_sample_sequence_id, - size_t latencies_to_reserve); - void RecordSampleCompletion(uint64_t sample_sequence_id, - PerfClock::time_point completion_time, - QuerySampleLatency latency, int64_t n_tokens); - void RecordTokenCompletion(uint64_t sample_sequence_id, - PerfClock::time_point completion_time, - QuerySampleLatency latency); - std::vector GetLatenciesBlocking(size_t expected_count); - std::vector GetTokenLatencies(size_t expected_count); - std::vector GetTimePerOutputToken(size_t expected_count); - std::vector GetTokensPerSample(size_t expected_count); - PerfClock::time_point GetMaxCompletionTime(); - QuerySampleLatency GetMaxLatencySoFar(); - void SetUseTokens(bool use_tokens); - void SetNeedsFirstToken(bool needs_first_token); - size_t GetErrorCount() { return log_error_count_; }; - - private: - void WriteAccuracyHeaderLocked(); - void WriteAccuracyFooterLocked(); - - void LogArgs(std::ostream*) {} - - template - void LogArgs(std::ostream* out, const T& value_only) { - *out << ArgValueTransformWithoutQuote(value_only); - } - - template - void LogArgs(std::ostream* out, const std::string& arg_name, - const T& arg_value) { - *out << "\"" << arg_name - << "\" : " << ArgValueTransformWithoutQuote(arg_value); - } - - template - void LogArgs(std::ostream* out, const std::string& arg_name, - const T& arg_value, const Args... args) { - *out << "\"" << arg_name - << "\" : " << ArgValueTransformWithoutQuote(arg_value) << ", "; - LogArgs(out, args...); - } - - std::mutex log_mutex_; - std::ostream* summary_out_ = &std::cerr; - std::ostream* detail_out_ = &std::cerr; - std::ostream* accuracy_out_ = &std::cerr; - // TODO: Instead of these bools, use a class that forwards to two streams. - bool copy_detail_to_stdout_ = false; - bool copy_summary_to_stdout_ = false; - bool accuracy_needs_comma_ = false; - PerfClock::time_point log_origin_; - size_t log_error_count_ = 0; - bool error_flagged_ = false; - size_t log_warning_count_ = 0; - bool warning_flagged_ = false; - bool use_tokens_ = false; - bool needs_first_token_ = false; - - std::mutex trace_mutex_; - std::unique_ptr tracer_; - - uint64_t current_pid_; - uint64_t current_tid_; - PerfClock::time_point log_detail_time_; - PerfClock::time_point scoped_start_; - PerfClock::time_point scoped_end_; - - std::mutex latencies_mutex_; - std::mutex token_latencies_mutex_; - std::mutex token_record_mutex_; - std::condition_variable all_latencies_recorded_; - uint64_t latencies_first_sample_sequence_id_ = 0; - std::vector latencies_; - std::vector token_latencies_; - std::vector time_per_output_token_; - std::vector token_records_; - std::vector tokens_per_sample_; - QuerySampleLatency max_latency_ = 0; - PerfClock::time_point max_completion_timstamp_; - size_t latencies_recorded_ = 0; - size_t latencies_expected_ = 0; - // Must be called with latencies_mutex_ held. - bool AllLatenciesRecorded() { - return latencies_recorded_ == latencies_expected_; - } -}; - -/// \brief The central logger that logs all threads belonging to a run. -class Logger { - public: - Logger(std::chrono::duration poll_period, size_t max_threads_to_log); - ~Logger(); - - void StartIOThread(); - void StopIOThread(); - - void StartLogging(std::ostream* summary, std::ostream* detail, - std::ostream* accuracy, bool copy_detail_to_stdout, - bool copy_summary_to_stdout); - void StopLogging(); - - void StartNewTrace(std::ostream* trace_out, PerfClock::time_point origin); - void StopTracing(); - - void LogContentionAndAllocations(); - - void RestartLatencyRecording(uint64_t first_sample_sequence_id, - size_t latencies_to_reserve); - std::vector GetLatenciesBlocking(size_t expected_count); - std::vector GetTokenLatencies(size_t expected_count); - std::vector GetTimePerOutputToken(size_t expected_count); - std::vector GetTokensPerSample(size_t expected_count); - PerfClock::time_point GetMaxCompletionTime(); - QuerySampleLatency GetMaxLatencySoFar(); - void SetUseTokens(bool use_tokens); - void SetNeedsFirstToken(bool needs_first_token); - - private: - friend AsyncLog; - friend TlsLogger; - friend TlsLoggerWrapper; - - void RegisterTlsLogger(TlsLogger* tls_logger); - void UnRegisterTlsLogger(std::unique_ptr tls_logger); - void RequestSwapBuffers(TlsLogger* tls_logger); - void CollectTlsLoggerStats(TlsLogger* tls_logger); - - TlsLogger* GetTlsLoggerThatRequestedSwap(size_t slot, size_t next_id); - void GatherRetrySwapRequests(std::vector* threads_to_swap); - void GatherNewSwapRequests(std::vector* threads_to_swap); - - /// \brief The main logging thread function that handles the serialization - /// and I/O to the stream or file. - /// - /// \todo Provide client hook to set logging thead affinity and priority. - void IOThread(); - -// Slow synchronous error logging for internals that may prevent -// async logging from working. -#if USE_NEW_LOGGING_FORMAT - template - void LogErrorSync(const std::string& key, const T& value, - const std::string file_name, const unsigned int line_no) { - /// \todo Acquire mutex once for FlagError + LogDetail to avoid - /// races. Better yet, switch to a non-stateful error API. - // This is better than nothing though. - async_logger_.FlagError(); - async_logger_.LogDetail(key, value, file_name, line_no); - } - template - void LogWarning(const std::string& key, const T& value, - const std::string file_name, const unsigned int line_no) { - async_logger_.FlagWarning(); - async_logger_.LogDetail(key, value, file_name, line_no); - } -#else - template - void LogErrorSync(const std::string& message, Args&&... args) { - /// \todo Acquire mutex once for FlagError + LogDetail to avoid - /// races. Better yet, switch to a non-stateful error API. - // This is better than nothing though. - async_logger_.FlagError(); - async_logger_.LogDetail(message, std::forward(args)...); - } -#endif - - // Accessed by IOThead only. - const std::chrono::duration poll_period_; - AsyncLog async_logger_; - - const size_t max_threads_to_log_; - std::thread io_thread_; - - // Accessed by producers and IOThead during thread registration and - // destruction. Protected by io_thread_mutex_. - std::mutex io_thread_mutex_; - std::condition_variable io_thread_cv_; - bool keep_io_thread_alive_ = false; - - std::mutex tls_loggers_registerd_mutex_; - std::unordered_set tls_loggers_registerd_; - - // Temporarily stores TlsLogger data for threads that have exited until - // all their log entries have been processed. - // Accessed by IOThread and producers as their threads exit. - std::mutex tls_logger_orphans_mutex_; - using OrphanContainer = std::list>; - OrphanContainer tls_logger_orphans_; - - // Accessed by producers and IOThead atomically. - std::atomic swap_request_id_{0}; - std::vector> thread_swap_request_slots_; - - // Accessed by IOThead only. - size_t swap_request_id_read_{0}; - struct SlotRetry { - size_t slot; - uintptr_t next_id; - }; - std::vector swap_request_slots_to_retry_; - std::vector threads_to_swap_deferred_; - std::vector threads_to_read_; - std::vector orphans_to_destroy_; - - // Counts for retries related to the lock-free scheme. - // Abnormally high counts could be an indicator of contention. - // Access on IOThread only. - size_t swap_request_slots_retry_count_ = 0; - size_t swap_request_slots_retry_retry_count_ = 0; - size_t swap_request_slots_retry_reencounter_count_ = 0; - size_t start_reading_entries_retry_count_ = 0; - size_t tls_total_log_cas_fail_count_ = 0; - size_t tls_total_swap_buffers_slot_retry_count_ = 0; -}; - -Logger& GlobalLogger(); - -/// \brief The generic way to add a log entry. -/// \details Supports all types of logs, which is useful for complex -/// lambdas that may wish to log in multiple places or log something other -/// than a simple summary, detail, or trace entry. -void Log(AsyncLogEntry&& entry); - -/// \brief The convenience proxy a LogSummary lambda uses to write to the -/// summary log. -class AsyncSummary { - public: - explicit AsyncSummary(AsyncLog& async_log) : async_log_(async_log) {} - AsyncLog& async_log() { return async_log_; } - - template - AsyncLog& operator()(Args&&... args) { - async_log_.LogSummary(std::forward(args)...); - return async_log_; - } - - private: - AsyncLog& async_log_; -}; - -/// \brief A helper to simplify adding a summary log entry. -template -void LogSummary(LambdaT&& lambda) { - Log([lambda = std::forward(lambda)](AsyncLog& log) mutable { - AsyncSummary async_summary(log); - lambda(async_summary); - }); -} - -/// \brief The convenience proxy a LogDetail lambda uses to write to the detail -/// log. -class AsyncDetail { - public: - explicit AsyncDetail(AsyncLog& async_log) : async_log_(async_log) {} - AsyncLog& async_log() { return async_log_; } - -#if USE_NEW_LOGGING_FORMAT - template - AsyncLog& Log(const std::string& key, const T& value, - const std::string file_name, const unsigned int line_no) { - async_log_.LogDetail(key, value, file_name, line_no); - return async_log_; - } - - template - AsyncLog& LogError(const std::string& key, const T& value, - const std::string file_name, const unsigned int line_no) { - async_log_.FlagError(); - async_log_.LogDetail(key, value, file_name, line_no); - return async_log_; - } - - template - AsyncLog& LogWarning(const std::string& key, const T& value, - const std::string file_name, - const unsigned int line_no) { - async_log_.FlagWarning(); - async_log_.LogDetail(key, value, file_name, line_no); - return async_log_; - } - - template - AsyncLog& LogIntervalStart(const std::string& key, const T& value, - const std::string file_name, - const unsigned int line_no) { - async_log_.LogDetail(key, value, file_name, line_no); - return async_log_; - } - - template - AsyncLog& LogIntervalEnd(const std::string& key, const T& value, - const std::string file_name, - const unsigned int line_no) { - async_log_.LogDetail(key, value, file_name, line_no); - return async_log_; - } -#else - template - AsyncLog& operator()(Args&&... args) { - async_log_.LogDetail(std::forward(args)...); - return async_log_; - } - - template - AsyncLog& Error(Args&&... args) { - async_log_.FlagError(); - async_log_.LogDetail(std::forward(args)...); - return async_log_; - } - - template - AsyncLog& Warning(Args&&... args) { - async_log_.FlagWarning(); - async_log_.LogDetail(std::forward(args)...); - return async_log_; - } -#endif - - private: - AsyncLog& async_log_; -}; - -/// \brief A helper to simplify adding a detail log entry. -template -void LogDetail(LambdaT&& lambda) { - Log([lambda = std::forward(lambda), - timestamp = PerfClock::now()](AsyncLog& log) mutable { - log.SetLogDetailTime(timestamp); - AsyncDetail async_detail(log); - lambda(async_detail); - }); -} - -/// \brief The convenience proxy a ScopedTracer lambda uses to write to the -/// detail log. -class AsyncTrace { - public: - explicit AsyncTrace(AsyncLog& async_log) : async_log_(async_log) {} - AsyncLog& async_log() { return async_log_; } - - template - AsyncLog& operator()(Args&&... args) { - async_log_.ScopedTrace(std::forward(args)...); - return async_log_; - } - - private: - AsyncLog& async_log_; -}; - -/// \brief ScopedTracer is an RAII object that traces the start and end -/// of its lifetime. -template -class ScopedTracer { - public: - ScopedTracer(LambdaT&& lambda) - : start_(PerfClock::now()), lambda_(std::forward(lambda)) {} - - ~ScopedTracer() { - Log([start = start_, lambda = std::move(lambda_), - end = PerfClock::now()](AsyncLog& log) { - log.SetScopedTraceTimes(start, end); - AsyncTrace async_trace(log); - lambda(async_trace); - }); - } - - private: - PerfClock::time_point start_; - LambdaT lambda_; -}; - -/// \brief Helper that creates a ScopeTracer with automatic type deduction. -/// \details Helps with automatic template type deduction, which has been -/// supported for functions for a long time. -/// C++17 will support deduction for classes, which will neutralize the utility -/// of a helper function like this. -/// \todo Determine which traces to keep for submission purposes. -template -auto MakeScopedTracer(LambdaT&& lambda) -> ScopedTracer { - return ScopedTracer(std::forward(lambda)); -} - -template -void AsyncLog::LogSummary(const std::string& message, const Args... args) { - auto tracer = MakeScopedTracer([message](AsyncTrace& trace) { - std::string sanitized_message = message; - std::replace(sanitized_message.begin(), sanitized_message.end(), '"', '\''); - std::replace(sanitized_message.begin(), sanitized_message.end(), '\n', ';'); - trace("LogSummary", "message", "\"" + sanitized_message + "\""); - }); - std::unique_lock lock(log_mutex_); - *summary_out_ << message; - LogArgs(summary_out_, args...); - *summary_out_ << "\n"; - - if (copy_summary_to_stdout_) { - std::cout << message; - LogArgs(&std::cout, args...); - std::cout << "\n"; - } -} - -#if USE_NEW_LOGGING_FORMAT -template -void AsyncLog::LogDetail(const std::string& key, const T& value, - const std::string file_name, - const unsigned int line_no) { - auto tracer = MakeScopedTracer([key](AsyncTrace& trace) { - std::string sanitized_key = key; - std::replace(sanitized_key.begin(), sanitized_key.end(), '"', '\''); - std::replace(sanitized_key.begin(), sanitized_key.end(), '\n', ';'); - trace("LogDetail", "key", "\"" + sanitized_key + "\""); - }); - std::unique_lock lock(log_mutex_); - std::vector detail_streams{detail_out_, &std::cout}; - if (!copy_detail_to_stdout_) { - detail_streams.pop_back(); - } - auto time_ns = (log_detail_time_ - log_origin_).count(); - for (auto os : detail_streams) { - *os << ":::MLLOG {" << "\"key\": " << ArgValueTransform(key) << ", " - << "\"value\": " << ArgValueTransform(value) << ", " - << "\"time_ms\": " << ArgValueTransform(time_ns / 1000000ULL) << "." - << std::setfill('0') << std::setw(6) - << ArgValueTransform(time_ns % 1000000ULL) << ", " - << "\"namespace\": \"mlperf::logging\", " - << "\"event_type\": \"POINT_IN_TIME\", " << "\"metadata\": {" - << "\"is_error\": " << ArgValueTransform(error_flagged_) << ", " - << "\"is_warning\": " << ArgValueTransform(warning_flagged_) << ", " - << "\"file\": \"" << file_name << "\", " - << "\"line_no\": " << ArgValueTransform(line_no) << ", " - << "\"pid\": " << ArgValueTransform(current_pid_) << ", " - << "\"tid\": " << ArgValueTransform(current_tid_) << "}}\n"; - if (error_flagged_) { - os->flush(); - } - } - error_flagged_ = false; - warning_flagged_ = false; -} -#else -template -void AsyncLog::LogDetail(const std::string& message, const Args... args) { - auto tracer = MakeScopedTracer([message](AsyncTrace& trace) { - std::string sanitized_message = message; - std::replace(sanitized_message.begin(), sanitized_message.end(), '"', '\''); - std::replace(sanitized_message.begin(), sanitized_message.end(), '\n', ';'); - trace("LogDetail", "message", "\"" + sanitized_message + "\""); - }); - std::unique_lock lock(log_mutex_); - std::vector detail_streams{detail_out_, &std::cout}; - if (!copy_detail_to_stdout_) { - detail_streams.pop_back(); - } - for (auto os : detail_streams) { - *os << "\"pid\": " << current_pid_ << ", " << "\"tid\": " << current_tid_ - << ", " << "\"ts\": " << (log_detail_time_ - log_origin_).count() - << "ns : "; - if (error_flagged_) { - *os << "ERROR : "; - } else if (warning_flagged_) { - *os << "WARNING : "; - } - *os << message; - LogArgs(os, args...); - *os << "\n"; - if (error_flagged_) { - os->flush(); - } - } - error_flagged_ = false; - warning_flagged_ = false; -} -#endif - -} // namespace logging - -// Export some things out of the logging namespace to simplify call sites. - -const auto GlobalLogger = logging::GlobalLogger; -const auto Log = logging::Log; - -using PerfClock = logging::PerfClock; - -using LogBinaryAsHexString = logging::LogBinaryAsHexString; - -using AsyncLog = logging::AsyncLog; - -using AsyncSummary = logging::AsyncSummary; -template -void LogSummary(LambdaT&& lambda) { - logging::LogSummary(std::forward(lambda)); -} - -using AsyncDetail = logging::AsyncDetail; -template -void LogDetail(LambdaT&& lambda) { - logging::LogDetail(std::forward(lambda)); -} - -using AsyncTrace = logging::AsyncTrace; - -template -using ScopedTracer = logging::ScopedTracer; - -template -auto MakeScopedTracer(LambdaT&& lambda) -> ScopedTracer { - return ScopedTracer(std::forward(lambda)); -} - -} // namespace mlperf - -#endif // MLPERF_LOADGEN_LOGGING_H_ diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/mlperf.conf b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/mlperf.conf deleted file mode 100644 index 1b825514b..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/mlperf.conf +++ /dev/null @@ -1,164 +0,0 @@ -# The format of this config file is 'key = value'. -# The key has the format 'model.scenario.key'. Value is mostly int64_t. -# Model maybe '*' as wildcard. In that case the value applies to all models. -# All times are in milli seconds - -# Set performance_sample_count for each model. -# User can optionally set this to higher values in user.conf. -resnet50.*.performance_sample_count_override = 1024 -ssd-mobilenet.*.performance_sample_count_override = 256 -retinanet.*.performance_sample_count_override = 64 -bert.*.performance_sample_count_override = 10833 -dlrm.*.performance_sample_count_override = 204800 -dlrm-v2.*.performance_sample_count_override = 204800 -rnnt.*.performance_sample_count_override = 2513 -gptj.*.performance_sample_count_override = 13368 -mixtral-8x7b.*.performance_sample_count_override = 15000 -llama2-70b.*.performance_sample_count_override = 24576 -llama2-70b-interactive.*.performance_sample_count_override = 24576 -llama3_1-405b.*.performance_sample_count_override = 8313 -llama3_1-405b-interactive.*.performance_sample_count_override = 8313 -llama3_1-8b.*.performance_sample_count_override = 13368 -llama3_1-8b-edge.*.performance_sample_count_override = 5000 -llama3_1-8b-interactive.*.performance_sample_count_override = 13368 -stable-diffusion-xl.*.performance_sample_count_override = 5000 -rgat.*.performance_sample_count_override = 788379 -pointpainting.*.performance_sample_count_override = 1024 -deepseek-r1.*.performance_sample_count_override = 4388 -whisper.*.performance_sample_count_override = 1633 -# set to 0 to let entire sample set to be performance sample -3d-unet.*.performance_sample_count_override = 0 - -# Set seeds. -*.*.qsl_rng_seed = 1780908523862526354 -*.*.sample_index_rng_seed = 14771362308971278857 -*.*.schedule_rng_seed = 18209322760996052031 - -# Set seeds for TEST_05 (not needed from v5.0 onwards) -*.*.test05_qsl_rng_seed = 7975553102935885558 -*.*.test05_sample_index_rng_seed = 11403566307062068064 -*.*.test05_schedule_rng_seed = 15816800565822761601 - -*.SingleStream.target_latency_percentile = 90 -pointpainting.SingleStream.target_latency_percentile = 99.9 -*.SingleStream.min_duration = 600000 - -*.MultiStream.target_latency_percentile = 99 -*.MultiStream.samples_per_query = 8 -*.MultiStream.min_duration = 600000 -*.MultiStream.min_query_count = 662 -retinanet.MultiStream.target_latency = 528 - -# 3D-UNet uses equal issue mode because it has non-uniform inputs -3d-unet.*.sample_concatenate_permutation = 1 - -# R-GAT uses equal issue mode because it may have non-uniform inputs -rgat.*.sample_concatenate_permutation = 1 - -# LLM benchmarks have non-uniform inputs and outputs, and use equal issue mode for all latency scenario -gptj.*.sample_concatenate_permutation = 1 -llama2-70b.*.sample_concatenate_permutation = 1 -llama2-70b-interactive.*.sample_concatenate_permutation = 1 -mixtral-8x7b.*.sample_concatenate_permutation = 1 -llama3_1-405b.*.sample_concatenate_permutation = 1 -llama3_1-405b-interactive.*.sample_concatenate_permutation = 1 -llama3_1-8b.*.sample_concatenate_permutation = 1 -llama3_1-8b-edge.*.sample_concatenate_permutation = 1 -llama3_1-8b-interactive.*.sample_concatenate_permutation = 1 -deepseek-r1.*.sample_concatenate_permutation = 1 -whisper.*.sample_concatenate_permutation = 1 - -*.Server.target_latency = 10 -*.Server.target_latency_percentile = 99 -*.Server.target_duration = 0 -*.Server.min_duration = 600000 -resnet50.Server.target_latency = 15 -retinanet.Server.target_latency = 100 -bert.Server.target_latency = 130 -dlrm.Server.target_latency = 60 -dlrm-v2.Server.target_latency = 60 -rnnt.Server.target_latency = 1000 -gptj.Server.target_latency = 20000 -stable-diffusion-xl.Server.target_latency = 20000 -# Benchmarks that measure token latencies -llama2-70b.*.use_token_latencies = 1 -llama2-70b-interactive.*.use_token_latencies = 1 -mixtral-8x7b.*.use_token_latencies = 1 -llama3_1-405b.*.use_token_latencies = 1 -llama3_1-405b-interactive.*.use_token_latencies = 1 -llama3_1-8b.*.use_token_latencies = 1 -llama3_1-8b-edge.*.use_token_latencies = 1 -llama3_1-8b-interactive.*.use_token_latencies = 1 -deepseek-r1.*.use_token_latencies = 1 -whisper.*.use_token_latencies = 1 - -# gptj benchmark infers token latencies -gptj.*.infer_token_latencies = 1 -gptj.*.token_latency_scaling_factor = 69 -# Only ttft and tpot are tracked for the llama2-70b, mixtral-8x7B & llama3_1-405b benchmark therefore target_latency = 0 -llama2-70b.Server.target_latency = 0 -llama2-70b.Server.ttft_latency = 2000 -llama2-70b.Server.tpot_latency = 200 - -# Target Latencies for interactive setting -llama2-70b-interactive.Server.target_latency = 0 -llama2-70b-interactive.Server.ttft_latency = 450 -llama2-70b-interactive.Server.tpot_latency = 40 - -mixtral-8x7b.Server.target_latency = 0 -mixtral-8x7b.Server.ttft_latency = 2000 -mixtral-8x7b.Server.tpot_latency = 200 - -llama3_1-405b.Server.target_latency = 0 -llama3_1-405b.Server.ttft_latency = 6000 -llama3_1-405b.Server.tpot_latency = 175 - -# Target Latencies for interactive setting -llama3_1-405b-interactive.Server.target_latency = 0 -llama3_1-405b-interactive.Server.ttft_latency = 4500 -llama3_1-405b-interactive.Server.tpot_latency = 80 - - -llama3_1-8b.Server.target_latency = 0 -llama3_1-8b.Server.ttft_latency = 2000 -llama3_1-8b.Server.tpot_latency = 100 - -# Target Latencies for interactive setting -llama3_1-8b-interactive.Server.target_latency = 0 -llama3_1-8b-interactive.Server.ttft_latency = 500 -llama3_1-8b-interactive.Server.tpot_latency = 30 - -deepseek-r1.Server.target_latency = 0 -deepseek-r1.Server.ttft_latency = 2000 -deepseek-r1.Server.tpot_latency = 80 - -*.Offline.target_latency_percentile = 90 -*.Offline.min_duration = 600000 - -# In Offline scenario, we always have one query. But LoadGen maps this to -# min_sample_count internally in Offline scenario. If the dataset size is larger -# than 24576 we limit the min_query_count to 24576 and otherwise we use -# the dataset size as the limit - -resnet50.Offline.min_query_count = 24576 -retinanet.Offline.min_query_count = 24576 -dlrm-v2.Offline.min_query_count = 24576 -bert.Offline.min_query_count = 10833 -gptj.Offline.min_query_count = 13368 -rnnt.Offline.min_query_count = 2513 -3d-unet.Offline.min_query_count = 43 -stable-diffusion-xl.Offline.min_query_count = 5000 -llama2-70b.Offline.min_query_count = 24576 -llama3_1-405b.Offline.min_query_count = 8313 -llama3_1-8b.Offline.min_query_count = 13368 -llama3_1-8b-edge.Offline.min_query_count = 5000 -mixtral-8x7b.Offline.min_query_count = 15000 -rgat.Offline.min_query_count = 788379 -deepseek-r1.Offline.min_query_count = 4388 -whisper.Offline.min_query_count = 1633 - -# These fields should be defined and overridden by user.conf. -*.SingleStream.target_latency = 10 -*.MultiStream.target_latency = 80 -*.Server.target_qps = 1.0 -*.Offline.target_qps = 1.0 diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/mlperf_conf.h b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/mlperf_conf.h deleted file mode 100644 index 7859e0139..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/mlperf_conf.h +++ /dev/null @@ -1,167 +0,0 @@ -const char* mlperf_conf = -"# The format of this config file is 'key = value'.\n" -"# The key has the format 'model.scenario.key'. Value is mostly int64_t.\n" -"# Model maybe '*' as wildcard. In that case the value applies to all models.\n" -"# All times are in milli seconds\n" -"\n" -"# Set performance_sample_count for each model.\n" -"# User can optionally set this to higher values in user.conf.\n" -"resnet50.*.performance_sample_count_override = 1024\n" -"ssd-mobilenet.*.performance_sample_count_override = 256\n" -"retinanet.*.performance_sample_count_override = 64\n" -"bert.*.performance_sample_count_override = 10833\n" -"dlrm.*.performance_sample_count_override = 204800\n" -"dlrm-v2.*.performance_sample_count_override = 204800\n" -"rnnt.*.performance_sample_count_override = 2513\n" -"gptj.*.performance_sample_count_override = 13368\n" -"mixtral-8x7b.*.performance_sample_count_override = 15000\n" -"llama2-70b.*.performance_sample_count_override = 24576\n" -"llama2-70b-interactive.*.performance_sample_count_override = 24576\n" -"llama3_1-405b.*.performance_sample_count_override = 8313\n" -"llama3_1-405b-interactive.*.performance_sample_count_override = 8313\n" -"llama3_1-8b.*.performance_sample_count_override = 13368\n" -"llama3_1-8b-edge.*.performance_sample_count_override = 5000\n" -"llama3_1-8b-interactive.*.performance_sample_count_override = 13368\n" -"stable-diffusion-xl.*.performance_sample_count_override = 5000\n" -"rgat.*.performance_sample_count_override = 788379\n" -"pointpainting.*.performance_sample_count_override = 1024\n" -"deepseek-r1.*.performance_sample_count_override = 4388\n" -"whisper.*.performance_sample_count_override = 1633\n" -"# set to 0 to let entire sample set to be performance sample\n" -"3d-unet.*.performance_sample_count_override = 0\n" -"\n" -"# Set seeds.\n" -"*.*.qsl_rng_seed = 1780908523862526354\n" -"*.*.sample_index_rng_seed = 14771362308971278857\n" -"*.*.schedule_rng_seed = 18209322760996052031\n" -"\n" -"# Set seeds for TEST_05 (not needed from v5.0 onwards)\n" -"*.*.test05_qsl_rng_seed = 7975553102935885558\n" -"*.*.test05_sample_index_rng_seed = 11403566307062068064\n" -"*.*.test05_schedule_rng_seed = 15816800565822761601\n" -"\n" -"*.SingleStream.target_latency_percentile = 90\n" -"pointpainting.SingleStream.target_latency_percentile = 99.9\n" -"*.SingleStream.min_duration = 600000\n" -"\n" -"*.MultiStream.target_latency_percentile = 99\n" -"*.MultiStream.samples_per_query = 8\n" -"*.MultiStream.min_duration = 600000\n" -"*.MultiStream.min_query_count = 662\n" -"retinanet.MultiStream.target_latency = 528\n" -"\n" -"# 3D-UNet uses equal issue mode because it has non-uniform inputs\n" -"3d-unet.*.sample_concatenate_permutation = 1\n" -"\n" -"# R-GAT uses equal issue mode because it may have non-uniform inputs\n" -"rgat.*.sample_concatenate_permutation = 1\n" -"\n" -"# LLM benchmarks have non-uniform inputs and outputs, and use equal issue mode for all latency scenario\n" -"gptj.*.sample_concatenate_permutation = 1\n" -"llama2-70b.*.sample_concatenate_permutation = 1\n" -"llama2-70b-interactive.*.sample_concatenate_permutation = 1\n" -"mixtral-8x7b.*.sample_concatenate_permutation = 1\n" -"llama3_1-405b.*.sample_concatenate_permutation = 1\n" -"llama3_1-405b-interactive.*.sample_concatenate_permutation = 1\n" -"llama3_1-8b.*.sample_concatenate_permutation = 1\n" -"llama3_1-8b-edge.*.sample_concatenate_permutation = 1\n" -"llama3_1-8b-interactive.*.sample_concatenate_permutation = 1\n" -"deepseek-r1.*.sample_concatenate_permutation = 1\n" -"whisper.*.sample_concatenate_permutation = 1\n" -"\n" -"*.Server.target_latency = 10\n" -"*.Server.target_latency_percentile = 99\n" -"*.Server.target_duration = 0\n" -"*.Server.min_duration = 600000\n" -"resnet50.Server.target_latency = 15\n" -"retinanet.Server.target_latency = 100\n" -"bert.Server.target_latency = 130\n" -"dlrm.Server.target_latency = 60\n" -"dlrm-v2.Server.target_latency = 60\n" -"rnnt.Server.target_latency = 1000\n" -"gptj.Server.target_latency = 20000\n" -"stable-diffusion-xl.Server.target_latency = 20000\n" -"# Benchmarks that measure token latencies\n" -"llama2-70b.*.use_token_latencies = 1\n" -"llama2-70b-interactive.*.use_token_latencies = 1\n" -"mixtral-8x7b.*.use_token_latencies = 1\n" -"llama3_1-405b.*.use_token_latencies = 1\n" -"llama3_1-405b-interactive.*.use_token_latencies = 1\n" -"llama3_1-8b.*.use_token_latencies = 1\n" -"llama3_1-8b-edge.*.use_token_latencies = 1\n" -"llama3_1-8b-interactive.*.use_token_latencies = 1\n" -"deepseek-r1.*.use_token_latencies = 1\n" -"whisper.*.use_token_latencies = 1\n" -"\n" -"# gptj benchmark infers token latencies\n" -"gptj.*.infer_token_latencies = 1\n" -"gptj.*.token_latency_scaling_factor = 69\n" -"# Only ttft and tpot are tracked for the llama2-70b, mixtral-8x7B & llama3_1-405b benchmark therefore target_latency = 0\n" -"llama2-70b.Server.target_latency = 0\n" -"llama2-70b.Server.ttft_latency = 2000\n" -"llama2-70b.Server.tpot_latency = 200\n" -"\n" -"# Target Latencies for interactive setting\n" -"llama2-70b-interactive.Server.target_latency = 0\n" -"llama2-70b-interactive.Server.ttft_latency = 450\n" -"llama2-70b-interactive.Server.tpot_latency = 40\n" -"\n" -"mixtral-8x7b.Server.target_latency = 0\n" -"mixtral-8x7b.Server.ttft_latency = 2000\n" -"mixtral-8x7b.Server.tpot_latency = 200\n" -"\n" -"llama3_1-405b.Server.target_latency = 0\n" -"llama3_1-405b.Server.ttft_latency = 6000\n" -"llama3_1-405b.Server.tpot_latency = 175\n" -"\n" -"# Target Latencies for interactive setting\n" -"llama3_1-405b-interactive.Server.target_latency = 0\n" -"llama3_1-405b-interactive.Server.ttft_latency = 4500\n" -"llama3_1-405b-interactive.Server.tpot_latency = 80\n" -"\n" -"\n" -"llama3_1-8b.Server.target_latency = 0\n" -"llama3_1-8b.Server.ttft_latency = 2000\n" -"llama3_1-8b.Server.tpot_latency = 100\n" -"\n" -"# Target Latencies for interactive setting\n" -"llama3_1-8b-interactive.Server.target_latency = 0\n" -"llama3_1-8b-interactive.Server.ttft_latency = 500\n" -"llama3_1-8b-interactive.Server.tpot_latency = 30\n" -"\n" -"deepseek-r1.Server.target_latency = 0\n" -"deepseek-r1.Server.ttft_latency = 2000\n" -"deepseek-r1.Server.tpot_latency = 80\n" -"\n" -"*.Offline.target_latency_percentile = 90\n" -"*.Offline.min_duration = 600000\n" -"\n" -"# In Offline scenario, we always have one query. But LoadGen maps this to\n" -"# min_sample_count internally in Offline scenario. If the dataset size is larger\n" -"# than 24576 we limit the min_query_count to 24576 and otherwise we use\n" -"# the dataset size as the limit\n" -"\n" -"resnet50.Offline.min_query_count = 24576\n" -"retinanet.Offline.min_query_count = 24576\n" -"dlrm-v2.Offline.min_query_count = 24576\n" -"bert.Offline.min_query_count = 10833\n" -"gptj.Offline.min_query_count = 13368\n" -"rnnt.Offline.min_query_count = 2513\n" -"3d-unet.Offline.min_query_count = 43\n" -"stable-diffusion-xl.Offline.min_query_count = 5000\n" -"llama2-70b.Offline.min_query_count = 24576\n" -"llama3_1-405b.Offline.min_query_count = 8313\n" -"llama3_1-8b.Offline.min_query_count = 13368\n" -"llama3_1-8b-edge.Offline.min_query_count = 5000\n" -"mixtral-8x7b.Offline.min_query_count = 15000\n" -"rgat.Offline.min_query_count = 788379\n" -"deepseek-r1.Offline.min_query_count = 4388\n" -"whisper.Offline.min_query_count = 1633\n" -"\n" -"# These fields should be defined and overridden by user.conf.\n" -"*.SingleStream.target_latency = 10\n" -"*.MultiStream.target_latency = 80\n" -"*.Server.target_qps = 1.0\n" -"*.Offline.target_qps = 1.0\n" -"\n" -""; diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/pyproject.toml b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/pyproject.toml deleted file mode 100755 index 6f0ae06f0..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/pyproject.toml +++ /dev/null @@ -1,7 +0,0 @@ -[build-system] -requires = ["setuptools>=42", "wheel", "pybind11==2.11.1"] -build-backend = "setuptools.build_meta:__legacy__" - -[tool.cibuildwheel] -environment = "CFLAGS='-std=c++14'" -build = "cp3{7,8,9,10,11,12,13}-*" diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/query_dispatch_library.h b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/query_dispatch_library.h deleted file mode 100644 index 6c594efe0..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/query_dispatch_library.h +++ /dev/null @@ -1,42 +0,0 @@ -/* Copyright 2019 The MLPerf Authors. All Rights Reserved. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -/// \file -/// \brief Defines the QueryDispatchLibrary interface. - -#ifndef MLPERF_LOADGEN_QUERY_DISPATCH_LIBRARY_H -#define MLPERF_LOADGEN_QUERY_DISPATCH_LIBRARY_H - -#include - -#include "system_under_test.h" - -namespace mlperf { - -/// \addtogroup LoadgenAPI -/// @{ - -/// \brief The interface a client implements for the LoadGen over the network to -/// test. The API inherits the System_under_test.h API When working in LON mode -/// the QueryDispatchLibrary class is used and natively Upcasted to the -/// QueryDispatchLibrary class. - -class QueryDispatchLibrary : public SystemUnderTest { - public: - virtual ~QueryDispatchLibrary() = default; -}; - -/// @} - -} // namespace mlperf - -#endif // MLPERF_LOADGEN_QUERY_DISPATCH_LIBRARY_H diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/query_sample.h b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/query_sample.h deleted file mode 100644 index e740be99e..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/query_sample.h +++ /dev/null @@ -1,91 +0,0 @@ -/* Copyright 2019 The MLPerf Authors. All Rights Reserved. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -/// \file -/// \brief Defines the structs involved in issuing a query and responding to -/// a query. -/// \details These are broken out into their own files since they are exposed -/// as part of the C API and we want to avoid C clients including C++ code. - -#ifndef MLPERF_LOADGEN_QUERY_SAMPLE_H_ -#define MLPERF_LOADGEN_QUERY_SAMPLE_H_ - -#include -#include - -#include - -namespace mlperf { - -/// \addtogroup LoadgenAPI -/// @{ - -/// \brief Represents a unique identifier for a sample of an issued query. -/// \details As currently implemented, the id is a pointer to an internal -/// loadgen struct whose value will never be zero/null. -typedef uintptr_t ResponseId; -constexpr ResponseId kResponseIdReserved = 0; - -/// \brief An index into the QuerySampleLibrary corresponding to a -/// single sample. -typedef size_t QuerySampleIndex; - -/// \brief Represents the smallest unit of input inference can run on. -/// A query consists of one or more samples. -struct QuerySample { - ResponseId id; - QuerySampleIndex index; -}; - -/// \brief Represents a single response to QuerySample -struct QuerySampleResponse { - ResponseId id; - uintptr_t data; - size_t size; ///< Size in bytes. - int64_t n_tokens; - - public: - QuerySampleResponse(ResponseId id, uintptr_t data, size_t size, - int64_t n_tokens) - : id(id), - data(data), - size(size), - n_tokens(n_tokens){ - // std::cout << "Initialized with 4 arguments, n_tokens: " << - // n_tokens <<"\n"; - }; - QuerySampleResponse(ResponseId id, uintptr_t data, size_t size) - : id(id), - data(data), - size(size), - n_tokens(0){ - // std::cout << "Initialized with 3 arguments, n_tokens: " << - // n_tokens <<"\n"; - }; - QuerySampleResponse() - : id(0), - data(0), - size(0), - n_tokens(0){ - // std::cout << "Initialized with 0 arguments, n_tokens: " << - // n_tokens <<"\n"; - }; -}; - -/// \brief A latency in nanoseconds, as recorded by the loadgen. -typedef int64_t QuerySampleLatency; - -/// @} - -} // namespace mlperf - -#endif // MLPERF_LOADGEN_QUERY_SAMPLE_H_ diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/query_sample_library.h b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/query_sample_library.h deleted file mode 100644 index 7258068cb..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/query_sample_library.h +++ /dev/null @@ -1,75 +0,0 @@ -/* Copyright 2019 The MLPerf Authors. All Rights Reserved. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -/// \file -/// \brief Defines the QuerySampleLibrary interface. - -#ifndef MLPERF_LOADGEN_QUERY_SAMPLE_LIBRARY_H -#define MLPERF_LOADGEN_QUERY_SAMPLE_LIBRARY_H - -#include -#include -#include - -#include "query_sample.h" - -namespace mlperf { - -/// \addtogroup LoadgenAPI -/// @{ - -/// \brief The interface a client implements to coordinate with the loadgen -/// which samples should be loaded. -class QuerySampleLibrary { - public: - virtual ~QuerySampleLibrary() {} - - /// \brief A human readable name for the model. - virtual const std::string& Name() = 0; - - /// \brief Total number of samples in library. - virtual size_t TotalSampleCount() = 0; - - /// \brief The number of samples that are guaranteed to fit in RAM. - virtual size_t PerformanceSampleCount() = 0; - - /// \brief Loads the requested query samples into memory. - /// \details Paired with calls to UnloadSamplesFromRam. - /// In the MultiStream scenarios: - /// * Samples will appear more than once. - /// * SystemUnderTest::IssueQuery will only be called with a set of samples - /// that are neighbors in the vector of samples here, which helps - /// SUTs that need the queries to be contiguous. - /// In all other scenarios: - /// * A previously loaded sample will not be loaded again. - virtual void LoadSamplesToRam( - const std::vector& samples) = 0; - - /// \brief Unloads the requested query samples from memory. - /// \details In the MultiStream scenarios: - /// * Samples may be unloaded the same number of times they were loaded; - /// however, if the implementation de-dups loaded samples rather than - /// loading samples into contiguous memory, it may unload a sample the - /// first time they see it unloaded without a refcounting scheme, ignoring - /// subsequent unloads. A refcounting scheme would also work, but is not - /// a requirement. - /// In all other scenarios: - /// * A previously unloaded sample will not be unloaded again. - virtual void UnloadSamplesFromRam( - const std::vector& samples) = 0; -}; - -/// @} - -} // namespace mlperf - -#endif // MLPERF_LOADGEN_QUERY_SAMPLE_LIBRARY_H diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/requirements.txt b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/requirements.txt deleted file mode 100644 index e47c59fd7..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -pybind11 diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/results.cc b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/results.cc deleted file mode 100644 index f7c61af43..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/results.cc +++ /dev/null @@ -1,856 +0,0 @@ -/* Copyright 2019 The MLPerf Authors. All Rights Reserved. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "results.h" - -#include "early_stopping.h" -#include "utils.h" - -namespace mlperf { -namespace loadgen { - -void PerformanceSummary::ProcessLatencies() { - if (pr.sample_latencies.empty()) { - return; - } - - sample_count = pr.sample_latencies.size(); - - QuerySampleLatency accumulated_sample_latency = 0; - for (auto latency : pr.sample_latencies) { - accumulated_sample_latency += latency; - } - sample_latency_mean = accumulated_sample_latency / sample_count; - - std::sort(pr.sample_latencies.begin(), pr.sample_latencies.end()); - - target_latency_percentile.sample_latency = - pr.sample_latencies[sample_count * target_latency_percentile.percentile]; - sample_latency_min = pr.sample_latencies.front(); - sample_latency_max = pr.sample_latencies.back(); - for (auto& lp : latency_percentiles) { - assert(lp.percentile >= 0.0); - assert(lp.percentile < 1.0); - lp.sample_latency = pr.sample_latencies[sample_count * lp.percentile]; - } - - query_count = pr.queries_issued; - - // Count the number of overlatency queries. Only for Server scenario. Since in - // this scenario the number of samples per query is 1, sample_latencies are - // used. - if (settings.scenario == TestScenario::Server) { - QuerySampleLatency max_latency = settings.target_latency.count() + 1; - overlatency_query_count = - pr.sample_latencies.end() - - std::lower_bound(pr.sample_latencies.begin(), pr.sample_latencies.end(), - max_latency); - } - - if (settings.use_token_latencies) { - ProcessTokenLatencies(); - } - - // MultiStream only after this point. - if (settings.scenario != TestScenario::MultiStream) { - return; - } - - // Calculate per-query stats. - size_t query_count = pr.queries_issued; - assert(pr.query_latencies.size() == query_count); - std::sort(pr.query_latencies.begin(), pr.query_latencies.end()); - QuerySampleLatency accumulated_query_latency = 0; - for (auto latency : pr.query_latencies) { - accumulated_query_latency += latency; - } - query_latency_mean = accumulated_query_latency / query_count; - query_latency_min = pr.query_latencies.front(); - query_latency_max = pr.query_latencies.back(); - target_latency_percentile.query_latency = - pr.query_latencies[query_count * target_latency_percentile.percentile]; - for (auto& lp : latency_percentiles) { - lp.query_latency = pr.query_latencies[query_count * lp.percentile]; - } -} - -void PerformanceSummary::ProcessTokenLatencies() { - constexpr auto nTokenInvalid = std::numeric_limits::min(); - token_count = 0; - for (auto n_tokens : pr.token_results.tokens_per_sample) { - if (n_tokens != nTokenInvalid) token_count += n_tokens; - } - if (pr.token_results.first_token_latencies.empty()) { - return; - } - QuerySampleLatency accumulated_first_token_latency = 0; - for (auto latency : pr.token_results.first_token_latencies) { - accumulated_first_token_latency += latency; - } - first_token_latency_mean = accumulated_first_token_latency / sample_count; - QuerySampleLatency accumulated_tpot = 0; - for (auto latency : pr.token_results.time_per_output_token_arr) { - accumulated_tpot += latency; - } - time_per_output_token_mean = accumulated_tpot / sample_count; - std::sort(pr.token_results.first_token_latencies.begin(), - pr.token_results.first_token_latencies.end()); - std::sort(pr.token_results.time_per_output_token_arr.begin(), - pr.token_results.time_per_output_token_arr.end()); - - token_target_latency_percentile.sample_latency = - pr.token_results - .first_token_latencies[sample_count * - token_target_latency_percentile.percentile]; - first_token_latency_min = pr.token_results.first_token_latencies.front(); - first_token_latency_max = pr.token_results.first_token_latencies.back(); - for (auto& lp : token_latency_percentiles) { - assert(lp.percentile >= 0.0); - assert(lp.percentile < 1.0); - lp.sample_latency = - pr.token_results.first_token_latencies[sample_count * lp.percentile]; - } - - target_tpot_percentile.sample_latency = - pr.token_results - .time_per_output_token_arr[sample_count * - target_tpot_percentile.percentile]; - time_per_output_token_min = - pr.token_results.time_per_output_token_arr.front(); - time_per_output_token_max = pr.token_results.time_per_output_token_arr.back(); - for (auto& lp : tpot_percentiles) { - assert(lp.percentile >= 0.0); - assert(lp.percentile < 1.0); - lp.sample_latency = - pr.token_results - .time_per_output_token_arr[sample_count * lp.percentile]; - } - - if (settings.scenario == TestScenario::Server) { - // TODO: Maybe another target latency needs to be added? - QuerySampleLatency max_latency = settings.target_latency.count() + 1; - overlatency_first_token_count = - pr.token_results.first_token_latencies.end() - - std::lower_bound(pr.token_results.first_token_latencies.begin(), - pr.token_results.first_token_latencies.end(), - max_latency); - } -} - -bool PerformanceSummary::EarlyStopping( - std::string* recommendation, int64_t queries_issued, - std::vector* sample_latencies, - std::vector* query_latencies, - std::chrono::nanoseconds target_latency) { - recommendation->clear(); - - MinPassingQueriesFinder find_min_passing; - double confidence = 0.99; - double tolerance = 0.0; - - ProcessLatencies(); - switch (settings.scenario) { - case TestScenario::SingleStream: { - // TODO: Grab multistream percentile from settings, instead of hardcoding. - double multi_stream_percentile = 0.99; - int64_t t = 1; - int64_t h_min = find_min_passing(1, target_latency_percentile.percentile, - tolerance, confidence); - int64_t h = h_min; - if (queries_issued < h_min + 1) { - *recommendation = - " * Only processed " + std::to_string(queries_issued) + - " queries.\n * Need to process at least " + - std::to_string(h_min + 1) + " queries for early stopping."; - return false; - } else { - for (int64_t i = 2; i < queries_issued + 1; ++i) { - h = find_min_passing(i, target_latency_percentile.percentile, - tolerance, confidence); - if (queries_issued < h + i) { - t = i - 1; - break; - } - } - } - QuerySampleLatency percentile_estimate = - (*sample_latencies)[queries_issued - t]; - *recommendation = - " * Processed at least " + std::to_string(h_min + 1) + " queries (" + - std::to_string(queries_issued) + ").\n" + " * Would discard " + - std::to_string(t - 1) + " highest latency queries.\n" + - " * Early stopping " + - DoubleToString(target_latency_percentile.percentile * 100, 1) + - "th percentile estimate: " + std::to_string(percentile_estimate); - early_stopping_latency_ss = percentile_estimate; - - // Early stopping estimate for 99%ile (used for infering multi-stream from - // single-stream) - t = 1; - h_min = - find_min_passing(1, multi_stream_percentile, tolerance, confidence); - h = h_min; - if (queries_issued < h_min + 1) { - *recommendation += - "\n * Not enough queries processed for " + - DoubleToString(multi_stream_percentile * 100, 1) + - "th percentile\n" + - " early stopping estimate (would need to process at\n least " + - std::to_string(h_min + 1) + " total queries)."; - } else { - for (int64_t i = 2; i < queries_issued + 1; ++i) { - h = find_min_passing(i, multi_stream_percentile, tolerance, - confidence); - if (queries_issued < h + i) { - t = i - 1; - break; - } - } - percentile_estimate = (*sample_latencies)[queries_issued - t]; - *recommendation += - "\n * Early stopping " + - DoubleToString(multi_stream_percentile * 100, 1) + - "th percentile estimate: " + std::to_string(percentile_estimate); - early_stopping_latency_ms = percentile_estimate; - } - break; - } - case TestScenario::Server: { - int64_t t = - std::count_if((*sample_latencies).begin(), (*sample_latencies).end(), - [=](auto const& latency) { - return latency > target_latency.count(); - }); - int64_t h = find_min_passing(t, target_latency_percentile.percentile, - tolerance, confidence); - if (queries_issued >= h + t) { - *recommendation = " * Run successful."; - } else { - *recommendation = " * Run unsuccessful.\n * Processed " + - std::to_string(queries_issued) + " queries.\n" + - " * Would need to run at least " + - std::to_string(h + t - queries_issued) + - " more queries,\n with the run being successful if " - "every additional\n query were under latency."; - return false; - } - break; - } - case TestScenario::MultiStream: { - int64_t t = 1; - int64_t h_min = find_min_passing(1, target_latency_percentile.percentile, - tolerance, confidence); - int64_t h = h_min; - if (queries_issued < h_min + 1) { - *recommendation = - " * Only processed " + std::to_string(queries_issued) + - " queries.\n * Need to process at least " + - std::to_string(h_min + 1) + " queries for early stopping."; - return false; - } else { - for (int64_t i = 2; i < queries_issued + 1; ++i) { - h = find_min_passing(i, target_latency_percentile.percentile, - tolerance, confidence); - if (queries_issued < h + i) { - t = i - 1; - break; - } - } - } - QuerySampleLatency percentile_estimate = - (*query_latencies)[queries_issued - t]; - *recommendation = - " * Processed at least " + std::to_string(h_min + 1) + " queries (" + - std::to_string(queries_issued) + ").\n" + " * Would discard " + - std::to_string(t - 1) + " highest latency queries.\n" + - " * Early stopping " + - DoubleToString(target_latency_percentile.percentile * 100, 1) + - "th percentile estimate: " + std::to_string(percentile_estimate); - early_stopping_latency_ms = percentile_estimate; - break; - } - case TestScenario::Offline: - break; - } - return true; -} - -bool PerformanceSummary::MinDurationMet(std::string* recommendation) { - recommendation->clear(); - const double min_duration = DurationToSeconds(settings.min_duration); - bool min_duration_met = false; - switch (settings.scenario) { - case TestScenario::Offline: - min_duration_met = pr.max_latency >= min_duration; - break; - case TestScenario::Server: - min_duration_met = pr.final_query_scheduled_time >= min_duration; - break; - case TestScenario::SingleStream: - case TestScenario::MultiStream: - min_duration_met = pr.final_query_issued_time >= min_duration; - break; - } - if (min_duration_met) { - return true; - } - - switch (settings.scenario) { - case TestScenario::SingleStream: - case TestScenario::MultiStream: - *recommendation = - "Decrease the expected latency so the loadgen pre-generates more " - "queries."; - break; - case TestScenario::Server: - *recommendation = - "Increase the target QPS so the loadgen pre-generates more queries."; - break; - case TestScenario::Offline: - *recommendation = - "Increase expected QPS so the loadgen pre-generates a larger " - "(coalesced) query."; - break; - } - return false; -} - -bool PerformanceSummary::MinQueriesMet() { - return pr.queries_issued >= settings.min_query_count; -} - -bool PerformanceSummary::MinSamplesMet() { - return sample_count >= settings.min_sample_count; -} - -bool PerformanceSummary::HasPerfConstraints() { - return settings.scenario == TestScenario::Server; -} - -bool PerformanceSummary::PerfConstraintsMet(std::string* recommendation) { - recommendation->clear(); - bool perf_constraints_met = true; - switch (settings.scenario) { - case TestScenario::SingleStream: - case TestScenario::MultiStream: - break; - case TestScenario::Server: - ProcessLatencies(); - if (!settings.use_token_latencies) { - if (target_latency_percentile.sample_latency > - settings.target_latency.count()) { - *recommendation = "Reduce target QPS to improve latency."; - perf_constraints_met = false; - } - } else { - if (token_target_latency_percentile.sample_latency > - settings.server_ttft_latency) { - *recommendation = - "TTFT constrain not met: Reduce target QPS to improve latency."; - perf_constraints_met = false; - } - - if (target_tpot_percentile.sample_latency > - settings.server_tpot_latency) { - if (recommendation->empty()) { - *recommendation = - "TPOT constrain not met: Reduce target QPS to improve latency."; - } else { - recommendation->append( - "\n * TPOT constrain not met: Reduce target QPS to improve " - "latency."); - } - perf_constraints_met = false; - } - } - break; - case TestScenario::Offline: - break; - } - return perf_constraints_met; -} - -void PerformanceSummary::LogSummary(AsyncSummary& summary) { - ProcessLatencies(); - - summary( - "================================================\n" - "MLPerf Results Summary\n" - "================================================"); - summary("SUT name : ", sut_name); - summary("Scenario : ", ToString(settings.scenario)); - summary("Mode : ", ToString(settings.mode)); - - switch (settings.scenario) { - case TestScenario::SingleStream: { - summary(DoubleToString(target_latency_percentile.percentile * 100, 1) + - "th percentile latency (ns) : ", - target_latency_percentile.sample_latency); - break; - } - case TestScenario::MultiStream: { - summary(DoubleToString(target_latency_percentile.percentile * 100, 1) + - "th percentile latency (ns) : ", - target_latency_percentile.query_latency); - break; - } - case TestScenario::Server: { - // Subtract 1 from sample count since the start of the final sample - // represents the open end of the time range: i.e. [begin, end). - // This makes sense since: - // a) QPS doesn't apply if there's only one sample; it's pure latency. - // b) If you have precisely 1k QPS, there will be a sample exactly on - // the 1 second time point; but that would be the 1001th sample in - // the stream. Given the first 1001 queries, the QPS is - // 1000 queries / 1 second. - // TODO: make a more permanent solution - double qps_as_completed = - (sample_count - 1) / pr.final_query_all_samples_done_time; - summary("Completed samples per second : ", - DoubleToString(qps_as_completed)); - break; - } - case TestScenario::Offline: { - double samples_per_second = sample_count / pr.max_latency; - summary("Samples per second: ", samples_per_second); - break; - } - } - - if (settings.use_token_latencies) { - switch (settings.scenario) { - case TestScenario::SingleStream: { - summary(DoubleToString(token_target_latency_percentile.percentile * 100, - 1) + - "th first token percentile latency (ns) : ", - token_target_latency_percentile.sample_latency); - break; - } - case TestScenario::MultiStream: { - summary(DoubleToString(token_target_latency_percentile.percentile * 100, - 1) + - "th first token percentile latency (ns) : ", - token_target_latency_percentile.sample_latency); - break; - } - case TestScenario::Offline: { - double tokens_per_second = token_count / pr.max_latency; - summary("Tokens per second: ", tokens_per_second); - break; - } - case TestScenario::Server: - double tps_as_completed = - token_count / pr.final_query_all_samples_done_time; - summary("Completed tokens per second: ", - DoubleToString(tps_as_completed)); - break; - } - } - - if (settings.infer_token_latencies) { - switch (settings.scenario) { - case TestScenario::SingleStream: { - break; - } - case TestScenario::MultiStream: { - break; - } - case TestScenario::Offline: { - double tokens_per_second = settings.token_latency_scaling_factor * - sample_count / pr.max_latency; - summary("Tokens per second (inferred): ", tokens_per_second); - break; - } - case TestScenario::Server: - double tps_as_completed = settings.token_latency_scaling_factor * - (sample_count - 1) / - pr.final_query_all_samples_done_time; - summary("Completed tokens per second (inferred): ", - DoubleToString(tps_as_completed)); - break; - } - } - - std::string min_duration_recommendation; - std::string perf_constraints_recommendation; - std::string early_stopping_recommendation; - std::string early_stopping_ttft_recommendation; - std::string early_stopping_tpot_recommendation; - - bool min_duration_met = MinDurationMet(&min_duration_recommendation); - bool min_queries_met = MinQueriesMet() && MinSamplesMet(); - bool early_stopping_met = true; - if (!settings.use_token_latencies) { - early_stopping_met = EarlyStopping( - &early_stopping_recommendation, pr.queries_issued, &pr.sample_latencies, - &pr.query_latencies, settings.target_latency); - } else { - early_stopping_met = - EarlyStopping(&early_stopping_tpot_recommendation, pr.queries_issued, - &pr.token_results.time_per_output_token_arr, - &pr.query_latencies, - std::chrono::nanoseconds(settings.server_tpot_latency)) && - EarlyStopping(&early_stopping_ttft_recommendation, pr.queries_issued, - &pr.token_results.first_token_latencies, - &pr.query_latencies, - std::chrono::nanoseconds(settings.server_ttft_latency)); - } - bool perf_constraints_met = - PerfConstraintsMet(&perf_constraints_recommendation); - bool all_constraints_met = min_duration_met && min_queries_met && - perf_constraints_met && early_stopping_met; - summary("Result is : ", all_constraints_met ? "VALID" : "INVALID"); - if (HasPerfConstraints()) { - summary(" Performance constraints satisfied : ", - perf_constraints_met ? "Yes" : "NO"); - } - summary(" Min duration satisfied : ", min_duration_met ? "Yes" : "NO"); - summary(" Min queries satisfied : ", min_queries_met ? "Yes" : "NO"); - summary(" Early stopping satisfied: ", early_stopping_met ? "Yes" : "NO"); - - if (!all_constraints_met) { - summary("Recommendations:"); - if (!perf_constraints_met) { - summary(" * " + perf_constraints_recommendation); - } - if (!min_duration_met) { - summary(" * " + min_duration_recommendation); - } - if (!min_queries_met) { - summary( - " * The test exited early, before enough queries were issued.\n" - " See the detailed log for why this may have occurred."); - } - } - // Early stopping results - if (settings.scenario == TestScenario::SingleStream || - settings.scenario == TestScenario::Server || - settings.scenario == TestScenario::MultiStream) { - if (!settings.use_token_latencies) { - summary("Early Stopping Result:"); - summary(early_stopping_recommendation); - } else { - summary("TTFT Early Stopping Result:"); - summary(early_stopping_ttft_recommendation); - summary("TPOT Early Stopping Result:"); - summary(early_stopping_tpot_recommendation); - } - } - - summary( - "\n" - "================================================\n" - "Additional Stats\n" - "================================================"); - - if (settings.scenario == TestScenario::SingleStream) { - double qps_w_lg = (sample_count - 1) / pr.final_query_issued_time; - double qps_wo_lg = 1 / QuerySampleLatencyToSeconds(sample_latency_mean); - summary("QPS w/ loadgen overhead : " + DoubleToString(qps_w_lg)); - summary("QPS w/o loadgen overhead : " + DoubleToString(qps_wo_lg)); - summary(""); - } else if (settings.scenario == TestScenario::Server) { - // Scheduled samples per second as an additional stat - double qps_as_scheduled = - (sample_count - 1) / pr.final_query_scheduled_time; - summary("Scheduled samples per second : ", - DoubleToString(qps_as_scheduled)); - } else if (settings.scenario == TestScenario::MultiStream) { - summary("Per-query latency: "); - summary("Min latency (ns) : ", query_latency_min); - summary("Max latency (ns) : ", query_latency_max); - summary("Mean latency (ns) : ", query_latency_mean); - for (auto& lp : latency_percentiles) { - summary( - DoubleToString(lp.percentile * 100) + " percentile latency (ns) : ", - lp.query_latency); - } - } - - if (settings.scenario != TestScenario::MultiStream) { - summary("Min latency (ns) : ", sample_latency_min); - summary("Max latency (ns) : ", sample_latency_max); - summary("Mean latency (ns) : ", sample_latency_mean); - for (auto& lp : latency_percentiles) { - summary( - DoubleToString(lp.percentile * 100) + " percentile latency (ns) : ", - lp.sample_latency); - } - } - if (settings.use_token_latencies) { - summary(""); - if (settings.scenario == TestScenario::SingleStream) { - double tps_w_lg = token_count / pr.final_query_issued_time; - double tps_wo_lg = - ((double)token_count) / - (QuerySampleLatencyToSeconds(sample_latency_mean) * sample_count); - summary("TPS w/ loadgen overhead : " + DoubleToString(tps_w_lg)); - summary("TPS w/o loadgen overhead : " + DoubleToString(tps_wo_lg)); - - } else if (settings.scenario == TestScenario::Server) { - double tps_as_completed = - token_count / pr.final_query_all_samples_done_time; - summary("Completed tokens per second : ", - DoubleToString(tps_as_completed)); - } - - if (settings.scenario != TestScenario::Offline) { - summary("Min First Token latency (ns) : ", - first_token_latency_min); - summary("Max First Token latency (ns) : ", - first_token_latency_max); - summary("Mean First Token latency (ns) : ", - first_token_latency_mean); - for (auto& lp : token_latency_percentiles) { - summary(DoubleToString(lp.percentile * 100) + - " percentile first token latency (ns) : ", - lp.sample_latency); - } - summary(""); - summary("Min Time per Output Token (ns) : ", - time_per_output_token_min); - summary("Max Time per Output Token (ns) : ", - time_per_output_token_max); - summary("Mean Time per Output Token (ns) : ", - time_per_output_token_mean); - for (auto& lp : tpot_percentiles) { - summary(DoubleToString(lp.percentile * 100) + - " percentile time to output token (ns) : ", - lp.sample_latency); - } - } - } - - summary( - "\n" - "================================================\n" - "Test Parameters Used\n" - "================================================"); - settings.LogSummary(summary); -} - -void PerformanceSummary::LogDetail(AsyncDetail& detail) { -#if USE_NEW_LOGGING_FORMAT - ProcessLatencies(); - - // General validity checking - std::string min_duration_recommendation; - std::string perf_constraints_recommendation; - std::string early_stopping_recommendation; - std::string early_stopping_ttft_recommendation; - std::string early_stopping_tpot_recommendation; - bool min_duration_met = MinDurationMet(&min_duration_recommendation); - bool min_queries_met = MinQueriesMet() && MinSamplesMet(); - bool perf_constraints_met = - PerfConstraintsMet(&perf_constraints_recommendation); - bool early_stopping_met = true; - if (!settings.use_token_latencies) { - early_stopping_met = EarlyStopping( - &early_stopping_recommendation, pr.queries_issued, &pr.sample_latencies, - &pr.query_latencies, settings.target_latency); - } else { - early_stopping_met = - EarlyStopping(&early_stopping_tpot_recommendation, pr.queries_issued, - &pr.token_results.time_per_output_token_arr, - &pr.query_latencies, - std::chrono::nanoseconds(settings.server_tpot_latency)) && - EarlyStopping(&early_stopping_ttft_recommendation, pr.queries_issued, - &pr.token_results.first_token_latencies, - &pr.query_latencies, - std::chrono::nanoseconds(settings.server_ttft_latency)); - } - bool all_constraints_met = min_duration_met && min_queries_met && - perf_constraints_met && early_stopping_met; - - MLPERF_LOG(detail, "result_validity", - all_constraints_met ? "VALID" : "INVALID"); - if (HasPerfConstraints()) { - MLPERF_LOG(detail, "result_perf_constraints_met", perf_constraints_met); - } - MLPERF_LOG(detail, "result_min_duration_met", min_duration_met); - MLPERF_LOG(detail, "result_min_queries_met", min_queries_met); - MLPERF_LOG(detail, "early_stopping_met", early_stopping_met); - if (!all_constraints_met) { - std::string recommendation; - if (!perf_constraints_met) { - recommendation += perf_constraints_recommendation + " "; - } - if (!min_duration_met) { - recommendation += min_duration_recommendation + " "; - } - if (!min_queries_met) { - recommendation += - "The test exited early, before enough queries were issued."; - } - std::replace(recommendation.begin(), recommendation.end(), '\n', ' '); - MLPERF_LOG(detail, "result_invalid_reason", recommendation); - } - std::replace(early_stopping_recommendation.begin(), - early_stopping_recommendation.end(), '\n', ' '); - if (!settings.use_token_latencies) { - MLPERF_LOG(detail, "early_stopping_result", early_stopping_recommendation); - } else { - std::replace(early_stopping_ttft_recommendation.begin(), - early_stopping_ttft_recommendation.end(), '\n', ' '); - std::replace(early_stopping_tpot_recommendation.begin(), - early_stopping_tpot_recommendation.end(), '\n', ' '); - MLPERF_LOG(detail, "early_stopping_ttft_result", - early_stopping_ttft_recommendation); - MLPERF_LOG(detail, "early_stopping_tpot_result", - early_stopping_tpot_recommendation); - } - // Report number of queries - MLPERF_LOG(detail, "result_query_count", query_count); - if (settings.scenario == TestScenario::Server) { - MLPERF_LOG(detail, "result_overlatency_query_count", - overlatency_query_count); - } - - auto reportPerQueryLatencies = [&]() { - MLPERF_LOG(detail, "result_min_query_latency_ns", query_latency_min); - MLPERF_LOG(detail, "result_max_query_latency_ns", query_latency_max); - MLPERF_LOG(detail, "result_mean_query_latency_ns", query_latency_mean); - for (auto& lp : latency_percentiles) { - std::string percentile = DoubleToString(lp.percentile * 100); - MLPERF_LOG(detail, - "result_" + percentile + "_percentile_per_query_latency_ns", - lp.query_latency); - } - }; - - // Per-scenario performance results. - switch (settings.scenario) { - case TestScenario::SingleStream: { - double qps_w_lg = (sample_count - 1) / pr.final_query_issued_time; - double qps_wo_lg = 1 / QuerySampleLatencyToSeconds(sample_latency_mean); - MLPERF_LOG(detail, "result_qps_with_loadgen_overhead", qps_w_lg); - MLPERF_LOG(detail, "result_qps_without_loadgen_overhead", qps_wo_lg); - MLPERF_LOG(detail, "early_stopping_latency_ss", - early_stopping_latency_ss); - MLPERF_LOG(detail, "early_stopping_latency_ms", - early_stopping_latency_ms); - break; - } - case TestScenario::MultiStream: { - reportPerQueryLatencies(); - MLPERF_LOG(detail, "early_stopping_latency_ms", - early_stopping_latency_ms); - break; - } - case TestScenario::Server: { - // Subtract 1 from sample count since the start of the final sample - // represents the open end of the time range: i.e. [begin, end). - // This makes sense since: - // a) QPS doesn't apply if there's only one sample; it's pure latency. - // b) If you have precisely 1k QPS, there will be a sample exactly on - // the 1 second time point; but that would be the 1001th sample in - // the stream. Given the first 1001 queries, the QPS is - // 1000 queries / 1 second. - double qps_as_scheduled = - (sample_count - 1) / pr.final_query_scheduled_time; - MLPERF_LOG(detail, "result_scheduled_samples_per_sec", qps_as_scheduled); - double qps_as_completed = - (sample_count - 1) / pr.final_query_all_samples_done_time; - MLPERF_LOG(detail, "result_completed_samples_per_sec", qps_as_completed); - break; - } - case TestScenario::Offline: { - double samples_per_second = sample_count / pr.max_latency; - MLPERF_LOG(detail, "result_samples_per_second", samples_per_second); - break; - } - } - - // Detailed latencies - MLPERF_LOG(detail, "result_min_latency_ns", sample_latency_min); - MLPERF_LOG(detail, "result_max_latency_ns", sample_latency_max); - MLPERF_LOG(detail, "result_mean_latency_ns", sample_latency_mean); - for (auto& lp : latency_percentiles) { - MLPERF_LOG(detail, - "result_" + DoubleToString(lp.percentile * 100) + - "_percentile_latency_ns", - lp.sample_latency); - } - // Detailed first token latencies - if (settings.use_token_latencies) { - if (settings.scenario != TestScenario::Offline) { - MLPERF_LOG(detail, "result_first_token_min_latency_ns", - first_token_latency_min); - MLPERF_LOG(detail, "result_first_token_max_latency_ns", - first_token_latency_max); - MLPERF_LOG(detail, "result_first_token_mean_latency_ns", - first_token_latency_mean); - for (auto& lp : token_latency_percentiles) { - MLPERF_LOG(detail, - "result_first_token_" + DoubleToString(lp.percentile * 100) + - "_percentile_latency_ns", - lp.sample_latency); - } - double tps_w_lg = ((double)token_count) / pr.final_query_issued_time; - double tps_wo_lg = - ((double)token_count) / (sample_latency_mean * sample_count); - MLPERF_LOG(detail, "result_token_throughput_with_loadgen_overhead", - tps_w_lg); - MLPERF_LOG(detail, "result_token_throughput", tps_wo_lg); - for (auto& lp : tpot_percentiles) { - MLPERF_LOG(detail, - "result_time_per_output_token_" + - DoubleToString(lp.percentile * 100) + "_percentile_ns", - lp.sample_latency); - } - MLPERF_LOG(detail, "result_time_to_output_token_min", - time_per_output_token_min); - MLPERF_LOG(detail, "result_time_to_output_token_max", - time_per_output_token_max); - MLPERF_LOG(detail, "result_time_to_output_token_mean", - time_per_output_token_mean); - double tps_as_completed = - token_count / pr.final_query_all_samples_done_time; - MLPERF_LOG(detail, "result_completed_tokens_per_second", - tps_as_completed); - } else { - double tokens_per_second = token_count / pr.max_latency; - MLPERF_LOG(detail, "result_tokens_per_second", tokens_per_second); - } - } - - if (settings.infer_token_latencies) { - switch (settings.scenario) { - case TestScenario::Server: { - double completed_tokens_per_second = - (sample_count - 1) * settings.token_latency_scaling_factor / - pr.final_query_all_samples_done_time; - MLPERF_LOG(detail, "result_inferred_completed_tokens_per_second", - completed_tokens_per_second); - break; - } - case TestScenario::Offline: { - double tokens_per_second = sample_count * - settings.token_latency_scaling_factor / - pr.max_latency; - MLPERF_LOG(detail, "result_inferred_tokens_per_second", - tokens_per_second); - break; - } - case TestScenario::SingleStream: { - break; - } - case TestScenario::MultiStream: { - break; - } - } - } - MLPERF_LOG(detail, "num_errors", detail.async_log().GetErrorCount()); -#endif -} -} // namespace loadgen -} // namespace mlperf diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/results.h b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/results.h deleted file mode 100644 index 6befea2c0..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/results.h +++ /dev/null @@ -1,128 +0,0 @@ -/* Copyright 2019 The MLPerf Authors. All Rights Reserved. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -/// \file -/// \brief Defines PerformanceResult and PerformanceSummary. - -#ifndef MLPERF_LOADGEN_RESULTS_H_ -#define MLPERF_LOADGEN_RESULTS_H_ - -#include -#include - -#include "query_sample.h" -#include "test_settings_internal.h" - -namespace mlperf { -namespace loadgen { - -/// \brief Contains the performance results for benchmarks that have -/// token based metrics -struct TokenPerformanceResults { - std::vector first_token_latencies; - std::vector time_per_output_token_arr; - std::vector tokens_per_sample; -}; - -/// \brief Provides performance results that are independent of scenario -/// and other context. -struct PerformanceResult { - std::vector sample_latencies; - std::vector query_latencies; - size_t queries_issued; - double max_latency; - double final_query_scheduled_time; // seconds from start. - double final_query_issued_time; // seconds from start. - double final_query_all_samples_done_time; // seconds from start. - TokenPerformanceResults token_results; -}; - -/// \brief Wraps PerformanceResult with relevant context to change how -/// it's interpreted and reported. -struct PerformanceSummary { - std::string sut_name; - TestSettingsInternal settings; - PerformanceResult pr; - - // Set by ProcessLatencies. - size_t sample_count = 0; - size_t query_count = 0; - size_t overlatency_query_count = 0; - QuerySampleLatency sample_latency_min = 0; - QuerySampleLatency sample_latency_max = 0; - QuerySampleLatency sample_latency_mean = 0; - QuerySampleLatency query_latency_min = 0; - QuerySampleLatency query_latency_max = 0; - QuerySampleLatency query_latency_mean = 0; - - /// \brief The latency at a given percentile. - struct PercentileEntry { - const double percentile; - QuerySampleLatency sample_latency = 0; - QuerySampleLatency query_latency = 0; // MultiStream only. - }; - - // Latency target percentile - PercentileEntry target_latency_percentile{settings.target_latency_percentile}; - PercentileEntry latency_percentiles[6] = {{.50}, {.90}, {.95}, - {.97}, {.99}, {.999}}; - - // Early stopping percentile estimates for SingleStream and MultiStream - QuerySampleLatency early_stopping_latency_ss = 0; - QuerySampleLatency early_stopping_latency_ms = 0; - - // Set by ProcessTokenLatencies - size_t token_count = 0; - size_t overlatency_first_token_count = 0; - QuerySampleLatency first_token_latency_min = 0; - QuerySampleLatency first_token_latency_max = 0; - QuerySampleLatency first_token_latency_mean = 0; - QuerySampleLatency time_per_output_token_min = 0; - QuerySampleLatency time_per_output_token_max = 0; - QuerySampleLatency time_per_output_token_mean = 0; - - // Latency token target percentile - PercentileEntry token_target_latency_percentile{ - settings.target_latency_percentile}; - PercentileEntry token_latency_percentiles[6] = {{.50}, {.90}, {.95}, - {.97}, {.99}, {.999}}; - PercentileEntry target_tpot_percentile{settings.target_latency_percentile}; - PercentileEntry tpot_percentiles[6] = {{.50}, {.90}, {.95}, - {.97}, {.99}, {.999}}; - -#if defined(_WIN32) || defined(WIN32) || defined(_WIN64) || defined(WIN64) - // MSVC complains if there is no explicit constructor. - // (target_latency_percentile above depends on construction with settings) - PerformanceSummary(const std::string& sut_name_arg, - const TestSettingsInternal& settings_arg, - const PerformanceResult& pr_arg) - : sut_name(sut_name_arg), settings(settings_arg), pr(pr_arg){}; -#endif - void ProcessLatencies(); - void ProcessTokenLatencies(); - - bool MinDurationMet(std::string* recommendation); - bool EarlyStopping(std::string* recommendation, int64_t queries_issued, - std::vector* sample_latencies, - std::vector* query_latencies, - std::chrono::nanoseconds target_latency); - bool MinQueriesMet(); - bool MinSamplesMet(); - bool HasPerfConstraints(); - bool PerfConstraintsMet(std::string* recommendation); - void LogSummary(AsyncSummary& summary); - void LogDetail(AsyncDetail& detail); -}; -} // namespace loadgen -} // namespace mlperf - -#endif diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/setup.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/setup.py deleted file mode 100644 index 6254eea17..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/setup.py +++ /dev/null @@ -1,136 +0,0 @@ -# Copyright 2019 The MLPerf Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -# \file -# \brief MLPerf Inference LoadGen python module setup. -# \details Creates a module that python can import. -# All source files are compiled by python"s C++ toolchain without depending -# on a loadgen lib. -# -# This setup.py can be used stand-alone, without the use of an external -# build system. This will polute your source tree with output files -# and binaries. Use one of the gn build targets instead if you want -# to avoid poluting the source tree. - -from setuptools import Extension, setup -from pathlib import Path -from pybind11 import get_include -from pybind11.setup_helpers import Pybind11Extension, build_ext -from version_generator import generate_loadgen_version_definitions -import subprocess - -generated_version_source_filename = "generated/version_generated.cc" -generate_loadgen_version_definitions(generated_version_source_filename, ".") - -public_headers = [ - "loadgen.h", - "query_sample.h", - "query_sample_library.h", - "system_under_test.h", - "test_settings.h", - "issue_query_controller.h", - "early_stopping.h", - "query_dispatch_library.h" -] - -lib_headers = [ - "logging.h", - "test_settings_internal.h", - "trace_generator.h", - "utils.h", - "version.h", - "results.h", - "bindings/c_api.h", - "version_generator.py", - "mlperf_conf.h" -] - -lib_sources = [ - "early_stopping.cc", - "issue_query_controller.cc", - "loadgen.cc", - "logging.cc", - "test_settings_internal.cc", - "utils.cc", - "version.cc", - "results.cc", -] - -lib_bindings = [ - "bindings/c_api.cc", - "bindings/python_api.cc", -] - -this_directory = Path(__file__).parent -mlperf_loadgen_headers = public_headers + lib_headers -mlperf_loadgen_sources_no_gen = lib_sources + lib_bindings -mlperf_loadgen_sources = mlperf_loadgen_sources_no_gen + [ - generated_version_source_filename -] -mlperf_long_description = ( - this_directory / - "README.md").read_text( - encoding="utf-8") - -with open("VERSION.txt", "r") as f: - version = f.read() -version_split = version.split(".") - -if len(version_split) < 2: - print("Version is incomplete. Needs a format like 4.1.1 in VERSION file") - - -try: - with open("mlperf.conf", 'r') as file: - conf_contents = file.read() - - # Escape backslashes and double quotes - conf_contents = conf_contents.replace('\\', '\\\\').replace('"', '\\"') - - # Convert newlines - conf_contents = conf_contents.replace('\n', '\\n"\n"') - - formatted_content = f'const char* mlperf_conf =\n"{conf_contents}";\n' - - with open("mlperf_conf.h", 'w') as header_file: - header_file.write(formatted_content) - -except IOError as e: - raise RuntimeError(f"Failed to generate header file: {e}") - -mlperf_loadgen_module = Pybind11Extension( - "mlperf_loadgen", - define_macros=[ - ("MAJOR_VERSION", - version_split[0]), - ("MINOR_VERSION", - version_split[1]) - ], - include_dirs=[".", get_include()], - sources=mlperf_loadgen_sources, - depends=mlperf_loadgen_headers, -) - -setup(name="mlcommons_loadgen", - version=version, - description="MLPerf Inference LoadGen python bindings", - url="https://mlcommons.org/", - cmdclass={"build_ext": build_ext}, - ext_modules=[mlperf_loadgen_module], - packages=['mlcommons_loadgen'], - package_dir={'mlcommons_loadgen': '.'}, - include_package_data=True, - long_description=mlperf_long_description, - long_description_content_type='text/markdown') diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/system_under_test.h b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/system_under_test.h deleted file mode 100644 index 843453962..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/system_under_test.h +++ /dev/null @@ -1,67 +0,0 @@ -/* Copyright 2019 The MLPerf Authors. All Rights Reserved. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -/// \file -/// \brief Defines the SystemUnderTest interface. - -#ifndef MLPERF_LOADGEN_SYSTEM_UNDER_TEST_H -#define MLPERF_LOADGEN_SYSTEM_UNDER_TEST_H - -#include -#include - -#include "query_sample.h" - -namespace mlperf { - -/// \addtogroup LoadgenAPI -/// @{ - -/// \brief The interface a client implements for the loadgen to test. -/// \todo Add hook for an untimed warm up period for the SUT. -/// \todo Add hook for an untimed warm up period for the loadgen logic. -/// \todo Support power hooks for cool-down period before runing performance -/// traffic. -/// \todo Support power hooks for correlating test timeline with power -/// measurment timeline. -class SystemUnderTest { - public: - virtual ~SystemUnderTest() {} - - /// \brief A human-readable string for logging purposes. - virtual const std::string& Name() = 0; - - /// \brief Lets the loadgen issue N samples to the SUT. - /// \details The SUT may either a) return immediately and signal completion - /// at a later time on another thread or b) it may block and signal - /// completion on the current stack. The load generator will handle both - /// cases properly. - /// Note: The data for neighboring samples may or may not be contiguous - /// depending on the scenario. - virtual void IssueQuery(const std::vector& samples) = 0; - - /// \brief Called immediately after the last call to IssueQuery - /// in a series is made. - /// \details This doesn't necessarily signify the end of the - /// test since there may be multiple series involved during a test; for - /// example in accuracy mode. - /// Clients can use this to flush any deferred queries immediately, rather - /// than waiting for some timeout. - /// This is especially useful in the server scenario. - virtual void FlushQueries() = 0; -}; - -/// @} - -} // namespace mlperf - -#endif // MLPERF_LOADGEN_SYSTEM_UNDER_TEST_H diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/test_settings.h b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/test_settings.h deleted file mode 100644 index 584d073bb..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/test_settings.h +++ /dev/null @@ -1,329 +0,0 @@ -/* Copyright 2019 The MLPerf Authors. All Rights Reserved. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -/// \file -/// \brief Provides ways for a client to change the behavior and -/// constraints of the load generator. -/// \details Note: The MLPerf specification takes precedent over any of the -/// comments in this file if there are inconsistencies in regards to how the -/// loadgen *should* work. -/// The comments in this file are indicative of the loadgen implementation. - -#ifndef MLPERF_LOADGEN_TEST_SETTINGS_H -#define MLPERF_LOADGEN_TEST_SETTINGS_H - -#include -#include - -namespace mlperf { - -/// \addtogroup LoadgenAPI -/// @{ - -/// \addtogroup LoadgenAPITestSettings Test Settings -/// \brief This page contains a description of all the scenarios, modes, -/// and log settings as implemented by the LoadGen. -/// @{ - -/// -/// \enum TestScenario -/// * **SingleStream** -/// + Issues queries containing a single sample. -/// + The next query is only issued once the previous one has completed. -/// + Internal LoadGen latency between queries is not included in the -/// latency results. -/// + **Final performance result is:** a percentile of the latency. -/// * **MultiStream** -/// + Issues queries containing N samples. -/// - N is specified by \link -/// mlperf::TestSettings::multi_stream_samples_per_query -/// multi_stream_samples_per_query \endlink. -/// + The next query is only issued once the previous one has completed. -/// + The samples of each query are guaranteed to be contiguous with respect -/// to the order they were loaded in the QuerySampleLibrary. -/// + Latency is tracked and reported on a per-query and per-sample basis. -/// + The latency of a query is the maximum latency of its samples, including -/// any cross-thread communication within the loadgen. -/// + Internal LoadGen latency between queries is not included in the -/// latency results. -/// + **Final performance result is:** a percentile of the query latency. -/// * **Server** -/// + Sends queries with a single sample. -/// + Queries have a random poisson (non-uniform) arrival rate that, when -/// averaged, hits the target QPS. -/// + There is no limit on the number of outstanding queries, as long as -/// the latency constraints are met. -/// + **Final performance result is:** PASS if the a percentile of the latency -/// is under a given threshold. FAIL otherwise. -/// - Threshold is specified by \link -/// mlperf::TestSettings::server_target_latency_ns server_target_latency_ns -/// \endlink. -/// * **Offline** -/// + Sends all N samples to the SUT inside of a single query. -/// + The samples of the query are guaranteed to be contiguous with respect -/// to the order they were loaded in the QuerySampleLibrary. -/// + **Final performance result is:** samples per second. -/// -enum class TestScenario { - SingleStream, - MultiStream, - Server, - Offline, -}; - -/// -/// \enum TestMode -/// * **SubmissionRun** -/// + Runs accuracy mode followed by performance mode. -/// + TODO: Implement further requirements as decided by MLPerf. -/// * **AccuracyOnly** -/// + Runs each sample from the QSL through the SUT a least once. -/// + Outputs responses to an accuracy json that can be parsed by a model + -/// sample library specific script. -/// * **PerformanceOnly** -/// + Runs the performance traffic for the given scenario, as described in -/// the comments for TestScenario. -/// * **FindPeakPerformance** -/// + Determines the maximumum QPS for the Server scenario. -/// + Not applicable for SingleStream, MultiStream or Offline scenarios. -/// -enum class TestMode { - SubmissionRun, - AccuracyOnly, - PerformanceOnly, - FindPeakPerformance, -}; - -/// -/// \brief Top-level struct specifing the modes and parameters of the test. -/// -struct TestSettings { - TestScenario scenario = TestScenario::SingleStream; - TestMode mode = TestMode::PerformanceOnly; - - // ================================== - /// \name SingleStream-specific - /**@{*/ - /// \brief A hint used by the loadgen to pre-generate enough samples to - /// meet the minimum test duration. - double single_stream_expected_latency_ns = 1000000; - /// \brief The latency percentile reported as the final result. - double single_stream_target_latency_percentile = 0.90; - /**@}*/ - - // ================================== - /// \name MultiStream-specific - /**@{*/ - /// \brief A hint used by the loadgen to pre-generate enough samples to - /// meet the minimum test duration. - /// \brief MultiStream latency is for query (not sample) latency - double multi_stream_expected_latency_ns = 8000000; - /// \brief The latency percentile for MultiStream mode. - double multi_stream_target_latency_percentile = 0.99; - /// \brief The number of samples in each query. - /// \details How many samples are bundled in a query - uint64_t multi_stream_samples_per_query = 8; - /**@}*/ - - // ================================== - /// \name Server-specific - /**@{*/ - /// \brief The average QPS of the poisson distribution. - /// \details note: This field is used as a FindPeakPerformance's lower bound. - /// When you run FindPeakPerformanceMode, you should make sure that this value - /// satisfies performance constraints. - double server_target_qps = 1; - /// \brief The latency constraint for the Server scenario. - uint64_t server_target_latency_ns = 100000000; - /// \brief The latency percentile for server mode. This value is combined with - /// server_target_latency_ns to determine if a run is valid. - /// \details 99% is the default value, which is correct for image models. GNMT - /// should be set to 0.97 (97%) in v0.5.(As always, check the policy page for - /// updated values for the benchmark you are running.) - double server_target_latency_percentile = 0.99; - /// \brief If this flag is set to true, LoadGen will combine samples from - /// multiple queries into a single query if their scheduled issue times have - /// passed. - bool server_coalesce_queries = false; - /// \brief The decimal places of QPS precision used to terminate - /// FindPeakPerformance mode. - int server_find_peak_qps_decimals_of_precision = 1; - /// \brief A step size (as a fraction of the QPS) used to widen the lower and - /// upper bounds to find the initial boundaries of binary search. - double server_find_peak_qps_boundary_step_size = 1; - /// \brief The maximum number of outstanding queries to allow before earlying - /// out from a performance run. Useful for performance tuning and speeding up - /// the FindPeakPerformance mode. - uint64_t server_max_async_queries = 0; ///< 0: Infinity. - /// \brief The number of issue query threads that will be registered and used - /// to call SUT's IssueQuery(). If this is 0, the same thread calling - /// StartTest() will be used to call IssueQuery(). See also - /// mlperf::RegisterIssueQueryThread(). - uint64_t server_num_issue_query_threads = 0; - /**@}*/ - - // ================================== - /// \name Offline-specific - /**@{*/ - /// \brief Specifies the QPS the SUT expects to hit for the offline load. - /// The loadgen generates 10% more queries than it thinks it needs to meet - /// the minimum test duration. - double offline_expected_qps = 1; - /// \brief Affects the order in which the samples of the dataset are chosen. - /// If false it concatenates a single permutation of the dataset (or part - /// of it depending on QSL->PerformanceSampleCount()) several times up to the - /// number of samples requested. - /// If true it concatenates a multiple permutation of the dataset (or a - /// part of it depending on QSL->PerformanceSampleCount()) several times - /// up to the number of samples requested. - bool sample_concatenate_permutation = false; - /**@}*/ - - // ================================== - /// \name Test duration - /// The test runs until **both** min duration and min query count have been - /// met. However, it will exit before that point if **either** max duration or - /// max query count have been reached. - /**@{*/ - uint64_t min_duration_ms = 10000; - uint64_t max_duration_ms = 0; ///< 0: Infinity. - uint64_t min_query_count = 100; - uint64_t max_query_count = 0; ///< 0: Infinity. - /**@}*/ - - // ================================== - /// \name Random number generation - /// There are 4 separate seeds, so each dimension can be changed - /// independently. - /**@{*/ - /// \brief Affects which subset of samples from the QSL are chosen for - /// the performance sample set and accuracy sample sets. - uint64_t qsl_rng_seed = 0; - /// \brief Affects the order in which samples from the performance set will - /// be included in queries. - uint64_t sample_index_rng_seed = 0; - /// \brief Affects the poisson arrival process of the Server scenario. - /// \details Different seeds will appear to "jitter" the queries - /// differently in time, but should not affect the average issued QPS. - uint64_t schedule_rng_seed = 0; - /// \brief Affects which samples have their query returns logged to the - /// accuracy log in performance mode. - uint64_t accuracy_log_rng_seed = 0; - - /// \brief Probability of the query response of a sample being logged to the - /// accuracy log in performance mode - double accuracy_log_probability = 0.0; - - /// \brief Target number of samples that will have their results printed to - /// accuracy log in performance mode for compliance testing - uint64_t accuracy_log_sampling_target = 0; - - /// \brief Variables for running test05 from native config. A boolean that - /// determines whether or not to run test05 and three random seed to run the - /// test - bool test05 = false; - uint64_t test05_qsl_rng_seed = 0; - uint64_t test05_sample_index_rng_seed = 0; - uint64_t test05_schedule_rng_seed = 0; - - /// \brief Load mlperf parameter config from file. - int FromConfig(const std::string &path, const std::string &model, - const std::string &scenario, int conf_type = 1); - /**@}*/ - - // ================================== - /// \name Performance Sample modifiers - /// \details These settings can be used to Audit Performance mode runs. - /// In order to detect sample caching by SUT, performance of runs when only - /// unique queries (with non-repeated samples) are issued can be compared with - /// that when the same query is repeatedly issued. - /**@{*/ - /// \brief Prints measurement interval start and stop timestamps to std::cout - /// for the purpose of comparison against an external timer - bool print_timestamps = false; - /// \brief Allows issuing only unique queries in Performance mode of any - /// scenario \details This can be used to send non-repeat & hence unique - /// samples to SUT - bool performance_issue_unique = false; - /// \brief If true, the same query is chosen repeatedley for Inference. - /// In offline scenario, the query is filled with the same sample. - bool performance_issue_same = false; - /// \brief Offset to control which sample is repeated in - /// performance_issue_same mode. - /// Value should be within [0, performance_sample_count) - uint64_t performance_issue_same_index = 0; - /// \brief Overrides QSL->PerformanceSampleCount() when non-zero - uint64_t performance_sample_count_override = 0; - /// \brief Measure token latencies - bool use_token_latencies = false; - /// Token latency parameters - uint64_t server_ttft_latency = 100000000; - uint64_t server_tpot_latency = 100000000; - /// \brief Infer token latencies - bool infer_token_latencies = false; - uint64_t token_latency_scaling_factor; - /**@}*/ -}; - -/// -/// \enum LoggingMode -/// Specifies how and when logging should be sampled and stringified at -/// runtime. -/// * **AsyncPoll** -/// + Logs are serialized and output on an IOThread that polls for new logs at -/// a fixed interval. This is the only mode currently implemented. -/// * **EndOfTestOnly** -/// + TODO: Logs are serialzied and output only at the end of the test. -/// * **Synchronous** -/// + TODO: Logs are serialized and output inline. -enum class LoggingMode { - AsyncPoll, - EndOfTestOnly, - Synchronous, -}; - -/// -/// \brief Specifies where log outputs should go. -/// -/// By default, the loadgen outputs its log files to outdir and -/// modifies the filenames of its logs with a prefix and suffix. -/// Filenames will take the form: -/// "/summary.txt" -/// -/// Affordances for outputing logs to stdout are also provided. -/// -struct LogOutputSettings { - std::string outdir = "."; - std::string prefix = "mlperf_log_"; - std::string suffix = ""; - bool prefix_with_datetime = false; - bool copy_detail_to_stdout = false; - bool copy_summary_to_stdout = false; -}; - -/// -/// \brief Top-level log settings. -/// -struct LogSettings { - LogOutputSettings log_output; - LoggingMode log_mode = LoggingMode::AsyncPoll; - uint64_t log_mode_async_poll_interval_ms = 1000; ///< TODO: Implement this. - bool enable_trace = true; -}; - -/// @} - -/// @} - -} // namespace mlperf - -#endif // MLPERF_LOADGEN_TEST_SETTINGS_H diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/test_settings_internal.cc b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/test_settings_internal.cc deleted file mode 100644 index 3f2cd8847..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/test_settings_internal.cc +++ /dev/null @@ -1,800 +0,0 @@ -/* Copyright 2019 The MLPerf Authors. All Rights Reserved. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "test_settings_internal.h" - -#include -#include -#include -#include - -#include "logging.h" -#include "mlperf_conf.h" -#include "utils.h" - -namespace mlperf { -namespace loadgen { - -TestSettingsInternal::TestSettingsInternal( - const TestSettings &requested_settings, size_t qsl_performance_sample_count) - : requested(requested_settings), - scenario(requested.scenario), - mode(requested.mode), - samples_per_query(1), - target_qps(1), - max_async_queries(0), - target_duration(std::chrono::milliseconds(requested.min_duration_ms)), - min_duration(std::chrono::milliseconds(requested.min_duration_ms)), - max_duration(std::chrono::milliseconds(requested.max_duration_ms)), - min_query_count(requested.min_query_count), - max_query_count(requested.max_query_count), - min_sample_count(0), - qsl_rng_seed(requested.qsl_rng_seed), - sample_index_rng_seed(requested.sample_index_rng_seed), - schedule_rng_seed(requested.schedule_rng_seed), - accuracy_log_rng_seed(requested.accuracy_log_rng_seed), - accuracy_log_probability(requested.accuracy_log_probability), - accuracy_log_sampling_target(requested.accuracy_log_sampling_target), - print_timestamps(requested.print_timestamps), - performance_issue_unique(requested.performance_issue_unique), - performance_issue_same(requested.performance_issue_same), - performance_issue_same_index(requested.performance_issue_same_index), - performance_sample_count(0), - sample_concatenate_permutation(false), - use_token_latencies(requested.use_token_latencies), - server_ttft_latency(requested.server_ttft_latency), - server_tpot_latency(requested.server_tpot_latency), - infer_token_latencies(requested.infer_token_latencies), - token_latency_scaling_factor(requested.token_latency_scaling_factor) { - // Target QPS, target latency, and max_async_queries. - switch (requested.scenario) { - case TestScenario::SingleStream: - target_qps = static_cast(std::nano::den) / - requested.single_stream_expected_latency_ns; - max_async_queries = 1; - target_latency_percentile = - requested.single_stream_target_latency_percentile; - break; - case TestScenario::MultiStream: - target_qps = static_cast(std::nano::den) / - requested.multi_stream_expected_latency_ns; - max_async_queries = 1; - target_latency_percentile = - requested.multi_stream_target_latency_percentile; - break; - case TestScenario::Server: - if (requested.server_target_qps >= 0.0) { - target_qps = requested.server_target_qps; - } else { - LogDetail([server_target_qps = requested.server_target_qps, - target_qps = target_qps](AsyncDetail &detail) { -#if USE_NEW_LOGGING_FORMAT - std::stringstream ss; - ss << "Invalid value for server_target_qps requested." - << " requested: " << server_target_qps << " using: " << target_qps; - MLPERF_LOG_ERROR(detail, "error_invalid_test_settings", ss.str()); -#else - detail.Error("Invalid value for server_target_qps requested.", - "requested", server_target_qps, "using", target_qps); -#endif - }); - } - target_latency = - std::chrono::nanoseconds(requested.server_target_latency_ns); - target_latency_percentile = requested.server_target_latency_percentile; - max_async_queries = requested.server_max_async_queries; - break; - case TestScenario::Offline: - // target_latency_percentile is not used in Offline, but set it to - // 0.99 anyway to avoid garbage value. - target_latency_percentile = 0.99; - if (requested.offline_expected_qps >= 0.0) { - target_qps = requested.offline_expected_qps; - } else { - LogDetail([offline_expected_qps = requested.offline_expected_qps, - target_qps = target_qps](AsyncDetail &detail) { -#if USE_NEW_LOGGING_FORMAT - std::stringstream ss; - ss << "Invalid value for offline_expected_qps requested." - << " requested: " << offline_expected_qps - << " using: " << target_qps; - MLPERF_LOG_ERROR(detail, "error_invalid_test_settings", ss.str()); -#else - detail.Error("Invalid value for offline_expected_qps requested.", - "requested", offline_expected_qps, "using", target_qps); -#endif - }); - } - max_async_queries = 1; - break; - } - - // Performance Sample Count: TestSettings override QSL -> - // PerformanceSampleCount - performance_sample_count = (requested.performance_sample_count_override == 0) - ? qsl_performance_sample_count - : requested.performance_sample_count_override; - - // Sample by concatentating several permutations of the dataset - // sample_concatenate_permutation - sample_concatenate_permutation = - (requested.sample_concatenate_permutation == 0) - ? false - : requested.sample_concatenate_permutation; - - // Samples per query. - if (requested.scenario == TestScenario::MultiStream) { - samples_per_query = requested.multi_stream_samples_per_query; - } - - // In the offline scenario, coalesce all queries into a single query. - if (requested.scenario == TestScenario::Offline) { - // TODO: Should the spec require a max duration for large query counts? - // kSlack is used to make sure we generate enough samples for the SUT - // to take longer than than the minimum test duration required by the - // MLPerf spec. - constexpr double kSlack = 1.1; - uint64_t target_sample_count = - kSlack * DurationToSeconds(target_duration) * target_qps; - samples_per_query = - (requested.performance_issue_unique) - ? performance_sample_count - : std::max(min_query_count, target_sample_count); - min_query_count = 1; - target_duration = std::chrono::milliseconds(0); - } - - // FIXME: Only do this for 3D-UNet SingleStream, for v2.0 - // TODO: consolidate after v2.0 - // make min_queries to be multiple of performance_sample_count - // performance_sample_count == 0 makes it to be equal to loaded_samples.size() - if (sample_concatenate_permutation && - requested.scenario == TestScenario::SingleStream) { - // set slack larger for 3D-UNet KiTS19 distribution, i.e. 50% latency << 90% - // latency - constexpr double kSlack = 2.0; - uint64_t expected_queries = - kSlack * DurationToSeconds(target_duration) * target_qps; - min_query_count = - min_query_count > expected_queries ? min_query_count : expected_queries; - min_query_count += qsl_performance_sample_count - - (min_query_count % qsl_performance_sample_count); - } - - min_sample_count = min_query_count * samples_per_query; - - // Validate TestSettings - if (requested.performance_issue_same && - (requested.performance_issue_same_index >= performance_sample_count)) { - LogDetail([performance_issue_same_index = - requested.performance_issue_same_index, - performance_sample_count = - performance_sample_count](AsyncDetail &detail) { -#if USE_NEW_LOGGING_FORMAT - std::stringstream ss; - ss << "Sample Idx to be repeated in performance_issue_same mode" - << " cannot be greater than loaded performance_sample_count." - << " performance_issue_same_index: " << performance_issue_same_index - << " performance_sample_count: " << performance_sample_count; - MLPERF_LOG_ERROR(detail, "error_invalid_test_settings", ss.str()); -#else - detail.Error( - "Sample Idx to be repeated in performance_issue_same mode" - " cannot be greater than loaded performance_sample_count.", - "performance_issue_same_index", performance_issue_same_index, - "performance_sample_count", performance_sample_count); -#endif - }); - } - - if (requested.performance_issue_unique && requested.performance_issue_same) { - LogDetail([performance_issue_unique = requested.performance_issue_unique, - performance_issue_same = - requested.performance_issue_same](AsyncDetail &detail) { -#if USE_NEW_LOGGING_FORMAT - std::stringstream ss; - ss << "Performance_issue_unique and performance_issue_same, both" - << " cannot be true at the same time." - << " performance_issue_unique: " << performance_issue_unique - << " performance_issue_same: " << performance_issue_same; - MLPERF_LOG_ERROR(detail, "error_invalid_test_settings", ss.str()); -#else - detail.Error( - "Performance_issue_unique and performance_issue_same, both" - " cannot be true at the same time.", - "performance_issue_unique", performance_issue_unique, - "performance_issue_same", performance_issue_same); -#endif - }); - } -} - -std::string ToString(TestScenario scenario) { - switch (scenario) { -#if USE_NEW_LOGGING_FORMAT - case TestScenario::SingleStream: - return "SingleStream"; - case TestScenario::MultiStream: - return "MultiStream"; -#else - case TestScenario::SingleStream: - return "Single Stream"; - case TestScenario::MultiStream: - return "Multi Stream"; -#endif - case TestScenario::Server: - return "Server"; - case TestScenario::Offline: - return "Offline"; - } - assert(false); - return "InvalidScenario"; -} - -std::string ToString(TestMode mode) { - switch (mode) { -#if USE_NEW_LOGGING_FORMAT - case TestMode::SubmissionRun: - return "SubmissionRun"; - case TestMode::AccuracyOnly: - return "AccuracyOnly"; - case TestMode::PerformanceOnly: - return "PerformanceOnly"; - case TestMode::FindPeakPerformance: - return "FindPeakPerformance"; -#else - case TestMode::SubmissionRun: - return "Submission"; - case TestMode::AccuracyOnly: - return "Accuracy"; - case TestMode::PerformanceOnly: - return "Performance"; - case TestMode::FindPeakPerformance: - return "Find Peak Performance"; -#endif - } - assert(false); - return "InvalidMode"; -} - -void LogRequestedTestSettings(const TestSettings &s) { - LogDetail([s](AsyncDetail &detail) { -#if USE_NEW_LOGGING_FORMAT - MLPERF_LOG(detail, "requested_scenario", ToString(s.scenario)); - MLPERF_LOG(detail, "requested_test_mode", ToString(s.mode)); - - // Scenario-specific - switch (s.scenario) { - case TestScenario::SingleStream: - MLPERF_LOG(detail, "requested_single_stream_expected_latency_ns", - s.single_stream_expected_latency_ns); - MLPERF_LOG(detail, "requested_single_stream_target_latency_percentile", - s.single_stream_target_latency_percentile); - break; - case TestScenario::MultiStream: - MLPERF_LOG(detail, "requested_multi_stream_expected_latency_ns", - s.multi_stream_expected_latency_ns); - MLPERF_LOG(detail, "requested_multi_stream_target_latency_percentile", - s.multi_stream_target_latency_percentile); - MLPERF_LOG(detail, "requested_multi_stream_samples_per_query", - s.multi_stream_samples_per_query); - break; - case TestScenario::Server: - MLPERF_LOG(detail, "requested_server_target_qps", s.server_target_qps); - MLPERF_LOG(detail, "requested_server_target_latency_ns", - s.server_target_latency_ns); - MLPERF_LOG(detail, "requested_server_target_latency_percentile", - s.server_target_latency_percentile); - MLPERF_LOG(detail, "requested_server_coalesce_queries", - s.server_coalesce_queries); - MLPERF_LOG(detail, - "requested_server_find_peak_qps_decimals_of_precision", - s.server_find_peak_qps_decimals_of_precision); - MLPERF_LOG(detail, "requested_server_find_peak_qps_boundary_step_size", - s.server_find_peak_qps_boundary_step_size); - MLPERF_LOG(detail, "requested_server_max_async_queries", - s.server_max_async_queries); - MLPERF_LOG(detail, "requested_server_num_issue_query_threads", - s.server_num_issue_query_threads); - break; - case TestScenario::Offline: - MLPERF_LOG(detail, "requested_offline_expected_qps", - s.offline_expected_qps); - break; - } - - // Overrides - MLPERF_LOG(detail, "requested_min_duration_ms", s.min_duration_ms); - MLPERF_LOG(detail, "requested_max_duration_ms", s.max_duration_ms); - MLPERF_LOG(detail, "requested_min_query_count", s.min_query_count); - MLPERF_LOG(detail, "requested_max_query_count", s.max_query_count); - MLPERF_LOG(detail, "requested_qsl_rng_seed", s.qsl_rng_seed); - MLPERF_LOG(detail, "requested_sample_index_rng_seed", - s.sample_index_rng_seed); - MLPERF_LOG(detail, "requested_schedule_rng_seed", s.schedule_rng_seed); - MLPERF_LOG(detail, "requested_accuracy_log_rng_seed", - s.accuracy_log_rng_seed); - MLPERF_LOG(detail, "requested_accuracy_log_probability", - s.accuracy_log_probability); - MLPERF_LOG(detail, "requested_accuracy_log_sampling_target", - s.accuracy_log_sampling_target); - MLPERF_LOG(detail, "requested_print_timestamps", s.print_timestamps); - MLPERF_LOG(detail, "requested_performance_issue_unique", - s.performance_issue_unique); - MLPERF_LOG(detail, "requested_performance_issue_same", - s.performance_issue_same); - MLPERF_LOG(detail, "requested_performance_issue_same_index", - s.performance_issue_same_index); - MLPERF_LOG(detail, "requested_performance_sample_count_override", - s.performance_sample_count_override); - MLPERF_LOG(detail, "requested_sample_concatenate_permutation", - s.sample_concatenate_permutation); - // Token latencies specific values - if (s.use_token_latencies) { - MLPERF_LOG(detail, "requested_use_token_latencies", - s.use_token_latencies); - if (s.scenario != TestScenario::Offline) { - MLPERF_LOG(detail, "requested_server_ttft_latency", - s.server_ttft_latency); - MLPERF_LOG(detail, "requested_server_tpot_latency", - s.server_tpot_latency); - } - } -#else - detail(""); - detail("Requested Settings:"); - detail("Scenario : " + ToString(s.scenario)); - detail("Test mode : " + ToString(s.mode)); - - // Scenario-specific - switch (s.scenario) { - case TestScenario::SingleStream: - detail("single_stream_expected_latency_ns : ", - s.single_stream_expected_latency_ns); - detail("single_stream_target_latency_percentile : ", - s.single_stream_target_latency_percentile); - break; - case TestScenario::MultiStream: - detail("multi_stream_expected_latency_ns : ", - s.multi_stream_expected_latency_ns); - detail("multi_stream_target_latency_percentile : ", - s.multi_stream_target_latency_percentile); - detail("multi_stream_samples_per_query : ", - s.multi_stream_samples_per_query); - break; - case TestScenario::Server: - detail("server_target_qps : ", s.server_target_qps); - detail("server_target_latency_ns : ", s.server_target_latency_ns); - detail("server_target_latency_percentile : ", - s.server_target_latency_percentile); - detail("server_coalesce_queries : ", s.server_coalesce_queries); - detail("server_find_peak_qps_decimals_of_precision : ", - s.server_find_peak_qps_decimals_of_precision); - detail("server_find_peak_qps_boundary_step_size : ", - s.server_find_peak_qps_boundary_step_size); - detail("server_max_async_queries : ", s.server_max_async_queries); - detail("server_num_issue_query_threads : ", - s.server_num_issue_query_threads); - break; - case TestScenario::Offline: - detail("offline_expected_qps : ", s.offline_expected_qps); - break; - } - - // Overrides - detail("min_duration_ms : ", s.min_duration_ms); - detail("max_duration_ms : ", s.max_duration_ms); - detail("min_query_count : ", s.min_query_count); - detail("max_query_count : ", s.max_query_count); - detail("qsl_rng_seed : ", s.qsl_rng_seed); - detail("sample_index_rng_seed : ", s.sample_index_rng_seed); - detail("schedule_rng_seed : ", s.schedule_rng_seed); - detail("accuracy_log_rng_seed : ", s.accuracy_log_rng_seed); - detail("accuracy_log_probability : ", s.accuracy_log_probability); - detail("accuracy_log_sampling_target : ", s.accuracy_log_sampling_target); - detail("print_timestamps : ", s.print_timestamps); - detail("performance_issue_unique : ", s.performance_issue_unique); - detail("performance_issue_same : ", s.performance_issue_same); - detail("performance_issue_same_index : ", s.performance_issue_same_index); - detail("performance_sample_count_override : ", - s.performance_sample_count_override); - detail(""); -#endif - }); -} - -void TestSettingsInternal::LogEffectiveSettings() const { - LogDetail([s = *this](AsyncDetail &detail) { -#if USE_NEW_LOGGING_FORMAT - MLPERF_LOG(detail, "effective_scenario", ToString(s.scenario)); - MLPERF_LOG(detail, "effective_test_mode", ToString(s.mode)); - - MLPERF_LOG(detail, "effective_samples_per_query", s.samples_per_query); - MLPERF_LOG(detail, "effective_target_qps", s.target_qps); - MLPERF_LOG(detail, "effective_target_latency_ns", s.target_latency.count()); - MLPERF_LOG(detail, "effective_target_latency_percentile", - s.target_latency_percentile); - MLPERF_LOG(detail, "effective_max_async_queries", s.max_async_queries); - MLPERF_LOG(detail, "effective_target_duration_ms", - s.target_duration.count()); - MLPERF_LOG(detail, "effective_min_duration_ms", s.min_duration.count()); - MLPERF_LOG(detail, "effective_max_duration_ms", s.max_duration.count()); - MLPERF_LOG(detail, "effective_min_query_count", s.min_query_count); - MLPERF_LOG(detail, "effective_max_query_count", s.max_query_count); - MLPERF_LOG(detail, "effective_min_sample_count", s.min_sample_count); - MLPERF_LOG(detail, "effective_qsl_rng_seed", s.qsl_rng_seed); - MLPERF_LOG(detail, "effective_sample_index_rng_seed", - s.sample_index_rng_seed); - MLPERF_LOG(detail, "effective_schedule_rng_seed", s.schedule_rng_seed); - MLPERF_LOG(detail, "effective_accuracy_log_rng_seed", - s.accuracy_log_rng_seed); - MLPERF_LOG(detail, "effective_accuracy_log_probability", - s.accuracy_log_probability); - MLPERF_LOG(detail, "effective_accuracy_log_sampling_target", - s.accuracy_log_sampling_target); - MLPERF_LOG(detail, "effective_print_timestamps", s.print_timestamps); - MLPERF_LOG(detail, "effective_performance_issue_unique", - s.performance_issue_unique); - MLPERF_LOG(detail, "effective_performance_issue_same", - s.performance_issue_same); - MLPERF_LOG(detail, "effective_performance_issue_same_index", - s.performance_issue_same_index); - MLPERF_LOG(detail, "effective_performance_sample_count", - s.performance_sample_count); - MLPERF_LOG(detail, "effective_sample_concatenate_permutation", - s.sample_concatenate_permutation); -#else - detail(""); - detail("Effective Settings:"); - - detail("Scenario : " + ToString(s.scenario)); - detail("Test mode : " + ToString(s.mode)); - - detail("samples_per_query : ", s.samples_per_query); - detail("target_qps : ", s.target_qps); - detail("target_latency (ns): ", s.target_latency.count()); - detail("target_latency_percentile : ", s.target_latency_percentile); - detail("max_async_queries : ", s.max_async_queries); - detail("target_duration (ms): ", s.target_duration.count()); - detail("min_duration (ms): ", s.min_duration.count()); - detail("max_duration (ms): ", s.max_duration.count()); - detail("min_query_count : ", s.min_query_count); - detail("max_query_count : ", s.max_query_count); - detail("min_sample_count : ", s.min_sample_count); - detail("qsl_rng_seed : ", s.qsl_rng_seed); - detail("sample_index_rng_seed : ", s.sample_index_rng_seed); - detail("schedule_rng_seed : ", s.schedule_rng_seed); - detail("accuracy_log_rng_seed : ", s.accuracy_log_rng_seed); - detail("accuracy_log_probability : ", s.accuracy_log_probability); - detail("accuracy_log_sampling_target : ", s.accuracy_log_sampling_target); - detail("print_timestamps : ", s.print_timestamps); - detail("performance_issue_unique : ", s.performance_issue_unique); - detail("performance_issue_same : ", s.performance_issue_same); - detail("performance_issue_same_index : ", s.performance_issue_same_index); - detail("performance_sample_count : ", s.performance_sample_count); -#endif - }); -} - -void TestSettingsInternal::LogAllSettings() const { - LogRequestedTestSettings(requested); - LogEffectiveSettings(); -} - -void TestSettingsInternal::LogSummary(AsyncSummary &summary) const { - summary("samples_per_query : ", samples_per_query); - summary("target_qps : ", target_qps); - if (!use_token_latencies) { - summary("target_latency (ns): ", target_latency.count()); - } else { - summary("ttft_latency (ns): ", server_ttft_latency); - summary("tpot_latency (ns): ", server_tpot_latency); - } - summary("max_async_queries : ", max_async_queries); - summary("min_duration (ms): ", min_duration.count()); - summary("max_duration (ms): ", max_duration.count()); - summary("min_query_count : ", min_query_count); - summary("max_query_count : ", max_query_count); - summary("qsl_rng_seed : ", qsl_rng_seed); - summary("sample_index_rng_seed : ", sample_index_rng_seed); - summary("schedule_rng_seed : ", schedule_rng_seed); - summary("accuracy_log_rng_seed : ", accuracy_log_rng_seed); - summary("accuracy_log_probability : ", accuracy_log_probability); - summary("accuracy_log_sampling_target : ", accuracy_log_sampling_target); - summary("print_timestamps : ", print_timestamps); - summary("performance_issue_unique : ", performance_issue_unique); - summary("performance_issue_same : ", performance_issue_same); - summary("performance_issue_same_index : ", performance_issue_same_index); - summary("performance_sample_count : ", performance_sample_count); - if (sample_concatenate_permutation) { - summary( - "WARNING: sample_concatenate_permutation was set to true. \n" - "Generated samples per query might be different as the one in the " - "setting.\n" - "Check the generated_samples_per_query line in the detailed log for " - "the real\n" - "samples_per_query value"); - } -} - -} // namespace loadgen - -int TestSettings::FromConfig(const std::string &path, const std::string &model, - const std::string &scenario, int conf_type) { - std::map kv; - static int configCount = 0; - - if (conf_type == 1) { - if (configCount == 0) { - // Only allow userConf as the single configFile and loadgen loads the - // mlperfConf automatically for perf and accuracy runs - FromConfig("", model, scenario, 0); - } - - else { - LogDetail([](AsyncDetail &detail) { - std::stringstream ss; - ss << "Multiple conf files are used. This is not valid for official " - "submission."; - MLPERF_LOG_ERROR(detail, "error_invalid_config", ss.str()); - }); - } - configCount++; - } - - // lookup key/value pairs from config - auto lookupkv = [&](const std::string &model, const std::string &scenario, - const std::string &key, uint64_t *val_l, double *val_d, - double multiplier = 1.0) { - std::map::iterator it; - std::string found; - // lookup exact key first - it = kv.find(model + "." + scenario + "." + key); - if (it != kv.end()) { - found = it->second; - } else { - // lookup key with model wildcard - it = kv.find("*." + scenario + "." + key); - if (it != kv.end()) { - found = it->second; - } else { - it = kv.find(model + ".*." + key); - if (it != kv.end()) { - found = it->second; - } else { - it = kv.find("*.*." + key); - if (it != kv.end()) { - found = it->second; - } else { - return false; - } - } - } - } - // if we get here, found will be set - if (val_l) { - *val_l = strtoull(found.c_str(), nullptr, 0) * - static_cast(multiplier); - } - if (val_d) *val_d = strtod(found.c_str(), nullptr) * multiplier; - return true; - }; - - int line_nr = 0; - int errors = 0; - // Declare the input stream before the if-else block - std::unique_ptr fss; - std::string line; - - if (conf_type != 0) { - // dirt simple config parser - fss = std::make_unique(path); - if (!static_cast(fss.get())->is_open()) { - LogDetail([p = path](AsyncDetail &detail) { -#if USE_NEW_LOGGING_FORMAT - std::stringstream ss; - ss << "can't open file " << p; - MLPERF_LOG_ERROR(detail, "error_invalid_config", ss.str()); -#else - detail.Error("can't open file ", p); -#endif - }); - return -ENOENT; - } - } else { - // Convert unsigned char array to std::string - std::string config_str(mlperf_conf); - fss = std::make_unique(config_str); - } - while (std::getline(*fss, line)) { - line_nr++; - std::istringstream iss(line); - std::string s, k; - int looking_for = 0; // 0=key, 1=equal, 2=value - while (iss >> s) { - if (s == "#" && looking_for != 2) { - // done with this line - break; - } - if (looking_for == 2) { - // got key and value - const char *start = s.c_str(); - char *stop; - (void)strtoul(start, &stop, 0); - if (start + s.size() == stop) { - kv[k] = s; - continue; - } - (void)strtod(start, &stop); - if (start + s.size() == stop) { - kv[k] = s; - continue; - } - errors++; - LogDetail([l = line_nr](AsyncDetail &detail) { -#if USE_NEW_LOGGING_FORMAT - std::stringstream ss; - ss << "value needs to be integer or double, line=" << l; - MLPERF_LOG_ERROR(detail, "error_invalid_config", ss.str()); -#else - detail.Error("value needs to be integer or double, line=", l); -#endif - }); - break; - } - if (looking_for == 1 && s != "=") { - errors++; - LogDetail([l = line_nr](AsyncDetail &detail) { -#if USE_NEW_LOGGING_FORMAT - std::stringstream ss; - ss << "expected 'key=value', line=" << l; - MLPERF_LOG_ERROR(detail, "error_invalid_config", ss.str()); -#else - detail.Error("expected 'key=value', line=", l); -#endif - }); - break; - } - if (looking_for == 0) k = s; - looking_for++; - } - } - if (errors != 0) return -EINVAL; - - uint64_t val; - - // keys that apply to all scenarios - if (lookupkv(model, scenario, "mode", &val, nullptr)) { - switch (val) { - case 0: - mode = TestMode::SubmissionRun; - break; - case 1: - mode = TestMode::AccuracyOnly; - break; - case 2: - mode = TestMode::PerformanceOnly; - break; - case 3: - mode = TestMode::FindPeakPerformance; - break; - default: - LogDetail([](AsyncDetail &detail) { -#if USE_NEW_LOGGING_FORMAT - std::stringstream ss; - ss << "Invalid value passed to Mode key in config."; - MLPERF_LOG_ERROR(detail, "error_invalid_config", ss.str()); -#else - detail.Error("Invalid value passed to Mode key in config."); -#endif - }); - break; - } - } - - if (conf_type == 0) { - lookupkv(model, scenario, "qsl_rng_seed", &qsl_rng_seed, nullptr); - lookupkv(model, scenario, "sample_index_rng_seed", &sample_index_rng_seed, - nullptr); - lookupkv(model, scenario, "schedule_rng_seed", &schedule_rng_seed, nullptr); - lookupkv(model, scenario, "accuracy_log_probability", nullptr, - &accuracy_log_probability, 0.01); - if (lookupkv(model, scenario, "test05", &val, nullptr)) - test05 = (val == 1) ? true : false; - lookupkv(model, scenario, "test05_qsl_rng_seed", &test05_qsl_rng_seed, - nullptr); - lookupkv(model, scenario, "test05_sample_index_rng_seed", - &test05_sample_index_rng_seed, nullptr); - lookupkv(model, scenario, "test05_schedule_rng_seed", - &test05_schedule_rng_seed, nullptr); - } - - // keys that can be overriden in user.conf but will make the results eligible - // only for open submissions - - // keys to measure token metrics - if (lookupkv(model, scenario, "use_token_latencies", &val, nullptr)) { - use_token_latencies = (val == 1) ? true : false; - } - if (use_token_latencies) { - lookupkv(model, "Server", "ttft_latency", &server_ttft_latency, nullptr, - 1000 * 1000); - lookupkv(model, "Server", "tpot_latency", &server_tpot_latency, nullptr, - 1000 * 1000); - } - - // keys to infer token metrics - if (lookupkv(model, scenario, "infer_token_latencies", &val, nullptr)) { - infer_token_latencies = (val == 1) ? true : false; - } - if (infer_token_latencies) { - lookupkv(model, scenario, "token_latency_scaling_factor", - &token_latency_scaling_factor, nullptr, 1); - } - // keys that apply to SingleStream - lookupkv(model, "SingleStream", "target_latency_percentile", nullptr, - &single_stream_target_latency_percentile, 0.01); - - // keys that apply to MultiStream - lookupkv(model, "MultiStream", "target_latency_percentile", nullptr, - &multi_stream_target_latency_percentile, 0.01); - lookupkv(model, "MultiStream", "samples_per_query", - &multi_stream_samples_per_query, nullptr, 1); - - // keys that apply to Server - lookupkv(model, "Server", "target_latency_percentile", nullptr, - &server_target_latency_percentile, 0.01); - lookupkv(model, "Server", "target_latency", &server_target_latency_ns, - nullptr, 1000 * 1000); - - // keys that can be overriden in user.conf (the provided values still need to - // pass the submission checker rules) - if (lookupkv(model, scenario, "performance_issue_unique", &val, nullptr)) - performance_issue_unique = (val == 0) ? false : true; - if (lookupkv(model, scenario, "performance_issue_same", &val, nullptr)) - performance_issue_same = (val == 0) ? false : true; - lookupkv(model, scenario, "performance_issue_same_index", - &performance_issue_same_index, nullptr); - - if (lookupkv(model, scenario, "sample_concatenate_permutation", &val, - nullptr)) - sample_concatenate_permutation = (val == 1) ? true : false; - if (lookupkv(model, "Server", "coalesce_queries", &val, nullptr)) - server_coalesce_queries = (val == 0) ? false : true; - if (lookupkv(model, "Server", "max_async_queries", &val, nullptr)) - server_max_async_queries = int(val); - - lookupkv(model, scenario, "min_duration", &min_duration_ms, nullptr); - lookupkv(model, scenario, "max_duration", &max_duration_ms, nullptr); - lookupkv(model, scenario, "min_query_count", &min_query_count, nullptr); - lookupkv(model, scenario, "max_query_count", &max_query_count, nullptr); - lookupkv(model, scenario, "performance_sample_count_override", - &performance_sample_count_override, nullptr); - lookupkv(model, "SingleStream", "target_latency", nullptr, - &single_stream_expected_latency_ns, 1000 * 1000); - lookupkv(model, "MultiStream", "target_latency", nullptr, - &multi_stream_expected_latency_ns, 1000 * 1000); - lookupkv(model, "Server", "target_qps", nullptr, &server_target_qps); - lookupkv(model, "Offline", "target_qps", 0, &offline_expected_qps); - - if (lookupkv(model, scenario, "print_timestamps", &val, nullptr)) - print_timestamps = (val == 0) ? false : true; - - // keys that are used in audit.conf - lookupkv(model, scenario, "accuracy_log_rng_seed", &accuracy_log_rng_seed, - nullptr); - lookupkv(model, scenario, "accuracy_log_sampling_target", - &accuracy_log_sampling_target, nullptr); - return 0; -} - -} // namespace mlperf diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/test_settings_internal.h b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/test_settings_internal.h deleted file mode 100644 index ab2773bd1..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/test_settings_internal.h +++ /dev/null @@ -1,182 +0,0 @@ -/* Copyright 2019 The MLPerf Authors. All Rights Reserved. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -/// \file -/// \brief The internal representation of user-provided settings. - -#ifndef MLPERF_LOADGEN_TEST_SETTINGS_INTERNAL_H -#define MLPERF_LOADGEN_TEST_SETTINGS_INTERNAL_H - -#include -#include -#include - -#include "logging.h" -#include "test_settings.h" - -namespace mlperf { - -namespace logging { -class AsyncSummary; -} - -namespace loadgen { - -using AsyncSummary = logging::AsyncSummary; - -std::string ToString(TestScenario scenario); -std::string ToString(TestMode mode); - -/// \brief takes the user-friendly TestSettings and normalizes it -/// for consumption by the loadgen. -/// \details It does things like remove scenario-specific naming and introduce -/// the concept of target_duration used to pre-generate queries. -struct TestSettingsInternal { - explicit TestSettingsInternal(const TestSettings &requested_settings, - size_t qsl_performance_sample_count); - void LogEffectiveSettings() const; - void LogAllSettings() const; - void LogSummary(AsyncSummary &summary) const; - - const TestSettings requested; - const TestScenario scenario; // Copied here for convenience. - const TestMode mode; // Copied here for convenience. - - uint64_t samples_per_query; - double target_qps; - std::chrono::nanoseconds target_latency{0}; - double target_latency_percentile; // Single, multistream, and server modes. - uint64_t max_async_queries; - - // Target duration is used to generate queries of a minimum duration before - // the test run. - std::chrono::milliseconds target_duration{0}; - - // Min duration/query_count/sample_count are used to validate the test - // duration at the end of the run. - std::chrono::milliseconds min_duration{0}; - std::chrono::milliseconds max_duration{0}; - uint64_t min_query_count; - uint64_t max_query_count; - uint64_t min_sample_count; // Offline only. - - uint64_t qsl_rng_seed; - uint64_t sample_index_rng_seed; - uint64_t schedule_rng_seed; - uint64_t accuracy_log_rng_seed; - double accuracy_log_probability; - uint64_t accuracy_log_sampling_target; - bool print_timestamps; - bool performance_issue_unique; - bool performance_issue_same; - uint64_t performance_issue_same_index; - uint64_t performance_sample_count; - - bool sample_concatenate_permutation; - bool use_token_latencies = false; - int64_t server_ttft_latency; - int64_t server_tpot_latency; - - bool infer_token_latencies = false; - int64_t token_latency_scaling_factor; -}; - -/// \brief A namespace of collections of FindPeakPerformance helper functions, -/// mainly about binary search. -namespace find_peak_performance { - -constexpr char const *kNotSupportedMsg = - "Finding peak performance is only supported in Server scenarios."; - -template -TestSettingsInternal MidOfBoundaries( - const TestSettingsInternal &lower_bound_settings, - const TestSettingsInternal &upper_bound_settings) { - TestSettingsInternal mid_settings = lower_bound_settings; - if (scenario == TestScenario::Server) { - assert(lower_bound_settings.target_qps < upper_bound_settings.target_qps); - mid_settings.target_qps = - lower_bound_settings.target_qps + - (upper_bound_settings.target_qps - lower_bound_settings.target_qps) / 2; - } else { - LogDetail([](AsyncDetail &detail) { -#if USE_NEW_LOGGING_FORMAT - MLPERF_LOG_ERROR(detail, "error_invalid_test_settings", kNotSupportedMsg); -#else - detail(kNotSupportedMsg); -#endif - }); - } - return mid_settings; -} - -template -bool IsFinished(const TestSettingsInternal &lower_bound_settings, - const TestSettingsInternal &upper_bound_settings) { - if (scenario == TestScenario::Server) { - uint8_t precision = lower_bound_settings.requested - .server_find_peak_qps_decimals_of_precision; - double l = - std::floor(lower_bound_settings.target_qps * std::pow(10, precision)); - double u = - std::floor(upper_bound_settings.target_qps * std::pow(10, precision)); - return l + 1 >= u; - } else { - LogDetail([](AsyncDetail &detail) { -#if USE_NEW_LOGGING_FORMAT - MLPERF_LOG_ERROR(detail, "error_invalid_test_settings", kNotSupportedMsg); -#else - detail(kNotSupportedMsg); -#endif - }); - return true; - } -} - -template -std::string ToStringPerformanceField(const TestSettingsInternal &settings) { - if (scenario == TestScenario::Server) { - return std::to_string(settings.target_qps); - } else { - LogDetail([](AsyncDetail &detail) { -#if USE_NEW_LOGGING_FORMAT - MLPERF_LOG_ERROR(detail, "error_invalid_test_settings", kNotSupportedMsg); -#else - detail(kNotSupportedMsg); -#endif - }); - return ToString(settings.scenario); - } -} - -template -void WidenPerformanceField(TestSettingsInternal *settings) { - if (scenario == TestScenario::Server) { - settings->target_qps = - settings->target_qps * - (1 + settings->requested.server_find_peak_qps_boundary_step_size); - } else { - LogDetail([](AsyncDetail &detail) { -#if USE_NEW_LOGGING_FORMAT - MLPERF_LOG_ERROR(detail, "error_invalid_test_settings", kNotSupportedMsg); -#else - detail(kNotSupportedMsg); -#endif - }); - } -} - -} // namespace find_peak_performance -} // namespace loadgen -} // namespace mlperf - -#endif // MLPERF_LOADGEN_TEST_SETTINGS_INTERNAL_H diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/BUILD.gn b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/BUILD.gn deleted file mode 100644 index d73bf831a..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/BUILD.gn +++ /dev/null @@ -1,25 +0,0 @@ -static_library("mlperf_loadgen_tests_loadgen_test_main") { - sources = [ "loadgen_test.h", "loadgen_test_main.cc" ] - configs += [ "//build/config/compiler:exceptions" ] -} - -executable("mlperf_loadgen_perftests") { - sources = [ "perftests_null_sut.cc" ] - deps = [ "..:mlperf_loadgen" ] -} - -executable("mlperf_loadgen_tests_basic") { - sources = [ "basic.cc" ] - deps = [ "..:mlperf_loadgen", - ":mlperf_loadgen_tests_loadgen_test_main" ] - configs += [ "//build/config/compiler:exceptions" ] -} - -source_set("mlperf_loadgen_perftests_py") { - sources = [ "perftests_null_sut.py" ] - deps = [ "../..:loadgen_pymodule_wheel_lib" ] -} - -source_set("docs") { - sources = [ "README.md" ] -} diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/README.md b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/README.md deleted file mode 100644 index 41056b457..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/README.md +++ /dev/null @@ -1,42 +0,0 @@ -# Building and Running the Tests {#ReadmeTests} - -The unit and performance tests are only supported via gn/ninja at the moment. - -See the [top-level build readme](@ref ReadmeBuild) for details but, from a clean checkout, you must first run: - - make bootstrap_gn_ninja - third_party/gn/gn gen out/Release --args="is_debug=false" - -This will build the gn and ninja build tools and create a release project. - -## Unit Tests - -To build: - - third_party/ninja/ninja -C out/Release mlperf_loadgen_tests_basic - -To run all tests: - - out/Release/mlperf_loadgen_tests_basic . - -To run specific tests: - - out/Release/mlperf_loadgen_tests_basic - e.g.: - out/Release/mlperf_loadgen_tests_basic SingleStream - -## Performance Tests - -To build: - - third_party/ninja/ninja -C out/Release mlperf_loadgen_perftests - -To run all tests: - - out/Release/mlperf_loadgen_perftests . - -To run specific tests: - - out/Release/mlperf_loadgen_perftests - e.g.: - out/Release/mlperf_loadgen_tests_basic ServerPool diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/basic.cc b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/basic.cc deleted file mode 100644 index 97c6a0bb1..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/basic.cc +++ /dev/null @@ -1,314 +0,0 @@ -/* Copyright 2019 The MLPerf Authors. All Rights Reserved. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -/// \file -/// \brief Basic functionality unit tests. - -#include -#include -#include -#include -#include -#include -#include - -#include "../loadgen.h" -#include "../query_sample_library.h" -#include "../system_under_test.h" -#include "../test_settings.h" -#include "loadgen_test.h" - -/// \brief Correctness unit tests. -namespace unit_tests { - -/// \defgroup LoadgenTestsBasic Test Coverage: Basic - -/// \brief Implements the client interfaces of the loadgen and -/// has some basic sanity checks that are enabled for all tests. -/// \details It also forwards calls to overrideable *Ext methods and implements -/// the TestProxy concept. -struct SystemUnderTestBasic : public mlperf::QuerySampleLibrary, - public mlperf::SystemUnderTest { - const std::string& Name() const override { return name_; } - - size_t TotalSampleCount() override { return total_sample_count_; } - size_t PerformanceSampleCount() override { return performance_sample_count_; } - - void LoadSamplesToRam( - const std::vector& samples) override { - for (auto s : samples) { - samples_load_count_.at(s)++; - loaded_samples_.push_back(s); - } - LoadSamplesToRamExt(samples); - } - virtual void LoadSamplesToRamExt( - const std::vector& samples) {} - - void UnloadSamplesFromRam( - const std::vector& samples) override { - for (auto s : samples) { - FAIL_IF(loaded_samples_.front() != s) && - FAIL_EXP(loaded_samples_.front()) && FAIL_EXP(s); - loaded_samples_.pop_front(); - size_t prev_load_count = samples_load_count_.at(s)--; - FAIL_IF(prev_load_count == 0) && FAIL_EXP(prev_load_count); - } - UnloadSamplesFromRamExt(samples); - } - virtual void UnloadSamplesFromRamExt( - const std::vector& samples) {} - - void IssueQuery(const std::vector& samples) override { - std::vector responses; - query_sizes_.push_back(samples.size()); - samples_between_flushes_.back() += samples.size(); - responses.reserve(samples.size()); - for (auto s : samples) { - FAIL_IF(samples_load_count_.at(s.index) == 0) && - FAIL_MSG("Issued unloaded sample:") && FAIL_EXP(s.index); - samples_issue_count_.at(s.index)++; - issued_samples_.push_back(s.index); - responses.push_back({s.id, 0, 0}); - } - mlperf::QuerySamplesComplete(responses.data(), responses.size()); - IssueQueryExt(samples); - } - virtual void IssueQueryExt(const std::vector& samples) {} - - void FlushQueries() override { - samples_between_flushes_.push_back(0); - FlushQueriesExt(); - } - virtual void FlushQueriesExt() {} - - virtual void RunTest() { - samples_load_count_.resize(total_sample_count_, 0); - samples_issue_count_.resize(total_sample_count_, 0); - samples_between_flushes_.resize(1, 0); - mlperf::StartTest(this, this, test_settings_, log_settings_); - } - - virtual void EndTest() {} - - protected: - mlperf::TestSettings test_settings_; - mlperf::LogSettings log_settings_; - - std::string name_{"BasicSUT"}; - size_t total_sample_count_; - size_t performance_sample_count_; - std::vector issued_samples_; - std::deque loaded_samples_; - std::vector samples_load_count_; - std::vector samples_issue_count_; - - std::vector query_sizes_; - std::vector samples_between_flushes_; -}; - -/// \brief Provides common test set up logic. -struct SystemUnderTestAccuracy : public SystemUnderTestBasic { - virtual void SetUpTest(size_t samples_per_query, - size_t samples_per_query_remainder, - size_t accuracy_remainder, - mlperf::TestScenario scenario) { - performance_sample_count_ = - samples_per_query * 16 + samples_per_query_remainder; - total_sample_count_ = performance_sample_count_ * 32 + accuracy_remainder; - - log_settings_.log_output.prefix_with_datetime = false; - - test_settings_.scenario = scenario; - test_settings_.mode = mlperf::TestMode::AccuracyOnly; - test_settings_.multi_stream_samples_per_query = samples_per_query; - - double qps = 1e3; - test_settings_.server_target_qps = qps; - } -}; - -/// \brief Verifies all samples from the QSL are included at least once -/// in accuracy mode. -/// \ingroup LoadgenTestsBasic -struct TestAccuracyIncludesAllSamples : public SystemUnderTestAccuracy { - void EndTest() override { - std::sort(issued_samples_.begin(), issued_samples_.end()); - - FAIL_IF(issued_samples_.size() < total_sample_count_) && - FAIL_EXP(issued_samples_.size()) && FAIL_EXP(total_sample_count_); - FAIL_IF(issued_samples_.front() != 0) && FAIL_EXP(issued_samples_.front()); - FAIL_IF(issued_samples_.back() != total_sample_count_ - 1) && - FAIL_EXP(issued_samples_.back()) && FAIL_EXP(total_sample_count_); - - mlperf::QuerySampleIndex prev = -1; - size_t discontinuities = 0; - size_t dupes = 0; - for (auto s : issued_samples_) { - if (s == prev) { - dupes++; - } else if (s - prev > 1) { - discontinuities++; - } - prev = s; - } - - FAIL_IF(discontinuities != 0) && FAIL_EXP(discontinuities); - FAIL_IF(dupes != 0) && FAIL_EXP(dupes); - } -}; - -REGISTER_TEST_ALL_SCENARIOS(AccuracyIncludesAllSamples, - TestProxy(), 4, 0, - 0); - -/// \brief Verifies samples from the QSL aren't included too many times. -/// \details This is a regression test for: -/// https://github.com/mlperf/inference/pull/386 -/// The root cause was using different values for samples_per_query while -/// generating queries for the GNMT dataset. -/// \ingroup LoadgenTestsBasic -struct TestAccuracyDupesAreLimitted : public SystemUnderTestAccuracy { - void SetUpTest(bool, mlperf::TestScenario scenario) { - SystemUnderTestAccuracy::SetUpTest(4, 0, 0, scenario); - total_sample_count_ = 3003; - performance_sample_count_ = 1001; - } - - void EndTest() override { - std::sort(issued_samples_.begin(), issued_samples_.end()); - - FAIL_IF(issued_samples_.size() < total_sample_count_) && - FAIL_EXP(issued_samples_.size()) && FAIL_EXP(total_sample_count_); - FAIL_IF(issued_samples_.front() != 0) && FAIL_EXP(issued_samples_.front()); - FAIL_IF(issued_samples_.back() != total_sample_count_ - 1) && - FAIL_EXP(issued_samples_.back()) && FAIL_EXP(total_sample_count_); - - std::vector issue_counts(total_sample_count_, 0); - for (auto s : issued_samples_) { - issue_counts.at(s)++; - } - - const size_t max_count = 1; - for (size_t i = 0; i < issue_counts.size(); i++) { - FAIL_IF(issue_counts[i] > max_count) && FAIL_EXP(i) && - FAIL_EXP(max_count) && FAIL_EXP(issue_counts[i]); - } - } -}; - -REGISTER_TEST_ALL_SCENARIOS(TestAccuracyDupesAreLimitted, - TestProxy(), true); - -/// \brief Verifies offline + accuracy doesn't hang if the last set -/// in the accuracy series is smaller than others. -/// \ingroup LoadgenTestsBasic -struct TestOfflineRemainderAccuracySet : public SystemUnderTestAccuracy { - void SetUpTest() { - SystemUnderTestAccuracy::SetUpTest(4, 0, 7, mlperf::TestScenario::Offline); - } - - void EndTest() override { - auto& flush_samples = samples_between_flushes_; - - FAIL_IF(flush_samples.size() < 3) && FAIL_EXP(flush_samples.size()) && - BAD_TEST_MSG("Test should generate multiple query sets.") && ABORT_TEST; - - // The last counter will be 0, since a test ends with a call to - // FlushQuery. - FAIL_IF(flush_samples.back() != 0) && FAIL_EXP(flush_samples.back()) && - FAIL_MSG( - "Detected stray calls to IssueQuery after the last call to " - "FlushQuery."); - flush_samples.pop_back(); - - // Verify the test ran with a smaller last accuracy set. - size_t first_size = flush_samples.front(); - size_t last_size = flush_samples.back(); - FAIL_IF(first_size <= last_size) && FAIL_EXP(first_size) && - FAIL_EXP(last_size) && BAD_TEST_MSG(); - - flush_samples.pop_back(); // Don't check the last set for equality. - for (size_t query_size : flush_samples) { - FAIL_IF(query_size != first_size) && FAIL_EXP(query_size) && - FAIL_EXP(first_size); - } - } -}; - -REGISTER_TEST(Offline_RemainderAccuracySets, - TestProxy()); - -/// \brief Verifies all queries only contain samples that are contiguous, -/// even if the set size is not a multiple of samples_per_query. -/// \ingroup LoadgenTestsBasic -struct TestMultiStreamContiguousRemainderQuery - : public SystemUnderTestAccuracy { - void SetUpTest(mlperf::TestScenario scenario) { - SystemUnderTestAccuracy::SetUpTest(4, 1, 0, scenario); - first_qsl_offsets_.resize(total_sample_count_, kBadQslOffset); - - auto spq = test_settings_.multi_stream_samples_per_query; - FAIL_IF(performance_sample_count_ % spq == 0) && - FAIL_EXP(performance_sample_count_) && FAIL_EXP(spq) && - BAD_TEST_MSG("There is no remainder."); - } - - void LoadSamplesToRamExt( - const std::vector& samples) override { - FAIL_IF(loaded_samples_.size() != samples.size()) && - FAIL_MSG("Contiguous sample order is likely ambiguous."); - for (size_t i = 0; i < samples.size(); i++) { - auto& offset = first_qsl_offsets_.at(samples.at(i)); - // Samples may be loaded into multiple slots for padding purposes, - // so make sure to only index the first time a sample appears in a - // loaded set. - if (offset == kBadQslOffset) { - offset = i; - } - } - } - - void UnloadSamplesFromRamExt( - const std::vector& samples) override { - FAIL_IF(!loaded_samples_.empty()) && - FAIL_MSG("Contiguous sample order is likely ambiguous."); - for (size_t i = 0; i < samples.size(); i++) { - first_qsl_offsets_.at(samples.at(i)) = kBadQslOffset; - } - } - - void IssueQueryExt(const std::vector& samples) override { - size_t expected_offset = first_qsl_offsets_[samples[0].index]; - for (auto s : samples) { - FAIL_IF(loaded_samples_[expected_offset] != s.index) && - FAIL_MSG("Samples are not contiguous."); - expected_offset++; - } - } - - void FlushQueriesExt() override {} - - void EndTest() override {} - - private: - static const size_t kBadQslOffset; - std::vector first_qsl_offsets_; -}; - -constexpr size_t TestMultiStreamContiguousRemainderQuery::kBadQslOffset = - std::numeric_limits::max(); - -REGISTER_TEST(MultiStream_RemainderQueryContiguous, - TestProxy(), - mlperf::TestScenario::MultiStream); -} // namespace unit_tests diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/loadgen_test.h b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/loadgen_test.h deleted file mode 100644 index 777029b99..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/loadgen_test.h +++ /dev/null @@ -1,198 +0,0 @@ -/* Copyright 2019 The MLPerf Authors. All Rights Reserved. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -/// \file -/// \brief A minimal test framework. - -#ifndef MLPERF_LOADGEN_TESTS_LOADGEN_TEST_H_ -#define MLPERF_LOADGEN_TESTS_LOADGEN_TEST_H_ - -#include -#include -#include -#include -#include -#include - -#define REGISTER_TEST(name, ...) \ - static Test::StaticRegistrant test##name(#name, __VA_ARGS__); - -#define REGISTER_TEST_SCENARIO(name, scenario, test, ...) \ - static Test::StaticRegistrant t##name##scenario( \ - #name "_" #scenario, test, __VA_ARGS__, mlperf::TestScenario::scenario) - -#define REGISTER_TEST_ALL_SCENARIOS(name, test, ...) \ - REGISTER_TEST_SCENARIO(name, SingleStream, test, __VA_ARGS__); \ - REGISTER_TEST_SCENARIO(name, MultiStream, test, __VA_ARGS__); \ - REGISTER_TEST_SCENARIO(name, Server, test, __VA_ARGS__); \ - REGISTER_TEST_SCENARIO(name, Offline, test, __VA_ARGS__); - -#define FAIL_IF(exp) \ - [&]() { \ - const bool v = exp; \ - if (v) { \ - std::cerr << "\n ERROR: (" << __FILE__ << "@" << __LINE__ \ - << ") : " #exp; \ - Test::AddFailure(); \ - } \ - return v; \ - }() - -#define FAIL_MSG(...) \ - [&]() { \ - std::cerr << "\n Info: (" << __FILE__ << "@" << __LINE__ << ") : "; \ - Test::Log(__VA_ARGS__); \ - return true; \ - }() - -#define FAIL_EXP(exp) \ - [&]() { \ - std::cerr << "\n Info: (" << __FILE__ << "@" << __LINE__ << ") : "; \ - std::cerr << #exp << " is " << (exp); \ - return true; \ - }() - -#define BAD_TEST_MSG(...) \ - [&]() { \ - FAIL_MSG("The test isn't testing what it claims to test. "); \ - Test::Log(__VA_ARGS__); \ - return true; \ - }() - -#define ABORT_TEST \ - [&]() { \ - FAIL_MSG("ABORTING"); \ - throw std::logic_error("ABORT_TEST encountered."); \ - return false; \ - }(); - -/// \brief Testing utilities. -namespace testing { - -/// \brief Wraps a test class as a functor for easy registration. -/// Forwards registration args to a SetUpTest method. -/// \details Calls SetUpTest, RunTest, and EndTest. -template -struct TestProxy { - template - void operator()(Args&&... args) { - TestT test; - test.SetUpTest(std::forward(args)...); - test.RunTest(); - test.EndTest(); - } -}; - -/// \brief A collection of methods for registering and running tests. -class Test { - /// \brief Maps registered test names to a callback. - using TestMap = std::multimap>; - - /// \brief The registered tests. - /// \details Wraps a static local to avoid undefined initialization order - /// and guarantee it is initialized before the first test registers itself. - static TestMap& tests() { - static TestMap tests_; - return tests_; - } - - /// \brief The number of errors the current test has encountered. - static size_t& test_fails() { - static size_t test_fails_ = 0; - return test_fails_; - } - - public: - /// \brief Registers a test before main() starts during static initialization. - struct StaticRegistrant { - template - StaticRegistrant(Args&&... args) { - Test::Register(std::forward(args)...); - } - }; - - /// \brief Registers a test at runtime. - template - static void Register(const char* name, TestF test, Args&&... args) { - std::function test_closure = - std::bind(test, std::forward(args)...); - tests().insert({std::move(name), std::move(test_closure)}); - } - - /// \brief Runs all currently registered tests that match the given filter. - static int Run(std::function filter) { - // Determine which tests are enabled. - std::vector enabled_tests; - for (auto& test : tests()) { - if (filter(test.first)) { - enabled_tests.push_back(&test); - } - } - const size_t enabled = enabled_tests.size(); - std::cout << enabled << " of " << tests().size() << " tests enabled.\n"; - - // Run the tests. - std::vector failures; - for (size_t i = 0; i < enabled; i++) { - const char* name = enabled_tests[i]->first; - std::cout << "[" << (i + 1) << "/" << enabled << "] : " << name << " : "; - std::cout.flush(); - test_fails() = 0; - try { - enabled_tests[i]->second(); // Run the test. - } catch (std::exception& e) { - constexpr bool TestThrewException = true; - FAIL_IF(TestThrewException) && FAIL_EXP(e.what()); - } - if (test_fails() > 0) { - failures.push_back(name); - std::cerr << "\n FAILED: " << name << "\n"; - } else { - std::cout << "SUCCESS\n"; - } - } - - // Summarize. - if (enabled_tests.empty()) { - std::cerr << "Check your test filter.\n"; - } else if (failures.empty()) { - std::cout << "All " << enabled << " tests passed! \\o/\n"; - } else { - std::cout << failures.size() << " of " << enabled << " tests failed:\n"; - for (auto failed_test_name : failures) { - std::cout << " " << failed_test_name << "\n"; - } - } - return failures.size(); - } - - /// \brief Used by test macros to flag test failure. - static void AddFailure() { test_fails()++; } - - /// \brief Base case for the variadic version of Log. - static void Log() {} - - /// \brief Used by test macros to log an arbitrary list of args. - template - static void Log(T&& v, Args&&... args) { - std::cerr << v; - Log(std::forward(args)...); - } -}; - -} // namespace testing - -// The testing namespace exists for documentation purposes. -// Export the testing namespace for all files that define tests. -using namespace testing; - -#endif // MLPERF_LOADGEN_TESTS_LOADGEN_TEST_H_ diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/loadgen_test_main.cc b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/loadgen_test_main.cc deleted file mode 100644 index 3dc5afa80..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/loadgen_test_main.cc +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright 2019 The MLPerf Authors. All Rights Reserved. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -/// \file -/// \brief A main entry point a test binary can use if it just wants to execute -/// Test::Run on all statically registered tests. - -#include - -#include "loadgen_test.h" - -int main(int argc, char* argv[]) { - if (argc <= 1) { - std::cerr << "Usage: " << argv[0] << " \n"; - return -1; - } - std::regex include_regex(argc >= 2 ? argv[1] : ".*"); - std::regex exclude_regex(argc >= 3 ? std::regex(argv[2]) : std::regex()); - auto test_filter = [&](const char* test_name) { - return (std::regex_search(test_name, include_regex) && - !std::regex_search(test_name, exclude_regex)); - }; - return Test::Run(test_filter); -} diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/perftests_null_sut.cc b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/perftests_null_sut.cc deleted file mode 100644 index 56d562c3e..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/perftests_null_sut.cc +++ /dev/null @@ -1,230 +0,0 @@ -/* Copyright 2019 The MLPerf Authors. All Rights Reserved. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -/// \file -/// \brief Performance tests using a null backend. - -#include - -#include "../loadgen.h" -#include "../query_sample_library.h" -#include "../system_under_test.h" -#include "../test_settings.h" - -/// \brief Performance unit tests. -namespace perf_tests { - -/// \defgroup LoadgenTestsPerformance Test Coverage: Performance - -/// \brief A simple SUT implemenatation that immediately completes -/// issued queries sychronously ASAP. -class SystemUnderTestNull : public mlperf::SystemUnderTest { - public: - SystemUnderTestNull() = default; - ~SystemUnderTestNull() override = default; - const std::string& Name() override { return name_; } - void IssueQuery(const std::vector& samples) override { - std::vector responses; - responses.reserve(samples.size()); - for (auto s : samples) { - responses.push_back({s.id, 0, 0}); - } - mlperf::QuerySamplesComplete(responses.data(), responses.size()); - } - - void FlushQueries() override {} - - private: - std::string name_{"NullSUT"}; -}; - -/// \brief A stub implementation of QuerySampleLibrary. -class QuerySampleLibraryNull : public mlperf::QuerySampleLibrary { - public: - QuerySampleLibraryNull() = default; - ~QuerySampleLibraryNull() = default; - const std::string& Name() const override { return name_; } - - size_t TotalSampleCount() override { return 1024 * 1024; } - - size_t PerformanceSampleCount() override { return 1024; } - - void LoadSamplesToRam( - const std::vector& samples) override { - return; - } - - void UnloadSamplesFromRam( - const std::vector& samples) override { - return; - } - - private: - std::string name_{"NullQSL"}; -}; - -/// \brief Runs single stream traffic. -/// \ingroup LoadgenTestsPerformance -void TestSingleStream() { - SystemUnderTestNull null_sut; - QuerySampleLibraryNull null_qsl; - - mlperf::LogSettings log_settings; - log_settings.log_output.prefix_with_datetime = true; - - mlperf::TestSettings ts; - - mlperf::StartTest(&null_sut, &null_qsl, ts, log_settings); -} - -/// \brief A SUT implementation that completes queries asynchronously using -/// std::async. -class SystemUnderTestNullStdAsync : public mlperf::SystemUnderTest { - public: - SystemUnderTestNullStdAsync() { futures_.reserve(1000000); } - ~SystemUnderTestNullStdAsync() override = default; - const std::string& Name() const override { return name_; } - void IssueQuery(const std::vector& samples) override { - futures_.emplace_back(std::async(std::launch::async, [samples] { - std::vector responses; - responses.reserve(samples.size()); - for (auto s : samples) { - responses.push_back({s.id, 0, 0}); - } - mlperf::QuerySamplesComplete(responses.data(), responses.size()); - })); - } - - void FlushQueries() override {} - - private: - std::string name_{"NullStdAsync"}; - std::vector> futures_; -}; - -/// \brief Tests server traffic using SystemUnderTestNullStdAsync. -/// \ingroup LoadgenTestsPerformance -void TestServerStdAsync() { - SystemUnderTestNullStdAsync null_std_async_sut; - QuerySampleLibraryNull null_qsl; - - mlperf::LogSettings log_settings; - log_settings.log_output.prefix_with_datetime = true; - log_settings.log_output.copy_summary_to_stdout = true; - - mlperf::TestSettings ts; - ts.scenario = mlperf::TestScenario::Server; - ts.server_target_qps = 2000000; - ts.min_duration_ms = 100; - - mlperf::StartTest(&null_std_async_sut, &null_qsl, ts, log_settings); -} - -/// \brief A SUT implementation that completes queries asynchronously using -/// an explicitly managed thread pool. -class SystemUnderTestNullPool : public mlperf::SystemUnderTest { - public: - SystemUnderTestNullPool() { - samples_.reserve(kReserveSampleSize); - next_poll_time_ = std::chrono::high_resolution_clock::now() + poll_period_; - for (size_t i = 0; i < thread_count_; i++) { - threads_.emplace_back(&SystemUnderTestNullPool::WorkerThread, this); - } - } - - ~SystemUnderTestNullPool() override { - { - std::unique_lock lock(mutex_); - keep_workers_alive_ = false; - } - cv_.notify_all(); - for (auto& thread : threads_) { - thread.join(); - } - } - - const std::string& Name() const override { return name_; } - - void IssueQuery(const std::vector& samples) override { - std::unique_lock lock(mutex_); - samples_.insert(samples_.end(), samples.begin(), samples.end()); - } - - void FlushQueries() override {} - - private: - void WorkerThread() { - std::vector my_samples; - my_samples.reserve(kReserveSampleSize); - std::unique_lock lock(mutex_); - while (keep_workers_alive_) { - next_poll_time_ += poll_period_; - auto my_wakeup_time = next_poll_time_; - cv_.wait_until(lock, my_wakeup_time, - [&]() { return !keep_workers_alive_; }); - my_samples.swap(samples_); - lock.unlock(); - - std::vector responses; - responses.reserve(my_samples.size()); - for (auto s : my_samples) { - responses.push_back({s.id, 0, 0}); - } - mlperf::QuerySamplesComplete(responses.data(), responses.size()); - - lock.lock(); - my_samples.clear(); - } - } - - static constexpr size_t kReserveSampleSize = 1024 * 1024; - const std::string name_{"NullPool"}; - const size_t thread_count_ = 4; - const std::chrono::milliseconds poll_period_{1}; - std::chrono::high_resolution_clock::time_point next_poll_time_; - - std::mutex mutex_; - std::condition_variable cv_; - bool keep_workers_alive_ = true; - std::vector threads_; - - std::vector samples_; -}; - -/// \brief Tests server traffic using SystemUnderTestNullPool. -/// \ingroup LoadgenTestsPerformance -void TestServerPool() { - SystemUnderTestNullPool null_pool; - QuerySampleLibraryNull null_qsl; - - mlperf::LogSettings log_settings; - log_settings.log_output.prefix_with_datetime = true; - log_settings.log_output.copy_summary_to_stdout = true; - - mlperf::TestSettings ts; - ts.scenario = mlperf::TestScenario::Server; - ts.server_target_qps = 2000000; - ts.min_duration_ms = 100; - - mlperf::StartTest(&null_pool, &null_qsl, ts, log_settings); -} - -/// @} - -} // namespace perf_tests - -int main(int argc, char* argv[]) { - perf_tests::TestSingleStream(); - perf_tests::TestServerStdAsync(); - perf_tests::TestServerPool(); - return 0; -} diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/perftests_null_sut.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/perftests_null_sut.py deleted file mode 100644 index 115372e18..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tests/perftests_null_sut.py +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2019 The MLPerf Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -"""Python version of perftests_null_sut.cc. -""" - -from __future__ import print_function -from absl import app -import mlperf_loadgen - - -def load_samples_to_ram(query_samples): - del query_samples - return - - -def unload_samples_from_ram(query_samples): - del query_samples - return - - -def issue_query(query_samples): - responses = [] - for s in query_samples: - responses.append(mlperf_loadgen.QuerySampleResponse(s.id, 0, 0)) - mlperf_loadgen.QuerySamplesComplete(responses) - - -def flush_queries(): - pass - - -def main(argv): - del argv - settings = mlperf_loadgen.TestSettings() - settings.scenario = mlperf_loadgen.TestScenario.SingleStream - settings.mode = mlperf_loadgen.TestMode.PerformanceOnly - - sut = mlperf_loadgen.ConstructSUT(issue_query, flush_queries) - qsl = mlperf_loadgen.ConstructQSL( - 1024 * 1024, 1024, load_samples_to_ram, unload_samples_from_ram - ) - mlperf_loadgen.StartTest(sut, qsl, settings) - mlperf_loadgen.DestroyQSL(qsl) - mlperf_loadgen.DestroySUT(sut) - - -if __name__ == "__main__": - app.run(main) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tools/mlperf-trace.ipynb b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tools/mlperf-trace.ipynb deleted file mode 100644 index ab834d17a..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/tools/mlperf-trace.ipynb +++ /dev/null @@ -1,441 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Tool to extract usefull information from mlperf trace" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "%matplotlib inline\n", - "# Ignore warnings\n", - "import warnings\n", - "warnings.filterwarnings('ignore')\n", - "\n", - "import json\n", - "import os\n", - "import seaborn as sns\n", - "from operator import itemgetter\n", - "import pandas as pd\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "\n", - "figsize=(10, 5)\n", - "font=10\n", - "\n", - "plt.figure(dpi=600)\n", - "plt.rc('xtick', labelsize=font) \n", - "plt.rc('font', size=font)\n", - "sns.set(font_scale=1.4, style=\"whitegrid\");" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "def trace_to_df(fname):\n", - " with open(fname, \"r\") as f:\n", - " j = json.load(f)\n", - " if type(j) == dict:\n", - " j = j['traceEvents']\n", - " \n", - " result = []\n", - " for item in j:\n", - " name = item['name']\n", - " if name not in [\"Latency\", \"Sample\", \"QuerySamplesComplete\", \"IssueQuery\"]:\n", - " continue\n", - "\n", - " args = item.get('args')\n", - " d = {\"ts\": item['ts'], \"name\": name, \"dur\": item.get(\"dur\")}\n", - "\n", - " if name == \"Latency\":\n", - " d[\"issue_delay\"] = args[\"issue_delay\"]\n", - " d[\"issue_to_done\"] = args[\"issue_to_done\"] / 1e3\n", - " result.append(d)\n", - " elif name == \"Sample\":\n", - " if args:\n", - " d[\"issue_start_ns\"] = args[\"issue_start_ns\"]\n", - " d[\"complete_ns\"] = args[\"complete_ns\"]\n", - " d[\"issue_to_done\"] = (args[\"complete_ns\"] - args[\"issue_start_ns\"]) / 1e3\n", - " result.append(d)\n", - " elif name == \"QuerySamplesComplete\":\n", - " result.append(d)\n", - " elif name == \"IssueQuery\":\n", - " result.append(d)\n", - "\n", - " df = pd.DataFrame(result)\n", - " df = df.sort_values(by=[\"ts\"])\n", - " return df\n", - "\n", - "BINS = 10" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
tsdurissue_delayissue_to_doneissue_start_nscomplete_ns
count2.000000e+0410000.0000005.000000e+0310000.0000005.000000e+035.000000e+03
mean4.894584e+0717.7316827.001508e+046112.5544917.001508e+046.182570e+06
std2.839099e+0725.5786399.666462e+042254.0772359.666462e+042.263719e+06
min4.102560e+031.1520008.810000e+022754.9670008.810000e+022.780383e+06
25%2.463025e+073.9747505.806250e+044100.4730005.806250e+044.166623e+06
50%4.881766e+077.3640006.159800e+046089.8800006.159800e+046.155939e+06
75%7.373552e+0727.4410006.835175e+047337.2570006.835175e+047.408272e+06
max9.832065e+07508.5520006.522433e+0622234.1010006.522433e+062.414005e+07
\n", - "
" - ], - "text/plain": [ - " ts dur issue_delay issue_to_done \\\n", - "count 2.000000e+04 10000.000000 5.000000e+03 10000.000000 \n", - "mean 4.894584e+07 17.731682 7.001508e+04 6112.554491 \n", - "std 2.839099e+07 25.578639 9.666462e+04 2254.077235 \n", - "min 4.102560e+03 1.152000 8.810000e+02 2754.967000 \n", - "25% 2.463025e+07 3.974750 5.806250e+04 4100.473000 \n", - "50% 4.881766e+07 7.364000 6.159800e+04 6089.880000 \n", - "75% 7.373552e+07 27.441000 6.835175e+04 7337.257000 \n", - "max 9.832065e+07 508.552000 6.522433e+06 22234.101000 \n", - "\n", - " issue_start_ns complete_ns \n", - "count 5.000000e+03 5.000000e+03 \n", - "mean 7.001508e+04 6.182570e+06 \n", - "std 9.666462e+04 2.263719e+06 \n", - "min 8.810000e+02 2.780383e+06 \n", - "25% 5.806250e+04 4.166623e+06 \n", - "50% 6.159800e+04 6.155939e+06 \n", - "75% 6.835175e+04 7.408272e+06 \n", - "max 6.522433e+06 2.414005e+07 " - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df = trace_to_df('/tmp/mlperf_log_trace.json')\n", - "df.describe()" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAoIAAAFKCAYAAACJoz5RAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAAAstklEQVR4nO3deZxcVZ338Q8ECIQASVgNyjIIP8AoPkRGUfRBkSXgOiIMjAsoKIroIAgzKItsowODKCIoI4IzqICODgjigiwqKtCAEiA/MLI8EEFIAoJgB0ieP84tUhSVdHelu6vS9/N+vfp1U/eeunWqTi/fnHvOuSssWrQISZIk1c+K3a6AJEmSusMgKEmSVFMGQUmSpJoyCEqSJNWUQVCSJKmmDIKSJEk1tVK3KyBJdRIRmwB3A/tn5nndrc3z9ULdIuJdwDeAjTLz0W7UYbAiYhpwC/DKzJzZ5epIHTEISmNIROxH+SO6fWb+psvVGZSIWAc4Eng7sBHwJHAD8MXMvLybdRurIuKjwJM9GETHAScAZ/V6CATIzJkRcTlwPPAP3a6P1AkvDUvqmogISo/KIcCVwMeAfwPWAy6LiM91r3Zj2keB/drsvxdYDfivUa3NYm8BtgK+2qXX78TZwDsj4qXdrojUCXsEJXVFRKwMfBeYDLwhM69vOnYacAFwZET0ZebFo1y31TPzr6P5mp2qetFWysz+ZT1XZi4C/rbsterYB4AbM/OPXazDUP0UmE8J1p/pblWkoTMISmNcRKwPnATsSulpexS4CTg8M2+rymwLnAhsB6wBPARcC3woM5+KiB2Bq4A3ZubVTefehDZjyiJii+p8OwGrA3cAJ2Xmd5uq9i5gGnBMcwgEyMxnI+LDVZ0/C1xcnXfY69F0OX0n4B3A3sB61XPvBA7LzNNaPtNXAL8DPpqZZ7EEETEJOB14J7AI+F/gC23KXV297x1b9p8H7JiZm7S8z38FngA+AWwKvBm4OiIOq15rS2AicBflEvt/Np3zHmDj6t+Ne4zem5mbLOVz3IbSU7sDMI5y6f7ozPxFU5n9KJ/jjpTL/O8FJgA/oXwfPbykz6l6/qrAbsB/tDm2CPhsZh7Xsv8e4OrM3K96vBLwL8D7gJcAT1Wfwecz83+anjeY708iYi1KuHsXsCHwCHAN8KnMfAAgM5+u2u+dGAS1HPLSsDT2fRfYEzifckmwEUS2AIiIdSm9GpsB/065PHseJaStPtQXi4itgN8CL6/OdxgwF7g4It7TVPSt1fab7c6TmY9RgtNWEbHZCNaj4QxgW0poPiYz7wJ+DbQr+x5gAXDhUl5/har+76X0bn4GmEpph2X1XuBwSvA6FPhTtf9QYCZlzNqnKIH+nIg4qOm5/wzcD8yqzvPeat+S3sdWwC+A/wOcAhxXvY+fRcQb2jzldGAbSoA/i9LOXx7Ee5oOrALcOIiyS3Is5b1fA3y8+vcs4O8bBQb7fRERq1fnORT4OSV0f4USolsvA/dRvk8nL0Pdpa6wR1Aaw6oeqR0oPRinNh1qHnv3WmAKsGtmNv8RPrbDl/0iMAd4VWY+Ve07MyJ+AnwuIi6oLkFuDTyWmfcu5Vy/q7ZbA7NHqB4NT1B6355p2vdN4KyI2DozbweIiBWBfYDLMnPeUl7/bcAbgCMz89+r554F/GyI76OdjYHNM/NPLfu3yMwnmx6fUb3fwylj2cjMH0TEicAjmfnfg3itk4BVgelVOCYivkEJWKcBr2opPxfYufHZVp/XxyNirSrcL8mW1XZZLgu/Bbg8Mw9cSpnBfl98ihJo92oZmnBSFfKb/RFYgTK+8bplqL806uwRlMa2pyg9VztGxJQllGn8cX5LNW6vY9VrvBm4CFg9ItZpfAFXUC6vbVEVXwN4fIBTNo6vMYL1aDinJQRC6fHrp/SaNewIvJiBJ1TsDiyk9IoB5ZI3cOZQ3ssS/KBNCKQRAiNi5YiYUr3fq4DNqsucQ1KNP9wVuLQRAqvXeYTSazy9GnrQ7OstAfsXlMvJGw/wcmtX2/lDrWeTx4CXVZd+X2CI3xd7Are1G5/a8v6a67zOMtRd6gqDoDSGVRMIjqSMvXooIn4ZEUdFxEuail1DuXx8LDA3Ii6NiAOrS2ND9VJKz8hxwMMtX42xX+tV28cZOOA1jv95BOvR8IIex8ycD1wC7NvUC/QeYB5w2QB12Bh4MDNbw+6dg3sLS9W2dzQi3h4RN1L+AzCX8n5Prg4POQgC61LG+WWbY3dU201a9t/X8rgRkgZ72bS1t20ojqG8z4yI2yLitIho7rEcyvfFZpTL7EOpc2tAlHqeQVAa4zLzdGBzyqWux4CjgTuqiRdk5qLMfDfwasr4rnWArwG3RkTjj+KS/sCNa3nc+J3yBWDnJXw1/rjeDqwVERstpfqvqLaNy4UjUY+Gp2jvm5T1Dd9QTWh4F3BRZi5YSr2HarDvq+EFdY2IHYDvU9ZhPAjYg/I+G2NCR+v3/bNL2D9QwHuk2g5lnN3zPp/MvJYS4N4P3EyZNHJ9RBxRFenk+2IwGnV+ZKmlpB7kGEGpBjLzbkrIOz0iXkxZu+/TwNVNZa4HrgeOiYgZwOXAgZQxYo1enUktp2693NcIbM9k5kBj4S4F9qX8sT6x9WBErEmZfXpT03IiI1GPgVxB6ZF8L7A+sCaDW2fvXmDniFijpVew3WXL+cDftdk/0OXUZntSln7ZJTOfWwImIt7Ypuxge64epgTLaHOsMabvniHUcWkaPYybUkJcs/m0tHlErAK8qPUkVS/uN4FvRsRqlO/jz0bEfzC074vZlAlTg7Ep5TOdNcjyUs+wR1AawyJiQvXH8DmZeT8l2EyqykxuM/j9pmo7qdreS+npaZ0l+tGWc/+ZMibtwIjYsE191m16+D3gNuBfWi7fNcamnUXpaTmp6dBI1GOpqnGDF1CC1geBP2TmYCYEXE75HfuRptddETi4TdnZwJbN9aqWbHndYOtJ+VwW0fR7vZrF+oE2Zf/KIHreqjGNVwBvbZ65XY21ez9lzb+HhlDHpemjBNnWySdQPp/WNv8QLT2CEbF28+NqMsgsymSX1Yb4ffFdynjDd7cp1/rzMh2YVYVQablij6A0tm0B/DwiLqaErn7KJIatKDNJofxBPzgivk/5g7sasD8lWHwXylIu1TkOqdZ0m02Zodk6zg5K8PkV8PuIOKcqux7l0vPWVEtvVOuvvYuyNMcvI+JcShiYTOkp/D/Aic3rv41EPQbpm5RlRHahjC8bjEur1/+3an2+2yjrFLabtHMu8EngxxHx9aqeB1XPWXMIr/dJ4KcR8V/V6xwIPAhs0FL2RuCjEXEsZcziE5l56RLO+xnK+/5lRJxJCWsHUv6TsOcg6zagzFwQEVdQLs8e1XL4P4GzI+J7lKWOtqFMYmm9FHtHRFxLWefwkarcAcAPM/OJqsxgvy9OoQwD+HZE7EL53pwEzKCMRbwGnlsY/f+yfN0NRXqOPYLS2Pb/KL1Zr6f0rJ1CmRn5wcxsDI6/hnJJeC/K0hpHUcLDmzLzt03nOoSyLt5BlEu591FC5PNkZlJ6dS6hXPY9k9JjtxJlfGJr2W0o68ztTFmn7RRKCHx/Zj6v/EjVYyCZeQvw++rhYJZcITMXUpaQuQD4J8rn/6cl1PWOqo5rUZZkeRvlUvRNrWWX8npXV+eeQhkG8AHK2ohfalP8eBYHx29V5ZZ03jsoSxDdTJl49FnK98ebqzF5w+lc4FURsWnL/nOAz1N6Bf+Dcil2Z0rPZrPTKTO6j6R8T+1GWSppn0aBwX5fVHeWeUN1fDfK5/gxyhqMz82gpsxCnkKZRS0td1ZYtMhJTpJ6R0S8nLLkyL2UW88tbe25URMRNwALMnMol2s1BNWl85mU5WqO7HZ9BiMiLgEWZuY7ul0XqRP2CErqKZl5K2WSSADfryYFdFVEvJLSizQcdwXRElS9qEcDH6kWQ+9pETGNMtTCW8tpuWWPoCQtQfWHfjplfOCLgE1b7twhScs1ewQlacn2pNzPdzXgHw2BksYaewQlSZJqyuVjOtDX1zce2I4yA3BJq+hLkiT1gnGU4S03TJ8+vb/5gEGwM9tRZjVKkiQtL14P/LJ5h0GwM38C2GKLLVhlleGd0Dhz5kymTRvsXY00GmyT3mJ79Bbbo/fYJr2lF9pjwYIF3HnnnVDll2YGwc48C7DKKqswfvz4YT/5SJxTy8Y26S22R2+xPXqPbdJbeqg9XjCczVnDkiRJNWUQlCRJqimDoCRJUk0ZBCVJkmrKIChJklRTBkFJkqSaMghKkiTVlEFQkiSppgyCkiRJNeWdRXrY439dwJP9z3S7GsNiwviVWGP14b0dnyRJWjYGwR72ZP8zXHnDfd2uxrDYabuNDIKSJPUYLw1LkiTVlEFQkiSppgyCkiRJNWUQlCRJqimDoCRJUk0ZBCVJkmrKIChJklRTBkFJkqSaMghKkiTVlEFQkiSppgyCkiRJNWUQlCRJqimDoCRJUk0ZBCVJkmrKIChJklRTBkFJkqSaMghKkiTVlEFQkiSppgyCkiRJNWUQlCRJqimDoCRJUk0ZBCVJkmrKIChJklRTBkFJkqSaMghKkiTVlEFQkiSppgyCkiRJNWUQlCRJqimDoCRJUk0ZBCVJkmrKIChJklRTBkFJkqSaMghKkiTVlEFQkiSpplbqdgUaImIiMAvYENguM29sOvY+4ChgE2A2cHxmXtjy/JWB44H3A5OAG4BPZOYtLeU2AL4I7AYsAn4I/HNmPjIS70uSJKlX9VKP4HG0CaYRsSdwPvB9YAbwM+DbETGjpegXgIOBY4G3AwuAKyNiatO5VgKuAF4OvA84AHgtcElErDDM70eSJKmn9USPYERMAw4CPgl8teXwCcDFmfmv1eOrImIr4LPAj6rnb1g9/+OZeU617zfA3cA/A0dUz30XsA0wLTNvq8rNAX5FCZmXj8T7kyRJ6kW90iN4JvBl4M7mnRGxKbAl8J2W8t8CtouIdavHuwDjgOcuF2fm45TLvrs3PW934NZGCKzKXQfc21JOkiRpzOt6EIyI9wIvBU5sc3irant7y/5GkIumcg9l5tw25baIiBWbyrWeq1Fuy6HUW5IkaXnX1SAYEWsBpwBHZOYTbYpMrraPtuyfX22nNJVrLdMotzIwcRDlprTZL0mSNGZ1e4zgicBdmXlBl+vRkZkzZ47Iefv6+gBYbc11mTNnzoi8xmibO3cC99/9cLer0bFGm6g32B69xfboPbZJb+nl9uhaEIyIl1EmeOwcEZOq3Y2eu4kRsQaLe/4mAQ82Pb3RUziv2s6vyrSaDDwNPDGIcvPa7F+qadOmMX78+KE+ban6+vqYPn06AA/Ne5KpU58c1vN3y9prr8P6m2/U7Wp0pLlN1H22R2+xPXqPbdJbeqE9+vv7l9h51c1Lw5tTguhVlIA2H7i0OnYV8AvgjurxVi3P3braZrW9A1gvIlov724N3JmZC5vKtZ6rUW5WB+9BkiRpudXNIPhL4I0tX4dWxw4CDsjMuykBbe+W5+4D3JCZjWuNPwEWAns1ClQLVL+V5y8Jcznw8mr5mUa511AWqnbpGEmSVCtduzRc3cnj6uZ9EY1JwPQ13VnkGODCiJgN/JSyWPQuwB5N53ogIs4GPh8Rz1CWgzkcWAE4veklvgf8HvhuRPwr5f2fAvyaak1CSZKkuuj68jEDycyLgf2BPYEfA7sC+2Zma3A7FDiLMgHlEmA14M2ZOafpXM9Qbi03E/hv4BvAb4C3ZeaiEX4rkiRJPaXbs4afJzOvpvTite4/n3KbuaU992ngX6qvpZV7kBdeapYkSaqdnu8RlCRJ0sgwCEqSJNWUQVCSJKmmDIKSJEk1ZRCUJEmqKYOgJElSTRkEJUmSasogKEmSVFMGQUmSpJoyCEqSJNWUQVCSJKmmDIKSJEk1ZRCUJEmqKYOgJElSTRkEJUmSasogKEmSVFMGQUmSpJoyCEqSJNWUQVCSJKmmDIKSJEk1ZRCUJEmqKYOgJElSTRkEJUmSasogKEmSVFMGQUmSpJoyCEqSJNWUQVCSJKmmDIKSJEk1ZRCUJEmqKYOgJElSTRkEJUmSasogKEmSVFMGQUmSpJoyCEqSJNWUQVCSJKmmhhwEI2LXiFhhJCojSZKk0dNJj+CPgPsj4pSI2Ga4KyRJkqTR0UkQfAfwK+Bg4KaI+H1EHB4RU4e1ZpIkSRpRQw6CmXlJZu4FrA8cCDwMfA64NyJ+EhHviYgJw1xPSZIkDbOOJ4tk5uOZeW5m7gRsDBwFrAecDzwUEd+MiJ2GqZ6SJEkaZsM1a3gcsDIwHlgBeAp4M/DTiLg5IqYN0+tIkiRpmKzU6RMjYi1gL+A9wOuAZ4DLgH+ptguBtwFfAL4BbLeslZUkSdLwGXIQjIh3UMLf7sCqwA3AJ4BvZ+a8luI/iIh1gK+0Oc8/AJ8EtgQmAg8A3wdOyMzHmsrNAE4Ctq7KnJ6ZZ7Q53+GUCSwbALcBR2bmlS1l1gBOAfas6n4VcEhm3jOkD0GSJGkM6OTS8P8Arwa+CGydma/OzDPbhMCG3wMXtNk/BbgW+BCwW3W+DwAXNwpExPbAJcDNwAxKz+LpEXFQ84mqEHgycCawB3AXcFmb5W2+TemlPATYG5gKXOnkFkmSVEedXBreBbgyMxcNpnBmXg9c32b/f7bsujoi/gZ8NSKmZuYc4Bjgpsz8YFXmqojYCDg2Ir6WmQsjYjzwGUpP4akAEXENcCvwacrlayLi1ZSQuEdmXl7tuxWYDexHm15LSZKksayT5WN+NtgQ2IFHqu0qVcB7E3BhS5lvUS7/bls9fi2wFvCdpjo+C1wEzGi6C8ruwGPAFU3l7qOsibj78L4NSZKk3tfJLea+EBF3LeX4nRFxyhDONy4iVo2I6ZQewEuqMXubAasAt7c85bZqu2W13ara3tGm3ERgw6ZyszJzYZtyWyJJklQznVwa3oMX9tI1uxB4N/CpQZ5vLqVHD0pv3b7VvydX20dbys+vtlOayvVn5lNLKXd/Va71XI1yU9rsH9DMmTM7edqA+vr6AFhtzXWZM2fOiLzGaJs7dwL33/1wt6vRsUabqDfYHr3F9ug9tklv6eX26CQIvgS4ZynH763KDNaOwARgGmWs36URsXMH9Rp106ZNY/z48cN6zr6+PqZPnw7AQ/OeZOrUJ4f1/N2y9trrsP7mG3W7Gh1pbhN1n+3RW2yP3mOb9JZeaI/+/v4ldl51EgT/Amy6lON/R1lQelAy85bqn9dFRB9wI/BOFl8SntTylEZPYWOW8nxgfESsmpl/G6BcuyQyuamMJElSbXSyfMzPgQ9Xs3efJyI2AT5clenELZSFqF9Kmc27gMVjABu2rrazqm1jbGC7co9T1h5slIumySPN5WYhSZJUM50EwWMoPYkzI+KLEfGh6utLlDUDVwSO7rA+21fP/2Nm9lMC5V4tZfYBHgRuqh5fR5kNvHejQESMq553RdMM58spvYu7NpV7CbBDdUySJKlWhnxpODPviojXURZvPqTl8DWUO3XkQOeJiB8DV1Jm7f4NeCVlgsnvgR9UxY4Hro2IcyiLUr8OOBA4uDH7NzP7I+JE4OSIeJgSEA+gzDpuTDwhM38bEZcBX4+IwyiXuI8H7gPOG9qnIEmStPzr6F7DmXkbsGN1+7i/q3bPzsy5QzjN9ZRb1TXGG94DnA2clpkLqtf5dUS8nXLXkPcBc4BDM/PslvqcGhEAHwfWp4TLPTLzdy2vuQ9wKmXx6PGUW8y9OzPHxowMSZKkIegoCDZk5iMsXgR6qM89mkFcQq7uAjLgpdvqriKnDlDmccoYxg8PspqSJEljVkdBsBqDtyulN3Ay0DoBY1FmnrCMdZMkSdIIGnIQjIhXAd8DXswLA2DDIsAgKEmS1MM66RH8CrAa8A7gF5n56HBWSJIkSaOjkyD4CuDTmXnpcFdGkiRJo6eTdQTvZ8mXhCVJkrSc6CQIfg44MCLWHO7KSJIkafR0cml4CvBX4A8R8V3g/wHPtpRZlJmnLGvlJEmSNHI6CYKfa/r3QUsoswgwCEqSJPWwToLgpgMXkSRJUq/r5F7D945ERSRJkjS6Or7FXERsDuwIrAdckJn3RMQqwAbAg437BUuSJKk3dXJnkRWBs4EPUpaRWQT8GrgHWAW4FTge+I9hq6UkSZKGXSfLxxwFfAA4GtiepjUFM/MJyu3n/mFYaidJkqQR00kQ3B84NzNPBv7Q5vitwObLVCtJkiSNuE6C4IuB65dy/Clgjc6qI0mSpNHSSRB8ENh4KcenA84sliRJ6nGdBMHvAR+pZg03LAKIiBnA+4CLhqFukiRJGkGdBMHjgPuAm4ELKCHwqIj4DfBD4HfAvw1XBSVJkjQyhhwEM/MvwGuBk4H1gb8BOwATKSHxDZn51DDWUZIkSSOgowWlM/NvlCB48vBWR5IkSaOlk0vDkiRJGgM6ubPIuYMotigzP9hBfSRJkjRKOrk0/CaqWcJNxgEvqrYPA39dxnpJkiRphA05CGbmJu32R8TKwIeBfwZ2XqZaSZIkacQN2xjBzHw6M78M/AT48nCdV5IkSSNjJCaL/A54wwicV5IkScNoJILgzsCTI3BeSZIkDaNOZg0fs4RDkyg9gdsCn1uGOkmSJGkUdDJr+Lgl7J8PzAYOAs7ptEKSJEkaHZ3MGnYRakmSpDHAUCdJklRTnYwR3KiTF8rM+zp5niRJkkZGJ2ME7+GFdxYZjHEdPEeSJEkjpJMgeADwceAlwLeAO6v9AewD3Ad8CVg4HBWUJEnSyOgkCL4IGA+8NDPnNx+IiGOBXwEbZOa/DUP9JEmSNEI6mSxyEPC11hAIkJlzKUvHfGRZKyZJkqSR1UkQXBuYuJTjq1dlJEmS1MM6CYK/AT4REdNbD0TEq4BPAL9d1opJkiRpZHUyRvBjwNXA9RFxA3BXtX9zYDtgHnDIsNROkiRJI2bIPYKZeTvwcsrM4EnAntXXJOCLwMsz87bhq6IkSZJGQic9gmTmQ8Ch1ZckSZKWQx0FwYaI2BxYD5iZmY8N8bnvBv4JmA5MAWYDZwFfzcyFTeVmACcBWwMPAKdn5hltznc4cDCwAXAbcGRmXtlSZg3gFEoP5qrAVcAhmXnPUOouSZI0FnR0r+GI2Dci7gNmAddSwhwRsU5E3BkRew3iNIcB/cCngLcAP6Bcbv580+tsD1wC3AzMAL4BnB4RB7XU53DgZOBMYA/KuMXLImKbltf8NvA2yhjGvYGpwJURMWHQb16SJGmM6ORew+8C/hv4KXA6cGrjWGY+EhF3AO8DLhrgVG/NzIebHl8VEROBj0XEZzKzHzgGuCkzP9hUZiPg2Ij4WmYujIjxwGcoPYWnVnW8BrgV+DSwV7Xv1ZSQuEdmXl7tu5XSE7kf8JWhfhaSJEnLs056BD8N/CwzdwXOb3P8t0BrT9wLtITAhpspl2ynVAHvTcCFLWW+Rbn8u231+LXAWsB3ms79LCWIzoiIFarduwOPAVc0lbuPcieU3QeqryRJ0ljTSRDcCvj+Uo7/GVi3s+rwesryM38GNgNWAW5vKdOYkbxlU30A7mhTbiKwYVO5Wc3jD5vKbYkkSVLNdDJZ5K8s/c4imwGPDPWk1WLU+wOfzcxnI2JydejRlqKNW9tNqbaTgf7MfGop5e6vyrWeq1FuSpv9A5o5c2YnTxtQX18fAKutuS5z5swZkdcYbXPnTuD+u9t1Ai8fGm2i3mB79Bbbo/fYJr2ll9ujkyD4c2C/iPhi64GImAocCPzvUE4YERsA3wOup2mySK+bNm0a48ePH9Zz9vX1MX16uWnLQ/OeZOrUJ4f1/N2y9trrsP7mG3W7Gh1pbhN1n+3RW2yP3mOb9JZeaI/+/v4ldl51OkbwRcCNwEeBRcDuEfE5ygSNhcBnB3uyiFgL+BHwJPC2zHy6OtTo0ZvU8pRGT+G8pnLjI2LVQZRrPVej3Lw2+yVJksa0Tu4schfwOuBB4DhgBeCTwBHALcAO1SSMAVXh7RLKWoS7ZebcpsOzgQUsHgPYsHW1nVVtG2MD25V7nLL2YKNcNE0eaS43C0mSpJoZUhCMiHHV8i0PZeYuwDrAq4HtgfUzc6fMvHOQ51qJMrP3FcCMzLy3+Xi1fMzPqZZ/abIPJYTeVD2+jjIbeO/melbPuyIzF1W7L6f0CO7aVO4lwA7VMUmSpFoZ6hjBFSk9dUcCp2XmfOCGDl/7TOCtlJ7ECRHxmqZjt2fmX4DjgWsj4hzgAkpP5IHAwY3Zv5nZHxEnAidHxMOUgHgAZdLKvo0TZuZvI+Iy4OsRcRjQOP99wHkdvgdJkqTl1pB6BKvxe3Mo4wKXVaNn7t+BX7d8bVu93q+BtwPbAT+mBLxDM/PslnqdChwFfJwy3nBLysLRv2t5zX2AH1IWj76Y0rP45swcGzMyJEmShqCTWcPfoMwaPisz/9bpC2fmJoMsdzmDuHRbhcFTByjzOPDh6kuSJKnWOgmCdwLjgFkRcT7wR6B1DT8yc6BbzEmSJKmLOgmC/93076OXUGYRA99rWJIkSV00qCAYEV8Czs/MPuCN1e6JlJ7AZ0eobpIkSRpBg+0R/BjwG6AvM6+JiLUp9wPeOTOvGbHaSZIkacR0cmeRhtaFmSVJkrQcWZYgKEmSpOWYQVCSJKmmhjJr+O8i4u+rf69VbbeMiCfaFc7M65epZpIkSRpRQwmCn62+mp3RptwKlOVjxnVaKUmSJI28wQbB/Ue0FpIkSRp1gwqCmXn+SFdEkiRJo8vJIpIkSTVlEJQkSaopg6AkSVJNGQQlSZJqyiAoSZJUUwZBSZKkmjIISpIk1ZRBUJIkqaYMgpIkSTVlEJQkSaopg6AkSVJNGQQlSZJqyiAoSZJUUwZBSZKkmjIISpIk1ZRBUJIkqaYMgpIkSTVlEJQkSaopg6AkSVJNGQQlSZJqyiAoSZJUUwZBSZKkmjIISpIk1ZRBUJIkqaYMgpIkSTVlEJQkSaopg6AkSVJNGQQlSZJqyiAoSZJUUwZBSZKkmjIISpIk1dRK3XzxiHgpcDjwGmAaMCszp7UpNwM4CdgaeAA4PTPPaFPucOBgYAPgNuDIzLyypcwawCnAnsCqwFXAIZl5z/C9M0mSpN7X7R7BlwF7AH8Abm9XICK2By4BbgZmAN8ATo+Ig1rKHQ6cDJxZnfMu4LKI2KbllN8G3gYcAuwNTAWujIgJw/SeJEmSlgtd7REELs3M/wWIiPOAV7UpcwxwU2Z+sHp8VURsBBwbEV/LzIURMR74DKWn8NTqfNcAtwKfBvaq9r2aEhL3yMzLq323ArOB/YCvjMi7lCRJ6kFd7RHMzIVLO14FvDcBF7Yc+hbl8u+21ePXAmsB32k697PARcCMiFih2r078BhwRVO5+4BfVcckSZJqo9uXhgeyGbAKL7xsfFu13bLablVt72hTbiKwYVO5WW0C6G1N55IkSaqFbl8aHsjkavtoy/751XZKU7n+zHxqKeXur8q1nqtRbkqb/Us1c+bMoT5lUPr6+gBYbc11mTNnzoi8xmibO3cC99/9cLer0bFGm6g32B69xfboPbZJb+nl9uj1INjTpk2bxvjx44f1nH19fUyfPh2Ah+Y9ydSpTw7r+btl7bXXYf3NN+p2NTrS3CbqPtujt9gevcc26S290B79/f1L7Lzq9UvDjR69SS37Gz2F85rKjY+IVQdRrvVcjXLz2uyXJEkas3o9CM4GFrB4DGDD1tV2VrVtjA1sV+5xytqDjXLRNHmkudwsJEmSaqSng2Bm9gM/p1r+pck+wIPATdXj6yizgfduFIiIcdXzrsjMRdXuyyk9grs2lXsJsEN1TJIkqTa6fWeRCSxetmVjYM2I2LN6fENm3gscD1wbEecAFwCvAw4EDm7M/s3M/og4ETg5Ih6mBMQDKLOO9228Xmb+NiIuA74eEYcBf6nOfx9w3oi+WUmSpB7T7cki6wEXt+xrPN4fOC8zfx0Rb6fcNeR9wBzg0Mw8u/lJmXlqRAB8HFifsiTMHpn5u5bz7wOcSlk8ejzlFnPvzsyxMStDkiRpkLoaBKv7+7aO12tX7nIGcem2uqvIqQOUeRz4cPUlSZJUWz09RlCSJEkjxyAoSZJUUwZBSZKkmjIISpIk1ZRBUJIkqaYMgpIkSTVlEJQkSaopg6AkSVJNGQQlSZJqyiAoSZJUUwZBSZKkmjIISpIk1ZRBUJIkqaYMgpIkSTVlEJQkSaopg6AkSVJNGQQlSZJqyiAoSZJUUwZBSZKkmjIISpIk1ZRBUJIkqaYMgpIkSTVlEJQkSaopg6AkSVJNGQQlSZJqyiAoSZJUUwZBSZKkmjIISpIk1ZRBUJIkqaYMgpIkSTVlEJQkSaopg6AkSVJNGQQlSZJqyiAoSZJUUwZBSZKkmjIISpIk1ZRBUJIkqaZW6nYFVA8LFy7ioXlPdrsaHVltzXWfV/cJ41dijdVX6WKNJEkaHgZBjYr+p5/lut/P6XY1OjJnzhymTl0cBHfabiODoCRpTPDSsCRJUk0ZBCVJkmqqdpeGI2Jz4AxgB+Ap4DvAkZm5fA5gkyRJ6lCtgmBETAKuAu4F9gTWA04D1gX+sXs1kyRJGn21CoLAh4HJwCsz8xGAiHgGuCAiTsjM27paO0mSpFFUtzGCuwNXNkJg5XtAPzCjO1WSJEnqjrr1CG4FnNu8IzP7I2I2sOUQzjMOYMGCBcNYtcX6+/sBeObpBay04sIReY3R9uwzTy+372XVlVd4Xt0XLOjngYdGpu1H07gVV+DZhYu6XY0hGz9xCg889Ojz9q22ykqsPmHl7lRIz/3OUu+wTXpLt9ujKa+Maz1WtyA4GXi0zf75wJQhnOdFAHfeeecwVOmFZs6c+dy/NxtKrXrYo3++Z7l9L5tNWZMyr6j48wN/7F5lBMCDT8zrdhXUpPl3lnqDbdJbeqg9XgTMbt5RtyA4XG4AXg/8CXi2y3WRJElamnGUEHhD64G6BcH5wKQ2+ycDswZ7kunTp/cDvxymOkmSJI202e121m2yyB2UcYLPiYjxwGYMIQhKkiSNBXULgpcDO0XE2k373gmMr45JkiTVxgqLFi1/swY7VS0oPRO4BziBxQtKX5mZLigtSZJqpVY9gpn5KPAm4Angf4AvABcCH+hitSRJkrqiVj2CkiRJWqxWPYKSJElazCAoSZJUUwZBSZKkmqrbgtI9KyI2B84AdqDcz+w7wJGZ+WRXKzbGRMRLgcOB1wDTgFmZOa1NuRnAScDWwAPA6Zl5RptyhwMHAxsAt1Ha7MqRewdjR0S8G/gnYDrlFo+zgbOAr2bmwqZytsUoiYh/AD5Juff6RMrn/X3ghMx8rKmcbdIFETGRsubthsB2mXlj07H3AUcBm1B+lo7PzAtbnr8ycDzwfsrNFW4APpGZt4xC9Zd7EbEf8I02h87MzI81lVuufj7sEewB1bI2VwFrAHsChwH7AOd2sVpj1cuAPYA/ALe3KxAR2wOXADcDMyg/+KdHxEEt5Q4HTgbOrM55F3BZRGwzYrUfWw4D+oFPAW8BfgB8Cfh8o4BtMeqmANcCHwJ2A75IWVXh4kYB26SrjqNNB05E7AmcTwntM4CfAd+uAkmzL1CCx7HA24EFwJURMXUE6zwW7QZs3/R1auPA8vjz4azhHhARRwLHABtn5iPVvn2BC4BpmXlbN+s3lkTEio3epog4D3hVa49gRPwImJKZr27a9zXgrcCGmbmwuiPNQ8DXMvOIqsw44FZgZmbuNSpvaDkWEetm5sMt+04DPgJMysx+26L7IuJDwFcpn/cc26Q7ImIa8BtKj+1XaeoRjIg7gFubP9eI+Anl5+jvq8cbAvcCH8/Mr1T71gDuBs5ttJOWrKlHcN3G3+o2ZZa7nw97BHvD7pRFrZu/sb5H6S1p/R+dlkHzJcd2qh/QN1HWl2z2LUr3/bbV49cCa1Eu4TfO/SxwETAjIlYYrjqPVa0hsHIzsCowxbboGY3fS6vYJl11JvBl4M7mnRGxKeVS/ndayn8L2C4i1q0e7wKMo6ntMvNx4IeUv0FaRsvrz4dBsDdsRctlyszsp4zz2LIrNaqvzYBVeOFl40avbKM9GvesvqNNuYmUMTwautcD84A/Y1t0TUSMi4hVI2I65WrFJZl5D7ZJV0TEe4GXAie2Odz4rJfUJtFU7qHMnNum3BYRYR4YvJkR8WxE3B0Rx0ZE43L9cvnzYcP3hsnAo232z6eM2dHomVxtH23ZP7/aTmkq15+ZTw1QToMUEa8C9ge+UP3v2LbonrmUSWs3An8C9q322yajLCLWAk4BjsjMJ9oUGUqbtJZplFuZEkC0dH+ijK/cjzJO8PvA0cB/VseXy58PZw1L6rqI2IAyHOJ6miaLqGt2BCZQZtZ/Brg0Inbuao3q60Tgrsy8oNsVqbvM/DHw46ZdP42Ix4DjIuKELlVrmdkj2BvmU6byt5pMuUym0dP4H9mklv2N/+nNayo3PiJWHaCcBlD1ePwIeBJ4W2Y+XR2yLbokM2/JzOsy82vAO4E3VlvbZBRFxMuAg4CjI2JStcJEo+duYjXZYyht0lqmUe5poF1vowZ2UbXdluX058Mg2BvuYPGYAeC5QaebUdaM0uiZTVlSYauW/VtX20Z7NMZ2tCv3OGXtKA2g+kV4CbAesFvL+CXbojfcAiykjFGzTUbX5pQrd1dRwsN84NLq2FXAL1j6Zw2Q1fYOYL2IaL3suDVw50AT6TQoy+XPh0GwN1wO7BQRazfteycwvjqmUVJN0vk50Dp9fx/gQeCm6vF1wGPA3o0C1fT/vYArMtN1mQZQDbC+CHgFMCMz720+blv0jO0pfyv+aJuMul9SemObvw6tjh0EHJCZd1MCxt4tz90HuKFpdv5PKIG+eYmZiZRlTfw707l/BBYBfcvrz4djBHvDV4FDgP+txhmsB5wGXJiZbRc9VmciYgKLl0rYGFizWowVyi/Neykr718bEedQ1nJ8HXAgcHDjf83VGncnAidHxMOUH/ADKL24+6LBOJPyR+gIYEJEvKbp2O2Z+Rdsi1EVET8GrqTMXvwb8ErKgt+/pyz4DbbJqKmWFLu6eV9EYxIwfU13FjkGuDAiZgM/pSwWvQtloeLGuR6IiLOBz0fEM5Q1BQ8HVgBOH7l3MXZUPx8/B2ZSQvUM4KPA1zPzj1Wx5e7nwwWle0REbEG5q8LrWXyLuSO8xdzwiohNKAuotrN/Zp5Xldudsur7VsAcykzWL7U53+GUEL8+5Y/nEd5Ca3Ai4h5KGG/njZl5dVXOthgl1X9E3w5sWu26hzKJ57QqmDfK2SZdEhE7Ui4Lt95i7v288BZz32l57srACZRZr2ux+BZzN49G3Zd3EXE6Jfy9mNKRdhfVnUOqlQ4a5Zarnw+DoCRJUk05RlCSJKmmDIKSJEk1ZRCUJEmqKYOgJElSTRkEJUmSasogKEmSVFMGQUmSpJoyCEqSJNXU/we2dyrPLzUr8gAAAABJRU5ErkJggg==\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAApsAAAFKCAYAAABSGJRzAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAAA3OklEQVR4nO3debgcVZn48W8EuRDZQtiMIy4sr2DUGQIjICiKigFFHFnEUQSF0ZHFBRQXREDEhYyiAu4KjiCL/lQQxAUQHRDBwAAB8oIIZDAjsoRFYS6Y5PfHqSZFc29yb9+u23f5fp4nT6er3q46dbq679unzjk1ZenSpUiSJElNeEqvCyBJkqSJy2RTkiRJjTHZlCRJUmNMNiVJktQYk01JkiQ1xmRTkiRJjTHZlMagiDg6IsbUvGQRsTQivtLrckgjERFTIuLaiPhEr8syFBFxTkSc3etySCOxcq8LIE0Ww0ge92+0IGNEREwFPgj8KjN/1ePiNCIidgQuAfbJzDN7XJwhiYinAe8H9gI2Bh4DrgW+BpyemWPqR1AH9gE2Ab7Q64IM0aeA30fEizLz2l4XRuqEyaY0et7a9vzfgG2At7ctvxz4LvDp0ShUD00FPl79/1c9LIcqEbEBcBGwOfA94EvAasC/AP8JzI6It2bmkt6VcsQ+AHw/M+/pdUGGIjOvjojfA4fz5O8QaVww2ZRGSWZ+t/48Il4J/HP78pq/N18q6QlOoySab8jMc2vLvxARJ1ASnv8GThjNQkXEKsCSzBzRZyIi/gn4R+DIbpRrFJ0FHBsRB2Xmg70ujDRcJpvSGBQRRwMfz8wptWW3A/MpLZ5zgOcDtwKHZubFEbE7cCywGXAjcGBmzm3b7mbAccBOwNOAm4BPZub3h1G2vSktks8FEjgiMy9si1mritkD2BC4E/g2cHxmLo6IZwO3VeEfj4hWC+dpwH8A1wF7ZOYPqu1Fdex/yMxNa/v5T+Clmfms2rKtgWOAlwCrAHOBj2XmJW1lfDrwCeC1wDTgj8AXM/PLtZgdKZfB3ww8B3g3sC5wGfDOzPzDUOutts3VgaOBNwIzgAeBG4CjMvPXVcwmwPHADsA6wL3Ab4GDM/N/a/W3f2ae2rb9pcAxmXn0MI91G2Bn4FttiWbLh4HXAx+KiJMy85GGyrEjpc7fAmxKafl/BvBPEXEF8I3MPLRtX9OB/wU+n5lHDFD2lt2BxcDFba8/mrbPW7V8P8p5+5zMvL1atiXlM7Q1sAZwF/Br4N8y85EqZgpwMOXqxaaU9/g8ymflnrZ9vIpSt1sBU4CbgS9n5jdqYb+gfOZ3Bs5ZzvFJY5IDhKTx5bmUy5vnAx8C1gbOjYg3A18EzgCOquLOiYiVWi+MiM2B3wEvAD4LHEZJYs6JiLcMcf8vAb4MnA18FFgVOC8itq/tZzVKsrAfpTvAwZQ/7kcDX63C7gb+vfr/DymXB99arZ8HLAJeWtvvS4ElwCZVwtKyA+UPfWvfLwN+Q0nQjgWOAPqAn1dJTCtufeAK4DXAKcB7qv2eEhEDtXp9kHIpeQ6lD902wOmD1tLyfRk4hHLc7wY+Q6mPF1VleyrwM2B74OQq5hRgA0pyOizDONbXVY/fGWg7VaviGZS63a7BcrR8hJKQf4HyPi4EfgTsHRHtDSV7A08drOw12wE3tpLCDo5hPUritzHlM3QwcCowk/LjreXLwOcon7f3UPq77gFcEhGr1rb3Vsp7vUG1vQ8CVwK7tu36RuARyudPGnds2ZTGl00pLXm/AYiImyh/rL4FbJ6Zt1XL76ckbi8Hflm99guUP9hb1f7YnhwRPwc+HRFDGfwxE9guM39b7edU4BZKa2sr4Xwf8Dxgy8ycXy37WkTcBhwXESdkZkbE9yl/lK8boIvBZTwx2dwB+CmwY7X8rIh4JvAs4JPVa6ZUx/xfwKtax1KNoL+G0lLYSpKOoyShL8jMu6tlX4mIrwMfqVru7q/tf1XgRZn5aLXNRZRLyzMzc94K6qzda4GvZ+b7B1m/BeXHwp5tLc7HDXM/9dcN5Vi3qNYtbxBKa90WlL6dTZSjZQ3KOf231oKI+A5lgM+rgQtqsW8BrsnMG1ZQhudRWro7tR0l2d45M39fW95qmScitgPeCbwtM79TW34h5YfQvpTPw5rAScDVwA71BLg6lx+XmX+PiP9h2XskjSu2bErjy82tRLPyu+rxV61Es235cwEiYh3glZQWyadFxLqtf8CFlMuUmw1h/79vJZoAmXkvpbXrJRExrVq8FyXhu6dtP62kd8ch7Oc3wAury/FQEsyLKS1jrSR0h1oslJbBqMozvbbfNSmtUS+OiKnVH/I9KK3DS9vK+HPKgJgXt5XnO61Es22fzx3CsbR7oCrLMwZZ3+qTt3M1MrxjwzzWNarHh5azyda6NZYTM9JytHynnmhWfkG5XP74QJmIeC6wLWUA04pMp7Sad+qB6vG1VQv0QPYC/gpc2Hac8ymX3F9exb2acm5+ur2ldZAffYsoXTikcceWTWl8WVB/kpkPlO6M/E9bXOuPYisB3ITSH+zo6t9A1qf0wVyeWwZYdnP1+CzKH8TNKInf3QPEtvazIr+h/BjePiKuq7b9a2B1YM8qZgfgL7XW01ay/M3lbHc60E+pl7fz5JkABivjgrbnrYRlGsP3AUrf1AURcQ0l2f/PzEyAzLwtIj5HmX7oLVUr73nAd6vkfjjWY+jHWk8k7x8ktpVk/qXBcrTc2h5Q9ff9LnBQRKyRmQ9RWjUXU7qXDMWUFYcM6lLg+5SWzPdHxKXAucAZtcR4M8p5etcg22gd58bV41BbxqcA433aKU1SJpvS+LJ4mMtbf1hbVzE+zxMvP9YN93LwYJ5CaYX81CDr/ziEbfye0kftpZR+qQ9RLoWvARxdtdTuQGlBre8XSl/WwS6V3l1tD0py8q1B4tovx66ofocsM8+JiN9QBtu8GjgU+GBE7JeZZ1Qxh0XEt4Ddqpj/AI6MiJdl5o0MknTU++hWWnUylGO9kTKA5oXU+sG2eWH12HoPmyhHy2D9Kr9DSdj/hZK0/yvwi8z88yDxdfcw8A+EwZK4JxxH1eK4Z0T8M6U7xKso/TE/HBHbZOZfKMd6L/CmQbbZacvqNJYNqpPGFZNNaXJoJQd/z8xfLjdy+TYdYFmrRfGO6vFWYI0h7GfQVprMfKwaefxSYC3g8qpV6wrKlFCvp/Rf+3rtZa2WsIeWt++IuJuSvK48wrroWJUYfRX4akSsTekecAylC0Ar5gZKAvapiHghJYF+H3AgyxKWtds2/ay258M51vMog3L2ZYBks0og38yy0dc0VI7lysx5EXE18Naqz/JmlLobipsoswq0WwQQEWu39RttP45WGa6kDOQ5KiJmU37AHUjpP3wrJQm9IjP/upyytM7XmZRL7IOqBkQ9k8F/KEpjmn02pUmganG5BDhwoL6C1SjbodgqIratvW46JQG5PDNbicdZwNYRscsA+1kjIvqqpw9Xj4Ndiv4NMIvyh/vX1XE8Qmn1PILSqlhPiuYCf6Bc3nxSn8LWMWbmYsql0N0j4kWDxTUhIlaq9UOlKs/9lBartauYNQcYbX0TpaVv7eo1D1Ja6V7aFvfutm0P+Vgz8wpK/8n9I6J9NDSURGoz4DOt+S6bKMcQnUbp+/hBShL7wyG+7jJgi2rGhLpW4vf4cVT9Zd/WVs5p7YN3KAN8YFnCfRblb+tR7Tuv3v/W+f5zSv/cD7WXZ4B9bEEZpHb5wIcljW22bEqTx79T/theV40AvpXSf+zFlD9mmwxhG/OAn0TElyh/5P+Ncmn7w7WYEyjT6Pw4Ik6jJIGrUVpw9qRMvXR7NU/jDcCbIuJmyqXH2zKzNbjpNyybxqmeVP6akmw+SG3kdGYuiYh3UPpA3lhdhr6TMl3QyyjJaWtwxocoA5V+W9XFDZSk9x+BN1D+sDdhDeBPEfGDquwPUqazeQ1lZDLAKyizBHyf0od2CmVqnzUoiUzLNyiJyjcoCfhLGXiQ13COdV9KF4hzI+IMynuwKuWS9csog3Y+37b9JsqxIt+jTEP1RuDUYUxl9GNKK+grKIOVWn5O6Zf7zSiT1y+m9C29G9ioFvc2Sn/RH1I+P6tRbi/bSqbJzF9HxMnAB6oW6Z9R+glvQhkkdVRV5gcj4j2UbgW/r+r7Xsr8uc+g1HnLqyg/Nn42xOOUxhRbNqVJohqAshVlQMO+LJvDcWXgY0PczGXVa/amTCXUD+zemoy82s8jlKTiM5TE40TK5dnNKRN61/vWvQO4ndIn8Xssm3sTyiTmfwf+j3LJsqU1Evyy9tsmVuXYhnJZ+t2UBO7twH1VeVpxf6Ek2d+g9FM8iXKJekPK/KNNeZhS7y+g1PmJlPfk8Gr/UJLQnwK7UBKqT1ASzt3bpkI6ljIYag/KHI0rAbPbdzicY83Mu6rYY4B/okyX9QVKovnxzHxCS19T5ViRauqkn1ZPhzIKvfW6ayktkXu1LX+MkvDeSqnvQ6tyntS2iUsp5+JelHr5COV8fkXtRxKZeTDl3F6H0iL8aUrf27OpTShfTYT/Wsr5+RFK/W1L6dJQtxfww8x8AGkcmrJ0qYPbJEkDq7pdXE5pnNg2M+/scZEAiIhzKD8sntX+o2MFr9uH0tf3WR2M7h91Ue5Y9HtgVmZe0+vySJ2wZVOSNKjM/BPlMv9UytyRa/e2RI/fjWg3ypRRQ040K2dSWjDf2+1yNeTDwPdNNDWe2bIpSRoXIuI5lD6ub6dcbt50rLS0ShqcA4QkSePFy4BvU25isJ+JpjQ+2LIpSZKkxtiy2YG5c+f2AVtT7tE72J1FJEmSxoKVgKcDV82aNat/tHdustmZrVk2/YokSdJ40H6b31FhstmZ/wXYbLPNWGWVVRrbybx585g5c2Zj2x8vrIfCeiish8J6KKyHwnoorIeivR4effRRbr75Zqjyl9FmstmZxQCrrLIKfX19K4odkaa3P15YD4X1UFgPhfVQWA+F9VBYD8Ug9dCTrn/OsylJkqTGmGxKkiSpMSabkiRJaozJpiRJkhpjsilJkqTGmGxKkiSpMSabkiRJaozJpiRJkhpjsilJkqTGeAehMWz1tdblrvse7nUxumJq38qs8bTmbu0pSZLGJpPNMWzx0ilcdNWCXhejK3baeiOTTUmSJiEvo0uSJKkxJpuSJElqjMmmJEmSGmOyKUmSpMaYbEqSJKkxJpuSJElqjMmmJEmSGmOyKUmSpMaYbEqSJKkxJpuSJElqjMmmJEmSGmOyKUmSpMaYbEqSJKkxJpuSJElqjMmmJEmSGrNyr3YcEXsC/wrMAtYBbgW+DHw1M5dUMacCbxvg5Xtm5vfbtnc4cBCwIXADcERmXtQWswZwArAHsCpwCXBIZt7etQOTJEnS43rZsnkY0A98AHgt8CPgi8Bn2uL+CGzb9u/iekCVaB4PnAzsCtwCnB8RL2rb1veA3YBDgL2BGcBFETG1WwclSZKkZXrWsgm8LjPvrj2/JCJWBw6OiCMzs79a/khmXjHYRiKiDzgSODEz51TLLgWuBz4K7FUtezElEd01My+oll1PaVHdDzilmwcnSZKkHrZstiWaLddQLm+vM4xNbQesBZxZ2/Zi4GxgdkRMqRbvAjwAXFiLWwBcVq2TJElSl/WyZXMgOwD3AX+pLds4Iu4HngbMAz6dmWfV1m9ePd7Utq0bgNWBZwB3VnHzW/1B2+J27krpJUmS9ARjJtmMiK2A/YFjqpZJKC2dV1ESwrWAA4AzI2K1zDy1ipkG9GfmI22bXFQ9rkNJNqcB9w+w60UMryX1cfPmzevkZUO22prrsXDhwkb3MVruvXcqd942UGP20MydO7eLpRm/rIfCeiish8J6KKyHwnooxlI9jIlkMyI2BH4AXEltgFBmfqEt9McRcTFwDHDqqBVwEDNnzqSvr6+x7d94ywJmzJjR2PZH0/Tp67LBpht19Nq5c+cya9asLpdo/LEeCuuhsB4K66GwHgrroWivh/7+/sYbyJan5/NsRsRawE+Bh4HdMvOxFbzkHGCjiFiver4I6IuIVdviplWP99Xi1h5ge9NqMZIkSeqiniabVYJ4LrA+8JrMvLeDzbT6am7etnwL4CHgT7W4qA0YqsfN72C/kiRJWoGeJZsRsTJlxPgLgdmZeccQXjOFMpXRHbXR7JdTRpnvXYtbqYq7MDOXVosvoLRs7lyLeyawfbVOkiRJXdbLPpsnA68DPghMjYhtautupFzePo0yEfsfKIniAcCOwFtbgZnZHxHHAcdHxN3A1VXcxsCba3G/i4jzgW9GxGHAg8CxwALGQP9PSZKkiaiXyWarhfGzA6x7OXAdpcXySMpl9scoieRumXlePTgz50QEwKHABpTR67tm5rVt290HmEOZwL2PcrvKPTPz4W4ckCRJkp6oZ8lmZj57CGGvH8b25lASyeXFPAS8s/onSZKkhvV8NLokSZImLpNNSZIkNcZkU5IkSY0x2ZQkSVJjTDYlSZLUGJNNSZIkNcZkU5IkSY0x2ZQkSVJjTDYlSZLUGJNNSZIkNcZkU5IkSY0x2ZQkSVJjTDYlSZLUGJNNSZIkNcZkU5IkSY0x2ZQkSVJjTDYlSZLUGJNNSZIkNcZkU5IkSY0x2ZQkSVJjTDYlSZLUGJNNSZIkNcZkU5IkSY0x2ZQkSVJjTDYlSZLUGJNNSZIkNcZkU5IkSY0x2ZQkSVJjTDYlSZLUGJNNSZIkNcZkU5IkSY0x2ZQkSVJjTDYlSZLUGJNNSZIkNWblXu04IvYE/hWYBawD3Ap8GfhqZi6pxc0GPglsAfwJODEzvzTA9g4HDgI2BG4AjsjMi9pi1gBOAPYAVgUuAQ7JzNu7fXySJEnqbcvmYUA/8AHgtcCPgC8Cn2kFRMS2wLnANcBs4NvAiRHxrvqGqkTzeOBkYFfgFuD8iHhR2z6/B+wGHALsDcwALoqIqV0+NkmSJNHDlk3gdZl5d+35JRGxOnBwRByZmf3AUcDVmfmOWsxGwMcj4muZuSQi+oAjKS2ecwAi4lLgeuCjwF7VshdTEtFdM/OCatn1lBbV/YBTGj5eSZKkSadnLZttiWbLNZTL2+tUSeQrgLPaYs6gXCrfsnq+HbAWcGZt24uBs4HZETGlWrwL8ABwYS1uAXBZtU6SJEldNtYGCO0A3Af8BdgYWAW4sS3mhurxedXj5tXjTQPErQ48oxY3v94ftBb3PCRJktR1vbyM/gQRsRWwP3BMZi6OiGnVqvvbQhdVj+tUj9OA/sx8ZDlxd1Zx7dtqxa0zwPIVmjdvXicvG7LV1lyPhQsXNrqP0XLvvVO587aBGrOHZu7cuV0szfhlPRTWQ2E9FNZDYT0U1kMxluphTCSbEbEh8APgSmoDhMa6mTNn0tfX19j2b7xlATNmzGhs+6Np+vR12WDTjTp67dy5c5k1a1aXSzT+WA+F9VBYD4X1UFgPhfVQtNdDf39/4w1ky9Pzy+gRsRbwU+BhYLfMfKxa1WqZXLvtJa0Wz/tqcX0RseoQ4tq31Yq7b4DlkiRJGqGeJptVgngusD7wmsy8t7b6VuBRlvXJbNmiepxfPbb6ag4U9xBlbs5WXNQGDNXj5iNJkqSu61myGRErU0aMvxCYnZl31NdXUx9dTDV1Uc0+wJ+Bq6vnl1NGme9d2/ZK1esuzMyl1eILKC2bO9finglsX62TJElSl/Wyz+bJwOuADwJTI2Kb2robM/NB4Fjg1xHxdeB04CXAgcBBrVHlmdkfEccBx0fE3ZQk9ADKaPY3tzaYmb+LiPOBb0bEYUBr+wuAUxs9UkmSpEmql8lmq4XxswOseznwq8z8bUS8nnJ3oH2BhcD7MvMr9eDMnBMRAIcCG1CmM9o1M69t2+4+wBzKBO59lNtV7pmZD3fnkCRJklTXs2QzM589xLgLGMJl7uruQXNWEPMQ8M7qnyRJkhrW89HokiRJmrhMNiVJktQYk01JkiQ1xmRTkiRJjTHZlCRJUmNMNiVJktQYk01JkiQ1ZtjJZkTsPMD9xSVJkqQn6aRl86fAnRFxQkS8qNsFkiRJ0sTRSbK5O3AZcBBwdURcFxGHR8SMrpZMkiRJ496wk83MPDcz96Lcg/xA4G7g08AdEfHziHhLREztcjklSZI0DnU8QCgzH8rMb2XmTsCzgI8A6wOnAXdFxHciYqculVOSJEnjULdGo68EPBXoA6YAjwCvBH4REddExMwu7UeSJEnjyMqdvjAi1gL2At4CvAT4O3A+8KHqcQmwG/B54NvA1iMtrCRJksaXYSebEbE7JcHcBVgVuAp4D/C9zLyvLfxHEbEucMoIyylJkqRxqJOWzf8H/An4AnBaZs5fQfx1wOkd7EeSJEnjXCfJ5quBizJz6VCCM/NK4MoO9iNJkqRxbtjJZmb+somCSJIkaeLp5HaVn4+IW5az/uaIOGFkxZIkSdJE0MnUR7sCZy1n/VnA6zorjiRJkiaSTpLNZwK3L2f9HVWMJEmSJrlOks0HgecsZ/1zKZO6S5IkaZLrJNm8GHhnRGzUviIing28s4qRJEnSJNfJ1EdHAbOBeRHxbeCGavlMYD9gMfCxrpROkiRJ41onUx/dEhEvAU4GDmlbfSlwSGZmNwonSZKk8a2je6Nn5g3AjtWtKJ9bLb41M+/tWskkSZI07nWUbLZk5j3APV0qiyRJkiaYjpLNiFgJ2JnSqjkNmNIWsjQzPzHCskmSJGmcG3ayGRFbAT8A/oEnJ5ktSwGTTUmSpEmuk5bNU4DVgN2B32Tm/d0skCRJkiaOTpLNFwIfzczzul0YSZIkTSydTOp+J4NfPpckSZIe10my+WngwIhYs9uFkSRJ0sTSyWX0dYC/AX+IiO8D/0O5a1Dd0sw8YaSFkyRJ0vjWSbL56dr/3zVIzFLAZFOSJGmS6yTZfE63dh4RmwCHA9tQ7q0+PzNntsWcCrxtgJfvmZnfb4s9HDgI2JByz/YjMvOitpg1KInwHsCqwCWUW2ze3oVDkiRJUk0n90a/o4v7fz6wK/A7Sv/RwfqQ/hH417ZlN9efVInm8cBHgKuBA4HzI+LFmXltLfR7wJaU+7o/CBwLXBQRL8jMh0d2OJIkSarr+HaVEbEpsCOwPnB6Zt4eEatQWhX/nJmPDmEz52Xmj6vtnQpsNUjcI5l5xXLK0gccCZyYmXOqZZcC1wMfBfaqlr2YktzumpkXVMuuB24F9qPMISpJkqQuGfZo9Ih4SkR8DZgPfJXSMvjcavUqlATvkKFsKzOXDHf/g9gOWAs4s7btxcDZwOyIaE3VtAvwAHBhLW4BcFm1TpIkSV3UydRHHwHeDnwM2JbanJuZ+VfKrSz/pSulW2bjiLg/Ih6LiGsiYu+29ZtXjze1Lb8BWB14Ri1u/gBJ7g3A87paYkmSJHV0GX1/4FuZeXxETB9g/fXAa0dWrCe4BriKkhCuBRwAnBkRq2XmqVXMNKA/Mx9pe+2i6nEdymT004D7B9jHoipmWObNmzfclwzLamuux8KFCxvdx2i5996p3Hnb3R2/fu7cuV0szfhlPRTWQ2E9FNZDYT0U1kMxluqhk2TzH4Arl7P+EWCNzorzZJn5hbZFP46Ii4FjgFO7tZ9OzJw5k76+vsa2f+MtC5gxY0Zj2x9N06evywabbtTRa+fOncusWbO6XKLxx3oorIfCeiish8J6KKyHor0e+vv7G28gW55OLqP/GXjWctbPAro5Yn0g5wAbRcR61fNFQF9ErNoWN616vK8Wt/YA25tWi5EkSVKXdJJs/gD492o0estSgIiYDexLGZgzmlp9NTdvW74F8BDwp1pc1AYM1ePmN1c8SZKkyamTZPNoYAGlL+XplETzIxFxBfAT4FrgU90qYLsqUdwLuCMzW50AL6eMMt+7FrdSFXdhZi6tFl9AadncuRb3TGD7ap0kSZK6qJNJ3R+MiO2A9wN7Av9HSdZupSSiJ2Tm/w1lWxExlWVTDj0LWDMi9qieX1U9nkaZiP0PlETxAMr8nm+tlak/Io4Djo+IuymTuh8AbAy8uRb3u4g4H/hmRBzGskndF9Dj/p+SJEkTUUeTulfJ5PHVv5FYn9L/sq71fH/gXEqL5ZFV7GOURHK3zDyvrUxzIgLgUGADyuj1XdvuHgSwDzCHMoF7H+V2lXt69yBJkqTu6/gOQt1Q3Y+8vf9ku9cPY3tzKInk8mIeAt5Z/ZMkSVKDhp1sRsS3hhC2NDPf0UF5JEmSNIF00rL5CqrR5zUrAU+vHu8G/jbCckmSJGkC6GSA0LMHWh4RT6Vcmn4v8KoRlUqSJEkTQidTHw0oMx/LzJOAnwMndWu7kiRJGr+6lmzWXAu8tIHtSpIkaZxpItl8FeA0QpIkSepoNPpRg6xam9KiuSXw6RGUSZIkSRNEJ6PRjx5k+SLKXYTeBXy90wJJkiRp4uhkNHoTl94lSZI0AZk4SpIkqTGd9NncqJMdZeaCTl4nSZKk8auTPpu38+Q7CA3FSh28RpIkSeNYJ8nmAcChwDOBM4Cbq+UB7AMsAL4ILOlGASVJkjR+dZJsPh3oAzbJzEX1FRHxceAyYMPM/FQXyidJkqRxrJMBQu8CvtaeaAJk5r2UaY/+faQFkyRJ0vjXSbI5HVh9OeufVsVIkiRpkusk2bwCeE9EzGpfERFbAe8BfjfSgkmSJGn866TP5sHAr4ArI+Iq4JZq+abA1sB9wCFdKZ0mjCVLlnLXfQ939NrV1lyv49c2YWrfyqzxtFV6XQxJksaFTu4gdGNEvAD4EDAb2KNadQfwBeCzmfnn7hVRE0H/Y4u5/LqFHb124cKFzJgxdpLNnbbeyGRTkqQh6qRlk8y8C3hf9U+SJEkaUEfJZktEbAqsD8zLzAe6UyRJkiRNFB3dGz0i3hwRC4D5wK+BWdXydSPi5ojYq4tllCRJ0jg17GQzIt4IfBe4CfgAMKW1LjPvqZbv260CSpIkafzqpGXzo8AvM3Nn4LQB1v8OeNGISiVJkqQJoZNkc3Pgh8tZ/xdgvc6KI0mSpImkk2Tzbyz/DkIbA/d0VhxJkiRNJJ0kmxcD+0XEkyYajIgZwIHAz0ZaMEmSJI1/nfbZfDrwe+DdwFJgl4j4NHA9sAQ4pmsllCRJ0rg17GQzM28BXgL8GTiaMhr9/cAHgf8Gts/MBd0roiRJksarYU3qHhErAc8A7srMV0fENGATStL6x8y8u4EySpIkaZwa7h2EngLcChwBfC4zFwFXdb1UkiRJmhCGdRk9Mx8DFlL6aUqSJEnL1ckAoW9TRqOv2u3CSJIkaWIZ7mV0gJuBlYD5EXEa8EfgkfagzDx7hGWTJEnSONdJsvnd2v8/NkjMUmCFyWZEbAIcDmwDzATmZ+bMAeJmA58EtgD+BJyYmV8aIO5w4CBgQ+AG4IjMvKgtZg3gBGAPYFXgEuCQzLx9ReWVJEnS8AzpMnpEfDEiZlVPX179ex3wytrz+r9XDHH/zwd2Bf4A3DjIvrcFzgWuAWZTLuOfGBHvaos7HDgeOLna5i3A+RHRfp/27wG7AYcAewMzgIsiYuoQyyxJkqQhGmrL5sHAFcDczLw0IqZT7oH+qsy8dAT7Py8zfwwQEacCWw0QcxRwdWa+o3p+SURsBHw8Ir6WmUsiog84ktLiOafa3qWUSeY/CuxVLXsxJRHdNTMvqJZdTxlhvx9wygiORZIkSW06GSDUMmWkO8/MJctbXyWRrwDOalt1BuVS+ZbV8+2AtYAza9teTLmUPzsiWmXdBXgAuLAWtwC4rFonSZKkLhpJsjkaNgZW4cmX2G+oHp9XPW5ePd40QNzqlInoW3HzB0hyb6htS5IkSV3SyQCh0TStery/bfmi6nGdWlx/ZraPiq/H3VnFtW+rFbfOAMuXa968ecN9ybCstuZ6LFy4sNF9jJb+WGdExzKW6uHee6dy5229uVnW3Llze7LfscZ6KKyHwnoorIfCeijGUj0MJ9l8bkT8c/X/tarH50XEXwcKzswrR1SycWDmzJn09fU1tv0bb1nAjBkzGtv+aOrrW7XjY1m4cOGYqofp09dlg003GvX9zp07l1mzZq04cIKzHgrrobAeCuuhsB6K9nro7+9vvIFseYaTbB5T/at70vRDlL6cSylzcY5Uq2Vy7bblrRbP+2pxfRGxamb+3wriBsoSptViJEmS1CVDTTb3b7QUg7sVeJTS1/LC2vItqsf51WOrr+bmlCmS6nEPUebmbMW9KiKmZObStrj5SJIkqauGlGxm5mlNF2SQ/fZHxMWUqYs+X1u1D/Bn4Orq+eWUUeZ7UyWbEbFS9boLa4nlBZSplHamSl4j4pnA9sB7Gj0YSZKkSainA4SqidRbUw49C1gzIvaonl+VmXcAxwK/joivA6cDLwEOBA5qjSqvktLjgOMj4m5KEnoAZTT7m1v7y8zfRcT5wDcj4jDgwWr7C4BTGz1YSZKkSajXo9HXB85pW9Z6vj9wamb+NiJeT7k70L7AQuB9mfmV+osyc05EABwKbECZzmjXzLy2bfv7AHMoE7j3UW5XuWdmPty1o5IkSRLQ42Szuh/5CieHr+72c8EQ4uZQEsnlxTwEvLP6J0mSpAaN9UndJUmSNI6ZbEqSJKkxJpuSJElqjMmmJEmSGmOyKUmSpMaYbEqSJKkxJpuSJElqjMmmJEmSGmOyKUmSpMaYbEqSJKkxJpuSJElqjMmmJEmSGmOyKUmSpMaYbEqSJKkxJpuSJElqjMmmJEmSGmOyKUmSpMaYbEqSJKkxJpuSJElqjMmmJEmSGmOyKUmSpMaYbEqSJKkxJpuSJElqjMmmJEmSGmOyKUmSpMaYbEqSJKkxJpuSJElqjMmmJEmSGmOyKUmSpMaYbEqSJKkxJpuSJElqjMmmJEmSGmOyKUmSpMaYbEqSJKkxJpuSJElqzMq9LsCKRMR+wLcHWHVyZh5ci5sNfBLYAvgTcGJmfmmA7R0OHARsCNwAHJGZFzVQdE1QS5Ys5a77Hh71/a625npd3e/UvpVZ42mrdG17kiQNZMwnmzWvAR6oPf9z6z8RsS1wLvAd4DDgJcCJEfFYZn6lFnc4cDzwEeBq4EDg/Ih4cWZe2/whaCLof2wxl1+3cNT3u3DhQmbM6F6yudPWG5lsSpIaN56SzbmZec8g644Crs7Md1TPL4mIjYCPR8TXMnNJRPQBR1JaPOcARMSlwPXAR4G9Gi6/JEnSpDPu+2xWSeQrgLPaVp1BuVS+ZfV8O2At4MxWQGYuBs4GZkfElOZLK0mSNLmMp5bNeRGxHrAAOBX4ZGb+HdgYWAW4sS3+hurxecDvgc2r5zcNELc68Azgzu4XW5IkafIaD8nm/wIfB64EFgOzgY8BzwH2A6ZVcfe3vW5R9bhO9TgN6M/MR5YTN6xkc968ecMJH7bV1lyPhQtHv29gE/pjnREdy1iqh5Eey0h0c7/33juVO2+7u2vbG01z587tdRHGBOuhsB4K66GwHoqxVA9jPtnMzJ8BP6st+kVEPAAcHRGf6FGxAJg5cyZ9fX2Nbf/GWxYwY8aMxrY/mvr6Vu34WMrAmLFTDyM5lpHodj1Mn74uG2y6Ude2N1rmzp3LrFmzel2MnrMeCuuhsB4K66For4f+/v7GG8iWZ7z22Ty7etySZS2Ta7fFtFo876seFwF9EbHqCuIkSZLUJeM12ay7FXiUZX0yW7aoHudXj62+mgPFPUSZm1OSJEldNF6TzTcBSynTIfUDF/PkqYv2oczFeXX1/HLKPJ17twIiYqXqdRdm5tKmCy1JkjTZjPk+mxHxM0oyOQ9YQhkg9G7gm5n5xyrsWODXEfF14HTKpO4HAgdl5hKAzOyPiOOA4yPibkoSegBlNPubR/GQJEmSJo0xn2xSLn+/HfgHSnlvAY4ATmwFZOZvI+L1lLsD7QssBN5Xv3tQFTcnIgAOBTagTHu0q3cPkiRJasaYTzYz873Ae4cQdwFwwRDi5gBzRlwwSZIkrdB47bMpSZKkccBkU5IkSY0x2ZQkSVJjTDYlSZLUGJNNSZIkNcZkU5IkSY0x2ZQkSVJjTDYlSZLUGJNNSZIkNcZkU5IkSY0x2ZQkSVJjTDYlSZLUGJNNSZIkNcZkU5IkSY0x2ZQkSVJjTDYlSZLUmJV7XQBJvbFkyVLuuu/hXhdj2FZbc70nlXtq38qs8bRVelQiSdLymGxKk1T/Y4u5/LqFvS7GsC1cuJAZM56YbO609UYmm5I0RnkZXZIkSY0x2ZQkSVJjTDYlSZLUGJNNSZIkNcZkU5IkSY0x2ZQkSVJjTDYlSZLUGJNNSZIkNcZkU5IkSY0x2ZQkSVJjTDYlSZLUGO+NLmncW7JkKXfd9/CKA8eBqX0re593SROKyaakca//scVcft3CXhejK3baeiOTTUkTipfRJUmS1BhbNiVpDBlql4DV1lxvTHcdsDuApBaTTUkaQ4baJWDhwoXMmDF2k027A0hqmXTJZkRsCnwJ2B54BDgTOCIzx+63tiRJ0jg1qZLNiFgbuAS4A9gDWB/4HLAe8KbelUySJGlimlTJJvBOYBrwj5l5D0BE/B04PSI+kZk39LR0kjRBjNZ0VKPRd9X+p9LITLZkcxfgolaiWfkB8C1gNmCyKUldMFrTUY1G31X7n0ojM9mSzc0pieXjMrM/Im4FnjeM7awE8Oijj3axaE+2ZMliVn7Kkkb3MVoW//2xjo9l1adOGVP1MJJjGYlu10OvjmOkBqqH8XosAxnqsYy1z0W70XpPRqMe/v7Yo/T3r9ToPrqhv7+/10UYE6yHol4PtXylJyfylKVLl/Zivz0REY8BH8vMT7ct/y/gL5n5L0PZzty5c7cHftNAESVJkpqyw6xZs/5rtHc62Vo2u+UqYAfgf4HFPS6LJEnS8qwEPJ2Sv4y6yZZsLgLWHmD5NGD+UDcya9asfmDUfxlIkiR16NZe7Xiy3a7yJkq/zcdFRB+wMcNINiVJkjQ0ky3ZvADYKSKm15a9Aeir1kmSJKmLJtsAobWBecDtwCdYNqn7RZnppO6SJEldNqlaNjPzfuAVwF+B/wd8HjgLeHsPiyVJkjRhTaqWTUmSJI2uSdWyKUmSpNFlsilJkqTGmGxKkiSpMZNtUvcxLyI2Bb4EbA88ApwJHJGZD/e0YCsQEXsC/wrMAtahTB77ZeCrmbmkFjcb+CSwBfAn4MTM/NIA2zscOAjYELiBUgcXtcWsAZwA7AGsClwCHJKZt7fF9aROI2J1yvytzwC2zszf19btC3wEeDalro7NzLPaXv9U4FjgbZSbEVwFvCcz/7stbkPgC8BrgKXAT4D3ZuY9bXH/TJl9YRZwH/CNar+N3QUrIt4KvJfyfj8MXA3s0yrbZDgfImJ3ynu9OfA34DLgQ5l5S1vchDknImIT4HBgG2AmMD8zZw4QN2bf/6GWbST1EBErAYcBu1b7WRm4Hjim/fgmcj0MED8LuBJ4JDNXb1vXk8/AUD6fKzKMz8WqwIeAtwL/ANwDXJCZB7bFjZvzwZbNMaSamukSYA3KiXEYsA/wrR4Wa6gOA/qBDwCvBX4EfBH4TCsgIrYFzgWuAWYD3wZOjIh31TdUfYCOB06mfAnfApwfES9q2+f3gN2AQ4C9gRnARRExtbatteldnR7NAD/oImIP4DTgh5R6+CXwvepDXPd5yhfJx4HXA49Sjm9GbVsrAxcCLwD2BQ4AtgPOjYgptbjnVvu5j/L+HE95rz7ZheMcUER8lPKD4/9RjvMdlC/Evmr9hD8fImInyvHPB/6lKtvzgF9GxJq1uIl2Tjyf8l79AbhxoICx/P4PtWxDsKJ6WI2SwPw3sD/wJsof8F9ExGvbyjSR66G+z6dQvjfuHiRk1D8Dw/h8rshQPhdPofz93Lcqz6uBD1Jm0anHjavzwZbNseWdlFtn/mOt5efvwOkR8YnMvKGnpVu+12Vm/cvhkqpl7+CIODIz+4GjgKsz8x21mI2Aj0fE1zJzSXVHpyMpv5bmAETEpZRf+x8F9qqWvZjyAds1My+oll1P+cW5H3BKtY+e1GlEzATeBbwf+Grb6k8A52Tmh6vnl0TE5sAxwE+r1z+jev2hmfn1atkVwG2UlsIPVq99I/AiYGbrWCJiIaX1bDbLblbwAeB+YM/qvbgoItYCjoqIz2bmfd07eoiIoCTbb8jMn9RW/aj2/8lwPuwD3AG8LTOXVvu7A/gd8BKq95uJd06cl5k/rvZ9KrDVADFj+f1fYdm6VA+PAM/JzEWtBRHxc2Azyh/8n1TLJno91B0IrEVJdA6tr+jhZ2CFn88u1sP+wLbAFpn5p9ry02v1MO7OB1s2x5ZdKBPM15v5f0BpMRzuL6hR1ZZotlxDabZfp/pwvIIyr2ndGZRLAFtWz7ejfNGcWdv2YuBsYHbtV+kuwAOUX6+tuAWUL5NdatvvVZ2eDJwE3FxfGBHPobRsndkWfwawdUSsVz1/NbAStfrKzIcof3zaj+/6epKUmZdTEpz2uB9VX6j1fbbel27bH7ijLdF83CQ6H54KPNRKNCv3V49TYGKeEytKQsby+z+Msq3QiuohMxfXE81q2VJKS+eM2uIJXQ8tEbEupbXuPZQWy3aj/hkYxudzhYZYDwdSEts/LSdm3J0PJptjy+a0Na1XH4JbKSf7eLMD5fLEXyj3n1+FJ186aH0ZtI6vde/6mwaIW53S/7EVN3+AD+8NPLGuRr1Oo/RT3AQ4boDVreMbrB6iFndXZt47QNxm1aWWVtxAl2Mer4eIeBqwUXtc1WfnYZqph22A6yLiyIj4c0Q8FhFXRsTLqvWT5Xw4Fdg8Ig6JiLUj4tnAHMrxtPpWTZZzom4sv/9DLVsjqvdxO554zJOlHj4D/FdmXjjI+l58Bob6+RyxKP1RtwRuj4jTIuKvEfG3iPhR1YLYMu7OB5PNsWUay1o96hZRBt2MGxGxFaV16/PVL65p1ar720Jbv+pbxzcN6M/MR4YQ176tVly9rka1TqtLMCcAH8zMvw4QMpx6aI9pxT2V8oWyorjWttYeZJ/tcd20IfAqyjlwKPA64EHgwirhmhTnQ2ZeQumr+clqH7cBzwFeVWtNmSznRN1Yfv+HWramHEJJYP6jtmzC10PVH3Af4H3LCevFZ2A062E65TiOoHyHvpHS1/1FwAVR+qK2yjSuzgeTTXVdlFGAP6CMJvzMCsInmuOAWzLz9BVGTmxPoXzxvzEzz65aKnajJJwf6GnJRlFEbAd8B/gm5RLUnsASykCF1XpZNo09Vcv/Z4E5mfmbXpdntEQZlX8K8LnM/GOvy9NDrZzsr8DumfmzzDyT8r3xfOANPSvZCJlsji2LWPZrq24a5XL0mFe17P2Uchlit8x8rFrV+vWzdttLWr+W7qvF9UWZ+mFFce3basXV62rU6jQink/pvP6x6pLp2iz7pb16lCkohlMP7TGtuMdYNjJxKMd3/yD7bI/rpkXAvVmbjiTLFBpXUKb8mPDnQ+WLwCWZ+b7MvCQzv0/psP9PlGlNWmVigHJNtHOibiy//0MtW1dFxAuBH1MG0R3Rtnqi18OBwNOBU2rfnatCGSld+2HWi8/AaNbD/ZRpmi6rt1pmmTbvQcp3Z6tM4+p8MNkcW25iWV8M4PFOuRtTpk4Z06oT/1xgfeA1bf1qbqV0+N687WVbVI+t42v1QRko7iHKtCCtuKh1hK7H1etqNOt0U8oMD5dQPpiLgPOqdZcAv2H5xweQ1eNNwPoR0X5pYgvg5lofnCcdXy1uPkBm/g1Y0B4XEc8CptLMubW8Ud2rMjnOh9b+/7u+IDPvpMybt3GtTLSXi4l3TtSN5fd/qGXrmojYGPgZZR7at7YNKIOJXw/PAzagHEfru/MI4GnV/z9VK/dofwaG+vkcseoH+e2DrF5KlYCvoExj8nww2RxbLgB2iojptWVvoIyMu2Dgl4wNVV+Ss4EXArMz8476+qp/2sVUUzLU7AP8mfIlC3A5ZfTc3rVtr1S97sLal/AFlF9YO9finkmZkLZeV6NZp/8FvLztX6v/0buAAzLzNsqHcu+21+4DXFUb1f9zyuXWx+srylRSr+PJx/eCahqOVtw2lImH2+N2j4hV2vbZz7KBKt30E2B6RDw+QrHqkL8tMHeSnA9QRr/Oqi+o/pitS/VHZRKdE48by+//MMrWFVW3o59X2949MwcahT3R6+EknvzdeRrwf9X/T6riRv0zMIzPZ7f8BNi+3s0mysTzawFzq0Xj7nyYsnRp+w8o9Up16WAe5Y/QJygthJ+jTEnwpt6VbMUi4qvAv1HmOWvva3RjZj5YdQD/NWWE7umUeQaPBQ7KzK/UttWarPbDlBP4AEpH6Rdn5rW1uJ9QLkceRrnEcCylSf8F1S/EntdpROxIadV8/A5CUe62dBbl1/ovKBMTv4cyF9pPa689iXKp9TBK0nI4ZV62F2TmwipmZeD3lE7lH6a0rJ4A3AW8JJfN7fhcSgvbxZS7REQV96XM/FADx/0U4LfAepR53x6qjmNrylxuf5gM50NEHEyp75Mol0inU+bHWw94fqv1f6KdE1EmjG5NrXIQpYXk/dXzqzLzjrH8/g+1bCOtB8pMHb+tlr+F8h49LjOvmAz10N44Ub3maODwfPIdhEb9MzDUz2c36qFKBq+lvMefpySLx1Peyy1bXdPG2/lgy+YYkpn3UwYR/JVy15HPU07wt/ewWEPV+uX0WcqXZ/3flgCZ+VvKh3RryiWjA4D3tZ+kWSap/QhlFPNPKZdYdq1/gCr7UH4FngKcQ/l19cqs3V5rLNZpZp5DGaW9B6UedgbePMCX1vsod9I4jtI9YTXK8S2sbevvlNuxzQO+S7mjwxWU/rJLa3F/BF5JSXLOpyQ8/0FJBLuuupy1K+WLqfX+AOyYmX+oYibD+XAyZcLkHSh98U6k3D3k5fVuJhPwnFif8h6cA+wIPLP2/OXV/sfs+z/UsnWhHjagjDRenXJ+tH93TpZ6GI5R/wwM4/O5IkP5XPxP9f8p1fKTKA04r8xlYyDG3flgy6YkSZIaY8umJEmSGmOyKUmSpMaYbEqSJKkxJpuSJElqjMmmJEmSGmOyKUmSpMaYbEqSJKkxJpuSJElqzP8HVvNfDeBqWn8AAAAASUVORK5CYII=\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "df1 = df[df[\"name\"].isin([\"IssueQuery\"])]\n", - "df1['delta'] = df1['ts'].diff()\n", - "ax = df1['dur'].plot.hist(bins=BINS, alpha=0.5, figsize=figsize)\n", - "ax.set_title('IssueQuery duration (usec)');\n", - "plt.show()\n", - "ax = df1['delta'].plot.hist(bins=BINS, alpha=0.5, figsize=figsize)\n", - "ax.set_title('Time between IssueQuery (usec)');\n", - "\n", - "# df1['delta'].describe()" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "# for SingleStream\n", - "if False:\n", - " df1 = df[df[\"name\"].isin([\"QuerySamplesComplete\"])]\n", - " ax = df1['dur'].plot.hist(bins=BINS, alpha=0.5, figsize=figsize)\n", - " ax.set_title('Inference time (usec)');\n", - " plt.show()\n", - " ax = df1['dur'].plot(figsize=figsize)\n", - " ax.set(ylim=(0, 100))\n", - " ax.set_title('Individual inference time (usec)');" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "scrolled": false - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAoMAAAFtCAYAAAB8yGDhAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAABA20lEQVR4nO3deZwcZbX/8U8WMkmAkIWwBAkgwgGM4DXyExQF5SKyCFxlEQQFRUAQRETZd81FiQsoKKAsV5TVK8tlEWVfJQ4IJJAjW4hhIIQkLJIwZJnfH+fppNL0bD29Tdf3/Xrl1emq6uqnz9TUnH7qeU4N6OjoQERERETyaWC9GyAiIiIi9aNkUERERCTHlAyKiIiI5JiSQREREZEcUzIoIiIikmNKBkVERERybHC9GyAi+WFmw4GfArsDawHnuvvRdW1UAzKz9YEXgIPc/bL6tmZFZvZd4AhgY3dfXO/2dMXMdgGuBjZw9zn1bo9Io1LPoIj0mJkdaGYdZrZVmbs4BjgEuBg4APhdxRrXD5nZ4WZ2YL3b0VNmtipwAnBOoyeCAO5+M/Ac0WYR6YR6BkWklrYDHnf3U+vdkAZxOPAacFnR8heBYcCiWjeoG18j2vU/9W5IL1wInGNmp7v7m/VujEgjUjIoIrW0BjCvUjszswHAUHdfWKl9NgJ37wDeqXc7SvgacIu7v13vhvTCdcB5wN7Ab+rcFpGGpGRQRPrEzC4DvgRsCJwP/CewELgcOM7dl5jZdsBdmdcU7oO5gbvPMLMW4Hhgf2A80Vt2DXCSuy8oet2FaV8nA0Zcdr7MzFYDTgP2JMYjzgIuBSa5+5L0+vWJsXgnAHPTe74PeAI43N2nFH22jYEzgO2BEcC/gJuz4xzNbG3gLGBXYBTwPHCeu/+qm7jNANYriseL7r5+qTGDZnZ6+nybAScBnyd6Di9Kz8cR8f8MEf/J7n5O0Xv2KM6dtHcDYPP0Htnl72lrZl0HcIa7n56erwKcDnwxtfdNYBpwqrvfm3ndlkTcPwEMAVqBU9z9rqL9r532twswFngZ+AtwjLu/BeDur5rZE8B/oWRQpCSNGRSRShgI3EYkWMcC9wDfJRI1gKeJMYKzgOnp/wcAc1Lv3p+A7wM3A0cSCcrhwPVpfdaniITkj8BRwHQzG0YkiAcCVwDfAu4kEoULS7R3n/R+FxJJ5frA/5rZSoUNzOyDwCPAjsAl6b2uJZKwwjZrAA8DnwMuAL4NTAUuMLOTu4nZ0SXicXQ3rwG4kvgifzzwEJHYfpdIgl4BjgOeAX5sZp/JtLW3cS728fT49x60sTO/Su/7p/S+PwLmAFtk2rktcB8wGjgzfZ4W4Pb0paKw3VrEz+crxLFwJHG5/f8BY4retxXYugefUSSX1DMoIpWwEnCtu5+Znv/azB4Fvg78yt1nA1eY2fHAa+5+ReGFZrYfkUx92t3vySz/O5HY7QDcnnmvTYCPuPs/MtuemFk+PS2+yMxeAH5gZue4u2f2sS6wkbvPT6934AYi8fu/tM35xDnyQ+7+Qua9Tsrs5wdEovKhzGzVX5vZxcCJZvZLd3+9VMDc/Xoz+0FxPHqg1d2/ntpyETAD+DHRc/bDtPxKoI24rHtnet2+9C7OxTZJj8/3oq3FdgUudvdjSq1MydqFwP3ADulyOWb2a+AxYBLLk9Kzid7Fj7v73zK7Ob1E0vc80Wu7NhEXEclQMigilXJx0fP7iN6u7uwN/BOYZmarZ5bfA3QAn2bFJOXBbCKY2cf9wGtF+/grkbBtB2STwT8WEsFMWwHeD2BmY4FtgfOziSAsG89XSFz2JHq5Oore93bgYOBjwJ87++BlWnapM12C/ztxqfu3meWvpwT3/ZnX9TbOxcYAS4E3+tD2N4CPmdk67v5SifVbEJf+fwyMMbPsur8AR6byRO8Ql31vLUoEgeU/o4zCz3p1lAyKvIeSQRGphEXu/nLRsvlEb0x3NiYSgM7qwK1R9Py5TvaxRS/2MTP7xN3np8Sj0N5CEjW1k/1BjFEbRfS+fa2H71sJM4uev0HE/5USy9fMPO9tnEsZkP4VJ1s99T1iLOlMM3uMGFrwu0yv7cbp8belXpyMAd4lxnB29fPJKvQUlttukaamZFBEKmFpH147EHiKGG9XSnFPTqmZwwOJy6H/3ck+ii9tLulku96MKSuMub6SGFNYyrRe7K+nSrW9s/hnP09v41zstbS/1Vje0wadJFhmNqh4mbtfa2b3EUXHP0uMw/y+mR3o7n9geUyPJ8b5lTIntaE3Ckn+a718nUguKBkUkXp7DpgI3FHi8l5v9rGqu/+1gm0CmNDFNnOAt4DBfXjfWvZU9TXOT6fHDVgxGSz8f2TR9uuV2knqwbwQuNDMRhITcM4A/sDyuL/VVUzN7F1iJnJXP5+sQpuLe09FBM0mFpH6u5q4nPnN4hVm1pLuetGTfWxpZjuX2MeqqaRKj7n7a8RYugNTSZXs/gakbZYQNez2MLMtiveRxh125216dim9Evoa5wfS40ezC1Mh59eIWd5Zhxe9x6BU/if72teJsjQj06JW4FngmFLtKcTU3ZcSYzV3MrOPldiuuId3IvBwH75siDQ19QyKSL1dQUzEOD+VFbmfuBxpxKSHvYC7u9nHOUTJlxvM7HIiqRhG9BztBXyImHXbG0emtrSa2YXEpebxRE3FjdI2xxOTUx5KM4inEcndh4kJDkO7eY+/A4eb2WnE5I5/u/tNvWxnT/Upzu4+08z+Qcw6vqho9W+A483sN8Rn+hTLx/8VrAq8ZGZ/BB4nevY+Qcxw/mV6j6Vm9nViLOFTZnYJUX5nHDGhZwAx0QWipM4OwN3p5/MUkex+gYj9DFhW/mdz4Nfdh0gkn5QMikhdpQTgC0SNva8S48kWEsnXBURB6O72sTDVoDuBSGwOIC7hPkMUhO715UF3fzLdg/ks4FAiufwXcFNmm1dTz9QpwB5Er9s84pLqd3vwNmcSZW6OISZEvJjdfyVVIs7E2MizzWzloruQnElMqNmTiP+twE7Aq5ltFhDlenYAdiPKEb1A1KU8N9POe1PcTyF6F0cQP78prDiT+uUU+7OIsjkjiXGPt7Pi2MAvEhNOru7B5xPJpQEdHeo1FxGR7qVLt88Tdwzp8g4rjSL1Zt6dvWuMiKxIYwZFRKRH0i3ezga+Z2YNf2XJzHYBPkAUqxaRTqhnUERERCTH1DMoIiIikmNKBkVERERyrOHHfDSi1tbWFmBL4GU6v5OBiIiISCMYBKwNTJk4cWJ78Uolg+XZkuU3thcRERHpDz5J1BhdgZLB8rwMsPHGGzNkyJA+7Wjq1KlMmNDTOyo1L8VBMQDFABQDUAxAMQDFACoXg3fffZd//vOfkPKXYkoGy7MEYMiQIbS09OouVyVVYh/NQHFQDEAxAMUAFANQDEAxgIrHoOTQNk0gEREREckxJYMiIiIiOaZkUERERCTHlAyKiIiI5JiSQREREZEcUzIoIiIikmNKBkVERERyTMmgiIiISI4pGRQRERHJMSWDIiIiIjmm29E1sLfefpcF7Yvr3YyKGN4ymFVX7tt9nEVERKTylAw2sAXti7ljysx6N6Mitt9yvJJBERGRBqTLxCIiIiI5pmRQREREJMeUDIqIiIjkmJJBERERkRxTMigiIiKSY0oGRURERHJMyaCIiIhIjikZFBEREckxJYMiIiIiOaZkUERERCTHlAyKiIiI5JiSQREREZEcUzIoIiIikmNKBkVERERyTMmgiIiISI4Nruebm9kHgGOBrYAJwHR3n1C0zWXAV0u8fC93v65o22OBI4C1gGnAce5+R9E2qwLnAHsCQ4G7gCPdfUYFPpKIiIhIv1LvnsEPArsAzwJPdbHd88DWRf/uzG6QEsFJwPlpn88AN5vZFkX7uhLYDTgS2AcYB9xhZsP7+mFERERE+pu69gwCN7n7DbCsB/CjnWy30N0f7mwnZtYCnAz83N0np2X3AE8CJwF7p2UfIxLFXdz9lrTsSeA54EDggr5/JBEREZH+o649g+6+tEK7+jiwGnBVZt9LgGuAncxsQFq8M/AGcFtmu5nAA2mdiIiISK7Uu2ewpzY0s9eBlYGpwNnufnVm/abp8emi100DVgHWAWal7aaXSEKnATtWutEiIiIija7eYwZ74jFikskexKSPWcBVZnZgZptRQLu7Lyx67fz0ODqz3esl3mN+ZhsRERGR3Gj4nkF3P7do0Q1mdidwBnBZ7Vu03NSpUyuyn9bW1pLLh40YS1tbW0Xeo97mzh3OrBfmdLlNZ3HIE8VAMQDFABQDUAxAMYDaxKDhk8FOXAtcYGZj3X0O0bPXYmZD3f2dzHaj0uO89DgfGF9if6My2/TYhAkTaGlp6e3LVtDa2srEiRNLrps9bwHjxi3o0/4bxZgxq7PmRqVCH7qKQ14oBooBKAagGIBiAIoBVC4G7e3tXXZg9YfLxD1RGCu4adHyzYC3gJcy21lmQkl2u+nVa56IiIhIY+p3yWBK5PYGXky9ggAPErOE98lsNyhtd5u7d6TFtwAjyUwWMbN1gW3SOhEREZFcqfcdSIazvKTLesAIM9szPZ+SHi8nCkU/SyRyBwPbAQcU9uPu7Wb2A2CSmc0BHk3bbQjsl9nub2Z2M/BbM/su8CZwJjCTOo8/FBEREamHeo8ZXIMY/5dVeH4QcCPR43dy2nYRkejt5u43ZV/k7pPNDOAoYE2iXMwu7v540f73BSYTBaZbiNvR7eXuzTE4T0RERKQX6poMpvsBF4/fK7Z7L/Y3mUj0utrmLeDQ9E9EREQk1/rdmEERERERqRwlgyIiIiI5pmRQREREJMeUDIqIiIjkmJJBERERkRxTMigiIiKSY0oGRURERHJMyaCIiIhIjikZFBEREckxJYMiIiIiOaZkUERERCTH6npvYpH+6K2332VB++KK73fYiLHMnreg4vvtzPCWway68pCavZ+IiDQmJYMivbSgfTF3TJlZ8f22tbUxblztksHttxyvZFBERHSZWERERCTPlAyKiIiI5JiSQREREZEcUzIoIiIikmNKBkVERERyTLOJpSaWLu3osmxKrcuq9MWixUvq3QQREZGKUTIoNdG+aAkPPtHW6fpal1Xpi49vPq7eTRAREakYXSYWERERyTElgyIiIiI5pmRQREREJMeUDIqIiIjkmJJBERERkRxTMigiIiKSY0oGRURERHKsrnUGzewDwLHAVsAEYLq7T8isHwR8F9gF2Ixo75PAGe5+R9G+ZgDrlXibse7+Wma7VYFzgD2BocBdwJHuPqNiH0xERESkn6h3z+AHiUTvWeCpEuuHAScC/wAOAr4EvAT8xcx2LbH9dcDWRf9eL9rmSmA34EhgH2AccIeZDe/bRxERERHpf+p9B5Kb3P0GADO7DPho0fqFwAbuPr+wwMxuBzYmegz/r2j72e7+cGdvZmYfI5LPXdz9lrTsSeA54EDggr58GBEREZH+pq49g+6+tJv1S7KJYFrWQfQUlnNPsJ2BN4DbMvubCTyQ1omIiIjkSr17BnvNzAYCHweeLrH6y2Z2MLAEuB84wd0fzazflBiXWJyETgN2rEZ7RURERBpZv0sGibF+BhxStPxG4G/ATGIiyQnAfWa2pbsXxiOO4r1jCAHmA6N725CpU6f29iUltba2llw+bMRY2traKvIe9dZuo7v9LP3ls/bks5SrljGYO3c4s16YU7P366nOfh/yRDFQDEAxAMUAahODfpUMmtm2wI+Bye5+X3adux+VeXqfmd0KTAeOB75SjfZMmDCBlpaWPu2jtbWViRMnllw3e94Cxo1b0Kf9N4qWlqGMG9f5lf22trYu1zeS7j5LuWodgzFjVmfNjcbX7P16oqvfh7xQDBQDUAxAMYDKxaC9vb3LDqx6zybuMTPbHLgBuB44rrvt3X0ucCeQjeJ8YGSJzUcB8/rcSBEREZF+pl8kg2a2IfBn4FHggDSJpBxPx+5sQNHyzYheRBEREZFcafhk0MzWAm4HXgH2cPd3e/i61YHtgSmZxbcQPYM7ZrZbF9gmrRMRERHJlXrfgWQ4y0u6rAeMMLM90/MpwKtEGZg1gGOAzcxs2esLNQXNbF9gV+BWoij1+sSl5Bbg7Mz2fzOzm4Hfmtl3gTeBM4lJJ5dV4zOKiIiINLJ6TyBZA7i2aFnh+UHA3cAW6fn1JV5fuNz7AlF38KfE+L83gHuAPd29+PLvvsBkosB0C3E7ur3cvTlmaoiIiIj0Ql2TwXQ/4OLxe8W6W1/oIfx0D9/zLeDQ9E9EREQk1xp+zKCIiIiIVI+SQREREZEcUzIoIiIikmNKBkVERERyTMmgiIiISI4pGRQRERHJMSWDIiIiIjmmZFBEREQkx5QMioiIiOSYkkERERGRHFMyKCIiIpJjSgZFREREckzJoIiIiEiO9ToZNLMdzWxANRojIiIiIrVVTs/grcAsMzvHzLaodINEREREpHbKSQb3AB4AjgAeNbMnzOxYMxtX0ZaJiIiISNX1Ohl09xvdfW9gTeAbwBzgbOBFM7vdzPY3s+EVbqeIiIiIVEHZE0jc/S13v8TdtwfWA04E1gAuB2ab2f+Y2fYVaqeIiIiIVEGlZhMPAlYCWoABwELgP4G/mNljZjahQu8jIiIiIhU0uNwXmtlqwN7A/sAngMXAzcDx6XEpsBvwM+BSYMu+NlZEREREKqvXyaCZ7UEkgDsDQ4EpwLeBK919XtHm15vZ6sAFfWyniIiIiFRBOT2D/wu8BJwLXO7u07vZ/gng92W8j4iIiIhUWTnJ4GeBO9y9oycbu/sjwCNlvI+IiIiIVFmvk0F3/2s1GiIiIiIitVfO7eh+ZmbPdLH+n2Z2Tt+aJSIiIiK1UE5pmV2Aq7tYfzXw+fKaIyIiIiK1VM6YwXWBGV2sfzFt0y0z+wBwLLAVMAGY7u7vqUloZjsBPwQ2Iyav/Nzdf1Fiu2OJ2+StBUwDjnP3O4q2WRU4B9iTmA19F3Cku3f1mURERESaUjk9g28CG3Sx/v1E0eme+CDR0/gs8FSpDcxsa+BG4DFgJ6Jm4c/N7LCi7Y4FJgHnp30+A9xsZlsU7fJKov7hkcA+wDjgDt1CT0RERPKonGTwTuBQMxtfvMLM1gcOTdv0xE3uvq677wk82sk2pwKPuvvX3f0ud/8B8FvgNDMbmN63BTiZ6DGc7O53ErUQnwdOyrTvY0SieLC7X+nuNwP/BYwHDuxhm0VERESaRjnJ4KnE5eWpZnaumR2S/p1H1BQcCJzSkx25+9Ku1qck7zO8d4ziH4hLwR9Jzz8OrAZcldn3EuAaYCczG5AW7wy8AdyW2W4m8EBaJyIiIpIrvU4G3f0Z4vZzjxKXWn+d/n0LaAU+6e5eofZtCAzhvZeQp6XHTdLjpunx6RLbrQKsk9lueokkdFpmXyIiIiK5Uda9id19GrBdutXc+9Pi59x9bsVaFkalx9eLls9Pj6Mz27W7e/FYxex2s9J2xfsqbDe6xPIuTZ06tbcvKam1tbXk8mEjxtLW1laR96i3dhvd7WfpL5+1J5+lXLWMwdy5w5n1wpyavV9Pdfb7kCeKgWIAigEoBlCbGJSVDBa4+2vAaxVqS78zYcIEWlpa+rSP1tZWJk6cWHLd7HkLGDduQZ/23yhaWoYybty4Tte3tbV1ub6RdPdZylXrGIwZszprbvSeob911dXvQ14oBooBKAagGEDlYtDe3t5lB1ZZyaCZDQJ2JHoFRwEDijbpcPezytl3kULP3sii5YUew3mZ7VrMbKi7v9PNdqX++o3KbCMiIiKSG71OBs3so8Afgffx3iSwoAOoRDL4HPAuMdbvtszyzdLj9PRYGCu4KVGCJrvdW0RtwsJ2O5jZgKJ7K2+W2ZeIiIhIbpTTM3gBMAzYA7jP3V+vZIOy3L3dzO4E9gZ+llm1L/AKy8vRPEjMEt6HlAym3su9gdsyid8txGzoHUnJpZmtC2wDfLtan0NERESkUZWTDG4OnOTuN/X1zVOh50JJl/WAEWa2Z3o+xd1fBM4E7jWzi4HfEzOZvwEcUZgVnJLGHwCTzGwOkSQeTMxG3q/wfu7+NzO7GfitmX2XKKB9JjATuKyvn0dERESkvyknGZxF55eHe2sN4NqiZYXnBwGXuftDZrY7cXeRrwBtwHfc/dfZF7n7ZDMDOApYkygXs4u7P160/32ByUQPZwtxO7q93L05ZmqIiIiI9EI5yeDZwPfM7CJ3f7Mvb57uB9xtYunutxCXeLvbbjKR6HW1zVvEXVIO7VkrRURERJpXOcngaOBt4Fkzuw74F7CkaJsOdz+nr40TERERkeoqt2ew4LBOtukAlAyKiIiINLhyksENKt4KEREREamLXieDaYaviIiIiDSBsm9HZ2YbAdsRM4J/7+4zzGwIsBbwiru/W5kmioiIiEi1lHMHkoHAr4GvEzOBO4CHgBnAEOBJonbfTyrWShERERGpioFlvOZE4GvAKcDWZErDuPu/iVvVfaEirRMRERGRqionGTwIuMTdJwHPllj/JLBRn1olIiIiIjVRTjL4PuCRLtYvBFYtrzkiIiIiUkvlJIOvEPcR7sxEQDOORURERPqBcpLBPwLfTLOJCzoAzGwn4v7B11SgbSIiIiJSZeUkg6cDM4HHgN8TieCJZvYw8H/A48B/V6qBIiIiIlI9vU4G3f1N4OPAJGBN4B1gG2AVIlH8lLsvrGAbRURERKRKyio67e7vEMngpMo2R0RERERqqZzLxCIiIiLSJMq5A8klPdisw92/XkZ7RERERKSGyrlM/BnS7OGMQcDa6XEO8HYf2yUiIiIiNdDrZNDd1y+13MxWAg4FjgZ26FOrRERERKQmKjZm0N0XufsvgduBX1ZqvyIiIiJSPdWYQPI48Kkq7FdEREREKqwayeAOwIIq7FdEREREKqyc2cSndrJqJNEj+BHg7D60SURERERqpJzZxKd3snw+8BxwGHBxuQ0SERERkdopZzaxClWLiIiINAkldiIiIiI5Vs6YwfHlvJG7zyzndSIiIiJSPeWMGZzBe+9A0hODyngNZnY3sG0nq09w97PN7HTgtBLrv+fuk4v29xXgRGB9Yozjme5+dTltExEREenvykkGDwaOAtYF/gD8My03YF9gJnAesLQSDQQOB0YULTsgLb8ls2whcau8rBezT8xsT+ByYrbz7cAewJVm9qa731qh9oqIiIj0G+Ukg2sDLcAH3H1+doWZnQY8AKzl7v9dgfbh7k8VLzOz84An3f2JzOKl7v5wN7s7C7jW3U9Iz+8ys02BMwAlgyIiIpI75UwgOQy4qDgRBHD3uURZmW/2tWGdMbONgC2BK3r5ug2ATYCrilb9AdjSzMZWpoUiIiIi/Uc5PYNjgFW6WL9y2qZa9icuQf+haPkwM3sVGA08C/zC3c/PrN80PRb3NE5LjwbMqXBbRURERBpaOT2DDwPfNrOJxSvM7KPAt4G/9bVhXfgycI+7z8osexY4jhizuBvwEPDLNLGkYFR6fL1of4UeztEVb6mIiIhIgyunZ/BbwN3AI2Y2BXgmLS9cvp0HHFmR1hUxs62ADYFJ2eXuXnzJ+BYzAzjOzM5x97er0Z6pU6dWZD+tra0llw8bMZa2traKvEe9tdvobj9Lf/msPfks5aplDObOHc6sFxqvM7yz34c8UQwUA1AMQDGA2sSgnDuQPGVmHwKOB3YC9kyrXgTOBX7s7q9Urokr2B94B7iuB9teAxwIbAZMYXkP4Egg275Cj+G83jZmwoQJtLS09PZlK2htbWXixPd0sgIwe94Cxo1b0Kf9N4qWlqGMGzeu0/VtbW1drm8k3X2WctU6BmPGrM6aG5VVNrRquvp9yAvFQDEAxQAUA6hcDNrb27vswCqnZxB3nw18J/2rCTMbDOwD3OTub5axi6fT46bA9MzyzdKj96F5IiIiIv1Sn25HZ2YbmdknzGy1SjWoCzsCq9PzWcRfImoPTgNw9xeIJHCfou32Baa4e+NdLxMRERGpsrJ6Bs1sP6Jw8zpp0Q7AnWa2OvAgcLK7X1OZJi6zPzCXEvUAzayVKCbtwBAi4ftyakf2OuupwNVm9hzwF2B34LPALhVuq4iIiEi/0OueQTP7ItE79zTwPWBAYZ27v5aWf6VSDUzvuQoxS/gad19UYpNngaOB64mxgpsAX3P3H2Y3cvdrgYOIcY5/Jnob99PdR0RERCSvyukZPAn4q7vvaGZjgMlF6/9GhYtOu/u/ifqFna0vvvTb1b4uJ3oRRURERHKvnDGDmwJ/6mL9q4Du5iEiIiLSD5STDL5N13cg2RB4rbzmiIiIiEgtlZMM3gkcaGZDileY2TjgG8R4PBERERFpcOUkgycBawN/Bw4HOoCdzexs4EnivsFnVKyFIiIiIlI1vU4G3f0Z4BPEXTxOJ2YTHwN8H/gHsI27z6xcE0VERESkWno1m9jMBhG1BWe7+2fNbBTwASKpfF6Fm0VERET6l96WlhkIPAccB/zU3ecT9/0VERERkX6oV5eJU8HnNmKcoIiIiIj0c+VMILmUmE08tNKNEREREZHaKucOJP8EBgHTzexy4HlgYfFGVbg3sYiIiIhUWDnJ4BWZ/5/SyTYdxD2CRURERKSB9SgZNLPzgMvdvRX4dFq8CtEjuKRKbRMRERGRKutpz+C3gIeBVne/x8zGEPcg3sHd76la60RERESkqsqZQFIwoGKtEBEREZG66EsyKCIiIiL9nJJBERERkRzrzWzi95vZ/0v/Xy09bmJm/y61sbs/0qeWiYiIiEjV9SYZPCP9y/pFie0GEKVlBpXbKBERERGpjZ4mgwdVtRUiIiIiUhc9Sgbd/fJqN0REREREak8TSERERERyTMmgiIiISI4pGRQRERHJMSWDIiIiIjmmZFBEREQkx5QMioiIiORYb4pO14WZHQhcWmLV+e7+rcx2OwE/BDYDXgJ+7u7vKYptZscCRwBrAdOA49z9jio0XURERKTh9aeewc8BW2f+TS6sMLOtgRuBx4CdiOTx52Z2WHYHKRGcBJwP7AI8A9xsZlvU4gOIiIiINJqG7xnMaHX31zpZdyrwqLt/PT2/y8zGA6eZ2UXuvtTMWoCTiR7DyQBmdg/wJHASsHeV2y8iIiLScPpTz2BJKcn7DHB10ao/EJeCP5KefxxYDbiqsIG7LwGuAXYyswHVb62IiIhIY+lPPYNTzWwsMBO4DPihuy8GNgSGAE8VbT8tPW4C/B3YND1/usR2qwDrALMq32wRERGRxtUfksGXgdOAR4AlxJjAU4ANgAOBUWm714teNz89jk6Po4B2d1/YxXZKBkVERCRXGj4ZdPc/A3/OLPqLmb0BnG5mZ9WpWQBMnTq1IvtpbW0tuXzYiLG0tbVV5D3qrd1Gd/tZ+stn7clnKVctYzB37nBmvTCnZu/XU539PuSJYqAYgGIAigHUJgYNnwx24hrgdGI8YOFy8MiibQo9hvPS43ygxcyGuvs7XWzXYxMmTKClpaW3L1tBa2srEydOLLlu9rwFjBu3oE/7bxQtLUMZN25cp+vb2tq6XN9Iuvss5ap1DMaMWZ01Nxpfs/fria5+H/JCMVAMQDEAxQAqF4P29vYuO7D6/QQS4DngXZaPCSzYLD1OT4+FsYKltnuLqE0oIiIikiv9NRn8EtBBlJtpB+7kvaVh9gVeAR5Nzx8E3gD2KWxgZoPS625z945qN1pERESk0TT8ZWIz+zOR7E0FlhITSA4Hfuvuz6fNzgTuNbOLgd8DnwC+ARzh7ksB3L3dzH4ATDKzOUSSeDAxG3m/Gn4kERERkYbR8MkgcXn3a8D7iPY+AxwH/Lywgbs/ZGa7E3cX+QrQBnzH3X+d3ZG7TzYzgKOANYnxhru4++PV/xgiIiIijafhk0F3Pxo4ugfb3QLc0oPtJpO5lZ2IiIhInvXXMYMiIiIiUgEN3zMoItWxdGkHs+c1VumiYSPGltWm4S2DWXXlIVVokYhI81MyKJJT7YuW8OATjVXoO2ot9j4Z3H7L8UoGRUTKpMvEIiIiIjmmZFBEREQkx5QMioiIiOSYkkERERGRHFMyKCIiIpJjSgZFREREckzJoIiIiEiOKRkUERERyTElgyIiIiI5pmRQREREJMeUDIqIiIjkmJJBERERkRxTMigiIiKSY0oGRURERHJMyaCIiIhIjikZFBEREckxJYMiIiIiOaZkUERERCTHlAyKiIiI5JiSQREREZEcUzIoIiIikmNKBkVERERyTMmgiIiISI4pGRQRERHJscH1bkB3zGwv4MvARGA08BzwK+BCd1+atrkM+GqJl+/l7tcV7e9Y4AhgLWAacJy731G1DyAiIiLSwPpDz+B3gXbge8CuwPXAecCPirZ7Hti66N+d2Q1SIjgJOB/YBXgGuNnMtqhe80VEREQaV8P3DAKfd/c5med3mdkqwLfM7GR3b0/LF7r7w53txMxagJOBn7v75LTsHuBJ4CRg7+o0X0RERKRxNXzPYFEiWPAYMJS4bNxTHwdWA67K7HsJcA2wk5kN6Es7RURERPqj/tAzWMongXnAq5llG5rZ68DKwFTgbHe/OrN+0/T4dNG+pgGrAOsAs6rSWhEREZEG1e+SQTP7KHAQcEbq2YPoKZxCJHarAQcDV5nZMHe/LG0zCmh394VFu5yfHkfTy2Rw6tSpvf8AJbS2tpZcPmzEWNra2iryHvXWbqO7/Sz95bP25LOUq5YxqObn6Ity2jR37nBmvVDqIkL/1Nk5IU8UA8UAFAOoTQz6VTJoZmsBfwQeITOBxN3PLdr0BjO7EzgDuKxa7ZkwYQItLS192kdraysTJ04suW72vAWMG7egT/tvFC0tQxk3blyn69va2rpc30i6+yzlqnUMqvU5+qLcGIwZszprbjS+Ci2qva7OCXmhGCgGoBhA5WLQ3t7eZQdWw48ZLDCz1YBbgQXAbu6+qJuXXAuMN7Ox6fl8oMXMhhZtNyo9zqtYY0VERET6iX6RDKYE7kZgDeBz7j63jN0UxgpuWrR8M+At4KXyWygiIiLSPzV8Mmhmg4kZv5sDO7n7iz14zQCiVMyLmdnIDwJvAPtkthuUtrvN3Tsq3XYRERGRRtcfxgyeD3we+D4w3My2yqx7irjMezlwJfAsMJKYQLIdcEBhQ3dvN7MfAJPMbA7waNpuQ2C/qn8KERERkQbUH5LBHdPjj0us+zTwBNHjdzJxGXkRkejt5u43ZTd298lmBnAUsCYx+3gXd3+8Ok0XERERaWwNnwy6+/o92Gz3XuxvMjC57AaJiIiINJGGHzMoIiIiItWjZFBEREQkx5QMioiIiOSYkkERERGRHFMyKCIiIpJjSgZFREREckzJoIiIiEiOKRkUERERyTElgyIiIiI5pmRQREREJMeUDIqIiIjkWMPfm1hEpDtLl3Ywe96CejejIlZZbfV6N0FEckbJoIj0e+2LlvDgE231bkZFfPj9w+vdBBHJGV0mFhEREckxJYMiIiIiOaZkUERERCTHlAyKiIiI5JiSQREREZEcUzIoIiIikmNKBkVERERyTMmgiIiISI4pGRQRERHJMSWDIiIiIjmmZFBEREQkx5QMioiIiOSYkkERERGRHBtc7wbUmpltBPwC2AZYCFwFHOfuC+raMBEREZE6yFUyaGYjgbuAF4E9gTWAnwJjgS/Vr2UiImHo0GHMntf/v5sObxnMqisPqXczRKQHcpUMAocCo4APu/trAGa2GPi9mZ3l7tPq2joRyb1FSzq4Y8rMejejz7bfcrySQZF+Im9jBncG7igkgskfgXZgp/o0SURERKR+8tYzuClwSXaBu7eb2XPAJr3YzyCAd999tyKNam9vL7l88aJ3GTxwaUXeo96WLF7U5WcZutKAfvNZu/ss5ap1DKr1Ofqi3Bg04mcp19Ili5vis7z7bjsvzS7vHNmyymhemv16ZRvUB8OGDGbl4SvV/H07+9uQJ4pBZWKQyVcGlVo/oKOjo89v0l+Y2SLgFHc/u2j5/cCr7v6FnuyntbV1G+C+KjRRREREpFo+OXHixPuLF+atZ7BSpgCfBF4GltS5LSIiIiJdGQSsTeQv75G3ZHA+MLLE8lHA9J7uZOLEie3AezJrERERkQb1XGcr8jaB5Gli3OAyZtYCbEgvkkERERGRZpG3ZPAWYHszG5NZ9l9AS1onIiIikit5m0AyEpgKzADOYnnR6TvcXUWnRUREJHdy1TPo7q8DnwH+Dfwv8DPgauBrdWyWiIiISN3kqmdQRERERFaUq55BEREREVmRkkERERGRHFMyKCIiIpJjSgZFREREckzJoIiIiEiOKRlsEGY2oOgxdz8bxeC9FIOgOCgGoBiAYgCKAVQ+BrkPaKNw945UFHukma3s7ksBzGxQfVtWO4oBmNkQM1vXzDY2s5GZGAyod9tqLfuZC3EoXt7sFAPFoFheY6DjoLoxUJ3BBmBmewN7AbsALwLPAH919/My2wzM/vCbjWIAZnYgsC+wA/Av4A3g/4AfuPuCtE1TxyDLzFYHNgG2BZyIyTPuPi+tH+DuTX0CUwwUAwAzGwdMBHYG/gnMAh5095fS+qY/L+g4qG4MlAzWmZltADwG3AHcBGwBbAZsTtwp5SR3v6Z+Law+xWBZDJ4AriHujmPAB4E9gAHAKe5+ftq26U96AGZ2K/EHcAEwHngZeJBIkK9y9/Y6Nq8mFAPFAMDM7gPeD8wF3kecF9uAG4BzM18Wm/bcoOOgujFQMlhnZnYpsDqwT+YXeh1gO2A/YGvgT0Qy0FavdlaTYgBmdgGwAfAFd1+Ylo0EPgR8FTgQuAc41N2frVMza8bMziJ6io8A/pEWHwHsDqwK3A9c4O5/r0sDa0AxUAwAzOwMYB/gQHd/2MyGAfsDOwH/AcwAfuTut9WvldWl46D6MVAyWEdpLNylwEjgi8BSoCMzTmwjIhHYB7jJ3Y+pU1OrRjFYNt7jXOIb3w7uviB72cfMxgK7AUcTl4gOKCTNzcjMhhKJ71/c/eSidesC3wb2JGJxuLs/22w9IoqBYgAxhhi4GZjq7t8pWjcW+BKRGA4krqDc3oQx0HFQgxgoGawzMzsJOAyY6O6vpmUrufui9P9BwPeAScBe7v7HujW2ShQDMLOvAj8Gdnb31rSsOAZfBi4DvunuF9arrbVgZtcBS9197/R8JWBJJkHeAbgCeA7YtTBmppkoBooBgJn9BtjQ3T+dnhfH4GPARcBQYDt3f7luja0SHQfVj4FmE9ff5cT4j/vN7FMA7r7IzAaa2RB3X+LuZwN/Bz5az4ZWkWIQYz6eA243s8/DshgMSknhEnf/H+DPwEfq2dAauRf4opkdYmaD3H2Ruy9NJ0Dc/S/A54jB1J+pZ0OrSDFQDADuBLY1s9PTFYPiGPwN+E9gFHF1pRnpOKhyDJQM1lH6xZ4FHAq8Bvwq/cKv4+5L3f3dwnbAC8AH6tjcqlAMgrvPJWYSPwhcZmaXmNmGKQks9A4OAF4F1qtjU2vlUuBq4vLH0WY2HpZ/SUjbzCAS6C3q0sLqUwwUA9z9D8CPiOEy55nZh9PywpfFgcSEgieATaw5S63k/jigyjFQMlhHhe5dd78XOAmYDnwFuMrMTjCzUWkixReJwcJX1a2xVaIYLOfuLwLfJ37pPwHcbWbnmtkmZvYfwEHE7OLf1K+V1WOZYuPu/hZwKnFimwSca2ZfMLO1fXkJjSHEwOnX69HealAMFIOsTGL3S+LqwR5EDI4xsw+kL4tLgRHAOkBbs4yV03FQ2xhozGCNmdmqwJZEnaABwAvufmla10LMGv08MUtsTaInaBFwu7t/ox5trjTFYNlM4c8S3fmrAE8CV7j7S2l84GeJGOxE9ATOBd4CbnT3o+vR5lpIcRkILEonP8zsIOC/gcHAQ0R9rZeBXYH13X2D+rS2OhQDxcBi4siawHBgji+vI7cH8YVxHWAe8eV5FnEuHenuG9elwVWS9+MAahcDJYM1ZmbXELNGVyH+wI8n/sj/DPiFuy80szWJy6FrpvW3AS96KjnS3ykGYGY3ABsTyXAb8GFgEPA/wPnuPt3MVgFGEyf+dYCHiT8MTVdPy0oXHb/H3X+S2eY44FPE8TAO+D3wB3d/uPYtrjzFQDGATovP3wqc4ctLbx1EnEM3B9YCrgSud/fH6tHmStNxUPsYKBmsITPbnZgs8WV3v9nM3kckBHsRl0afB45Il0yLX9sUU+UVAzCzXYlf2t3c/R4zWw0YC+wNHAu8DZzo7r8r8dqmiEGWdV10/G3gdHe/Im27CpFAd7j7v+vT4spTDBQD6Lb4/EAiIfx52naou79jmaoDzUDHQX1ioGSwhszsZ8Qlvz19xfsKDiUum55GFFo+AZgMDHT3JXVoatUoBmBmpxDf5nYpTJDJrFuX6P7fD/ipux9bhybWlPWs6PiNRNHxf1nMpGu2Y0IxUAywnhWfvw84zN29SWOg46AOMdAEkhrIDAKeSSQB66XlgwHc/R13vw84BLiEqLk3oZkOcMVgBf8CPk1841+Bu/+LmFl9KvBlM9upxm2rqTQ+chDQASybHenuL7n774lC2xcQE2q+A9Bsx4RioBjAsnPkYmIySHbiwOvp3HgCcW5YHZhkZsObMAY6DuoUAyWDNZC5rHcv8ct+RFq+2MwGpx8+7v48cDwxI+g7pfbVXykGK/gr8BRwnJmtBSsky7j728BPgFeAb1pzlooAlp3EnJgsNKowO9KW1856hugt/i1RTqHp6qgpBooBLDtHthJjpTdNy7IxmEMUnT8H+C/ggPq0tHp0HNQvBkoGa8jjzhJnAUeZ2S1pSvhid1+S+UHPA64FVre4B2VTyXsM0pi/WcAPiZlf15vZR4vHAaZLRFcTZQJG1L6lNaWi44oBKAag4vOg4wDqEAONGawyMxvs7ouLlu0PnA6sDfwUOMuXF1dehRg8vMDd96xxc2smxeBMYrZw08egMNi7aNnWxGf/f0TtwJ8Ar7j7m2a2OjGzeJG7717zBtdIuvyxNJ3wzgZWI74IXOzuL2W3I2ZMDnT3verT2spLvb4DUgy2IW5JOAK4jpzEAHQcZFkUE/4lcRnwBuCH7v5cZv0AoodwTXf/XF0aWSV5Pw7qeT5QMlhlFvfdvdbd/5lZthIxAHTf9K8D+BPwDjF77EPAVumSab9nUTtwbWCEuz+Rlg0gkqCvETdbX0pzx+BnxHHwYGbZAGAjoqD2YcAawN3EZfS1iJh90t1fqHmD68DMPg18i7g88hJwC/Brotbax4nLIgd5k9yb2sxGe9H9Q81sW+AootRQG00eg1JyeBwY8LxnZgSb2QeJIvO7ECW4/hf4FTCMiMvPgK+7+3W1b3Ft5PA4qOv5QMlgFZnZN4ALiZuMv2BFZUHMbCwxXXxbYE+i1t6TRNJwRz3aXGlmth1x+5ztiGEJc0k1sYjxMS1EvaztiaSoGWNwMPELbO7+XInjYDjRQ7odkRi/QVwq+l93n1KHJleVqeg4FneUuRfYvPjcYDGz/mvAzsQfgbVozhjoOIhbyz0ATPSoLZo9DoYQyeAONHHxeR0HjXE+UDJYRWY2B/g5cHYaEzeA6PIdD8zwVE08s/1q7v5G7VtaPWb2EnGj9b8RB/A2xIG9gPim+xuPGbSF7ZsxBnOAc4H/TsfBYOKm8psD09z9laLtm6puWDFT0XHM7AHiXtz7FA8fyGyzFrAhzRsDHQdxHMwBvpQ9DrLDiyxKy6xGFBVuuuLzOg4a43wwuBI7kfcys58Ss0Ev8eXTvo8Gvk5k9kMsagkdX/hhFpKg4p6j/srMDgPaiQLKhYTvGjM7mZgRdwrwSTM7uHA5uAlj8FPiNkG/zRwHpwH7E7/UQ83sWuBUd/e0fvF799QcLIqOf5bSRcdPAw4ws0LR8dlFr22KWdUWdxb4MHGZpz0tW5vo/RhJ9I5c6e4ziXNI9rXNEgMdB8uPg0+w/DjYmJgpvE4aE/Zzd3+WuNfsi5nXNksMdBw0yPlAPYNVkC7/zgYmufvJadmpRDJY6CV7H/Bl4lvhbh7TxZuKmR0AnAx8xN3fTgfusiLSaQLFJcT4h8+5+9P1a23lpUkgrwLnuvt30rIziLI61wH3EOMCv0l889/Z3f9ep+bWhKnoOGY2E/i9u5+Qnm9L9BxvDrwJLCTuMvAjd7/YmrOoro4Ds1nA7zLHwWeIqyXrE+PDWog7E51BTCJpuj/WOg4a53yg0jLVsQ3xLW4/MzsvjQv5NlFIeF93P4c4yA8nuv73rldDq+xZ4hf9MDMb5u4d6TJpoYjmQ0Q19QVEdf1mY8AjwDfM7EYz+xjwDaJH9Ah3vxL4BdFL2A4cXLeWVpmp6DgAZvZN4ovgSma2WVp8EfACMWZqDaK+5kzgRDPbuJlioOMgmNmXiXP/+y1mjQKcDzwO/D933wDYB7gKOIY4NpqGjoPQSOcDJYNV4O5/Iga93kp0gT9AFBm+pjAWzKOq/DXEfSi3slR0uclMAX4HfBfYM02UwFMRzfT/x4iBs582s6app2dmK7v7A8RxcDox1uMh4u4jNxR+od19kbv/jYjVZmY2rFkuf2S5io4X3E3MCNwfmJyGCKxE/I7c7+7vuvtVxHCStYk7TjQNHQfLPETcReKDwM/M7G6iJ/B4YgIdKRk6iojTV+rTzOrQcbDM3TTI+UDJYJW4+z3EN7pTiUKidxBdvsu+FaXZYs8TYzebJhksfMNJA6CPJu6leSnwSzPbxFJx6YwnieLKzfSt7ydm9ll3n050+R9CXOa4iRj/kz0OBgOzCi9sxstBBZ7zouPu/rS770oMDVgD+AIxZOAld+/I9Iy8QMy2X7lJvxzk9jhI46Gfd/dvAUcCM4irCNcTdUaXZpKh+cTVhSElzpv9Xp6PA2is84HGDFaQxRT5CcAniVmiN6fl44CR7v5Uujy6NC1fj0gSL3f3s+rV7kpKk0Z2JC77/jvzWQ8jLo8OIwqm3k3ccmdz4EdEr+nxdWhyxZnZocTYn+eJYQFT0vKRxO2FXig6DjYgblH3O3c/vT6tri3LWdHxzqTfi8Xu/pui5WOI5OAxdz+qHm2rNFMB/kJB6TW9qGSUme1DjIe7smj5KOI4cHc/pGYNrTGdD0I9zwdKBiskfZO7lKgF1AGMIS4Pf8lXrBo+KH3r2ZAYB7Gvu7+vHm2utPQt5m3gJHefXGL9eOKy6beIntAWooTAXe6+Xw2bWjUpBnOJX9ytidlfX/WiwtGZ42ATotfwi+6+Xq3bWwumouPAshP6Ku7+YtHyAakXYCWPW04NIeqOXkbUpmyKouOmAvyY2WPEbNnt3f3h7GSAEsdBC3EcXEpzHQc6H9B45wMlgxViZucRs57OIsYB/gfR43Wfu+9ftO0goqL8x4DD3P36mja2SszsQmAr4kT3Wlq2CZEgrwrcBTzh7q+b2W7AfKLA8j+9k9pK/U0mBtsSJSIuJi5xHFCiV2Qg0SO4CXCou99U4+ZWnanoOGa2BTFR6gBiqMhjwMleogB52v7bxBem6939ezVvcBWYCvBjZl8lErvZxFWRA9z9X6V6TNP2pxBfnq9z9+Nq2tgq0fmgcc8HSgYrwKI20hTgSE+3B0o9RMcAk4BPufuDtmJV8S2A8c2SAJjZB4DpxCXie9JA4G8RA6DXJWYMjyQGyx6W7S1tFma2EXGS/0IhwU9/BM8l/gic6O5vFF0iHgP8h7v/tU7NripT0XHM7ElgHvAoMUxie+AfwFe8qGCsma0LfI8oSv6VZhk/airAj5nNI+41+wJwBXAjUV/vPV+EzWw0kTSNAw5pouNA54MGPR9oAkllfJYoJfMMLOvmXUyMfZhB3FKITCL4PqKCfFMkgsmxxPE0OiWCo4kK8jcR3/Y3I7r/twJusKim3mwuAm4mZpEXXAdcTcwG+yJAJhEc5O5zmzgRzBYd/6W7X5PGu4wjLgGdAlxuZu8vvMYzRcfr0eZKs6gruRT4hketycOBnxDHwr5pm2WTx9IfwnOAo5ooAeisAP8DxNWCl8zs3OzkgCY8Dn5K9IL9xt2vJnp6diQmmg1L2yz7e5wmTZwNfLuJjgOdDxr4fKBksDL+TdxKZwZE0pd6fxYTycCyQa8W5VNuIsaJNYV0EruR+FxXm9mfiG++twI/cPdHgFfd/Vric38I+Ei92lsNaRzM08SJbtltotx9vrsfRCSEF5rZ18xsYPrC0Eyzp0t5m7h/5jyIE3pKgN9090OJOy+sDdxlZptmX9gMfwDT7/qOwP8Qk4lw96Xu/mvij9+X07JlY8bS8395zCLt99Ll36OJckovp2WnEn/4pxNDaS4lYvFo6l1fpkmOg3WJ0iknFYbPEJdGryBmkR4Iy78kFrj7QndfUMOmVpvOBw18PlAyWAEe9QJ3TZcACz/Awi/2bcAHzGyr9HwvorbUebVvaXWkA/oWYgzEgcAGRG/gLaQyKhlPEL0E69awiVXn7u3ufri7P5ldnvmWdw4x9uVE4IPNcHLrgbwXHV83/Vvsy+8zWzjn/gH4qJl9MLP9p8zsZmuu8hkqwB+fdRpRYqxw5ehNdz+MSIQnm9kh6UtiM/9N1vmggc8HzXzg1ZS7z0iPxX/kHwJeArYzszVYfmuhebVtYfW5+5tEkendiMvGj6Re0sLsqAHERJJBvDdJbEq+vLj0VOIP3WLgT2b2H3VtWG3ktug4gLtPI7703QHLkoDCl8T7ibqan0vrRhC9ZaOKxw31Zx4F+A8gxwX43f0bxK0mF6TnHZkk4GdED+nxwCbFvYNNppXoDc3z+eB8GvR8oGSwitIPu52YMfp54DhgkLufUd+WVU/6tjcTuMjdH02LC8fZCKLncEkaN5Mb6XLIs8SkonWJnpCmlOkdLxQdv5+cFR235UWDf5S+CKzwRdHdZwN/BnZNi3YnZlh+sbYtrR5bXjD3fuLuESeTvwL8QwDc/ZXs8kzyM5X4mS8E/s/ilpVNyaNu4NHAg+TsfFDg7pPcfWrqCW2o84FmE1dRpkfs88TU+QFEPbk/1bdltZe+BR5J/FE4xN1vrHOT6sbMjgGedPe/1Lst1ZKSoYGF3h8zO4T4pjucqJd1D01adLwgJUNLOhsSYGb/RfSUfAr4PXCTN0kpmYJMQli4LLYGMMbdn7YmL8BfkH4XlpY6DgoxMLPdiQlojwL7NdGY0c5uxPBN4CSW34Sgac8HRTF43N3/nJYvO/7T87qeD5QM1oDFnSceB55298/VuTl1YWYbE5cM72qWX/LeshI1pJpJ+qN3EnBxZrLASpmE8P1Ez/A3iS9GQ2m+ouOlYrCssHBmu4HEDeqnECU21nL3sbVubzV0dhwQiXH2j18zF+Dv0XFQ9JpvAke7u9WomVVlnd+IYW93fzn93L/K8qskzXo+KI7BvcSx/nJmu7qfD5QM1kg6Ga7s7q/Xuy31kn4xBntmtq00DzM7G/g+cZnnUuBcX15OqbjI8OeJcaNvAdO9eYqOdxWDUknhX4HPAHu5+x9r3d5q6CYGKxRYtuUF+LciCq9fX/sWV15vjoPMFaSViF7TV0rutJ+x0jdi+DFwr7/3RgyF88GbxK33muV80OObUaTt63Y+UDIoIn2WSojcS5SNmE9c7pkBTC4MCUjjw4Y065eBXsRgoC8vH7EJccvK0+vR5krrbQzS/ycQBfhvrk+rK6uc46DZWPc3YtjW3R8o/nLQTHoQg1I3o9iU6DU8tdbtHVzrNxSRpvQRYC1i9uyVwMHAHsAFZrYvMMmj7E67mQ0lak22AA800aXznsZgiUVdyg8Tl46aaYxcb2IwhEiUBhFlqJpFOTEYSnP9LpS8EYNF8e1DiMumD2TGkq5LJMcvdrbDfqi7GOwCPJhJBNcjekdPq0djNZtYRCphOjEQ/JpUKmQyMWHoSqLn5wYzOzuVTBhO1NXauYn++EHvYrAyMVh8jybrHepNDFZJy3fP8XFQiEGz/S709kYMN5KKbzeR3sbgemKoRF2OA10mFpGKMLMh7v5udtJIWv45opjsNsAbRN3NHYhB0k0xa7JAMVAMQDEAMLP13X1GifHC2xEzxz/h7g+b2deJ+xKv5U1Wf7c/xUDJoIhURVHpkBHAF4ib0m8DHOvuP61n+2pBMVAMQDHISkMkngEuAC4hyulc7E1cf7dYI8ZAyaCIVFWmhMhg4CpgC3ffqLvXNRPFQDEAxSAzc/oSwICHibqKa9e5aTXTqDHQBBIRqar0x28g8J9Ej8ie3byk6SgGigEoBhl/IsbIbU0T3XWnlxoqBuoZFJGaMLPxwP7uPqnebakXxUAxAMXAdCOGhouBkkERERGpKd2IobFioGRQREREJMdUZ1BEREQkx5QMioiIiOSYkkERERGRHFMyKCIiIpJjSgZFREREckzJoIiIiEiO/X/icxdMgxke1QAAAABJRU5ErkJggg==\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAnYAAAFKCAYAAACQBBKyAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAACHR0lEQVR4nO2dd5wURdrHf7MLLDkHEUkCFiKY1hzOHFAx5/CennreneE8MZzhxHzmnD3jmdBTCZJEohIUlpyKuOSwwO4Cm0O/f1T3THdPdZye6ZnZ5/v56DLd1RW6u6qefup5noooigKCIAiCIAgi88kJuwIEQRAEQRBEMJBgRxAEQRAEkSWQYEcQBEEQBJElkGBHEARBEASRJZBgRxAEQRAEkSWQYEcQBEEQBJElkGBHEBkAY6wXY0xhjN2oO/YYY8xXvCLGWCFj7BMX6U5Vyz3VTzlJqM9UxtjUBMo5gzE2jzFWobarrd+8shnG2CeMscKw62GGMdaCMbaNMXZz2HVxA2NsDmPs+bDrQTQsSLAjiIBhjN2oCg3HhV0XIgZjrBWAbwHUA7gDwA0AykKtVIgwxgaoHwe9wq6LB/4OoAbAf8OuiEv+DeB2xth+YVeEaDg0CrsCBEH45ikAz/q8lkEIOJnG2QlcexiAdgCe5JyPDKg+mcwAAMMATAVQaDp3K9Lsw58x1hjA3QDe4ZxXh1wdt4wAsAfA7QD+FW5ViIYCCXYEkaFwzmsB1Pq8tirg6qSEBCf0zurf0iDqAoilQc551mn9OOc1YddBwgUAOgH4JuyKuIVzXs8Y+x+APzLGhnHOM/FjisgwSLAjiBSg2o9dDaAPgLcAnAmgAsCnAB7gnNfp0rYF8CqASwAoAEYCeEWS52MAhnHOI+rvHwEcCqAn51wxpf0ZQB/OeW/1dyGAqZzzG3VpDgDwJoCzIJYovwAwXlJu3LXq8akAwDk/Vf3dBMDDAM4D0BdAUwCLATzDOR8hv1P2SMroBWAdgAcB7ALwTwAHAFgE4G+c8zm6605Rs5nCGAOAT7U2MMaOBvA4gBMBNAFQAOBfnPMpurIfg9BwDQLwAIDzIYRE7Z6erbY3X73kVwD/5Jwv0OXxCdy/BxEAf4PQnjGIZzIfwBOc81906a4F8A8AAwFUAvgZwP2c83U29/FGAB+b7gcA3MQ5/0St56mc8166axQA76n5Pw7gQIj7fBvnfAFj7FYA9wPoDuA3Na+1pnId77MNFwPYyjlfasozrq7q8ceg6x/qsTMQe4Z5ALYCGMc5v0OXJg/iPboeQA8AOyGEyYc55+WmMq6G0CIOglgiXgLgBZNGeCLE0n8+gDku2kkQCZFWqnaCyHJyIASlXQDuBTANwFAAf9YSqJP5SAj7ry8APAJgf4iJ34mvISbV4/UHGWOdAZwKYLjVhYyxZgAmATgHQrh7GmLyTcTwuzWA2wDMgBB4Hoa4Bz8wxgYnkK+MqyCEivcg7lkvAN+ry3eAaM/r6r+fgbi/7wEAY+wUAL8AaA/gCQihLQ/ATxZOI8MhlnQfAfCamse1EM+2EkLIfAxC8PmFMdbfdL3je6DyPsSz2Kbm+TSAEgB/0BIwxv4J4HMI4XYogBcBnARgBmOsk/xWAQCmS+7HDepxO06A+Mj4r9pGBuBHxthfIITLdyDemeMAfKK/0Md9lpU910U6KYyxAQDGAGim1v3vEDaXJ+rSRAD8APEujQFwJ4RQ9zcAI9TzWtpHAHwF8fH1OMRS62qIPqSnQP17IggiBZDGjiBSR2MA33LOn1B/v8sYmwfgZogJEQAuhJi4H+CcPw8AjLF3ILQkToyE0P5cBWCm7vjlAHIhBD8r/gzgIABXcc6/Uct9H0JD5JdiCO1hdNmXMfYmgHkQQsi4BPI20x1AP855sVoOh7gf5wD4kXM+kTHWBsBdACZyzqeq6SIQAt6vAM7SNJ2MsXch2v4MhEChZwXn/DJdm1pACGCfcM7/pDv+IQAO4FEA1+qud3wPVEHnFgBvc85v1137iiZcMMZ6AHgSwGO6vMAY+xrAUghB6yHZzeKcr2WM/WK+Hy7oD+BgzvkataxiiPv3BMT9L1WPNwLwIGOsL+d8tc/7HEXNrw+EsOWXsyAEycGc85264//U/fsaAOcCOI1zPk1X/lwIAfosCEG0D4QwNwrApRJNaxTO+WbGWDWETSNBJB3S2BFEavnA9PsXCM2OxnkQTg2aoAd10njLKWPO+V4AYwFcwRjT9+2rIISRBTaXnwdgO4D/6fKrAPAfp3Jt6lOnCXWMsSaMsfYQWrzpiC1XBsV3mlCnoi1VHihLrOMwCK3TlwA6MMY6MsY6qvWcCOBYxlhz0zXvmH6fBaHB+1K7Xs0jV63HaZJynd6Dy9W/w8wX6pbZL4X4OB9uKrcUYslbVm6iTNGEOpXf1L/fa0Kd6bjWJj/3WU97ABGIjwW/aPW72NQ/9FwJYCWApaZ7Og1CM6fd00sg5s8n9UIdYHg+eooBdEyg7gThGtLYEUTqqOGcbzUdK4YQCjR6AtimCml6Vros42sAl0Fo/aYyxvaHWJp70uG6ngDWSIy73ZYrhTF2C4Tm6GCIiVnDV/w9Gzbof3DOi1W7sXby5FEOUv9+aJOmAwC9bdUa03ktj4kW15vvqZv3oA+A7SbNkhmt3BUW59daHE+EDabfmrC00eK41iY/91lGxOG8HcMhtKIfAHiWMTYZwmv1G9URSasnA1BkkYfmgNNH/bvUIp2ZCIJ/5wlCCgl2BJE6UuERNwbAXggt3VQIDUQO7JdhvWI1QeUC0C9JXQcxiY4G8ByAHRBevDfBuDQZBHUWx50EAU1z80/EbKHMmCf5Cos8bgSw2aE8ILj3QCt3MOTe0eZ6BoHVfXa6/37us55dEO+dTFC3ex+jcM4rVDu/P0BoqM+BsGO9hzF2sqqhzgGwDML+TsYWmzra0RbCCYMgkg4JdgSRXqwHcBZjrJVJa3eQ1QV61MlrFIDLGGN3QAh4CznnVlodfbmHMcZyTFo7WbnFEBOVmZ4waomuUH9fpF+eYozd5NySlKFp3/Zyzt3YMdrlUZRAHrI8z2WMdeKcWwk8WrkbOOfLfJSRSg1SQveZc17HGFsF1QPZhN37aM6nHuKDZyqA+xljfwXwNsSy9hdqPfMBTLJYUtXQ2nMIHBw6GGPdIDyAl9ulI4igIBs7gkgvxkL0y79qB1R7oNstr4jna4h4XzdBeCe60daNBdAFMdsuzVP2FknaNQCOU8OZaGkvgHBg0KNpcfSehAdC2CelCwUQnoz3qDtTGHDwLNWYAOGt+pD+nnjMw4xm6/iYJD/tfn4HcY8fNRvsq+mcbLq0+HtOy9VBEMR9ngHgKMnxNQDaMMYO1eXXFab3jDHWQXLtPPVvW/XvcIh+8FdzQsZYnq7uP0BoXh9ljOWa0pmfhWZPOhMEkQJIY0cQ6cVoiAns32qMtqUQ8bvae8hjAoQW42X1txvB7gOIWFufMsbyIZYUrwcgC2T8HwgBcDxj7BsIe6PrEW97NgpCEzJK1SJ2gwgbwQEc7qE9SUMNIHszRPiRZYyxjwBsgggxcwqEUGrrhMA536OG+/gCwHzG2FcQjig9IDwsl0Is03qp11Q1PtvfVA9MzYP4eIjYcc+onq3/BPACgJ6MsREQAmZvABdBCCmP2RQzH0IwfJCJ2IkVAH6zi3/nlyDuM4SX802MsUNMsey+hljq/4Ex9jqA5hCC2UoAR+rS/Uv1Nh4DsdNGOwB/gRBwf1TTfA7xbr+lLtv+qtaNQZg1XAERw3ENY+wJiPv7K2Psewj7wCMhQt7oP8TOUtvqO1QLQXiBNHYEkUaoS0UXQggJ10HELtsK4I8e8qgB8D2AVhATdaGLa8oBnAHgJwgB718QGob7JWknQIQrOQgikPLxELsCbDKl+1S9fgBEzLTLIBwp0mo7L875dAjN5mwIwfNNAH8CsBtCYHCTx3AIwWQDxL15HcKOcDnUeHk+uBnAPRCa0Ochnkl7CA9NrdwXIQT/aoi4ei9DaKqmQsRos6vzdojgx+0gBPuvEAviHDgB3OcxEHaaV5ry3QXR5nKI+/RHiLh/o03Xj4QwDfijWvbdEMLtiZzz9Wpe9RAfI/dBvLcvQIQ1OQ5iyXaRrtzH1bwaQ4R7eQpCAJygpVG17ZcD+Ix2nSBSRURRyFGHIAiCSH8YYw9CaOP6pOm2ZwYYY5dCaAH7SDyhCSIpkMaOIAiCyBReg3BE+L+wK+KSBwG8SUIdkUpIY0cQBEEQBJElkMaOIAiCIAgiS2jwXrEFBQWNABwAYFN+fr4syCdBEARBEERa4CS3NHjBDiKI5WoAJxcUFGxySkwQBEEQBBEiB0DsL90X8WGmSLAD0FX9+4ttKoIgCIIgiPShK0iwk7IVAA466CA0aRIXND5QlixZgoEDBya1jHSlIbcdaNjtp7Y3zLYDDbv9DbntQMNuf7LbXl1djZUrVwKq/GKGBDt126MmTZogLy8v6YWloox0pSG3HWjY7ae2N1wacvsbctuBht3+FLW9TnaQvGIJgiAIgiCyBBLsCIIgCIIgsgQS7AiCIAiCILIEEuwIgiAIgiCyBBLsCIIgCIIgsgQS7AiCIAiCILIEEuwIgiAIgiCyBBLsCIIgCIIgsgQS7AiCIAgiC9hStA/XDxuHouKKsKtChAgJdgRBEASRBYyfvR6l+6rxy4JNYVeFCBES7AiCIAiCILIEEuwIgiAIgiCyBBLsCIIgCCILUBQl7CoQaQAJdgRBEASRVUTCrgARIiTYEQRBEARBZAkk2BEEQRAEQWQJJNgRBEEQBEFkCSTYEQRBEARBZAkk2BEEQRAEQWQJJNgRBEEQBEFkCSTYEQRBEEQWEaFoJw0aEuwIgiAIgiCyBBLsCIIgCIIgsgQS7AiCIAiCILIEEuwIgiAIgiCyBBLsCIIgCIIgsgQS7AiCIAgiC1CUsGtApAMk2BEEQRBEFkHhTho2JNgRBEEQBEFkCSTYEQRBEARBZAkk2BEEQRAEQWQJJNgRBEEQBEFkCY2cEjDGrgBwHYB8AO0BrAHwDoD3OOf1unSDATwNYACAzQBe5Zy/IcnvXgC3A9gPwFIAD3DOJ5nStALwAoDLATQFMAXAnZzzQlO6fgDeAHASgAoAX6v5lbtoO0EQBEEQRFbhRmM3FEAVgPsAXABgBIDXATynJWCMHQ9gFID5AAYD+BjAq4yxv+gzUoW6ZwC8BeB8AKsAjGGMHWYq8ysAFwK4E8BVAPYHMIkx1lyXV1sIga8VhAA4FMA1AD5y0SaCIAiCyCoUULwTwoXGDsAQznmR7vcUxlhLAHcwxh7hnFcBeBTAPM75zbo0PQAMY4y9zzmvZ4zlAXgEQpP3IgAwxqYBWAzgYQBXqseOhRD6zuecj1WPLYbQFN4I4G21jNsAtANwOOd8p5quFsAXjLEnOedL/dwQgiAIgshsKN5JQ8ZRY2cS6jTmQyyRtlcFttMBDDel+RJiufVI9fcJANpALJdqedcB+AbAYMaY9iaeB6AUwHhdug0AZqjnoEs3SRPqVL6D0C4OdmoXQRAEQRBEtuHXeeJkALsB7ADQB0ATAMtMaTSNWX/178Hq3+WSdC0BdNOlW6G339Ol66/7fbC5TFV7uMaUjiAIgiAIokHgZinWAGPsKAA3AXicc17HGGunnioxJS1W/7ZX/7YDUMU5r7BJt0lNZ85LS9de99ttOlcsWbLE6yW+KCgoSEk56UhDbjvQsNtPbW+4NOT2p7rtO7aXAAA2bdqIgoKSlJYtg559OHgS7Bhj+0Esd/4OnfNENjBw4EDk5eUltYyCggLk5+cntYx0pSG3HWjY7ae2N8y2Aw27/WG0fd6mxQDfhwMO6I78/D4pLdsMPfvktb2qqspWGeV6KZYx1gbAOADlAC7knNeopzSNW1vTJZomb7cuXR5jrKmLdOa8tHS7db/dpiMIgiCI7IecYgm4FOxUYWwUgM4AzuWc79KdXgOgGjEbOo0B6t8V6l/Ntk6Wbi9E7DstHdM5U+jTrdD9Xm7OS3Xk6GNKRxAEQRANhgg5xTZoHAU7xlgjCM/VQwEM5pyv159XHRYmQw1XouMaANsAzFN/z4Twdr1Kl3euet14zrn2rTEWQhN3ji5dd4ggxGN1+Y8FcAZjrIPu2CUA8kzpCIIgCIIgGgRubOzeAjAEwP0AmjPGjtOdW8Y53wPgCQDTGWMfAPgCwIkAbgVwu+bdyjmvYow9BeAZxlgRhMB3C4SG7VotQ875b4yxMQA+ZIwNBaDlvwHAJ7qy34MIYDySMfYkhDbxZQDDOedmD12CIAiCIIisx81SrKY5ex7ALNN/RwIA53wWgIsAHA1gAoTA9g/O+bv6jNTAxA8BuAvCXq8/RCDihaYyrwHwI0Qw4m8hNH9n6rcK45yXQMTP2wfgewCvQMTS+5OLNhEEQRAEQWQdjho7znkvNxmpu0Q4LoGqwt2LDmn2QuwscZtDupUAznVTP4IgCIIgiGzHb4BigiAIgiAIIs0gwY4gCIIgsgCKdkIAJNgRBEEQRFZB0U4aNiTYEQRBEARBZAkk2BEEQRAEQWQJJNgRBEEQBEFkCSTYEQRBEARBZAkk2BEEQRAEQWQJJNgRBEEQRBagKBTwhCDBjiAIgiCyC4p30qAhwY4gCIIgCCJLIMGOIAiCyDh2FJfT0iNBSCDBjiAIgsgo+PrduPmpifjpt/VhV4Ug0g4S7AiCIIiMYuP2fQCAZet2h1wTgkg/SLAjCIIgMgxagiUIK0iwIwiCIDKSCHl/GiF5lwAJdgRBEESGQT4T9kQo3kmDhgQ7giAMlFXU4LclW8OuBkE4QgIMQcRDgh1BEAZe/KIAT338O7bvLg+7KkSCjPplDdZv3RN2NQiCSCGNwq4AQRDpxdadZQCA6pq6kGtCJMoHI5YgJwKMfPGisKsSKLQSSxDWkMaOIAgTYtokw/TsoD4LpSDNxo7eUYKIhzR2BEEYiE2aNGv6pbK6Ftt30VJ28shCaZUgAoI0dgRBGNCmTBLr/PP8f+fijhenoLq2PuyqZDX08WGExF0CIMGOIAgT2v6bNGn6Z8maXQCycxk0HaBwJ/ZQ123YkGBHEIQBsl8KApI8UgG9owQRDwl2BEEYIJEkcaLCcbjVyFroHbWHNJoNGxLsCIIwQkuxBEEQGQsJdgRBGCDnicRJB4WJks1qG/r4sIVuS8OGBDtCypChI/HcZ3PCrgYRAgpJdglDdorJhV5RgrCGBDvCkl8Xbgm7CkQYaNoQmjb9kwbasjSoApFislpLS7iGBDuCIAxEtSEk1/kmplGim5gMSH6xh966hg0JdgRBGKBlxAAJ8R42CNmH3lGCiIMEO4IgTDQIkSCpkEYpydANJghLSLAjCMIA7RUbBJqdYphVyH7hh95QgoiHBDuCIAyQx2HipINMlQZVSBrZ3DaCSBQS7AiCMEKSXcKQA0pyIa0yQVhDgh1BEAbqKdxJVpAOWsNkoaTDUncaksWPnPAACXZE4IybVYiSvVVhV4PwCXnFBkA2S1XpBL2jcqjzNmhIsCMCZUvRPrz9v4V4lnatyGBIKAGAD0YuxvT5mxLO59IHRuPOF6cEUCOv0HMkiIZIo7ArQGQXNbX1AIA9ZdUh14TwC9kvCUZNXwsA+MMRB3i+Vi9S1dTWo3DrnoBqRQAgmZUgbCCNHREoZDSe+dAzTJx0WIlNhzokm4b+8UEQMkiwIwjCiEKG6dlANst12dw2gkgUEuwIgjAQ09iRaEekJ1FzgXCrQRBpCQl2DZAxM9bhq594UvJWSNuT8ZBXbHagZPVaLEl2UrL5kROuIeeJBsi73y8CAFxzNktaGaTtyWRodiAyA4q1KIeG34YNaewIgjBAXrFZQhbL51mtjCSIBCHBjiAIA7SjWHAk6x5WVtVixLQ1qK9PLwnn2c/m4JWv5qWsPPr2IIh4SLAjAoW+pDMfhSS7tOfTscvw4aglmLVkq2WaMLrijIVbMHnuxqSXQ+MMQVhDgh2RFOhLOoPRHGDoIaYt+8prAABV1XWWabLbeYIgCCtIsCMIwgAp7DIHkr0JgjDjyiuWMdYXwL0AjgMwEMAKzvlAU5pPAPxRcvkVnPP/mdLeC+B2APsBWArgAc75JFOaVgBeAHA5gKYApgC4k3NeaErXD8AbAE4CUAHgazW/cjdtI4KFtASZD4U7IdKf7NAq7yguBxSgc/vmgeRHoy8BuNfYHQLgfACrASyzSbcWwPGm/ybrE6hC3TMA3lLzXAVgDGPsMFNeXwG4EMCdAK4CsD+ASYyx5rq82kIIfK0gBMChAK4B8JHLdhFJgsIQZC4KGdkFB93CpJAtAYpvfmoibn56YuD5Zvp9IRLDbRy70ZzzkUBUM3eURboKzvlsq0wYY3kAHgHwKuf8RfXYNACLATwM4Er12LEQQt/5nPOx6rHFANYAuBHA22qWtwFoB+BwzvlONV0tgC8YY09yzpe6bB8REPTFmPnQXrHpjxvFeDYrz7O4aQSRMK40dpzz+oDKOwFAG4jlUi3vOgDfABjMGNOmkvMAlAIYr0u3AcAM9Rx06SZpQp3KdwCqAAwOqM4E0bCg3UOyimwW0LO5bQThl6CdJ/owxkoYYzWMsfmMsatM5w9W/y43HV8KoCWAbrp0KyQC5VIA/U35GZaGOedVEJo9fToi1dCAm7FEtSE0a6aMEdPWYN2W0kDzzGatFtnyEoQ1QQp28yEcLC6GsHfbBOBrxtiNujTtAFRxzitM1xarf9vr0pVIyijWpfGSjkgVNN5mPNliv5RJfDhqCe56aarn62yfEWleCaJBEthesZzz10yHRjLGJgN4HMAnQZWTLJYsWZKScgoKClJSjhvc1MVrfbfsrgYAVJSXx12bTm0Pg0xr/7x5BYF5HWZa2/UEVfcg+9uu3bsAAOvWFaIVdkjTVFSLBQ/FQ75BoS8vGWVv3rwHALBt+3YUFFQGnn9QuG17UPeoqEjoSNZv2ICCvN2B5JkImdzvEyXMtgcm2FnwLYC3GWOdOOdFEJq0PMZYU865vje2U/9qb2IxgB6S/Nrp0mjp2lqkW+GlogMHDkReXp6XSzxTUFCA/Pz8pJbhii83AYB9XdykkdBmYwkwfgeat2huuDZt2h4SGdV+9dkfdZSVj5Q3Mqrtenz2Af21erE4yP42eflcYP1m9O7dC/n53aVp9pZXA//bgoiHfBPG1I5kPfs1xSuBhXvQdb/9kJ8/IPD8g8BV2xN5xyTMWrsAWF2Gnj16ID+/dyB5+iVj+30AJLvtVVVVtsqoVAco1mzrDjYdHwBgL4DNunRM50yhT6cX2Jab81I9b/vAo2BHBIOShLXY14fPx0+/rQ88X4JoEGShraQSjWMXckXSFRc3ZmdJBd75biHq6oLyjSTShaQJdqpQdiWA9aq2DgBmQni7XqVLl6umG88516SCsRCauHN06bpDBCEeqytmLIAzGGMddMcuAZBnSkekiGTYZ038fQPe+GZBgDkSRPqQkCOAzQRO/gWEHW9+uwBjZxZi4aqdzomJjMLtzhPNEQsz0hNAa8bY5ervOerfTyGCCq+GEMpuAXAqgBu0fDjnVYyxpwA8wxgrAjBPTdcHwLW6dL8xxsYA+JAxNhTAHgBPANgAo73eexABjEcyxp4E0BnAywCGc87tAikTyYY+pQnCFb4EMBLaiATR3rt6+gLIOtza2HWGsJfTo/2+CcAoCE3cI2raGgih7ULO+Wj9RZzzFxljAHAXgC4QIUzO55wvNOV/DYAXIYIR50HsMHGFfqswznkJY+x0AK8D+B6xLcXud9kugiCIUElW6I6sDgmSxU1LGfTtnbW4EuzU/VmdXoOL3Baq7jrxokOavRA7S9zmkG4lgHPdlk0QBJFOJFtGyeb5O9P3ig0T7c5l9QdAAyXVzhNElqNQ7CyCiOJG8Ij2GR+dpqH2MxJFEkd7N+leZh8k2BEEQYSIH4cjN5NxQ1DENFTBNlAawHvS0CDBjkg7vp20EisKww+uaWbx6p0YMnQkdu+pRHllDW4YNh6LVhc5X0gQNmjzarKWFbNxtTIqtGZh24LAzW3J0TR2DeELoIFBgh0RKLFJyn8en41djvve+CWQ+gTJ6F/XAgCWF+7G+q17UbKvCv8da972mCC8odRrS7HBSinJiCmZNkRNPkiy84v2umXxW9JgIcGOSAo04DYctu0qo6/+BNDuXI6HLuPKLi/6SLKvL5rftlUbi1FUbN6CnHADdd3sgwQ7gnCJzDGkoY+JfP1u3PrMzxg3qzDsqmQsUaE4G9dMk4x2y+55dTr+9NRP4VYmY2noo1j2QYIdESxZPEbo59/oHJzF7XXD5qIyAGJ5mvBHsuS6bH4100nLdNXDYzD6l7VhV8Mz0aXYNLqXRDCQYEcESsMwao5E25fVdkyuaOjtTxw/S7EabkweslkRGHbTFEVBeWUt3h+xOOSaCLwIaRTuJHshwY5ICmEPuMkm29vnFbof/lF8fA25C3dCU3ayqa9Pz3vsSZhPzyYQCUCCHUF4RD9o0tzpTF29gro0nQDTAe0d8qOxSwd2labeaSGqKQ9ZHants5qTgQ8vQqsOWQsJdkSgNJRBgpYx3PPk15vxwJvpF74mXYh5uGaecLBi/W7c+MRPmDx3Q2oLTg+5DnV1qmAXdkV8EB3DaBDLOkiwI5JCKiepyupaLFu3K+nl0ADoH76+OOwqpC0JOU/YXONnRwuvrN+6BwCwdG1qnWfSpStmtMZO/UtL9tkHCXZEoIQxRrz5zUI88Oav2FFcntRyNG2kYQhv4INitjY/lZNd9L3yItml2X0PSzgIW5zSbOwyUK7LSA0x4Q4S7IiMZ+2WUgBARWVtUsuJaVYiFLXdRLZNEqmUU5IW7iQlbQjnuaeLlimInXbCJk1uJREgJNgRhFcitLOGGW2iLa+swazFW0KuTeKkcq5Luo1dJksdToTcNP3HXjrgReClUJzZCwl2hC3fTV6F8sqasKvhDoex9bOxyzBu5rpAi8zmr10/WpHXhs/HM5/Mwcbte5NQo+wkEa9YO3mioTgyeWXj9r0YPpEHkpdsNxonxs8qxCX3j0qyp7iLGsWM7JJYDyIMSLAjbPlkzDJ8NHqp6/Su9rB0cb2/i+1PfztpFd7+bpH//PXQ564BTWOxY7ewc6yoSu6yeNJJpY2djz1d00VoC3v3Aj+a84fenoHPx69AWUXiH6yKj7XYD0YuQW2dgpqauoTLNzPxd8072fmBaPcuPd4kIkhIsEszdu+pxJChI7Fg5Y7Q6mAWrvzYrvldmsiEj8cI9HJdBlTYJ26eRVyaNFmSSpSULsUiSQb4KWhEWE87EbvE6trgBCo//V97zvVJHOw87UCRtFoQYUGCXZrB14uwAWNmBLtkmCmks5ikF3jTxaYmmfh5FtkSQiEM54mgheJsMOy3IpH3K9Bn6yOkjBYaJZnvmKudSdJ6tCUSgQS7NCXMeTGJq6GW/Hfccnw1YYWvwlM1ccUmykjoS1DpSjYKEclGE1KStles92w901CFBD/CcywwcPw9W72pJJCPIld5pJnjhx8mz92A1ZtKwq5G2kGCHeFMCvYd/ObnlfjyJ57e00NaVy4J+BGys8ZuJ/U2dnYT7NadZfhs7DJPk75fAWHx6p2uPZszUSYIss5+9vnVBHiz88TiNTvxj1emYeT0tQHUyzmNthScLs+wdF8VKqvdm/3U1Nbjla/m44E3aFcbMyTYEXGEOSmTBizD0TSZ9eFWI1FSuhQL5wn2yY9+w7eTVmHrzrKk1+ehd2bgmU/mJL2cIPCjbUr2s527fDse+2CWpWBttRSrOR6t3VySzOpF0a9ApAPXDxuP+153L6Rp0RqqazN8sEkCJNilHenRycIjsyS7TLcls8Ndy4ypssWpJKW1d6Gxq60zTl7p9tqluj5BlDfs/Vn4+ff1CeVRrz6WHN1M+sSHs1GwYoflOxTVapsaEajtnYtMEo1gkAwK1S3q3FCjCnRNGucmqzoZS6OwK0CkIYkYJrvQPrgpOp0GGw39QBi1kwmzQmkIbSzuneiSmIu0cbc19H4SbgUSKZ1vKAbfUIwzj+npO4/YNoOSmlj0AW1sM3vFan0nCG9ZNyHyYnsJh/4S+ULzbm7ahAQ7M6SxIxxJZcdPt+UBGRHdHclmASab2+ZESveKdaGx89Mbsvn5pU3TJCZ2Th99Vpq5aBiUAAIXe9KYBzTUVlTVYsjQkZixKDU7z9TUCI1dHgl2cZBgR8QRro2d90ju0WuDrYp9/ukrd4aCJpPEvIXTZur1R0rDnThrub1UR1EUzFm2Le2M44MkXZYRZa4TTlWKaubq5Ro7P11nReFuY5+T5PH9lFVYu7k0liSBsVbGtl3C/vOrCSsCytEeTWOXR0uxcZBgRwRKwur9dHbBlzrAZbgAY4uf0DPZsUSd2gDFgqDe+ZmLt+KJD3/DiGlrtJwDyTc9Cbdtsg8Yp1BImsbOvORqddyJ35dtw31v/IKxutinMqXfxz8uw99fnhr9HfQ+tzkp7vuaYNwol8QYM3RH0ox0kGeCULb4trFL4Ppk37qYPU2srCCWTTIZy3elYd8WT7jZK9bqlOz4rtIKAMCO4vKE6uWGsDW06TBeAuZ62As4VmYcfpdiNU3Zph37dEfdOE+o9QnqHqb4XdBKyQl8y5bMhwQ7Iq2ILbH4CGMQdGVsmMeLAACbi5IffiIs/IzP0Yk+wyU7N5NTRVVtdFINoiw373yllz14XTyCRAPihiVYpctKv2yfX6d7EtVsxe/H57MS6tUeBRz9h2oQ5ESXmAPK0CXpItwDQE1tHYqKK8KuBgl2hIwERs2ABtx06qwydu+p9H3tnGXbUhKPLAxioRwSy+fbSSt1S4npycPvzMCtz/yccD5uNCfaubtfmaZek3hH+3XhZvzjlWmYNn9zwnmFhb9xIjipULbPbyR2UooWGiWovWJlFiJe9nn2KhBakXLtbZoI93pe/nIe/vTUT6itC7dyFO6EcMTL4JmopiZmo5d+6CfgROr3xIe/IRIBRr14USD1ShZenqQm0EXflQTHtc/GLgcAXHxKn8Qy8okbYWvVxpJAyoqFO7EuzGqulNZPcZEGsaW7Tdv3OlXRkTScY1OCdJ/fqK2Z/K4EHRJIlo8b4Spo54lU29hphKW9raiqxZ6yanRp3zx67Lel20SdwqlSFNLYEXGkh42d+wxSLQRGEElYoxj0YLRwZRFqVC+xdCDjl2LVvyl5t3wUlsrJrK6u3marp2Du0LotpRijM/53Il3eLz/OE1b4H1Piv4Y9bBUbnKNaqm3sQn4FHnz7V9zy9ETDMc0+MmzFBAl2aUrYL21opEkYA1siAQ6GAbBmUwkeeW8mPhy1NNB8/QzQfie1sCneU4lp8zbFn0jBc9Y0drbOEz6qEdQE+8RHv+GKB8c4FJZYGXe9NBXvfr/I/QUS27YwkGl2E62R38em1/h21mmRRJ4ytZ6/cqzICVgTme6s2VQad0zb/zfs6YEEOyKORPplop068K/IANG3LZ2qV1pWDQDYXLTPIaVH/DhPOHgEmqmuqcO6LfEDZKp57D+z8eIXBdhbLu4lAl6msmPDNrGNUrLe+URznbdih3XeIfWD6apdYIUXZ5IkYohjF3UgkhO04KPP77B+HQEATRrlWKaJHUuOEJI6r9j0lSDDnh5IsCOSgt84dmHb2G3cvhfPfTYnbm9OPZE009hp6v+cEOqkDa0/z9kg/uHRxu6t/y3EXS9NRXECzihO1NbVY/Qva1Fn80x3lghPtjrV6DmRsDteeW34AgBOz89LwBPB/JVFvuuU7pTsqwIAbN0ZzMfM356fhP+MXOL5OkWmsoODZBcwbuxBZVUJ+iM66hWbciO7FJeXAZBgl2akg7iQyBdXon0skXAnQfDa8Pn4deEWrHYwjE+H56SRtC9vH9fE5Dp3V/P1xQCAfRU1Pkpzx8hpa/D+iMUYP6vQMo05TIssjEXSsS0qvWevsJbfWjVvEkg+G7fvw8jp9l7YKwp346mPfosutwHyp+KksbPCa9+J1kE3ZlrHlYw/YXae2FFcjh9/XeupbD2xrdIaho2dLbQUS6QdCYRX0jq1X1f+wINmeq6A3Sn9Wmzyq+KWoCPI+ylbI2KxD6YV0Wj7SfzML6usUf9aL9tZaZhTeUuDKipZd1LqKJCkspw48dD9AQAH9WiXsjL//ekc/LZ0m1G7rN4SQ7iTkAYvu1KlGjv14MzFWwEAw96fhfd+WIxSVRvqufxUhzuJFhx/qHRfVUIhqTIdEuzSlHS2H7Bjwuz1AIBFq3f6uj4aNDMkY3G+QWiQ6iSChn67NN9bpiUBTYjODTgCuy/nCY/pc31uo+SWhSuL8OtCsSm5l/YEHQrCDfYCQQI1CUjQCDs2lx7tgyC1MlT82CQLVRPVvKVqKdZNGqmNnfg7flYh1m/bg33l4gPIb18MOoyLW2rr6qO7rWhcP2w8/vj4hJTVYduuMkNg4rBnBxLsiDgS6ZfGbW38Fx62DVtVtU3okEh6OU/Uh+iJZVWmWyFKuz5ZGrtH3psZDQbtqoQ4bXXqbmpOTgQjp6/BkKEjsU9z4nAgmdUbY1qWs7M7TfWHqNX7Vbh1TxLLtDkpfQ6W66LyLHw+TJkphuSz1DYP2/HOJSkXaNWCNmzbixuf+ClUR5pbn/kZf3rqp+jvsOcvEuyynJraOluj8aBxOyFZUR+CpkSG7S4ACL/j6kmrpViPQUqTrbHT42Zi1pK42b81aCIRYMLsQgDArgSWkeKeic983v1hseG3dBzxa1CWIFHDf1PrErERc1um3slFu9dbd5bpQl2401zpu+u+8mp8MHKxdeIEkUc7UWx/e0Vrd6J9ma/f7es6T9vtZTkk2KUZQU/Olz7wIx58e4anaxJZ0pQtYfqheG8VZqhLaE4kQ6Bx2lg6jeQ6XRy0NKqUy9cgOhmkwpXO5r229DlNqY1d/HJeWpGO75epSl620vJcpMRlXz9WLlgpwsL4uU2fjl2ObbvKfdVP/2Fnda2djZ0Zv2YmQdnY3fv6L67SmUtJxcdhpkCCXQNgeaG/L6Aw0G8g/exnc0Krh0xI0g9Y6TTHRcOdBG5j5/0a6w3O7UmFttGdLZIW7iT1k8TiNTtdC7h2t3fR6uSEOQnahjMRErHF9V2mzsY2Vg/JeR9KzERWVfT3Qvv3JC38ULRu1l6xcf/OUPvuoJQK2QAJdoQjqRzOUzGoTJ8v2WHAjN1SbCS9nCeUNNLY+V2ZS4UnnV0Rcbcumja193RzkbAHNAu6+p/3v/ELqtXt42S1m7Nsu/Tawq17ErJDapQbX1pozusSIUscT957JPXYlxQXDdLtoSpePmw2bt8r/QD4fsrqqNZv9pJtsWDbFvWMX7JP7Glq5bUMKASNI6b6p0TrnyGQYJemNFStsrndyRioX/i8wDGNbIhL150ntPEsEnBvTuTOu31sKdW4eGhRzJ4qOXVJhOWFuz05CeyrqMGu0grc+eKUhLTgOTlpOF2kVGNntKEDjO9U9HAS67R6Uwn+9vxkfD91ta5i4k9dvRINtG1GvhQbrI2dFouyZbPG0WOP/2c2PhzlPfCzHxrqnCkjDXsqEQZlFTWoqhFagDA7SNxgE1JdFABTCzZG70k6E67GzviAYlVIfEkxcGzLstD8pJME7xNFAW58Qnjs8QTMMuyE3OkLNvvON12prK5FTW2s/0fN+gw2drF/axovpzdG9s67fc22qxq5lRuKUbBiO1ZuKLZ8rY1OHrKlWN2/9XXxK5lGHY5i189dvh0jptkHft5TVo3rh43zUZyxTWRjF4MEOwIAcPUjY/H3l6aEXY20+epatGonXvpyHj4ZvTR6TB9WINHQBEESRLiTrTvL4u18/MSxS+ONwN3Z2Bl/p6tY5/c9atI4N+CahINVnMFE37tq3YfcFQ+Owd+enxyXecRKYFIPx5wI7MvyM4ZEHaVyInjsg9kY+tp0y3KCtrl1jcdil67dhdJ9iUVTAGgpVg8JdkQUzb4nFUtwltfHueCHgxa2pXhvfBT2CCK+l+iSIfDUS76UvVBUXIE///tnfDJmWWB1ctvM1Bq/23jFupyMg6S6pg41tXKjeaf74ree2SLYaZjvU6LLieZYfZrNGqAzeTCUp6uL6V/JsBfWhJdcK7WhDr1g5yrcSVDV9ZhPUGOpX+eJiqpaTJhdmPodM5IICXYhs2nHXrzw+Vzb4J9mvKTNONKkb9U57ubgU2Pnsz62eeq+4v2gbSG0eI1xtxA/dY3ONy4vTpex1Erzk0zB87J//oi/PjdJXh9dwdt3l8cF/vYbL1Im2A3/eaWra1P5rGYu2hK3m4C8Lql17QKsnSe0Dyu3fcDoZW+/bKqxUt0ZxyC0WaQ1LMVKzhvkoDTph4ngV2P37veL8Oa3C7FsXeZEj3CCBLuQefXr+Zg+fzNWbSgRBxzGqR27y3HJ/aPx02/rk1cp08CSysC3cXYSIc380RAiufLB0fctScZSrGSJyA9ubn3pvipTmASLvNJwpvBy64MSHJy0ANt3l0uP19crUW3eUx/9Fn/e5+3Naxwb8s1V27G7PC44bBjLW+WVNfj3p3Pw1Ee/YUvRPmwpit/NxkrwTuZwYbctl6iM+schSLesm7p9y0b9sjYuD8t4dAYB1NktNrBhPuDporqmTq6NM3vFenj4C1buwIX3jsTe8upoH5Rd/9VPHA+/4y0ObDpAgl3INMoVj2BywUZX6bUv918CMFYeN3MdPhq9NNStWMzECRfhVCMm2ElGO2Fj5y9fd3Zeird9TaN19Vcnt2zbVYbrh43HD1OtjaG92tily1JsNE3Ab5zf3F77ej4ufWA0AATaPxs3sl6KvfnpiXHBYYdP5NF/p6ovllWI9hbvrcJtz07Cbc/KtZpAMDKEWwcprf1GTVj8XfG1jO6xIW608w5RWQwfB0EIxF7z2FK0D9U1dY4fpJf980e8/EV8JINE+ur/Jq+CogBrNpVEn3+eRJv95YQVvvc9DxMS7EJGW+obP6sQG7fvdb4gwInw7e8W4Yepqw2DN5DoAJ7g1QmMMEF+rWtfb4alWL2dtG/nCec0F947ylNYCm3ZOFGv2Hjth7Gy2ibXc5Zvs84jenFCVUk9pgB8YWscubrk9sPU1ZZaPSD+Njttx6QXCNy8Lobg5gF0sJra+ujSv4w9ZdW6gLvWFbR6PvX1CuYu3+5pHNm+q8xVOu0D6relsfffLgSSF+cJJyHMjJNwaa6Qmxh8svqWVdRgyNCR+H7K6viTCVBRVYvbnp2E14cvcPWs3Hhda0oSN8S2P4vtk5vXJHvsTxu5ScQY6wvgXgDHARgIYAXnfKAk3WAATwMYAGAzgFc5529I0t0L4HYA+wFYCuABzvkkU5pWAF4AcDmApgCmALiTc15oStcPwBsATgJQAeBrNT/r0TCN0A+0ldW1+Fj1wmyortvmVptvw7R5m9Bjv1bovX8bd/kpCr4YvwInH9HNUz3qJLs5RAfQiH/52u2EM3PRVg95ir+RJKvstHtRV2fnhJA8w/FE8RegONFCE7v8I51XthsKt7r4OPRA0MPQc5/NwW9Lt2H0SxdJz1/36Di88o9TXNfLLPxNnbcJU+dtwiM3HYNjB3aVX2v67dY2VbvuxS8KcMqRB8Sdj777AXipO2Fot5Vcp/u3NNyJxYX647vVPYt/+m09Lj2tr+d6WlGiOqWtWL8bJx++v688zE2SBdC2Qr9DjhbSprEHwTDdcduSQwCcD2A1AKnrHGPseACjAMwHMBjAxwBeZYz9xZTuXgDPAHhLzXMVgDGMscNMWX4F4EIAdwK4CsD+ACYxxprr8moLIfC1ghAAhwK4BsBHLtsVOmYziKiRtNOAGuCA20IXUFKrh18S9op1uP7FLwpw10tTMWp6/HKgbPAqq6zF8J9X4sG3PNpJqFnJNvyORP9nzZaifRgydGRcINlkiDv6MCx6Zi7agiFDR0YDhzrnY3/eabuw9Vv3RG9LOn6XuFoGN/1NdHJO9m0w3+fGjYKVJvSTfBBt0Wu7rNijhr5wde8t0ui92Z3sBN1qumXvtP4D3Owz4eXd8boC4MZ5wsmWz7KPGpZovQupbpJqu1S0at44MBtuJ43dtl1lKNlbhbf/txDzuNjXV1F0S9LpGtvIB24Fu9Gc8+6c88sBzLNI8yiAeZzzmznnUzjnTwH4EMAwxlgOADDG8gA8AqHJe5FzPhnA9QDWAnhYy4gxdiyE0HcL5/wrzvkYAJcA6AHgRl2ZtwFoB+Aizvl4zvlnAO4CcBVj7BCXbQsV/UvtRkuXjHcvyMC2xw8SX8n7dWjukFJOvNAgvycfjIyPZm5nuuLbk9ji1jgF8dSWDsy2kMmJYyf+mp+jtnyyyWmJX3eZnb2RtvGA/j3Vt2brrrJoXmHLdcV7KqPaBjdky5ie67A7hNeubghi6+Gh7iguxw6bJWTbMrWlWF9XC8ora7CvvBrT5m3CRfeNwtad1sutrjV2uhsQFRalPgla/d23QJ9y3oodlmFwNAwWIm5sRx2kTQWKdDtAL0KqFy19eaX42GzetLFDSrvyjNi9+/P4Dtz6zM+44bHxGDerMJaHR3vmTMGVYMc5t33LVIHtdADDTae+hFhuPVL9fQKANhDLpVredQC+ATCYMaa9PucBKAUwXpduA4AZ6jno0k3inOutG78DUAWhNUx/9B00pCgm8cuf/l/0owd0AQD06NI6gRrp65LY9f9861ctJ28XOgxkTgOdFui0SWNTF0vCGBINWmqqVKNGomynSULPJz/qAzIbz0WXYi2srhVFv09muIPl/z0+AX98fILn67R6h11/vzi9l153FfBrEnLzUxNx89MTLc8nen+tAhRrfPzjMlzzr3HRDyu7LdicNEa/q1pGfY21PmDcUszoOJTI9/KE2YW25105T6gVGPPrWtzw2Pi48/Wmvit9JBZL3l6pqa3D4/+ZHX0OicbelGGXVeGWUulxRdEtnWfN511wzhN9ADRB/DKtNkv0V/8erP5dLknXEkA3XboVEoFyqS4vLZ2hTM55FYA1pnQZgWEQdfKs8ighvPPdQgwZOlJ6LkjTLD/j9QqdgXZC4RUkl2oDSSBeX+rfSCSCH39da5/WYuBK5lKsebBvogp21bUOXn+6ShXvsTZsd/J4/XTM0szdgStJFU++fGgswG95Uyy88o35xX7sKq3Au98vQl29v69Ru27upF3S1ySIZTwn4eJJLdyM7mZou7TInCcMJhsSpE03Ja528NT1IhB9OjY23bpZijVoJh0EaLes2liCucu3461vF6iFSCrkFVP9/bz79Uri1sDzVuxIMIfgceU84YJ26t8S0/Fi9W97Xboqzrk58qQ+3SY1nTkvLV173W+36RxZsiQ1GxUXFBjdtveUxr4kVvCYd2ppaWlcWgBYs00sL+3Zs1d63oqxMwul5QPApk2x5cKCggKUVRoHlV27drkua80GsfxSWlpiuKa8qh4zZs9BU7MGC8BjX26K/nvZcqPMP2/efEPd9Gi/KyoqoteW7mgirZd5uyyn9uzeJYTNoh1F0bTlZWI5Z8WKFdFdOqzy2rpNPNfNmzejoGBvNF1VTb3tdV7qqLFpkxBet2/fhoKCWNcqKxPlruCrgLJN0msBYMtuYe9SXl6O4pxY4NuFCxeiZbNcaTqtbus3xGKMbS4qQ4cWYphct24dWsM44MnaU6a7p/t25sWd9/KOO7Ft2zYUFMiXZ6ur1CDNS5Zgc8tG2L1XhNyora0N7DkF2RaNNWvWokl1zNFmXaH98ufefbFxY8vmmBbr5S/lFjb79sWW8QvmzUdj1UD9q2k7wTdXYsOmZrHzFu2THZ87t8DS2H3VqlUAYs9ElkepOm6uXr0KkXLrd7u4pAQAsHbtGjSp3gIAqDeNBYsXL4r+e/6CBdIxqqCgwCCMzps/H3mNc7B2W+x94pyjYnceqmtUm9aIvO1VlaJd+jmnaEeRIc3GTZtRUGCtZSwqivWtrVvldota2fX1uv1uFSVu3NTqXlsr3vlFixajdXPR77cWiz5fWVnp+P5uKBLtKisri0vL1bmtZM8+fPbDL9GPzr179mD16niPW+16vZBpznP1FmNfXrR4EVo3z8WCteVx16xcUyKt8+rVa1BVLdq4ZMkSbGklF4ns2v7GcHkEg2T0d7cEJdhlPAMHDkReXvzEEiQFBQXIz883HBs1bxawVXTSfv36AT+LDt6mdZu4tACQu3IHMHknWrduFXe+oqoWNbX1aN1CJ9x8aRz0Duh9MITsrDt2wAHAAjFQ5ufni3AE38cmiw4dOiA//0i4obLRFuDX3Wjbtm20fuWVNbjq4bEAIPeG09Wxf//+wPjYoHXEkUcA32yO1k2fVsu/2ZQpQEkNDj64P/p1bwcDanrxZa/EXWtVl44dOwCF5ejUqRPy84VfT/PpU4HdpaKOE2J1lOW1eOtSYNledOvWDfn5B0WffXllDfDtFld1kJ2fPHcjunVqAdazPX78dS2OOWQ/7LdrI7BoD/bv2hX5+QdjyZqdqKtX0KlDPVZu3oIePXsj/3Brr+C534uJbVtxDfr06ARsFAP+oYcdinatmkbTtdlYAozfgebNm0frtq1yHTCnJJqmQ4f2QGE5evXqhfz8Ho7t+fLXacCuEvTv3x+V1XWYUrARd199pO01GhNmF6J1i7yoXacx4/jJvkuXLsjPjznzb99djsqqWvTs2hp54ycCZeUYeMhAdO3YAlt27gNGb0Pjxo18PSdZ+XHpJGm8cuCBByL/0JhX4b7IJmCmdciT1q1i48aq3RxYbC08AECrVq2AHbsAAEcecUR054qxC34DNm9D23btgfXy/qlhaLd6/vAjjjDGDdNd17dvX2DaLjRtmgeUlcfnAWDk3JnA1iL069cP+f27WN7LNm3aAJsr0a9vX+Qfsh8AIOd/W4G6mLAzcOAgYKQQjo44/PCY3Zd5rNH9PuKII9Asr1F0PAaAg/v3R/9e7ZE7YgeAakQk9QaAvJ9+BvbV4pBDDgF+3A4A6NKlM7Ay9pEkxo1+8Q1S69ClSxdgxb7Yv5fF29FqZTf6YTuqVGEzJycSPZ43cRIAkQdjDI1m/Q5UVWPQoEHo2FYI7Ks3lQDjdqCFrs9b0XTtLmBiEVq2bIn8/HyDYMMYAyYWYVtxDb79dTeuOZsB2Ik2bdqgb9/ewPRd0rrX1yvAV5sNx6K02A5MjVlhDRo4CEvW7sKo3+bF5fPYl/LVqt4HHohGCxYBqMKgQQOxX4cWxgQu+njTpk2BPfFBtJ3uVyJUVVXZKqOCWorVNG5tTce1WXa3Ll0eY6ypi3TmvLR0+lHLbbq0xVIT7bQUK9Ef3/7CZFz36Djb62SBjZ3iL3lZ7YjFoIod21Pmfvuj+N0PnBXlm9XI9HZJfavbJVEFEvF2S3Rp7pWv5uHe139B6b4qvPfDYjz63kyd55oo68G3Z+CRd2eisUsbu+3FFloeta7bd5dbb2FlsUuJUzt37C7Hq1/PQ3llbbSoR96diUlz3AXqBoA3v12IZz753XV6M7c8PRF3vDgFQPJCU2SmpV4Mvd2Rvi2xZUd/LVRs1mK95OhkF+WmegZnAZeF3/3yVDzzye/SnSfsPEm37y6POnIYHFPMdXKoiBdnMGtTGw83J8H+YX5OZRUxraZd3lb3YcvOfVHbx2haeJtrohdleieVEJRgtwZANWI2dBoD1L8r1L/aOpss3V6I2HdaOqZzptCnW6H7vdycl+rI0ceULm0xBql0YRBrk0YLIGuHrJMHOalpDiD6dskielte72PrCTfOAUHaOrl5Tpb2KwHVQQucvHtPVTSERPFe49JETLBzF1kfEN6kZm55eiJuf2GKp8HdaWL6fupqTJqzMW4P1LQhQwb7OIHA5XVT523CF+Odh0jj1lWx3DV7Tr/96t7Xp1ufjNqoyV+48soaVFa7f6cBYzvMdrx+hNMtO8swa/FWSOQ6W/u/WYu3eC5Lhj6WpJdAyPpObCVbGwVOzTHLaw3tcWu7J6tiXV09bvv3pKh5UTStoqDeo82nAiUrY8YGItipDguTAVxpOnUNgG2IhUiZCeHtepWWgDGWq143nnOu3eGxEJq4c3TpukMEIR6ry38sgDMYYx10xy4BkGdKR6jIjY3NBv7+X3Sna52MgpPXxVLbebXSauvqsWh1zH4mKG9LbWKtqKrF2s1iGX3C7PWGNFpcJ+k+izr0E6h+I2z9VVahQ8w5u/1IKHMZWy/VaO9vTFGR4IyW6jnD4f3Sns9Lki2avKAZ7/udFNdvsw7BE+0jFrf+qofHxnbEcFzZMGqzAW9e4k7o+3NMW219T+yCexvztT+v19g55WjlaKGv56qNJVFtlyF2oWmZoqKqVpgpmFi5oTjuw9IOqwDTVnXUJ7Mazwwx6TyQqR7wdrjdeaI5YmFGegJozRi7XP09h3O+HsATAKYzxj4A8AWAEwHcCuB2zbuVc17FGHsKwDOMsSIIge8WCA3btVp5nPPfGGNjAHzIGBsKYI+a/wYAn+iq9h5EAOORjLEnAXQG8DKA4ZxzaSDlho5TRyreW5nQZCTrI/pDl/3zR3z37AVRWx3bxPE/Hcq2Th3EZuZW2a/bUop1W/bg9KO6x53TtCJDL+lqm0cyiC6X2bT9X+/OxIJVRZbnnTC3p0Z1DvGyvY8XXvqiAHx9Md5/6EzP19ruPGEKwBfUYJ/0HThM2e+xWjJX8SqoWm1BpR0Pol+Z0bKU1VSLfxath0Nebmone9SV1e7255WOd3aCXb3iKp3Te+P0sablH4lEDM/QqIGN/fvDUXJ7rajHvXrdYx/MwrJ1u+NspYe+ZqOBRfwHn9MHgVZ3swC4cFURtu2Sm44oUHy9j4m+wukYCcCt80RnAN+ajmm/bwLwCed8FmPsIohdJf4PwBYA/+Ccv6u/iHP+ImMMEIGEu0CEMDmfc77QlP81AF4E8DaEBm4KgCv0W4VxzksYY6cDeB3A94htKXa/y3aFRl1dPdZsNsbWWVa4yyK1Po34UvW7MbEshqNezb5qYwn6HdDWV96AOzu06po6S8HO3OG9TLB2KQOZfiwyueulqQDEzgs3DRFxsc31rlW/1AMTGFxko32p19kkthPqXAU+Nd2U0jLhGde6pX9HJKtyd5ZUYOo8/w4HtpOlUa5LCgtW7sDhB3VOYgnAByMcvPs9TkJmQbCouAJzV2wPNP6YZdmSIsxbrDlWIyoY2CXRCVvq33Wb7Z1KbIu0eYkMgp1tJvZlGDR2LrW0gPM2Y+ayY8pTcaVeow+IUFV2MQItcXgudfUKGuVGdDE6xfFH3p1pm6cbgddwidKANXbq/qyOPZlzPhYulkA55y9CCG12afZC7Cxxm0O6lQDOdSoz3fh64kp8PZGjTcuYB+vn4+JtXvaUVRu8XFeqm4P7JddhJDT6jvpAEnXd3HFsN/cOsI/py/War1bFcTMLcfOFA5HXONfxK/r7qaujgp1Vfma27NyHXSWVGNS3Y1ydE8Zs9JMElq41foxUqxo7za7Sysjb1tHF4ty/3rMZ1AMmGWP9v96bhe+fuwCNGwW32Xgq9+RVAAz7YCY2bt+HI5kQUL1OpO5LAmRTTkWlUZPm7Dwh8pq3YgeaNmmEQw7sIEnkq5KG/PX/jm5KIcnXEPfPptzPx6/A2cf2RLvWZj9DgVGws6qbNu5YLMValK0/rlg/CgDAfW/8YpGLPdruD3OWbceh6thnqINWsMslWy2pZ42dTrALamuzdCB7dr3NMAq3Cm1d6T7rpZPl63bjukfHYcbCmMFtohO/dKN4g6o+/ryX5Zto7Wwusf16TqR9pkuDmnNGTI2Ps+QXc/Nu+/ckPPRObB9bfZ2HDB1psM8z5OOiLG3SzQ3Y8nnt5lK8/d1CrN5YgpmLttqmfVMLSGpi5UbrDxSrtun3/0wZiZrYmRrj1sbKLc99NtdTes/NMTlP7C0TS6H1USEmeMHObueGOO1zxGlJUzDql7W6XWjkafSFu53jFdm/XZqEON25d39YZHnO4DxhkZP0qGTv67jrlPi8E9bQ2lz+4aillueiGjsXY5ii+HOESPwVTj+BkAS7kHDzdbBqk5j8lqyJLbt6+SKRaUucBLecSCQh4Upm92zOzm6QSKSP+QmVomfbrljQYf09qZFEmfdal5jyzNm2RM+YGevcF2pCe1dydPZupfuqXGtZ7No7bmahq9ACMxfFewHW1yu2e3daEbRnnp7o81GCXTI3E7YHnletxHrzMlvE8Md1hx0ydCS+nGBckVhRuBtDho7Ehm3GMuzMOfxoZByTSJ6J68ckkez0VTSPwUZvVqexwN25UdMtdsFxaoSLRu4scY60YCYIxVf8nAHsdbAfVRR/Np9JUTqHDAl2IbGv3L1XYPHeKjz2wSzsq6jxJFy8+338F59juJOENRTOam2p1tB0fex34nVxi9Wku7u0Eu/9sEi6P6QVI6evsaiU/LBsiyLZb+cTMbRJpZF6v8sqanD9sPH4yMJQ2isO+80DkA+ac1dst7/Ism2Je6iWV9Zg8Zp4+9RUrcKEPYlUVLlzCtAwa0m1rqv1by8T6Vc/ccNvLabmPO7svDNj4RaU7quSmHU4CEAu+qoxvIc35EKhOLZrTw0uuX+0wS7Uy9L1rMX22nDHukmOeX3PX/hceE+70Zglg6gjTSTiausu2f3dUmQdUkmBEpuz/FVRBPRPM2jniZCoqHIh2Knv6AxV6zF57gZPX/wFko4g05ZFDOddZy/FjcbOrog4wcZL2abUTmPoisLdKNlXheMGCo9VK03ixN83eKiFVd2Mf83U1NYjNzdHItj6lwSiS7Hq1k1lqkfhzEVbcOvFgxLSGAAul0ckD6HWIdzEtPnxQbSBxIUvBcBz/52LeSt24PPH5Wa5Wps320wGXsvUkwwvUi8sL9ztnMgCYbNlfAiJeFRrxD1XiT3Hs5/NwSEHdkALbVcIl/jtPm5tF/WPM2oWpv5jW4nob7MWb8GpRx7gKd8gcFTY+bzOD167bnS81CkKnMcrRZpm2AezHK9LBM9BkVMAaexSyIjZu1Ggaiscl0Qs3jUvgp1sEnGajO3s6eYu347Pxxn3ct22q8zUMSRfsKZjtk1PSENn+u0wid73xi94+uPYzgX6e+P0eJo28WcAbzmIaOEjXGos3dym+jrNPkV0c01wjQ2aLjKxIddGZRcNtSK9zv7mvvKVfN/SILRqhVuEbWu8mUIs8+XrduOZT+aYjvrEdAPqPAZQTTeC1NtEl1xNx7UlN/PzLiqpkI5/Mg2sp3q4cLKynvx116r/NodrsRoeEzMndnNxfBo3z0+Wd1h+BVFzkoiLECmQz3nVNfZ9LmwtejIgwS6FLFhbjsc+mA3AnTFq3PumuBsMFq/ZiXkrdkgHI6lAqd/9Ise6jMf/MxvDf14Z/b1uSylufeZnjNTZeEi9qDx0nLjO6+IrzXVeDrg1EJ7Pi3zLn1ZV0oKmBvm1bHae8Bp7zGnycOOUIXs+fpd1UuW1tlVna1m8twoLA9BKadTXK/j59w3YtMM6QG86MaB3++i/FejMKAKxo5Ko9wG89T9z5CtBbk5E+u7ahcBw059ciUiWAp+Liy3zjF08+hcLOzkL3PThAEzsosjGRqtdNKR23D77bsyRJgLHbyLFaqnbuqGGcCcRYaqxw2qLxQyCBLuQcPOeyzqeVYf+8dfYwPDQ2zMw7INZUsFm9C/xtl/GOEeyTimvn7ZctVwXfy8m1/nsyB7T241vdgPXlIL4PUndChz/HbccVR63NIot08jPX/uvcep583JyIkuxYiTUBLCoxk6ri++cBW7ul+z5+BbsfF0Vw+4jQP+Om6v3no13omOZpt/19cBrw+dHYx+GwYr17pdju3bUbYquKNFnEIiI7fACmsvIicgFOztc9R+DFk1R/xqTrN5U4nSpZyEvkf4nM7Oxyt8qQLFVDYZPXGm504yeNZtKHdP4xezEtKes2nkpFvL50YuAe8+r03HzUxNd1y9dIcEuJJwmt+27y7F6Y4nhmAL5QFVbV4/3flgcd1z2kq+WdEaD74SDMbIsf/3XnOxaLwsCcTZmTpWwC3Vgc+7lL+OX+1KhD7LTgr357QLXzhNunpH29bpx+148/p/Z8c4ffo1wVGy9m6NFyJ5+/HWlLkKZJMPbTkN7l/eV10j6pvj90eilGDJ0ZELla8J2kNtaeeW+1/3FHgOAHep+1EFoT50+As1l5OoC1kbTBNBr3Qx3ljsr6C6urK7F356fHDugmSOEJATIy41gzaYSWyeayXM3xm05J+vrqWiV/nk7yfSKooCvLzYcq6mt91RPt7a1aS7XkWCXKtbqdplYt6U0LoK3mc1F+zBtvjHKvkFtrCMuJIGK649b/VKshwE7KtgZJkJxLKJ7szztHuFtJda206aj7QQvLLY8N2H2+kAnAS2rz8evwNzl27FWtS/TljTMJR3er5On/GUfJ369mp/S2TomDV1dzO/5FjX8ysTf18ed037+EEA8w7DDnSSCf/MD+ZWyfUDtyInEC3ZBoK+fV0N4vSZv1cYSbNzuYYk92a+CJP+a2jrc/co0/PuT3237ZpVpT+/q2jrsM4UbSarAKvn2dN6GDNExTmNL0T7beurP2H0kmDcGII0dAQCGWE1O++rZIRNWrF54t8sWcRo71x5hIp3e1spun0c3eO0wxk24E8vLD7Ko6XY8/7l9QFnzI3OaFO0wp3n1q/mG4+YsWjY3ehwqANbr3luzHZDX+7t1Zxk+HRPuFs5WNT7q4C4AgEP7dYoTWBMJzmq+RWF7xYaBkwOQ27urKArWb/Vmm+j1HTVo3Fzwv8mrdGVZ1UH3b/1xj5Kd112HZLlrWntH72jTxcvW7cY1qrkIIByMUiHbGDR2Dn1H+qw9rEDZMfS16SjRrSqkey8mwS4E/C/DyDc5tnrh3Q5qTjZ2VtRFvS511+jc092UZ8Zc40RCcnj9uvfTWc3CEAAMn8jjjrnNO8g4fmbMAT65ydZK9szueGFK9N+T5xrtEu2qJntuT340G/+bvMrgnJAuaF7OOZEAouwbMN6H5GzBlRr0j9TLLXJssUVm5sPrt+11DFIbV3Yqb3fcMrGn5FIWrirC3OUikoJXhUAyP2zvf/OXlHw4G4RiFzZ2ZiJwuM8e2lBZHVu+dhN6JUxIsEsRQYzniiIX4qwmCz/agR9/XetaColp7GKvkexSu3d84m/rjWm9buKs+7dZKE1F35JtqfX5+Pg9f93eU3Od5/EdWGdaXkgUrYgH355hn87hBsrerxUmGxc9idqVWVVnV2kFvp200tPAv2N3zPPt20krDfZe5hXmSASGJbZEBm3zXqeZhO/JzEllZ0EQ9nNu4vYFNU7EZ2N0VhL/9lbYI+/OxOP/mZ1QveJrJOpqK+8koJOSjQtev5W0HPR5Oc1n1TXxDm3iY9W7gkN+3t6WXE/Yn28k2GUQVjZ2Vi+8exkp9sJOXyAPDivNX2JjJ3eesK7IhNlGwc6rLGq2+9DjeTAxLOsG70pR6SLqv+z56pd7AiGgmcyt7Yq52KCD2L/weQE+G7schRa2psZKiD/6zcs/G7vcaH9nqmAEEdslOr39bFxxphvxwJv+HRfSHa/ym3b8s7EWy/Mpip0WlHZlwUqrfZ1F/jW1ddiwLZgPBN/ox7UEi7d674O0g9TfI6e5QbYvdSQSnK21/nV0bGPIkh0Jdikj8Se9bVeZ9IWy1Nj5WIoF3NdUE+zGzyrEtf8aq16sLcVaZ6g/xePsRrwtRf770zmxfE3tSGRJLWi5ToG75+H2mXmM4mCgzEJr5LXJTnGlVm00PttYvKhgb265uqOGc1gDFzdNshQbMY2S5mze/k4ed02WNoNXYg3ItGmypq3cXGH5YaI9j/IUajFlzhFBPRIn7eCb3y7EotWJBVP2gqKIuWFXaSx0SUR3LtFdMOZbCbIBCHZaHvrx0CnfjdvjPVpzHMI8eKqph9isYXdzEuxSRBAB58fNKpTmY6mxczmLFBV72+hZ29dUL1DuVfe+1S9p+cHrxKd3b99ZWoE/PfUTtqk2XInsbxj4x3QiApvfuiTYBqcq2wqhiogJJatOsDZs3rCazPRtiRPsHOpr18/CHuCDxOgEIFPNxx/7ctoufCEzTXCBq9fE46t03aPj4g8m+SFpt2W5KRKCl2LNoa9clQsbT24fzldukXUH33FNdXn5cjyKONkC65IGaTcaMiTYZRiyzpaoQfbXJmN/pw7960IRcVw2sWvHbDX+kQjKK2uwq1QiUCrmn+7bNnnuRhQVV+An1W7PbT9Nxq4PfnFbh1TsOblT9nz0dbD9Eo4/p9lPBr0U6/aejZ1ZaHlOmzQikfgB3qm6dgJuOrxTiaC3i3TaeivopoYn/idnmTRuBz4PRfzj1WneC1SUuPArsViWri73RZD3zhjHznu+W3eWBRYzUi+cJrrPdrJpFG7xDYlgnrRVgOJkYtZYaDGONK9YDUXo9+OQdYLbX5iCnSXxgkP8lmLu6xlb6Yt4urRkbxU6tWtmOJaMpVhX6ZJkn+IKU5udYnr59Tr2vb2Qw3k32Vo5L+g/jsyaXidTBVuNXdgjfIJMnx+zuXWOI+bxfQjigQZAKj6SgNiezdFyk60phP+PDouh3F25sunI56M0Op54v17bwjMIDMqKNO/WJNiliKBsa2T5pDqEgjaRyTasl64kSg7KhDqR2Pjz9W8WWNbDvMSjD3g6duY6adgRGbm58d5rqUC2J6ZsIA5rDHGMG2XzPSHV6MjsLwMkEbtD7eNIeMXGO0/4LTfNx39PGJbFZO+px8ami9CbdAErGj3A5LWf5LdDUaz7qHDEc5GBD6TCpI+s1m/bg607Y3Zzyblf3vOsrqlDmWrXG1yuwUKCXYoIavCQhjtJssYurg6aYaupLgqMWjNFUfDpmGXo172dIZ3dNGnuvFoMJxnmJeQN6rJDTiSCd75zv79nWDZf42cVxh1z/Z4kIMS4vcBp4q3zKNEkyXfCm22MRZXrPWjsRCaxg3YfVmu3OW+VlikYDNlle3IGXF6oS7EB5KG9N1peidj8+kFRFEthyM0qj2+NnaLgv+OWo7kSe/e368ILucvDGEMzWejHg6kFm6wTIvY8//HqNIN3szzjBCuWICTYpYxgnrR+sl29sQR9u7dFbV2wb5Fs8tMHs9UG9TghU1GgtTMSEQ4V302JN961+9pJxB6iyOc+llGNXRJ7YyK2c9ozX7p2l8dCE0vuZm9GK2RCX+xIsBNcEB9N+onO3C7n98m6AttL7L/sMwmj84TsfLD9J1XfW+Z6+3FUcINZsKupSe4H+YZtex0c45JjJ1avAN/8vNJw7NnP5liklrNph0RwSrKw9IluV5y5y7dbxg90FOqQuuV9K8h5IkUkQ2O3bXeZeiz5Grt7dRuHaxoKs6ZCr7EDrAd6u44RhKGr1wkhSI2dfgsuP1i9J1U1dfjnW7/G0iVUijuclmLtzg97f1b8wQQ0dmN+XWvY0kdGIo8x+i5H4peSnGzs7Pp28b7MDUhsxhBTTKax8/hSWoXLSDU7SyoNv305Ktih3hfzUuxz/7XfXjBRHnpnhqsAzTJK91V53uVDIwgBXz/fhMFHo5fEHUtGbNNkQYJdighqItZPOtqLFrSNnflrwxxYVyvOzs4mWQbybvAaXiH6JR1A4XbLB27Gu0S3hzNck2CDXvlqnn3+njWCmjbX+7vx7g+LPV9jVQsZeo2d2SnIqb6KAqzZVILvJdrplZsrJVdkJkE7T1ja2aoEsfOEG8wmHYDYPD5RamrF/dDui/kD0q/gFBR2j2tHcQX++PgEX/kmYz/kJo1zUZMEk6PPxy2XHs+Nc2H2RtjmoyTYpYqAnrR+0smJAKs3ldhGvg8C824U9RZLscIgVxwbP6vQ4FHnljA2SU/F9OG2VTLbl1Qse/mhzqOmWEueYlMjA1a3LfpBAucPpTjHbUXB3a9Mw8c/LgUgdhhIJTMWbUlJOfp2m7VPycCN/F9Vnfi9lu1YsqKwOOF8xxWUJJxHJpKMoam6pg7DJ650TuiRnaXyDy+ZPWTm6OtIsEsZydDY1dYp+Mcr02zjc/nCobJaHeInQKOO6P0Rci2L3YBtJyzMWhy/L2sQvPntQtTXh20VIbjrpalxx+rrFXwwIn5pwIlky37vfu/eQQWIaSh27wnWocDVVmIqbm6J+ePCrGkxC9XmPC994EfX9QmCZz+dgwoX29Uliv6+VEtMJoJ+3xo1cp6eRk1fE2yhKhXVwd1PBcLBze+yaLJI1vCQHiNpYsg+XGYvcT//hH0HyHkiRSTDxu5Vh6UyvzhVdcS01dhZUhF1OtAoKq5IWJNkpwR65pPfE8rbimnzN+Gc43vGBfMMlARuy4rC3Sh2sC8LAq/Pzq+A9uEo90JqbV09GuXGT/DJCvETicTnbf4QeX34Agzq2zF2QJd8nMTTORVc+dAYjH7poqSWoX8/ZI48QT8RN7avyfp4mbEwOC3ogpVFqE3HveSSdPPCWHUJGplg9/Z3i/DLApfvBS3FNhAC6kR6D1jZV3MqKN1XjTEz1mGvKYDt8J/dqcr//Ym1h9SYGWsTqptfHnp7hqcQKV6prKnHGzYx+dKBdDQOvuT+0dLjK/xqPyzXYmP/NE9M5tsybf4mw4bjeg3F25LYhKliyNCRSc3faQgL2jygsQuNXU5u+r2zMtLBdCJVZIFch1zJxyQALF7jbq/fsG8BaexSRFAPOtm7TADuB6Epkrg/bi4t2Wet6dm2y1u8o2QRtJDz0/xSbNrpz1haZu9hdZ9vfCJm8OxVq5WK5Tw/yN5H2Rd1MM8sgrr6OtMR/wGKswkn54myimBDu7Rv3dQxTSps/QKhgbwjQHYIsam2kw0a0tiliCx4113xXwsvo0wj6MHJr1AHeBNYdumMgb22wS4YdJisksQVK97rz9vUjaw7ae5G4wGH299AurbjGHbrMz+npiI6UuU5myhrkuzg5oek2dhlQYdYuaEkoevDvgck2KWIVi2ahF0F1/h9KSebJ8QMZsLs9WFXIYpMKeFq+6wsGGABYOhr0+OOPWOznO+HtVvExBuJAPNW7DCccwxPnC032oHXhs9PaXnutojLjHuvj0GZ7TSU/mDHO2PD/UimpdgU0altM+dEBCEjDW3f0pXaunpUJhAC4+mP4x10Ig7LffbR/bOHZIdV8kO6mg9kAuWVybl3Tkv2DYG9FeEu5ZJgRxBpzg6P+yxqaFooGXrj/2whAmtni0TzJVKPG23ckjUet9kjkk7pvnADLxO0FJsyzHvnEUQiuJn07IK3ptNSc1AkS0+wbVdZknImCCIb6dEpXNMrEuxSxDy+wzlRmkA2EkQmkqz3toQ0EKFAwxCRqcgiGaS0/FBLJwjCHzTppQzzXskEQRB2hB2FhwQ7Ig6SGYhMJFkanmTtckEQRHYStl0uCXZEHJPmZE/YkmyFRI14GooJwdnH9gy7CgRBpDEk2BFxLF3rbtsUgkgnGoZY13Ci3zQUQZ0ggoYEOyIOWnlKf2jSk9BAbkk67ulLEIQOsrEj0o3Vki2cCCLdyZRdCBKloch1DeNpEkTwkGBHEERW0FCUmDkNRbIjiAwl7B5Kgh1BZCD1tF7eYAl70kgVm3fsC7sKBOETimNHEIRHvvqJh12FtGPrzoaxQ4TT3rXZwioyCSEIX5BgRxAZyKLVRWFXIe148YuCsKuQEpK9EjvsluOSWwBBZDlhW0uQYEcQGUhDsScj4km2jV2fA9okNX+CIJILCXYEkYGQXEckDXq5CCKjIcGOIHzQvUurUMsv2VsVavlEeJBXLEEQdpBgRxA+oLmVCIuDe7dPav6ksCOIxAh7eiDBjiB8EHbHJRouzfIaJTV/2tWEIBKEnCcIgiAIt5C2mCAIO0iwIzKesO3dCCKVkEKNINKbsL+9AtPpM8ZuBPCx5NRbnPM7dOkGA3gawAAAmwG8yjl/Q5LfvQBuB7AfgKUAHuCcTzKlaQXgBQCXA2gKYAqAOznnhQE0icgQBh/fC++PWJzSMmkjdiJbIcGRIDKbZGjszgVwvO6/F7UTjLHjAYwCMB/AYAhB8FXG2F/0GahC3TMA3gJwPoBVAMYwxg4zlfUVgAsB3AngKgD7A5jEGGsefLOIdCW/f2fb8+ed0Cs1FSEIgiCIkL/7k2GFW8A532lx7lEA8zjnN6u/pzDGegAYxhh7n3NezxjLA/AIhCbvRQBgjE0DsBjAwwCuVI8dCyH0nc85H6seWwxgDYAbAbydhLYRaYiT9oz1bIexMwtTUxmCyHBIY0f4pUv75miUG8HmooaxvV+6kjIbO1VgOx3AcNOpLyGWW49Uf58AoA2Ar7UEnPM6AN8AGMwY02bx8wCUAhivS7cBwAz1HBECRw/okvIynVdFadmUINyiUMATwidiLKbxNuw7kAyN3RLGWCcAGwB8AuBpznktgD4AmgBYZkq/VP3bH8BcAAerv5dL0rUE0A3AJjXdCs55vSTdOYk3g/BDJIRX2ilgazL2TCcTO4IgCCMRRJBDLpmhE6RgtxXAMAC/A6iDsKH7F4DeEEuj7dR0JabritW/WtTNdgCqOOcVNuk2qenMeWnpPEfwXLJkiddLCAklpSUpL3PxEnvHicLCwsDLLC83v54EkRpWrlyZ1PwXL06tIxKRPVRVVaG2NuxapAcFBQWhlR2YYMc5nwBggu7QRMZYKYDHGGNPBlVOshg4cCDy8vKSV8CXm5KXdxrRrm1bYPO2lJZ52KGHAiOtyzzwwN7AzN2BltmieXOgpDTQPAnCiUgEOOigg4DJVmbMiTNw4CDb/kQQVjRtmofGjXKA0r1hVyVUIpEI8vPzk5Z/VVWVrTIq2UrTb9S/RyKmcWtrSqNp8rSZtxhAHmOsqYt05ry0dMHO4kRa47QsSqFJiGwhAiR9zy/aeYLwiwIab9OBVK6GrwFQjZgNncYA9e8K9a9mWydLtxci9p2WjumcKfTpVoBoMDjZ2CVlnKGxiwiDBF/mRrlkAEUkF5LrwifZvfxqCCG+gHNeBWAy1HAlOq4BsA3APPX3TAhv16u0BIyxXPW68Zxz7XNyLITG7hxduu4ATlLPESEQRqd2+kJMhkNHTjI8MggiydCkSySbMBzoCCNB7jwxAUJwWwKgHsJ54m8APuScr1WTPQFgOmPsAwBfADgRwK0Abte8WznnVYyxpwA8wxgrghD4boHwqr1WK49z/htjbAyADxljQwHsUfPXvHGJBoLjZJWEcSaXZkgiBBJ969xc77QSmxMB6jNgtbZ1iybYU1YddjUaHBFSCodOkI9gOYA/QdjVjYCIWfcAgOiuEpzzWQAuAnA0hKPFLQD+wTl/V5+RGpj4IQB3ARgHEQrlfM75QlOZ1wD4ESIY8bcQmr8zOeflAbaL8ICd9uy7Zy9IeZkAhTshsodE37tIEJ0hQ17+p/5yQthVaJBkxtuRXHbtDdc1OEiv2LsB3O0i3Vi4WCpVhbsXHdLsBXCb+h+R5jRpnJuUfJ3nKlqKJRoOZx7dAz/P2SA950pj5+CdkSlvPhnxhwPdd2BHSU2o5ZPSNA0ZfHwvtGjWOOxqZAxhaOxIsCPSFTvBzNWc67DMmm7zdqPcNKtQAyfd3o+GCAl2aQjr2Q4XnXxg2NXwRTjOE07nk6Cxo9GLCIVIgh99Qby39O4T1pDzRPiQYJeG1NaZd0nLHFo0Tb2m0VFwo3GGyCL6dm+Lf/3pWMvzds4PkQhwzID9bPPPAL8IV2Rqt7//+qPCroJ/FKC2PnPnr2yBBLs0RFGAfZXhrtH75eqzWMrLJLmOaCho7/oxh9gLZ1YoCvDIn45xSGMv2qWbFUK2xVPusV8r2/OXnto3RTXxx5pN2bsjT9Mm7uzEe3dJ4i5WLiDBLk058+gentIf3Eu+Pe5RB3cJojquadw49a+Uc4Bi+/MtyZ4xaeQ1ycURBzYPuxopp1Xz5LxTbmSqehtJp6a23rE/OApKSTZDGP3SRUnN3yu3XjTQdVrWs51zIiciwBv3nmZ5Ol3se3NyIvjb5YcZBc0UVu2vlx2KT4ed45wwQE4+vJvhd67Fs7jipA6pqI4lJNilKb33bxNIPq1bNAkkH7eEISQ5Bih2GGwOOTDcTpjNRADkSYT9Vs3Fe3nWMd4+YIJk6HXJ28vxlCMPsDyXbMeo+jprySwIMw9zf/IqaNh9tJ53Qi/P9bESZP0o8i6RaMMuOOlAPHv7SfjqqfPw5n3WAhcAdOvU0kepwEmH7R/9dwTBfxhce07/QPMDgMP6dsTg43vhpiGH4NGbVdMABdivQ2o+5Nq0yEP71uadR/3hVng/oLPx+V5xxkEAgL9ceqjhePO8cEUrEuyyHNkXResWTZL2Vdy4UXJCmtjh1nmiRbPGGPXihbjstL6m8+7KCUoA7Nw+uQNf144tkpq/mTuuOMzyXCQSwWmHto473rpFE/zw/BDcccXhSayZNa1bNMGpRx6Ac47rKT3vdsnFEhupwosc9Pb9p3suWm/j1Lypc0SrCxN01BpyUnCOXieZNCJu0Mt1+mVMP3vetpF8COfkRHDIgR3Qsllj9Nwv/l3W41eZuW7LHtdp3bSrp2k5d0Bv+YpOIug/qLvpBJ5zj+sVeFnyCgSXlVkT55brzu2PkS9ciIN6tA2uMgFAgl2Wk8q9If9whL/OkSjOW4qpKAoikUhc1Hyn6x+5SdgklVXE7B4Tses5bqA/+yi3NG6U2m7dsW0zw2/9pBKJyDV27VrnoVFujqW257LT+qZEk9rfYunsDI+mEGYmWsSREyQwI+neVav7U1RcEf13n25tHbO88syDDL+dBAdz7d0IM6ccEdNg2i0VD0zwmV+XgGaqzwFtEn7ufj1CNxftw/+dJ7ZHb9e6KTq0aeZwhT3P/O0kw++ke/HrHmluBu5H3LpFE5x+VHfpuUF9Otpem5MTCUWhYUfmPYEsxEpr4IVO7eQDQa4kxpPbPj7sluM81eE+G2+uls3Eq3bCoV095ekGxx3F1ASN1QDJ5onLadBrp6r795RV4e6rj/BVRz11NktlAHD94P5o18q/8W2TFAt25nn6jisPtzzndLxX19b4+F9n44bzBqBZXmDx0y2pqpEvTd584SGJ5VtdZ3kuJ4HHo39VrQS7VRtLov92CjYsw6uJ3fkn9o5LY/640NfDTnDUPrL8yiF6odHp48u8JH7PNUeiaV5iE3Qi8tPlp/fDd89eEDVTkHH4QZ1c5WX+Xkq2XV5NrehHrVo0lq4S+f1Is4tRaNeiPw1x339vvXigrTAaXWa2oVdXe01uqiHBLg2444rDceP5AxLK44bBB0uPd2lvvSz3wUNn2uZ5aF/7LxUv/HWwcOK4/4aj0UKyPGSeSJ+87XjXeTsNpi3VgVLTKNabVHZO17dtKYSsft3bRQVoBYrtFmmP/9m6/v17tTdoMMyclt8dHdp4sx3RL7/Kvh69LMV4tQFrqbMHatsqD/17xsqqqva2tc4RrDM6tm2G3JyIr6U0r5xpYeOX6Be4+X0+UWdDVesg2Gv03j9+sjj72NhH4CkmDfkFJwkBy24s6XuAs+1uba1R2L3+XKMWTK/hHv3SRehk0tge1q8j/nHNkZb561tvXjKUlaHxxRODLfPUaK4Lt+T0/uT37xxXrya65965fXN0tOmH+mdhxklr/rfLDo07FolEHHfnefgma49mg1mD6f7ZjXFu7qsezVGvdcuYANpjv1a46syD8OAfj5EKY3q7Oy9C0N1Xx96jl/7+B9fXebLzc/yScZ9VukCCXQg8+Mejo//+6knRqS47vV/0mJ/pzGoguegPMfsXs+Hnfh1aYPRLF1l6zgYZ2LdFUzFg5eZEcLtuALrijH4Y/dJFuPgUo93b4QcZB10Zrw89FW/ee5pjPVs0bYzPhp2DP10gJluzYOeksWvcKAdv3Xca7jUZ2+sH4SNZZ3z/XEzQO8Lmy/rUIw/AvdcHa7j/xr2n4cBuYtKWaUW9LMmfeXQP9OraGh89crb0/BCTTVb/nu2jE5V2J6PLswm8Q7J+cCRzfi/MPP3XE3DxKX2k5/Ia5+ICicYJAIY/fR7+drm1/aAd5vdZb8Mj0+Y9d8dJUcFM4/iBxuf4+ePnGoy8e+zX2jCWdGrbHKNfusgwlui1wx8+fBae/uuJjnWvNmkxI5EIjuxvfd/N/e/YQ7rGPzzdb0WX/aH9OuGea+OFQJlQZuUINuyW4/DS3/+A+284Ckcc1AlfPDEYnw47x3Ycvf+Go+K174rQbGmCyYcPn4WPH7X2upRpzzTNmJOzwgCfGqymTeRa7Iv+0MdoAmFRLzOd2zf37GD31F9OwJCTD8SfLx4UKy8SwfWDD0bHts2QI1FJ33JRLK2dTa6ZZnmNog4pzfIaGYTQPNUO9o+SDxmZTD/yhQt1dTjcdR3080OmhNYhwS4E9J0sL1EjbQcikQi6dhDaHNmSif21yaiRe0NVq2XPvt3bYvRLF6H3/m3Q0+XXX7vWTaPqdvOA7GaZosd+rdFUtzRotqVp3CjHoOVJSCi2GDxusfHcymucG71uoMQmxMotX0b71k3xxr2nWS7v5/fvjC4mBxAtrprW7DvVgVOmdQKME/eb950WtW8xTOiy++DjtuZEInHPQ++1etul8doTQGh/nDSnfsItyDxTB/TugM7t7LUMbVrmeX6v9GV1bt/coNGyorrWKHial3PNH5E5ORHDx6ECJc6OTjH8O/YrAqC5ZMndbgI1a/laNmuMg3q0w8mHd0MkEkHrFk2Et6RNHicf3s1SK/vds0MwQicEWGH3KGSrEgYCFhBYj3aGvmHO3urj1esKbf+e7dCkcS7+fPEgyyVjmcauWZNc9OveFoC3sdEs4Ldu0QTfPHM+brtkUPQj73Ldh0z0Okle+nHeSzdKl/AyXiDBLgSSYchq11natRZLiVYvqNWSRdivs6Uhs8fPJvOtOdakCXF6HLLSzJOd287vOuyApFJul2dlNfFm0Ox8f51sfZxslfSPsOd+raXehjIjez/vZCRiXNZ9959n4OYLXcYmc7gVXgRmDbNXts+i49IcNyjeKafO7CkkwTx21JiWYs2PQeYgoF/6UpT4utfbCOzNPSz9f/jIWXj+zpNt6xcrxp/0lJMTcfVcZfdBG9sVAJedELwnql+sBTtv768bMw2re+dXltWqqPXhZnmNcMFJB9oLiA6Feem2+nu0dN0u9xeGCAl2KeLlu2P2Afr3MTcRS2odbt5Tq45g2QdSsB/qhm17PV8TtDZcJpRpHmpmrLze3C51XnO2y505ZEKNU2BZmzuT6o3SA9kvUtIcP5pQ80TTspncwFuGnQenLG83yJaOgPjupijOXVCzAbv2bIb9O8bHUDObHcho1byxQZtfXSNx/FCzufAPB8afg/F17dCmafzHou6n+Z568YTt3C5e62gZxy6MZTNNCKlXMKiXtQY2kapJ2xUxLnGbE1l9eHrtT27SS+e0SCRaJy9FKroyvTxPZ6Fet7zqkFJ/7+Ys2+6+EiFCgl2K6Nc9FlZB3zlSqeb1WpIsvdfwKUf274y7dF6SGprn5m9Lt3msVfCYv1pbNGuMP9g4NwDxg4yfCd4re8qqXaWTDb5BhyAwt79tyzwc3Kt91Ng5kiNPZ4XmZJGns1v0q3Exk5NjXIpNZClIlneiHJhAMHLWsz1e+vsfcKXFVn49usidE/ReipFIxBBgNW4ZVffzSNZZOjBo1xx7yH448dD94+6b0Ss2drxZXiPp8zDfk2NttlCzekZ2nsmJcPygeBvW5+44CQ/deLRBY2eHH693W6cDxeQRbDptZYMdkF7BgCwSAxCrk/6jz2kVomPbZjGNnYc6OI07siHAzbBgDkScrpBgFwJBLcUathFzkaVlsWonMAdZlKV32sfQzOO3Ho+zJN5jiUyIijxChW+0ieWmCw7Ba/eciq+fOg9d2jePDr6GicOi2vrBTLMB8rIVkRsqKr15mOppFOAIrijxk2lubg6ev/NkHKHavTg9XfP1eapRePcu+gCz8df56Trma7zk4aTwct2X1Xw6t2sWJ8g8pHo6mrVr/Xu1QycHuzsAOKhHO8sPi7uvOQK9urbGq/84xXDcSuADgOMGdsWlp/aNOXPEhQeKv+bKMw7CoX074u6rj1CXvo3njSuxsR9XmGLoaTz11xMMv+3scq0eUXebNnrJxxL1PuREhI3k8YP2j773ToJFm5Z5+N+zF3iysTZvM2YW1vTvj7l8q3vhdcXITRgR2cd/BLoxISKCYv/9qsOjbXj/QXmEhr4HtNXdU+ub+sD/GUNt1TnssCL7mHDySAaAwcf3wu2qQ1UqwjH5hQS7EAhqhbONzt1c/xVk9orU+kOOhUpbC/kwoLdxSUSq2ZD0rYF9OuCMo+XBHa3QjLplAp75i9gcly0S8Fur1aFZXm7UsxSQPyfN/qixafDSBsjXh56KZ28XwUEv/EMfTx5gg4/vBUDsFnDpqRKDYIfZwu50x7ZN8dCN9pu/a+zvYlskrSwr7zKvSzxDTu6N+284yhDkOqiltJyI/9Ap+usO72ftBelIREw+z90h7MNOPfIAHHFQJ1x6al90Vp1UzLZt+f27RDWYXiLb6+vcuFEu3rj3NPQ5wHi9zBGne5eWuOy0vmiUm4ObhhyC6889GGcc3R0XmTyKZRNgx7bN8PRfT4yGFiq3+QjR97E8i8m0VfMmeEUnjPoZM9u2yjPssHPducFsq6V5abZtmYerzjoIL98dq6dmL+hG25zXOBeKzZeDlXOCdoXZrEO/y4hM6JCZY1hp1/QxTD9THYT269DclbDc0mI7tPtvOBoXnNQbB+7fBrdePAhnHhP/wS/DzVhi9haukpkT6Ni/Uyw8lNZdrDyO9eTkRHDu8b3w2K3H4am/nOCYPizSV+TMYpzsdtyiKMDVZzGUV9YYjj/4x2MwZOjIuPTN1I5vnkBOPrwbTj68G0ZMW+1Y5p8vGYSnP/4de8vFsqDZ7d0tfQ5oC76+2BBC5Lk7TkIL07Y9r9x9Ctq1zsONT/wEQNgnnXjo/nH5aYx4fgguvn+04ZjT5KuNG27skbTlHfPkpmlMzHv86uPLmXdoMPPnSwbhijMOQsvmTXDyEd3w/OdzDeed3ptjB+6Hwq174pZ5br7wEJx/Ym80bpSL9q2bYveeyrhr+x7QBqs3lQIQGhuNa85m+OonHpdem7isbm1UuLA0bDfSuFFunFbmxEO7YvGanYZjfmz3zM/fS/fTP897rhPLzDW19aisqsVvS7cZ3oPjB3XFrMVbo7+bNMpBta6vnXRYrH2yfWpr1I+dS07ta3Cw+Pqp89BEsnuHFW7aJxPO3r7/DMPvFs0aG+KIaWzbVe6Y/wmHdsX7Ixajc9vG2FFSA0VRMPzp8zB/ZRGOH9gVn49bEXeNef7ue0Bb5PfvjIIVO+LSfjrsHPzx8QkAgEN6u7PR269DMFvtXXdufxzatyMOObBDXODd0/IPAF+/Gycd1g3rVpWgSeNcg83iFWf0M3jj2g05Hzx0Jq5+ZGzcce1DwKzN1QvvjRvlYOQLF+Ki+0ZFj33++OC4/Mwa57OO6YH8g7sYvJzbtW6KP188KOr57kT/nu3xt8sPQ5tIEf797RYA4tl27dgCt13ifynT7r3O798Z3Tq1xOaifQBEyJ03v11oW0cz2kfGgd3aYO3mUtu65PePDxF24P5tsHZLKbp1Su2WjjJIYxcC7VoJuwLN/duMl6Xa687tj1svHhQdFO1svbR4RVZBWbUtJo8ZsF80eLGmPTvq4C74/PFzcciBHQyBIv0IdUBMXd9VN9gO6N0hzjuyb/e26NCmGW66YADOOa4nLj+9n+VeqAN6t5faksnCSHymC1Oh3e9mLkJBaLEATzzMqFU8V9W2mTm0b0zL06dbG2kajUa5OYYQI6NfusjTnr7Xnt0fnz9+Ltq1bmrYTP3iU/pGJxOzkKPFaTvCIj6clW2TNshavaqa8NhetaH5dNg5+ORRXVw8F8LH+ScdiM8fP9dwTF/e3686wjkTyMOduKVbp5bRD4m8xrlo16opOrdrjh77tY5uAP72/afjnmuPjAvo7dWuUQsM3LpFE7RpGRPOWzRr7ClgctSWyaHJ157TH6fl29uSRlHzcuNlCwAd2jTD6JcuwmmDRH9WFBE+5sRD95d+aN115eF46774fXG1pTrzvdRv/u7FrMPOVk+2T6yMRrk5lv3lgM6t8NRfTozWz7yd3v+dN8CgadPGk1YSLVeLZo2lQeLPO6E3ht1yXFyAajPm+yLzaNXmi/Zq5ISuHVtIP5yHnHxgXIgju3IHH99LupWgH2IrTdbvXiQSMezg1LaVWOo2k5sTwUV/kMe01PaHlsUgffimY3C/zc5KAPDEbcfjyduOx7v/tA/8nwpIsEshD1/VDV89ORi992+N68/tj3/qgosCwH3X56Nj22bRmF53Xnk4/mWznYl56RSwF+xyc3Iw/OnzLEM9aB2nW+eW0a/bv191BG67ZBAevfnY6GQThG3BXy87FENOPhC9HYQdjUtP62cbVPLjf50d3e3BTYTydq2b4tC+HTH0unxce05/XH56v7i9bo8fJAY4fXu7d2mFb545H6cfZRSOD3TRjjtNTiSP3XpcNHq9m51HmMW+pgP7iPcgJycSfUZ/vUy+BHymLoRMs7xcnH1MD/xpyCG4+iyG14eeahS+ABzQpRW6d2mJq3XG+U0a50S1glbhD/Iai3umCbPt1f0vtWUst4o3vYCjp1O7ZoYPFP1+p41ycww7HzRpnIuObWOCQFMHuybz7gz3XHsk3nngdMsYcN27tMJp+d3jtDf6pa+uLrRFJ6gT6jED5AHD3ZITEfa3D5rGFzPXnM1wz7XuAmVfqgZc7tOtDf6qBqN20880ecy85PfYrccZTAPOOrandJlP26VDFuvODw/deIwhRp32AXTHFYdFtw4MEm3pHRABr81oO0nceaX8I0Ubq1vq+pkWN9DNx8pNFxyCoyXvk7abiGZ28e/bT0LblnmG+I5B8PwdJxs+Ms1o5bXSCdUnH94Npx55QOzjOxruxL4sTbB77FaxjCxb5h/xwoVxZghHMCHInXdib5xxdHdcdno/dO9iNEc5bmBXnOwgSLdpmecqsH5KUBSlQf83d+7cXnPnzlUqKyuVZDN37lxf123cvkeZt2K7UlZRrfy2ZKuyt7xaWbu5RKmvr4+mqa6pVW56coLy/ZRViqIoSsneSqWqulZRFEV545v5ygX3jFDWbSm1Laesolp5+csCZW9ZlWOdtu0qU2pr66TnVm0sVh59b6ZywT0josf8tt0vF9wzQrnv9enK+z8s8p1HbW2dsntPhW2aucu3KRu27bFNc9u/fzbcCz319fWO1/++dKvyn5GLFUVRlMWri5QL7hkR/a++vt7wHuhZs6lEWbu5xHCstq5e2bRjr3LL0z8pMxZuti3XjL7M6ppaZeJvhZZl19fXKzMXbVFqa+sMz762rl559/uFytad+1yXO3nuhmjZS9fuVPaWVysVVTWKoijKzEWblS/GL1dqa+uiaTRWFO5Svpu8UlEURamrq1cmzC5UyitrpGUMfXWa8ta3C5QL7hmhfDlhheu6mVm3pVTZXLRXURRF+e/YZXF1amjMnP278spXBcrOknJf1+/YXa68/8Mi6VizZM1OZcmanY55LFmzU7nt3z8re8ur485VVNYoo39ZY/keJ4L23tfV1Su1df7znzJ3g+U49Mg7M5QL7hmhlOwV89cF94xQbn16omVeKzfsVkr3ifF99uIt0X4k4+ff1ysvf1ngu95uxvy6unqlQu2T2jxlfk7T5m2UHnfDlqJ9yoVD5X2wYMV227HXbmx1ItnzXWVlpTJ37lxl7ty5vRSJXBNRQgn2kz4UFBT0ArBu4MCByMvzv/G6y7KQnx/sVlJuqK6pw9K1uyyXD1JBWG1PB8orazDr93k44w/Om0m7oaKqFne/PBWXntYX5xzXK5A83TBq+hoM7NPRlXZSTxDPvrKqFjuKy9FDEshY45f5m3FQz3aul4xkbC7ah64dWgQSxkRRFMz+fS76DxgUNb9oaDTkfh9G20v3VaFJ49y08NhMl2dfXVOH+nrFsHNQskl226uqqrBkyRIA6J2fn19oPh/+0yeSTpPGuaEKdQ2d5k0bo22L4Lpas7xGeM8iPEAyudDCNiUVNM1rZCvUAXBcKnFDNxcewW6JRCJo0iinwQp1ROqxMl1oyLgJY5JtkI0dQRAEQRBElkCCHUEQBEEQRJZAgh1BEARBEESWQIIdQRAEQRBElkCCHUEQBEEQRJZAgh1BEARBEESWQIIdQRAEQRBElkCCHUEQBEEQRJZAgh1BEARBEESWQDtPALkAUF1dnZLCqqqqUlJOOtKQ2w407PZT2xsuDbn9DbntQMNufzLbrpNXpNtq0F6xBQUnAfgl7HoQBEEQBEF44OT8/PxfzQdJYwfMAXAygK0A6kKuC0EQBEEQhB25ALpCyC9xNHiNHUEQBEEQRLZAzhMEQRAEQRBZAgl2BEEQBEEQWQIJdgRBEARBEFkCCXYEQRAEQRBZAgl2BEEQBEEQWQIJdgRBEARBEFkCCXYEQRAEQRBZAgl2BEEQBEEQWQLtPJFkGGP9ALwB4CQAFQC+BvAA57w81Iq5hDF2BYDrAOQDaA9gDYB3ALzHOa9X03wC4I+Sy6/gnP/PlN+9AG4HsB+ApRD3YpIpTSsALwC4HEBTAFMA3Mk5LwysYS5gjN0I4GPJqbc453fo0g0G8DSAAQA2A3iVc/6GJL+Mabtal6kATrE4/SDn/FnG2GMAhknO38c5f9GU3/8BeAhAL4j36AnO+XBTmsYAnoB4n9pCRFb/O+d8gd92uIEx1hfAvQCOAzAQwArO+UBJupQ/62SPIU5tZ4zlAhgK4HyIdjcCsBjA45I2FQLoKSmmE+d8py5dWrRdLcPx2Yc1xoX97NU0drsYHM85n62mmwr5eHE053yuLj9XfZwxth+A1wCcC0AB8COAu/XvUSK4mdvUdBnX50ljl0QYY20hHlwriAc5FMA1AD4KsVpeGQqgCsB9AC4AMALA6wCeM6VbC+B403+T9QnUF/8ZAG9BTBKrAIxhjB1myusrABcCuBPAVQD2BzCJMdY8qEZ55FwY2xUVWBhjxwMYBWA+gMEQguCrjLG/6DPI0Lb/DfHP9G313FhdugpJui/0GTHGLgfwKYAfIO7TzwC+UgdNPa9ADI7DAFwEoBqi/fsH1io5h0A8l9UAlskShPGsUzSGOLW9GYRAvgDATQCuhpjgJjLGLpCk/x/i34cSU5p0aTvg4tmrpHSMS5NnD8S3+XgAswHsADDXlHaGJO1yUxrHPs4YawRgPIBBAP4PwC0ATgAwijEW8dFOGY5zW6b2edLYJZfbALQDcLj2lcEYqwXwBWPsSc750lBr544hnPMi3e8pjLGWAO5gjD3COa9Sj1doX24yGGN5AB6B+Np5UT02DeLL/2EAV6rHjoXoGOdzzseqxxZDfE3diJhgkUoKbL4SHwUwj3N+s/p7CmOsB4BhjLH3Oef1mdp2znncQM8Yex3AYs75It3hertnr/IkgG855w+qv6cwxg4G8DiAcWre3QD8BcBdnPMP1GOzAawDcDeA+xNojhOjOecj1TI/AXCUJE0YzzoVY4hT2ysA9OacF2sHGGM/ATgIYtL50ZR+u8NYkE5tB9w9eyD1Y1w6PHuY26wKHkcAeJ9zXmtKXuJwj9z28csAHAZgoNZOxtgWCMFxMIwfln5xM7dlZJ8njV1yOQ/AJJNQ8B3EV4JZU5GWmF58jfkQauT2HrI6AUAbCJWylncdgG8ADNZ9hZ0HoBTia01LtwGiQ5/nqfJJRu3QpwMYbjr1JYQ6/kj1d1a0XV0eOBrA5x6v6w2gP3TtV/kSwNGMsU7q77MhNreO3k/O+V4IwSGp7dcvvcgI8VknfQxxajvnvE4v1KnHFAgNnh9Natq0XS3btv0eyLpnb8EVAPLgcRxQcdvHz4P4gFyqSzcTwHoENBY4zW2Z3OdJsEsuB8Ok3la/AtZATHSZyskAdkOo4jX6MMZKGGM1jLH5jLGrTNccrP41q+WXAmgJoJsu3QrJgLMU4d2zJYyxOsbYOsbYMHWZAAD6AGiC+CUMbTDS6pvJbddzPYB6iIFNTzPG2A7GWC1jbAVj7HbTea39VveJ6dJt55zvkqQ7iDEW5ngV1rNOyzFEfRYnIL6dAHAdY6ySMVbGGJvAGDvSdD5T257qMS7d2q9xPYCVnPPfJedOYYztU5//r4yxM0zn3fbxuLbr0iWz7fq5LWP7PAl2yaUd4m1LAKAY3rRdaQNj7CgIO5tX1K8SQHzl3AvgYgibgE0AvmbC+UCjHYAqznmFKUtNE9Bel65EUnQY92wrhB3IjRB2dj8A+BeA/6jn26l/S0zXydqUaW2XcR2AaZzzTbpjqwE8AGEDciGAWQDeZMKpQsPLfTKn0dI1hhgkwyKsZ52u78SdEAL5S6bjowDcAeAsiCWlAwD8whgboEuTiW0PY4xLp/YDANRlyJNhsqFVmQaxnHoegBsARAD8xBg7XZfGbR9Pedslc1vG9nmysSNcw4SX0ncAfofOwJRz/pop6UjG2GQI+6lPUlbBgOGcTwAwQXdoImOsFMBjjLEnQ6pWKDDGjoP4gn1Gf5xzbl6OGcsYA4AHGGMvcM7LUlRFIkUwxk4B8DyAFznnv+jPcc7v0v38hTE2DsAKAP+EMILPSLJ1jPPBtRACW9wyLOfc4B3PGBsFYCGAx2ByMkk3rOa2TIU0dsmlGMKd20w7CHVvxsAYawNh5F4O4ELOeY3DJd8C6KGznyoGkMcYa2pKp30V7dalayvJL13u2Tfq3yMR+yJra0oja1Omt/16AJUQHo9OfANhp6JpabzcJ3MaLV0NgH3uqpoUwnrWafVOMMYOBTASwoPwAaf06pLbZIiQEhoZ2XYJyR7j0rH91wGYxTlf65RQXT4cCffPXt/HU9Z2m7ktY/s8CXbJZTli6+8AokbYfSC+YjMC9YUdBaAzgHMl9hFu0OwPDjYdHwBgL0T4BC0dY/Eu7QOQfvdsDYSrvqxNQKy+Gd121abwKggPuj0+srBrPwBwXbrOjDHzcsMACJueoIzc/RDWs06bMYQx1gdCgz0PwA2qA4UfMq7tLsnaZ6+WfThErDs/ThMabvt4XNt16QJru8PclrF9ngS75DIWwBmMsQ66Y5dAeBQF4a6ddNRJ/RsAhwIYzDlf7+KaCISL93qd59FMCI+gq3TpctV043WTxFiIr5VzdOm6QwRqTId7djVEsMwC9Yt0MlR3dh3XANgGMQECmd/2cwB0hPsB/WqIEBlLAYBzvg5iQDIbm18DYI7uHfkJwjkjej/V8ANDEPKzD/FZp8UYoi5V/QTR1os559Uur+sI4AyIILQaGdV2GSka49Kt/ddBaNXMXqJSVGHkYhifvds+PhbAICZCImnpjoMIbh5I253mtkzu8xFF8fvRRTjBRLyfJQAKIeJ4dQbwMoQr89Xh1cw9jLH3APwZIr7QL6bTyyDUw59CBF5cDfHi3gLhbHCD3gaLxYI4PgjRKW6BiFd0LOd8oS7djxBxkoYC2AMRpbwdgEE8hTt2MMYmQHTsJRCD0WCIoL0fc85vVdMcD2A6hJ3NFwBOVOt7O+f8XV1eGdV2PYyxryCM4bual+AZYwUQz59DeJBdBTEBPMI5f1qX7gqICeHfACZCBCb9O0RMp3G6dG9CGF4PhQhtcC9EbK1BnPMtSWxjc8RCDtwO8XV8j/p7Dud8fRjPOhVjiFPbITwEZ6nHrwewXX89j+08cA1EoNdxEFqKXhDLtd0gdh+IahvSpe1qOU7tB0IY49Lh2WvCjuqtugEiptuFknxOhgj0+4Na3/0g+nc+gLM451N1aR37uCp0zYVwqHgQwh/gBYh378QEtMX6OtvObZzzPZna58l5IolwzktUj6DXAXyP2NYgyQy0GjTa18XzknOnAVgE8bXyCMQLWAPxYl/IOR+tT8w5f1E1rL8LQBcIjc75+hdf5RqI3R3ehvhKmQKxdU+qBZvlAP4E4dnXCCKa+AMAXtUScM5nMcYugujU/wdgC4B/6Du9mi7T2g4g+jV9IYBPLewqV0N4wnVVfy8F8CfOuWErNs75t+ok8hDEQL4GwLV6oU7lHxB2Nk9BxIaaA+DMZAp1Kp0hbKb0aL9vAvBJGM86RWOIU9unQgSLBYRtnRltaWkdRFy7lyEmq1IIT8nL9UKdSrq0HXBu/yiEMMalybP/RP33qRAC+j2QsxXiw+4ZAB0g7NVmAziVcz7DlNaxj3POaxlj50JsKfY5YluK/T0IoU7FaW6bmql9njR2BEEQBEEQWQLZ2BEEQRAEQWQJJNgRBEEQBEFkCSTYEQRBEARBZAkk2BEEQRAEQWQJJNgRBEEQBEFkCSTYEQRBEARBZAkk2BEEQRAEQWQJJNgRBEEQBEFkCf8Pn/9H/koRoC8AAAAASUVORK5CYII=\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "df1 = df[df[\"name\"].isin([\"Latency\"])]\n", - "ax = df1['issue_to_done'].plot.hist(bins=BINS, alpha=0.5, figsize=figsize)\n", - "ax.set_title('Inference time (usec)');\n", - "#ax.set(xlim=(0, 25000))\n", - "plt.xticks(rotation=60)\n", - "plt.show()\n", - "\n", - "ax = df1['issue_to_done'].plot(figsize=figsize)\n", - "ax.set_title('Individual inference time (usec)');\n", - "#ax.set(ylim=(0, 200))\n", - "plt.show()\n", - "\n", - "\n", - "# df1['issue_to_done'].describe()" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAApwAAAFKCAYAAACwxI8KAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAAA8y0lEQVR4nO3deZgkRZn48e8A0oBcw+24ooL4Ao6iDKyKuIsiIqDIKoegIigs7iIeC4oiIiIiyqh4gNeq4G9REF1PEA9AVJDDwUWG4wURmMVZEbkVbJCZ3x+RNZMkNdPdNZVd3T3fz/P0U12ZUZmRkUe9FRkROW3hwoVIkiRJbVlh0BmQJEnS1GbAKUmSpFYZcEqSJKlVBpySJElqlQGnJEmSWmXAKUmSpFYZcGpSi4hjI2JCje0VEQsj4nODzofUi4g4LSJuGXQ+JqqIeEp1jh8w6Lw0RcS0iLgqIj446LyMRkScHRHfGHQ+ND5WGnQGpKYxBJAHtpqRCSIiVgPeBfwsM3824Oy0KiLWA44EXglsDDwAXAF8MjPPHWTexiIitgSOAZ4LPAG4C7gRuDAzjx1g1iaMiHg58GbgH4G1gbuBy4CvZOa3B5i11kTEDOBfge9k5v+0sIp9gacBn2xh2W34MPDriNgqM68adGbULgNOTUSvb7z/V+B5wBsb0y8B/gs4cTwyNUCrAe+v/v/ZAPPRqogI4HxgPeArwBxgOrAfcE5EfCQz3z3ALI5KRDwfuBCYD5wG/AGYAcwC3g0cO6i8TQQRMQ34PHAw8Fvg05QyWh/YFfjviHhtZn5tcLlszQzKuXwL8D8tLP+dwDcz888tLLvvMvPKiPg1cASPve5rijHg1ISTmf9Vfx8RLwH+sTm95u/t50ptiojHAd+kBJj/lJmX1+Z9HDgDODIi5mTm2eOct8dn5l/H8JGjgb8C22bmnY1lbdjXzE1O76AEm58B3paZC2rzPhIRLwMeN5CcTWIR8Rzg2ZTjbzI5CzguIg7NzPsGnRm1x4BTk1pEHAu8PzOn1abdAlxPqfmcDTwDuAl4a2ZeEBF7AMcBTweuBQ7OzDmN5T4dOB7YEXg8cB3wocz85hjytg+lNmMTIIEjM/O8Rpq1qjR7AhsBt1Fq907IzEci4inAzVXy90dEp6bzdOBjlBqiPTPzW9Xyotr232XmZrX1/D9KIPfk2rRtgQ8ALwBWptQovi8zL2zk8QnAB4GXUwLC3wOfyszP1tLsQKnV2w94KvDvlJrKi4FDMvN3IxTXq4GZwDH1YBOgKodDgJ2r/J7dWOeL6k0NamV2YGaeVps+4j6t2uV9pUqzB7APsEH12RuAwzPz443yeRZwFfDvVZlsClzbDDarbbm98dndKcHXcyg1fH8EvkHZD3+rpTsNeA3lmD0V2AG4DzgxMz8VEc8APkW5E3An8N7M/H9dtuvFlLJ+DTAE/BA4rJmvbiJiP0qwOBP4G/BT4F2ZeXMtzdOAE4AXAutUefkV8JbM/L+IWBU4inI+vKMRbHbKqHmOrFctc3fKrfffUZpYfLGW5imUff6eqlyOoJxPlwBvAuZV8/6Nclz+BHhjvSawdt34KHAS5boxDzg+M786ivJZ6nlSO14BvhIRX6n+/0CnmcUyXnf2AB4BLmjk61ga18hq+gGUY+KpmXlLNW3rav3bAmsAtwM/B/41Mx+s0kwD3kK587QZpby/T7m+/bmxjp0o5b4NMI1yDn02M/+zluwnlOv0zlTntqYmOw1pqtoE+DpwDuU25trA96ovzU8BX6O0sdsEODsiVux8MCK2oLQleybly+dwyhfn2RHxulGu/wXAZynBw3uBVYDvR8T2tfWsSvkCOoDSNOAtlC+LYym3HAHuoHxJAnybctvp9dX8uZR2b/9UW+8/AQuAp1VfgB0vpHxxdNb9z8AvKEHBcZR2k0PAj6svxk66DYBLgZdRAp23Ves9NSK61aS8C3gV5Qvkw5QA6IwlltJir6heu36xZ+a9wHeBLSJi01Es71F62KefBrYGPkQJgm+kBE7d0r4OeIhSUwPldulzImKrUWTtQGCYcky+lbL/30G5Fd+0AnAu5fbzOykBzScj4kDgR8CVlP14H3BaFfw1fZJya/844AuUIOXHEbHy0jIZEe+mHKM3U8puNrA9cHFErF+leVyVj+2BUyg/Ok4FNqTcSoZyXqwLfC0zR7wzERGrUM6RA4Ezq+2+HfhClaem1wCHUWpPP0Y57s+m/FB5BWXff54SFH68y+c3Ab5FadpxJHAPcHr143Fp+RzNeXId5ZoDpew75/J/V8tY1uvOdpQfOg+OIm23bVifEvxtWq3/LZTjcCYl+O34LKXsLqu28wuUH8wXVvurs7zXU46HDavlvQu4HNitseprgQcpx4amMGs4NVVtRqnR+wVARFxHufh9GdiiUysTEfdQvoBeRKmxgfKlPB/YpnbxPiUifgycGBFnZOZIHZtmAttl5q+q9ZxG6TRyIuULGUpgsTmwdWZeX037QkTcDBwfESdlZkbENykX+d92aW5wMY8OOF9IqbXaoZp+VkQ8CXgyJXiqt6H7JbBTZ1uqnvW/odQmbVct73hKIPrMzLyjmva5iPgicFREfCYz76mtfxVgq8x8qFrm3ZSgaGZmzl1KeW0J3JuZty4lzVW1tDctJV03Y92nfwF2aARFXwU+GxFbZua1ABGxAqWjxjmZeVeV7qPATkCnfdovKEHT+fVay8prM/OB2vvPR8SNlP3/zsz839q8xwFnZeYHq3V/vdqmLwGvz8wzquk/odTUHUD326s7ZOZwlfaa6vP7A//ZJS0RsTGl5u7YzDyuNv1M4BrKcXwUZb9sAuzVqJE7vvb/ltXrb7utq4t/pZxLB2Tm6dV6T6Wcy8dGxBcbNcn/ADytc0xWPyTfQ2kH/ZzMfLiavgHwmog4pBGgbQbsl5lfr9J9gXJOnBQRZ3erka1t40jnye0R8UNKsP+rLk2ElvW6sznlLkWvtqP8AN05M39dm965q0JEbAccAryhXusbEedRjvP9KdewNSlB/5XAC+tlXF1/FsnMv0fE/7L42NAUZQ2npqobOsFm5bLq9Wf1W4C16ZsARMQ6wEsoNZOPj4j1On/AecATKbc1R/LrTrAJUH0pfg14QURMrybvTQn6/txYTyfw3WEU6/kF8Kzq1jyUIPMCSm1LJxB9YS0twFZAVPlZt7beNSk1HM+NiNWqL4Y9KbXECxt5/DGwKqUXdt1XO8FmY52bjLAdawD3j5CmM3+NEdI9So/79ItdauDOotRG1js37EAJchbdvs7MCyhl/gNKsHR49f/tVW0ktbQPVHlcISLWqvL0S8rtx627bM5/1j57D+XW9N8otfmd6UmpmetW5p/vBJuVr1ZpX94lbcerKJUTZzXK7l7gasqPNSg1qwA7R8TjuywHyjEGI+/rjt0otfyLgrPMfAQ4mRLgvaSR/luNH0Cd8/u/OsFmbfrjgCc1Pv8nFtdUUwVK/1mle1a3DPZ4njSX0Y/rzrqUOx69urd6fXlVW93N3pQfY+c18ng9pea5cyy8lLKvT2zWuC4haL6b0tRBU5gBp6aqefU31S1ZgP9tpOtM7wSBT6N82R9L+aKr/32sSrPBKNZ/Y5dpN1SvnXaUT6fUhDXXc+kY1vMLynm8fa0m8+fVXz3g/FOtFrXzxfWlLut+W7W8dSltCqdTRgdopuuMndfM47zG+84X4HSW7n5GDiQ78/80QrqmXvbpY2pQM/Nu4HvAfrVamtdRhjw6p5H2ksx8JaUpx7MpNY0LgS9HxIs76SJiZkScS/kSv6fK00XV7LV4tIcz8/8a0+4F/tCl5u1eupf5o47LKqi+GXhKl7QdnePleh5bfttQlV31Q+7jwEGUH1E/jYi3RcS6tWV1gtLR/mh4MqU98iON6ddVr818N4+/0Z73HTd1KcvOedtcV0cv50lTv64700ZOskQXUTruvR+4MyK+HxEHN348PB1YnRJcNvO5YS2PnWYvS7ur0cz3hBpPWf3nLXVNVc0vqJGmdy7UnR9hn6C0l+tmtBfRkaxAqY388BLm/34Uy/g1pf3TP1GCm/sptwDXoNxyXIcScP6ysV4obVuXdAvujmp5UGrPvryEdNc03o9UvktyLfDsiNg4M5tBQ0enhqlTLkv6glqx8b6XfbqkdnBfBfYC/ikiLqN0wPlao1Z3kapW7Srgqoj4FaVt4OuAC6pa6QspPdrfS+kM8yClNus0HlshsKTbub2W+Wh18rEL3UeEWFRWmXl4RHyZ0sHnpZRg6eiI+OeqGUInUHwm8J0+5a+u1/N+WXTKZyznyZKWsSzXnT/T/UfGqM6TquZxr4j4R0qN906U9pnviYjnZeafqnzeSWkr202vNazTWdw5UlOUAaf0aJ1g5u+Z+dOlply6zbpM69QUddop3gSsMYr1LPGXf2Y+HBGd2+drAZdUvbovpQQHr6S0jfpi7WOd2rv7l7buiLiDEsCutIxlMRrfp/Rw359Ht/nr5GVNyrZcmZmdfdT5clu7kfzJjff92qdQbm/+iXJbfUPKbcP/t9RPLNbpfd/pQPMiym3EPTOzU6vZ6dnbls0ot3k761qJMqrARUv8xOLjZV6n7erSZOY1lADrw1F68M9h8VBIv6Tst/0i4oQuNZdNt1I6YK3YSLt59XrLSPkZo00jYoVGLWfnvF3SusZynizpXO7HMXodZV823Q0QEWs3mhs0zxMAsowScTlwTETsQgmAD6a0Ab+JEohempl/WUpeOsfMTErN+BJVx+CTWHKgrSnCW+pSTfUr/kLg4Ih4YnN+1ZNzNLaJMgB453PrUgKqS6pbs1Daim0bEbt2Wc8aETFUve10KlnSbelfUHoe70TVE71qN/VrSk/badR6qFMCgN8B/xERj7m12dnG6gv+m8Ae0aXH9RjKYjS+RQlS3h0R2zTWsyKl09R0qo5PlVspNVf1TlNQekcv0sd92rkFfQalzd6bKLd7L2ks78VVZ6Kmzn7ufAF3Aqj6kF4rAP8x2vz04JDacQUlwF+bRpOAhm9R8npMs8MHLBq2iIhYswoe6q6j1ICuDYuOyw9TAsaPLWF5L43yFCIobV/Xp5w7nfkrUJp+DLO4vXO/bEAZCquzrlUpTQRuYwkdncZ4nnTGc33UudynY/RiYMsqz3Wd4G/ReVLdJn9DYx3Tu+yPK6vXtavXsyhxwzGNdETEirX26T+mNJ94dzM/XdaxJaWz4SVoSrOGU3qsf6NcvH9b9TK9ifJF9FzKxbHbcDNNc4EfRMSnKbUf/0q5zf2eWpqTKEO1fDciTqcEgqtSagX2otx2vCUzH4zSm/g1EXED5ZbWzZnZ6RDxCxYP8VQPLH/O4mFyFj02LjMXRMSbKLV111a3QG+j1Lz9MyUA6jT+fzelY8yvqrK4hvJl+WzgXyhfFMusqql9NaWJwS+rPNWfNPQcyniI/137zL0RcTZwWJTHod5EuRXYra1bP/Zpx1cpNXYvpftTgz4FrB4R36YEXCtQOgC9nrLvTq7SXVy9P706Th6mBLKrjyEvvbgwSg/3p1CGEJpLGde1q8z8fZQhiE4CnhwR36G0N30qpdb5LEo5vJjSq/qblM5M0yjB2xrUOuJQhlTanBI07lDtw/mUwHLnajmdAPOLlHPnS1EGNv89ZSinHYH3ZJexTpfRjZShjJ5DOSdeR+lg99ql9FCH0Z8nN1FqHP8tIv5CuTbMrUZwWNZj9LuU4Z9ezKN/QPyY0rb1SxFxEuXHQ6e96ca1dG8ADq2O25so16IDq/TfBMjMn0fEKcA7q9rrH1EC/6dRjt1jgNMy876IeBulicGvI+JrlGP9GZQmI6+qrXcnyo+SH42wfZrkrOGUGqpevttQOojsz+IxBVcC3jfKxVxcfWYfyjBDw8AembkoIKxqe3YAPkKpfTiZMrzMFpRhaP5YW96bKLf0PkZpK/ZvtXm/otw+/xuLb9vC4h7iFze/LKt8PI/SQenfKUOYvJHSAeYjtXR/onzh/Sfli/4zlGBrI0rv676pyn2rah07UcYzPIkSbL4hM7uV/WGUL9o3U27Fz6NRc1Nb9rLu086y/ofFtV3dnn51BKW3/86U4OqT1facQXn60C3Vcu6i9ML+X0qgcBSl1/f+Y8nPGL2N0sb3/ZThbb4HvHRJbVA7MnM2Zf8/ROkA9XFKIPUzFg/WfRVlSK5dKdv9QUrQuUd9mKTMXJiZb6IEq/OBt1PaCh5O6Ty1e2dYomoYqRdRAuLXUo7/J1AGIm/jkba/p7TL3ZEyvNV0ygMElvqYzdGeJ1Wb3tdTztVTKOfyntW8ZTpGszyL/EpKT/L69Icp++omyj55a5XPzzQWcRHl+rE35Zg9inINenHtxy2Z+RbK9Wgdyh2HEyk/vr5BbdD5LA9deDnlmnIUpTyfT2k+U7c38O1ax05NUdMWLrRjmKSJKSKeSQmcb6WMqzohvpQi4grgocycFINVx+Knyjw/My8dIflyKaonDWXmywadl15FxL6UWuEnt1D723dRnmz0a2BWZv5m0PlRu6zhlDRhZebVlJqwAL4dIzwRZzxExLMpNVFLvA0tDciZlJrMtw84H6P1HuCbBpvLB9twSprQqh7cfWkruiwiYialc9Y7KD3Vu91OlwamGtpoNI9UnRAyc69B50HjxxpOSRqdPSm3pVcFXpOPfiSlJGkpbMMpSZKkVnlLvQdz5swZArYF/o8lP8FCkiRpIliRMsLDFbNmzRoeRAYMOHuzLYuHnJEkSZoMmo86HjcGnL35P4CnP/3prLxye51m586dy8yZM1tb/mRhORSWQ2E5FJZDYTkUlsNilkVRL4eHHnqIG264Aar4ZRAMOHvzCMDKK6/M0NDQSGmXSdvLnywsh8JyKCyHwnIoLIfCcljMsii6lMPAmgHaS12SJEmtMuCUJElSqww4JUmS1CoDTkmSJLXKgFOSJEmtMuCUJElSqww4JUmS1CoDTkmSJLXKgFOSJEmt8klDE9jqa63H7Xc9MOhs9MVqQyuxxuPbewyoJEmauAw4J7BHFk7j/CvmDTobfbHjthsbcEqStJzylrokSZJaZcApSZKkVhlwSpIkqVUGnJIkSWqVAackSZJaZcApSZKkVhlwSpIkqVUGnJIkSWqVAackSZJaZcApSZKkVhlwSpIkqVUGnJIkSWqVAackSZJaZcApSZKkVhlwSpIkqVUGnJIkSWrVSoNacUTsBbwWmAWsA9wEfBb4fGYuqNKcBryhy8f3ysxvNpZ3BHAosBFwDXBkZp7fSLMGcBKwJ7AKcCFwWGbe0rcNkyRJ0qMMsobzcGAYeCfwcuA7wKeAjzTS/R54fuPvgnqCKtg8ATgF2A24ETgnIrZqLOvrwO7AYcA+wAzg/IhYrV8bJUmSpEcbWA0n8IrMvKP2/sKIWB14S0QcnZnD1fQHM/PSJS0kIoaAo4GTM3N2Ne0i4GrgvcDe1bTnUoLR3TLz3Gra1ZSa1QOAU/u5cZIkSSoGVsPZCDY7fkO51b3OGBa1HbAWcGZt2Y8A3wB2iYhp1eRdgXuB82rp5gEXV/MkSZLUgkHWcHbzQuAu4E+1aZtGxD3A44G5wImZeVZt/hbV63WNZV0DrA48EbitSnd9p31oI93Ofcm9JEmSHmPCBJwRsQ1wIPCBqoYSSo3nFZSgcC3gIODMiFg1M0+r0kwHhjPzwcYi765e16EEnNOBe7qs+m7GVqO6yNy5c3v52Kituub6zJ8/v9V1jJc771yN227uVqk9OnPmzOljbiYvy6GwHArLobAcCsthMcuimEjlMCECzojYCPgWcDm1TkOZ+clG0u9GxAXAB4DTxi2DSzBz5kyGhoZaW/61N85jxowZrS1/PK277npsuNnGPX12zpw5zJo1q885mnwsh8JyKCyHwnIoLIfFLIuiXg7Dw8OtV5KNZODjcEbEWsAPgQeA3TPz4RE+cjawcUSsX72/GxiKiFUa6aZXr3fV0q3dZXnTa2kkSZLUZwMNOKsg8XvABsDLMvPOHhbTabu5RWP6lsD9wB9q6aLWiaie7voe1itJkqRRGFjAGRErUXqSPwvYJTNvHcVnplGGObq11sv9Ekrv831q6Vas0p2XmQuryedSajh3rqV7ErB9NU+SJEktGGQbzlOAVwDvAlaLiOfV5l1LudV9OmWw9t9RgsWDgB2A13cSZuZwRBwPnBARdwBXVuk2BfarpbssIs4BvhQRhwP3AccB85gA7UElSZKmqkEGnJ2axo92mfci4LeUmsujKbfcH6YEk7tn5vfriTNzdkQAvBXYkNKrfbfMvKqx3H2B2ZRB3ocoj7bcKzMf6McGSZIk6bEGFnBm5lNGkeyVY1jebEowubQ09wOHVH+SJEkaBwPvpS5JkqSpzYBTkiRJrTLglCRJUqsMOCVJktQqA05JkiS1yoBTkiRJrTLglCRJUqsMOCVJktQqA05JkiS1yoBTkiRJrTLglCRJUqsMOCVJktQqA05JkiS1yoBTkiRJrTLglCRJUqsMOCVJktQqA05JkiS1yoBTkiRJrTLglCRJUqsMOCVJktQqA05JkiS1yoBTkiRJrTLglCRJUqsMOCVJktQqA05JkiS1yoBTkiRJrTLglCRJUqsMOCVJktQqA05JkiS1yoBTkiRJrTLglCRJUqsMOCVJktQqA05JkiS1yoBTkiRJrVppUCuOiL2A1wKzgHWAm4DPAp/PzAW1dLsAHwK2BP4AnJyZn+6yvCOAQ4GNgGuAIzPz/EaaNYCTgD2BVYALgcMy85Z+b58kSZKKQdZwHg4MA+8EXg58B/gU8JFOgoh4PvA94DfALsBXgJMj4s31BVXB5gnAKcBuwI3AORGxVWOdXwd2Bw4D9gFmAOdHxGp93jZJkiRVBlbDCbwiM++ovb8wIlYH3hIRR2fmMHAMcGVmvqmWZmPg/RHxhcxcEBFDwNGUms/ZABFxEXA18F5g72racynB6G6ZeW417WpKzeoBwKktb68kSdJyaWA1nI1gs+M3lFvd61SB5IuBsxppvka5bb519X47YC3gzNqyHwG+AewSEdOqybsC9wLn1dLNAy6u5kmSJKkFE63T0AuBu4A/AZsCKwPXNtJcU71uXr1uUb1e1yXd6sATa+mur7cPraXbHEmSJLViwgScEbENcCDwiaqGcno1655G0rur13Wq1+nAcGY+OIp0zWV10q3TZbokSZL6YJBtOBeJiI2AbwGXU+s0NNHNnTu31eWvuub6zJ8/v9V1jJc771yN227u1opidObMmdPH3ExelkNhORSWQ2E5FJbDYpZFMZHKYeABZ0SsBfwQeADYPTMfrmZ1aijXbnykU/N5Vy3dUESskpl/GyHdxl2yML2WZkxmzpzJ0NBQLx8dlWtvnMeMGTNaW/54Wnfd9dhws27FP7I5c+Ywa9asPudo8rEcCsuhsBwKy6GwHBazLIp6OQwPD7deSTaSgd5Sj4hVKMMebQC8LDPvrM2+CXiIxW00O7asXq+vXjttN7ulu58ydmcnXdQ6EdXTXY8kSZJaMbCAMyJWovQkfxawS2beWp9fDYt0AdWwRjX7An8ErqzeX0Lpfb5PbdkrVp87LzMXVpPPpdSW7lxL9yRg+2qeJEmSWjDIW+qnAK8A3gWsFhHPq827NjPvA44Dfh4RXwTOAF4AHAwc2ultnpnDEXE8cEJE3EEJRA+i9HLfr7PAzLwsIs4BvhQRhwOd5c8DTmt1SyVJkpZjgww4OzWNH+0y70XAzzLzVxHxSspThPYH5gPvyMzP1RNn5uyIAHgrsCFlqKPdMvOqxnL3BWZTBnkfojzacq/MfKA/myRJkqSmgQWcmfmUUaY7l1Hc8q6eMjR7hDT3A4dUf5IkSRoHE2YcTkmSJE1NBpySJElqlQGnJEmSWmXAKUmSpFYZcEqSJKlVBpySJElq1ZgDzojYucvjISVJkqSueqnh/CFwW0ScFBFb9TtDkiRJmlp6CTj3AC4GDgWujIjfRsQRETGjrzmTJEnSlDDmgDMzv5eZe1MeIXkwcAdwInBrRPw4Il4XEav1OZ+SJEmapHruNJSZ92fmlzNzR+DJwFHABsDpwO0R8dWI2LFP+ZQkSdIk1a9e6isCjwOGgGnAg8BLgJ9ExG8iYmaf1iNJkqRJZqVePxgRawF7A68DXgD8HTgHeHf1ugDYHfgE8BVg22XNrCRJkiafMQecEbEHJcjcFVgFuAJ4G/D1zLyrkfw7EbEecOoy5lOSJEmTVC81nP8N/AH4JHB6Zl4/QvrfAmf0sB5JkiRNAb0EnC8Fzs/MhaNJnJmXA5f3sB5JkiRNAWMOODPzp21kRJIkSVNTL4+2/ERE3LiU+TdExEnLli1JkiRNFb0Mi7QbcNZS5p8FvKK37EiSJGmq6SXgfBJwy1Lm31qlkSRJknoKOO8DnrqU+ZtQBn6XJEmSego4LwAOiYiNmzMi4inAIVUaSZIkqadhkY4BdgHmRsRXgGuq6TOBA4BHgPf1JXeSJEma9HoZFunGiHgBcApwWGP2RcBhmZn9yJwkSZImv56epZ6Z1wA7VI+t3KSafFNm3tm3nEmSJGlK6Cng7MjMPwN/7lNeJEmSNAX1FHBGxIrAzpTazenAtEaShZn5wWXMmyRJkqaAMQecEbEN8C3gH3hsoNmxEDDglCRJUk81nKcCqwJ7AL/IzHv6mSFJkiRNLb0EnM8C3puZ3+93ZiRJkjT19DLw+20s+Va6JEmS9Ci9BJwnAgdHxJr9zowkSZKmnl5uqa8D/BX4XUR8E/hfytOF6hZm5knLmjlJkiRNfr0EnCfW/n/zEtIsBAw4JUmS1FPA+dR+rTwingYcATyP8iz26zNzZiPNacAbunx8r8z8ZiPtEcChwEaUZ7wfmZnnN9KsQQmG9wRWAS6kPI7zlj5skiRJkhp6eZb6rX1c/zOA3YDLKO1Jl9Sm9PfAaxvTbqi/qYLNE4CjgCuBg4FzIuK5mXlVLenXga0pz4G/DzgOOD8inpmZDyzb5kiSJKmp50dbRsRmwA7ABsAZmXlLRKxMqV38Y2Y+NIrFfD8zv1st7zRgmyWkezAzL11KXoaAo4GTM3N2Ne0i4GrgvcDe1bTnUgLc3TLz3Gra1cBNwAGUMUYlSZLUR2PupR4RK0TEF4Drgc9Tagg3qWavTAnyDhvNsjJzwVjXvwTbAWsBZ9aW/QjwDWCXiOgM47QrcC9wXi3dPODiap4kSZL6rJdhkY4C3gi8D3g+tTE5M/MvlMdevqovuVts04i4JyIejojfRMQ+jflbVK/XNaZfA6wOPLGW7vouge41wOZ9zbEkSZKA3gLOA4EvZ+YJwO+6zL8a2GyZcvVov6F0LNqD0tHnNuDMiDiglmY6MJyZDzY+e3f1uk4t3T1d1nF3LY0kSZL6qJc2nP8AXL6U+Q8Ca/SWncfKzE82Jn03Ii4APgCc1q/19GLu3LmtLn/VNddn/vz5ra5jvNx552rcdvMdPX9+zpw5fczN5GU5FJZDYTkUlkNhOSxmWRQTqRx6CTj/CDx5KfNnAf3syd7N2cCpEbF+Zt5BqaEciohVMvNvtXTTq9e7qte7gY27LG96Lc2ozZw5k6GhobF+bNSuvXEeM2bMaG3542nddddjw826Ff3I5syZw6xZs/qco8nHcigsh8JyKCyHwnJYzLIo6uUwPDzceiXZSHq5pf4t4N+qXuodCwEiYhdgf0pnnfHUabu5RWP6lsD9wB9q6aLWiaie7vr2sidJkrT86iXgPBaYR2lbeQYl2DwqIi4FfgBcBXy4XxlsqoLFvYFbq9pNgEsovc/3qaVbsUp3XmYurCafC6wN7FxL9yRg+2qeJEmS+qyXgd/vi4jtgP8A9gL+RgnYbqIEoyc1bmsvUUSsxuLhiJ4MrBkRe1bvr6heT6cM1v47SrB4EGX8z9fX8jQcEccDJ0TEHZSB3w8CNgX2q6W7LCLOAb4UEYezeOD3eQy4PagkSdJU1dPA71VAeUL1tyw2oLTHrOu8PxD4HqXm8ugq7cOUYHL3zPx+I0+zIwLgrcCGlKGOdms8ZQhgX2A2ZZD3IcqjLffyKUOSJEnt6PlJQ/1QPb+82Z6y6ZVjWN5sSjC5tDT3A4dUf5IkSWrZmAPOiPjyKJItzMw39ZAfSZIkTTG91HC+mKpXes2KwBOq1zuAvy5jviRJkjRF9NJp6CndpkfE4yi3qd8O7LRMuZIkSdKU0cuwSF1l5sOZ+Rngx8Bn+rVcSZIkTW59CzhrrgL+qYXlSpIkaRJqI+DcCXCIIUmSJAG99VI/Zgmz1qbUbG4NnLgMeZIkSdIU0ksv9WOXMP1uytOG3gx8sdcMSZIkaWrppZd6G7fhJUmSNEUZPEqSJKlVvbTh3LiXFWXmvF4+J0mSpMmtlzact/DYJw2Nxoo9fEaSJEmTXC8B50HAW4EnAV8DbqimB7AvMA/4FLCgHxmUJEnS5NZLwPkEYAh4WmbeXZ8REe8HLgY2yswP9yF/kiRJmuR66TT0ZuALzWATIDPvpAyJ9G/LmjFJkiRNDb0EnOsCqy9l/uOrNJIkSVJPAeelwNsiYlZzRkRsA7wNuGxZMyZJkqSpoZc2nG8BfgZcHhFXADdW0zcDtgXuAg7rS+4kSZI06Y25hjMzrwWeSemJvjawZ/W3NvBJ4JmZeU3/sihJkqTJrJcaTjLzduAd1Z8kSZK0RD0FnB0RsRmwATA3M+/tT5YkSZI0lfQUcEbEfsCJwBOrSTsBF0TEesAlwNGZ+Y3+ZFFTwYIFC7n9rgd6+uyqa67f82fbsNrQSqzx+JUHnQ1JkiaNXp6l/mrgv4CfACcDszvzMvPPEXEdsD9gwKlFhh9+hEt+O7+nz86fP58ZMyZOwLnjthsbcEqSNAa9DIv0XuCnmbkzcHqX+ZcBWy1TriRJkjRl9BJwbgF8eynz/wSs31t2JEmSNNX0EnD+laU/aWhT4M+9ZUeSJElTTS8B5wXAARHxmEZsETEDOBj40bJmTJIkSVNDr204nwD8Gvh3YCGwa0ScCFwNLAA+0LccSpIkaVLr5UlDNwIvAP4IHAtMA/4DeBfwP8D2mTmvf1mUJEnSZDamYZEiYkXK2Ju3Z+ZLI2I68DRK4Pr7zLyjhTxKkiRpEhvrOJwrADcBRwIfz8y7gSv6nitJkiRNGWO6pZ6ZDwPzKe02JUmSpBH10mnoK5Re6qv0OzOSJEmaenp5lvoNwIrA9RFxOvB74MFmIp+lLkmSJOgt4Pyv2v/vW0KahfgsdUmSJDHKgDMiPgWcnplzgBdVk1en1Gw+0uvKI+JpwBHA84CZwPWZObNLul2ADwFbAn8ATs7MT3dJdwRwKLARcA1wZGae30izBnASsCewCnAhcFhm3tLrdkiSJGnJRlvD+RbgUmBOZl4UEetSnpm+U2ZetAzrfwawG3AZpT3pY9qURsTzge8BXwUOp4wBenJEPJyZn6ulOwI4ATgKuJLyxKNzIuK5mXlVbZFfB7YGDgPuA44Dzo+IZ2bmA8uwLZIkSeqil1vqHdP6sP7vZ+Z3ASLiNGCbLmmOAa7MzDdV7y+MiI2B90fEFzJzQUQMAUdTaj5nV8u7iPLko/cCe1fTnksJcHfLzHOraVdThno6ADi1D9skSZKkml56qfdNZi5Y2vwqkHwxcFZj1tcot823rt5vB6wFnFlb9iOUdqS7REQnON4VuBc4r5ZuHnBxNU+SJEl9NtCAcxQ2BVYGrm1Mv6Z63bx63aJ6va5LutUpT0fqpLu+S6B7TW1ZkiRJ6qOx3FLfJCL+sfp/rep184j4S7fEmXn5MuWsmF693tOYfnf1uk4t3XBmNodnqqe7rUrXXFYn3Tpdpi/V3Llzx/qRMVl1zfWZP39+q+sYL8OxzjJty0QqhzvvXI3bbh7MU1znzJkzkPVONJZDYTkUlkNhOSxmWRQTqRzGEnB+oPqre0xPcUrbzoWUsTqntJkzZzI0NNTa8q+9cR4zZsxobfnjaWholZ63Zf78+ROqHNZddz023GzjcV/vnDlzmDVr1rivd6KxHArLobAcCsthMcuiqJfD8PBw65VkIxltwHlgq7lYsk4N5dqN6Z2az7tq6YYiYpXM/NsI6bpFCtNraSRJktRHowo4M/P0tjOyBDcBD1HaXp5Xm75l9Xp99dppu7kF8JtGuvspY3d20u0UEdMyc2Ej3fVIkiSp7yZ0p6HMHAYuoBrWqGZf4I+U8TYBLqH0Pt+nkyAiVqw+d14tuDyXUlu6cy3dk4Dtq3mSJEnqs2UZh3OZRcRqLB6O6MnAmhGxZ/X+isy8lTIw+88j4ovAGZSB3w8GDu30Ns/M4Yg4HjghIu6gBKIHUXq579dZX2ZeFhHnAF+KiMNZPPD7POC0VjdWkiRpOTXQgBPYADi7Ma3z/kDgtMz8VUS8kvIUof2B+cA76k8ZAsjM2REB8FZgQ8pQR7s1njIEpXZ0NmWQ9yHKoy338ilDkiRJ7RhowFk9v3zEJxZVTwUa8ZZ39ZSh2SOkuR84pPqTJElSyyZ0G05JkiRNfgackiRJapUBpyRJklplwClJkqRWGXBKkiSpVQackiRJapUBpyRJklplwClJkqRWGXBKkiSpVQackiRJapUBpyRJklplwClJkqRWGXBKkiSpVQackiRJapUBpyRJklplwClJkqRWGXBKkiSpVQackiRJapUBpyRJklplwClJkqRWGXBKkiSpVQackiRJapUBpyRJklplwClJkqRWGXBKkiSpVQackiRJapUBpyRJklplwClJkqRWGXBKkiSpVQackiRJapUBpyRJklplwClJkqRWGXBKkiSpVSsNOgPSZLNgwUJuv+uBcV/vqmuu39f1rja0Ems8fuW+LU+SpCWZ8AFnRBwAfKXLrFMy8y21dLsAHwK2BP4AnJyZn+6yvCOAQ4GNgGuAIzPz/Bayrilq+OFHuOS388d9vfPnz2fGjP4FnDtuu7EBpyRpXEymW+ovA55f+5vdmRERzwe+B/wG2IUSoJ4cEW+uL6AKNk8ATgF2A24EzomIrcZjAyRJkpZHE76Gs2ZOZv55CfOOAa7MzDdV7y+MiI2B90fEFzJzQUQMAUdTaj5nA0TERcDVwHuBvVvOvyRJ0nJpMtVwdlUFki8GzmrM+hrltvnW1fvtgLWAMzsJMvMR4BvALhExrf3cSpIkLX8mUw3n3IhYH5gHnAZ8KDP/DmwKrAxc20h/TfW6OfBrYIvq/XVd0q0OPBG4rf/ZliRJWr5NhoDz/4D3A5cDj1DaaL4PeCpwADC9SndP43N3V6/rVK/TgeHMfHAp6cYUcM6dO3csycds1TXXZ/788e+c0obhWGeZtmUilcOybsuy6Od677xzNW67+Y6+LW88zZkzZ9BZmBAsh8JyKCyHxSyLYiKVw4QPODPzR8CPapN+EhH3AsdGxAcHlC0AZs6cydDQUGvLv/bGecyYMaO15Y+noaFVet6W0jt74pTDsmzLsuh3Oay77npsuNnGfVveeJkzZw6zZs0adDYGznIoLIfCcljMsijq5TA8PNx6JdlIJmsbzm9Ur1uzuIZy7UaaTs3nXdXr3cBQRKwyQjpJkiT10WQNOOtuAh5icRvNji2r1+ur107bzW7p7qeM3SlJkqQ+m6wB52uAhZShkoaBC3jssEb7An8ErqzeXwLcC+zTSRARK1afOy8zF7adaUmSpOXRhG/DGRE/ogSUc4EFlE5D/w58KTN/XyU7Dvh5RHwROAN4AXAwcGhmLgDIzOGIOB44ISLuoASiB1F6ue83jpskSZK0XJnwASflVvgbgX+g5PdG4Ejg5E6CzPxVRLyS8hSh/YH5wDsy83P1BWXm7IgAeCuwIWVIpN0y86r2N0OSJGn5NOEDzsx8O/D2UaQ7Fzh3FOlmU3sspiRJkto1WdtwSpIkaZIw4JQkSVKrDDglSZLUKgNOSZIktcqAU5IkSa0y4JQkSVKrDDglSZLUKgNOSZIktcqAU5IkSa0y4JQkSVKrDDglSZLUKgNOSZIktcqAU5IkSa0y4JQkSVKrDDglSZLUKgNOSZIktWqlQWdA0mAsWLCQ2+96YNDZGLNV11z/MflebWgl1nj8ygPKkSRpJAac0nJq+OFHuOS38wedjTGbP38+M2Y8OuDccduNDTglaQLzlrokSZJaZcApSZKkVhlwSpIkqVUGnJIkSWqVAackSZJaZcApSZKkVhlwSpIkqVUGnJIkSWqVAackSZJaZcApSZKkVhlwSpIkqVU+S13SpLdgwUJuv+uBkRNOAqsNreRz4SVNOQackia94Ycf4ZLfzh90Nvpix203NuCUNOV4S12SJEmtMuCUJElSq7ylLkkTyGjbo6665voTut2qbVEl1S13AWdEbAZ8GtgeeBA4EzgyMyfulVvScmO07VHnz5/PjBkT97JlW1RJdctVwBkRawMXArcCewIbAB8H1gdeM7icSZIkTV3LVcAJHAJMB56dmX8GiIi/A2dExAcz85qB5k6SpojxGqpqPJoW2DxAWnbLW8C5K3B+J9isfAv4MrALYMApSX0wXkNVjUfTApsHSMtueQs4t6AEl4tk5nBE3ARsPoblrAjw0EMP9TFrj7VgwSOstMKCVtcxXh75+8M9b8sqj5s2ocphWbZlWfS7HAa1HcuqWzlM1m3pZrTbMtHOi6bx2ifjUQ5/f/ghhodXbHUd/TA8PDzoLEwYlkXRKYdavDKwA3nawoULB7XucRcRDwPvy8wTG9N/CfwpM181muXMmTNne+AXLWRRkiSpLS+cNWvWLwex4uWthrNfrgBeCPwf8MiA8yJJkrQ0KwJPoMQvA7G8BZx3A2t3mT4duH60C5k1a9YwMJBfCJIkST24aZArX96eNHQdpR3nIhExBGzKGAJOSZIkjd7yFnCeC+wYEevWpv0LMFTNkyRJUp8tb52G1gbmArcAH2TxwO/nZ6YDv0uSJLVguarhzMx7gBcDfwH+G/gEcBbwxgFmS5IkaUpbrmo4JUmSNP6WqxpOSZIkjT8DTkmSJLXKgFOSJEmtWt4Gfp/wImIz4NPA9sCDwJnAkZn5wEAzNoKI2At4LTALWIcywOxngc9n5oJaul2ADwFbAn8ATs7MT3dZ3hHAocBGwDWUMji/kWYN4CRgT2AV4ELgsMy8pZFuYGUaEatTxnh9IrBtZv66Nm9/4CjgKZTyOi4zz2p8/nHAccAbKA8tuAJ4W2b+TyPdRsAngZcBC4EfAG/PzD830v0jZWSGWcBdwH9W623liVkR8Xrg7ZT9/QBwJbBvJ1/Lw/EQEXtQ9vMWwF+Bi4F3Z+aNjXRT5niIiKcBRwDPA2YC12fmzC7pJuz+H23elqUcImJF4HBgt2o9KwFXAx9obt9ULocu6WcBlwMPZubqjXkDOQdGc36OZAznxSrAu4HXA/8A/Bk4NzMPbqSbVMeDNZwTSDVs04XAGpSD43BgX+DLA8zWaB0ODAPvBF4OfAf4FPCRToKIeD7wPeA3wC7AV4CTI+LN9QVVJ9EJwCmUC/GNwDkRsVVjnV8HdgcOA/YBZgDnR8RqtWWtzWDL9Fi6/LCLiD2B04FvU8rip8DXq5O57hOUC8r7gVcCD1G2cUZtWSsB5wHPBPYHDgK2A74XEdNq6Tap1nMXZR+dQNlfH+rDdj5GRLyX8qPjvynb+CbKRXGomj/lj4eI2JGy/dcDr6rytjnw04hYs5Zuqh0Pz6Dsq98B13ZLMJH3/2jzNgojlcOqlCDmf4ADgddQvsR/EhEvb+RpKpdDfZ0rUK4bdywhybifA2M4P0cymvNiBcr35/5Vfl4KvIsyuk493aQ7HqzhnFgOoTxm89m1GqC/A2dExAcz85qB5m7pXpGZ9QvEhVXt3lsi4ujMHAaOAa7MzDfV0mwMvD8ivpCZC6onPx1N+dU0GyAiLqL86n8vsHc17bmUk2y3zDy3mnY15ZfnAcCp1ToGVqYRMRN4M/AfwOcbsz8InJ2Z76neXxgRWwAfAH5Yff6J1effmplfrKZdCtxMqTV8V/XZVwNbATM72xMR8yk1abuw+KEG7wTuAfaq9sf5EbEWcExEfDQz7+rjtgcl2P6XzPxBbdZ3av8vD8fDvsCtwBsyc2G1vluBy4AXUO1rpt7x8P3M/G617tOAbbqkmcj7f8S89akcHgSempl3dyZExI+Bp1O+9H9QTZvq5VB3MLAWJdh5a33GAM+BEc/PPpbDgcDzgS0z8w+16WfUymFSHg/WcE4su1IGoa9X+X+LUnM41l9S46oRbHb8hlKFv051gryYMu5p3dcotwO2rt5vR7nYnFlb9iPAN4Bdar9OdwXupfyK7aSbR7mg7Fpb/iDL9BTgM8AN9YkR8VRKLdeZjfRfA7aNiPWr9y8FVqRWZpl5P+VLqLmNV9eDpcy8hBLoNNN9p7qw1tfZ2Tf9dCBwayPYXGQ5Oh4eB9zfCTYr91Sv02BqHg8jffFM5P0/hryNaKRyyMxH6sFmNW0hpcZzRm3ylC6HjohYj1Jr9zZKzWXTuJ8DYzg/RzTKcjiYEtz+YSlpJuXxYMA5sWxBo5q9OhFuohzwk80LKbcq/kR5Xv3KPPY2QueC0Nm+zrPur+uSbnVKW8hOuuu7nMDX8OiyGkiZRmm7+DTg+C6zO9u4pLKIWrrbM/POLumeXt166aTrdntmUVlExOOBjZvpqnY8D9D/snge8NuIODoi/hgRD0fE5RHxz9X85eV4OA3YIiIOi4i1I+IpwGzK9nTaWi0Px0PTRN7/o81bK6r9uB2P3ublpRw+AvwyM89bwvxBnAOjPT+XWZT2qVsDt0TE6RHxl4j4a0R8p6pJ7JiUx4MB58QyncW1H3V3UzriTBoRsQ2llusT1S+v6dWsexpJO7/uO9s3HRjOzAdHka65rE66elmNe5lWt2ROAt6VmX/pkmQsZdFM00n3OMqFZaR0nWWtvYR1NtP1y0bATpRj4K3AK4D7gPOqoGu5OB4y80JK280PVeu4GXgqsFOtVmV5OB6aJvL+H23e2nIYJYj5WG3alC+Hqn3gvsA7lpJsEOfAeJbDupTtOJJyDX01pe37VsC5UdqmdvI06Y4HA071XZTegd+i9DL8yAjJp6LjgRsz84wRU05dK1Au/q/OzG9UNRa7U4LOdw40Z+MoIrYDvgp8iXI7ai9gAaXzwqqDzJsmnuoOwEeB2Zn5i0HnZ7xE6a1/KvDxzPz9oPMzQJ2Y7C/AHpn5o8w8k3LdeAbwLwPLWR8YcE4sd7P4V1fddMqt6Qmvqt37IeWWxO6Z+XA1q/MraO3GRzq/mu6qpRuKMizESOmay+qkq5fVuJZpRDyD0qj9fdUt1LVZ/Kt79ShDVIylLJppOukeZnGvxdFs4z1LWGczXb/cDdyZtaFKsgyvcSllOJDl4nigjNRwYWa+IzMvzMxvUhrxP4cy5EknT3TJ11Q6Hpom8v4fbd76KiKeBXyX0rHuyMbsqV4OBwNPAE6tXTdXgdKDuvbjbBDnwHiWwz2UIZwurtdeZhlO7z7KtbOTp0l3PBhwTizXsbhtBrCooe6mlGFVJrTq4P8esAHwskY7m5sojcC3aHxsy+q1s32dNind0t1PGTKkky5qjaPr6eplNd5luhll9IcLKSfo3cD3q3kXAr9g6dsIkNXrdcAGEdG8VbElcEOtXc5jtrGW7nqAzPwrMK+ZLiKeDKxG/8tiab29V2H5OR62pHQAWSQzb6OMq7dpLU8088XUOh6aJvL+H23e+iYiNgV+RBmn9vWNTmYw9cthc2BDynZ0rptHAo+v/v9wLd/jfQ6M9vxcZtWP8luWMHshVRA+Qp4m7PFgwDmxnAvsGBHr1qb9C6XH3LndPzIxVG1LvgE8C9glM2+tz6/aq11ANVxDzb7AHykXWoBLKL3q9qkte8Xqc+fVLsTnUn5p7VxL9yTKoLX1shrvMv0l8KLGX6dN0puBgzLzZsrJuU/js/sCV9R6/P+Ycvt1UZlFGWrqFTx2G59ZDdPRSfc8ygDFzXR7RMTKjXUOs7gDS7/8AFg3Ihb1XKwa6T8fmLMcHQ+3UgaUXqT6QluP6otlOTkeHmUi7/8x5K0vqiZIP66WvUdmduudPdXL4TM89rp5OvC36v/PVOnG/RwYw/nZLz8Atq83uYkyOP1awJxq0qQ8HqYtXNj8IaVBqW4jzKV8EX2QUlP4ccpwBa8ZXM5GFhGfB/6VMg5as+3RtZl5X9Uo/OeUnrtnUMYhPA44NDM/V1tWZ0Db91AO4oMojaefm5lX1dL9gHJr8nDK7YbjKNX7z6x+KU6IMo2IHSi1m4ueNBTlyUxnUX65/4QygPHbKOOl/bD22c9Qbr0eTglejqCM3fbMzJxfpVkJ+DWlsfl7KDWsJwG3Ay/IxeM/bkKpbbuA8kSJqNJ9OjPf3edtXgH4FbA+ZVy4+6tt2JYy1tvvlofjISLeQinrz1Bul65LGT9vfeAZnbsAU+14iDKodGfYlUMpNSX/Ub2/IjNvncj7f7R5W9ZyoIzg8atq+uso+2iRzLx0eSiHZgVF9ZljgSPysU8aGvdzYLTnZz/KoQoIr6Ls409QAsYTKPty604ztcl4PFjDOYFk5j2UjgV/oTyd5BOUg/yNA8zWaHV+QX2UcgGt/20NkJm/opyo21JuHx0EvKN5oGYZyPYoSu/mH1Jut+xWP4kq+1J+DZ4KnE35lfWSrD2Ka6KWaWaeTenBvSelLHYG9uty8XoH5akbx1OaK6xK2cb5tWX9nfL4trnAf1Ge/nAppQ3twlq63wMvoQQ751ACn49RAsJ+b98CSlvFn7N4/wDskJm/q9IsD8fDKZRBlV9IaZt3MuUpIy+qNzmZgsfDBpR9cDawA/Ck2vsXVeufsPt/tHnrQzlsSOmBvDrl+GheO5eXchiLcT8HxnB+jmQ058X/Vv9Pq6Z/hlKJ85Jc3CdiUh4P1nBKkiSpVdZwSpIkqVUGnJIkSWqVAackSZJaZcApSZKkVhlwSpIkqVUGnJIkSWqVAackSZJaZcApSZKkVv1/xVCuu0pwNnUAAAAASUVORK5CYII=\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAoIAAAFKCAYAAACJoz5RAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAAAw2ElEQVR4nO3debxtc/348deNXMksEpUy9Ea3+HWpDPVrki6l4WuIypeifJMkU5Mh5Kf4lpKiCRWRVF8iDcZKGU4D1/CmK3y5kXmI7sW9vz8+a3eWbZ9pn2Gfe9br+Xicx7p7rc9a+7M++3PPfp/PtKYtXLgQSZIkNc8zep0BSZIk9YaBoCRJUkMZCEqSJDWUgaAkSVJDGQhKkiQ1lIGgJElSQxkISuMgIg6NCNdm0tNExOsiYmFEvK7XeZmsIuLiiLi41/noJCL2jYibI2LxXudlKBGxdUQ8EhEr9zovmrwmfUWWJosRBHa7jmtGRiEipgHvA3YDNgCeCcwBzgC+mJmP9jB7wxYRzwb2AbYH1gQeB+4Afgd8KTNv6GH2JoWIWAPYH9gSeD7wJHA98FPg+Mx8oGeZG0cR8WHg0cw8eRyuvQzwSeCgzHxirK8/1jLz3IiYQ8nzx3udH01OBoLS8L2v7fUHgVcD72/bfxnwfeCoicjUcEXEYsBplODpN8DBwGPAa4HPAttHxJsy8x+9y+XQIuKZwCXADOB7wNeAZwHrAlsDvwcaHQhGxJbAWZTg73vA1ZTf9xtRgoL/C7y5ZxkcXx8G7gFOHodrv59S1747DtceLycCR0fEoZn5UK8zo8nHQFAapsz8fv11RLwJeGX7/prJ1mJwACUIPCYz96/t/0ZE/BD4CXASJZiaMFUr5ZKZ+dgwT3kHMBPYtb3Vp+quW25MM7iIiYgXAWdSWkjfkJl3tB3/FKVFWCP3fuC8zPxnrzMyAj8CvkL5v/+tHudFk5CBoDQOIuJQ4JDMnFbbdwulpeoo4BjgpZRu2Y9m5oUR8Q7gMOAlwHXA7pnZ13bdlwBHAG8Enk3p6vtcZv5oiPw8i9JNeCOlRegpMvPsiDgF2DUiXpmZV9TyfHFm7tJ2vYur815X2zcd+ATwXuCFlFaZHwKfrnc5V13sJwIXAZ8BAvhgROwOLJOZL++Q/z8Cj2fmq4C1qt2/6XAfTwD31s5bgxIAvwFYA5gP/Bb4ZGZeU0v3uio/7wHWobT2Lgf8EvgA8E/K5/YeSrmfBexRD17b7usQSpd1Agdm5vntee1wj0N+tlWg+wlgZ+AFlBbdm4DPZ+aPq2QHAMsAW7UHgVUZ3Vm9T/299wA+AqwNPAD8T1VG99XSXAysCmwHfBXYGLizSndGRGxOqdcbALdR6vUvaucfWpXLSymf+9bAAkod2TczHxmifKZVefwg5TN6CDiHUr73VGluoXzO9aEct2bmi6p9w6qjA7z/i4GXA8e37X8R8Dc6/2GyEPhsZh5avV4aOBT4D2C16h6uBQ7OzEtr521MaaXfDFgC6KN0R1/Udv3nVdfbGlgZ+DvwK+DjmfkwQGb+IyKuBt6JgaA6cLKINLHWBH4AnEv5QloeODsidqL81X4apct2TeDMqjsXgIhYD7gceBnwBWBfStBzZkS8d4j33RxYAThtkLFNre6ut430pqov6Z9QgpBzgb0oX7AfBn5aHa97LeUL9Szgo5QA+RTgZRHxlECwuu//U8vfLdV25w7Xbbdx9V4/AvYGvgS8Arik+hJtdwAwixL0fZvS+vgN4JvA+pQv57MogdjTAmrKF/fXq3v/NLAkcE4VJA1oBJ/tIZQ/Fi6hlNthlLJ7ZS3NNsDfMvO3g71n7b0/U+X5LsofC6dTWr4urAKnuuUon++VlLJ6FDg1It5NKePzKfV6qSrvnVpnT6fUxU9V53yQUl5D+TrwRUo57U35XLYFLoqIJas0HwNup5TJ+6qfj1X3OdI62m7TanvVMPI62D3sVeXjw8DngbspwTNVPv8v5Y+cFSmf74HAdOCX9QlGEbEqcAWlLp5VXfdkSl1Yqe19+4BNhnGPaiBbBKWJtQ7w2sz8DUBEXA/8AvgOsF5m/q3a/wClden1wK+rc78MzAU2qrVEHR8RvwSOiohTM3OgCS3rV9u/DJK31rH1B0kzkB2BtwCvz8xLWjsj4irKeMktKK1rLesCr8jMP9fS3kC5x/dSvqxb3keZDHJG9fqnlC/6gyktmBdTWvnO7dACdm57a2lEfI/S4voB2lrGKK0vr8zM+VXalYF3U1pZ3lKV79ciYh1KsHRw2/kzgE0z8/fV+SdTWuyOogTjAxnuZ/tWStfk7p0uEhHLAqtTWvSGVN3fQcAFwJaZ+WS1/8+UYQK7U1r/WlYFds7M71XpfkX5LE4DXpOZv6v2t+r1djy9FeoOSmvlwirt34GDqvGpv6aDiNgU+BDwn5n53dr+8ylB087ANzLzpxFxBHBPhyEbI62j7dattjcPkmYobwW+mZkdJ25UgdqJlPq8Ra2MTgD+BBxJf0B6FKVVcdPMvLx2mUM7BHw3U4Lv51HqmfRvtghKE+vGVhBYaf0Cv7gVBLbtXxMgIlYE3kRpwXh2RDyn9UNphVmd0qU8kGWq7cODpGkdW2aQNAPZntLtfG1b3i4BFlIC2rrL6kEgQDWL9Wxgp4h4Bvz7i3En4Oet7r/M/BclqDoGaM2CPhH434j4fhUMta5Z77pdKiJWonTHJWWcYbvvtYLAyuXVe5zUFmRfDqzWocXsqlYQWL3/vZQgabOIWKHD+430s30QeGnVjdxJ694H+5zr3kQJfr/cCgIr36O0ELaPF30MOLV2f0npSr6xFQRWnlJ/23y1rSy/Um3fOkg+twceAc5vK58bqny216+BrjGSOtpuJUpX9oPDeK+BPAi8KiJWH+D4BpShEqcBK9XyuCzlj5FXVfX4GZSu3p+3BYEAdPiD8P5q+5xR5F1TlC2C0sS6rf4iMx+MCID/bUvX+rJpBQ9rUwKSQ6ufTlahBDidDCfIax3rZtbwSyhfYHcPcHyVttdzBkh3CuUL+/WUVqrXUMZ87VdPVAVY+wP7R8TzKd2/e1PG8C2gtBBRdRkeRmllbO8Kvpenu63tdetz6PT5TKN07d9V239Th2veWG3XoP8LuW4kn+3BlBbRjIjrKK1up2Vmq7uyNSt0uMH8GtX2KfUmM5+MiJuAF7WlvyMzF7Tte5C28qnV607B701tae+JiPs7vFfdS4CleWpZ17XXr4GuMZI62sm06qfbNUL3p9Tx2yLiT5RA/3tVQN3KI5RhCQNZiTLWdVlg9jDft9VC6NqmehoDQWliPTnC/a1f4K3W+y8B5w2QdrAvheuq7cspgUQnrbF59a6vgb44FuOpeX5G9R57D5C+vTtqoBnCv6B82b+XEgi+l9LidM4A6cnM24HTIuJHlIH3746I91djIY+jdOEeR1nW5wFKoHgsnXtEuv18RmPYn21mXhoRa1HGcb6ZEvB+LCI+kZlfyMyHImIuZazheOhF+UApo3sp3fSddAqwO11jJHW03T2U+1mu7f06/h+pj+9tycwzI+I3wNspn99HgQMiYpfMPI3+uvAJyri+Tu5m5DPjWwH5PSM8Tw1gICgtGlrB2RMDjaMawu8oQdBOEfG5tm7Alp2r7Zm1ffdTWr3arcFTA8Y5lK7WCwYZpzikqiXqVGD3iNiHMhngzMycN4xz50fEXygtbM+hzGjdDvhuZn6snrbqph2PL8V1OuxrtfLcOsA5I/psM/N+ysSZ70aZDX4e8NmI+O/qcz0b2CMiNmvrru2klaegv+WSqutxHcq4tLG2Ttt7PYcSqNwyyDlzKGP4/jDU7GIG/uNltHX0+mr7Yp4aCLb+vXxb+jXooJq1fSJwYkQsD/yBMgnpNPpbyh8erC5ExHxK6++MYea9lec7h5leDeIYQWkRUC3yfBElQHra+KIY4hFS1dIYX6B84X+uw/lbA7sAZ9eXVaF8Mb06IpaopX0rZemSujOA5wL/1eHa06M8kWG4TqF0bZ5ICRCesnhvRGzQ6X6rL9VNgPvo7/57krZWqYjYkTLIfjxsFBGb1N5rJcoYx8uqAO5pRvLZVtern/sYZZzckpSFjgGOpoyn+3ZEPO0+I+K51UxhKOPO5gMfbY3LrLyH8nn+bPDb7cpH2iYzfLTanjvIOWdQvq/aJ+cQEYu1jb/8J527pEdbR1tB9Ub1ndUizfdQhifUfbhDPp/SkleNi/0b/UFkH/BX4OOd8tOqC1X3/E+AWRHxqg7p2ltiZ1KCaLuG9TS2CEqLjv+ifBldHRHfpARpqwCvosz0XXuI878AbAgcGBGvBn4MtCZevAe4hhIM1n2L0ip3fpRFp9eidNe2j/H7fpXu+CjLX/yWEoAFZczfdsDFw7nJzLy6atnbnvIl2d6qtQVweEScQ3mKyIOUwHRnSoC3V63F82zKMjMPUbpXNwR2YHQzPwczG/hZRBxHGZf5QUpQ22mpmbrhfrbXR8SllOVb7qFMLtgN+FmrpSwzb46IHSgtu9dVs6RbTxZ5BaV79bIq7T0RcThwOGV5kp9SJnh8hDKLfDzWnVsdOC8iflblf3fgl5n5q4FOqLrEj6eMCX05ZQjBPEq5bEsJEE+ukl8FfDgiDqG0PD6SmecwyjqambdVs6m3oCxdU/ct4BMR8a3q/V/L0ydvLQPcERFnUcr2IcpyQ2+hmpmdmQsi4gOUsYPXRcR3KMvhrEZ5Gsw0+ie1fLLKy8URcSKl2/u5wLsoE0luAYiIVSjDPk4Y6N7UbLYISouIakD5RlTBDWUdvg9TvuAPGsb5T1KCgF2qc46gtLq9j9Iy9Mr2VqtqQeB9KV9qx1Ja3N5K+XKqp1tA+QLanxK4HE3p7no15RFwV4/wdk+ptt/v0IpxFmXpjNUpa6ydSGlVuhl4Z2bWlzvZmzLwfgfKGLyXUr542yd/jJXfUT6THShLfcwD3lFfLLiTEXy2x1KeG3wgJXh4C6Usdmy73nmUbsPTgK0oYySPpgTCn6MEPa20R1AC0ecB/035o+Bk4I3D6ZLvwo6Ubsoj6X/axXaDnlHy+RHKkj8rUu7hKMo4ux8CF9aSHkYZU/pxyv0fV50/FnX0O8BWUZ51XXcYpZ5tS/mDazHKepR1j1I+15dRPtNjKZ/5fpTnZrfu89IqT3+g1IGvUsa53kdZd7CV7u+UPxROp5RpazzsFTx12MN/UFp9z0DqYNrChbYUS00V5bm951CevLH1YK0yEyki9qR8AUZm3jhU+skgqieLZOYevc7LZBT9TxZ5XjVObpFTddfeTHkSyNd7nZ/hqFoxL24fJyu12CIoNVhmPk5pMfgLcFZEvKLHWWrZDfj9ohIEqhmqx7YdReminvRDq6qxv2tTWl+ljiZ9RZY0vjLzn5RHsfVU1d22DWUs1IaUbjZpUsnM/6Z0oU96mXkuZf1FaUAGgpImi5UpY7oeAL6QmWf1NjuSNPU5RlCSJKmhbBHsQl9f33RKV9rfGXhFfUmSpMlgMcrKAFfOnDnzKasBGAh2Z2PgN73OhCRJ0gi8hrKG5r8ZCHbn7wAveclLWGKJJQZNOHv2bGbMGO5TgKYuy6GwHArLoZ9lUVgOheVQWA79xqIs5s+fz4033ghV/FJnINidJwGWWGIJpk+fPmTi4aRpAsuhsBwKy6GfZVFYDoXlUFgO/cawLJ42nM11BCVJkhrKQFCSJKmhDAQlSZIaykBQkiSpoQwEJUmSGspAUJIkqaEMBCVJkhrKQFCSJKmhDAQlSZIayieLTGIP/3M+j857otfZGBNLL/ecXmdBkiS1mTSBYEQsDdwArA5snJlX1Y7tDHwKeBEwBzgsM89oO/+ZwGHAfwLLA1cCe2fmn9vSrQp8GXgLsBD4GfCxzLxnPO5rNB6d9wQXXHlbr7MxJjZcc6leZ0GSJLWZTF3Dh9IhMI2IbYFTgJ8As4BfAz+IiFltSb8E7AkcArwdmA9cEBGr1a61OHA+8DJgZ2A3YFPg7IiYNsb3I0mSNKlNihbBiJgB7AF8HDix7fDhwJmZ+cnq9UURsR7wWeDn1fmrV+d/NDO/We37A/A34GPAAdW5/wFsAMzIzGurdHOB31GCzPPG4/4kSZImo8nSIng88FXgxvrOiHgxsC5welv604CNI2Ll6vWbgcWAf3cXZ+bDlG7frWrnbQVc0woCq3SXAbe2pZMkSZryeh4IRsT7gLWBIzocXq/aXte2vxXIRS3dXZl5b4d0L4mIZ9TStV+rlW7dkeRbkiRpUdfTruGIWA44Gtg3Mx+JiPYkK1TbB9r2319tV6yla0/TSvdMYGngoSHSrT/8nBezZ88eVrq+vr6RXhqAZy27MnPnzu3q3MlmwzXX7rocphrLobAc+lkWheVQWA6F5dBvPMui12MEjwBuysxTe5yPrsyYMYPp06cPmqavr4+ZM2d2df277nuU1VZ7tKtzJ6Nuy2EqGU19mEosh36WRWE5FJZDYTn0G4uymDdv3oCNVz0LBCPipZQJHltExPLV7qVb24hYhv6Wv+WBO2unt1oK76u291dp2q0APA48Mox093XYL0mSNGX1cozgOpRA9CJKgHY/cE517CLgN8D11ev12s5tdeNmtb0eWCUiVuyQ7sbMXFBL136tVroburgHSZKkRVYvA8HfAq9v+9mnOrYHsFtm/o0SoO3Qdu6OwJWZeXf1+pfAAmD7VoJqgeq38dQlYc4DXlYtP9NK92rKQtUuHSNJkhqlZ13D1ZM8Lq7vq00W6as9WeRg4IyImAP8irJY9JuBrWvXuiMiTgA+HxFPUJaD2Q+YBhxbe4uzgKuBH0XEJyn3fzTwe6o1CSVJkpqi58vHDCUzzwR2BbYFfgFsCeyUme2B2z7A1ykTUM4GngW8KTPn1q71BOXRcrOB7wMnAX8AtsnMheN8K5IkSZNKr2cNP0VmXkxpxWvffwrlMXODnfs48InqZ7B0d/L0rmZJkqTGmfQtgpIkSRofBoKSJEkNZSAoSZLUUAaCkiRJDWUgKEmS1FAGgpIkSQ1lIChJktRQBoKSJEkNZSAoSZLUUAaCkiRJDWUgKEmS1FAGgpIkSQ1lIChJktRQBoKSJEkNZSAoSZLUUAaCkiRJDWUgKEmS1FAGgpIkSQ1lIChJktRQBoKSJEkNZSAoSZLUUAaCkiRJDWUgKEmS1FAGgpIkSQ1lIChJktRQBoKSJEkNZSAoSZLUUAaCkiRJDWUgKEmS1FAGgpIkSQ1lIChJktRQBoKSJEkNZSAoSZLUUAaCkiRJDWUgKEmS1FAGgpIkSQ1lIChJktRQBoKSJEkNZSAoSZLUUAaCkiRJDWUgKEmS1FAGgpIkSQ1lIChJktRQBoKSJEkNZSAoSZLUUAaCkiRJDWUgKEmS1FAGgpIkSQ1lIChJktRQBoKSJEkNtXiv3jgi3gV8HFgXWBq4A/gJcHhmPlhLNwv4HLB+lebYzDyuw/X2A/YEVgWuBQ7MzAva0iwDHA1sCywJXATslZm3jPX9SZIkTXa9bBFcEbgU+CDwFuDLwPuBM1sJImIT4GzgT8As4CTg2IjYo36hKgg8Ejge2Bq4CTg3IjZoe88fANsAewE7AKsBF0TEUmN9c5IkSZNdz1oEM/Nbbbsujoh/ASdGxGqZORc4GPhjZn6gSnNRRLwQOCQivpGZCyJiOvAZSkvhMQARcQlwDfBpYPtq36soQeLWmXlete8aYA6wC/C1cbxdSZKkSWeyjRG8p9ouUQV4bwDOaEtzGqX79xXV602B5YDTWwky80ngh8CsiJhW7d4KeBA4v5buNuB31TFJkqRG6VmLYEtELAY8E3gppQXw7My8JSLWB5YArms75dpquy5wFbBe9fr6DumWBlYHbq/S3ZCZCzqk23IMbkWSJGmR0vNAELiX0qIHpbVup+rfK1TbB9rS319tV6ylm5eZjw2S7vYqXfu1WulW7LB/SLNnzx5Wur6+vm4uz7OWXZm5c+d2de5ks+Gaa3ddDlON5VBYDv0si8JyKCyHwnLoN55lMRkCwdcBSwEzKGP9zomILXqao2GaMWMG06dPHzRNX18fM2fO7Or6d933KKut9mhX505G3ZbDVDKa+jCVWA79LIvCcigsh8Jy6DcWZTFv3rwBG696Hghm5p+rf14WEX2U7t530t8lvHzbKa2Wwvuq7f3A9IhYMjP/NUS6F3bIwgq1NJIkSY0x2SaL/BlYAKxNmc07n/4xgC3rV9sbqm1rbGCndA9T1h5spYva5JF6uhuQJElqmMkWCG5CydPNmTkPuJBq+ZeaHYE7gT9Wry+jzAbeoZWgmoCyPXB+Zi6sdp9HaV3cspbuBcDm1TFJkqRG6eWTRX4BXECZtfsvYENgf+Bq4KdVssOASyPim8CpwGbA7sCerdm/mTkvIo4AjoyIuykB4m7AWvRPPCEzL4+Ic4FvR8S+wEPV9W8DTh7Pe5UkSZqMejlG8ArgvcCLq9e3ACcAX8zM+QCZ+fuIeDvlqSE7A3OBfTLzhPqFMvOYiAD4KPBcSnC5dWb+pe09dwSOoSwePZ3yiLntMnPqzMiQJEkapl4+WeQg4KBhpDuPYXTdVk8VOWaINA8DH6p+JEmSGm2yjRGUJEnSBDEQlCRJaigDQUmSpIYyEJQkSWooA0FJkqSGMhCUJElqKANBSZKkhhpxIBgRW3Z4Xq8kSZIWMd20CP4cuD0ijo6IDcY6Q5IkSZoY3QSC7wB+B+wJ/DEiro6I/SJitTHNmSRJksbViAPBzDw7M7enPNN3d+Bu4Cjg1oj4ZUS8NyKWGuN8SpIkaYx1PVkkMx/OzO9k5huBNYBPAasApwB3RcR3I+KNY5RPSZIkjbGxmjW8GPBMYDowDXgMeBPwq4j4U0TMGKP3kSRJ0hhZvNsTI2I5YHvgvcBmwBPAucAnqu0CYBvgS8BJwMajzawkSZLGzogDwYh4ByX42wpYErgS2Bv4QWbe15b8pxHxHOBro8ynJEmSxlg3LYI/Bu4Avgyckpk3DJH+auDULt5HkiRJ46ibQPDNwAWZuXA4iTPzCuCKLt5HkiRJ42jEgWBm/no8MiJJkqSJ1c0j5r4UETcNcvzGiDh6dNmSJEnSeOtm+ZitgTMGOX4G8LbusiNJkqSJ0k0g+ALglkGO31qlkSRJ0iTWTSD4EPDiQY6vSVlQWpIkSZNYN4HghcCHIuKF7Qci4kXAh6o0kiRJmsS6WT7mYGAWMDsiTgKurfbPAHYBngQOGpPcSZIkadx0s3zMTRGxGXA8sFfb4UuAvTIzxyJzkiRJGj9dPWs4M68FXlc9Pm7NaveczLx3zHImSZKkcdVVINiSmfcA94xRXiRJkjSBugoEI2IxYEtKa+AKwLS2JAsz8/BR5k2SJEnjaMSBYERsBJwFPJ+nB4AtCwEDQUmSpEmsmxbBrwHPAt4B/CYzHxjLDEmSJGlidBMIvhz4dGaeM9aZkSRJ0sTpZkHp2xm4S1iSJEmLiG4CwaOA3SNi2bHOjCRJkiZON13DKwL/BP4aET8C/pfyNJG6hZl59GgzJ0mSpPHTTSB4VO3fewyQZiFgIChJkjSJdRMIvnjMcyFJkqQJ182zhm8dj4xIkiRpYnX9iLmIWAd4HbAKcGpm3hIRSwCrAndm5vyxyaIkSZLGQzdPFnkGcALwAcoyMguB3wO3AEsA1wCHAf89ZrmUJEnSmOtm+ZhPAe8HDgI2obamYGY+Qnn83LvGJHeSJEkaN90EgrsC38nMI4G/djh+DbDOqHIlSZKkcddNIPh84IpBjj8GLNNddiRJkjRRugkE7wTWGOT4TMCZxZIkSZNcN4HgWcB/VbOGWxYCRMQsYGfgh2OQN0mSJI2jbgLBQ4HbgD8Bp1KCwE9FxB+AnwF/Af7fWGVQkiRJ42PEgWBmPgRsChwJPBf4F7A5sDQlSHxtZj42hnmUJEnSOOhqQenM/BclEDxybLMjSZKkidJN17AkSZKmgG6eLPKdYSRbmJkf6CI/kiRJmiDddA2/gWqWcM1iwPOq7d3AP0eZL0mSJI2zEQeCmfmiTvsj4pnAh4CPAVsMdZ2I2A54D2XdwRWBOcDXgRMzc0Et3Szgc8D6wB3AsZl5XIfr7QfsCawKXAscmJkXtKVZBjga2BZYErgI2Cszbxkqv5IkSVPNmI0RzMzHM/OrwC+Brw7jlH2BecD+wFuBnwJfAT7fShARmwBnU5aqmQWcBBwbEXvUL1QFgUcCxwNbAzcB50bEBm3v+QNgG2AvYAdgNeCCiFhqJPcqSZI0FXQ1a3gIfwHeN4x0b8vMu2uvL4qIpYGPRMRnMnMecDDwx9p4w4si4oXAIRHxjcxcEBHTgc9QWgqPAYiISyjPPP40sH2171WUIHHrzDyv2ncNpSVyF+Bro7prSZKkRcx4zBreAnh0qERtQWDLnyhdtitWAd4bgDPa0pxG6f59RfV6U2A54PTatZ+kPN1kVkRMq3ZvBTwInF9Ldxvwu+qYJElSo3Qza/jgAQ4tD7yWEqAd1WV+XgPcB/wDCGAJ4Lq2NNdW23WBq4D1qtfXd0i3NLA6cHuV7ob6+MNaui27zK8kSdIiq5uu4UMH2H8/pZt1D+CbI71oRGwE7Ap8NjOfjIgVqkMPdHgfKBNMAFYA5nV4mkk93e1VuvZrtdKt2GG/JEnSlNbNrOEx706OiFWBs4ArqE0Wmexmz549rHR9fX1dXf9Zy67M3Llzuzp3stlwzbW7LoepxnIoLId+lkVhORSWQ2E59BvPshiPySIjEhHLAT+njCvcJjMfrw61WvSWbzul1VJ4Xy3d9IhYsnr03WDpXtghCyvU0ozIjBkzmD59+qBp+vr6mDlzZjeX5677HmW11YYcbrnI6LYcppLR1IepxHLoZ1kUlkNhORSWQ7+xKIt58+YN2HjVzRjBTsHUkKqJGe3XWpKyPMwqwKaZeW/t8BxgPmVs3/m1/etX2xuqbWts4HqUySb1dA9T1h5spdsiIqZl5sK2dDcgSZLUMN10894C/K2Ln6eIiMUpM3tfDszKzFvrx6vlYy6kWv6lZkfgTuCP1evLKLOBd6hde7HqvPNrQd95lNbFLWvpXgBsXh2TJElqlG66hncDPgq8gLKUy43V/qAEabdRFoZun53b7njgbcABwFIR8erasesy8yHgMODSiPgmcCqwGbA7sGdr9m9mzouII4AjI+JuSoC4G7AWsFPrgpl5eUScC3w7IvYFWte/DTi5i3KQJElapHUTCD4PmA6snZn31w9ExCGUdflWzcz/N8R1Wi1zX+hw7PXAxZn5+4h4O+WpITsDc4F9MvOEeuLMPCYioASoz6UsCbN1Zv6l7bo7AsdQFo+eTnnE3HaZOXUG4kmSJA1TN4HgHsAX24NAgMy8t2q92xsYNBAc6JnFHdKdxzC6bqunihwzRJqHKc9D/tBw3luSJGkq62aM4EqUhZoH8uwqjSRJkiaxbgLBPwB7R8TT5jJXi0LvDVw+2oxJkiRpfHXTNfwR4GLgioi4Erip2r8OsDFlTb69xiR3kiRJGjcjbhHMzOuAl1FmBi8PbFv9LA98GXhZZl470PmSJEmaHLp6skhm3gXsU/1IkiRpETSqR8xFxDqUp4LMzswHxyZLkiRJmgjdTBYhInaKiNsoj2a7FJhZ7X9ORNwYEe1PA5EkSdIkM+JAMCL+A/g+5dm9+wPTWscy855q/85jlUFJkiSNj25aBD8N/DoztwRO6XD8cmCDUeVKkiRJ466bQHA94CeDHP8HsHJ32ZEkSdJE6SYQ/CeDP1lkLeCe7rIjSZKkidJNIHghsEtELNF+ICJWA3YHfjHajEmSJGl8dTtG8HnAVcCHgYXAVhFxFHANsAD47JjlUJIkSeOimyeL3ARsBtwJHEqZNfxx4ADgz8DmmXnb2GVRkiRJ42FEC0pHxGLA6sBdmfnmiFgBWJsSUN6cmXePQx4lSZI0Dkb6ZJFnAHOAA4EvZub9wJVjnitJkiSNuxF1DWfm48BcyrhASZIkLcK6mSxyEmXW8JJjnRlJkiRNnJF2DQPcCCwG3BARpwA3A4+1J8rMH44yb5IkSRpH3QSC36/9+6AB0iwEDAQlSZImsWEFghHxFeCUzOwDXl/tXprSEvjkOOVNkiRJ42i4LYIfAf4A9GXmJRGxEuWZwltk5iXjljtJkiSNm24mi7RMG7NcSJIkacKNJhCUJEnSIsxAUJIkqaFGMmt4zYh4ZfXv5artuhHxSKfEmXnFqHImSZKkcTWSQPCz1U/dcR3STaMsH7NYt5mSJEnS+BtuILjruOZCkiRJE25YgWBmnjLeGZEkSdLEcrKIJElSQxkISpIkNZSBoCRJUkMZCEqSJDWUgaAkSVJDGQhKkiQ1lIGgJElSQxkISpIkNZSBoCRJUkMZCEqSJDWUgaAkSVJDGQhKkiQ1lIGgJElSQxkISpIkNZSBoCRJUkMZCEqSJDWUgaAkSVJDGQhKkiQ1lIGgJElSQxkISpIkNZSBoCRJUkMZCEqSJDXU4r1884hYG9gPeDUwA7ghM2d0SDcL+BywPnAHcGxmHtch3X7AnsCqwLXAgZl5QVuaZYCjgW2BJYGLgL0y85axuzNJkqTJr9ctgi8Ftgb+ClzXKUFEbAKcDfwJmAWcBBwbEXu0pdsPOBI4vrrmTcC5EbFB2yV/AGwD7AXsAKwGXBARS43RPUmSJC0SetoiCJyTmf8DEBEnAxt1SHMw8MfM/ED1+qKIeCFwSER8IzMXRMR04DOUlsJjqutdAlwDfBrYvtr3KkqQuHVmnlftuwaYA+wCfG1c7lKSJGkS6mmLYGYuGOx4FeC9ATij7dBplO7fV1SvNwWWA06vXftJ4IfArIiYVu3eCngQOL+W7jbgd9UxSZKkxuh11/BQ1gKW4OndxtdW23Wr7XrV9voO6ZYGVq+lu6FDAHpt7VqSJEmN0Ouu4aGsUG0faNt/f7VdsZZuXmY+Nki626t07ddqpVuxw/5BzZ49e1jp+vr6RnppAJ617MrMnTu3q3Mnmw3XXLvrcphqLIfCcuhnWRSWQ2E5FJZDv/Esi8keCE5qM2bMYPr06YOm6evrY+bMmV1d/677HmW11R7t6tzJqNtymEpGUx+mEsuhn2VRWA6F5VBYDv3GoizmzZs3YOPVZO8abrXoLd+2v9VSeF8t3fSIWHIY6dqv1Up3X4f9kiRJU9ZkDwTnAPPpHwPYsn61vaHatsYGdkr3MGXtwVa6qE0eqae7AUmSpAaZ1IFgZs4DLqRa/qVmR+BO4I/V68sos4F3aCWIiMWq887PzIXV7vMoLYJb1tK9ANi8OiZJktQYvX6yyFL0L9uyBrBsRGxbvb4yM28FDgMujYhvAqcCmwG7A3u2Zv9m5ryIOAI4MiLupgSIu1FmHe/Uer/MvDwizgW+HRH7Ag9V178NOHlcb1aSJGmS6fVkkVWAM9v2tV7vCpycmb+PiLdTnhqyMzAX2CczT6iflJnHRATAR4HnUpaE2Toz/9J2/R2BYyiLR0+nPGJuu8ycOrMyJqEll3wWd903NYp4qemLs8yzl+h1NiRJGrWeBoLV833bx+t1Sncew+i6rZ4qcswQaR4GPlT9aII8/uRCLrjytl5nY0y8ceMXGghKkqaEST1GUJIkSePHQFCSJKmhDAQlSZIaykBQkiSpoQwEJUmSGspAUJIkqaEMBCVJkhrKQFCSJKmhDAQlSZIaykBQkiSpoQwEJUmSGspAUJIkqaEMBCVJkhrKQFCSJKmhDAQlSZIaykBQkiSpoQwEJUmSGspAUJIkqaEMBCVJkhrKQFCSJKmhDAQlSZIaykBQkiSpoQwEJUmSGspAUJIkqaEMBCVJkhrKQFCSJKmhDAQlSZIaykBQkiSpoQwEJUmSGspAUJIkqaEMBCVJkhrKQFCSJKmhDAQlSZIaykBQkiSpoQwEJUmSGspAUJIkqaEMBCVJkhrKQFCSJKmhDAQlSZIaykBQkiSpoQwEJUmSGspAUJIkqaEMBCVJkhrKQFCSJKmhDAQlSZIaykBQkiSpoQwEJUmSGspAUJIkqaEMBCVJkhrKQFCSJKmhFu91BiZaRKwDHAdsDjwGnA4cmJmP9jRjkiRJE6xRgWBELA9cBNwKbAusAnwRWBl4d+9yJkmSNPEaFQgCHwJWADbMzHsAIuIJ4NSIODwzr+1p7iRJkiZQ08YIbgVc0AoCK2cB84BZvcmSJElSbzStRXA94Dv1HZk5LyLmAOuO4DqLAcyfP39YiefNmzeCS/d74vH5LP6MBV2dO9ksePKJKXMvTzw+n3nzFuv6/G7rw1RjOfSzLArLobAcCsuh32jLohavPO3La9rChQtHdfFFSUQ8DhyUmUe17f8t8I/MfNdwrtPX17c58JtxyKIkSdJ4ec3MmTN/W9/RtBbBsXIl8Brg78CTPc6LJEnSYBYDnkeJX56iaYHg/cDyHfavANww3IvMnDlzHvDbIRNKkiRNDnM67WzaZJHrKeME/y0ipgNrMYJAUJIkaSpoWiB4HvDGiFiptu+dwPTqmCRJUmM0bbLI8sBs4BbgcPoXlL4gM11QWpIkNUqjWgQz8wHgDcAjwI+BLwFnAO/vYbYkSZJ6olEtgpIkSerXqBZBSZIk9TMQlCRJaigDQUmSpIZq2oLSEyIi1gGOAzYHHgNOBw7MzEd7mrEJFBG7ACd1OHR8Zn5kgrMzYSJibWA/4NXADOCGzJzRId0s4HPA+sAdwLGZedxE5nU8DaccIuJk4D87nL5dZv5o3DM5ASJiO+A9wExgRcqCrl8HTszMBbV0U70+DFkOTagPABHxLuDjlOfbL035vH8CHJ6ZD9bSTfU6MWQ5NKVO1EXE0pR1jVcHNs7Mq2rHdgY+BbyI8n/osMw8Y7TvaSA4xqolai4CbgW2pX+JmpWBJi5R8xbgwdrrO3uVkQnyUmBr4HJKi/vTWt0jYhPgbOC7wL7AZsCxEfF4Zp4wgXkdT0OWQ+VmSoBQd+M45mui7Uv5XbA/cBfweuArwJrVvqbUhyHLoTLV6wOUQPhSyvfCfcDLgUOr7ZuhMXViyHKoNKFO1B1Kh9gsIrYFTgGOAn4JvAP4QUQ8lJk/H80bOmt4jEXEgcDBwBqZeU+1byfgVGBGZl7by/xNlFqL4MqtcmiCiHhGWwvHRh1awn4OrJiZr6rt+wbwNmD1ekvRomqY5dBx/1QSEStn5t1t+74I/BewfGbOa0h9GE45nMwUrw8DiYgPAidSPu+5TagTnXQoh5NpUJ2IiBnAHygtpSdSaxGMiOuBazJz+1r6X1L+/7xyNO/rGMGxtxVlgep68HMWMA+Y1ZssaaIM9Qu6eqThGyjrV9adBqwKvGKcsjahpuoX1Ui1Bz+VPwFLAis2qD4MWg4TnJ3JqPV9sURT6sQA/l0OPc1F7xwPfJW2Fs+IeDGlC/30tvSnARtHxMqjeVO7hsfeesB36juqv3bnUD7IppldVdLbgJOBz2XmE73NUk+tRfkld13b/lZL8brAVTTHWhHxAPBsylN/jhqLMS+T3GsoXWH/AILm1od6ObQ0pj5ExGLAMynDKA4Gzs7MWyJifRpUJwYqh1qSRtSJiHgfsDZlSM1GbYfXq7YD1YkAOv2xNSy2CI69FYAHOuy/n2b95ft34BBgF8o4wZ8ABwHf6mGeJoMVqu0Dbfvvr7ZNqiN/okwoeQdlPO3twOnVsIIpKSI2AnYFvpSZT9LQ+tChHKB59eFeymTCqyi/L3eq9jetTgxUDtCQOhERywFHAwdk5iMdkoxrnbBFUOMiM38B/KK261cR8SBwaEQcnplzepQ1TRKZ+eW2Xf8TERcCn6W0Hk8pEbEqZZjIFcDne5ydnhmoHJpWH4DXAUtRZtV/BjgnIrboaY5643V0KIfMfLJBdeII4KbMPLUXb26L4Ni7H1i+w/4VKN0gTfbDajuVx7gMpfUX3PJt+1t/8TW9jpwJvHC0Y14mm+ov/p8DjwLbZObj1aFG1YdBymEgU7I+AGTmnzPzssz8BvBOykzqd9KwOjFIOQxkStWJiHgpsAdwUEQsX608snR1eOmIWIZxrhMGgmPvevr784F/TxBYi7I2kJptDjCftjpCWSsMrCNTTkQsSVkKZBXgLZl5b+1wY+rDEOXQdH8GFlDGiDWmTnTwZ/rLoSnWofTOXkQJ+O4HzqmOXQT8hhJXwMB1IkeTAQPBsXce8MaIWKm2753A9OpYk70bWAj09TojvZKZ84ALge3bDu1IWWPxjxOeqUkiIqZRyuXWAWaZLnIiYnFKS/jLgVmZeWv9eFPqw1DlMMA5U64+DGITyvfxzU2pEwP4dzl0OjhF68RvKa2g9Z99qmN7ALtl5t8ofwDs0HbujsCVoy0LxwiOvROBvShjGQ6nf0HpMzKzfcbPlBURv6D8MptN+QtvFvBh4NuZ2fE/+VQQEUtRlhACWANYtloIFMp/2FuBw4BLI+KblPUlNwN2B/acKsuuDFUO1fYU4AfAXyldHrtRxgu9b8IyOv6Op6z9dgCwVES8unbsusx8iAbUB4YoB0oXVxPqQ+t34wWUGZ//AjakLKp9NfDTKtmUrxNDlUNErEED6kS11NzF9X0R0fpnX+3JIgcDZ1QrkPwKeDtl4e2tR5sHA8ExlpkPRMQbKKvm/5j+R8wd0NOMTbzrgfcDz6fUs5uAA4Fje5inibAKZQxLXev1rsDJmfn7iHg7cCSwMzAX2GcKPTEAhi6HsylPnPlMlfZxSkvHNpl5DlPHltX2Cx2OvR64uCH1YahyuJpm1Acok2TeC7y4en0LcALwxcycD9CQOjFoOUTEwzSnTgwpM8+s/sD+FGUm9Rxgp9E+VQR8sogkSVJjOUZQkiSpoQwEJUmSGspAUJIkqaEMBCVJkhrKQFCSJKmhDAQlSZIaykBQkiSpoQwEJUmSGur/A0w6fGYym9kTAAAAAElFTkSuQmCC\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "df1 = df[df[\"name\"].isin([\"QuerySamplesComplete\"])]\n", - "df1['delta'] = df1['ts'].diff()\n", - "ax = df1['delta'].plot.hist(bins=BINS, alpha=0.5, figsize=figsize)\n", - "ax.set_title('Time between QuerySamplesComplete (usec)');\n", - "plt.show()\n", - "\n", - "ax = df1['dur'].plot.hist(bins=BINS, alpha=0.5, figsize=figsize)\n", - "ax.set_title('Time QuerySamplesComplete (usec)');" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.3" - }, - "varInspector": { - "cols": { - "lenName": 16, - "lenType": 16, - "lenVar": 40 - }, - "kernels_config": { - "python": { - "delete_cmd_postfix": "", - "delete_cmd_prefix": "del ", - "library": "var_list.py", - "varRefreshCmd": "print(var_dic_list())" - }, - "r": { - "delete_cmd_postfix": ") ", - "delete_cmd_prefix": "rm(", - "library": "var_list.r", - "varRefreshCmd": "cat(var_dic_list()) " - } - }, - "types_to_exclude": [ - "module", - "function", - "builtin_function_or_method", - "instance", - "_Feature" - ], - "window_display": false - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/utils.cc b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/utils.cc deleted file mode 100644 index de74eb820..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/utils.cc +++ /dev/null @@ -1,124 +0,0 @@ -/* Copyright 2019 The MLPerf Authors. All Rights Reserved. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "utils.h" - -#include -#include -#include -#include -#include - -#include "logging.h" - -namespace mlperf { - -std::string DoubleToString(double value, int precision) { - std::stringstream ss; - ss.precision(precision); - ss << std::fixed << value; - return ss.str(); -} - -bool FileExists(const std::string filename) { - std::ifstream file_object(filename); - return file_object.good(); -} - -namespace { - -std::string DateTimeString(const char* format, - std::chrono::system_clock::time_point tp, - bool append_ms, bool utc) { - std::time_t tp_time_t = std::chrono::system_clock::to_time_t(tp); - std::tm date_time = - utc ? *std::gmtime(&tp_time_t) : *std::localtime(&tp_time_t); - constexpr size_t kDateTimeMaxSize = 256; - char date_time_cstring[kDateTimeMaxSize]; - std::strftime(date_time_cstring, kDateTimeMaxSize, format, &date_time); - std::string date_time_string(date_time_cstring); - if (!append_ms) { - return date_time_string; - } - - auto tp_time_t_part = std::chrono::system_clock::from_time_t(tp_time_t); - auto tp_remainder = tp - tp_time_t_part; - auto ms = std::chrono::duration_cast(tp_remainder) - .count(); - if (ms < 0 || ms >= 1000) { - LogDetail([ms](AsyncDetail& detail) { -#if USE_NEW_LOGGING_FORMAT - std::stringstream ss; - ss << "WARNING: Unexpected milliseconds getting date and time." - << " ms: " << ms; - MLPERF_LOG_WARNING(detail, "warning_generic_message", ss.str()); -#else - detail("WARNING: Unexpected milliseconds getting date and time.", "ms", - ms); -#endif - }); - } - std::string ms_string = std::to_string(ms); - // Prefix with zeros so length is always 3. - ms_string.insert(0, std::min(2, 3 - ms_string.length()), '0'); - return date_time_string + "." + ms_string; -} - -} // namespace - -std::string CurrentDateTimeISO8601() { - return DateTimeString("%FT%TZ", std::chrono::system_clock::now(), false, - false); -} - -std::string DateTimeStringForPower(std::chrono::system_clock::time_point tp) { - return DateTimeString("%m-%d-%Y %T", tp, true, true); -} - -std::string EscapeStringJson(const std::string& in) { - std::stringstream ss; - for (auto c = in.cbegin(); c != in.cend(); c++) { - int c_val = static_cast(*c); - switch (*c) { - case '"': - ss << "\\\""; - break; - case '\\': - ss << "\\\\"; - break; - case '\b': - ss << "\\b"; - break; - case '\f': - ss << "\\f"; - break; - case '\n': - ss << "\\n"; - break; - case '\r': - ss << "\\r"; - break; - case '\t': - ss << "\\t"; - break; - default: - if (c_val >= 0x00 && c_val < 0x20) { - ss << "\\u" << std::hex << std::setw(4) << std::setfill('0') << c_val; - } else { - ss << *c; - } - } - } - return ss.str(); -} - -} // namespace mlperf diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/utils.h b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/utils.h deleted file mode 100644 index c587e0cbe..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/utils.h +++ /dev/null @@ -1,70 +0,0 @@ -/* Copyright 2019 The MLPerf Authors. All Rights Reserved. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -/// \file -/// \brief Various shared utility functions. - -#ifndef MLPERF_LOADGEN_UTILS_H -#define MLPERF_LOADGEN_UTILS_H - -#include -#include -#include - -#include "query_sample.h" - -namespace mlperf { - -template -void RemoveValue(T* container, const typename T::value_type& value_to_remove) { - container->erase(std::remove_if(container->begin(), container->end(), - [&](typename T::value_type v) { - return v == value_to_remove; - }), - container->end()); -} - -template -double DurationToSeconds( - const std::chrono::duration& chrono_duration) { - return std::chrono::duration_cast>( - chrono_duration) - .count(); -} - -inline double QuerySampleLatencyToSeconds(QuerySampleLatency qsl) { - return static_cast(qsl) / std::nano::den; -} - -template -inline DurationT SecondsToDuration(double seconds) { - return std::chrono::duration_cast( - std::chrono::duration(seconds)); -} - -std::string CurrentDateTimeISO8601(); - -/// \brief Uses a format that matches the one used by SPEC power -/// measurement logging. -std::string DateTimeStringForPower(std::chrono::system_clock::time_point tp); - -std::string DoubleToString(double value, int precision = 2); - -bool FileExists(const std::string filename); - -// \brief Escape special characters in a string for JSON. -// Don't use this in performance critical path. -std::string EscapeStringJson(const std::string& in); - -} // namespace mlperf - -#endif // MLPERF_LOADGEN_UTILS_H diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/version.cc b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/version.cc deleted file mode 100644 index 3216c9d72..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/version.cc +++ /dev/null @@ -1,85 +0,0 @@ -/* Copyright 2019 The MLPerf Authors. All Rights Reserved. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -/// \file -/// \brief Non-generated version logic. - -#include "version.h" - -#include "logging.h" -#include "utils.h" - -namespace mlperf { - -/// Helper function to split a string based on a delimiting character. -std::vector splitString(const std::string& input, - const std::string& delimiter) { - std::vector result; - size_t start = 0; - size_t next = 0; - while (next != std::string::npos) { - next = input.find(delimiter, start); - result.emplace_back(input, start, next - start); - start = next + 1; - } - return result; -} - -/// Converts the hash-filename pairs to a dict. -std::map LoadgenSha1OfFilesToDict( - const std::string& in) { - std::map result; - auto files = splitString(in, "\n"); - for (const auto& file : files) { - auto hash_and_name = splitString(file, " "); - assert(hash_and_name.size() > 1); - result[hash_and_name[1]] = hash_and_name[0]; - } - return result; -} - -void LogLoadgenVersion() { - LogDetail([](AsyncDetail& detail) { -#if USE_NEW_LOGGING_FORMAT - MLPERF_LOG(detail, "loadgen_version", - LoadgenVersion() + " @ " + LoadgenGitRevision()); - MLPERF_LOG(detail, "loadgen_build_date_local", LoadgenBuildDateLocal()); - MLPERF_LOG(detail, "loadgen_build_date_utc", LoadgenBuildDateUtc()); - MLPERF_LOG(detail, "loadgen_git_commit_date", LoadgenGitCommitDate()); - MLPERF_LOG(detail, "loadgen_git_log_message", - EscapeStringJson(LoadgenGitLog())); - MLPERF_LOG(detail, "loadgen_git_status_message", - EscapeStringJson(LoadgenGitStatus())); - if (!LoadgenGitStatus().empty() && LoadgenGitStatus() != "NA") { - MLPERF_LOG_ERROR(detail, "error_uncommitted_loadgen_changes", - "Loadgen built with uncommitted changes!"); - ; - } - MLPERF_LOG(detail, "loadgen_file_sha1", - LoadgenSha1OfFilesToDict(LoadgenSha1OfFiles())); -#else - detail("LoadgenVersionInfo:"); - detail("version : " + LoadgenVersion() + " @ " + LoadgenGitRevision()); - detail("build_date_local : " + LoadgenBuildDateLocal()); - detail("build_date_utc : " + LoadgenBuildDateUtc()); - detail("git_commit_date : " + LoadgenGitCommitDate()); - detail("git_log :\n\n" + LoadgenGitLog() + "\n"); - detail("git_status :\n\n" + LoadgenGitStatus() + "\n"); - if (!LoadgenGitStatus().empty() && LoadgenGitStatus() != "NA") { - detail.Error("Loadgen built with uncommitted changes!"); - } - detail("SHA1 of files :\n\n" + LoadgenSha1OfFiles() + "\n"); -#endif - }); -} - -} // namespace mlperf diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/version.h b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/version.h deleted file mode 100644 index 87c3409aa..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/version.h +++ /dev/null @@ -1,39 +0,0 @@ -/* Copyright 2019 The MLPerf Authors. All Rights Reserved. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -/// \file -/// \brief Declares the version-related strings that will be defined in -/// a version_generated.cc as created by version_generator.py. - -#ifndef MLPERF_LOADGEN_VERSION_H -#define MLPERF_LOADGEN_VERSION_H - -#include - -namespace mlperf { - -// Non-generated. -void LogLoadgenVersion(); - -// Definitions generated at compile time. -const std::string& LoadgenVersion(); -const std::string& LoadgenGitRevision(); -const std::string& LoadgenBuildDateLocal(); -const std::string& LoadgenBuildDateUtc(); -const std::string& LoadgenGitCommitDate(); -const std::string& LoadgenGitStatus(); -const std::string& LoadgenGitLog(); -const std::string& LoadgenSha1OfFiles(); - -} // namespace mlperf - -#endif // MLPERF_LOADGEN_VERSION_H diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/version_generator.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/version_generator.py deleted file mode 100644 index 2e7524330..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/version_generator.py +++ /dev/null @@ -1,141 +0,0 @@ -# Copyright 2019 The MLPerf Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -# \file -# \brief A script run by the build to generate the version definitions -# expected at link time. - -import datetime -import errno -import hashlib -import os -import sys -import subprocess - - -# Creates a C++ raw string literal using a delimiter that is very -# unlikely to show up in a git stats. -def make_raw_string(str): - delimeter = "LGVG_RSLD" - return 'R"' + delimeter + "(" + str + ")" + delimeter + '"' - - -def func_def(name, string): - return ( - "const std::string& Loadgen" - + name - + "() {\n" - + " static const std::string str = " - + string - + ";\n" - + " return str;\n" - + "}\n\n" - ) - - -# For clients that build the loadgen from the git respository without -# any modifications. -def generate_loadgen_version_definitions_git(ofile, git_command): - git_rev = os.popen(git_command + "rev-parse --short=10 HEAD").read() - git_commit_date = os.popen( - git_command + - "log --format=\"%cI\" -n 1").read() - git_status = os.popen(git_command + "status -s -uno .").read() - git_log = subprocess.Popen( - git_command + "log --pretty=oneline -n 16 --no-decorate", stdout=subprocess.PIPE, shell=True, encoding='ascii', errors="ignore").stdout.read() - ofile.write(func_def("GitRevision", "\"" + git_rev[0:-1] + "\"")) - ofile.write(func_def("GitCommitDate", "\"" + git_commit_date[0:-1] + "\"")) - ofile.write(func_def("GitStatus", make_raw_string(git_status[0:-1]))) - ofile.write(func_def("GitLog", make_raw_string(git_log[0:-1]))) - - -# For clients that might not import the loadgen code as the original git -# repository. -def generate_loadgen_verstion_definitions_git_stubs(ofile): - na = '"NA"' - ofile.write(func_def("GitRevision", na)) - ofile.write(func_def("GitCommitDate", na)) - ofile.write(func_def("GitStatus", na)) - ofile.write(func_def("GitLog", na)) - - -# Always log the sha1 of the loadgen files, regardless of whether we are -# in the original git repository or not. -def generate_loadgen_version_definitions_sha1(ofile, loadgen_root): - """Writes definition for Sha1OfFiles.""" - sha1s = "" - loadgen_files = [ - "/bindings/" + s for s in os.listdir(loadgen_root + "/bindings") - ] + ["/" + s for s in os.listdir(loadgen_root)] - for fn in sorted(loadgen_files): - full_fn = loadgen_root + fn - if not os.path.isfile(full_fn): - continue - file_data = open(full_fn, "rb").read() - sha1s += hashlib.sha1(file_data).hexdigest() + " " + fn + "\n" - - ofile.write(func_def("Sha1OfFiles", make_raw_string(sha1s[0:-1]))) - - -# Outputs version function definitions to cc_filename. -# Includes SHA1's of the relevant dirs in the loadgen_root directory. -def generate_loadgen_version_definitions(cc_filename, loadgen_root): - """Generates the C++ source file with the loadgen version info.""" - try: - os.makedirs(os.path.dirname(cc_filename)) - except OSError as exc: - if exc.errno != errno.EEXIST: - raise - ofile = open(cc_filename, "w") - ofile.write("// DO NOT EDIT: Autogenerated by version_generator.py.\n\n") - ofile.write("#include \n\n") - ofile.write("namespace mlperf {\n\n") - # Open and read the VERSION.txt file - with open(os.path.join(loadgen_root, "VERSION.txt"), "r") as version_file: - # Read and strip any extra whitespace/newlines - version_contents = version_file.read().strip() - - # Write the version into the function definition - ofile.write(func_def("Version", f"\"{version_contents}\"")) - - date_time_now_local = datetime.datetime.now().isoformat() - date_time_now_utc = datetime.datetime.utcnow().isoformat() - ofile.write(func_def("BuildDateLocal", '"' + date_time_now_local + '"')) - ofile.write(func_def("BuildDateUtc", '"' + date_time_now_utc + '"')) - - git_dir = '--git-dir="' + loadgen_root + '/../.git" ' - git_work_tree = '--work-tree="' + loadgen_root + '/.." ' - git_command = "git " + git_dir + git_work_tree - git_status = os.popen(git_command + "status") - git_status.read() - is_git_repo = git_status.close() is None - if is_git_repo: - generate_loadgen_version_definitions_git(ofile, git_command) - else: - generate_loadgen_verstion_definitions_git_stubs(ofile) - generate_loadgen_version_definitions_sha1(ofile, loadgen_root) - - ofile.write("} // namespace mlperf\n") - ofile.close() - - -def main(): - if len(sys.argv) != 3: - raise ValueError("Incorrect command-line arguments.") - generate_loadgen_version_definitions(sys.argv[1], sys.argv[2]) - - -if __name__ == "__main__": - main() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/ts_types.py b/recommendation_v4/generative_recommenders/dlrm_v3/inference/ts_types.py deleted file mode 100644 index cb558726b..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/ts_types.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-strict - -""" -TorchScript-friendly boundary types for the HSTU sparse <-> dense interface. - -The eager path uses ``Dict[str, SequenceEmbedding]`` (a NamedTuple of -``lengths`` and ``embedding`` tensors). TorchScript supports ``NamedTuple`` but -does not script cleanly through ``Dict[str, NamedTuple]`` once the dict crosses -device boundaries. The packaged sparse / dense modules instead exchange two -parallel ``Dict[str, Tensor]`` dicts -- one of jagged values, one of lengths. - -These helpers convert between the two representations so we can keep the -existing eager code unchanged while the scripted modules use only TS-friendly -types at their boundaries. -""" - -from typing import Dict, Tuple - -import torch -from generative_recommenders.modules.dlrm_hstu import SequenceEmbedding - - -# Per-feature jagged values (concatenated across batch, [L_total, table_dim]). -SeqEmbValues = Dict[str, torch.Tensor] -# Per-feature per-batch lengths ([B]). -SeqEmbLengths = Dict[str, torch.Tensor] - - -def flatten_seq_embeddings( - seq_embeddings: Dict[str, SequenceEmbedding], -) -> Tuple[SeqEmbValues, SeqEmbLengths]: - """Split ``Dict[str, SequenceEmbedding]`` into parallel value/length dicts. - - Lossless and zero-copy -- the returned tensors alias the inputs. - """ - values: Dict[str, torch.Tensor] = {} - lengths: Dict[str, torch.Tensor] = {} - for k, v in seq_embeddings.items(): - values[k] = v.embedding - lengths[k] = v.lengths - return values, lengths - - -def unflatten_seq_embeddings( - values: SeqEmbValues, - lengths: SeqEmbLengths, -) -> Dict[str, SequenceEmbedding]: - """Inverse of :func:`flatten_seq_embeddings`. - - Reconstructs ``Dict[str, SequenceEmbedding]`` for code paths (e.g. - ``DlrmHSTU.main_forward``) that still consume the NamedTuple form. - """ - out: Dict[str, SequenceEmbedding] = {} - for k, val in values.items(): - out[k] = SequenceEmbedding(lengths=lengths[k], embedding=val) - return out diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/inference/user.conf b/recommendation_v4/generative_recommenders/dlrm_v3/inference/user.conf deleted file mode 100644 index c6ca854f9..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/inference/user.conf +++ /dev/null @@ -1,5 +0,0 @@ -# Please set these fields depending on the performance of your system to -# override default LoadGen settings. -*.Server.target_latency = 80 -# *.Server.min_duration = 20000 -# *.Offline.min_duration = 20000 diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/kuairand_1k.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/kuairand_1k.gin deleted file mode 100644 index 46d8e1272..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/kuairand_1k.gin +++ /dev/null @@ -1,41 +0,0 @@ -batch_size = 16 -dataset = "kuairand-1k" - -# model parameters -make_model.dataset = %dataset - -# dense model optimizer -dense_optimizer_factory_and_class.learning_rate = 0.001 -dense_optimizer_factory_and_class.optimizer_name = "Adam" -dense_optimizer_factory_and_class.momentum = 0 -dense_optimizer_factory_and_class.weight_decay = 0 -dense_optimizer_factory_and_class.eps = 1e-8 -dense_optimizer_factory_and_class.betas = (0.95, 0.999) - -# sparse model optimizer -sparse_optimizer_factory_and_class.learning_rate = 0.001 -sparse_optimizer_factory_and_class.optimizer_name = "RowWiseAdagrad" -sparse_optimizer_factory_and_class.momentum = 0 -sparse_optimizer_factory_and_class.weight_decay = 0 -sparse_optimizer_factory_and_class.eps = 1e-8 -sparse_optimizer_factory_and_class.betas = (0.95, 0.999) - -# dataloader configs -make_train_test_dataloaders.batch_size = %batch_size -make_train_test_dataloaders.dataset_type = %dataset -make_train_test_dataloaders.train_split_percentage = 0.75 -make_train_test_dataloaders.new_path_prefix = "/home/linjianma" -make_train_test_dataloaders.num_workers = 2 -make_train_test_dataloaders.prefetch_factor = 4 - -# train loop variables -train_loop.num_epochs = 5 -train_loop.output_trace = True -train_loop.metric_log_frequency = 10 - -# logger variables -MetricsLogger.tensorboard_log_path = "/tmp/tensorboard_log_path.log" - -# checkpoint -# save_dmp_checkpoint.path = "/home/linjianma/ckpts/kuairand_1k" -# load_dmp_checkpoint.path = "/home/linjianma/ckpts/kuairand_1k/2025_01_12_17_56_43/" diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/movielens_13b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/movielens_13b.gin deleted file mode 100644 index e2f371de4..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/movielens_13b.gin +++ /dev/null @@ -1,41 +0,0 @@ -batch_size = 128 -dataset = "movielens-13b" - -# model parameters -make_model.dataset = %dataset - -# dense model optimizer -dense_optimizer_factory_and_class.learning_rate = 0.001 -dense_optimizer_factory_and_class.optimizer_name = "Adam" -dense_optimizer_factory_and_class.momentum = 0 -dense_optimizer_factory_and_class.weight_decay = 0 -dense_optimizer_factory_and_class.eps = 1e-8 -dense_optimizer_factory_and_class.betas = (0.95, 0.999) - -# sparse model optimizer -sparse_optimizer_factory_and_class.learning_rate = 0.001 -sparse_optimizer_factory_and_class.optimizer_name = "RowWiseAdagrad" -sparse_optimizer_factory_and_class.momentum = 0 -sparse_optimizer_factory_and_class.weight_decay = 0 -sparse_optimizer_factory_and_class.eps = 1e-8 -sparse_optimizer_factory_and_class.betas = (0.95, 0.999) - -# dataloader configs -make_train_test_dataloaders.batch_size = %batch_size -make_train_test_dataloaders.dataset_type = %dataset -make_train_test_dataloaders.train_split_percentage = 0.75 -make_train_test_dataloaders.new_path_prefix = "/home/linjianma" -make_train_test_dataloaders.num_workers = 2 -make_train_test_dataloaders.prefetch_factor = 4 - -# train loop variables -train_loop.num_epochs = 1 -train_loop.output_trace = True -train_loop.metric_log_frequency = 10 -train_eval_loop.num_epochs = 1 -train_eval_loop.output_trace = True -train_eval_loop.metric_log_frequency = 10 - -# logger variables -MetricsLogger.tensorboard_log_path = "/tmp/tensorboard_log_path.log" -save_dmp_checkpoint.path = "/home/linjianma/ckpts/movielens_13b" diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/movielens_18b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/movielens_18b.gin deleted file mode 100644 index 094271b57..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/movielens_18b.gin +++ /dev/null @@ -1,56 +0,0 @@ -batch_size = 64 -dataset = "movielens-18b" - -# model parameters -make_model.dataset = %dataset - -# dense model optimizer -dense_optimizer_factory_and_class.learning_rate = 0.001 -dense_optimizer_factory_and_class.optimizer_name = "Adam" -dense_optimizer_factory_and_class.momentum = 0 -dense_optimizer_factory_and_class.weight_decay = 0 -dense_optimizer_factory_and_class.eps = 1e-8 -dense_optimizer_factory_and_class.betas = (0.95, 0.999) - -# sparse model optimizer -sparse_optimizer_factory_and_class.learning_rate = 0.001 -sparse_optimizer_factory_and_class.optimizer_name = "RowWiseAdagrad" -sparse_optimizer_factory_and_class.momentum = 0 -sparse_optimizer_factory_and_class.weight_decay = 0 -sparse_optimizer_factory_and_class.eps = 1e-8 -sparse_optimizer_factory_and_class.betas = (0.95, 0.999) - -# dataloader configs -make_train_test_dataloaders.batch_size = %batch_size -make_train_test_dataloaders.dataset_type = %dataset -make_train_test_dataloaders.train_split_percentage = 0.80 -make_train_test_dataloaders.new_path_prefix = "/home/linjianma" -make_train_test_dataloaders.num_workers = 2 -make_train_test_dataloaders.prefetch_factor = 4 -make_train_test_dataloaders.num_blocks = 20 - -# train loop variables -train_loop.num_epochs = 200 -train_loop.output_trace = False -train_loop.metric_log_frequency = 40 -train_loop.checkpoint_frequency = 4000 -train_loop.start_batch_idx = 0 - -# eval loop variables -eval_loop.metric_log_frequency = 40 - -# train eval loop variables -train_eval_loop.num_epochs = 20 -train_eval_loop.output_trace = False -train_eval_loop.start_train_batch_idx = 0 -train_eval_loop.start_eval_batch_idx = 0 -train_eval_loop.num_eval_batches = 200 -train_eval_loop.metric_log_frequency = 40 -train_eval_loop.checkpoint_frequency = 2000 -train_eval_loop.eval_frequency = 500 - - -# logger variables -MetricsLogger.tensorboard_log_path = "/home/linjianma/tensorboard/movielens_18b/" -save_dmp_checkpoint.path = "/home/linjianma/ckpts/movielens_18b/" -# load_dmp_checkpoint.path = "/home/linjianma/ckpts/movielens_18b/20000/" diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/movielens_1m.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/movielens_1m.gin deleted file mode 100644 index 2b6cd6b64..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/movielens_1m.gin +++ /dev/null @@ -1,38 +0,0 @@ -batch_size = 128 -dataset = "movielens-1m" - -# model parameters -make_model.dataset = %dataset - -# dense model optimizer -dense_optimizer_factory_and_class.learning_rate = 0.001 -dense_optimizer_factory_and_class.optimizer_name = "Adam" -dense_optimizer_factory_and_class.momentum = 0 -dense_optimizer_factory_and_class.weight_decay = 0 -dense_optimizer_factory_and_class.eps = 1e-8 -dense_optimizer_factory_and_class.betas = (0.9, 0.98) - -# sparse model optimizer -sparse_optimizer_factory_and_class.learning_rate = 0.001 -sparse_optimizer_factory_and_class.optimizer_name = "RowWiseAdagrad" -sparse_optimizer_factory_and_class.momentum = 0 -sparse_optimizer_factory_and_class.weight_decay = 0 -sparse_optimizer_factory_and_class.eps = 1e-8 -sparse_optimizer_factory_and_class.betas = (0.95, 0.999) - -# dataloader configs -make_train_test_dataloaders.batch_size = %batch_size -make_train_test_dataloaders.dataset_type = %dataset -make_train_test_dataloaders.train_split_percentage = 0.75 -make_train_test_dataloaders.new_path_prefix = "/home/linjianma" -make_train_test_dataloaders.num_workers = 2 -make_train_test_dataloaders.prefetch_factor = 4 - -# train-eval loop variables -train_eval_loop.num_epochs = 101 -train_eval_loop.output_trace = True -train_eval_loop.metric_log_frequency = 10 -train_eval_loop.eval_frequency = 1 - -# logger variables -MetricsLogger.tensorboard_log_path = "/tmp/tensorboard_log_path.log" diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/movielens_20m.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/movielens_20m.gin deleted file mode 100644 index c01fab5af..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/movielens_20m.gin +++ /dev/null @@ -1,56 +0,0 @@ -batch_size = 64 -dataset = "movielens-20m" - -# model parameters -make_model.dataset = %dataset - -# dense model optimizer -dense_optimizer_factory_and_class.learning_rate = 0.001 -dense_optimizer_factory_and_class.optimizer_name = "Adam" -dense_optimizer_factory_and_class.momentum = 0 -dense_optimizer_factory_and_class.weight_decay = 0 -dense_optimizer_factory_and_class.eps = 1e-8 -dense_optimizer_factory_and_class.betas = (0.95, 0.999) - -# sparse model optimizer -sparse_optimizer_factory_and_class.learning_rate = 0.001 -sparse_optimizer_factory_and_class.optimizer_name = "RowWiseAdagrad" -sparse_optimizer_factory_and_class.momentum = 0 -sparse_optimizer_factory_and_class.weight_decay = 0 -sparse_optimizer_factory_and_class.eps = 1e-8 -sparse_optimizer_factory_and_class.betas = (0.95, 0.999) - -# dataloader configs -make_train_test_dataloaders.batch_size = %batch_size -make_train_test_dataloaders.dataset_type = %dataset -make_train_test_dataloaders.train_split_percentage = 0.80 -make_train_test_dataloaders.new_path_prefix = "/home/linjianma" -make_train_test_dataloaders.num_workers = 2 -make_train_test_dataloaders.prefetch_factor = 4 -make_train_test_dataloaders.num_blocks = 1 - -# train loop variables -train_loop.num_epochs = 200 -train_loop.output_trace = False -train_loop.metric_log_frequency = 40 -train_loop.checkpoint_frequency = 4000 -train_loop.start_batch_idx = 0 - -# eval loop variables -eval_loop.metric_log_frequency = 10 - -# train eval loop variables -train_eval_loop.num_epochs = 20 -train_eval_loop.output_trace = False -train_eval_loop.start_train_batch_idx = 0 -train_eval_loop.start_eval_batch_idx = 0 -train_eval_loop.num_eval_batches = 100 -train_eval_loop.metric_log_frequency = 40 -train_eval_loop.checkpoint_frequency = 2000 -train_eval_loop.eval_frequency = 200 - - -# logger variables -MetricsLogger.tensorboard_log_path = "/home/linjianma/tensorboard/movielens_20m/" -# save_dmp_checkpoint.path = "/home/linjianma/ckpts/movielens_18b/0.5T" -# load_dmp_checkpoint.path = "/home/linjianma/ckpts/movielens_18b/20000/" diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/streaming_100b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/streaming_100b.gin deleted file mode 100644 index 7d1df4bce..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/streaming_100b.gin +++ /dev/null @@ -1,52 +0,0 @@ -batch_size = 64 -num_workers = 2 -prefetch_factor = 4 -dataset = "streaming-100b" - -# model parameters -make_model.dataset = %dataset - -# dense model optimizer -dense_optimizer_factory_and_class.learning_rate = 0.001 -dense_optimizer_factory_and_class.optimizer_name = "Adam" -dense_optimizer_factory_and_class.momentum = 0 -dense_optimizer_factory_and_class.weight_decay = 0 -dense_optimizer_factory_and_class.eps = 1e-8 -dense_optimizer_factory_and_class.betas = (0.95, 0.999) - -# sparse model optimizer -sparse_optimizer_factory_and_class.learning_rate = 0.001 -sparse_optimizer_factory_and_class.optimizer_name = "RowWiseAdagrad" -sparse_optimizer_factory_and_class.momentum = 0 -sparse_optimizer_factory_and_class.weight_decay = 0 -sparse_optimizer_factory_and_class.eps = 1e-8 -sparse_optimizer_factory_and_class.betas = (0.95, 0.999) - -# dataloader configs -make_train_test_dataloaders.batch_size = %batch_size -make_train_test_dataloaders.dataset_type = %dataset -make_train_test_dataloaders.train_split_percentage = 0.80 -make_train_test_dataloaders.new_path_prefix = "/home/linjianma" -make_train_test_dataloaders.num_workers = %num_workers -make_train_test_dataloaders.prefetch_factor = %prefetch_factor -make_train_test_dataloaders.num_blocks = 20 - -get_dataset.name = %dataset -get_dataset.new_path_prefix = "/home/linjianma" - -make_streaming_dataloader.batch_size = %batch_size -make_streaming_dataloader.num_workers = %num_workers -make_streaming_dataloader.prefetch_factor = %prefetch_factor - -# train eval loop variables -streaming_train_eval_loop.num_train_ts = 90 -streaming_train_eval_loop.output_trace = False -streaming_train_eval_loop.num_eval_batches = 500 -streaming_train_eval_loop.metric_log_frequency = 40 -streaming_train_eval_loop.checkpoint_frequency = 3 - - -# logger variables -MetricsLogger.tensorboard_log_path = "/home/linjianma/tensorboard/streaming_100b/run4/" -save_dmp_checkpoint.path = "/home/linjianma/ckpts/streaming_100b/" -# load_dmp_checkpoint.path = "/home/linjianma/ckpts/streaming_100b/20000/" diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/streaming_200b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/streaming_200b.gin deleted file mode 100644 index 872019962..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/streaming_200b.gin +++ /dev/null @@ -1,63 +0,0 @@ -batch_size = 64 -num_workers = 2 -prefetch_factor = 4 -dataset = "streaming-200b" - -# model parameters -make_model.dataset = %dataset - -# dense model optimizer -dense_optimizer_factory_and_class.learning_rate = 0.001 -dense_optimizer_factory_and_class.optimizer_name = "Adam" -dense_optimizer_factory_and_class.momentum = 0 -dense_optimizer_factory_and_class.weight_decay = 0 -dense_optimizer_factory_and_class.eps = 1e-8 -dense_optimizer_factory_and_class.betas = (0.95, 0.999) - -# sparse model optimizer -sparse_optimizer_factory_and_class.learning_rate = 0.001 -sparse_optimizer_factory_and_class.optimizer_name = "RowWiseAdagrad" -sparse_optimizer_factory_and_class.momentum = 0 -sparse_optimizer_factory_and_class.weight_decay = 0 -sparse_optimizer_factory_and_class.eps = 1e-8 -sparse_optimizer_factory_and_class.betas = (0.95, 0.999) - -# dataloader configs -make_train_test_dataloaders.batch_size = %batch_size -make_train_test_dataloaders.dataset_type = %dataset -make_train_test_dataloaders.train_split_percentage = 0.80 -make_train_test_dataloaders.new_path_prefix = "/home/linjianma" -make_train_test_dataloaders.num_workers = %num_workers -make_train_test_dataloaders.prefetch_factor = %prefetch_factor -make_train_test_dataloaders.num_blocks = 20 - -get_dataset.name = %dataset -get_dataset.new_path_prefix = "/home/linjianma" - -make_streaming_dataloader.batch_size = %batch_size -make_streaming_dataloader.num_workers = %num_workers -make_streaming_dataloader.prefetch_factor = %prefetch_factor - -# train loop variables -train_loop.num_epochs = 200 -train_loop.output_trace = False -train_loop.metric_log_frequency = 40 -train_loop.checkpoint_frequency = 4000 -train_loop.start_batch_idx = 0 - -# eval loop variables -eval_loop.metric_log_frequency = 40 - -# train eval loop variables -streaming_train_eval_loop.num_train_ts = 90 -streaming_train_eval_loop.output_trace = False -streaming_train_eval_loop.num_train_batches = 5000 -streaming_train_eval_loop.num_eval_batches = 200 -streaming_train_eval_loop.metric_log_frequency = 40 -streaming_train_eval_loop.checkpoint_frequency = 2000 - - -# logger variables -MetricsLogger.tensorboard_log_path = "/home/linjianma/tensorboard/streaming_200b/" -# save_dmp_checkpoint.path = "/home/linjianma/ckpts/streaming_200b/" -# load_dmp_checkpoint.path = "/home/linjianma/ckpts/streaming_400m/20000/" diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/streaming_400m.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/streaming_400m.gin deleted file mode 100644 index eba17bc23..000000000 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/streaming_400m.gin +++ /dev/null @@ -1,61 +0,0 @@ -batch_size = 64 -num_workers = 2 -prefetch_factor = 4 -dataset = "streaming-400m" - -# model parameters -make_model.dataset = %dataset - -# dense model optimizer -dense_optimizer_factory_and_class.learning_rate = 0.001 -dense_optimizer_factory_and_class.optimizer_name = "Adam" -dense_optimizer_factory_and_class.momentum = 0 -dense_optimizer_factory_and_class.weight_decay = 0 -dense_optimizer_factory_and_class.eps = 1e-8 -dense_optimizer_factory_and_class.betas = (0.95, 0.999) - -# sparse model optimizer -sparse_optimizer_factory_and_class.learning_rate = 0.001 -sparse_optimizer_factory_and_class.optimizer_name = "RowWiseAdagrad" -sparse_optimizer_factory_and_class.momentum = 0 -sparse_optimizer_factory_and_class.weight_decay = 0 -sparse_optimizer_factory_and_class.eps = 1e-8 -sparse_optimizer_factory_and_class.betas = (0.95, 0.999) - -# dataloader configs -make_train_test_dataloaders.batch_size = %batch_size -make_train_test_dataloaders.dataset_type = %dataset -make_train_test_dataloaders.train_split_percentage = 0.80 -make_train_test_dataloaders.new_path_prefix = "/home/linjianma" -make_train_test_dataloaders.num_workers = %num_workers -make_train_test_dataloaders.prefetch_factor = %prefetch_factor -make_train_test_dataloaders.num_blocks = 20 - -get_dataset.name = %dataset -get_dataset.new_path_prefix = "/home/linjianma" - -make_streaming_dataloader.batch_size = %batch_size -make_streaming_dataloader.num_workers = %num_workers -make_streaming_dataloader.prefetch_factor = %prefetch_factor - -# train loop variables -train_loop.num_epochs = 200 -train_loop.output_trace = False -train_loop.metric_log_frequency = 40 -train_loop.checkpoint_frequency = 4000 -train_loop.start_batch_idx = 0 - -# eval loop variables -eval_loop.metric_log_frequency = 40 - -# train eval loop variables -streaming_train_eval_loop.num_train_ts = 8 -streaming_train_eval_loop.output_trace = False -streaming_train_eval_loop.metric_log_frequency = 40 -streaming_train_eval_loop.checkpoint_frequency = 2000 - - -# logger variables -MetricsLogger.tensorboard_log_path = "/home/linjianma/tensorboard/streaming_400m/" -# save_dmp_checkpoint.path = "/home/linjianma/ckpts/streaming_400m/" -# load_dmp_checkpoint.path = "/home/linjianma/ckpts/streaming_400m/20000/" diff --git a/recommendation_v4/generative_recommenders/ops/benchmarks/hstu_attention_bench.py b/recommendation_v4/generative_recommenders/ops/benchmarks/hstu_attention_bench.py deleted file mode 100644 index cc7fbede7..000000000 --- a/recommendation_v4/generative_recommenders/ops/benchmarks/hstu_attention_bench.py +++ /dev/null @@ -1,406 +0,0 @@ -# pyre-strict -import os -from typing import List, Optional, Tuple - -import click -import pandas as pd -import torch - -# @manual=//triton:triton -import triton -from generative_recommenders.common import ( - apply_sampling, - blackwell_tlx_unavailable, - generate_sparse_seq_len, - HammerKernel, -) -from generative_recommenders.ops.cpp.cuda_hstu_attention import cuda_hstu_mha -from generative_recommenders.ops.hstu_attention import delta_hstu_mha, hstu_mha - -try: - from hammer.ops.ragged_hstu_attention import ragged_hstu_mha - from hammer.utils import HammerKernel as HammerKernel2 -except ImportError: - pass - - -def _get_kernel(provider: str) -> HammerKernel: - if provider == "triton": - return HammerKernel.TRITON - elif provider == "tlx": - return HammerKernel.TLX - elif provider == "pytorch": - return HammerKernel.PYTORCH - else: - raise ValueError(f"Unknown provider {provider}") - - -def _flops( - batch_size: int, - max_seqlen: int, - attn_dim: int, - hidden_dim: int, - nheads: int, - seq_offsets: torch.Tensor, - mode: str = "fwd", -) -> float: - assert mode in ["fwd", "bwd", "fwd_bwd"] - ratio = 2.0 # triangular masking - f1 = 0.0 - f2 = 0.0 - for i in range(batch_size): - seq_len = int((seq_offsets[i + 1] - seq_offsets[i]).item()) - # (QK^T), dQ = d(QK^T)K, dK^T = Q^Td(QK^T) - f1 += 2 * nheads * attn_dim * seq_len**2 // ratio - # (QK^T)V, d(QK^T) = dOV^T, dV = (QK^T)^TdO, - f2 += 2 * nheads * hidden_dim * seq_len**2 // ratio - if mode == "fwd": - return f1 + f2 # computes (QK^T) and (QK^T)V - elif mode == "bwd": - return 3 * f1 + 2 * f2 # computes (QK^T), dQ, dK, dV, d(QK^T) - else: - return 4 * f1 + 3 * f2 - - -@click.command() -@click.option( - "--batch-size", - type=int, - default=512, -) -@click.option("--heads", type=int, default=4) -@click.option("--attn-dim", type=int, default=128) -@click.option("--hidden-dim", type=int, default=128) -@click.option("--max-seq-len-log2", type=int, default=13) -@click.option("--data-type", type=str, default="bf16") -@click.option("--seq-sparsity", type=float, default=0.95) -@click.option("--has-delta-q", type=bool, default=False) -@click.option("--delta-size", type=int, default=256) -@click.option("--target-size", type=int, default=20) -@click.option("--bench-backward", type=bool, default=True) -@click.option("--bench-forward", type=bool, default=True) -@click.option("--bench-tlx", type=bool, default=False) -@click.option("--bench-pytorch", type=bool, default=False) -@click.option("--bench-ragged", type=bool, default=True) -@click.option("--report-flops", type=bool, default=False) -@click.option("--return-result", type=bool, default=False) -@click.option("--max-attn-len", type=int, default=0) -@click.option("--min-full-attn-seq-len", type=int, default=0) -@click.option("--contextual-seq-len", type=int, default=0) -@click.option("--sampling-alpha", type=float, default=2.0) -@click.option("--triton-enable-tma", type=bool, default=False) -@click.option("--dynamic-attn-scale", type=bool, default=False) -@click.option("--num-softmax-heads", type=int, default=0) -def main( # noqa: C901 - batch_size: int, - heads: int, - attn_dim: int, - hidden_dim: int, - max_seq_len_log2: int, - data_type: str, - seq_sparsity: float, - has_delta_q: bool, - delta_size: int, - target_size: int, - bench_backward: bool, - bench_forward: bool, - bench_tlx: bool, - bench_pytorch: bool, - bench_ragged: bool, - report_flops: bool, - return_result: bool, - max_attn_len: int, - min_full_attn_seq_len: int, - contextual_seq_len: int, - sampling_alpha: float, - triton_enable_tma: bool, - dynamic_attn_scale: bool, - num_softmax_heads: int, -) -> Optional[Tuple[List[triton.testing.Benchmark], List[pd.DataFrame]]]: - torch.backends.cudnn.allow_tf32 = True - torch.backends.cuda.matmul.allow_tf32 = True - if data_type == "fp32": - dtype = torch.float32 - elif data_type == "fp16": - dtype = torch.float16 - elif data_type == "bf16": - dtype = torch.bfloat16 - else: - raise ValueError(f"Unsupported data type: {data_type}.") - - line_vals = ["triton", "flash_cuda_jagged"] - line_names = ["triton", "flash_cuda_jagged"] - styles = [("blue", "-"), ("green", "-")] - if bench_pytorch: - line_vals.append("pytorch") - line_names.append("PyTorch") - styles.append(("green", "-")) - if bench_ragged: - line_vals.append("ragged") - line_names.append("ragged") - styles.append(("red", "-")) - if bench_tlx and not blackwell_tlx_unavailable[0]: - line_vals.append("tlx") - line_names.append("tlx") - styles.append(("cyan", "-")) - - bench_backward = False if has_delta_q else bench_backward - modes = [] - if bench_forward: - modes.append("fwd") - if bench_backward: - modes.append("bwd") - assert len(modes) > 0 - - configs: List[triton.testing.Benchmark] = [ - triton.testing.Benchmark( - x_names=["seq_len"], - x_vals=[2**i for i in range(8, max_seq_len_log2)], - line_arg="provider", - line_vals=line_vals, - line_names=line_names, - styles=styles, - ylabel="ms", - plot_name=f"hstu-attn-b{batch_size}-h{heads}-d{attn_dim}-v{hidden_dim}--sparsity{seq_sparsity}-{mode}-{dtype}-target{target_size}-mattn{max_attn_len}-full{min_full_attn_seq_len}-c{contextual_seq_len}-sl_alpha{sampling_alpha}-triton_tma{triton_enable_tma}-dynamic_scale{dynamic_attn_scale}-num_softmax_heads{num_softmax_heads}", - args={ - "batch_size": batch_size, - "heads": heads, - "attn_dim": attn_dim, - "hidden_dim": hidden_dim, - "dtype": dtype, - "mode": mode, - "seq_sparsity": seq_sparsity, - "has_delta_q": has_delta_q, - "delta_size": delta_size, - "target_size": target_size, - "bench_backward": bench_backward, - "report_flops": report_flops, - "max_attn_len": max_attn_len, - "min_full_attn_seq_len": min_full_attn_seq_len, - "contextual_seq_len": contextual_seq_len, - "sampling_alpha": sampling_alpha, - "triton_enable_tma": triton_enable_tma, - "dynamic_attn_scale": dynamic_attn_scale, - "num_softmax_heads": num_softmax_heads, - }, - ) - for mode in modes - ] - - @triton.testing.perf_report(configs) - def _bench_hstu_attention( - batch_size: int, - heads: int, - seq_len: int, - attn_dim: int, - hidden_dim: int, - mode: str, - provider: str, - dtype: torch.dtype, - seq_sparsity: float, - has_delta_q: bool, - delta_size: int, - target_size: int, - bench_backward: bool, - report_flops: bool, - max_attn_len: int, - min_full_attn_seq_len: int, - contextual_seq_len: int, - sampling_alpha: float, - triton_enable_tma: bool, - dynamic_attn_scale: bool, - num_softmax_heads: int, - ) -> float: - assert mode in ["fwd", "bwd"] - warmup = 25 - rep = 1000 - torch.manual_seed(1001) # for reproducibility - alpha = 1.0 / attn_dim - causal = True - lengths = generate_sparse_seq_len( - size=batch_size, - max_seq_len=seq_len, - sparsity=seq_sparsity, - device=torch.device("cuda"), - ) - lengths = apply_sampling(lengths, sampling_alpha, max_seq_len=seq_len) - if has_delta_q: - lengths = lengths + delta_size - num_targets = torch.ones_like(lengths) * delta_size - seq_len = seq_len + delta_size - else: - delta_size = 0 - num_targets = None - if target_size != 0: - num_targets = torch.randint( - 1, - target_size + 1, - (batch_size,), - device=lengths.device, - dtype=lengths.dtype, - ) - num_targets = torch.where( - num_targets > lengths, lengths, num_targets - ).to(torch.int32) - max_attn_len = max_attn_len if max_attn_len < seq_len else seq_len - seq_offsets = torch.zeros( - (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") - ) - seq_offsets[1:] = torch.cumsum(lengths, dim=0) - L = int(seq_offsets[-1].item()) - x = torch.empty( - (L, heads, attn_dim * 2 + hidden_dim), - dtype=dtype, - device=torch.device("cuda"), - ).uniform_(-0.01, 0.01) - q, k, v = torch.split(x, [attn_dim, attn_dim, hidden_dim], dim=-1) - delta_q = torch.empty( - (batch_size * delta_size, heads, attn_dim), - dtype=dtype, - device=torch.device("cuda"), - ).uniform_(-0.1, 0.1) - delta_x_offsets = torch.arange(0, delta_size, device=torch.device("cuda")) - delta_x_offsets = (seq_offsets[1:] - delta_size).view( - batch_size, 1 - ) + delta_x_offsets.view(1, delta_size) - delta_x_offsets = delta_x_offsets.view(-1) - attn_scale = torch.empty( - (L,), - dtype=torch.float32, - device=torch.device("cuda"), - ).uniform_(0.5, 1.0) - - if bench_backward: - q = q.requires_grad_(True) - k = k.requires_grad_(True) - v = v.requires_grad_(True) - assert provider in [ - "triton", - "pytorch", - "flash_cuda_jagged", - "flash_cuda", - "tlx", - "ragged", - ] - if has_delta_q: - fn = lambda: delta_hstu_mha( # noqa E731 - max_seq_len=seq_len, - alpha=alpha, - delta_q=delta_q, - k=k, - v=v, - seq_offsets=seq_offsets, - num_targets=num_targets, - kernel=_get_kernel(provider), - ) - else: - if provider == "flash_cuda_jagged": - fn = lambda: cuda_hstu_mha( # noqa E731 - q=q, - k=k, - v=v, - alpha=alpha, - causal=True, - seq_offsets=seq_offsets.to(torch.int32), - attn_scale=attn_scale if dynamic_attn_scale else None, - max_seq_len=seq_len, - max_attn_len=max_attn_len, - min_full_attn_seq_len=min_full_attn_seq_len, - contextual_seq_len=contextual_seq_len, - num_targets=num_targets, - sort_by_length=False, - num_softmax_heads=num_softmax_heads, - ) - elif provider == "flash_cuda": - q, k, v = [ - torch.randn( - batch_size, - seq_len, - heads, - attn_dim, - device="cuda", - dtype=dtype, - requires_grad=True, - ) - for _ in range(3) - ] - fn = lambda: cuda_hstu_mha( # noqa E731 - q=q, - k=k, - v=v, - alpha=alpha, - causal=True, - max_seq_len=seq_len, - max_attn_len=max_attn_len, - min_full_attn_seq_len=min_full_attn_seq_len, - contextual_seq_len=contextual_seq_len, - num_targets=num_targets, - sort_by_length=False, - num_softmax_heads=num_softmax_heads, - ) - elif provider == "ragged": - fn = lambda: ragged_hstu_mha( # noqa E731 - max_seq_len=seq_len, - alpha=alpha, - q=q, - k=k, - v=v, - seq_offsets=seq_offsets, - dropout_pr=0.0, - training=True, - invalid_attn_mask_type="lower_triangular", - num_targets=num_targets, - attn_scale=attn_scale if dynamic_attn_scale else None, - max_attn_len=max_attn_len, - contextual_seq_len=contextual_seq_len, - full_attn_size=min_full_attn_seq_len, - sort_by_length=True, - kernel=HammerKernel2.TRITON, - num_softmax_heads=num_softmax_heads, - ) - else: - fn = lambda: hstu_mha( # noqa E731 - max_seq_len=seq_len, - alpha=alpha, - q=q, - k=k, - v=v, - seq_offsets=seq_offsets, - causal=causal, - dropout_pr=0.0, - training=True, - num_targets=num_targets, - max_attn_len=max_attn_len, - contextual_seq_len=contextual_seq_len, - sort_by_length=True, - kernel=_get_kernel(provider), - enable_tma=triton_enable_tma, - ) - if mode == "bwd": - o = fn() - do = torch.randn_like(o) - fn = lambda: o.backward(do, retain_graph=True) # noqa E731 - ms = triton.testing.do_bench(fn, warmup=warmup, rep=rep) - all_flops = _flops( - batch_size, seq_len, attn_dim, hidden_dim, heads, seq_offsets, mode - ) - if has_delta_q: - all_flops = all_flops / seq_len * delta_size - if report_flops: - return all_flops / ms / 1e9 - else: - return ms - - df = _bench_hstu_attention.run( - print_data=True, - show_plots=False, - save_path="/tmp/" + os.environ["USER"], - return_df=return_result, - ) - - if return_result: - return configs, df - - -if __name__ == "__main__": - main() diff --git a/recommendation_v4/generative_recommenders/ops/cpp/benchmarks/concat_1d_jagged_jagged_bench.py b/recommendation_v4/generative_recommenders/ops/cpp/benchmarks/concat_1d_jagged_jagged_bench.py deleted file mode 100644 index 95c43853f..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/benchmarks/concat_1d_jagged_jagged_bench.py +++ /dev/null @@ -1,125 +0,0 @@ -# pyre-strict -from typing import List - -import click -import torch - -# @manual=//triton:triton -import triton -from hammer.ops.jagged import concat_1D_jagged_jagged - -# buck2 run @//mode/opt -c fbcode.nvcc_arch=h100 //generative_recommenders/ops/cpp/benchmarks:concat_1d_jagged_jagged_bench - -torch.ops.load_library("//generative_recommenders/ops/cpp:cpp_ops") -torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops") -torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops_cpu") - - -@click.command() -@click.option("--data-type", type=str, default="float32") -@click.option("--batch-size", type=int, default=512) -@click.option("--max-seq-len-log2", type=int, default=20) -@click.option("--seq-sparsity", type=float, default=0.8) -def main( - data_type: str, - batch_size: int, - max_seq_len_log2: int, - seq_sparsity: float, -) -> None: - if data_type == "float32": - dtype = torch.float32 - elif data_type == "float16": - dtype = torch.float16 - elif data_type == "bfloat16": - dtype = torch.bfloat16 - else: - raise ValueError(f"Unsupported data type: {data_type}.") - - configs: List[triton.testing.Benchmark] = [ - triton.testing.Benchmark( - x_names=["max_seq_len"], - x_vals=[2**i for i in range(6, max_seq_len_log2)], - line_arg="method", - line_vals=[ - "custom_cuda", - "hammer_pytorch", - ], - line_names=["Custom CUDA", "Hammer PyTorch"], - styles=[("green", "-"), ("orange", "--")], - ylabel="ms", - plot_name=f"concat_1d_jagged_jagged_batch{batch_size}_sparsity{seq_sparsity}_{data_type}", - args={ - "dtype": dtype, - "batch_size": batch_size, - "seq_sparsity": seq_sparsity, - }, - ) - ] - - @triton.testing.perf_report(configs) - def bench_concat_1d_jagged_jagged( - max_seq_len: int, - batch_size: int, - method: str, - dtype: torch.dtype, - seq_sparsity: float, - ) -> float: - warmup = 50 - rep = 500 - torch.manual_seed(1001) - - lengths_left = torch.randint( - 1, int(max_seq_len * seq_sparsity) + 1, (batch_size,), dtype=torch.int32 - ) - lengths_right = torch.randint( - 1, int(max_seq_len * seq_sparsity) + 1, (batch_size,), dtype=torch.int32 - ) - - total_left = int(lengths_left.sum().item()) - total_right = int(lengths_right.sum().item()) - - values_left = torch.randn(total_left, dtype=dtype) - values_right = torch.randn(total_right, dtype=dtype) - - offsets_left = torch.zeros( - (batch_size + 1,), dtype=lengths_left.dtype, device=lengths_left.device - ) - offsets_left[1:] = torch.cumsum(lengths_left.view(-1), dim=0) - offsets_right = torch.zeros( - (batch_size + 1,), dtype=lengths_right.dtype, device=lengths_right.device - ) - offsets_right[1:] = torch.cumsum(lengths_right.view(-1), dim=0) - max_seq_len_left = int(lengths_left.max().item()) - max_seq_len_right = int(lengths_right.max().item()) - - lengths_left = lengths_left.cuda() - lengths_right = lengths_right.cuda() - values_left = values_left.cuda() - values_right = values_right.cuda() - offsets_left = offsets_left.cuda() - offsets_right = offsets_right.cuda() - - if method == "custom_cuda": - fn = lambda: torch.ops.hstu.concat_1d_jagged_jagged( # noqa E731 - lengths_left, values_left, lengths_right, values_right - ) - elif method == "hammer_pytorch": - fn = lambda: concat_1D_jagged_jagged( # noqa E731 - max_seq_len_left, - offsets_left, - values_left, - max_seq_len_right, - offsets_right, - values_right, - ) - else: - raise ValueError(f"unknown method: {method}") - - ms = triton.testing.do_bench(fn, warmup=warmup, rep=rep) - return ms - - bench_concat_1d_jagged_jagged.run(print_data=True) - - -if __name__ == "__main__": - main() diff --git a/recommendation_v4/generative_recommenders/ops/cpp/benchmarks/jagged_transpose_1d_bench.py b/recommendation_v4/generative_recommenders/ops/cpp/benchmarks/jagged_transpose_1d_bench.py deleted file mode 100644 index 7806d6970..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/benchmarks/jagged_transpose_1d_bench.py +++ /dev/null @@ -1,117 +0,0 @@ -# pyre-strict -from typing import List - -import click -import torch - -# @manual=//triton:triton -import triton -from hammer.ops.jagged import jagged_transpose_1D - -# buck2 run @//mode/opt -c fbcode.nvcc_arch=h100 //generative_recommenders/ops/cpp/benchmarks:jagged_transpose_1d_bench - -torch.ops.load_library("//generative_recommenders/ops/cpp:cpp_ops") -torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops") -torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops_cpu") - - -@click.command() -@click.option("--data-type", type=str, default="float32") -@click.option("--size1", type=int, default=32) -@click.option("--size2", type=int, default=16) -@click.option("--max-len-log2", type=int, default=19) -@click.option("--seq-sparsity", type=float, default=0.8) -def main( - data_type: str, - size1: int, - size2: int, - max_len_log2: int, - seq_sparsity: float, -) -> None: - if data_type == "float32": - dtype = torch.float32 - elif data_type == "float16": - dtype = torch.float16 - elif data_type == "bfloat16": - dtype = torch.bfloat16 - else: - raise ValueError(f"Unsupported data type: {data_type}.") - - configs: List[triton.testing.Benchmark] = [ - triton.testing.Benchmark( - x_names=["max_len"], - x_vals=[2**i for i in range(4, max_len_log2)], - line_arg="method", - line_vals=[ - "custom_cuda", - "hammer_pytorch", - ], - line_names=["Custom CUDA", "Hammer PyTorch"], - styles=[("green", "-"), ("orange", "--")], - ylabel="ms", - plot_name=f"jagged_transpose_1d_size1_{size1}_size2_{size2}_sparsity{seq_sparsity}_{data_type}", - args={ - "dtype": dtype, - "size1": size1, - "size2": size2, - "seq_sparsity": seq_sparsity, - }, - ) - ] - - @triton.testing.perf_report(configs) - def bench_jagged_transpose_1d( - max_len: int, - size1: int, - size2: int, - method: str, - dtype: torch.dtype, - seq_sparsity: float, - ) -> float: - warmup = 50 - rep = 500 - torch.manual_seed(1001) - - lengths = torch.randint( - 1, int(max_len * seq_sparsity) + 1, (size1 * size2,), dtype=torch.int32 - ) - offsets = torch.zeros( - (size1 * size2 + 1,), dtype=lengths.dtype, device=lengths.device - ) - offsets[1:] = torch.cumsum(lengths.view(-1), dim=0) - - values = torch.randn(int(offsets[-1].item()), dtype=dtype) - - lengths = lengths.cuda() - offsets = offsets.cuda() - values = values.cuda() - - if method == "custom_cuda": - fn = lambda: torch.ops.hstu.jagged_transpose_1d( # noqa E731 - values=values, - offsets=offsets, - lengths=lengths, - max_len=max_len, - size1=size1, - size2=size2, - ) - elif method == "hammer_pytorch": - fn = lambda: jagged_transpose_1D( # noqa E731 - values=values, - offsets=offsets, - lengths=lengths, - max_len=max_len, - size1=size1, - size2=size2, - ) - else: - raise ValueError(f"unknown method: {method}") - - ms = triton.testing.do_bench(fn, warmup=warmup, rep=rep) - return ms - - bench_jagged_transpose_1d.run(print_data=True) - - -if __name__ == "__main__": - main() diff --git a/recommendation_v4/generative_recommenders/ops/cpp/benchmarks/replace_last_n_with_jagged_bench.py b/recommendation_v4/generative_recommenders/ops/cpp/benchmarks/replace_last_n_with_jagged_bench.py deleted file mode 100644 index a3f2483fa..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/benchmarks/replace_last_n_with_jagged_bench.py +++ /dev/null @@ -1,150 +0,0 @@ -# pyre-strict -from typing import List - -import click -import torch - -# @manual=//triton:triton -import triton -from hammer.ops.jagged import replace_last_n_with_jagged -from hammer.utils import HammerKernel - -# buck2 run @//mode/opt -c fbcode.nvcc_arch=h100 //generative_recommenders/ops/cpp/benchmarks:replace_last_n_with_jagged_bench - -torch.ops.load_library("//generative_recommenders/ops/cpp:cpp_ops") -torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops") -torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops_cpu") - - -@click.command() -@click.option("--data-type", type=str, default="float32") -@click.option("--batch-size", type=int, default=512) -@click.option("--embedding-dim", type=int, default=64) -@click.option("--max-seq-len-log2", type=int, default=16) -@click.option("--seq-sparsity", type=float, default=0.8) -def main( - data_type: str, - batch_size: int, - embedding_dim: int, - max_seq_len_log2: int, - seq_sparsity: float, -) -> None: - if data_type == "float32": - dtype = torch.float32 - elif data_type == "float16": - dtype = torch.float16 - elif data_type == "bfloat16": - dtype = torch.bfloat16 - else: - raise ValueError(f"Unsupported data type: {data_type}.") - - configs: List[triton.testing.Benchmark] = [ - triton.testing.Benchmark( - x_names=["max_seq_len"], - x_vals=[2**i for i in range(6, max_seq_len_log2)], - line_arg="method", - line_vals=[ - "custom_cuda", - "hammer_pytorch", - "hammer_triton", - ], - line_names=[ - "Custom CUDA", - "Hammer PyTorch", - "Hammer Triton", - ], - styles=[ - ("green", "-"), - ("orange", "--"), - ("purple", "-."), - ], - ylabel="ms", - plot_name=f"replace_last_n_with_jagged_batch{batch_size}_dim{embedding_dim}_sparsity{seq_sparsity}_{data_type}", - args={ - "dtype": dtype, - "batch_size": batch_size, - "embedding_dim": embedding_dim, - "seq_sparsity": seq_sparsity, - }, - ) - ] - - @triton.testing.perf_report(configs) - def bench_replace_last_n_with_jagged( - max_seq_len: int, - batch_size: int, - method: str, - dtype: torch.dtype, - embedding_dim: int, - seq_sparsity: float, - ) -> float: - warmup = 50 - rep = 500 - torch.manual_seed(1001) - - min_left_len = max(1, int(max_seq_len * seq_sparsity * 0.3)) - max_left_len = int(max_seq_len * seq_sparsity) - - lengths_left = torch.randint( - min_left_len, max_left_len + 1, (batch_size,), dtype=torch.int32 - ) - lengths_right = torch.randint( - 1, min_left_len + 1, (batch_size,), dtype=torch.int32 - ) - - lengths_right = torch.min(lengths_right, lengths_left) - - total_left = int(lengths_left.sum().item()) - total_right = int(lengths_right.sum().item()) - - values_left = torch.randn(total_left, embedding_dim, dtype=dtype) - values_right = torch.randn(total_right, embedding_dim, dtype=dtype) - - offsets_left = torch.zeros( - (batch_size + 1,), dtype=lengths_left.dtype, device=lengths_left.device - ) - offsets_left[1:] = torch.cumsum(lengths_left.view(-1), dim=0) - offsets_right = torch.zeros( - (batch_size + 1,), dtype=lengths_right.dtype, device=lengths_right.device - ) - offsets_right[1:] = torch.cumsum(lengths_right.view(-1), dim=0) - - lengths_left = lengths_left.cuda() - lengths_right = lengths_right.cuda() - values_left = values_left.cuda() - values_right = values_right.cuda() - offsets_left = offsets_left.cuda() - offsets_right = offsets_right.cuda() - - if method == "custom_cuda": - fn = lambda: torch.ops.hstu.replace_last_n_with_jagged( # noqa E731 - lengths_left, values_left, lengths_right, values_right - ) - elif method == "hammer_pytorch": - fn = lambda: replace_last_n_with_jagged( # noqa E731 - max_seq_len_left=max_seq_len, - offsets_left=offsets_left, - values_left=values_left, - offsets_right=offsets_right, - values_right=values_right, - ) - elif method == "hammer_triton": - fn = lambda: replace_last_n_with_jagged( # noqa E731 - max_seq_len_left=max_seq_len, - offsets_left=offsets_left, - values_left=values_left, - offsets_right=offsets_right, - values_right=values_right, - kernel=HammerKernel.TRITON, - ) - else: - raise ValueError(f"unknown method: {method}") - - ms = triton.testing.do_bench(fn, warmup=warmup, rep=rep) - return ms - - bench_replace_last_n_with_jagged.run(print_data=True) - - -if __name__ == "__main__": - main() diff --git a/recommendation_v4/generative_recommenders/ops/cpp/benchmarks/split_1d_jagged_jagged_bench.py b/recommendation_v4/generative_recommenders/ops/cpp/benchmarks/split_1d_jagged_jagged_bench.py deleted file mode 100644 index 4aaa9d77c..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/benchmarks/split_1d_jagged_jagged_bench.py +++ /dev/null @@ -1,116 +0,0 @@ -# pyre-strict -from typing import List - -import click -import torch - -# @manual=//triton:triton -import triton -from hammer.ops.jagged import split_1D_jagged_jagged - -# buck2 run @//mode/opt -c fbcode.nvcc_arch=h100 //generative_recommenders/ops/cpp/benchmarks:split_1d_jagged_jagged_bench - -torch.ops.load_library("//generative_recommenders/ops/cpp:cpp_ops") -torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops") -torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops_cpu") - - -@click.command() -@click.option("--data-type", type=str, default="float32") -@click.option("--batch-size", type=int, default=512) -@click.option("--max-seq-len-log2", type=int, default=20) -@click.option("--seq-sparsity", type=float, default=0.8) -def main( - data_type: str, - batch_size: int, - max_seq_len_log2: int, - seq_sparsity: float, -) -> None: - if data_type == "float32": - dtype = torch.float32 - elif data_type == "float16": - dtype = torch.float16 - elif data_type == "bfloat16": - dtype = torch.bfloat16 - else: - raise ValueError(f"Unsupported data type: {data_type}.") - - configs: List[triton.testing.Benchmark] = [ - triton.testing.Benchmark( - x_names=["max_seq_len"], - x_vals=[2**i for i in range(6, max_seq_len_log2)], - line_arg="method", - line_vals=[ - "custom_cuda", - "hammer_pytorch", - ], - line_names=["Custom CUDA", "Hammer PyTorch"], - styles=[("green", "-"), ("orange", "--")], - ylabel="ms", - plot_name=f"split_1d_jagged_jagged_batch{batch_size}_sparsity{seq_sparsity}_{data_type}", - args={ - "dtype": dtype, - "batch_size": batch_size, - "seq_sparsity": seq_sparsity, - }, - ) - ] - - @triton.testing.perf_report(configs) - def bench_split_1d_jagged_jagged( - max_seq_len: int, - batch_size: int, - method: str, - dtype: torch.dtype, - seq_sparsity: float, - ) -> float: - warmup = 50 - rep = 500 - torch.manual_seed(1001) - - lengths_left = torch.randint( - 1, int(max_seq_len * seq_sparsity) + 1, (batch_size,), dtype=torch.int32 - ) - lengths_right = torch.randint( - 1, int(max_seq_len * seq_sparsity) + 1, (batch_size,), dtype=torch.int32 - ) - - offsets_left = torch.zeros( - (batch_size + 1,), dtype=lengths_left.dtype, device=lengths_left.device - ) - offsets_left[1:] = torch.cumsum(lengths_left.view(-1), dim=0) - offsets_right = torch.zeros( - (batch_size + 1,), dtype=lengths_right.dtype, device=lengths_right.device - ) - offsets_right[1:] = torch.cumsum(lengths_right.view(-1), dim=0) - - combined_offsets = offsets_left + offsets_right - combined_values = torch.randn(int(combined_offsets[-1].item()), dtype=dtype) - - max_seq_len_combined = int((lengths_left + lengths_right).max().item()) - - lengths_left = lengths_left.cuda() - lengths_right = lengths_right.cuda() - combined_values = combined_values.cuda() - offsets_left = offsets_left.cuda() - offsets_right = offsets_right.cuda() - - if method == "custom_cuda": - fn = lambda: torch.ops.hstu.split_1d_jagged_jagged( # noqa E731 - lengths_left, lengths_right, combined_values - ) - elif method == "hammer_pytorch": - fn = lambda: split_1D_jagged_jagged( # noqa E731 - max_seq_len_combined, combined_values, offsets_left, offsets_right - ) - else: - raise ValueError(f"unknown method: {method}") - - ms = triton.testing.do_bench(fn, warmup=warmup, rep=rep) - return ms - - bench_split_1d_jagged_jagged.run(print_data=True) - - -if __name__ == "__main__": - main() diff --git a/recommendation_v4/generative_recommenders/ops/cpp/common.h b/recommendation_v4/generative_recommenders/ops/cpp/common.h deleted file mode 100644 index 1c4b43768..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/common.h +++ /dev/null @@ -1,60 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include -#include - -#define AT_DISPATCH_CASE_FLOATING_TYPES_AND4( \ - SCALARTYPE1, SCALARTYPE2, SCALARTYPE3, SCALARTYPE4, ...) \ - AT_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__) \ - AT_DISPATCH_CASE(SCALARTYPE1, __VA_ARGS__) \ - AT_DISPATCH_CASE(SCALARTYPE2, __VA_ARGS__) \ - AT_DISPATCH_CASE(SCALARTYPE3, __VA_ARGS__) \ - AT_DISPATCH_CASE(SCALARTYPE4, __VA_ARGS__) - -#define AT_DISPATCH_FLOATING_TYPES_AND4( \ - SCALARTYPE1, SCALARTYPE2, SCALARTYPE3, SCALARTYPE4, TYPE, NAME, ...) \ - AT_DISPATCH_SWITCH( \ - TYPE, \ - NAME, \ - AT_DISPATCH_CASE_FLOATING_TYPES_AND4( \ - SCALARTYPE1, SCALARTYPE2, SCALARTYPE3, SCALARTYPE4, __VA_ARGS__)) - -inline __attribute__((always_inline)) uint32_t -div_round_up(uint32_t a, uint32_t b) { - return (a + b - 1) / b; -}; - -inline __attribute__((always_inline)) uint32_t next_power_of_2(uint32_t n) { - n--; - n |= n >> 1; - n |= n >> 2; - n |= n >> 4; - n |= n >> 8; - n |= n >> 16; - n++; - return n; -} - -/* - * Because different .SO may include the same CUDA CUB kernels, this results in - * confusion, where libA may end up calling libB's cub kernel and causing - * failures when we static link libcudart_static.a. To avoid this, we annotate - * only the public functions and hide the rest. - */ -#define DLL_PUBLIC __attribute__((visibility("default"))) diff --git a/recommendation_v4/generative_recommenders/ops/cpp/complete_cumsum.cpp b/recommendation_v4/generative_recommenders/ops/cpp/complete_cumsum.cpp deleted file mode 100644 index 4ebd426d7..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/complete_cumsum.cpp +++ /dev/null @@ -1,44 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include - -#include "fbgemm_gpu/sparse_ops.h" // @manual - -namespace hstu { - -at::Tensor complete_cumsum_cpu(const at::Tensor& values) { - TORCH_CHECK(values.dim() == 1); - auto len = values.size(0); - const torch::Tensor index = at::range(0, len, at::kLong).cpu(); - auto output = fbgemm_gpu::asynchronous_complete_cumsum_cpu(values); - return output; -} - -at::Tensor complete_cumsum_meta(const at::Tensor& values) { - auto len = values.sym_size(0); - auto output = at::native::empty_meta_symint( - {len + 1}, - /*dtype=*/::std::make_optional(values.scalar_type()), - /*layout=*/::std::make_optional(values.layout()), - /*device=*/::std::make_optional(c10::Device(c10::kMeta)), - /*pin_memory=*/::std::nullopt); - return output; -} - -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/complete_cumsum.cu b/recommendation_v4/generative_recommenders/ops/cpp/complete_cumsum.cu deleted file mode 100644 index 06d1abdd6..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/complete_cumsum.cu +++ /dev/null @@ -1,51 +0,0 @@ -#include "common.h" - -#include - -namespace hstu { - -DLL_PUBLIC at::Tensor complete_cumsum_cuda(const at::Tensor& values) { - at::cuda::OptionalCUDAGuard device_guard; - device_guard.set_index(values.get_device()); - - TORCH_CHECK(values.numel() < std::numeric_limits::max()); - TORCH_CHECK(values.dim() == 1); - const auto values_contig = values.contiguous(); - - auto cumsum = at::empty({values_contig.numel() + 1}, values_contig.options()); - cumsum[0].zero_(); - - AT_DISPATCH_FLOATING_TYPES_AND4( - at::ScalarType::Int, - at::ScalarType::Long, - at::ScalarType::Half, - at::ScalarType::BFloat16, - values_contig.scalar_type(), - "complete_cumsum_cuda", - [&] { - size_t temp_storage_bytes = 0; - AT_CUDA_CHECK( - cub::DeviceScan::InclusiveSum( - nullptr, - temp_storage_bytes, - values_contig.data_ptr(), - cumsum.data_ptr() + 1, - values_contig.numel(), - at::cuda::getCurrentCUDAStream())); - auto temp_storage = at::empty( - {static_cast(temp_storage_bytes)}, - values_contig.options().dtype(at::kByte)); - AT_CUDA_CHECK( - cub::DeviceScan::InclusiveSum( - temp_storage.data_ptr(), - temp_storage_bytes, - values_contig.data_ptr(), - cumsum.data_ptr() + 1, - values_contig.numel(), - at::cuda::getCurrentCUDAStream())); - }); - - return cumsum; -} - -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/concat_1d_jagged_jagged.cpp b/recommendation_v4/generative_recommenders/ops/cpp/concat_1d_jagged_jagged.cpp deleted file mode 100644 index 51b313443..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/concat_1d_jagged_jagged.cpp +++ /dev/null @@ -1,111 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "fbgemm_gpu/sparse_ops.h" // @manual - -namespace hstu { - -template -void _concat_1d_jagged_jagged_cpu_kernel( - int32_t B, - const at::TensorAccessor& offsets_left, - const at::TensorAccessor& values_left, - const at::TensorAccessor& offsets_right, - const at::TensorAccessor& values_right, - at::TensorAccessor combined_values) { - for (auto b : c10::irange(B)) { - auto left_start = offsets_left[b]; - auto left_len = offsets_left[b + 1] - left_start; - auto right_start = offsets_right[b]; - auto right_len = offsets_right[b + 1] - right_start; - auto combined_start = left_start + right_start; - for (auto i = 0; i < left_len; ++i) { - combined_values[combined_start + i] = values_left[left_start + i]; - } - for (auto i = 0; i < right_len; ++i) { - combined_values[left_len + combined_start + i] = - values_right[right_start + i]; - } - } -} - -at::Tensor concat_1d_jagged_jagged_cpu( - const at::Tensor& lengths_left, - const at::Tensor& values_left, - const at::Tensor& lengths_right, - const at::Tensor& values_right) { - TORCH_INTERNAL_ASSERT(lengths_left.device().type() == at::DeviceType::CPU); - TORCH_INTERNAL_ASSERT(values_left.device().type() == at::DeviceType::CPU); - TORCH_INTERNAL_ASSERT(lengths_right.device().type() == at::DeviceType::CPU); - TORCH_INTERNAL_ASSERT(values_right.device().type() == at::DeviceType::CPU); - auto L = values_left.numel() + values_right.numel(); - TORCH_CHECK(L < std::numeric_limits::max()); - TORCH_CHECK(lengths_left.size(0) == lengths_right.size(0)); - auto B = lengths_left.size(0); - auto combined_values = at::empty({L}, values_left.options()); - if (L == 0) { - return combined_values; - } - const auto offsets_left = - fbgemm_gpu::asynchronous_complete_cumsum_cpu(lengths_left.view({-1})); - const auto offsets_right = - fbgemm_gpu::asynchronous_complete_cumsum_cpu(lengths_right.view({-1})); - AT_DISPATCH_INTEGRAL_TYPES( - lengths_left.scalar_type(), - "concat_1d_jagged_jagged_values_cpu_kernel_input1", - [&] { - using index_t = scalar_t; - AT_DISPATCH_ALL_TYPES_AND2( - at::ScalarType::BFloat16, - at::ScalarType::Half, - values_left.scalar_type(), - "concat_1d_jagged_jagged_values_cpu_kernel_input2", - [&] { - using val_t = scalar_t; - _concat_1d_jagged_jagged_cpu_kernel( - B, - offsets_left.accessor(), - values_left.accessor(), - offsets_right.accessor(), - values_right.accessor(), - combined_values.accessor()); - }); - }); - return combined_values; -} - -at::Tensor concat_1d_jagged_jagged_meta( - const at::Tensor& lengths_left, - const at::Tensor& values_left, - const at::Tensor& lengths_right, - const at::Tensor& values_right) { - auto L = values_left.numel() + values_right.numel(); - return at::native::empty_meta_symint( - {L}, - /*dtype=*/::std::make_optional(values_left.scalar_type()), - /*layout=*/::std::make_optional(values_left.layout()), - /*device=*/::std::make_optional(c10::Device(c10::kMeta)), - /*pin_memory=*/::std::nullopt); -} -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/concat_1d_jagged_jagged.cu b/recommendation_v4/generative_recommenders/ops/cpp/concat_1d_jagged_jagged.cu deleted file mode 100644 index 8eeae9d59..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/concat_1d_jagged_jagged.cu +++ /dev/null @@ -1,130 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include - -#include "common.h" -#include "fbgemm_gpu/sparse_ops.h" // @manual -#include "fbgemm_gpu/utils/fixed_divisor.cuh" // @manual - -namespace hstu { - -static constexpr int32_t kMaxThreads = 1024; - -template -__global__ -__launch_bounds__(kMaxThreads) void _concat_1d_jagged_jagged_cuda_kernel( - int32_t B, - const at::PackedTensorAccessor32 - offsets_left, - const at::PackedTensorAccessor32 - values_left, - const at::PackedTensorAccessor32 - offsets_right, - const at::PackedTensorAccessor32 - values_right, - at::PackedTensorAccessor32 - combined_values) { - for (auto b = blockIdx.x * blockDim.y + threadIdx.y; - b < static_cast(B); - b += gridDim.x * blockDim.y) { - auto left_start = offsets_left[b]; - auto left_len = offsets_left[b + 1] - left_start; - auto right_start = offsets_right[b]; - auto right_len = offsets_right[b + 1] - right_start; - auto combined_start = left_start + right_start; - for (auto i = threadIdx.x; i < static_cast(left_len + right_len); - i += blockDim.x) { - if (i < static_cast(left_len)) { - combined_values[combined_start + i] = values_left[left_start + i]; - } else { - combined_values[combined_start + i] = - values_right[right_start + i - left_len]; - } - } - } -} - -at::Tensor concat_1d_jagged_jagged_cuda( - const at::Tensor& lengths_left, - const at::Tensor& values_left, - const at::Tensor& lengths_right, - const at::Tensor& values_right) { - at::cuda::OptionalCUDAGuard device_guard; - device_guard.set_index(values_left.get_device()); - TORCH_INTERNAL_ASSERT(lengths_left.device().type() == at::DeviceType::CUDA); - TORCH_INTERNAL_ASSERT(values_left.device().type() == at::DeviceType::CUDA); - TORCH_INTERNAL_ASSERT(lengths_right.device().type() == at::DeviceType::CUDA); - TORCH_INTERNAL_ASSERT(values_right.device().type() == at::DeviceType::CUDA); - auto L = values_left.numel() + values_right.numel(); - TORCH_CHECK(L < std::numeric_limits::max()); - TORCH_CHECK(values_left.get_device() == lengths_left.get_device()); - TORCH_CHECK(values_left.get_device() == lengths_right.get_device()); - TORCH_CHECK(values_left.get_device() == values_right.get_device()); - auto B = lengths_left.size(0); - auto combined_values = at::empty({L}, values_left.options()); - if (L == 0) { - return combined_values; - } - const auto offsets_left = - fbgemm_gpu::asynchronous_complete_cumsum_gpu(lengths_left.view({-1})); - const auto offsets_right = - fbgemm_gpu::asynchronous_complete_cumsum_gpu(lengths_right.view({-1})); - // Optimized thread block configuration based on benchmark results - uint32_t B_blocks = 4; - dim3 threads(256, B_blocks); - auto blocks = div_round_up(B, B_blocks); - AT_DISPATCH_INTEGRAL_TYPES( - lengths_left.scalar_type(), - "concat_1d_jagged_jagged_values_cuda_kernel_input1", - [&] { - using index_t = scalar_t; - AT_DISPATCH_ALL_TYPES_AND2( - at::ScalarType::BFloat16, - at::ScalarType::Half, - values_left.scalar_type(), - "concat_1d_jagged_jagged_values_cuda_kernel_input2", - [&] { - using val_t = scalar_t; - _concat_1d_jagged_jagged_cuda_kernel - <<>>( - B, - offsets_left.packed_accessor32< - index_t, - 1, - at::RestrictPtrTraits>(), - values_left - .packed_accessor32(), - offsets_right.packed_accessor32< - index_t, - 1, - at::RestrictPtrTraits>(), - values_right - .packed_accessor32(), - combined_values.packed_accessor32< - val_t, - 1, - at::RestrictPtrTraits>()); - }); - }); - return combined_values; -} -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/cpp_ops.cpp b/recommendation_v4/generative_recommenders/ops/cpp/cpp_ops.cpp deleted file mode 100644 index 155cc7572..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/cpp_ops.cpp +++ /dev/null @@ -1,207 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -/* - * Because different .SO may include the same CUDA CUB kernels, this results in - * confusion, where libA may end up calling libB's cub kernel and causing - * failures when we static link libcudart_static.a. To avoid this, we annotate - * only the public functions and hide the rest. - */ -#define DLL_PUBLIC __attribute__((visibility("default"))) - -namespace hstu { -at::Tensor expand_1d_jagged_to_dense_cpu( - const at::Tensor& values, - const at::Tensor& offsets, - const int64_t max_len); - -at::Tensor expand_1d_jagged_to_dense_meta( - const at::Tensor& values, - const at::Tensor& offsets, - const c10::SymInt max_len); - -at::Tensor expand_1d_jagged_to_dense_cuda( - const at::Tensor& values, - const at::Tensor& offsets, - const int64_t max_len); - -at::Tensor complete_cumsum_cpu(const at::Tensor& values); - -at::Tensor complete_cumsum_cuda(const at::Tensor& values); - -at::Tensor complete_cumsum_meta(const at::Tensor& values); - -at::Tensor concat_1d_jagged_jagged_cpu( - const at::Tensor& lengths_left, - const at::Tensor& values_left, - const at::Tensor& lengths_right, - const at::Tensor& values_right); - -at::Tensor concat_1d_jagged_jagged_cuda( - const at::Tensor& lengths_left, - const at::Tensor& values_left, - const at::Tensor& lengths_right, - const at::Tensor& values_right); - -at::Tensor concat_1d_jagged_jagged_meta( - const at::Tensor& lengths_left, - const at::Tensor& values_left, - const at::Tensor& lengths_right, - const at::Tensor& values_right); - -std::tuple split_1d_jagged_jagged_cpu( - const at::Tensor& lengths_left, - const at::Tensor& lengths_right, - const at::Tensor& combined_values); - -std::tuple split_1d_jagged_jagged_cuda( - const at::Tensor& lengths_left, - const at::Tensor& lengths_right, - const at::Tensor& combined_values); - -std::tuple split_1d_jagged_jagged_meta( - const at::Tensor& lengths_left, - const at::Tensor& lengths_right, - const at::Tensor& combined_values); - -at::Tensor replace_last_n_with_jagged_cpu( - const at::Tensor& lengths_left, - const at::Tensor& values_left, - const at::Tensor& lengths_right, - const at::Tensor& values_right); - -at::Tensor replace_last_n_with_jagged_cuda( - const at::Tensor& lengths_left, - const at::Tensor& values_left, - const at::Tensor& lengths_right, - const at::Tensor& values_right); - -at::Tensor replace_last_n_with_jagged_meta( - const at::Tensor& lengths_left, - const at::Tensor& values_left, - const at::Tensor& lengths_right, - const at::Tensor& values_right); - -std::tuple jagged_transpose_1d_cpu( - const at::Tensor& values, - const at::Tensor& offsets, - const at::Tensor& lengths, - const int64_t max_len, - const int64_t size1, - const int64_t size2); - -std::tuple jagged_transpose_1d_cuda( - const at::Tensor& values, - const at::Tensor& offsets, - const at::Tensor& lengths, - const int64_t max_len, - const int64_t size1, - const int64_t size2); - -std::tuple jagged_transpose_1d_meta( - const at::Tensor& values, - const at::Tensor& offsets, - const at::Tensor& lengths, - const int64_t max_len, - const int64_t size1, - const int64_t size2); - -DLL_PUBLIC std::tuple sort_kv_pairs_meta( - const at::Tensor& keys, - const at::Tensor& values, - const std::optional& end_bit, - const bool descending = false) { - TORCH_CHECK( - keys.dtype() == at::kInt || keys.dtype() == at::kLong || - keys.dtype() == at::kByte || keys.dtype() == at::kShort); - TORCH_CHECK(keys.dim() == 1); - TORCH_CHECK(values.dim() == 1); - return {at::empty_like(keys), at::empty_like(values)}; -} - -std::tuple sort_kv_pairs_cuda( - const at::Tensor& keys, - const at::Tensor& values, - const std::optional& end_bit, - const bool descending = false); - -} // namespace hstu - -TORCH_LIBRARY_FRAGMENT(hstu, m) { - m.def( - "expand_1d_jagged_to_dense(Tensor values, Tensor offsets, SymInt max_len) -> Tensor"); - m.def( - "concat_1d_jagged_jagged(Tensor lengths_left, Tensor values_left, Tensor lengths_right, Tensor values_right) -> Tensor"); - m.def( - "split_1d_jagged_jagged(Tensor lengths_left, Tensor lengths_right, Tensor combined_values) -> (Tensor, Tensor)"); - m.def( - "replace_last_n_with_jagged(Tensor lengths_left, Tensor values_left, Tensor lengths_right, Tensor values_right) -> Tensor"); - m.def( - "jagged_transpose_1d(Tensor values, Tensor offsets, Tensor lengths, int max_len, int size1, int size2) -> (Tensor, Tensor, Tensor)"); - m.def("complete_cumsum(Tensor values) -> Tensor"); - m.def( - "sort_kv_pairs(Tensor keys, Tensor values, int? end_bit=None, bool descending=False) -> (Tensor, Tensor)"); -} - -TORCH_LIBRARY_IMPL(hstu, CPU, m) { - m.impl("expand_1d_jagged_to_dense", hstu::expand_1d_jagged_to_dense_cpu); - m.impl("concat_1d_jagged_jagged", hstu::concat_1d_jagged_jagged_cpu); - m.impl("split_1d_jagged_jagged", hstu::split_1d_jagged_jagged_cpu); - m.impl("replace_last_n_with_jagged", hstu::replace_last_n_with_jagged_cpu); - m.impl("jagged_transpose_1d", hstu::jagged_transpose_1d_cpu); - m.impl("complete_cumsum", hstu::complete_cumsum_cpu); -} - -TORCH_LIBRARY_IMPL(hstu, CUDA, m) { - m.impl("expand_1d_jagged_to_dense", hstu::expand_1d_jagged_to_dense_cuda); - m.impl("concat_1d_jagged_jagged", hstu::concat_1d_jagged_jagged_cuda); - m.impl("split_1d_jagged_jagged", hstu::split_1d_jagged_jagged_cuda); - m.impl("replace_last_n_with_jagged", hstu::replace_last_n_with_jagged_cuda); - m.impl("jagged_transpose_1d", hstu::jagged_transpose_1d_cuda); - m.impl("complete_cumsum", hstu::complete_cumsum_cuda); - m.impl( - "sort_kv_pairs", - torch::dispatch( - c10::DispatchKey::CUDA, TORCH_FN(hstu::sort_kv_pairs_cuda))); -} - -TORCH_LIBRARY_IMPL(hstu, Meta, m) { - m.impl("expand_1d_jagged_to_dense", hstu::expand_1d_jagged_to_dense_meta); - m.impl("concat_1d_jagged_jagged", hstu::concat_1d_jagged_jagged_meta); - m.impl("split_1d_jagged_jagged", hstu::split_1d_jagged_jagged_meta); - m.impl("replace_last_n_with_jagged", hstu::replace_last_n_with_jagged_meta); - m.impl("jagged_transpose_1d", hstu::jagged_transpose_1d_meta); - m.impl("complete_cumsum", hstu::complete_cumsum_meta); - m.impl( - "sort_kv_pairs", - torch::dispatch( - c10::DispatchKey::Meta, TORCH_FN(hstu::sort_kv_pairs_meta))); -} - -TORCH_LIBRARY_IMPL(hstu, Autograd, m) { - m.impl( - "expand_1d_jagged_to_dense", - torch::autograd::autogradNotImplementedFallback()); - m.impl("complete_cumsum", torch::autograd::autogradNotImplementedFallback()); -} diff --git a/recommendation_v4/generative_recommenders/ops/cpp/cuda_hstu_attention.py b/recommendation_v4/generative_recommenders/ops/cpp/cuda_hstu_attention.py deleted file mode 100644 index 0f9458c8b..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/cuda_hstu_attention.py +++ /dev/null @@ -1,193 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-strict - -from typing import Optional - -import torch -from generative_recommenders.ops.utils import is_sm100_plus - -try: - # We need to import the CUDA kernels after importing torch - import hstu._C # pyre-ignore [21] -except: - pass -try: - torch.ops.load_library( - "//generative_recommenders/fb/ultra/ops/blackwell/hstu_attention:hstu_flash_attention" - ) - torch.ops.load_library( - "//generative_recommenders/ops/cpp/hstu_attention:hstu_flash_attention" - ) -except: - pass - - -def cuda_hstu_mha( - max_seq_len: int, - alpha: float, - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - seq_offsets: Optional[torch.Tensor] = None, - causal: bool = False, - num_targets: Optional[torch.Tensor] = None, - attn_scale: Optional[torch.Tensor] = None, - max_attn_len: int = 0, - min_full_attn_seq_len: int = 0, - contextual_seq_len: int = 0, - q_descale: Optional[torch.Tensor] = None, - k_descale: Optional[torch.Tensor] = None, - v_descale: Optional[torch.Tensor] = None, - sort_by_length: bool = False, - deterministic: bool = False, - sm_margin: int = 0, - max_q_len: int = 0, - seq_offsets_q: Optional[torch.Tensor] = None, - num_softmax_heads: int = 0, - training: bool = True, - max_seq_len_tensor: Optional[torch.Tensor] = None, - contextual_seq_len_tensor: Optional[torch.Tensor] = None, - max_attn_len_tensor: Optional[torch.Tensor] = None, - min_full_attn_seq_len_tensor: Optional[torch.Tensor] = None, - num_groups: int = 1, - is_inference: bool = False, -) -> torch.Tensor: - """ - Arguments: - q, k, v: (batch_size, seqlen, nheads, headdim) or (total_seqlen, nheads, headdim) - """ - if is_sm100_plus() and not is_inference: - return torch.ops.bw_hstu.bw_hstu_mha( - max_seq_len, - alpha, - q, - k, - v, - seq_offsets, - causal, - num_targets, - attn_scale, - max_attn_len, - min_full_attn_seq_len, - contextual_seq_len, - q_descale, - k_descale, - v_descale, - sort_by_length, - deterministic, - sm_margin, - max_q_len, - seq_offsets_q, - max_seq_len_tensor, - contextual_seq_len_tensor, - max_attn_len_tensor, - min_full_attn_seq_len_tensor, - num_groups, - num_softmax_heads, - ) - else: - return cuda_hstu_mha_inference_wrapper( - max_seq_len, - alpha, - q, - k, - v, - seq_offsets, - causal, - num_targets, - attn_scale, - max_attn_len, - min_full_attn_seq_len, - contextual_seq_len, - q_descale, - k_descale, - v_descale, - sort_by_length, - deterministic, - sm_margin, - max_q_len, - seq_offsets_q, - num_softmax_heads, - training, - max_seq_len_tensor, - contextual_seq_len_tensor, - max_attn_len_tensor, - min_full_attn_seq_len_tensor, - num_groups, - ) - - -@torch.fx.wrap -def cuda_hstu_mha_inference_wrapper( - max_seq_len: int, - alpha: float, - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - seq_offsets: Optional[torch.Tensor] = None, - causal: bool = False, - num_targets: Optional[torch.Tensor] = None, - attn_scale: Optional[torch.Tensor] = None, - max_attn_len: int = 0, - min_full_attn_seq_len: int = 0, - contextual_seq_len: int = 0, - q_descale: Optional[torch.Tensor] = None, - k_descale: Optional[torch.Tensor] = None, - v_descale: Optional[torch.Tensor] = None, - sort_by_length: bool = False, - deterministic: bool = False, - sm_margin: int = 0, - max_q_len: int = 0, - seq_offsets_q: Optional[torch.Tensor] = None, - num_softmax_heads: int = 0, - training: bool = True, - max_seq_len_tensor: Optional[torch.Tensor] = None, - contextual_seq_len_tensor: Optional[torch.Tensor] = None, - max_attn_len_tensor: Optional[torch.Tensor] = None, - min_full_attn_seq_len_tensor: Optional[torch.Tensor] = None, - num_groups: int = 1, -) -> torch.Tensor: - attn_scale = attn_scale.to(torch.float32) if attn_scale is not None else attn_scale - - return torch.ops.hstu.hstu_mha( - max_seq_len, - alpha, - q, - k, - v, - seq_offsets, - causal, - num_targets, - attn_scale, - max_attn_len, - min_full_attn_seq_len, - contextual_seq_len, - q_descale, - k_descale, - v_descale, - sort_by_length, - deterministic, - sm_margin, - max_q_len, - seq_offsets_q, - num_softmax_heads, - training, - max_seq_len_tensor, - contextual_seq_len_tensor, - max_attn_len_tensor, - min_full_attn_seq_len_tensor, - num_groups, - ) diff --git a/recommendation_v4/generative_recommenders/ops/cpp/cuda_hstu_preprocess_and_attention.py b/recommendation_v4/generative_recommenders/ops/cpp/cuda_hstu_preprocess_and_attention.py deleted file mode 100644 index 2184ef2a5..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/cuda_hstu_preprocess_and_attention.py +++ /dev/null @@ -1,668 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -#!/usr/bin/env python3 - -# pyre-strict - -from typing import Optional, Tuple - -import torch -from generative_recommenders.ops.triton.triton_addmm import ( - maybe_triton_addmm_fwd, - triton_addmm_bwd, -) -from generative_recommenders.ops.triton.triton_layer_norm import ( - triton_weighted_layer_norm_bwd, -) -from generative_recommenders.ops.utils import copy_if_different_ptr, is_sm100_plus -from torch.nn import functional as F - -try: - from generative_recommenders.fb.ultra.ops.fp8.fp8_addmm import ( - fp8_rowwise_quantize_addmm, - ) - from generative_recommenders.fb.ultra.ops.fp8.layer_norm_quantization import ( - triton_weighted_layer_norm_quantization_fwd, - ) - from hammer.ops.triton.triton_apply_rope import ( - triton_apply_rope_bwd, - triton_apply_rope_fwd, - ) - - if is_sm100_plus(): - print("is sm100_plus architecture, loading hstu flash attention for blackwell") - torch.ops.load_library( - "//generative_recommenders/fb/ultra/ops/blackwell/hstu_attention:hstu_flash_attention" - ) - print("loading hstu flash attention for general architecture") - torch.ops.load_library( - "//generative_recommenders/ops/cpp/hstu_attention:hstu_flash_attention" - ) -except Exception as ex: - print(f"Library importing error when importing library: {ex}") - - -class _HSTUPreprocessAndAttentionFunction(torch.autograd.Function): - @staticmethod - # pyre-ignore [14] - def forward( - ctx, # pyre-ignore [2] - x: torch.Tensor, - norm_weight: torch.Tensor, - norm_bias: torch.Tensor, - norm_eps: float, - num_heads: int, - attn_dim: int, - hidden_dim: int, - uvqk_weight: torch.Tensor, - uvqk_bias: Optional[torch.Tensor], - max_seq_len: int, - seq_offsets: torch.Tensor, - alpha: float, - invalid_attn_mask_type: str, - num_targets: Optional[torch.Tensor], - rotary_weights: Optional[ - Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor] - ] = None, - attn_scale: Optional[torch.Tensor] = None, - recompute_uvqk_in_backward: bool = False, - recompute_normed_x_in_backward: bool = False, - contextual_seq_len: int = 0, - sort_by_length: bool = False, - max_attn_len: Optional[int] = None, - full_attn_size: Optional[int] = None, - silu_u: bool = True, - fp8_in_addmm_fwd: bool = False, - num_softmax_heads: int = 0, - ) -> Tuple[torch.Tensor, torch.Tensor]: - max_attn_len = max_attn_len or 0 - full_attn_size = full_attn_size or 0 - normed_x, x_mean, x_rstd, BLOCK_D, x_scale, normed_x_fp8 = ( - triton_weighted_layer_norm_quantization_fwd( - x=x, - weight=norm_weight, - bias=norm_bias, - eps=norm_eps, - quantize_output=fp8_in_addmm_fwd, - ) - ) - # When silu_u is False and we want to recompute in backward, we split the weight - # for u and vqk separately during training to compute them independently. - # This avoids needing to clone u (which would otherwise keep the whole uvqk alive). - if not silu_u and recompute_uvqk_in_backward: - # Split the weights/biases to compute u and vqk separately - u_weight, vqk_weight = uvqk_weight.split( - [ - hidden_dim * num_heads, - hidden_dim * num_heads - + attn_dim * num_heads - + attn_dim * num_heads, - ], - dim=1, - ) - if uvqk_bias is not None: - u_bias, vqk_bias = uvqk_bias.split( - [ - hidden_dim * num_heads, - hidden_dim * num_heads - + attn_dim * num_heads - + attn_dim * num_heads, - ], - dim=0, - ) - else: - u_bias, vqk_bias = None, None - if fp8_in_addmm_fwd: - assert x_scale is not None and normed_x_fp8 is not None - u = fp8_rowwise_quantize_addmm( - x=normed_x, - x_fp8=normed_x_fp8, - w=u_weight, - y=u_bias, - x_scale=x_scale, - custom_kernel=False, - is_inference=False, - ).contiguous() - vqk = fp8_rowwise_quantize_addmm( - x=normed_x, - x_fp8=normed_x_fp8, - w=vqk_weight, - y=vqk_bias, - x_scale=x_scale, - custom_kernel=False, - is_inference=False, - ).contiguous() - else: - u = maybe_triton_addmm_fwd(normed_x, u_weight, u_bias).contiguous() - vqk = maybe_triton_addmm_fwd( - normed_x, vqk_weight, vqk_bias - ).contiguous() - v, q, k = vqk.split( - [ - hidden_dim * num_heads, - attn_dim * num_heads, - attn_dim * num_heads, - ], - dim=1, - ) - # uvqk is not used since we split the computation, but we need it - # for saving in case recompute_uvqk_in_backward is False in a - # different code path. Set to None to satisfy type checker. - uvqk = None - else: - if fp8_in_addmm_fwd: - assert ( - x_scale is not None - and normed_x_fp8 is not None - and uvqk_bias is not None - ) - uvqk = fp8_rowwise_quantize_addmm( - x=normed_x, - x_fp8=normed_x_fp8, - w=uvqk_weight, - y=uvqk_bias, - x_scale=x_scale, - custom_kernel=False, - is_inference=False, - ).contiguous() - else: - uvqk = maybe_triton_addmm_fwd( - normed_x, uvqk_weight, uvqk_bias - ).contiguous() - u, v, q, k = uvqk.split( - [ - hidden_dim * num_heads, - hidden_dim * num_heads, - attn_dim * num_heads, - attn_dim * num_heads, - ], - dim=1, - ) - if silu_u: - u = F.silu(u) - if rotary_weights is not None: - q_cos_weights = rotary_weights[0] - q_sin_weights = rotary_weights[1] - k_cos_weights = rotary_weights[2] - k_sin_weights = rotary_weights[3] - _q = triton_apply_rope_fwd( - x=q.view(-1, num_heads, attn_dim), - N=max_seq_len, - seq_offsets=seq_offsets, - cos_rope=q_cos_weights, - sin_rope=q_sin_weights, - ).view(-1, num_heads * attn_dim) - _k = triton_apply_rope_fwd( - x=k.view(-1, num_heads, attn_dim), - N=max_seq_len, - seq_offsets=seq_offsets, - cos_rope=k_cos_weights, - sin_rope=k_sin_weights, - ).view(-1, num_heads * attn_dim) - copy_if_different_ptr(q, _q) - copy_if_different_ptr(k, _k) - q = q.view(-1, num_heads, attn_dim) - k = k.view(-1, num_heads, attn_dim) - v = v.view(-1, num_heads, hidden_dim) - if is_sm100_plus(): - out, softmax_lse = torch.ops.bw_hstu.bw_hstu_mha_fwd( - max_seq_len, - alpha, - q, - k, - v, - seq_offsets, - True, # causal - num_targets, - attn_scale, - max_attn_len, - full_attn_size, - contextual_seq_len, - None, # q_descale - None, # k_descale - None, # v_descale - 0, # sm_margin - max_seq_len, # max_q_len, - None, # seq_offsets_q, - None, # max_seq_len_tensor, - None, # contextual_seq_len_tensor, - None, # max_attn_len_tensor, - None, # min_full_attn_seq_len_tensor, - 1, # num_groups - num_softmax_heads, # num_softmax_heads - ) - else: - out, softmax_lse = torch.ops.hstu.hstu_mha_fwd( - max_seq_len, - alpha, - q, - k, - v, - seq_offsets, - True, # causal - num_targets, - attn_scale, - max_attn_len, - full_attn_size, - contextual_seq_len, - None, # q_descale - None, # k_descale - None, # v_descale - 0, # sm_margin - 0, # max_q_len, - None, # seq_offsets_q, - num_softmax_heads, # num_softmax_heads, - ) - # update ctx - saved_tensors = [ - x, - norm_weight, - norm_bias, - x_mean, - x_rstd, - uvqk_weight, - seq_offsets, - out, - ] - if num_softmax_heads > 0: - saved_tensors.append(softmax_lse) - if num_targets is not None: - saved_tensors.append(num_targets) - if attn_scale is not None: - saved_tensors.append(attn_scale) - if not recompute_normed_x_in_backward: - saved_tensors.append(normed_x) - if recompute_uvqk_in_backward: - if uvqk_bias is not None: - saved_tensors.append(uvqk_bias) - if fp8_in_addmm_fwd: - saved_tensors.append(x_scale) # pyre-ignore - saved_tensors.append(normed_x_fp8) # pyre-ignore - else: - saved_tensors.append(uvqk) - if rotary_weights is not None: - saved_tensors.append(rotary_weights[0]) - saved_tensors.append(rotary_weights[1]) - saved_tensors.append(rotary_weights[2]) - saved_tensors.append(rotary_weights[3]) - ctx.save_for_backward(*saved_tensors) - ctx.alpha = alpha - ctx.invalid_attn_mask_type = invalid_attn_mask_type - ctx.has_multiple_targets = num_targets is not None - ctx.has_rotary_weights = rotary_weights is not None - ctx.has_attn_scale = attn_scale is not None - ctx.max_seq_len = max_seq_len - ctx.max_attn_len = max_attn_len - ctx.full_attn_size = full_attn_size - ctx.recompute_normed_x_in_backward = recompute_normed_x_in_backward - ctx.recompute_uvqk_in_backward = recompute_uvqk_in_backward - ctx.hidden_dim = hidden_dim - ctx.attn_dim = attn_dim - ctx.num_heads = num_heads - ctx.has_uvqk_bias = uvqk_bias is not None - ctx.uvqk_bias_1d = uvqk_bias.dim() == 1 if uvqk_bias is not None else False - ctx.norm_eps = norm_eps - ctx.norm_BLOCK_D = BLOCK_D - ctx.contextual_seq_len = contextual_seq_len - ctx.sort_by_length = sort_by_length - ctx.silu_u = silu_u - ctx.fp8_in_addmm_fwd = fp8_in_addmm_fwd - ctx.num_softmax_heads = num_softmax_heads - return u, out - - @staticmethod - # pyre-ignore[14] - def backward( - ctx, # pyre-ignore[2] - _du: torch.Tensor, - dout: torch.Tensor, - ) -> Tuple[ - torch.Tensor, # d_x - torch.Tensor, # d_norm_weight - torch.Tensor, # d_norm_bias - None, - None, - None, - None, - torch.Tensor, # d_uvqk_weight - torch.Tensor, # d_uvqk_bias - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - ]: - x, norm_weight, norm_bias, x_mean, x_rstd, uvqk_weight, seq_offsets, out = ( - ctx.saved_tensors[:8] - ) - idx = 8 - if ctx.num_softmax_heads > 0: - softmax_lse = ctx.saved_tensors[idx] - idx += 1 - else: - softmax_lse = None - if ctx.has_multiple_targets: - num_targets = ctx.saved_tensors[idx] - idx += 1 - else: - num_targets = None - if ctx.has_attn_scale: - attn_scale = ctx.saved_tensors[idx] - idx += 1 - else: - attn_scale = None - if ctx.recompute_normed_x_in_backward: - normed_x, _, _, _, _, _ = triton_weighted_layer_norm_quantization_fwd( - x=x, - weight=norm_weight, - bias=norm_bias, - eps=ctx.norm_eps, - mean=x_mean, - rstd=x_rstd, - quantize_output=ctx.fp8_in_addmm_fwd, - ) - else: - normed_x = ctx.saved_tensors[idx] - idx += 1 - if ctx.recompute_uvqk_in_backward: - if ctx.has_uvqk_bias: - uvqk_bias = ctx.saved_tensors[idx] - idx += 1 - else: - uvqk_bias = None - if not ctx.silu_u: - # When silu_u is False, we only recompute vqk (not u) - # Split the weights/biases to extract vqk portion - _, vqk_weight = uvqk_weight.split( - [ - ctx.hidden_dim * ctx.num_heads, - ctx.hidden_dim * ctx.num_heads - + ctx.attn_dim * ctx.num_heads - + ctx.attn_dim * ctx.num_heads, - ], - dim=1, - ) - vqk_bias = None - if ctx.has_uvqk_bias: - _, vqk_bias = uvqk_bias.split( - [ - ctx.hidden_dim * ctx.num_heads, - ctx.hidden_dim * ctx.num_heads - + ctx.attn_dim * ctx.num_heads - + ctx.attn_dim * ctx.num_heads, - ], - dim=0, - ) - if ctx.fp8_in_addmm_fwd: - x_scale, normed_x_fp8 = ctx.saved_tensors[idx : idx + 2] - vqk = fp8_rowwise_quantize_addmm( - x=normed_x, - x_fp8=normed_x_fp8, - w=vqk_weight, - y=vqk_bias, - x_scale=x_scale, - custom_kernel=False, - is_inference=False, - ) - idx += 2 - else: - vqk = maybe_triton_addmm_fwd( - normed_x, vqk_weight, vqk_bias - ).contiguous() - # Split vqk into v, q, k components - v, q, k = vqk.split( - [ - ctx.hidden_dim * ctx.num_heads, - ctx.attn_dim * ctx.num_heads, - ctx.attn_dim * ctx.num_heads, - ], - dim=1, - ) - u = None - else: - # When silu_u is True, we recompute uvqk (all components) - if ctx.fp8_in_addmm_fwd: - x_scale, normed_x_fp8 = ctx.saved_tensors[idx : idx + 2] - uvqk = fp8_rowwise_quantize_addmm( - x=normed_x, - x_fp8=normed_x_fp8, - w=uvqk_weight, - y=uvqk_bias, - x_scale=x_scale, - custom_kernel=False, - is_inference=False, - ) - idx += 2 - else: - uvqk = maybe_triton_addmm_fwd( - normed_x, uvqk_weight, uvqk_bias - ).contiguous() - # Split uvqk into u, v, q, k components - u, v, q, k = uvqk.split( - [ - ctx.hidden_dim * ctx.num_heads, - ctx.hidden_dim * ctx.num_heads, - ctx.attn_dim * ctx.num_heads, - ctx.attn_dim * ctx.num_heads, - ], - dim=1, - ) - else: - uvqk = ctx.saved_tensors[idx] - idx += 1 - # Split saved uvqk into u, v, q, k components - u, v, q, k = uvqk.split( - [ - ctx.hidden_dim * ctx.num_heads, - ctx.hidden_dim * ctx.num_heads, - ctx.attn_dim * ctx.num_heads, - ctx.attn_dim * ctx.num_heads, - ], - dim=1, - ) - if ctx.has_rotary_weights: - q_cos_weights, q_sin_weights, k_cos_weights, k_sin_weights = ( - ctx.saved_tensors[idx : idx + 4] - ) - idx += 4 - else: - q_cos_weights, q_sin_weights, k_cos_weights, k_sin_weights = ( - None, - None, - None, - None, - ) - - duvqk = torch.empty( - [ - x.size(0), - ctx.hidden_dim * ctx.num_heads * 2 + ctx.attn_dim * ctx.num_heads * 2, - ], - device=x.device, - dtype=x.dtype, - ) - du, dv, dq, dk = duvqk.split( - [ - ctx.hidden_dim * ctx.num_heads, - ctx.hidden_dim * ctx.num_heads, - ctx.attn_dim * ctx.num_heads, - ctx.attn_dim * ctx.num_heads, - ], - dim=1, - ) - q = q.view(-1, ctx.num_heads, ctx.attn_dim) - k = k.view(-1, ctx.num_heads, ctx.attn_dim) - v = v.view(-1, ctx.num_heads, ctx.hidden_dim) - dq = dq.view(-1, ctx.num_heads, ctx.attn_dim) - dk = dk.view(-1, ctx.num_heads, ctx.attn_dim) - dv = dv.view(-1, ctx.num_heads, ctx.hidden_dim) - if ( - ctx.recompute_uvqk_in_backward and ctx.has_rotary_weights - ): # recompute ROPE on qk - q = triton_apply_rope_fwd( - x=q, - N=ctx.max_seq_len, - seq_offsets=seq_offsets, - cos_rope=q_cos_weights, - sin_rope=q_sin_weights, - ) - k = triton_apply_rope_fwd( - x=k, - N=ctx.max_seq_len, - seq_offsets=seq_offsets, - cos_rope=k_cos_weights, - sin_rope=k_sin_weights, - ) - dq = dq.view(-1, ctx.num_heads, ctx.attn_dim) - dk = dk.view(-1, ctx.num_heads, ctx.attn_dim) - dv = dv.view(-1, ctx.num_heads, ctx.hidden_dim) - # Note: the two operations below update duvqk in place - if is_sm100_plus(): - _dq, _dk, _dv = torch.ops.bw_hstu.bw_hstu_mha_bwd( - ctx.max_seq_len, - ctx.alpha, - dout, - q, - k, - v, - dq, - dk, - dv, - seq_offsets, - True, # causal - num_targets, - attn_scale, - ctx.max_attn_len, - ctx.full_attn_size, - ctx.contextual_seq_len, - ctx.sort_by_length, - False, # deterministic - 0, # sm_margin - ctx.max_seq_len, # max_q_len, - None, # seq_offsets_q, - None, # max_seq_len_tensor, - None, # contextual_seq_len_tensor, - None, # max_attn_len_tensor, - None, # min_full_attn_seq_len_tensor, - 1, # num_groups - ctx.num_softmax_heads, # num_softmax_heads - out, # out - softmax_lse, # lse - ) - else: - _dq, _dk, _dv = torch.ops.hstu.hstu_mha_bwd( - ctx.max_seq_len, - ctx.alpha, - dout, - q, - k, - v, - dq, - dk, - dv, - out, - seq_offsets, - True, # causal - num_targets, - attn_scale, - ctx.max_attn_len, - ctx.full_attn_size, - ctx.contextual_seq_len, - ctx.sort_by_length, - False, # deterministic - 0, # sm_margin - 0, # max_q_len, - None, # seq_offsets_q, - ctx.num_softmax_heads, # num_softmax_heads, - softmax_lse, - ) - if ctx.has_rotary_weights: - _dq = triton_apply_rope_bwd( - grad=_dq, - N=ctx.max_seq_len, - seq_offsets=seq_offsets, - cos_rope=q_cos_weights, - sin_rope=q_sin_weights, - ) - _dk = triton_apply_rope_bwd( - grad=_dk, - N=ctx.max_seq_len, - seq_offsets=seq_offsets, - cos_rope=k_cos_weights, - sin_rope=k_sin_weights, - ) - copy_if_different_ptr(dq, _dq) - copy_if_different_ptr(dk, _dk) - copy_if_different_ptr(dv, _dv) - if ctx.silu_u: - torch.ops.aten.silu_backward(_du, u, grad_input=du) - else: - copy_if_different_ptr(du, _du) - d_normed_x, d_uvqk_weight, d_uvqk_bias = triton_addmm_bwd( - x=normed_x, - w=uvqk_weight, - dz=duvqk, - is_y_1d=ctx.uvqk_bias_1d and ctx.has_uvqk_bias, - ) - d_x, d_norm_weight, d_norm_bias = triton_weighted_layer_norm_bwd( - dy=d_normed_x, - x=x, - weight=norm_weight, - bias=norm_bias, - mean=x_mean, - rstd=x_rstd, - learnable=True, - eps=ctx.norm_eps, - BLOCK_D=ctx.norm_BLOCK_D, - ) - # pyre-ignore[7] - return ( - d_x, - d_norm_weight, - d_norm_bias, - None, - None, - None, - None, - d_uvqk_weight, - d_uvqk_bias if ctx.has_uvqk_bias else None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - ) diff --git a/recommendation_v4/generative_recommenders/ops/cpp/expand_1d_jagged_to_dense.cpp b/recommendation_v4/generative_recommenders/ops/cpp/expand_1d_jagged_to_dense.cpp deleted file mode 100644 index 4730078e6..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/expand_1d_jagged_to_dense.cpp +++ /dev/null @@ -1,97 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace hstu { - -template -void expand_1d_jagged_to_dense_cpu_kernel_( - int64_t B, - int64_t max_len, - const at::TensorAccessor& values, - const at::TensorAccessor& offsets, - at::TensorAccessor output) { - for (auto i : c10::irange(B)) { - int64_t begin = offsets[i]; - int64_t end = offsets[i + 1]; - if (end - begin == 0) { - for (int64_t j : c10::irange(max_len)) { - output[i][j] = 0; - continue; - } - } else { - int64_t j = 0; - for (; j < std::min(end - begin, max_len); ++j) { - output[i][j] = values[begin + j]; - } - for (; j < max_len; ++j) { - output[i][j] = values[end - 1]; - } - } - } // for each i -} - -at::Tensor expand_1d_jagged_to_dense_cpu( - const at::Tensor& values, - const at::Tensor& offsets, - const int64_t max_len) { - TORCH_INTERNAL_ASSERT(values.device().type() == at::DeviceType::CPU); - TORCH_INTERNAL_ASSERT(offsets.device().type() == at::DeviceType::CPU); - TORCH_CHECK(values.numel() < std::numeric_limits::max()); - TORCH_CHECK(max_len >= 0); - auto B = offsets.size(0) - 1; - auto output = at::empty({B, max_len}, values.options()); - if (values.numel() == 0 || max_len == 0) { - return output; - } - AT_DISPATCH_ALL_TYPES_AND2( - at::ScalarType::BFloat16, - at::ScalarType::Half, - values.scalar_type(), - "expand_1d_jagged_to_dense_cpu_input1", - [&] { - using val_t = scalar_t; - AT_DISPATCH_INTEGRAL_TYPES( - offsets.scalar_type(), "expand_1d_jagged_to_dense_cpu_input2", [&] { - using index_t = scalar_t; - expand_1d_jagged_to_dense_cpu_kernel_( - B, - max_len, - values.accessor(), - offsets.accessor(), - output.accessor()); - }); - }); - return output; -} - -at::Tensor expand_1d_jagged_to_dense_meta( - const at::Tensor& values, - const at::Tensor& offsets, - const c10::SymInt max_len) { - auto B = offsets.sym_size(0) - 1; - auto output = at::empty_symint({B, max_len}, values.options()); - return output; -} - -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/expand_1d_jagged_to_dense.cu b/recommendation_v4/generative_recommenders/ops/cpp/expand_1d_jagged_to_dense.cu deleted file mode 100644 index aa3678d2b..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/expand_1d_jagged_to_dense.cu +++ /dev/null @@ -1,103 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include - -#include "common.h" - -static constexpr int32_t kMaxThreads = 1024; - -namespace hstu { - -template -__global__ -__launch_bounds__(kMaxThreads) void expand_1d_jagged_to_dense_cuda_kernel_( - int64_t B, - int64_t max_len, - const at::PackedTensorAccessor32 values, - const at::PackedTensorAccessor32 offsets, - at::PackedTensorAccessor32 output) { - int64_t b = blockIdx.y; - int64_t begin = offsets[b]; - int64_t i = blockIdx.x * blockDim.x + threadIdx.x; - int64_t end = offsets[b + 1]; - if (end - begin == 0) { - if (i < max_len) { - output[b][i] = 0; - } - } else { - if (i < std::min(end - begin, max_len)) { - output[b][i] = values[i + begin]; - } else if (i < max_len) { - output[b][i] = values[end - 1]; - } - } -} - -at::Tensor expand_1d_jagged_to_dense_cuda( - const at::Tensor& values, - const at::Tensor& offsets, - const int64_t max_len) { - at::cuda::OptionalCUDAGuard device_guard; - device_guard.set_index(values.get_device()); - TORCH_INTERNAL_ASSERT(values.device().type() == at::DeviceType::CUDA); - TORCH_INTERNAL_ASSERT(offsets.device().type() == at::DeviceType::CUDA); - TORCH_CHECK(values.numel() < std::numeric_limits::max()); - TORCH_CHECK(values.get_device() == offsets.get_device()); - TORCH_CHECK(max_len >= 0); - auto B = offsets.size(0) - 1; - auto output = at::empty({B, max_len}, values.options()); - if (values.numel() == 0 || max_len == 0) { - return output; - } - uint32_t nthreads_per_block = max_len > 64 ? 64 : max_len; - dim3 grid_size = dim3(div_round_up(max_len, nthreads_per_block), B); - AT_DISPATCH_ALL_TYPES_AND2( - at::ScalarType::BFloat16, - at::ScalarType::Half, - values.scalar_type(), - "expand_1d_jagged_to_dense_cuda_input1", - [&] { - using val_t = scalar_t; - AT_DISPATCH_INTEGRAL_TYPES( - offsets.scalar_type(), - "expand_1d_jagged_to_dense_cuda_input2", - [&] { - using index_t = scalar_t; - expand_1d_jagged_to_dense_cuda_kernel_<<< - grid_size, - nthreads_per_block, - 0, - at::cuda::getCurrentCUDAStream()>>>( - B, - max_len, - values.packed_accessor32(), - offsets - .packed_accessor32(), - output.packed_accessor32()); - C10_CUDA_KERNEL_LAUNCH_CHECK(); - }); - }); - - return output; -} - -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/copy_sm90_bulk_reduce.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/copy_sm90_bulk_reduce.h deleted file mode 100644 index a22ae7745..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/copy_sm90_bulk_reduce.h +++ /dev/null @@ -1,66 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/****************************************************************************** - * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, - *Pradeep Ramani, Tri Dao. - ******************************************************************************/ - -#pragma once - -#include - -namespace cute { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -struct SM90_BULK_REDUCE_ADD { - CUTE_HOST_DEVICE static void - copy(float const* smem_ptr, float* gmem_ptr, int32_t store_bytes) { -#if defined(CUTE_ARCH_TMA_SM90_ENABLED) - uint32_t smem_int_ptr = cast_smem_ptr_to_uint(smem_ptr); - asm volatile( - "cp.reduce.async.bulk.global.shared::cta.bulk_group.add.f32 [%0], [%1], %2;\n" - : - : "l"(gmem_ptr), "r"(smem_int_ptr), "r"(store_bytes) - : "memory"); -#else - CUTE_INVALID_CONTROL_PATH( - "Trying to use BULK_REDUCE_ADD without CUTE_ARCH_TMA_SM90_ENABLED."); -#endif - } - - CUTE_HOST_DEVICE static void copy( - float const* smem_ptr, - float* gmem_ptr, - int32_t store_bytes, - uint64_t cache_hint) { -#if defined(CUTE_ARCH_TMA_SM90_ENABLED) - uint32_t smem_int_ptr = cast_smem_ptr_to_uint(smem_ptr); - asm volatile( - "cp.reduce.async.bulk.global.shared::cta.bulk_group.L2::cache_hint.add.f32 [%0], [%1], %2, %3;\n" - : - : "l"(gmem_ptr), "r"(smem_int_ptr), "r"(store_bytes), "l"(cache_hint) - : "memory"); -#else - CUTE_INVALID_CONTROL_PATH( - "Trying to use BULK_REDUCE_ADD without CUTE_ARCH_TMA_SM90_ENABLED."); -#endif - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // end namespace cute diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/epilogue_bwd.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/epilogue_bwd.h deleted file mode 100644 index 833f3ae28..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/epilogue_bwd.h +++ /dev/null @@ -1,481 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/****************************************************************************** - * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, - *Pradeep Ramani, Tri Dao. - ******************************************************************************/ - -#pragma once - -#include -#include - -#include "cutlass/gemm/collective/builders/sm90_common.inl" - -#include "copy_sm90_bulk_reduce.h" -#include "named_barrier.h" -#include "seqlen.h" -#include "utils.h" - -namespace hstu { - -using namespace cute; - -template < - class TileShape_MNK_, - class Element_, - class ArchTag_, - int NumEpilogueThreads_, - bool Jagged, - bool dKV_swapAB_, - int AtomLayoutKdKV = 1> -struct CollectiveEpilogueBwd { - using TileShape_MNK = TileShape_MNK_; - using Element = Element_; - using ArchTag = ArchTag_; - static constexpr int NumEpilogueThreads = NumEpilogueThreads_; - static constexpr bool dKV_swapAB = dKV_swapAB_; - static constexpr bool Use_TMA = - !Jagged && ArchTag::kMinComputeCapability >= 90; - - static_assert(ArchTag::kMinComputeCapability >= 80); - - using GmemTiledCopydKVTMA = cute::SM90_TMA_STORE; - - // These are for storing the output tensor without TMA (e.g., for setting - // output to zero) - static constexpr int kGmemElemsPerLoad = - sizeof(cute::uint128_t) / sizeof(Element); - static_assert( - get<2>(TileShape_MNK{}) % kGmemElemsPerLoad == 0, - "Headdim must be a multiple of kGmemElemsPerLoad"); - static constexpr int kHeadDim = get<2>(TileShape_MNK{}); - static constexpr int kGmemThreadsPerRow = - cutlass::gcd(kHeadDim / kGmemElemsPerLoad, NumEpilogueThreads); - static_assert( - NumEpilogueThreads % kGmemThreadsPerRow == 0, - "NumEpilogueThreads must be a multiple of kGmemThreadsPerRow"); - using GmemLayoutAtom = Layout< - Shape< - Int, - Int>, - Stride, _1>>; - using GmemTiledCopydKV = decltype(make_tiled_copy( - Copy_Atom, Element>{}, - GmemLayoutAtom{}, - Layout>>{})); // Val layout, 8 or 16 vals - // per store - - using SmemLayoutAtomdKVTMA = - decltype(cutlass::gemm::collective::detail::ss_smem_selector< - GMMA::Major::K, - Element, - // TODO: do we have to change this if dKV_swapAB is true? - decltype(cute::get<1>(TileShape_MNK{})), - Int(TileShape_MNK{})) / - AtomLayoutKdKV>>()); - using SmemLayoutdKVTMA = decltype(tile_to_shape( - SmemLayoutAtomdKVTMA{}, - select<1, 2>(TileShape_MNK{}))); - using SmemLayoutdKVtTMA = decltype(cute::composition( - SmemLayoutdKVTMA{}, - make_layout( - make_shape(get<2>(TileShape_MNK{}), get<1>(TileShape_MNK{})), - make_stride(decltype(get<1>(TileShape_MNK{})){}, _1{})))); - - // If we don't use TMA - static constexpr int kBlockKSmem = - kHeadDim % 64 == 0 ? 64 : (kHeadDim % 32 == 0 ? 32 : 16); - static constexpr int kSwizzle = - kBlockKSmem == 64 ? 3 : (kBlockKSmem == 32 ? 2 : 1); - using SmemLayoutAtomdKVSTG = decltype(composition( - Swizzle{}, - Layout, Int>, Stride, _1>>{})); - - using SmemLayoutAtomdKV = - std::conditional_t; - using SmemLayoutdKV = decltype(tile_to_shape( - SmemLayoutAtomdKV{}, - select<1, 2>(TileShape_MNK{}))); - using SmemLayoutdKVt = decltype(cute::composition( - SmemLayoutdKV{}, - make_layout( - make_shape(get<2>(TileShape_MNK{}), get<1>(TileShape_MNK{})), - make_stride(decltype(get<1>(TileShape_MNK{})){}, _1{})))); - - using SmemCopyAtomdKV = Copy_Atom< - std::conditional_t< - ArchTag::kMinComputeCapability >= 90, - std::conditional_t< - !dKV_swapAB, - cute::SM90_U32x4_STSM_N, - cute::SM90_U16x8_STSM_T>, - AutoVectorizingCopyWithAssumedAlignment<128>>, - Element>; - - static constexpr size_t SmemAlignmentdKV = - ArchTag::kMinComputeCapability >= 90 - ? cutlass::detail::alignment_for_swizzle(SmemLayoutdKV{}) - : 128; - static_assert(SmemAlignmentdKV >= 128, "Require at least 128B alignment"); - - struct TensorStorage : cute::aligned_struct { - cute:: - array_aligned, SmemAlignmentdKV> - smem_dk; - cute:: - array_aligned, SmemAlignmentdKV> - smem_dv; - }; - - using ShapedKV = - cute::Shape; // (seqlen_k, d, head, - // batch) - using StridedKV = cute::Stride; - - using TMA_dKV = std::conditional_t< - Use_TMA, - decltype(make_tma_copy( - GmemTiledCopydKVTMA{}, - make_tensor( - make_gmem_ptr(static_cast(nullptr)), - ShapedKV{}, - StridedKV{}), - SmemLayoutdKVTMA{}, - select<1, 2>(TileShape_MNK{}), - _1{})), // no mcast for dKV - std::nullptr_t>; - - // Host side kernel arguments - struct Arguments { - Element* ptr_dK; - ShapedKV const shape_dK; - StridedKV const stride_dK; - Element* ptr_dV; - StridedKV const stride_dV; - int const num_heads_q; - int const* seq_offsets; - }; - - // Device side kernel params - struct Params { - Element* ptr_dK; - ShapedKV const shape_dK; - StridedKV const stride_dK; - Element* ptr_dV; - StridedKV const stride_dV; - TMA_dKV tma_store_dK, tma_store_dV; - int const* seq_offsets = nullptr; - }; - - static Params to_underlying_arguments(Arguments const& args) { - Tensor mdK = - make_tensor(make_gmem_ptr(args.ptr_dK), args.shape_dK, args.stride_dK); - Tensor mdV = - make_tensor(make_gmem_ptr(args.ptr_dV), args.shape_dK, args.stride_dV); - TMA_dKV tma_store_dK = [&] { - if constexpr (Use_TMA) { - return make_tma_copy( - GmemTiledCopydKVTMA{}, - mdK, - SmemLayoutdKVTMA{}, - select<1, 2>(TileShape_MNK{}), - _1{}); // no mcast for dKV - } else { - return nullptr; - } - }(); - TMA_dKV tma_store_dV = [&] { - if constexpr (Use_TMA) { - return make_tma_copy( - GmemTiledCopydKVTMA{}, - mdV, - SmemLayoutdKVTMA{}, - select<1, 2>(TileShape_MNK{}), - _1{}); // no mcast for dKV - } else { - return nullptr; - } - }(); - return { - args.ptr_dK, - args.shape_dK, - args.stride_dK, - args.ptr_dV, - args.stride_dV, - tma_store_dK, - tma_store_dV, - args.seq_offsets}; - } - - /// Issue Tma Descriptor Prefetch -- ideally from a single thread for best - /// performance - CUTLASS_DEVICE - static void prefetch_tma_descriptors(Params const& params) { - if constexpr (Use_TMA) { - cute::prefetch_tma_descriptor(params.tma_store_dK.get_tma_descriptor()); - cute::prefetch_tma_descriptor(params.tma_store_dV.get_tma_descriptor()); - } - } - - template - CUTLASS_DEVICE void store( - Params const& params, - FrgTensorO const& tdKrdK, - FrgTensorO const& tdVrdV, - SharedStorage& shared_storage, - TiledMma tiled_mma, - int thread_idx, - cute::tuple const& block_coord) { - auto [n_block, bidh, bidb] = block_coord; - Tensor sdK = cute::as_position_independent_swizzle_tensor(make_tensor( - make_smem_ptr(shared_storage.tensors.epilogue.smem_dk.data()), - SmemLayoutdKV{})); - Tensor sdV = cute::as_position_independent_swizzle_tensor(make_tensor( - make_smem_ptr(shared_storage.tensors.epilogue.smem_dv.data()), - SmemLayoutdKV{})); - Tensor sdKt = cute::as_position_independent_swizzle_tensor(make_tensor( - make_smem_ptr(shared_storage.tensors.epilogue.smem_dk.data()), - SmemLayoutdKVt{})); - Tensor sdVt = cute::as_position_independent_swizzle_tensor(make_tensor( - make_smem_ptr(shared_storage.tensors.epilogue.smem_dv.data()), - SmemLayoutdKVt{})); - auto smem_tiled_copy_dKV = make_tiled_copy_C(SmemCopyAtomdKV{}, tiled_mma); - auto smem_thr_copy_dKV = smem_tiled_copy_dKV.get_thread_slice(thread_idx); - - Tensor tdVrdV_out = make_tensor_like(tdVrdV); - hstu::convert_type_out(tdVrdV, tdVrdV_out); - Tensor tdKrdK_out = make_tensor_like(tdKrdK); - hstu::convert_type_out(tdKrdK, tdKrdK_out); - Tensor taccdKrdK = smem_thr_copy_dKV.retile_S( - tdKrdK_out); // ((Atom,AtomNum), MMA_M, MMA_N) - Tensor taccdVrdV = smem_thr_copy_dKV.retile_S( - tdVrdV_out); // ((Atom,AtomNum), MMA_M, MMA_N) - // if (blockIdx.x == 0 && threadIdx.x == 128) { print(smem_thr_copy_dKV); - // print(sdK); printf("\n"); print(sdKt); printf("\n"); } - Tensor taccdKsdK = smem_thr_copy_dKV.partition_D( - cute::conditional_return( - sdK, sdKt)); // ((Atom,AtomNum),PIPE_M,PIPE_N) - Tensor taccdVsdV = smem_thr_copy_dKV.partition_D( - cute::conditional_return( - sdV, sdVt)); // ((Atom,AtomNum),PIPE_M,PIPE_N) - - // Make sure all WGs have finished reading K and V - hstu::named_barrier_sync( - NumEpilogueThreads, - cutlass::arch::ReservedNamedBarriers::EpilogueBarrier); - cute::copy(smem_tiled_copy_dKV, taccdVrdV, taccdVsdV); - cute::copy(smem_tiled_copy_dKV, taccdKrdK, taccdKsdK); - if constexpr (Use_TMA) { - cutlass::arch::fence_view_async_shared(); // ensure smem writes are - // visible to TMA - cutlass::arch::NamedBarrier::arrive( - NumEpilogueThreads + cutlass::NumThreadsPerWarp, - cutlass::arch::ReservedNamedBarriers::EpilogueBarrier); - - Tensor mdK = params.tma_store_dK.get_tma_tensor(params.shape_dK); - Tensor mdV = params.tma_store_dV.get_tma_tensor(params.shape_dK); - Tensor gdK = local_tile( - mdK(_, _, bidh, bidb), - select<1, 2>(TileShape_MNK{}), - make_coord(n_block, _0{})); // (M, K) - Tensor gdV = local_tile( - mdV(_, _, bidh, bidb), - select<1, 2>(TileShape_MNK{}), - make_coord(n_block, _0{})); // (M, K) - auto block_tma_dK = params.tma_store_dK.get_slice(_0{}); - auto block_tma_dV = params.tma_store_dV.get_slice(_0{}); - Tensor tdKgdK = block_tma_dK.partition_D(gdK); // (TMA, TMA_M, TMA_K) - Tensor tdKsdK = block_tma_dK.partition_S(sdK); // (TMA, TMA_M, TMA_K) - Tensor tdVgdV = block_tma_dV.partition_D(gdV); // (TMA, TMA_M, TMA_K) - Tensor tdVsdV = block_tma_dV.partition_S(sdV); // (TMA, TMA_M, TMA_K) - int warp_idx_sync = - __shfl_sync(0xffffffff, thread_idx / cutlass::NumThreadsPerWarp, 0); - if (warp_idx_sync == - NumEpilogueThreads / cutlass::NumThreadsPerWarp - 1) { - cutlass::arch::NamedBarrier::sync( - NumEpilogueThreads + cutlass::NumThreadsPerWarp, - cutlass::arch::ReservedNamedBarriers::EpilogueBarrier); - if (cute::elect_one_sync()) { - cute::copy(params.tma_store_dV, tdVsdV, tdVgdV); - cute::copy(params.tma_store_dK, tdKsdK, tdKgdK); - tma_store_arrive(); - } - } - tma_store_wait<0>(); - // // Tell warp 0 that smem_k and smem_v are ready - // cutlass::arch::NamedBarrier::arrive(NumEpilogueThreads + - // cutlass::NumThreadsPerWarp, - // static_cast(BwdNamedBarriers::KVEmpty) /*id*/); - - } else { - hstu::named_barrier_sync( - NumEpilogueThreads, - cutlass::arch::ReservedNamedBarriers::EpilogueBarrier); - static constexpr int kBlockN = get<1>(TileShape_MNK{}); - hstu::SeqlenInfo seqlen_info{ - bidb, size<0>(params.shape_dK), params.seq_offsets}; - Tensor mdK = make_tensor( - make_gmem_ptr(params.ptr_dK), params.shape_dK, params.stride_dK)( - _, _, bidh, !Jagged ? bidb : 0); - Tensor gdK = local_tile( - cute::domain_offset(make_coord(seqlen_info.offset, _0{}), mdK), - select<1, 2>(TileShape_MNK{}), - make_coord(n_block, _0{})); // (M, K) - Tensor mdV = make_tensor( - make_gmem_ptr(params.ptr_dV), params.shape_dK, params.stride_dV)( - _, _, bidh, !Jagged ? bidb : 0); - Tensor gdV = local_tile( - cute::domain_offset(make_coord(seqlen_info.offset, _0{}), mdV), - select<1, 2>(TileShape_MNK{}), - make_coord(n_block, _0{})); // (M, K) - - GmemTiledCopydKV gmem_tiled_copy_dKV; - auto gmem_thr_copy_dKV = gmem_tiled_copy_dKV.get_thread_slice(thread_idx); - Tensor tdKVgdV = gmem_thr_copy_dKV.partition_D(gdV); - Tensor tdKVsdV = - gmem_thr_copy_dKV.partition_S(sdV); // (TMA, TMA_M, TMA_K) - Tensor tdKVgdK = gmem_thr_copy_dKV.partition_D(gdK); - Tensor tdKVsdK = - gmem_thr_copy_dKV.partition_S(sdK); // (TMA, TMA_M, TMA_K) - Tensor tdKVrdV = make_fragment_like(tdKVgdV); - Tensor tdKVrdK = make_fragment_like(tdKVgdK); - Tensor cdKV = cute::make_identity_tensor( - select<1, 2>(TileShape_MNK{})); // (BLK_M,BLK_K) -> (blk_m,blk_k) - // Repeat the partitioning with identity layouts - Tensor tdKVcdKV = gmem_thr_copy_dKV.partition_D(cdKV); - Tensor tdKVpdKV = make_tensor(make_shape(size<2>(tdKVgdV))); -#pragma unroll - for (int k = 0; k < size(tdKVpdKV); ++k) { - tdKVpdKV(k) = get<1>(tdKVcdKV(_0{}, _0{}, k)) < get<1>(params.shape_dK); - } - // Need to check OOB when reading from smem if kBlockN isn't evenly tiled - static constexpr bool EvenN = - kBlockN % CUTE_STATIC_V(size<0>(GmemLayoutAtom{})) == 0; - hstu::copy< - /*Is_even_MN=*/EvenN, - /*Is_even_K=*/true, - /*Clear_OOB_MN=*/false>( - gmem_tiled_copy_dKV, tdKVsdV, tdKVrdV, tdKVcdKV, tdKVpdKV, kBlockN); - hstu::copy< - /*Is_even_MN=*/EvenN, - /*Is_even_K=*/true, - /*Clear_OOB_MN=*/false>( - gmem_tiled_copy_dKV, tdKVsdK, tdKVrdK, tdKVcdKV, tdKVpdKV, kBlockN); - // // Tell warp 0 that smem_k and smem_v are ready - // cutlass::arch::fence_view_async_shared(); // ensure smem reads are done - // before next TMA to smem_k/v - // hstu::named_barrier_arrive(NumEpilogueThreads + - // cutlass::NumThreadsPerWarp, - // static_cast(BwdNamedBarriers::KVEmpty) /*id*/); Construct - // identity layout for gdKV Clear_OOB_K must be false since we don't want - // to write zeros to gmem - hstu::copy< - /*Is_even_MN=*/false, - /*Is_even_K=*/false, - /*Clear_OOB_MN=*/false, - /*Clear_OOB_K=*/false>( - gmem_tiled_copy_dKV, - tdKVrdV, - tdKVgdV, - tdKVcdKV, - tdKVpdKV, - std::min(seqlen_info.seqlen - n_block * kBlockN, kBlockN)); - hstu::copy< - /*Is_even_MN=*/false, - /*Is_even_K=*/false, - /*Clear_OOB_MN=*/false, - /*Clear_OOB_K=*/false>( - gmem_tiled_copy_dKV, - tdKVrdK, - tdKVgdK, - tdKVcdKV, - tdKVpdKV, - std::min(seqlen_info.seqlen - n_block * kBlockN, kBlockN)); - } - } - - CUTLASS_DEVICE void store_tail() { - // if constexpr (Use_TMA) { tma_store_wait<0>(); } - } - - // Write 0 to dK and dV - CUTLASS_DEVICE void store_zero( - Params const& params, - int thread_idx, - cute::tuple const& block_coord) { - static constexpr int kBlockN = get<1>(TileShape_MNK{}); - auto [n_block, bidh, bidb] = block_coord; - hstu::SeqlenInfo seqlen_info{ - bidb, size<0>(params.shape_dK), params.seq_offsets}; - Tensor mdK = make_tensor( - make_gmem_ptr(params.ptr_dK), params.shape_dK, params.stride_dK)( - _, _, bidh, !Jagged ? bidb : 0); - Tensor gdK = local_tile( - cute::domain_offset(make_coord(seqlen_info.offset, _0{}), mdK), - select<1, 2>(TileShape_MNK{}), - make_coord(n_block, _0{})); // (M, K) - Tensor mdV = make_tensor( - make_gmem_ptr(params.ptr_dV), params.shape_dK, params.stride_dV)( - _, _, bidh, !Jagged ? bidb : 0); - Tensor gdV = local_tile( - cute::domain_offset(make_coord(seqlen_info.offset, _0{}), mdV), - select<1, 2>(TileShape_MNK{}), - make_coord(n_block, _0{})); // (M, K) - - GmemTiledCopydKV gmem_tiled_copy_dKV; - auto gmem_thr_copy_dKV = gmem_tiled_copy_dKV.get_thread_slice(thread_idx); - Tensor tdKVgdK = gmem_thr_copy_dKV.partition_D(gdK); - Tensor tdKVgdV = gmem_thr_copy_dKV.partition_D(gdV); - Tensor tdKVrdKV = make_fragment_like(tdKVgdK); - clear(tdKVrdKV); - // Construct identity layout for gdKV - Tensor cdKV = cute::make_identity_tensor( - select<1, 2>(TileShape_MNK{})); // (BLK_M,BLK_K) -> (blk_m,blk_k) - // Repeat the partitioning with identity layouts - Tensor tdKVcdKV = gmem_thr_copy_dKV.partition_D(cdKV); - Tensor tdKVpdKV = make_tensor(make_shape(size<2>(tdKVgdK))); -#pragma unroll - for (int k = 0; k < size(tdKVpdKV); ++k) { - tdKVpdKV(k) = get<1>(tdKVcdKV(_0{}, _0{}, k)) < get<1>(params.shape_dK); - } - // Clear_OOB_K must be false since we don't want to write zeros to gmem - hstu::copy< - /*Is_even_MN=*/false, - /*Is_even_K=*/false, - /*Clear_OOB_MN=*/false, - /*Clear_OOB_K=*/false>( - gmem_tiled_copy_dKV, - tdKVrdKV, - tdKVgdK, - tdKVcdKV, - tdKVpdKV, - seqlen_info.seqlen - n_block * kBlockN); - hstu::copy< - /*Is_even_MN=*/false, - /*Is_even_K=*/false, - /*Clear_OOB_MN=*/false, - /*Clear_OOB_K=*/false>( - gmem_tiled_copy_dKV, - tdKVrdKV, - tdKVgdV, - tdKVcdKV, - tdKVpdKV, - seqlen_info.seqlen - n_block * kBlockN); - } -}; - -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/epilogue_fwd.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/epilogue_fwd.h deleted file mode 100644 index c794a114c..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/epilogue_fwd.h +++ /dev/null @@ -1,550 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/****************************************************************************** - * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, - *Pradeep Ramani, Tri Dao. - ******************************************************************************/ - -#pragma once - -#include -#include // For FastDivMod -#include "cute/tensor.hpp" - -#include "cutlass/epilogue/collective/builders/sm90_common.inl" -#include "cutlass/gemm/collective/builders/sm90_common.inl" - -#include "named_barrier.h" -#include "seqlen.h" -#include "utils.h" - -namespace hstu { - -using namespace cute; - -template < - class TileShape_MNK_, - class ClusterShape_, - class Element_, - class ArchTag_, - int NumEpilogueThreads_, - bool Jagged, - bool FP8PermuteCol = false> -struct CollectiveEpilogueFwd { - using TileShape_MNK = TileShape_MNK_; - using ClusterShape = ClusterShape_; - using Element = Element_; - using ArchTag = ArchTag_; - static constexpr int NumEpilogueThreads = NumEpilogueThreads_; - static constexpr bool Use_smem = sizeof(Element) <= 2; - static constexpr bool Use_TMA_O = - ArchTag::kMinComputeCapability >= 90 && !Jagged && Use_smem; - - static_assert(ArchTag::kMinComputeCapability >= 80); - static_assert( - ArchTag::kMinComputeCapability >= 90 || - CUTE_STATIC_V(size(ClusterShape{})) == 1); - - static constexpr int kBlockM = get<0>(TileShape_MNK{}); - static constexpr int kHeadDim = get<2>(TileShape_MNK{}); - - using GmemTiledCopyOTMA = cute::SM90_TMA_STORE; - - // These are for storing the output tensor without TMA (e.g., for setting - // output to zero) - static constexpr int kGmemElemsPerStore = - sizeof(cute::uint128_t) / sizeof(Element); - static_assert( - kHeadDim % kGmemElemsPerStore == 0, - "Headdim must be a multiple of kGmemElemsPerStore"); - // We want each "row" to have 64 elements (128 bytes, i.e. 1 cache line). We - // want each thread to have 4 elements in the M direction and 2 elements in - // the K direction. In the case of PackGQA, this reduces the number of times - // we need to call divmod. - static constexpr int kBytePerRow = kHeadDim * sizeof(Element); - static constexpr int kBlockKGmem = - (kBytePerRow % 128 == 0 ? 128 : (kBytePerRow % 64 == 0 ? 64 : 32)) / - sizeof(Element); - // static constexpr int kBlockKGmem = kHeadDim % 128 == 0 ? 128 : (kHeadDim % - // 64 == 0 ? 64 : 32); static constexpr int kGmemThreadsPerRow = - // cutlass::gcd(kHeadDim / kGmemElemsPerStore, NumEpilogueThreads); - static constexpr int kGmemThreadsPerRow = kBlockKGmem / kGmemElemsPerStore; - // If PackGQA, we split the work of compute O_ptr among threads in the same - // row, so we need this to within a warp - static_assert(cutlass::NumThreadsPerWarp % kGmemThreadsPerRow == 0); - static_assert( - NumEpilogueThreads % kGmemThreadsPerRow == 0, - "NumEpilogueThreads must be a multiple of kGmemThreadsPerRow"); - using GmemLayoutAtom = Layout< - Shape< - Int, - Int>, - Stride, _1>>; - static_assert( - kBlockM % CUTE_STATIC_V(shape<0>(GmemLayoutAtom{})) == 0, - "kBlockM must be a multiple of NumEpilogueThreads / kGmemThreadsPerRow"); - using GmemTiledCopyO = decltype(make_tiled_copy( - Copy_Atom, Element>{}, - GmemLayoutAtom{}, - Layout>>{})); // Val layout, 8 or 16 - // vals per store - - using SmemLayoutAtomOTMA = - decltype(cutlass::gemm::collective::detail::ss_smem_selector< - GMMA::Major::K, - Element, - decltype(cute::get<0>(TileShape_MNK{})), - decltype(cute::get<2>(TileShape_MNK{}))>()); - using SmemLayoutOTMA = decltype(tile_to_shape( - SmemLayoutAtomOTMA{}, - select<0, 2>(TileShape_MNK{}))); - static constexpr int kSwizzle = kBlockKGmem == 128 - ? 4 - : (kBlockKGmem == 64 ? 3 : (kBlockKGmem == 32 ? 2 : 1)); - static constexpr int kSwizzleBase = - sizeof(Element) == 4 ? 2 : (sizeof(Element) == 2 ? 3 : 4); - using SmemLayoutAtomO = decltype(composition( - Swizzle{}, - Layout>, Stride, _1>>{})); - using SmemLayoutOSTS = - decltype(tile_to_shape(SmemLayoutAtomO{}, select<0, 2>(TileShape_MNK{}))); - using SmemLayoutO = std::conditional_t< - ArchTag::kMinComputeCapability >= 90, - SmemLayoutOTMA, - SmemLayoutOSTS>; - - using ShapeO = - cute::Shape; // (seqlen_q, d, - // head, batch, - // num_splits) - using StrideO = cute::Stride; - // ((qhead_per_khead, seqlen_q), d, nheads, batch, num_splits) - using ShapeOPacked = ShapeO; - using StrideOPacked = StrideO; - // ((qhead_per_khead, seqlen_q), nheads, batch, num_splits) - using StrideLSE = - cute::Stride<_1, int64_t, int64_t, int64_t>; // (seqlen_q, head, batch, - // num_splits) - using ShapeLSEPacked = cute::Shape; - using StrideLSEPacked = StrideLSE; - using EpilogueTile_MN = decltype(select<0, 1>(TileShape_MNK{})); - using CopyOpR2S = std::conditional_t< - ArchTag::kMinComputeCapability >= 90, - // cute::SM90_U32x4_STSM_N if Element size is 2 bytes (fp16, bf16) - decltype(cutlass::epilogue::collective::detail:: - sm90_get_smem_store_op_for_accumulator< - StrideO, - Element, - EpilogueTile_MN>()), - AutoVectorizingCopyWithAssumedAlignment<128>>; - using SmemCopyAtomO = Copy_Atom; - - // static constexpr size_t SmemAlignmentO = - // cutlass::detail::alignment_for_swizzle(SmemLayoutO{}); - // static_assert(SmemAlignmentO >= 128, "Require at least 128B alignment"); - // struct TensorStorage : cute::aligned_struct { - // cute::array_aligned : - // 0, SmemAlignmentO> smem_o; - // }; - struct TensorStorage : cute::aligned_struct<128> { - cute::array_aligned : 0> - smem_o; - }; - - using TMA_O = std::conditional_t< - Use_TMA_O, - decltype(make_tma_copy( - GmemTiledCopyOTMA{}, - make_tensor( - make_gmem_ptr(static_cast(nullptr)), - ShapeO{}, - StrideO{}), - SmemLayoutOTMA{}, - select<0, 2>(TileShape_MNK{}), - _1{})), // no mcast for O - std::nullptr_t>; - - // Host side kernel arguments - struct Arguments { - Element* ptr_O; - ShapeO const shape_O; - StrideO const stride_O; - int32_t const nheads; - int32_t const num_softmax_heads; - StrideLSE const stride_lse; - float* ptr_lse = nullptr; - int const* seq_offsets = nullptr; - }; - - // Device side kernel params - struct Params { - Element* ptr_O; - ShapeO const shape_O; - StrideO const stride_O; - ShapeOPacked const shape_O_packed; - StrideOPacked const stride_O_packed; - float* ptr_lse; - StrideLSE const stride_lse; - ShapeLSEPacked const shape_lse_packed; - StrideLSEPacked const stride_lse_packed; - TMA_O tma_store_O; - int const* seq_offsets = nullptr; - }; - - static Params to_underlying_arguments(Arguments const& args) { - Tensor mO = - make_tensor(make_gmem_ptr(args.ptr_O), args.shape_O, args.stride_O); - TMA_O tma_store_O = [&] { - if constexpr (Use_TMA_O) { - return make_tma_copy( - GmemTiledCopyOTMA{}, - mO, - SmemLayoutO{}, - select<0, 2>(TileShape_MNK{}), - _1{}); // no mcast - } else { - return nullptr; - } - }(); - // If PackGQA, reshape O to be ((qhead_per_khead, seqlen_q), head_size, - // nhead_k, batch_size, num_splits) - int const qhead_per_khead = 1; - auto const shape_O_packed = cute::conditional_return( - args.shape_O, - make_shape( - make_shape(qhead_per_khead, get<0>(args.shape_O)), - get<1>(args.shape_O), - args.nheads, - get<3>(args.shape_O), - get<4>(args.shape_O))); - auto const stride_O_packed = cute::conditional_return( - args.stride_O, - make_stride( - make_stride(get<2>(args.stride_O), get<0>(args.stride_O)), - get<1>(args.stride_O), - get<2>(args.stride_O) * qhead_per_khead, - get<3>(args.stride_O), - get<4>(args.stride_O))); - auto const shape_lse_packed = select<0, 2, 3, 4>(args.shape_O); - auto const stride_lse_packed = args.stride_lse; - return { - args.ptr_O, - args.shape_O, - args.stride_O, - shape_O_packed, - stride_O_packed, - args.ptr_lse, - args.stride_lse, - shape_lse_packed, - stride_lse_packed, - tma_store_O, - args.seq_offsets}; - } - - /// Issue Tma Descriptor Prefetch -- ideally from a single thread for best - /// performance - CUTLASS_DEVICE - static void prefetch_tma_descriptors(Params const& params) { - if constexpr (Use_TMA_O) { - cute::prefetch_tma_descriptor(params.tma_store_O.get_tma_descriptor()); - } - } - - template - CUTLASS_DEVICE void store( - Params const& params, - FrgTensorO const& tOrO, - SharedStorage& shared_storage, - TiledMma tiled_mma, - int thread_idx, - cute::tuple const& block_coord) { - auto [m_block, bidh, bidb, split_idx] = block_coord; - Tensor sO = make_tensor( - make_smem_ptr(shared_storage.tensors.epilogue.smem_o.data()), - SmemLayoutO{}); - // Tensor sO_pi = cute::as_position_independent_swizzle_tensor(sO); - - Tensor tOrO_out = make_tensor_like(tOrO); - hstu::convert_type_out(tOrO, tOrO_out); - if constexpr ( - FP8PermuteCol && (sizeof(Element) == 2 || sizeof(Element) == 4)) { - hstu::permute_output_fp8_Vcolmajor(tOrO_out); - } - - // Make sure all WGs have finished reading V - // Technically we don't need this if we're not using smem, but the mainloop - // makes the assumption that all epilogue threads sync at least once during - // the epilogue (so that we can start loading Q with cp.async if we need). - hstu::named_barrier_sync( - NumEpilogueThreads, - cutlass::arch::ReservedNamedBarriers::EpilogueBarrier); - - // Step 1: Write O from rmem -> smem - if constexpr (Use_smem) { - auto smem_tiled_copy_O = make_tiled_copy_C(SmemCopyAtomO{}, tiled_mma); - auto smem_thr_copy_O = smem_tiled_copy_O.get_thread_slice(thread_idx); - Tensor taccOrO = - smem_thr_copy_O.retile_S(tOrO_out); // ((Atom,AtomNum), MMA_M, MMA_N) - Tensor taccOsO = - smem_thr_copy_O.partition_D(sO); // ((Atom,AtomNum),PIPE_M,PIPE_N) - // Tensor taccOsO = smem_thr_copy_O.partition_D(sO_pi); // - // ((Atom,AtomNum),PIPE_M,PIPE_N) - cute::copy(smem_tiled_copy_O, taccOrO, taccOsO); - if constexpr (Use_TMA_O) { - cutlass::arch::fence_view_async_shared(); // ensure smem writes are - // visible to TMA - cutlass::arch::NamedBarrier::arrive( - NumEpilogueThreads + cutlass::NumThreadsPerWarp, - cutlass::arch::ReservedNamedBarriers::EpilogueBarrier); - } else { - hstu::named_barrier_sync( - NumEpilogueThreads, - cutlass::arch::ReservedNamedBarriers::EpilogueBarrier); - } - } else { - if constexpr (ArchTag::kMinComputeCapability >= 90) { -#pragma unroll - for (uint32_t cta_id = 0; cta_id < size(ClusterShape{}); ++cta_id) { - shared_storage.pipelines.barrier_O.arrive(cta_id); - } - } - } - - hstu::SeqlenInfo seqlen_info{ - bidb, size<0>(params.shape_O), params.seq_offsets}; - int offset_o = seqlen_info.offset; - int seqlen_o = seqlen_info.seqlen; - - // Step 2: Write LSE from rmem -> gmem - auto thread_mma = tiled_mma.get_thread_slice(thread_idx); - // (MMA,MMA_M,MMA_K) - Tensor taccOcO = thread_mma.partition_C( - cute::make_identity_tensor(select<0, 2>(TileShape_MNK{}))); - static_assert(decltype(size<0, 0>(taccOcO))::value == 2); - static_assert(decltype(size<0, 1>(taccOcO))::value == 2); - Tensor taccOcO_rowcol = make_tensor( - taccOcO.data(), hstu::convert_layout_acc_rowcol(taccOcO.layout())); - Tensor taccOcO_row = taccOcO_rowcol(_, _0{}); - // Step 3: Write O from smem -> gmem - if constexpr (Use_TMA_O) { - Tensor mO = params.tma_store_O.get_tma_tensor(params.shape_O)( - _, _, bidh, bidb, split_idx); - Tensor gO = local_tile( - mO, - select<0, 2>(TileShape_MNK{}), - make_coord(m_block, _0{})); // (M, K) - auto block_tma_O = params.tma_store_O.get_slice(_0{}); - Tensor tOgO = block_tma_O.partition_D(gO); // (TMA, TMA_M, TMA_K) - Tensor tOsO = block_tma_O.partition_S(sO); // (TMA, TMA_M, TMA_K) - int warp_idx_sync = - __shfl_sync(0xffffffff, thread_idx / cutlass::NumThreadsPerWarp, 0); - if (warp_idx_sync == - NumEpilogueThreads / cutlass::NumThreadsPerWarp - 1) { - cutlass::arch::NamedBarrier::sync( - NumEpilogueThreads + cutlass::NumThreadsPerWarp, - cutlass::arch::ReservedNamedBarriers::EpilogueBarrier); - if (cute::elect_one_sync()) { - cute::copy(params.tma_store_O, tOsO, tOgO); - tma_store_arrive(); - tma_store_wait<0>(); -#pragma unroll - for (uint32_t cta_id = 0; cta_id < size(ClusterShape{}); ++cta_id) { - shared_storage.pipelines.barrier_O.arrive(cta_id); - } - } - } - } else { // Don't use TMA in Jagged case since we don't want to overwrite - // the output of another sequence - Tensor mO = make_tensor( - make_gmem_ptr(params.ptr_O + offset_o * get<0>(params.stride_O)), - params.shape_O_packed, - params.stride_O_packed)(_, _, bidh, !Jagged ? bidb : 0, split_idx); - Tensor gO = local_tile( - mO, - select<0, 2>(TileShape_MNK{}), - make_coord(m_block, _0{})); // (M, K) - // if (thread_idx == 0) { printf("Before O write, m_block: %d, bidh: %d, - // bidb: %d, split_idx: %d, offset_o: %d, seqlen_o: %d, mO_addr = %p, addr - // diff = %d\n", m_block, bidh, bidb, split_idx, offset_o, seqlen_o, - // mO.data(), reinterpret_cast(&mO(0)) - - // reinterpret_cast(params.ptr_O)); } - if constexpr (Use_smem) { - GmemTiledCopyO gmem_tiled_copy_O; - auto gmem_thr_copy_O = gmem_tiled_copy_O.get_thread_slice(thread_idx); - Tensor tOsO = - gmem_thr_copy_O.partition_S(sO); // ((Atom,AtomNum),ATOM_M,ATOM_N) - // Tensor tOsO = gmem_thr_copy_O.partition_S(sO_pi); // - // ((Atom,AtomNum),ATOM_M,ATOM_N) - Tensor tOrO = make_fragment_like(tOsO); - cute::copy(gmem_tiled_copy_O, tOsO, tOrO); - if constexpr (ArchTag::kMinComputeCapability >= 90) { - cutlass::arch::fence_view_async_shared(); // ensure smem reads are - // done before next TMA to - // smem_v -#pragma unroll - for (uint32_t cta_id = 0; cta_id < size(ClusterShape{}); ++cta_id) { - shared_storage.pipelines.barrier_O.arrive(cta_id); - } - } - // (BLK_M,BLK_K) -> (blk_m,blk_k) - Tensor tOcO = gmem_thr_copy_O.partition_D( - cute::make_identity_tensor(select<0, 2>(TileShape_MNK{}))); - Tensor tOpO = make_tensor(make_shape(size<2>(tOsO))); -#pragma unroll - for (int k = 0; k < size(tOpO); ++k) { - tOpO(k) = get<1>(tOcO(_0{}, _0{}, k)) < get<1>(params.shape_O); - } - Tensor tOgO = gmem_thr_copy_O.partition_D(gO); - // Clear_OOB_K must be false since we don't want to write zeros to - // gmem - hstu::copy< - /*Is_even_MN=*/false, - /*Is_even_K=*/false, - /*Clear_OOB_MN=*/false, - /*Clear_OOB_K=*/false>( - gmem_tiled_copy_O, - tOrO, - tOgO, - tOcO, - tOpO, - seqlen_o - m_block * kBlockM); - } else { - // We already arrived on barrier_O earlier - static constexpr int kGmemElemsPerStoreDirect = 2; - cute::Copy_Atom, Element> - gmem_copy_direct; - // Reshape acc from ((2, 2, V), MMA_M, MMA_N) to (nrow=(2, MMA_M), - // ncol=(2, V, MMA_N)) - Tensor tOrO_rowcol = make_tensor( - tOrO_out.data(), hstu::convert_layout_acc_rowcol(tOrO.layout())); - Tensor tOrO_copy = cute::tiled_divide( - tOrO_rowcol, Shape<_1, Int>{}); - Tensor tOgO = thread_mma.partition_C(gO); - Tensor tOgO_rowcol = make_tensor( - tOgO.data(), hstu::convert_layout_acc_rowcol(tOgO.layout())); - Tensor tOgO_copy = cute::tiled_divide( - tOgO_rowcol, Shape<_1, Int>{}); - Tensor taccOcO_col = taccOcO_rowcol(_0{}, _); -#pragma unroll - for (int m = 0; m < size(taccOcO_row); ++m) { - if (get<0>(taccOcO_row(m)) < seqlen_o - m_block * kBlockM) { -#pragma unroll - for (int k = 0; k < size(taccOcO_col) / kGmemElemsPerStoreDirect; - ++k) { - if (get<1>(taccOcO_col(k * kGmemElemsPerStoreDirect)) < - get<1>(params.shape_O)) { - cute::copy( - gmem_copy_direct, tOrO_copy(_, m, k), tOgO_copy(_, m, k)); - } - } - } - } - } - } - } - - template - CUTLASS_DEVICE void store_softmax( - Params const& params, - FrgTensorLSE const& lse, - TiledMma tiled_mma, - int thread_idx, - cute::tuple const& block_coord) { - auto [m_block, bidh, bidb, split_idx] = block_coord; - hstu::SeqlenInfo seqlen_info{ - bidb, size<0>(params.shape_O), params.seq_offsets}; - int offset_o = seqlen_info.offset; - int seqlen_o = seqlen_info.seqlen; - // Step 2: Write LSE from rmem -> gmem - auto thread_mma = tiled_mma.get_thread_slice(thread_idx); - // (MMA,MMA_M,MMA_K) - Tensor taccOcO = thread_mma.partition_C( - cute::make_identity_tensor(select<0, 2>(TileShape_MNK{}))); - static_assert(decltype(size<0, 0>(taccOcO))::value == 2); - static_assert(decltype(size<0, 1>(taccOcO))::value == 2); - Tensor taccOcO_rowcol = make_tensor( - taccOcO.data(), hstu::convert_layout_acc_rowcol(taccOcO.layout())); - Tensor taccOcO_row = taccOcO_rowcol(_, _0{}); - CUTE_STATIC_ASSERT_V(size(lse) == size(taccOcO_row)); // MMA_M - Tensor mLSE = make_tensor( - make_gmem_ptr(params.ptr_lse + offset_o * get<0>(params.stride_lse)), - params.shape_lse_packed, - params.stride_lse_packed)(_, bidh, !Jagged ? bidb : 0, 0); -#pragma unroll - for (int mi = 0; mi < size(lse); ++mi) { - int const row = m_block * kBlockM + get<0>(taccOcO_row(mi)); - if (get<1>(taccOcO_row(_0{})) == 0 && row < seqlen_o) { - mLSE(row) = lse(mi); - } - } - } - - CUTLASS_DEVICE void store_tail() { - // Don't need to do tma_store_wait<0>() here since we already did in @store - } - - // Write 0 to output and -inf to LSE - template - CUTLASS_DEVICE void store_zero( - Params const& params, - int thread_idx, - cute::tuple const& block_coord) { - static constexpr int kBlockM = get<0>(TileShape_MNK{}); - auto [m_block, bidh, bidb, split_idx] = block_coord; - hstu::SeqlenInfo seqlen_info{ - bidb, size<0>(params.shape_O), params.seq_offsets}; - int offset_o = seqlen_info.offset; - int seqlen_o = seqlen_info.seqlen; - Tensor mO = make_tensor( - make_gmem_ptr(params.ptr_O + offset_o * get<0>(params.stride_O)), - params.shape_O_packed, - params.stride_O_packed)(_, _, bidh, !Jagged ? bidb : 0, split_idx); - - static_assert(kBlockM <= NumEpilogueThreads); - if constexpr (!Clear_O) { - return; - } - - GmemTiledCopyO gmem_tiled_copy_O; - auto gmem_thr_copy_O = gmem_tiled_copy_O.get_thread_slice(thread_idx); - Tensor tOcO = gmem_thr_copy_O.partition_D( - cute::make_identity_tensor(select<0, 2>(TileShape_MNK{}))); - Tensor tOpO = make_tensor(make_shape(size<2>(tOcO))); -#pragma unroll - for (int k = 0; k < size(tOpO); ++k) { - tOpO(k) = get<1>(tOcO(_0{}, _0{}, k)) < get<1>(params.shape_O); - } - Tensor gO = local_tile( - mO, select<0, 2>(TileShape_MNK{}), make_coord(m_block, _0{})); // (M, K) - Tensor tOgO = gmem_thr_copy_O.partition_D(gO); - Tensor tOrO = make_fragment_like(tOgO); - cute::clear(tOrO); - // Clear_OOB_K must be false since we don't want to write zeros to gmem - hstu::copy< - /*Is_even_MN=*/false, - /*Is_even_K=*/false, - /*Clear_OOB_MN=*/false, - /*Clear_OOB_K=*/false>( - gmem_tiled_copy_O, - tOrO, - tOgO, - tOcO, - tOpO, - seqlen_o - m_block * kBlockM); - } -}; - -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash.h deleted file mode 100644 index ef37e3408..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash.h +++ /dev/null @@ -1,157 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include - -//////////////////////////////////////////////////////////////////////////////////////////////////// -namespace hstu { - -struct Qkv_params { - using index_t = int64_t; - // The QKV matrices. - void* __restrict__ q_ptr; - void* __restrict__ k_ptr; - void* __restrict__ v_ptr; - - // The stride between rows of the Q, K and V matrices. - index_t q_batch_stride; - index_t k_batch_stride; - index_t v_batch_stride; - index_t q_row_stride; - index_t k_row_stride; - index_t v_row_stride; - index_t q_head_stride; - index_t k_head_stride; - index_t v_head_stride; - index_t v_dim_stride; - - // The number of heads. - int h; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -struct Flash_fwd_params : public Qkv_params { - using index_t = int64_t; - - // The O matrix (output). - void* __restrict__ o_ptr; - - // The stride between rows of O. - index_t o_batch_stride; - index_t o_row_stride; - index_t o_head_stride; - - // For FP8 scaling - float* __restrict__ q_descale_ptr; - float* __restrict__ k_descale_ptr; - float* __restrict__ v_descale_ptr; - index_t q_descale_batch_stride; - index_t q_descale_head_stride; - index_t k_descale_batch_stride; - index_t k_descale_head_stride; - index_t v_descale_batch_stride; - index_t v_descale_head_stride; - - // The dimensions. - int b, max_kv_len, max_q_len, qk_d, v_d, total_seq_len_q, total_seq_len_kv; - - // groups - int num_groups, batch_size_per_group; - int* __restrict__ max_seq_len_tensor; - int* __restrict__ contextual_seq_len_tensor; - int* __restrict__ max_attn_len_tensor; - int* __restrict__ min_full_attn_seq_len_tensor; - - // The scaling factors for the kernel. - float alpha; - - int* __restrict__ seq_offsets; - int* __restrict__ seq_offsets_q; - float* __restrict__ softmax_lse; - int* __restrict__ num_targets; - float* __restrict__ attn_scale; - - // Local window size - int max_attn_len, contextual_seq_len, min_full_attn_seq_len, - num_softmax_heads; - - // Pointer to the RNG seed (idx 0) and offset (idx 1). - uint64_t* rng_state; - - bool is_bf16; - bool is_fp32; - bool is_e4m3; - bool is_causal; - bool is_local; - bool has_contexual_mask; - bool scalar_scale; - bool training; - - int* __restrict__ tile_count_semaphore; - - int arch; - int num_sm; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -struct Flash_bwd_params : public Flash_fwd_params { - using index_t = int64_t; - - // The dO and dQKV matrices. - void* __restrict__ do_ptr; - void* __restrict__ dq_ptr; - void* __restrict__ dk_ptr; - void* __restrict__ dv_ptr; - float* __restrict__ softmax_lse_log2; - float* __restrict__ softmax_d; - - // To accumulate dQ - void* __restrict__ dq_accum_ptr; - int* __restrict__ dq_semaphore; - - // The stride between rows of the dO, dQ, dK and dV matrices. - index_t do_batch_stride; - index_t do_row_stride; - index_t do_head_stride; - index_t dq_batch_stride; - index_t dk_batch_stride; - index_t dv_batch_stride; - index_t dq_row_stride; - index_t dk_row_stride; - index_t dv_row_stride; - index_t dq_head_stride; - index_t dk_head_stride; - index_t dv_head_stride; - - int* __restrict__ sort_by_length_indices; - - int max_q_len_rounded, qk_d_rounded, v_d_rounded; - - bool deterministic; - index_t dq_accum_split_stride; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template -void run_mha_fwd_(Flash_fwd_params& params, cudaStream_t stream); -template -void run_mha_bwd_(Flash_bwd_params& params, cudaStream_t stream); -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_api.cpp b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_api.cpp deleted file mode 100644 index 389e6620f..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_api.cpp +++ /dev/null @@ -1,322 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/****************************************************************************** - * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, - *Pradeep Ramani, Tri Dao. - ******************************************************************************/ - -#include -#include -#include -#include -#include -#include -#include // @manual -#include -#include "flash_common.h" - -extern "C" { -/* Creates a dummy empty _C module that can be imported from Python. - The import from Python will load the .so consisting of this file - in this extension, so that the TORCH_LIBRARY static initializers - below are run. */ -PyObject* PyInit__C(void) { - static struct PyModuleDef module_def = { - PyModuleDef_HEAD_INIT, - "_C", /* name of module */ - NULL, /* module documentation, may be NULL */ - -1, /* size of per-interpreter state of the module, - or -1 if the module keeps state in global variables. */ - NULL, /* methods */ - }; - return PyModule_Create(&module_def); -} -} - -namespace hstu { - -class HSTUFlashAttentionFunctionGPU - : public torch::autograd::Function { - public: - static at::Tensor forward( - torch::autograd::AutogradContext* ctx, - int64_t max_seq_len, - double alpha, - at::Tensor& q, // (b, s, h, d) or (total_s, h, d) - at::Tensor& k, // (b, s, h, d) or (total_s, h, d) - at::Tensor& v, // (b, s, h, d) or (total_s, h, d) - const std::optional& seq_offsets, - bool causal, - const std::optional& num_targets, - const std::optional& attn_scale, - int64_t max_attn_len, - int64_t min_full_attn_seq_len, - int64_t contextual_seq_len, - const std::optional& q_descale, // (b, h_k), not (b, h) - const std::optional& k_descale, // (b, h_k) - const std::optional& v_descale, // (b, h_k) - bool sort_by_length, - bool deterministic, - const int64_t sm_margin, - int64_t max_q_len, - const std::optional& seq_offsets_q, - int64_t num_softmax_heads, - bool training, - const std::optional& max_seq_len_tensor = std::nullopt, - const std::optional& contextual_seq_len_tensor = std::nullopt, - const std::optional& max_attn_len_tensor = std::nullopt, - const std::optional& min_full_attn_seq_len_tensor = - std::nullopt, - int64_t num_groups = 1) { - ctx->saved_data["max_seq_len"] = max_seq_len; - ctx->saved_data["alpha"] = alpha; - ctx->saved_data["causal"] = causal; - ctx->saved_data["max_attn_len"] = max_attn_len; - ctx->saved_data["min_full_attn_seq_len"] = min_full_attn_seq_len; - ctx->saved_data["contextual_seq_len"] = contextual_seq_len; - ctx->saved_data["deterministic"] = deterministic; - ctx->saved_data["sort_by_length"] = sort_by_length; - ctx->saved_data["sm_margin"] = sm_margin; - ctx->saved_data["max_q_len"] = max_q_len; - ctx->saved_data["num_softmax_heads"] = num_softmax_heads; - ctx->saved_data["num_groups"] = num_groups; - auto fwd_out = hstu::hstu_mha_fwd( - max_seq_len, // max_seq_len - alpha, // alpha - q, // q - k, // k - v, // v - seq_offsets, // seq_offsets - causal, // causal - num_targets, // num_targets - attn_scale, // attn_scale - max_attn_len, // max_attn_len - min_full_attn_seq_len, // min_full_attn_seq_len - contextual_seq_len, // contextual_seq_len - q_descale, // q_descale - k_descale, // k_descale - v_descale, // v_descale - sm_margin, // sm_margin - max_q_len, // max_q_len - seq_offsets_q, // seq_offsets_q - num_softmax_heads, // num_softmax_heads - training, - max_seq_len_tensor, - contextual_seq_len_tensor, - max_attn_len_tensor, - min_full_attn_seq_len_tensor, - num_groups); - auto out = get<0>(fwd_out); - auto softmax_lse = get<1>(fwd_out); - ctx->save_for_backward( - {q, - k, - v, - out, - seq_offsets.value_or(at::Tensor()), - num_targets.value_or(at::Tensor()), - attn_scale.value_or(at::Tensor()), - seq_offsets_q.value_or(at::Tensor()), - softmax_lse.value_or(at::Tensor()), - max_seq_len_tensor.value_or(at::Tensor()), - contextual_seq_len_tensor.value_or(at::Tensor()), - max_attn_len_tensor.value_or(at::Tensor()), - min_full_attn_seq_len_tensor.value_or(at::Tensor())}); - return out; - } - - static torch::autograd::variable_list backward( - torch::autograd::AutogradContext* ctx, - torch::autograd::variable_list grad_outputs) { - auto saved_tensors = ctx->get_saved_variables(); - auto saved_data = ctx->saved_data; - auto q = saved_tensors[0]; - auto k = saved_tensors[1]; - auto v = saved_tensors[2]; - auto out = saved_tensors[3]; - auto seq_offsets = saved_tensors[4]; - auto num_targets = saved_tensors[5]; - auto attn_scale = saved_tensors[6]; - auto seq_offsets_q = saved_tensors[7]; - auto softmax_lse = saved_tensors[8]; - auto max_seq_len_tensor = saved_tensors[9]; - auto contextual_seq_len_tensor = saved_tensors[10]; - auto max_attn_len_tensor = saved_tensors[11]; - auto min_full_attn_seq_len_tensor = saved_tensors[12]; - auto seq_offsets_opt = - seq_offsets.defined() ? std::optional(seq_offsets) : std::nullopt; - auto num_targets_opt = - num_targets.defined() ? std::optional(num_targets) : std::nullopt; - auto attn_scale_opt = - attn_scale.defined() ? std::optional(attn_scale) : std::nullopt; - auto seq_offsets_q_opt = - seq_offsets_q.defined() ? std::optional(seq_offsets_q) : std::nullopt; - auto softmax_lse_opt = - softmax_lse.defined() ? std::optional(softmax_lse) : std::nullopt; - auto max_seq_len_tensor_opt = max_seq_len_tensor.defined() - ? std::optional(max_seq_len_tensor) - : std::nullopt; - auto contextual_seq_len_tensor_opt = contextual_seq_len_tensor.defined() - ? std::optional(contextual_seq_len_tensor) - : std::nullopt; - auto max_attn_len_tensor_opt = max_attn_len_tensor.defined() - ? std::optional(max_attn_len_tensor) - : std::nullopt; - auto min_full_attn_seq_len_tensor_opt = - min_full_attn_seq_len_tensor.defined() - ? std::optional(min_full_attn_seq_len_tensor) - : std::nullopt; - - auto dq = at::empty_like(q); - auto dk = at::empty_like(k); - auto dv = at::empty_like(v); - - auto bwd_res = hstu::hstu_mha_bwd( - saved_data["max_seq_len"].toInt(), // max_seq_len - saved_data["alpha"].toDouble(), // alpha - grad_outputs[0], // dout - q, // q - k, // k - v, // v - dq, // dq - dk, // dk - dv, // dv - out, // out - seq_offsets_opt, // seq_offsets - saved_data["causal"].toBool(), // causal - num_targets_opt, // num_targets - attn_scale_opt, // attn_scale - saved_data["max_attn_len"].toInt(), // max_attn_len - saved_data["min_full_attn_seq_len"].toInt(), // min_full_attn_seq_len - saved_data["contextual_seq_len"].toInt(), // contextual_seq_len - saved_data["sort_by_length"].toBool(), // sort_by_length - saved_data["deterministic"].toBool(), // deterministic - saved_data["sm_margin"].toInt(), // sm_margin - saved_data["max_q_len"].toInt(), // max_q_len - seq_offsets_q_opt, // seq_offsets_q - saved_data["num_softmax_heads"].toInt(), // num_softmax_heads - softmax_lse_opt, - max_seq_len_tensor_opt, - contextual_seq_len_tensor_opt, - max_attn_len_tensor_opt, - min_full_attn_seq_len_tensor_opt, - saved_data["num_groups"].toInt()); - - return { - torch::autograd::Variable(), // max_seq_len - torch::autograd::Variable(), // alpha - bwd_res[0], // dq - bwd_res[1], // dk - bwd_res[2], // dv - torch::autograd::Variable(), // seq_offsets - torch::autograd::Variable(), // causal - torch::autograd::Variable(), // num_targets - torch::autograd::Variable(), // attn_scale - torch::autograd::Variable(), // max_attn_len - torch::autograd::Variable(), // min_full_attn_seq_len - torch::autograd::Variable(), // contextual_seq_len - torch::autograd::Variable(), // q_descale - torch::autograd::Variable(), // k_descale - torch::autograd::Variable(), // v_descale - torch::autograd::Variable(), // sort_by_length - torch::autograd::Variable(), // deterministic - torch::autograd::Variable(), // sm_margin - torch::autograd::Variable(), // max_q_len - torch::autograd::Variable(), // seq_offsets_q - torch::autograd::Variable(), // num_softmax_heads - torch::autograd::Variable(), // training - torch::autograd::Variable(), // max_seq_len_tensor - torch::autograd::Variable(), // contextual_seq_len_tensor - torch::autograd::Variable(), // max_attn_len_tensor - torch::autograd::Variable(), // min_full_attn_seq_len_tensor - torch::autograd::Variable(), // num_groups - }; - } -}; - -at::Tensor cuda_hstu_mha( - int64_t max_seq_len, - double alpha, - at::Tensor& q, // (b, s, h, d) or (total_s, h, d) - at::Tensor& k, // (b, s, h, d) or (total_s, h, d) - at::Tensor& v, // (b, s, h, d) or (total_s, h, d) - const std::optional& seq_offsets, - bool causal, - const std::optional& num_targets, - const std::optional& attn_scale, - int64_t max_attn_len, - int64_t min_full_attn_seq_len, - int64_t contextual_seq_len, - const std::optional& q_descale, // (b, h_k), not (b, h) - const std::optional& k_descale, // (b, h_k) - const std::optional& v_descale, // (b, h_k) - bool sort_by_length, - bool deterministic, - const int64_t sm_margin = 0, - int64_t max_q_len = 0, - const std::optional& seq_offsets_q = std::nullopt, - int64_t num_softmax_heads = 0, - bool training = true, - const std::optional& max_seq_len_tensor = std::nullopt, - const std::optional& contextual_seq_len_tensor = std::nullopt, - const std::optional& max_attn_len_tensor = std::nullopt, - const std::optional& min_full_attn_seq_len_tensor = - std::nullopt, - int64_t num_groups = 1) { - return hstu::HSTUFlashAttentionFunctionGPU::apply( - max_seq_len, - alpha, - q, - k, - v, - seq_offsets, - causal, - num_targets, - attn_scale, - max_attn_len, - min_full_attn_seq_len, - contextual_seq_len, - q_descale, - k_descale, - v_descale, - sort_by_length, - deterministic, - sm_margin, - max_q_len, - seq_offsets_q, - num_softmax_heads, - training, - max_seq_len_tensor, - contextual_seq_len_tensor, - max_attn_len_tensor, - min_full_attn_seq_len_tensor, - num_groups); -} - -TORCH_LIBRARY_FRAGMENT(hstu, m) { - m.impl( - "hstu_mha", - torch::dispatch(c10::DispatchKey::CUDA, TORCH_FN(cuda_hstu_mha))); - - m.impl( - "hstu_mha_fwd", - torch::dispatch(c10::DispatchKey::CUDA, TORCH_FN(hstu::hstu_mha_fwd))); - - m.impl( - "hstu_mha_bwd", - torch::dispatch(c10::DispatchKey::CUDA, TORCH_FN(hstu::hstu_mha_bwd))); -} -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_api_cpu.cpp b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_api_cpu.cpp deleted file mode 100644 index c02424efe..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_api_cpu.cpp +++ /dev/null @@ -1,256 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/****************************************************************************** - * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, - *Pradeep Ramani, Tri Dao. - ******************************************************************************/ - -#include // @manual -#include -#include "flash_common_cpu.h" - -namespace hstu { - -at::Tensor hstu_mha_cpu( - int64_t max_seq_len, - double alpha, - at::Tensor& q, // (b, s, h, d) or (total_s, h, d) - at::Tensor& k, // (b, s, h, d) or (total_s, h, d) - at::Tensor& v, // (b, s, h, d) or (total_s, h, d) - const std::optional& seq_offsets, - bool causal, - const std::optional& num_targets, - const std::optional& attn_scale, - int64_t max_attn_len, - int64_t min_full_attn_seq_len, - int64_t contextual_seq_len, - const std::optional& q_descale, // (b, h_k), not (b, h) - const std::optional& k_descale, // (b, h_k) - const std::optional& v_descale, // (b, h_k) - bool sort_by_length, - bool deterministic, - const int64_t sm_margin = 0, - int64_t max_q_len = 0, - const std::optional& seq_offsets_q = std::nullopt, - int64_t num_softmax_heads = 0, - bool training = true, - const std::optional& max_seq_len_tensor = std::nullopt, - const std::optional& contextual_seq_len_tensor = std::nullopt, - const std::optional& max_attn_len_tensor = std::nullopt, - const std::optional& min_full_attn_seq_len_tensor = - std::nullopt, - int64_t num_groups = 1) { - auto fwd_out = hstu::hstu_mha_fwd_dummy( - max_seq_len, - alpha, - q, - k, - v, - seq_offsets, - causal, - num_targets, - attn_scale, - max_attn_len, - min_full_attn_seq_len, - contextual_seq_len, - q_descale, - k_descale, - v_descale, - sm_margin, - max_q_len, - seq_offsets_q, - num_softmax_heads, - training); - return get<0>(fwd_out); -} - -at::Tensor hstu_mha_meta( - const at::SymInt max_seq_len, - double alpha, - at::Tensor& q, // (b, s, h, d) or (total_s, h, d) - at::Tensor& k, // (b, s, h, d) or (total_s, h, d) - at::Tensor& v, // (b, s, h, d) or (total_s, h, d) - const std::optional& seq_offsets, - bool causal, - const std::optional& num_targets, - const std::optional& attn_scale, - int64_t max_attn_len, - int64_t min_full_attn_seq_len, - int64_t contextual_seq_len, - const std::optional& q_descale, // (b, h_k), not (b, h) - const std::optional& k_descale, // (b, h_k) - const std::optional& v_descale, // (b, h_k) - bool sort_by_length, - bool deterministic, - const int64_t sm_margin = 0, - int64_t max_q_len = 0, - const std::optional& seq_offsets_q = std::nullopt, - int64_t num_softmax_heads = 0, - bool training = true, - const std::optional& max_seq_len_tensor = std::nullopt, - const std::optional& contextual_seq_len_tensor = std::nullopt, - const std::optional& max_attn_len_tensor = std::nullopt, - const std::optional& min_full_attn_seq_len_tensor = - std::nullopt, - int64_t num_groups = 1) { - auto fwd_out = hstu::hstu_mha_fwd_meta( - max_seq_len, - alpha, - q, - k, - v, - seq_offsets, - causal, - num_targets, - attn_scale, - max_attn_len, - min_full_attn_seq_len, - contextual_seq_len, - q_descale, - k_descale, - v_descale, - sm_margin, - max_q_len, - seq_offsets_q, - num_softmax_heads, - training); - return get<0>(fwd_out); -} - -// CPU-only implementation that registers under main hstu namespace -// This provides fallback implementations when GPU code is not compiled -TORCH_LIBRARY_FRAGMENT(hstu, m) { - // Only register operators if they haven't been registered by GPU code - // This allows CPU-only builds to work while GPU builds use GPU - // implementations - - m.def( - "hstu_mha_fwd(" - "SymInt max_seq_len, " - "float alpha, " - "Tensor q, " - "Tensor k, " - "Tensor v, " - "Tensor? seq_offsets, " - "bool causal, " - "Tensor? num_targets, " - "Tensor? attn_scale, " - "int max_attn_len, " - "int min_full_attn_seq_len, " - "int contextual_seq_len, " - "Tensor? q_descale, " - "Tensor? k_descale, " - "Tensor? v_descale, " - "int sm_margin = 0," - "int max_q_len = 0," - "Tensor? seq_offsets_q = None," - "int num_softmax_heads = 0," - "bool training = True," - "Tensor? max_seq_len_tensor = None," - "Tensor? contextual_seq_len_tensor = None," - "Tensor? max_attn_len_tensor = None," - "Tensor? min_full_attn_seq_len_tensor = None," - "int num_groups = 1" - ") -> (Tensor, Tensor?)"); - - m.def( - "hstu_mha_bwd(" - "int max_seq_len, " - "float alpha, " - "Tensor dout, " - "Tensor q, " - "Tensor k, " - "Tensor v, " - "Tensor dq, " - "Tensor dk, " - "Tensor dv, " - "Tensor out, " - "Tensor? seq_offsets, " - "bool causal, " - "Tensor? num_targets, " - "Tensor? attn_scale, " - "int max_attn_len, " - "int min_full_attn_seq_len, " - "int contextual_seq_len, " - "bool sort_by_length," - "bool deterministic," - "int sm_margin = 0," - "int max_q_len = 0," - "Tensor? seq_offsets_q = None," - "int num_softmax_heads = 0," - "Tensor? softmax_lse = None," - "Tensor? max_seq_len_tensor = None," - "Tensor? contextual_seq_len_tensor = None," - "Tensor? max_attn_len_tensor = None," - "Tensor? min_full_attn_seq_len_tensor = None," - "int num_groups = 1" - ") -> Tensor[]"); - - m.def( - "hstu_mha(" - "SymInt max_seq_len, " - "float alpha, " - "Tensor q, " - "Tensor k, " - "Tensor v, " - "Tensor? seq_offsets, " - "bool causal, " - "Tensor? num_targets, " - "Tensor? attn_scale, " - "int max_attn_len, " - "int min_full_attn_seq_len, " - "int contextual_seq_len, " - "Tensor? q_descale, " - "Tensor? k_descale, " - "Tensor? v_descale, " - "bool sort_by_length, " - "bool deterministic, " - "int sm_margin = 0," - "int max_q_len = 0," - "Tensor? seq_offsets_q = None," - "int num_softmax_heads = 0," - "bool training = True," - "Tensor? max_seq_len_tensor = None," - "Tensor? contextual_seq_len_tensor = None," - "Tensor? max_attn_len_tensor = None," - "Tensor? min_full_attn_seq_len_tensor = None," - "int num_groups = 1" - ") -> Tensor"); - - // Register CPU implementations - m.impl( - "hstu_mha", - torch::dispatch(c10::DispatchKey::CPU, TORCH_FN(hstu_mha_cpu))); - m.impl( - "hstu_mha", - torch::dispatch(c10::DispatchKey::Meta, TORCH_FN(hstu_mha_meta))); - - m.impl( - "hstu_mha_fwd", - torch::dispatch( - c10::DispatchKey::CPU, TORCH_FN(hstu::hstu_mha_fwd_dummy))); - m.impl( - "hstu_mha_fwd", - torch::dispatch( - c10::DispatchKey::Meta, TORCH_FN(hstu::hstu_mha_fwd_meta))); - - m.impl( - "hstu_mha_bwd", - torch::dispatch( - c10::DispatchKey::CPU, TORCH_FN(hstu::hstu_mha_bwd_dummy))); -} - -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_bwd_kernel_sm90.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_bwd_kernel_sm90.h deleted file mode 100644 index 051d5141b..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_bwd_kernel_sm90.h +++ /dev/null @@ -1,402 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/****************************************************************************** - * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, - *Pradeep Ramani, Tri Dao. - ******************************************************************************/ - -#pragma once - -#include "cute/tensor.hpp" - -#include -#include -#include -#include -#include -#include -#include "cutlass/pipeline/pipeline.hpp" - -#include "tile_scheduler.h" -#include "utils.h" - -namespace hstu { - -using namespace cute; - -template < - bool Softmax, - class CollectiveMainloop_, - class CollectiveEpilogue_, - class TileScheduler_> -class FlashAttnBwdSm90 { - public: - // Mainloop derived types - using CollectiveMainloop = CollectiveMainloop_; - using TileShape_MNK = typename CollectiveMainloop::TileShape_MNK; - using TiledMmaSdP = typename CollectiveMainloop::TiledMmaSdP; - using TiledMmadKV = typename CollectiveMainloop::TiledMmadKV; - using ArchTag = typename CollectiveMainloop::ArchTag; - using ClusterShape = typename CollectiveMainloop::ClusterShape; - using MainloopArguments = typename CollectiveMainloop::Arguments; - using MainloopParams = typename CollectiveMainloop::Params; - static constexpr bool dKV_swapAB = CollectiveMainloop::dKV_swapAB; - - // Epilogue derived types - using CollectiveEpilogue = CollectiveEpilogue_; - using EpilogueArguments = typename CollectiveEpilogue::Arguments; - using EpilogueParams = typename CollectiveEpilogue::Params; - - static_assert(ArchTag::kMinComputeCapability >= 90); - - using TileScheduler = TileScheduler_; - using TileSchedulerArguments = typename hstu::TileSchedulerArguments; - using TileSchedulerParams = typename TileScheduler::Params; - - static constexpr uint32_t NumLoadWarpGroups = 1; - static constexpr uint32_t NumMmaWarpGroups = - CUTE_STATIC_V(size(TiledMmaSdP{})) / cutlass::NumThreadsPerWarpGroup; - static constexpr uint32_t MaxThreadsPerBlock = - CUTE_STATIC_V(size(TiledMmaSdP{})) + - (NumLoadWarpGroups * cutlass::NumThreadsPerWarpGroup); - static constexpr uint32_t MinBlocksPerMultiprocessor = 1; - static_assert(NumMmaWarpGroups == 2 || NumMmaWarpGroups == 3); - - /// Register requirement for Load and Math WGs - static constexpr uint32_t LoadRegisterRequirement = - NumMmaWarpGroups == 2 ? 24 : 32; - static constexpr uint32_t MmaRegisterRequirement = - NumMmaWarpGroups == 2 ? 240 : 160; - // If you want to print from the producer warp, you'd need to increase the - // number of registers Otherwise you'll get CUDA error. static constexpr - // uint32_t LoadRegisterRequirement = 40; static constexpr uint32_t - // MmaRegisterRequirement = NumMmaWarpGroups == 2 ? 232 : 152; - - // Kernel level shared memory storage - struct SharedStorage { - struct TensorStorage : cute::aligned_struct<128> { - union { - typename CollectiveMainloop::TensorStorage mainloop; - typename CollectiveEpilogue::TensorStorage epilogue; - }; - } tensors; - - struct PipelineStorage : cute::aligned_struct<16> { - alignas(16) cutlass::arch::ClusterTransactionBarrier barrier_KV; - alignas(16) typename CollectiveMainloop::MainloopPipeline::SharedStorage - pipeline_q; - alignas(16) - typename CollectiveMainloop::MainloopPipeline_dO::SharedStorage - pipeline_do; - alignas(16) typename TileScheduler::SharedStorage smem_scheduler; - } pipelines; - }; - - static constexpr int SharedStorageSize = sizeof(SharedStorage); - - // Device side arguments - struct Arguments { - MainloopArguments mainloop{}; - EpilogueArguments epilogue{}; - cutlass::KernelHardwareInfo hw_info{}; - TileSchedulerArguments scheduler{}; - }; - - // Kernel entry point API - struct Params { - MainloopParams mainloop{}; - EpilogueParams epilogue{}; - cutlass::KernelHardwareInfo hw_info{}; - TileSchedulerParams scheduler{}; - }; - - // - // Methods - // - - // Convert to underlying arguments. In this case, a simple copy for the - // aliased type. - static Params to_underlying_arguments(Arguments const& args) { - CUTLASS_TRACE_HOST("to_underlying_arguments():"); - - // Get SM count if needed, otherwise use user supplied SM count - int sm_count = args.hw_info.sm_count; - if (sm_count <= 0) { - CUTLASS_TRACE_HOST( - " WARNING: Arguments do not include a valid SM count.\n" - " For optimal performance, populate the arguments KernelHardwareInfo struct with the SM count."); - sm_count = cutlass::KernelHardwareInfo::query_device_multiprocessor_count( - args.hw_info.device_id); - } - - CUTLASS_TRACE_HOST( - "to_underlying_arguments(): Setting persistent grid SM count to " - << sm_count); - - cutlass::KernelHardwareInfo hw_info{args.hw_info.device_id, sm_count}; - return { - CollectiveMainloop::to_underlying_arguments(args.mainloop), - CollectiveEpilogue::to_underlying_arguments(args.epilogue), - hw_info, - TileScheduler::to_underlying_arguments(args.scheduler)}; - } - - // Computes the kernel launch grid shape based on runtime parameters - static dim3 get_grid_shape(Params const& params) { - return TileScheduler::get_grid_shape( - params.scheduler, params.hw_info.sm_count); - } - - static dim3 get_block_shape() { - return dim3(MaxThreadsPerBlock, 1, 1); - } - - CUTLASS_DEVICE - void operator()(Params const& params, char* smem_buf) { - static constexpr int NumMmaThreads = - NumMmaWarpGroups * cutlass::NumThreadsPerWarpGroup; - static constexpr int NumCopyThreads = - NumLoadWarpGroups * cutlass::NumThreadsPerWarpGroup; - - using MainloopPipeline = typename CollectiveMainloop::MainloopPipeline; - using PipelineParams = typename MainloopPipeline::Params; - using PipelineState = typename MainloopPipeline::PipelineState; - using MainloopPipeline_dO = - typename CollectiveMainloop::MainloopPipeline_dO; - using PipelineParams_dO = typename MainloopPipeline_dO::Params; - using PipelineState_dO = typename MainloopPipeline_dO::PipelineState; - static constexpr bool Q_dO_same_stages = - std::is_same_v; - - SharedStorage& shared_storage = *reinterpret_cast(smem_buf); - - int const lane_predicate = cute::elect_one_sync(); - int const warp_idx = cutlass::canonical_warp_idx_sync(); - - // Issue Tma Descriptor Prefetch from a single thread - if (warp_idx == 0 && lane_predicate) { - CollectiveMainloop::prefetch_tma_descriptors(params.mainloop); - CollectiveEpilogue::prefetch_tma_descriptors(params.epilogue); - } - - // Obtain warp index - int const warp_group_thread_idx = - threadIdx.x % cutlass::NumThreadsPerWarpGroup; - - PipelineParams pipeline_params; - if constexpr (Softmax) { - pipeline_params.transaction_bytes = - CollectiveMainloop::TmaTransactionBytesQ + - CollectiveMainloop::TmaTransactionBytesLSE; - } else { - pipeline_params.transaction_bytes = - CollectiveMainloop::TmaTransactionBytesQ; - } - int warp_group_idx = cutlass::canonical_warp_group_idx(); - pipeline_params.role = warp_group_idx == 0 - ? MainloopPipeline::ThreadCategory::Producer - : MainloopPipeline::ThreadCategory::Consumer; - pipeline_params.is_leader = warp_group_thread_idx == 0; - pipeline_params.num_consumers = NumMmaThreads; - - if (warp_idx == 0 && lane_predicate) { - shared_storage.pipelines.barrier_KV.init(1 /*numThreads*/); - } - // We're counting on pipeline_q to call cutlass::arch::fence_barrier_init(); - MainloopPipeline pipeline_q( - shared_storage.pipelines.pipeline_q, pipeline_params, ClusterShape{}); - auto role_dO = warp_group_idx == 0 - ? MainloopPipeline_dO::ThreadCategory::Producer - : MainloopPipeline_dO::ThreadCategory::Consumer; - PipelineParams_dO pipeline_params_dO{ - pipeline_params.transaction_bytes, - role_dO, - pipeline_params.is_leader, - pipeline_params.num_consumers}; - MainloopPipeline_dO pipeline_do( - shared_storage.pipelines.pipeline_do, - cute::conditional_return( - pipeline_params, pipeline_params_dO), - ClusterShape{}); - - CollectiveMainloop collective_mainloop; - CollectiveEpilogue collective_epilogue; - - // We need this to guarantee that the Pipeline init is visible to all - // producers and consumer blocks in the Cluster - if constexpr (size(ClusterShape{}) > 1) { - cute::cluster_arrive_relaxed(); - cute::cluster_wait(); - } else { - __syncthreads(); - } - - if (warp_group_idx == 0) { // Producer - cutlass::arch::warpgroup_reg_dealloc(); - - int warp_idx_in_warpgroup = - __shfl_sync(0xffffffff, (threadIdx.x / 32) % 4, 0); - if (warp_idx_in_warpgroup == 0) { // Load K, V, and do TMA on Q and dO - PipelineState smem_pipe_write = - cutlass::make_producer_start_state(); - PipelineState_dO smem_pipe_write_do = - cutlass::make_producer_start_state(); - - TileScheduler scheduler( - reinterpret_cast( - &shared_storage.pipelines.smem_scheduler)); - for (auto work_tile_info = - scheduler.template get_initial_work( - params.scheduler); - work_tile_info.is_valid(params.scheduler); - work_tile_info = - scheduler.template get_next_work( - params.scheduler, work_tile_info)) { - auto block_coord_ = work_tile_info.get_block_coord(params.scheduler); - auto [n_block, bidh, bidb, _ /*split_idx*/] = block_coord_; - cute::tuple block_coord = { - n_block, bidh, bidb}; - auto scheduler_prefetch = [&scheduler, ¶ms, &work_tile_info]() { - scheduler.prefetch_next_work(params.scheduler, work_tile_info); - }; - collective_mainloop.load( - params.mainloop, - pipeline_q, - pipeline_do, - smem_pipe_write, - smem_pipe_write_do, - shared_storage, - scheduler_prefetch, - block_coord); - } - collective_mainloop.load_tail( - pipeline_q, pipeline_do, smem_pipe_write, smem_pipe_write_do); - } else if (warp_idx_in_warpgroup == 1) { - TileScheduler scheduler( - reinterpret_cast( - &shared_storage.pipelines.smem_scheduler)); - for (auto work_tile_info = - scheduler.template get_initial_work( - params.scheduler); - work_tile_info.is_valid(params.scheduler); - work_tile_info = - scheduler.template get_next_work( - params.scheduler, work_tile_info)) { - auto block_coord_ = work_tile_info.get_block_coord(params.scheduler); - auto [n_block, bidh, bidb, _ /*split_idx*/] = block_coord_; - cute::tuple block_coord = { - n_block, bidh, bidb}; - collective_mainloop.store_dq( - params.mainloop, shared_storage, block_coord); - } - } - } else { // Consumer - cutlass::arch::warpgroup_reg_alloc(); - - TileScheduler scheduler( - reinterpret_cast( - &shared_storage.pipelines.smem_scheduler)); - // Initialize matmul objects. - TiledMmadKV tiled_mma_dKV; - - PipelineState smem_pipe_read; - PipelineState_dO smem_pipe_read_do; - - collective_mainloop.mma_init(); - scheduler.init_consumer(); - - int work_idx = 0; - CUTLASS_PRAGMA_NO_UNROLL - for (auto work_tile_info = - scheduler.template get_initial_work( - params.scheduler); - work_tile_info.is_valid(params.scheduler); - work_tile_info = - scheduler.template get_next_work( - params.scheduler, work_tile_info)) { - auto block_coord_ = work_tile_info.get_block_coord(params.scheduler); - auto [n_block, bidh, bidb, _ /*split_idx*/] = block_coord_; -#ifdef HSTU_FLASH_ATTN_DEBUG_INFO - if (threadIdx.x == 0 || threadIdx.x == 128) { - std::printf( - "n_block: (%d), bidh: (%d), bidb: (%d), blockIdx.x: (%d), blockIdx.y: (%d), blockIdx.z: (%d)\n", - n_block, - bidh, - bidb, - blockIdx.x, - blockIdx.y, - blockIdx.z); - } -#endif - cute::tuple block_coord = { - n_block, bidh, bidb}; - - // dK and dV output accumulator. - Tensor tdKrdK = partition_fragment_C( - tiled_mma_dKV, - select(TileShape_MNK{})); - Tensor tdVrdV = partition_fragment_C( - tiled_mma_dKV, - select(TileShape_MNK{})); - - bool tile_valid; - if constexpr (Softmax) { - tile_valid = collective_mainloop.mma_softmax( - params.mainloop, - pipeline_q, - pipeline_do, - smem_pipe_read, - smem_pipe_read_do, - tdKrdK, - tdVrdV, - threadIdx.x - NumCopyThreads, - work_idx, - block_coord, - shared_storage); - } else { - tile_valid = collective_mainloop.mma( - params.mainloop, - pipeline_q, - pipeline_do, - smem_pipe_read, - smem_pipe_read_do, - tdKrdK, - tdVrdV, - threadIdx.x - NumCopyThreads, - work_idx, - block_coord, - shared_storage); - } - if (tile_valid) { - collective_epilogue.store( - params.epilogue, - tdKrdK, - tdVrdV, - shared_storage, - tiled_mma_dKV, - threadIdx.x - NumCopyThreads, - block_coord); - } else { - collective_epilogue.store_zero( - params.epilogue, threadIdx.x - NumCopyThreads, block_coord); - } - } - collective_epilogue.store_tail(); - } - } -}; - -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_bwd_launch_template.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_bwd_launch_template.h deleted file mode 100644 index 6900852df..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_bwd_launch_template.h +++ /dev/null @@ -1,492 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/****************************************************************************** - * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, - *Pradeep Ramani, Tri Dao. - ******************************************************************************/ - -#pragma once - -#include "cute/tensor.hpp" - -#include "cutlass/cluster_launch.hpp" // For ClusterLauncher -#include "cutlass/device_kernel.h" // For device_kernel -#include "cutlass/kernel_launch.h" // For kernel_launch - -#include "epilogue_bwd.h" -#include "flash.h" -#include "flash_bwd_kernel_sm90.h" -#include "flash_bwd_postprocess_kernel.h" -#include "flash_bwd_preprocess_kernel.h" -#include "mainloop_bwd_sm90_tma_gmma_ws.h" -#include "static_switch.h" -#include "tile_scheduler.h" -#include "tile_size.h" - -namespace hstu { - -using namespace cute; - -template < - int Arch, - int kHeadDim, - int kBlockM, - int kBlockN, - typename Element, - bool Causal, - bool Local, - bool Contexual_mask, - bool Jagged, - bool Has_targets, - bool Deterministic, - int Stages_dO = 2, - int Stages_dS_or_QSm80 = 2, - bool SdP_swapAB = true, - bool dKV_swapAB = false, - bool dQ_swapAB = false, - int NumMmaWarpGroups = 2, - int AtomLayoutMSdP = 1, - int AtomLayoutNdKV = 2, - int AtomLayoutMdQ = 1, - bool V_in_regs = false, - bool Cross = false, - bool Softmax = false> -void run_flash_bwd(hstu::Flash_bwd_params& params, cudaStream_t stream) { -#ifdef HSTU_FLASH_ATTN_DEBUG_INFO - std::printf( - "[flash_bwd_launch_template] Local: (%d), Jagged: (%d), Has_targets: (%d), Causal: (%d), max_kv_len: (%d), kHeadDim: (%d), kBlockM: (%d), kBlockN: (%d)\n", - Local, - Jagged, - Has_targets, - Causal, - params.max_kv_len, - kHeadDim, - kBlockM, - kBlockN); -#endif - static_assert( - !(Causal && Local), "Causal and Local cannot be true at the same time."); - using ElementAccum = float; - using ArchTag = - std::conditional_t= 90, cutlass::arch::Sm90, cutlass::arch::Sm80>; - - int const total_q_padded_rounded = - cute::round_up(params.total_seq_len_q + params.b * kBlockM, kBlockM); - int seqlen_q = !Jagged ? params.max_q_len : params.total_seq_len_q; - int seqlen_kv = !Jagged ? params.max_kv_len : params.total_seq_len_kv; - int seqlen_q_rounded = - !Jagged ? params.max_q_len_rounded : total_q_padded_rounded; - int batch = !Jagged ? params.b : 1; - - using TileShape_MK = cute::Shape, Int>; - using PreprocessKernel = hstu::FlashAttnBwdPreprocess< - TileShape_MK, - Element, - ElementAccum, - ArchTag, - /*Clear_dQaccum=*/true, - Jagged, - Softmax>; - typename PreprocessKernel::Arguments preprocess_args{ - static_cast(params.o_ptr), - {seqlen_q, params.v_d, params.h, batch}, // shape_O - {params.o_row_stride, - _1{}, - params.o_head_stride, - !Jagged ? params.o_batch_stride : 0}, // stride_O - static_cast(params.do_ptr), - {params.do_row_stride, - _1{}, - params.do_head_stride, - !Jagged ? params.do_batch_stride : 0}, // stride_dO - static_cast(params.softmax_d), - {seqlen_q_rounded, params.num_softmax_heads, batch}, // shape_dPsum - {_1{}, - seqlen_q_rounded, - !Jagged ? params.num_softmax_heads * params.max_q_len_rounded - : 0}, // stride_dPsum - static_cast(params.softmax_lse), - {_1{}, - seqlen_q, - !Jagged ? params.num_softmax_heads * params.max_q_len_rounded - : 0}, // stride_LSE - static_cast(params.softmax_lse_log2), - {_1{}, - seqlen_q_rounded, - !Jagged ? params.num_softmax_heads * params.max_q_len_rounded - : 0}, // stride_LSE_log2 - static_cast(params.dq_accum_ptr), - {seqlen_q_rounded * params.qk_d_rounded, - params.h, - batch}, // shape_dQaccum - {_1{}, - seqlen_q_rounded * params.qk_d_rounded, - !Jagged ? params.qk_d_rounded * params.max_q_len_rounded * params.h - : 0}, // stride_dQaccum - params.b, - params.h, - params.num_softmax_heads, - params.max_q_len, - params.dq_semaphore, - Cross ? params.seq_offsets_q : params.seq_offsets}; - typename PreprocessKernel::Params preprocess_params = - PreprocessKernel::to_underlying_arguments(preprocess_args); - int num_m_block = cute::ceil_div(params.max_q_len, kBlockM); - dim3 grid_m(num_m_block, params.h, params.b); - cutlass::kernel_launch( - grid_m, - PreprocessKernel::MaxThreadsPerBlock, - PreprocessKernel::SharedStorageSize, - stream, - preprocess_params, - false /*launch_with_pdl*/); - CHECK_CUDA_KERNEL_LAUNCH(); - - using TileShape_MNK = cute::Shape, Int, Int>; - using ClusterShape = - cute::Shape<_1, Int<1>, _1>; // Currently doesn't not support cluster - // Stages_dS_or_QSm80 is Stages_dS if Sm90 and Stages if Sm80 - static constexpr int Stages = Arch >= 90 ? 2 : Stages_dS_or_QSm80; - static constexpr int Stages_dS = Arch >= 90 ? Stages_dS_or_QSm80 : 1; - using CollectiveMainloop = hstu::CollectiveMainloopBwdSm90< - Stages, - Stages_dO, - Stages_dS, - ClusterShape, - TileShape_MNK, - Element, - ElementAccum, - cutlass::arch::Sm90, - Causal, - Local, - Contexual_mask, - Jagged, - Has_targets, - Deterministic, - SdP_swapAB, - dKV_swapAB, - dQ_swapAB, - NumMmaWarpGroups, - AtomLayoutMSdP, - AtomLayoutNdKV, - AtomLayoutMdQ, - V_in_regs, - Cross, - Softmax>; - using CollectiveEpilogue = hstu::CollectiveEpilogueBwd< - TileShape_MNK, - Element, - ArchTag, - CollectiveMainloop::NumMmaThreads, - Jagged, - dKV_swapAB, - NumMmaWarpGroups*(Arch >= 90 ? 1 : cutlass::NumWarpsPerWarpGroup) / - AtomLayoutNdKV>; - using Scheduler = - hstu::SingleTileScheduler; - using AttnKernel = hstu::enable_sm90_or_later>; - - typename CollectiveMainloop::Arguments mainloop_args{ - static_cast(params.q_ptr), - {seqlen_q, params.qk_d, params.h, batch}, // shape_Q - {params.q_row_stride, - _1{}, - params.q_head_stride, - !Jagged ? params.q_batch_stride : 0}, // stride_Q - static_cast(params.k_ptr), - {seqlen_kv, params.qk_d, params.h, batch}, // shape_K - {params.k_row_stride, - _1{}, - params.k_head_stride, - !Jagged ? params.k_batch_stride : 0}, // stride_K - static_cast(params.v_ptr), - {seqlen_kv, params.v_d, params.h, batch}, // shape_V - {params.v_row_stride, - _1{}, - params.v_head_stride, - !Jagged ? params.v_batch_stride : 0}, // stride_V - static_cast(params.do_ptr), - {seqlen_q, params.v_d, params.h, batch}, // shape_dO - {params.do_row_stride, - _1{}, - params.do_head_stride, - !Jagged ? params.do_batch_stride : 0}, // stride_dO - static_cast(params.dq_accum_ptr), - {seqlen_q_rounded * params.qk_d_rounded, - params.h, - batch}, // shape_dQaccum - {_1{}, - seqlen_q_rounded * params.qk_d_rounded, - !Jagged ? params.qk_d_rounded * params.max_q_len_rounded * params.h - : 0}, // stride_dQaccum - static_cast(params.softmax_lse_log2), - {seqlen_q_rounded, params.num_softmax_heads, batch}, // shape_LSE - {_1{}, - seqlen_q_rounded, - !Jagged ? params.num_softmax_heads * params.max_q_len_rounded - : 0}, // stride_LSE_log2 - static_cast(params.softmax_d), - {_1{}, - seqlen_q_rounded, - !Jagged ? params.num_softmax_heads * params.max_q_len_rounded - : 0}, // stride_dPsum - params.max_attn_len, - params.min_full_attn_seq_len, - params.contextual_seq_len, - 1.0f / params.max_kv_len, - params.alpha, - params.b, - params.num_softmax_heads, - params.num_groups, - params.batch_size_per_group, - params.dq_semaphore, - params.seq_offsets, - params.seq_offsets_q, - params.num_targets, - params.max_seq_len_tensor, - params.contextual_seq_len_tensor, - params.max_attn_len_tensor, - params.min_full_attn_seq_len_tensor, - params.attn_scale, - params.scalar_scale}; - typename CollectiveEpilogue::Arguments epilogue_args{ - static_cast(params.dk_ptr), - [&] { - return typename CollectiveEpilogue::ShapedKV{ - seqlen_kv, params.qk_d, params.h, batch}; // shape_dK - }(), - [&] { - return typename CollectiveEpilogue::StridedKV{ - params.dk_row_stride, - _1{}, - params.dk_head_stride, - !Jagged ? params.dk_batch_stride : 0}; // stride_dK - }(), - static_cast(params.dv_ptr), - [&] { - return typename CollectiveEpilogue::StridedKV{ - params.dv_row_stride, - _1{}, - params.dv_head_stride, - !Jagged ? params.dv_batch_stride : 0}; // stride_dV - }(), - params.h, - params.seq_offsets}; - - int num_blocks_n = - cutlass::ceil_div(params.max_kv_len, get<1>(TileShape_MNK{})); - num_blocks_n = cutlass::round_up(num_blocks_n, size<1>(ClusterShape{})); - typename hstu::TileSchedulerArguments scheduler_args{ - num_blocks_n, - params.h, - params.b, - params.max_kv_len, - params.qk_d, - sizeof(Element), - params.tile_count_semaphore, - params.seq_offsets, - params.sort_by_length_indices}; - - int device; - cudaGetDevice(&device); - typename AttnKernel::Params kernel_params = - AttnKernel::to_underlying_arguments( - {mainloop_args, - epilogue_args, - {device, params.num_sm}, - scheduler_args}); - - dim3 grid_dims = AttnKernel::get_grid_shape(kernel_params); - dim3 block_dims = AttnKernel::get_block_shape(); - int smem_size = AttnKernel::SharedStorageSize; - if constexpr (size(ClusterShape{}) > 1) { - void const* kernel = (void const*)cutlass::device_kernel; - if (smem_size >= 48 * 1024) { - CHECK_CUDA(cudaFuncSetAttribute( - kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size)); - } - dim3 cluster_dims( - size<0>(ClusterShape{}), - size<1>(ClusterShape{}), - size<2>(ClusterShape{})); - cutlass::ClusterLauncher::launch( - grid_dims, - cluster_dims, - block_dims, - smem_size, - stream, - kernel, - kernel_params, - false /*launch_with_pdl*/); - } else { - if (smem_size >= 48 * 1024) { - CHECK_CUDA(cudaFuncSetAttribute( - cutlass::device_kernel, - cudaFuncAttributeMaxDynamicSharedMemorySize, - smem_size)); - } - cutlass::kernel_launch( - grid_dims, - block_dims, - smem_size, - stream, - kernel_params, - false /*launch_with_pdl*/); - } - CHECK_CUDA_KERNEL_LAUNCH(); - - using PostprocessKernel = hstu::FlashAttnBwdPostprocessConvertdQ< - TileShape_MK, - Element, - ElementAccum, - ArchTag, - AttnKernel::CollectiveMainloop::NumMmaThreads, - typename AttnKernel::CollectiveMainloop::TiledMmadQ, - AttnKernel::CollectiveMainloop::dQ_swapAB, - Jagged, - Softmax>; - typename PostprocessKernel::Arguments postprocess_args{ - static_cast(params.dq_accum_ptr), - {seqlen_q_rounded * params.qk_d_rounded, - params.h, - batch}, // shape_dQaccum - {_1{}, - seqlen_q_rounded * params.qk_d_rounded, - !Jagged ? params.qk_d_rounded * params.max_q_len_rounded * params.h - : 0}, // stride_dQaccum - static_cast(params.dq_ptr), - {seqlen_q, params.qk_d, params.h, batch}, // shape_dQ - {params.dq_row_stride, - _1{}, - params.dq_head_stride, - params.dq_batch_stride}, // stride_dQ - Cross ? params.seq_offsets_q : params.seq_offsets}; - typename PostprocessKernel::Params postprocess_params = - PostprocessKernel::to_underlying_arguments(postprocess_args); - int num_m_block_postprocess = - cute::ceil_div(params.max_q_len, get<0>(TileShape_MK{})); - dim3 grid_m_postprocess(num_m_block_postprocess, params.h, params.b); - int smem_size_postprocess = PostprocessKernel::SharedStorageSize; - if (smem_size_postprocess >= 48 * 1024) { - CHECK_CUDA(cudaFuncSetAttribute( - cutlass::device_kernel, - cudaFuncAttributeMaxDynamicSharedMemorySize, - smem_size_postprocess)); - } - cutlass::kernel_launch( - grid_m_postprocess, - PostprocessKernel::MaxThreadsPerBlock, - smem_size_postprocess, - stream, - postprocess_params, - false /*launch_with_pdl*/); - CHECK_CUDA_KERNEL_LAUNCH(); -} - -template < - int Arch, - typename T, - int kBlockM, - int kBlockN, - int kHeadDim, - bool Causal, - bool Local, - int Stages_dO = 2, - int Stages_dS_or_QSm80 = 2, - bool SdP_swapAB = true, - bool dKV_swapAB = false, - bool dQ_swapAB = false, - int NumMmaWarpGroups = 2, - int AtomLayoutMSdP = 1, - int AtomLayoutNdKV = 2, - int AtomLayoutMdQ = 1, - bool V_in_regs = false, - bool Softmax = false> -void run_mha_bwd_dispatch(hstu::Flash_bwd_params& params, cudaStream_t stream) { - BOOL_SWITCH(params.seq_offsets != nullptr, Jagged, [&] { - BOOL_SWITCH(params.num_targets != nullptr, Has_targets, [&] { - BOOL_SWITCH(params.has_contexual_mask, Contexual_mask, [&] { - BOOL_SWITCH(params.seq_offsets_q, Cross, [&] { - run_flash_bwd< - Arch, - kHeadDim, - kBlockM, - kBlockN, - T, - Causal, - Local, - Contexual_mask, - Jagged, - Has_targets, - false /*Deterministic*/, - Stages_dO, - Stages_dS_or_QSm80, - SdP_swapAB, - dKV_swapAB, - dQ_swapAB, - NumMmaWarpGroups, - AtomLayoutMSdP, - AtomLayoutNdKV, - AtomLayoutMdQ, - V_in_regs, - Cross, - Softmax>(params, stream); - }); - }); - }); - }); -} - -template -void run_mha_bwd_(hstu::Flash_bwd_params& params, cudaStream_t stream) { - CAUSAL_LOCAL_SWITCH(params.is_causal, params.is_local, Causal, Local, [&] { - int const kBlockM = hstu::kBlockM_bwd(Arch, kHeadDim, Causal, Local); - int const kBlockN = hstu::kBlockN_bwd(Arch, kHeadDim); - bool const V_in_regs = hstu::V_in_regs_bwd(Arch, kHeadDim); - static constexpr std::tuple Stages = - hstu::Stages_bwd(Arch, kHeadDim); - static constexpr std::tuple swapAB = - hstu::swapAB_bwd(Arch, kHeadDim, Causal, Local); - int const NumMmaWarpGroups = hstu::NumMmaWarpGroups_bwd(Arch, kHeadDim); - static constexpr std::tuple AtomLayout = - hstu::AtomLayout_bwd(Arch, kHeadDim); - run_mha_bwd_dispatch< - Arch, - T, - kBlockM, - kBlockN, - kHeadDim, - Causal, - Local, - std::get<0>(Stages), /*Stages_dO*/ - std::get<1>(Stages), /*Stages_dS_or_QSm80*/ - std::get<0>(swapAB), /*SdP_swapAB*/ - std::get<1>(swapAB), /*dKV_swapAB*/ - std::get<2>(swapAB), /*dQ_swapAB*/ - NumMmaWarpGroups, - std::get<0>(AtomLayout), /*AtomLayoutMSdP*/ - std::get<1>(AtomLayout), /*AtomLayoutNdKV*/ - std::get<2>(AtomLayout), /*AtomLayoutMdQ*/ - V_in_regs, - Softmax>(params, stream); - }); -} - -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_bwd_postprocess_kernel.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_bwd_postprocess_kernel.h deleted file mode 100644 index ca04a1456..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_bwd_postprocess_kernel.h +++ /dev/null @@ -1,348 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/****************************************************************************** - * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, - *Pradeep Ramani, Tri Dao. - ******************************************************************************/ - -#pragma once - -#include "cute/tensor.hpp" - -#include -#include -#include -#include -#include "cutlass/arch/barrier.h" - -#include "seqlen.h" -#include "utils.h" - -namespace hstu { - -using namespace cute; - -template < - class TileShape_MK_, - class Element, - class ElementAccum, - class ArchTag_, - int kNThreads, - class TiledMma, - bool dQ_swapAB, - bool Jagged, - bool Softmax> -class FlashAttnBwdPostprocessConvertdQ { - public: - // Type Aliases - using TileShape_MK = TileShape_MK_; - using ArchTag = ArchTag_; - - static_assert(ArchTag::kMinComputeCapability >= 75); - static constexpr bool IsSm90 = ArchTag::kMinComputeCapability >= 90; - - static constexpr uint32_t MaxThreadsPerBlock = kNThreads; - static constexpr uint32_t MinBlocksPerMultiprocessor = 2; - - static constexpr int kBlockM = get<0>(TileShape_MK{}); - static constexpr int kHeadDim = get<1>(TileShape_MK{}); - static_assert( - !IsSm90 || kNThreads % cutlass::NumThreadsPerWarpGroup == 0, - "kNThreads must be a multiple of NumThreadsPerWarpGroup"); - static constexpr int NumdQWarpGgroups = - kNThreads / cutlass::NumThreadsPerWarpGroup; - using R2SLayoutAtomdQaccum = std::conditional_t< - IsSm90, - Layout< - Shape, Int>>, - Layout>>>; - using R2STiledCopydQaccum = decltype(make_tiled_copy( - Copy_Atom, ElementAccum>{}, - R2SLayoutAtomdQaccum{}, - Layout>>{})); // Val layout, 1 or 4 vals per - // read - using G2SLayoutAtomdQaccum = Layout>>; - // UniversalCopy instead of AutoVectorizingCopyWithAssumedAlignment as the - // latter generates cp.async instructions - using G2STiledCopydQaccum = decltype(make_tiled_copy( - Copy_Atom, ElementAccum>{}, - G2SLayoutAtomdQaccum{}, - Layout>{})); // Val layout, 4 vals per read - // We don't do bound checking for the gmem -> smem load so we just assert - // here. - static_assert(IsSm90 || (kBlockM * kHeadDim) % (kNThreads * 4) == 0); - static constexpr int SmemdQaccumSize = size(TileShape_MK{}); - using SmemLayoutdQaccumFlat = Layout>>; - using SmemLayoutdQaccum = std::conditional_t< - IsSm90, - Layout, - Int>>, - Layout>>>; - - // We can't just use kHeadDim here. E.g. if MMA shape is 64 x 96 but split - // across 2 WGs, then setting kBlockKSmem to 32 will cause "Static shape_div - // failure". We want to treat it as 64 x 48, so kBlockKSmem should be 16. - static constexpr int MmaShapeN = get<1>(typename TiledMma::AtomShape_MNK{}); - static constexpr int kBlockKSmem = - MmaShapeN % 64 == 0 ? 64 : (MmaShapeN % 32 == 0 ? 32 : 16); - static constexpr int kSwizzle = - kBlockKSmem == 64 ? 3 : (kBlockKSmem == 32 ? 2 : 1); - using SmemLayoutAtomdQ = decltype(composition( - Swizzle{}, - Layout, Int>, Stride, _1>>{})); - using SmemLayoutdQ = - decltype(tile_to_shape(SmemLayoutAtomdQ{}, TileShape_MK{})); - using SmemLayoutdQt = decltype(cute::composition( - SmemLayoutdQ{}, - make_layout( - make_shape(get<1>(TileShape_MK{}), get<0>(TileShape_MK{})), - make_stride(Int(TileShape_MK{})>{}, _1{})))); - - using SmemCopyAtomdQ = Copy_Atom< - std::conditional_t< - IsSm90, - std::conditional_t< - !dQ_swapAB, - cute::SM90_U32x4_STSM_N, - cute::SM90_U16x8_STSM_T>, - AutoVectorizingCopyWithAssumedAlignment<128>>, - Element>; - - static constexpr int kGmemElemsPerLoad = - sizeof(cute::uint128_t) / sizeof(Element); - static_assert( - kHeadDim % kGmemElemsPerLoad == 0, - "Headdim must be a multiple of kGmemElemsPerLoad"); - static constexpr int kGmemThreadsPerRow = - cutlass::gcd(kHeadDim / kGmemElemsPerLoad, int(MaxThreadsPerBlock)); - static_assert( - MaxThreadsPerBlock % kGmemThreadsPerRow == 0, - "MaxThreadsPerBlock must be a multiple of kGmemThreadsPerRow"); - using GmemLayoutAtom = Layout< - Shape< - Int, - Int>, - Stride, _1>>; - using GmemTiledCopy = decltype(make_tiled_copy( - Copy_Atom, Element>{}, - GmemLayoutAtom{}, - Layout>>{})); // Val layout, 8 or 16 vals - // per load - - struct SharedStorage : cute::aligned_struct<128> { - cute::array_aligned> - smem_dqacc; - cute::array_aligned> smem_dq; - alignas(16) cutlass::arch::ClusterTransactionBarrier barrier_dQaccum; - }; - - static constexpr int SharedStorageSize = sizeof(SharedStorage); - - using ShapedQ = - cute::Shape; // (seqlen_q, d, head, - // batch) - using StridedQ = cute::Stride; - using ShapedQaccum = - cute::Shape; // (seqlen_q * d, head, batch) - using StridedQaccum = cute::Stride<_1, int64_t, int64_t>; - - // Device side arguments - struct Arguments { - ElementAccum const* ptr_dQaccum; - ShapedQaccum const shape_dQaccum; - StridedQaccum const stride_dQaccum; - Element* ptr_dQ; - ShapedQ const shape_dQ; - StridedQ const stride_dQ; - int const* seq_offsets = nullptr; - }; - - // Kernel entry point API - struct Params { - ElementAccum const* ptr_dQaccum; - ShapedQaccum const shape_dQaccum; - StridedQaccum const stride_dQaccum; - Element* ptr_dQ; - ShapedQ const shape_dQ; - StridedQ const stride_dQ; - int const* seq_offsets = nullptr; - }; - - // Convert to underlying arguments. In this case, a simple copy for the - // aliased type. - static Params to_underlying_arguments(Arguments const& args) { - return { - args.ptr_dQaccum, - args.shape_dQaccum, - args.stride_dQaccum, - args.ptr_dQ, - args.shape_dQ, - args.stride_dQ, - args.seq_offsets}; - } - - CUTLASS_DEVICE - void operator()(Params const& params, char* smem_buf) { - static constexpr int kBlockM = get<0>(TileShape_MK{}); - SharedStorage& shared_storage = *reinterpret_cast(smem_buf); - - Tensor sdQaccum = make_tensor( - make_smem_ptr(shared_storage.smem_dqacc.data()), SmemLayoutdQaccum{}); - Tensor sdQaccum_flat = make_tensor( - make_smem_ptr(shared_storage.smem_dqacc.data()), - SmemLayoutdQaccumFlat{}); - Tensor sdQ = make_tensor( - make_smem_ptr(shared_storage.smem_dq.data()), SmemLayoutdQ{}); - Tensor sdQt = make_tensor( - make_smem_ptr(shared_storage.smem_dq.data()), SmemLayoutdQt{}); - - int const thread_idx = threadIdx.x; - int const m_block = blockIdx.x; - int const bidh = blockIdx.y; - int const bidb = blockIdx.z; - - hstu::SeqlenInfo seqlen_info( - bidb, size<0>(params.shape_dQ), params.seq_offsets); - if (Jagged && m_block * kBlockM >= seqlen_info.seqlen) { - return; - } - - // Step 1: load dQaccum from gmem to smem - Tensor mdQaccum = make_tensor( - make_gmem_ptr( - reinterpret_cast(params.ptr_dQaccum)), - params.shape_dQaccum, - params.stride_dQaccum)(_, bidh, !Jagged ? bidb : 0); - Tensor gdQaccum = local_tile( - domain_offset( - make_coord(seqlen_info.offset_padded * kHeadDim), mdQaccum), - Shape>{}, - make_coord(m_block)); // (M * K) - if constexpr (IsSm90) { // Use BulkCopy - static constexpr uint32_t TmaTransactionBytesdQaccum = - static_cast( - size(SmemLayoutdQaccumFlat{}) * - cute::sizeof_bits_v / 8); - auto bulk_copy = Copy_Traits{}; - // if (thread0()) { print(gdQaccum); printf("\n"); print(sdQaccum_flat); - // printf("\n"); } - if (thread_idx == 0) { - shared_storage.barrier_dQaccum.init(1 /*numThreads*/); - shared_storage.barrier_dQaccum.arrive_and_expect_tx( - TmaTransactionBytesdQaccum); - copy( - bulk_copy.with( - *reinterpret_cast(&shared_storage.barrier_dQaccum)), - gdQaccum, - sdQaccum_flat); - } - __syncthreads(); - shared_storage.barrier_dQaccum.wait(0); - } else { - G2STiledCopydQaccum g2s_tiled_copy_dQaccum; - auto g2s_thr_copy_dQaccum = - g2s_tiled_copy_dQaccum.get_thread_slice(thread_idx); - Tensor tdQgdQaccumg2s = g2s_thr_copy_dQaccum.partition_S(gdQaccum); - Tensor tdQsdQaccumg2s = g2s_thr_copy_dQaccum.partition_D(sdQaccum); - cute::copy(g2s_tiled_copy_dQaccum, tdQgdQaccumg2s, tdQsdQaccumg2s); - __syncthreads(); - } - - // __syncthreads(); if (cute::thread0()) { print_tensor(sdQaccum); } - - // Step 2: Load dQaccum from smem to register, then convert fp32 -> - // fp16/bf16 - R2STiledCopydQaccum s2r_tiled_copy_dQaccum; - auto s2r_thr_copy_dQaccum = - s2r_tiled_copy_dQaccum.get_thread_slice(thread_idx); - Tensor tdQsdQaccum = s2r_thr_copy_dQaccum.partition_S(sdQaccum); - TiledMma tiled_mma_dQ; - Tensor taccdQrdQaccum = partition_fragment_C( - tiled_mma_dQ, - select(TileShape_MK{})); - // if (blockIdx.x == 0 && blockIdx.y == 0 && threadIdx.x == 1) { - // print(tiled_mma_dQ); printf("\n"); } if (blockIdx.x == 0 && blockIdx.y == - // 0 && threadIdx.x == 1) { print(tdQsdQaccum); } if (blockIdx.x == 0 && - // blockIdx.y == 0 && threadIdx.x == 1) { print(taccdQrdQaccum); } - CUTE_STATIC_ASSERT_V(size(taccdQrdQaccum) == size(tdQsdQaccum)); - Tensor tdQrdQaccum = s2r_thr_copy_dQaccum.retile_D(taccdQrdQaccum); - cute::copy(s2r_tiled_copy_dQaccum, tdQsdQaccum, tdQrdQaccum); - // Convert tdQrdQ from fp32 to fp16 - Tensor rdQ = make_tensor_like(taccdQrdQaccum); - hstu::convert_type_out(taccdQrdQaccum, rdQ); - - // Step 3: Copy dQ from register to smem - auto smem_tiled_copy_dQ = make_tiled_copy_C(SmemCopyAtomdQ{}, tiled_mma_dQ); - auto smem_thr_copy_dQ = smem_tiled_copy_dQ.get_thread_slice(thread_idx); - Tensor taccdQrdQ = - smem_thr_copy_dQ.retile_S(rdQ); // ((Atom,AtomNum), MMA_N, MMA_N) - // if (cute::thread0()) { print(smem_tiled_copy_dQ); } - // if (cute::thread0()) { print(smem_thr_copy_dQ); } - // if (cute::thread0()) { print(sdQ); } - Tensor taccdQsdQ = smem_thr_copy_dQ.partition_D( - cute::conditional_return( - sdQ, sdQt)); // ((Atom,AtomNum),PIPE_M,PIPE_N) - cute::copy(smem_tiled_copy_dQ, taccdQrdQ, taccdQsdQ); - __syncthreads(); - - // Step 4: Copy dQ from smem to register to prepare for coalesced write to - // gmem - Tensor mdQ = make_tensor( - make_gmem_ptr(params.ptr_dQ), params.shape_dQ, params.stride_dQ)( - _, _, bidh, !Jagged ? bidb : 0); - Tensor gdQ = local_tile( - domain_offset(make_coord(seqlen_info.offset, _0{}), mdQ), - TileShape_MK{}, - make_coord(m_block, _0{})); // (M, K) - GmemTiledCopy gmem_tiled_copy_dQ; - auto gmem_thr_copy_dQ = gmem_tiled_copy_dQ.get_thread_slice(thread_idx); - Tensor tdQsdQ = - gmem_thr_copy_dQ.partition_S(sdQ); // ((Atom,AtomNum),ATOM_M,ATOM_N) - Tensor tdQgdQ = gmem_thr_copy_dQ.partition_D(gdQ); - - Tensor tdQrdQ = make_fragment_like(tdQsdQ); - Tensor tdQcdQ = gmem_thr_copy_dQ.partition_D( - cute::make_identity_tensor(TileShape_MK{})); - Tensor tdQpdQ = make_tensor(make_shape(size<2>(tdQgdQ))); -#pragma unroll - for (int k = 0; k < size(tdQpdQ); ++k) { - tdQpdQ(k) = get<1>(tdQcdQ(_0{}, _0{}, k)) < get<1>(params.shape_dQ); - } - // Need to check OOB when reading from smem if kBlockM isn't evenly tiled - static constexpr bool EvenM = - kBlockM % CUTE_STATIC_V(size<0>(GmemLayoutAtom{})) == 0; - hstu:: - copy( - gmem_tiled_copy_dQ, tdQsdQ, tdQrdQ, tdQcdQ, tdQpdQ, kBlockM); - - // Step 5: Copy dQ from register to gmem - // Clear_OOB_K must be false since we don't want to write zeros to gmem - hstu::copy< - /*Is_even_MN=*/false, - /*Is_even_K=*/false, - /*Clear_OOB_MN=*/false, - /*Clear_OOB_K=*/false>( - gmem_tiled_copy_dQ, - tdQrdQ, - tdQgdQ, - tdQcdQ, - tdQpdQ, - std::min(seqlen_info.seqlen - m_block * kBlockM, kBlockM)); - } -}; - -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_bwd_preprocess_kernel.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_bwd_preprocess_kernel.h deleted file mode 100644 index 8d29778af..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_bwd_preprocess_kernel.h +++ /dev/null @@ -1,349 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/****************************************************************************** - * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, - *Pradeep Ramani, Tri Dao. - ******************************************************************************/ - -#pragma once - -#include "cute/tensor.hpp" - -#include -#include -#include -#include - -#include "seqlen.h" - -namespace hstu { - -using namespace cute; - -template < - class TileShape_MK_, - class Element, - class ElementAccum, - class ArchTag_, - bool Clear_dQaccum, - bool Jagged, - bool Softmax> -class FlashAttnBwdPreprocess { - public: - // Type Aliases - using TileShape_MK = TileShape_MK_; - using ArchTag = ArchTag_; - - static_assert( - std::is_same_v && - ArchTag::kMinComputeCapability >= 75 || - std::is_same_v && - ArchTag::kMinComputeCapability >= 80 || - std::is_same_v && - ArchTag::kMinComputeCapability >= 89); - - static constexpr uint32_t MaxThreadsPerBlock = 256; - static constexpr uint32_t MinBlocksPerMultiprocessor = 2; - static constexpr int SharedStorageSize = 0; - - static constexpr int kGmemElemsPerLoad = - sizeof(cute::uint128_t) / sizeof(Element); - static_assert( - get<1>(TileShape_MK{}) % kGmemElemsPerLoad == 0, - "Headdim must be a multiple of kGmemElemsPerLoad"); - static constexpr int kBlockM = get<0>(TileShape_MK{}); - static constexpr int kHeadDim = get<1>(TileShape_MK{}); - // We want kBlockKGmem to be a power of 2 so that when we do the summing, - // it's just between threads in the same warp - static constexpr int kBlockKGmem = - kHeadDim % 128 == 0 ? 128 : (kHeadDim % 64 == 0 ? 64 : 32); - static constexpr int kGmemThreadsPerRow = kBlockKGmem / kGmemElemsPerLoad; - static_assert( - MaxThreadsPerBlock % kGmemThreadsPerRow == 0, - "MaxThreadsPerBlock must be a multiple of kGmemThreadsPerRow"); - using GmemLayoutAtom = Layout< - Shape< - Int, - Int>, - Stride, _1>>; - using GmemTiledCopy = decltype(make_tiled_copy( - Copy_Atom, Element>{}, - GmemLayoutAtom{}, - Layout>>{})); // Val layout, 8 or 16 vals - // per load - - static constexpr int kGmemElemsPerLoadAccum = - sizeof(cute::uint128_t) / sizeof(ElementAccum); - static_assert( - (kBlockM * kHeadDim / kGmemElemsPerLoadAccum) % MaxThreadsPerBlock == 0, - "MaxThreadsPerBlock must divide kBlockM * kHeadDim / kGmemElemsPerLoadAccum"); - using GmemLayoutAtomAccum = Layout>>; - using GmemTiledCopyAccum = decltype(make_tiled_copy( - Copy_Atom, ElementAccum>{}, - GmemLayoutAtomAccum{}, - Layout>>{})); // Val layout, 4 vals per - // store - - using ShapeO = - cute::Shape; // (seqlen_q, d, head, - // batch) - using StrideO = cute::Stride; - using ShapedPsum = - cute::Shape; // (seqlen_q, head, batch) - using StridedPsum = cute::Stride<_1, int64_t, int64_t>; - using ShapedQaccum = - cute::Shape; // (seqlen_q * d, head, batch) - using StridedQaccum = cute::Stride<_1, int64_t, int64_t>; - - // Device side arguments - struct Arguments { - Element const* ptr_O; - ShapeO const shape_O; - StrideO const stride_O; - Element const* ptr_dO; - StrideO const stride_dO; - float* ptr_dPsum; - ShapedPsum const shape_dPsum; - StridedPsum const stride_dPsum; - float const* ptr_LSE; - StridedPsum const stride_LSE; - float* ptr_LSE_log2; - StridedPsum const stride_LSE_log2; - ElementAccum* ptr_dQaccum; - ShapedQaccum const shape_dQaccum; - StridedQaccum const stride_dQaccum; - int num_batch; // We need this to know the size of dq_semaphore in case of - // jagged - int num_heads; - int num_softmax_heads; - int max_seq_len; - int* dq_semaphore; - int const* seq_offsets = nullptr; - }; - - // Kernel entry point API - struct Params { - Element const* ptr_O; - ShapeO const shape_O; - StrideO const stride_O; - Element const* ptr_dO; - StrideO const stride_dO; - float* ptr_dPsum; - ShapedPsum const shape_dPsum; - StridedPsum const stride_dPsum; - float const* ptr_LSE; - StridedPsum const stride_LSE; - float* ptr_LSE_log2; - StridedPsum const stride_LSE_log2; - ElementAccum* ptr_dQaccum; - ShapedQaccum const shape_dQaccum; - StridedQaccum const stride_dQaccum; - int num_batch; - int num_heads; - int num_softmax_heads; - int max_seq_len; - int* dq_semaphore; - int const* seq_offsets = nullptr; - }; - - // Convert to underlying arguments. In this case, a simple copy for the - // aliased type. - static Params to_underlying_arguments(Arguments const& args) { - return {args.ptr_O, args.shape_O, args.stride_O, - args.ptr_dO, args.stride_dO, args.ptr_dPsum, - args.shape_dPsum, args.stride_dPsum, args.ptr_LSE, - args.stride_LSE, args.ptr_LSE_log2, args.stride_LSE_log2, - args.ptr_dQaccum, args.shape_dQaccum, args.stride_dQaccum, - args.num_batch, args.num_heads, args.num_softmax_heads, - args.max_seq_len, args.dq_semaphore, args.seq_offsets}; - } - - CUTLASS_DEVICE - void operator()(Params const& params, [[maybe_unused]] char* smem_buf) { - static constexpr int kBlockM = get<0>(TileShape_MK{}); - - int const thread_idx = threadIdx.x; - int const m_block = blockIdx.x; - int const bidh = blockIdx.y; - int const bidb = blockIdx.z; - - hstu::SeqlenInfo seqlen_info( - bidb, params.max_seq_len, params.seq_offsets); - int const seqlen_o = seqlen_info.seqlen; - if (Jagged && m_block * kBlockM >= seqlen_o) { - return; - } - - if constexpr (Softmax) { - Tensor mO = make_tensor( - make_gmem_ptr(params.ptr_O), params.shape_O, params.stride_O)( - _, _, bidh, !Jagged ? bidb : 0); - Tensor gO = local_tile( - cute::domain_offset(make_coord(seqlen_info.offset, _0{}), mO), - TileShape_MK{}, - make_coord(m_block, _0{})); // (M, K) - Tensor mdO = make_tensor( - make_gmem_ptr(params.ptr_dO), params.shape_O, params.stride_dO)( - _, _, bidh, !Jagged ? bidb : 0); - Tensor gdO = local_tile( - cute::domain_offset(make_coord(seqlen_info.offset, _0{}), mdO), - TileShape_MK{}, - make_coord(m_block, _0{})); // (M, K) - - auto shape_LSE = select<0, 2, 3>(params.shape_O); - Tensor mLSE = make_tensor( - make_gmem_ptr(params.ptr_LSE), shape_LSE, params.stride_LSE)( - _, bidh, !Jagged ? bidb : 0); - Tensor gLSE = local_tile( - cute::domain_offset(make_coord(seqlen_info.offset), mLSE), - Shape>{}, - make_coord(m_block)); - static_assert(kBlockM <= MaxThreadsPerBlock); - float lse = - thread_idx < seqlen_o - m_block * kBlockM && thread_idx < kBlockM - ? gLSE(thread_idx) - : 0.0f; - - GmemTiledCopy gmem_tiled_copy_O; - auto gmem_thr_copy_O = gmem_tiled_copy_O.get_thread_slice(thread_idx); - - Tensor tOgO = gmem_thr_copy_O.partition_S(gO); - Tensor tOgdO = gmem_thr_copy_O.partition_S(gdO); - // Construct identity layout for gO - Tensor cO = cute::make_identity_tensor( - TileShape_MK{}); // (BLK_M,BLK_K) -> (blk_m,blk_k) - // Repeat the partitioning with identity layouts - Tensor tOcO = gmem_thr_copy_O.partition_D(cO); - Tensor tOpO = make_tensor(make_shape(size<2>(tOgO))); -#pragma unroll - for (int k = 0; k < size(tOpO); ++k) { - tOpO(k) = get<1>(tOcO(_0{}, _0{}, k)) < get<1>(params.shape_O); - } - - // (8, kBlockM / 32, kHeadDim / 64) or (8, kBlockM / 16, kHeadDim / 128) - Tensor tOrO = make_fragment_like(tOgO); - Tensor tOrdO = make_fragment_like(tOgdO); - hstu::copy< - /*Is_even_MN=*/false, - /*Is_even_K=*/false, - /*Clear_OOB_MN=*/true, - /*Clearn_OOB_K=*/true>( - gmem_tiled_copy_O, - tOgO, - tOrO, - tOcO, - tOpO, - seqlen_o - m_block * kBlockM); - hstu::copy< - /*Is_even_MN=*/false, - /*Is_even_K=*/false, - /*Clear_OOB_MN=*/true, - /*Clearn_OOB_K=*/true>( - gmem_tiled_copy_O, - tOgdO, - tOrdO, - tOcO, - tOpO, - seqlen_o - m_block * kBlockM); - // Reshape from e.g. (8, kBlockM / 32, kHeadDim / 64) to (kBlockM / 32, - // (8, kHeadDim / 64)) - Layout l = make_layout( - get<1>(tOrO.layout()), - make_layout(get<0>(tOrO.layout()), get<2>(tOrO.layout()))); - Tensor tOrO_l = make_tensor(tOrO.data(), l); - Tensor o_fp32 = make_tensor_like(tOrO_l); - hstu::convert_type_out(tOrO_l, o_fp32); - Tensor tOrdO_l = make_tensor(tOrdO.data(), l); - Tensor do_fp32 = make_tensor_like(tOrdO_l); - hstu::convert_type_out(tOrdO_l, do_fp32); - // Sum across the last dimension - Tensor dP_sum = make_tensor(make_shape(size<0>(o_fp32))); -#pragma unroll - for (int mi = 0; mi < size<0>(o_fp32); ++mi) { - float dP_sum_cur = do_fp32(mi, 0) * o_fp32(mi, 0); -#pragma unroll - for (int ni = 1; ni < size<1>(o_fp32); ni++) { - dP_sum_cur += do_fp32(mi, ni) * o_fp32(mi, ni); - } - hstu::SumOp sum_op; - dP_sum(mi) = - hstu::Allreduce::run(dP_sum_cur, sum_op); - } - - Tensor mdPsum = make_tensor( - make_gmem_ptr(params.ptr_dPsum), - params.shape_dPsum, - params.stride_dPsum)(_, bidh, !Jagged ? bidb : 0); - Tensor gdPsum = local_tile( - cute::domain_offset(make_coord(seqlen_info.offset_padded), mdPsum), - Shape>{}, - make_coord(m_block)); - if (get<1>(tOcO(_0{}, _0{}, _0{})) == 0) { -#pragma unroll - for (int mi = 0; mi < size(dP_sum); ++mi) { - int const row = get<0>(tOcO(_0{}, mi, _0{})); - gdPsum(row) = row < seqlen_o - m_block * kBlockM ? dP_sum(mi) : 0; - } - } - - int const seqlen_rounded = cute::round_up(seqlen_o, kBlockM); - Tensor mLSElog2 = make_tensor( - make_gmem_ptr(params.ptr_LSE_log2), - params.shape_dPsum, - params.stride_LSE_log2)(_, bidh, !Jagged ? bidb : 0); - Tensor gLSElog2 = local_tile( - cute::domain_offset(make_coord(seqlen_info.offset_padded), mLSElog2), - Shape>{}, - make_coord(m_block)); - if (thread_idx < seqlen_rounded - m_block * kBlockM && - thread_idx < kBlockM) { - gLSElog2(thread_idx) = lse * float(M_LOG2E); - } - } - if constexpr (Clear_dQaccum) { - Tensor mdQaccum = make_tensor( - make_gmem_ptr(params.ptr_dQaccum), - params.shape_dQaccum, - params.stride_dQaccum)(_, bidh, !Jagged ? bidb : 0); - Tensor gdQaccum = local_tile( - cute::domain_offset( - make_coord(seqlen_info.offset_padded * kHeadDim), mdQaccum), - Shape>{}, - make_coord(m_block)); - GmemTiledCopyAccum gmem_tiled_copy_dQaccum; - auto gmem_thr_copy_dQaccum = - gmem_tiled_copy_dQaccum.get_thread_slice(thread_idx); - Tensor tdQgdQaccum = gmem_thr_copy_dQaccum.partition_D(gdQaccum); - Tensor zero = make_fragment_like(tdQgdQaccum); - clear(zero); - cute::copy( - Copy_Atom< - AutoVectorizingCopyWithAssumedAlignment<128>, - ElementAccum>{}, - zero, - tdQgdQaccum); - } - - if (params.dq_semaphore != nullptr && thread_idx == 0) { - int const num_batch = params.num_batch; - int const num_head = params.num_heads; - params.dq_semaphore - [bidh + bidb * num_head + m_block * num_head * num_batch] = 0; - } - } -}; - -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_common.cpp b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_common.cpp deleted file mode 100644 index 66ec445be..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_common.cpp +++ /dev/null @@ -1,1165 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/****************************************************************************** - * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, - *Pradeep Ramani, Tri Dao. - ******************************************************************************/ - -// Include these 2 headers instead of torch/extension.h since we don't need all -// of the torch headers. -#include -#include -#include -#include -#include // For TORCH_VERSION* macros - -#include - -#include "flash.h" -#include "flash_common.h" -#include "static_switch.h" -#include "tile_size.h" - -#define CHECK_DEVICE(x) TORCH_CHECK(x.is_cuda(), #x " must be on CUDA") -#define CHECK_SHAPE(x, ...) \ - TORCH_CHECK( \ - x.sizes() == torch::IntArrayRef({__VA_ARGS__}), \ - #x " must have shape (" #__VA_ARGS__ ")") -#define CHECK_CONTIGUOUS(x) \ - TORCH_CHECK(x.is_contiguous(), #x " must be contiguous") - -at::Tensor switch_to_contiguous_if_needed(const at::Tensor& x) { - if (x.stride(x.dim() - 1) == 1) { - return x; - } - return x.contiguous(); -} - -namespace hstu { - -void set_params_fprop( - hstu::Flash_fwd_params& params, - // sizes - const size_t b, - const size_t total_seq_len_kv, - const size_t total_seq_len_q, - const size_t max_seq_len, - const size_t max_q_len, - const size_t h, - const size_t qk_d, - const size_t v_d, - // device pointers - const at::Tensor& q, - const at::Tensor& k, - const at::Tensor& v, - const at::Tensor& out, - void* seq_offsets, - void* num_targets, - void* attn_scale, - void* seq_offsets_q, - void* softmax_lse, - void* max_seq_len_tensor, - void* contextual_seq_len_tensor, - void* max_attn_len_tensor, - void* min_full_attn_seq_len_tensor, - const int num_groups, - bool causal, - float alpha, - const bool scalar_scale, - const int max_attn_len, - const int min_full_attn_seq_len, - const int contextual_seq_len, - const int num_softmax_heads, - const bool training, - const int sm_margin = 0) { - // Reset the parameters - params = {}; - - params.is_bf16 = q.dtype() == torch::kBFloat16; - params.is_e4m3 = q.dtype() == torch::kFloat8_e4m3fn; - - // Set the pointers and strides. - params.q_ptr = q.data_ptr(); - params.k_ptr = k.data_ptr(); - params.v_ptr = v.data_ptr(); - params.o_ptr = out.data_ptr(); - // All stride are in elements, not bytes. - params.q_row_stride = q.stride(-3); - params.k_row_stride = k.stride(-3); - params.v_row_stride = v.stride(-3); - params.o_row_stride = out.stride(-3); - params.q_head_stride = q.stride(-2); - params.k_head_stride = k.stride(-2); - params.v_head_stride = v.stride(-2); - params.o_head_stride = out.stride(-2); - params.v_dim_stride = v.stride(-1); - - if (seq_offsets == nullptr) { - params.q_batch_stride = q.stride(0); - params.k_batch_stride = k.stride(0); - params.v_batch_stride = v.stride(0); - params.o_batch_stride = out.stride(0); - } - - params.seq_offsets = static_cast(seq_offsets); - params.seq_offsets_q = static_cast(seq_offsets_q); - params.num_targets = static_cast(num_targets); - params.attn_scale = static_cast(attn_scale); - params.softmax_lse = static_cast(softmax_lse); - params.max_seq_len_tensor = static_cast(max_seq_len_tensor); - params.contextual_seq_len_tensor = - static_cast(contextual_seq_len_tensor); - params.max_attn_len_tensor = static_cast(max_attn_len_tensor); - params.min_full_attn_seq_len_tensor = - static_cast(min_full_attn_seq_len_tensor); - params.num_groups = num_groups; - params.batch_size_per_group = b / num_groups; - - // Set the dimensions. - params.b = b; - params.h = h; - params.total_seq_len_q = total_seq_len_q; - params.total_seq_len_kv = total_seq_len_kv; - params.max_kv_len = max_seq_len; - params.max_q_len = max_q_len; - params.qk_d = qk_d; - params.v_d = v_d; - - params.alpha = alpha; - - // Note: when num_groups > 1, max_attn_len, contextual_seq_len, - // min_full_attn_seq_len represent the max value in the tensor. - params.is_local = max_attn_len > 0; - params.is_causal = causal && (!params.is_local); - params.has_contexual_mask = contextual_seq_len > 0; - params.scalar_scale = scalar_scale; - params.num_softmax_heads = num_softmax_heads; - params.training = training; - - params.max_attn_len = max_attn_len; - params.min_full_attn_seq_len = min_full_attn_seq_len; - params.contextual_seq_len = contextual_seq_len; - - params.arch = at::cuda::getCurrentDeviceProperties()->major * 10 + - at::cuda::getCurrentDeviceProperties()->minor; - params.num_sm = - at::cuda::getCurrentDeviceProperties()->multiProcessorCount - sm_margin; - -#ifdef FLASHATTENTION_DISABLE_LOCAL - TORCH_CHECK( - !params.is_local, - "This flash attention build does not support local attention."); -#endif -} - -void set_params_dgrad( - hstu::Flash_bwd_params& params, - // sizes - const size_t b, - const size_t total_seq_len_kv, - const size_t total_seq_len_q, - const size_t max_seq_len, - const size_t max_q_len, - const size_t max_q_len_rounded, - const size_t h, - const size_t qk_d, - const size_t v_d, - const size_t qk_d_rounded, - const size_t v_d_rounded, - // device pointers - const at::Tensor& q, - const at::Tensor& k, - const at::Tensor& v, - const at::Tensor& out, - const at::Tensor& dout, - const at::Tensor& dq, - const at::Tensor& dk, - const at::Tensor& dv, - void* dq_accum_d, - void* seq_offsets, - void* num_targets, - void* attn_scale, - void* sort_by_length_indices, - void* seq_offsets_q, - void* softmax_lse, - void* softmax_d, - void* softmax_lse_log2, - void* max_seq_len_tensor, - void* contextual_seq_len_tensor, - void* max_attn_len_tensor, - void* min_full_attn_seq_len_tensor, - const int num_groups, - const bool scalar_scale, - const bool causal, - const float alpha, - const int max_attn_len, - const int min_full_attn_seq_len, - const int contextual_seq_len, - const int num_softmax_heads, - bool deterministic = false, - int const sm_margin = 0) { - hstu::set_params_fprop( - params, - b, - total_seq_len_kv, - total_seq_len_q, - max_seq_len, - max_q_len, - h, - qk_d, - v_d, - q, - k, - v, - out, - seq_offsets, - num_targets, - attn_scale, - seq_offsets_q, - softmax_lse, - max_seq_len_tensor, - contextual_seq_len_tensor, - max_attn_len_tensor, - min_full_attn_seq_len_tensor, - num_groups, - causal, - alpha, - scalar_scale, - max_attn_len, - min_full_attn_seq_len, - contextual_seq_len, - num_softmax_heads, - false /* training */, - sm_margin); - - // Set the pointers and strides. - params.do_ptr = dout.data_ptr(); - params.do_row_stride = dout.stride(-3); - params.do_head_stride = dout.stride(-2); - params.dq_ptr = dq.data_ptr(); - params.dk_ptr = dk.data_ptr(); - params.dv_ptr = dv.data_ptr(); - params.dq_row_stride = dq.stride(-3); - params.dk_row_stride = dk.stride(-3); - params.dv_row_stride = dv.stride(-3); - params.dq_head_stride = dq.stride(-2); - params.dk_head_stride = dk.stride(-2); - params.dv_head_stride = dv.stride(-2); - - params.qk_d_rounded = qk_d_rounded; - params.v_d_rounded = v_d_rounded; - params.max_q_len_rounded = max_q_len_rounded; - - params.sort_by_length_indices = static_cast(sort_by_length_indices); - - if (seq_offsets == nullptr) { - params.do_batch_stride = dout.stride(0); - params.dq_batch_stride = dq.stride(0); - params.dk_batch_stride = dk.stride(0); - params.dv_batch_stride = dv.stride(0); - } - params.dq_accum_ptr = dq_accum_d; - params.softmax_lse_log2 = static_cast(softmax_lse_log2); - params.softmax_d = static_cast(softmax_d); - params.deterministic = deterministic; -} - -void run_mha_fwd(hstu::Flash_fwd_params& params, cudaStream_t stream) { - // HEADDIM_SWITCH(params.d, [&] { - // hstu::run_mha_fwd_(params, stream); - // }); - ARCH_SWITCH(params.arch, Arch, [&] { - BOOL_SWITCH(params.num_softmax_heads == params.h, Softmax, [&] { - if (!params.is_e4m3) { - if (params.is_bf16) { -#ifndef FLASHATTENTION_DISABLE_HDIM64 - if (params.qk_d <= 64) { - return hstu::run_mha_fwd_( - params, stream); - } -#endif -#ifndef FLASHATTENTION_DISABLE_HDIM96 - if (params.qk_d <= 96) { - return hstu::run_mha_fwd_( - params, stream); - } -#endif -#ifndef FLASHATTENTION_DISABLE_HDIM128 - if (params.qk_d <= 128) { - return hstu::run_mha_fwd_( - params, stream); - } -#endif -#ifndef FLASHATTENTION_DISABLE_HDIM192 - if (params.qk_d <= 192) { - return hstu::run_mha_fwd_( - params, stream); - } -#endif -#ifndef FLASHATTENTION_DISABLE_HDIM256 - if (params.qk_d <= 256) { - return hstu::run_mha_fwd_( - params, stream); - } -#endif - } else { -#ifndef FLASHATTENTION_DISABLE_FP16 -#ifndef FLASHATTENTION_DISABLE_HDIM64 - if (params.qk_d <= 64) { - return hstu::run_mha_fwd_( - params, stream); - } -#endif -#ifndef FLASHATTENTION_DISABLE_HDIM96 - if (params.qk_d <= 96) { - return hstu::run_mha_fwd_( - params, stream); - } -#endif -#ifndef FLASHATTENTION_DISABLE_HDIM128 - if (params.qk_d <= 128) { - return hstu::run_mha_fwd_( - params, stream); - } -#endif -#ifndef FLASHATTENTION_DISABLE_HDIM192 - if (params.qk_d <= 192) { - return hstu::run_mha_fwd_( - params, stream); - } -#endif -#ifndef FLASHATTENTION_DISABLE_HDIM256 - if (params.qk_d <= 256) { - return hstu::run_mha_fwd_( - params, stream); - } -#endif -#else - TORCH_CHECK(false, "This flash attention build does not support FP16."); -#endif - } - } else { -#ifndef FLASHATTENTION_DISABLE_FP8 -#ifndef FLASHATTENTION_DISABLE_HDIM64 - if (params.qk_d <= 64) { - return hstu::run_mha_fwd_<90, cutlass::float_e4m3_t, 64, Softmax>( - params, stream); - } -#endif -#ifndef FLASHATTENTION_DISABLE_HDIM96 - if (params.qk_d <= 96) { - return hstu::run_mha_fwd_<90, cutlass::float_e4m3_t, 96, Softmax>( - params, stream); - } -#endif -#ifndef FLASHATTENTION_DISABLE_HDIM128 - if (params.qk_d <= 128) { - return hstu::run_mha_fwd_<90, cutlass::float_e4m3_t, 128, Softmax>( - params, stream); - } -#endif -#ifndef FLASHATTENTION_DISABLE_HDIM192 - if (params.qk_d <= 192) { - return hstu::run_mha_fwd_<90, cutlass::float_e4m3_t, 192, Softmax>( - params, stream); - } -#endif -#ifndef FLASHATTENTION_DISABLE_HDIM256 - if (params.qk_d <= 256) { - return hstu::run_mha_fwd_<90, cutlass::float_e4m3_t, 256, Softmax>( - params, stream); - } -#endif -#else - TORCH_CHECK(false, "This flash attention build does not support FP8."); -#endif - } - }); - }); -} - -std::tuple> hstu_mha_fwd( - int64_t max_seq_len, - double alpha, - at::Tensor& q, // (b, s, h, d) or (total_s, h, d) - at::Tensor& k, // (b, s, h, d) or (total_s, h, d) - at::Tensor& v, // (b, s, h, d) or (total_s, h, d) - const std::optional& seq_offsets, - bool causal, - const std::optional& num_targets, - const std::optional& attn_scale, - int64_t max_attn_len, - int64_t min_full_attn_seq_len, - int64_t contextual_seq_len, - const std::optional& q_descale, // (b, h_k), not (b, h) - const std::optional& k_descale, // (b, h_k) - const std::optional& v_descale, // (b, h_k) - const int64_t sm_margin, - int64_t max_q_len, - const std::optional& seq_offsets_q, - int64_t num_softmax_heads, - bool training, - const std::optional& max_seq_len_tensor, - const std::optional& contextual_seq_len_tensor, - const std::optional& max_attn_len_tensor, - const std::optional& min_full_attn_seq_len_tensor, - int64_t num_groups) { - auto dprops = at::cuda::getCurrentDeviceProperties(); - bool is_sm9x = dprops->major >= 9; - TORCH_CHECK(is_sm9x, "HSTU Attention only supports Hopper GPUs or newer."); - - q = switch_to_contiguous_if_needed(q); - k = switch_to_contiguous_if_needed(k); - v = switch_to_contiguous_if_needed(v); - - auto q_type = q.scalar_type(); - TORCH_CHECK( - q_type == at::ScalarType::Half || q_type == at::ScalarType::BFloat16 || - q_type == at::ScalarType::Float8_e4m3fn, - "FlashAttention only supports fp16, bf16, and fp8_e4m3 data type"); - if (dprops->major < 9) { - TORCH_CHECK( - q_type == at::ScalarType::Half || q_type == at::ScalarType::BFloat16, - "FlashAttention on Ampere/Ada cards only supports fp16 and bf16 data type"); - } - TORCH_CHECK( - k.scalar_type() == q_type, "query and key must have the same dtype"); - TORCH_CHECK( - v.scalar_type() == q_type, "query and value must have the same dtype"); - - CHECK_DEVICE(q); - CHECK_DEVICE(k); - CHECK_DEVICE(v); - - TORCH_CHECK( - q.stride(-1) == 1, "Input tensor must have contiguous last dimension"); - TORCH_CHECK( - k.stride(-1) == 1, "Input tensor must have contiguous last dimension"); - TORCH_CHECK( - v.stride(-1) == 1, "Input tensor must have contiguous last dimension"); - - at::Tensor seq_offsets_; - bool const is_jagged = seq_offsets.has_value(); - if (is_jagged) { - seq_offsets_ = seq_offsets.value(); - CHECK_DEVICE(seq_offsets_); - CHECK_CONTIGUOUS(seq_offsets_); - TORCH_CHECK( - seq_offsets_.dtype() == torch::kInt32, - "seq_offsets_ must have dtype torch.int32"); - } - at::Tensor num_targets_; - bool const has_multiple_targets = num_targets.has_value(); - if (has_multiple_targets) { - num_targets_ = num_targets.value(); - CHECK_DEVICE(num_targets_); - CHECK_CONTIGUOUS(num_targets_); - TORCH_CHECK( - num_targets_.dtype() == torch::kInt32, - "num_targets_ must have dtype torch.int32"); - } - at::Tensor seq_offsets_q_; - bool const is_cross_attn = seq_offsets_q.has_value(); - if (is_cross_attn) { - seq_offsets_q_ = seq_offsets_q.value(); - CHECK_DEVICE(seq_offsets_q_); - CHECK_CONTIGUOUS(seq_offsets_q_); - TORCH_CHECK( - seq_offsets_q_.dtype() == torch::kInt32, - "seq_offsets_q_ must have dtype torch.int32"); - } else { - max_q_len = max_seq_len; - } - at::Tensor attn_scale_; - bool scalar_scale = true; - bool const has_attn_scale = attn_scale.has_value(); - if (has_attn_scale) { - attn_scale_ = attn_scale.value(); - scalar_scale = attn_scale_.numel() == num_groups; - CHECK_DEVICE(attn_scale_); - TORCH_CHECK( - attn_scale_.dtype() == torch::kFloat32, - "attn_scale_ must have dtype torch.float32"); - } - at::Tensor max_seq_len_tensor_; - at::Tensor contextual_seq_len_tensor_; - at::Tensor max_attn_len_tensor_; - at::Tensor min_full_attn_seq_len_tensor_; - if (num_groups > 1) { - TORCH_CHECK( - max_seq_len_tensor.has_value(), - "max_seq_len_tensor cannot be empty for num_groups > 1."); - max_seq_len_tensor_ = max_seq_len_tensor.value(); - CHECK_DEVICE(max_seq_len_tensor_); - TORCH_CHECK(max_seq_len_tensor_.dtype() == torch::kInt32); - if (!is_cross_attn) { - TORCH_CHECK( - contextual_seq_len_tensor.has_value(), - "contextual_seq_len_tensor cannot be empty for num_groups > 1 and not cross_attn."); - TORCH_CHECK( - max_attn_len_tensor.has_value(), - "max_attn_len_tensor cannot be empty for num_groups > 1 and not cross_attn."); - TORCH_CHECK( - min_full_attn_seq_len_tensor.has_value(), - "min_full_attn_seq_len_tensor cannot be empty for num_groups > 1 and not cross_attn."); - contextual_seq_len_tensor_ = contextual_seq_len_tensor.value(); - max_attn_len_tensor_ = max_attn_len_tensor.value(); - min_full_attn_seq_len_tensor_ = min_full_attn_seq_len_tensor.value(); - CHECK_DEVICE(contextual_seq_len_tensor_); - CHECK_DEVICE(max_attn_len_tensor_); - CHECK_DEVICE(min_full_attn_seq_len_tensor_); - TORCH_CHECK(contextual_seq_len_tensor_.dtype() == torch::kInt32); - TORCH_CHECK(max_attn_len_tensor_.dtype() == torch::kInt32); - TORCH_CHECK(min_full_attn_seq_len_tensor_.dtype() == torch::kInt32); - } - } -#ifdef HSTU_FLASH_ATTN_DEBUG_INFO - if (is_jagged && has_multiple_targets) { - auto uih_lengths = seq_offsets_.slice(0, 1) - .sub(seq_offsets_.slice(0, 0, -1)) - .sub(num_targets_); - TORCH_CHECK( - (uih_lengths.gt(0)).sum().item() == num_targets_.size(0), - "some uih seqlen is 0"); - TORCH_CHECK( - (uih_lengths.greater_equal(contextual_seq_len)).sum().item() == - num_targets_.size(0), - "some uih seqlen is less than contextual_seq_len"); - } -#endif - TORCH_CHECK( - q.size(-1) == k.size(-1) && k.size(-1) == v.size(-1), - "only attndim == hidden_dim is supported"); - - auto const sizes_q = q.sizes(); - auto const sizes_k = k.sizes(); - const int batch_size = !is_jagged ? sizes_q[0] : seq_offsets_.size(0) - 1; - TORCH_CHECK( - batch_size % num_groups == 0, "batch_size not divisible by num_groups"); - int total_seq_len_q = !is_jagged ? batch_size * max_q_len : sizes_q[0]; - int total_seq_len_kv = !is_jagged ? batch_size * max_seq_len : sizes_k[0]; - int num_heads = q.size(-2); - int const qk_head_size = q.size(-1); - int const v_head_size = v.size(-1); - int const max_headdim = get_max_headdim(); - TORCH_CHECK( - qk_head_size <= max_headdim && v_head_size <= max_headdim, - "FlashAttention forward only supports head dimension at most " + - std::to_string(max_headdim)); - TORCH_CHECK(max_attn_len >= 0, "max_attn_len must be at least 0"); - TORCH_CHECK( - min_full_attn_seq_len >= 0, "min_full_attn_seq_len must be at least 0"); - TORCH_CHECK(contextual_seq_len >= 0, "contextual_seq_len must be at least 0"); - if (max_attn_len > 0) { - TORCH_CHECK( - min_full_attn_seq_len > 0, - "min_full_attn_seq_len=0 not supported when max_attn_len > 0"); - } - TORCH_CHECK( - 0 == num_softmax_heads || num_softmax_heads == num_heads, - "num_softmax_heads must be either 0 or num_heads"); - if (!is_jagged) { - CHECK_SHAPE(q, batch_size, max_q_len, num_heads, qk_head_size); - CHECK_SHAPE(k, batch_size, max_seq_len, num_heads, qk_head_size); - CHECK_SHAPE(v, batch_size, max_seq_len, num_heads, v_head_size); - } else { - CHECK_SHAPE(q, total_seq_len_q, num_heads, qk_head_size); - CHECK_SHAPE(k, total_seq_len_kv, num_heads, qk_head_size); - CHECK_SHAPE(v, total_seq_len_kv, num_heads, v_head_size); - CHECK_SHAPE(seq_offsets_, batch_size + 1); - } - if (has_multiple_targets) { - CHECK_SHAPE(num_targets_, batch_size); - } - if (is_cross_attn) { - CHECK_SHAPE(seq_offsets_q_, batch_size + 1); - } - - int const alignment = q_type == torch::kFloat8_e4m3fn ? 16 : 8; - TORCH_CHECK( - qk_head_size % alignment == 0 && v_head_size % alignment == 0, - "head_size should be a multiple of " + std::to_string(alignment)); - - auto opts = q.options(); - auto out_type = q_type == at::ScalarType::Float8_e4m3fn - ? at::ScalarType::BFloat16 - : q_type; - at::Tensor out; - if (!is_jagged) { - out = torch::empty( - {batch_size, max_q_len, num_heads, v_head_size}, opts.dtype(out_type)); - } else { - out = torch::empty( - {total_seq_len_q, num_heads, v_head_size}, opts.dtype(out_type)); - } - std::optional softmax_lse = std::nullopt; - - // Early return for empty sequences to avoid TMA descriptor - // initialization failure - if (total_seq_len_kv == 0 || total_seq_len_q == 0) { - return {out, std::nullopt}; - } - - if (num_softmax_heads > 0) { - if (!is_jagged) { - softmax_lse = torch::empty( - {batch_size, num_softmax_heads, max_q_len}, opts.dtype(at::kFloat)); - } else { - softmax_lse = torch::empty( - {num_softmax_heads, total_seq_len_q}, opts.dtype(at::kFloat)); - } - } - - // Otherwise the kernel will be launched from cuda:0 device - // Cast to char to avoid compiler warning about narrowing - at::cuda::CUDAGuard device_guard{(char)q.get_device()}; - hstu::Flash_fwd_params params; - hstu::set_params_fprop( - params, - batch_size, - total_seq_len_kv, - total_seq_len_q, - max_seq_len, - max_q_len, - num_heads, - qk_head_size, - v_head_size, - q, - k, - v, - out, - !is_jagged ? nullptr : seq_offsets_.data_ptr(), - !has_multiple_targets ? nullptr : num_targets_.data_ptr(), - !has_attn_scale ? nullptr : attn_scale_.data_ptr(), - !is_cross_attn ? nullptr : seq_offsets_q_.data_ptr(), - (num_softmax_heads == 0) ? nullptr : softmax_lse.value().data_ptr(), - num_groups > 1 ? max_seq_len_tensor_.data_ptr() : nullptr, - ((num_groups > 1) && (!is_cross_attn)) - ? contextual_seq_len_tensor_.data_ptr() - : nullptr, - ((num_groups > 1) && (!is_cross_attn)) ? max_attn_len_tensor_.data_ptr() - : nullptr, - ((num_groups > 1) && (!is_cross_attn)) - ? min_full_attn_seq_len_tensor_.data_ptr() - : nullptr, - num_groups, - causal, - alpha, - scalar_scale, - max_attn_len, - min_full_attn_seq_len, - contextual_seq_len, - num_softmax_heads, - training, - sm_margin); - at::Tensor tile_count_semaphore; - // We don't use the persistent scheduler if not jagged - bool const persistent_scheduler = params.arch >= 90 - ? (params.is_causal || params.is_local || is_jagged) - : (params.is_causal || is_jagged); - if (persistent_scheduler) { - tile_count_semaphore = torch::zeros({1}, opts.dtype(torch::kInt32)); - params.tile_count_semaphore = tile_count_semaphore.data_ptr(); - } else { - params.tile_count_semaphore = nullptr; - } - - if (q_type == at::ScalarType::Float8_e4m3fn) { - if (q_descale.has_value()) { - auto q_descale_ = q_descale.value(); - CHECK_DEVICE(q_descale_); - CHECK_SHAPE(q_descale_, batch_size, num_heads); - params.q_descale_ptr = q_descale_.data_ptr(); - params.q_descale_batch_stride = q_descale_.stride(0); - params.q_descale_head_stride = q_descale_.stride(1); - } else { - params.q_descale_ptr = nullptr; - } - if (k_descale.has_value()) { - auto k_descale_ = k_descale.value(); - CHECK_DEVICE(k_descale_); - CHECK_SHAPE(k_descale_, batch_size, num_heads); - params.k_descale_ptr = k_descale_.data_ptr(); - params.k_descale_batch_stride = k_descale_.stride(0); - params.k_descale_head_stride = k_descale_.stride(1); - } else { - params.k_descale_ptr = nullptr; - } - if (v_descale.has_value()) { - auto v_descale_ = v_descale.value(); - CHECK_DEVICE(v_descale_); - CHECK_SHAPE(v_descale_, batch_size, num_heads); - params.v_descale_ptr = v_descale_.data_ptr(); - params.v_descale_batch_stride = v_descale_.stride(0); - params.v_descale_head_stride = v_descale_.stride(1); - } else { - params.v_descale_ptr = nullptr; - } - } - -#ifdef FLASHATTENTION_DISABLE_LOCAL - TORCH_CHECK( - !params.is_local, - "This flash attention build does not support local attention."); -#endif - - if (total_seq_len_q > 0 && num_heads > 0) { - auto stream = at::cuda::getCurrentCUDAStream().stream(); - run_mha_fwd(params, stream); - } - return {out, softmax_lse}; -} - -void run_mha_bwd(hstu::Flash_bwd_params& params, cudaStream_t stream) { -#ifndef FLASHATTENTION_DISABLE_BACKWARD - // FP16_SWITCH(!params.is_bf16, [&] { - // HEADDIM_SWITCH(params.d, [&] { - // hstu::run_mha_bwd_(params, stream); - // }); - // }); - ARCH_SWITCH(params.arch, Arch, [&] { - BOOL_SWITCH(params.num_softmax_heads == params.h, Softmax, [&] { - if (!params.is_bf16) { -#ifndef FLASHATTENTION_DISABLE_FP16 -#ifndef FLASHATTENTION_DISABLE_HDIM64 - if (params.qk_d <= 64) { - return hstu::run_mha_bwd_( - params, stream); - } -#endif -#ifndef FLASHATTENTION_DISABLE_HDIM96 - if (params.qk_d <= 96) { - return hstu::run_mha_bwd_( - params, stream); - } -#endif -#ifndef FLASHATTENTION_DISABLE_HDIM128 - if (params.qk_d <= 128) { - return hstu::run_mha_bwd_( - params, stream); - } -#endif -#ifndef FLASHATTENTION_DISABLE_HDIM192 - if (params.qk_d <= 192) { - return hstu::run_mha_bwd_( - params, stream); - } -#endif -#ifndef FLASHATTENTION_DISABLE_HDIM256 - if (params.qk_d <= 256) { - return hstu::run_mha_bwd_( - params, stream); - } -#endif -#else - TORCH_CHECK(false, "This flash attention build does not support FP16."); -#endif - } else { -#ifndef FLASHATTENTION_DISABLE_HDIM64 - if (params.qk_d <= 64) { - return hstu::run_mha_bwd_( - params, stream); - } -#endif -#ifndef FLASHATTENTION_DISABLE_HDIM96 - if (params.qk_d <= 96) { - return hstu::run_mha_bwd_( - params, stream); - } -#endif -#ifndef FLASHATTENTION_DISABLE_HDIM128 - if (params.qk_d <= 128) { - return hstu::run_mha_bwd_( - params, stream); - } -#endif -#ifndef FLASHATTENTION_DISABLE_HDIM192 - if (params.qk_d <= 192) { - return hstu::run_mha_bwd_( - params, stream); - } -#endif -#ifndef FLASHATTENTION_DISABLE_HDIM256 - if (params.qk_d <= 256) { - return hstu::run_mha_bwd_( - params, stream); - } -#endif - } - }); - }); -#endif -} - -std::vector hstu_mha_bwd( - int64_t max_seq_len, - double alpha, - at::Tensor& dout, - at::Tensor& q, - at::Tensor& k, - at::Tensor& v, - at::Tensor& dq, - at::Tensor& dk, - at::Tensor& dv, - at::Tensor& out, - const std::optional& seq_offsets, - bool causal, - const std::optional& num_targets, - const std::optional& attn_scale, - int64_t max_attn_len, - int64_t min_full_attn_seq_len, - int64_t contextual_seq_len, - bool sort_by_length, - bool const deterministic, - const int64_t sm_margin, - int64_t max_q_len, - const std::optional& seq_offsets_q, - int64_t num_softmax_heads, - const std::optional& softmax_lse, - const std::optional& max_seq_len_tensor, - const std::optional& contextual_seq_len_tensor, - const std::optional& max_attn_len_tensor, - const std::optional& min_full_attn_seq_len_tensor, - int64_t num_groups) { -#ifdef FLASHATTENTION_DISABLE_BACKWARD - TORCH_CHECK(false, "This flash attention build does not support backward."); -#endif - - auto dprops = at::cuda::getCurrentDeviceProperties(); - bool is_sm9x = dprops->major >= 9; - TORCH_CHECK(is_sm9x, "HSTU Attention only supports Hopper GPUs or newer."); - - q = switch_to_contiguous_if_needed(q); - k = switch_to_contiguous_if_needed(k); - v = switch_to_contiguous_if_needed(v); - out = switch_to_contiguous_if_needed(out); - dout = switch_to_contiguous_if_needed(dout); - - auto q_type = q.dtype(); - TORCH_CHECK( - q_type == torch::kFloat16 || q_type == torch::kBFloat16, - "FlashAttention only support fp16 and bf16 data type"); - TORCH_CHECK(k.dtype() == q_type, "query and key must have the same dtype"); - TORCH_CHECK(v.dtype() == q_type, "query and value must have the same dtype"); - TORCH_CHECK( - dout.dtype() == q_type, "query and dout must have the same dtype"); - - CHECK_DEVICE(q); - CHECK_DEVICE(k); - CHECK_DEVICE(v); - CHECK_DEVICE(dout); - - TORCH_CHECK( - q.stride(-1) == 1, "Input tensor must have contiguous last dimension"); - TORCH_CHECK( - k.stride(-1) == 1, "Input tensor must have contiguous last dimension"); - TORCH_CHECK( - v.stride(-1) == 1, "Input tensor must have contiguous last dimension"); - TORCH_CHECK( - dout.stride(-1) == 1, "dout tensor must have contiguous last dimension"); - - at::Tensor seq_offsets_; - bool const is_jagged = seq_offsets.has_value(); - if (is_jagged) { - seq_offsets_ = seq_offsets.value(); - CHECK_DEVICE(seq_offsets_); - CHECK_CONTIGUOUS(seq_offsets_); - TORCH_CHECK( - seq_offsets_.dtype() == torch::kInt32, - "seq_offsets_ must have dtype torch.int32"); - } - at::Tensor sort_by_length_indices_; - if (sort_by_length && is_jagged) { - auto seq_lengths = - seq_offsets_.slice(0, 1).sub(seq_offsets_.slice(0, 0, -1)); - std::tuple sort_result = torch::sort( - seq_lengths, false /*stable*/, 0 /*dim*/, true /*descending*/); - sort_by_length_indices_ = std::get<1>(sort_result).to(torch::kInt32); - CHECK_DEVICE(sort_by_length_indices_); - CHECK_CONTIGUOUS(sort_by_length_indices_); - TORCH_CHECK( - sort_by_length_indices_.dtype() == torch::kInt32, - "sort_by_length_indices_ must have dtype torch.int32"); - } - at::Tensor num_targets_; - bool const has_multiple_targets = num_targets.has_value(); - if (has_multiple_targets) { - num_targets_ = num_targets.value(); - CHECK_DEVICE(num_targets_); - CHECK_CONTIGUOUS(num_targets_); - TORCH_CHECK( - num_targets_.dtype() == torch::kInt32, - "num_targets_ must have dtype torch.int32"); - } - at::Tensor attn_scale_; - bool scalar_scale = true; - bool const has_attn_scale = attn_scale.has_value(); - if (has_attn_scale) { - attn_scale_ = attn_scale.value(); - scalar_scale = attn_scale_.numel() == num_groups; - CHECK_DEVICE(attn_scale_); - TORCH_CHECK( - attn_scale_.dtype() == torch::kFloat32, - "attn_scale_ must have dtype torch.float32"); - } - at::Tensor seq_offsets_q_; - bool const is_cross_attn = seq_offsets_q.has_value(); - if (is_cross_attn) { - seq_offsets_q_ = seq_offsets_q.value(); - CHECK_DEVICE(seq_offsets_q_); - CHECK_CONTIGUOUS(seq_offsets_q_); - TORCH_CHECK( - seq_offsets_q_.dtype() == torch::kInt32, - "seq_offsets_q_ must have dtype torch.int32"); - } else { - max_q_len = max_seq_len; - } - at::Tensor max_seq_len_tensor_; - at::Tensor contextual_seq_len_tensor_; - at::Tensor max_attn_len_tensor_; - at::Tensor min_full_attn_seq_len_tensor_; - if (num_groups > 1) { - TORCH_CHECK( - max_seq_len_tensor.has_value(), - "max_seq_len_tensor cannot be empty for num_groups > 1."); - max_seq_len_tensor_ = max_seq_len_tensor.value(); - CHECK_DEVICE(max_seq_len_tensor_); - TORCH_CHECK(max_seq_len_tensor_.dtype() == torch::kInt32); - if (!is_cross_attn) { - TORCH_CHECK( - contextual_seq_len_tensor.has_value(), - "contextual_seq_len_tensor cannot be empty for num_groups > 1 and not cross_attn."); - TORCH_CHECK( - max_attn_len_tensor.has_value(), - "max_attn_len_tensor cannot be empty for num_groups > 1 and not cross_attn."); - TORCH_CHECK( - min_full_attn_seq_len_tensor.has_value(), - "min_full_attn_seq_len_tensor cannot be empty for num_groups > 1 and not cross_attn."); - contextual_seq_len_tensor_ = contextual_seq_len_tensor.value(); - max_attn_len_tensor_ = max_attn_len_tensor.value(); - min_full_attn_seq_len_tensor_ = min_full_attn_seq_len_tensor.value(); - CHECK_DEVICE(contextual_seq_len_tensor_); - CHECK_DEVICE(max_attn_len_tensor_); - CHECK_DEVICE(min_full_attn_seq_len_tensor_); - TORCH_CHECK(contextual_seq_len_tensor_.dtype() == torch::kInt32); - TORCH_CHECK(max_attn_len_tensor_.dtype() == torch::kInt32); - TORCH_CHECK(min_full_attn_seq_len_tensor_.dtype() == torch::kInt32); - } - } - auto const sizes_q = q.sizes(); - auto const sizes_kv = k.sizes(); - int const batch_size = !is_jagged ? sizes_q[0] : seq_offsets_.size(0) - 1; - TORCH_CHECK( - batch_size % num_groups == 0, "batch_size not divisible by num_groups"); - if (!is_jagged) { - max_seq_len = sizes_kv[1]; - } - int const total_seq_len_q = !is_jagged ? batch_size * sizes_q[1] : sizes_q[0]; - int const total_seq_len_kv = - !is_jagged ? batch_size * sizes_kv[1] : sizes_kv[0]; - int const num_heads = q.size(-2); - int const qk_head_size = q.size(-1); - int const v_head_size = v.size(-1); - TORCH_CHECK( - qk_head_size % 8 == 0 && v_head_size % 8 == 0, - "head_size should be a multiple of 8"); - int const max_headdim = get_max_headdim(); - TORCH_CHECK( - qk_head_size <= max_headdim && v_head_size <= max_headdim, - "FlashAttention backward only supports head dimension at most " + - std::to_string(max_headdim)); - TORCH_CHECK(max_attn_len >= 0, "max_attn_len must be at least 0"); - TORCH_CHECK( - min_full_attn_seq_len >= 0, "min_full_attn_seq_len must be at least 0"); - TORCH_CHECK(contextual_seq_len >= 0, "contextual_seq_len must be at least 0"); - if (!is_jagged) { - CHECK_SHAPE(q, batch_size, max_q_len, num_heads, qk_head_size); - CHECK_SHAPE(k, batch_size, max_seq_len, num_heads, qk_head_size); - CHECK_SHAPE(v, batch_size, max_seq_len, num_heads, v_head_size); - CHECK_SHAPE(dout, batch_size, max_q_len, num_heads, v_head_size); - CHECK_SHAPE(dq, batch_size, max_q_len, num_heads, qk_head_size); - CHECK_SHAPE(dk, batch_size, max_seq_len, num_heads, qk_head_size); - CHECK_SHAPE(dv, batch_size, max_seq_len, num_heads, v_head_size); - } else { - CHECK_SHAPE(q, total_seq_len_q, num_heads, qk_head_size); - CHECK_SHAPE(k, total_seq_len_kv, num_heads, qk_head_size); - CHECK_SHAPE(v, total_seq_len_kv, num_heads, v_head_size); - CHECK_SHAPE(dout, total_seq_len_q, num_heads, v_head_size); - CHECK_SHAPE(dq, total_seq_len_q, num_heads, qk_head_size); - CHECK_SHAPE(dk, total_seq_len_kv, num_heads, qk_head_size); - CHECK_SHAPE(dv, total_seq_len_kv, num_heads, v_head_size); - CHECK_SHAPE(seq_offsets_, batch_size + 1); - } - if (has_multiple_targets) { - CHECK_SHAPE(num_targets_, batch_size); - } - if (is_cross_attn) { - CHECK_SHAPE(seq_offsets_q_, batch_size + 1); - } - int const arch = at::cuda::getCurrentDeviceProperties()->major * 10 + - at::cuda::getCurrentDeviceProperties()->minor; - int const qk_head_size_rounded = round_up_headdim(qk_head_size); - int const v_head_size_rounded = round_up_headdim(v_head_size); - // Very important that these match the kernel configs - bool const is_local = max_attn_len > 0; - int const kBlockM = - hstu::kBlockM_bwd(arch, qk_head_size_rounded, causal, is_local); - auto round_multiple = [](int x, int m) { return (x + m - 1) / m * m; }; - int const max_q_len_rounded = round_multiple(max_q_len, kBlockM); - int const total_seq_len_q_padded_rounded = - round_multiple(total_seq_len_q + batch_size * kBlockM, kBlockM); - - TORCH_CHECK(dq.dtype() == q_type, "dq must have the same dtype as q"); - CHECK_DEVICE(dq); - TORCH_CHECK(dq.stride(-1) == 1, "dq must have contiguous last dimension"); - if (!is_jagged) { - CHECK_SHAPE(dq, batch_size, max_q_len, num_heads, qk_head_size); - } else { - CHECK_SHAPE(dq, total_seq_len_q, num_heads, qk_head_size); - } - TORCH_CHECK(dk.dtype() == q_type, "dk must have the same dtype as q"); - CHECK_DEVICE(dk); - TORCH_CHECK(dk.stride(-1) == 1, "dk must have contiguous last dimension"); - if (!is_jagged) { - CHECK_SHAPE(dk, batch_size, max_seq_len, num_heads, qk_head_size); - } else { - CHECK_SHAPE(dk, total_seq_len_kv, num_heads, qk_head_size); - } - TORCH_CHECK(dv.dtype() == q_type, "dv must have the same dtype as q"); - CHECK_DEVICE(dv); - TORCH_CHECK(dv.stride(-1) == 1, "dv must have contiguous last dimension"); - if (!is_jagged) { - CHECK_SHAPE(dv, batch_size, max_seq_len, num_heads, v_head_size); - } else { - CHECK_SHAPE(dv, total_seq_len_kv, num_heads, v_head_size); - } - - // Otherwise the kernel will be launched from cuda:0 device - // Cast to char to avoid compiler warning about narrowing - at::cuda::CUDAGuard device_guard{(char)q.get_device()}; - auto opts = q.options(); - - at::Tensor dq_accum; - if (!is_jagged) { - dq_accum = torch::empty( - {batch_size, num_heads, max_q_len_rounded * qk_head_size_rounded}, - opts.dtype(at::kFloat)); - } else { - dq_accum = torch::empty( - {num_heads, total_seq_len_q_padded_rounded * qk_head_size_rounded}, - opts.dtype(at::kFloat)); - } - at::Tensor softmax_d, softmax_lse_log2; - if (!is_jagged) { - // Need softmax_d to have seqlen_q_rounded since we want its address to be - // aligned by 16/8 bytes for TMA / LDG.64 - softmax_d = torch::empty( - {batch_size, num_softmax_heads, max_q_len_rounded}, - opts.dtype(at::kFloat)); - softmax_lse_log2 = torch::empty( - {batch_size, num_softmax_heads, max_q_len_rounded}, - opts.dtype(at::kFloat)); - } else { - softmax_d = torch::empty( - {num_softmax_heads, total_seq_len_q_padded_rounded}, - opts.dtype(at::kFloat)); - softmax_lse_log2 = torch::empty( - {num_softmax_heads, total_seq_len_q_padded_rounded}, - opts.dtype(at::kFloat)); - } - - // Early return for empty sequences; analog to TMA prevention guard - // in hstu_mha_fwd - if (total_seq_len_kv == 0 || total_seq_len_q == 0) { - return {dq, dk, dv}; - } - - hstu::Flash_bwd_params params; - hstu::set_params_dgrad( - params, - batch_size, - total_seq_len_kv, - total_seq_len_q, - max_seq_len, - max_q_len, - max_q_len_rounded, - num_heads, - qk_head_size, - v_head_size, - qk_head_size_rounded, - v_head_size_rounded, - q, - k, - v, - out, - dout, - dq, - dk, - dv, - dq_accum.data_ptr(), - !is_jagged ? nullptr : seq_offsets_.data_ptr(), - !has_multiple_targets ? nullptr : num_targets_.data_ptr(), - !has_attn_scale ? nullptr : attn_scale_.data_ptr(), - !(sort_by_length && is_jagged) ? nullptr - : sort_by_length_indices_.data_ptr(), - !is_cross_attn ? nullptr : seq_offsets_q_.data_ptr(), - num_softmax_heads == 0 ? nullptr : softmax_lse.value().data_ptr(), - num_softmax_heads == 0 ? nullptr : softmax_d.data_ptr(), - num_softmax_heads == 0 ? nullptr : softmax_lse_log2.data_ptr(), - num_groups > 1 ? max_seq_len_tensor_.data_ptr() : nullptr, - ((num_groups > 1) && (!is_cross_attn)) - ? contextual_seq_len_tensor_.data_ptr() - : nullptr, - ((num_groups > 1) && (!is_cross_attn)) ? max_attn_len_tensor_.data_ptr() - : nullptr, - ((num_groups > 1) && (!is_cross_attn)) - ? min_full_attn_seq_len_tensor_.data_ptr() - : nullptr, - num_groups, - scalar_scale, - causal, - alpha, - max_attn_len, - min_full_attn_seq_len, - contextual_seq_len, - num_softmax_heads, - deterministic, - sm_margin); - - // auto tile_count_semaphore = (params.is_causal || params.is_local) ? - // torch::zeros({1}, opts.dtype(torch::kInt32)) : torch::empty({1}, - // opts.dtype(torch::kInt32)); params.tile_count_semaphore = - // tile_count_semaphore.data_ptr(); Will be zero'ed out in the - // backward preprocess kernel - at::Tensor dq_semaphore = torch::empty( - {(max_seq_len + kBlockM - 1) / kBlockM, batch_size, num_heads}, - opts.dtype(torch::kInt32)); - params.dq_semaphore = dq_semaphore.data_ptr(); - -#ifdef FLASHATTENTION_DISABLE_LOCAL - TORCH_CHECK( - !params.is_local, - "This flash attention build does not support local attention."); -#endif - - if (total_seq_len_q > 0 && num_heads > 0) { - auto stream = at::cuda::getCurrentCUDAStream().stream(); - run_mha_bwd(params, stream); - } - return {dq, dk, dv}; -} - -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_common.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_common.h deleted file mode 100644 index 98ca009f5..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_common.h +++ /dev/null @@ -1,149 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/****************************************************************************** - * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, - *Pradeep Ramani, Tri Dao. - ******************************************************************************/ - -// Include these 2 headers instead of torch/extension.h since we don't need all -// of the torch headers. -#include -#include -#include - -#include - -#include // @manual - -#define CHECK_DEVICE(x) TORCH_CHECK(x.is_cuda(), #x " must be on CUDA") -#define CHECK_SHAPE(x, ...) \ - TORCH_CHECK( \ - x.sizes() == torch::IntArrayRef({__VA_ARGS__}), \ - #x " must have shape (" #__VA_ARGS__ ")") -#define CHECK_CONTIGUOUS(x) \ - TORCH_CHECK(x.is_contiguous(), #x " must be contiguous") - -inline int round_up_headdim(int head_size) { -#ifndef FLASHATTENTION_DISABLE_HDIM64 - if (head_size <= 64) { - return 64; - } -#endif -#ifndef FLASHATTENTION_DISABLE_HDIM96 - if (head_size <= 96) { - return 96; - } -#endif -#ifndef FLASHATTENTION_DISABLE_HDIM128 - if (head_size <= 128) { - return 128; - } -#endif -#ifndef FLASHATTENTION_DISABLE_HDIM192 - if (head_size <= 192) { - return 192; - } -#endif -#ifndef FLASHATTENTION_DISABLE_HDIM256 - if (head_size <= 256) { - return 256; - } -#endif - return 256; -} - -inline int get_max_headdim() { -#ifndef FLASHATTENTION_DISABLE_HDIM256 - return 256; -#endif -#ifndef FLASHATTENTION_DISABLE_HDIM192 - return 192; -#endif -#ifndef FLASHATTENTION_DISABLE_HDIM128 - return 128; -#endif -#ifndef FLASHATTENTION_DISABLE_HDIM96 - return 96; -#endif -#ifndef FLASHATTENTION_DISABLE_HDIM64 - return 64; -#endif - return 0; -} - -namespace hstu { - -std::tuple> hstu_mha_fwd( - int64_t max_seq_len, - double alpha, - at::Tensor& q, // (b, s, h, d) or (total_s, h, d) - at::Tensor& k, // (b, s, h, d) or (total_s, h, d) - at::Tensor& v, // (b, s, h, d) or (total_s, h, d) - const std::optional& seq_offsets, - bool causal, - const std::optional& num_targets, - const std::optional& attn_scale, - int64_t max_attn_len, - int64_t min_full_attn_seq_len, - int64_t contextual_seq_len, - const std::optional& q_descale, // (b, h_k), not (b, h) - const std::optional& k_descale, // (b, h_k) - const std::optional& v_descale, // (b, h_k) - const int64_t sm_margin = 0, - int64_t max_q_len = 0, - const std::optional& seq_offsets_q = std::nullopt, - int64_t num_softmax_heads = 0, - bool training = true, - const std::optional& max_seq_len_tensor = std::nullopt, - const std::optional& contextual_seq_len_tensor = std::nullopt, - const std::optional& max_attn_len_tensor = std::nullopt, - const std::optional& min_full_attn_seq_len_tensor = - std::nullopt, - int64_t num_groups = 1); - -std::vector hstu_mha_bwd( - int64_t max_seq_len, - double alpha, - at::Tensor& dout, - at::Tensor& q, - at::Tensor& k, - at::Tensor& v, - at::Tensor& dq, - at::Tensor& dk, - at::Tensor& dv, - at::Tensor& out, - const std::optional& seq_offsets, - bool causal, - const std::optional& num_targets, - const std::optional& attn_scale, - int64_t max_attn_len, - int64_t min_full_attn_seq_len, - int64_t contextual_seq_len, - bool sort_by_length, - bool const deterministic, - const int64_t sm_margin = 0, - int64_t max_q_len = 0, - const std::optional& seq_offsets_q = std::nullopt, - int64_t num_softmax_heads = 0, - const std::optional& softmax_lse = std::nullopt, - const std::optional& max_seq_len_tensor = std::nullopt, - const std::optional& contextual_seq_len_tensor = std::nullopt, - const std::optional& max_attn_len_tensor = std::nullopt, - const std::optional& min_full_attn_seq_len_tensor = - std::nullopt, - int64_t num_groups = 1); - -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_common_cpu.cpp b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_common_cpu.cpp deleted file mode 100644 index 30d4f792c..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_common_cpu.cpp +++ /dev/null @@ -1,172 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/****************************************************************************** - * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, - *Pradeep Ramani, Tri Dao. - ******************************************************************************/ - -#include -#include - -#include "flash_common_cpu.h" - -namespace hstu { - -std::tuple> hstu_mha_fwd_meta( - const at::SymInt max_seq_len, - double alpha, - at::Tensor& q, // (b, s, h, d) or (total_s, h, d) - at::Tensor& k, // (b, s, h, d) or (total_s, h, d) - at::Tensor& v, // (b, s, h, d) or (total_s, h, d) - const std::optional& seq_offsets, - bool causal, - const std::optional& num_targets, - const std::optional& attn_scale, - int64_t max_attn_len, - int64_t min_full_attn_seq_len, - int64_t contextual_seq_len, - const std::optional& q_descale, // (b, h_k), not (b, h) - const std::optional& k_descale, // (b, h_k) - const std::optional& v_descale, // (b, h_k) - const int64_t sm_margin, - int64_t max_q_len, - const std::optional& seq_offsets_q, - int64_t num_softmax_heads, - bool training, - const std::optional& max_seq_len_tensor, - const std::optional& contextual_seq_len_tensor, - const std::optional& max_attn_len_tensor, - const std::optional& min_full_attn_seq_len_tensor, - int64_t num_groups) { - auto q_type = q.scalar_type(); - auto const sizes = q.sym_sizes(); - at::Tensor seq_offsets_; - bool const is_jagged = seq_offsets.has_value(); - if (is_jagged) { - seq_offsets_ = seq_offsets.value(); - } - const c10::SymInt batch_size = - !is_jagged ? sizes[0] : seq_offsets_.sym_sizes()[0] - 1; - auto total_seq_len = !is_jagged ? batch_size * max_seq_len : sizes[0]; - const auto& num_heads = sizes[sizes.size() - 2]; - auto v_head_size = v.sym_sizes()[v.sym_sizes().size() - 1]; - auto out_type = q_type == at::ScalarType::Float8_e4m3fn - ? at::ScalarType::BFloat16 - : q_type; - auto opts = q.options(); - - at::Tensor out; - if (!is_jagged) { - out = at::empty_symint( - {batch_size, max_seq_len, num_heads, v_head_size}, - opts.dtype(out_type)); - } else { - out = at::empty_symint( - {total_seq_len, num_heads, v_head_size}, opts.dtype(out_type)); - } - return {out, std::nullopt}; -}; - -std::tuple> hstu_mha_fwd_dummy( - int64_t max_seq_len, - double alpha, - at::Tensor& q, // (b, s, h, d) or (total_s, h, d) - at::Tensor& k, // (b, s, h, d) or (total_s, h, d) - at::Tensor& v, // (b, s, h, d) or (total_s, h, d) - const std::optional& seq_offsets, - bool causal, - const std::optional& num_targets, - const std::optional& attn_scale, - int64_t max_attn_len, - int64_t min_full_attn_seq_len, - int64_t contextual_seq_len, - const std::optional& q_descale, // (b, h_k), not (b, h) - const std::optional& k_descale, // (b, h_k) - const std::optional& v_descale, // (b, h_k) - const int64_t sm_margin, - const int64_t max_q_len, - const std::optional& seq_offsets_q, - int64_t num_softmax_heads, - bool training, - const std::optional& max_seq_len_tensor, - const std::optional& contextual_seq_len_tensor, - const std::optional& max_attn_len_tensor, - const std::optional& min_full_attn_seq_len_tensor, - int64_t num_groups) { - auto q_type = q.scalar_type(); - auto const sizes = q.sizes(); - at::Tensor seq_offsets_; - bool const is_jagged = seq_offsets.has_value(); - if (is_jagged) { - seq_offsets_ = seq_offsets.value(); - } - const int batch_size = !is_jagged ? sizes[0] : seq_offsets_.size(0) - 1; - int total_seq_len = !is_jagged ? batch_size * max_seq_len : sizes[0]; - int num_heads = q.size(-2); - // int const qk_head_size = q.size(-1); - int const v_head_size = v.size(-1); - // int const max_headdim = get_max_headdim(); - auto out_type = q_type == at::ScalarType::Float8_e4m3fn - ? at::ScalarType::BFloat16 - : q_type; - auto opts = q.options(); - - at::Tensor out; - if (!is_jagged) { - out = torch::empty( - {batch_size, max_seq_len, num_heads, v_head_size}, - opts.dtype(out_type)); - } else { - out = torch::empty( - {total_seq_len, num_heads, v_head_size}, opts.dtype(out_type)); - } - return {out, std::nullopt}; -}; - -std::vector hstu_mha_bwd_dummy( - int64_t max_seq_len, - double alpha, - at::Tensor& dout, - at::Tensor& q, - at::Tensor& k, - at::Tensor& v, - at::Tensor& dq, - at::Tensor& dk, - at::Tensor& dv, - at::Tensor& out, - const std::optional& seq_offsets, - bool causal, - const std::optional& num_targets, - const std::optional& attn_scale, - int64_t max_attn_len, - int64_t min_full_attn_seq_len, - int64_t contextual_seq_len, - bool sort_by_length, - bool const deterministic, - const int64_t sm_margin, - const int64_t max_q_len, - const std::optional& seq_offsets_q, - int64_t num_softmax_heads, - const std::optional& softmax_lse, - const std::optional& max_seq_len_tensor, - const std::optional& contextual_seq_len_tensor, - const std::optional& max_attn_len_tensor, - const std::optional& min_full_attn_seq_len_tensor, - int64_t num_groups) { - return {dq, dk, dv}; -}; - -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_common_cpu.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_common_cpu.h deleted file mode 100644 index 9d0e18a71..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_common_cpu.h +++ /dev/null @@ -1,114 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/****************************************************************************** - * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, - *Pradeep Ramani, Tri Dao. - ******************************************************************************/ - -#include -#include // @manual -#include - -namespace hstu { - -std::tuple> hstu_mha_fwd_dummy( - int64_t max_seq_len, - double alpha, - at::Tensor& q, // (b, s, h, d) or (total_s, h, d) - at::Tensor& k, // (b, s, h, d) or (total_s, h, d) - at::Tensor& v, // (b, s, h, d) or (total_s, h, d) - const std::optional& seq_offsets, - bool causal, - const std::optional& num_targets, - const std::optional& attn_scale, - int64_t max_attn_len, - int64_t min_full_attn_seq_len, - int64_t contextual_seq_len, - const std::optional& q_descale, // (b, h_k), not (b, h) - const std::optional& k_descale, // (b, h_k) - const std::optional& v_descale, // (b, h_k) - const int64_t sm_margin = 0, - int64_t max_q_len = 0, - const std::optional& seq_offsets_q = std::nullopt, - int64_t num_softmax_heads = 0, - bool training = true, - const std::optional& max_seq_len_tensor = std::nullopt, - const std::optional& contextual_seq_len_tensor = std::nullopt, - const std::optional& max_attn_len_tensor = std::nullopt, - const std::optional& min_full_attn_seq_len_tensor = - std::nullopt, - int64_t num_groups = 1); - -std::vector hstu_mha_bwd_dummy( - int64_t max_seq_len, - double alpha, - at::Tensor& dout, - at::Tensor& q, - at::Tensor& k, - at::Tensor& v, - at::Tensor& dq, - at::Tensor& dk, - at::Tensor& dv, - at::Tensor& out, - const std::optional& seq_offsets, - bool causal, - const std::optional& num_targets, - const std::optional& attn_scale, - int64_t max_attn_len, - int64_t min_full_attn_seq_len, - int64_t contextual_seq_len, - bool sort_by_length, - bool const deterministic, - const int64_t sm_margin = 0, - int64_t max_q_len = 0, - const std::optional& seq_offsets_q = std::nullopt, - int64_t num_softmax_heads = 0, - const std::optional& softmax_lse = std::nullopt, - const std::optional& max_seq_len_tensor = std::nullopt, - const std::optional& contextual_seq_len_tensor = std::nullopt, - const std::optional& max_attn_len_tensor = std::nullopt, - const std::optional& min_full_attn_seq_len_tensor = - std::nullopt, - int64_t num_groups = 1); - -std::tuple> hstu_mha_fwd_meta( - const at::SymInt max_seq_len, - double alpha, - at::Tensor& q, // (b, s, h, d) or (total_s, h, d) - at::Tensor& k, // (b, s, h, d) or (total_s, h, d) - at::Tensor& v, // (b, s, h, d) or (total_s, h, d) - const std::optional& seq_offsets, - bool causal, - const std::optional& num_targets, - const std::optional& attn_scale, - int64_t max_attn_len, - int64_t min_full_attn_seq_len, - int64_t contextual_seq_len, - const std::optional& q_descale, // (b, h_k), not (b, h) - const std::optional& k_descale, // (b, h_k) - const std::optional& v_descale, // (b, h_k) - const int64_t sm_margin = 0, - int64_t max_q_len = 0, - const std::optional& seq_offsets_q = std::nullopt, - int64_t num_softmax_heads = 0, - bool training = true, - const std::optional& max_seq_len_tensor = std::nullopt, - const std::optional& contextual_seq_len_tensor = std::nullopt, - const std::optional& max_attn_len_tensor = std::nullopt, - const std::optional& min_full_attn_seq_len_tensor = - std::nullopt, - int64_t num_groups = 1); -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_fwd_kernel_sm90.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_fwd_kernel_sm90.h deleted file mode 100644 index 2e3d0916b..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_fwd_kernel_sm90.h +++ /dev/null @@ -1,511 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/****************************************************************************** - * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, - *Pradeep Ramani, Tri Dao. - ******************************************************************************/ - -#pragma once - -#include "cute/tensor.hpp" - -#include -#include -#include -#include -#include -#include -#include - -#include "seqlen.h" -#include "softmax.h" -#include "tile_scheduler.h" -#include "utils.h" - -namespace hstu { - -using namespace cute; - -template < - bool Softmax, - class CollectiveMainloop_, - class CollectiveEpilogue_, - class TileScheduler_> -class FlashAttnFwdSm90 { - public: - // Type Aliases - using CollectiveMainloop = CollectiveMainloop_; - using CollectiveEpilogue = CollectiveEpilogue_; - static constexpr bool Is_FP8 = CollectiveMainloop::Is_FP8; - static constexpr bool Transpose_V = CollectiveMainloop::Transpose_V; - static constexpr bool Use_TMA_O = CollectiveEpilogue::Use_TMA_O; - static constexpr int NumProducerThreads = - CollectiveMainloop::NumProducerThreads; - using SeqlenInfo_t = typename CollectiveMainloop::SeqlenInfo_t; - - // Mainloop derived types - using TileShape_MNK = typename CollectiveMainloop::TileShape_MNK; - using TiledMma0 = typename CollectiveMainloop::TiledMma0; - using TiledMma1 = typename CollectiveMainloop::TiledMma1; - using ArchTag = typename CollectiveMainloop::ArchTag; - using ClusterShape = typename CollectiveMainloop::ClusterShape; - using MainloopArguments = typename CollectiveMainloop::Arguments; - using MainloopParams = typename CollectiveMainloop::Params; - using BarrierQ = cutlass::arch::ClusterTransactionBarrier; - - // Epilogue derived types - using EpilogueArguments = typename CollectiveEpilogue::Arguments; - using EpilogueParams = typename CollectiveEpilogue::Params; - - static_assert(ArchTag::kMinComputeCapability >= 90); - - using TileScheduler = TileScheduler_; - using TileSchedulerArguments = typename hstu::TileSchedulerArguments; - using TileSchedulerParams = typename TileScheduler::Params; - - static constexpr uint32_t NumLoadWarpGroups = 1; - static constexpr uint32_t NumMmaWarpGroups = - CUTE_STATIC_V(size(TiledMma0{})) / cutlass::NumThreadsPerWarpGroup; - static constexpr uint32_t MaxThreadsPerBlock = - CUTE_STATIC_V(size(TiledMma0{})) + - (NumLoadWarpGroups * cutlass::NumThreadsPerWarpGroup); - static constexpr uint32_t MinBlocksPerMultiprocessor = 1; - static_assert( - NumMmaWarpGroups == 1 || NumMmaWarpGroups == 2 || NumMmaWarpGroups == 3); - - /// Register requirement for Load and Math WGs - // If we use cp.async to load K and V, we need more registers for the producer - // WG. - static constexpr uint32_t LoadRegisterRequirement = - NumMmaWarpGroups == 1 ? 56 : (NumMmaWarpGroups == 2 ? 24 : 32); - static constexpr uint32_t MmaRegisterRequirement = - NumMmaWarpGroups == 1 ? 256 : (NumMmaWarpGroups == 2 ? 240 : 160); - // If you want to print from the producer warp, you'd need to increase the - // number of registers Otherwise you'll get CUDA error. static constexpr - // uint32_t LoadRegisterRequirement = 40; static constexpr uint32_t - // MmaRegisterRequirement = NumMmaWarpGroups == 2 ? 232 : 152; - - // Kernel level shared memory storage - // We overlap the shared memory for the mainloop and epilogue. However, we - // only want smem_o to overlap with smem_v and nothing else, so we'll pad in - // case sizeof(smem_o) > sizeof(smem_v). - static constexpr int mainloop_smem_padding_ = - int(sizeof(typename CollectiveEpilogue::TensorStorage)) - - int(sizeof( - decltype((typename CollectiveMainloop::TensorStorage{}).smem_v))); - static constexpr int mainloop_smem_padding = - mainloop_smem_padding_ < 0 ? 0 : mainloop_smem_padding_; - struct SharedStorage { - struct TensorStorage : cute::aligned_struct<128> { - union { - struct { - cute::array - padding_; - typename CollectiveMainloop::TensorStorage mainloop; - }; - // We want smem_o to line up with the start of smem_v - typename CollectiveEpilogue::TensorStorage epilogue; - }; - } tensors; - - struct PipelineStorage : cute::aligned_struct<16> { - alignas(16) BarrierQ barrier_Q; - alignas(16) cutlass::arch::ClusterBarrier barrier_O; - alignas(16) typename CollectiveMainloop::MainloopPipelineK::SharedStorage - pipeline_k; - alignas(16) typename CollectiveMainloop::MainloopPipelineV::SharedStorage - pipeline_v; - alignas(16) typename CollectiveMainloop::MainloopPipelineVt::SharedStorage - pipeline_vt; - alignas(16) typename TileScheduler::SharedStorage smem_scheduler; - } pipelines; - }; - - static constexpr int SharedStorageSize = sizeof(SharedStorage); - - // Device side arguments - struct Arguments { - MainloopArguments mainloop{}; - EpilogueArguments epilogue{}; - cutlass::KernelHardwareInfo hw_info{}; - TileSchedulerArguments scheduler{}; - }; - - // Kernel entry point API - struct Params { - MainloopParams mainloop{}; - EpilogueParams epilogue{}; - cutlass::KernelHardwareInfo hw_info{}; - TileSchedulerParams scheduler{}; - }; - - // - // Methods - // - - // Convert to underlying arguments. In this case, a simple copy for the - // aliased type. - static Params to_underlying_arguments(Arguments const& args) { - CUTLASS_TRACE_HOST("to_underlying_arguments():"); - - // Get SM count if needed, otherwise use user supplied SM count - int sm_count = args.hw_info.sm_count; - if (sm_count <= 0) { - CUTLASS_TRACE_HOST( - " WARNING: Arguments do not include a valid SM count.\n" - " For optimal performance, populate the arguments KernelHardwareInfo struct with the SM count."); - sm_count = cutlass::KernelHardwareInfo::query_device_multiprocessor_count( - args.hw_info.device_id); - } - - CUTLASS_TRACE_HOST( - "to_underlying_arguments(): Setting persistent grid SM count to " - << sm_count); - - cutlass::KernelHardwareInfo hw_info{args.hw_info.device_id, sm_count}; - return { - CollectiveMainloop::to_underlying_arguments(args.mainloop), - CollectiveEpilogue::to_underlying_arguments(args.epilogue), - hw_info, - TileScheduler::to_underlying_arguments(args.scheduler)}; - } - - // Computes the kernel launch grid shape based on runtime parameters - static dim3 get_grid_shape(Params const& params) { - return TileScheduler::get_grid_shape( - params.scheduler, params.hw_info.sm_count); - } - - static dim3 get_block_shape() { - return dim3(MaxThreadsPerBlock, 1, 1); - } - - CUTLASS_DEVICE - void operator()(Params const& params, char* smem_buf) { - static constexpr int NumMmaThreads = - NumMmaWarpGroups * cutlass::NumThreadsPerWarpGroup; - static constexpr int MmaThreadOffset = - NumLoadWarpGroups * cutlass::NumThreadsPerWarpGroup; - static constexpr int kBlockM = get<0>(TileShape_MNK{}); - - using MainloopPipelineK = typename CollectiveMainloop::MainloopPipelineK; - using MainloopPipelineV = typename CollectiveMainloop::MainloopPipelineV; - using MainloopPipelineVt = typename CollectiveMainloop::MainloopPipelineVt; - using MainloopPipelineKVNew = - typename CollectiveMainloop::MainloopPipelineKVNew; - using PipelineState = typename CollectiveMainloop::PipelineState; - using PipelineParamsK = typename MainloopPipelineK::Params; - using PipelineParamsV = typename MainloopPipelineV::Params; - using PipelineParamsVt = typename MainloopPipelineVt::Params; - using PipelineParamsKVNew = typename MainloopPipelineKVNew::Params; - - SharedStorage& shared_storage = *reinterpret_cast(smem_buf); - - int const lane_predicate = cute::elect_one_sync(); - int const warp_idx = cutlass::canonical_warp_idx_sync(); - - // Issue Tma Descriptor Prefetch from a single thread - if (warp_idx == 0 && lane_predicate) { - CollectiveMainloop::prefetch_tma_descriptors(params.mainloop); - CollectiveEpilogue::prefetch_tma_descriptors(params.epilogue); - } - - // Obtain warp index - int const warp_group_thread_idx = - threadIdx.x % cutlass::NumThreadsPerWarpGroup; - int warp_group_idx = cutlass::canonical_warp_group_idx(); - - if (warp_idx == 0 && lane_predicate) { - shared_storage.pipelines.barrier_Q.init(1 /*numThreads*/); - shared_storage.pipelines.barrier_O.init( - size(ClusterShape{}) * - (Use_TMA_O ? 1 : NumMmaThreads) /*numThreads*/); - } - - // We're counting on pipeline_k to call cutlass::arch::fence_barrier_init(); - PipelineParamsK pipeline_params_k; - pipeline_params_k.role = warp_group_idx == 0 - ? MainloopPipelineK::ThreadCategory::Producer - : MainloopPipelineK::ThreadCategory::Consumer; - pipeline_params_k.transaction_bytes = - CollectiveMainloop::TmaTransactionBytesK; - pipeline_params_k.is_leader = warp_group_thread_idx == 0; - pipeline_params_k.num_consumers = NumMmaThreads; - - MainloopPipelineK pipeline_k = [&] { - return MainloopPipelineK( - shared_storage.pipelines.pipeline_k, - pipeline_params_k, - ClusterShape{}); - }(); - // MainloopPipelineV pipeline_v(shared_storage.pipelines.pipeline_v, - // pipeline_params_v, ClusterShape{}); - MainloopPipelineV pipeline_v = [&] { - if constexpr (!Transpose_V) { - static_assert(is_same_v); - return MainloopPipelineV( - shared_storage.pipelines.pipeline_v, - pipeline_params_k, - ClusterShape{}); - } else { - PipelineParamsV pipeline_params_v; - pipeline_params_v.role = warp_group_idx == 0 - ? MainloopPipelineV::ThreadCategory::Producer - : MainloopPipelineV::ThreadCategory::Consumer; - pipeline_params_v.producer_arv_count = NumProducerThreads; - pipeline_params_v.consumer_arv_count = NumMmaThreads; - return MainloopPipelineV( - shared_storage.pipelines.pipeline_v, pipeline_params_v); - } - }(); - static_assert(is_same_v); - // If we need to transpose V (e.g. FP8 and V is row-major), we use - // pipeline_vt for the TMA, then the producer WG will read from pipeline_vt - // and write to pipeline_v. If we don't need to transpose V, we use - // pipeline_v for the TMA, and pipeline_vt won't be used. Technically for - // pipeline_params_vt, warp0 of WG0 is the producer and all of WG0 are - // consumers. However, the thread role isn't used in the pipeline - // implementation. - MainloopPipelineVt pipeline_vt = [&] { - pipeline_params_k.num_consumers = - NumProducerThreads; // TMA_V is only consumed by the producer WG - return MainloopPipelineVt( - shared_storage.pipelines.pipeline_vt, - pipeline_params_k, - ClusterShape{}); - }(); - - CollectiveMainloop collective_mainloop; - CollectiveEpilogue collective_epilogue; - - // We need this to guarantee that the Pipeline init is visible to all - // producers and consumer blocks in the Cluster - if constexpr (size(ClusterShape{}) > 1) { - cute::cluster_arrive_relaxed(); - cute::cluster_wait(); - } else { - __syncthreads(); - } - - if (warp_group_idx == 0) { // Producer - cutlass::arch::warpgroup_reg_dealloc(); - - // The pipelines for AppendKV and main attention are different, since e.g. - // main attention might use cp.async to load KV (if PagedKV) while - // AppendKV always uses TMA to load KV_new. Since the pipeline states are - // different, we have to manually sync to make sure the two pipelines - // don't race when accessing smem_k and smem_v. - PipelineState smem_pipe_write = - cutlass::make_producer_start_state(); - PipelineState smem_pipe_write_new = - cutlass::make_producer_start_state(); - int work_idx = 0; - - TileScheduler scheduler( - reinterpret_cast( - &shared_storage.pipelines.smem_scheduler)); - int warp_idx_in_warpgroup = - __shfl_sync(0xffffffff, (threadIdx.x / 32) % 4, 0); - static constexpr bool SingleProducerWarp = - NumProducerThreads == cutlass::NumThreadsPerWarp; - if constexpr (SingleProducerWarp) { - if (warp_idx_in_warpgroup != 0) { - return; - } - } - if (!SingleProducerWarp && warp_idx_in_warpgroup != 0) { - scheduler.init_consumer(); - } - - // Load Q, K, V - for (auto work_tile_info = SingleProducerWarp || - warp_idx_in_warpgroup == 0 - ? scheduler.template get_initial_work( - params.scheduler) - : scheduler.template get_initial_work( - params.scheduler); - work_tile_info.is_valid(params.scheduler); - work_tile_info = SingleProducerWarp || warp_idx_in_warpgroup == 0 - ? scheduler.template get_next_work( - params.scheduler, work_tile_info) - : scheduler.template get_next_work( - params.scheduler, work_tile_info)) { - auto block_coord = work_tile_info.get_block_coord(params.scheduler); - SeqlenInfo_t seqlen_info{ - get<2>(block_coord) /*bidb*/, - get<0>(params.mainloop.shape_Q), - get<0>(params.mainloop.shape_K), - params.mainloop.seq_offsets, - params.mainloop.seq_offsets_q, - params.mainloop.num_targets, - }; - auto scheduler_prefetch = [&scheduler, ¶ms, &work_tile_info]() { - scheduler.prefetch_next_work(params.scheduler, work_tile_info); - }; - // pipeline_vt won't be used if we don't need to transpose V. - collective_mainloop.load( - params.mainloop, - pipeline_k, - pipeline_v, - pipeline_vt, - smem_pipe_write, - shared_storage, - scheduler_prefetch, - seqlen_info, - block_coord, - work_idx); - } - collective_mainloop.load_tail( - pipeline_k, - pipeline_v, - pipeline_vt, - smem_pipe_write, - shared_storage, - work_idx); - } else { // Consumer - cutlass::arch::warpgroup_reg_alloc(); - - TileScheduler scheduler( - reinterpret_cast( - &shared_storage.pipelines.smem_scheduler)); - // Initialize matmul objects. - TiledMma1 tiled_mma1; - - PipelineState smem_pipe_read; - // We don't need separate variables smem_pipe_release_k and - // smem_pipe_release_v (like in Cutlass's gemm) because the read and - // release pipeline states are always the same. - - scheduler.init_consumer(); - collective_mainloop.mma_init(); - - int work_idx = 0; - CUTLASS_PRAGMA_NO_UNROLL - for (auto work_tile_info = - scheduler.template get_initial_work( - params.scheduler); - work_tile_info.is_valid(params.scheduler); - work_tile_info = - scheduler.template get_next_work( - params.scheduler, work_tile_info)) { - // Attention output (GEMM-II) accumulator. - Tensor tOrO = - partition_fragment_C(tiled_mma1, select<0, 2>(TileShape_MNK{})); - // If there's tanh softcap, the scaling will be done before tanh. - auto block_coord = work_tile_info.get_block_coord(params.scheduler); - int const bidb = get<2>(block_coord); - int const bidh = get<1>(block_coord); - if constexpr (Is_FP8) { - int const bidh_kv = bidh; - float const q_descale = params.mainloop.ptr_q_descale == nullptr - ? 1.0f - : params.mainloop.ptr_q_descale - [bidb * get<0>(params.mainloop.stride_q_descale) + - bidh_kv * get<1>(params.mainloop.stride_q_descale)]; - float const k_descale = params.mainloop.ptr_k_descale == nullptr - ? 1.0f - : params.mainloop.ptr_k_descale - [bidb * get<0>(params.mainloop.stride_k_descale) + - bidh_kv * get<1>(params.mainloop.stride_k_descale)]; - } - - SeqlenInfo_t seqlen_info{ - bidb, - get<0>(params.mainloop.shape_Q), - get<0>(params.mainloop.shape_K), - params.mainloop.seq_offsets, - params.mainloop.seq_offsets_q, - params.mainloop.num_targets, - }; - float alpha_log2 = params.mainloop.alpha_log2; - bool tile_valid; - if constexpr (Softmax) { - hstu::Softmax< - 2 * (2 * kBlockM / NumMmaThreads), - /*Max_offset=*/!Is_FP8 ? 0 : 8> - softmax(alpha_log2); - tile_valid = collective_mainloop.mma_softmax( - params.mainloop, - pipeline_k, - pipeline_v, - smem_pipe_read, - tOrO, - softmax, - threadIdx.x - MmaThreadOffset, - work_idx, - seqlen_info, - block_coord, - shared_storage); - if (tile_valid) { - collective_epilogue.store( - params.epilogue, - tOrO, - shared_storage, - tiled_mma1, - threadIdx.x - MmaThreadOffset, - block_coord); - collective_epilogue.store_softmax( - params.epilogue, - softmax.row_sum, - tiled_mma1, - threadIdx.x - MmaThreadOffset, - block_coord); - } else { - // Write 0 to gO and -inf to gLSE. - // If Split, we don't have to write 0 to O if the mha_combine kernel - // is used, since it will not use the value of O if LSE is -inf. - collective_epilogue.template store_zero( - params.epilogue, threadIdx.x - MmaThreadOffset, block_coord); - // collective_epilogue.store_zero(params.epilogue, threadIdx.x - - // MmaThreadOffset, block_coord); - } - } else { - tile_valid = collective_mainloop.mma( - params.mainloop, - pipeline_k, - pipeline_v, - smem_pipe_read, - tOrO, - threadIdx.x - MmaThreadOffset, - work_idx, - seqlen_info, - block_coord, - shared_storage); - if (tile_valid) { - collective_epilogue.store( - params.epilogue, - tOrO, - shared_storage, - tiled_mma1, - threadIdx.x - MmaThreadOffset, - block_coord); - } else { - // Write 0 to gO and -inf to gLSE. - // If Split, we don't have to write 0 to O if the mha_combine kernel - // is used, since it will not use the value of O if LSE is -inf. - collective_epilogue.template store_zero( - params.epilogue, threadIdx.x - MmaThreadOffset, block_coord); - // collective_epilogue.store_zero(params.epilogue, threadIdx.x - - // MmaThreadOffset, block_coord); - } - } - } - collective_epilogue.store_tail(); - } - } -}; - -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_fwd_launch_template.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_fwd_launch_template.h deleted file mode 100644 index c79ea3a3f..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/flash_fwd_launch_template.h +++ /dev/null @@ -1,376 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/****************************************************************************** - * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, - *Pradeep Ramani, Tri Dao. - ******************************************************************************/ - -#pragma once - -// clang-format off -#include "cute/tensor.hpp" - -#include "cutlass/cutlass.h" -#include "cutlass/device_kernel.h" // For device_kernel -#include -#include "cutlass/cluster_launch.hpp" -#include "cutlass/kernel_launch.h" - -#include "static_switch.h" -#include "flash.h" -#include "tile_size.h" -#include "tile_scheduler.h" -#include "flash_fwd_kernel_sm90.h" -#include "mainloop_fwd_sm90_tma_gmma_ws.h" -#include "epilogue_fwd.h" -// clang-format on - -namespace hstu { - -using namespace cute; - -template < - int Arch, - int kHeadDim, - int ClusterM, - typename Element, - typename ElementOut, - bool Causal, - bool Local, - bool Contexual_mask, - bool Jagged, - bool Has_targets, - bool V_colmajor, - bool Cross, - bool Softmax, - bool Training> -void run_flash_fwd(hstu::Flash_fwd_params& params, cudaStream_t stream) { - static_assert( - !(Causal && Local), - "Causal and Local cannot be enabled at the same time"); - static constexpr bool Is_FP8 = - cute::is_same_v || - cute::is_same_v; - static constexpr bool FP8_TransposeV = Is_FP8 && !V_colmajor; - using ArchTag = - std::conditional_t= 90, cutlass::arch::Sm90, cutlass::arch::Sm80>; - - // Can't use structured binding since it's not compatible with constexpr - static constexpr std::tuple kBlockMN_RS = - hstu::tile_size_fwd_sm90( - kHeadDim, - Causal, - Local, - sizeof(Element) /*element_size*/, - V_colmajor, - Cross, - Training); - static constexpr std::tuple - kBlockMN_kNWarps_Stages_RS = hstu::tile_size_fwd_sm8x( - Arch == 86 || Arch == 89, - kHeadDim, - Causal, - Local, - sizeof(Element) /*element_size*/); - static constexpr int kBlockM = Arch >= 90 - ? std::get<0>(kBlockMN_RS) - : std::get<0>(kBlockMN_kNWarps_Stages_RS); - static constexpr int kBlockN = Arch >= 90 - ? std::get<1>(kBlockMN_RS) - : std::get<1>(kBlockMN_kNWarps_Stages_RS); - static constexpr bool Mma1_is_RS = std::get<2>(kBlockMN_RS); - static constexpr int kNWarps = std::get<2>(kBlockMN_kNWarps_Stages_RS); - static constexpr int kStages = - Arch >= 90 ? 2 : std::get<3>(kBlockMN_kNWarps_Stages_RS); - static constexpr bool Q_in_regs = - Arch >= 90 ? false : std::get<4>(kBlockMN_kNWarps_Stages_RS); - -#ifdef HSTU_FLASH_ATTN_DEBUG_INFO - std::printf( - "kBlockM: (%d), kBlockN: (%d), Mma1_is_RS: (%d), kNWarps: (%d), kStages: (%d), Q_in_regs: (%d)\n", - kBlockM, - kBlockN, - Mma1_is_RS, - kNWarps, - kStages, - Q_in_regs); -#endif - - using TileShape_MNK = cute::Shape, Int, Int>; - using ClusterShape = cute::Shape, _1, _1>; - using CollectiveMainloop = hstu::CollectiveMainloopFwdSm90< - kStages, - ClusterShape, - TileShape_MNK, - Element, - float, - cutlass::arch::Sm90, - Causal, - Local, - Contexual_mask, - Jagged, - Has_targets, - Mma1_is_RS, - V_colmajor, - Cross>; - using CollectiveEpilogue = hstu::CollectiveEpilogueFwd< - TileShape_MNK, - ClusterShape, - ElementOut, - ArchTag, - CollectiveMainloop::NumMmaThreads, - Jagged, - FP8_TransposeV>; - - static constexpr int NumProducerThreads = Arch >= 90 - ? CollectiveMainloop::NumProducerThreads - : CollectiveMainloop::NumMmaThreads; - using SchedulerPersistent = std::conditional_t< - Jagged, - hstu::VarlenDynamicPersistentTileScheduler< - kBlockM, - CollectiveMainloop::NumMmaThreads, - NumProducerThreads, - Arch >= 90 /*WarpSpecialized*/>, - std::conditional_t< - !Causal && !Local, - hstu::StaticPersistentTileScheduler, - hstu::DynamicPersistentTileScheduler< - CollectiveMainloop::NumMmaThreads, - NumProducerThreads, - Arch >= 90 /*WarpSpecialized*/>>>; - using SchedulerSingleTile = hstu:: - SingleTileScheduler; - // If Split then we probably don't have enough work for PersistentScheduler to - // be useful. However, if Jagged (e.g., during decode where we have - // max_seqlens), using PersistentScheduler is better since we'll avoid - // launching a bunch of thread blocks that immediately exit. On Sm80, - // noncausal persistent seems a bit slower. - using Scheduler = std::conditional_t< - Arch >= 90 ? false : !(Causal && !Jagged), - SchedulerSingleTile, - SchedulerPersistent>; - using AttnKernel = hstu::enable_sm90_or_later>; - - int seqlen_q = !Jagged ? params.max_q_len : params.total_seq_len_q; - int seqlen_kv = !Jagged ? params.max_kv_len : params.total_seq_len_kv; - int batch = !Jagged ? params.b : 1; -#ifdef HSTU_FLASH_ATTN_DEBUG_INFO - std::printf("max/total seqlen: (%d), batch: (%d)\n", seqlen, batch); -#endif - typename CollectiveMainloop::StrideV v_strides = - cute::conditional_return( - make_stride( - params.v_row_stride, - _1{}, - params.v_head_stride, - !Jagged ? params.v_batch_stride : 0), - make_stride( - _1{}, - params.v_dim_stride, - params.v_head_stride, - !Jagged ? params.v_batch_stride : 0)); - typename CollectiveMainloop::Arguments mainloop_args{ - static_cast(params.q_ptr), - {seqlen_q, params.qk_d, params.h, batch}, // shape_Q - {params.q_row_stride, - _1{}, - params.q_head_stride, - !Jagged ? params.q_batch_stride : 0}, // stride_Q - static_cast(params.k_ptr), - {seqlen_kv, params.qk_d, params.h, batch}, // shape_K - {params.k_row_stride, - _1{}, - params.k_head_stride, - !Jagged ? params.k_batch_stride : 0}, // stride_K - static_cast(params.v_ptr), - v_strides, // stride_V - params.q_descale_ptr, - params.k_descale_ptr, - params.v_descale_ptr, - {params.q_descale_batch_stride, params.q_descale_head_stride}, - {params.k_descale_batch_stride, params.k_descale_head_stride}, - {params.v_descale_batch_stride, params.v_descale_head_stride}, - 1.0f / params.max_kv_len, - params.alpha, - params.max_attn_len, - params.min_full_attn_seq_len, - params.contextual_seq_len, - params.num_softmax_heads, - params.num_groups, - params.batch_size_per_group, - params.seq_offsets, - params.seq_offsets_q, - params.num_targets, - params.max_seq_len_tensor, - params.contextual_seq_len_tensor, - params.max_attn_len_tensor, - params.min_full_attn_seq_len_tensor, - params.attn_scale, - params.scalar_scale, - }; - typename CollectiveEpilogue::Arguments epilogue_args{ - static_cast(params.o_ptr), - {seqlen_q, params.v_d, params.h, batch, 1}, // shape_O - {params.o_row_stride, - _1{}, - params.o_head_stride, - !Jagged ? params.o_batch_stride : 0, - 0}, // stride_O - params.h, - params.num_softmax_heads, - {_1{}, seqlen_q, !Jagged ? params.h * seqlen_q : 0, 0}, // stride_LSE} - static_cast(params.softmax_lse), - Cross ? params.seq_offsets_q : params.seq_offsets}; - - int num_blocks_m = - cutlass::ceil_div(params.max_q_len, get<0>(TileShape_MNK{})); - num_blocks_m = cutlass::round_up(num_blocks_m, size<0>(ClusterShape{})); - typename hstu::TileSchedulerArguments scheduler_args{ - num_blocks_m, - params.h, - params.b, - params.max_q_len, - params.qk_d, - sizeof(Element), - params.tile_count_semaphore, - Cross ? params.seq_offsets_q : params.seq_offsets, - nullptr /*sort_by_length_indices*/}; - - int device; - CHECK_CUDA(cudaGetDevice(&device)); - typename AttnKernel::Params kernel_params = - AttnKernel::to_underlying_arguments( - {mainloop_args, - epilogue_args, - {device, params.num_sm}, - scheduler_args}); - - dim3 grid_dims = AttnKernel::get_grid_shape(kernel_params); - dim3 block_dims = AttnKernel::get_block_shape(); - int smem_size = AttnKernel::SharedStorageSize; - // int smem_size_q = sizeof(decltype((typename - // CollectiveMainloop::TensorStorage{}).smem_q)); int smem_size_k = - // sizeof(decltype((typename CollectiveMainloop::TensorStorage{}).smem_k)); - // int smem_size_v = sizeof(decltype((typename - // CollectiveMainloop::TensorStorage{}).smem_v)); printf("smem_size = %d, q = - // %d, k = %d, v = %d\n", smem_size, smem_size_q, smem_size_k, smem_size_v); - // Get the ptr to kernel function. - if constexpr (size(ClusterShape{}) > 1) { - void const* kernel = (void const*)cutlass::device_kernel; - if (smem_size >= 48 * 1024) { - CHECK_CUDA(cudaFuncSetAttribute( - kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size)); - } - dim3 cluster_dims( - size<0>(ClusterShape{}), - size<1>(ClusterShape{}), - size<2>(ClusterShape{})); - cutlass::ClusterLaunchParams launch_params{ - grid_dims, block_dims, cluster_dims, smem_size, stream}; - cutlass::launch_kernel_on_cluster(launch_params, kernel, kernel_params); - } else { -#ifdef HSTU_FLASH_ATTN_DEBUG_INFO - std::cout << "ClusterShape = 1" << std::endl; - std::cout << "grid_dims = " << grid_dims << std::endl; - std::cout << "block_dims = " << block_dims << std::endl; - std::cout << "smem_size = " << smem_size << std::endl; -#endif - auto kernel = cutlass::device_kernel; - if (smem_size >= 48 * 1024) { - CHECK_CUDA(cudaFuncSetAttribute( - kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size)); - } - kernel<<>>(kernel_params); - } - CHECK_CUDA_KERNEL_LAUNCH(); -} - -template < - int Arch, - int kHeadDim, - bool Causal, - bool Local, - bool Softmax, - typename T, - typename T_out> -void run_mha_fwd_dispatch(hstu::Flash_fwd_params& params, cudaStream_t stream) { - static constexpr bool V_colmajor = false; // V_colmajor_ && sizeof(T) == 1; - BOOL_SWITCH(params.num_targets, Has_targets, [&] { - BOOL_SWITCH(params.seq_offsets, Jagged, [&] { - BOOL_SWITCH(params.seq_offsets_q, Cross, [&] { - BOOL_SWITCH(params.has_contexual_mask, Contexual_mask, [&] { - BOOL_SWITCH(params.training, Training, [&] { -#ifdef HSTU_FLASH_ATTN_DEBUG_INFO - std::printf( - "[flash_fwd_launch_template] Local: (%d), Jagged: (%d), Has_targets: (%d), Causal: (%d), max_kv_len: (%d), kHeadDim: (%d)\n", - Local, - Jagged, - Has_targets, - Causal, - params.max_kv_len, - kHeadDim); -#endif - // static constexpr bool Enable_cluster = Arch >= 90 && - // (sizeof(T) == 2 ? (kHeadDim >= 128) : (kHeadDim == 192)) && - // !Causal && !Local && !Jagged; - // static constexpr bool Enable_cluster = false; - // CLUSTER_SWITCH( - // cutlass::ceil_div(params.max_q_len, kBlockM) % 2 == 0, - // Use_cluster, - // [&] { - // static constexpr int ClusterM = - // Enable_cluster && Use_cluster ? 2 : 1; - run_flash_fwd< - Arch, - kHeadDim, - 1, // ClusterM, - T, - T_out, - Causal, - Local, - Contexual_mask, - Jagged, - Has_targets, - V_colmajor, - Cross, - Softmax, - Training>(params, stream); - }); - }); - }); - }); - }); -} - -template -void run_mha_fwd_(hstu::Flash_fwd_params& params, cudaStream_t stream) { - static_assert( - sizeof(T) == 2 || sizeof(T) == 1, "Only 16bit and 8bit are supported"); - static constexpr bool Is_FP8 = cute::is_same_v || - cute::is_same_v; - using T_out = std::conditional_t; - CAUSAL_LOCAL_SWITCH(params.is_causal, params.is_local, Causal, Local, [&] { - // VCOLMAJOR_SWITCH(params.v_dim_stride != 1, V_colmajor_, [&] { - run_mha_fwd_dispatch( - params, stream); - }); -} -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/generate_kernels.py b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/generate_kernels.py deleted file mode 100644 index 6c3a03188..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/generate_kernels.py +++ /dev/null @@ -1,236 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-unsafe - - -# Copied from Driss Guessous's PR in PyTorch: https://github.com/pytorch/pytorch/pull/105602 - -# This file is run to generate the kernel instantiations for the flash_attn kernels -# They are written to several files in order to speed up compilation - -import argparse -import itertools -from dataclasses import dataclass -from pathlib import Path -from typing import List, Optional, Union - - -DTYPE_MAP = { - "fp16": "cutlass::half_t", - "bf16": "cutlass::bfloat16_t", - "e4m3": "cutlass::float_e4m3_t", -} - -DTYPE_MAP_FWD_SM8x = { - "fp16": "cutlass::half_t", - "bf16": "cutlass::bfloat16_t", -} - -DTYPE_MAP_BWD = { - "fp16": "cutlass::half_t", - "bf16": "cutlass::bfloat16_t", -} - -SM = [90] # Sm kernels support up to -SOFTMAX = ["true", "false"] -HEAD_DIMENSIONS = [64, 96, 128, 192, 256] - -KERNEL_IMPL_TEMPLATE_FWD_SM90 = """ -#ifdef OSS_ENV -#include "hstu_attention/flash_fwd_launch_template.h" -#else -#include "flash_fwd_launch_template.h" -#endif - -namespace hstu {{ -#ifndef FLASHATTENTION_DISABLE_HDIM{HEAD_DIM} -template void run_mha_fwd_<{ARCH}, {DTYPE}, {HEAD_DIM}, {SOFTMAX}>(Flash_fwd_params ¶ms, cudaStream_t stream); -#endif -}} // namespace hstu -""" - -KERNEL_IMPL_TEMPLATE_FWD_SM8x = """ -#ifdef OSS_ENV -#include "hstu_attention/flash_fwd_launch_template.h" -#else -#include "flash_fwd_launch_template.h" -#endif - -namespace hstu {{ -#ifndef FLASHATTENTION_DISABLE_SM8x -#ifndef FLASHATTENTION_DISABLE_HDIM{HEAD_DIM} -template void run_mha_fwd_<80, {DTYPE}, {HEAD_DIM}, {SOFTMAX}>(Flash_fwd_params ¶ms, cudaStream_t stream); -#endif -#endif -}} // namespace hstu -""" - -KERNEL_IMPL_TEMPLATE_BWD_SM90 = """ -#ifdef OSS_ENV -#include "hstu_attention/flash_bwd_launch_template.h" -#else -#include "flash_bwd_launch_template.h" -#endif - -namespace hstu {{ -#ifndef FLASHATTENTION_DISABLE_HDIM{HEAD_DIM} -template void run_mha_bwd_<{ARCH}, {DTYPE}, {HEAD_DIM}, {SOFTMAX}>(Flash_bwd_params ¶ms, cudaStream_t stream); -#endif -}} // namespace hstu -""" - -KERNEL_IMPL_TEMPLATE_BWD_SM8x = """ -#ifdef OSS_ENV -#include "hstu_attention/flash_bwd_launch_template.h" -#else -#include "flash_bwd_launch_template.h" -#endif - -namespace hstu {{ -#ifndef FLASHATTENTION_DISABLE_SM8x -#ifndef FLASHATTENTION_DISABLE_HDIM{HEAD_DIM} -template void run_mha_bwd_<80, {DTYPE}, {HEAD_DIM}, {SOFTMAX}>(Flash_bwd_params ¶ms, cudaStream_t stream); -#endif -#endif -}} // namespace hstu -""" - - -@dataclass -class Kernel: - sm: int - dtype: str - head_dim: int - softmax: str - direction: str - - @property - def template(self) -> str: - if self.direction == "fwd": - if self.sm == 90: - return KERNEL_IMPL_TEMPLATE_FWD_SM90.format( - ARCH=str(self.sm), - DTYPE=DTYPE_MAP[self.dtype], - HEAD_DIM=self.head_dim, - SOFTMAX=self.softmax, - ) - else: - # Always enable PackGQA for Sm8x to reduce compilation - return KERNEL_IMPL_TEMPLATE_FWD_SM8x.format( - DTYPE=DTYPE_MAP[self.dtype], - HEAD_DIM=self.head_dim, - SOFTMAX=self.softmax, - ) - else: - assert self.direction == "bwd" - if self.sm == 90: - return KERNEL_IMPL_TEMPLATE_BWD_SM90.format( - ARCH=str(self.sm), - DTYPE=DTYPE_MAP[self.dtype], - HEAD_DIM=self.head_dim, - SOFTMAX=self.softmax, - ) - else: - return KERNEL_IMPL_TEMPLATE_BWD_SM8x.format( - DTYPE=DTYPE_MAP[self.dtype], - HEAD_DIM=self.head_dim, - SOFTMAX=self.softmax, - ) - - @property - def filename(self) -> str: - return f"flash_{self.direction}_hdim{self.head_dim}_{self.dtype}_softmax{self.softmax}_sm{self.sm}.cu" - - -def get_all_kernels() -> List[Kernel]: - kernels: List[Kernel] = [] - for dtype, head_dim, sm, softmax in itertools.product( - DTYPE_MAP.keys(), HEAD_DIMENSIONS, SM, SOFTMAX - ): - # We always enable PackGQA for Sm8x or Split - # so we should just pass in packgqa=False to avoid the `_packgqa` in the filename. - if sm >= 90 or dtype in DTYPE_MAP_FWD_SM8x: - kernels.append( - Kernel( - sm=sm, - dtype=dtype, - head_dim=head_dim, - direction="fwd", - softmax=softmax, - ) - ) - for dtype, head_dim, sm, softmax in itertools.product( - DTYPE_MAP_BWD.keys(), HEAD_DIMENSIONS, SM, SOFTMAX - ): - kernels.append( - Kernel( - sm=sm, - dtype=dtype, - head_dim=head_dim, - direction="bwd", - softmax=softmax, - ) - ) - return kernels - - -def write_kernel(kernel: Union[Kernel], autogen_dir: Path) -> None: - prelude = """ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ \n -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, Pradeep Ramani, Tri Dao. -// Splitting the different template instantiations to different files to speed up compilation. -// This file is auto-generated. See "generate_kernels.py"\n -""" - (autogen_dir / kernel.filename).write_text(prelude + kernel.template) - - -def main(output_dir_name: Optional[str]) -> None: - output_dir = ( - Path(output_dir_name) if output_dir_name is not None else Path(__file__).parent - ) - output_dir.mkdir(parents=True, exist_ok=True) - kernels_all = list(get_all_kernels()) - for kernel in kernels_all: - write_kernel(kernel, output_dir) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - prog="generate_kernels", - description="Generate the flash_attention kernels template instantiations", - ) - # Set an optional output directory - parser.add_argument( - "-o", - "--output_dir", - default="instantiations", - required=False, - help="Where to generate the kernels will default to the current directory ", - ) - args = parser.parse_args() - main(args.output_dir) diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim128_bf16_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim128_bf16_softmaxfalse_sm90.cu deleted file mode 100644 index da0eeb2df..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim128_bf16_softmaxfalse_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_bwd_launch_template.h" -#else -#include "flash_bwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM128 -template void run_mha_bwd_<90, cutlass::bfloat16_t, 128, false>( - Flash_bwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim128_bf16_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim128_bf16_softmaxtrue_sm90.cu deleted file mode 100644 index 8d85c2235..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim128_bf16_softmaxtrue_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_bwd_launch_template.h" -#else -#include "flash_bwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM128 -template void run_mha_bwd_<90, cutlass::bfloat16_t, 128, true>( - Flash_bwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim128_fp16_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim128_fp16_softmaxfalse_sm90.cu deleted file mode 100644 index 09226cd80..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim128_fp16_softmaxfalse_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_bwd_launch_template.h" -#else -#include "flash_bwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM128 -template void run_mha_bwd_<90, cutlass::half_t, 128, false>( - Flash_bwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim128_fp16_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim128_fp16_softmaxtrue_sm90.cu deleted file mode 100644 index 63e451d14..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim128_fp16_softmaxtrue_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_bwd_launch_template.h" -#else -#include "flash_bwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM128 -template void run_mha_bwd_<90, cutlass::half_t, 128, true>( - Flash_bwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim192_bf16_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim192_bf16_softmaxfalse_sm90.cu deleted file mode 100644 index e379d9918..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim192_bf16_softmaxfalse_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_bwd_launch_template.h" -#else -#include "flash_bwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM192 -template void run_mha_bwd_<90, cutlass::bfloat16_t, 192, false>( - Flash_bwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim192_bf16_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim192_bf16_softmaxtrue_sm90.cu deleted file mode 100644 index 7faa31376..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim192_bf16_softmaxtrue_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_bwd_launch_template.h" -#else -#include "flash_bwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM192 -template void run_mha_bwd_<90, cutlass::bfloat16_t, 192, true>( - Flash_bwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim192_fp16_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim192_fp16_softmaxfalse_sm90.cu deleted file mode 100644 index 5ddc7d7fc..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim192_fp16_softmaxfalse_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_bwd_launch_template.h" -#else -#include "flash_bwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM192 -template void run_mha_bwd_<90, cutlass::half_t, 192, false>( - Flash_bwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim192_fp16_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim192_fp16_softmaxtrue_sm90.cu deleted file mode 100644 index 530deae2b..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim192_fp16_softmaxtrue_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_bwd_launch_template.h" -#else -#include "flash_bwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM192 -template void run_mha_bwd_<90, cutlass::half_t, 192, true>( - Flash_bwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim256_bf16_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim256_bf16_softmaxfalse_sm90.cu deleted file mode 100644 index 185907c5e..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim256_bf16_softmaxfalse_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_bwd_launch_template.h" -#else -#include "flash_bwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM256 -template void run_mha_bwd_<90, cutlass::bfloat16_t, 256, false>( - Flash_bwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim256_bf16_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim256_bf16_softmaxtrue_sm90.cu deleted file mode 100644 index 39df173bb..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim256_bf16_softmaxtrue_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_bwd_launch_template.h" -#else -#include "flash_bwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM256 -template void run_mha_bwd_<90, cutlass::bfloat16_t, 256, true>( - Flash_bwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim256_fp16_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim256_fp16_softmaxfalse_sm90.cu deleted file mode 100644 index cdc0a9f7e..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim256_fp16_softmaxfalse_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_bwd_launch_template.h" -#else -#include "flash_bwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM256 -template void run_mha_bwd_<90, cutlass::half_t, 256, false>( - Flash_bwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim256_fp16_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim256_fp16_softmaxtrue_sm90.cu deleted file mode 100644 index 6f3182d34..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim256_fp16_softmaxtrue_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_bwd_launch_template.h" -#else -#include "flash_bwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM256 -template void run_mha_bwd_<90, cutlass::half_t, 256, true>( - Flash_bwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim64_bf16_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim64_bf16_softmaxfalse_sm90.cu deleted file mode 100644 index 89285d0be..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim64_bf16_softmaxfalse_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_bwd_launch_template.h" -#else -#include "flash_bwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM64 -template void run_mha_bwd_<90, cutlass::bfloat16_t, 64, false>( - Flash_bwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim64_bf16_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim64_bf16_softmaxtrue_sm90.cu deleted file mode 100644 index ab39c7e06..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim64_bf16_softmaxtrue_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_bwd_launch_template.h" -#else -#include "flash_bwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM64 -template void run_mha_bwd_<90, cutlass::bfloat16_t, 64, true>( - Flash_bwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim64_fp16_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim64_fp16_softmaxfalse_sm90.cu deleted file mode 100644 index 8d62b8827..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim64_fp16_softmaxfalse_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_bwd_launch_template.h" -#else -#include "flash_bwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM64 -template void run_mha_bwd_<90, cutlass::half_t, 64, false>( - Flash_bwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim64_fp16_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim64_fp16_softmaxtrue_sm90.cu deleted file mode 100644 index 5192d945f..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim64_fp16_softmaxtrue_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_bwd_launch_template.h" -#else -#include "flash_bwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM64 -template void run_mha_bwd_<90, cutlass::half_t, 64, true>( - Flash_bwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim96_bf16_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim96_bf16_softmaxfalse_sm90.cu deleted file mode 100644 index cbeeac64a..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim96_bf16_softmaxfalse_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_bwd_launch_template.h" -#else -#include "flash_bwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM96 -template void run_mha_bwd_<90, cutlass::bfloat16_t, 96, false>( - Flash_bwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim96_bf16_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim96_bf16_softmaxtrue_sm90.cu deleted file mode 100644 index b654969e4..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim96_bf16_softmaxtrue_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_bwd_launch_template.h" -#else -#include "flash_bwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM96 -template void run_mha_bwd_<90, cutlass::bfloat16_t, 96, true>( - Flash_bwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim96_fp16_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim96_fp16_softmaxfalse_sm90.cu deleted file mode 100644 index ea81f7ee4..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim96_fp16_softmaxfalse_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_bwd_launch_template.h" -#else -#include "flash_bwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM96 -template void run_mha_bwd_<90, cutlass::half_t, 96, false>( - Flash_bwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim96_fp16_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim96_fp16_softmaxtrue_sm90.cu deleted file mode 100644 index 7439f322e..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_bwd_hdim96_fp16_softmaxtrue_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_bwd_launch_template.h" -#else -#include "flash_bwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM96 -template void run_mha_bwd_<90, cutlass::half_t, 96, true>( - Flash_bwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_bf16_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_bf16_softmaxfalse_sm90.cu deleted file mode 100644 index a39bcd505..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_bf16_softmaxfalse_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_fwd_launch_template.h" -#else -#include "flash_fwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM128 -template void run_mha_fwd_<90, cutlass::bfloat16_t, 128, false>( - Flash_fwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_bf16_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_bf16_softmaxtrue_sm90.cu deleted file mode 100644 index 464a0f443..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_bf16_softmaxtrue_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_fwd_launch_template.h" -#else -#include "flash_fwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM128 -template void run_mha_fwd_<90, cutlass::bfloat16_t, 128, true>( - Flash_fwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_e4m3_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_e4m3_softmaxfalse_sm90.cu deleted file mode 100644 index 3075657bb..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_e4m3_softmaxfalse_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_fwd_launch_template.h" -#else -#include "flash_fwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM128 -template void run_mha_fwd_<90, cutlass::float_e4m3_t, 128, false>( - Flash_fwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_e4m3_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_e4m3_softmaxtrue_sm90.cu deleted file mode 100644 index 1ab6e4394..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_e4m3_softmaxtrue_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_fwd_launch_template.h" -#else -#include "flash_fwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM128 -template void run_mha_fwd_<90, cutlass::float_e4m3_t, 128, true>( - Flash_fwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_fp16_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_fp16_softmaxfalse_sm90.cu deleted file mode 100644 index be5a6cb0d..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_fp16_softmaxfalse_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_fwd_launch_template.h" -#else -#include "flash_fwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM128 -template void run_mha_fwd_<90, cutlass::half_t, 128, false>( - Flash_fwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_fp16_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_fp16_softmaxtrue_sm90.cu deleted file mode 100644 index 7c303e7ef..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim128_fp16_softmaxtrue_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_fwd_launch_template.h" -#else -#include "flash_fwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM128 -template void run_mha_fwd_<90, cutlass::half_t, 128, true>( - Flash_fwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_bf16_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_bf16_softmaxfalse_sm90.cu deleted file mode 100644 index 6e8d906d5..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_bf16_softmaxfalse_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_fwd_launch_template.h" -#else -#include "flash_fwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM192 -template void run_mha_fwd_<90, cutlass::bfloat16_t, 192, false>( - Flash_fwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_bf16_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_bf16_softmaxtrue_sm90.cu deleted file mode 100644 index 80367708f..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_bf16_softmaxtrue_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_fwd_launch_template.h" -#else -#include "flash_fwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM192 -template void run_mha_fwd_<90, cutlass::bfloat16_t, 192, true>( - Flash_fwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_e4m3_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_e4m3_softmaxfalse_sm90.cu deleted file mode 100644 index 67ade004b..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_e4m3_softmaxfalse_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_fwd_launch_template.h" -#else -#include "flash_fwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM192 -template void run_mha_fwd_<90, cutlass::float_e4m3_t, 192, false>( - Flash_fwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_e4m3_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_e4m3_softmaxtrue_sm90.cu deleted file mode 100644 index 9f40d2726..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_e4m3_softmaxtrue_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_fwd_launch_template.h" -#else -#include "flash_fwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM192 -template void run_mha_fwd_<90, cutlass::float_e4m3_t, 192, true>( - Flash_fwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_fp16_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_fp16_softmaxfalse_sm90.cu deleted file mode 100644 index 1779657c0..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_fp16_softmaxfalse_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_fwd_launch_template.h" -#else -#include "flash_fwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM192 -template void run_mha_fwd_<90, cutlass::half_t, 192, false>( - Flash_fwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_fp16_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_fp16_softmaxtrue_sm90.cu deleted file mode 100644 index 0037dbc17..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim192_fp16_softmaxtrue_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_fwd_launch_template.h" -#else -#include "flash_fwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM192 -template void run_mha_fwd_<90, cutlass::half_t, 192, true>( - Flash_fwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_bf16_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_bf16_softmaxfalse_sm90.cu deleted file mode 100644 index 93440571c..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_bf16_softmaxfalse_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_fwd_launch_template.h" -#else -#include "flash_fwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM256 -template void run_mha_fwd_<90, cutlass::bfloat16_t, 256, false>( - Flash_fwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_bf16_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_bf16_softmaxtrue_sm90.cu deleted file mode 100644 index c0634db8f..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_bf16_softmaxtrue_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_fwd_launch_template.h" -#else -#include "flash_fwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM256 -template void run_mha_fwd_<90, cutlass::bfloat16_t, 256, true>( - Flash_fwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_e4m3_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_e4m3_softmaxfalse_sm90.cu deleted file mode 100644 index a0eb625f5..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_e4m3_softmaxfalse_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_fwd_launch_template.h" -#else -#include "flash_fwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM256 -template void run_mha_fwd_<90, cutlass::float_e4m3_t, 256, false>( - Flash_fwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_e4m3_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_e4m3_softmaxtrue_sm90.cu deleted file mode 100644 index 8b7216302..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_e4m3_softmaxtrue_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_fwd_launch_template.h" -#else -#include "flash_fwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM256 -template void run_mha_fwd_<90, cutlass::float_e4m3_t, 256, true>( - Flash_fwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_fp16_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_fp16_softmaxfalse_sm90.cu deleted file mode 100644 index fe89b532f..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_fp16_softmaxfalse_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_fwd_launch_template.h" -#else -#include "flash_fwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM256 -template void run_mha_fwd_<90, cutlass::half_t, 256, false>( - Flash_fwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_fp16_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_fp16_softmaxtrue_sm90.cu deleted file mode 100644 index c0857f941..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim256_fp16_softmaxtrue_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_fwd_launch_template.h" -#else -#include "flash_fwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM256 -template void run_mha_fwd_<90, cutlass::half_t, 256, true>( - Flash_fwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_bf16_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_bf16_softmaxfalse_sm90.cu deleted file mode 100644 index 841e9359e..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_bf16_softmaxfalse_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_fwd_launch_template.h" -#else -#include "flash_fwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM64 -template void run_mha_fwd_<90, cutlass::bfloat16_t, 64, false>( - Flash_fwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_bf16_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_bf16_softmaxtrue_sm90.cu deleted file mode 100644 index 3da54d69f..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_bf16_softmaxtrue_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_fwd_launch_template.h" -#else -#include "flash_fwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM64 -template void run_mha_fwd_<90, cutlass::bfloat16_t, 64, true>( - Flash_fwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_e4m3_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_e4m3_softmaxfalse_sm90.cu deleted file mode 100644 index 4761ca635..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_e4m3_softmaxfalse_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_fwd_launch_template.h" -#else -#include "flash_fwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM64 -template void run_mha_fwd_<90, cutlass::float_e4m3_t, 64, false>( - Flash_fwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_e4m3_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_e4m3_softmaxtrue_sm90.cu deleted file mode 100644 index 33e66d0a7..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_e4m3_softmaxtrue_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_fwd_launch_template.h" -#else -#include "flash_fwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM64 -template void run_mha_fwd_<90, cutlass::float_e4m3_t, 64, true>( - Flash_fwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_fp16_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_fp16_softmaxfalse_sm90.cu deleted file mode 100644 index fab2951ee..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_fp16_softmaxfalse_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_fwd_launch_template.h" -#else -#include "flash_fwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM64 -template void run_mha_fwd_<90, cutlass::half_t, 64, false>( - Flash_fwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_fp16_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_fp16_softmaxtrue_sm90.cu deleted file mode 100644 index 2ef1f29c9..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim64_fp16_softmaxtrue_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_fwd_launch_template.h" -#else -#include "flash_fwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM64 -template void run_mha_fwd_<90, cutlass::half_t, 64, true>( - Flash_fwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_bf16_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_bf16_softmaxfalse_sm90.cu deleted file mode 100644 index bc52514e9..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_bf16_softmaxfalse_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_fwd_launch_template.h" -#else -#include "flash_fwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM96 -template void run_mha_fwd_<90, cutlass::bfloat16_t, 96, false>( - Flash_fwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_bf16_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_bf16_softmaxtrue_sm90.cu deleted file mode 100644 index 11ea3bb20..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_bf16_softmaxtrue_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_fwd_launch_template.h" -#else -#include "flash_fwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM96 -template void run_mha_fwd_<90, cutlass::bfloat16_t, 96, true>( - Flash_fwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_e4m3_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_e4m3_softmaxfalse_sm90.cu deleted file mode 100644 index 9e0b05a31..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_e4m3_softmaxfalse_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_fwd_launch_template.h" -#else -#include "flash_fwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM96 -template void run_mha_fwd_<90, cutlass::float_e4m3_t, 96, false>( - Flash_fwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_e4m3_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_e4m3_softmaxtrue_sm90.cu deleted file mode 100644 index 7fa79aa76..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_e4m3_softmaxtrue_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_fwd_launch_template.h" -#else -#include "flash_fwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM96 -template void run_mha_fwd_<90, cutlass::float_e4m3_t, 96, true>( - Flash_fwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_fp16_softmaxfalse_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_fp16_softmaxfalse_sm90.cu deleted file mode 100644 index 83a25a649..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_fp16_softmaxfalse_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_fwd_launch_template.h" -#else -#include "flash_fwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM96 -template void run_mha_fwd_<90, cutlass::half_t, 96, false>( - Flash_fwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_fp16_softmaxtrue_sm90.cu b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_fp16_softmaxtrue_sm90.cu deleted file mode 100644 index e0526dec8..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/instantiations/flash_fwd_hdim96_fp16_softmaxtrue_sm90.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, -// Pradeep Ramani, Tri Dao. Splitting the different template instantiations to -// different files to speed up compilation. This file is auto-generated. See -// "generate_kernels.py" - -#ifdef OSS_ENV -#include "hstu_attention/flash_fwd_launch_template.h" -#else -#include "flash_fwd_launch_template.h" -#endif - -namespace hstu { -#ifndef FLASHATTENTION_DISABLE_HDIM96 -template void run_mha_fwd_<90, cutlass::half_t, 96, true>( - Flash_fwd_params& params, - cudaStream_t stream); -#endif -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/mainloop_bwd_sm90_tma_gmma_ws.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/mainloop_bwd_sm90_tma_gmma_ws.h deleted file mode 100644 index e702faf0b..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/mainloop_bwd_sm90_tma_gmma_ws.h +++ /dev/null @@ -1,3166 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/****************************************************************************** - * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, - *Pradeep Ramani, Tri Dao. - ******************************************************************************/ - -#pragma once - -#include -#include -#include -#include -#include -#include "cutlass/pipeline/pipeline.hpp" - -#include "cute/tensor.hpp" - -#include "cutlass/gemm/collective/builders/sm90_common.inl" - -#include "copy_sm90_bulk_reduce.h" -#include "mask.h" -#include "named_barrier.h" -#include "seqlen.h" -#include "utils.h" - -namespace hstu { - -using namespace cute; - -template < - int Stages, - int Stages_dO, - int Stages_dS, - class ClusterShape_, - class TileShape_MNK_, - class Element_, - class ElementAccum_, - class ArchTag_, - bool Causal, - bool Local, - bool Contexual_mask, - bool Jagged, - bool Has_targets, - bool Deterministic, - bool SdP_swapAB_, - bool dKV_swapAB_, - bool dQ_swapAB_, - int NumMmaWarpGroups = 2, - int AtomLayoutMSdP = 1, - int AtomLayoutNdKV = 2, - int AtomLayoutMdQ = 1, - bool Mma_dP_is_RS = false, - bool Cross = false, - bool Softmax = false> -struct CollectiveMainloopBwdSm90 { - static constexpr int kStages = Stages; - static constexpr int kStages_dO = Stages_dO; - static constexpr int kStages_dS = Stages_dS; - static_assert(kStages >= kStages_dO); - static_assert(Stages_dS == 1 || Stages_dS == kStages); - static_assert( - !Mma_dP_is_RS || SdP_swapAB_); // If Mma_dP_is_RS, we need SdP_SwapAB - using ClusterShape = ClusterShape_; - using TileShape_MNK = TileShape_MNK_; - using Element = Element_; - using ElementAccum = ElementAccum_; - using ArchTag = ArchTag_; - using SeqlenInfo_t = hstu::SeqlenInfoQKBwd< - Jagged, - Cross, - Has_targets, - CUTE_STATIC_V(get<0>(TileShape_MNK{}))>; - - static constexpr bool SdP_swapAB = SdP_swapAB_; - static constexpr bool dKV_swapAB = dKV_swapAB_; - static constexpr bool dQ_swapAB = dQ_swapAB_; - - static constexpr bool Q_dO_same_stages = kStages == kStages_dO; - - static constexpr int kBlockM = get<0>(TileShape_MNK{}); - static constexpr int kBlockN = get<1>(TileShape_MNK{}); - static constexpr int kHeadDim = get<2>(TileShape_MNK{}); - - static_assert(ArchTag::kMinComputeCapability >= 90); - static_assert(get<0>(ClusterShape{}) == 1 && get<2>(ClusterShape{}) == 1); - - static constexpr int NumMmaThreads = - NumMmaWarpGroups * cutlass::NumThreadsPerWarpGroup; - static constexpr int NumProducerThreads = cutlass::NumThreadsPerWarp * 2; - - static_assert(NumMmaWarpGroups % AtomLayoutMSdP == 0); - static_assert(NumMmaWarpGroups % AtomLayoutNdKV == 0); - static_assert(NumMmaWarpGroups % AtomLayoutMdQ == 0); - static constexpr bool Mma_dKV_is_RS = AtomLayoutMSdP == 1 && - AtomLayoutNdKV == NumMmaWarpGroups && SdP_swapAB && !dKV_swapAB; - static constexpr bool Mma_dQ_is_RS = AtomLayoutMSdP == NumMmaWarpGroups && - AtomLayoutMdQ == NumMmaWarpGroups && !SdP_swapAB && - !dQ_swapAB; // If dQ_swapAB we can't use RS - - static constexpr GMMA::Major PdS_Major = GMMA::Major::K; - // static constexpr GMMA::Major PdS_Major = GMMA::Major::MN; - static constexpr GMMA::Major PdSt_Major = - PdS_Major == GMMA::Major::K ? GMMA::Major::MN : GMMA::Major::K; - - using TileShapeAtomSdP = std::conditional_t< - !SdP_swapAB, - Shape< - Int, - Int, - Int>, - Shape, Int, Int>>; - using AtomLayoutSdP = std::conditional_t< - !SdP_swapAB, - Layout, - Int, - _1>>, - Layout, - Int, - _1>>>; - using TiledMmaSdP = decltype(cute::make_tiled_mma( - cute::GMMA:: - ss_op_selector(), - AtomLayoutSdP{})); - - using TiledMmadPRS = decltype(cute::make_tiled_mma( - cute::GMMA:: - rs_op_selector(), - AtomLayoutSdP{})); - - using TileShapeAtomdKV = std::conditional_t< - !dKV_swapAB, - Shape< - Int, - Int, - Int>, - Shape, Int, Int>>; - using AtomLayoutdKV = std::conditional_t< - !dKV_swapAB, - Layout, - Int, - _1>>, - Layout, - Int, - _1>>>; - using TiledMmadKV = decltype(cute::make_tiled_mma( - std::conditional_t< - Mma_dKV_is_RS, - decltype(cute::GMMA::rs_op_selector< - Element, - Element, - ElementAccum, - TileShapeAtomdKV, - GMMA::Major::K, - GMMA::Major::MN>()), - decltype(cute::GMMA::ss_op_selector< - Element, - Element, - ElementAccum, - TileShapeAtomdKV, - !dKV_swapAB ? PdSt_Major : GMMA::Major::MN, - !dKV_swapAB ? GMMA::Major::MN : PdSt_Major>())>{}, - AtomLayoutdKV{})); - - static constexpr bool dQacc_use_TMA = kHeadDim < 256; - // For hdim256, we want to slice the dQ MMA (64 x 256 on 2 WGs) into two (64 x - // 128 on 2 WGs) so that we can do atomic add on one half before doing the - // other half of the MMA, to reduce register pressure. - static constexpr bool Slice_dQKV_Mma = kHeadDim == 256 && !dQacc_use_TMA && - dQ_swapAB && AtomLayoutMdQ == 1 && NumMmaWarpGroups == 2; - static_assert( - !(Deterministic && Slice_dQKV_Mma), - "Deterministic mode not supported with Slice_dQKV_Mma"); - - static constexpr int TileShapeAtomdQ_BlockM = kBlockM / AtomLayoutMdQ; - static constexpr int TileShapeAtomdQ_HeadDim = - (Slice_dQKV_Mma ? kHeadDim / 2 : kHeadDim) / - (NumMmaWarpGroups / AtomLayoutMdQ); - static_assert( - !dQ_swapAB ? TileShapeAtomdQ_BlockM == 64 : TileShapeAtomdQ_HeadDim == 64, - "Tile_M must be 64."); - using TileShapeAtomdQ = std::conditional_t< - !dQ_swapAB, - Shape< - Int, - Int, - Int>, - Shape< - Int, - Int, - Int>>; - using AtomLayoutdQ = std::conditional_t< - !dQ_swapAB, - Layout< - Shape, Int, _1>>, - Layout, - Int, - _1>>>; - using TiledMmadQ = decltype(cute::make_tiled_mma( - std::conditional_t< - Mma_dQ_is_RS, - decltype(cute::GMMA::rs_op_selector< - Element, - Element, - ElementAccum, - TileShapeAtomdQ, - GMMA::Major::K, - GMMA::Major::MN>()), - decltype(cute::GMMA::ss_op_selector< - Element, - Element, - ElementAccum, - TileShapeAtomdQ, - !dQ_swapAB ? PdS_Major : GMMA::Major::MN, - !dQ_swapAB ? GMMA::Major::MN : PdS_Major>())>{}, - AtomLayoutdQ{})); - - // We need to accommodate both Q and Q^T (and dO and dO^T) in shared memory. - // Q & dO are used in the SdP Mma and Q^T and dO^T are used in the dKV Mma. - // Since this is GMMA::Major::K, the M dimension (kBlockM) doesn't matter for - // the layout, only the K dimension changes the layout. - using SmemLayoutAtomQdO = - decltype(cutlass::gemm::collective::detail::ss_smem_selector< - GMMA::Major::K, - Element, - Int, - Int>()); // for dKV_Mma - using SmemLayoutQ = decltype(tile_to_shape( - SmemLayoutAtomQdO{}, - make_shape( - shape<0>(TileShape_MNK{}), - shape<2>(TileShape_MNK{}), - Int{}))); - using SmemLayoutdO = decltype(tile_to_shape( - SmemLayoutAtomQdO{}, - make_shape( - shape<0>(TileShape_MNK{}), - shape<2>(TileShape_MNK{}), - Int{}))); - - using SmemLayoutAtomK = - decltype(cutlass::gemm::collective::detail::ss_smem_selector< - GMMA::Major::K, - Element, - Int, - Int>()); - using SmemLayoutK = - decltype(tile_to_shape(SmemLayoutAtomK{}, select<1, 2>(TileShape_MNK{}))); - - using SmemLayoutAtomV = - decltype(cutlass::gemm::collective::detail::ss_smem_selector< - GMMA::Major::K, - Element, - decltype(cute::get<1>(TileShape_MNK{})), - decltype(cute::get<2>(TileShape_MNK{}))>()); - using SmemLayoutV = - decltype(tile_to_shape(SmemLayoutAtomV{}, select<1, 2>(TileShape_MNK{}))); - - using SmemLayoutAtomPdS = - decltype(cutlass::gemm::collective::detail::ss_smem_selector< - PdS_Major, - Element, - Int, - Int>()); - using SmemLayoutPdS = decltype(tile_to_shape( - SmemLayoutAtomPdS{}, - make_shape(Int{}, Int{}, Int{}), - std::conditional_t< - PdS_Major == GMMA::Major::K, - cute::Step<_1, _2, _3>, - cute::Step<_2, _1, _3>>{})); - // Need stride to be multiple of 32, otherwise we get error (misaligned - // address) when doing TMA if e.g. kBlockM=80 We set stride to be multiple of - // 64 so that if ShuffleLSE, even if threads read from sLSE but out of bounds, - // it's still a valid smem address. - using SmemLayoutLSE = cute::Layout< - cute::Shape, Int>, - cute::Stride<_1, Int>>; - using SmemLayoutLSEMma = std::conditional_t< - SdP_swapAB, - cute::Layout< - cute::Shape, Int, Int>, - cute::Stride<_0, _1, Int>>, - cute::Layout< - cute::Shape, Int, Int>, - cute::Stride<_1, _0, Int>>>; - - // Note this is the transpose in terms of the view, not in terms of memory. - using SmemLayoutQt = decltype(cute::composition( - SmemLayoutQ{}, - make_layout( - make_shape( - get<2>(TileShape_MNK{}), - get<0>(TileShape_MNK{}), - Int{}), - make_stride(Int{}, _1{}, Int{})))); - using SmemLayoutdOt = decltype(cute::composition( - SmemLayoutdO{}, - make_layout( - make_shape( - get<2>(TileShape_MNK{}), - get<0>(TileShape_MNK{}), - Int{}), - make_stride(Int{}, _1{}, Int{})))); - using SmemLayoutKt = decltype(cute::composition( - SmemLayoutK{}, - make_layout( - make_shape(get<2>(TileShape_MNK{}), get<1>(TileShape_MNK{})), - make_stride(Int{}, _1{})))); - using SmemLayoutPdSt = decltype(cute::composition( - SmemLayoutPdS{}, - make_layout( - make_shape(Int{}, Int{}, Int{}), - make_stride(Int{}, _1{}, Int{})))); - - // Thread layout, 256 or 384 threads per row - // We split into NumMmaWarpGroups so that we can do Bulk reduce add for each - // WG separately. - using R2SLayoutAtomdQaccum = Layout< - Shape, Int>>; - using R2STiledCopydQaccum = decltype(make_tiled_copy( - Copy_Atom, ElementAccum>{}, - R2SLayoutAtomdQaccum{}, - Layout>{})); // Val layout, 4 vals per store - using SmemLayoutdQaccum = Layout< - Shape, Int>>; - - static constexpr int kNumPdSStore = kBlockM * kBlockN / NumMmaThreads; - // If !SdP_swapAB, the accum registers hold P / dS, otherwise they hold Pt / - // dSt. If PdS_major is MN, then we need to "transpose" the write. - using SmemCopyAtomPdS = Copy_Atom< - std::conditional_t< - (!SdP_swapAB) ^ (PdS_Major == GMMA::Major::MN), - std::conditional_t< - kNumPdSStore % 8 == 0, - cute::SM90_U32x4_STSM_N, - cute::SM90_U32x2_STSM_N>, - std::conditional_t< - kNumPdSStore % 8 == 0, - cute::SM90_U16x8_STSM_T, - cute::SM90_U16x4_STSM_T>>, - Element>; - - using GmemTiledCopyQdO = - decltype(cutlass::gemm::collective::detail:: - sm90_cluster_shape_to_tma_atom(shape<1>(ClusterShape{}))); - using GmemTiledCopyKV = cute::SM90_TMA_LOAD; - - using ShapeQKV = - cute::Shape; // (seqlen, d, head, - // batch) - using StrideQKV = cute::Stride; - using ShapeLSE = - cute::Shape; // (seqlen, head, batch) - using StrideLSE = cute::Stride<_1, int64_t, int64_t>; // (seqlen, head, batch) - using ShapedQaccum = - cute::Shape; // (seqlen * d, head, batch) - using StridedQaccum = cute::Stride<_1, int64_t, int64_t>; - - using TMA_QdO = decltype(make_tma_copy_A_sm90( - GmemTiledCopyQdO{}, - make_tensor( - make_gmem_ptr(static_cast(nullptr)), - ShapeQKV{}, - StrideQKV{}), - take<0, 2>(SmemLayoutQ{}), - TileShape_MNK{}, - ClusterShape{})); // mcast along N mode for this M load, if any - - using TMA_K = decltype(make_tma_copy_B_sm90( - GmemTiledCopyKV{}, - make_tensor( - make_gmem_ptr(static_cast(nullptr)), - ShapeQKV{}, - StrideQKV{}), - SmemLayoutK{}, - TileShape_MNK{}, - ClusterShape{})); // no mcast for KV - - using TMA_V = decltype(make_tma_copy_B_sm90( - GmemTiledCopyKV{}, - make_tensor( - make_gmem_ptr(static_cast(nullptr)), - ShapeQKV{}, - StrideQKV{}), - SmemLayoutV{}, - TileShape_MNK{}, - ClusterShape{})); // no mcast for KV - - using MainloopPipeline = typename cutlass::PipelineTmaAsync; - using PipelineState = typename MainloopPipeline::PipelineState; - using MainloopPipeline_dO = typename cutlass::PipelineTmaAsync; - using PipelineState_dO = typename MainloopPipeline_dO::PipelineState; - - // Set the bytes transferred in this TMA transaction (may involve multiple - // issues) - static constexpr uint32_t TmaTransactionBytesQ = static_cast( - size(take<0, 2>(SmemLayoutQ{})) * cutlass::sizeof_bits_v / 8); - static constexpr uint32_t TmaTransactionBytesK = static_cast( - size(SmemLayoutK{}) * cutlass::sizeof_bits_v / 8); - static constexpr uint32_t TmaTransactionBytesV = static_cast( - size(SmemLayoutV{}) * cutlass::sizeof_bits_v / 8); - static constexpr uint32_t TmaTransactionBytesLSE = static_cast( - size(select<0>(SmemLayoutLSE{})) * cutlass::sizeof_bits_v / - 8); - - // These are tuned for speed. They don't affect correctness. - // We have separate iterations with causal masking. Not necessary for hdim 128 - // but for hdim 64 this helps quite a bit to not have to do causal masking for - // most of the iterations. For hdim 192, separating masking iterations results - // in register spills. - static constexpr bool SeparateMaskingIterations = false; - // Do we keep the LSE and dPsum in each thread, or split them across 8 threads - // that share them and then shuffle to get the value whenever we need? This - // can reduce register pressure when SdP_swapAB, where each thread needs to - // keep statistics for (kBlockM / 4) rows. If !SdP_swapAB, each thread only - // needs to keep statistic for 2 rows. - static constexpr bool ShuffleLSE = SdP_swapAB && kHeadDim <= 64; - static constexpr bool ShuffledPsum = SdP_swapAB && kHeadDim <= 64; - static constexpr size_t SmemAlignmentP = - cutlass::detail::alignment_for_swizzle(SmemLayoutPdS{}); - static constexpr size_t SmemAlignmentdS = - cutlass::detail::alignment_for_swizzle(SmemLayoutPdS{}); - // Without this SmemAlignment, with hdim 256 we get "misaligned address" error - // in TMA - static constexpr size_t SmemAlignmentQKVdO = kHeadDim % 256 == 0 ? 256 : 128; - static constexpr size_t SmemAlignmentV = !Mma_dP_is_RS - ? SmemAlignmentQKVdO - : cutlass::detail::alignment_for_swizzle(SmemLayoutV{}); - static_assert( - SmemAlignmentP >= 128 && SmemAlignmentdS >= 128, - "Require at least 128B alignment"); - - // TODO: do we have to worry that smem_dk and smem_dv in the epilogue don't - // line up w smem_k and smem_v due to alignment? - using SmemdQacc_t = std::conditional_t< - !dQacc_use_TMA, - cute::array, - cute::array_aligned>>; - using SmemP_t = std::conditional_t< - Mma_dKV_is_RS, - cute::array, - cute::array_aligned< - Element, - cute::cosize_v, - SmemAlignmentP>>; - struct TensorStorage - : cute::aligned_struct< - cute::max(SmemAlignmentP, SmemAlignmentdS, SmemAlignmentQKVdO)> { - cute:: - array_aligned, SmemAlignmentQKVdO> - smem_k; - cute::array_aligned, SmemAlignmentV> - smem_v; - SmemdQacc_t smem_dqacc; - cute:: - array_aligned, SmemAlignmentQKVdO> - smem_q; - cute:: - array_aligned, SmemAlignmentQKVdO> - smem_do; - cute::array_aligned, 128> - smem_lse; - cute::array_aligned, 128> - smem_dpsum; - SmemP_t smem_p; - cute::array_aligned, SmemAlignmentdS> - smem_ds; - }; - - // Host side kernel arguments - struct Arguments { - Element const* const ptr_Q; - ShapeQKV const shape_Q; - StrideQKV const stride_Q; - Element const* const ptr_K; - ShapeQKV const shape_K; - StrideQKV const stride_K; - Element const* const ptr_V; - ShapeQKV const shape_V; - StrideQKV const stride_V; - Element const* const ptr_dO; - ShapeQKV const shape_dO; - StrideQKV const stride_dO; - ElementAccum* const ptr_dQaccum; - ShapedQaccum const shape_dQaccum; - StridedQaccum const stride_dQaccum; - float const* const ptr_LSE_log2; - ShapeLSE const shape_LSE; - StrideLSE const stride_LSE_log2; - float const* const ptr_dPsum; - StrideLSE const stride_dPsum; - int const max_attn_len; - int const min_full_attn_seq_len; - int const contextual_seq_len; - float const max_seq_len_inv; - float const alpha; - int const num_batch; - int const num_softmax_heads; - int const num_groups; - int const batch_size_per_group; - int* const dq_semaphore; - int const* const seq_offsets = nullptr; - int const* const seq_offsets_q = nullptr; - int const* const num_targets = nullptr; - int const* const max_seq_len_tensor = nullptr; - int const* const contextual_seq_len_tensor = nullptr; - int const* const max_attn_len_tensor = nullptr; - int const* const min_full_attn_seq_len_tensor = nullptr; - float const* const attn_scale = nullptr; - bool const scalar_scale = true; - }; - - // Device side kernel params - struct Params { - ShapeQKV const shape_Q; - ShapeQKV const shape_K; - ShapeQKV const shape_V; - ShapeQKV const shape_dO; - ElementAccum* const ptr_dQaccum; - ShapedQaccum const shape_dQaccum; - StridedQaccum stride_dQaccum; - TMA_QdO tma_load_Q, tma_load_dO; - TMA_K tma_load_K; - TMA_V tma_load_V; - float const* const ptr_LSE_log2; - ShapeLSE const shape_LSE; - StrideLSE const stride_LSE_log2; - float const* const ptr_dPsum; - StrideLSE const stride_dPsum; - int const max_attn_len; - int const min_full_attn_seq_len; - int const contextual_seq_len; - float const max_seq_len_inv; - float const alpha; - float const alpha_log2; - int const num_batch; - int const num_softmax_heads; - int const num_groups; - int const batch_size_per_group; - int* const dq_semaphore; - int const* const seq_offsets = nullptr; - int const* const seq_offsets_q = nullptr; - int const* const num_targets; - int const* const max_seq_len_tensor = nullptr; - int const* const contextual_seq_len_tensor = nullptr; - int const* const max_attn_len_tensor = nullptr; - int const* const min_full_attn_seq_len_tensor = nullptr; - float const* const attn_scale; - bool const scalar_scale = true; - }; - - static Params to_underlying_arguments(Arguments const& args) { - Tensor mQ = - make_tensor(make_gmem_ptr(args.ptr_Q), args.shape_Q, args.stride_Q); - TMA_QdO tma_load_Q = make_tma_copy_A_sm90( - GmemTiledCopyQdO{}, - mQ, - SmemLayoutQ{}(_, _, _0{}), - TileShape_MNK{}, - ClusterShape{}); // mcast along N mode for this M load, if any - Tensor mdO = - make_tensor(make_gmem_ptr(args.ptr_dO), args.shape_Q, args.stride_dO); - TMA_QdO tma_load_dO = make_tma_copy_A_sm90( - GmemTiledCopyQdO{}, - mdO, - SmemLayoutdO{}(_, _, _0{}), - TileShape_MNK{}, - ClusterShape{}); // mcast along N mode for this M load, if any - Tensor mK = - make_tensor(make_gmem_ptr(args.ptr_K), args.shape_K, args.stride_K); - TMA_K tma_load_K = make_tma_copy_B_sm90( - GmemTiledCopyKV{}, - mK, - SmemLayoutK{}, - TileShape_MNK{}, - ClusterShape{}); // no mcast for KV - Tensor mV = - make_tensor(make_gmem_ptr(args.ptr_V), args.shape_K, args.stride_V); - TMA_V tma_load_V = make_tma_copy_B_sm90( - GmemTiledCopyKV{}, - mV, - SmemLayoutV{}, - TileShape_MNK{}, - ClusterShape{}); // no mcast for KV - if constexpr (Deterministic) { - assert(args.dq_semaphore != nullptr); - } - return { - args.shape_Q, - args.shape_K, - args.shape_V, - args.shape_dO, - args.ptr_dQaccum, - args.shape_dQaccum, - args.stride_dQaccum, - tma_load_Q, - tma_load_dO, - tma_load_K, - tma_load_V, - args.ptr_LSE_log2, - args.shape_LSE, - args.stride_LSE_log2, - args.ptr_dPsum, - args.stride_dPsum, - args.max_attn_len, - args.min_full_attn_seq_len, - args.contextual_seq_len, - args.max_seq_len_inv, - args.alpha, - float(args.alpha * M_LOG2E), - args.num_batch, - args.num_softmax_heads, - args.num_groups, - args.batch_size_per_group, - args.dq_semaphore, - args.seq_offsets, - args.seq_offsets_q, - args.num_targets, - args.max_seq_len_tensor, - args.contextual_seq_len_tensor, - args.max_attn_len_tensor, - args.min_full_attn_seq_len_tensor, - args.attn_scale, - args.scalar_scale}; - } - - /// Issue Tma Descriptor Prefetch -- ideally from a single thread for best - /// performance - CUTLASS_DEVICE - static void prefetch_tma_descriptors(Params const& params) { - cute::prefetch_tma_descriptor(params.tma_load_Q.get_tma_descriptor()); - cute::prefetch_tma_descriptor(params.tma_load_dO.get_tma_descriptor()); - cute::prefetch_tma_descriptor(params.tma_load_K.get_tma_descriptor()); - cute::prefetch_tma_descriptor(params.tma_load_V.get_tma_descriptor()); - } - - CUTLASS_DEVICE - cute::tuple get_m_block_min_max( - int const max_attn_len, - int const contextual_seq_len, - int const uihlen, - int const seqlen, - int const n_block) { - static constexpr int kBlockM = get<0>(TileShape_MNK{}); - static constexpr int kBlockN = get<1>(TileShape_MNK{}); - if constexpr (Has_targets) { - int n_idx_min = n_block * kBlockN; - if (n_idx_min >= uihlen) { - int n_idx_max = (n_block + 1) * kBlockN; - return { - std::max(0, n_idx_min / kBlockM), - cute::ceil_div(std::min(n_idx_max, seqlen), kBlockM)}; - } - } - // uih part - int m_block_max = cute::ceil_div(seqlen, kBlockM); - if constexpr (Local) { - int local_m_block_max = - cute::ceil_div((n_block + 1) * kBlockN + max_attn_len, kBlockM); - if constexpr (Contexual_mask) { - // row contexual without sink - if (n_block * kBlockN < contextual_seq_len) { - local_m_block_max = std::max( - local_m_block_max, - cute::ceil_div(contextual_seq_len + max_attn_len, kBlockM)); - } - } - m_block_max = std::min(m_block_max, local_m_block_max); - } - int m_block_min = 0; - if constexpr (Causal || Local) { - m_block_min = std::max(m_block_min, (n_block * kBlockN) / kBlockM); - } - return {m_block_min, m_block_max}; - } - - CUTLASS_DEVICE - cute::tuple get_full_m_block_min_max( - int const uihlen, - int const seqlen, - int const min_full_attn_seq_len, - int const m_block_max, - int const n_block) { - static constexpr int kBlockM = get<0>(TileShape_MNK{}); - static constexpr int kBlockN = get<1>(TileShape_MNK{}); - if constexpr (Cross) { - return {0, 0}; - } - if constexpr (!Local) { - return {0, 0}; - } - if constexpr (Has_targets) { - int n_idx_min = n_block * kBlockN; - if (n_idx_min >= uihlen) { - return {0, 0}; - } - } - if constexpr (Local) { - int full_m_block_max = cute::ceil_div(seqlen, kBlockM); - int full_m_block_min = - std::max(m_block_max, (uihlen - min_full_attn_seq_len) / kBlockM); - return {full_m_block_min, full_m_block_max}; - } - return {0, 0}; - } - - CUTLASS_DEVICE - int get_contexual_m_block_max( - int const uihlen, - int const contextual_seq_len, - int const m_block_min, - int const n_block) { - static constexpr int kBlockM = get<0>(TileShape_MNK{}); - static constexpr int kBlockN = get<1>(TileShape_MNK{}); - if constexpr (Cross) { - return 0; - } - if constexpr (!Contexual_mask) { - return 0; - } - if constexpr (Has_targets) { - int n_idx_min = n_block * kBlockN; - if (n_idx_min >= uihlen) { - return 0; - } - } - if constexpr (Causal || Local) { - int contexual_m_block_max = - std::min(m_block_min, cute::ceil_div(contextual_seq_len, kBlockM)); - return contexual_m_block_max; - } - return 0; - } - - CUTLASS_DEVICE - int get_next_m_block( - int const m_block, - int const m_block_min, - int const m_block_max, - int const contexual_m_block_max, - int const full_m_block_min, - int const full_m_block_max) { - int const out_m_block = m_block + 1; - if constexpr (Contexual_mask || Local) { - if (out_m_block == m_block_max) { - if (contexual_m_block_max > 0) { - return 0; - } - if (full_m_block_max > full_m_block_min) { - return full_m_block_min; - } - return -1; - } - if (out_m_block == contexual_m_block_max) { - if (full_m_block_max > full_m_block_min) { - return full_m_block_min; - } - return -1; - } - if (out_m_block == full_m_block_max) { - return -1; - } - return out_m_block; - } - if (out_m_block == m_block_max) { - return -1; - } - return out_m_block; - } - - CUTLASS_DEVICE - cute::tuple get_cross_m_block_min_max( - int const uihlen_q, - int const seqlen_q, - int const seqlen_kv, - int const n_block) { - static constexpr int kBlockM = get<0>(TileShape_MNK{}); - static constexpr int kBlockN = get<1>(TileShape_MNK{}); - int m_block_max = cute::ceil_div(seqlen_q, kBlockM); - if constexpr (!Causal) { - return {0, m_block_max}; - } - int m_block_min = - std::max(0, (n_block * kBlockN + uihlen_q - seqlen_kv) / kBlockM); - return {m_block_min, m_block_max}; - } - - template - CUTLASS_DEVICE void load( - Params const& params, - MainloopPipeline pipeline_q, - MainloopPipeline_dO pipeline_do, - PipelineState& smem_pipe_write, - PipelineState_dO& smem_pipe_write_do, - SharedStorage& shared_storage, - SchedulerPrefetch const& scheduler_prefetch, - cute::tuple block_coord) { - auto [n_block, bidh, bidb] = block_coord; - SeqlenInfo_t seqlen_info{ - bidb, - get<0>(params.shape_Q), - get<0>(params.shape_K), - params.seq_offsets, - params.seq_offsets_q, - params.num_targets}; - if constexpr (Jagged) { - static constexpr int kBlockN = get<1>(TileShape_MNK{}); - if (n_block * kBlockN >= seqlen_info.seqlen_kv) { - scheduler_prefetch(); - return; - } - } - int min_full_attn_seq_len_, max_attn_len_, contextual_seq_len_; - if constexpr (!Cross) { - if (params.num_groups > 1) { - int group_id = bidb / params.batch_size_per_group; - min_full_attn_seq_len_ = params.min_full_attn_seq_len_tensor[group_id]; - max_attn_len_ = params.max_attn_len_tensor[group_id]; - contextual_seq_len_ = params.contextual_seq_len_tensor[group_id]; - } else { - min_full_attn_seq_len_ = params.min_full_attn_seq_len; - max_attn_len_ = params.max_attn_len; - contextual_seq_len_ = params.contextual_seq_len; - } - } - int m_block_min, m_block_max; - if constexpr (Cross) { - auto m_block_min_max = get_cross_m_block_min_max( - seqlen_info.uihlen_q, - seqlen_info.seqlen_q, - seqlen_info.seqlen_kv, - n_block); - m_block_min = get<0>(m_block_min_max); - m_block_max = get<1>(m_block_min_max); - } else { - auto m_block_min_max = get_m_block_min_max( - max_attn_len_, - contextual_seq_len_, - seqlen_info.uihlen_q, - seqlen_info.seqlen_q, - n_block); - m_block_min = get<0>(m_block_min_max); - m_block_max = get<1>(m_block_min_max); - } - auto full_m_block_min_max = get_full_m_block_min_max( - seqlen_info.uihlen_q, - seqlen_info.seqlen_q, - min_full_attn_seq_len_, - m_block_max, - n_block); - int const full_m_block_min = get<0>(full_m_block_min_max); - int const full_m_block_max = get<1>(full_m_block_min_max); - int contexual_m_block_max = get_contexual_m_block_max( - seqlen_info.uihlen_q, contextual_seq_len_, m_block_min, n_block); - - Tensor sQ = make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_q.data()), - SmemLayoutQ{}); - Tensor sdO = make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_do.data()), - SmemLayoutdO{}); - Tensor sK = make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_k.data()), - SmemLayoutK{}); - Tensor sV = make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_v.data()), - SmemLayoutV{}); - Tensor sLSE = make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_lse.data()), - SmemLayoutLSE{}); - Tensor sdPsum = make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_dpsum.data()), - SmemLayoutLSE{}); - - // Prepare the TMA loads - uint32_t block_rank_in_cluster = cute::block_rank_in_cluster(); - constexpr uint32_t cluster_shape_x = get<0>(ClusterShape()); - uint2 cluster_local_block_id = { - block_rank_in_cluster % cluster_shape_x, - block_rank_in_cluster / cluster_shape_x}; - Tensor mQ = params.tma_load_Q.get_tma_tensor(params.shape_Q)( - _, _, bidh, !Jagged ? bidb : 0); - Tensor mdO = params.tma_load_dO.get_tma_tensor(params.shape_Q)( - _, _, bidh, !Jagged ? bidb : 0); - Tensor mK = params.tma_load_K.get_tma_tensor(params.shape_K)( - _, _, bidh, !Jagged ? bidb : 0); - Tensor mV = params.tma_load_V.get_tma_tensor(params.shape_K)( - _, _, bidh, !Jagged ? bidb : 0); - Tensor mLSE = make_tensor( - make_gmem_ptr(params.ptr_LSE_log2), - params.shape_LSE, - params.stride_LSE_log2)(_, bidh, !Jagged ? bidb : 0); - Tensor mdPsum = make_tensor( - make_gmem_ptr(params.ptr_dPsum), params.shape_LSE, params.stride_dPsum)( - _, bidh, !Jagged ? bidb : 0); - - Tensor gQ = local_tile( - domain_offset(make_coord(seqlen_info.offset_q, _0{}), mQ), - select<0, 2>(TileShape_MNK{}), - make_coord(_, _0{})); // (M, K, _) - Tensor gdO = local_tile( - domain_offset(make_coord(seqlen_info.offset_q, _0{}), mdO), - select<0, 2>(TileShape_MNK{}), - make_coord(_, _0{})); // (M, K, _) - Tensor gK = local_tile( - domain_offset(make_coord(seqlen_info.offset_k, _0{}), mK), - select<1, 2>(TileShape_MNK{}), - make_coord(n_block, _0{})); // (N, K) - Tensor gV = local_tile( - domain_offset(make_coord(seqlen_info.offset_k, _0{}), mV), - select<1, 2>(TileShape_MNK{}), - make_coord(n_block, _0{})); // (N, K) - Tensor gLSE = local_tile( - domain_offset(make_coord(seqlen_info.offset_q_padded), mLSE), - select<0>(TileShape_MNK{}), - make_coord(_)); // (M, _) - Tensor gdPsum = local_tile( - domain_offset(make_coord(seqlen_info.offset_q_padded), mdPsum), - select<0>(TileShape_MNK{}), - make_coord(_)); // (M, _) - - Tensor sK_x = - make_tensor(sK.data(), make_layout(sK.layout(), Layout<_1>{})); - Tensor gK_x = - make_tensor(gK.data(), make_layout(gK.layout(), Layout<_1>{})); - Tensor sV_x = - make_tensor(sV.data(), make_layout(sV.layout(), Layout<_1>{})); - Tensor gV_x = - make_tensor(gV.data(), make_layout(gV.layout(), Layout<_1>{})); - // auto [tQgQ, tQsQ] = tma_partition(params.tma_load_Q, - // block_rank_in_cluster, Layout{}, - // group_modes<0, 2>(sQ), group_modes<0, - // 2>(gQ)); // (TMA, k), (TMA, PIPE) - // auto [tdOgdO, tdOsdO] = tma_partition(params.tma_load_dO, - // block_rank_in_cluster, Layout{}, - // group_modes<0, 2>(sdO), group_modes<0, - // 2>(gdO)); // (TMA, k), (TMA, PIPE) - auto block_tma_Q = params.tma_load_Q.get_slice(cluster_local_block_id.y); - auto block_tma_dO = params.tma_load_dO.get_slice(cluster_local_block_id.y); - Tensor tQgQ = group_modes<0, 3>(block_tma_Q.partition_S(gQ)); - Tensor tQsQ = group_modes<0, 3>(block_tma_Q.partition_D(sQ)); - Tensor tdOgdO = group_modes<0, 3>(block_tma_dO.partition_S(gdO)); - Tensor tdOsdO = group_modes<0, 3>(block_tma_dO.partition_D(sdO)); - auto [tKgK, tKsK] = tma_partition( - params.tma_load_K, - _0{}, - Layout<_1>{}, - group_modes<0, 2>(sK_x), - group_modes<0, 2>(gK_x)); // (TMA), (TMA) - auto [tVgV, tVsV] = tma_partition( - params.tma_load_V, - _0{}, - Layout<_1>{}, - group_modes<0, 2>(sV_x), - group_modes<0, 2>(gV_x)); // (TMA), (TMA) - auto bulk_copy = Copy_Traits{}; - - uint16_t mcast_mask_qdo = 0; - if constexpr (cute::is_same_v) { - auto block_layout = Layout{}; // (m,n) -> block_id - for (int n = 0; n < size<1>(block_layout); ++n) { - mcast_mask_qdo |= - (uint16_t(1) << block_layout(cluster_local_block_id.x, n, _0{})); - } - } - - int m_block = m_block_min; - int next_m_block = -1; - int lane_predicate = cute::elect_one_sync(); - - if (lane_predicate) { - pipeline_q.producer_acquire(smem_pipe_write); - copy( - params.tma_load_Q.with( - *pipeline_q.producer_get_barrier(smem_pipe_write), - mcast_mask_qdo, - TMA::CacheHintSm90::EVICT_LAST), - tQgQ(_, m_block), - tQsQ(_, smem_pipe_write.index())); - if constexpr (Softmax) { - copy( - bulk_copy.with(*pipeline_q.producer_get_barrier(smem_pipe_write)), - gLSE(_, m_block), - sLSE(_, smem_pipe_write.index())); - } - } - - // // Wait for the MMA warpgroups to say that smem_k and smem_v are ready - // cutlass::arch::NamedBarrier::sync(NumMmaThreads + - // cutlass::NumThreadsPerWarp, - // static_cast(BwdNamedBarriers::KVEmpty) /*id*/); - - auto load_step = [&](int m_block) { - // If Q and dO have the same number of stages, we can use the same - // pipeline state variable to reduce registers - PipelineState_dO smem_pipe_write_do_cur = - cute::conditional_return( - smem_pipe_write, smem_pipe_write_do); - pipeline_do.producer_acquire(smem_pipe_write_do_cur); - copy( - params.tma_load_dO.with( - *pipeline_do.producer_get_barrier(smem_pipe_write_do_cur), - mcast_mask_qdo, - TMA::CacheHintSm90::EVICT_LAST), - tdOgdO(_, m_block), - tdOsdO(_, smem_pipe_write_do_cur.index())); - if constexpr (Softmax) { - copy( - bulk_copy.with( - *pipeline_do.producer_get_barrier(smem_pipe_write_do_cur)), - gdPsum(_, m_block), - sdPsum(_, smem_pipe_write_do_cur.index())); - } - if constexpr (!Q_dO_same_stages) { - ++smem_pipe_write_do; - } - ++smem_pipe_write; - next_m_block = get_next_m_block( - m_block, - m_block_min, - m_block_max, - contexual_m_block_max, - full_m_block_min, - full_m_block_max); - if (next_m_block != -1) { - pipeline_q.producer_acquire(smem_pipe_write); - copy( - params.tma_load_Q.with( - *pipeline_q.producer_get_barrier(smem_pipe_write), - mcast_mask_qdo, - TMA::CacheHintSm90::EVICT_LAST), - tQgQ(_, next_m_block), - tQsQ(_, smem_pipe_write.index())); - if constexpr (Softmax) { - copy( - bulk_copy.with(*pipeline_q.producer_get_barrier(smem_pipe_write)), - gLSE(_, next_m_block), - sLSE(_, smem_pipe_write.index())); - } - } - }; - - if (lane_predicate) { - // Copy K tile and V tile from GMEM to SMEM. - shared_storage.pipelines.barrier_KV.arrive_and_expect_tx( - TmaTransactionBytesK + TmaTransactionBytesV); - copy( - params.tma_load_K.with( - reinterpret_cast< - cutlass::arch::ClusterTransactionBarrier::ValueType&>( - shared_storage.pipelines.barrier_KV), - 0 /*mcast_mask*/), - tKgK, - tKsK); - copy( - params.tma_load_V.with( - reinterpret_cast< - cutlass::arch::ClusterTransactionBarrier::ValueType&>( - shared_storage.pipelines.barrier_KV), - 0 /*mcast_mask*/), - tVgV, - tVsV); - -#pragma unroll(kHeadDim < 256 ? 2 : 1) - for (; m_block < m_block_max; ++m_block) { - load_step(m_block); - } - } - scheduler_prefetch(); - m_block = next_m_block; - if constexpr (Contexual_mask) { - if (lane_predicate) { - if (m_block >= 0) { -#pragma unroll(kHeadDim < 256 ? 2 : 1) - for (; m_block < contexual_m_block_max; ++m_block) { - load_step(m_block); - } - } - } - } - m_block = next_m_block; - if constexpr (Local) { - if (lane_predicate) { - if (m_block >= 0) { -#pragma unroll(kHeadDim < 256 ? 2 : 1) - for (; m_block < full_m_block_max; ++m_block) { - load_step(m_block); - } - } - } - } - if constexpr (Q_dO_same_stages) { - smem_pipe_write_do = smem_pipe_write; - } - } - - /// Perform a Producer Epilogue to prevent early exit of blocks in a Cluster - CUTLASS_DEVICE void load_tail( - MainloopPipeline pipeline_q, - MainloopPipeline_dO pipeline_do, - PipelineState& smem_pipe_write) { - static_assert( - Q_dO_same_stages, "Q and dO must have the same number of stages"); - // Need to copy since pipeline_q.producer_tail(smem_pipe_write) will - // increment smem_pipe_write - PipelineState smem_pipe_write_do = smem_pipe_write; - // Issue the epilogue waits - if (cute::elect_one_sync()) { - /* This helps avoid early exit of blocks in Cluster - * Waits for all stages to either be released (all Consumer UNLOCKs), or - * if the stage was never used then would just be acquired since the phase - * was still inverted from make_producer_start_state - */ - pipeline_q.producer_tail(smem_pipe_write); - pipeline_do.producer_tail(smem_pipe_write_do); - } - } - - /// Perform a Producer Epilogue to prevent early exit of blocks in a Cluster - CUTLASS_DEVICE void load_tail( - MainloopPipeline pipeline_q, - MainloopPipeline_dO pipeline_do, - PipelineState& smem_pipe_write, - PipelineState_dO& smem_pipe_write_do) { - // Issue the epilogue waits - if (cute::elect_one_sync()) { - /* This helps avoid early exit of blocks in Cluster - * Waits for all stages to either be released (all Consumer UNLOCKs), or - * if the stage was never used then would just be acquired since the phase - * was still inverted from make_producer_start_state - */ - pipeline_q.producer_tail(smem_pipe_write); - pipeline_do.producer_tail(smem_pipe_write_do); - } - } - - template - CUTLASS_DEVICE void store_dq( - Params const& params, - SharedStorage& shared_storage, - cute::tuple block_coord) { - if constexpr (!dQacc_use_TMA) { - return; - } - - auto [n_block, bidh, bidb] = block_coord; - SeqlenInfo_t seqlen_info{ - bidb, - get<0>(params.shape_Q), - get<0>(params.shape_K), - params.seq_offsets, - params.seq_offsets_q, - params.num_targets}; - if constexpr (Jagged) { - static constexpr int kBlockN = get<1>(TileShape_MNK{}); - if (n_block * kBlockN >= seqlen_info.seqlen_kv) { - return; - } - } - int min_full_attn_seq_len_, max_attn_len_, contextual_seq_len_; - if constexpr (!Cross) { - if (params.num_groups > 1) { - int group_id = bidb / params.batch_size_per_group; - min_full_attn_seq_len_ = params.min_full_attn_seq_len_tensor[group_id]; - max_attn_len_ = params.max_attn_len_tensor[group_id]; - contextual_seq_len_ = params.contextual_seq_len_tensor[group_id]; - } else { - min_full_attn_seq_len_ = params.min_full_attn_seq_len; - max_attn_len_ = params.max_attn_len; - contextual_seq_len_ = params.contextual_seq_len; - } - } - int m_block_min, m_block_max; - if constexpr (Cross) { - auto m_block_min_max = get_cross_m_block_min_max( - seqlen_info.uihlen_q, - seqlen_info.seqlen_q, - seqlen_info.seqlen_kv, - n_block); - m_block_min = get<0>(m_block_min_max); - m_block_max = get<1>(m_block_min_max); - } else { - auto m_block_min_max = get_m_block_min_max( - max_attn_len_, - contextual_seq_len_, - seqlen_info.uihlen_q, - seqlen_info.seqlen_q, - n_block); - m_block_min = get<0>(m_block_min_max); - m_block_max = get<1>(m_block_min_max); - } - auto full_m_block_min_max = get_full_m_block_min_max( - seqlen_info.uihlen_q, - seqlen_info.seqlen_q, - min_full_attn_seq_len_, - m_block_max, - n_block); - int const full_m_block_min = get<0>(full_m_block_min_max); - int const full_m_block_max = get<1>(full_m_block_min_max); - int contexual_m_block_max = get_contexual_m_block_max( - seqlen_info.uihlen_q, contextual_seq_len_, m_block_min, n_block); - - Tensor sdQ = make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_dqacc.data()), - SmemLayoutdQaccum{}); - static constexpr int dQ_TMA_num_bytes = - CUTE_STATIC_V(size<0>(sdQ)) * sizeof(ElementAccum); - - Tensor mdQaccum = make_tensor( - make_gmem_ptr(reinterpret_cast(params.ptr_dQaccum)), - params.shape_dQaccum, - params.stride_dQaccum)(_, bidh, !Jagged ? bidb : 0); - Tensor gdQaccum_ = local_tile( - domain_offset( - make_coord(seqlen_info.offset_q_padded * kHeadDim), mdQaccum), - Shape>{}, - make_coord(_)); // (M * K, _) - Tensor gdQaccum = cute::flat_divide( - gdQaccum_, - Int{}); // (M * K / WG, WG, _) - - int const num_batch = params.num_batch; - int const num_head = get<2>(params.shape_Q); - int* lock_ptr = - !Deterministic ? nullptr : params.dq_semaphore + bidb * num_head + bidh; - using Barrier = cutlass::GenericBarrier; - bool const lane_predicate = cute::elect_one_sync(); - - auto store_dq_step = [&](int m_block) { - if constexpr (Deterministic) { - Barrier::wait_eq( - lock_ptr, - threadIdx.x % cutlass::NumThreadsPerWarp, - m_block * num_batch * num_head, - n_block); - } -#pragma unroll - for (int warpgroup_idx = 0; warpgroup_idx < NumMmaWarpGroups; - ++warpgroup_idx) { - cutlass::arch::NamedBarrier::sync( - cutlass::NumThreadsPerWarpGroup + cutlass::NumThreadsPerWarp, - static_cast(BwdNamedBarriers::dQFullWG1) + - warpgroup_idx /*id*/); // sdQ full, to be written to gmem - if (lane_predicate) { - SM90_BULK_REDUCE_ADD::copy( - raw_pointer_cast(sdQ(_, warpgroup_idx).data()), - raw_pointer_cast(gdQaccum(_, warpgroup_idx, m_block).data()), - dQ_TMA_num_bytes, - static_cast(TMA::CacheHintSm90::EVICT_LAST)); - tma_store_arrive(); - } - } - // Note, the for_each() function is required here to ensure - // `warpgroup_idx` is of type Int. - for_each(make_int_sequence{}, [&](auto warpgroup_idx) { - if (lane_predicate) { - tma_store_wait(); - } - cutlass::arch::NamedBarrier::arrive( - cutlass::NumThreadsPerWarpGroup + cutlass::NumThreadsPerWarp, - static_cast(BwdNamedBarriers::dQEmptyWG1) + - warpgroup_idx /*id*/); // sdQ empty, ready to be written to - }); - if constexpr (Deterministic) { - Barrier::arrive_inc( - lock_ptr, - threadIdx.x % cutlass::NumThreadsPerWarp, - m_block * num_batch * num_head); - } - }; - -#pragma unroll 2 - for (int m_block = m_block_min; m_block < m_block_max; ++m_block) { - store_dq_step(m_block); - } - if constexpr (Contexual_mask) { -#pragma unroll 2 - for (int m_block = 0; m_block < contexual_m_block_max; ++m_block) { - store_dq_step(m_block); - } - } - if constexpr (Local) { -#pragma unroll 2 - for (int m_block = full_m_block_min; m_block < full_m_block_max; - ++m_block) { - store_dq_step(m_block); - } - } - if constexpr (Local && Deterministic) { - constexpr int kBlockM = get<0>(TileShape_MNK{}); - int const m_block_global_max = - cute::ceil_div(seqlen_info.seqlen_q, kBlockM); -#pragma unroll 2 - for (int m_block = m_block_max; m_block < m_block_global_max; ++m_block) { - Barrier::arrive_inc( - lock_ptr, - threadIdx.x % cutlass::NumThreadsPerWarp, - m_block * num_batch * num_head); - } - } - } - - CUTLASS_DEVICE void mma_init() { - // We're not currently using this bc we're not using persistent scheduler - // // Tell producer (warp 0) that smem_k and smem_v are ready - // cutlass::arch::NamedBarrier::arrive(NumMmaThreads + - // cutlass::NumThreadsPerWarp, - // static_cast(BwdNamedBarriers::KVEmpty) /*id*/); - int warp_idx_in_warpgroup = - __shfl_sync(0xffffffff, (threadIdx.x / 32) % 4, 0); - if constexpr (dQacc_use_TMA) { - if (warp_idx_in_warpgroup == 0) { - cutlass::arch::NamedBarrier::arrive( - cutlass::NumThreadsPerWarpGroup + cutlass::NumThreadsPerWarp, - static_cast(BwdNamedBarriers::dQEmptyWG1) - 1 + - hstu::canonical_warp_group_idx_nosync() /*id*/); // sdQ empty, - // ready to be - // written to - } - } - } - - template - CUTLASS_DEVICE bool mma( - Params const& params, - MainloopPipeline pipeline_q, - MainloopPipeline_dO pipeline_do, - PipelineState& smem_pipe_read, - PipelineState_dO& smem_pipe_read_do, - FrgTensordKV& tdKrdK, - FrgTensordKV& tdVrdV, - int thread_idx, - int& work_idx, - cute::tuple block_coord, - SharedStorage& shared_storage) { - static_assert( - is_rmem::value, - "dK and dV tensor must be rmem resident."); - - int n_block = get<0>(block_coord); - int bidb = get<2>(block_coord); - SeqlenInfo_t seqlen_info{ - bidb, - get<0>(params.shape_Q), - get<0>(params.shape_K), - params.seq_offsets, - params.seq_offsets_q, - params.num_targets}; - if constexpr (Jagged) { - static constexpr int kBlockN = get<1>(TileShape_MNK{}); - if (n_block * kBlockN >= seqlen_info.seqlen_kv) { - return false; - } - } - int min_full_attn_seq_len_, max_attn_len_, contextual_seq_len_; - float scalar_scale_val_; - if constexpr (!Cross) { - if (params.num_groups > 1) { - int group_id = bidb / params.batch_size_per_group; - min_full_attn_seq_len_ = params.min_full_attn_seq_len_tensor[group_id]; - max_attn_len_ = params.max_attn_len_tensor[group_id]; - contextual_seq_len_ = params.contextual_seq_len_tensor[group_id]; - } else { - min_full_attn_seq_len_ = params.min_full_attn_seq_len; - max_attn_len_ = params.max_attn_len; - contextual_seq_len_ = params.contextual_seq_len; - } - } - if (params.num_groups > 1) { - int group_id = bidb / params.batch_size_per_group; - int max_seq_len_per_group = params.max_seq_len_tensor[group_id]; - // attention scale - scalar_scale_val_ = params.scalar_scale - ? (params.attn_scale == nullptr ? 1.0f / max_seq_len_per_group - : params.attn_scale[group_id]) - : 0; - } else { - // attention scale - scalar_scale_val_ = params.scalar_scale - ? (params.attn_scale == nullptr ? params.max_seq_len_inv - : params.attn_scale[0]) - : 0; - } - int m_block_min, m_block_max; - if constexpr (Cross) { - auto m_block_min_max = get_cross_m_block_min_max( - seqlen_info.uihlen_q, - seqlen_info.seqlen_q, - seqlen_info.seqlen_kv, - n_block); - m_block_min = get<0>(m_block_min_max); - m_block_max = get<1>(m_block_min_max); - } else { - auto m_block_min_max = get_m_block_min_max( - max_attn_len_, - contextual_seq_len_, - seqlen_info.uihlen_q, - seqlen_info.seqlen_q, - n_block); - m_block_min = get<0>(m_block_min_max); - m_block_max = get<1>(m_block_min_max); - } - auto full_m_block_min_max = get_full_m_block_min_max( - seqlen_info.uihlen_q, - seqlen_info.seqlen_q, - min_full_attn_seq_len_, - m_block_max, - n_block); - int const full_m_block_min = get<0>(full_m_block_min_max); - int const full_m_block_max = get<1>(full_m_block_min_max); - int contexual_m_block_max = get_contexual_m_block_max( - seqlen_info.uihlen_q, contextual_seq_len_, m_block_min, n_block); - - Tensor sQ = make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_q.data()), - SmemLayoutQ{}); - Tensor sdO = make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_do.data()), - SmemLayoutdO{}); - Tensor sK = make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_k.data()), - SmemLayoutK{}); - Tensor sV = make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_v.data()), - SmemLayoutV{}); - Tensor sQt = make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_q.data()), - SmemLayoutQt{}); - Tensor sdOt = make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_do.data()), - SmemLayoutdOt{}); - Tensor sKt = make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_k.data()), - SmemLayoutKt{}); - Tensor sP = make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_p.data()), - SmemLayoutPdS{}); - Tensor sP_pi = cute::as_position_independent_swizzle_tensor(sP); - Tensor sPt = make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_p.data()), - SmemLayoutPdSt{}); - Tensor sPt_pi = cute::as_position_independent_swizzle_tensor(sPt); - Tensor sdS = make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_ds.data()), - SmemLayoutPdS{}); - Tensor sdS_pi = cute::as_position_independent_swizzle_tensor(sdS); - Tensor sdSt = make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_ds.data()), - SmemLayoutPdSt{}); - Tensor sdSt_pi = cute::as_position_independent_swizzle_tensor(sdSt); - Tensor sdQ = make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_dqacc.data()), - SmemLayoutdQaccum{}); - - static_assert( - stride<0>(typename TiledMmaSdP::ALayout{}) == 0 and - stride<0>(typename TiledMmaSdP::BLayout{}) == 0 and - size<0>(typename TiledMmaSdP::ALayout{}) == - cutlass::NumThreadsPerWarpGroup and - size<0>(typename TiledMmaSdP::BLayout{}) == - cutlass::NumThreadsPerWarpGroup, - "Stride of the first mode must be 0 and the size of the mode must be NumThreadsPerWarpGroup"); - constexpr int MmaWarpGroups = - NumMmaThreads / cutlass::NumThreadsPerWarpGroup; - Layout warp_group_thread_layout = make_layout( - make_shape(Int{}), - make_stride(Int{})); - Layout warp_group_thread_layout_dq = make_layout( - make_shape(Int{}), - make_stride(Int{})); - - int warp_group_idx = __shfl_sync( - 0xFFFFFFFF, thread_idx / cutlass::NumThreadsPerWarpGroup, 0); - TiledMmaSdP tiled_mma_SdP; - using TiledMmadP = - std::conditional_t; - TiledMmadP tiled_mma_dP; - TiledMmadKV tiled_mma_dKV; - TiledMmadQ tiled_mma_dQ; - - auto wg_mma_SdP = - tiled_mma_SdP.get_slice(warp_group_thread_layout(warp_group_idx)); - auto wg_mma_dP = - tiled_mma_dP.get_slice(warp_group_thread_layout(warp_group_idx)); - auto thread_mma_SdP = tiled_mma_SdP.get_thread_slice(thread_idx); - auto wg_mma_dKV = - tiled_mma_dKV.get_slice(warp_group_thread_layout(warp_group_idx)); - auto wg_mma_dQ = - tiled_mma_dQ.get_slice(warp_group_thread_layout_dq(warp_group_idx)); - - auto smem_tiled_copy_PdS = - make_tiled_copy_C(SmemCopyAtomPdS{}, tiled_mma_SdP); - auto smem_thr_copy_PdS = smem_tiled_copy_PdS.get_thread_slice(thread_idx); - - R2STiledCopydQaccum r2s_tiled_copy_dQaccum; - auto r2s_thr_copy_dQaccum = - r2s_tiled_copy_dQaccum.get_thread_slice(thread_idx); - Tensor tdQsdQaccum = r2s_thr_copy_dQaccum.partition_D(sdQ); - // if (thread_idx == 0) { print(sdQ); printf("\n"); print(tdQsdQaccum); - // printf("\n"); } - - // Allocate "fragments/descriptors" - // We have to use the templated mma_partition_fragment_AB instead of - // cute::conditional_return or lambda, because some partition_fragment_A/B - // don't compile. - // https://stackoverflow.com/questions/50051473/if-constexpr-in-c17-does-not-work-in-a-non-templated-function - Tensor tSrQ = mma_partition_fragment_AB(wg_mma_SdP, sQ); - Tensor tSrK = mma_partition_fragment_AB(wg_mma_SdP, sK); - Tensor tdPrdO = - mma_partition_fragment_AB(wg_mma_SdP, sdO); - Tensor tdPrV = mma_partition_fragment_AB(wg_mma_dP, sV); - Tensor tdVrdO = - mma_partition_fragment_AB(wg_mma_dKV, sdOt); - Tensor tdKrQ = mma_partition_fragment_AB(wg_mma_dKV, sQt); - Tensor tdQrdS = mma_partition_fragment_AB(wg_mma_dQ, sdS); - Tensor tdQrK = mma_partition_fragment_AB(wg_mma_dQ, sKt); - - Tensor tPsP = smem_thr_copy_PdS.partition_D( - cute::conditional_return( - sP_pi, sPt_pi)); // ((Atom,AtomNum),PIPE_M,PIPE_N) - Tensor tdSsdS = smem_thr_copy_PdS.partition_D( - cute::conditional_return( - sdS_pi, sdSt_pi)); // ((Atom,AtomNum),PIPE_M,PIPE_N) - // if (blockIdx.x == 0 && threadIdx.x == 128) { print(smem_thr_copy_PdS); - // print(sP_pi); printf("\n"); print(sPt_pi); printf("\n"); print(tPsP); - // printf("\n"); print(tdSsdS); printf("\n"); } - - auto consumer_wait = [](auto& pipeline, auto& smem_pipe_read) { - auto barrier_token = pipeline.consumer_try_wait(smem_pipe_read); - pipeline.consumer_wait(smem_pipe_read, barrier_token); - }; - - int bidh = get<1>(block_coord); - // For the case where we do atomicAdd directly to gdQaccum instead of using - // TMA - Tensor mdQaccum = make_tensor( - make_gmem_ptr(reinterpret_cast(params.ptr_dQaccum)), - params.shape_dQaccum, - params.stride_dQaccum)(_, bidh, !Jagged ? bidb : 0); - Tensor gdQaccum_ = local_tile( - domain_offset( - make_coord(seqlen_info.offset_q_padded * kHeadDim), mdQaccum), - Shape>{}, - make_coord(_)); // (M * K, _) - Tensor gdQaccum = cute::flat_divide( - gdQaccum_, - Int{}); // (M * K / WG, WG, _) - // We can reuse r2s_thr_copy_dQaccum for this partitioning - Tensor tdQgdQaccum = r2s_thr_copy_dQaccum.partition_D(gdQaccum); - // if (blockIdx.x == 0 && threadIdx.x == 128) { print(mdQaccum); - // printf("\n"); print(gdQaccum_); printf("\n"); print(gdQaccum); - // printf("\n"); print(tdQgdQaccum); printf("\n"); } - - hstu::Mask mask( - thread_idx, - seqlen_info.seqlen_q, - seqlen_info.seqlen_kv, - max_attn_len_, - min_full_attn_seq_len_, - contextual_seq_len_, - seqlen_info.uihlen_q); - - int m_block = m_block_min; - - clear(tdKrdK); - clear(tdVrdV); - // tiled_mma_dKV.accumulate_ = GMMA::ScaleOut::Zero; - - cutlass::ConsumerToken barrier_token = static_cast( - shared_storage.pipelines.barrier_KV.try_wait(work_idx % 2)); - if (barrier_token == cutlass::BarrierStatus::WaitAgain) { - shared_storage.pipelines.barrier_KV.wait(work_idx % 2); - } - - if constexpr (Mma_dP_is_RS) { - using SmemCopyAtomV = Copy_Atom; - auto smem_tiled_copy_V = make_tiled_copy_A(SmemCopyAtomV{}, tiled_mma_dP); - auto smem_thr_copy_V = smem_tiled_copy_V.get_thread_slice(thread_idx); - Tensor tdPrV_copy_view = smem_thr_copy_V.retile_D(tdPrV); - Tensor tdPsV_copy_view = smem_thr_copy_V.partition_S( - cute::as_position_independent_swizzle_tensor(sV)); - cute::copy(smem_tiled_copy_V, tdPsV_copy_view, tdPrV_copy_view); - } - static constexpr int Qdim = !SdP_swapAB ? 0 : 1; - auto thread0_mma_SdP = tiled_mma_SdP.get_thread_slice(_0{}); - Tensor cS = cute::make_identity_tensor( - Shape< - Int, - Int>{}); - Tensor tScS = thread_mma_SdP.partition_C(cS); - Tensor tScS_rowcol = make_tensor( - tScS.data(), - hstu::convert_layout_acc_rowcol( - tScS.layout())); - Tensor t0ScS = thread0_mma_SdP.partition_C(cS); - Tensor t0ScS_rowcol = make_tensor( - t0ScS.data(), - hstu::convert_layout_acc_rowcol( - t0ScS.layout())); - int const thread_qdim_offset = get(tScS_rowcol(_0{}, _0{})); - - auto bwd_step = [&](int m_block, auto mask_fn) { - Tensor tSrS = partition_fragment_C( - tiled_mma_SdP, - select(TileShape_MNK{})); - consumer_wait(pipeline_q, smem_pipe_read); - hstu::gemm( - tiled_mma_SdP, tSrQ(_, _, _, smem_pipe_read.index()), tSrK, tSrS); - Tensor tdPrdP = partition_fragment_C( - tiled_mma_SdP, - select(TileShape_MNK{})); - PipelineState_dO smem_pipe_read_do_cur = - cute::conditional_return( - smem_pipe_read, smem_pipe_read_do); - consumer_wait(pipeline_do, smem_pipe_read_do_cur); - hstu::gemm( - tiled_mma_dP, - tdPrdO(_, _, _, smem_pipe_read_do_cur.index()), - tdPrV, - tdPrdP); - warpgroup_wait<1>(); - // Reshape tSrS from ((2, 2, V), MMA_N, MMA_M) to (nrow=(2, V, MMA_M), - // ncol=(2, MMA_N)) - Tensor scores = make_tensor( - tSrS.data(), - hstu::convert_layout_acc_rowcol( - tSrS.layout())); - Tensor tSrS_sigmoid = make_tensor_like(tSrS); - Tensor sigmoid = make_tensor( - tSrS_sigmoid.data(), - hstu::convert_layout_acc_rowcol( - tSrS_sigmoid.layout())); - int qdim_offset = params.scalar_scale - ? 0 - : m_block * kBlockM + thread_qdim_offset + seqlen_info.offset_q; - mask_fn(tSrS, m_block); -#pragma unroll - for (int mi = 0; mi < size<0>(scores); ++mi) { - float scale = scalar_scale_val_; - if (!params.scalar_scale) { - int q_index = qdim_offset + int(get(t0ScS_rowcol(mi, _0{}))); - int q_local = q_index - seqlen_info.offset_q; - if (q_local < seqlen_info.seqlen_q) { - scale = params.attn_scale[q_index]; - } - } -#pragma unroll - for (int ni = 0; ni < size<1>(scores); ++ni) { - scores(mi, ni) = scores(mi, ni) * params.alpha; - sigmoid(mi, ni) = - __fdividef(1., 1.0f + cutlass::fast_exp(-scores(mi, ni))); - scores(mi, ni) = sigmoid(mi, ni) * scores(mi, ni) * scale; - } - } - mask_fn(tSrS_sigmoid, m_block); - - warpgroup_wait<0>(); - // Reshape tdPrdP from ((2, 2, V), MMA_N, MMA_M) to (nrow=(2, V, MMA_M), - // ncol=(2, MMA_N)) - Tensor dS = make_tensor(tdPrdP.data(), scores.layout()); -#pragma unroll - for (int mi = 0; mi < size<0>(dS); ++mi) { - float scale = scalar_scale_val_; - if (!params.scalar_scale) { - int q_index = qdim_offset + int(get(t0ScS_rowcol(mi, _0{}))); - int q_local = q_index - seqlen_info.offset_q; - if (q_local < seqlen_info.seqlen_q) { - scale = params.attn_scale[q_index]; - } - } -#pragma unroll - for (int ni = 0; ni < size<1>(dS); ++ni) { - dS(mi, ni) = dS(mi, ni) * sigmoid(mi, ni) * scale + - dS(mi, ni) * scores(mi, ni) * (1.f - sigmoid(mi, ni)); - dS(mi, ni) = dS(mi, ni) * params.alpha; - // if (dS(mi, ni) > 0.0001) { - // std::printf( - // "dS(mi, ni) is (%f), (m, n) is (%d, %d), thread_idx is - // (%d), blockIdx.z is (%d)\n", dS(mi, ni), mi, ni, - // threadIdx.x, - // blockIdx.z); - // } - } - } - // Convert scores from fp32 to fp16/bf16 - Tensor rP = make_tensor_like(tSrS); - hstu::convert_type_out(tSrS, rP); - if constexpr (!Mma_dKV_is_RS) { - // Need to sync to make sure P has already been used in the previous - // iteration before writing new values - if constexpr (kStages_dS == 1) { - cutlass::arch::NamedBarrier::sync( - NumMmaThreads, - static_cast(BwdNamedBarriers::PdS) /*id*/); - } - Tensor tPaP = - smem_thr_copy_PdS.retile_S(rP); // ((Atom,AtomNum), MMA_N, MMA_N) - cute::copy( - smem_tiled_copy_PdS, - tPaP, - tPsP( - _, - _, - _, - cute::conditional_return( - _0{}, smem_pipe_read.index()))); - } - Tensor rdS = make_tensor_like(tdPrdP); - hstu::convert_type_out(tdPrdP, rdS); - // If there's double buffering on dS, we don't need to sync here. - // Otherwise we might have WG1 writing to dS before WG2 is done reading - // from it during MmadQ. But because both WGs have to sync at the end of - // the loop and double buffering, this race condition is not possible. - // This sync is to ensure (1) P is written in case of !Mma_dKV_is_RS and - // (2) dS is already read by the Mma in the previous iteration in case of - // Mma_dKV_is_RS. - if constexpr (!Mma_dKV_is_RS || (kStages_dS == 1 && Mma_dKV_is_RS)) { - cutlass::arch::fence_view_async_shared(); - cutlass::arch::NamedBarrier::sync( - NumMmaThreads, static_cast(BwdNamedBarriers::PdS) /*id*/); - } - // For hdim 64, It's faster to write to smem_dS first before the dV gemm - Tensor tdSadS = - smem_thr_copy_PdS.retile_S(rdS); // ((Atom,AtomNum), MMA_N, MMA_N) - cute::copy( - smem_tiled_copy_PdS, - tdSadS, - tdSsdS( - _, - _, - _, - cute::conditional_return( - _0{}, smem_pipe_read.index()))); - - if constexpr (!Slice_dQKV_Mma) { - // Most cases take this path, except for hdim256 where we want to slice - // to reduce register pressure - if constexpr (Mma_dKV_is_RS) { - Tensor tdVrP = make_tensor( - rP.data(), convert_layout_acc_Aregs(tSrS.layout())); - hstu::gemm( - tiled_mma_dKV, - tdVrP, - tdVrdO(_, _, _, smem_pipe_read_do_cur.index()), - tdVrdV); - } else { - Tensor tdVrP = - mma_partition_fragment_AB(wg_mma_dKV, sPt); - Tensor tdVrP_cur = tdVrP( - _, - _, - _, - cute::conditional_return( - _0{}, smem_pipe_read.index())); - hstu:: - gemm( - tiled_mma_dKV, - tdVrP_cur, - tdVrdO(_, _, _, smem_pipe_read_do_cur.index()), - tdVrdV); - } - // SMEM fence to make sure sdS is written before it's read by WGMMA - cutlass::arch::fence_view_async_shared(); - cutlass::arch::NamedBarrier::sync( - NumMmaThreads, static_cast(BwdNamedBarriers::PdS) /*id*/); - Tensor tdQrdQ = partition_fragment_C( - tiled_mma_dQ, - select(TileShape_MNK{})); - Tensor tdQrdS_cur = tdQrdS( - _, - _, - _, - cute::conditional_return( - _0{}, smem_pipe_read.index())); - hstu::gemm( - tiled_mma_dQ, tdQrdS_cur, tdQrK, tdQrdQ); - pipeline_do.consumer_release(smem_pipe_read_do_cur); // release dQ - - if constexpr (Mma_dKV_is_RS) { - Tensor tdKrdS = make_tensor( - rdS.data(), - convert_layout_acc_Aregs(tdPrdP.layout())); - hstu::gemm( - tiled_mma_dKV, - tdKrdS, - tdKrQ(_, _, _, smem_pipe_read.index()), - tdKrdK); - } else { - Tensor tdKrdS = - mma_partition_fragment_AB(wg_mma_dKV, sdSt); - Tensor tdKrdS_cur = tdKrdS( - _, - _, - _, - cute::conditional_return( - _0{}, smem_pipe_read.index())); - hstu::gemm( - tiled_mma_dKV, - tdKrdS_cur, - tdKrQ(_, _, _, smem_pipe_read.index()), - tdKrdK); - } - if constexpr (dQacc_use_TMA) { - int const warp_group_idx = - hstu::canonical_warp_group_idx_nosync() - 1; - cutlass::arch::NamedBarrier::sync( - cutlass::NumThreadsPerWarpGroup + cutlass::NumThreadsPerWarp, - static_cast(BwdNamedBarriers::dQEmptyWG1) + - warp_group_idx /*id*/); // sdQ full, to be written to gmem - Tensor taccdQrdQ = r2s_thr_copy_dQaccum.retile_S(tdQrdQ); - cute::copy(r2s_tiled_copy_dQaccum, taccdQrdQ, tdQsdQaccum); - cutlass::arch::fence_view_async_shared(); - cutlass::arch::NamedBarrier::arrive( - cutlass::NumThreadsPerWarpGroup + cutlass::NumThreadsPerWarp, - static_cast(BwdNamedBarriers::dQFullWG1) + - warp_group_idx /*id*/); // sdQ full, to be written to gmem - } else { - // We can reuse r2s_thr_copy_dQaccum for this partitioning - Tensor tdQrdQ_atomic = - recast(r2s_thr_copy_dQaccum.retile_S(tdQrdQ)); - Tensor tdQgdQaccum_atomic = - recast(tdQgdQaccum(_, _, _, m_block)); - static_assert( - CUTE_STATIC_V(size(tdQrdQ_atomic)) == - CUTE_STATIC_V(size(tdQgdQaccum_atomic))); -#pragma unroll - for (int i = 0; i < size(tdQrdQ_atomic); ++i) { - atomicAdd(&tdQgdQaccum_atomic(i), tdQrdQ_atomic(i)); - } - } - - } else { // Slice_dQKV_Mma - - static_assert(!(Slice_dQKV_Mma && Mma_dKV_is_RS)); - Tensor tdVrP = - mma_partition_fragment_AB(wg_mma_dKV, sPt); - Tensor tdVrP_cur = tdVrP( - _, - _, - _, - cute::conditional_return( - _0{}, smem_pipe_read.index())); - hstu::gemm< - /*zero_init=*/false, - /*wg_wait=*/-1, - /*SwapAB=*/dKV_swapAB, - /*M_slice=*/0>( - tiled_mma_dKV, - tdVrP_cur, - tdVrdO(_, _, _, smem_pipe_read_do_cur.index()), - tdVrdV); - - cutlass::arch::fence_view_async_shared(); - cutlass::arch::NamedBarrier::sync( - NumMmaThreads, static_cast(BwdNamedBarriers::PdS) /*id*/); - Tensor tdQrdQ = partition_fragment_C( - tiled_mma_dQ, - select(TileShape_MNK{})); - Tensor tdQrdS_cur = tdQrdS( - _, - _, - _, - cute::conditional_return( - _0{}, smem_pipe_read.index())); - hstu::gemm< - /*zero_init=*/true, - /*wg_wait=*/-1, - /*SwapAB=*/dQ_swapAB, - /*M_slice=*/0>(tiled_mma_dQ, tdQrdS_cur, tdQrK, tdQrdQ); - hstu::gemm< - /*zero_init=*/false, - /*wg_wait=*/1, - /*SwapAB=*/dKV_swapAB, - /*M_slice=*/1>( - tiled_mma_dKV, - tdVrP_cur, - tdVrdO(_, _, _, smem_pipe_read_do_cur.index()), - tdVrdV); - Tensor tdQrdQ_atomic = - recast(r2s_thr_copy_dQaccum.retile_S(tdQrdQ)); - Tensor tdQgdQaccum_atomic = - recast(tdQgdQaccum(_, _, _, m_block)); -#pragma unroll - for (int i = 0; i < size(tdQrdQ_atomic) / 2; ++i) { - atomicAdd(&tdQgdQaccum_atomic(i), tdQrdQ_atomic(i)); - } - - Tensor tdKrdS = - mma_partition_fragment_AB(wg_mma_dKV, sdSt); - Tensor tdKrdS_cur = tdKrdS( - _, - _, - _, - cute::conditional_return( - _0{}, smem_pipe_read.index())); - hstu::gemm< - /*zero_init=*/false, - /*wg_wait=*/1, - /*SwapAB=*/dKV_swapAB, - /*M_slice=*/0>( - tiled_mma_dKV, - tdKrdS_cur, - tdKrQ(_, _, _, smem_pipe_read.index()), - tdKrdK); - pipeline_do.consumer_release(smem_pipe_read_do_cur); // release dO - - hstu::gemm< - /*zero_init=*/true, - /*wg_wait=*/0, - /*SwapAB=*/dQ_swapAB, - /*M_slice=*/1>(tiled_mma_dQ, tdQrdS_cur, tdQrK, tdQrdQ); -#pragma unroll - for (int i = size(tdQrdQ_atomic) / 2; i < size(tdQrdQ_atomic); ++i) { - atomicAdd(&tdQgdQaccum_atomic(i), tdQrdQ_atomic(i)); - } - - hstu::gemm< - /*zero_init=*/false, - /*wg_wait=*/-1, - /*SwapAB=*/dKV_swapAB, - /*M_slice=*/1>( - tiled_mma_dKV, - tdKrdS_cur, - tdKrQ(_, _, _, smem_pipe_read.index()), - tdKrdK); - } - - warpgroup_wait<0>(); - pipeline_q.consumer_release(smem_pipe_read); // release Q - ++smem_pipe_read; - if constexpr (!Q_dO_same_stages) { - ++smem_pipe_read_do; - } - }; - static constexpr int kBlockM = get<0>(TileShape_MNK{}); - static constexpr int kBlockN = get<1>(TileShape_MNK{}); - - if constexpr (Cross) { - auto mask_fn = [&](auto& tSrS, int m_block) { - mask.template apply< - true /*Seqlenq_mask*/, - true /*Seqlenk_mask*/, - Causal, - false /*Local*/, - false /*Contexual_mask*/, - false /*Target_mask*/, - Cross, - false /*Softmax*/>(tSrS, m_block, n_block); - }; - CUTLASS_PRAGMA_NO_UNROLL - for (; m_block < m_block_max; ++m_block) { - bwd_step(m_block, mask_fn); - } - if constexpr (Q_dO_same_stages) { - smem_pipe_read_do = smem_pipe_read; - } - ++work_idx; - return true; - } - if constexpr (Has_targets) { - if (n_block * kBlockN >= seqlen_info.uihlen_q) { - auto mask_fn = [&](auto& tSrS, int m_block) { - mask.template apply< - true /*Seqlenq_mask*/, - true /*Seqlenk_mask*/, - false /*Causal*/, - false /*Local*/, - false /*Contexual_mask*/, - Has_targets /*Target_mask*/, - false /*Cross*/, - false /*Softmax*/>(tSrS, m_block, n_block); - }; - CUTLASS_PRAGMA_NO_UNROLL - for (; m_block < m_block_max; ++m_block) { - bwd_step(m_block, mask_fn); - } - if constexpr (Q_dO_same_stages) { - smem_pipe_read_do = smem_pipe_read; - } - ++work_idx; - return true; - } else if ((n_block + 1) * kBlockN >= seqlen_info.uihlen_q) { - if constexpr ((Causal || Local) && SeparateMaskingIterations) { - auto mask_fn = [&](auto& tSrS, int m_block) { - mask.template apply< - true /*Seqlenq_mask*/, - true /*Seqlenk_mask*/, - Causal, - Local, - Contexual_mask, - Has_targets /*Target_mask*/, - false /*Cross*/, - false /*Softmax*/>(tSrS, m_block, n_block); - }; - int const m_block_masking_max = - ((n_block + 1) * kBlockN - 1) / kBlockM + 1; - CUTLASS_PRAGMA_NO_UNROLL - for (; m_block < std::min(m_block_max, m_block_masking_max); - ++m_block) { - bwd_step(m_block, mask_fn); - } - } - - auto mask_fn = [&](auto& tSrS, int m_block) { - mask.template apply< - true /*Seqlenq_mask*/, - true /*Seqlenk_mask*/, - Causal && !SeparateMaskingIterations, - Local && !SeparateMaskingIterations, - Contexual_mask, - Has_targets /*Target_mask*/, - false /*Cross*/, - false /*Softmax*/>(tSrS, m_block, n_block); - }; - if constexpr (SeparateMaskingIterations) { - int const m_block_max_before_local_mask = - !Local || !SeparateMaskingIterations - ? m_block_max - : std::min( - m_block_max, (n_block * kBlockN + max_attn_len_) / kBlockM); - CUTLASS_PRAGMA_NO_UNROLL - for (; m_block < m_block_max_before_local_mask; ++m_block) { - bwd_step(m_block, mask_fn); - } - } else { - int num_m_block = m_block_max - m_block_min; - CUTLASS_PRAGMA_NO_UNROLL - for (int i = 0; i < num_m_block + full_m_block_max - - full_m_block_min + contexual_m_block_max; - ++i) { - if (i < num_m_block) { - m_block = m_block_min + i; - } else if (i < num_m_block + contexual_m_block_max) { - m_block = i - num_m_block; - } else { - m_block = - i - num_m_block - contexual_m_block_max + full_m_block_min; - } - bwd_step(m_block, mask_fn); - } - } - - if constexpr (Local && SeparateMaskingIterations) { - auto mask_fn = [&](auto& tSrS, int m_block) { - mask.template apply< - true /*Seqlenq_mask*/, - true /*Seqlenk_mask*/, - false /*Causal_mask*/, - Local, - Contexual_mask, - Has_targets /*Target_mask*/, - false /*Cross*/, - false /*Softmax*/>(tSrS, m_block, n_block); - }; - CUTLASS_PRAGMA_NO_UNROLL - for (; m_block < m_block_max; ++m_block) { - bwd_step(m_block, mask_fn); - } - } - if constexpr (Contexual_mask && SeparateMaskingIterations) { - auto mask_fn = [&](auto& tSrS, int m_block) { - mask.template apply< - true /*Seqlenq_mask*/, - true /*Seqlenk_mask*/, - Causal /*Causal_mask*/, - Local /*Local_mask*/, - Contexual_mask, - Has_targets, - false /*Cross*/, - false /*Softmax*/>(tSrS, m_block, n_block); - }; - CUTLASS_PRAGMA_NO_UNROLL - for (m_block = 0; m_block < contexual_m_block_max; ++m_block) { - bwd_step(m_block, mask_fn); - } - } - - if constexpr (Local && SeparateMaskingIterations) { - auto mask_fn = [&](auto& tSrS, int m_block) { - mask.template apply< - true /*Seqlenq_mask*/, - true /*Seqlenk_mask*/, - false /*Causal_mask*/, - Local, - Contexual_mask, - Has_targets, - false /*Cross*/, - false /*Softmax*/>(tSrS, m_block, n_block); - }; - CUTLASS_PRAGMA_NO_UNROLL - for (m_block = full_m_block_min; m_block < full_m_block_max; - ++m_block) { - bwd_step(m_block, mask_fn); - } - } - if constexpr (Q_dO_same_stages) { - smem_pipe_read_do = smem_pipe_read; - } - ++work_idx; - return true; - } - } - // We have separate iterations with causal masking. Not necessary for hdim - // 128 but for hdim 64 this helps quite a bit to not have to do causal - // masking for most of the iterations. - if constexpr ((Causal || Local) && SeparateMaskingIterations) { - auto mask_fn = [&](auto& tSrS, int m_block) { - mask.template apply< - true /*Seqlenq_mask*/, - true /*Seqlenk_mask*/, - Causal, - Local, - Contexual_mask, - false /*Target_mask*/, - false /*Cross*/, - false /*Softmax*/>(tSrS, m_block, n_block); - }; - static constexpr int kBlockM = get<0>(TileShape_MNK{}); - int const m_block_masking_max = - ((n_block + 1) * kBlockN - 1) / kBlockM + 1; - CUTLASS_PRAGMA_NO_UNROLL - for (; m_block < std::min(m_block_max, m_block_masking_max); ++m_block) { - bwd_step(m_block, mask_fn); - } - } - - auto mask_fn = [&](auto& tSrS, int m_block) { - mask.template apply< - true /*Seqlenq_mask*/, - true /*Seqlenk_mask*/, - Causal && !SeparateMaskingIterations, - Local && !SeparateMaskingIterations, - Contexual_mask, - false /*Target_mask*/, - false /*Cross*/, - false /*Softmax*/>(tSrS, m_block, n_block); - }; - if constexpr (SeparateMaskingIterations) { - int const m_block_max_before_local_mask = - !Local || !SeparateMaskingIterations - ? m_block_max - : std::min( - m_block_max, (n_block * kBlockN + max_attn_len_) / kBlockM); - CUTLASS_PRAGMA_NO_UNROLL - for (; m_block < m_block_max_before_local_mask; ++m_block) { - bwd_step(m_block, mask_fn); - } - } else { - int num_m_block = m_block_max - m_block_min; - CUTLASS_PRAGMA_NO_UNROLL - for (int i = 0; i < num_m_block + full_m_block_max - full_m_block_min + - contexual_m_block_max; - ++i) { - if (i < num_m_block) { - m_block = m_block_min + i; - } else if (i < num_m_block + contexual_m_block_max) { - m_block = i - num_m_block; - } else { - m_block = i - num_m_block - contexual_m_block_max + full_m_block_min; - } - bwd_step(m_block, mask_fn); - } - } - - if constexpr (Local && SeparateMaskingIterations) { - auto mask_fn = [&](auto& tSrS, int m_block) { - mask.template apply< - true /*Seqlenq_mask*/, - true /*Seqlenk_mask*/, - false /*Causal_mask*/, - Local, - Contexual_mask, - false /*Target_mask*/, - false /*Cross*/, - false /*Softmax*/>(tSrS, m_block, n_block); - }; - CUTLASS_PRAGMA_NO_UNROLL - for (; m_block < m_block_max; ++m_block) { - bwd_step(m_block, mask_fn); - } - } - if constexpr (Contexual_mask && SeparateMaskingIterations) { - auto mask_fn = [&](auto& tSrS, int m_block) { - mask.template apply< - true /*Seqlenq_mask*/, - true /*Seqlenk_mask*/, - Causal /*Causal_mask*/, - Local /*Local_mask*/, - Contexual_mask, - false /*Target_mask*/, - false /*Cross*/, - false /*Softmax*/>(tSrS, m_block, n_block); - }; - CUTLASS_PRAGMA_NO_UNROLL - for (m_block = 0; m_block < contexual_m_block_max; ++m_block) { - bwd_step(m_block, mask_fn); - } - } - - if constexpr (Local && SeparateMaskingIterations) { - auto mask_fn = [&](auto& tSrS, int m_block) { - mask.template apply< - true /*Seqlenq_mask*/, - true /*Seqlenk_mask*/, - false /*Causal_mask*/, - Local, - Contexual_mask, - false /*Target_mask*/, - false /*Cross*/, - false /*Softmax*/>(tSrS, m_block, n_block); - }; - CUTLASS_PRAGMA_NO_UNROLL - for (m_block = full_m_block_min; m_block < full_m_block_max; ++m_block) { - bwd_step(m_block, mask_fn); - } - } - - // if (blockIdx.x == 0 && threadIdx.x == 128) { print_tensor(tdVrdV); } - if constexpr (Q_dO_same_stages) { - smem_pipe_read_do = smem_pipe_read; - } - ++work_idx; - return true; - } - - template - CUTLASS_DEVICE bool mma_softmax( - Params const& params, - MainloopPipeline pipeline_q, - MainloopPipeline_dO pipeline_do, - PipelineState& smem_pipe_read, - PipelineState_dO& smem_pipe_read_do, - FrgTensordKV& tdKrdK, - FrgTensordKV& tdVrdV, - int thread_idx, - int& work_idx, - cute::tuple block_coord, - SharedStorage& shared_storage) { - static_assert( - is_rmem::value, - "dK and dV tensor must be rmem resident."); - - int n_block = get<0>(block_coord); - int bidb = get<2>(block_coord); - SeqlenInfo_t seqlen_info{ - bidb, - get<0>(params.shape_Q), - get<0>(params.shape_K), - params.seq_offsets, - params.seq_offsets_q, - params.num_targets}; - if constexpr (Jagged) { - static constexpr int kBlockN = get<1>(TileShape_MNK{}); - if (n_block * kBlockN >= seqlen_info.seqlen_kv) { - return false; - } - } - int min_full_attn_seq_len_, max_attn_len_, contextual_seq_len_; - if constexpr (!Cross) { - if (params.num_groups > 1) { - int group_id = bidb / params.num_groups; - min_full_attn_seq_len_ = params.min_full_attn_seq_len_tensor[group_id]; - max_attn_len_ = params.max_attn_len_tensor[group_id]; - contextual_seq_len_ = params.contextual_seq_len_tensor[group_id]; - } else { - min_full_attn_seq_len_ = params.min_full_attn_seq_len; - max_attn_len_ = params.max_attn_len; - contextual_seq_len_ = params.contextual_seq_len; - } - } - int m_block_min, m_block_max; - if constexpr (Cross) { - auto m_block_min_max = get_cross_m_block_min_max( - seqlen_info.uihlen_q, - seqlen_info.seqlen_q, - seqlen_info.seqlen_kv, - n_block); - m_block_min = get<0>(m_block_min_max); - m_block_max = get<1>(m_block_min_max); - } else { - auto m_block_min_max = get_m_block_min_max( - max_attn_len_, - contextual_seq_len_, - seqlen_info.uihlen_q, - seqlen_info.seqlen_q, - n_block); - m_block_min = get<0>(m_block_min_max); - m_block_max = get<1>(m_block_min_max); - } - auto full_m_block_min_max = get_full_m_block_min_max( - seqlen_info.uihlen_q, - seqlen_info.seqlen_q, - min_full_attn_seq_len_, - m_block_max, - n_block); - int const full_m_block_min = get<0>(full_m_block_min_max); - int const full_m_block_max = get<1>(full_m_block_min_max); - int contexual_m_block_max = get_contexual_m_block_max( - seqlen_info.uihlen_q, contextual_seq_len_, m_block_min, n_block); - - Tensor sQ = make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_q.data()), - SmemLayoutQ{}); - Tensor sdO = make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_do.data()), - SmemLayoutdO{}); - Tensor sK = make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_k.data()), - SmemLayoutK{}); - Tensor sV = make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_v.data()), - SmemLayoutV{}); - Tensor sQt = make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_q.data()), - SmemLayoutQt{}); - Tensor sdOt = make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_do.data()), - SmemLayoutdOt{}); - Tensor sKt = make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_k.data()), - SmemLayoutKt{}); - Tensor sP = make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_p.data()), - SmemLayoutPdS{}); - Tensor sP_pi = cute::as_position_independent_swizzle_tensor(sP); - Tensor sPt = make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_p.data()), - SmemLayoutPdSt{}); - Tensor sPt_pi = cute::as_position_independent_swizzle_tensor(sPt); - Tensor sdS = make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_ds.data()), - SmemLayoutPdS{}); - Tensor sdS_pi = cute::as_position_independent_swizzle_tensor(sdS); - Tensor sdSt = make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_ds.data()), - SmemLayoutPdSt{}); - Tensor sdSt_pi = cute::as_position_independent_swizzle_tensor(sdSt); - Tensor sdQ = make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_dqacc.data()), - SmemLayoutdQaccum{}); - Tensor sLSEMma = make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_lse.data()), - SmemLayoutLSEMma{}); - Tensor sdPsumMma = make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_dpsum.data()), - SmemLayoutLSEMma{}); - - static_assert( - stride<0>(typename TiledMmaSdP::ALayout{}) == 0 and - stride<0>(typename TiledMmaSdP::BLayout{}) == 0 and - size<0>(typename TiledMmaSdP::ALayout{}) == - cutlass::NumThreadsPerWarpGroup and - size<0>(typename TiledMmaSdP::BLayout{}) == - cutlass::NumThreadsPerWarpGroup, - "Stride of the first mode must be 0 and the size of the mode must be NumThreadsPerWarpGroup"); - constexpr int MmaWarpGroups = - NumMmaThreads / cutlass::NumThreadsPerWarpGroup; - Layout warp_group_thread_layout = make_layout( - make_shape(Int{}), - make_stride(Int{})); - Layout warp_group_thread_layout_dq = make_layout( - make_shape(Int{}), - make_stride(Int{})); - - int warp_group_idx = __shfl_sync( - 0xFFFFFFFF, thread_idx / cutlass::NumThreadsPerWarpGroup, 0); - TiledMmaSdP tiled_mma_SdP; - using TiledMmadP = - std::conditional_t; - TiledMmadP tiled_mma_dP; - TiledMmadKV tiled_mma_dKV; - TiledMmadQ tiled_mma_dQ; - - auto wg_mma_SdP = - tiled_mma_SdP.get_slice(warp_group_thread_layout(warp_group_idx)); - auto wg_mma_dP = - tiled_mma_dP.get_slice(warp_group_thread_layout(warp_group_idx)); - auto thread_mma_SdP = tiled_mma_SdP.get_thread_slice(thread_idx); - auto wg_mma_dKV = - tiled_mma_dKV.get_slice(warp_group_thread_layout(warp_group_idx)); - auto wg_mma_dQ = - tiled_mma_dQ.get_slice(warp_group_thread_layout_dq(warp_group_idx)); - - auto smem_tiled_copy_PdS = - make_tiled_copy_C(SmemCopyAtomPdS{}, tiled_mma_SdP); - auto smem_thr_copy_PdS = smem_tiled_copy_PdS.get_thread_slice(thread_idx); - - R2STiledCopydQaccum r2s_tiled_copy_dQaccum; - auto r2s_thr_copy_dQaccum = - r2s_tiled_copy_dQaccum.get_thread_slice(thread_idx); - Tensor tdQsdQaccum = r2s_thr_copy_dQaccum.partition_D(sdQ); - // if (thread_idx == 0) { print(sdQ); printf("\n"); print(tdQsdQaccum); - // printf("\n"); } - - // Allocate "fragments/descriptors" - // We have to use the templated mma_partition_fragment_AB instead of - // cute::conditional_return or lambda, because some partition_fragment_A/B - // don't compile. - // https://stackoverflow.com/questions/50051473/if-constexpr-in-c17-does-not-work-in-a-non-templated-function - Tensor tSrQ = mma_partition_fragment_AB(wg_mma_SdP, sQ); - Tensor tSrK = mma_partition_fragment_AB(wg_mma_SdP, sK); - Tensor tdPrdO = - mma_partition_fragment_AB(wg_mma_SdP, sdO); - Tensor tdPrV = mma_partition_fragment_AB(wg_mma_dP, sV); - Tensor tdVrdO = - mma_partition_fragment_AB(wg_mma_dKV, sdOt); - Tensor tdKrQ = mma_partition_fragment_AB(wg_mma_dKV, sQt); - Tensor tdQrdS = mma_partition_fragment_AB(wg_mma_dQ, sdS); - Tensor tdQrK = mma_partition_fragment_AB(wg_mma_dQ, sKt); - - Tensor tPsP = smem_thr_copy_PdS.partition_D( - cute::conditional_return( - sP_pi, sPt_pi)); // ((Atom,AtomNum),PIPE_M,PIPE_N) - Tensor tdSsdS = smem_thr_copy_PdS.partition_D( - cute::conditional_return( - sdS_pi, sdSt_pi)); // ((Atom,AtomNum),PIPE_M,PIPE_N) - // if (blockIdx.x == 0 && threadIdx.x == 128) { print(smem_thr_copy_PdS); - // print(sP_pi); printf("\n"); print(sPt_pi); printf("\n"); print(tPsP); - // printf("\n"); print(tdSsdS); printf("\n"); } - - // thread_mma_SdP.partition_C(sLSEMma) has shape ((2, 2, V), MMA_M, MMA_N, - // PIPE), we only take the col indices or row indices, depending on whether - // SdP_swapAB. - Tensor tLSEsLSE = cute::conditional_return( - group_modes<0, 2>(thread_mma_SdP.partition_C(sLSEMma)( - make_coord(_0{}, _, _0{}), _, _0{}, _)), // (2, MMA_M, PIPE) - group_modes<0, 3>(thread_mma_SdP.partition_C(sLSEMma)( - make_coord(_, _0{}, _), _0{}, _, _))); // (2, V, MMA_N, PIPE) - Tensor tLSEsdPsum = cute::conditional_return( - group_modes<0, 2>(thread_mma_SdP.partition_C(sdPsumMma)( - make_coord(_0{}, _, _0{}), _, _0{}, _)), - group_modes<0, 3>(thread_mma_SdP.partition_C(sdPsumMma)( - make_coord(_, _0{}, _), _0{}, _, _))); - // if (blockIdx.x == 0 && threadIdx.x == 128) { print(sLSEMma); - // printf("\n"); print(tLSEsLSE); printf("\n"); } If we want to split the - // stats among the 8 threads that share the same rows. - static constexpr int kStatsPerThread = - cute::ceil_div(decltype(size(tLSEsLSE))::value, 8); - - auto consumer_wait = [](auto& pipeline, auto& smem_pipe_read) { - auto barrier_token = pipeline.consumer_try_wait(smem_pipe_read); - pipeline.consumer_wait(smem_pipe_read, barrier_token); - }; - - int bidh = get<1>(block_coord); - // For the case where we do atomicAdd directly to gdQaccum instead of using - // TMA - Tensor mdQaccum = make_tensor( - make_gmem_ptr(reinterpret_cast(params.ptr_dQaccum)), - params.shape_dQaccum, - params.stride_dQaccum)(_, bidh, !Jagged ? bidb : 0); - Tensor gdQaccum_ = local_tile( - domain_offset( - make_coord(seqlen_info.offset_q_padded * kHeadDim), mdQaccum), - Shape>{}, - make_coord(_)); // (M * K, _) - Tensor gdQaccum = cute::flat_divide( - gdQaccum_, - Int{}); // (M * K / WG, WG, _) - // We can reuse r2s_thr_copy_dQaccum for this partitioning - Tensor tdQgdQaccum = r2s_thr_copy_dQaccum.partition_D(gdQaccum); - // if (blockIdx.x == 0 && threadIdx.x == 128) { print(mdQaccum); - // printf("\n"); print(gdQaccum_); printf("\n"); print(gdQaccum); - // printf("\n"); print(tdQgdQaccum); printf("\n"); } - - hstu::Mask mask( - thread_idx, - seqlen_info.seqlen_q, - seqlen_info.seqlen_kv, - max_attn_len_, - min_full_attn_seq_len_, - contextual_seq_len_, - seqlen_info.uihlen_q); - - int m_block = m_block_min; - - clear(tdKrdK); - clear(tdVrdV); - // tiled_mma_dKV.accumulate_ = GMMA::ScaleOut::Zero; - - cutlass::ConsumerToken barrier_token = static_cast( - shared_storage.pipelines.barrier_KV.try_wait(work_idx % 2)); - if (barrier_token == cutlass::BarrierStatus::WaitAgain) { - shared_storage.pipelines.barrier_KV.wait(work_idx % 2); - } - - if constexpr (Mma_dP_is_RS) { - using SmemCopyAtomV = Copy_Atom; - auto smem_tiled_copy_V = make_tiled_copy_A(SmemCopyAtomV{}, tiled_mma_dP); - auto smem_thr_copy_V = smem_tiled_copy_V.get_thread_slice(thread_idx); - Tensor tdPrV_copy_view = smem_thr_copy_V.retile_D(tdPrV); - Tensor tdPsV_copy_view = smem_thr_copy_V.partition_S( - cute::as_position_independent_swizzle_tensor(sV)); - cute::copy(smem_tiled_copy_V, tdPsV_copy_view, tdPrV_copy_view); - } - static constexpr int Qdim = !SdP_swapAB ? 0 : 1; - auto thread0_mma_SdP = tiled_mma_SdP.get_thread_slice(_0{}); - Tensor cS = cute::make_identity_tensor( - Shape< - Int, - Int>{}); - Tensor tScS = thread_mma_SdP.partition_C(cS); - Tensor tScS_rowcol = make_tensor( - tScS.data(), - hstu::convert_layout_acc_rowcol( - tScS.layout())); - Tensor t0ScS = thread0_mma_SdP.partition_C(cS); - Tensor t0ScS_rowcol = make_tensor( - t0ScS.data(), - hstu::convert_layout_acc_rowcol( - t0ScS.layout())); - int const thread_qdim_offset = get(tScS_rowcol(_0{}, _0{})); - - auto bwd_step = [&](int m_block, auto mask_fn) { - Tensor tSrS = partition_fragment_C( - tiled_mma_SdP, - select(TileShape_MNK{})); - consumer_wait(pipeline_q, smem_pipe_read); - hstu::gemm( - tiled_mma_SdP, tSrQ(_, _, _, smem_pipe_read.index()), tSrK, tSrS); - Tensor tLSErLSE = cute::conditional_return( - make_fragment_like(tLSEsLSE(_, _0{})), - make_tensor(Int{})); - if constexpr (!ShuffleLSE) { - cute::copy(tLSEsLSE(_, smem_pipe_read.index()), tLSErLSE); - } else { -#pragma unroll - for (int i = 0; i < kStatsPerThread; ++i) { - // It's ok to read OOB, since we made sure sLSE is large enough and we - // won't use the OOB values - tLSErLSE(i) = - tLSEsLSE((thread_idx % 32) / 4 + i * 8, smem_pipe_read.index()); - } - } - Tensor tdPrdP = partition_fragment_C( - tiled_mma_SdP, - select(TileShape_MNK{})); - PipelineState_dO smem_pipe_read_do_cur = - cute::conditional_return( - smem_pipe_read, smem_pipe_read_do); - consumer_wait(pipeline_do, smem_pipe_read_do_cur); - hstu::gemm( - tiled_mma_dP, - tdPrdO(_, _, _, smem_pipe_read_do_cur.index()), - tdPrV, - tdPrdP); - warpgroup_wait<1>(); - // Reshape tSrS from ((2, 2, V), MMA_N, MMA_M) to (nrow=(2, V, MMA_M), - // ncol=(2, MMA_N)) - Tensor scores = make_tensor( - tSrS.data(), - hstu::convert_layout_acc_rowcol( - tSrS.layout())); - mask_fn(tSrS, m_block); -#pragma unroll - for (int mi = 0; mi < size<0>(scores); ++mi) { - float const lse_scaled = [&] { - if constexpr (!ShuffleLSE) - return tLSErLSE(mi); - else - return __shfl_sync( - 0xffffffff, tLSErLSE(mi / 8), (mi % 8) * 4 + (thread_idx % 4)); - }(); -#pragma unroll - for (int ni = 0; ni < size<1>(scores); ++ni) { - scores(mi, ni) = - exp2f(scores(mi, ni) * params.alpha_log2 - lse_scaled); - } - } - Tensor tLSErdPsum = cute::conditional_return( - make_fragment_like(tLSEsdPsum(_, _0{})), - make_tensor(Int{})); - if constexpr (!ShuffledPsum) { - cute::copy(tLSEsdPsum(_, smem_pipe_read_do_cur.index()), tLSErdPsum); - } else { -#pragma unroll - for (int i = 0; i < kStatsPerThread; ++i) { - tLSErdPsum(i) = tLSEsdPsum( - (thread_idx % 32) / 4 + i * 8, smem_pipe_read_do_cur.index()); - } - } - - warpgroup_wait<0>(); - // Reshape tdPrdP from ((2, 2, V), MMA_N, MMA_M) to (nrow=(2, V, MMA_M), - // ncol=(2, MMA_N)) - Tensor dS = make_tensor(tdPrdP.data(), scores.layout()); -#pragma unroll - for (int mi = 0; mi < size<0>(dS); ++mi) { - float const dP_sum_cur = [&] { - if constexpr (!ShuffledPsum) - return tLSErdPsum(mi); - else - return __shfl_sync( - 0xffffffff, - tLSErdPsum(mi / 8), - (mi % 8) * 4 + (thread_idx % 4)); - }(); -#pragma unroll - for (int ni = 0; ni < size<1>(dS); ++ni) { - dS(mi, ni) = - scores(mi, ni) * (dS(mi, ni) - dP_sum_cur) * params.alpha; - } - } - // Convert scores from fp32 to fp16/bf16 - Tensor rP = make_tensor_like(tSrS); - hstu::convert_type_out(tSrS, rP); - if constexpr (!Mma_dKV_is_RS) { - // Need to sync to make sure P has already been used in the previous - // iteration before writing new values - if constexpr (kStages_dS == 1) { - cutlass::arch::NamedBarrier::sync( - NumMmaThreads, - static_cast(BwdNamedBarriers::PdS) /*id*/); - } - Tensor tPaP = - smem_thr_copy_PdS.retile_S(rP); // ((Atom,AtomNum), MMA_N, MMA_N) - cute::copy( - smem_tiled_copy_PdS, - tPaP, - tPsP( - _, - _, - _, - cute::conditional_return( - _0{}, smem_pipe_read.index()))); - } - Tensor rdS = make_tensor_like(tdPrdP); - hstu::convert_type_out(tdPrdP, rdS); - // If there's double buffering on dS, we don't need to sync here. - // Otherwise we might have WG1 writing to dS before WG2 is done reading - // from it during MmadQ. But because both WGs have to sync at the end of - // the loop and double buffering, this race condition is not possible. - // This sync is to ensure (1) P is written in case of !Mma_dKV_is_RS and - // (2) dS is already read by the Mma in the previous iteration in case of - // Mma_dKV_is_RS. - if constexpr (!Mma_dKV_is_RS || (kStages_dS == 1 && Mma_dKV_is_RS)) { - cutlass::arch::fence_view_async_shared(); - cutlass::arch::NamedBarrier::sync( - NumMmaThreads, static_cast(BwdNamedBarriers::PdS) /*id*/); - } - // For hdim 64, It's faster to write to smem_dS first before the dV gemm - Tensor tdSadS = - smem_thr_copy_PdS.retile_S(rdS); // ((Atom,AtomNum), MMA_N, MMA_N) - cute::copy( - smem_tiled_copy_PdS, - tdSadS, - tdSsdS( - _, - _, - _, - cute::conditional_return( - _0{}, smem_pipe_read.index()))); - - if constexpr (!Slice_dQKV_Mma) { - // Most cases take this path, except for hdim256 where we want to slice - // to reduce register pressure - if constexpr (Mma_dKV_is_RS) { - Tensor tdVrP = make_tensor( - rP.data(), convert_layout_acc_Aregs(tSrS.layout())); - hstu::gemm( - tiled_mma_dKV, - tdVrP, - tdVrdO(_, _, _, smem_pipe_read_do_cur.index()), - tdVrdV); - } else { - Tensor tdVrP = - mma_partition_fragment_AB(wg_mma_dKV, sPt); - Tensor tdVrP_cur = tdVrP( - _, - _, - _, - cute::conditional_return( - _0{}, smem_pipe_read.index())); - hstu:: - gemm( - tiled_mma_dKV, - tdVrP_cur, - tdVrdO(_, _, _, smem_pipe_read_do_cur.index()), - tdVrdV); - } - // SMEM fence to make sure sdS is written before it's read by WGMMA - cutlass::arch::fence_view_async_shared(); - cutlass::arch::NamedBarrier::sync( - NumMmaThreads, static_cast(BwdNamedBarriers::PdS) /*id*/); - Tensor tdQrdQ = partition_fragment_C( - tiled_mma_dQ, - select(TileShape_MNK{})); - Tensor tdQrdS_cur = tdQrdS( - _, - _, - _, - cute::conditional_return( - _0{}, smem_pipe_read.index())); - hstu::gemm( - tiled_mma_dQ, tdQrdS_cur, tdQrK, tdQrdQ); - pipeline_do.consumer_release(smem_pipe_read_do_cur); // release dQ - - if constexpr (Mma_dKV_is_RS) { - Tensor tdKrdS = make_tensor( - rdS.data(), - convert_layout_acc_Aregs(tdPrdP.layout())); - hstu::gemm( - tiled_mma_dKV, - tdKrdS, - tdKrQ(_, _, _, smem_pipe_read.index()), - tdKrdK); - } else { - Tensor tdKrdS = - mma_partition_fragment_AB(wg_mma_dKV, sdSt); - Tensor tdKrdS_cur = tdKrdS( - _, - _, - _, - cute::conditional_return( - _0{}, smem_pipe_read.index())); - hstu::gemm( - tiled_mma_dKV, - tdKrdS_cur, - tdKrQ(_, _, _, smem_pipe_read.index()), - tdKrdK); - } - if constexpr (dQacc_use_TMA) { - int const warp_group_idx = - hstu::canonical_warp_group_idx_nosync() - 1; - cutlass::arch::NamedBarrier::sync( - cutlass::NumThreadsPerWarpGroup + cutlass::NumThreadsPerWarp, - static_cast(BwdNamedBarriers::dQEmptyWG1) + - warp_group_idx /*id*/); // sdQ full, to be written to gmem - Tensor taccdQrdQ = r2s_thr_copy_dQaccum.retile_S(tdQrdQ); - cute::copy(r2s_tiled_copy_dQaccum, taccdQrdQ, tdQsdQaccum); - cutlass::arch::fence_view_async_shared(); - cutlass::arch::NamedBarrier::arrive( - cutlass::NumThreadsPerWarpGroup + cutlass::NumThreadsPerWarp, - static_cast(BwdNamedBarriers::dQFullWG1) + - warp_group_idx /*id*/); // sdQ full, to be written to gmem - } else { - // We can reuse r2s_thr_copy_dQaccum for this partitioning - Tensor tdQrdQ_atomic = - recast(r2s_thr_copy_dQaccum.retile_S(tdQrdQ)); - Tensor tdQgdQaccum_atomic = - recast(tdQgdQaccum(_, _, _, m_block)); - static_assert( - CUTE_STATIC_V(size(tdQrdQ_atomic)) == - CUTE_STATIC_V(size(tdQgdQaccum_atomic))); -#pragma unroll - for (int i = 0; i < size(tdQrdQ_atomic); ++i) { - atomicAdd(&tdQgdQaccum_atomic(i), tdQrdQ_atomic(i)); - } - } - - } else { // Slice_dQKV_Mma - - static_assert(!(Slice_dQKV_Mma && Mma_dKV_is_RS)); - Tensor tdVrP = - mma_partition_fragment_AB(wg_mma_dKV, sPt); - Tensor tdVrP_cur = tdVrP( - _, - _, - _, - cute::conditional_return( - _0{}, smem_pipe_read.index())); - hstu::gemm< - /*zero_init=*/false, - /*wg_wait=*/-1, - /*SwapAB=*/dKV_swapAB, - /*M_slice=*/0>( - tiled_mma_dKV, - tdVrP_cur, - tdVrdO(_, _, _, smem_pipe_read_do_cur.index()), - tdVrdV); - - cutlass::arch::fence_view_async_shared(); - cutlass::arch::NamedBarrier::sync( - NumMmaThreads, static_cast(BwdNamedBarriers::PdS) /*id*/); - Tensor tdQrdQ = partition_fragment_C( - tiled_mma_dQ, - select(TileShape_MNK{})); - Tensor tdQrdS_cur = tdQrdS( - _, - _, - _, - cute::conditional_return( - _0{}, smem_pipe_read.index())); - hstu::gemm< - /*zero_init=*/true, - /*wg_wait=*/-1, - /*SwapAB=*/dQ_swapAB, - /*M_slice=*/0>(tiled_mma_dQ, tdQrdS_cur, tdQrK, tdQrdQ); - hstu::gemm< - /*zero_init=*/false, - /*wg_wait=*/1, - /*SwapAB=*/dKV_swapAB, - /*M_slice=*/1>( - tiled_mma_dKV, - tdVrP_cur, - tdVrdO(_, _, _, smem_pipe_read_do_cur.index()), - tdVrdV); - Tensor tdQrdQ_atomic = - recast(r2s_thr_copy_dQaccum.retile_S(tdQrdQ)); - Tensor tdQgdQaccum_atomic = - recast(tdQgdQaccum(_, _, _, m_block)); -#pragma unroll - for (int i = 0; i < size(tdQrdQ_atomic) / 2; ++i) { - atomicAdd(&tdQgdQaccum_atomic(i), tdQrdQ_atomic(i)); - } - - Tensor tdKrdS = - mma_partition_fragment_AB(wg_mma_dKV, sdSt); - Tensor tdKrdS_cur = tdKrdS( - _, - _, - _, - cute::conditional_return( - _0{}, smem_pipe_read.index())); - hstu::gemm< - /*zero_init=*/false, - /*wg_wait=*/1, - /*SwapAB=*/dKV_swapAB, - /*M_slice=*/0>( - tiled_mma_dKV, - tdKrdS_cur, - tdKrQ(_, _, _, smem_pipe_read.index()), - tdKrdK); - pipeline_do.consumer_release(smem_pipe_read_do_cur); // release dO - - hstu::gemm< - /*zero_init=*/true, - /*wg_wait=*/0, - /*SwapAB=*/dQ_swapAB, - /*M_slice=*/1>(tiled_mma_dQ, tdQrdS_cur, tdQrK, tdQrdQ); -#pragma unroll - for (int i = size(tdQrdQ_atomic) / 2; i < size(tdQrdQ_atomic); ++i) { - atomicAdd(&tdQgdQaccum_atomic(i), tdQrdQ_atomic(i)); - } - - hstu::gemm< - /*zero_init=*/false, - /*wg_wait=*/-1, - /*SwapAB=*/dKV_swapAB, - /*M_slice=*/1>( - tiled_mma_dKV, - tdKrdS_cur, - tdKrQ(_, _, _, smem_pipe_read.index()), - tdKrdK); - } - - warpgroup_wait<0>(); - pipeline_q.consumer_release(smem_pipe_read); // release Q - ++smem_pipe_read; - if constexpr (!Q_dO_same_stages) { - ++smem_pipe_read_do; - } - }; - static constexpr int kBlockM = get<0>(TileShape_MNK{}); - static constexpr int kBlockN = get<1>(TileShape_MNK{}); - - if constexpr (Cross) { - auto mask_fn = [&](auto& tSrS, int m_block) { - mask.template apply< - true /*Seqlenq_mask*/, - true /*Seqlenk_mask*/, - Causal, - false /*Local*/, - false /*Contexual_mask*/, - false /*Target_mask*/, - Cross, - true /*Softmax*/>(tSrS, m_block, n_block); - }; - CUTLASS_PRAGMA_NO_UNROLL - for (; m_block < m_block_max; ++m_block) { - bwd_step(m_block, mask_fn); - } - if constexpr (Q_dO_same_stages) { - smem_pipe_read_do = smem_pipe_read; - } - ++work_idx; - return true; - } - if constexpr (Has_targets) { - if (n_block * kBlockN >= seqlen_info.uihlen_q) { - auto mask_fn = [&](auto& tSrS, int m_block) { - mask.template apply< - true /*Seqlenq_mask*/, - true /*Seqlenk_mask*/, - false /*Causal*/, - false /*Local*/, - false /*Contexual_mask*/, - Has_targets /*Target_mask*/, - false /*Cross*/, - true /*Softmax*/>(tSrS, m_block, n_block); - }; - CUTLASS_PRAGMA_NO_UNROLL - for (; m_block < m_block_max; ++m_block) { - bwd_step(m_block, mask_fn); - } - if constexpr (Q_dO_same_stages) { - smem_pipe_read_do = smem_pipe_read; - } - ++work_idx; - return true; - } else if ((n_block + 1) * kBlockN >= seqlen_info.uihlen_q) { - if constexpr ((Causal || Local) && SeparateMaskingIterations) { - auto mask_fn = [&](auto& tSrS, int m_block) { - mask.template apply< - true /*Seqlenq_mask*/, - true /*Seqlenk_mask*/, - Causal, - Local, - Contexual_mask, - Has_targets /*Target_mask*/, - false /*Cross*/, - true /*Softmax*/>(tSrS, m_block, n_block); - }; - int const m_block_masking_max = - ((n_block + 1) * kBlockN - 1) / kBlockM + 1; - CUTLASS_PRAGMA_NO_UNROLL - for (; m_block < std::min(m_block_max, m_block_masking_max); - ++m_block) { - bwd_step(m_block, mask_fn); - } - } - - auto mask_fn = [&](auto& tSrS, int m_block) { - mask.template apply< - true /*Seqlenq_mask*/, - true /*Seqlenk_mask*/, - Causal && !SeparateMaskingIterations, - Local && !SeparateMaskingIterations, - Contexual_mask, - Has_targets /*Target_mask*/, - false /*Cross*/, - true /*Softmax*/>(tSrS, m_block, n_block); - }; - if constexpr (SeparateMaskingIterations) { - int const m_block_max_before_local_mask = - !Local || !SeparateMaskingIterations - ? m_block_max - : std::min( - m_block_max, (n_block * kBlockN + max_attn_len_) / kBlockM); - CUTLASS_PRAGMA_NO_UNROLL - for (; m_block < m_block_max_before_local_mask; ++m_block) { - bwd_step(m_block, mask_fn); - } - } else { - int num_m_block = m_block_max - m_block_min; - CUTLASS_PRAGMA_NO_UNROLL - for (int i = 0; i < num_m_block + full_m_block_max - - full_m_block_min + contexual_m_block_max; - ++i) { - if (i < num_m_block) { - m_block = m_block_min + i; - } else if (i < num_m_block + contexual_m_block_max) { - m_block = i - num_m_block; - } else { - m_block = - i - num_m_block - contexual_m_block_max + full_m_block_min; - } - bwd_step(m_block, mask_fn); - } - } - - if constexpr (Local && SeparateMaskingIterations) { - auto mask_fn = [&](auto& tSrS, int m_block) { - mask.template apply< - true /*Seqlenq_mask*/, - true /*Seqlenk_mask*/, - false /*Causal_mask*/, - Local, - Contexual_mask, - Has_targets /*Target_mask*/, - false /*Cross*/, - true /*Softmax*/>(tSrS, m_block, n_block); - }; - CUTLASS_PRAGMA_NO_UNROLL - for (; m_block < m_block_max; ++m_block) { - bwd_step(m_block, mask_fn); - } - } - if constexpr (Contexual_mask && SeparateMaskingIterations) { - auto mask_fn = [&](auto& tSrS, int m_block) { - mask.template apply< - true /*Seqlenq_mask*/, - true /*Seqlenk_mask*/, - Causal /*Causal_mask*/, - Local /*Local_mask*/, - Contexual_mask, - Has_targets, - false /*Cross*/, - true /*Softmax*/>(tSrS, m_block, n_block); - }; - CUTLASS_PRAGMA_NO_UNROLL - for (m_block = 0; m_block < contexual_m_block_max; ++m_block) { - bwd_step(m_block, mask_fn); - } - } - - if constexpr (Local && SeparateMaskingIterations) { - auto mask_fn = [&](auto& tSrS, int m_block) { - mask.template apply< - true /*Seqlenq_mask*/, - true /*Seqlenk_mask*/, - false /*Causal_mask*/, - Local, - Contexual_mask, - Has_targets, - false /*Cross*/, - true /*Softmax*/>(tSrS, m_block, n_block); - }; - CUTLASS_PRAGMA_NO_UNROLL - for (m_block = full_m_block_min; m_block < full_m_block_max; - ++m_block) { - bwd_step(m_block, mask_fn); - } - } - if constexpr (Q_dO_same_stages) { - smem_pipe_read_do = smem_pipe_read; - } - ++work_idx; - return true; - } - } - // We have separate iterations with causal masking. Not necessary for hdim - // 128 but for hdim 64 this helps quite a bit to not have to do causal - // masking for most of the iterations. - if constexpr ((Causal || Local) && SeparateMaskingIterations) { - auto mask_fn = [&](auto& tSrS, int m_block) { - mask.template apply< - true /*Seqlenq_mask*/, - true /*Seqlenk_mask*/, - Causal, - Local, - Contexual_mask, - false /*Target_mask*/, - false /*Cross*/, - true /*Softmax*/>(tSrS, m_block, n_block); - }; - static constexpr int kBlockM = get<0>(TileShape_MNK{}); - int const m_block_masking_max = - ((n_block + 1) * kBlockN - 1) / kBlockM + 1; - CUTLASS_PRAGMA_NO_UNROLL - for (; m_block < std::min(m_block_max, m_block_masking_max); ++m_block) { - bwd_step(m_block, mask_fn); - } - } - - auto mask_fn = [&](auto& tSrS, int m_block) { - mask.template apply< - true /*Seqlenq_mask*/, - true /*Seqlenk_mask*/, - Causal && !SeparateMaskingIterations, - Local && !SeparateMaskingIterations, - Contexual_mask, - false /*Target_mask*/, - false /*Cross*/, - true /*Softmax*/>(tSrS, m_block, n_block); - }; - if constexpr (SeparateMaskingIterations) { - int const m_block_max_before_local_mask = - !Local || !SeparateMaskingIterations - ? m_block_max - : std::min( - m_block_max, (n_block * kBlockN + max_attn_len_) / kBlockM); - CUTLASS_PRAGMA_NO_UNROLL - for (; m_block < m_block_max_before_local_mask; ++m_block) { - bwd_step(m_block, mask_fn); - } - } else { - int num_m_block = m_block_max - m_block_min; - CUTLASS_PRAGMA_NO_UNROLL - for (int i = 0; i < num_m_block + full_m_block_max - full_m_block_min + - contexual_m_block_max; - ++i) { - if (i < num_m_block) { - m_block = m_block_min + i; - } else if (i < num_m_block + contexual_m_block_max) { - m_block = i - num_m_block; - } else { - m_block = i - num_m_block - contexual_m_block_max + full_m_block_min; - } - bwd_step(m_block, mask_fn); - } - } - - if constexpr (Local && SeparateMaskingIterations) { - auto mask_fn = [&](auto& tSrS, int m_block) { - mask.template apply< - true /*Seqlenq_mask*/, - true /*Seqlenk_mask*/, - false /*Causal_mask*/, - Local, - Contexual_mask, - false /*Target_mask*/, - false /*Cross*/, - true /*Softmax*/>(tSrS, m_block, n_block); - }; - CUTLASS_PRAGMA_NO_UNROLL - for (; m_block < m_block_max; ++m_block) { - bwd_step(m_block, mask_fn); - } - } - if constexpr (Contexual_mask && SeparateMaskingIterations) { - auto mask_fn = [&](auto& tSrS, int m_block) { - mask.template apply< - true /*Seqlenq_mask*/, - true /*Seqlenk_mask*/, - Causal /*Causal_mask*/, - Local /*Local_mask*/, - Contexual_mask, - false /*Target_mask*/, - false /*Cross*/, - true /*Softmax*/>(tSrS, m_block, n_block); - }; - CUTLASS_PRAGMA_NO_UNROLL - for (m_block = 0; m_block < contexual_m_block_max; ++m_block) { - bwd_step(m_block, mask_fn); - } - } - - if constexpr (Local && SeparateMaskingIterations) { - auto mask_fn = [&](auto& tSrS, int m_block) { - mask.template apply< - true /*Seqlenq_mask*/, - true /*Seqlenk_mask*/, - false /*Causal_mask*/, - Local, - Contexual_mask, - false /*Target_mask*/, - false /*Cross*/, - true /*Softmax*/>(tSrS, m_block, n_block); - }; - CUTLASS_PRAGMA_NO_UNROLL - for (m_block = full_m_block_min; m_block < full_m_block_max; ++m_block) { - bwd_step(m_block, mask_fn); - } - } - - // if (blockIdx.x == 0 && threadIdx.x == 128) { print_tensor(tdVrdV); } - if constexpr (Q_dO_same_stages) { - smem_pipe_read_do = smem_pipe_read; - } - ++work_idx; - return true; - } -}; - -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/mainloop_fwd_sm90_tma_gmma_ws.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/mainloop_fwd_sm90_tma_gmma_ws.h deleted file mode 100644 index 7c8a447af..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/mainloop_fwd_sm90_tma_gmma_ws.h +++ /dev/null @@ -1,2180 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/****************************************************************************** - * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, - *Pradeep Ramani, Tri Dao. - ******************************************************************************/ - -#pragma once - -#include -#include -#include -#include -#include "cutlass/pipeline/pipeline.hpp" - -#include "cute/tensor.hpp" - -#include "cutlass/gemm/collective/builders/sm90_common.inl" - -#include "mask.h" -#include "named_barrier.h" -#include "seqlen.h" -#include "sm90_pipeline_no_cluster.h" -#include "utils.h" - -namespace hstu { - -using namespace cute; - -template < - int Stages, - class ClusterShape_, - class TileShape_MNK_, - class Element_, - class ElementAccum_, - class ArchTag_, - bool Causal, - bool Local, - bool Contexual_mask, - bool Jagged, - bool Has_targets, - bool Mma1_is_RS, - bool V_colmajor_, - bool Cross> -struct CollectiveMainloopFwdSm90 { - static constexpr int kStages = Stages; - using ClusterShape = ClusterShape_; - using TileShape_MNK = TileShape_MNK_; - using Element = Element_; - using ElementAccum = ElementAccum_; - using ArchTag = ArchTag_; - static constexpr bool Is_FP8 = - cute::is_same_v || - cute::is_same_v; - ; - static constexpr bool V_colmajor = V_colmajor_; - static constexpr bool Transpose_V = Is_FP8 && !V_colmajor; - using SeqlenInfo_t = hstu::SeqlenInfoQKFwd; - - static_assert(ArchTag::kMinComputeCapability >= 90); - - static constexpr cute::GMMA::Major MmaMajorV = - !Is_FP8 && !V_colmajor ? GMMA::Major::MN : GMMA::Major::K; - static constexpr cute::GMMA::Major TmaMajorV = - !V_colmajor ? GMMA::Major::MN : GMMA::Major::K; - - static constexpr int kBlockM = get<0>(TileShape_MNK{}); - static constexpr int kBlockN = get<1>(TileShape_MNK{}); - static constexpr int kHeadDim = get<2>(TileShape_MNK{}); - - // Register bandwidth is actually a bottleneck so we don't want Q to be in - // registers. Leaving this option here for reference. - static constexpr bool Mma0_is_RS = false; - // We can have Mma1 (P @ V) with P in smem in rmem to reduce register pressure - // at the cost of more smem. - static_assert(!(!Mma1_is_RS && Is_FP8), "Mma1 must be RS if FP8"); - static_assert( - !(!Mma1_is_RS && Transpose_V), - "Mma1 must be RS if Transpose_V"); - - using AtomLayoutMNK = Layout, _1, _1>>; - using TiledMma0 = decltype(cute::make_tiled_mma( - std::conditional_t< - !Mma0_is_RS, - decltype(cute::GMMA::ss_op_selector< - Element, - Element, - ElementAccum, - TileShape_MNK>()), - decltype(cute::GMMA::rs_op_selector< - Element, - Element, - ElementAccum, - TileShape_MNK>())>{}, - AtomLayoutMNK{})); - using TiledMma1 = decltype(cute::make_tiled_mma( - std::conditional_t< - !Mma1_is_RS, - decltype(cute::GMMA::ss_op_selector< - Element, - Element, - ElementAccum, - decltype(select<0, 2, 1>(TileShape_MNK{})), - GMMA::Major::K, - MmaMajorV>()), - decltype(cute::GMMA::rs_op_selector< - Element, - Element, - ElementAccum, - decltype(select<0, 2, 1>(TileShape_MNK{})), - GMMA::Major::K, - MmaMajorV>())>{}, - AtomLayoutMNK{})); - - static constexpr int NumMmaThreads = size(TiledMma0{}); - static constexpr int NumProducerThreads = !Transpose_V - ? cutlass::NumThreadsPerWarp - : cutlass::NumThreadsPerWarpGroup; - static_assert(NumMmaThreads % cutlass::NumThreadsPerWarpGroup == 0); - static constexpr int NumMmaWarpGroups = - NumMmaThreads / cutlass::NumThreadsPerWarpGroup; - static_assert( - NumMmaWarpGroups == 1 || NumMmaWarpGroups == 2 || NumMmaWarpGroups == 3); - - using SmemLayoutAtomQ = - decltype(cutlass::gemm::collective::detail::ss_smem_selector< - GMMA::Major::K, - Element, - decltype(cute::get<0>(TileShape_MNK{})), - decltype(cute::get<2>(TileShape_MNK{}))>()); - using SmemLayoutQ = - decltype(tile_to_shape(SmemLayoutAtomQ{}, select<0, 2>(TileShape_MNK{}))); - - using SmemLayoutAtomK = - decltype(cutlass::gemm::collective::detail::ss_smem_selector< - GMMA::Major::K, - Element, - decltype(cute::get<1>(TileShape_MNK{})), - decltype(cute::get<2>(TileShape_MNK{}))>()); - using SmemLayoutK = decltype(tile_to_shape( - SmemLayoutAtomK{}, - make_shape( - shape<1>(TileShape_MNK{}), - shape<2>(TileShape_MNK{}), - Int{}))); - - using SmemLayoutAtomVt = - decltype(cutlass::gemm::collective::detail::ss_smem_selector< - TmaMajorV, - Element, - decltype(cute::get<2>(TileShape_MNK{})), - decltype(cute::get<1>(TileShape_MNK{}))>()); - using SmemLayoutVt = decltype(tile_to_shape( - SmemLayoutAtomVt{}, - make_shape( - shape<2>(TileShape_MNK{}), - shape<1>(TileShape_MNK{}), - Int{}), - std::conditional_t< - TmaMajorV == GMMA::Major::K, - cute::Step<_1, _2, _3>, - cute::Step<_2, _1, _3>>{})); - - using SmemLayoutAtomVtMma = - decltype(cutlass::gemm::collective::detail::ss_smem_selector< - MmaMajorV, - Element, - decltype(cute::get<2>(TileShape_MNK{})), - decltype(cute::get<1>(TileShape_MNK{}))>()); - using SmemLayoutVtMma = decltype(tile_to_shape( - SmemLayoutAtomVtMma{}, - make_shape( - shape<2>(TileShape_MNK{}), - shape<1>(TileShape_MNK{}), - Int{}), - std::conditional_t< - MmaMajorV == GMMA::Major::K, - cute::Step<_1, _2, _3>, - cute::Step<_2, _1, _3>>{})); - - // Only used if we're using cp.async to load V - using SmemLayoutAtomVCpAsync = - decltype(cutlass::gemm::collective::detail::ss_smem_selector< - GMMA::Major::K, - Element, - decltype(cute::get<1>(TileShape_MNK{})), - decltype(cute::get<2>(TileShape_MNK{}))>()); - using SmemLayoutVCpAsync = decltype(tile_to_shape( - SmemLayoutAtomVCpAsync{}, - make_shape( - shape<1>(TileShape_MNK{}), - shape<2>(TileShape_MNK{}), - Int{}))); - - using SmemLayoutAtomP = - decltype(cutlass::gemm::collective::detail::ss_smem_selector< - GMMA::Major::K, - Element, - decltype(cute::get<0>(TileShape_MNK{})), - decltype(cute::get<1>(TileShape_MNK{}))>()); - using SmemLayoutP = - decltype(tile_to_shape(SmemLayoutAtomP{}, select<0, 1>(TileShape_MNK{}))); - - using SmemCopyAtomP = Copy_Atom; - - // Use LDSM.T and STSM to transpose V in the case of FP8 and V being - // row-major. For FP16/BF16 we don't do any transposing. - static_assert(!Transpose_V || (kHeadDim % 32 == 0 && kBlockN % 32 == 0)); - static constexpr bool kHeadDim_multiple_64 = kHeadDim % 64 == 0; - // Either kHeadDim is a multiple of 64 (in which case we use a block size of - // 64 x 32 for the transpose), or we need kBlockN to be a multiple of 64 (in - // which case we use a block size of 32 x 64 for the transpose). - static_assert(!Transpose_V || (kHeadDim_multiple_64 || kBlockN % 64 == 0)); - using LDSM_thread_shape = std::conditional_t< - kHeadDim_multiple_64, - Shape<_32, _4, _1, _1>, - Shape<_16, _4, _1, _2>>; - using LDSM_thread_stride = std::conditional_t< - kHeadDim_multiple_64, - Stride<_4, _1, _0, _0>, - Stride<_4, _1, _0, _64>>; - using LDSM_value_shape = Shape<_2, _2, _1, _4>; - using LDSM_value_stride = Stride<_1, _2, _16, _4>; - using LDSM_divide_shape = - std::conditional_t, Shape<_32, _8>>; - using S2RTiledCopyVt = decltype(make_tiled_copy( - Copy_Atom{}, - Layout{}, - Layout{})); - - using STSM_thread_shape = std::conditional_t< - kHeadDim_multiple_64, - Shape<_8, _4, _4, _1>, - Shape<_8, _4, _2, _2>>; - using STSM_thread_stride = std::conditional_t< - kHeadDim_multiple_64, - Stride<_4, _1, _32, _0>, - Stride<_4, _1, _32, _64>>; - using STSM_value_shape = Shape<_1, _4, _2, _2>; - using STSM_value_stride = Stride<_0, _1, _4, _8>; - using STSM_divide_shape = Shape<_8, _16>; - // These will not permute the columns of V (the kHeadDim dimension) but incur - // bank conflicts so a little slower (e.g. 1150 TFLOPS for hdim 256 instead of - // 1200 TFLOPS). Instead we will permute the cols of V, and un-permute the - // cols of O in the epilogue. using STSM_value_shape = Shape<_2, _4, _1, _2>; - // using STSM_value_stride = Stride<_4, _1, _0, _8>; - // using STSM_divide_shape = Shape<_16, _16>; - using R2STiledCopyV = decltype(make_tiled_copy( - Copy_Atom{}, - Layout{}, - Layout{})); - - using GmemTiledCopyQ = cute::SM90_TMA_LOAD; - using GmemTiledCopyKV = - decltype(cutlass::gemm::collective::detail:: - sm90_cluster_shape_to_tma_atom(shape<0>(ClusterShape{}))); - - // We use CpAsync for K and V if PagedKV and AppendKV, since TMA doesn't work - // there - static constexpr int kGmemElemsPerLoad = - sizeof(cute::uint128_t) / sizeof(Element); - static_assert( - kHeadDim % kGmemElemsPerLoad == 0, - "Headdim must be a multiple of kGmemElemsPerLoad"); - // We want each "row" to have 64 elements (128 bytes, i.e. 1 cache line). E.g. - // if hdim=128, we want each thread to have 4 loads in the M direction and 2 - // vectorized load in the K direction. We want each thread to have at least 2 - // loads in the K direction since in the case of non-interleaved rotary - // (combining elements at indices 0 and rotary_dim/2, 1 and rotary_dim/2+1, - // etc), each thread will load twice from the same row. - static constexpr int kBytePerHalfRow = kHeadDim / 2 * sizeof(Element); - static constexpr int kBlockKGmem = - (kBytePerHalfRow % 128 == 0 ? 128 - : (kBytePerHalfRow % 64 == 0 ? 64 : 32)) / - sizeof(Element); - static constexpr int kGmemThreadsPerRow = kBlockKGmem / kGmemElemsPerLoad; - static_assert( - NumMmaThreads % kGmemThreadsPerRow == 0, - "NumMmaThreads must be a multiple of kGmemThreadsPerRow"); - // We assume threads loading the same row are in the same warp. This is for an - // optimization in PagedKV where these threads share the same page table entry - // and share the work of computing pointers to paged K and paged V. - static_assert( - cutlass::NumThreadsPerWarp % kGmemThreadsPerRow == 0, - "kGmemThreadsPerRow must divide NumThreadsPerWarp"); - using GmemLayoutAtom = Layout< - Shape, Int>, - Stride, _1>>; - // If AppendKV, we'll be loading Q for rotary, and we assume divisibility to - // avoid predication - static_assert( - kBlockM % CUTE_STATIC_V(shape<0>(GmemLayoutAtom{})) == 0, - "kBlockM must be a multiple of NumMmaThreads / kGmemThreadsPerRow"); - - using ShapeQKV = - cute::Shape; // (seqlen, d, head, - // batch) - using StrideQK = cute::Stride; - using StrideV = std::conditional_t< - !V_colmajor, - StrideQK, - cute::Stride<_1, int64_t, int64_t, int64_t>>; - // ((qhead_per_khead, seqlen), d, nheads_kv, batch, num_splits) - using ShapeQPacked = ShapeQKV; - using StrideQPacked = StrideQK; - using StrideDescale = cute::Stride; - - using TMA_Q = decltype(make_tma_copy_A_sm90( - GmemTiledCopyQ{}, - make_tensor( - make_gmem_ptr(static_cast(nullptr)), - ShapeQKV{}, - StrideQK{}), - SmemLayoutQ{}, - TileShape_MNK{}, - ClusterShape{})); - - using TMA_K = decltype(make_tma_copy_B_sm90( - GmemTiledCopyKV{}, - make_tensor( - make_gmem_ptr(static_cast(nullptr)), - ShapeQKV{}, - StrideQK{}), - take<0, 2>(SmemLayoutK{}), - TileShape_MNK{}, - ClusterShape{})); // mcast along M mode for this N load, if any - - using TMA_V = decltype(make_tma_copy( - GmemTiledCopyKV{}, - make_tensor( - make_gmem_ptr(static_cast(nullptr)), - ShapeQKV{}, - select<1, 0, 2, 3>(StrideV{})), - take<0, 2>(SmemLayoutVt{}), - select<2, 1>(TileShape_MNK{}), - size<0>(ClusterShape{}))); // mcast along M mode for this N load, if any - - // Set the bytes transferred in this TMA transaction (may involve multiple - // issues) - static constexpr uint32_t TmaTransactionBytesQ = static_cast( - size(SmemLayoutQ{}) * cutlass::sizeof_bits_v / 8); - static constexpr uint32_t TmaTransactionBytesK = static_cast( - size(take<0, 2>(SmemLayoutK{})) * cutlass::sizeof_bits_v / 8); - static constexpr uint32_t TmaTransactionBytesV = static_cast( - size(take<0, 2>(SmemLayoutVt{})) * cutlass::sizeof_bits_v / 8); - static_assert(TmaTransactionBytesK == TmaTransactionBytesV); - - using PipelineTmaAsync = std::conditional_t< - CUTE_STATIC_V(size(ClusterShape{})) == 1, - typename cutlass::PipelineTmaAsyncNoCluster, - typename cutlass::PipelineTmaAsync>; - using MainloopPipelineK = PipelineTmaAsync; - using MainloopPipelineV = std::conditional_t< - !Transpose_V, - PipelineTmaAsync, - typename cutlass::PipelineAsync>; - using MainloopPipelineVt = PipelineTmaAsync; - // We always use TMA for K_new and V_new - using MainloopPipelineKVNew = PipelineTmaAsync; - using PipelineState = cutlass::PipelineState; - - // If PackGQA, we use cp.async (instead of TMA) to load Q, so we want smem_q - // to be aligned and have sQ being position_independent_swizzle_tensor. If - // !Use_TMA_KV, we use cp.async (instead of TMA) to load K & V, so we want - // smem_k and smem_v to be aligned. - static constexpr size_t SmemAlignmentQ = - !Mma0_is_RS ? 128 : cutlass::detail::alignment_for_swizzle(SmemLayoutQ{}); - static constexpr size_t SmemAlignmentK = 128; - static constexpr size_t SmemAlignmentVtNoTranspose = - cutlass::detail::alignment_for_swizzle(SmemLayoutVt{}); - static_assert( - SmemAlignmentQ >= 128 and SmemAlignmentK >= 128 && - SmemAlignmentVtNoTranspose >= 128, - "Require at least 128B alignment"); - static constexpr size_t SmemAlignmentP = - cutlass::detail::alignment_for_swizzle(SmemLayoutP{}); - static_assert(SmemAlignmentP >= 128, "Require at least 128B alignment"); - - using SmemP_t = std::conditional_t< - Mma1_is_RS, - cute::array, - cute:: - array_aligned, SmemAlignmentP>>; - // Sometimes even with SmemP_t = cute::array, putting it in the - // TensorStorage struct causes smem size to go from 227KB to 228KB and we get - // "invalid argument". - - struct TensorStorageWithoutPNoTranspose : cute::aligned_struct { - cute::array_aligned< - Element, - cute::cosize_v, - SmemAlignmentVtNoTranspose> - smem_v; - cute::array_aligned, SmemAlignmentQ> - smem_q; - cute::array_aligned, SmemAlignmentK> - smem_k; - }; - - struct TensorStorageWithPNoTranspose : cute::aligned_struct { - cute::array_aligned< - Element, - cute::cosize_v, - SmemAlignmentVtNoTranspose> - smem_v; - cute::array_aligned, SmemAlignmentQ> - smem_q; - cute::array_aligned, SmemAlignmentK> - smem_k; - SmemP_t smem_p; - }; - - using TensorStorageNoTranspose = std::conditional_t< - Mma1_is_RS, - TensorStorageWithoutPNoTranspose, - TensorStorageWithPNoTranspose>; - - static constexpr size_t SmemAlignmentVt = - cutlass::detail::alignment_for_swizzle(SmemLayoutVt{}); - static constexpr size_t SmemAlignmentV = - cutlass::detail::alignment_for_swizzle(SmemLayoutVtMma{}); - static_assert( - SmemAlignmentVt >= 128 and SmemAlignmentV >= 128, - "Require at least 128B alignment"); - struct TensorStorageTransposeV - : cute::aligned_struct< - cute::max(SmemAlignmentQ, SmemAlignmentK, SmemAlignmentV)> { - cute:: - array_aligned, SmemAlignmentV> - smem_v; - cute::array_aligned, SmemAlignmentVt> - smem_vt; - cute::array_aligned, SmemAlignmentQ> - smem_q; - cute::array_aligned, SmemAlignmentK> - smem_k; - }; - - using TensorStorage = std::conditional_t< - !Transpose_V, - TensorStorageNoTranspose, - TensorStorageTransposeV>; - - // These are tuned for speed. They don't affect correctness. - static constexpr bool UseSchedulerBarrier = - (NumMmaWarpGroups >= 2) && (!Is_FP8 ? kHeadDim <= 128 : kHeadDim >= 128); - static constexpr bool RescaleOBeforeGemm = - kHeadDim > 128 && (!Is_FP8 || V_colmajor); - - // Host side kernel arguments - struct Arguments { - Element const* const ptr_Q; - ShapeQKV const shape_Q; - StrideQK const stride_Q; - Element* const - ptr_K; // Not Element const* since we might append to KV cache in-place - ShapeQKV const shape_K; - StrideQK const stride_K; - Element* const ptr_V; - StrideV const stride_V; - float const *ptr_q_descale, *ptr_k_descale, *ptr_v_descale; - StrideDescale const stride_q_descale, stride_k_descale, stride_v_descale; - float const max_seq_len_inv; - float const alpha; - int const max_attn_len; - int const min_full_attn_seq_len; - int const contextual_seq_len; - int const num_softmax_heads; - int const num_groups; - int const batch_size_per_group; - int const* const seq_offsets = nullptr; - int const* const seq_offsets_q = nullptr; - int const* const num_targets = nullptr; - int const* const max_seq_len_tensor = nullptr; - int const* const contextual_seq_len_tensor = nullptr; - int const* const max_attn_len_tensor = nullptr; - int const* const min_full_attn_seq_len_tensor = nullptr; - float const* const attn_scale = nullptr; - bool const scalar_scale = true; - }; - - // Device side kernel params - struct Params { - Element const* const ptr_Q; - ShapeQKV const shape_Q; - StrideQK const stride_Q; - ShapeQPacked const shape_Q_packed; - StrideQPacked const stride_Q_packed; - Element* const ptr_K; - ShapeQKV const shape_K; - StrideQK const stride_K; - Element* const ptr_V; - StrideV const stride_V; - TMA_Q tma_load_Q; - TMA_K tma_load_K; - TMA_V tma_load_V; - float const *ptr_q_descale, *ptr_k_descale, *ptr_v_descale; - StrideDescale const stride_q_descale, stride_k_descale, stride_v_descale; - float const max_seq_len_inv; - float const alpha; - float const alpha_log2; - int const max_attn_len; - int const min_full_attn_seq_len; - int const contextual_seq_len; - int const num_softmax_heads; - int const num_groups; - int const batch_size_per_group; - int const* const seq_offsets = nullptr; - int const* const seq_offsets_q = nullptr; - int const* const num_targets = nullptr; - int const* const max_seq_len_tensor = nullptr; - int const* const contextual_seq_len_tensor = nullptr; - int const* const max_attn_len_tensor = nullptr; - int const* const min_full_attn_seq_len_tensor = nullptr; - float const* const attn_scale = nullptr; - bool const scalar_scale = true; - }; - - static Params to_underlying_arguments(Arguments const& args) { - Tensor mQ = - make_tensor(make_gmem_ptr(args.ptr_Q), args.shape_Q, args.stride_Q); - TMA_Q tma_load_Q = make_tma_copy_A_sm90( - GmemTiledCopyQ{}, - mQ, - SmemLayoutQ{}, - TileShape_MNK{}, - ClusterShape{}); // no mcast for Q - Tensor mK = - make_tensor(make_gmem_ptr(args.ptr_K), args.shape_K, args.stride_K); - TMA_K tma_load_K = make_tma_copy_B_sm90( - GmemTiledCopyKV{}, - mK, - take<0, 2>(SmemLayoutK{}), - TileShape_MNK{}, - ClusterShape{}); // mcast along M mode for this N load, if any - Tensor mV = make_tensor( - make_gmem_ptr(args.ptr_V), - select<1, 0, 2, 3>(args.shape_K), - select<1, 0, 2, 3>(args.stride_V)); - TMA_V tma_load_V = make_tma_copy( - GmemTiledCopyKV{}, - mV, - take<0, 2>(SmemLayoutVt{}), - select<2, 1>(TileShape_MNK{}), - size<0>(ClusterShape{})); // mcast along M mode for this N load, if any - auto const shape_Q_packed = cute::conditional_return( - args.shape_Q, - make_shape( - make_shape(1, get<0>(args.shape_Q)), - get<1>(args.shape_Q), - get<2>(args.shape_K), - get<3>(args.shape_Q))); - auto const stride_Q_packed = cute::conditional_return( - args.stride_Q, - make_stride( - make_stride(get<2>(args.stride_Q), get<0>(args.stride_Q)), - get<1>(args.stride_Q), - get<2>(args.stride_Q), - get<3>(args.stride_Q))); - return { - args.ptr_Q, - args.shape_Q, - args.stride_Q, - shape_Q_packed, - stride_Q_packed, - args.ptr_K, - args.shape_K, - args.stride_K, - args.ptr_V, - args.stride_V, - tma_load_Q, - tma_load_K, - tma_load_V, - args.ptr_q_descale, - args.ptr_k_descale, - args.ptr_v_descale, - args.stride_q_descale, - args.stride_k_descale, - args.stride_v_descale, - args.max_seq_len_inv, - args.alpha, - float(args.alpha * M_LOG2E), - args.max_attn_len, - args.min_full_attn_seq_len, - args.contextual_seq_len, - args.num_softmax_heads, - args.num_groups, - args.batch_size_per_group, - args.seq_offsets, - args.seq_offsets_q, - args.num_targets, - args.max_seq_len_tensor, - args.contextual_seq_len_tensor, - args.max_attn_len_tensor, - args.min_full_attn_seq_len_tensor, - args.attn_scale, - args.scalar_scale}; - } - - /// Issue Tma Descriptor Prefetch -- ideally from a single thread for best - /// performance - CUTLASS_DEVICE - static void prefetch_tma_descriptors(Params const& params) { - cute::prefetch_tma_descriptor(params.tma_load_Q.get_tma_descriptor()); - cute::prefetch_tma_descriptor(params.tma_load_K.get_tma_descriptor()); - cute::prefetch_tma_descriptor(params.tma_load_V.get_tma_descriptor()); - } - - CUTLASS_DEVICE - cute::tuple get_n_block_min_max( - int max_attn_len, - int min_full_attn_seq_len, - int contextual_seq_len, - int uihlen, - int m_block) { - static constexpr int kBlockM = get<0>(TileShape_MNK{}); - static constexpr int kBlockN = get<1>(TileShape_MNK{}); - if constexpr (Contexual_mask) { - if (m_block * kBlockM < contextual_seq_len) { - return {0, cute::ceil_div(uihlen, kBlockN)}; - } - } - if constexpr (Has_targets) { - int m_idx_max = (m_block + 1) * kBlockM; - if (m_idx_max > uihlen) { - return {0, cute::ceil_div(uihlen, kBlockN)}; - } - } - int n_block_max; - int n_block_min; - // Non-target part, n_block_max - if constexpr (Causal || Local) { - int m_idx_max = (m_block + 1) * kBlockM; - n_block_max = cute::ceil_div(std::min(m_idx_max, uihlen), kBlockN); - } else { - n_block_max = cute::ceil_div(uihlen, kBlockN); - } - // Non-target part, n_block_min - if constexpr (Local) { - int m_idx_min = m_block * kBlockM; - int m_idx_max = (m_block + 1) * kBlockM; - if (min_full_attn_seq_len == 0 || - m_idx_max <= uihlen - min_full_attn_seq_len) { - n_block_min = std::max(int(0), (m_idx_min - max_attn_len) / kBlockN); - if constexpr (Contexual_mask) { - // row contexual without sink - if (n_block_min * kBlockN < contextual_seq_len) { - n_block_min = 0; - } - } - } else { - n_block_min = 0; - } - } else { - n_block_min = 0; - } - return {n_block_min, n_block_max}; - } - - CUTLASS_DEVICE - cute::tuple get_target_n_block_min_max( - int n_block_max, - int uihlen, - int seqlen, - int m_block) { - static constexpr int kBlockM = get<0>(TileShape_MNK{}); - static constexpr int kBlockN = get<1>(TileShape_MNK{}); - int m_idx_max = (m_block + 1) * kBlockM; - if (m_idx_max <= uihlen) { // Non-target part - return {n_block_max, n_block_max}; - } else { // Target part - int m_idx_min = m_block * kBlockM; - return { - std::max(n_block_max, m_idx_min / kBlockN), - cute::ceil_div(std::min(m_idx_max, seqlen), kBlockN)}; - } - } - - CUTLASS_DEVICE - int get_contexual_n_block_max( - int n_block_min, - int min_full_attn_seq_len, - int contextual_seq_len, - int uihlen, - int m_block) { - return 0; - // TODO: reenable below once contexual + semi local implementation is - // finalized - static constexpr int kBlockM = get<0>(TileShape_MNK{}); - static constexpr int kBlockN = get<1>(TileShape_MNK{}); - if constexpr (!Local) { - return 0; - } - if (m_block * kBlockM < contextual_seq_len) { - return 0; - } - int m_idx_max = (m_block + 1) * kBlockM; - if constexpr (Has_targets) { - if (m_idx_max > uihlen) { - return 0; - } - } - if (min_full_attn_seq_len == 0 || - m_idx_max <= uihlen - min_full_attn_seq_len) { - return std::min(n_block_min, cute::ceil_div(contextual_seq_len, kBlockN)); - } - return 0; - } - - CUTLASS_DEVICE - cute::tuple get_cross_n_block_min_max( - int const uihlen_q, - int const seqlen_q, - int const seqlen_kv, - int const m_block) { - static constexpr int kBlockM = get<0>(TileShape_MNK{}); - static constexpr int kBlockN = get<1>(TileShape_MNK{}); - if constexpr (!Causal) { - return {0, cute::ceil_div(seqlen_kv, kBlockN)}; - } - int n_block_max = - std::min(seqlen_kv, (m_block + 1) * kBlockM + seqlen_kv - uihlen_q); - return {0, cute::ceil_div(n_block_max, kBlockN)}; - } - - template - CUTLASS_DEVICE void load( - Params const& params, - MainloopPipelineK pipeline_k, - MainloopPipelineV pipeline_v, - MainloopPipelineVt pipeline_vt, - PipelineState& smem_pipe_write, - SharedStorage& shared_storage, - SchedulerPrefetch const& scheduler_prefetch, - SeqlenInfo_t const& seqlen_info, - cute::tuple block_coord, - int& work_idx) { - auto [m_block, bidh, bidb, split_idx] = block_coord; - if constexpr (Jagged) { - static constexpr int kBlockM = get<0>(TileShape_MNK{}); - if (m_block * kBlockM >= seqlen_info.seqlen_q) { - scheduler_prefetch(); - return; - } - } - int min_full_attn_seq_len_, max_attn_len_, contextual_seq_len_; - if constexpr (!Cross) { - if (params.num_groups > 1) { - int group_id = bidb / params.batch_size_per_group; - min_full_attn_seq_len_ = params.min_full_attn_seq_len_tensor[group_id]; - max_attn_len_ = params.max_attn_len_tensor[group_id]; - contextual_seq_len_ = params.contextual_seq_len_tensor[group_id]; - } else { - min_full_attn_seq_len_ = params.min_full_attn_seq_len; - max_attn_len_ = params.max_attn_len; - contextual_seq_len_ = params.contextual_seq_len; - } - } - int n_block_min, n_block_max; - if constexpr (Cross) { - auto n_block_min_max = get_cross_n_block_min_max( - seqlen_info.uihlen_q, - seqlen_info.seqlen_q, - seqlen_info.seqlen_kv, - m_block); - n_block_min = get<0>(n_block_min_max); - n_block_max = get<1>(n_block_min_max); - } else { - auto n_block_min_max = get_n_block_min_max( - max_attn_len_, - min_full_attn_seq_len_, - contextual_seq_len_, - seqlen_info.uihlen_q, - m_block); - n_block_min = get<0>(n_block_min_max); - n_block_max = get<1>(n_block_min_max); - } -#ifdef HSTU_FLASH_ATTN_DEBUG_INFO - if (n_block_max <= n_block_min) { - std::printf( - "mainloop_fwd_sm90: n_block_max <= n_block_min not expected."); - scheduler_prefetch(); - return; - } -#endif - - Tensor sQ = make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_q.data()), - SmemLayoutQ{}); - Tensor sK = make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_k.data()), - SmemLayoutK{}); - Tensor sK_pi = as_position_independent_swizzle_tensor(sK); - // as_position_independent_swizzle_tensor makes address calculation easier - // when we do LDSM & STSM to transpose. But it requires smem_vt and smem_v - // to be aligned to e.g 512 bytes. - Tensor sVt = [&] { - if constexpr (!Transpose_V) { - return make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_v.data()), - SmemLayoutVt{}); - } else { - return cute::as_position_independent_swizzle_tensor(make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_vt.data()), - SmemLayoutVt{})); - } - }(); - // Only used if Transpose_V - Tensor sV = cute::as_position_independent_swizzle_tensor(make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_v.data()), - SmemLayoutVtMma{})); - - int const thread_idx = threadIdx.x % NumProducerThreads; - - // Prepare the TMA loads - uint32_t block_rank_in_cluster = cute::block_rank_in_cluster(); - constexpr uint32_t cluster_shape_x = get<0>(ClusterShape()); - uint2 cluster_local_block_id = { - block_rank_in_cluster % cluster_shape_x, - block_rank_in_cluster / cluster_shape_x}; - - Tensor mQ = params.tma_load_Q.get_tma_tensor(params.shape_Q)( - _, _, bidh, !Jagged ? bidb : 0); - Tensor mK_TMA = params.tma_load_K.get_tma_tensor(params.shape_K)( - _, _, bidh, !Jagged ? bidb : 0); - Tensor mVt_TMA = params.tma_load_V.get_tma_tensor( - select<1, 0, 2, 3>(params.shape_K))(_, _, bidh, !Jagged ? bidb : 0); - - Tensor gQ = local_tile( - domain_offset(make_coord(seqlen_info.offset_q, _0{}), mQ), - select<0, 2>(TileShape_MNK{}), - make_coord(m_block, _0{})); // (M, K) - // if (cute::thread0()) { printf("Jagged = %d, params.leftpad_k = %p, - // leftpad_k = %d\n", Jagged, params.leftpad_k, leftpad_k); } - Tensor gK_TMA = local_tile( - domain_offset(make_coord(seqlen_info.offset_k, _0{}), mK_TMA), - select<1, 2>(TileShape_MNK{}), - make_coord(_, _0{})); // (N, K, _) - Tensor gVt_TMA = local_tile( - domain_offset(make_coord(_0{}, seqlen_info.offset_k), mVt_TMA), - select<2, 1>(TileShape_MNK{}), - make_coord(_0{}, _)); // (K, N, _) - - auto block_tma_Q = params.tma_load_Q.get_slice(_0{}); - Tensor tQgQ = group_modes<0, 3>(block_tma_Q.partition_S(gQ)); // (TMA) - Tensor tQsQ = group_modes<0, 3>(block_tma_Q.partition_D(sQ)); // (TMA) - // tma_partition doesn't handle position_independent_swizzle_tensor - // correctly, so we need to do it manually - auto block_tma_K = params.tma_load_K.get_slice(cluster_local_block_id.x); - Tensor tKgK_TMA = - group_modes<0, 3>(block_tma_K.partition_S(gK_TMA)); // (TMA, k) - Tensor tKsK_TMA = - group_modes<0, 3>(block_tma_K.partition_D(sK)); // (TMA, PIPE) - auto block_tma_V = params.tma_load_V.get_slice(cluster_local_block_id.x); - Tensor tVgVt_TMA = - group_modes<0, 3>(block_tma_V.partition_S(gVt_TMA)); // (TMA, k) - Tensor tVsVt_TMA = - group_modes<0, 3>(block_tma_V.partition_D(sVt)); // (TMA, PIPE) - - // Set up for transposing V, only used if Transpose_V - S2RTiledCopyVt s2r_tiled_copy_vt; - R2STiledCopyV r2s_tiled_copy_v; - auto s2r_thr_copy_vt = s2r_tiled_copy_vt.get_thread_slice(thread_idx); - auto r2s_thr_copy_v = r2s_tiled_copy_v.get_thread_slice(thread_idx); - // flat_divide(sVt, LDSM_divide_shape{}): (64, 8, kHeadDim / 64, kBlockN / - // 8, kStages) - Tensor tTranssVt_ = s2r_thr_copy_vt.partition_S( - flat_divide(sVt, LDSM_divide_shape{})); // ((16, 1), 1, 1, kHeadDim / - // 64, kBlockN / 32, kStages) - // flat_divide(sV, STSM_divide_shape{}): (8, 16, kHeadDim / 8, (4, kBlockN - // / 64), kStages) - Tensor tTranssV_ = r2s_thr_copy_v.partition_D( - flat_divide(sV, STSM_divide_shape{})); // ((16, 1), 1, 1, kHeadDim / 64, - // (2, kBlockN / 64), kStages) - CUTE_STATIC_ASSERT_V(rank(tTranssVt_) == rank(tTranssV_)); - CUTE_STATIC_ASSERT_V(size<0>(tTranssVt_) == size<0>(tTranssV_)); - CUTE_STATIC_ASSERT_V(size<1>(tTranssVt_) == size<1>(tTranssV_)); - CUTE_STATIC_ASSERT_V(size<2>(tTranssVt_) == size<2>(tTranssV_)); - CUTE_STATIC_ASSERT_V(size<3>(tTranssVt_) == size<3>(tTranssV_)); - CUTE_STATIC_ASSERT_V(size<4>(tTranssVt_) == size<4>(tTranssV_)); - // Faster to have 2 LDSM.T, byte permute, STSM for better ILP - static constexpr int Transpose_ILP = - (size<2>(tTranssVt_) * size<3>(tTranssVt_)) % 2 == 0 ? 2 : 1; - Tensor tTranssVt = logical_divide( - group_modes<1, rank(tTranssVt_) - 1>(tTranssVt_), - Shape>{}); // ((16, 1), (2, kHeadDim / 64 - // * kBlockN / 32 / 2), - // kStages) - Tensor tTranssV = logical_divide( - group_modes<1, rank(tTranssV_) - 1>(tTranssV_), - Shape>{}); // ((16, 1), (2, kHeadDim / 64 - // * kBlockN / 32 / 2), - // kStages) - auto transpose_V = [&](int stage) { - if constexpr (Transpose_V) { -#pragma unroll - for (int i = 0; i < size<1, 1>(tTranssVt); ++i) { - Tensor tTransrV = - make_fragment_like(tTranssV(_, make_coord(_, _0{}), _0{})); - static_assert(size<0>(tTransrV) == 16); - Tensor tTransrV_64 = recast(tTransrV); - cute::copy( - s2r_tiled_copy_vt, - tTranssVt(_, make_coord(_, i), stage), - tTransrV); -#pragma unroll - for (int j = 0; j < size(tTransrV_64); ++j) { - uint32_t upper = tTransrV_64[j].x; - uint32_t lower = tTransrV_64[j].y; - tTransrV_64[j].x = __byte_perm(upper, lower, 0x6420); - tTransrV_64[j].y = __byte_perm(upper, lower, 0x7531); - } - cute::copy( - r2s_tiled_copy_v, tTransrV, tTranssV(_, make_coord(_, i), stage)); - } - } - }; - - uint16_t mcast_mask_kv = 0; - if constexpr (cute::is_same_v) { - auto block_layout = Layout{}; // (m,n) -> block_id - for (int m = 0; m < size<0>(block_layout); ++m) { - mcast_mask_kv |= - (uint16_t(1) << block_layout(m, cluster_local_block_id.y, _0{})); - } - } - - auto load_K = [&](int const n_block, auto const& smem_pipe_write) { - pipeline_k.producer_acquire(smem_pipe_write); - copy( - params.tma_load_K.with( - *pipeline_k.producer_get_barrier(smem_pipe_write), - mcast_mask_kv, - TMA::CacheHintSm90::EVICT_LAST), - tKgK_TMA(_, n_block), - tKsK_TMA(_, smem_pipe_write.index())); - }; - - auto load_V = [&](int const n_block, auto const& smem_pipe_write) { - auto pipeline_v_load = - cute::conditional_return(pipeline_v, pipeline_vt); - pipeline_v_load.producer_acquire(smem_pipe_write); - copy( - params.tma_load_V.with( - *pipeline_v_load.producer_get_barrier(smem_pipe_write), - mcast_mask_kv, - TMA::CacheHintSm90::EVICT_LAST), - tVgVt_TMA(_, n_block), - tVsVt_TMA(_, smem_pipe_write.index())); - }; - - auto copy_Vt_to_V = [&](auto const& smem_pipe_write) { - // Instead of maintaining smem_pipe_read as a separate variable, we can - // just use smem_pipe_write, and exploit the invariance that - // smem_pipe_write.phase() == smem_pipe_read.phase() ^ 1. This saves 1 or - // 2 registers. - PipelineState smem_pipe_read{ - smem_pipe_write.index(), - smem_pipe_write.phase() ^ 1, - smem_pipe_write.count()}; - pipeline_vt.consumer_wait(smem_pipe_read); - pipeline_v.producer_acquire(smem_pipe_write); - transpose_V(smem_pipe_write.index()); - // SMEM fence to make sure V is transposed before math - cutlass::arch::fence_view_async_shared(); - pipeline_v.producer_commit(smem_pipe_write); - // Very important: PipelineTmaAsync::consumer_release assumes that the - // warpgroup is synchronized before calling. Without this we get race - // conditions. - cutlass::arch::NamedBarrier::sync( - cutlass::NumThreadsPerWarpGroup, - static_cast(FwdNamedBarriers::ProducerWG) /*id*/); - pipeline_vt.consumer_release(smem_pipe_read); - }; - - int n_block = n_block_max - 1; - - int warp_idx_in_warpgroup = - __shfl_sync(0xffffffff, (threadIdx.x / 32) % 4, 0); - // If this is true, we're guaranteed that only the first warp will execute - // this function - static constexpr bool SingleProducerWarp = - NumProducerThreads == cutlass::NumThreadsPerWarp; - bool should_load_KV = - ((SingleProducerWarp || warp_idx_in_warpgroup == 0) && - cute::elect_one_sync()); - - if (should_load_KV) { - if constexpr (Transpose_V) { - load_V(n_block, smem_pipe_write); - } - // if (thread_idx == 0) { printf("Producer: main load, before load_K, - // index = %d\n", smem_pipe_write.index());} - load_K(n_block, smem_pipe_write); - // if (thread_idx == 0) { printf("Producer: main load, after load K, index - // = %d\n", smem_pipe_write.index());} - } - - // TMA_Q, Wait for the MMA warpgroups to signal that smem_q is ready - if (SingleProducerWarp || warp_idx_in_warpgroup == 0) { - cutlass::arch::NamedBarrier::sync( - NumMmaThreads + cutlass::NumThreadsPerWarp, - static_cast(FwdNamedBarriers::QueryEmpty) /*id*/); - } - if ((SingleProducerWarp || warp_idx_in_warpgroup == 0) && - cute::elect_one_sync()) { - shared_storage.pipelines.barrier_Q.arrive_and_expect_tx( - TmaTransactionBytesQ); - copy( - params.tma_load_Q.with( - reinterpret_cast( - shared_storage.pipelines.barrier_Q), - 0 /*mcast_mask*/, - TMA::CacheHintSm90::EVICT_FIRST), - tQgQ, - tQsQ); - } - - // Wait for the MMA WGs to signal that smem_v are ready and V can be copied - // from gmem Need ClusterBarrier, not just NamedBarrier. Otherwise we might - // have CTA 0 finishing the TMA store on O first, call TMA multicast load on - // V, before CTA 1 can finishing TMA store on O. if (thread_idx == 0) { - // printf("Producer: main load, before barrier_O, work_idx = %d\n", - // work_idx);} - shared_storage.pipelines.barrier_O.wait((work_idx + 1) % 2); - // if (thread_idx == 0) { printf("Producer: main load, after barrier_O\n");} - - int n_block_prev = n_block; - --n_block; -#pragma unroll(!Transpose_V ? 2 : 1) - for (; n_block >= n_block_min; --n_block) { - PipelineState smem_pipe_write_v = - smem_pipe_write; // copy the state, write_v is always 1 step behind - ++smem_pipe_write; - if (should_load_KV) { - if constexpr (Transpose_V) { - load_V(n_block, smem_pipe_write); - } else { - load_V(n_block_prev, smem_pipe_write_v); - } - load_K(n_block, smem_pipe_write); - } - n_block_prev = n_block; - if constexpr (Transpose_V) { - copy_Vt_to_V(smem_pipe_write_v); - } - } - scheduler_prefetch(); - if constexpr (!Transpose_V) { - if (should_load_KV) { - load_V(n_block_prev, smem_pipe_write); - } - } - if constexpr (Transpose_V) { - copy_Vt_to_V(smem_pipe_write); - } - ++smem_pipe_write; - if constexpr (!Cross) { - if constexpr (Has_targets) { - auto [target_n_block_min, target_n_block_max] = - get_target_n_block_min_max( - n_block_max, - seqlen_info.uihlen_q, - seqlen_info.seqlen_kv, - m_block); -#pragma unroll 1 - for (n_block = target_n_block_max - 1; n_block >= target_n_block_min; - --n_block) { - if (should_load_KV) { - load_V(n_block, smem_pipe_write); - load_K(n_block, smem_pipe_write); - } - if constexpr (Transpose_V) { - copy_Vt_to_V(smem_pipe_write); - } - ++smem_pipe_write; - } - } - if constexpr (Contexual_mask) { - int contexual_n_block_max = get_contexual_n_block_max( - n_block_min, - min_full_attn_seq_len_, - contextual_seq_len_, - seqlen_info.uihlen_q, - m_block); -#pragma unroll 1 - for (n_block = contexual_n_block_max - 1; n_block >= 0; --n_block) { - if (should_load_KV) { - load_V(n_block, smem_pipe_write); - load_K(n_block, smem_pipe_write); - } - if constexpr (Transpose_V) { - copy_Vt_to_V(smem_pipe_write); - } - ++smem_pipe_write; - } - } - } - // At the end, all threads have the correct smem_pipe_write. - ++work_idx; - } - - template - CUTLASS_DEVICE void load_tail( - MainloopPipelineK pipeline_k, - MainloopPipelineV pipeline_v, - MainloopPipelineVt pipeline_vt, - PipelineState& smem_pipe_write, - SharedStorage& shared_storage, - int const work_idx) { - // If we don't wait for barrier_O here, when using Cluster, CTA0 might exit - // early and CTA1 will try to arrive on barrier_O of CTA0, causing - // "unspecified launch failure". - shared_storage.pipelines.barrier_O.wait((work_idx + 1) % 2); - int warp_idx_in_warpgroup = - __shfl_sync(0xffffffff, (threadIdx.x / 32) % 4, 0); - // Issue the epilogue waits - // TODO: check if this should be called by 1 thread or more - if (warp_idx_in_warpgroup == 0 && cute::elect_one_sync()) { - /* This helps avoid early exit of blocks in Cluster - * Waits for all stages to either be released (all Consumer UNLOCKs), or - * if the stage was never used then would just be acquired since the phase - * was still inverted from make_producer_start_state - */ - pipeline_k.producer_tail(smem_pipe_write); - pipeline_v.producer_tail(smem_pipe_write); - if constexpr (Transpose_V) { - pipeline_vt.producer_tail(smem_pipe_write); - } - } - } - - CUTLASS_DEVICE void warp_scheduler_barrier_sync() { - if constexpr (UseSchedulerBarrier) { - cutlass::arch::NamedBarrier::sync( - 2 * cutlass::NumThreadsPerWarpGroup, - static_cast(FwdNamedBarriers::WarpSchedulerWG1) - 1 + - hstu::canonical_warp_group_idx_nosync() /*id*/); - } - } - - CUTLASS_DEVICE void warp_scheduler_barrier_arrive() { - if constexpr (UseSchedulerBarrier) { - static_assert(NumMmaWarpGroups == 2 || NumMmaWarpGroups == 3); - int const cur_WG = hstu::canonical_warp_group_idx_nosync() - 1; - int const next_WG = NumMmaWarpGroups == 2 - ? 1 - cur_WG - : (cur_WG < NumMmaWarpGroups - 1 ? cur_WG + 1 : 0); - cutlass::arch::NamedBarrier::arrive( - 2 * cutlass::NumThreadsPerWarpGroup, - static_cast(FwdNamedBarriers::WarpSchedulerWG1) + - next_WG /*id*/); - } - } - - CUTLASS_DEVICE void mma_init() { - // Tell producers that smem_q is ready - cutlass::arch::NamedBarrier::arrive( - NumMmaThreads + cutlass::NumThreadsPerWarp, - static_cast(FwdNamedBarriers::QueryEmpty) /*id*/); - if constexpr (UseSchedulerBarrier) { - // We have NamedBarrier for up to 3 WGs - static_assert(NumMmaWarpGroups == 2 || NumMmaWarpGroups == 3); - // WG1 needs the very first signal to start - if (hstu::canonical_warp_group_idx_nosync() == 1) { - cutlass::arch::NamedBarrier::arrive( - 2 * cutlass::NumThreadsPerWarpGroup, - static_cast(FwdNamedBarriers::WarpSchedulerWG1) /*id*/); - } - } - } - - template - CUTLASS_DEVICE bool mma( - Params const& params, - MainloopPipelineK pipeline_k, - MainloopPipelineV pipeline_v, - PipelineState& smem_pipe_read, - FrgTensorO& tOrO, - int const thread_idx, - int& work_idx, - SeqlenInfo_t const& seqlen_info, - cute::tuple block_coord, - SharedStorage& shared_storage) { - static_assert( - is_rmem::value, "O tensor must be rmem resident."); - static constexpr int kBlockM = get<0>(TileShape_MNK{}); - static constexpr int kBlockN = get<1>(TileShape_MNK{}); - - // can't use auto [m_block, ...] = block_coord since structured binding - // cannot be captured in lambda - int const m_block = get<0>(block_coord); - int const bidh = get<1>(block_coord); - int const bidb = get<2>(block_coord); - int const split_idx = get<3>(block_coord); - if constexpr (Jagged) { - static constexpr int kBlockM = get<0>(TileShape_MNK{}); - if (m_block * kBlockM >= seqlen_info.seqlen_q) { - return false; - } - } - int min_full_attn_seq_len_, max_attn_len_, contextual_seq_len_; - float scalar_scale_val_; - if constexpr (!Cross) { - if (params.num_groups > 1) { - int group_id = bidb / params.batch_size_per_group; - min_full_attn_seq_len_ = params.min_full_attn_seq_len_tensor[group_id]; - max_attn_len_ = params.max_attn_len_tensor[group_id]; - contextual_seq_len_ = params.contextual_seq_len_tensor[group_id]; - } else { - min_full_attn_seq_len_ = params.min_full_attn_seq_len; - max_attn_len_ = params.max_attn_len; - contextual_seq_len_ = params.contextual_seq_len; - } - } - if (params.num_groups > 1) { - int group_id = bidb / params.batch_size_per_group; - int max_seq_len_per_group = params.max_seq_len_tensor[group_id]; - // attention scale - scalar_scale_val_ = params.scalar_scale - ? (params.attn_scale == nullptr ? 1.0f / max_seq_len_per_group - : params.attn_scale[group_id]) - : 0; - } else { - // attention scale - scalar_scale_val_ = params.scalar_scale - ? (params.attn_scale == nullptr ? params.max_seq_len_inv - : params.attn_scale[0]) - : 0; - } - int n_block_min, n_block_max; - if constexpr (Cross) { - auto n_block_min_max = get_cross_n_block_min_max( - seqlen_info.uihlen_q, - seqlen_info.seqlen_q, - seqlen_info.seqlen_kv, - m_block); - n_block_min = get<0>(n_block_min_max); - n_block_max = get<1>(n_block_min_max); - } else { - auto n_block_min_max = get_n_block_min_max( - max_attn_len_, - min_full_attn_seq_len_, - contextual_seq_len_, - seqlen_info.uihlen_q, - m_block); - n_block_min = get<0>(n_block_min_max); - n_block_max = get<1>(n_block_min_max); - } - -#ifdef HSTU_FLASH_ATTN_DEBUG_INFO - if (n_block_max <= n_block_min) { - std::printf( - "mainloop_fwd_sm90: n_block_max <= n_block_min not expected."); - return false; - } -#endif - - Tensor sQ = make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_q.data()), - SmemLayoutQ{}); - Tensor sK = make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_k.data()), - SmemLayoutK{}); - Tensor sV = make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_v.data()), - SmemLayoutVtMma{}); - Tensor sP = [&] { - if constexpr (Mma1_is_RS) { - // We might not have smem_p if !Mma1_is_RS1, just use smem_q as a - // placeholder since we don't use it - return make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_q.data()), - SmemLayoutP{}); - } else { - return make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_p.data()), - SmemLayoutP{}); - } - }(); - - if constexpr (!Mma0_is_RS) { - static_assert( - stride<0>(typename TiledMma0::ALayout{}) == 0 and - stride<0>(typename TiledMma0::BLayout{}) == 0 and - size<0>(typename TiledMma0::ALayout{}) == - cutlass::NumThreadsPerWarpGroup and - size<0>(typename TiledMma0::BLayout{}) == - cutlass::NumThreadsPerWarpGroup, - "Stride of the first mode must be 0 and the size of the mode must be NumThreadsPerWarpGroup"); - } - constexpr int MmaWarpGroups = - size(TiledMma0{}) / cutlass::NumThreadsPerWarpGroup; - Layout warp_group_thread_layout = make_layout( - make_shape(Int{}), - make_stride(Int{})); - - int warp_group_idx = __shfl_sync( - 0xFFFFFFFF, thread_idx / cutlass::NumThreadsPerWarpGroup, 0); - TiledMma0 tiled_mma0; - TiledMma1 tiled_mma1; - auto wg_mma0 = - tiled_mma0.get_slice(warp_group_thread_layout(warp_group_idx)); - auto wg_mma1 = - tiled_mma1.get_slice(warp_group_thread_layout(warp_group_idx)); - - auto smem_tiled_copy_P = make_tiled_copy_C(SmemCopyAtomP{}, tiled_mma0); - auto smem_thr_copy_P = smem_tiled_copy_P.get_thread_slice(thread_idx); - - // Allocate "fragments/descriptors" - Tensor tSrQ = wg_mma0.partition_fragment_A(sQ); - Tensor tSrK = wg_mma0.partition_fragment_B(sK); - Tensor tOrV = wg_mma1.partition_fragment_B(sV); - Tensor tOsP = wg_mma1.partition_fragment_A(sP); - Tensor tPsP = smem_thr_copy_P.partition_D( - cute::as_position_independent_swizzle_tensor(sP)); - - auto consumer_wait = [](auto& pipeline, auto& smem_pipe_read) { - auto barrier_token = pipeline.consumer_try_wait(smem_pipe_read); - pipeline.consumer_wait(smem_pipe_read, barrier_token); - }; - - clear(tOrO); - - int n_block = n_block_max - 1; - - hstu::Mask mask( - thread_idx, - seqlen_info.seqlen_q, - seqlen_info.seqlen_kv, - max_attn_len_, - min_full_attn_seq_len_, - contextual_seq_len_, - seqlen_info.uihlen_q); - - auto& barrier_Q = shared_storage.pipelines.barrier_Q; - barrier_Q.wait(work_idx % 2); - - static constexpr int Qdim = 0; - auto thread_mma = tiled_mma0.get_thread_slice(thread_idx); - auto thread0_mma = tiled_mma0.get_thread_slice(_0{}); - Tensor cS = cute::make_identity_tensor(Shape, Int>{}); - Tensor tScS = thread_mma.partition_C(cS); - Tensor tScS_rowcol = make_tensor( - tScS.data(), - hstu::convert_layout_acc_rowcol(tScS.layout())); - Tensor t0ScS = thread0_mma.partition_C(cS); - Tensor t0ScS_rowcol = make_tensor( - t0ScS.data(), - hstu::convert_layout_acc_rowcol(t0ScS.layout())); - int const thread_qdim_offset = get(tScS_rowcol(_0{}, _0{})); - SiluScaleOp silu_scale_op; - int qdim_offset = params.scalar_scale - ? 0 - : m_block * kBlockM + thread_qdim_offset + seqlen_info.offset_q; - - if constexpr (Mma0_is_RS) { - using SmemCopyAtomQ = Copy_Atom; - auto smem_tiled_copy_Q = make_tiled_copy_A(SmemCopyAtomQ{}, tiled_mma0); - auto smem_thr_copy_Q = smem_tiled_copy_Q.get_thread_slice(thread_idx); - Tensor tSrQ_copy_view = smem_thr_copy_Q.retile_D(tSrQ); - Tensor tSsQ_copy_view = smem_thr_copy_Q.partition_S( - cute::as_position_independent_swizzle_tensor(sQ)); - cute::copy(smem_tiled_copy_Q, tSsQ_copy_view, tSrQ_copy_view); - } - - Tensor tSrS = - partition_fragment_C(tiled_mma0, select<0, 1>(TileShape_MNK{})); - Tensor tSrS_rowcol = make_tensor( - tSrS.data(), - hstu::convert_layout_acc_rowcol(tSrS.layout())); - consumer_wait(pipeline_k, smem_pipe_read); - hstu::gemm( - tiled_mma0, tSrQ, tSrK(_, _, _, smem_pipe_read.index()), tSrS); - warpgroup_wait<0>(); - pipeline_k.consumer_release(smem_pipe_read); -#pragma unroll - for (int mi = 0; mi < size<0>(tSrS_rowcol); ++mi) { - float scale = scalar_scale_val_; - if (!params.scalar_scale) { - int q_index = qdim_offset + int(get(t0ScS_rowcol(mi, _0{}))); - // Convert global index to local sequence position for bounds checking - int q_local = q_index - seqlen_info.offset_q; - if (q_local < seqlen_info.seqlen_q) { - scale = params.attn_scale[q_index]; - } - } -#pragma unroll - for (int ni = 0; ni < size<1>(tSrS_rowcol); ++ni) { - tSrS_rowcol(mi, ni) = - silu_scale_op(tSrS_rowcol(mi, ni) * params.alpha, scale); - } - } - int const m_idx_max = (m_block + 1) * kBlockM; - if constexpr (Cross) { - mask.template apply< - false /*Seqlenq_mask*/, - true /*Seqlenk_mask*/, - Causal, - false /*Local*/, - false /*Contexual_mask*/, - false /*Target_mask*/, - Cross, - false /*Softmax*/>(tSrS, m_block, n_block); - } else { - if (m_idx_max <= seqlen_info.uihlen_q) { - mask.template apply< - false /*Seqlenq_mask*/, - false /*Seqlenk_mask*/, - Causal, - Local, - Contexual_mask, - false /*Target_mask*/, - Cross, - false /*Softmax*/>(tSrS, m_block, n_block); - } else if ( - m_idx_max <= - cute::ceil_div(seqlen_info.uihlen_q, kBlockM) * kBlockM) { - mask.template apply< - false /*Seqlenq_mask*/, - true /*Seqlenk_mask*/, - Causal, - Local, - Contexual_mask, - Has_targets, - Cross, - false /*Softmax*/>(tSrS, m_block, n_block); - } else { - mask.template apply< - false /*Seqlenq_mask*/, - true /*Seqlenk_mask*/, - false /*Causal*/, - false, - Contexual_mask, - Has_targets, - Cross, - false /*Softmax*/>(tSrS, m_block, n_block); - } - } - if constexpr (Is_FP8 && !V_colmajor) { - hstu::permute_Cregs_fp8(tSrS); - } - Tensor tOrP_acc = make_tensor( - tSrS.data(), hstu::convert_layout_acc_Aregs(tSrS.layout())); - Tensor tOrP = make_tensor_like(tOrP_acc); - convert_type_out(tOrP_acc, tOrP); - if constexpr (Is_FP8 && V_colmajor) { - hstu::permute_Aregs_fp8(tOrP); - } - if constexpr (!Mma1_is_RS) { - cute::copy(smem_tiled_copy_P, smem_thr_copy_P.retile_S(tOrP), tPsP); - cutlass::arch::fence_view_async_shared(); - __syncwarp(); // Only need syncwarp since each warp is using its own P - // values for Mma1 - } - --n_block; - - // Each step does gemm0 and silu for iter n_block and gemm1 for prev iter. - auto fwd_step_intra_warp_pipeline = [&](int const n_block, auto mask_fn) { - PipelineState smem_pipe_read_v( - smem_pipe_read.index(), - smem_pipe_read.phase(), - smem_pipe_read.count()); - ++smem_pipe_read; - Tensor tSrS = - partition_fragment_C(tiled_mma0, select<0, 1>(TileShape_MNK{})); - Tensor tSrS_rowcol = make_tensor( - tSrS.data(), - hstu::convert_layout_acc_rowcol(tSrS.layout())); - if (!UseSchedulerBarrier || warp_group_idx == 0) { - consumer_wait(pipeline_k, smem_pipe_read); - } - warp_scheduler_barrier_sync(); - hstu::gemm( - tiled_mma0, tSrQ, tSrK(_, _, _, smem_pipe_read.index()), tSrS); - if (!UseSchedulerBarrier || warp_group_idx == 0) { - consumer_wait(pipeline_v, smem_pipe_read_v); - } - hstu::gemm( - tiled_mma1, - cute::conditional_return(tOrP, tOsP), - tOrV(_, _, _, smem_pipe_read_v.index()), - tOrO); - warp_scheduler_barrier_arrive(); - warpgroup_wait<1>(); - pipeline_k.consumer_release(smem_pipe_read); // release K -#pragma unroll - for (int mi = 0; mi < size<0>(tSrS_rowcol); ++mi) { - float scale = scalar_scale_val_; - if (!params.scalar_scale) { - int q_index = qdim_offset + int(get(t0ScS_rowcol(mi, _0{}))); - // Convert global index to local sequence position for bounds checking - int q_local = q_index - seqlen_info.offset_q; - if (q_local < seqlen_info.seqlen_q) { - scale = params.attn_scale[q_index]; - } - } -#pragma unroll - for (int ni = 0; ni < size<1>(tSrS_rowcol); ++ni) { - tSrS_rowcol(mi, ni) = - silu_scale_op(tSrS_rowcol(mi, ni) * params.alpha, scale); - } - } - mask_fn(tSrS, n_block); - warpgroup_wait<0>(); - pipeline_v.consumer_release(smem_pipe_read_v); // release V - if constexpr (Is_FP8 && !V_colmajor) { - hstu::permute_Cregs_fp8(tSrS); - } - convert_type_out(make_tensor(tSrS.data(), tOrP.layout()), tOrP); - if constexpr (Is_FP8 && V_colmajor) { - hstu::permute_Aregs_fp8(tOrP); - } - if constexpr (!Mma1_is_RS) { - cute::copy(smem_tiled_copy_P, smem_thr_copy_P.retile_S(tOrP), tPsP); - } - if constexpr (!Mma1_is_RS) { - cutlass::arch::fence_view_async_shared(); - __syncwarp(); - } - }; - - if constexpr (Cross) { - if constexpr (Causal) { - if (m_idx_max <= - cute::ceil_div(seqlen_info.uihlen_q, kBlockM) * kBlockM) { - auto mask_fn = [&](auto& tSrS, int n_block) { - mask.template apply< - false /*Seqlenq_mask*/, - false /*Seqlenk_mask*/, - Causal, - false /*Local*/, - false /*Contexual_mask*/, - false /*Target_mask*/, - Cross, - false /*Softmax*/>(tSrS, m_block, n_block); - }; - int const m_idx_min = m_block * kBlockM; - int const n_block_min_causal_mask = std::max( - n_block_min, - (m_idx_min + seqlen_info.seqlen_kv - seqlen_info.uihlen_q) / - kBlockN); -#pragma unroll 1 - for (; n_block >= n_block_min_causal_mask; --n_block) { - fwd_step_intra_warp_pipeline(n_block, mask_fn); - } - } - } - auto no_mask_fn = [](auto& tSrS, int n_block) {}; -#pragma unroll 1 - for (; n_block >= n_block_min; --n_block) { - fwd_step_intra_warp_pipeline(n_block, no_mask_fn); - } - } else { - if constexpr (Causal || Local) { // Separate iterations with causal - // or local masking - if (m_idx_max <= - cute::ceil_div(seqlen_info.uihlen_q, kBlockM) * kBlockM) { - auto mask_fn = [&](auto& tSrS, int n_block) { - mask.template apply< - false /*Seqlenq_mask*/, - false /*Seqlenk_mask*/, - Causal, - Local, - Contexual_mask, - false /*Has_targets*/, - Cross, - false /*Softmax*/>(tSrS, m_block, n_block); - }; - int const m_idx_min = m_block * kBlockM; - int const n_block_min_causal_local_mask = - std::max(n_block_min, m_idx_min / kBlockN); -#pragma unroll 1 - for (; n_block >= n_block_min_causal_local_mask; --n_block) { - fwd_step_intra_warp_pipeline(n_block, mask_fn); - } - } - } - int n_block_min_before_local_mask = n_block_min; - if constexpr (Local) { - if (m_idx_max <= - cute::ceil_div( - seqlen_info.uihlen_q - min_full_attn_seq_len_, kBlockM) * - kBlockM) { - n_block_min_before_local_mask = std::max( - n_block_min, cute::ceil_div(m_idx_max - max_attn_len_, kBlockN)); - } - } - auto no_mask_fn = [](auto& tSrS, int n_block) {}; -#pragma unroll 1 - for (; n_block >= n_block_min_before_local_mask; --n_block) { - fwd_step_intra_warp_pipeline(n_block, no_mask_fn); - } - // Separate masking iterations on the left for local attention - if constexpr (Local) { - auto local_mask_fn = [&](auto& tSrS, int n_block) { - mask.template apply< - false /*Seqlenq_mask*/, - false /*Seqlenk_mask*/, - false /*Causal_mask*/, - Local, - Contexual_mask, - false /*Has_targets*/, - Cross, - false /*Softmax*/>(tSrS, m_block, n_block); - }; -#pragma unroll 1 - for (; n_block >= n_block_min; --n_block) { - fwd_step_intra_warp_pipeline(n_block, local_mask_fn); - } - } - // Target part GEMM - if constexpr (Has_targets) { - auto [target_n_block_min, target_n_block_max] = - get_target_n_block_min_max( - n_block_max, - seqlen_info.uihlen_q, - seqlen_info.seqlen_kv, - m_block); - auto target_mask_fn = [&](auto& tSrS, int n_block) { - mask.template apply< - false /*Seqlenq_mask*/, - true /*Seqlenk_mask*/, - false /*Causal_mask*/, - false /*Local*/, - Contexual_mask, - Has_targets, - Cross, - false /*Softmax*/>(tSrS, m_block, n_block); - }; -#pragma unroll 1 - for (n_block = target_n_block_max - 1; n_block >= target_n_block_min; - --n_block) { - fwd_step_intra_warp_pipeline(n_block, target_mask_fn); - } - } - if constexpr (Contexual_mask) { - int contexual_n_block_max = get_contexual_n_block_max( - n_block_min, - min_full_attn_seq_len_, - contextual_seq_len_, - seqlen_info.uihlen_q, - m_block); - auto contexual_mask_fn = [&](auto& tSrS, int n_block) { - mask.template apply< - false /*Seqlenq_mask*/, - true /*Seqlenk_mask*/, - false /*Causal_mask*/, - Local, - Contexual_mask, - Has_targets, - Cross, - false /*Softmax*/>(tSrS, m_block, n_block); - }; -#pragma unroll 1 - for (n_block = contexual_n_block_max - 1; n_block >= 0; --n_block) { - fwd_step_intra_warp_pipeline(n_block, contexual_mask_fn); - } - } - } - // Tell producers that smem_q is ready - cutlass::arch::NamedBarrier::arrive( - NumMmaThreads + cutlass::NumThreadsPerWarp, - static_cast(FwdNamedBarriers::QueryEmpty) /*id*/); - consumer_wait(pipeline_v, smem_pipe_read); - hstu::gemm( - tiled_mma1, - cute::conditional_return(tOrP, tOsP), - tOrV(_, _, _, smem_pipe_read.index()), - tOrO); - warpgroup_wait<0>(); - pipeline_v.consumer_release( - smem_pipe_read); // release V, otherwise producers will hang - if constexpr (Is_FP8 && !V_colmajor) { - hstu::permute_output_fp8(tOrO); - } - ++smem_pipe_read; - ++work_idx; - return true; - } - - template - CUTLASS_DEVICE bool mma_softmax( - Params const& params, - MainloopPipelineK pipeline_k, - MainloopPipelineV pipeline_v, - PipelineState& smem_pipe_read, - FrgTensorO& tOrO, - Softmax& softmax, - int const thread_idx, - int& work_idx, - SeqlenInfo_t const& seqlen_info, - cute::tuple block_coord, - SharedStorage& shared_storage) { - static_assert( - is_rmem::value, "O tensor must be rmem resident."); - static constexpr int kBlockM = get<0>(TileShape_MNK{}); - static constexpr int kBlockN = get<1>(TileShape_MNK{}); - - // can't use auto [m_block, ...] = block_coord since structured binding - // cannot be captured in lambda - int const m_block = get<0>(block_coord); - int const bidh = get<1>(block_coord); - int const bidb = get<2>(block_coord); - int const split_idx = get<3>(block_coord); - if constexpr (Jagged) { - static constexpr int kBlockM = get<0>(TileShape_MNK{}); - if (m_block * kBlockM >= seqlen_info.seqlen_q) { - return false; - } - } - int min_full_attn_seq_len_, max_attn_len_, contextual_seq_len_; - if constexpr (!Cross) { - if (params.num_groups > 1) { - int group_id = bidb / params.batch_size_per_group; - min_full_attn_seq_len_ = params.min_full_attn_seq_len_tensor[group_id]; - max_attn_len_ = params.max_attn_len_tensor[group_id]; - contextual_seq_len_ = params.contextual_seq_len_tensor[group_id]; - } else { - min_full_attn_seq_len_ = params.min_full_attn_seq_len; - max_attn_len_ = params.max_attn_len; - contextual_seq_len_ = params.contextual_seq_len; - } - } - int n_block_min, n_block_max; - if constexpr (Cross) { - auto n_block_min_max = get_cross_n_block_min_max( - seqlen_info.uihlen_q, - seqlen_info.seqlen_q, - seqlen_info.seqlen_kv, - m_block); - n_block_min = get<0>(n_block_min_max); - n_block_max = get<1>(n_block_min_max); - } else { - auto n_block_min_max = get_n_block_min_max( - max_attn_len_, - min_full_attn_seq_len_, - contextual_seq_len_, - seqlen_info.uihlen_q, - m_block); - n_block_min = get<0>(n_block_min_max); - n_block_max = get<1>(n_block_min_max); - } - -#ifdef HSTU_FLASH_ATTN_DEBUG_INFO - if (n_block_max <= n_block_min) { - std::printf( - "mainloop_fwd_sm90: n_block_max <= n_block_min not expected."); - return false; - } -#endif - - Tensor sQ = make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_q.data()), - SmemLayoutQ{}); - Tensor sK = make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_k.data()), - SmemLayoutK{}); - Tensor sV = make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_v.data()), - SmemLayoutVtMma{}); - Tensor sP = [&] { - if constexpr (Mma1_is_RS) { - // We might not have smem_p if !Mma1_is_RS1, just use smem_q as a - // placeholder since we don't use it - return make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_q.data()), - SmemLayoutP{}); - } else { - return make_tensor( - make_smem_ptr(shared_storage.tensors.mainloop.smem_p.data()), - SmemLayoutP{}); - } - }(); - - if constexpr (!Mma0_is_RS) { - static_assert( - stride<0>(typename TiledMma0::ALayout{}) == 0 and - stride<0>(typename TiledMma0::BLayout{}) == 0 and - size<0>(typename TiledMma0::ALayout{}) == - cutlass::NumThreadsPerWarpGroup and - size<0>(typename TiledMma0::BLayout{}) == - cutlass::NumThreadsPerWarpGroup, - "Stride of the first mode must be 0 and the size of the mode must be NumThreadsPerWarpGroup"); - } - constexpr int MmaWarpGroups = - size(TiledMma0{}) / cutlass::NumThreadsPerWarpGroup; - Layout warp_group_thread_layout = make_layout( - make_shape(Int{}), - make_stride(Int{})); - - int warp_group_idx = __shfl_sync( - 0xFFFFFFFF, thread_idx / cutlass::NumThreadsPerWarpGroup, 0); - TiledMma0 tiled_mma0; - TiledMma1 tiled_mma1; - auto wg_mma0 = - tiled_mma0.get_slice(warp_group_thread_layout(warp_group_idx)); - auto wg_mma1 = - tiled_mma1.get_slice(warp_group_thread_layout(warp_group_idx)); - - auto smem_tiled_copy_P = make_tiled_copy_C(SmemCopyAtomP{}, tiled_mma0); - auto smem_thr_copy_P = smem_tiled_copy_P.get_thread_slice(thread_idx); - - // Allocate "fragments/descriptors" - Tensor tSrQ = wg_mma0.partition_fragment_A(sQ); - Tensor tSrK = wg_mma0.partition_fragment_B(sK); - Tensor tOrV = wg_mma1.partition_fragment_B(sV); - Tensor tOsP = wg_mma1.partition_fragment_A(sP); - Tensor tPsP = smem_thr_copy_P.partition_D( - cute::as_position_independent_swizzle_tensor(sP)); - - auto consumer_wait = [](auto& pipeline, auto& smem_pipe_read) { - auto barrier_token = pipeline.consumer_try_wait(smem_pipe_read); - pipeline.consumer_wait(smem_pipe_read, barrier_token); - }; - - clear(tOrO); - - int n_block = n_block_max - 1; - - hstu::Mask mask( - thread_idx, - seqlen_info.seqlen_q, - seqlen_info.seqlen_kv, - max_attn_len_, - min_full_attn_seq_len_, - contextual_seq_len_, - seqlen_info.uihlen_q); - - auto& barrier_Q = shared_storage.pipelines.barrier_Q; - barrier_Q.wait(work_idx % 2); - static constexpr int Qdim = 0; - auto thread_mma = tiled_mma0.get_thread_slice(thread_idx); - auto thread0_mma = tiled_mma0.get_thread_slice(_0{}); - Tensor cS = cute::make_identity_tensor(Shape, Int>{}); - Tensor tScS = thread_mma.partition_C(cS); - Tensor tScS_rowcol = make_tensor( - tScS.data(), - hstu::convert_layout_acc_rowcol(tScS.layout())); - Tensor t0ScS = thread0_mma.partition_C(cS); - Tensor t0ScS_rowcol = make_tensor( - t0ScS.data(), - hstu::convert_layout_acc_rowcol(t0ScS.layout())); - int const thread_qdim_offset = get(tScS_rowcol(_0{}, _0{})); - int qdim_offset = params.scalar_scale - ? 0 - : m_block * kBlockM + thread_qdim_offset + seqlen_info.offset_q; - - if constexpr (Mma0_is_RS) { - using SmemCopyAtomQ = Copy_Atom; - auto smem_tiled_copy_Q = make_tiled_copy_A(SmemCopyAtomQ{}, tiled_mma0); - auto smem_thr_copy_Q = smem_tiled_copy_Q.get_thread_slice(thread_idx); - Tensor tSrQ_copy_view = smem_thr_copy_Q.retile_D(tSrQ); - Tensor tSsQ_copy_view = smem_thr_copy_Q.partition_S( - cute::as_position_independent_swizzle_tensor(sQ)); - cute::copy(smem_tiled_copy_Q, tSsQ_copy_view, tSrQ_copy_view); - } - - Tensor tSrS = - partition_fragment_C(tiled_mma0, select<0, 1>(TileShape_MNK{})); - Tensor tSrS_rowcol = make_tensor( - tSrS.data(), - hstu::convert_layout_acc_rowcol(tSrS.layout())); - consumer_wait(pipeline_k, smem_pipe_read); - hstu::gemm( - tiled_mma0, tSrQ, tSrK(_, _, _, smem_pipe_read.index()), tSrS); - warpgroup_wait<0>(); - pipeline_k.consumer_release(smem_pipe_read); - int const m_idx_max = (m_block + 1) * kBlockM; - if constexpr (Cross) { - mask.template apply< - false /*Seqlenq_mask*/, - true /*Seqlenk_mask*/, - Causal, - false /*Local*/, - false /*Contexual_mask*/, - false /*Target_mask*/, - Cross, - true /*Softmax*/>(tSrS, m_block, n_block); - } else { - if (m_idx_max <= seqlen_info.uihlen_q) { - mask.template apply< - false /*Seqlenq_mask*/, - false /*Seqlenk_mask*/, - Causal, - Local, - Contexual_mask, - false /*Target_mask*/, - Cross, - true /*Softmax*/>(tSrS, m_block, n_block); - } else if ( - m_idx_max <= - cute::ceil_div(seqlen_info.uihlen_q, kBlockM) * kBlockM) { - mask.template apply< - false /*Seqlenq_mask*/, - true /*Seqlenk_mask*/, - Causal, - Local, - Contexual_mask, - Has_targets, - Cross, - true /*Softmax*/>(tSrS, m_block, n_block); - } else { - mask.template apply< - false /*Seqlenq_mask*/, - true /*Seqlenk_mask*/, - false /*Causal*/, - false, - Contexual_mask, - Has_targets, - Cross, - true /*Softmax*/>(tSrS, m_block, n_block); - } - } - Tensor scores_scale = softmax.template max_get_scale< - /*Is_first=*/true, - /*Check_inf=*/true>(tSrS); - softmax.template online_softmax( - tSrS); - if constexpr (Is_FP8 && !V_colmajor) { - hstu::permute_Cregs_fp8(tSrS); - } - Tensor tOrP_acc = make_tensor( - tSrS.data(), hstu::convert_layout_acc_Aregs(tSrS.layout())); - Tensor tOrP = make_tensor_like(tOrP_acc); - convert_type_out(tOrP_acc, tOrP); - if constexpr (Is_FP8 && V_colmajor) { - hstu::permute_Aregs_fp8(tOrP); - } - if constexpr (!Mma1_is_RS) { - cute::copy(smem_tiled_copy_P, smem_thr_copy_P.retile_S(tOrP), tPsP); - cutlass::arch::fence_view_async_shared(); - __syncwarp(); // Only need syncwarp since each warp is using its own P - // values for Mma1 - } - --n_block; - - // Each step does gemm0 and softmax for iter n_block and gemm1 for prev - auto fwd_step_intra_warp_pipeline = [&](int const n_block, - auto mask_fn, - auto check_inf_type) { - static constexpr bool Check_inf = decltype(check_inf_type)::value; - PipelineState smem_pipe_read_v( - smem_pipe_read.index(), - smem_pipe_read.phase(), - smem_pipe_read.count()); - ++smem_pipe_read; - Tensor tSrS = - partition_fragment_C(tiled_mma0, select<0, 1>(TileShape_MNK{})); - Tensor tSrS_rowcol = make_tensor( - tSrS.data(), - hstu::convert_layout_acc_rowcol(tSrS.layout())); - if (!UseSchedulerBarrier || warp_group_idx == 0) { - consumer_wait(pipeline_k, smem_pipe_read); - } - warp_scheduler_barrier_sync(); - hstu::gemm( - tiled_mma0, tSrQ, tSrK(_, _, _, smem_pipe_read.index()), tSrS); - if (!UseSchedulerBarrier || warp_group_idx == 0) { - consumer_wait(pipeline_v, smem_pipe_read_v); - } - hstu::gemm( - tiled_mma1, - cute::conditional_return(tOrP, tOsP), - tOrV(_, _, _, smem_pipe_read_v.index()), - tOrO); - warp_scheduler_barrier_arrive(); - warpgroup_wait<1>(); - pipeline_k.consumer_release(smem_pipe_read); // release K - mask_fn(tSrS, n_block); - cute::copy( - softmax.template max_get_scale(tSrS), - scores_scale); - softmax.template online_softmax(tSrS); - warpgroup_wait<0>(); - pipeline_v.consumer_release(smem_pipe_read_v); // release V - if constexpr (Is_FP8 && !V_colmajor) { - hstu::permute_Cregs_fp8(tSrS); - } - convert_type_out(make_tensor(tSrS.data(), tOrP.layout()), tOrP); - if constexpr (Is_FP8 && V_colmajor) { - hstu::permute_Aregs_fp8(tOrP); - } - softmax.rescale_o(tOrO, scores_scale); - if constexpr (!Mma1_is_RS) { - cute::copy(smem_tiled_copy_P, smem_thr_copy_P.retile_S(tOrP), tPsP); - } - if constexpr (!Mma1_is_RS) { - cutlass::arch::fence_view_async_shared(); - __syncwarp(); - } - }; - - if constexpr (Cross) { - if constexpr (Causal) { - if (m_idx_max <= - cute::ceil_div(seqlen_info.uihlen_q, kBlockM) * kBlockM) { - auto mask_fn = [&](auto& tSrS, int n_block) { - mask.template apply< - false /*Seqlenq_mask*/, - false /*Seqlenk_mask*/, - Causal, - false /*Local*/, - false /*Contexual_mask*/, - false /*Target_mask*/, - Cross, - true /*Softmax*/>(tSrS, m_block, n_block); - }; - int const m_idx_min = m_block * kBlockM; - int const n_block_min_causal_mask = std::max( - n_block_min, - (m_idx_min + seqlen_info.seqlen_kv - seqlen_info.uihlen_q) / - kBlockN); -#pragma unroll 1 - for (; n_block >= n_block_min_causal_mask; --n_block) { - fwd_step_intra_warp_pipeline(n_block, mask_fn, cute::true_type{}); - } - } - } - auto no_mask_fn = [](auto& tSrS, int n_block) {}; -#pragma unroll 1 - for (; n_block >= n_block_min; --n_block) { - fwd_step_intra_warp_pipeline(n_block, no_mask_fn, cute::false_type{}); - } - } else { - if constexpr (Causal || Local) { // Separate iterations with causal - // or local masking - if (m_idx_max <= - cute::ceil_div(seqlen_info.uihlen_q, kBlockM) * kBlockM) { - auto mask_fn = [&](auto& tSrS, int n_block) { - mask.template apply< - false /*Seqlenq_mask*/, - false /*Seqlenk_mask*/, - Causal, - Local, - Contexual_mask, - false /*Has_targets*/, - Cross, - true /*Softmax*/>(tSrS, m_block, n_block); - }; - int const m_idx_min = m_block * kBlockM; - int const n_block_min_causal_local_mask = - std::max(n_block_min, m_idx_min / kBlockN); -#pragma unroll 1 - for (; n_block >= n_block_min_causal_local_mask; --n_block) { - fwd_step_intra_warp_pipeline(n_block, mask_fn, cute::true_type{}); - } - } - } - int n_block_min_before_local_mask = n_block_min; - if constexpr (Local) { - if (m_idx_max <= - cute::ceil_div( - seqlen_info.uihlen_q - min_full_attn_seq_len_, kBlockM) * - kBlockM) { - n_block_min_before_local_mask = std::max( - n_block_min, cute::ceil_div(m_idx_max - max_attn_len_, kBlockN)); - } - } - auto no_mask_fn = [](auto& tSrS, int n_block) {}; -#pragma unroll 1 - for (; n_block >= n_block_min_before_local_mask; --n_block) { - fwd_step_intra_warp_pipeline(n_block, no_mask_fn, cute::false_type{}); - } - // Separate masking iterations on the left for local attention - if constexpr (Local) { - auto local_mask_fn = [&](auto& tSrS, int n_block) { - mask.template apply< - false /*Seqlenq_mask*/, - false /*Seqlenk_mask*/, - false /*Causal_mask*/, - Local, - Contexual_mask, - false /*Has_targets*/, - Cross, - true /*Softmax*/>(tSrS, m_block, n_block); - }; -#pragma unroll 1 - for (; n_block >= n_block_min; --n_block) { - fwd_step_intra_warp_pipeline( - n_block, local_mask_fn, cute::true_type{}); - } - } - // Target part GEMM - if constexpr (Has_targets) { - auto [target_n_block_min, target_n_block_max] = - get_target_n_block_min_max( - n_block_max, - seqlen_info.uihlen_q, - seqlen_info.seqlen_kv, - m_block); - auto target_mask_fn = [&](auto& tSrS, int n_block) { - mask.template apply< - false /*Seqlenq_mask*/, - true /*Seqlenk_mask*/, - false /*Causal_mask*/, - false /*Local*/, - Contexual_mask, - Has_targets, - Cross, - true /*Softmax*/>(tSrS, m_block, n_block); - }; -#pragma unroll 1 - for (n_block = target_n_block_max - 1; n_block >= target_n_block_min; - --n_block) { - fwd_step_intra_warp_pipeline( - n_block, target_mask_fn, cute::true_type{}); - } - } - if constexpr (Contexual_mask) { - int contexual_n_block_max = get_contexual_n_block_max( - n_block_min, - min_full_attn_seq_len_, - contextual_seq_len_, - seqlen_info.uihlen_q, - m_block); - auto contexual_mask_fn = [&](auto& tSrS, int n_block) { - mask.template apply< - false /*Seqlenq_mask*/, - true /*Seqlenk_mask*/, - false /*Causal_mask*/, - Local, - Contexual_mask, - Has_targets, - Cross, - true /*Softmax*/>(tSrS, m_block, n_block); - }; -#pragma unroll 1 - for (n_block = contexual_n_block_max - 1; n_block >= 0; --n_block) { - fwd_step_intra_warp_pipeline( - n_block, contexual_mask_fn, cute::true_type{}); - } - } - } - // Tell producers that smem_q is ready - cutlass::arch::NamedBarrier::arrive( - NumMmaThreads + cutlass::NumThreadsPerWarp, - static_cast(FwdNamedBarriers::QueryEmpty) /*id*/); - consumer_wait(pipeline_v, smem_pipe_read); - hstu::gemm( - tiled_mma1, - cute::conditional_return(tOrP, tOsP), - tOrV(_, _, _, smem_pipe_read.index()), - tOrO); - cute::copy(softmax.finalize(1.0f), scores_scale); - warpgroup_wait<0>(); - pipeline_v.consumer_release( - smem_pipe_read); // release V, otherwise producers will hang - softmax.rescale_o(tOrO, scores_scale); - if constexpr (Is_FP8 && !V_colmajor) { - hstu::permute_output_fp8(tOrO); - } - ++smem_pipe_read; - ++work_idx; - return true; - } -}; - -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/mask.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/mask.h deleted file mode 100644 index e35af5193..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/mask.h +++ /dev/null @@ -1,396 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/****************************************************************************** - * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, - *Pradeep Ramani, Tri Dao. - ******************************************************************************/ - -#pragma once - -#include - -#include "utils.h" - -namespace hstu { - -using namespace cute; - -template -struct Mask { - int const thread_idx; - int const max_q_len; - int const max_kv_len; - int const max_attn_len; - int const min_full_attn_seq_len; - int const contextual_seq_len; - int const max_uih_len; - - CUTLASS_DEVICE - Mask( - const int thread_idx, - const int max_q_len, - const int max_kv_len, - const int max_attn_len, - const int min_full_attn_seq_len, - const int contextual_seq_len, - const int max_uih_len) - : thread_idx(thread_idx), - max_q_len(max_q_len), - max_kv_len(max_kv_len), - max_attn_len(max_attn_len), - min_full_attn_seq_len(min_full_attn_seq_len), - contextual_seq_len(contextual_seq_len), - max_uih_len(max_uih_len) {}; - - template < - bool Seqlenq_mask = false, - bool Seqlenk_mask = false, - bool Causal_mask = false, - bool Local_mask = false, - bool Contexual_mask = false, - bool Target_mask = false, // If Target_mask, Seqlenk_mask will be disabled - bool Cross = false, - bool Softmax = false, - typename Engine, - typename Layout> - CUTLASS_DEVICE void apply( - Tensor& tSrS, - const int m_block, - const int n_block) const { - static_assert( - !(Causal_mask && Local_mask), "Cannot be both causal and local"); - static_assert(Layout::rank == 3, "Only support 3D Tensor"); - if constexpr (Cross) { - static_assert( - (!Local_mask) && (!Contexual_mask) && (!Target_mask), - "Local, contexual, and target masks not supported under cross attention"); - } - if (!Seqlenq_mask && !Seqlenk_mask && !Causal_mask && !Local_mask && - !Target_mask) { - return; - } - - auto thread_mma = TiledMma{}.get_thread_slice(thread_idx); - auto thread0_mma = TiledMma{}.get_thread_slice(_0{}); - - static constexpr int Qdim = !SwapAB ? 0 : 1, Kdim = !SwapAB ? 1 : 0; - - Tensor cS = cute::make_identity_tensor( - Shape< - Int, - Int>{}); - Tensor tScS = thread_mma.partition_C(cS); - Tensor tSrS_rowcol = make_tensor( - tSrS.data(), - hstu::convert_layout_acc_rowcol(tSrS.layout())); - Tensor tScS_rowcol = make_tensor( - tScS.data(), - hstu::convert_layout_acc_rowcol(tScS.layout())); - Tensor t0ScS = thread0_mma.partition_C(cS); - Tensor t0ScS_rowcol = make_tensor( - t0ScS.data(), - hstu::convert_layout_acc_rowcol(t0ScS.layout())); - // We want to use the col indices of thread0 to compare, since that is known - // at compile time. So we subtract the limit by the first col index of this - // thread - int const thread_kdim_offset = get(tScS_rowcol(_0{}, _0{})); - int const thread_qdim_offset = get(tScS_rowcol(_0{}, _0{})); - int const seqlen_k_limit = - max_kv_len - n_block * BlockN - thread_kdim_offset; - int const uihlen_k_limit = - max_uih_len - n_block * BlockN - thread_kdim_offset; - int const seqlen_q_limit = - max_q_len - m_block * BlockM - thread_qdim_offset; - if constexpr (Seqlenq_mask) { -#pragma unroll - for (int m = 0; m < size<0>(tSrS_rowcol); ++m) { - if (int(get(t0ScS_rowcol(m, _0{}))) >= seqlen_q_limit) { -#pragma unroll - for (int n = 0; n < size<1>(tSrS_rowcol); ++n) { - if constexpr (Softmax) { - tSrS_rowcol(m, n) = -INFINITY; - } else { - tSrS_rowcol(m, n) = 0.0f; - } - } - } - } - } - if constexpr (Cross) { - if constexpr (!Causal_mask) { - if constexpr (Seqlenk_mask) { -#pragma unroll - for (int n = 0; n < size<1>(tSrS_rowcol); ++n) { - int const t0_col_idx = int(get(t0ScS_rowcol(_0{}, n))); - if (t0_col_idx >= seqlen_k_limit) { -#pragma unroll - for (int m = 0; m < size<0>(tSrS_rowcol); ++m) { - if constexpr (Softmax) { - tSrS_rowcol(m, n) = -INFINITY; - } else { - tSrS_rowcol(m, n) = 0.0f; - } - } - } - } - } - } else { - int const causal_row_offset = max_kv_len - max_uih_len + 1 - - n_block * BlockN - thread_kdim_offset; -#pragma unroll - for (int m = 0; m < size<0>(tSrS_rowcol); ++m) { - if constexpr (Seqlenq_mask) { - if (int(get(t0ScS_rowcol(m, _0{}))) >= seqlen_q_limit) { - continue; - } - } - int const row_idx = get(t0ScS_rowcol(m, _0{})) + - m_block * BlockM + thread_qdim_offset; - int const col_limit_right = !Seqlenk_mask - ? row_idx + causal_row_offset - : __viaddmin_s32(row_idx, causal_row_offset, seqlen_k_limit); -#pragma unroll - for (int n = 0; n < size<1>(tSrS_rowcol); ++n) { - int const t0_col_idx = int(get(t0ScS_rowcol(_0{}, n))); - if (t0_col_idx >= col_limit_right) { - if constexpr (Softmax) { - tSrS_rowcol(m, n) = -INFINITY; - } else { - tSrS_rowcol(m, n) = 0.0f; - } - } - } - } - } - } else { - if constexpr (!Causal_mask && !Local_mask) { - if constexpr (Seqlenk_mask || Target_mask) { -#pragma unroll - for (int n = 0; n < size<1>(tSrS_rowcol); ++n) { - int const t0_col_idx = int(get(t0ScS_rowcol(_0{}, n))); - if constexpr (Target_mask) { - if (t0_col_idx >= uihlen_k_limit) { - bool const oob_predicate = (t0_col_idx >= seqlen_k_limit); - int const col_offset = - t0_col_idx - seqlen_k_limit + seqlen_q_limit; -#pragma unroll - for (int m = 0; m < size<0>(tSrS_rowcol); ++m) { - int const t0_row_idx = int(get(t0ScS_rowcol(m, _0{}))); - if ((t0_row_idx != col_offset) || oob_predicate) { - if constexpr (Softmax) { - tSrS_rowcol(m, n) = -INFINITY; - } else { - tSrS_rowcol(m, n) = 0.0f; - } - } - } - } - } else if constexpr (Seqlenk_mask) { - if (t0_col_idx >= seqlen_k_limit) { -#pragma unroll - for (int m = 0; m < size<0>(tSrS_rowcol); ++m) { - if constexpr (Softmax) { - tSrS_rowcol(m, n) = -INFINITY; - } else { - tSrS_rowcol(m, n) = 0.0f; - } - } - } - } - } - } - } else { // Causal_mask or Local_mask - int const causal_row_offset = 1 - n_block * BlockN - thread_kdim_offset; - if constexpr (Causal_mask) { -#pragma unroll - for (int m = 0; m < size<0>(tSrS_rowcol); ++m) { - if constexpr (Seqlenq_mask) { - if (int(get(t0ScS_rowcol(m, _0{}))) >= seqlen_q_limit) { - continue; - } - } - if constexpr (Contexual_mask) { - if (int(get(t0ScS_rowcol(m, _0{}))) < - contextual_seq_len - m_block * BlockM - thread_qdim_offset) { -#pragma unroll - for (int n = 0; n < size<1>(tSrS_rowcol); ++n) { - int const t0_col_idx = int(get(t0ScS_rowcol(_0{}, n))); - if (t0_col_idx >= uihlen_k_limit) { - if constexpr (Softmax) { - tSrS_rowcol(m, n) = -INFINITY; - } else { - tSrS_rowcol(m, n) = 0.0f; - } - } - } - continue; - } - } - int const row_idx = get(t0ScS_rowcol(m, _0{})) + - m_block * BlockM + thread_qdim_offset; - if constexpr (!Target_mask) { - int const col_limit_right = !Seqlenk_mask - ? row_idx + causal_row_offset - : __viaddmin_s32(row_idx, causal_row_offset, seqlen_k_limit); -#pragma unroll - for (int n = 0; n < size<1>(tSrS_rowcol); ++n) { - int const t0_col_idx = int(get(t0ScS_rowcol(_0{}, n))); - if (t0_col_idx >= col_limit_right) { - if constexpr (Softmax) { - tSrS_rowcol(m, n) = -INFINITY; - } else { - tSrS_rowcol(m, n) = 0.0f; - } - } - } - } else { -#pragma unroll - for (int n = 0; n < size<1>(tSrS_rowcol); ++n) { - int const t0_col_idx = int(get(t0ScS_rowcol(_0{}, n))); - int const col_idx = - t0_col_idx + n_block * BlockN + thread_kdim_offset; - bool const uih_cond = - (t0_col_idx >= row_idx + causal_row_offset) && - (row_idx < max_uih_len); - bool const target_cond = (row_idx != col_idx) && - (row_idx >= max_uih_len) && (col_idx >= max_uih_len); - bool const seqlen_k_cond = (t0_col_idx >= seqlen_k_limit); - if (uih_cond || target_cond || seqlen_k_cond) { - if constexpr (Softmax) { - tSrS_rowcol(m, n) = -INFINITY; - } else { - tSrS_rowcol(m, n) = 0.0f; - } - } - } - } - } - } else { // Local_mask - int const local_row_offset_left = - causal_row_offset - 1 - max_attn_len; - int const col_limit_sink = 0 - n_block * BlockN - thread_kdim_offset; -#pragma unroll - for (int m = 0; m < size<0>(tSrS_rowcol); ++m) { - if constexpr (Seqlenq_mask) { - if (int(get(t0ScS_rowcol(m, _0{}))) >= seqlen_q_limit) { - continue; - } - } - if constexpr (Contexual_mask) { - if (int(get(t0ScS_rowcol(m, _0{}))) < - contextual_seq_len - m_block * BlockM - thread_qdim_offset) { -#pragma unroll - for (int n = 0; n < size<1>(tSrS_rowcol); ++n) { - int const t0_col_idx = int(get(t0ScS_rowcol(_0{}, n))); - if (t0_col_idx >= uihlen_k_limit) { - if constexpr (Softmax) { - tSrS_rowcol(m, n) = -INFINITY; - } else { - tSrS_rowcol(m, n) = 0.0f; - } - } - } - continue; - } - } - int const row_idx = get(t0ScS_rowcol(m, _0{})) + - m_block * BlockM + thread_qdim_offset; - int col_limit_left = row_idx + local_row_offset_left; - if constexpr (Contexual_mask) { - // row contexual without sink - if (col_limit_left + n_block * BlockN + thread_kdim_offset < - contextual_seq_len) { - col_limit_left = 0; - } - } - if constexpr (!Target_mask) { - int const col_limit_right = !Seqlenk_mask - ? row_idx + causal_row_offset - : __viaddmin_s32(row_idx, causal_row_offset, seqlen_k_limit); -#pragma unroll - for (int n = 0; n < size<1>(tSrS_rowcol); ++n) { - int const t0_col_idx = int(get(t0ScS_rowcol(m, n))); - if (row_idx < max_uih_len - min_full_attn_seq_len) { - bool const local_left_cond = Contexual_mask - ? (t0_col_idx < col_limit_left && - t0_col_idx >= col_limit_sink) - : (t0_col_idx < col_limit_left); - if (local_left_cond) { - if constexpr (Softmax) { - tSrS_rowcol(m, n) = -INFINITY; - } else { - tSrS_rowcol(m, n) = 0.0f; - } - } - } - if (t0_col_idx >= col_limit_right) { - if constexpr (Softmax) { - tSrS_rowcol(m, n) = -INFINITY; - } else { - tSrS_rowcol(m, n) = 0.0f; - } - } - } - } else { - int const col_limit_right = row_idx + causal_row_offset; -#pragma unroll - for (int n = 0; n < size<1>(tSrS_rowcol); ++n) { - int const t0_col_idx = int(get(t0ScS_rowcol(_0{}, n))); - if (row_idx < max_uih_len) { - if (row_idx < max_uih_len - min_full_attn_seq_len) { - bool const local_left_cond = Contexual_mask - ? (t0_col_idx < col_limit_left && - t0_col_idx >= col_limit_sink) - : (t0_col_idx < col_limit_left); - if (local_left_cond) { - if constexpr (Softmax) { - tSrS_rowcol(m, n) = -INFINITY; - } else { - tSrS_rowcol(m, n) = 0.0f; - } - } - } - if (t0_col_idx >= col_limit_right) { - if constexpr (Softmax) { - tSrS_rowcol(m, n) = -INFINITY; - } else { - tSrS_rowcol(m, n) = 0.0f; - } - } - } else { - int const col_idx = - t0_col_idx + n_block * BlockN + thread_kdim_offset; - bool const target_cond = (row_idx != col_idx) && - (row_idx >= max_uih_len) && (col_idx >= max_uih_len); - bool const seqlen_k_cond = (t0_col_idx >= seqlen_k_limit); - if (target_cond || seqlen_k_cond) { - if constexpr (Softmax) { - tSrS_rowcol(m, n) = -INFINITY; - } else { - tSrS_rowcol(m, n) = 0.0f; - } - } - } - } - } - } - } - } - } - }; -}; - -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/named_barrier.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/named_barrier.h deleted file mode 100644 index 79dce0dd9..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/named_barrier.h +++ /dev/null @@ -1,101 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/****************************************************************************** - * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, - *Pradeep Ramani, Tri Dao. - ******************************************************************************/ - -#pragma once - -#include "cutlass/arch/barrier.h" - -namespace hstu { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -// cutlass::arch::NamedBarrier::sync/arrive are only enabled Sm90 even though -// they work for Sm80 as well. We reimplement them here, enabled for both Sm90 -// and Sm80. - -CUTLASS_DEVICE -static void named_barrier_sync(uint32_t num_threads, uint32_t barrier_id_) { - static constexpr uint32_t ReservedNamedBarrierCount = static_cast( - cutlass::arch::ReservedNamedBarriers::FirstUserBarrier); - uint32_t barrier_id = barrier_id_ + ReservedNamedBarrierCount; - asm volatile("bar.sync %0, %1;" : : "r"(barrier_id), "r"(num_threads)); - cutlass::arch::synclog_emit_named_barrier_arrive_and_wait( - __LINE__, num_threads, barrier_id); -} - -CUTLASS_DEVICE -static void named_barrier_sync( - uint32_t num_threads, - cutlass::arch::ReservedNamedBarriers reserved_named_barriers) { - uint32_t barrier_id = static_cast(reserved_named_barriers); - asm volatile("bar.sync %0, %1;" : : "r"(barrier_id), "r"(num_threads)); - cutlass::arch::synclog_emit_named_barrier_arrive_and_wait( - __LINE__, num_threads, barrier_id); -} - -CUTLASS_DEVICE -static void named_barrier_arrive(uint32_t num_threads, uint32_t barrier_id_) { - static constexpr uint32_t ReservedNamedBarrierCount = static_cast( - cutlass::arch::ReservedNamedBarriers::FirstUserBarrier); - uint32_t barrier_id = barrier_id_ + ReservedNamedBarrierCount; - cutlass::arch::synclog_emit_named_barrier_arrive( - __LINE__, num_threads, barrier_id); - asm volatile("bar.arrive %0, %1;" : : "r"(barrier_id), "r"(num_threads)); -} - -CUTLASS_DEVICE -static void named_barrier_arrive( - uint32_t num_threads, - cutlass::arch::ReservedNamedBarriers reserved_named_barriers) { - uint32_t barrier_id = static_cast(reserved_named_barriers); - cutlass::arch::synclog_emit_named_barrier_arrive( - __LINE__, num_threads, barrier_id); - asm volatile("bar.arrive %0, %1;" : : "r"(barrier_id), "r"(num_threads)); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////// -// Enumerates the reserved named barriers to avoid potential conflicts - -enum class FwdNamedBarriers { - QueryEmpty = 0, - ProducerWG = 1, - TileCountSmemEmpty = 2, - TileCountSmemFull = 3, - WarpSchedulerWG1 = 4, - WarpSchedulerWG2 = 5, - WarpSchedulerWG3 = 6, -}; - -enum class BwdNamedBarriers { - KVEmpty = 0, - PdS = 1, - // This needs to match FwdNamedBarriers::TileCountSmemEmpty since - // TileScheduler uses it - TileCountSmemEmpty = 2, - TileCountSmemFull = 3, - dQEmptyWG1 = 4, - dQEmptyWG2 = 5, - dQEmptyWG3 = 6, - dQFullWG1 = 7, - dQFullWG2 = 8, - dQFullWG3 = 9, -}; - -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/seqlen.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/seqlen.h deleted file mode 100644 index c5721b272..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/seqlen.h +++ /dev/null @@ -1,134 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/****************************************************************************** - * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, - *Pradeep Ramani, Tri Dao. - ******************************************************************************/ - -#pragma once - -#include - -namespace hstu { - -// We consolidate all the info related to sequence length here. This is so that -// we can do all the gmem reads once at the beginning of each tile, rather than -// having to repeat these reads to compute various things like n_block_min, -// n_block_max, etc. - -template -struct SeqlenInfo { - int const offset, offset_padded; - int const seqlen; - - CUTLASS_DEVICE - SeqlenInfo( - int const bidb, - int const seqlen_static, - int const* const seq_offsets) - : offset(!Jagged ? 0 : seq_offsets[bidb]), - offset_padded( - !Jagged ? 0 - : (seq_offsets[bidb] + bidb * kBlock) / kBlock * kBlock), - seqlen( - !Jagged ? seqlen_static - : (seq_offsets[bidb + 1] - seq_offsets[bidb])) {} -}; - -template -struct SeqlenInfoQKBwd { - int const offset_q, offset_k, offset_q_padded; - int const seqlen_q, seqlen_kv, uihlen_q; - - CUTLASS_DEVICE - SeqlenInfoQKBwd( - int const bidb, - int const max_q_len, - int const max_kv_len, - int const* const seq_offsets, - int const* const seq_offsets_q, - int const* const num_targets) - : offset_q( - !Jagged ? 0 : (Cross ? seq_offsets_q[bidb] : seq_offsets[bidb])), - offset_k(!Jagged ? 0 : seq_offsets[bidb]) - // If jagged, the layout for dQaccum is that we pad - // each sequence in the batch by an extra kBlockM, so that the write for - // each sequence doesn't touch the next sequence. Sequence i starts at - // seq_offsets[i] + i * kBlockM and ends at seq_offsets[i + 1] + i * - // kBlockM However, the start must align to multiples of kBlockM. - , - offset_q_padded( - !Jagged ? 0 - : Cross - ? ((seq_offsets_q[bidb] + bidb * kBlockM) / kBlockM * kBlockM) - : ((seq_offsets[bidb] + bidb * kBlockM) / kBlockM * kBlockM)), - seqlen_q( - !Jagged ? max_q_len - : (Cross ? (seq_offsets_q[bidb + 1] - seq_offsets_q[bidb]) - : (seq_offsets[bidb + 1] - seq_offsets[bidb]))), - seqlen_kv( - !Jagged ? max_kv_len : (seq_offsets[bidb + 1] - seq_offsets[bidb])), - uihlen_q( - !Jagged - ? (Has_targets ? max_q_len - num_targets[bidb] : max_q_len) - : (Has_targets - ? (Cross ? (seq_offsets_q[bidb + 1] - - seq_offsets_q[bidb] - num_targets[bidb]) - : (seq_offsets[bidb + 1] - seq_offsets[bidb] - - num_targets[bidb])) - : (Cross - ? (seq_offsets_q[bidb + 1] - seq_offsets_q[bidb]) - : (seq_offsets[bidb + 1] - seq_offsets[bidb])))) { - } -}; - -template -struct SeqlenInfoQKFwd { - int const offset_q, offset_k; - int const seqlen_q, seqlen_kv, uihlen_q; - - CUTLASS_DEVICE - SeqlenInfoQKFwd( - int const bidb, - int const max_q_len, - int const max_kv_len, - int const* const seq_offsets, - int const* const seq_offsets_q, - int const* const num_targets) - : offset_q( - !Jagged ? 0 : (Cross ? seq_offsets_q[bidb] : seq_offsets[bidb])), - offset_k(!Jagged ? 0 : seq_offsets[bidb]), - seqlen_q( - !Jagged ? max_q_len - : (Cross ? (seq_offsets_q[bidb + 1] - seq_offsets_q[bidb]) - : (seq_offsets[bidb + 1] - seq_offsets[bidb]))), - seqlen_kv( - !Jagged ? max_kv_len : (seq_offsets[bidb + 1] - seq_offsets[bidb])), - uihlen_q( - !Jagged - ? (Has_targets ? max_q_len - num_targets[bidb] : max_q_len) - : (Has_targets - ? (Cross ? (seq_offsets_q[bidb + 1] - - seq_offsets_q[bidb] - num_targets[bidb]) - : (seq_offsets[bidb + 1] - seq_offsets[bidb] - - num_targets[bidb])) - : (Cross - ? (seq_offsets_q[bidb + 1] - seq_offsets_q[bidb]) - : (seq_offsets[bidb + 1] - seq_offsets[bidb])))) { - } -}; - -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/sm90_pipeline_no_cluster.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/sm90_pipeline_no_cluster.h deleted file mode 100644 index b6428e79c..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/sm90_pipeline_no_cluster.h +++ /dev/null @@ -1,150 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/****************************************************************************** - * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, - *Pradeep Ramani, Tri Dao. - ******************************************************************************/ - -#pragma once - -#include - -namespace cutlass { - -using namespace cute; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -// As of Cutlass v3.6.0, if size(ClusterShape) == 1, PipelineTmaAsync has all -// threads signaling the barrier during consumer_release. This causes a perf -// regression in FA3 forward pass (especially hdim 128 causal). We instead -// reimplement the version of PipelineTmaAsync before v3.6.0 where only 1 out of -// 128 threads signals the barrier. -// -// Assumption: params.num_consumers % NumThreadsPerWarpGroup == 0 -template > -class PipelineTmaAsyncNoCluster : public Base { - public: - using FullBarrier = typename Base::FullBarrier; - using EmptyBarrier = typename Base::EmptyBarrier; - static constexpr uint32_t Stages = Stages_; - using PipelineState = typename Base::PipelineState; - - using SharedStorage = typename Base::SharedStorage; - using ThreadCategory = typename Base::ThreadCategory; - using Params = typename Base::Params; - - static CUTLASS_DEVICE void init_barriers( - SharedStorage& storage, - Params params) { - int warp_idx = canonical_warp_idx_sync(); - bool is_initializing_warp = (warp_idx == 0); - if (is_initializing_warp) { - // Barrier FULL and EMPTY init - constexpr int producer_arv_cnt = 1; - uint32_t const num_consumer_warpgroups_per_cluster = - params.num_consumers / NumThreadsPerWarpGroup; - uint32_t const multicast_consumer_arrival_count = - num_consumer_warpgroups_per_cluster; - - cutlass::arch::detail::initialize_barrier_array_pair_aligned< - decltype(storage.full_barrier_), - decltype(storage.empty_barrier_), - Stages>( - storage.full_barrier_, - storage.empty_barrier_, - producer_arv_cnt, - multicast_consumer_arrival_count); - } - cutlass::arch::fence_barrier_init(); - } - - template - CUTLASS_DEVICE PipelineTmaAsyncNoCluster( - SharedStorage& storage, - Params params, - ClusterShape cluster_shape, - InitBarriers = {}, - InitMasks = {}) - : Base( - storage, - params, - make_shape(_1{}, _1{}, _1{}) /*cluster_shape*/, - cute::false_type{} /*init_barriers*/, - cute::false_type{} /*init_masks*/), - empty_barrier_ptr_(&storage.empty_barrier_[0]) { - int warp_idx = canonical_warp_idx_sync(); - int lane_predicate = cute::elect_one_sync(); - - static_assert( - cute::is_same_v || - cute::is_same_v); - static_assert( - cute::is_same_v || - cute::is_same_v); - if constexpr (cute::is_same_v) { - init_barriers(storage, params); - } - } - - // Constructor - template - CUTLASS_DEVICE PipelineTmaAsyncNoCluster( - SharedStorage& storage, - Params params, - ClusterShape cluster_shape) - : PipelineTmaAsyncNoCluster( - storage, - params, - cluster_shape, - cute::true_type{}, - cute::true_type{}) {} - - template - CUTLASS_DEVICE PipelineTmaAsyncNoCluster( - SharedStorage& storage, - Params params, - ClusterShape cluster_shape, - InitBarriers = {}) - : PipelineTmaAsyncNoCluster( - storage, - params, - cluster_shape, - InitBarriers{}, - cute::true_type{}) {} - - CUTLASS_DEVICE - void consumer_release(PipelineState state) { - consumer_release(state.index()); - } - - private: - EmptyBarrier* const empty_barrier_ptr_ = nullptr; - - // Consumer signalling Producer of completion - // Ensures all blocks in the Same Row and Column get notifed. - CUTLASS_DEVICE - void consumer_release(uint32_t stage, uint32_t skip = false) { - empty_barrier_ptr_[stage].arrive( - 0 /*dst_blockid_*/, - uint32_t(threadIdx.x % cutlass::NumThreadsPerWarpGroup == 0) & - (!skip) /*is_signaling_thread*/); - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // end namespace cutlass diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/softmax.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/softmax.h deleted file mode 100644 index 1bd6131c6..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/softmax.h +++ /dev/null @@ -1,256 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/****************************************************************************** - * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, - *Pradeep Ramani, Tri Dao. - ******************************************************************************/ - -#pragma once - -#include -#include -#include -#include "utils.h" - -namespace hstu { - -using namespace cute; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template < - bool zero_init = true, - typename Engine0, - typename Layout0, - typename Engine1, - typename Layout1, - typename Operator> -__device__ __forceinline__ void thread_reduce_( - Tensor const& tensor, - Tensor& summary, - Operator& op) { - static_assert(Layout0::rank == 2, "Only support 2D Tensor"); - static_assert(Layout1::rank == 1, "Only support 1D Tensor"); - CUTE_STATIC_ASSERT_V(size<0>(summary) == size<0>(tensor)); -#pragma unroll - for (int ni = 0; ni < size<1>(tensor); ni++) { -#pragma unroll - for (int mi = 0; mi < size<0>(tensor); mi++) { - summary(mi) = zero_init && ni == 0 ? tensor(mi, ni) - : op(summary(mi), tensor(mi, ni)); - } - } -} - -template < - typename Engine0, - typename Layout0, - typename Engine1, - typename Layout1, - typename Operator> -__device__ __forceinline__ void quad_allreduce_( - Tensor& dst, - Tensor& src, - Operator& op) { - CUTE_STATIC_ASSERT_V(size(dst) == size(src)); -#pragma unroll - for (int i = 0; i < size(dst); i++) { - dst(i) = Allreduce<4>::run(src(i), op); - } -} - -template < - bool zero_init = true, - typename Engine0, - typename Layout0, - typename Engine1, - typename Layout1, - typename Operator> -__device__ __forceinline__ void reduce_( - Tensor const& tensor, - Tensor& summary, - Operator& op) { - thread_reduce_(tensor, summary, op); - quad_allreduce_(summary, summary, op); -} - -template < - bool zero_init = true, - typename Engine0, - typename Layout0, - typename Engine1, - typename Layout1> -__device__ __forceinline__ void reduce_max( - Tensor const& tensor, - Tensor& max) { - MaxOp max_op; - reduce_(tensor, max, max_op); -} - -template < - bool zero_init = true, - bool warp_reduce = true, - typename Engine0, - typename Layout0, - typename Engine1, - typename Layout1> -__device__ __forceinline__ void reduce_sum( - Tensor const& tensor, - Tensor& sum) { - SumOp sum_op; - thread_reduce_(tensor, sum, sum_op); - if constexpr (warp_reduce) { - quad_allreduce_(sum, sum, sum_op); - } -} - -// Apply the exp to all the elements. -template < - bool Scale_max = true, - bool Check_inf = true, - int Max_offset = 0, - typename Engine0, - typename Layout0, - typename Engine1, - typename Layout1> -__forceinline__ __device__ void scale_apply_exp2( - Tensor& tensor, - Tensor const& max, - const float scale) { - // For FP8, we can subtract max by 8.0 so that the value after exp2 is in the - // range of [0, 256]. This lets us use more of the FP8 range (instead of just - // [0, 1]) to reduce underflow. - static constexpr float max_offset = - float(Max_offset); // We can only template on int, not float - static_assert(Layout0::rank == 2, "Only support 2D Tensor"); - static_assert(Layout1::rank == 1, "Only support 1D Tensor"); - CUTE_STATIC_ASSERT_V(size<0>(max) == size<0>(tensor)); -#pragma unroll - for (int mi = 0; mi < size<0>(tensor); ++mi) { - // If max is -inf, then all elements must have been -inf (possibly due to - // masking). We don't want (-inf - (-inf)) since that would give NaN. - const float max_scaled = Check_inf - ? (max(mi) == -INFINITY - ? 0.f - : (!Scale_max ? max(mi) : max(mi) * scale) - max_offset) - : (!Scale_max ? max(mi) : max(mi) * scale) - max_offset; -#pragma unroll - for (int ni = 0; ni < size<1>(tensor); ++ni) { - // Instead of computing exp(x - max), we compute exp2(x * log_2(e) - - // max * log_2(e)). This allows the compiler to use the ffma - // instruction instead of fadd and fmul separately. - tensor(mi, ni) = exp2f(tensor(mi, ni) * scale - max_scaled); - } - } -} - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template -struct Softmax { - using TensorT = decltype(make_tensor(Shape>{})); - TensorT row_max, row_sum; - float const softmax_scale_log2; - - CUTLASS_DEVICE Softmax(float const softmax_scale_log2_) - : softmax_scale_log2(softmax_scale_log2_) {}; - - template - __forceinline__ __device__ TensorT max_get_scale(Tensor0& acc_s) { - // Reshape acc_s from ((2, 2, V), MMA_M, MMA_N) to (nrow=(2, MMA_M), - // ncol=(2, V, MMA_N)) - Tensor scores = make_tensor( - acc_s.data(), hstu::convert_layout_acc_rowcol(acc_s.layout())); - static_assert(CUTE_STATIC_V(size<0>(scores)) == kNRows); - TensorT scores_scale; - if constexpr (Is_first) { - hstu::template reduce_max(scores, row_max); - cute::fill(scores_scale, 1.f); - } else { - Tensor scores_max_prev = make_fragment_like(row_max); - cute::copy(row_max, scores_max_prev); - hstu::template reduce_max(scores, row_max); -#pragma unroll - for (int mi = 0; mi < size(row_max); ++mi) { - float scores_max_cur = !Check_inf - ? row_max(mi) - : (row_max(mi) == -INFINITY ? 0.0f : row_max(mi)); - scores_scale(mi) = - exp2f((scores_max_prev(mi) - scores_max_cur) * softmax_scale_log2); - row_sum(mi) *= scores_scale(mi); - } - } - return scores_scale; - }; - - template - __forceinline__ __device__ void online_softmax(Tensor0& acc_s) { - // Reshape acc_s from ((2, 2, V), MMA_M, MMA_N) to (nrow=(2, MMA_M), - // ncol=(2, V, MMA_N)) - Tensor scores = make_tensor( - acc_s.data(), hstu::convert_layout_acc_rowcol(acc_s.layout())); - static_assert(CUTE_STATIC_V(size<0>(scores)) == kNRows); - hstu::template scale_apply_exp2( - scores, row_max, softmax_scale_log2); - // We don't do the reduce across threads here since we don't need to use the - // row_sum. We do that reduce at the end when we need to normalize the - // softmax. - hstu::reduce_sum( - scores, row_sum); - }; - - __forceinline__ __device__ TensorT finalize(float const final_scale = 1.f) { - SumOp sum_op; - quad_allreduce_(row_sum, row_sum, sum_op); - TensorT scores_scale; -#pragma unroll - for (int mi = 0; mi < size(row_sum); ++mi) { - float sum = row_sum(mi); - float inv_sum = (sum == 0.f || sum != sum) ? 0.f : 1.f / sum; - scores_scale(mi) = inv_sum * final_scale; - // For FP8, we might have scaled the output of exp by 2**8 so we need to - // divide sum by that amount. - if constexpr (Max_offset != 0) { - static constexpr float sum_scale = 1.f / float(1 << Max_offset); - sum *= sum_scale; - } - row_sum(mi) = (sum == 0.f || sum != sum) - ? -INFINITY - : row_max(mi) * (softmax_scale_log2 * float(M_LN2)) + __logf(sum); - } - return scores_scale; - }; - - template - __forceinline__ __device__ void rescale_o( - Tensor1& acc_o, - TensorT const& scores_scale) { - // Reshape acc_o from (MMA=4, MMA_M, MMA_K) to (nrow=(2, MMA_M), ncol=(2, - // MMA_K)) - Tensor acc_o_rowcol = make_tensor( - acc_o.data(), hstu::convert_layout_acc_rowcol(acc_o.layout())); - static_assert(CUTE_STATIC_V(size<0>(acc_o_rowcol)) == kNRows); -#pragma unroll - for (int mi = 0; mi < size<0>(acc_o_rowcol); ++mi) { -#pragma unroll - for (int ni = 0; ni < size<1>(acc_o_rowcol); ++ni) { - acc_o_rowcol(mi, ni) *= scores_scale(mi); - } - } - }; -}; - -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/static_switch.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/static_switch.h deleted file mode 100644 index c5759c9d2..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/static_switch.h +++ /dev/null @@ -1,135 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Inspired by -// https://github.com/NVIDIA/DALI/blob/main/include/dali/core/static_switch.h -// and https://github.com/pytorch/pytorch/blob/master/aten/src/ATen/Dispatch.h - -#pragma once - -/// @param COND - a boolean expression to switch by -/// @param CONST_NAME - a name given for the constexpr bool variable. -/// @param ... - code to execute for true and false -/// -/// Usage: -/// ``` -/// BOOL_SWITCH(flag, BoolConst, [&] { -/// some_function(...); -/// }); -/// ``` -// - -#define BOOL_SWITCH(COND, CONST_NAME, ...) \ - [&] { \ - if (COND) { \ - constexpr static bool CONST_NAME = true; \ - return __VA_ARGS__(); \ - } else { \ - constexpr static bool CONST_NAME = false; \ - return __VA_ARGS__(); \ - } \ - }() - -#ifdef FLASHATTENTION_DISABLE_LOCAL -#define CAUSAL_LOCAL_SWITCH( \ - CAUSAL_COND, LOCAL_COND, CAUSAL_CONST_NAME, LOCAL_CONST_NAME, ...) \ - [&] { \ - constexpr static bool LOCAL_CONST_NAME = false; \ - if (CAUSAL_COND) { \ - constexpr static bool CAUSAL_CONST_NAME = true; \ - return __VA_ARGS__(); \ - } else { \ - constexpr static bool CAUSAL_CONST_NAME = false; \ - return __VA_ARGS__(); \ - } \ - }() -#else -#define CAUSAL_LOCAL_SWITCH( \ - CAUSAL_COND, LOCAL_COND, CAUSAL_CONST_NAME, LOCAL_CONST_NAME, ...) \ - [&] { \ - if (CAUSAL_COND) { \ - constexpr static bool CAUSAL_CONST_NAME = true; \ - constexpr static bool LOCAL_CONST_NAME = false; \ - return __VA_ARGS__(); \ - } else if (LOCAL_COND) { \ - constexpr static bool CAUSAL_CONST_NAME = false; \ - constexpr static bool LOCAL_CONST_NAME = true; \ - return __VA_ARGS__(); \ - } else { \ - constexpr static bool CAUSAL_CONST_NAME = false; \ - constexpr static bool LOCAL_CONST_NAME = false; \ - return __VA_ARGS__(); \ - } \ - }() -#endif - -#ifdef FLASHATTENTION_DISABLE_CLUSTER -#define CLUSTER_SWITCH(COND, CONST_NAME, ...) \ - [&] { \ - constexpr static bool CONST_NAME = false; \ - return __VA_ARGS__(); \ - }() -#else -#define CLUSTER_SWITCH BOOL_SWITCH -#endif - -// #ifdef FLASHATTENTION_DISABLE_SM8x -#define ARCH_SWITCH(ARCH, ARCH_NAME, ...) \ - [&] { \ - constexpr static int ARCH_NAME = 90; \ - return __VA_ARGS__(); \ - }() -// #else -// #define ARCH_SWITCH(ARCH, ARCH_NAME, ...) \ -// [&] { \ -// if (ARCH < 90) { \ -// constexpr static int ARCH_NAME = 80; \ -// return __VA_ARGS__(); \ -// } else { \ -// constexpr static int ARCH_NAME = 90; \ -// return __VA_ARGS__(); \ -// } \ -// }() -// #endif - -#ifndef FLASHATTENTION_ENABLE_VCOLMAJOR -#define VCOLMAJOR_SWITCH(COND, CONST_NAME, ...) \ - [&] { \ - constexpr static bool CONST_NAME = false; \ - return __VA_ARGS__(); \ - }() -#else -#define VCOLMAJOR_SWITCH BOOL_SWITCH -#endif - -#define HEADDIM_SWITCH(HEADDIM, ...) \ - [&] { \ - if (HEADDIM == 64) { \ - constexpr static int kHeadSize = 64; \ - return __VA_ARGS__(); \ - } else if (HEADDIM == 96) { \ - constexpr static int kHeadSize = 96; \ - return __VA_ARGS__(); \ - } else if (HEADDIM == 128) { \ - constexpr static int kHeadSize = 128; \ - return __VA_ARGS__(); \ - } else if (HEADDIM == 96) { \ - constexpr static int kHeadSize = 96; \ - return __VA_ARGS__(); \ - } else if (HEADDIM == 256) { \ - constexpr static int kHeadSize = 256; \ - return __VA_ARGS__(); \ - } \ - }() diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/tile_scheduler.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/tile_scheduler.h deleted file mode 100644 index cde7837ce..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/tile_scheduler.h +++ /dev/null @@ -1,616 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/****************************************************************************** - * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, - *Pradeep Ramani, Tri Dao. - ******************************************************************************/ - -#pragma once - -#include "cute/tensor.hpp" -#include "cutlass/arch/barrier.h" -#include "cutlass/fast_math.h" - -#include "named_barrier.h" - -namespace hstu { - -/////////////////////////////////////////////////////////////////////////////// - -// Host side kernel arguments -struct TileSchedulerArguments { - int const num_blocks, num_head, num_batch; - int const max_seq_len, headdim, - element_size; // Used to calculate L2 swizzling - int* const tile_count_semaphore = nullptr; - int* const seq_offsets = nullptr; - int* const sort_by_length_indices = nullptr; -}; - -/////////////////////////////////////////////////////////////////////////////// - -template < - bool Jagged = false, - int kBlock = 128, - bool Sort_by_length_indices = false> -class SingleTileScheduler { - public: - using SharedStorage = int; - - // Device side kernel params - struct Params { - int const num_blocks, num_head, num_batch; - int const max_seq_len; - int* const seq_offsets; - int* const sort_by_length_indices; - }; - - static Params to_underlying_arguments(TileSchedulerArguments const& args) { - return { - args.num_blocks, - args.num_head, - args.num_batch, - args.max_seq_len, - !Jagged ? nullptr : args.seq_offsets, - !Sort_by_length_indices ? nullptr : args.sort_by_length_indices}; - } - - static dim3 get_grid_shape(Params const& params, int num_sm) { -#ifdef HSTU_FLASH_ATTN_DEBUG_INFO - std::printf( - "SingleTileScheduler::get_grid_shape: %d, %d, %d\n", - params.num_blocks, - params.num_head, - params.num_batch); -#endif - return { - uint32_t(params.num_blocks), - uint32_t(params.num_head), - uint32_t(params.num_batch)}; - } - - struct WorkTileInfo { - int block_idx = 0; - int bidh = 0; - int bidb = 0; - bool is_valid_tile = false; - - CUTLASS_DEVICE - bool is_valid(Params const& params) const { - return is_valid_tile; - } - - CUTLASS_DEVICE - cute::tuple get_block_coord( - Params const& params) const { - return {block_idx, bidh, bidb, 0 /*split_idx*/}; - } - }; - - CUTLASS_DEVICE - SingleTileScheduler(SharedStorage* const smem_scheduler) {} - - template - CUTLASS_DEVICE WorkTileInfo get_initial_work(Params const& params) const { - int bidb = int(blockIdx.z); - if constexpr (Sort_by_length_indices) { - bidb = params.sort_by_length_indices[bidb]; - } - WorkTileInfo work_info{int(blockIdx.x), int(blockIdx.y), bidb, true}; - if constexpr (Jagged) { - int seqlen = - (params.seq_offsets ? params.seq_offsets[work_info.bidb + 1] - - params.seq_offsets[work_info.bidb] - : params.max_seq_len); - work_info.is_valid_tile = work_info.block_idx * kBlock < seqlen; - } - return work_info; - } - - CUTLASS_DEVICE - void init_consumer() const {} - - CUTLASS_DEVICE - void prefetch_next_work(Params const& params, WorkTileInfo& current_work) - const {} - - template - CUTLASS_DEVICE WorkTileInfo - get_next_work(Params const& params, WorkTileInfo const& current_work) const { - return {-1, -1, -1, false}; - } -}; - -/////////////////////////////////////////////////////////////////////////////// - -class StaticPersistentTileScheduler { - public: - using SharedStorage = int; - - // Device side kernel params - struct Params { - int total_blocks; - cutlass::FastDivmod m_block_divmod, head_divmod; - cutlass::FastDivmod nsplits_divmod; - }; - - static Params to_underlying_arguments(TileSchedulerArguments const& args) { - return { - args.num_blocks * args.num_head * args.num_batch, - cutlass::FastDivmod(args.num_blocks), - cutlass::FastDivmod(args.num_head), - cutlass::FastDivmod(1)}; - } - - static dim3 get_grid_shape(Params const& params, int num_sm) { -#ifdef HSTU_FLASH_ATTN_DEBUG_INFO - std::printf("StaticPersistentTileScheduler::get_grid_shape %d\n", num_sm); -#endif - return {uint32_t(num_sm)}; - } - - struct WorkTileInfo { - int tile_idx; - - CUTLASS_DEVICE - bool is_valid(Params const& params) const { - return tile_idx < params.total_blocks; - } - - CUTLASS_DEVICE - cute::tuple get_block_coord( - Params const& params) const { - int block, bidh, bidb; - bidb = params.head_divmod.divmod( - bidh, params.m_block_divmod.divmod(block, tile_idx)); - int split_idx = 0; - return {block, bidh, bidb, split_idx}; - } - }; - - CUTLASS_DEVICE - StaticPersistentTileScheduler(SharedStorage* const smem_scheduler) {}; - - template - CUTLASS_DEVICE WorkTileInfo get_initial_work(Params const& params) const { - return {int(blockIdx.x)}; - } - - CUTLASS_DEVICE - void init_consumer() const {} - - CUTLASS_DEVICE - void prefetch_next_work(Params const& params, WorkTileInfo& current_work) - const {} - - template - CUTLASS_DEVICE WorkTileInfo - get_next_work(Params const& params, WorkTileInfo const& current_work) const { - return {current_work.tile_idx + int(gridDim.x)}; - } -}; - -template < - int NumMmaThreads = 2 * cutlass::NumThreadsPerWarpGroup, - int NumProducerThreads = cutlass::NumThreadsPerWarp, - bool WarpSpecialized = true> -class DynamicPersistentTileScheduler { - // This scheduler targets the causal (or local) case where each tile takes - // different amount of time. We use longest-processing-time-first scheduling: - // the longest remaining tile is assigned to the first SM that's free. - // SM indicates they are free by incrementing a semaphore. - // However, we have to make sure K & V still fit into L2 cache, so we perform - // scheduling on "sections" of the head & batch dimension, each section - // consisting of e.g. 8 heads. This is the L2 swizzling part. The size of each - // section is precomputed based on the size of K & V and the L2 cache size. - - static_assert(WarpSpecialized || NumProducerThreads == NumMmaThreads); - static constexpr int NumThreads = - WarpSpecialized ? NumMmaThreads + NumProducerThreads : NumMmaThreads; - - public: - using SharedStorage = int; - - protected: - SharedStorage* const tile_count_smem; - - public: - // Device side kernel params - struct Params { - int const total_blocks; - cutlass::FastDivmod const m_block_divmod, head_divmod; - cutlass::FastDivmod const l2_minor_divmod, l2_major_divmod; - cutlass::FastDivmod const l2_minor_residual_divmod; - int const num_hb_quotient; - int* const tile_count_semaphore; - }; - - static Params to_underlying_arguments(TileSchedulerArguments const& args) { - int const size_one_kv_head = - args.max_seq_len * args.headdim * args.element_size * 2; - int const size_l2 = 32 * 1024 * 1024; // 32 MB for K & V - // Swizzle is the size of each "section". Round swizzle to a power of 2 - // If not PackGQA already, the size of each section can increase by - // qhead_per_khead - int const swizzle = (1 << cutlass::find_log2(size_l2 / size_one_kv_head)); - // If we're in the last section (called residual), we don't want to divide - // by swizzle. Instead we want to divide by the remainder. - int const num_hb_remainder = (args.num_head * args.num_batch) % swizzle; - int const num_split_blocks = args.num_blocks; - // printf("num_split_blocks = %d, num_head = %d, num_batch = %d, swizzle = - // %d, PackGQA = %d, qhead_per_khead = %d, num_hb_remainder = %d\n", - // num_split_blocks, args.num_head, args.num_batch, swizzle, int(PackGQA), - // args.qhead_per_khead, num_hb_remainder); - assert(args.tile_count_semaphore != nullptr); - return { - num_split_blocks * args.num_head * args.num_batch, - cutlass::FastDivmod(args.num_blocks), - cutlass::FastDivmod(args.num_head), - cutlass::FastDivmod(swizzle), - cutlass::FastDivmod(swizzle * num_split_blocks), - // don't divide by 0 - cutlass::FastDivmod(num_hb_remainder > 0 ? num_hb_remainder : 1), - (args.num_head * args.num_batch) / swizzle, - args.tile_count_semaphore}; - } - - static dim3 get_grid_shape(Params const& params, int num_sm) { -#ifdef HSTU_FLASH_ATTN_DEBUG_INFO - std::printf("DynamicPersistentTileScheduler::get_grid_shape %d\n", num_sm); -#endif - return {uint32_t(num_sm)}; - } - - struct WorkTileInfo { - int tile_idx; - - CUTLASS_DEVICE - bool is_valid(Params const& params) const { - return tile_idx < params.total_blocks; - } - - CUTLASS_DEVICE - cute::tuple get_block_coord( - Params const& params) const { - int block, bidh, bidb; - int l2_mod, bidhb, bidhb_residual; - bidhb = params.l2_major_divmod.divmod(l2_mod, tile_idx); - // If we're in the last section (called residual), we don't want to divide - // by swizzle. Instead we want to divide by the remainder. - if (bidhb < params.num_hb_quotient) { - block = params.l2_minor_divmod.divmod(bidhb_residual, l2_mod); - } else { - block = params.l2_minor_residual_divmod.divmod(bidhb_residual, l2_mod); - } - bidb = params.head_divmod.divmod( - bidh, bidhb * params.l2_minor_divmod.divisor + bidhb_residual); - int split_idx = 0; - // Longest-processing-time-first - block = params.m_block_divmod.divisor - 1 - block; - return {block, bidh, bidb, split_idx}; - } - }; - - CUTLASS_DEVICE - DynamicPersistentTileScheduler(SharedStorage* const smem_scheduler) - : tile_count_smem(smem_scheduler) {}; - - template - CUTLASS_DEVICE WorkTileInfo get_initial_work(Params const& params) const { - return {int(blockIdx.x)}; - } - - CUTLASS_DEVICE - void init_consumer() const { - if (WarpSpecialized || cutlass::canonical_warp_idx_sync() > 0) { - hstu::named_barrier_arrive( - NumThreads, - static_cast(FwdNamedBarriers::TileCountSmemEmpty) /*id*/); - } - } - - CUTLASS_DEVICE - void prefetch_next_work(Params const& params, WorkTileInfo& current_work) - const { - if (threadIdx.x % NumProducerThreads == 0) { - current_work.tile_idx = - atomicAdd(params.tile_count_semaphore, 1) + int(gridDim.x); - } - } - - template - CUTLASS_DEVICE WorkTileInfo - get_next_work(Params const& params, WorkTileInfo const& current_work) const { - if constexpr (IsProducerWarp) { - // thread 0 already has the right tile_idx, just need to broadcast to the - // rest of warp 0 - int new_tile_idx = - __shfl_sync(0xffffffff, current_work.tile_idx, 0 /*lane*/); - hstu::named_barrier_sync( - NumThreads, - static_cast(FwdNamedBarriers::TileCountSmemEmpty) /*id*/); - if (threadIdx.x % NumProducerThreads == 0) { - *tile_count_smem = current_work.tile_idx; - } - hstu::named_barrier_arrive( - NumThreads, - static_cast(FwdNamedBarriers::TileCountSmemFull) /*id*/); - return {new_tile_idx}; - } else { - hstu::named_barrier_sync( - NumThreads, - static_cast(FwdNamedBarriers::TileCountSmemFull) /*id*/); - int tile_idx = *tile_count_smem; - hstu::named_barrier_arrive( - NumThreads, - static_cast(FwdNamedBarriers::TileCountSmemEmpty) /*id*/); - return {tile_idx}; - } - } -}; - -template < - int kBlock, - int NumMmaThreads = 2 * cutlass::NumThreadsPerWarpGroup, - int NumProducerThreads = cutlass::NumThreadsPerWarp, - bool WarpSpecialized = true> -class VarlenDynamicPersistentTileScheduler { - static_assert(WarpSpecialized || NumProducerThreads == NumMmaThreads); - static constexpr int NumThreads = - WarpSpecialized ? NumMmaThreads + NumProducerThreads : NumMmaThreads; - - public: - using SharedStorage = int4; - - protected: - SharedStorage* const work_info_smem; - - public: - // Device side kernel params - struct Params { - int num_head, num_batch; - int const max_seq_len; - cutlass::FastDivmod nsplits_divmod; - int* const tile_count_semaphore; - int* const seq_offsets; - }; - - static Params to_underlying_arguments(TileSchedulerArguments const& args) { - // If Split, for the purpose of scheduling, we pretend that instead there - // are (args.num_splits * args.num_head) number of heads. - assert(args.tile_count_semaphore != nullptr); - return { - args.num_head, - args.num_batch, - args.max_seq_len, - cutlass::FastDivmod(1), - args.tile_count_semaphore, - args.seq_offsets}; - } - - static dim3 get_grid_shape(Params const& params, int num_sm) { -#ifdef HSTU_FLASH_ATTN_DEBUG_INFO - std::printf( - "VarlenDynamicPersistentTileScheduler::get_grid_shape %d\n", num_sm); -#endif - return {uint32_t(num_sm)}; - } - - struct WorkTileInfo { - int tile_idx, block, bidh, bidb; - - CUTLASS_DEVICE - bool is_valid(Params const& params) const { - // if (blockIdx.x >= 0 && (threadIdx.x == 128 || threadIdx.x == 0)) { - // printf("blockIdx.x = %d, threadIdx.x = %d, checking valid, bidb = %d, - // params.num_batch = %d\n", blockIdx.x, threadIdx.x, bidb, - // params.num_batch); } - return bidb < params.num_batch; - } - - CUTLASS_DEVICE - cute::tuple get_block_coord( - Params const& params) const { - return {block, bidh, bidb, 0 /*split_idx*/}; - } - }; - - CUTLASS_DEVICE - VarlenDynamicPersistentTileScheduler(SharedStorage* const smem_scheduler) - : work_info_smem(smem_scheduler) {}; - - CUTLASS_DEVICE - WorkTileInfo tile_idx_to_work_tile( - Params const& params, - int next_tile_idx, - WorkTileInfo const& current_work) const { - auto prefix_sum = [](int val) { - auto lane = threadIdx.x % cutlass::NumThreadsPerWarp; - CUTLASS_PRAGMA_UNROLL - for (int i = 1; i < cutlass::NumThreadsPerWarp; i <<= 1) { - int32_t partial_sum = __shfl_up_sync(0xffffffff, val, i); - if (lane >= i) { - val += partial_sum; - } - } - return val; - }; - - auto get_num_m_blocks = [&](int bidb_start) { - auto lane = threadIdx.x % cutlass::NumThreadsPerWarp; - int seqlen; - if (params.seq_offsets) { - int cur_cu_seqlen = lane + bidb_start <= params.num_batch - ? params.seq_offsets[lane + bidb_start] - : 0; - int next_cu_seqlen = __shfl_down_sync(0xffffffff, cur_cu_seqlen, 1); - seqlen = next_cu_seqlen - cur_cu_seqlen; - } else { - seqlen = params.max_seq_len; - } - return lane + bidb_start < params.num_batch && - lane < cutlass::NumThreadsPerWarp - 1 - ? cute::ceil_div(seqlen, kBlock) - : 0; - }; - - int num_m_blocks = - get_num_m_blocks(current_work.bidb); // Different for each lane - // Cumulative number of blocks for the next 31 batches - int num_m_blocks_cumulative = prefix_sum(num_m_blocks); - // Total number of blocks for the next 31 batches - int m_blocks_in_group = __shfl_sync( - 0xffffffff, num_m_blocks_cumulative, cutlass::NumThreadsPerWarp - 1); - int group_end_tile = current_work.tile_idx - current_work.block - - current_work.bidh * __shfl_sync(0xffffffff, num_m_blocks, 0 /*lane*/) + - m_blocks_in_group * params.num_head; // Same for all lanes - int bidb = current_work.bidb; - // if (blockIdx.x <= 9 && threadIdx.x == 0) { - // printf("Before while, blockIdx.x = %d, threadIdx.x = %d, bidb = %d, - // num_m_blocks = %d, next_tile_idx = %d, group_end_tile = %d, - // m_blocks_in_group = %d\n", blockIdx.x, threadIdx.x, bidb, - // num_m_blocks, next_tile_idx, group_end_tile, m_blocks_in_group); - // } - while (group_end_tile <= next_tile_idx) { - bidb += cutlass::NumThreadsPerWarp - 1; - if (bidb >= params.num_batch) { - // if (blockIdx.x <= 9 && threadIdx.x == 0) { - // printf("Returning early, blockIdx.x = %d, threadIdx.x = %d, bidb - // = %d, num_m_blocks = %d, next_tile_idx = %d, group_end_tile = %d, - // m_blocks_in_group = %d\n", blockIdx.x, threadIdx.x, bidb, - // num_m_blocks, next_tile_idx, group_end_tile, m_blocks_in_group); - // } - return {next_tile_idx, 0, 0, params.num_batch}; - } - num_m_blocks = get_num_m_blocks(bidb); - num_m_blocks_cumulative = prefix_sum(num_m_blocks); - m_blocks_in_group = __shfl_sync( - 0xffffffff, num_m_blocks_cumulative, cutlass::NumThreadsPerWarp - 1); - group_end_tile += m_blocks_in_group * params.num_head; - // if (blockIdx.x <= 9 && threadIdx.x == 0) { - // printf("Bottom of while, blockIdx.x = %d, threadIdx.x = %d, bidb = - // %d, num_m_blocks = %d, next_tile_idx = %d, group_end_tile = %d, - // m_blocks_in_group = %d\n", blockIdx.x, threadIdx.x, bidb, - // num_m_blocks, next_tile_idx, group_end_tile, m_blocks_in_group); - // } - } - int group_start_tile = group_end_tile - m_blocks_in_group * params.num_head; - // The next problem to process is the first one that does not have ending - // tile position that is greater than or equal to tile index. - int batch_idx_in_group = __popc(__ballot_sync( - 0xffffffff, - group_start_tile + num_m_blocks_cumulative * params.num_head <= - next_tile_idx)); - bidb += batch_idx_in_group; - num_m_blocks = __shfl_sync(0xffffffff, num_m_blocks, batch_idx_in_group); - int mh_block = next_tile_idx - group_start_tile - - (batch_idx_in_group == 0 ? 0 - : __shfl_sync( - 0xffffffff, - num_m_blocks_cumulative, - batch_idx_in_group - 1)) * - params.num_head; - int bidh = mh_block / num_m_blocks; - int block = mh_block - bidh * num_m_blocks; - // if (blockIdx.x <= 9 && threadIdx.x == 0) { - // printf("blockIdx.x = %d, threadIdx.x = %d, batch_idx_in_group = %d, - // bidb = %d, num_m_blocks = %d, next_tile_idx = %d, group_end_tile = - // %d, m_blocks_in_group = %d, mh_block = %d, bidh = %d, block = %d\n", - // blockIdx.x, threadIdx.x, batch_idx_in_group, bidb, num_m_blocks, - // next_tile_idx, group_end_tile, m_blocks_in_group, mh_block, bidh, - // block); - // } - return {next_tile_idx, block, bidh, bidb}; - } - - template - CUTLASS_DEVICE WorkTileInfo get_initial_work(Params const& params) const { - if constexpr (IsProducerWarp) { - WorkTileInfo work_info = - tile_idx_to_work_tile(params, int(blockIdx.x), {0, 0, 0, 0}); - if (threadIdx.x % cutlass::NumThreadsPerWarp == 0) { - *work_info_smem = make_int4( - work_info.tile_idx, - work_info.block, - work_info.bidh, - work_info.bidb); - } - hstu::named_barrier_arrive( - NumThreads, - static_cast(FwdNamedBarriers::TileCountSmemFull) /*id*/); - return work_info; - } else { - return get_next_work(params, {0, 0, 0, 0}); - } - } - - CUTLASS_DEVICE - void init_consumer() const { - // Don't arrive at the TileCountSmemEmpty barrier here, because - // get_initial_work will do that - } - - CUTLASS_DEVICE - void prefetch_next_work(Params const& params, WorkTileInfo& current_work) - const { - if (threadIdx.x % NumProducerThreads == 0) { - current_work.tile_idx = - atomicAdd(params.tile_count_semaphore, 1) + int(gridDim.x); - } - } - - template - CUTLASS_DEVICE WorkTileInfo - get_next_work(Params const& params, WorkTileInfo const& current_work) const { - if constexpr (IsProducerWarp) { - // thread 0 has the next tile_idx, just need to broadcast to the rest of - // warp 0 - int new_tile_idx = - __shfl_sync(0xffffffff, current_work.tile_idx, 0 /*lane*/); - WorkTileInfo work_info = { - __shfl_sync(0xffffffff, current_work.tile_idx, 1 /*lane*/), - current_work.block, - current_work.bidh, - current_work.bidb}; - work_info = tile_idx_to_work_tile(params, new_tile_idx, work_info); - hstu::named_barrier_sync( - NumThreads, - static_cast(FwdNamedBarriers::TileCountSmemEmpty) /*id*/); - if (threadIdx.x % cutlass::NumThreadsPerWarp == 0) { - *work_info_smem = make_int4( - work_info.tile_idx, - work_info.block, - work_info.bidh, - work_info.bidb); - } - hstu::named_barrier_arrive( - NumThreads, - static_cast(FwdNamedBarriers::TileCountSmemFull) /*id*/); - return work_info; - } else { - hstu::named_barrier_sync( - NumThreads, - static_cast(FwdNamedBarriers::TileCountSmemFull) /*id*/); - int4 work_info = *work_info_smem; - hstu::named_barrier_arrive( - NumThreads, - static_cast(FwdNamedBarriers::TileCountSmemEmpty) /*id*/); - return WorkTileInfo{work_info.x, work_info.y, work_info.z, work_info.w}; - } - } -}; - -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/tile_size.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/tile_size.h deleted file mode 100644 index 3c8968bda..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/tile_size.h +++ /dev/null @@ -1,220 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/****************************************************************************** - * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, - *Pradeep Ramani, Tri Dao. - ******************************************************************************/ - -#pragma once - -#include - -namespace hstu { - -constexpr int kBlockM_bwd( - const int arch, - const int headdim, - const bool causal, - const bool is_local) { - int const kBlockM_sm90 = headdim <= 64 - ? 64 - : (headdim <= 96 - ? 64 - : (headdim <= 128 ? (causal || is_local ? 64 : 80) : 64)); - int const kBlockM_sm80 = headdim <= 64 ? 128 : 64; - int const kBlockM = arch >= 90 ? kBlockM_sm90 : kBlockM_sm80; - return kBlockM; -} - -constexpr int kBlockN_bwd(const int arch, const int headdim) { - int const kBlockN_sm90 = headdim <= 128 ? 128 : (headdim <= 192 ? 96 : 80); - int const kBlockN_sm80 = headdim <= 128 ? 128 : (headdim <= 192 ? 80 : 64); - int const kBlockN = arch >= 90 ? kBlockN_sm90 : kBlockN_sm80; - return kBlockN; -} - -constexpr int NumMmaWarpGroups_bwd(const int arch, const int headdim) { - if (headdim <= 128) { - return 2; - } else if (headdim == 192) { - return arch >= 90 ? 3 : 2; - } else { - return 2; - } -} - -constexpr bool V_in_regs_bwd(const int arch, const int headdim) { - if (arch >= 90 && headdim == 96) { - return true; - } - return false; -} - -// Stages_dO, Stages_dS_or_QSm80 -constexpr std::tuple Stages_bwd(const int arch, const int headdim) { - if (headdim <= 128) { - return {2, 2}; - } - if (headdim == 192) { - if (arch >= 90) { - return {1, 1}; - } else { - return {1, 2}; - } - } else { - return {1, 1}; - } -} - -// AtomLayoutMSdP, AtomLayoutNdKV, AtomLayoutMdQ -constexpr std::tuple AtomLayout_bwd( - const int arch, - const int headdim) { - if (headdim <= 64) { - if (arch >= 90) { - return {1, 2, 1}; - } else { - return {4, 4, 4}; - } - } else if (headdim <= 96) { - if (arch >= 90) { - return {1, 2, 1}; - } else { - return {2, 4, 2}; - } - } else if (headdim <= 128) { - if (arch >= 90) { - return {1, 2, 1}; - } else { - return {2, 2, 2}; - } - } else { - if (arch >= 90) { - return {1, 1, 1}; - } else { - return {4, 2, 2}; - } - } -} - -// SdP_swapAB, dKV_swapAB, dQ_swapAB -constexpr std::tuple swapAB_bwd( - const int arch, - const int headdim, - const bool causal, - const bool local) { - if (headdim <= 96) { - return {arch >= 90 ? true : false, false, false}; - } else if (headdim == 128) { - bool SdP_swapAB = arch >= 90 ? true : false; - bool dKV_swapAB = false; - bool dQ_swapAB = arch >= 90 ? ((causal || local) ? false : true) : false; - return {SdP_swapAB, dKV_swapAB, dQ_swapAB}; - } else if (headdim == 192) { - return {false, true, false}; - } else { - return {false, arch >= 90 ? true : false, arch >= 90 ? true : false}; - } -} - -// Return {kBlockM, kBlockN, Mma1_is_RS} -constexpr std::tuple tile_size_fwd_sm90( - int headdim, - bool is_causal, - bool is_local, - int element_size = 2, - bool v_colmajor = false, - bool Cross = false, - bool Training = true) { - // for cross attention, q is usually much smaller than k/v, so we reduce the - // BlockM size to increase parallelism - bool small_blockm = Cross && (!Training); - if (element_size == 2) { - if (headdim <= 64) { - return {small_blockm ? 64 : 192, 128, true}; - // Good for long seqlen (>= 4k) but suffers from tile quantization at - // short seqlen return {192, is_causal || is_local ? 192 : 176, true, - // false}; - } else if (headdim <= 96) { - return {small_blockm ? 64 : 192, is_local ? 128 : 144, false}; - } else if (headdim <= 128) { - return {small_blockm ? 64 : 128, is_causal || is_local ? 128 : 176, true}; - // {128, 192, false, false} and {192, 128, false, true} are quite good too - // 128 x 192 hits the limit of smem if Mma1_is_RS, 128 x 144 hits the - // limit if !Mma1_is_RS - } else if (headdim <= 192) { - return { - small_blockm ? 64 : 128, - is_local ? 96 : 112, - true}; // 128 x 112 hits the limit of smem - } else { - return { - small_blockm ? 64 : 128, - is_local ? 64 : 80, - true}; // 128 x 80 hits the limit of smem - } - } else { - if (headdim <= 64) { - return {192, 160, true}; - } else if (headdim <= 96) { - return {192, 128, true}; - } else if (headdim <= 128) { - return {128, (v_colmajor ? 192 : 224), true}; - } else if (headdim <= 192) { - return {128, 160, true}; - } else { - return {128, is_local ? 64 : 128, true}; - } - } -} - -// Return {kBlockM, kBlockN, kNWarps, kStages, Q_in_regs} -constexpr std::tuple tile_size_fwd_sm8x( - bool sm86_or_89, - int headdim, - bool is_causal, - bool is_local, - int element_size = 2) { - if (element_size == 2) { - if (headdim <= 64) { - return {128, (is_local ? 96 : 112), 4, 1, false}; - } else if (headdim <= 96) { - return {128, is_local ? 48 : 64, 4, 1, false}; - } else if (headdim <= 128) { - bool const use_8_warps = sm86_or_89; - return { - 128, - use_8_warps ? (is_local ? 96 : 128) : (is_local ? 48 : 64), - use_8_warps ? 8 : 4, - 1, - use_8_warps}; - } else if (headdim <= 192) { - bool const kBlockN_64 = is_local; - return {128, kBlockN_64 ? 64 : 96, 8, sm86_or_89 ? 1 : 2, !kBlockN_64}; - } else { - return { - 128, - sm86_or_89 ? (is_local ? 48 : 64) : (is_local ? 64 : 96), - 8, - 1, - false}; - } - } else { - // Placeholder for now - return {128, 64, 8, 2, false}; - } -} -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/utils.h b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/utils.h deleted file mode 100644 index 50a065ed4..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/utils.h +++ /dev/null @@ -1,789 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/****************************************************************************** - * Copyright (c) 2024, Tri Dao. - ******************************************************************************/ - -#pragma once - -#include -#include -#include - -#include - -#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 -#include -#endif - -#include -#include - -#include -#include -#include -#include - -#define CHECK_CUDA(call) \ - do { \ - cudaError_t status_ = call; \ - if (status_ != cudaSuccess) { \ - fprintf( \ - stderr, \ - "CUDA error (%s:%d): %s\n", \ - __FILE__, \ - __LINE__, \ - cudaGetErrorString(status_)); \ - exit(1); \ - } \ - } while (0) - -#define CHECK_CUDA_KERNEL_LAUNCH() CHECK_CUDA(cudaGetLastError()) - -#ifndef M_LOG2E -#define M_LOG2E 1.44269504088896340735992468100 /* log_2 (e) */ -#endif - -namespace hstu { - -using namespace cute; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -// A wrapper for the kernel that is used to guard against compilation on -// architectures that will never use the kernel. The purpose of this is to -// reduce the size of the compiled binary. -// Adapted from -// https://github.com/vllm-project/vllm/blob/4d29e91be84d27ca313d657eee92c067439a4c23/csrc/quantization/cutlass_w8a8/scaled_mm_c2x.cuh#L55 -template -struct enable_sm90_or_later : Kernel { - template - CUTLASS_DEVICE void operator()(Args&&... args) { -#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) - Kernel::operator()(std::forward(args)...); -#endif - } -}; - -template -struct enable_sm80_to_sm89 : Kernel { - template - CUTLASS_DEVICE void operator()(Args&&... args) { -#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 800) && (__CUDA_ARCH__ <= 890) - Kernel::operator()(std::forward(args)...); -#endif - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template -struct MaxOp { - __device__ __forceinline__ T operator()(T const& x, T const& y) { - return x > y ? x : y; - } -}; - -template <> -struct MaxOp { - // This is slightly faster - __device__ __forceinline__ float operator()(float const& x, float const& y) { - return max(x, y); - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template -struct SumOp { - __device__ __forceinline__ T operator()(T const& x, T const& y) { - return x + y; - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template -struct SiluScaleOp { - cutlass::epilogue::thread::SiLu silu; - __device__ __forceinline__ T operator()(T const& t, T const& scale) { - float t2 = t / 2; - return t2 * (1 + cutlass::fast_tanh(t2)) * - scale; // __fdividef(t, 1.0f + cutlass::fast_exp(-t)) * scale - } -}; - -template -CUTLASS_DEVICE void inplace_silu_scale( - Tensor& tensor, - T const& scale_before, - T const& scale_after) { - SiluScaleOp silu_scale_op; -#pragma unroll - for (int i = 0; i < size(tensor); ++i) { - tensor(i) = silu_scale_op(tensor(i) * scale_before, scale_after); - } -} - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template -struct Allreduce { - static_assert(THREADS == 32 || THREADS == 16 || THREADS == 8 || THREADS == 4); - template - static __device__ __forceinline__ T run(T x, Operator& op) { - constexpr int OFFSET = THREADS / 2; - x = op(x, __shfl_xor_sync(uint32_t(-1), x, OFFSET)); - return Allreduce::run(x, op); - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <> -struct Allreduce<2> { - template - static __device__ __forceinline__ T run(T x, Operator& op) { - x = op(x, __shfl_xor_sync(uint32_t(-1), x, 1)); - return x; - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -// For SM80, convert acc_layout from (MMA=4, MMA_M, MMA_N) to (nrow=(2, -// MMA_M), ncol=(2, MMA_N)). For SM90, convert acc_layout from ((2, 2, V), -// MMA_M, MMA_N) to (nrow=(2, MMA_M), ncol=(2, V, MMA_N)) -template -CUTLASS_DEVICE auto convert_layout_acc_rowcol(Layout0 acc_layout) { - if constexpr (decltype(rank<0>(acc_layout))::value == 3) { // SM90 - static_assert(decltype(size<0, 0>(acc_layout))::value == 2); - static_assert(decltype(size<0, 1>(acc_layout))::value == 2); - static_assert(decltype(rank(acc_layout))::value == 3); - auto l = acc_layout; - if constexpr (!Transposed) { - return make_layout( - make_layout(get<0, 1>(l), get<1>(l)), - make_layout(get<0, 0>(l), get<0, 2>(l), get<2>(l))); - } else { - return make_layout( - make_layout(get<0, 0>(l), get<0, 2>(l), get<2>(l)), - make_layout(get<0, 1>(l), get<1>(l))); - } - - } else { // SM80 - static_assert(decltype(size<0>(acc_layout))::value == 4); - static_assert(decltype(rank(acc_layout))::value == 3); - auto l = logical_divide(acc_layout, Shape<_2>{}); // ((2, 2), MMA_M, MMA_N) - if constexpr (!Transposed) { - return make_layout( - make_layout(get<0, 1>(l), get<1>(l)), - make_layout(get<0, 0>(l), get<2>(l))); - } else { - return make_layout( - make_layout(get<0, 0>(l), get<2>(l)), - make_layout(get<0, 1>(l), get<1>(l))); - } - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -// For SM80, convert acc_layout from (MMA=4, MMA_M, MMA_N) to ((4, 2), MMA_M, -// MMA_N / 2) if using m16n8k16, or to (4, MMA_M, MMA_N) if using m16n8k8. For -// SM90, FP16/BF16, convert acc_layout from ((2, 2, N / 8), MMA_M, MMA_N) to -// ((2, 2, 2), MMA_M, (N / 16, MMA_N)) For SM90, FP8, convert acc_layout from -// ((2, 2, N / 8), MMA_M, MMA_N) to ((4, 2, 2), MMA_M, (N / 32, MMA_N)) -template -CUTLASS_DEVICE auto convert_layout_acc_Aregs(Layout0 acc_layout) { - using X = Underscore; - if constexpr (decltype(rank<0>(acc_layout))::value == 3) { // SM90 - static_assert(decltype(size<0, 0>(acc_layout))::value == 2); - static_assert(decltype(size<0, 1>(acc_layout))::value == 2); - static_assert(decltype(rank(acc_layout))::value == 3); - static_assert(decltype(rank(get<0>(acc_layout)))::value == 3); - if constexpr (sizeof(typename MMA_Traits::ValTypeA) == 2) { - auto l = - logical_divide(get<0, 2>(acc_layout), Tile<_2>{}); // ((2, N / 16)) - return make_layout( - make_layout( - get<0, 0>(acc_layout), get<0, 1>(acc_layout), get<0, 0>(l)), - get<1>(acc_layout), - coalesce(make_layout(get<0, 1>(l), get<2>(acc_layout)))); - } else { - static_assert(sizeof(typename MMA_Traits::ValTypeA) == 1); - static_assert(decltype(stride<0, 0>(acc_layout))::value == 1); - static_assert(decltype(stride<0, 1>(acc_layout))::value == 2); - auto l = logical_divide( - get<0, 2>(acc_layout), - Tile>>{}); // (((2, 2), N / 32)) - // This combines the first two modes (<0, 0> and <0, 1>) into one mode. - // Will require register shuffling later to be correct. - return make_layout( - make_layout(Layout<_4>{}, get<0, 0, 0>(l), get<0, 0, 1>(l)), - get<1>(acc_layout), - coalesce(make_layout( - get<0, 1>(l), - get<2>(acc_layout)))); // ((4, 2, 2), MMA_M, N / 32 * MMA_N) - // This combination is right but doesn't work with register shuffling. - // return make_layout(make_layout(coalesce(make_layout(get<0, - // 0>(acc_layout), get<0, 0, 0>(l))), get<0, 1>(acc_layout), get<0, 0, - // 1>(l)), - // get<1>(acc_layout), - // coalesce(make_layout(get<0, 1>(l), - // get<2>(acc_layout)))); - } - } else { // SM80 - static_assert(decltype(size<0>(acc_layout))::value == 4); - static_assert(decltype(rank(acc_layout))::value == 3); - constexpr int mma_shape_K = get<2>(typename MMA_Traits::Shape_MNK{}); - static_assert(mma_shape_K == 8 || mma_shape_K == 16); - if constexpr (mma_shape_K == 8) { - return acc_layout; - } else { - auto l = logical_divide( - acc_layout, Shape{}); // (4, MMA_M, (2, MMA_N / 2))) - return make_layout( - make_layout(get<0>(l), get<2, 0>(l)), get<1>(l), get<2, 1>(l)); - } - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template -CUTLASS_DEVICE auto convert_type_unsafe(Tensor const& tensor) { - using From_type = typename Engine::value_type; - static constexpr int numel = decltype(size(tensor))::value; - cutlass::NumericArrayConverter convert_op; - // HACK: this requires tensor to be "contiguous" - auto frag = - convert_op(*reinterpret_cast*>( - tensor.data())); - return make_tensor(make_rmem_ptr(&frag), tensor.layout()); - // Unsafe because we're returning a tensor with memory allocated on the - // stack. If the compiler does not inline this function, then the memory - // might not be valid. -} - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template -CUTLASS_DEVICE void convert_type_out( - Tensor const& tensor, - Tensor& out) { - // Somehow if we allocate out inside this function and return it, e2e is - // slower and the output can be wrong. - using From_type = typename Engine::value_type; - using To_type = typename EngineOut::value_type; - static constexpr int FragmentSize = std::max( - sizeof(From_type) / sizeof(To_type), sizeof(To_type) / sizeof(From_type)); - static_assert( - CUTE_STATIC_V(size(tensor)) % FragmentSize == 0, - "Fragment size does not vectorize properly"); - Tensor frag = recast const>(tensor); - Tensor out_frg = recast>(out); - static_assert(size(frag) == size(out_frg)); - cutlass::NumericArrayConverter convert_op; -#pragma unroll - for (int i = 0; i < size(frag); ++i) { - out_frg[i] = convert_op(frag[i]); - } -} - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -// Blocks until all but N previous cp.async.commit_group operations have -// committed. This differs from cute::cp_async_wait in that when N = 0 we -// don't call cp.async.wait_all (which is equivalent to commit_group then -// wait_group 0). Instead we just call cp.async.wait_group 0, which is -// slightly faster. -// https://github.com/NVIDIA/cutlass/blob/master/include/cute/arch/copy_sm80.hpp#L113 -template -CUTE_HOST_DEVICE void cp_async_wait() { -#if defined(CUTE_ARCH_CP_ASYNC_SM80_ENABLED) - asm volatile("cp.async.wait_group %0;\n" ::"n"(N)); -#endif -} - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template -CUTLASS_DEVICE auto mma_partition_fragment_AB( - Mma const& mma, - Tensor0 const& tensor0) { - if constexpr (A) { - return mma.partition_fragment_A(tensor0); - } else { - return mma.partition_fragment_B(tensor0); - } -} - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template < - bool zero_init = false, - int wg_wait = 0, - bool SwapAB = false, - int M_slice = -1, - typename Tensor0, - typename Tensor1, - typename Tensor2, - typename TiledMma> -CUTLASS_DEVICE void gemm( - TiledMma& tiled_mma, - Tensor0 const& tCrA, - Tensor1 const& tCrB, - Tensor2& tCrC) { - if constexpr (M_slice >= 0) { - static constexpr int MMA_M = decltype(size<1>(tCrC))::value; - static_assert(M_slice < MMA_M); - // After logical_divide, C has shape ((2,2,V), (MMA_M, 1), MMA_N) - Tensor tCrC_slice = - cute::logical_divide(tCrC, Shape>{})( - _, make_coord(Int{}, _), _); - if constexpr (!SwapAB) { - Tensor tCrA_slice = - cute::logical_divide(tCrA, Shape>{})( - _, make_coord(Int{}, _), _); - gemm( - tiled_mma, tCrA_slice, tCrB, tCrC_slice); - } else { - Tensor tCrB_slice = - cute::logical_divide(tCrB, Shape>{})( - _, make_coord(Int{}, _), _); - gemm( - tiled_mma, tCrA, tCrB_slice, tCrC_slice); - } - } else { - constexpr bool Is_RS = !cute::is_base_of< - cute::GMMA::DescriptorIterator, - typename TiledMma::FrgTypeA>::value; - // Need to cast away const on tCrA since warpgroup_fence_operand doesn't - // take const - if constexpr (Is_RS) { - if constexpr (!SwapAB) { - warpgroup_fence_operand(const_cast(tCrA)); - } else { - warpgroup_fence_operand(const_cast(tCrB)); - } - } - warpgroup_fence_operand(tCrC); - warpgroup_arrive(); - if constexpr (zero_init) { - tiled_mma.accumulate_ = GMMA::ScaleOut::Zero; - } - // Unroll the K mode manually to set scale D to 1 - CUTLASS_PRAGMA_UNROLL - for (int k_block = 0; k_block < size<2>(tCrA); ++k_block) { - if constexpr (!SwapAB) { - cute::gemm(tiled_mma, tCrA(_, _, k_block), tCrB(_, _, k_block), tCrC); - } else { - cute::gemm(tiled_mma, tCrB(_, _, k_block), tCrA(_, _, k_block), tCrC); - } - tiled_mma.accumulate_ = GMMA::ScaleOut::One; - } - warpgroup_commit_batch(); - if constexpr (wg_wait >= 0) { - warpgroup_wait(); - } - warpgroup_fence_operand(tCrC); - if constexpr (Is_RS) { - if constexpr (!SwapAB) { - warpgroup_fence_operand(const_cast(tCrA)); - } else { - warpgroup_fence_operand(const_cast(tCrB)); - } - } - } -} - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template < - bool A_in_regs = false, - bool B_in_regs = false, - bool SwapAB = false, - typename Tensor0, - typename Tensor1, - typename Tensor2, - typename Tensor3, - typename Tensor4, - typename TiledMma, - typename TiledCopyA, - typename TiledCopyB, - typename ThrCopyA, - typename ThrCopyB, - typename Hook> -CUTLASS_DEVICE void gemm_sm80( - Tensor0& acc, - Tensor1& tCrA, - Tensor2& tCrB, - Tensor3 const& tCsA, - Tensor4 const& tCsB, - TiledMma tiled_mma, - TiledCopyA smem_tiled_copy_A, - TiledCopyB smem_tiled_copy_B, - ThrCopyA smem_thr_copy_A, - ThrCopyB smem_thr_copy_B, - Hook fn) { - if constexpr (SwapAB) { - gemm_sm80( - acc, - tCrB, - tCrA, - tCsB, - tCsA, - tiled_mma, - smem_tiled_copy_B, - smem_tiled_copy_A, - smem_thr_copy_B, - smem_thr_copy_A, - fn); - } else { - CUTE_STATIC_ASSERT_V(size<1>(tCrA) == size<1>(acc)); // MMA_M - CUTE_STATIC_ASSERT_V(size<1>(tCrB) == size<2>(acc)); // MMA_N - CUTE_STATIC_ASSERT_V(size<2>(tCrA) == size<2>(tCrB)); // MMA_K - Tensor tCrA_copy_view = smem_thr_copy_A.retile_D(tCrA); - CUTE_STATIC_ASSERT_V(size<1>(tCsA) == size<1>(tCrA_copy_view)); // M - Tensor tCrB_copy_view = smem_thr_copy_B.retile_D(tCrB); - CUTE_STATIC_ASSERT_V(size<1>(tCsB) == size<1>(tCrB_copy_view)); // N - if (!A_in_regs) { - cute::copy( - smem_tiled_copy_A, tCsA(_, _, _0{}), tCrA_copy_view(_, _, _0{})); - } - if (!B_in_regs) { - cute::copy( - smem_tiled_copy_B, tCsB(_, _, _0{}), tCrB_copy_view(_, _, _0{})); - } -#pragma unroll - for (int i = 0; i < size<2>(tCrA); ++i) { - if (i < size<2>(tCrA) - 1) { - if (!A_in_regs) { - cute::copy( - smem_tiled_copy_A, - tCsA(_, _, i + 1), - tCrA_copy_view(_, _, i + 1)); - } - if (!B_in_regs) { - cute::copy( - smem_tiled_copy_B, - tCsB(_, _, i + 1), - tCrB_copy_view(_, _, i + 1)); - } - } - if constexpr (!std::is_same_v) { - if (i == 0) { - fn(); - } - } - cute::gemm(tiled_mma, tCrA(_, _, i), tCrB(_, _, i), acc); - } - } -} - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template < - typename Tensor0, - typename Tensor1, - typename Tensor2, - typename Tensor3, - typename TiledMma, - typename TiledCopy, - typename ThrCopy> -CUTLASS_DEVICE void gemm_rs_sm80( - Tensor0& acc, - Tensor1& tCrA, - Tensor2& tCrB, - Tensor3 const& tCsB, - TiledMma tiled_mma, - TiledCopy smem_tiled_copy_B, - ThrCopy smem_thr_copy_B) { - CUTE_STATIC_ASSERT_V(size<1>(tCrA) == size<1>(acc)); // MMA_M - CUTE_STATIC_ASSERT_V(size<1>(tCrB) == size<2>(acc)); // MMA_N - CUTE_STATIC_ASSERT_V(size<2>(tCrA) == size<2>(tCrB)); // MMA_K - Tensor tCrB_copy_view = smem_thr_copy_B.retile_D(tCrB); - CUTE_STATIC_ASSERT_V(size<1>(tCsB) == size<1>(tCrB_copy_view)); // N - cute::copy(smem_tiled_copy_B, tCsB(_, _, _0{}), tCrB_copy_view(_, _, _0{})); -#pragma unroll - for (int i = 0; i < size<2>(tCrA); ++i) { - if (i < size<2>(tCrA) - 1) { - cute::copy( - smem_tiled_copy_B, tCsB(_, _, i + 1), tCrB_copy_view(_, _, i + 1)); - } - cute::gemm(tiled_mma, tCrA(_, _, i), tCrB(_, _, i), acc); - } -} - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template < - bool Is_even_MN = true, - bool Is_even_K = true, - bool Clear_OOB_MN = false, - bool Clear_OOB_K = true, - class CopyAtom, - class TV, - class Tiler, - typename Engine0, - typename Layout0, - typename Engine1, - typename Layout1, - typename Engine2, - typename Layout2, - typename Engine3, - typename Layout3> -CUTLASS_DEVICE void copy( - TiledCopy const& tiled_copy, - Tensor const& S, - Tensor& D, - Tensor const& identity_MN, - Tensor const& predicate_K, - const int max_MN = 0) { - // Decay TiledCopy to CopyAtom - auto copy_atom = static_cast(tiled_copy); - CUTE_STATIC_ASSERT_V(rank(S) == Int<3>{}); - CUTE_STATIC_ASSERT_V(rank(D) == Int<3>{}); - CUTE_STATIC_ASSERT_V(size<0>(S) == size<0>(D)); // MMA - CUTE_STATIC_ASSERT_V(size<1>(S) == size<1>(D)); // MMA_M - CUTE_STATIC_ASSERT_V(size<2>(S) == size<2>(D)); // MMA_K - // There's no case where !Clear_OOB_K && Clear_OOB_MN - static_assert(!(Clear_OOB_MN && !Clear_OOB_K)); - auto has_with_bool = cute::is_valid( - [](auto t) -> void_t() - .with(true))> {}, - copy_atom); -#pragma unroll - for (int m = 0; m < size<1>(S); ++m) { - bool predicate_mn = - Is_even_MN || get<0>(identity_MN(_0{}, m, _0{})) < max_MN; - if constexpr (Is_even_MN || !Clear_OOB_MN) { - if (Is_even_MN || predicate_mn) { -#pragma unroll - for (int k = 0; k < size<2>(S); ++k) { - if constexpr (Is_even_K || !Clear_OOB_K) { - if (Is_even_K || predicate_K(k)) { - cute::copy(copy_atom, S(_, m, k), D(_, m, k)); - } - } else { // Clear_OOB_K == true && Is_even_K == false - // If copy traits can be transformed with a predicate value, do - // it, otherwise branch here - if constexpr (has_with_bool) { - cute::copy( - copy_atom.with(predicate_K(k)), S(_, m, k), D(_, m, k)); - } else { - if (predicate_K(k)) { - cute::copy(copy_atom, S(_, m, k), D(_, m, k)); - } else { - cute::clear(D(_, m, k)); - } - } - } - } - } - } else { // Clear_OOB_MN == true && Is_even_MN == false, also implies - // Clear_OOB_K == true - if constexpr (!has_with_bool) { - if (predicate_mn) { -#pragma unroll - for (int k = 0; k < size<2>(S); ++k) { - if (Is_even_K || predicate_K(k)) { - cute::copy(copy_atom, S(_, m, k), D(_, m, k)); - } else if (Clear_OOB_K) { - cute::clear(D(_, m, k)); - } - } - } else { - cute::clear(D(_, m, _)); - } - } else { // combine the mn predicate with the k predicate -#pragma unroll - for (int k = 0; k < size<2>(S); ++k) { - cute::copy( - copy_atom.with(predicate_mn && (Is_even_K || predicate_K(k))), - S(_, m, k), - D(_, m, k)); - } - } - } - } -} - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -// Byte permute and shuffle to match register layout of -// (FP8 downcasted) accumulator of GEMM-I to FP8 operand A of GEMM-II. -template -CUTLASS_DEVICE void permute_Aregs_fp8(Fragment& frag) { - // frag has shape ((4, 2, 2), MMA_M, MMA_N), each element is 8 bits - static_assert(decltype(size<0, 0>(frag))::value == 4); - static_assert(decltype(size<0, 1>(frag))::value == 2); - static_assert(decltype(stride<0, 0>(frag))::value == 1); - static_assert(decltype(stride<0, 1>(frag))::value == 4); - static_assert(sizeof(typename Fragment::value_type) == 1); - - auto quad_idx = threadIdx.x % 4; - bool lane_03 = quad_idx == 0 || quad_idx == 3; - int selector_upper = lane_03 ? 0x5410 : 0x1054; - int selector_lower = lane_03 ? 0x7632 : 0x3276; - - static constexpr int upper_map[4] = {0, 3, 1, 2}; - // static constexpr int lower_map[4] = {1, 2, 0, 3}; - - Tensor frag_64b = recast(frag); // ((1, 1, 2), MMA_M, MMA_N) -#pragma unroll - for (int i = 0; i < size(frag_64b); ++i) { - uint32_t upper = frag_64b[i].x; - uint32_t lower = frag_64b[i].y; - uint32_t upper0 = lane_03 ? upper : lower; - uint32_t lower0 = lane_03 ? lower : upper; - upper0 = __shfl_sync(uint32_t(-1), upper0, upper_map[quad_idx], 4); - // lower0 = __shfl_sync(uint32_t(-1), lower0, lower_map[quad_idx], 4); - lower0 = __shfl_sync(uint32_t(-1), lower0, upper_map[quad_idx] ^ 1, 4); - frag_64b[i].x = __byte_perm(upper0, lower0, selector_upper); - frag_64b[i].y = __byte_perm(upper0, lower0, selector_lower); - } -} - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template -CUTLASS_DEVICE void permute_Cregs_fp8(Fragment& frag) { - // frag has shape ((2, 2, N / 8), MMA_M, MMA_N), each element is 32 bits - static_assert(decltype(size<0, 0>(frag))::value == 2); - static_assert(decltype(size<0, 1>(frag))::value == 2); - static_assert(decltype(size<0, 2>(frag))::value % 2 == 0); - static_assert(decltype(stride<0, 0>(frag))::value == 1); - static_assert(sizeof(typename Fragment::value_type) == 4); - Tensor frag_64b = - group_modes<1, 3>(recast(frag)); // ((1, 2, N / 8), (MMA_M, MMA_N)) -#pragma unroll - for (int mi = 0; mi < size<1>(frag_64b); ++mi) { -#pragma unroll - for (int i = 0; i < size<0, 2>(frag_64b) / 2; ++i) { - cutlass::swap( - frag_64b(make_coord(_0{}, _1{}, 2 * i), mi), - frag_64b(make_coord(_0{}, _0{}, 2 * i + 1), mi)); - } - } -} - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template -CUTLASS_DEVICE void permute_output_fp8(Fragment& out) { - // out has shape ((2, 2, N / 8), MMA_M, MMA_N), each element is 32 bits - static_assert(decltype(size<0, 0>(out))::value == 2); - static_assert(decltype(size<0, 1>(out))::value == 2); - static_assert(decltype(size<0, 2>(out))::value % 2 == 0); - static_assert(decltype(stride<0, 0>(out))::value == 1); - static_assert(sizeof(typename Fragment::value_type) == 4); - Tensor frag = group_modes<1, 3>(out); // ((2, 2, N / 8), (MMA_M, MMA_N)) -#pragma unroll - for (int mi = 0; mi < size<1>(frag); ++mi) { -#pragma unroll - for (int j = 0; j < size<0, 1>(frag); ++j) { -#pragma unroll - for (int i = 0; i < size<0, 2>(frag) / 2; ++i) { - cutlass::swap( - frag(make_coord(_1{}, j, 2 * i), mi), - frag(make_coord(_0{}, j, 2 * i + 1), mi)); - } - } - } -} - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template -CUTLASS_DEVICE void permute_output_fp8_Vcolmajor(Fragment& frag) { - // frag has shape ((2, 2, N / 8), MMA_M, MMA_N), each element is 16 bits - static_assert(decltype(size<0, 0>(frag))::value == 2); - static_assert(decltype(size<0, 1>(frag))::value == 2); - static_assert(decltype(stride<0, 0>(frag))::value == 1); - static_assert( - sizeof(typename Fragment::value_type) == 2 || - sizeof(typename Fragment::value_type) == 4); - - auto quad_idx = threadIdx.x % 4; - bool lane_03 = quad_idx == 0 || quad_idx == 3; - - static constexpr int upper_map[4] = {0, 2, 3, 1}; - // static constexpr int lower_map[4] = {2, 0, 1, 3}; - - // if (blockIdx.x == 0 && threadIdx.x == 128) { print_tensor(frag); } - using type2 = std::conditional_t< - sizeof(typename Fragment::value_type) == 2, - uint32_t, - uint64_t>; - Tensor frag_2 = - group_modes<1, 3>(recast(frag)); // ((1, 2, N / 8), (MMA_M, MMA_N)) -// if (blockIdx.x == 0 && threadIdx.x == 128) { print(frag); printf("\n"); -// print(frag_2); } -#pragma unroll - for (int mi = 0; mi < size<1>(frag_2); ++mi) { -#pragma unroll - for (int j = 0; j < size<0, 1>(frag_2); ++j) { -#pragma unroll - for (int i = 0; i < size<0, 2>(frag_2) / 2; ++i) { - type2 upper = frag_2(make_coord(_0{}, j, 2 * i), mi); - type2 lower = frag_2(make_coord(_0{}, j, 2 * i + 1), mi); - type2 upper0 = lane_03 ? upper : lower; - type2 lower0 = lane_03 ? lower : upper; - upper0 = __shfl_sync(uint32_t(-1), upper0, upper_map[quad_idx], 4); - // lower0 = __shfl_sync(uint32_t(-1), lower0, lower_map[quad_idx], 4); - lower0 = __shfl_sync(uint32_t(-1), lower0, upper_map[quad_idx] ^ 2, 4); - frag_2(make_coord(_0{}, j, 2 * i), mi) = lane_03 ? upper0 : lower0; - frag_2(make_coord(_0{}, j, 2 * i + 1), mi) = lane_03 ? lower0 : upper0; - } - } - } - // if (blockIdx.x == 0 && threadIdx.x == 128) { print_tensor(frag); } -} - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template -CUTLASS_DEVICE void apply_softcap( - Tensor& tensor, - float const softcap) { -#pragma unroll - for (int i = 0; i < size(tensor); ++i) { - tensor(i) = cutlass::fast_tanh(tensor(i) * softcap); - } -} - -template -CUTLASS_DEVICE auto calculate_dtanh(Tensor& tensor) { - Tensor out = make_fragment_like(tensor); -#pragma unroll - for (int i = 0; i < size(tensor); ++i) { - out(i) = 1.f - (tensor(i) * tensor(i)); - } - return out; -} - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -CUTLASS_DEVICE -int canonical_warp_group_idx_nosync() { - return threadIdx.x / cutlass::NumThreadsPerWarpGroup; -} - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/version.txt b/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/version.txt deleted file mode 100644 index 04d34e1a3..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/hstu_attention/version.txt +++ /dev/null @@ -1 +0,0 @@ -5231d95 diff --git a/recommendation_v4/generative_recommenders/ops/cpp/jagged_transpose_1d.cpp b/recommendation_v4/generative_recommenders/ops/cpp/jagged_transpose_1d.cpp deleted file mode 100644 index d3a2e9421..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/jagged_transpose_1d.cpp +++ /dev/null @@ -1,130 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "fbgemm_gpu/sparse_ops.h" // @manual - -namespace hstu { - -template -void _jagged_transpose_1d_cpu_kernel( - int32_t size1, - int32_t size2, - int32_t max_len, - const at::TensorAccessor& offsets, - const at::TensorAccessor& values, - const at::TensorAccessor& lengths, - const at::TensorAccessor& trans_offsets, - at::TensorAccessor trans_values) { - for (auto i : c10::irange(size1)) { - for (auto j : c10::irange(size2)) { - auto src_idx = i * size2 + j; - auto dst_idx = j * size1 + i; - auto src_offset = offsets[src_idx]; - auto src_length = lengths[src_idx]; - auto dst_offset = trans_offsets[dst_idx]; - - for (auto k = 0; k < src_length; ++k) { - trans_values[dst_offset + k] = values[src_offset + k]; - } - } - } -} - -std::tuple jagged_transpose_1d_cpu( - const at::Tensor& values, - const at::Tensor& offsets, - const at::Tensor& lengths, - const int64_t max_len, - const int64_t size1, - const int64_t size2) { - TORCH_INTERNAL_ASSERT(values.device().type() == at::DeviceType::CPU); - TORCH_INTERNAL_ASSERT(offsets.device().type() == at::DeviceType::CPU); - TORCH_INTERNAL_ASSERT(lengths.device().type() == at::DeviceType::CPU); - TORCH_CHECK(offsets.size(0) == size1 * size2 + 1); - TORCH_CHECK(lengths.size(0) == size1 * size2); - - auto trans_lengths = - lengths.view({size1, size2}).transpose(0, 1).contiguous().view({-1}); - auto trans_offsets = - fbgemm_gpu::asynchronous_complete_cumsum_cpu(trans_lengths); - auto L_out = trans_offsets[-1].item(); - auto trans_values = at::empty({L_out}, values.options()); - - if (L_out == 0) { - return std::make_tuple(trans_values, trans_offsets, trans_lengths); - } - - AT_DISPATCH_INTEGRAL_TYPES( - lengths.scalar_type(), "jagged_transpose_1d_cpu_kernel_input1", [&] { - using index_t = scalar_t; - AT_DISPATCH_ALL_TYPES_AND2( - at::ScalarType::BFloat16, - at::ScalarType::Half, - values.scalar_type(), - "jagged_transpose_1d_cpu_kernel_input2", - [&] { - using val_t = scalar_t; - _jagged_transpose_1d_cpu_kernel( - size1, - size2, - max_len, - offsets.accessor(), - values.accessor(), - lengths.accessor(), - trans_offsets.accessor(), - trans_values.accessor()); - }); - }); - - return std::make_tuple(trans_values, trans_offsets, trans_lengths); -} - -std::tuple jagged_transpose_1d_meta( - const at::Tensor& values, - const at::Tensor& offsets, - const at::Tensor& lengths, - const int64_t max_len, - const int64_t size1, - const int64_t size2) { - auto trans_lengths = - lengths.view({size1, size2}).transpose(0, 1).contiguous().view({-1}); - auto L_out = trans_lengths.sum().item(); - - auto trans_values = at::native::empty_meta_symint( - {L_out}, - /*dtype=*/::std::make_optional(values.scalar_type()), - /*layout=*/::std::make_optional(values.layout()), - /*device=*/::std::make_optional(c10::Device(c10::kMeta)), - /*pin_memory=*/::std::nullopt); - - auto trans_offsets = at::native::empty_meta_symint( - {size1 * size2 + 1}, - /*dtype=*/::std::make_optional(lengths.scalar_type()), - /*layout=*/::std::make_optional(lengths.layout()), - /*device=*/::std::make_optional(c10::Device(c10::kMeta)), - /*pin_memory=*/::std::nullopt); - - return std::make_tuple(trans_values, trans_offsets, trans_lengths); -} -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/jagged_transpose_1d.cu b/recommendation_v4/generative_recommenders/ops/cpp/jagged_transpose_1d.cu deleted file mode 100644 index 380100962..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/jagged_transpose_1d.cu +++ /dev/null @@ -1,127 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include - -#include "common.h" -#include "fbgemm_gpu/sparse_ops.h" // @manual -#include "fbgemm_gpu/utils/fixed_divisor.cuh" // @manual - -namespace hstu { - -static constexpr int32_t kMaxThreads = 1024; - -template -__global__ __launch_bounds__(kMaxThreads) void _jagged_transpose_1d_cuda_kernel( - int32_t size1, - int32_t size2, - int32_t max_len, - const at::PackedTensorAccessor32 offsets, - const at::PackedTensorAccessor32 values, - const at::PackedTensorAccessor32 lengths, - const at::PackedTensorAccessor32 - trans_offsets, - at::PackedTensorAccessor32 trans_values) { - for (auto idx = blockIdx.x * blockDim.y + threadIdx.y; - idx < static_cast(size1 * size2); - idx += gridDim.x * blockDim.y) { - auto i = idx / size2; - auto j = idx % size2; - auto src_idx = i * size2 + j; - auto dst_idx = j * size1 + i; - auto src_offset = offsets[src_idx]; - auto src_length = lengths[src_idx]; - auto dst_offset = trans_offsets[dst_idx]; - - for (auto k = threadIdx.x; k < static_cast(src_length); - k += blockDim.x) { - trans_values[dst_offset + k] = values[src_offset + k]; - } - } -} - -std::tuple jagged_transpose_1d_cuda( - const at::Tensor& values, - const at::Tensor& offsets, - const at::Tensor& lengths, - const int64_t max_len, - const int64_t size1, - const int64_t size2) { - at::cuda::OptionalCUDAGuard device_guard; - device_guard.set_index(values.get_device()); - TORCH_INTERNAL_ASSERT(values.device().type() == at::DeviceType::CUDA); - TORCH_INTERNAL_ASSERT(offsets.device().type() == at::DeviceType::CUDA); - TORCH_INTERNAL_ASSERT(lengths.device().type() == at::DeviceType::CUDA); - TORCH_CHECK(offsets.size(0) == size1 * size2 + 1); - TORCH_CHECK(lengths.size(0) == size1 * size2); - TORCH_CHECK(values.get_device() == offsets.get_device()); - TORCH_CHECK(values.get_device() == lengths.get_device()); - - auto trans_lengths = - lengths.view({size1, size2}).transpose(0, 1).contiguous().view({-1}); - auto trans_offsets = - fbgemm_gpu::asynchronous_complete_cumsum_gpu(trans_lengths); - auto L_out = trans_offsets[-1].item(); - TORCH_CHECK(L_out < std::numeric_limits::max()); - auto trans_values = at::empty({L_out}, values.options()); - - if (L_out == 0) { - return std::make_tuple(trans_values, trans_offsets, trans_lengths); - } - - // Optimized thread block configuration based on benchmark results - uint32_t B_blocks = 4; - dim3 threads(256, B_blocks); - auto blocks = div_round_up(size1 * size2, B_blocks); - - AT_DISPATCH_INTEGRAL_TYPES( - lengths.scalar_type(), "jagged_transpose_1d_cuda_kernel_input1", [&] { - using index_t = scalar_t; - AT_DISPATCH_ALL_TYPES_AND2( - at::ScalarType::BFloat16, - at::ScalarType::Half, - values.scalar_type(), - "jagged_transpose_1d_cuda_kernel_input2", - [&] { - using val_t = scalar_t; - _jagged_transpose_1d_cuda_kernel<<< - blocks, - threads, - 0, - at::cuda::getCurrentCUDAStream()>>>( - size1, - size2, - max_len, - offsets - .packed_accessor32(), - values.packed_accessor32(), - lengths - .packed_accessor32(), - trans_offsets - .packed_accessor32(), - trans_values - .packed_accessor32()); - }); - }); - - return std::make_tuple(trans_values, trans_offsets, trans_lengths); -} -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/replace_last_n_with_jagged.cpp b/recommendation_v4/generative_recommenders/ops/cpp/replace_last_n_with_jagged.cpp deleted file mode 100644 index fa8eaac09..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/replace_last_n_with_jagged.cpp +++ /dev/null @@ -1,139 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "fbgemm_gpu/sparse_ops.h" // @manual - -namespace hstu { - -template -void _replace_last_n_with_jagged_cpu_kernel( - int32_t B, - const at::TensorAccessor& lengths_left, - const at::TensorAccessor& offsets_left, - const at::TensorAccessor& values_left, - const at::TensorAccessor& lengths_right, - const at::TensorAccessor& offsets_right, - const at::TensorAccessor& values_right, - const at::TensorAccessor& output_offsets, - at::TensorAccessor output) { - for (auto b : c10::irange(B)) { - auto left_start = offsets_left[b]; - auto left_len = lengths_left[b]; - auto right_start = offsets_right[b]; - auto right_len = lengths_right[b]; - auto output_start = output_offsets[b]; - - auto keep_len = left_len - right_len; - - for (auto i = 0; i < left_len; ++i) { - for (auto d = 0; d < values_left.size(1); ++d) { - if (i < keep_len) { - output[output_start + i][d] = values_left[left_start + i][d]; - } else { - auto right_idx = i - keep_len; - if (right_idx < right_len) { - output[output_start + i][d] = - values_right[right_start + right_idx][d]; - } - } - } - } - } -} - -at::Tensor replace_last_n_with_jagged_cpu( - const at::Tensor& lengths_left, - const at::Tensor& values_left, - const at::Tensor& lengths_right, - const at::Tensor& values_right) { - TORCH_INTERNAL_ASSERT(lengths_left.device().type() == at::DeviceType::CPU); - TORCH_INTERNAL_ASSERT(lengths_right.device().type() == at::DeviceType::CPU); - TORCH_INTERNAL_ASSERT(values_left.device().type() == at::DeviceType::CPU); - TORCH_INTERNAL_ASSERT(values_right.device().type() == at::DeviceType::CPU); - TORCH_CHECK(lengths_left.size(0) == lengths_right.size(0)); - TORCH_CHECK(values_left.size(1) == values_right.size(1)); - - auto B = lengths_left.size(0); - auto D = values_left.size(1); - - auto L_out = lengths_left.sum().item(); - - auto output = at::empty({L_out, D}, values_left.options()); - - if (L_out == 0) { - return output; - } - - const auto offsets_left = - fbgemm_gpu::asynchronous_complete_cumsum_cpu(lengths_left.view({-1})); - const auto offsets_right = - fbgemm_gpu::asynchronous_complete_cumsum_cpu(lengths_right.view({-1})); - const auto output_offsets = offsets_left; - - AT_DISPATCH_INTEGRAL_TYPES( - lengths_left.scalar_type(), - "replace_last_n_with_jagged_cpu_kernel_input1", - [&] { - using index_t = scalar_t; - AT_DISPATCH_ALL_TYPES_AND2( - at::ScalarType::BFloat16, - at::ScalarType::Half, - values_left.scalar_type(), - "replace_last_n_with_jagged_cpu_kernel_input2", - [&] { - using val_t = scalar_t; - _replace_last_n_with_jagged_cpu_kernel( - B, - lengths_left.accessor(), - offsets_left.accessor(), - values_left.accessor(), - lengths_right.accessor(), - offsets_right.accessor(), - values_right.accessor(), - output_offsets.accessor(), - output.accessor()); - }); - }); - - return output; -} - -at::Tensor replace_last_n_with_jagged_meta( - const at::Tensor& lengths_left, - const at::Tensor& values_left, - const at::Tensor& lengths_right, - const at::Tensor& values_right) { - auto L_out = lengths_left.sum().item(); - auto D = values_left.size(1); - - auto output = at::native::empty_meta_symint( - {L_out, D}, - /*dtype=*/::std::make_optional(values_left.scalar_type()), - /*layout=*/::std::make_optional(values_left.layout()), - /*device=*/::std::make_optional(c10::Device(c10::kMeta)), - /*pin_memory=*/::std::nullopt); - - return output; -} -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/replace_last_n_with_jagged.cu b/recommendation_v4/generative_recommenders/ops/cpp/replace_last_n_with_jagged.cu deleted file mode 100644 index 00a589eb9..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/replace_last_n_with_jagged.cu +++ /dev/null @@ -1,156 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include - -#include "common.h" -#include "fbgemm_gpu/sparse_ops.h" // @manual -#include "fbgemm_gpu/utils/fixed_divisor.cuh" // @manual - -namespace hstu { - -static constexpr int32_t kMaxThreads = 1024; - -template -__global__ -__launch_bounds__(kMaxThreads) void _replace_last_n_with_jagged_cuda_kernel( - int32_t B, - int32_t D, - const at::PackedTensorAccessor32 - lengths_left, - const at::PackedTensorAccessor32 - offsets_left, - const at::PackedTensorAccessor32 - values_left, - const at::PackedTensorAccessor32 - lengths_right, - const at::PackedTensorAccessor32 - offsets_right, - const at::PackedTensorAccessor32 - values_right, - at::PackedTensorAccessor32 output) { - for (auto b = blockIdx.x * blockDim.y + threadIdx.y; - b < static_cast(B); - b += gridDim.x * blockDim.y) { - auto left_start = offsets_left[b]; - auto left_len = lengths_left[b]; - auto right_start = offsets_right[b]; - auto right_len = lengths_right[b]; - auto output_start = offsets_left[b]; - auto keep_len = left_len - right_len; - - for (auto i = threadIdx.x; i < static_cast(left_len * D); - i += blockDim.x) { - auto seq_pos = i / D; - auto dim_pos = i % D; - if (seq_pos < static_cast(keep_len)) { - output[output_start + seq_pos][dim_pos] = - values_left[left_start + seq_pos][dim_pos]; - } else { - auto right_idx = seq_pos - keep_len; - if (right_idx < static_cast(right_len)) { - output[output_start + seq_pos][dim_pos] = - values_right[right_start + right_idx][dim_pos]; - } - } - } - } -} - -at::Tensor replace_last_n_with_jagged_cuda( - const at::Tensor& lengths_left, - const at::Tensor& values_left, - const at::Tensor& lengths_right, - const at::Tensor& values_right) { - at::cuda::OptionalCUDAGuard device_guard; - device_guard.set_index(values_left.get_device()); - TORCH_INTERNAL_ASSERT(lengths_left.device().type() == at::DeviceType::CUDA); - TORCH_INTERNAL_ASSERT(lengths_right.device().type() == at::DeviceType::CUDA); - TORCH_INTERNAL_ASSERT(values_left.device().type() == at::DeviceType::CUDA); - TORCH_INTERNAL_ASSERT(values_right.device().type() == at::DeviceType::CUDA); - TORCH_CHECK(lengths_left.size(0) == lengths_right.size(0)); - TORCH_CHECK(values_left.size(1) == values_right.size(1)); - - auto B = lengths_left.size(0); - auto D = values_left.size(1); - auto L_out = lengths_left.sum().item(); - TORCH_CHECK(L_out < std::numeric_limits::max()); - TORCH_CHECK(values_left.get_device() == lengths_left.get_device()); - TORCH_CHECK(values_left.get_device() == lengths_right.get_device()); - TORCH_CHECK(values_left.get_device() == values_right.get_device()); - - auto output = at::empty({L_out, D}, values_left.options()); - - if (L_out == 0) { - return output; - } - - const auto offsets_left = - fbgemm_gpu::asynchronous_complete_cumsum_gpu(lengths_left.view({-1})); - const auto offsets_right = - fbgemm_gpu::asynchronous_complete_cumsum_gpu(lengths_right.view({-1})); - - // Optimized thread block configuration based on benchmark results - uint32_t B_blocks, threads_x; - B_blocks = 4; - threads_x = 256; - - dim3 threads(threads_x, B_blocks); - auto blocks = div_round_up(B, B_blocks); - - AT_DISPATCH_INTEGRAL_TYPES( - lengths_left.scalar_type(), - "replace_last_n_with_jagged_cuda_kernel_input1", - [&] { - using index_t = scalar_t; - AT_DISPATCH_ALL_TYPES_AND2( - at::ScalarType::BFloat16, - at::ScalarType::Half, - values_left.scalar_type(), - "replace_last_n_with_jagged_cuda_kernel_input2", - [&] { - using val_t = scalar_t; - _replace_last_n_with_jagged_cuda_kernel<<< - blocks, - threads, - 0, - at::cuda::getCurrentCUDAStream()>>>( - B, - D, - lengths_left - .packed_accessor32(), - offsets_left - .packed_accessor32(), - values_left - .packed_accessor32(), - lengths_right - .packed_accessor32(), - offsets_right - .packed_accessor32(), - values_right - .packed_accessor32(), - output.packed_accessor32()); - }); - }); - - return output; -} -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/setup.py b/recommendation_v4/generative_recommenders/ops/cpp/setup.py deleted file mode 100644 index 2a06a9d05..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/setup.py +++ /dev/null @@ -1,487 +0,0 @@ -# pyre-unsafe -""" -Modified from https://github.com/Dao-AILab/flash-attention/blob/main/hopper/setup.py -""" - -import itertools -import os -import platform -import subprocess -import sys -import sysconfig -import warnings -from pathlib import Path - -import torch -from packaging.version import parse, Version -from setuptools import find_packages, setup -from torch.utils.cpp_extension import BuildExtension, CUDA_HOME, CUDAExtension - -# ninja build does not work unless include_dirs are abs path -this_dir = os.path.dirname(os.path.abspath(__file__)) -PACKAGE_NAME = "hstu" -# FORCE_BUILD: Force a fresh build locally, instead of attempting to find prebuilt wheels -# SKIP_CUDA_BUILD: Intended to allow CI to use a simple `python setup.py sdist` run to copy over raw files, without any cuda compilation -FORCE_BUILD = os.getenv("FLASH_ATTENTION_FORCE_BUILD", "FALSE") == "TRUE" -SKIP_CUDA_BUILD = os.getenv("FLASH_ATTENTION_SKIP_CUDA_BUILD", "FALSE") == "TRUE" -# For CI, we want the option to build with C++11 ABI since the nvcr images use C++11 ABI -FORCE_CXX11_ABI = os.getenv("FLASH_ATTENTION_FORCE_CXX11_ABI", "FALSE") == "TRUE" - -# HACK: we monkey patch pytorch's _write_ninja_file to pass -# "-gencode arch=compute_sm90a,code=sm_90a" to files ending in '_sm90.cu', -# and pass "-gencode arch=compute_sm80,code=sm_80" to files ending in '_sm80.cu' -from torch.utils.cpp_extension import ( - _is_cuda_file, - _join_cuda_home, - _join_rocm_home, - _maybe_write, - COMMON_HIP_FLAGS, - get_cxx_compiler, - IS_HIP_EXTENSION, - IS_WINDOWS, - SUBPROCESS_DECODE_ARGS, -) - -DISABLE_BACKWARD = os.getenv("FLASH_ATTENTION_DISABLE_BACKWARD", "FALSE") == "TRUE" -DISABLE_FP16 = os.getenv("FLASH_ATTENTION_DISABLE_FP16", "TRUE") == "TRUE" -DISABLE_HDIM64 = os.getenv("FLASH_ATTENTION_DISABLE_HDIM64", "TRUE") == "TRUE" -DISABLE_HDIM96 = os.getenv("FLASH_ATTENTION_DISABLE_HDIM96", "TRUE") == "TRUE" -DISABLE_HDIM128 = os.getenv("FLASH_ATTENTION_DISABLE_HDIM128", "FALSE") == "TRUE" -DISABLE_HDIM192 = os.getenv("FLASH_ATTENTION_DISABLE_HDIM192", "TRUE") == "TRUE" -DISABLE_HDIM256 = os.getenv("FLASH_ATTENTION_DISABLE_HDIM256", "TRUE") == "TRUE" -DISABLE_SM8x = os.getenv("FLASH_ATTENTION_DISABLE_SM80", "TRUE") == "TRUE" - - -def _write_ninja_file( - path, - cflags, - post_cflags, - cuda_cflags, - cuda_post_cflags, - cuda_dlink_post_cflags, - sources, - objects, - ldflags, - library_target, - with_cuda, - **kwargs, # kwargs (ignored) to absorb new flags in torch.utils.cpp_extension -) -> None: - r"""Write a ninja file that does the desired compiling and linking. - - `path`: Where to write this file - `cflags`: list of flags to pass to $cxx. Can be None. - `post_cflags`: list of flags to append to the $cxx invocation. Can be None. - `cuda_cflags`: list of flags to pass to $nvcc. Can be None. - `cuda_postflags`: list of flags to append to the $nvcc invocation. Can be None. - `sources`: list of paths to source files - `objects`: list of desired paths to objects, one per source. - `ldflags`: list of flags to pass to linker. Can be None. - `library_target`: Name of the output library. Can be None; in that case, - we do no linking. - `with_cuda`: If we should be compiling with CUDA. - """ - - def sanitize_flags(flags): - if flags is None: - return [] - else: - return [flag.strip() for flag in flags] - - cflags = sanitize_flags(cflags) - post_cflags = sanitize_flags(post_cflags) - cuda_cflags = sanitize_flags(cuda_cflags) - cuda_post_cflags = sanitize_flags(cuda_post_cflags) - cuda_dlink_post_cflags = sanitize_flags(cuda_dlink_post_cflags) - ldflags = sanitize_flags(ldflags) - - # Sanity checks... - assert len(sources) == len(objects) - assert len(sources) > 0 - - compiler = get_cxx_compiler() - - # Version 1.3 is required for the `deps` directive. - config = ["ninja_required_version = 1.3"] - config.append(f"cxx = {compiler}") - if with_cuda or cuda_dlink_post_cflags: - if IS_HIP_EXTENSION: - nvcc = _join_rocm_home("bin", "hipcc") - else: - nvcc = _join_cuda_home("bin", "nvcc") - if "PYTORCH_NVCC" in os.environ: - nvcc_from_env = os.getenv( - "PYTORCH_NVCC" - ) # user can set nvcc compiler with ccache using the environment variable here - else: - nvcc_from_env = nvcc - config.append(f"nvcc_from_env = {nvcc_from_env}") - config.append(f"nvcc = {nvcc}") - - if IS_HIP_EXTENSION: - post_cflags = COMMON_HIP_FLAGS + post_cflags - flags = [f"cflags = {' '.join(cflags)}"] - flags.append(f"post_cflags = {' '.join(post_cflags)}") - if with_cuda: - flags.append(f"cuda_cflags = {' '.join(cuda_cflags)}") - flags.append(f"cuda_post_cflags = {' '.join(cuda_post_cflags)}") - cuda_post_cflags_sm80 = [ - s if s != "arch=compute_90a,code=sm_90a" else "arch=compute_80,code=sm_80" - for s in cuda_post_cflags - ] - flags.append(f"cuda_post_cflags_sm80 = {' '.join(cuda_post_cflags_sm80)}") - cuda_post_cflags_sm80_sm90 = cuda_post_cflags + [ - "-gencode", - "arch=compute_80,code=sm_80", - ] - flags.append( - f"cuda_post_cflags_sm80_sm90 = {' '.join(cuda_post_cflags_sm80_sm90)}" - ) - cuda_post_cflags_sm100 = [ - s - if s != "arch=compute_90a,code=sm_90a" - else "arch=compute_100a,code=sm_100a" - for s in cuda_post_cflags - ] - flags.append(f"cuda_post_cflags_sm100 = {' '.join(cuda_post_cflags_sm100)}") - flags.append(f"cuda_dlink_post_cflags = {' '.join(cuda_dlink_post_cflags)}") - flags.append(f"ldflags = {' '.join(ldflags)}") - - # Turn into absolute paths so we can emit them into the ninja build - # file wherever it is. - sources = [os.path.abspath(file) for file in sources] - - # See https://ninja-build.org/build.ninja.html for reference. - compile_rule = ["rule compile"] - if IS_WINDOWS: - compile_rule.append( - " command = cl /showIncludes $cflags -c $in /Fo$out $post_cflags" - ) - compile_rule.append(" deps = msvc") - else: - compile_rule.append( - " command = $cxx -MMD -MF $out.d $cflags -c $in -o $out $post_cflags" - ) - compile_rule.append(" depfile = $out.d") - compile_rule.append(" deps = gcc") - - if with_cuda: - cuda_compile_rule = ["rule cuda_compile"] - nvcc_gendeps = "" - # --generate-dependencies-with-compile is not supported by ROCm - # Nvcc flag `--generate-dependencies-with-compile` is not supported by sccache, which may increase build time. - if ( - torch.version.cuda is not None - and os.getenv("TORCH_EXTENSION_SKIP_NVCC_GEN_DEPENDENCIES", "0") != "1" - ): - cuda_compile_rule.append(" depfile = $out.d") - cuda_compile_rule.append(" deps = gcc") - # Note: non-system deps with nvcc are only supported - # on Linux so use --generate-dependencies-with-compile - # to make this work on Windows too. - nvcc_gendeps = ( - "--generate-dependencies-with-compile --dependency-output $out.d" - ) - cuda_compile_rule_sm80 = ( - ["rule cuda_compile_sm80"] - + cuda_compile_rule[1:] - + [ - f" command = $nvcc_from_env {nvcc_gendeps} $cuda_cflags -c $in -o $out $cuda_post_cflags_sm80" - ] - ) - cuda_compile_rule_sm80_sm90 = ( - ["rule cuda_compile_sm80_sm90"] - + cuda_compile_rule[1:] - + [ - f" command = $nvcc_from_env {nvcc_gendeps} $cuda_cflags -c $in -o $out $cuda_post_cflags_sm80_sm90" - ] - ) - cuda_compile_rule_sm100 = ( - ["rule cuda_compile_sm100"] - + cuda_compile_rule[1:] - + [ - f" command = $nvcc_from_env {nvcc_gendeps} $cuda_cflags -c $in -o $out $cuda_post_cflags_sm100" - ] - ) - cuda_compile_rule.append( - f" command = $nvcc_from_env {nvcc_gendeps} $cuda_cflags -c $in -o $out $cuda_post_cflags" - ) - - # Emit one build rule per source to enable incremental build. - build = [] - for source_file, object_file in zip(sources, objects): - is_cuda_source = _is_cuda_file(source_file) and with_cuda - if is_cuda_source: - if source_file.endswith("_sm90.cu"): - rule = "cuda_compile" - elif source_file.endswith("_sm80.cu"): - rule = "cuda_compile_sm80" - elif source_file.endswith("_sm100.cu"): - rule = "cuda_compile_sm100" - else: - rule = "cuda_compile_sm80_sm90" - else: - rule = "compile" - if IS_WINDOWS: - source_file = source_file.replace(":", "$:") - object_file = object_file.replace(":", "$:") - source_file = source_file.replace(" ", "$ ") - object_file = object_file.replace(" ", "$ ") - build.append(f"build {object_file}: {rule} {source_file}") - - if cuda_dlink_post_cflags: - devlink_out = os.path.join(os.path.dirname(objects[0]), "dlink.o") - devlink_rule = ["rule cuda_devlink"] - devlink_rule.append(" command = $nvcc $in -o $out $cuda_dlink_post_cflags") - devlink = [f"build {devlink_out}: cuda_devlink {' '.join(objects)}"] - objects += [devlink_out] - else: - devlink_rule, devlink = [], [] - - if library_target is not None: - link_rule = ["rule link"] - if IS_WINDOWS: - cl_paths = ( - subprocess.check_output(["where", "cl"]) - .decode(*SUBPROCESS_DECODE_ARGS) - .split("\r\n") - ) - if len(cl_paths) >= 1: - cl_path = os.path.dirname(cl_paths[0]).replace(":", "$:") - else: - raise RuntimeError("MSVC is required to load C++ extensions") - link_rule.append( - f' command = "{cl_path}/link.exe" $in /nologo $ldflags /out:$out' - ) - else: - link_rule.append(" command = $cxx $in $ldflags -o $out") - - link = [f"build {library_target}: link {' '.join(objects)}"] - - default = [f"default {library_target}"] - else: - link_rule, link, default = [], [], [] - - # 'Blocks' should be separated by newlines, for visual benefit. - blocks = [config, flags, compile_rule] - if with_cuda: - blocks.append(cuda_compile_rule) # type: ignore[possibly-undefined] - blocks.append(cuda_compile_rule_sm80) # type: ignore[possibly-undefined] - blocks.append(cuda_compile_rule_sm80_sm90) # type: ignore[possibly-undefined] - blocks.append(cuda_compile_rule_sm100) # type: ignore[possibly-undefined] - blocks += [devlink_rule, link_rule, build, devlink, link, default] - content = "\n\n".join("\n".join(b) for b in blocks) - # Ninja requires a new lines at the end of the .ninja file - content += "\n" - _maybe_write(path, content) - - -# Monkey patching -torch.utils.cpp_extension._write_ninja_file = _write_ninja_file - - -def get_platform(): - """ - Returns the platform name as used in wheel filenames. - """ - if sys.platform.startswith("linux"): - return "linux_x86_64" - elif sys.platform == "darwin": - mac_version = ".".join(platform.mac_ver()[0].split(".")[:2]) - return f"macosx_{mac_version}_x86_64" - elif sys.platform == "win32": - return "win_amd64" - else: - raise ValueError("Unsupported platform: {}".format(sys.platform)) - - -def get_cuda_bare_metal_version(cuda_dir): - raw_output = subprocess.check_output( - [cuda_dir + "/bin/nvcc", "-V"], universal_newlines=True - ) - output = raw_output.split() - release_idx = output.index("release") + 1 - bare_metal_version = parse(output[release_idx].split(",")[0]) - - return raw_output, bare_metal_version - - -def check_if_cuda_home_none(global_option: str) -> None: - if CUDA_HOME is not None: - return - # warn instead of error because user could be downloading prebuilt wheels, so nvcc won't be necessary - # in that case. - warnings.warn( - f"{global_option} was requested, but nvcc was not found. Are you sure your environment has nvcc available? " - "If you're installing within a container from https://hub.docker.com/r/pytorch/pytorch, " - "only images whose names contain 'devel' will provide nvcc." - ) - - -def nvcc_threads_args(): - nvcc_threads = os.getenv("NVCC_THREADS") or "4" - return ["--threads", nvcc_threads] - - -exe_extension = sysconfig.get_config_var("EXE") - - -cmdclass = {} -ext_modules = [] - -# We want this even if SKIP_CUDA_BUILD because when we run python setup.py sdist we want the .hpp -# files included in the source distribution, in case the user compiles from source. -subprocess.run(["git", "submodule", "update", "--init", "cutlass"]) - -if not SKIP_CUDA_BUILD: - print("\n\ntorch.__version__ = {}\n\n".format(torch.__version__)) - TORCH_MAJOR = int(torch.__version__.split(".")[0]) - TORCH_MINOR = int(torch.__version__.split(".")[1]) - - check_if_cuda_home_none(PACKAGE_NAME) - _, bare_metal_version = get_cuda_bare_metal_version(CUDA_HOME) - if bare_metal_version < Version("12.3"): - raise RuntimeError( - f"FlashAttention-3 is only supported on CUDA 12.3 and above, get {bare_metal_version} from {CUDA_HOME}" - ) - - cc_flag = [] - cc_flag.append("-gencode") - cc_flag.append("arch=compute_90a,code=sm_90a") - - # HACK: The compiler flag -D_GLIBCXX_USE_CXX11_ABI is set to be the same as - # torch._C._GLIBCXX_USE_CXX11_ABI - # https://github.com/pytorch/pytorch/blob/8472c24e3b5b60150096486616d98b7bea01500b/torch/utils/cpp_extension.py#L920 - if FORCE_CXX11_ABI: - torch._C._GLIBCXX_USE_CXX11_ABI = True - repo_dir = Path(this_dir).parent - cutlass_dir = repo_dir / "cpp" / "cutlass" - - feature_args = ( - [] - + ["-DOSS_ENV"] - + (["-DFLASHATTENTION_DISABLE_BACKWARD"] if DISABLE_BACKWARD else []) - + (["-DFLASHATTENTION_DISABLE_FP16"] if DISABLE_FP16 else []) - + ["-DFLASHATTENTION_DISABLE_FP8"] - + (["-DFLASHATTENTION_DISABLE_HDIM64"] if DISABLE_HDIM64 else []) - + (["-DFLASHATTENTION_DISABLE_HDIM96"] if DISABLE_HDIM96 else []) - + (["-DFLASHATTENTION_DISABLE_HDIM128"] if DISABLE_HDIM128 else []) - + (["-DFLASHATTENTION_DISABLE_HDIM192"] if DISABLE_HDIM192 else []) - + (["-DFLASHATTENTION_DISABLE_HDIM256"] if DISABLE_HDIM256 else []) - + (["-DFLASHATTENTION_DISABLE_SM8x"] if DISABLE_SM8x else []) - ) - - DTYPE = ["bf16"] + (["fp16"] if not DISABLE_FP16 else []) - HEAD_DIMENSIONS = ( - [] - + ([64] if not DISABLE_HDIM64 else []) - + ([96] if not DISABLE_HDIM96 else []) - + ([128] if not DISABLE_HDIM128 else []) - + ([192] if not DISABLE_HDIM192 else []) - + ([256] if not DISABLE_HDIM256 else []) - ) - sources_fwd_sm80 = [ - f"hstu_attention/instantiations/flash_fwd_hdim{hdim}_{dtype}_sm80.cu" - for hdim, dtype in itertools.product(HEAD_DIMENSIONS, DTYPE) - ] - sources_bwd_sm80 = [ - f"hstu_attention/instantiations/flash_bwd_hdim{hdim}_{dtype}_sm80.cu" - for hdim, dtype in itertools.product(HEAD_DIMENSIONS, DTYPE) - ] - sources_fwd_sm90 = [ - f"hstu_attention/instantiations/flash_fwd_hdim{hdim}_{dtype}_sm90.cu" - for hdim, dtype in itertools.product(HEAD_DIMENSIONS, DTYPE) - ] - sources_bwd_sm90 = [ - f"hstu_attention/instantiations/flash_bwd_hdim{hdim}_{dtype}_sm90.cu" - for hdim, dtype in itertools.product(HEAD_DIMENSIONS, DTYPE) - ] - if DISABLE_BACKWARD: - sources_bwd_sm90 = [] - sources_bwd_sm80 = [] - sources = ( - [ - "hstu_attention/flash_api.cpp", - "hstu_attention/flash_common.cpp", - "hstu_attention/flash_cpu_dummy.cpp", - "hstu_attention/flash_meta.cpp", - ] - + (sources_fwd_sm80 if not DISABLE_SM8x else []) - + sources_fwd_sm90 - + (sources_bwd_sm80 if not DISABLE_SM8x else []) - + sources_bwd_sm90 - ) - nvcc_flags = [ - "-O3", - "-std=c++17", - "--ftemplate-backtrace-limit=0", # To debug template code - "--use_fast_math", - # "--keep", - # "--ptxas-options=--verbose,--register-usage-level=5,--warn-on-local-memory-usage", # printing out number of registers - "--resource-usage", # printing out number of registers - # f"--split-compile={os.getenv('NVCC_THREADS', '4')}", # split-compile is faster - "-lineinfo", # TODO: disable this for release to reduce binary size - "-DCUTE_SM90_EXTENDED_MMA_SHAPES_ENABLED", # Necessary for the WGMMA shapes that we use - "-DCUTLASS_ENABLE_GDC_FOR_SM90", # For PDL - "-DCUTLASS_DEBUG_TRACE_LEVEL=0", # Can toggle for debugging - "-DNDEBUG", # Important, otherwise performance is severely impacted - "-Xfatbin", # compress all binary sections - "-compress-all", - ] - if get_platform() == "win_amd64": - nvcc_flags.extend( - [ - "-D_USE_MATH_DEFINES", # for M_LN2 - "-Xcompiler=/Zc:__cplusplus", # sets __cplusplus correctly, CUTLASS_CONSTEXPR_IF_CXX17 needed for cutlass::gcd - ] - ) - include_dirs = [ - Path(this_dir), - cutlass_dir / "include", - ] - - ext_modules.append( - CUDAExtension( - name=f"{PACKAGE_NAME}._C", - sources=sources, - extra_compile_args={ - "cxx": ["-O3", "-std=c++17", "-DPy_LIMITED_API=0x03090000"] - + feature_args, - "nvcc": nvcc_threads_args() + nvcc_flags + cc_flag + feature_args, - }, - include_dirs=include_dirs, - py_limited_api=True, - ) - ) - - -setup( - name=PACKAGE_NAME, - version="0.1.0", - packages=find_packages( - exclude=( - "build", - "csrc", - "include", - "tests", - "dist", - "docs", - "benchmarks", - ) - ), - py_modules=["cuda_hstu_attention"], - description="FlashAttention HSTU", - classifiers=[ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: Apache Software License", - "Operating System :: Unix", - ], - ext_modules=ext_modules, - cmdclass={"build_ext": BuildExtension}, - python_requires=">=3.8", - install_requires=[ - "torch", - "einops", - "packaging", - "ninja==1.11.1.1", - ], -) diff --git a/recommendation_v4/generative_recommenders/ops/cpp/sort_kv_pairs_cuda.cpp b/recommendation_v4/generative_recommenders/ops/cpp/sort_kv_pairs_cuda.cpp deleted file mode 100644 index 3925aefd7..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/sort_kv_pairs_cuda.cpp +++ /dev/null @@ -1,40 +0,0 @@ -#include "common.h" -#include "sort_kv_pairs_cuda_kernels_template.h" - -namespace hstu { - -DLL_PUBLIC std::tuple sort_kv_pairs_cuda( - const at::Tensor& keys, - const at::Tensor& values, - const std::optional& end_bit, - const bool descending = false) { - at::cuda::OptionalCUDAGuard device_guard; - device_guard.set_index(keys.get_device()); - TORCH_CHECK( - keys.dtype() == at::kInt || keys.dtype() == at::kLong || - keys.dtype() == at::kByte || keys.dtype() == at::kShort); - TORCH_CHECK(keys.numel() < std::numeric_limits::max()); - TORCH_CHECK(keys.dim() == 1); - TORCH_CHECK(values.dim() == 1); - at::Tensor sorted_keys; - at::Tensor sorted_values; - - AT_DISPATCH_INTEGRAL_TYPES(keys.scalar_type(), "sort_pairs_cuda_input1", [&] { - using key_t = scalar_t; - AT_DISPATCH_ALL_TYPES_AND2( - at::ScalarType::Half, - at::ScalarType::BFloat16, - values.scalar_type(), - "sort_pairs_cuda_input2", - [&] { - using val_t = scalar_t; - std::tie(sorted_keys, sorted_values) = - sort_kv_pairs_cuda_dispatched( - keys, values, end_bit, descending); - }); - }); - - return {std::move(sorted_keys), std::move(sorted_values)}; -} - -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/sort_kv_pairs_cuda_kernels_template.cu b/recommendation_v4/generative_recommenders/ops/cpp/sort_kv_pairs_cuda_kernels_template.cu deleted file mode 100644 index 8cd175c71..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/sort_kv_pairs_cuda_kernels_template.cu +++ /dev/null @@ -1,82 +0,0 @@ -#include -#include - -#include - -namespace hstu { - -template <> -DLL_PUBLIC std::tuple -sort_kv_pairs_cuda_dispatched( - const at::Tensor& keys, - const at::Tensor& values, - const std::optional& end_bit, - const bool descending) { - size_t temp_storage_bytes = 0; - auto keys_contig = keys.contiguous(); - auto values_contig = values.contiguous(); - auto sorted_keys = at::empty_like(keys_contig); - auto sorted_values = at::empty_like(values_contig); - - if (descending) { - AT_CUDA_CHECK( - cub::DeviceRadixSort::SortPairsDescending( - nullptr, - temp_storage_bytes, - keys_contig.data_ptr(), - sorted_keys.data_ptr(), - values_contig.data_ptr(), - sorted_values.data_ptr(), - keys_contig.numel(), - 0, - end_bit.has_value() ? end_bit.value() : sizeof(SUB_KEY_T) * 8, - at::cuda::getCurrentCUDAStream())); - auto temp_storage = at::empty( - {static_cast(temp_storage_bytes)}, - keys_contig.options().dtype(at::kByte)); - AT_CUDA_CHECK( - cub::DeviceRadixSort::SortPairsDescending( - temp_storage.data_ptr(), - temp_storage_bytes, - keys_contig.data_ptr(), - sorted_keys.data_ptr(), - values_contig.data_ptr(), - sorted_values.data_ptr(), - keys_contig.numel(), - 0, - end_bit.has_value() ? end_bit.value() : sizeof(SUB_KEY_T) * 8, - at::cuda::getCurrentCUDAStream())); - } else { - AT_CUDA_CHECK( - cub::DeviceRadixSort::SortPairs( - nullptr, - temp_storage_bytes, - keys_contig.data_ptr(), - sorted_keys.data_ptr(), - values_contig.data_ptr(), - sorted_values.data_ptr(), - keys_contig.numel(), - 0, - end_bit.has_value() ? end_bit.value() : sizeof(SUB_KEY_T) * 8, - at::cuda::getCurrentCUDAStream())); - auto temp_storage = at::empty( - {static_cast(temp_storage_bytes)}, - keys_contig.options().dtype(at::kByte)); - AT_CUDA_CHECK( - cub::DeviceRadixSort::SortPairs( - temp_storage.data_ptr(), - temp_storage_bytes, - keys_contig.data_ptr(), - sorted_keys.data_ptr(), - values_contig.data_ptr(), - sorted_values.data_ptr(), - keys_contig.numel(), - 0, - end_bit.has_value() ? end_bit.value() : sizeof(SUB_KEY_T) * 8, - at::cuda::getCurrentCUDAStream())); - } - - return {std::move(sorted_keys), std::move(sorted_values)}; -} - -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/sort_kv_pairs_cuda_kernels_template.h b/recommendation_v4/generative_recommenders/ops/cpp/sort_kv_pairs_cuda_kernels_template.h deleted file mode 100644 index e599eccb0..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/sort_kv_pairs_cuda_kernels_template.h +++ /dev/null @@ -1,15 +0,0 @@ -#pragma once - -#include -#include - -namespace hstu { - -template -std::tuple sort_kv_pairs_cuda_dispatched( - const at::Tensor& keys_contig, - const at::Tensor& values_contig, - const std::optional& end_bit, - const bool descending); - -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/split_1d_jagged_jagged.cpp b/recommendation_v4/generative_recommenders/ops/cpp/split_1d_jagged_jagged.cpp deleted file mode 100644 index c361488fa..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/split_1d_jagged_jagged.cpp +++ /dev/null @@ -1,136 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "fbgemm_gpu/sparse_ops.h" // @manual - -namespace hstu { - -template -void _split_1d_jagged_jagged_cpu_kernel( - int32_t B, - const at::TensorAccessor& combined_offsets, - const at::TensorAccessor& combined_values, - const at::TensorAccessor& lengths_left, - const at::TensorAccessor& offsets_left, - const at::TensorAccessor& offsets_right, - at::TensorAccessor values_left, - at::TensorAccessor values_right) { - for (auto b : c10::irange(B)) { - auto combined_start = combined_offsets[b]; - auto left_len = lengths_left[b]; - auto left_start = offsets_left[b]; - auto right_start = offsets_right[b]; - - for (auto i = 0; i < left_len; ++i) { - values_left[left_start + i] = combined_values[combined_start + i]; - } - - auto right_len = combined_offsets[b + 1] - combined_offsets[b] - left_len; - for (auto i = 0; i < right_len; ++i) { - values_right[right_start + i] = - combined_values[combined_start + left_len + i]; - } - } -} - -std::tuple split_1d_jagged_jagged_cpu( - const at::Tensor& lengths_left, - const at::Tensor& lengths_right, - const at::Tensor& combined_values) { - TORCH_INTERNAL_ASSERT(lengths_left.device().type() == at::DeviceType::CPU); - TORCH_INTERNAL_ASSERT(lengths_right.device().type() == at::DeviceType::CPU); - TORCH_INTERNAL_ASSERT(combined_values.device().type() == at::DeviceType::CPU); - TORCH_CHECK(lengths_left.size(0) == lengths_right.size(0)); - auto B = lengths_left.size(0); - - auto L_left = lengths_left.sum().item(); - auto L_right = lengths_right.sum().item(); - TORCH_CHECK(L_left + L_right == combined_values.numel()); - - auto values_left = at::empty({L_left}, combined_values.options()); - auto values_right = at::empty({L_right}, combined_values.options()); - - if (L_left == 0 && L_right == 0) { - return std::make_tuple(values_left, values_right); - } - - const auto combined_lengths = lengths_left + lengths_right; - const auto combined_offsets = - fbgemm_gpu::asynchronous_complete_cumsum_cpu(combined_lengths.view({-1})); - const auto offsets_left = - fbgemm_gpu::asynchronous_complete_cumsum_cpu(lengths_left.view({-1})); - const auto offsets_right = - fbgemm_gpu::asynchronous_complete_cumsum_cpu(lengths_right.view({-1})); - - AT_DISPATCH_INTEGRAL_TYPES( - lengths_left.scalar_type(), - "split_1d_jagged_jagged_values_cpu_kernel_input1", - [&] { - using index_t = scalar_t; - AT_DISPATCH_ALL_TYPES_AND2( - at::ScalarType::BFloat16, - at::ScalarType::Half, - combined_values.scalar_type(), - "split_1d_jagged_jagged_values_cpu_kernel_input2", - [&] { - using val_t = scalar_t; - _split_1d_jagged_jagged_cpu_kernel( - B, - combined_offsets.accessor(), - combined_values.accessor(), - lengths_left.accessor(), - offsets_left.accessor(), - offsets_right.accessor(), - values_left.accessor(), - values_right.accessor()); - }); - }); - - return std::make_tuple(values_left, values_right); -} - -std::tuple split_1d_jagged_jagged_meta( - const at::Tensor& lengths_left, - const at::Tensor& lengths_right, - const at::Tensor& combined_values) { - auto L_left = lengths_left.sum().item(); - auto L_right = lengths_right.sum().item(); - - auto values_left = at::native::empty_meta_symint( - {L_left}, - /*dtype=*/::std::make_optional(combined_values.scalar_type()), - /*layout=*/::std::make_optional(combined_values.layout()), - /*device=*/::std::make_optional(c10::Device(c10::kMeta)), - /*pin_memory=*/::std::nullopt); - - auto values_right = at::native::empty_meta_symint( - {L_right}, - /*dtype=*/::std::make_optional(combined_values.scalar_type()), - /*layout=*/::std::make_optional(combined_values.layout()), - /*device=*/::std::make_optional(c10::Device(c10::kMeta)), - /*pin_memory=*/::std::nullopt); - - return std::make_tuple(values_left, values_right); -} -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/split_1d_jagged_jagged.cu b/recommendation_v4/generative_recommenders/ops/cpp/split_1d_jagged_jagged.cu deleted file mode 100644 index 181489bae..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/split_1d_jagged_jagged.cu +++ /dev/null @@ -1,147 +0,0 @@ -/* Copyright (c) Meta Platforms, Inc. and affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include - -#include "common.h" -#include "fbgemm_gpu/sparse_ops.h" // @manual -#include "fbgemm_gpu/utils/fixed_divisor.cuh" // @manual - -namespace hstu { - -static constexpr int32_t kMaxThreads = 1024; - -template -__global__ -__launch_bounds__(kMaxThreads) void _split_1d_jagged_jagged_cuda_kernel( - int32_t B, - const at::PackedTensorAccessor32 - combined_offsets, - const at::PackedTensorAccessor32 - combined_values, - const at::PackedTensorAccessor32 - lengths_left, - const at::PackedTensorAccessor32 - offsets_left, - const at::PackedTensorAccessor32 - offsets_right, - at::PackedTensorAccessor32 values_left, - at::PackedTensorAccessor32 values_right) { - for (auto b = blockIdx.x * blockDim.y + threadIdx.y; - b < static_cast(B); - b += gridDim.x * blockDim.y) { - auto combined_start = combined_offsets[b]; - auto left_len = lengths_left[b]; - auto right_len = combined_offsets[b + 1] - combined_offsets[b] - left_len; - auto left_start = offsets_left[b]; - auto right_start = offsets_right[b]; - - for (auto i = threadIdx.x; i < static_cast(left_len + right_len); - i += blockDim.x) { - if (i < static_cast(left_len)) { - values_left[left_start + i] = combined_values[combined_start + i]; - } else { - values_right[right_start + i - left_len] = - combined_values[combined_start + i]; - } - } - } -} - -std::tuple split_1d_jagged_jagged_cuda( - const at::Tensor& lengths_left, - const at::Tensor& lengths_right, - const at::Tensor& combined_values) { - at::cuda::OptionalCUDAGuard device_guard; - device_guard.set_index(combined_values.get_device()); - TORCH_INTERNAL_ASSERT(lengths_left.device().type() == at::DeviceType::CUDA); - TORCH_INTERNAL_ASSERT(lengths_right.device().type() == at::DeviceType::CUDA); - TORCH_INTERNAL_ASSERT( - combined_values.device().type() == at::DeviceType::CUDA); - TORCH_CHECK(lengths_left.size(0) == lengths_right.size(0)); - - auto B = lengths_left.size(0); - auto L_left = lengths_left.sum().item(); - auto L_right = lengths_right.sum().item(); - TORCH_CHECK(L_left + L_right == combined_values.numel()); - TORCH_CHECK(L_left < std::numeric_limits::max()); - TORCH_CHECK(L_right < std::numeric_limits::max()); - TORCH_CHECK(combined_values.get_device() == lengths_left.get_device()); - TORCH_CHECK(combined_values.get_device() == lengths_right.get_device()); - - auto values_left = at::empty({L_left}, combined_values.options()); - auto values_right = at::empty({L_right}, combined_values.options()); - - if (L_left == 0 && L_right == 0) { - return std::make_tuple(values_left, values_right); - } - - const auto combined_lengths = lengths_left + lengths_right; - const auto combined_offsets = - fbgemm_gpu::asynchronous_complete_cumsum_gpu(combined_lengths.view({-1})); - const auto offsets_left = - fbgemm_gpu::asynchronous_complete_cumsum_gpu(lengths_left.view({-1})); - const auto offsets_right = - fbgemm_gpu::asynchronous_complete_cumsum_gpu(lengths_right.view({-1})); - - // Optimized thread block configuration based on benchmark results - uint32_t B_blocks = 4; - dim3 threads(256, B_blocks); - auto blocks = div_round_up(B, B_blocks); - - AT_DISPATCH_INTEGRAL_TYPES( - lengths_left.scalar_type(), - "split_1d_jagged_jagged_values_cuda_kernel_input1", - [&] { - using index_t = scalar_t; - AT_DISPATCH_ALL_TYPES_AND2( - at::ScalarType::BFloat16, - at::ScalarType::Half, - combined_values.scalar_type(), - "split_1d_jagged_jagged_values_cuda_kernel_input2", - [&] { - using val_t = scalar_t; - _split_1d_jagged_jagged_cuda_kernel<<< - blocks, - threads, - 0, - at::cuda::getCurrentCUDAStream()>>>( - B, - combined_offsets - .packed_accessor32(), - combined_values - .packed_accessor32(), - lengths_left - .packed_accessor32(), - offsets_left - .packed_accessor32(), - offsets_right - .packed_accessor32(), - values_left - .packed_accessor32(), - values_right - .packed_accessor32()); - }); - }); - - return std::make_tuple(values_left, values_right); -} -} // namespace hstu diff --git a/recommendation_v4/generative_recommenders/ops/cpp/tests/concat_1d_jagged_jagged_test.py b/recommendation_v4/generative_recommenders/ops/cpp/tests/concat_1d_jagged_jagged_test.py deleted file mode 100644 index 8c27a787b..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/tests/concat_1d_jagged_jagged_test.py +++ /dev/null @@ -1,135 +0,0 @@ -#!/usr/bin/env python3 - -# pyre-strict - -import unittest - -import torch -from generative_recommenders.common import gpu_unavailable -from hammer.ops.jagged import concat_1D_jagged_jagged -from hypothesis import given, settings, strategies as st, Verbosity - -# buck2 test @mode/opt -c fbcode.nvcc_arch=h100 fbcode//generative_recommenders/ops/cpp/tests:concat_1d_jagged_jagged_test - -torch.ops.load_library("//generative_recommenders/ops/cpp:cpp_ops") -torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops") -torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops_cpu") - - -class OpsTest(unittest.TestCase): - @unittest.skipIf(*gpu_unavailable) - # pyre-ignore - @given( - batch_size=st.integers(10, 500), - max_seq_len_left=st.integers(10, 1000), - max_seq_len_right=st.integers(10, 1000), - val_dtype=st.sampled_from([torch.float32, torch.float16, torch.bfloat16]), - ) - @settings( - verbosity=Verbosity.verbose, - max_examples=100, - deadline=None, - ) - def test_concat_1d_jagged_jagged( - self, - batch_size: int, - max_seq_len_left: int, - max_seq_len_right: int, - val_dtype: torch.dtype, - ) -> None: - batch_size = 3 - max_seq_len_left = 4 - max_seq_len_right = 2 - lengths_left = torch.randint( - 0, max_seq_len_left + 1, (batch_size,), device="cpu" - ) - values_left = torch.rand( - (int(torch.sum(lengths_left).cpu().item()),), dtype=val_dtype, device="cpu" - ) - offsets_left = torch.zeros( - (batch_size + 1,), - dtype=lengths_left.dtype, - device=lengths_left.device, - ) - offsets_left[1:] = torch.cumsum(lengths_left.view(-1), dim=0) - lengths_right = torch.randint( - 0, max_seq_len_right + 1, (batch_size,), device="cpu" - ) - values_right = torch.rand( - (int(torch.sum(lengths_right).cpu().item()),), dtype=val_dtype, device="cpu" - ) - offsets_right = torch.zeros( - (batch_size + 1,), - dtype=lengths_right.dtype, - device=lengths_right.device, - ) - offsets_right[1:] = torch.cumsum(lengths_right.view(-1), dim=0) - custom_cpu_result = torch.ops.hstu.concat_1d_jagged_jagged( - lengths_left=lengths_left, - values_left=values_left, - lengths_right=lengths_right, - values_right=values_right, - ) - - custom_cuda_result = torch.ops.hstu.concat_1d_jagged_jagged( - lengths_left=lengths_left.cuda(), - values_left=values_left.cuda(), - lengths_right=lengths_right.cuda(), - values_right=values_right.cuda(), - ) - torch.testing.assert_close(custom_cuda_result.cpu(), custom_cpu_result) - - @unittest.skipIf(*gpu_unavailable) - def test_concat_1d_jagged_jagged_vs_hammer(self) -> None: - torch.manual_seed(42) - batch_size = 8 - max_seq_len_left = 50 - max_seq_len_right = 30 - - lengths_left = torch.randint( - 0, max_seq_len_left + 1, (batch_size,), dtype=torch.int32 - ) - lengths_right = torch.randint( - 0, max_seq_len_right + 1, (batch_size,), dtype=torch.int32 - ) - - total_left = int(lengths_left.sum().item()) - total_right = int(lengths_right.sum().item()) - - values_left = ( - torch.randn(total_left, dtype=torch.float32) - if total_left > 0 - else torch.empty(0, dtype=torch.float32) - ) - values_right = ( - torch.randn(total_right, dtype=torch.float32) - if total_right > 0 - else torch.empty(0, dtype=torch.float32) - ) - - offsets_left = torch.zeros( - (batch_size + 1,), dtype=lengths_left.dtype, device=lengths_left.device - ) - offsets_left[1:] = torch.cumsum(lengths_left.view(-1), dim=0) - offsets_right = torch.zeros( - (batch_size + 1,), dtype=lengths_right.dtype, device=lengths_right.device - ) - offsets_right[1:] = torch.cumsum(lengths_right.view(-1), dim=0) - - combined_values_ref = concat_1D_jagged_jagged( - max_seq_len_left=max_seq_len_left, - offsets_left=offsets_left, - values_left=values_left, - max_seq_len_right=max_seq_len_right, - offsets_right=offsets_right, - values_right=values_right, - ) - - custom_cuda_result = torch.ops.hstu.concat_1d_jagged_jagged( - lengths_left=lengths_left.cuda(), - values_left=values_left.cuda(), - lengths_right=lengths_right.cuda(), - values_right=values_right.cuda(), - ) - - torch.testing.assert_close(custom_cuda_result.cpu(), combined_values_ref) diff --git a/recommendation_v4/generative_recommenders/ops/cpp/tests/hstu_mha_cpu_test.py b/recommendation_v4/generative_recommenders/ops/cpp/tests/hstu_mha_cpu_test.py deleted file mode 100644 index cb787ea61..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/tests/hstu_mha_cpu_test.py +++ /dev/null @@ -1,39 +0,0 @@ -# (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -# pyre-strict - -# cmd: buck2 run @//mode/opt -c fbcode.enable_gpu_sections=true -c fbcode.platform010_cuda_version=12.8 -c fbcode.nvcc_arch=b200a //generative_recommenders/ops/cpp/tests:hstu_mha_cpu_test - -import unittest - -import torch - -torch.ops.load_library( - "//generative_recommenders/ops/cpp/hstu_attention:hstu_flash_attention" -) - - -class TestHstuMhaFwd(unittest.TestCase): - def test_hstu_mha_fwd(self) -> None: - q: torch.Tensor = torch.randn([100, 4, 64], dtype=torch.bfloat16, device="cpu") - k: torch.Tensor = torch.randn([100, 4, 64], dtype=torch.bfloat16, device="cpu") - v: torch.Tensor = torch.randn([100, 4, 64], dtype=torch.bfloat16, device="cpu") - res = torch.ops.hstu.hstu_mha_fwd( - 10, - 0.25, - q, - k, - v, - torch.empty([0], dtype=torch.int32, device="cpu"), - True, # causal - None, - None, - 0, - 0, - 0, - None, # q_descale - None, # k_descale - None, # v_descale - 0, # sm_margin - ) - self.assertIsNotNone(res) diff --git a/recommendation_v4/generative_recommenders/ops/cpp/tests/jagged_transpose_1d_test.py b/recommendation_v4/generative_recommenders/ops/cpp/tests/jagged_transpose_1d_test.py deleted file mode 100644 index 6a5f5997b..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/tests/jagged_transpose_1d_test.py +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/env python3 - -# pyre-strict - -import unittest - -import torch -from generative_recommenders.common import gpu_unavailable -from hammer.ops.jagged import jagged_transpose_1D -from hypothesis import given, settings, strategies as st, Verbosity - -# buck2 test @mode/opt -c fbcode.nvcc_arch=h100 fbcode//generative_recommenders/ops/cpp/tests:jagged_transpose_1d_test - -torch.ops.load_library("//generative_recommenders/ops/cpp:cpp_ops") -torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops") -torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops_cpu") - - -class OpsTest(unittest.TestCase): - @unittest.skipIf(*gpu_unavailable) - # pyre-ignore - @given( - size1=st.integers(2, 10), - size2=st.integers(2, 10), - max_len=st.integers(5, 50), - val_dtype=st.sampled_from([torch.float32, torch.float16, torch.bfloat16]), - ) - @settings( - verbosity=Verbosity.verbose, - max_examples=100, - deadline=None, - ) - def test_jagged_transpose_1d( - self, - size1: int, - size2: int, - max_len: int, - val_dtype: torch.dtype, - ) -> None: - lengths = torch.randint( - 0, max_len + 1, (size1 * size2,), dtype=torch.int32, device="cpu" - ) - offsets = torch.zeros( - (size1 * size2 + 1,), dtype=lengths.dtype, device=lengths.device - ) - offsets[1:] = torch.cumsum(lengths.view(-1), dim=0) - - values = torch.randn(int(offsets[-1].item()), dtype=val_dtype, device="cpu") - - ( - custom_cpu_values, - custom_cpu_offsets, - custom_cpu_lengths, - ) = torch.ops.hstu.jagged_transpose_1d( - values=values, - offsets=offsets, - lengths=lengths, - max_len=max_len, - size1=size1, - size2=size2, - ) - - ( - custom_cuda_values, - custom_cuda_offsets, - custom_cuda_lengths, - ) = torch.ops.hstu.jagged_transpose_1d( - values=values.cuda(), - offsets=offsets.cuda(), - lengths=lengths.cuda(), - max_len=max_len, - size1=size1, - size2=size2, - ) - - torch.testing.assert_close(custom_cuda_values.cpu(), custom_cpu_values) - torch.testing.assert_close(custom_cuda_offsets.cpu(), custom_cpu_offsets) - torch.testing.assert_close(custom_cuda_lengths.cpu(), custom_cpu_lengths) - - @unittest.skipIf(*gpu_unavailable) - # pyre-ignore - @given( - size1=st.integers(2, 10), - size2=st.integers(2, 10), - max_len=st.integers(5, 50), - val_dtype=st.sampled_from([torch.float32, torch.float16, torch.bfloat16]), - ) - @settings( - verbosity=Verbosity.verbose, - max_examples=100, - deadline=None, - ) - def test_jagged_transpose_1d_vs_hammer( - self, - size1: int, - size2: int, - max_len: int, - val_dtype: torch.dtype, - ) -> None: - lengths = torch.randint(0, max_len + 1, (size1 * size2,), dtype=torch.int32) - offsets = torch.zeros( - (size1 * size2 + 1,), dtype=lengths.dtype, device=lengths.device - ) - offsets[1:] = torch.cumsum(lengths.view(-1), dim=0) - - values = torch.randn(int(offsets[-1].item()), dtype=val_dtype) - - values_ref, offsets_ref, lengths_ref = jagged_transpose_1D( - values=values, - offsets=offsets, - lengths=lengths, - max_len=max_len, - size1=size1, - size2=size2, - ) - - ( - custom_cuda_values, - custom_cuda_offsets, - custom_cuda_lengths, - ) = torch.ops.hstu.jagged_transpose_1d( - values=values.cuda(), - offsets=offsets.cuda(), - lengths=lengths.cuda(), - max_len=max_len, - size1=size1, - size2=size2, - ) - - torch.testing.assert_close(custom_cuda_values.cpu(), values_ref) - torch.testing.assert_close(custom_cuda_offsets.cpu(), offsets_ref) - torch.testing.assert_close(custom_cuda_lengths.cpu(), lengths_ref) diff --git a/recommendation_v4/generative_recommenders/ops/cpp/tests/replace_last_n_with_jagged_test.py b/recommendation_v4/generative_recommenders/ops/cpp/tests/replace_last_n_with_jagged_test.py deleted file mode 100644 index 9826f199d..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/tests/replace_last_n_with_jagged_test.py +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env python3 - -# pyre-strict - -import unittest - -import torch -from generative_recommenders.common import gpu_unavailable -from hammer.ops.jagged import replace_last_n_with_jagged - -# buck2 test @mode/opt -c fbcode.nvcc_arch=h100 fbcode//generative_recommenders/ops/cpp/tests:replace_last_n_with_jagged_test - -torch.ops.load_library("//generative_recommenders/ops/cpp:cpp_ops") -torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops") -torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops_cpu") - - -class OpsTest(unittest.TestCase): - @unittest.skipIf(*gpu_unavailable) - def test_replace_last_n_with_jagged(self) -> None: - torch.manual_seed(42) - batch_size = 8 - embedding_dim = 64 - max_seq_len_left = 25 - max_seq_len_right = 10 - - lengths_left = torch.randint( - max_seq_len_right, max_seq_len_left + 1, (batch_size,), dtype=torch.int32 - ) - lengths_right = torch.randint( - 1, max_seq_len_right + 1, (batch_size,), dtype=torch.int32 - ) - - lengths_right = torch.min(lengths_right, lengths_left) - - total_left = int(lengths_left.sum().item()) - total_right = int(lengths_right.sum().item()) - - values_left = torch.randn(total_left, embedding_dim, dtype=torch.float32) - values_right = torch.randn(total_right, embedding_dim, dtype=torch.float32) - - custom_cpu_result = torch.ops.hstu.replace_last_n_with_jagged( - lengths_left=lengths_left, - values_left=values_left, - lengths_right=lengths_right, - values_right=values_right, - ) - - custom_cuda_result = torch.ops.hstu.replace_last_n_with_jagged( - lengths_left=lengths_left.cuda(), - values_left=values_left.cuda(), - lengths_right=lengths_right.cuda(), - values_right=values_right.cuda(), - ) - - torch.testing.assert_close(custom_cuda_result.cpu(), custom_cpu_result) - - @unittest.skipIf(*gpu_unavailable) - def test_replace_last_n_with_jagged_vs_hammer(self) -> None: - torch.manual_seed(42) - batch_size = 8 - embedding_dim = 32 - max_seq_len_left = 20 - max_seq_len_right = 8 - - lengths_left = torch.randint( - max_seq_len_right, max_seq_len_left + 1, (batch_size,), dtype=torch.int32 - ) - lengths_right = torch.randint( - 1, max_seq_len_right + 1, (batch_size,), dtype=torch.int32 - ) - - lengths_right = torch.min(lengths_right, lengths_left) - - total_left = int(lengths_left.sum().item()) - total_right = int(lengths_right.sum().item()) - - values_left = torch.randn(total_left, embedding_dim, dtype=torch.float32) - values_right = torch.randn(total_right, embedding_dim, dtype=torch.float32) - - offsets_left = torch.zeros( - (batch_size + 1,), dtype=lengths_left.dtype, device=lengths_left.device - ) - offsets_left[1:] = torch.cumsum(lengths_left.view(-1), dim=0) - offsets_right = torch.zeros( - (batch_size + 1,), dtype=lengths_right.dtype, device=lengths_right.device - ) - offsets_right[1:] = torch.cumsum(lengths_right.view(-1), dim=0) - - result_ref = replace_last_n_with_jagged( - max_seq_len_left=max_seq_len_left, - offsets_left=offsets_left, - values_left=values_left, - offsets_right=offsets_right, - values_right=values_right, - ) - - custom_cuda_result = torch.ops.hstu.replace_last_n_with_jagged( - lengths_left=lengths_left.cuda(), - values_left=values_left.cuda(), - lengths_right=lengths_right.cuda(), - values_right=values_right.cuda(), - ) - - torch.testing.assert_close(custom_cuda_result.cpu(), result_ref) diff --git a/recommendation_v4/generative_recommenders/ops/cpp/tests/split_1d_jagged_jagged_test.py b/recommendation_v4/generative_recommenders/ops/cpp/tests/split_1d_jagged_jagged_test.py deleted file mode 100644 index 24f12c4a2..000000000 --- a/recommendation_v4/generative_recommenders/ops/cpp/tests/split_1d_jagged_jagged_test.py +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env python3 - -# pyre-strict - -import unittest - -import torch -from generative_recommenders.common import gpu_unavailable -from hammer.ops.jagged import split_1D_jagged_jagged - -# buck2 test @mode/opt -c fbcode.nvcc_arch=h100 fbcode//generative_recommenders/ops/cpp/tests:split_1d_jagged_jagged_test - -torch.ops.load_library("//generative_recommenders/ops/cpp:cpp_ops") -torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops") -torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops_cpu") - - -class OpsTest(unittest.TestCase): - @unittest.skipIf(*gpu_unavailable) - def test_split_1d_jagged_jagged(self) -> None: - torch.manual_seed(42) - batch_size = 8 - max_seq_len_left = 25 - max_seq_len_right = 20 - - lengths_left = torch.randint( - 0, max_seq_len_left + 1, (batch_size,), dtype=torch.int32 - ) - lengths_right = torch.randint( - 0, max_seq_len_right + 1, (batch_size,), dtype=torch.int32 - ) - - combined_lengths = lengths_left + lengths_right - combined_offsets = torch.zeros( - (batch_size + 1,), dtype=lengths_left.dtype, device=lengths_left.device - ) - combined_offsets[1:] = torch.cumsum(combined_lengths.view(-1), dim=0) - - combined_values = torch.randn( - int(combined_offsets[-1].item()), dtype=torch.float32 - ) - - custom_cpu_left, custom_cpu_right = torch.ops.hstu.split_1d_jagged_jagged( - lengths_left=lengths_left, - lengths_right=lengths_right, - combined_values=combined_values, - ) - - custom_cuda_left, custom_cuda_right = torch.ops.hstu.split_1d_jagged_jagged( - lengths_left=lengths_left.cuda(), - lengths_right=lengths_right.cuda(), - combined_values=combined_values.cuda(), - ) - - torch.testing.assert_close(custom_cuda_left.cpu(), custom_cpu_left) - torch.testing.assert_close(custom_cuda_right.cpu(), custom_cpu_right) - - @unittest.skipIf(*gpu_unavailable) - def test_split_1d_jagged_jagged_vs_hammer(self) -> None: - torch.manual_seed(42) - batch_size = 8 - max_seq_len_left = 25 - max_seq_len_right = 20 - - lengths_left = torch.randint( - 0, max_seq_len_left + 1, (batch_size,), dtype=torch.int32 - ) - lengths_right = torch.randint( - 0, max_seq_len_right + 1, (batch_size,), dtype=torch.int32 - ) - - offsets_left = torch.zeros( - (batch_size + 1,), dtype=lengths_left.dtype, device=lengths_left.device - ) - offsets_left[1:] = torch.cumsum(lengths_left.view(-1), dim=0) - offsets_right = torch.zeros( - (batch_size + 1,), dtype=lengths_right.dtype, device=lengths_right.device - ) - offsets_right[1:] = torch.cumsum(lengths_right.view(-1), dim=0) - - combined_offsets = offsets_left + offsets_right - combined_values = torch.randn( - int(combined_offsets[-1].item()), dtype=torch.float32 - ) - - left_ref, right_ref = split_1D_jagged_jagged( - max_seq_len=max_seq_len_left + max_seq_len_right, - values=combined_values, - offsets_left=offsets_left, - offsets_right=offsets_right, - ) - - custom_cuda_left, custom_cuda_right = torch.ops.hstu.split_1d_jagged_jagged( - lengths_left=lengths_left.cuda(), - lengths_right=lengths_right.cuda(), - combined_values=combined_values.cuda(), - ) - - torch.testing.assert_close(custom_cuda_left.cpu(), left_ref) - torch.testing.assert_close(custom_cuda_right.cpu(), right_ref) diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/README.md b/recommendation_v4/generative_recommenders/ops/triton_aot/README.md deleted file mode 100644 index 2b0b1a834..000000000 --- a/recommendation_v4/generative_recommenders/ops/triton_aot/README.md +++ /dev/null @@ -1,54 +0,0 @@ -# Local Triton AOT Support - -This package is a minimal local copy of the Triton AOT pieces needed by the -DLRM v3 HSTU inference end-to-end test. It avoids depending on the standalone -`fbcode/triton_aot` package while preserving the compile, transform, and -runtime-loading flow used by `generative_recommenders`. - -This is not intended to be a full fork of `fbcode/triton_aot`. Keep changes -scoped to the GR inference use case unless a broader migration plan exists. - -## Code Structure - -- `types.py`: local `TritonAOT` registration object and `triton_aot` helper used - by GR AOT wrapper modules. -- `preprocess.py`: FX graph preprocessing helpers, including wrapper-node - unwrapping before compile/transform. -- `triton_*.py`: GR kernel-specific AOT wrapper modules for addmm, jagged - concat/split, layer norm variants, HSTU attention, and timestamp position - embeddings. -- `compile/`: compile-time state, Triton signature/spec processing, generated - C++ codegen, and the `TritonAOTCompile` context manager. -- `transform/`: FX graph transformation and generated Python wrapper code that - swaps Python AOT wrappers for `torch.ops.triton_aot.*` calls backed by built - shared libraries. -- `build/`: extension builders and CUBIN embedding utilities used to create - loadable kernel libraries from compiled Triton artifacts. -- `templates/`: C++ template files used by the compile/codegen path for kernel - entry points, embedded CUBIN data, and Torch operator registration. -- `shared/`: compatibility helpers and type/spec conversion utilities shared by - compile and transform code. - -## Runtime Flow - -1. GR `triton_*.py` wrappers expose Triton kernels through local `triton_aot` - descriptors. -2. `TritonAOTCompile` runs representative CUDA inputs, records kernel specs, and - compiles the collected Triton kernels into shared libraries. -3. `transform_kernels` rewrites the FX graph so wrapper calls dispatch through - `torch.ops.triton_aot.*`. -4. The e2e test copies the generated libraries into its workdir and passes them - to the C++ runner before executing the scripted sparse/dense modules. - - -## Authors - -- Chang Pan -- Zhiyong Wang (MRS) -- Chenzhi Yu -- Runming Lu -- Chun-Wei Chen -- Michael He -- Linjian Ma -- Xing Liu -- Zhuoran Zhao diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/compile/arg_descriptor.py b/recommendation_v4/generative_recommenders/ops/triton_aot/compile/arg_descriptor.py deleted file mode 100644 index bc5963674..000000000 --- a/recommendation_v4/generative_recommenders/ops/triton_aot/compile/arg_descriptor.py +++ /dev/null @@ -1,146 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. - -# pyre-strict - -"""ArgDescriptor — per-arg codegen descriptor for AOT-T. - -Centralises arg classification (pointer / scalar / constant) so every -``gen_*`` function in ``codegen.py`` iterates descriptors instead of -doing its own dict lookups into ``OpsUnit`` fields. - -Also provides type-mapping helpers that convert ``ArgDescriptor`` -metadata into context-specific C++ / TorchScript type strings. -""" - -from __future__ import annotations - -from dataclasses import dataclass -from typing import Any - -from generative_recommenders.ops.triton_aot.compile.spec_processing import OpsUnit -from triton.runtime.jit import JITFunction - - -# --------------------------------------------------------------------------- -# Type-mapping helpers -# --------------------------------------------------------------------------- - -CONSTANT_SELECTOR_CTYPE: dict[type[Any], str] = { - bool: "bool", - int: "int", - str: "const std::string&", -} - -CONSTANT_CPP_OP_CTYPE: dict[type[Any], str] = { - bool: "bool", - int: "int64_t", - str: "const std::string&", -} - -CONSTANT_TORCH_SCHEMA: dict[type[Any], str] = { - bool: "bool", - int: "int", - str: "str", -} - - -def scalar_cpp_op_ctype(triton_dtype: str) -> str: - """Triton scalar dtype → widened C++ type for cpp_op / torch_op params.""" - if triton_dtype.startswith("i"): - return "int64_t" - if triton_dtype.startswith("f"): - return "double" - if triton_dtype == "bool": - return "bool" - raise ValueError(f"Unsupported scalar dtype for cpp_op: {triton_dtype}") - - -def scalar_torch_schema(triton_dtype: str) -> str: - """Triton scalar dtype → TorchScript schema type string.""" - if triton_dtype.startswith("i"): - return "int" - if triton_dtype.startswith("f"): - return "float" - if triton_dtype == "bool": - return "bool" - raise ValueError(f"Unsupported scalar dtype for torch schema: {triton_dtype}") - - -# --------------------------------------------------------------------------- -# ArgDescriptor hierarchy -# --------------------------------------------------------------------------- - - -@dataclass(frozen=True) -class ArgDescriptor: - """Base class for per-arg codegen descriptors. - - Built once by ``build_arg_descriptors`` and consumed by all ``gen_*`` - functions. Use ``isinstance`` to dispatch on arg kind: - - - ``PointerArg`` — tensor pointer (required or optional) - - ``ScalarArg`` — non-pointer signature arg with a Triton dtype - - ``ConstantArg`` — compile-time constant with a Python type - """ - - name: str - index: int - - -@dataclass(frozen=True) -class PointerArg(ArgDescriptor): - """Tensor pointer arg (required or optional).""" - - is_optional: bool - - -@dataclass(frozen=True) -class ScalarArg(ArgDescriptor): - """Non-pointer signature arg with a Triton dtype (e.g., ``"i32"``, ``"fp32"``). - - ``triton_dtype`` is the **widest** type across all specs for this - position, computed by ``_compute_invariants`` via ``_wider_type``. - Individual specs may use a narrower type (e.g., ``"i32"`` when - ``triton_dtype`` is ``"i64"``); codegen adds ``fits_i32`` guards - and ``static_cast`` for narrowing. - """ - - triton_dtype: str - - -@dataclass(frozen=True) -class ConstantArg(ArgDescriptor): - """Compile-time constant arg with a Python type (``int``, ``str``, ``bool``).""" - - python_type: type[Any] - - -def build_arg_descriptors( - func: JITFunction[list[Any]], - unit: OpsUnit, -) -> list[ArgDescriptor]: - """Build ordered arg descriptors from func arg names + OpsUnit invariants. - - Single source of truth for arg classification. Called once in - ``compile_to_cpp`` and passed to all downstream codegen functions. - """ - result: list[ArgDescriptor] = [] - for i, name in enumerate(func.arg_names): - if i in unit.pointer_args: - result.append( - PointerArg(name=name, index=i, is_optional=i in unit.optional) - ) - elif i in unit.scalar_dtypes: - result.append( - ScalarArg(name=name, index=i, triton_dtype=unit.scalar_dtypes[i]) - ) - elif i in unit.constant_types: - result.append( - ConstantArg(name=name, index=i, python_type=unit.constant_types[i]) - ) - else: - raise ValueError( - f"Arg {name} (index {i}) not classified as pointer, scalar, " - f"or constant in OpsUnit" - ) - return result diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/compile/codegen.py b/recommendation_v4/generative_recommenders/ops/triton_aot/compile/codegen.py deleted file mode 100644 index f1b03848f..000000000 --- a/recommendation_v4/generative_recommenders/ops/triton_aot/compile/codegen.py +++ /dev/null @@ -1,780 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. - -# pyre-strict - -"""C++ and Python code generation for AOT-T compiled kernels. - -Generates: - - kernel.h (header with gridDims, tuner meta, selector proto) - - kernel.cpp (cubin externs, loaders, launchers, selector) - - _torch_op.cpp (torch op registration) - - _meta.py (Python autotuner meta function) -""" - -import textwrap -from collections import Counter -from typing import Any - -# @manual=//triton:triton -import triton -from generative_recommenders.ops.triton_aot.compile.arg_descriptor import ( - ArgDescriptor, - CONSTANT_CPP_OP_CTYPE, - CONSTANT_SELECTOR_CTYPE, - CONSTANT_TORCH_SCHEMA, - ConstantArg, - PointerArg, - scalar_cpp_op_ctype, - scalar_torch_schema, - ScalarArg, -) -from generative_recommenders.ops.triton_aot.compile.spec_processing import ( - KernelSpec, - OpsUnit, -) -from generative_recommenders.ops.triton_aot.compile.stable_types import ( - PY_TYPES_TO_CPP_TYPES, - SCALAR_TYPES, -) -from generative_recommenders.ops.triton_aot.compile.utils import ( - hash_kernel_name, - unwrap_heuristic, -) -from generative_recommenders.ops.triton_aot.shared.compat import get_scratch_parameters -from generative_recommenders.ops.triton_aot.shared.types import AUTOTUNE_ATTRs, CTYPES -from generative_recommenders.ops.triton_aot.templates.template_utils import ( - load_template, - render_template, -) -from triton.runtime.jit import JITFunction - - -# --------------------------------------------------------------------------- -# Kernel naming and binary generation -# --------------------------------------------------------------------------- - - -def gen_kernel_name( - fn: Any, - spec: KernelSpec, - cc: int | str, -) -> str: - name = fn.__name__ - sig = "_".join([p.replace("*", "p") for p in spec.signature.values()]) - const = "_".join(map(str, spec.constants.values())) - cc_str = f"sm{cc}" - autotune_configs = [] - autotune_configs.append(f"w{spec.num_warps}") - autotune_configs.append(f"s{spec.num_stages}") - # AMD only - autotune_configs.append(f"matrix{spec.matrix_instr_nonkdim}") - autotune_configs.append(f"wave{spec.waves_per_eu}") - autotune_configs.append(f"kpack{spec.kpack}") - # See kernel_suffix in triton/compiler/code_generator.py - suffix = "" - for i, _ in enumerate(spec.signature): - suffix += str(i) - if i in spec.divisible_by_16: - suffix += "d" - if i in spec.divisible_by_8: - suffix += "e" - return "_".join([name, cc_str, sig, const] + autotune_configs + [suffix]) - - -def gen_cubin(kernel_name: str, kernel: Any, install_dir: str, backend: str) -> str: - """Generate kernel binary file (.cubin or .hsaco) and return extern declaration. - - Args: - kernel_name: Full kernel name including specialization suffix. - kernel: Compiled Triton kernel object containing binary in kernel.asm. - install_dir: Directory to write binary file. - backend: GPU backend ("cuda" or "hip"). - - Returns: - C++ extern declaration for the kernel binary array. - """ - hashed = hash_kernel_name(kernel_name) - if backend == "hip": - binary_file = f"{install_dir}/{hashed}.hsaco" - with open(binary_file, "wb") as hsaco: - hsaco.write(kernel.asm["hsaco"]) - target_symbol_name = f"{kernel_name}_cubin" - else: - binary_file = f"{install_dir}/{hashed}.cubin" - with open(binary_file, "wb") as cubin: - cubin.write(kernel.asm["cubin"]) - target_symbol_name = f"{kernel_name}_cubin" - - # We return extern declarations for both the array and its pointer. - # The pointer is used by gen_loader() to generate R_X86_64_64 relocations - # instead of R_X86_64_32, which allows the .triton section to be placed - # beyond the 4GB address limit in large binaries. - # Note: The pointer is volatile to prevent optimizer constant-propagation. - return f'extern "C" {{ extern unsigned char {target_symbol_name}[]; extern const void* volatile {target_symbol_name}_ptr; }}' - - -def gen_loader(kernel_name: str, cubin_name: str, shared: int) -> str: - # TODO(changpan): Extract inline cuModuleLoadData/cuModuleGetFunction error - # handling into a shared helper to reduce generated code size. - return textwrap.dedent( - f""" - CUfunction load_{kernel_name}(void) - {{ - thread_local std::unordered_map cache; - auto idx = torch::stable::accelerator::getCurrentDeviceIndex(); - auto res = cache.find(idx); - if (res != cache.end()) {{ - return res->second; - }} - CUfunction func; - CUmodule mod_ptr; - CUresult err; - // Use pointer to cubin data to generate R_X86_64_64 relocation - // instead of R_X86_64_32, allowing cubin data to be placed beyond 4GB - const void *image = {kernel_name}_cubin_ptr; - - err = cuModuleLoadData(&mod_ptr, image); - if (err != 0) {{ - const char* errStr; - cuGetErrorString(err, &errStr); - throw std::runtime_error("cuModuleLoadData failed for {kernel_name}: error " + std::to_string(err) + " (" + (errStr ? errStr : "unknown") + ")"); - }} - - err = cuModuleGetFunction(&func, mod_ptr, "{cubin_name}"); - if (err != 0) {{ - const char* errStr; - cuGetErrorString(err, &errStr); - throw std::runtime_error("cuModuleGetFunction failed for {kernel_name}: error " + std::to_string(err) + " (" + (errStr ? errStr : "unknown") + ")"); - }} - - check_errors({shared}, func); - cache.emplace(idx, func); - return func; - }} - """ - ) - - -# --------------------------------------------------------------------------- -# Launcher codegen (per-spec) -# --------------------------------------------------------------------------- - - -def gen_launcher_params( - descriptors: list[ArgDescriptor], - signature: dict[int, str], -) -> str: - args = ["gridDims grid"] - for d in descriptors: - if d.index in signature: - if isinstance(d, PointerArg): - ctype = "void*" - else: - ctype = CTYPES[signature[d.index]] - args.append(f"{ctype} {d.name}") - return ", ".join(args) - - -def gen_launch_args( - func: JITFunction[list[Any]], - spec: KernelSpec, -) -> list[str]: - """Generate kernel launch argument list (pointers to non-constant arguments).""" - args = [] - for i, arg in enumerate(func.arg_names): - if i in spec.constants: - continue - assert i in spec.signature, f"Argument {i} ({arg}) does not appear in signature" - args.append(f"&{arg}") - return args - - -def gen_launcher( - kernel_name: str, - func: JITFunction[list[Any]], - kernel: Any, - shared: int, - warp_size: int, - spec: KernelSpec, - descriptors: list[ArgDescriptor], -) -> str: - params = gen_launcher_params(descriptors, spec.signature) - args = gen_launch_args(func, spec) - - scratch_declarations, scratch_args = get_scratch_parameters(kernel) - args.extend(scratch_args) - - args_str = ", ".join(args) - - return textwrap.dedent( - f""" - void {kernel_name}({params}) {{ - CUfunction func = load_{kernel_name}(); - cudaStream_t stream = grid.stream ? grid.stream : triton_aot_get_current_stream(); - {scratch_declarations} - void *args[] = {{ {args_str} }}; - auto res = cuLaunchKernel(func, grid.x, grid.y, grid.z, {warp_size} * {spec.num_warps}, 1, 1, {shared}, stream, args, NULL); - TRITON_AOT_CU_CHECK(res); - }} - """ - ) - - -# --------------------------------------------------------------------------- -# Selector codegen (invariant) -# --------------------------------------------------------------------------- - - -def gen_selector_params( - descriptors: list[ArgDescriptor], -) -> str: - """Generate C++ selector function parameter list.""" - args = ["gridDims grid"] - for d in descriptors: - if isinstance(d, PointerArg): - args.append(f"const std::optional& {d.name}") - elif isinstance(d, ScalarArg): - args.append(f"{CTYPES[d.triton_dtype]} {d.name}") - elif isinstance(d, ConstantArg): - args.append(f"{CONSTANT_SELECTOR_CTYPE[d.python_type]} {d.name}") - - for name, value in AUTOTUNE_ATTRs.items(): - args.append(f"{type(value).__name__} {name}") - return ", ".join(args) - - -def gen_launcher_call_args( - descriptors: list[ArgDescriptor], - signature: dict[int, str], -) -> str: - args = ["grid"] - for d in descriptors: - if d.index in signature: - if isinstance(d, PointerArg): - args.append(f"{d.name}.value().data_ptr()") - elif isinstance(d, ScalarArg) and signature[d.index] != d.triton_dtype: - args.append(f"static_cast<{CTYPES[signature[d.index]]}>({d.name})") - else: - args.append(d.name) - return ", ".join(args) - - -def gen_guarded_calls( # noqa: C901 - func: JITFunction[list[Any]], - unit: OpsUnit, - descriptors: list[ArgDescriptor], -) -> str: - desc_by_idx: dict[int, ArgDescriptor] = {d.index: d for d in descriptors} - calls = [] - for spec in unit.specs: - kernel_name = gen_kernel_name(func, spec, unit.cc) - args = gen_launcher_call_args(descriptors, spec.signature) - guards = "" - - # Guard on tensor dtypes (per-spec: different specs may have different dtypes) - for i, ttype in spec.signature.items(): - d = desc_by_idx[i] - if not isinstance(d, PointerArg): - continue - arg = d.name - atype = SCALAR_TYPES[ttype] - guards += f"if ({arg}.has_value()) " - guards += f"if ({arg}.value().scalar_type() == {atype}) " - - # Guard on int range (spec uses narrower type than selector) - for i, dtype in spec.signature.items(): - d = desc_by_idx[i] - if isinstance(d, ScalarArg) and dtype != d.triton_dtype: - if dtype == "i32": - guards += f"if (fits_i32({d.name})) " - - # Guard on constant values. - for i, val in spec.constants.items(): - arg = desc_by_idx[i].name - if isinstance(val, bool): - guards += f"if ({arg}) " if val else f"if (!({arg})) " - elif isinstance(val, str): - guards += f'if ({arg} == "{val}") ' - elif val is None: - guards += f"if (!{arg}.has_value()) " - else: - guards += f"if ({arg} == {val}) " - - # Guard on special constants - for name in AUTOTUNE_ATTRs.keys(): - guards += f"if ({name} == {getattr(spec, name)}) " - - # Guard on divisible_by_16 - for i in spec.divisible_by_16: - arg = desc_by_idx[i].name - if i in spec.signature: - ttype = spec.signature[i] - if ttype.startswith("*"): - guards += f"if ((((uintptr_t){arg}.value().data_ptr()) % 16) == 0) " - else: - guards += f"if (({arg} % 16) == 0) " - elif i in spec.constants: - assert (spec.constants[i] % 16) == 0 - - # Guard on divisible_by_8 - for i in spec.divisible_by_8: - arg = desc_by_idx[i].name - if i in spec.signature: - ttype = spec.signature[i] - # divisible_by_8 is only applied to int - if not ttype.startswith("*"): - guards += f"if (({arg} % 8) == 0) " - elif i in spec.constants: - assert (spec.constants[i] % 8) == 0 - - # Call the specialization. - calls.append(f"{guards}return {kernel_name}({args});\n") - return "".join(calls) - - -def gen_selector_proto( - descriptors: list[ArgDescriptor], - func_name: str, -) -> str: - params = gen_selector_params(descriptors) - # Add Triton's default values for num warps/stages, etc - for name, value in AUTOTUNE_ATTRs.items(): - params = params.replace(name, f"{name}={value}") - return f"void {func_name}({params});" - - -def gen_failure_msg( - descriptors: list[ArgDescriptor], -) -> str: - """Generate C++ ``<<``-chain for the dispatch-failure error message. - - Groups parameters by category (Tensors / Scalars / Constants / - Autotune / Device). Tensor entries include aligned16 status. - """ - tensors: list[str] = [] - scalars: list[str] = [] - constants: list[str] = [] - - for d in descriptors: - if isinstance(d, PointerArg): - dtype_expr = ( - f"({d.name}.has_value()" - f" ? c10::toString({d.name}.value().scalar_type())" - f' : "nullptr")' - ) - align_expr = ( - f"(({d.name}.has_value()" - f" && (((uintptr_t){d.name}.value().data_ptr()) % 16) == 0)" - f' ? "true" : "false")' - ) - tensors.append( - f'" {d.name}=" << {dtype_expr} << "(aligned16=" << {align_expr} << ")"' - ) - elif isinstance(d, ScalarArg): - scalars.append(f'" {d.name}=" << {d.name}') - elif isinstance(d, ConstantArg): - constants.append(f'" {d.name}=" << {d.name}') - - autotune: list[str] = [f'" {n}=" << {n}' for n in AUTOTUNE_ATTRs] - - sections: list[str] = [] - if tensors: - sections.append('"\\n Tensors:" << ' + " << ".join(tensors)) - if scalars: - sections.append('"\\n Scalars:" << ' + " << ".join(scalars)) - if constants: - sections.append('"\\n Constants:" << ' + " << ".join(constants)) - sections.append('"\\n Autotune:" << ' + " << ".join(autotune)) - sections.append('"\\n Device: cc=" << cc') - - return " << ".join(sections) - - -def gen_selector( - func: JITFunction[list[Any]], - unit: OpsUnit, - descriptors: list[ArgDescriptor], -) -> str: - params = gen_selector_params(descriptors) - guarded_calls = gen_guarded_calls(func, unit, descriptors) - failure_msg = gen_failure_msg(descriptors) - return f""" - void {func.__name__}({params}) {{ - auto cc = compute_capability(); - if (grid.x * grid.y * grid.z > 0) {{ - {guarded_calls} - std::stringstream ss; - ss << "[TritonAOT] No implementation found for {func.__name__}" << {failure_msg}; - throw std::runtime_error(ss.str()); - }} - }} - """ - - -# --------------------------------------------------------------------------- -# Torch op codegen (invariant) -# --------------------------------------------------------------------------- - - -def gen_cpp_op_params( - descriptors: list[ArgDescriptor], -) -> str: - args = [] - for d in descriptors: - if isinstance(d, PointerArg): - args.append(f"std::optional {d.name}") - elif isinstance(d, ScalarArg): - args.append(f"{scalar_cpp_op_ctype(d.triton_dtype)} {d.name}") - elif isinstance(d, ConstantArg): - args.append(f"{CONSTANT_CPP_OP_CTYPE[d.python_type]} {d.name}") - for name, value in AUTOTUNE_ATTRs.items(): - args.append(f"{PY_TYPES_TO_CPP_TYPES[type(value)]} {name}") - return ", ".join(args) - - -def gen_torch_op_params( - descriptors: list[ArgDescriptor], - default_values: dict[str, Any], -) -> str: - args = [] - - def gen_str_wrap(value: Any) -> Any: - return f'\\"{value}\\"' if isinstance(value, str) else value - - def gen_default_str(arg: str) -> str: - return ( - f" = {gen_str_wrap(default_values[arg])}" if arg in default_values else "" - ) - - for d in descriptors: - df_str = gen_default_str(d.name) - if isinstance(d, PointerArg): - t = chr(ord("a") + d.index) - args.append(f"Tensor({t}!)? {d.name}") - elif isinstance(d, ScalarArg): - args.append(f"{scalar_torch_schema(d.triton_dtype)} {d.name}{df_str}") - elif isinstance(d, ConstantArg): - args.append(f"{CONSTANT_TORCH_SCHEMA[d.python_type]} {d.name}{df_str}") - for name, value in AUTOTUNE_ATTRs.items(): - args.append(f"{type(value).__name__} {name}={value}") - return ", ".join(args) - - -def gen_torch_op( - func: JITFunction[list[Any]], - descriptors: list[ArgDescriptor], - default_values: dict[str, Any], -) -> str: - cpp_params = gen_cpp_op_params(descriptors) - torch_params = gen_torch_op_params(descriptors, default_values) - arg_names = list(func.arg_names) + list(AUTOTUNE_ATTRs.keys()) - args = ", ".join(arg_names) - - # Generate a comment noting which tensor params are non-optional but - # promoted to Tensor? for TorchScript compatibility. - promoted = [ - d.name for d in descriptors if isinstance(d, PointerArg) and not d.is_optional - ] - type_comment = "" - if promoted: - type_comment = ( - f"// Note: {', '.join(promoted)} are non-optional but use Tensor? " - "for TorchScript compatibility.\n" - "// Dispatch uses HAS_XXX constexpr ints, not tensor presence.\n" - ) - return textwrap.dedent( - f""" - namespace {{ - triton::aot::gridDims dims_from_vec( - const std::vector& grid - ) {{ - return triton::aot::gridDims( - grid.size() > 0 ? grid[0] : 1, - grid.size() > 1 ? grid[1] : 1, - grid.size() > 2 ? grid[2] : 1 - ); - }} - - {type_comment}void {func.__name__}_op( - std::vector grid, - {cpp_params} - ) {{ - triton::aot::{func.__name__}( - dims_from_vec(grid), - {args} - ); - }} - - void {func.__name__}_dummy_op( - std::vector grid, - {cpp_params} - ) {{ - // Do nothing. The op is a dummy for model transform, - // processing, and splitting services. - }} - }} - - STABLE_TORCH_LIBRARY_FRAGMENT(triton_aot, m) {{ - m.def("{func.__name__}(int[] grid, {torch_params}) -> ()"); - }} - STABLE_TORCH_LIBRARY_IMPL(triton_aot, CUDA, m) {{ - m.impl("{func.__name__}", TORCH_BOX(&{func.__name__}_op)); - }} - - STABLE_TORCH_LIBRARY_IMPL(triton_aot, CPU, m) {{ - m.impl("{func.__name__}", TORCH_BOX(&{func.__name__}_dummy_op)); - }} - - STABLE_TORCH_LIBRARY_IMPL(triton_aot, Meta, m) {{ - m.impl("{func.__name__}", TORCH_BOX(&{func.__name__}_dummy_op)); - }} - """ - ) - - -# --------------------------------------------------------------------------- -# Tuner meta codegen -# --------------------------------------------------------------------------- - - -def key_names_and_idx(func: Any) -> tuple[list[str], list[int]]: - if hasattr(func, "key_idx"): - arg_names = [func.arg_names[idx] for idx in func.key_idx] - key_idx = func.key_idx - else: - arg_names = func.keys - key_idx = [func.arg_names.index(arg) for arg in arg_names] - return arg_names, key_idx - - -def is_non_empty_mapping_of_type(obj: object, value_type: type[Any]) -> bool: - """Check if object is a non-empty dict with all values of specific type""" - if not obj or not isinstance(obj, dict): - return False - - return all(isinstance(value, value_type) for value in obj.values()) - - -_LAUNCH_PARAM_NAMES: list[str] = ["num_warps", "num_stages"] - - -def gen_tuner_meta_py( - func: Any, - tuner_fallback: bool, - unit: OpsUnit, -) -> str: - vals = [] - - guard_list = [] - - # Use custom meta generation function if available - if hasattr(func, "gen_autotune_select_meta_src"): - return func.gen_autotune_select_meta_src(unit.constant_types) - - if hasattr(func, "cache") and is_non_empty_mapping_of_type( - func.cache, triton.runtime.autotuner.Config - ): - # auto tuned configs - arg_names, key_idx = key_names_and_idx(func) - - in_args = ", ".join( - [ - f"{name}: {unit.constant_types[idx].__name__ if idx in unit.constant_types else 'int'}" - for idx, name in zip(key_idx, arg_names) - ] - ) - - cfg_first = next(iter(func.cache.values())) - return_names = list(cfg_first.kwargs.keys()) + _LAUNCH_PARAM_NAMES - - for key, cfg in func.cache.items(): - val = list(cfg.kwargs.values()) + [cfg.num_warps, cfg.num_stages] - val = tuple(val) - vals.append(val) - equations = [] - for arg, value in zip(arg_names, key): - if isinstance(value, str): - equations.append(f"{arg} == '{value}'") - elif isinstance(value, bool): - equations.append(f"{arg} == {int(value)}") - else: - equations.append(f"{arg} == {value}") - guard_list.append(f"if {' and '.join(equations)}: return {val}") - - else: - # default configs — single spec, use specs[0] - in_args = "" - arg_names = list(_LAUNCH_PARAM_NAMES) - return_names = list(_LAUNCH_PARAM_NAMES) - val = unit.specs[0].num_warps, unit.specs[0].num_stages - vals.append(val) - - name = unwrap_heuristic(func, JITFunction).__name__ - meta = name + "_meta" - - guards = "\n ".join(guard_list) - - fmt_args = ", ".join([f"{{{arg_name}}}" for arg_name in arg_names]) - - raise_runtime_error_str = ( - f"""raise RuntimeError(f"No autotuning config found for {name}({fmt_args})")""" - ) - fallback_str = f"""return {Counter(vals).most_common(1)[0][0]}""" - - returns_comment = f"# Returns: ({', '.join(return_names)})" - - return textwrap.dedent( - f""" - def {meta}({in_args}): - {returns_comment} - {guards} - {fallback_str if tuner_fallback else raise_runtime_error_str} - """ - ) - - -def gen_tuner_meta_cpp( - func: Any, - tuner_fallback: bool, - constant_types: dict[int, type[Any]], -) -> str: - # TODO(changpan): This C++ inline _meta is currently dead code — no C++ caller - # invokes it. The Python _meta.py (gen_tuner_meta_py) is the only consumer. - # Double check, try remove this and the TUNER_META_CPP template region. - def infer_arg_type(idx: int) -> str: - if idx in constant_types: - return PY_TYPES_TO_CPP_TYPES[constant_types[idx]] - else: - return "int64_t" - - arg_names, key_idx = key_names_and_idx(func) - - in_args = ", ".join( - [f"{infer_arg_type(idx)} {name}" for idx, name in zip(key_idx, arg_names)] - ) - - vals = [] - guard_list = [] - for key, cfg in func.cache.items(): - val = list(cfg.kwargs.values()) + [cfg.num_warps, cfg.num_stages] - val = tuple(val) - vals.append(val) - equations = [] - for arg, value in zip(arg_names, key): - if isinstance(value, str): - equations.append(f'{arg} == "{value}"') - elif isinstance(value, bool): - equations.append(f"{arg} == {int(value)}") - else: - equations.append(f"{arg} == {value}") - guard_list.append(f"if ({' && '.join(equations)}) return std::make_tuple{val};") - guards = "\n ".join(guard_list) - name = unwrap_heuristic(func, JITFunction).__name__ - meta = name + "_meta" - fmt_args = ", ".join([f"{arg_name}" for arg_name in arg_names]) - raise_runtime_error_str = f"""throw std::runtime_error("No autotuning config found for {name}({fmt_args})");""" - fallback_str = f"""return std::make_tuple{Counter(vals).most_common(1)[0][0]};""" - # Infer the return type from the actual values - return_type = _infer_return_type(vals[0]) - return textwrap.dedent( - f""" - inline std::tuple<{return_type}> {meta}({in_args}) {{ - {guards} - {fallback_str if tuner_fallback else raise_runtime_error_str} - }} - """ - ) - - -def _infer_return_type(vals: tuple[Any, ...]) -> str: - types = [PY_TYPES_TO_CPP_TYPES.get(type(val)) for val in vals] - try: - # pyre-fixme[6]: For 1st argument expected - # `Iterable[typing_extensions.LiteralString]` but got `List[Optional[str]]`. - return ", ".join(types) - except TypeError: # one of the types cannot be inferred, e.g. `None` - raise ValueError("Cannot infer return type from `vals`") - - -# --------------------------------------------------------------------------- -# Top-level codegen entry points -# --------------------------------------------------------------------------- - - -def generate_header_content( - tuned_func: triton.runtime.autotuner.Autotuner | None, - func: JITFunction[list[Any]], - unit: OpsUnit, - descriptors: list[ArgDescriptor], - tuner_fallback: bool, -) -> str: - """Generate the content of the .h header file.""" - h_template = load_template("kernel.h") - tuner_meta_cpp = ( - gen_tuner_meta_cpp(tuned_func, tuner_fallback, unit.constant_types) - if tuned_func - else "" - ) - selector_proto = gen_selector_proto(descriptors, func.__name__) - return render_template( - h_template, - { - "TUNER_META_CPP": tuner_meta_cpp, - "SELECTOR_PROTO": selector_proto, - }, - ) - - -def generate_kernel_cpp_content( - func: JITFunction[list[Any]], - unit: OpsUnit, - descriptors: list[ArgDescriptor], - prefix: str, - generated_specs: list[str], - backend: str, -) -> str: - """Generate the content of the kernel .cpp file. - - All tensor params use Tensor? for TorchScript compatibility. - Dispatch relies on HAS_XXX constexpr ints, not tensor presence. - """ - cpp_template = load_template("kernel.cpp") - kernel_specs = "\n".join(generated_specs) - selector = gen_selector(func, unit, descriptors) - - # On AMD, apply hipification to generated code (KERNEL_SPECS, SELECTOR) - # Templates are already hipified at load time (from hip/ subdirectory) - if backend == "hip": - from torch._inductor.codegen.aoti_hipify_utils import maybe_hipify_code_wrapper - - kernel_specs = maybe_hipify_code_wrapper(kernel_specs, force_hipify=True) - selector = maybe_hipify_code_wrapper(selector, force_hipify=True) - - cpp_content = render_template( - cpp_template, - { - "HEADER_INCLUDE": f'#include "{prefix}.h"\n', - "KERNEL_SPECS": kernel_specs, - "SELECTOR": selector, - }, - ) - return cpp_content - - -def generate_torch_op_content( - func: JITFunction[list[Any]], - descriptors: list[ArgDescriptor], - prefix: str, - default_values: dict[str, Any], -) -> str: - """Generate the content of the torch_op .cpp file.""" - torch_template = load_template("torch_op.cpp") - torch_op_content = gen_torch_op(func, descriptors, default_values) - torch_content = render_template( - torch_template, - { - "HEADER_INCLUDE": f'#include "{prefix}.h"\n', - "TORCH_OP": torch_op_content, - }, - ) - return torch_content diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/compile/compile_state.py b/recommendation_v4/generative_recommenders/ops/triton_aot/compile/compile_state.py deleted file mode 100644 index 231cb52f1..000000000 --- a/recommendation_v4/generative_recommenders/ops/triton_aot/compile/compile_state.py +++ /dev/null @@ -1,409 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# pyre-strict - -#!/usr/bin/env python3 - -from __future__ import annotations - -import hashlib -import json -import os -import tempfile -from inspect import getcallargs, Parameter, signature -from typing import Any, Callable, Dict, List, Optional, Set - -import torch - -# @manual=//triton:triton -import triton.language as tl -from generative_recommenders.ops.triton_aot.compile.stable_types import SCALAR_TYPES -from generative_recommenders.ops.triton_aot.compile.utils import is_autotuner -from generative_recommenders.ops.triton_aot.types import ( - Annotation, - AnnotationHint, - TritonAOT, -) - -# @manual=//triton:triton -from triton.runtime.jit import KernelInterface, mangle_type - - -class CustomEncoder(json.JSONEncoder): - # pyre-ignore[14]: Inconsistent override - def default(self, obj: object) -> Any: - if isinstance(obj, set): - return {"__set__": True, "items": sorted(obj)} - # Handle other non-serializable types - return super().default(obj) - - -def hash_spec(spec: Dict[str, Any]) -> str: - serialized_dict = json.dumps(spec, cls=CustomEncoder, sort_keys=True) - return hashlib.sha256(serialized_dict.encode("utf-8")).hexdigest() - - -class AOTTCompileState: - """ - Singleton state container for Triton AOT compilation. - - Description: - This singleton pattern enables state sharing between code loaded via - torch.package (which creates isolated module namespaces) and the regular - Python import system. Without this pattern, the packaged module would have - its own copy of global state, leading to inconsistencies. - - Usage: - # Normal usage - get the singleton instance - state = AOTTCompileState.get_instance() - - # For torch.package integration - inject shared instance into packaged module - packaged_module = package_importer.import_module("triton_aot.compile.compile_state") - packaged_module.AOTTCompileState.set_instance(AOTTCompileState.get_instance()) - """ - - _instance: Optional["AOTTCompileState"] = None - - kernel_specs: Dict[KernelInterface[List[Any]], List[Dict[str, List[Any]]]] = {} - specs_hashset: Dict[KernelInterface[List[Any]], Set[str]] = {} - enable_aott_compile: bool = False - compile_base_dir: str = "" - compile_path: str = "" - - def __new__(cls) -> "AOTTCompileState": - if cls._instance is None: - instance = super().__new__(cls) - instance._initialize() - cls._instance = instance - return cls._instance - - def _initialize(self) -> None: - """Initialize the singleton state. Called only once.""" - self.kernel_specs: Dict[ - KernelInterface[List[Any]], List[Dict[str, List[Any]]] - ] = {} - self.specs_hashset: Dict[KernelInterface[List[Any]], Set[str]] = {} - self.enable_aott_compile: bool = False - self.compile_base_dir: str = os.getenv("TRITON_AOT_PATH_PREFIX", "/var/tmp") - self.compile_path: str = tempfile.mkdtemp( - dir=self.compile_base_dir, prefix="triton_aot_compile_" - ) - - @classmethod - def get_instance(cls) -> "AOTTCompileState": - """Get the singleton instance, creating it if necessary.""" - if cls._instance is None: - cls._instance = cls() - return cls._instance - - @classmethod - def set_instance(cls, instance: "AOTTCompileState") -> None: - """ - Set the singleton instance. Used for torch.package integration. - - When code is loaded via torch.package, it creates a separate module - namespace with its own class objects. This method allows injecting - a shared instance from the main module into the packaged module. - """ - cls._instance = instance - - def reset(self) -> None: - """Reset all state to initial values.""" - self.kernel_specs = {} - self.specs_hashset = {} - self.disable() - self.compile_base_dir = os.getenv("TRITON_AOT_PATH_PREFIX", "/var/tmp") - self.compile_path = tempfile.mkdtemp( - dir=self.compile_base_dir, prefix="triton_aot_compile_" - ) - - def add_kernel_spec( - self, - fn: KernelInterface[List[Any]], - spec: Dict[str, List[Any]], - hashed_spec: str, - ) -> None: - """Add a kernel spec if not already present (based on hash). - If the same Triton kernel is used at multiple locations in a model: - - All calls share one spec list under the same kernel function key - - Specs with identical signatures (same dtypes, shapes) are deduplicated via hash - - Specs with different signatures (e.g., fp32 vs bf16) are recorded separately - - Example: - # Two call sites using the same kernel: - my_kernel[grid](tensor_fp32, ...) # Records spec with "*fp32" - my_kernel[grid](tensor_bf16, ...) # Records spec with "*bf16" - my_kernel[grid](tensor_fp32, ...) # Deduplicated, same hash as first call - - # Result: kernel_specs[my_kernel] = [fp32_spec, bf16_spec] - """ - if fn not in self.kernel_specs: - self.kernel_specs[fn] = [] - self.specs_hashset[fn] = set() - if hashed_spec not in self.specs_hashset[fn]: - self.kernel_specs[fn].append(spec) - self.specs_hashset[fn].add(hashed_spec) - - def _collect_spec( - self, - fn: KernelInterface[List[Any]], - annotations: Dict[str, Annotation], - *args: Any, - **kwargs: Any, - ) -> None: - """Spec collection callback registered on TritonAOT during compile. - - Always collects the annotated spec (which equals the inferred spec - when no annotations are present). Also collects the inferred spec - when it differs and either: - - annotations conflict with sample (fallback for safety), or - - inferred has perf hints the annotation lacks (perf variant). - """ - spec = infer_spec(fn, annotations, *args, **kwargs) - annotated_hash = hash_spec(spec) - self.add_kernel_spec(fn, spec, annotated_hash) - - if annotations: - inferred = infer_spec(fn, {}, *args, **kwargs) - inferred_hash = hash_spec(inferred) - if inferred_hash == annotated_hash: - return - if _annotation_conflicts_with_sample( - fn, annotations, *args, **kwargs - ) or _inferred_has_perf_advantage(spec, inferred): - self.add_kernel_spec(fn, inferred, inferred_hash) - - def enable(self) -> None: - """Enable AOT compile and register the spec collection hook.""" - self.enable_aott_compile = True - TritonAOT.set_spec_collector(self._collect_spec) - - def disable(self) -> None: - """Disable AOT compile and unregister the spec collection hook.""" - self.enable_aott_compile = False - TritonAOT.set_spec_collector(None) - - -def get_aott_compile_state() -> AOTTCompileState: - """Get the current AOTTCompileState singleton. - - Uses get_instance() so injected instances (via set_instance() for - torch.package integration) are respected. - """ - return AOTTCompileState.get_instance() - - -######## -# Module-level global accessors that delegate to singleton -######## - - -def get_triton_aot_kernel_specs() -> Dict[ - KernelInterface[List[Any]], List[Dict[str, List[Any]]] -]: - return get_aott_compile_state().kernel_specs - - -def get_triton_aot_specs_hashset() -> Dict[KernelInterface[List[Any]], Set[str]]: - return get_aott_compile_state().specs_hashset - - -def get_aott_compile_path() -> str: - return get_aott_compile_state().compile_path - - -def add_kernel_spec( - fn: KernelInterface[List[Any]], spec: Dict[str, List[Any]], hashed_spec: str -) -> None: - get_aott_compile_state().add_kernel_spec(fn, spec, hashed_spec) - - -def _unwrap_triton_fn( - fn: KernelInterface[List[Any]], -) -> Callable[..., Any]: - while isinstance(fn, KernelInterface): - # pyre-ignore[16]: KernelInterface has `fn` attribute at runtime - fn = fn.fn - return fn - - -def _inferred_has_perf_advantage( - annotated_spec: Dict[str, List[Any]], - inferred_spec: Dict[str, List[Any]], -) -> bool: - """True if inferred spec has alignment/divisibility hints the annotated lacks. - - A tuple element ``(type, N)`` carries alignment or divisibility info - that a bare string does not. When inference adds such hints (e.g., - tensor alignment from ``data_ptr() % 16 == 0``), the inferred spec - produces a more optimized cubin worth keeping as a perf variant. - """ - for ann_elem, inf_elem in zip( - annotated_spec["signature"], inferred_spec["signature"] - ): - if isinstance(inf_elem, tuple) and not isinstance(ann_elem, tuple): - return True - return False - - -# Triton-internal kwargs injected by KernelInterface.__getitem__ -# (triton/runtime/jit.py). These are not kernel parameters and must -# be stripped before getcallargs. -_TRITON_INTERNAL_KWARGS: frozenset[str] = frozenset({"warmup", "grid"}) - - -def _resolve_call_args( - fn: KernelInterface[List[Any]], - *args: Any, - **kwargs: Any, -) -> tuple[Callable[..., Any], dict[str, Any]]: - """Unwrap kernel and resolve call args with autotune placeholder fill.""" - triton_fn = _unwrap_triton_fn(fn) - # Filter Triton-internal kwargs injected by KernelInterface.__getitem__ - # (triton/runtime/jit.py) — not part of the kernel signature. - clean_kwargs = {k: v for k, v in kwargs.items() if k not in _TRITON_INTERNAL_KWARGS} - if is_autotuner(fn): - # pyre-ignore[16]: Attributes checked by is_autotuner - for arg_name in fn.configs[0].kwargs.keys(): - if arg_name not in clean_kwargs: - clean_kwargs[arg_name] = -1 - return triton_fn, getcallargs(triton_fn, *args, **clean_kwargs) - - -_I32_MIN: int = -(2**31) -_I32_MAX: int = 2**31 - 1 - - -def _sample_satisfies_int_type(sample: int, ann_type: str) -> bool: - """True if sample int fits the annotated type range.""" - if ann_type == "i32": - return _I32_MIN <= sample <= _I32_MAX - return True - - -def _sample_satisfies_annotation(sample: Any, ann: Annotation) -> bool: - """True if a single sample value satisfies its annotation constraint.""" - if isinstance(ann, AnnotationHint): - if isinstance(sample, torch.Tensor): - return sample.data_ptr() % ann.hint == 0 - if isinstance(sample, int): - if ann.hint == 1: - return sample == 1 - if not _sample_satisfies_int_type(sample, ann.dtype): - return False - if ann.hint > 1: - return sample % ann.hint == 0 - return True - if isinstance(ann, str) and not ann.startswith("*") and isinstance(sample, int): - return _sample_satisfies_int_type(sample, ann) - return True - - -def _annotation_conflicts_with_sample( - fn: KernelInterface[List[Any]], - annotations: Dict[str, Annotation], - *args: Any, - **kwargs: Any, -) -> bool: - """True if any annotated param's sample value doesn't satisfy the annotation. - - Used by ``_collect_spec`` to decide whether to generate an inferred - fallback spec. When the sample satisfies all annotations, only the - annotated spec is needed (the user's constraints hold for this input). - """ - _, sample_args = _resolve_call_args(fn, *args, **kwargs) - - for param_name, ann in annotations.items(): - sample = sample_args.get(param_name) - if sample is None: - continue - if not _sample_satisfies_annotation(sample, ann): - return True - - return False - - -def _infer_spec_entry( - arg_name: str, - arg: Any, - arg_annotation: Any, - annotations: Dict[str, Annotation], -) -> Any: - if arg_annotation != Parameter.empty: - if arg_annotation == tl.constexpr: - return arg - raise RuntimeError( - f"TritonAOT: unsupported scalar annotation {arg_annotation}." - ) - - if arg_name in annotations: - ann = annotations[arg_name] - # Convert to tuple for raw spec format (shared/spec_conversion - # processes plain tuples). - return ann.to_tuple() if isinstance(ann, AnnotationHint) else ann - - if arg is None: - return None - - if isinstance(arg, torch.Tensor): - # Reject dtypes SCALAR_TYPES can't render (e.g. *u1, *u16, *fp8e5) - # so codegen doesn't KeyError downstream. - type_str = mangle_type(arg) - if type_str not in SCALAR_TYPES: - raise RuntimeError( - f"TritonAOT: unsupported tensor type for {arg_name}: " - f"{arg.dtype} (Triton mangled to {type_str!r}). " - f"Supported tensor dtypes: {sorted(SCALAR_TYPES.keys())}." - ) - return (type_str, 16) if arg.data_ptr() % 16 == 0 else type_str - - if isinstance(arg, bool): - # bool is subclass of int; must check before int. - # Non-constexpr bools have no CTYPES entry for codegen. - raise RuntimeError( - f"TritonAOT: parameter {arg_name} is a bool without " - f"tl.constexpr annotation. Add `{arg_name}: tl.constexpr` " - f"to the kernel signature." - ) - - if isinstance(arg, int): - # Always i64 for safety; users annotate "i32" for narrower variant via - # annotation-as-variant. - if not -(2**63) <= arg <= 2**63 - 1: - raise RuntimeError( - f"TritonAOT: unsupported int value for {arg_name}: " - f"value exceeds i64 range. Use a smaller value or tl.constexpr." - ) - return "i64" - - if isinstance(arg, float): - return "fp32" - - raise RuntimeError(f"TritonAOT: parameter {arg_name} needs annotation.") - - -def infer_spec( - fn: KernelInterface[List[Any]], - annotations: Dict[str, Annotation], - *args: Any, - **kwargs: Any, -) -> Dict[str, List[Any]]: - """Infer kernel spec from sample args. - - Tensor dtype: ``mangle_type``, alignment: ``data_ptr() % 16``. - Scalar int: always ``"i64"`` (safe default; user can annotate ``"i32"`` - to get a narrower variant via annotation-as-variant). - Float: ``mangle_type`` → fp32. - """ - triton_fn, call_args = _resolve_call_args(fn, *args, **kwargs) - fn_sig = signature(triton_fn) - arg_annotations = { - name: param.annotation for name, param in fn_sig.parameters.items() - } - spec = [] - - for arg_name in fn_sig.parameters.keys(): - arg = call_args[arg_name] - spec.append( - _infer_spec_entry(arg_name, arg, arg_annotations[arg_name], annotations) - ) - return {"signature": spec} diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/compile/pipeline.py b/recommendation_v4/generative_recommenders/ops/triton_aot/compile/pipeline.py deleted file mode 100644 index cdf4d1821..000000000 --- a/recommendation_v4/generative_recommenders/ops/triton_aot/compile/pipeline.py +++ /dev/null @@ -1,300 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. - -# pyre-strict - -"""AOT-T compilation pipeline. - -Orchestrates: spec processing → Triton native compile → C++ / Python codegen. -""" - -from __future__ import annotations - -import logging -import multiprocessing as mp -import os -import signal -import threading -from concurrent.futures import ThreadPoolExecutor -from types import FrameType, ModuleType -from typing import Any, Callable - -# @manual=//triton:triton -import triton -import triton.compiler -from generative_recommenders.ops.triton_aot.compile.arg_descriptor import ( - ArgDescriptor, - build_arg_descriptors, -) -from generative_recommenders.ops.triton_aot.compile.codegen import ( - gen_cubin, - gen_kernel_name, - gen_launcher, - gen_loader, - gen_tuner_meta_py, - generate_header_content, - generate_kernel_cpp_content, - generate_torch_op_content, -) -from generative_recommenders.ops.triton_aot.compile.spec_processing import ( - gen_compile_arg, - KernelSpec, - OpsUnit, - RawKernelSpec, -) -from generative_recommenders.ops.triton_aot.compile.utils import ( - is_autotuner, - unwrap_heuristic, -) -from generative_recommenders.ops.triton_aot.shared.types import AUTOTUNE_ATTRs -from triton.backends.compiler import GPUTarget -from triton.runtime.jit import JITFunction, KernelInterface - -logger: logging.Logger = logging.getLogger(__name__) - - -def compile_specs_parallel( - specs: list[KernelSpec], - install_dir: str, - module: str, - name: str, - gpu_target: GPUTarget, - import_module: Callable[[str], ModuleType], - descriptors: list[ArgDescriptor], -) -> list[str]: - """Compile kernel specs in parallel using multiprocessing. - - When TRITON_AOT_DEBUG=1 is set, compiles sequentially for easier debugging. - - Args: - specs: List of kernel specifications to compile - install_dir: Directory to install generated files - module: The module name of the function - name: The function name - gpu_target: GPU target for compilation - import_module: Function to import modules (e.g., importlib.import_module or PackageImporter.import_module) - - Returns: - List of generated code strings for each spec (cubin, loader, launcher) - """ - - debug = os.environ.get("TRITON_AOT_DEBUG", "0") == "1" - if debug: - outputs = [ - spec_gen( - install_dir, - spec, - module, - name, - gpu_target, - import_module, - descriptors, - ) - for spec in specs - ] - else: - max_workers = mp.cpu_count() // 2 + 1 - with ThreadPoolExecutor(max_workers=min(len(specs), max_workers)) as executor: - outputs = list( - executor.map( - lambda spec: spec_gen( - install_dir, - spec, - module, - name, - gpu_target, - import_module, - descriptors, - ), - specs, - ) - ) - return outputs - - -# For each spec, generate a kernel: -# - cubin -# - loader -# - launcher -def spec_gen( - install_dir: str, - spec: KernelSpec, - module: str, - name: str, - gpu_target: GPUTarget, - import_module: Callable[[str], ModuleType], - descriptors: list[ArgDescriptor], -) -> str: - # To run this function with multiprocessing, we need to import the function by name, - # since JITFunction cannot be pickled. - # we have the case where the func name is injected with a suffix, like "_cuda" or "_amd", - # we should use the original name to import the func in such case - original_name = name - splits = name.split("_") - end_idx = len(splits) - - while end_idx > 0: - original_name = "_".join(splits[:end_idx]) - if hasattr(import_module(module), original_name): - break - end_idx -= 1 - func = unwrap_heuristic(getattr(import_module(module), original_name), JITFunction) - func.__name__ = name - - # Generate cubin. - kernel_name = gen_kernel_name(func, spec, gpu_target.arch) - - compile_arg = gen_compile_arg(spec, func) - options = {name: getattr(spec, name) for name in AUTOTUNE_ATTRs.keys()} - compile_kwargs = { - "target": gpu_target, - "options": options, - } - kernel = triton.compiler.compile(*compile_arg, **compile_kwargs) - if getattr(kernel.metadata, "global_scratch_size", 0) > 0: - raise RuntimeError(f"{kernel_name=} with global scratch is not supported.") - - metadata_name = kernel.metadata.name - metadata_shared = kernel.metadata.shared - - cubin = gen_cubin(kernel_name, kernel, install_dir, gpu_target.backend) - out = [ - cubin, - # Generate loader. - gen_loader(kernel_name, metadata_name, metadata_shared), - # Generate launcher. - gen_launcher( - kernel_name, - func, - kernel, - metadata_shared, - gpu_target.warp_size, - spec, - descriptors, - ), - ] - return "".join(out) - - -def sigchld_handler(signum: int, frame: FrameType | None) -> None: - sketchy_signals = map(int, [signal.SIGSEGV, signal.SIGABRT, signal.SIGBUS]) - try: - # Consume all pending SIGCHLDs, looking for unexpected failures - while True: - pid, status = os.waitpid(-1, os.WNOHANG) - if pid == 0: - break - if os.WIFSIGNALED(status) and os.WTERMSIG(status) in sketchy_signals: - logger.error( - f"Child process {pid} exited catastrophically with signal {os.WTERMSIG(status)}, terminating!" - ) - - # Avoid triggering atexit etc which can get stuck and behave improperly - # because multiprocessing sets up an atexit handler to join workers - # (sigh). We want to exit, now, so use os._exit instead of sys.exit. - os._exit(1) - except ChildProcessError: - pass - - -def compile_to_cpp( - func: KernelInterface[list[Any]] | triton.runtime.autotuner.Autotuner, - base_specs: list[RawKernelSpec], - install_dir: str, - prefix: str, - *, - gpu_target: GPUTarget, - import_module: Callable[[str], ModuleType], - default_values: dict[str, Any] | None = None, - tuner_fallback: bool = False, -) -> None: - """Compile a Triton kernel into .cpp, .h, _torch_op.cpp, _meta.py files. - - Args: - func: Triton JITFunction or Autotuner to compile. - base_specs: List of kernel specialization specs. - install_dir: Directory to output generated files. - prefix: Kernel name prefix, e.g., "_addmm_fwd". - gpu_target: GPU target for compilation. - import_module: torch.package importer for loading kernels source code. - default_values: Default values for kernel arguments. - tuner_fallback: If True, generate fallback tuner code. - """ - tuned_func = func if is_autotuner(func) else None - # pyre-ignore[6]: Attributes verified by is_autotuner - unit = OpsUnit.from_raw_specs(base_specs, gpu_target, tuned_func) - default_values = {} if default_values is None else default_values - - func_unwrapped = unwrap_heuristic(func, JITFunction) - descriptors = build_arg_descriptors(func_unwrapped, unit) - - # Python's multiprocessing.Pool class is not great at handling unexpected child - # failures such as segfaults. Account for this by temporarily installing a signal - # handler that considers such signals a catastrophic compilation failure. If not - # for this, the Pool will deadlock. - if threading.current_thread() is threading.main_thread(): - previous_child_handler = signal.signal(signal.SIGCHLD, sigchld_handler) - else: - previous_child_handler = None - - func = func_unwrapped - - # sanity check to make sure args with default values are always at the end - has_default_value_arg = False - for name in func.arg_names: - if name in default_values: - has_default_value_arg = True - elif has_default_value_arg: - raise RuntimeError( - f"default values must be at the end of the argument list. {func.arg_names=} {default_values=}" - ) - - h_out = f"{install_dir}/{prefix}.h" - cu_out = f"{install_dir}/{prefix}.cpp" - torch_out = f"{install_dir}/{prefix}_torch_op.cpp" - py_out = f"{install_dir}/{prefix}_meta.py" - - # Generate kernel.h file - h_content = generate_header_content( - tuned_func, # pyre-ignore[6]: Autotuner when set (verified by is_autotuner) - func, - unit, - descriptors, - tuner_fallback, - ) - - with open(h_out, "w") as fp: - fp.write(h_content) - - generated_specs = compile_specs_parallel( - unit.specs, - install_dir, - func.__module__, - func.__name__, - gpu_target, - import_module, - descriptors, - ) - # Generate kernel.cpp file - cu_content = generate_kernel_cpp_content( - func, unit, descriptors, prefix, generated_specs, gpu_target.backend - ) - with open(cu_out, "w") as fp: - fp.write(cu_content) - - # Generate torch_op.cpp file - torch_op_content = generate_torch_op_content( - func, descriptors, prefix, default_values - ) - - with open(torch_out, "w") as fp: - fp.write(torch_op_content) - - if tuned_func: - with open(py_out, "w") as fp: - fp.write(gen_tuner_meta_py(tuned_func, tuner_fallback, unit)) - else: - with open(py_out, "w") as fp: - fp.write(gen_tuner_meta_py(func, tuner_fallback, unit)) - - if previous_child_handler is not None: - signal.signal(signal.SIGCHLD, previous_child_handler) diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/compile/spec_processing.py b/recommendation_v4/generative_recommenders/ops/triton_aot/compile/spec_processing.py deleted file mode 100644 index e8c181121..000000000 --- a/recommendation_v4/generative_recommenders/ops/triton_aot/compile/spec_processing.py +++ /dev/null @@ -1,593 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. - -# pyre-strict - -"""Kernel spec processing for AOT-T compilation. - -Transforms raw kernel specs (from infer_spec) into compiled specs ready -for Triton native compile and C++ codegen. -""" - -from __future__ import annotations - -import copy -import dataclasses -import logging -from dataclasses import dataclass -from typing import Any, cast - -# @manual=//triton:triton -import triton -from generative_recommenders.ops.triton_aot.compile.compile_state import hash_spec -from generative_recommenders.ops.triton_aot.shared.spec_conversion import ( - collect_constraints, - extract_constants, - get_fp8_replacement_signature_for_amd, - get_fp8_replacement_signature_for_sm80, - signature_list_to_dict, - SignatureElement, -) -from generative_recommenders.ops.triton_aot.shared.types import AUTOTUNE_ATTRs -from triton.backends.compiler import BaseBackend, GPUTarget -from triton.compiler.compiler import ASTSource -from triton.runtime.jit import JITFunction - -logger: logging.Logger = logging.getLogger(__name__) - -TRITON_VERSION: str = triton.__version__ - -# A raw kernel spec produced by infer_spec. The only key is "signature". -RawKernelSpec = dict[str, list[SignatureElement]] - - -@dataclass -class KernelSpec: - """A single compilation variant for a kernel. - - Each variant represents one combination of dtypes, constant values, - alignment constraints, and autotune configuration. Multiple variants - are grouped together in an ``OpsUnit``. - - Attributes: - signature: Non-constant arg index → dtype string (e.g., ``{0: "*fp32", 4: "i32"}``). - constants: Arg index → compile-time constant value. Includes bare - literals (128, ``"leaky_relu"``), absent optional tensors (None), - and equal-to-1 specializations (stride=1 → constexpr folding). - divisible_by_16: Indices of args whose values are divisible by 16. - For pointers this means the address is 16-byte aligned; - for scalars it means the value itself is a multiple of 16. - divisible_by_8: Indices of args whose values are divisible by 8. - Only meaningful for scalars (pointer alignment is always ≥16). - num_warps: Number of warps per block. - num_stages: Number of pipeline stages. - matrix_instr_nonkdim: AMD matrix instruction non-K dimension. - waves_per_eu: AMD waves per execution unit. - kpack: AMD kpack factor. - """ - - signature: dict[int, str] - constants: dict[int, Any] - divisible_by_16: set[int] - divisible_by_8: set[int] - num_warps: int = 4 - num_stages: int = 3 - matrix_instr_nonkdim: int = 0 - waves_per_eu: int = 1 - kpack: int = 1 - - -@dataclass -class OpsUnit: - """All compilation variants for a single kernel op. - - Groups per-kernel invariants with the list of ``KernelSpec`` variants. - Use ``OpsUnit.from_raw_specs()`` to build — it performs the complete - spec processing pipeline (convert → detect optional → validate → - autotune → dedup → compute invariants). - - Attributes: - cc: Compute capability (int for NVIDIA, str for AMD). - optional: Indices of optional tensor args (unified across all call sites). - pointer_args: Indices of all tensor pointer args (required + optional). - Invariant across specs — a pointer arg never becomes a non-pointer. - scalar_dtypes: Non-pointer signature arg index → widest dtype string - across all specs (e.g., ``"i32"``, ``"i64"``, ``"fp32"``). - Computed by ``_wider_type`` — individual specs may use narrower types. - constant_types: Python type per constant arg position (e.g., ``{15: int, 19: bool}``). - Excludes optional tensor args (None constants). - Invariant across specs — same Python type for each position. - specs: Per-variant compilation specs. - """ - - cc: int | str - optional: set[int] - pointer_args: set[int] - scalar_dtypes: dict[int, str] - constant_types: dict[int, type[Any]] - specs: list[KernelSpec] - - @classmethod - def from_raw_specs( - cls, - base_specs: list[RawKernelSpec], - gpu_target: GPUTarget, - tuned_func: triton.runtime.autotuner.Autotuner | None = None, - ) -> OpsUnit: - """Build an OpsUnit from raw kernel specs. - - Performs the complete spec processing pipeline: - 1. Convert raw specs to KernelSpecs - 2. Detect optional tensor args (cross-spec + 3-tuple) - 3. Validate consistency across converted specs - 4. Apply autotuning (if tuned_func provided) - 5. Deduplicate specs - 6. Compute shared invariants (pointer_args, scalar_dtypes, constant_types) - """ - # Validate raw specs upfront, before any rewriting. - num_params = _check_uniform_signature_length(base_specs) - specs, three_tuple_optional = _convert_raw_specs(base_specs, gpu_target) - optional = _detect_optional_args(specs) | three_tuple_optional - - _validate_converted_specs(specs, optional, num_params) - - # Plain @triton.jit kernels (no @triton.autotune) skip config expansion. - if tuned_func is not None: - specs = _autotune_specs(tuned_func, gpu_target, specs) - - specs = _dedup_specs(specs) - - pointer_args, scalar_dtypes, constant_types = _compute_invariants( - specs, optional - ) - - return cls( - cc=gpu_target.arch, - optional=optional, - pointer_args=pointer_args, - scalar_dtypes=scalar_dtypes, - constant_types=constant_types, - specs=specs, - ) - - -# --------------------------------------------------------------------------- -# Public helpers (used outside spec processing) -# --------------------------------------------------------------------------- - - -def gen_compile_arg( - spec: KernelSpec, - func: JITFunction[list[Any]], -) -> tuple[ASTSource]: - # ASTSource expects tuple-keyed dicts: {(idx,): value} for constants, - # {(idx,): [[attr_name, attr_val], ...]} for attrs. Tuple keys support - # nested paths into structured types (asserted by ASTSource.__init__). - new_signature = {} - new_constants = {} - param_names = list(func.signature.parameters.keys()) - for idx, param in enumerate(param_names): - if idx in spec.signature: - new_signature[param] = spec.signature[idx] - if idx in spec.constants: - new_constants[(idx,)] = spec.constants[idx] - new_signature[param] = "constexpr" - - # parse_attr("D") returns a fresh [["tt.divisibility", 16]] each call. - new_attrs = {(idx,): BaseBackend.parse_attr("D") for idx in spec.divisible_by_16} - - return ( - ASTSource( - func, - new_signature, - constexprs=new_constants, - attrs=new_attrs, - ), - ) - - -# --------------------------------------------------------------------------- -# Int width helpers -# --------------------------------------------------------------------------- - -_INT_WIDTH_RANK: dict[str, int] = {"i32": 0, "i64": 1} - - -def _wider_type(t1: str, t2: str) -> str: - """Return the wider of two scalar dtypes. - - Only i32/i64 widening is supported. All other types must match exactly. - """ - if t1 == t2: - return t1 - r1 = _INT_WIDTH_RANK.get(t1) - r2 = _INT_WIDTH_RANK.get(t2) - if r1 is not None and r2 is not None: - return t1 if r1 >= r2 else t2 - raise ValueError(f"Cannot widen incompatible types: {t1!r} vs {t2!r}") - - -# --------------------------------------------------------------------------- -# Private helpers — called by OpsUnit.from_raw_specs -# --------------------------------------------------------------------------- - - -def _detect_optional_args(specs: list[KernelSpec]) -> set[int]: - """Detect optional tensor args by cross-spec comparison. - - An arg at index ``i`` is optional if: - - Some specs have ``i`` in ``signature`` as a pointer type (``*...``) - - Other specs have ``constants[i] = None`` - - Single-spec None args (always-absent tensors) are NOT detected here - but are handled by ``_compute_invariants`` which adds any - ``constants[i] = None`` to ``pointer_args``. - """ - if len(specs) <= 1: - return set() - optional: set[int] = set() - all_indices: set[int] = set() - for spec in specs: - all_indices |= spec.signature.keys() - all_indices |= spec.constants.keys() - for i in all_indices: - has_pointer = any( - i in s.signature and s.signature[i].startswith("*") for s in specs - ) - has_none_const = any(i in s.constants and s.constants[i] is None for s in specs) - if has_pointer and has_none_const: - optional.add(i) - return optional - - -def _check_uniform_signature_length(base_specs: list[RawKernelSpec]) -> int: - """All raw specs must declare the same param count; return that count. - - Each raw spec is one ``infer_spec`` call site for the same kernel, - so all should have ``len(fn.signature.parameters)`` entries. Differing - lengths means upstream bug (mixed kernels, truncated spec, etc.) and - would surface later as silent IndexError or wrong bound checks. - """ - if not base_specs: - return 0 - sig_lens = {len(spec["signature"]) for spec in base_specs} - if len(sig_lens) != 1: - raise ValueError( - f"Raw specs declare inconsistent signature lengths: " - f"{sorted(sig_lens)}. All specs for the same kernel must have " - f"one entry per declared param." - ) - return sig_lens.pop() - - -def _check_arg_indices_in_range( - specs: list[KernelSpec], - num_params: int, -) -> None: - """Every spec arg index must be in ``[0, num_params)``. - - Out-of-range indices would silently drop in ``gen_compile_arg``'s - ``enumerate(param_names)`` loop. ``num_params <= 0`` disables the check. - """ - if num_params <= 0: - return - for idx, spec in enumerate(specs): - all_indices = ( - spec.signature.keys() - | spec.constants.keys() - | spec.divisible_by_16 - | spec.divisible_by_8 - ) - for i in all_indices: - if not 0 <= i < num_params: - raise ValueError( - f"Spec {idx}: arg index {i} out of range " - f"[0, {num_params}) — kernel has {num_params} declared params" - ) - - -def _collect_pointer_args( - specs: list[KernelSpec], - optional: set[int], -) -> set[int]: - """Collect all tensor pointer indices across all specs. - - Includes optional args (from _detect_optional_args) AND any arg - whose constant value is None (single-spec optional tensor case - where _detect_optional_args didn't fire). - """ - pointer_args: set[int] = set(optional) - for spec in specs: - for i, dtype in spec.signature.items(): - if dtype.startswith("*"): - pointer_args.add(i) - for i, val in spec.constants.items(): - if val is None: - pointer_args.add(i) - return pointer_args - - -def _collect_scalar_dtypes( - specs: list[KernelSpec], - pointer_args: set[int], -) -> dict[int, str]: - """Collect non-pointer signature arg dtypes, widening compatible int types. - - Invariant across specs (validated by _validate_converted_specs). - """ - scalar_dtypes: dict[int, str] = {} - for spec in specs: - for i, dtype in spec.signature.items(): - if i not in pointer_args: - if i in scalar_dtypes: - scalar_dtypes[i] = _wider_type(scalar_dtypes[i], dtype) - else: - scalar_dtypes[i] = dtype - return scalar_dtypes - - -def _collect_constant_types( - specs: list[KernelSpec], -) -> dict[int, type[Any]]: - """Collect Python type per constant position. - - Excludes None constants (optional tensor args — already in pointer_args). - """ - constant_types: dict[int, type[Any]] = {} - for spec in specs: - for i, val in spec.constants.items(): - if val is not None and i not in constant_types: - constant_types[i] = type(val) - return constant_types - - -def _compute_invariants( - specs: list[KernelSpec], - optional: set[int], -) -> tuple[set[int], dict[int, str], dict[int, type[Any]]]: - """Compute shared invariants from processed specs. - - Returns (pointer_args, scalar_dtypes, constant_types). - - When annotation-as-variant produces mixed partitions (arg in - ``signature`` in some specs, ``constants`` in others), the arg - appears in both ``scalar_dtypes`` and ``constant_types``. The - selector must receive it as a runtime parameter for dispatch, - so ``scalar_dtypes`` wins and the arg is removed from - ``constant_types``. - """ - pointer_args = _collect_pointer_args(specs, optional) - scalar_dtypes = _collect_scalar_dtypes(specs, pointer_args) - constant_types = _collect_constant_types(specs) - - # Resolve overlap: if any spec has the arg in signature (scalar), - # the selector needs it as a runtime parameter → not a constant. - for i in scalar_dtypes: - constant_types.pop(i, None) - - return pointer_args, scalar_dtypes, constant_types - - -def _validate_converted_specs( - specs: list[KernelSpec], - optional: set[int], - num_params: int = 0, -) -> None: - """Validate that converted specs are consistent before further processing. - - Checks that all specs produce identical C++ function signatures: - - All arg indices are in ``[0, num_params)`` (when ``num_params > 0``) - - Optional args: each spec has either a pointer in signature or None in constants - - Non-optional scalar args: same dtype (or compatible int widths) - - Non-optional constant args: same Python type - - Called after _convert_raw_specs + _detect_optional_args, before autotuning. - """ - _check_arg_indices_in_range(specs, num_params) - if len(specs) <= 1: - return - ref = specs[0] - for idx, spec in enumerate(specs[1:], 1): - _check_optional_consistency(ref, spec, idx, optional) - _check_signature_consistency(ref, spec, idx, optional) - _check_constants_consistency(ref, spec, idx, optional) - - -def _check_optional_consistency( - ref: KernelSpec, - spec: KernelSpec, - idx: int, - optional: set[int], -) -> None: - """Optional positions must be pointer-in-signature or None-in-constants. - - Validates that optional tensor args are not misclassified as scalars - or non-None constants, which would produce incompatible C++ types. - """ - for i in optional: - for label, s in [("spec 0", ref), (f"spec {idx}", spec)]: - if i in s.signature: - if not s.signature[i].startswith("*"): - raise ValueError( - f"Arg {i}: optional position has non-pointer type " - f"'{s.signature[i]}' in {label}" - ) - elif i in s.constants: - if s.constants[i] is not None: - raise ValueError( - f"Arg {i}: optional position has non-None constant " - f"{s.constants[i]!r} in {label}" - ) - - -def _check_signature_consistency( - ref: KernelSpec, - spec: KernelSpec, - idx: int, - optional: set[int], -) -> None: - """Non-optional, non-pointer scalar args must have compatible dtypes. - - Pointer args are skipped (different tensor dtypes are dispatched by - the dtype guard in ``gen_guarded_calls``). Compatible int widths - (i32/i64) are allowed — handled by ``_wider_type`` and int range guards. - Optional positions are validated by ``_check_optional_consistency``. - - Partition differences are allowed: an arg may be in ``signature`` in - one spec and in ``constants`` in another (e.g., annotation-as-variant - where stride=1 is constexpr in one spec but a runtime parameter in - another). The per-spec codegen handles this correctly. - """ - for i in ref.signature.keys() | spec.signature.keys(): - if i in optional: - continue - if (i in ref.signature and ref.signature[i].startswith("*")) or ( - i in spec.signature and spec.signature[i].startswith("*") - ): - continue - # Allow partition differences: arg in signature in one spec, - # in constants in another (annotation-as-variant pattern). - if i not in ref.signature or i not in spec.signature: - continue - if ref.signature[i] != spec.signature[i]: - r1 = _INT_WIDTH_RANK.get(ref.signature[i]) - r2 = _INT_WIDTH_RANK.get(spec.signature[i]) - if r1 is not None and r2 is not None: - continue - raise ValueError( - f"Arg {i}: dtype mismatch '{ref.signature[i]}' vs " - f"'{spec.signature[i]}' (spec 0 vs spec {idx})" - ) - - -def _check_constants_consistency( - ref: KernelSpec, - spec: KernelSpec, - idx: int, - optional: set[int], -) -> None: - """Non-optional constant args must have the same Python type across specs. - - C++ codegen uses one type per constant arg position (``PY_TYPES_TO_CPP_TYPES``), - so ``BLOCK_M=64`` (int) and ``BLOCK_M=64.0`` (float) would produce - incompatible launchers. Optional positions are validated separately - by ``_check_optional_consistency``. - """ - for i in ref.constants.keys() | spec.constants.keys(): - if i in optional: - continue - if ref.constants.get(i) is None or spec.constants.get(i) is None: - continue - if type(ref.constants[i]) is not type(spec.constants[i]): - raise ValueError( - f"Arg {i}: constant type mismatch " - f"{type(ref.constants[i]).__name__} vs " - f"{type(spec.constants[i]).__name__} (spec 0 vs spec {idx})" - ) - - -def _convert_raw_specs( - base_specs: list[RawKernelSpec], - gpu_target: GPUTarget, -) -> tuple[list[KernelSpec], set[int]]: - """Convert raw specs to KernelSpecs. - - Returns (specs, three_tuple_optional) where three_tuple_optional is the - union of optional_args detected from 3-tuple signature elements across - all specs (backward compat with ``collect_constraints``). - """ - raw_specs = cast(list[dict[str, Any]], copy.deepcopy(base_specs)) - is_amd = gpu_target.backend == "hip" - - result: list[KernelSpec] = [] - three_tuple_optional: set[int] = set() - for raw_spec in raw_specs: - constraints = collect_constraints(raw_spec["signature"]) - constants = extract_constants(raw_spec["signature"], constraints) - signature: dict[int, str] = signature_list_to_dict( - raw_spec["signature"], constants - ) - three_tuple_optional |= constraints.optional_args - - spec = KernelSpec( - signature=signature, - constants=constants, - divisible_by_16=constraints.divisible_by_16, - divisible_by_8=constraints.divisible_by_8, - ) - - if constraints.has_fp8: - if is_amd: - spec.signature = get_fp8_replacement_signature_for_amd( - {"signature": spec.signature}, {str(gpu_target.arch)} - ) - elif gpu_target.arch == 80: - spec.signature = get_fp8_replacement_signature_for_sm80( - {"signature": spec.signature} - ) - - result.append(spec) - - return result, three_tuple_optional - - -def _autotune_specs( - func: triton.runtime.autotuner.Autotuner, - target: GPUTarget, - specs: list[KernelSpec], -) -> list[KernelSpec]: - tuned_specs: list[KernelSpec] = [] - for spec in specs: - for cfg in func.cache.values(): - constants = spec.constants.copy() - for arg_name, arg_val in cfg.kwargs.items(): - if arg_name in AUTOTUNE_ATTRs: - continue - arg_idx = func.arg_names.index(arg_name) - if constants.get(arg_idx, -1) == -1: - constants[arg_idx] = arg_val - - autotune_values: dict[str, int] = {} - for name, default in AUTOTUNE_ATTRs.items(): - if name in cfg.kwargs: - autotune_values[name] = cfg.kwargs[name] - else: - autotune_values[name] = getattr(cfg, name, default) - # AMD has changed their software pipeliner in Triton - # It now expects num_stages == 2 instead of 0 - # see: https://github.com/pytorch/pytorch/pull/139881 - # if we see someone try to set num_stages == 0, set it to the default (2) instead - # We can't use the Triton hook to get the default value because it requires the AMD runtime to be loaded - if ( - target.backend == "hip" - and name == "num_stages" - and autotune_values[name] == 0 - and TRITON_VERSION >= "3.2.0" - ): - autotune_values[name] = 2 - - tuned_spec = dataclasses.replace( - spec, - constants=constants, - # pyrefly: ignore [bad-argument-type] - **autotune_values, - ) - tuned_specs.append(tuned_spec) - return tuned_specs - - -def _dedup_specs(specs: list[KernelSpec]) -> list[KernelSpec]: - deduped_specs: list[KernelSpec] = [] - duplicated_specs: list[KernelSpec] = [] - hash_spec_ids: set[str] = set() - for spec in specs: - id = hash_spec(dataclasses.asdict(spec)) - if id in hash_spec_ids: - duplicated_specs.append(spec) - else: - hash_spec_ids.add(id) - deduped_specs.append(spec) - - logger.debug( - f"[TritonAOT Dedup] {len(specs)=} {len(deduped_specs)=} {len(duplicated_specs)=}" - ) - return deduped_specs diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/compile/stable_types.py b/recommendation_v4/generative_recommenders/ops/triton_aot/compile/stable_types.py deleted file mode 100644 index 33038a534..000000000 --- a/recommendation_v4/generative_recommenders/ops/triton_aot/compile/stable_types.py +++ /dev/null @@ -1,35 +0,0 @@ -# pyre-strict - -"""AOTT-local type mappings for stable ABI codegen. - -These replace ``shared.types.ATYPES`` and ``shared.types.PY_TYPES_TO_CPP_TYPES`` -with versions that have zero link dependency on ATen. The shared dicts are kept -unchanged so TritonCC is not affected. -""" - -from typing import Any - -# Stable ABI scalar type mapping: Triton pointer dtype → c10::ScalarType enum. -# Uses c10::ScalarType:: (from torch/headeronly/core/ScalarType.h) instead of -# at::kFloat aliases (which require ATen headers). -SCALAR_TYPES: dict[str, str] = { - "*i1": "c10::ScalarType::Bool", - "*u8": "c10::ScalarType::Byte", - "*i8": "c10::ScalarType::Char", - "*i16": "c10::ScalarType::Short", - "*i32": "c10::ScalarType::Int", - "*i64": "c10::ScalarType::Long", - "*fp16": "c10::ScalarType::Half", - "*fp32": "c10::ScalarType::Float", - "*fp64": "c10::ScalarType::Double", - "*bf16": "c10::ScalarType::BFloat16", - "*fp8e4nv": "c10::ScalarType::Float8_e4m3fn", - "*fp8e4b8": "c10::ScalarType::Float8_e4m3fnuz", -} - -# Stable ABI override: str → "std::string" instead of "at::string". -PY_TYPES_TO_CPP_TYPES: dict[type[Any], str] = { - int: "int64_t", - str: "std::string", - float: "double", -} diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/compile/triton_aot_compile.py b/recommendation_v4/generative_recommenders/ops/triton_aot/compile/triton_aot_compile.py deleted file mode 100644 index 74179435a..000000000 --- a/recommendation_v4/generative_recommenders/ops/triton_aot/compile/triton_aot_compile.py +++ /dev/null @@ -1,149 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# pyre-strict - -import importlib -import logging -import os -import pickle -from types import ModuleType, TracebackType -from typing import Any, Callable, Optional, Type - -from generative_recommenders.ops.triton_aot.build.extension_builder import ( - build_triton_aot_extension, -) -from generative_recommenders.ops.triton_aot.compile.codegen import ( - is_non_empty_mapping_of_type, -) -from generative_recommenders.ops.triton_aot.compile.compile_state import ( - get_aott_compile_path, - get_aott_compile_state, - get_triton_aot_kernel_specs, -) -from generative_recommenders.ops.triton_aot.compile.pipeline import compile_to_cpp -from generative_recommenders.ops.triton_aot.compile.utils import unwrap_heuristic -from torch import package -from triton.backends.compiler import GPUTarget -from triton.runtime import driver, JITFunction - -# @manual=//triton:triton -from triton.runtime.autotuner import Config - -logger: logging.Logger = logging.getLogger(__name__) - - -class TritonAOTCompile: - """ - Context manager to compile Triton kernels to C++ and build a shared library. - The compiled kernels are cached in a temporary directory. - - - package_importer: - torch.package importer for loading kernels source code (aott/ops). - If not provided, the default importlib is used (for local use cases) - - gpu_target: - GPU target to compile for (default: active GPU target, determined by Triton driver) - This local copy intentionally omits Manifold autotune-cache overrides. The - HSTU e2e path only needs representative-input autotuning captured during - the compile context. - """ - - def __init__( - self, - package_importer: Optional[package.PackageImporter] = None, - gpu_target: Optional[GPUTarget] = None, - auto_tune_cache_override_path: Optional[str] = None, - ) -> None: - self._import_module: Callable[[str], ModuleType] = ( - package_importer.import_module - if package_importer is not None - else importlib.import_module - ) - self.gpu_target: GPUTarget = gpu_target or driver.active.get_current_target() - self.auto_tune_cache_override_path: Optional[str] = ( - auto_tune_cache_override_path - ) - - def _load_autotune_cache_overrides( - self, - ) -> dict[str, Any]: - if self.auto_tune_cache_override_path is None: - return {} - raise NotImplementedError( - "Local generative_recommenders AOT-T compile does not support " - "auto_tune_cache_override_path." - ) - - def __enter__(self) -> None: - state = get_aott_compile_state() - state.reset() - state.enable() - logger.info( - f"Start AOTT compile, output dir: {get_aott_compile_path()}, gpu_target: {self.gpu_target}" - ) - - def _resolve_autotune_cache( - self, - fn: Any, - fn_name: str, - fn_dir: str, - overrides: dict[str, Any], - ) -> None: - """Apply override (if matched) and dump the autotune cache to fn_dir.""" - override = overrides.get(fn_name) - if override is not None: - logger.info( - f"[AOTT]: Overriding autotune cache for {fn_name} " - f"from {self.auto_tune_cache_override_path}" - ) - fn.cache = override - - # cache are dumped just for testing - if hasattr(fn, "cache") and is_non_empty_mapping_of_type(fn.cache, Config): - with open(f"{fn_dir}/{fn_name}_autotune_cache", "wb") as data: - # @lint-ignore PYTHONPICKLEISBAD - pickle.dump(fn.cache, data) - - def __exit__( - self, - exc_type: Optional[Type[BaseException]], - exc_value: Optional[BaseException], - traceback: Optional[TracebackType], - ) -> None: - compile_path = get_aott_compile_path() - if not os.path.exists(compile_path): - os.makedirs(compile_path) - - kernel_specs = get_triton_aot_kernel_specs() - auto_tune_overrides = self._load_autotune_cache_overrides() - - logger.info(f"[AOTT]: compiling {len(kernel_specs)} kernels") - - for fn, specs in kernel_specs.items(): - jit_fn = unwrap_heuristic(fn, JITFunction) - fn_name = jit_fn.__name__ - - logger.info(f"[AOTT]: compiling {fn_name} with specs: {specs}") - - module_suffix = jit_fn.__module__.rsplit(".", 1)[-1] - fn_dir = f"{compile_path}/{module_suffix}_{fn_name}" - if not os.path.exists(fn_dir): - os.makedirs(fn_dir) - - self._resolve_autotune_cache(fn, fn_name, fn_dir, auto_tune_overrides) - - compile_to_cpp( - func=fn, - base_specs=specs, - install_dir=f"{fn_dir}", - prefix=f"{fn_name}", - gpu_target=self.gpu_target, - tuner_fallback=True, - import_module=self._import_module, - ) - - build_triton_aot_extension( - source_dir=fn_dir, - kernel_name=fn_name, - output_dir=fn_dir, - ) - - get_aott_compile_state().disable() diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/compile/utils.py b/recommendation_v4/generative_recommenders/ops/triton_aot/compile/utils.py deleted file mode 100644 index e3848fd2e..000000000 --- a/recommendation_v4/generative_recommenders/ops/triton_aot/compile/utils.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. - -# pyre-strict - -import hashlib -from typing import Any, Type, TypeVar - -T = TypeVar("T") - - -def unwrap_heuristic(func: Any, return_type: Type[T]) -> T: - while not isinstance(func, return_type): - func = func.fn - if not hasattr(func, "fn"): - # pyre-fixme[7]: Incompatible return type [7]: Expected `Variable[T]` but got `None`. - return None - return func - - -def is_autotuner(obj: Any) -> bool: - """Check whether *obj* is a Triton Autotuner using duck typing. - - In Buck builds the ``Autotuner`` class can be loaded from multiple module - paths (e.g. via ``torch.package`` re-imports), causing ``isinstance`` to - return ``False`` for genuine Autotuner instances. We combine a class-name - check with duck-typing on the attributes that callers actually need - (``cache``, ``configs``, ``arg_names``), making detection robust against - module-path aliasing. - """ - return "Autotuner" in type(obj).__name__ and all( - hasattr(obj, attr) for attr in ("cache", "configs", "arg_names") - ) - - -def hash_kernel_name(kernel_name: str) -> str: - """Hash kernel name to create shorter, filesystem-safe names. - - Args: - kernel_name: Full kernel name (can be very long with specialization suffixes). - e.g., "_addmm_fwd_sm80_pfp32_pfp32_pfp32_pfp32_i32_..." - - Returns: - Hashed name in format "kernel_". - e.g., "kernel_a1b2c3d4e5f6..." - - """ - return "kernel_" + hashlib.sha256(kernel_name.encode("utf-8")).hexdigest() diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/preprocess.py b/recommendation_v4/generative_recommenders/ops/triton_aot/preprocess.py deleted file mode 100644 index ce2e63d43..000000000 --- a/recommendation_v4/generative_recommenders/ops/triton_aot/preprocess.py +++ /dev/null @@ -1,76 +0,0 @@ -# (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -# pyre-strict - -""" -Preprocessing utilities for triton_aot models before AOT compilation. -""" - -import logging - -from tgif.fx.tgif_tracer import TGIFTracer -from torch.fx import GraphModule - -logger: logging.Logger = logging.getLogger(__name__) - -# "aot_triton_kernel_wrapper_" is a pre-defined prefix for -# AOT-T triton kernel wrapper functions. This is required for -# AOT-T backend to recognize and trace correctly for ops transformation. -AOTT_WRAPPER_PREFIX: str = "aot_triton_kernel_wrapper_" - - -def unwrap_aott_wrapper_nodes(fx_m: GraphModule, tracer: TGIFTracer) -> GraphModule: - """Mark ``aot_triton_kernel_wrapper_*`` FX nodes as unwrapped and re-trace. - - In the traced FX graph, outer wrapper functions (prefixed with - ``aot_triton_kernel_wrapper_``) are ``@torch.fx.wrap`` leaves. - Setting ``node.meta["is_wrapped"] = False`` causes a subsequent - ``symbolic_trace`` to trace *through* them, exposing the inner - ``@torch.fx.wrap`` functions (e.g., ``_triton_aot_grouped_gemm``) - that contain the actual kernel calls. - - Any ``_body_transformer`` hook (e.g. one registered by - ``early_return_fx_code_transform``) is temporarily removed before - re-tracing to avoid injecting un-traceable control flow - (``if Proxy: …``) into the generated ``forward``. After re-trace - the hook is restored on the new module. See P2266562545. - - Args: - fx_m: The FX GraphModule to modify **in-place** before re-trace. - tracer: Tracer instance used for the re-trace step. - - Returns: - The re-traced ``GraphModule`` with AOTT wrappers expanded. - """ - logger.info("Re-trace to get the AOTT node exposed.") - - # Save and clear the body transformer so that re-trace does not hit - # ``if Proxy:`` from code-level hooks like early_return_fx_code_transform. - saved_body_transformer = fx_m.graph._codegen._body_transformer - fx_m.graph._codegen._body_transformer = None - - unwrap_count = 0 - for node in fx_m.graph.nodes: - if node.op == "call_function": - target = node.target - if hasattr(target, "__name__") and target.__name__.startswith( - AOTT_WRAPPER_PREFIX - ): - logger.info(f"[AOTT] Found inference wrapper node: {node=}") - node.meta["is_wrapped"] = False - unwrap_count += 1 - - if unwrap_count > 0: - logger.info(f"[AOTT] Found {unwrap_count} inference wrapper nodes.") - fx_m.recompile() - else: - logger.warning("[AOTT] No inference wrapper node found. Skip re-compile.") - - result = tracer.symbolic_trace(fx_m) - - # Restore the body transformer on the new module. - if saved_body_transformer is not None: - result.graph._codegen._body_transformer = saved_body_transformer - result.recompile() - - return result diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/shared/compat.py b/recommendation_v4/generative_recommenders/ops/triton_aot/shared/compat.py deleted file mode 100644 index be6235701..000000000 --- a/recommendation_v4/generative_recommenders/ops/triton_aot/shared/compat.py +++ /dev/null @@ -1,91 +0,0 @@ -# pyre-strict -""" -This module provides shared utilities that handle differences between -Triton versions. -""" - -from typing import Any - -# @manual=//triton:triton -import triton -from packaging.version import Version -from triton.runtime.jit import JITFunction - -TRITON_VERSION: str = triton.__version__ - - -def version_gte(version: str, target: str) -> bool: - """ - Check if version >= target using semantic version comparison. - Simple string comparison fails for versions like "3.10" vs "3.5" - """ - return Version(version) >= Version(target) - - -def get_kernel_name(jit_fn: JITFunction[Any]) -> str: - """ - Get the simple kernel name from a JITFunction. - - In Triton 3.5+, JITFunction._fn_name returns the full qualified name - (e.g., "generative_recommenders.ops.triton_aot.triton_addmm._addmm_fwd"). - In older versions, it returns just the simple name (e.g., "_addmm_fwd"). - - This function normalizes the behavior to always return the simple name. - - Args: - jit_fn: A Triton JITFunction - - Returns: - The simple kernel name (e.g., "_addmm_fwd") - """ - fn_name = jit_fn._fn_name - if version_gte(TRITON_VERSION, "3.5"): - # Triton 3.5+ uses get_full_name(fn) which returns qualified name - return fn_name.rsplit(".", 1)[-1] - else: - # Older versions use fn.__name__ which is already simple - return fn_name - - -def get_scratch_parameters(kernel: Any) -> tuple[str, list[str]]: - """ - Get scratch parameter declarations and argument pointers for the kernel launcher. - - Scratch parameters are backend and version-specific features for profiling - and global memory management. - - Detection Strategy: - 1. Check metadata first for each parameter - 2. Fall back to version-based detection if metadata unavailable - - Version Requirements (fallback): - - v3.4+: both global_scratch and profile_scratch - - v3.3: only global_scratch - - v3.2 and earlier: no scratch parameters - - Args: - kernel: Compiled Triton kernel with metadata attribute - - Returns: - Tuple of (declarations, arg_pointers): - - declarations: C++ variable declarations for scratch parameters - - arg_pointers: List of argument pointers to append to kernel args - """ - declarations = [] - arg_pointers = [] - - if hasattr(kernel.metadata, "global_scratch_size"): - declarations.append("CUdeviceptr global_scratch = 0;") - arg_pointers.append("&global_scratch") - elif version_gte(TRITON_VERSION, "3.3"): - declarations.append("CUdeviceptr global_scratch = 0;") - arg_pointers.append("&global_scratch") - - if hasattr(kernel.metadata, "profile_scratch_size"): - declarations.append("CUdeviceptr profile_scratch = 0;") - arg_pointers.append("&profile_scratch") - elif version_gte(TRITON_VERSION, "3.4"): - declarations.append("CUdeviceptr profile_scratch = 0;") - arg_pointers.append("&profile_scratch") - - return ("\n ".join(declarations), arg_pointers) diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/shared/spec_conversion.py b/recommendation_v4/generative_recommenders/ops/triton_aot/shared/spec_conversion.py deleted file mode 100644 index 4a1ebb133..000000000 --- a/recommendation_v4/generative_recommenders/ops/triton_aot/shared/spec_conversion.py +++ /dev/null @@ -1,389 +0,0 @@ -# pyre-strict - -"""Functions for converting kernel specs to architecture-specific formats. - -A "spec" (specification) describes how to compile a Triton kernel for a specific -set of input shapes and types. Users provide "base specs" in a human-friendly -format that describes kernel arguments: - - {"signature": [("*fp32", 16), ("*bf16", 16), ("i32", None), 128]} - -This format encodes dtypes, alignment hints, and constant values together. -Before compilation, base specs must be converted to "compiled specs" that -separate this information into distinct fields the compiler understands: - - {"signature": {0: "*fp32", 1: "*bf16"}, - "constants": {2: None, 3: 128}, - "configs": (instance_descriptor(...),), - "cc": 80} - -This module provides the functionality to perform this transformation, -extracting constraints, identifying constants, and preparing specs for each -target GPU architecture. -""" - -from collections import namedtuple -from dataclasses import dataclass -from typing import Any, TypeAlias - -from generative_recommenders.ops.triton_aot.shared.types import CTYPES - -# Compile-time constant values that can appear in signatures or be returned -# by constexpr(). These are values the compiler can fold into generated code. -ConstantValue: TypeAlias = str | int | float | bool | None - -# A single element in a kernel signature list. -# Can be: dtype string, (dtype, alignment) tuple, (dtype, alignment, has_value) -# triple for optional args, or a bare literal constant. -SignatureElement: TypeAlias = ( - ConstantValue | tuple[str, int | None] | tuple[str, int | None, bool] -) - - -instance_descriptor = namedtuple( - "instance_descriptor", - [ - "divisible_by_16", - "equal_to_1", - "ids_of_folded_args", - "divisible_by_8", - ], -) - - -def constexpr(s: SignatureElement) -> ConstantValue: - """Identify compile-time constant expressions in signature elements. - - Args: - s: A signature element. - - Returns: - The constant value if s is a compile-time constant, None otherwise. - Constants are: int, float, bool, or strings that aren't dtype names. - """ - expr = s[0] if isinstance(s, tuple) and len(s) > 1 else s - - if expr is None: - return expr - - try: - ret = int(expr) - return ret - except (ValueError, TypeError): - pass - try: - ret = float(expr) - return ret - except (ValueError, TypeError): - pass - - if isinstance(expr, bool): - return expr - if isinstance(expr, str) and expr not in CTYPES and not expr.startswith("*"): - return expr - return None - - -@dataclass -class SignatureConstraints: - """Constraints extracted from parsing a kernel signature. - - When compiling a Triton kernel, the compiler can generate more efficient - code if it knows certain properties about the arguments: - - - Pointer alignment: If a pointer is always 16-byte aligned, the compiler - can use faster aligned memory operations. - - Constant values: Arguments known at compile time can be folded into the - generated code, eliminating runtime checks. - - FP8 dtypes: Some GPU architectures require dtype substitutions for FP8 - types (e.g., gfx942 needs fp8e4b8 instead of fp8e4nv). - - This dataclass collects all these constraints from a single pass over the - signature, so downstream code can use them without re-parsing. - - Attributes: - divisible_by_16: Indices of args with values divisible by 16. - divisible_by_8: Indices of args with values divisible by 8. - equal_to_1: Indices of args with value equal to 1. - none_args: Indices of args that are None (not provided). - optional_args: Indices of optional arguments. - has_fp8: Whether any argument has an FP8 dtype. - """ - - divisible_by_16: set[int] - divisible_by_8: set[int] - equal_to_1: set[int] - none_args: set[int] - optional_args: set[int] - has_fp8: bool - - -def collect_constraints(signature: list[SignatureElement]) -> SignatureConstraints: - """Collect divisibility and type constraints from a signature list. - - Iterates through signature elements and identifies: - - Arguments divisible by 16 or 8 (for memory alignment) - - Arguments equal to 1 (for optimization) - - Optional arguments and those not provided (None) - - Whether any FP8 dtypes are present - - Args: - signature: List of signature elements. The input format is unfortunately - variable; each element can be one of several types: - - 1. Plain string (dtype only, no alignment info): - "*fp32" - A float32 pointer - "i32" - A 32-bit integer scalar - - 2. Tuple of (dtype, value) where value indicates alignment or constness: - ("*fp32", 16) - Float32 pointer, 16-byte aligned - ("i32", None) - Integer arg not provided (becomes constant None) - ("*bf16", 1) - Pointer with value=1 (folded as constant) - - 3. Triple of (dtype, value, has_value) for optional arguments: - ("*fp32", 16, True) - Optional arg that IS provided, 16-byte aligned - ("*fp32", 16, False) - Optional arg NOT provided (becomes None) - - 4. Bare literals (become compile-time constants): - 128 - Integer constant - "leaky_relu" - String constant (e.g., activation name) - - Returns: - SignatureConstraints with all constraint sets populated. - - Example: - >>> sig = [("*fp32", 16), ("i32", None), ("*fp8e4nv", 8)] - >>> c = collect_constraints(sig) - >>> 0 in c.divisible_by_16 - True - >>> c.has_fp8 - True - """ - divisible_by_16: set[int] = set() - divisible_by_8: set[int] = set() - equal_to_1: set[int] = set() - none_args: set[int] = set() - optional_args: set[int] = set() - has_fp8: bool = False - - for i, s in enumerate(signature): - # Handle optional tensor case: tuple with 3 elements where s[2] indicates - # whether the optional arg has a value - if isinstance(s, tuple) and len(s) > 2: - optional_args.add(i) - # pyrefly: ignore [bad-index] - if not s[2]: # has_value is False - none_args.add(i) - continue - - # Extract dtype - dtype = s[0] if isinstance(s, tuple) else s - - # Check for FP8 types - if isinstance(dtype, str) and ("fp8e4nv" in dtype or "fp8e4b8" in dtype): - has_fp8 = True - - # Extract value (alignment or constant) - value = s[1] if isinstance(s, tuple) else s - - # Check divisibility and equality constraints - if isinstance(value, int): - if value % 16 == 0: - divisible_by_16.add(i) - if value % 8 == 0: - divisible_by_8.add(i) - if value == 1: - equal_to_1.add(i) - - if value is None: - none_args.add(i) - - return SignatureConstraints( - divisible_by_16=divisible_by_16, - divisible_by_8=divisible_by_8, - equal_to_1=equal_to_1, - none_args=none_args, - optional_args=optional_args, - has_fp8=has_fp8, - ) - - -def make_instance_descriptor( - constraints: SignatureConstraints, -) -> tuple[instance_descriptor]: - """Create an instance_descriptor tuple from constraints. - - Args: - constraints: The collected signature constraints. - - Returns: - A tuple containing a single instance_descriptor namedtuple with - divisible_by_16, equal_to_1, ids_of_folded_args, and divisible_by_8. - """ - ids_of_folded_args = constraints.equal_to_1 | constraints.none_args - return ( - instance_descriptor( - divisible_by_16=constraints.divisible_by_16, - equal_to_1=constraints.equal_to_1, - ids_of_folded_args=ids_of_folded_args, - divisible_by_8=constraints.divisible_by_8, - ), - ) - - -def extract_constants( - signature: list[SignatureElement], - constraints: SignatureConstraints, -) -> dict[int, ConstantValue]: - """Extract compile-time constant values from signature elements. - - Identifies arguments that can be folded into generated code at compile time. - Constants come from three sources: - - 1. Bare literals in the signature (e.g., 128 for block size, "leaky_relu" - for activation type, True for a boolean flag) - 2. Arguments with value=1 (tracked in constraints.equal_to_1) - 3. Arguments not provided (tracked in constraints.none_args) - - Args: - signature: List of signature elements in input format. - constraints: The collected signature constraints. - - Returns: - Dict mapping argument indices to their constant values. - """ - # Use constexpr to identify constant expressions - constexprs = {i: constexpr(s) for i, s in enumerate(signature)} - constants: dict[int, ConstantValue] = { - k: v for k, v in constexprs.items() if v is not None - } - - # Add equal_to_1 args with value 1 - for k in constraints.equal_to_1: - constants[k] = 1 - - # Add none_args with value None - for k in constraints.none_args: - constants[k] = None - - return constants - - -def signature_list_to_dict( - signature: list[SignatureElement], - constants: dict[int, ConstantValue], -) -> dict[int, str]: - """Convert signature from list format to dict format. - - Transforms the input signature list into a dict mapping argument - indices to dtype strings. Arguments that are constants are excluded - since they don't need runtime type information. - - Args: - signature: List of signature elements in input format. - constants: Dict of constant argument indices to exclude. - - Returns: - Dict mapping non-constant argument indices to their dtype strings. - """ - result: dict[int, str] = {} - for i, s in enumerate(signature): - if i in constants: - continue - # After filtering out constants, remaining elements are dtype declarations. - # For tuples like ("*fp32", 16), s[0] is the dtype string. - # For plain strings like "*fp32", the element itself is the dtype. - if isinstance(s, tuple) and len(s) > 1: - dtype = s[0] - else: - dtype = s - assert isinstance(dtype, str) - result[i] = dtype - return result - - -# CC (compute capability) to AMD GPU architecture mapping -# CC is a 2-digit shorthand: 94 -> gfx942, 95 -> gfx950 -HIP_CC_TO_ARCH_INFO: dict[int, str] = { - 90: "gfx90a", - 94: "gfx942", - 95: "gfx950", -} - -# Reverse mapping: architecture string -> CC string -HIP_ARCH_TO_CC: dict[str, str] = {v: str(k) for k, v in HIP_CC_TO_ARCH_INFO.items()} - -HIP_CC_MI350X: str = "95" # CC string for gfx950 (MI350X/MI355X) - - -def _normalize_cc(cc: set[str]) -> set[str]: - """Normalize CC values to 2-digit format for internal comparison. - - Accepts both tritoncc format ("94", "95") and Triton driver format - ("gfx942", "gfx950"). Returns 2-digit CC strings. - """ - return {HIP_ARCH_TO_CC.get(c, c) for c in cc} - - -def get_fp8_replacement_signature_for_amd( - spec: dict[str, Any], cc: set[str] -) -> dict[int, str]: - """Replace FP8 dtypes in signature for AMD architectures. - - Args: - spec: Compiled spec dict with 'signature' in dict format. - cc: Set of CC strings in either format: - - 2-digit tritoncc format: {"94"} for gfx942 - - Triton driver format: {"gfx942"} - See HIP_CC_TO_ARCH_INFO. - - Returns: - Dict mapping argument indices to dtype strings with FP8 types replaced. - """ - normalized_cc: set[str] = _normalize_cc(cc) - - def replace_fp8_type(dtype_str: str) -> str: - if "fp8e4nv" in dtype_str: - if HIP_CC_MI350X not in normalized_cc: - return dtype_str.replace("fp8e4nv", "fp8e4b8") - elif "fp8e4b8" in dtype_str and HIP_CC_MI350X in normalized_cc: - return dtype_str.replace("fp8e4b8", "fp8e4nv") - return dtype_str - - replace_fp8_signatures: dict[int, str] = {} - for key, value in spec["signature"].items(): - if isinstance(value, str): - replace_fp8_signatures[key] = replace_fp8_type(value) - else: - replace_fp8_signatures[key] = value - - return replace_fp8_signatures - - -def get_fp8_replacement_signature_for_sm80( - spec: dict[str, Any], -) -> dict[int, Any]: - """Replace FP8 dtypes with bf16 for SM80 (A100) which lacks native FP8 support. - - Args: - spec: Compiled spec dict with 'signature' in dict format. - - Returns: - Dict mapping argument indices to dtype strings with FP8 types replaced by bf16. - """ - - def replace_fp8_type(dtype_str: str) -> str: - if "fp8e4nv" in dtype_str: - return dtype_str.replace("fp8e4nv", "bf16") - return dtype_str - - replace_fp8_signatures: dict[int, Any] = {} - for key, value in spec["signature"].items(): - if isinstance(value, tuple) and isinstance(value[0], str): - replace_fp8_signatures[key] = (replace_fp8_type(value[0]), value[1]) - elif isinstance(value, str): - replace_fp8_signatures[key] = replace_fp8_type(value) - else: - replace_fp8_signatures[key] = value - - return replace_fp8_signatures diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/shared/types.py b/recommendation_v4/generative_recommenders/ops/triton_aot/shared/types.py deleted file mode 100644 index 6a870fc4d..000000000 --- a/recommendation_v4/generative_recommenders/ops/triton_aot/shared/types.py +++ /dev/null @@ -1,58 +0,0 @@ -# pyre-strict - -"""Shared type definitions for AOTT and Triton CC. - -This module contains fundamental type mappings used across the compiler. -""" - -from typing import Any - -# Mapping from Triton dtype names to C type names -CTYPES: dict[str, str] = { - "i1": "bool", - "u8": "uint8_t", - "i8": "int8_t", - "i16": "int16_t", - "i32": "int32_t", - "i64": "int64_t", - "fp16": "half", - "fp32": "float", - "fp64": "double", - "bf16": "__nv_bfloat16", - "fp8e4nv": "__nv_fp8_e4m3", - "fp8e4b8": "__hip_fp8_e4m3_fnuz", -} - -# Mapping from Triton pointer dtype names to ATen scalar types -ATYPES: dict[str, str] = { - "*i1": "at::kBool", - "*u8": "at::kByte", - "*i8": "at::kChar", - "*i16": "at::kShort", - "*i32": "at::kInt", - "*i64": "at::kLong", - "*fp16": "at::kHalf", - "*fp32": "at::kFloat", - "*fp64": "at::kDouble", - "*bf16": "at::kBFloat16", - "*fp8e4nv": "at::kFloat8_e4m3fn", - "*fp8e4b8": "at::kFloat8_e4m3fnuz", -} - -# Mapping from Python types to C++ type names -PY_TYPES_TO_CPP_TYPES: dict[type[Any], str] = { - int: "int64_t", - str: "at::string", - float: "double", -} - -# Default values for autotuning attributes. -# These are used as default kernel launch parameters. -AUTOTUNE_ATTRs: dict[str, int] = { - "num_warps": 4, - "num_stages": 3, - # AMD only - "matrix_instr_nonkdim": 0, - "waves_per_eu": 1, - "kpack": 1, -} diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/templates/embedded_cubins.cpp b/recommendation_v4/generative_recommenders/ops/triton_aot/templates/embedded_cubins.cpp deleted file mode 100644 index d269c3555..000000000 --- a/recommendation_v4/generative_recommenders/ops/triton_aot/templates/embedded_cubins.cpp +++ /dev/null @@ -1,7 +0,0 @@ -#include - -extern "C" { -// __TRITON_AOT_GENERATE_BEGIN__ CUBIN_ARRAYS -// placeholder -// __TRITON_AOT_GENERATE_END__ CUBIN_ARRAYS -} diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/templates/kernel.cpp b/recommendation_v4/generative_recommenders/ops/triton_aot/templates/kernel.cpp deleted file mode 100644 index 7f882f11f..000000000 --- a/recommendation_v4/generative_recommenders/ops/triton_aot/templates/kernel.cpp +++ /dev/null @@ -1,104 +0,0 @@ -// __TRITON_AOT_GENERATE_BEGIN__ HEADER_INCLUDE -#include "kernel.h" -// __TRITON_AOT_GENERATE_END__ HEADER_INCLUDE -// These headers are used by code generated at runtime in KERNEL_SPECS blocks -#include -#include // NOLINT(facebook-unused-include-check) - -inline void triton_aot_cu_check(CUresult err, const char* file, int line) { - if (err != CUDA_SUCCESS) { - const char* err_str; - cuGetErrorString(err, &err_str); - throw std::runtime_error( - std::string(file) + ":" + std::to_string(line) + - " CUDA driver error: " + (err_str ? err_str : "unknown")); - } -} -#define TRITON_AOT_CU_CHECK(EXPR) triton_aot_cu_check(EXPR, __FILE__, __LINE__) - -// NOLINTNEXTLINE(facebook-hte-NullableReturn): error path throws -inline cudaStream_t triton_aot_get_current_stream() { - auto device_idx = torch::stable::accelerator::getCurrentDeviceIndex(); - void* stream_ptr = nullptr; - // TODO: No torch::stable op provides the same functionality - // today. Revisit if torch exposes a proper stable::accelerator stream API. - if (aoti_torch_get_current_cuda_stream(device_idx, &stream_ptr) != 0) { - throw std::runtime_error("Failed to get current CUDA stream"); - } - return reinterpret_cast(stream_ptr); -} - -namespace triton { -namespace aot { - -namespace { -[[maybe_unused]] int compute_capability() { - // Cached: AOTT hosts use homogeneous GPUs. - static int cc = 0; - if (cc == 0) { - CUdevice device; - TRITON_AOT_CU_CHECK(cuCtxGetDevice(&device)); - int major, minor; - TRITON_AOT_CU_CHECK(cuDeviceGetAttribute( - &major, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, device)); - TRITON_AOT_CU_CHECK(cuDeviceGetAttribute( - &minor, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, device)); - cc = major * 10 + minor; - } - return cc; -} -} // namespace - -namespace { -#ifdef USE_ROCM -[[maybe_unused]] void check_errors(int shared, hipFunction_t func) { - // HIP doesn't need the same shared memory configuration as CUDA - return; -} -#else -[[maybe_unused]] void check_errors(int shared, CUfunction func) { - int shared_optin; - int device = 0; - TRITON_AOT_CU_CHECK(cuDeviceGetAttribute( - &shared_optin, - CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_BLOCK_OPTIN, - device)); - if (shared > 49152 && shared_optin > 49152) { - // If requested/shared_optin exceed 48 KB, it switches cache to prefer - // shared memory and sets the max dynamic shared memory so the kernel can - // allocate the larger amount needed. - TRITON_AOT_CU_CHECK( - cuFuncSetCacheConfig(func, CU_FUNC_CACHE_PREFER_SHARED)); - int shared_total, shared_static; - TRITON_AOT_CU_CHECK(cuDeviceGetAttribute( - &shared_total, - CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_MULTIPROCESSOR, - device)); - TRITON_AOT_CU_CHECK(cuFuncGetAttribute( - &shared_static, CU_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES, func)); - TRITON_AOT_CU_CHECK(cuFuncSetAttribute( - func, - CU_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES, - shared_optin - shared_static)); - } -} -#endif -} // namespace - -// __TRITON_AOT_GENERATE_BEGIN__ KERNEL_SPECS -// __TRITON_AOT_GENERATE_END__ KERNEL_SPECS - -// __TRITON_AOT_GENERATE_BEGIN__ SELECTOR -// __TRITON_AOT_GENERATE_END__ SELECTOR - -} // namespace aot -} // namespace triton - -// Anchor: keeps the inline `triton_aot_get_current_stream` (and its reference -// to `aoti_torch_get_current_cuda_stream`) from being dead-stripped at -// buck-build time, where KERNEL_SPECS is empty. `weak` dedups the symbol -// across the per-op .so files generated by the runtime template substitution. -extern "C" __attribute__((weak, visibility("default"))) cudaStream_t -__triton_aot_anchor_get_stream() { - return triton_aot_get_current_stream(); -} diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/templates/kernel.h b/recommendation_v4/generative_recommenders/ops/triton_aot/templates/kernel.h deleted file mode 100644 index 6b6f76e17..000000000 --- a/recommendation_v4/generative_recommenders/ops/triton_aot/templates/kernel.h +++ /dev/null @@ -1,36 +0,0 @@ -#pragma once - -#include - -#include -#include - -namespace triton { -namespace aot { - -#ifndef GRID_DIM_DEFINED_MACRO -struct gridDims { - int x = 1; - int y = 1; - int z = 1; - cudaStream_t stream = nullptr; - gridDims(int _x = 1, int _y = 1, int _z = 1, cudaStream_t _stream = nullptr) - : x(_x), y(_y), z(_z), stream(_stream) {} -}; -#define GRID_DIM_DEFINED_MACRO -#endif - -#ifndef FITS_I32_DEFINED_MACRO -constexpr bool fits_i32(int64_t v) { - return v >= INT32_MIN && v <= INT32_MAX; -} -#define FITS_I32_DEFINED_MACRO -#endif - -// __TRITON_AOT_GENERATE_BEGIN__ TUNER_META_CPP -// __TRITON_AOT_GENERATE_END__ TUNER_META_CPP -// __TRITON_AOT_GENERATE_BEGIN__ SELECTOR_PROTO -// __TRITON_AOT_GENERATE_END__ SELECTOR_PROTO - -} // namespace aot -} // namespace triton diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/templates/template_utils.py b/recommendation_v4/generative_recommenders/ops/triton_aot/templates/template_utils.py deleted file mode 100644 index d89bbe8b6..000000000 --- a/recommendation_v4/generative_recommenders/ops/triton_aot/templates/template_utils.py +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. - -# pyre-strict - -""" -Common utilities for template loading and rendering. - -This module provides functions to load template files from the templates -directory and render them by replacing marker blocks with actual values. -""" - -import re -from collections import Counter -from importlib import resources - - -def load_template(name: str) -> str: - """Load template file content from Buck resources. - - Templates are loaded from the resources bundled with this package via Buck's - select_accelerator mechanism: - - AMD builds: hipified templates with HIP APIs - - NVIDIA builds: original templates with CUDA APIs - - Args: - name: Template filename (e.g., 'kernel.cpp', 'embedded_cubins.cpp'). - - Returns: - The template file content as a string. - """ - return resources.files(__package__).joinpath(name).read_text() - - -def render_template(template: str, replacements: dict[str, str]) -> str: - """Replace block markers in template with actual values. - - Replaces content between "// __TRITON_AOT_GENERATE_BEGIN__ NAME" - and "// __TRITON_AOT_GENERATE_END__ NAME" with the value for key "NAME". - Each key must have exactly one BEGIN/END pair in the template. - The markers are preserved for easier debugging. - - Args: - template: Template string containing marker blocks. - replacements: Dict mapping marker names to replacement values. - - Returns: - Rendered template with all marker blocks replaced. - - Raises: - AssertionError: If markers are duplicated, mismatched, or keys don't match. - """ - BEGIN_PREFIX = "// __TRITON_AOT_GENERATE_BEGIN__ " - END_PREFIX = "// __TRITON_AOT_GENERATE_END__ " - - begin_keys = re.findall(r"// __TRITON_AOT_GENERATE_BEGIN__ (\w+)", template) - end_keys = re.findall(r"// __TRITON_AOT_GENERATE_END__ (\w+)", template) - - # Check for duplicate keys - begin_key_counts = Counter(begin_keys) - end_key_counts = Counter(end_keys) - for key, count in begin_key_counts.items(): - assert count == 1, f"Duplicate BEGIN marker for key: {key}" - for key, count in end_key_counts.items(): - assert count == 1, f"Duplicate END marker for key: {key}" - - # Check BEGIN and END keys match - template_keys = set(begin_keys) - assert template_keys == set(end_keys), ( - f"Mismatched BEGIN/END markers: BEGIN={template_keys}, END={set(end_keys)}" - ) - - # Validate keys match between template and replacements - replacement_keys = set(replacements.keys()) - assert template_keys == replacement_keys, ( - f"Keys mismatch: in template but not in replacements: {template_keys - replacement_keys}, " - f"in replacements but not in template: {replacement_keys - template_keys}" - ) - - # Do the replacements - result = template - for key, value in replacements.items(): - begin_marker = f"{BEGIN_PREFIX}{key}" - end_marker = f"{END_PREFIX}{key}" - - begin_idx = result.find(begin_marker) - newline_idx = result.find("\n", begin_idx) - assert newline_idx != -1, ( - f"BEGIN marker for key '{key}' must be followed by newline" - ) - content_start = newline_idx + 1 - end_idx = result.find(end_marker, begin_idx) - - result = result[:content_start] + value + result[end_idx:] - - return result diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/templates/torch_op.cpp b/recommendation_v4/generative_recommenders/ops/triton_aot/templates/torch_op.cpp deleted file mode 100644 index c2e5a4063..000000000 --- a/recommendation_v4/generative_recommenders/ops/triton_aot/templates/torch_op.cpp +++ /dev/null @@ -1,22 +0,0 @@ -// __TRITON_AOT_GENERATE_BEGIN__ HEADER_INCLUDE -#include "kernel.h" -// __TRITON_AOT_GENERATE_END__ HEADER_INCLUDE -#include -#include // NOLINT(facebook-unused-include-check) - -// __TRITON_AOT_GENERATE_BEGIN__ TORCH_OP -namespace { -// no-op, force link StableLibrary -torch::stable::Tensor _triton_aot_placeholder_noop( - torch::stable::Tensor input) { - return input; -} -} // namespace - -STABLE_TORCH_LIBRARY_FRAGMENT(triton_aot, m) { - m.def("_placeholder_noop(Tensor input) -> Tensor"); -} -STABLE_TORCH_LIBRARY_IMPL(triton_aot, CPU, m) { - m.impl("_placeholder_noop", TORCH_BOX(&_triton_aot_placeholder_noop)); -} -// __TRITON_AOT_GENERATE_END__ TORCH_OP diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/transform/import_utils.py b/recommendation_v4/generative_recommenders/ops/triton_aot/transform/import_utils.py deleted file mode 100644 index e3c4f1955..000000000 --- a/recommendation_v4/generative_recommenders/ops/triton_aot/transform/import_utils.py +++ /dev/null @@ -1,89 +0,0 @@ -# pyre-strict - -""" -Import-header utilities for triton_aot codegen. -""" - -import ast - -from torch import package - - -def get_original_import_header(source_code: str) -> str: - """Extract all import statements from *source_code* as a single string.""" - tree = ast.parse(source_code) - import_header = "" - for node in ast.walk(tree): - if isinstance(node, ast.ImportFrom): - import_header += ast.unparse(node) + "\n" - elif isinstance(node, ast.Import): - import_header += ast.unparse(node) + "\n" - return import_header - - -def _is_extern_module(module_name: str, extern_modules: set[str]) -> bool: - """Return True if *module_name* (or a parent) is in the extern set.""" - if module_name in extern_modules: - return True - parts = module_name.split(".") - for i in range(1, len(parts)): - if ".".join(parts[:i]) in extern_modules: - return True - return False - - -def rewrite_package_imports( - import_header: str, - package_importer: package.PackageImporter, -) -> str: - """Rewrite interned imports to use ``_package_importer``. - - Extern modules (``torch``, ``typing``, …) keep regular ``import`` - statements. Interned modules (for example, local - ``generative_recommenders.*`` modules) are rewritten to:: - - _pkg_mod = _package_importer.import_module( - 'generative_recommenders.ops.triton.triton_utils' - ) - helper = _pkg_mod.helper - - The ``_package_importer`` object is injected into the wrapper module's - namespace by ``replace_kernels`` before ``exec_module`` is called. - """ - extern_modules = set(package_importer.extern_modules) - header_tree = ast.parse(import_header) - - regular: list[str] = [] - from_package: list[str] = [] - - for node in header_tree.body: - if isinstance(node, ast.Import): - for alias in node.names: - if _is_extern_module(alias.name, extern_modules): - regular.append(ast.unparse(node)) - else: - local = alias.asname or alias.name - from_package.append( - f"{local} = _package_importer.import_module('{alias.name}')" - ) - elif isinstance(node, ast.ImportFrom): - mod = node.module or "" - if _is_extern_module(mod, extern_modules): - regular.append(ast.unparse(node)) - else: - var = f"_pkg_{mod.replace('.', '_')}" - from_package.append(f"{var} = _package_importer.import_module('{mod}')") - for alias in node.names: - local = alias.asname or alias.name - from_package.append(f"{local} = {var}.{alias.name}") - else: - # Non-import statement (should not appear, but preserve if it does) - regular.append(ast.unparse(node)) - - parts: list[str] = [] - if regular: - parts.append("\n".join(regular)) - if from_package: - parts.append("# Imports resolved from torch package via _package_importer") - parts.append("\n".join(from_package)) - return "\n".join(parts) + "\n" if parts else "" diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/transform/kernel_wrapper_codegen.py b/recommendation_v4/generative_recommenders/ops/triton_aot/transform/kernel_wrapper_codegen.py deleted file mode 100644 index 51d144f1b..000000000 --- a/recommendation_v4/generative_recommenders/ops/triton_aot/transform/kernel_wrapper_codegen.py +++ /dev/null @@ -1,500 +0,0 @@ -# pyre-strict -import ast -import inspect -import os -from typing import Any, Callable, Dict, List, Optional - -from generative_recommenders.ops.triton_aot.compile.compile_state import ( - get_aott_compile_path, - get_triton_aot_kernel_specs, -) -from generative_recommenders.ops.triton_aot.compile.utils import unwrap_heuristic -from generative_recommenders.ops.triton_aot.shared.compat import get_kernel_name -from generative_recommenders.ops.triton_aot.transform.import_utils import ( - get_original_import_header, - rewrite_package_imports, -) -from generative_recommenders.ops.triton_aot.types import TritonAOT -from pyre_extensions import none_throws -from torch import package -from torch.fx import GraphModule - -# @manual=//triton:triton -from triton.runtime.autotuner import Autotuner -from triton.runtime.jit import JITFunction, KernelInterface - - -def _is_torch_package_module(module_name: str) -> bool: - """Check if a module name is from torch.package namespace.""" - return module_name.startswith(" str: - """Strip the torch.package namespace prefix from a module name. - - Example: - '.generative_recommenders.ops.triton_aot.triton_layer_norm' - -> 'generative_recommenders.ops.triton_aot.triton_layer_norm' - """ - if _is_torch_package_module(module_name): - # Remove '.' prefix - return module_name.split(".", 1)[1] - return module_name - - -def _get_clean_module_basename(module_name: str) -> str: - """Get the basename of a module, stripping torch.package prefix if present. - - Example: - '.generative_recommenders.ops.triton_aot.triton_layer_norm' - -> 'triton_layer_norm' - 'generative_recommenders.ops.triton_aot.triton_layer_norm' - -> 'triton_layer_norm' - """ - clean_name = _strip_torch_package_prefix(module_name) - return clean_name.rsplit(".", 1)[-1] - - -def _extract_function_source(module_source: str, fn_name: str) -> str: - """Extract a function's source code from module source. - - Parses the module source and extracts just the function definition. - """ - tree = ast.parse(module_source) - for node in ast.walk(tree): - if isinstance(node, ast.FunctionDef) and node.name == fn_name: - return ast.unparse(node) - raise ValueError(f"Function '{fn_name}' not found in module source") - - -def _get_module_and_source( - target: Callable[..., Any], - package_importer: Optional[package.PackageImporter], -) -> tuple[Any, str, str]: - """Get module, module source, and function source for a callable. - - Handles both regular modules and torch.package loaded modules. - - Args: - target: The callable (function) to get source for - package_importer: Optional PackageImporter for torch.package modules - - Returns: - Tuple of (module, module_source, function_source) - """ - module_name = target.__module__ - fn_name = target.__name__ - - if _is_torch_package_module(module_name) and package_importer is not None: - # Handle torch.package namespace - real_module_name = _strip_torch_package_prefix(module_name) - assert real_module_name.startswith( - "generative_recommenders.ops.triton_aot" - ) or real_module_name.startswith("prime_perf_optimizer"), ( - f"Expected module under 'generative_recommenders.ops.triton_aot' or 'prime_perf_optimizer', got: {real_module_name}" - ) - - # Get module source from package - module_source = package_importer.get_source(real_module_name) - - # Import the module through the package importer - fn_module = package_importer.import_module(real_module_name) - - # Extract function source from module source - fn_source = _extract_function_source(module_source, fn_name) - - return fn_module, module_source, fn_source - else: - # Standard module handling - fn_module = inspect.getmodule(target) - module_source = inspect.getsource(none_throws(fn_module)) - fn_source = inspect.getsource(target) - - return fn_module, module_source, fn_source - - -def _calls_triton_aot_kernel(node: ast.FunctionDef, kernel_name: str) -> bool: - """ - kernel_name is the JIT function name (e.g. "_weighted_layer_norm_fwd"), - which may differ from the wrapper function name (e.g. - "_triton_aot_swish_layer_norm"). We match by looking for a - Subscript-call ``kernel_name[grid](...)`` inside the function body. - """ - for child in ast.walk(node): - if ( - isinstance(child, ast.Call) - and isinstance(child.func, ast.Subscript) - and isinstance(child.func.value, ast.Name) - and child.func.value.id == kernel_name - ): - return True - return False - - -def _is_torch_jit_unused(d: ast.expr) -> bool: - """Check if a decorator AST node represents @torch.jit.unused.""" - return ( - isinstance(d, ast.Attribute) - and d.attr == "unused" - and isinstance(d.value, ast.Attribute) - and d.value.attr == "jit" - and isinstance(d.value.value, ast.Name) - and d.value.value.id == "torch" - ) - - -def strip_jit_unused_decorator( - node: ast.FunctionDef, kernel_name: str -) -> ast.FunctionDef: - """Strip @torch.jit.unused if the function body calls ``kernel_name[grid](...)``. - - kernel_name is the TritonAOT kernel's JIT function name (e.g. - ``_weighted_layer_norm_fwd``), not the wrapper function name. This avoids - relying on a naming convention on the wrapper function itself. - """ - if _calls_triton_aot_kernel(node, kernel_name): - node.decorator_list = [ - d for d in node.decorator_list if not _is_torch_jit_unused(d) - ] - return node - - -class TritonAOTOperatorTransform(ast.NodeTransformer): - def __init__(self, kernel: Any) -> None: - super().__init__() - self._kernel: Any = kernel - self._kernel_jit_fn: JITFunction[List[Any]] = unwrap_heuristic( - kernel, return_type=JITFunction - ) - self._kernel_autotuner: Optional[Autotuner] = unwrap_heuristic( - kernel, return_type=Autotuner - ) - self._kernel_name: str = get_kernel_name(self._kernel_jit_fn) - # Only transform the function body - self._autotune_params: List[str] = ( - list(list(self._kernel_autotuner.cache.values())[0].kwargs.keys()) - if self._kernel_autotuner is not None - else [] - ) - self._autotune_params += ["num_warps", "num_stages"] - - self._lambda_arg_name: Optional[str] = None - self._grid_name: Optional[str] = None - self._autotune_key_id: Optional[Dict[str, int]] = None - self._autotune_key_map: Optional[Dict[str, ast.expr]] = None - self._kernel_meta: Optional[ast.Assign] = None - - if self._kernel_autotuner is not None: - autotune_key_id: Dict[str, int] = {} - self._autotune_key_id = autotune_key_id - # pyre-ignore[16]: JITFunction has arg_names at runtime - for key in self._kernel_autotuner.keys: - autotune_key_id[key] = self._kernel_jit_fn.arg_names.index(key) - - def generate_function_meta(self) -> None: - targets = [ - ast.Name(id=param, ctx=ast.Store()) for param in self._autotune_params - ] - autotune_key_map = self._autotune_key_map - kernel_autotuner = self._kernel_autotuner - call = ast.Call( - func=ast.Name(id=f"{self._kernel_name}_meta", ctx=ast.Load()), - args=[ - none_throws(autotune_key_map)[key] - for key in none_throws(kernel_autotuner).keys - ] - if kernel_autotuner is not None - else [], - keywords=[], - ) - self._kernel_meta = ast.Assign( - # pyre-ignore[6]: ast.Assign targets type - targets=[ast.Tuple(elts=targets, ctx=ast.Store())], - value=call, - ) - - def visit_FunctionDef(self, node: ast.FunctionDef) -> ast.FunctionDef: - strip_jit_unused_decorator(node, self._kernel_name) - - new_body: List[ast.stmt] = [] - stmts = node.body - for stmt in stmts: - if isinstance(stmt, ast.Assign): - for target in stmt.targets: - if isinstance(target, ast.Name) and target.id == self._grid_name: - assert self._kernel_meta is not None - self._kernel_meta.lineno = stmt.lineno - new_body.append(self._kernel_meta) - new_body.append(self.visit(stmt)) - node.body = new_body - return node - - def visit_Assign(self, node: ast.Assign) -> ast.Assign: - for target in node.targets: - if isinstance(target, ast.Name) and isinstance(node.value, ast.Lambda): - lambda_node = node.value - self._lambda_arg_name = lambda_node.args.args[0].arg - lambda_body = lambda_node.body - assert isinstance(lambda_body, ast.Tuple) - new_elts: List[ast.expr] = [] - for elt in lambda_body.elts: - new_elts.append(self.visit(elt)) - node.value = ast.Tuple(elts=new_elts, ctx=ast.Load()) - self._lambda_arg_name = None - return node - - def visit_Subscript(self, node: ast.Subscript) -> ast.expr: - if isinstance(node.value, ast.Name) and node.value.id == self._lambda_arg_name: - assert isinstance(node.slice, ast.Constant) - assert isinstance(node.slice.value, str) - var_name = node.slice.value - # pyre-ignore - node = ast.Name(id=var_name, ctx=ast.Load()) - return node - - def visit_Expr(self, node: ast.Expr) -> ast.Expr: - if isinstance(node.value, ast.Call): - call = node.value - if ( - isinstance(call.func, ast.Subscript) - and isinstance(call.func.value, ast.Name) - and call.func.value.id == self._kernel_name - ): - grid_arg = call.func.slice - new_func = ast.Attribute( - value=ast.Attribute( - value=ast.Attribute( - value=ast.Name(id="torch", ctx=ast.Load()), - attr="ops", - ctx=ast.Load(), - ), - attr="triton_aot", - ctx=ast.Load(), - ), - attr=self._kernel_name, - ctx=ast.Load(), - ) - new_args = [grid_arg] + call.args - new_keywords = call.keywords + [ - ast.keyword(arg=param, value=ast.Name(id=param, ctx=ast.Load())) - for param in self._autotune_params - ] - node.value = ast.Call( - func=new_func, - args=new_args, - keywords=new_keywords, - ) - return node - - def contains_triton_call(self, node: ast.AST) -> bool: - for child in ast.walk(node): - if ( - isinstance(child, ast.Call) - and isinstance(child.func, ast.Subscript) - # pyre-ignore[16]: ast.expr may have `id` attribute at runtime - and child.func.value.id == self._kernel_name - ): - # pyrefly: ignore [missing-attribute] - self._grid_name = child.func.slice.id - - if self._kernel_autotuner is not None: - autotune_key_map: Dict[str, ast.expr] = {} - self._autotune_key_map = autotune_key_map - # pyre-ignore[16]: Autotuner has keys at runtime - for key in self._kernel_autotuner.keys: - found_key = False - for keyword in child.keywords: - if keyword.arg == key: - autotune_key_map[key] = keyword.value - found_key = True - break - - if not found_key: - autotune_key_id = self._autotune_key_id - assert autotune_key_id is not None - assert key in autotune_key_id - key_id = autotune_key_id[key] - autotune_key_map[key] = child.args[key_id] - - self.generate_function_meta() - return True - return False - - def contains_lambda(self, node: ast.AST) -> bool: - for child in ast.walk(node): - if isinstance(child, ast.Lambda): - return True - return False - - def _get_grid_name(self, node: ast.AST) -> Optional[str]: - for child in ast.walk(node): - if ( - isinstance(child, ast.Call) - and isinstance(child.func, ast.Subscript) - # pyre-ignore[16]: ast.expr may have `id` attribute at runtime - and child.func.value.id == self._kernel_name - ): - # pyrefly: ignore [missing-attribute] - return child.func.slice.id - return None - - def generate_so_loading_code( - self, - node: ast.AST, - abs_triton_aot_path: str, - ) -> str: - """Return auto-generated code to load the compiled kernel at runtime. - - If *node* contains a call to this transformer's kernel, returns - ``import importlib.util`` + meta-module loading + ``torch.ops.load_library`` - code. Otherwise returns an empty string. - - This method also sets up internal transformer state (grid name, - autotune key map, etc.) via ``contains_triton_call`` as a side effect. - - Example for _addmm_fwd kernel: - kernel_dir = "triton_addmm__addmm_fwd" - meta_module_path = "/path/to/triton_aot_compile/triton_addmm__addmm_fwd/_addmm_fwd_meta.py" - so_path = "/path/to/triton_aot_compile/triton_addmm__addmm_fwd/addmm_fwd.so" - """ - if not self.contains_triton_call(node): - return "" - - kernel_dir = f"{_get_clean_module_basename(self._kernel_jit_fn.__module__)}_{self._kernel_name}" - - meta_module_path = os.path.join( - abs_triton_aot_path, kernel_dir, f"{self._kernel_name}_meta.py" - ) - - so_path = os.path.join( - abs_triton_aot_path, - kernel_dir, - f"{self._kernel_name.lstrip('_')}.so", - ) - - return f""" -# Auto-generated by triton_aot.kernel_wrapper_codegen -import importlib.util -_meta_spec = importlib.util.spec_from_file_location("{self._kernel_name}_meta", "{meta_module_path}") -_meta_module = importlib.util.module_from_spec(_meta_spec) -_meta_spec.loader.exec_module(_meta_module) -{self._kernel_name}_meta = _meta_module.{self._kernel_name}_meta - -torch.ops.load_library("{so_path}") -""" - - -def _find_triton_aot_kernel( - node_target: Any, - kernel_specs: Dict[KernelInterface[List[Any]], List[Dict[str, List[Any]]]], -) -> Optional[TritonAOT]: - """Find the single TritonAOT kernel referenced in a node target's globals. - - Scans ``node_target.__globals__`` for ``TritonAOT`` instances, validates - that every instance appears in *kernel_specs*, and asserts at most one - kernel is present (per the one-kernel-per-wrapper invariant). - - Returns the kernel, or ``None`` if the function references no kernels. - """ - kernels: set[TritonAOT] = set() - for _, var in node_target.__globals__.items(): - if isinstance(var, TritonAOT): - if var.fn in kernel_specs: - kernels.add(var) - else: - raise RuntimeError( - f"Cannot find TritonAOT kernel {var.fn} in TRITON_AOT_KERNEL_SPECS" - ) - - if len(kernels) == 0: - return None - - fn_name = node_target.__name__ - assert len(kernels) == 1, ( - f"Expected exactly 1 kernel per wrapper function '{fn_name}', " - f"got {len(kernels)}" - ) - (kernel_obj,) = kernels - return kernel_obj - - -def _generate_wrapper_files( - node_target: Any, - kernel: TritonAOT, - compile_path: str, - package_importer: Optional[package.PackageImporter], -) -> None: - """Generate ``_original.py`` and ``_wrapper.py`` for a single kernel. - - Creates a per-kernel subdirectory under *compile_path*, writes the - original function source, then AST-transforms the wrapper to replace - ``kernel[grid](...)`` with ``torch.ops.triton_aot.*`` calls. - """ - fn_name = node_target.__name__ - - jit_fn = none_throws( - unwrap_heuristic(kernel, return_type=JITFunction), - f"Failed to unwrap kernel to JITFunction: {kernel}", - ) - kernel_dir = ( - f"{_get_clean_module_basename(jit_fn.__module__)}_{get_kernel_name(jit_fn)}" - ) - output_dir = os.path.join(compile_path, kernel_dir) - os.makedirs(output_dir, exist_ok=True) - - _, module_code, wrapper_code = _get_module_and_source(node_target, package_importer) - import_header = get_original_import_header(module_code) - - with open(os.path.join(output_dir, f"{fn_name}_original.py"), "w") as f: - f.write(import_header) - f.write(wrapper_code) - - # When source comes from a torch package, rewrite interned - # imports to use _package_importer, which - # is injected by replace_kernels at load time. Must happen - # before auto-generated code is appended (so stdlib imports - # like ``import importlib.util`` are not touched). - if package_importer is not None: - import_header = rewrite_package_imports(import_header, package_importer) - - tree = ast.parse(wrapper_code) - transformer = TritonAOTOperatorTransform(kernel=kernel) - import_header += transformer.generate_so_loading_code(tree, compile_path) - tree = transformer.visit(tree) - - new_source_code = ast.unparse(tree) - - with open(os.path.join(output_dir, f"{fn_name}_wrapper.py"), "w") as f: - f.write(import_header) - f.write(new_source_code) - - -def kernel_wrapper_codegen( - module: GraphModule, packageImporter: package.PackageImporter | None = None -) -> None: - """ - Generate wrapper files for TritonAOT kernels. - Requirement: under wrapper.py, @triton.jit kernel/func is imported without 'as' alias. - - For each function containing TritonAOT kernels, generates: - - {fn_name}_original.py: Original source code with imports - - {fn_name}_wrapper.py: Transformed wrapper that uses torch.ops.triton_aot - """ - compile_path = get_aott_compile_path() - if not os.path.exists(compile_path): - os.makedirs(compile_path) - - transformed_ops: set[Callable[..., Any]] = set() - kernel_specs = get_triton_aot_kernel_specs() - for node in module.graph.nodes: - if node.op == "call_function" and hasattr(node.target, "__globals__"): - if node.target not in transformed_ops: - transformed_ops.add(node.target) - else: - continue - - kernel = _find_triton_aot_kernel(node.target, kernel_specs) - if kernel is not None: - _generate_wrapper_files( - node.target, kernel, compile_path, packageImporter - ) diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/transform/replace_kernels.py b/recommendation_v4/generative_recommenders/ops/triton_aot/transform/replace_kernels.py deleted file mode 100644 index 53e5d83e9..000000000 --- a/recommendation_v4/generative_recommenders/ops/triton_aot/transform/replace_kernels.py +++ /dev/null @@ -1,137 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files -# (the "Software"), to deal in the Software without restriction, -# including without limitation the rights to use, copy, modify, merge, -# publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, -# subject to the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -#!/usr/bin/env python3 - -# pyre-strict - -import importlib.util -import logging -import os -import sys -from typing import Any, Dict, Optional - -from generative_recommenders.ops.triton_aot.compile.compile_state import ( - get_aott_compile_path, -) -from torch import package -from torch.fx import GraphModule - -logger: logging.Logger = logging.getLogger(__name__) - - -def _find_wrapper_files( - compile_path: str, -) -> list[tuple[str, str, str]]: - """Find all ``*_wrapper.py`` files under *compile_path*. - - Walks one level deep into kernel subdirectories and returns a list of - ``(wrapper_name, fn_name, wrapper_path)`` tuples. - """ - results: list[tuple[str, str, str]] = [] - for dirpath, dirnames, filenames in os.walk(compile_path): - if dirpath != compile_path: - dirnames.clear() # only recurse one level into kernel subdirs - for item in filenames: - if item.endswith("_wrapper.py"): - wrapper_name = item.removesuffix(".py") - fn_name = wrapper_name.removesuffix("_wrapper") - wrapper_path = os.path.join(dirpath, item) - results.append((wrapper_name, fn_name, wrapper_path)) - return results - - -def _load_wrapper_module( - wrapper_name: str, - fn_name: str, - wrapper_path: str, - package_importer: Optional[package.PackageImporter], -) -> Optional[Any]: - """Dynamically import a single ``*_wrapper.py`` and return its wrapper callable. - - Returns ``None`` if the module does not expose a function named *fn_name*. - """ - spec = importlib.util.spec_from_file_location(wrapper_name, wrapper_path) - assert spec is not None, f"Failed to create spec for {wrapper_path}" - assert spec.loader is not None, f"Spec has no loader for {wrapper_path}" - - loader = spec.loader - wrapper_module = importlib.util.module_from_spec(spec) - - sys.modules[wrapper_name] = wrapper_module - - if package_importer is not None: - wrapper_module._package_importer = package_importer # type: ignore[attr-defined] - - loader.exec_module(wrapper_module) - - if hasattr(wrapper_module, fn_name): - return getattr(wrapper_module, fn_name) - return None - - -def replace_kernels( - fx_m: GraphModule, - eager: bool = False, - package_importer: Optional[package.PackageImporter] = None, -) -> GraphModule: - if eager: - raise NotImplementedError( - "Local generative_recommenders AOT-T transform does not support " - "eager replacement." - ) - - compile_path = get_aott_compile_path() - assert os.path.exists(compile_path), "triton_aot_compile dir does not exist" - - wrapper_dict: Dict[str, Any] = {} - for wrapper_name, fn_name, wrapper_path in _find_wrapper_files(compile_path): - wrapper_fn = _load_wrapper_module( - wrapper_name, fn_name, wrapper_path, package_importer - ) - if wrapper_fn is not None: - wrapper_dict[fn_name] = wrapper_fn - - logger.info(f"replace_kernels: {wrapper_dict=}") - - # Phase 2: Replace FX graph nodes - # Walk the FX graph, find call_function nodes whose target name - # matches a loaded wrapper, and swap the target so that - # kernel[grid](...) calls become torch.ops.triton_aot.* calls. - replaced_count = 0 - for nodes in fx_m.graph.nodes: - if nodes.op == "call_function" and nodes.target.__name__ in wrapper_dict.keys(): - logger.info( - f"Replaced node: {nodes.op} {nodes.target} -> {wrapper_dict[nodes.target.__name__]} {nodes.meta}" - ) - nodes.target = wrapper_dict[nodes.target.__name__] - replaced_count += 1 - - assert replaced_count > 0, ( - f"No ops were replaced with triton_aot wrappers. " - f"wrapper_dict={wrapper_dict}, compile_path={compile_path}" - ) - logger.info( - f"Successfully replaced {replaced_count} op(s) with triton_aot wrappers." - ) - - fx_m.recompile() - return fx_m diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/transform/transform_kernels.py b/recommendation_v4/generative_recommenders/ops/triton_aot/transform/transform_kernels.py deleted file mode 100644 index c2be78989..000000000 --- a/recommendation_v4/generative_recommenders/ops/triton_aot/transform/transform_kernels.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# pyre-strict - -from typing import Optional - -from generative_recommenders.ops.triton_aot.transform.kernel_wrapper_codegen import ( - kernel_wrapper_codegen, -) -from generative_recommenders.ops.triton_aot.transform.replace_kernels import ( - replace_kernels, -) -from torch import package -from torch.fx import GraphModule - - -def transform_kernels( - fx_m: GraphModule, - eager: bool = False, - package_importer: Optional[package.PackageImporter] = None, -) -> GraphModule: - """Generate AOT wrappers and replace FX graph nodes in one step. - - 1. kernel_wrapper_codegen: AST-transforms wrapper functions, - rewrites kernel[grid](...) -> torch.ops.triton_aot.kernel(...), - writes {fn}_wrapper.py - 2. replace_kernels: loads wrappers and replaces graph node targets - """ - kernel_wrapper_codegen(fx_m, package_importer) - return replace_kernels(fx_m, eager=eager, package_importer=package_importer) diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/triton_addmm.py b/recommendation_v4/generative_recommenders/ops/triton_aot/triton_addmm.py deleted file mode 100644 index b71ec8144..000000000 --- a/recommendation_v4/generative_recommenders/ops/triton_aot/triton_addmm.py +++ /dev/null @@ -1,347 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files -# (the "Software"), to deal in the Software without restriction, -# including without limitation the rights to use, copy, modify, merge, -# publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, -# subject to the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -# pyre-strict -# pyre-ignore-all-errors[2]: Triton has its own type system on func's input - -#!/usr/bin/env python3 - - -from typing import Any, List, Tuple - -import torch - -# @manual=//triton:triton -import triton - -# @manual=//triton:triton -import triton.language as tl -from generative_recommenders.common import ( - BACKEND_ALLOW_TF32, - cdiv, - should_trigger_eager_impl, -) -from generative_recommenders.ops.triton_aot.types import triton_aot - - -def get_mm_configs() -> List[triton.Config]: - return [ - triton.Config( - { - "BLOCK_M": 32, - "BLOCK_N": 64, - "BLOCK_K": 32, - "GROUP_M": 8, - }, - num_stages=5, - num_warps=2, - ), - triton.Config( - { - "BLOCK_M": 128, - "BLOCK_N": 256, - "BLOCK_K": 64, - "GROUP_M": 8, - }, - num_stages=3, - num_warps=8, - ), - triton.Config( - { - "BLOCK_M": 64, - "BLOCK_N": 256, - "BLOCK_K": 32, - "GROUP_M": 8, - }, - num_stages=4, - num_warps=4, - ), - triton.Config( - { - "BLOCK_M": 128, - "BLOCK_N": 128, - "BLOCK_K": 32, - "GROUP_M": 8, - }, - num_stages=4, - num_warps=4, - ), - triton.Config( - { - "BLOCK_M": 128, - "BLOCK_N": 64, - "BLOCK_K": 32, - "GROUP_M": 8, - }, - num_stages=4, - num_warps=4, - ), - triton.Config( - { - "BLOCK_M": 64, - "BLOCK_N": 128, - "BLOCK_K": 32, - "GROUP_M": 8, - }, - num_stages=4, - num_warps=4, - ), - triton.Config( - { - "BLOCK_M": 128, - "BLOCK_N": 32, - "BLOCK_K": 32, - "GROUP_M": 8, - }, - num_stages=4, - num_warps=4, - ), - triton.Config( - { - "BLOCK_M": 64, - "BLOCK_N": 32, - "BLOCK_K": 32, - "GROUP_M": 8, - }, - num_stages=5, - num_warps=2, - ), - ] - - -@triton_aot( - annotations={ - "M": "i32", - "N": ("i32", 16), - "K": ("i32", 16), - "stride_xm": ("i32", 16), - "stride_xk": ("i32", 1), - "stride_wk": ("i32", 16), - "stride_wn": ("i32", 1), - "stride_ym": ("i32", 16), - "stride_yn": ("i32", 1), - "stride_zm": ("i32", 16), - "stride_zn": ("i32", 1), - }, -) -# pyre-ignore[56]: Pyre cannot infer triton.autotune decorator type -@triton.autotune( - configs=get_mm_configs(), - key=["N", "K"], -) -@triton.jit -def _addmm_fwd( - x_ptr, - w_ptr, - y_ptr, - z_ptr, - M, - N, - K, - stride_xm, - stride_xk, - stride_wk, - stride_wn, - stride_ym, - stride_yn, - stride_zm, - stride_zn, - BLOCK_M: tl.constexpr, - BLOCK_N: tl.constexpr, - BLOCK_K: tl.constexpr, - GROUP_M: tl.constexpr, - ALLOW_TF32: tl.constexpr, - BROADCAST_Y: tl.constexpr, -) -> None: - pid_0, pid_1 = tl.program_id(axis=0), tl.program_id(axis=1) - pid = pid_0 * tl.num_programs(axis=1) + pid_1 - num_pid_m = tl.cdiv(M, BLOCK_M) - num_pid_n = tl.cdiv(N, BLOCK_N) - num_pid_in_group = GROUP_M * num_pid_n - group_id = pid // num_pid_in_group - first_pid_m = group_id * GROUP_M - group_size_m = min(num_pid_m - first_pid_m, GROUP_M) - pid_m = first_pid_m + (pid % group_size_m) - pid_n = (pid % num_pid_in_group) // group_size_m - - offs_m = tl.arange(0, BLOCK_M) - offs_k = tl.arange(0, BLOCK_K) - offs_n = tl.arange(0, BLOCK_N) - mask_m = (pid_m * BLOCK_M + offs_m)[:, None] < M - mask_n = (pid_n * BLOCK_N + offs_n)[None, :] < N - x_ptr += pid_m.to(tl.int64) * BLOCK_M * stride_xm - x_ptrs = x_ptr + (offs_m[:, None] * stride_xm + offs_k[None, :] * stride_xk) - w_ptr += pid_n.to(tl.int64) * BLOCK_N * stride_wn - w_ptrs = w_ptr + (offs_k[:, None] * stride_wk + offs_n[None, :] * stride_wn) - accumulator = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32) - for k in range(0, tl.cdiv(K, BLOCK_K)): - mask_k = offs_k[None, :] < K - k * BLOCK_K - x = tl.load(x_ptrs, mask=mask_k & mask_m, other=0.0) - mask_k = offs_k[:, None] < K - k * BLOCK_K - w = tl.load(w_ptrs, mask=mask_k & mask_n, other=0.0) - accumulator += tl.dot(x, w, allow_tf32=ALLOW_TF32) - x_ptrs += BLOCK_K * stride_xk - w_ptrs += BLOCK_K * stride_wk - - z_mask = mask_m & mask_n - if BROADCAST_Y: - # y is a vector, broadcast to add to z - y_ptr += pid_n.to(tl.int64) * BLOCK_N * stride_yn - y_ptrs = y_ptr + stride_yn * offs_n[None, :] - y = tl.load(y_ptrs, mask=mask_n) - else: - y_ptr += pid_m.to(tl.int64) * BLOCK_M * stride_ym - y_ptr += pid_n.to(tl.int64) * BLOCK_N * stride_yn - y_ptrs = y_ptr + stride_ym * offs_m[:, None] + stride_yn * offs_n[None, :] - y = tl.load(y_ptrs, mask=z_mask) - z = (accumulator + y.to(tl.float32)).to(z_ptr.dtype.element_ty) - z_ptr += pid_m.to(tl.int64) * BLOCK_M * stride_zm - z_ptr += pid_n.to(tl.int64) * BLOCK_N * stride_zn - z_ptrs = z_ptr + stride_zm * offs_m[:, None] + stride_zn * offs_n[None, :] - tl.store(z_ptrs, z, mask=z_mask) - - -@torch.jit.unused -@torch.fx.wrap -def _triton_aot_addmm_fwd( - x: torch.Tensor, - w: torch.Tensor, - y: torch.Tensor, - allow_tf32: bool = BACKEND_ALLOW_TF32, -) -> torch.Tensor: - M, K = x.shape - KB, N = w.shape - assert K == KB, f"incompatible dimensions {K}, {KB}" - - is_y_1d = y.dim() == 1 - NY = y.shape[0] if is_y_1d else y.shape[1] - assert N == NY, f"incompatible dimensions {N}, {NY}" - - # Allocate output - z = torch.empty((M, N), device=x.device, dtype=x.dtype) - if M == 0 or N == 0: - return z - - grid = lambda meta: ( # noqa E731 - cdiv(M, meta["BLOCK_M"]), - cdiv(N, meta["BLOCK_N"]), - ) - - _addmm_fwd[grid]( - x, - w, - y, - z, - M, - N, - K, - x.stride(0), - x.stride(1), - w.stride(0), - w.stride(1), - y.stride(0) if not is_y_1d else 0, - y.stride(1) if not is_y_1d else y.stride(0), - z.stride(0), - z.stride(1), - ALLOW_TF32=allow_tf32, - BROADCAST_Y=is_y_1d, - ) - return z - - -def _triton_aot_addmm_fwd_eager( - x: torch.Tensor, - w: torch.Tensor, - y: torch.Tensor, -) -> torch.Tensor: - return torch.addmm(y, x, w) - - -@torch.fx.wrap -def _triton_aot_addmm_fwd_maybe_eager( - x: torch.Tensor, - w: torch.Tensor, - y: torch.Tensor, -) -> torch.Tensor: - if torch.jit.is_scripting(): - # call eager - return torch.addmm(y, x, w) - else: - return _triton_aot_addmm_fwd(x, w, y) - - -def triton_addmm_bwd( - x: torch.Tensor, - w: torch.Tensor, - dz: torch.Tensor, - is_y_1d: bool, -) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - if is_y_1d: - dy = torch.sum(dz, dim=0) - else: - dy = dz - dw = torch.mm(x.t(), dz) - dx = torch.mm(dz, w.t()) - - return dx, dw, dy - - -class _AddMmFunction(torch.autograd.Function): - @staticmethod - # pyre-ignore[14]: autograd.Function signature override - def forward( - ctx: Any, - x: torch.Tensor, - w: torch.Tensor, - y: torch.Tensor, - ) -> torch.Tensor: - ctx.save_for_backward(x, w) - ctx.is_y_1d = y.dim() == 1 - return _triton_aot_addmm_fwd(x, w, y) - - @staticmethod - # pyre-ignore[14]: autograd.Function signature override - def backward( - ctx: Any, dz: torch.Tensor - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - (x, w) = ctx.saved_tensors - return triton_addmm_bwd(x, w, dz, ctx.is_y_1d) - - -def triton_addmm( - input: torch.Tensor, - mat1: torch.Tensor, - mat2: torch.Tensor, -) -> torch.Tensor: - return _AddMmFunction.apply(mat1, mat2, input) - - -@torch.fx.wrap -def aot_triton_kernel_wrapper_addmm( - input: torch.Tensor, - mat1: torch.Tensor, - mat2: torch.Tensor, - allow_tf32: bool = BACKEND_ALLOW_TF32, -) -> torch.Tensor: - if should_trigger_eager_impl(): - return torch.addmm(input, mat1, mat2) - else: - return _triton_aot_addmm_fwd(mat1, mat2, input, allow_tf32=allow_tf32) diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/triton_concat_2d_jagged.py b/recommendation_v4/generative_recommenders/ops/triton_aot/triton_concat_2d_jagged.py deleted file mode 100644 index 8afc3c18e..000000000 --- a/recommendation_v4/generative_recommenders/ops/triton_aot/triton_concat_2d_jagged.py +++ /dev/null @@ -1,183 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files -# (the "Software"), to deal in the Software without restriction, -# including without limitation the rights to use, copy, modify, merge, -# publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, -# subject to the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -# pyre-strict - -from typing import Optional - -import torch -from generative_recommenders.common import ( - fx_unwrap_optional_tensor, - next_power_of_2, - should_trigger_eager_impl, -) -from generative_recommenders.ops.pytorch.pt_jagged import ( - pytorch_replace_last_n_with_jagged, -) -from generative_recommenders.ops.pytorch.pt_jagged_tensors import ( - pytorch_concat_2D_jagged, -) -from generative_recommenders.ops.triton.triton_jagged import concat_2D_jagged -from generative_recommenders.ops.triton_aot.types import triton_aot - - -concat_2D_jagged = triton_aot( - annotations={ - "DenseSize": "i32", - "D": "i32", - "stride_ad": "i32", - "stride_bd": "i32", - "stride_dense_batch": "i32", - "stride_od": "i32", - }, - # pyrefly: ignore [bad-argument-type] -)(concat_2D_jagged) - - -@torch.jit.unused -@torch.fx.wrap -def _triton_aot_concat_2D_jagged( - max_seq_len: int, - values_a: torch.Tensor, - values_b: torch.Tensor, - offsets_a: Optional[torch.Tensor] = None, - offsets_b: Optional[torch.Tensor] = None, - is_replace: bool = False, -) -> torch.Tensor: - is_dense_a = offsets_a is None - is_dense_b = offsets_b is None - - dense_size: int = 0 - if is_dense_a: - B, dense_size, D = values_a.size() - offsets_b = fx_unwrap_optional_tensor(offsets_b) - jagged_seq_len, _ = values_b.shape - values_out = torch.empty( - (dense_size * B + jagged_seq_len, D), - device=values_b.device, - dtype=values_b.dtype, - ) - offsets_a = offsets_b.new_empty(0) - stride_dense_batch = values_a.stride(0) - elif is_dense_b: - B, dense_size, D = values_b.size() - offsets_a = fx_unwrap_optional_tensor(offsets_a) - jagged_seq_len, _ = values_a.shape - values_out = torch.empty( - (jagged_seq_len + dense_size * B, D), - device=values_a.device, - dtype=values_a.dtype, - ) - offsets_b = offsets_a.new_empty(0) - stride_dense_batch = values_b.stride(0) - else: - offsets_a = fx_unwrap_optional_tensor(offsets_a) - offsets_b = fx_unwrap_optional_tensor(offsets_b) - B = offsets_a.size(0) - 1 - seq_len_a, D = values_a.shape - seq_len_b, _ = values_b.shape - if is_replace: - values_out = torch.empty_like(values_a) - else: - values_out = torch.empty( - (seq_len_a + seq_len_b, D), device=values_a.device, dtype=values_a.dtype - ) - stride_dense_batch = 0 - - # Make sure offsets are alignted on 16-byte to match AOTT spec - if ( - offsets_a is not None - and (offsets_a.storage_offset() * offsets_a.element_size()) % 16 != 0 - ): - offsets_a = offsets_a.clone() - if ( - offsets_b is not None - and (offsets_b.storage_offset() * offsets_b.element_size()) % 16 != 0 - ): - offsets_b = offsets_b.clone() - - BLOCK_D = next_power_of_2(D) - - grid = (max_seq_len, B) - # pyrefly: ignore [not-callable] - concat_2D_jagged[grid]( - OffsetsA=offsets_a, - ValuesA=values_a, - OffsetsB=offsets_b, - ValuesB=values_b, - DenseSize=dense_size, - Out=values_out, - D=D, - stride_ad=(values_a.stride(1) if is_dense_a else values_a.stride(0)), - stride_bd=(values_b.stride(1) if is_dense_b else values_b.stride(0)), - stride_dense_batch=stride_dense_batch, - stride_od=values_out.stride(0), - # pyrefly: ignore [bad-argument-type] - IS_DENSE_A=is_dense_a, - # pyrefly: ignore [bad-argument-type] - IS_DENSE_B=is_dense_b, - # pyrefly: ignore [bad-argument-type] - BLOCK_D=BLOCK_D, - # pyrefly: ignore [bad-argument-type] - IS_REPLACE=is_replace, - ) - return values_out - - -@torch.fx.wrap -# "aot_triton_kernel_wrapper_" is a pre-defined prefix for -# AOT-T triton kernel wrapper functions. This is required for -# AOT-T backend to recognize and trace correctly for ops transformation. -def aot_triton_kernel_wrapper_concat_2D_jagged( - max_seq_len: int, - values_a: torch.Tensor, - values_b: torch.Tensor, - offsets_a: Optional[torch.Tensor] = None, - offsets_b: Optional[torch.Tensor] = None, - is_replace: bool = False, -) -> torch.Tensor: - if should_trigger_eager_impl(): - if is_replace: - assert offsets_a is not None and offsets_b is not None - return pytorch_replace_last_n_with_jagged( - max_seq_len_left=max_seq_len, - offsets_left=offsets_a, - values_left=values_a, - offsets_right=offsets_b, - values_right=values_b, - ) - return pytorch_concat_2D_jagged( - values_left=values_a, - values_right=values_b, - max_len_left=max_seq_len if offsets_a is None else None, - max_len_right=max_seq_len if offsets_b is None else None, - offsets_left=offsets_a, - offsets_right=offsets_b, - ) - else: - return _triton_aot_concat_2D_jagged( - max_seq_len=max_seq_len, - values_a=values_a, - values_b=values_b, - offsets_a=offsets_a, - offsets_b=offsets_b, - is_replace=is_replace, - ) diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/triton_group_norm_mul_dropout.py b/recommendation_v4/generative_recommenders/ops/triton_aot/triton_group_norm_mul_dropout.py deleted file mode 100644 index 15c609a3c..000000000 --- a/recommendation_v4/generative_recommenders/ops/triton_aot/triton_group_norm_mul_dropout.py +++ /dev/null @@ -1,124 +0,0 @@ -# pyre-strict - -import torch -from generative_recommenders.common import next_power_of_2, should_trigger_eager_impl -from generative_recommenders.ops.pytorch.pt_hstu_linear import pytorch_norm_mul_dropout -from generative_recommenders.ops.triton.triton_hstu_linear import ( - _group_norm_mul_dropout_fwd, -) -from generative_recommenders.ops.triton_aot.types import triton_aot - -_group_norm_mul_dropout_fwd = triton_aot( - annotations={ - "D": ("i32", 16), - "eps": "fp32", - "seed": "i64", - "dropout_ratio": "fp32", - "stride_x": ("i32", 16), - "stride_u": ("i32", 16), - "stride_y": ("i32", 16), - }, - # pyrefly: ignore [bad-argument-type] -)(_group_norm_mul_dropout_fwd) - - -@torch.jit.unused -@torch.fx.wrap -def _triton_aot_group_norm_mul_dropout( - x: torch.Tensor, - u: torch.Tensor, - weight: torch.Tensor, - bias: torch.Tensor, - eps: float, - silu_u: bool, - concat_ux: bool, - num_heads: int, - linear_dim: int, -) -> torch.Tensor: - x = x.contiguous() - u = u.contiguous() - N, _ = x.shape - if concat_ux: - y = torch.empty((N, 3 * num_heads * linear_dim), dtype=x.dtype, device=x.device) - else: - y = torch.empty((N, num_heads * linear_dim), dtype=x.dtype, device=x.device) - mean = torch.empty((N * num_heads,), dtype=x.dtype, device=x.device) - rstd = torch.empty((N * num_heads,), dtype=x.dtype, device=x.device) - - BLOCK_D = next_power_of_2(linear_dim) - BLOCK_H = next_power_of_2(num_heads) - - seed = 0 - dropout_ratio = 0.0 - - grid = (N,) - # pyrefly: ignore [not-callable] - _group_norm_mul_dropout_fwd[grid]( - x, # X - u, # U - y, # Y - weight, # W - bias, # B - mean, # Mean - rstd, # Rstd - linear_dim, # D - num_heads, # Heads - eps, # eps - seed, # seed - dropout_ratio, # dropout_ratio - x.stride(0), # stride_x - u.stride(0), # stride_u - y.stride(0), # stride_y - # pyrefly: ignore [bad-argument-type] - SILU_U=silu_u, - # pyrefly: ignore [bad-argument-type] - BLOCK_D=BLOCK_D, - # pyrefly: ignore [bad-argument-type] - BLOCK_H=BLOCK_H, - # pyrefly: ignore [bad-argument-type] - TRAINING=False, - # pyrefly: ignore [bad-argument-type] - CONCAT_UX=concat_ux, - ) - return y - - -@torch.fx.wrap -def aot_triton_kernel_wrapper_group_norm_mul_dropout( - x: torch.Tensor, - u: torch.Tensor, - weight: torch.Tensor, - bias: torch.Tensor, - eps: float, - silu_u: bool, - concat_ux: bool, - num_heads: int, - linear_dim: int, -) -> torch.Tensor: - if should_trigger_eager_impl(): - return pytorch_norm_mul_dropout( - x=x, - u=u, - weight=weight, - bias=bias, - eps=eps, - dropout_ratio=0.0, - training=False, - silu_u=silu_u, - concat_u=concat_ux, - concat_x=concat_ux, - group_norm=True, - num_heads=num_heads, - linear_dim=linear_dim, - ) - return _triton_aot_group_norm_mul_dropout( - x=x, - u=u, - weight=weight, - bias=bias, - eps=eps, - silu_u=silu_u, - concat_ux=concat_ux, - num_heads=num_heads, - linear_dim=linear_dim, - ) diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/triton_layer_norm.py b/recommendation_v4/generative_recommenders/ops/triton_aot/triton_layer_norm.py deleted file mode 100644 index c5339033f..000000000 --- a/recommendation_v4/generative_recommenders/ops/triton_aot/triton_layer_norm.py +++ /dev/null @@ -1,119 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files -# (the "Software"), to deal in the Software without restriction, -# including without limitation the rights to use, copy, modify, merge, -# publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, -# subject to the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -# pyre-strict - -#!/usr/bin/env python3 - -import torch -from generative_recommenders.common import ( - cdiv, - next_power_of_2, - should_trigger_eager_impl, - switch_to_contiguous_if_needed, -) -from generative_recommenders.ops.pytorch.pt_layer_norm import ( - pytorch_layer_norm, - pytorch_swish_layer_norm, -) -from generative_recommenders.ops.triton.triton_layer_norm import ( - _weighted_layer_norm_fwd, -) -from generative_recommenders.ops.triton_aot.types import triton_aot - - -_weighted_layer_norm_fwd = triton_aot( - annotations={ - "N": "i32", - "D": ("i32", 16), - "stride_x": ("i32", 16), - "stride_y": ("i32", 16), - }, -)(_weighted_layer_norm_fwd) - - -@torch.jit.unused -@torch.fx.wrap -def _triton_aot_swish_layer_norm( - x: torch.Tensor, - weight: torch.Tensor, - bias: torch.Tensor, - eps: float, - is_swish: bool, -) -> torch.Tensor: - assert x.dim() == 2, f"x.dim() == {x.dim()}, expected 2" - x = switch_to_contiguous_if_needed(x) - N, D = x.shape - - assert weight.dim() == 1 - assert bias.dim() == 1 - assert weight.numel() == D - assert bias.numel() == D - - y = torch.empty_like(x) - - BLOCK_D = next_power_of_2(D) - - grid = lambda meta: ( # noqa E731 - cdiv(N, meta["BLOCK_N"]), - ) - # pyrefly: ignore [not-callable] - _weighted_layer_norm_fwd[grid]( - x, - y, - weight, - bias, - torch.empty(0, dtype=torch.float32), - torch.empty(0, dtype=torch.float32), - N, - D, - eps, - stride_x=x.stride(0), - stride_y=y.stride(0), - IS_SWISH=is_swish, - TRAINING=False, - BLOCK_D=BLOCK_D, - COMPUTE_MEAN_AND_RSTD=True, - ) - - return y - - -@torch.fx.wrap -# "aot_triton_kernel_wrapper_" is a pre-defined prefix for -# AOT-T triton kernel wrapper functions. This is required for -# AOT-T backend to recognize and trace correctly for ops transformation. -def aot_triton_kernel_wrapper_swish_layer_norm( - x: torch.Tensor, - weight: torch.Tensor, - bias: torch.Tensor, - eps: float, - is_swish: bool, -) -> torch.Tensor: - if should_trigger_eager_impl(): - if is_swish: - return pytorch_swish_layer_norm(x, [x.shape[1]], weight, bias, eps).to( - x.dtype - ) - else: - return pytorch_layer_norm(x, [x.shape[1]], weight, bias, eps).to(x.dtype) - else: - return _triton_aot_swish_layer_norm(x, weight, bias, eps, is_swish) diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/triton_layer_norm_mul_dropout.py b/recommendation_v4/generative_recommenders/ops/triton_aot/triton_layer_norm_mul_dropout.py deleted file mode 100644 index 7f7a6c743..000000000 --- a/recommendation_v4/generative_recommenders/ops/triton_aot/triton_layer_norm_mul_dropout.py +++ /dev/null @@ -1,162 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files -# (the "Software"), to deal in the Software without restriction, -# including without limitation the rights to use, copy, modify, merge, -# publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, -# subject to the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -# pyre-strict - -#!/usr/bin/env python3 - - -import torch -from generative_recommenders.common import next_power_of_2, should_trigger_eager_impl -from generative_recommenders.ops.pytorch.pt_hstu_linear import pytorch_norm_mul_dropout -from generative_recommenders.ops.triton.triton_hstu_linear import _ln_mul_dropout_fwd -from generative_recommenders.ops.triton_aot.types import triton_aot - -_ln_mul_dropout_fwd = triton_aot( - annotations={ - "D": ("i32", 16), - "stride_x": ("i32", 16), - "stride_u": ("i32", 16), - "stride_y": ("i32", 16), - }, - # pyrefly: ignore [bad-argument-type] -)(_ln_mul_dropout_fwd) - - -@torch.jit.unused -@torch.fx.wrap -def _triton_aot_layer_norm_mul_dropout( - x: torch.Tensor, - u: torch.Tensor, - weight: torch.Tensor, - bias: torch.Tensor, - eps: float, - dropout_ratio: float, - training: bool, - silu_u: bool, - concat_ux: bool, - mul_u_activation_type: str, -) -> torch.Tensor: - assert x.dim() == 2 - if x.stride(1) != 1: - x = x.contiguous() - N, D = x.shape - assert weight.dim() == 1 - assert bias.dim() == 1 - assert weight.numel() == D - assert bias.numel() == D - - if concat_ux: - y = torch.empty((N, 3 * D), dtype=x.dtype, device=x.device) - else: - y = torch.empty_like(x) - if N == 0: - return y - mean = x.new_empty((N,)) - rstd = x.new_empty((N,)) - - # Less than 64KB per feature: enqueue fused kernel - MAX_FUSED_SIZE = 65536 // x.element_size() - BLOCK_D = min(MAX_FUSED_SIZE, next_power_of_2(D)) - if D > BLOCK_D: - raise RuntimeError("This layer norm doesn't support feature dim >= 64KB.") - - seed = 0 - # num_warps = min(max(BLOCK_D // 256, 1), 8) - grid = (N,) - # pyrefly: ignore [not-callable] - _ln_mul_dropout_fwd[grid]( - x, - u, - y, - weight, - bias, - mean, - rstd, - D, - eps, - seed, - dropout_ratio, - x.stride(0), - u.stride(0), - y.stride(0), - # pyrefly: ignore [bad-argument-type] - SILU_U=silu_u, - # pyrefly: ignore [bad-argument-type] - BLOCK_D=BLOCK_D, - # pyrefly: ignore [bad-argument-type] - TRAINING=training, - # pyrefly: ignore [bad-argument-type] - CONCAT_U=concat_ux, - # pyrefly: ignore [bad-argument-type] - CONCAT_X=concat_ux, - # pyrefly: ignore [bad-argument-type] - MUL_U_ACTIVATION_TYPE=mul_u_activation_type, - # pyrefly: ignore [bad-argument-type] - FAST_DROPOUT=False, - ) - return y - - -@torch.fx.wrap -# "aot_triton_kernel_wrapper_" is a pre-defined prefix for -# AOT-T triton kernel wrapper functions. This is required for -# AOT-T backend to recognize and trace correctly for ops transformation. -def aot_triton_kernel_wrapper_layer_norm_mul_dropout( - x: torch.Tensor, - u: torch.Tensor, - weight: torch.Tensor, - bias: torch.Tensor, - eps: float, - dropout_ratio: float, - training: bool, - silu_u: bool, - concat_ux: bool, - mul_u_activation_type: str, -) -> torch.Tensor: - if should_trigger_eager_impl(): - return pytorch_norm_mul_dropout( - x=x, - u=u, - weight=weight, - bias=bias, - eps=eps, - dropout_ratio=dropout_ratio, - training=training, - silu_u=silu_u, - concat_u=concat_ux, - concat_x=concat_ux, - mul_u_activation_type=mul_u_activation_type, - group_norm=False, - ) - else: - return _triton_aot_layer_norm_mul_dropout( - x=x, - u=u, - weight=weight, - bias=bias, - eps=eps, - dropout_ratio=dropout_ratio, - training=training, - silu_u=silu_u, - concat_ux=concat_ux, - mul_u_activation_type=mul_u_activation_type, - ) diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/triton_position.py b/recommendation_v4/generative_recommenders/ops/triton_aot/triton_position.py deleted file mode 100644 index 828a4f4d4..000000000 --- a/recommendation_v4/generative_recommenders/ops/triton_aot/triton_position.py +++ /dev/null @@ -1,176 +0,0 @@ -# (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -# pyre-strict - -from typing import Optional - -import torch -from generative_recommenders.common import ( - cdiv, - fx_unwrap_optional_tensor, - next_power_of_2, - prev_power_of_2, - should_trigger_eager_impl, -) -from generative_recommenders.ops.pytorch.pt_position import ( - pytorch_add_timestamp_positional_embeddings, -) -from generative_recommenders.ops.triton.triton_position import ( - _add_timestamp_position_embeddings_kernel, -) -from generative_recommenders.ops.triton_aot.types import triton_aot - - -_add_timestamp_position_embeddings_kernel = triton_aot( - annotations={ - "SeqEmb": ("*bf16", 16), - "Offsets": ("*i64", 16), - "Lengths": ("*i64", 16), - "PosEmb": ("*fp32", 16), - "TsEmb": ("*fp32", 16), - "Out": ("*bf16", 16), - "TS": ("*i64", 16), - "PosInds": ("*i32", 16), - "TsInds": ("*i32", 16), - "NumTargets": ("*i64", 16), - "AUTOTUNE_MAX_SEQ_LEN": "i32", - "D": "i32", - "num_time_buckets": "i32", - "time_bucket_increments": "fp32", - "time_bucket_scale": "fp32", - "time_delta": "i32", - "max_contextual_seq_len": "i32", - "max_pos_ind": "i32", - "stride_sn": ("i32", 16), - "stride_pn": ("i32", 16), - "stride_tn": ("i32", 16), - "stride_on": ("i32", 16), - }, -)(_add_timestamp_position_embeddings_kernel) - - -@torch.jit.unused -@torch.fx.wrap -def _triton_aot_position( - seq_embeddings: torch.Tensor, - seq_offsets: torch.Tensor, - pos_embeddings: torch.Tensor, - ts_embeddings: torch.Tensor, - timestamps: torch.Tensor, - max_seq_len: int, - max_contextual_seq_len: int, - seq_lengths: torch.Tensor, - num_targets: Optional[torch.Tensor], - interleave_targets: bool, - time_bucket_fn: str, -) -> torch.Tensor: - has_multiple_targets = num_targets is not None - if not has_multiple_targets: - num_targets_resolved = torch.empty( - 0, dtype=torch.int64, device=seq_embeddings.device - ) - else: - num_targets_resolved = fx_unwrap_optional_tensor(num_targets).to(torch.int64) - - seq_embeddings = seq_embeddings.contiguous() - pos_embeddings = pos_embeddings.contiguous() - ts_embeddings = ts_embeddings.contiguous() - - max_pos_ind = pos_embeddings.shape[0] - B = seq_lengths.shape[0] - - N, D = seq_embeddings.shape - out = torch.empty_like(seq_embeddings) - - timestamps = timestamps.contiguous() - ts_inds = torch.empty((N,), device=timestamps.device, dtype=torch.int32) - pos_inds = torch.empty((N,), device=timestamps.device, dtype=torch.int32) - - autotune_max_seq_len = prev_power_of_2(max_seq_len) - BLOCK_D = next_power_of_2(D) if D < 64 else 64 - - grid = lambda meta: ( # noqa E731 - B, - cdiv(max_seq_len, meta["BLOCK_N"]), - ) - # pyrefly: ignore [not-callable] - _add_timestamp_position_embeddings_kernel[grid]( - SeqEmb=seq_embeddings, - Offsets=seq_offsets, - Lengths=seq_lengths, - PosEmb=pos_embeddings, - TsEmb=ts_embeddings, - Out=out, - TS=timestamps, - PosInds=pos_inds, - TsInds=ts_inds, - NumTargets=num_targets_resolved, - AUTOTUNE_MAX_SEQ_LEN=autotune_max_seq_len, - D=D, - num_time_buckets=2048, - time_bucket_increments=60.0, - time_bucket_scale=1.0, - time_delta=0, - max_contextual_seq_len=max_contextual_seq_len, - max_pos_ind=max_pos_ind, - stride_sn=seq_embeddings.stride(0), - stride_pn=pos_embeddings.stride(0), - stride_tn=ts_embeddings.stride(0), - stride_on=out.stride(0), - TRAINING=False, - HAS_MULTIPLE_TARGETS=has_multiple_targets, - INTERLEAVE_TARGETS=interleave_targets, - TIME_BUCKET_FN=time_bucket_fn, - BLOCK_D=BLOCK_D, - ) - - return out - - -@torch.fx.wrap -# "aot_triton_kernel_wrapper_" is a pre-defined prefix for -# AOT-T triton kernel wrapper functions. This is required for -# AOT-T backend to recognize and trace correctly for ops transformation. -def aot_triton_kernel_wrapper_position( - alpha: float, - max_seq_len: int, - max_contextual_seq_len: int, - position_embeddings_weight: torch.Tensor, - timestamp_embeddings_weight: torch.Tensor, - seq_offsets: torch.Tensor, - seq_lengths: torch.Tensor, - seq_embeddings: torch.Tensor, - timestamps: torch.Tensor, - num_targets: Optional[torch.Tensor], - interleave_targets: bool, - time_bucket_fn: str, -) -> torch.Tensor: - seq_embeddings = seq_embeddings * alpha - if should_trigger_eager_impl(): - return pytorch_add_timestamp_positional_embeddings( - seq_embeddings=seq_embeddings, - seq_offsets=seq_offsets, - pos_embeddings=position_embeddings_weight, - ts_embeddings=timestamp_embeddings_weight, - timestamps=timestamps, - max_seq_len=max_seq_len, - max_contextual_seq_len=max_contextual_seq_len, - seq_lengths=seq_lengths, - num_targets=num_targets, - interleave_targets=interleave_targets, - time_bucket_fn=time_bucket_fn, - ) - else: - return _triton_aot_position( - seq_embeddings=seq_embeddings, - seq_offsets=seq_offsets, - pos_embeddings=position_embeddings_weight, - ts_embeddings=timestamp_embeddings_weight, - timestamps=timestamps, - max_seq_len=max_seq_len, - max_contextual_seq_len=max_contextual_seq_len, - seq_lengths=seq_lengths, - num_targets=num_targets, - interleave_targets=interleave_targets, - time_bucket_fn=time_bucket_fn, - ) diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/triton_ragged_hstu_attention.py b/recommendation_v4/generative_recommenders/ops/triton_aot/triton_ragged_hstu_attention.py deleted file mode 100644 index 4fa11dddc..000000000 --- a/recommendation_v4/generative_recommenders/ops/triton_aot/triton_ragged_hstu_attention.py +++ /dev/null @@ -1,366 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files -# (the "Software"), to deal in the Software without restriction, -# including without limitation the rights to use, copy, modify, merge, -# publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, -# subject to the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -# pyre-strict - -#!/usr/bin/env python3 - -from typing import Optional - -import torch -from generative_recommenders.common import ( - autotune_max_seq_len, - BACKEND_ALLOW_TF32, - cdiv, - prev_power_of_2, - should_trigger_eager_impl, -) -from generative_recommenders.ops.pytorch.pt_hstu_attention import ( - pytorch_cached_hstu_mha, - pytorch_hstu_mha, -) -from generative_recommenders.ops.triton.triton_hstu_attention import _hstu_attn_fwd -from generative_recommenders.ops.triton_aot.types import triton_aot - - -for _config in _hstu_attn_fwd.configs: - if isinstance(_config.kwargs.get("USE_TLX"), bool): - _config.kwargs["USE_TLX"] = int(_config.kwargs["USE_TLX"]) - - -_hstu_attn_fwd = triton_aot( - annotations={ - "stride_qm": ("i32", 16), - "stride_qh": ("i32", 16), - "stride_kn": ("i32", 16), - "stride_kh": ("i32", 16), - "stride_vn": ("i32", 16), - "stride_vh": ("i32", 16), - "stride_om": ("i32", 16), - "stride_oh": ("i32", 16), - "contextual_seq_len": "i32", - "max_attn_len": "i32", - "Z": "i32", - "AUTOTUNE_Z": "i32", - "H": "i32", - "MAX_SEQ_LEN": "i32", - "AUTOTUNE_MAX_SEQ_LEN": "i32", - "DimQ": "i32", - "DimV": "i32", - "DeltaSize": "i32", - "workspace_ptr": "*i8", - "sort_by_length_indices": "*i64", - } -)(_hstu_attn_fwd) - - -def _check_common_args( - invalid_attn_mask_type: str, - attn_scale: Optional[torch.Tensor], - full_attn_size: int, - num_softmax_heads: int, -) -> None: - assert invalid_attn_mask_type in ("causal", "lower_triangular"), ( - f"unsupported invalid_attn_mask_type: {invalid_attn_mask_type}" - ) - assert attn_scale is None, "attn_scale is not implemented for AOT-T HSTU MHA" - assert full_attn_size == 0, "full_attn_size is not implemented for AOT-T HSTU MHA" - assert num_softmax_heads == 0, ( - "num_softmax_heads is not implemented for AOT-T HSTU MHA" - ) - - -@torch.jit.unused -@torch.fx.wrap -def _triton_aot_ragged_hstu_mha( - N: int, - alpha: float, - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - seq_offsets: torch.Tensor, - invalid_attn_mask_type: str, - num_targets: Optional[torch.Tensor], - attn_scale: Optional[torch.Tensor], - max_attn_len: int, - contextual_seq_len: int, - full_attn_size: int, - num_softmax_heads: int = 0, - allow_tf32: bool = BACKEND_ALLOW_TF32, -) -> torch.Tensor: - assert invalid_attn_mask_type in ("causal", "lower_triangular"), ( - f"unsupported invalid_attn_mask_type: {invalid_attn_mask_type}" - ) - assert attn_scale is None, "attn_scale is not implemented for AOT-T HSTU MHA" - assert full_attn_size == 0, "full_attn_size is not implemented for AOT-T HSTU MHA" - assert num_softmax_heads == 0, ( - "num_softmax_heads is not implemented for AOT-T HSTU MHA" - ) - Z = seq_offsets.numel() - 1 - L, H, DimQ = q.shape - DimV = v.shape[2] - - q = q.contiguous() - k = k.contiguous() - v = v.contiguous() - seq_offsets = seq_offsets.contiguous() - - out = torch.empty_like(v) - if L == 0: - return out - workspace = torch.empty(0, dtype=torch.int8, device=q.device) - sort_by_length_indices = torch.empty( - 0, dtype=torch.int64, device=seq_offsets.device - ) - - grid = lambda meta: ( # noqa E731 - cdiv(N, meta["BLOCK_M"]), - Z * H, - ) - # pyrefly: ignore [not-callable] - _hstu_attn_fwd[grid]( - Q=q, - K=k, - V=v, - workspace_ptr=workspace, - sort_by_length_indices=sort_by_length_indices, - seq_offsets=seq_offsets, - num_targets=num_targets, - Out=out, - stride_qm=q.stride(0), - stride_qh=q.stride(1), - stride_kn=k.stride(0), - stride_kh=k.stride(1), - stride_vn=v.stride(0), - stride_vh=v.stride(1), - stride_om=out.stride(0), - stride_oh=out.stride(1), - alpha=alpha, - contextual_seq_len=contextual_seq_len, - max_attn_len=max_attn_len, - Z=Z, - AUTOTUNE_Z=prev_power_of_2(Z), - H=H, - MAX_SEQ_LEN=N, - AUTOTUNE_MAX_SEQ_LEN=autotune_max_seq_len(N), - DimQ=DimQ, - DimV=DimV, - DeltaSize=0, - HAS_MULTIPLE_TARGETS=num_targets is not None, - IS_DELTA_Q=False, - ALLOW_TF32=allow_tf32, - BLOCK_D_Q=DimQ, - BLOCK_D_V=DimV, - HAS_CONTEXTUAL_SEQ_LEN=contextual_seq_len > 0, - HAS_MAX_ATTN_LEN=max_attn_len > 0, - HAS_SORT_BY_LENGTH_INDICES=False, - ENABLE_TMA=False, - TMA_DESC_SIZE=128, - ) - return out - - -@torch.fx.wrap -def aot_triton_kernel_wrapper_ragged_hstu_mha( - N: int, - alpha: float, - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - seq_offsets: torch.Tensor, - invalid_attn_mask_type: str, - num_targets: Optional[torch.Tensor], - attn_scale: Optional[torch.Tensor], - max_attn_len: int, - contextual_seq_len: int, - full_attn_size: int, - num_softmax_heads: int, - allow_tf32: bool = BACKEND_ALLOW_TF32, -) -> torch.Tensor: - _check_common_args( - invalid_attn_mask_type=invalid_attn_mask_type, - attn_scale=attn_scale, - full_attn_size=full_attn_size, - num_softmax_heads=num_softmax_heads, - ) - if should_trigger_eager_impl(): - return pytorch_hstu_mha( - max_seq_len=N, - alpha=alpha, - q=q, - k=k, - v=v, - seq_offsets=seq_offsets, - causal=True, - dropout_pr=0.0, - training=False, - num_targets=num_targets, - attn_scale=attn_scale, - max_attn_len=max_attn_len, - contextual_seq_len=contextual_seq_len, - min_full_attn_seq_len=full_attn_size, - ) - return _triton_aot_ragged_hstu_mha( - N=N, - alpha=alpha, - q=q, - k=k, - v=v, - seq_offsets=seq_offsets, - invalid_attn_mask_type=invalid_attn_mask_type, - num_targets=num_targets, - attn_scale=attn_scale, - max_attn_len=max_attn_len, - contextual_seq_len=contextual_seq_len, - full_attn_size=full_attn_size, - num_softmax_heads=num_softmax_heads, - allow_tf32=allow_tf32, - ) - - -@torch.jit.unused -@torch.fx.wrap -def _triton_aot_cached_hstu_mha( - N: int, - alpha: float, - delta_q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - delta_x_offsets: torch.Tensor, - seq_offsets: torch.Tensor, - num_targets: Optional[torch.Tensor], - attn_scale: Optional[torch.Tensor], - max_attn_len: int, - full_attn_size: int, - allow_tf32: bool = BACKEND_ALLOW_TF32, -) -> torch.Tensor: - assert attn_scale is None, "attn_scale is not implemented for AOT-T HSTU MHA" - assert full_attn_size == 0, "full_attn_size is not implemented for AOT-T HSTU MHA" - Z = seq_offsets.size(0) - 1 - DELTA_L, H, DimQ = delta_q.shape - DeltaSize = DELTA_L // Z - DimV = v.shape[2] - - delta_q = delta_q.contiguous() - k = k.contiguous() - v = v.contiguous() - seq_offsets = seq_offsets.contiguous() - - out = torch.empty((DELTA_L, H, DimV), dtype=delta_q.dtype, device=delta_q.device) - if DELTA_L == 0: - return out - workspace = torch.empty(0, dtype=torch.int8, device=delta_q.device) - sort_by_length_indices = torch.empty( - 0, dtype=torch.int64, device=seq_offsets.device - ) - - grid = lambda meta: ( # noqa E731 - cdiv(DeltaSize, meta["BLOCK_M"]), - Z * H, - ) - # pyrefly: ignore [not-callable] - _hstu_attn_fwd[grid]( - Q=delta_q, - K=k, - V=v, - workspace_ptr=workspace, - sort_by_length_indices=sort_by_length_indices, - seq_offsets=seq_offsets, - num_targets=num_targets, - Out=out, - stride_qm=delta_q.stride(0), - stride_qh=delta_q.stride(1), - stride_kn=k.stride(0), - stride_kh=k.stride(1), - stride_vn=v.stride(0), - stride_vh=v.stride(1), - stride_om=out.stride(0), - stride_oh=out.stride(1), - alpha=alpha, - contextual_seq_len=0, - max_attn_len=max_attn_len, - Z=Z, - AUTOTUNE_Z=prev_power_of_2(Z), - H=H, - MAX_SEQ_LEN=N, - AUTOTUNE_MAX_SEQ_LEN=autotune_max_seq_len(N), - DimQ=DimQ, - DimV=DimV, - DeltaSize=DeltaSize, - HAS_MULTIPLE_TARGETS=num_targets is not None, - IS_DELTA_Q=True, - ALLOW_TF32=allow_tf32, - BLOCK_D_Q=DimQ, - BLOCK_D_V=DimV, - HAS_CONTEXTUAL_SEQ_LEN=False, - HAS_MAX_ATTN_LEN=max_attn_len > 0, - HAS_SORT_BY_LENGTH_INDICES=False, - ENABLE_TMA=False, - TMA_DESC_SIZE=128, - ) - return out - - -@torch.fx.wrap -def aot_triton_kernel_wrapper_cached_hstu_mha( - N: int, - alpha: float, - delta_q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - delta_x_offsets: torch.Tensor, - seq_offsets: torch.Tensor, - num_targets: Optional[torch.Tensor], - attn_scale: Optional[torch.Tensor], - max_attn_len: int, - full_attn_size: int, -) -> torch.Tensor: - _check_common_args( - invalid_attn_mask_type="causal", - attn_scale=attn_scale, - full_attn_size=full_attn_size, - num_softmax_heads=0, - ) - if should_trigger_eager_impl(): - return pytorch_cached_hstu_mha( - max_seq_len=N, - alpha=alpha, - delta_q=delta_q, - k=k, - v=v, - seq_offsets=seq_offsets, - num_targets=num_targets, - max_attn_len=max_attn_len, - contextual_seq_len=0, - ) - return _triton_aot_cached_hstu_mha( - N=N, - alpha=alpha, - delta_q=delta_q, - k=k, - v=v, - delta_x_offsets=delta_x_offsets, - seq_offsets=seq_offsets, - num_targets=num_targets, - attn_scale=attn_scale, - max_attn_len=max_attn_len, - full_attn_size=full_attn_size, - ) diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/triton_rms_norm.py b/recommendation_v4/generative_recommenders/ops/triton_aot/triton_rms_norm.py deleted file mode 100644 index e5d9e093e..000000000 --- a/recommendation_v4/generative_recommenders/ops/triton_aot/triton_rms_norm.py +++ /dev/null @@ -1,114 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files -# (the "Software"), to deal in the Software without restriction, -# including without limitation the rights to use, copy, modify, merge, -# publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, -# subject to the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -# pyre-strict - -#!/usr/bin/env python3 - -import torch -from generative_recommenders.common import ( - cdiv, - next_power_of_2, - should_trigger_eager_impl, - switch_to_contiguous_if_needed, -) -from generative_recommenders.ops.pytorch.pt_layer_norm import pytorch_rms_norm -from generative_recommenders.ops.triton.triton_layer_norm import _weighted_rms_norm_fwd -from generative_recommenders.ops.triton_aot.types import triton_aot - -_weighted_rms_norm_fwd = triton_aot( - annotations={ - "N": "i32", - "D": ("i32", 16), - "stride_x": ("i32", 16), - "stride_y": ("i32", 16), - }, -)(_weighted_rms_norm_fwd) - - -@torch.jit.unused -@torch.fx.wrap -def _triton_aot_rms_norm( - x: torch.Tensor, - weight: torch.Tensor, - eps: float, - silu: bool, -) -> torch.Tensor: - """Internal AOTT kernel function for RMS norm.""" - assert x.dim() == 2, f"x.dim() == {x.dim()}, expected 2" - x = switch_to_contiguous_if_needed(x) - N, D = x.shape - - assert weight.dim() == 1 - assert weight.numel() == D - - y = torch.empty_like(x) - rstd = torch.empty(N, dtype=torch.float32, device=x.device) - - BLOCK_D = next_power_of_2(D) - - grid = lambda meta: ( # noqa E731 - cdiv(N, meta["BLOCK_N"]), - ) - # pyrefly: ignore [not-callable] - _weighted_rms_norm_fwd[grid]( - x, - y, - weight, - rstd, - N, - D, - eps, - stride_x=x.stride(0), - stride_y=y.stride(0), - SILU=silu, - BLOCK_D=BLOCK_D, - ) - - return y - - -def _pytorch_rms_norm_fallback( - x: torch.Tensor, - weight: torch.Tensor, - eps: float, - silu: bool, -) -> torch.Tensor: - """PyTorch fallback for RMS norm in eager mode.""" - - return pytorch_rms_norm(x, [x.shape[-1]], weight, eps, silu) - - -@torch.fx.wrap -def aot_triton_kernel_wrapper_rms_norm( - x: torch.Tensor, - weight: torch.Tensor, - eps: float, - silu: bool, -) -> torch.Tensor: - """AOT-T wrapper for RMS norm. - - Routes between PyTorch fallback (for tracing/serialization) and AOTT kernel path. - """ - if should_trigger_eager_impl(): - return _pytorch_rms_norm_fallback(x, weight, eps, silu) - else: - return _triton_aot_rms_norm(x, weight, eps, silu) diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/triton_split_2d_jagged.py b/recommendation_v4/generative_recommenders/ops/triton_aot/triton_split_2d_jagged.py deleted file mode 100644 index 9aa0c655b..000000000 --- a/recommendation_v4/generative_recommenders/ops/triton_aot/triton_split_2d_jagged.py +++ /dev/null @@ -1,138 +0,0 @@ -# (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -# pyre-strict - -from typing import Optional, Tuple - -import torch -from generative_recommenders.common import ( - fx_unwrap_optional_tensor, - next_power_of_2, - should_trigger_eager_impl, -) -from generative_recommenders.ops.pytorch.pt_jagged_tensors import ( - pytorch_split_2D_jagged, -) -from generative_recommenders.ops.triton.triton_jagged import split_2D_jagged -from generative_recommenders.ops.triton_aot.types import triton_aot - - -split_2D_jagged = triton_aot( - annotations={ - "DenseSize": "i32", - "D": ("i32", 16), - "stride_id": ("i32", 16), - "stride_ad": ("i32", 16), - "stride_bd": ("i32", 16), - }, - # pyrefly: ignore [bad-argument-type] -)(split_2D_jagged) - - -@torch.jit.unused -@torch.fx.wrap -def _triton_aot_split_2D_jagged( - values: torch.Tensor, - max_seq_len: int, - offsets_a: torch.Tensor, - offsets_b: torch.Tensor, - dense_size: int = 0, - is_dense_a: bool = False, - is_dense_b: bool = False, -) -> Tuple[torch.Tensor, torch.Tensor]: - _, D = values.shape - BLOCK_D = next_power_of_2(D) - - if is_dense_a: - L, _ = values.shape - B = offsets_b.size(0) - 1 - seq_len_a = dense_size * B - seq_len_b = L - seq_len_a - elif is_dense_b: - L, _ = values.shape - B = offsets_a.size(0) - 1 - seq_len_b = dense_size * B - seq_len_a = L - seq_len_b - else: - B = offsets_a.size(0) - 1 - seq_len_a = int(offsets_a[-1].item()) - seq_len_b = int(offsets_b[-1].item()) - - values_a = torch.empty((seq_len_a, D), device=values.device, dtype=values.dtype) - values_b = torch.empty((seq_len_b, D), device=values.device, dtype=values.dtype) - - grid = (max_seq_len, B) - # pyre-ignore[29]: TritonAOT.__getitem__ is callable at runtime - split_2D_jagged[grid]( - JaggedIn=values, - DenseSize=dense_size, - OffsetsA=offsets_a, - OffsetsB=offsets_b, - OutA=values_a, - OutB=values_b, - D=D, - stride_id=values.stride(0), - stride_ad=values_a.stride(0), - stride_bd=values_b.stride(0), - # pyrefly: ignore [bad-argument-type] - IS_DENSE_A=is_dense_a, - # pyrefly: ignore [bad-argument-type] - IS_DENSE_B=is_dense_b, - # pyrefly: ignore [bad-argument-type] - BLOCK_D=BLOCK_D, - # pyrefly: ignore [bad-argument-type] - IS_REPLACE=False, - ) - - if is_dense_a: - values_a = values_a.reshape(B, dense_size, D) - if is_dense_b: - values_b = values_b.reshape(B, dense_size, D) - - return values_a, values_b - - -@torch.fx.wrap -def aot_triton_kernel_wrapper_split_2D_jagged( - values: torch.Tensor, - max_seq_len: int, - offsets_a: Optional[torch.Tensor] = None, - offsets_b: Optional[torch.Tensor] = None, - dense_size: int = 0, -) -> Tuple[torch.Tensor, torch.Tensor]: - if should_trigger_eager_impl(): - assert offsets_a is not None and offsets_b is not None, ( - "Eager fallback requires both offsets_a and offsets_b" - ) - return pytorch_split_2D_jagged( - max_seq_len=max_seq_len, - values=values, - max_len_left=None, - max_len_right=None, - offsets_left=offsets_a, - offsets_right=offsets_b, - ) - else: - is_dense_a: bool = offsets_a is None - is_dense_b: bool = offsets_b is None - resolved_offsets_a: torch.Tensor = values.new_empty(0) - resolved_offsets_b: torch.Tensor = values.new_empty(0) - if is_dense_a: - resolved_offsets_b = fx_unwrap_optional_tensor(offsets_b) - resolved_offsets_a = resolved_offsets_b.new_empty(0) - elif is_dense_b: - resolved_offsets_a = fx_unwrap_optional_tensor(offsets_a) - resolved_offsets_b = resolved_offsets_a.new_empty(0) - else: - resolved_offsets_a = fx_unwrap_optional_tensor(offsets_a) - resolved_offsets_b = fx_unwrap_optional_tensor(offsets_b) - - return _triton_aot_split_2D_jagged( - values=values, - max_seq_len=max_seq_len, - offsets_a=resolved_offsets_a, - offsets_b=resolved_offsets_b, - dense_size=dense_size, - is_dense_a=is_dense_a, - is_dense_b=is_dense_b, - ) diff --git a/recommendation_v4/generative_recommenders/ops/triton_aot/types.py b/recommendation_v4/generative_recommenders/ops/triton_aot/types.py deleted file mode 100644 index bd7b2a1ed..000000000 --- a/recommendation_v4/generative_recommenders/ops/triton_aot/types.py +++ /dev/null @@ -1,181 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. - -# pyre-strict - -from __future__ import annotations - -import logging -from dataclasses import dataclass -from typing import Any, Callable, ClassVar, Dict, List, Optional, Protocol, Union - -from generative_recommenders.ops.triton_aot.compile.utils import is_autotuner - -# @manual=//triton:triton -from triton.runtime.jit import KernelInterface -# triton.fb.triton_util depends on torch -# @dep=//caffe2:_torch - - -_VALID_HINTS: frozenset[int] = frozenset({1, 8, 16}) -_VALID_POINTER_HINTS: frozenset[int] = frozenset({16}) - - -@dataclass(frozen=True) -class AnnotationHint: - """Annotation with a value hint (dtype + divisibility/alignment). - - Valid hints: 16 (divisible_by_16), 8 (divisible_by_8), 1 (equal_to_1). - For pointers (dtype starts with ``*``), only 16 is valid — other values - would cause incorrect codegen (e.g. alignment=1 folds the pointer as a - constexpr constant, causing a segfault at launch). - """ - - dtype: str - hint: int - - def __post_init__(self) -> None: - if self.hint not in _VALID_HINTS: - raise RuntimeError( - f"TritonAOT: invalid annotation hint {self.hint!r} for " - f"dtype {self.dtype!r}. Valid hints: {sorted(_VALID_HINTS)}." - ) - if self.dtype.startswith("*") and self.hint not in _VALID_POINTER_HINTS: - raise RuntimeError( - f"TritonAOT: invalid pointer alignment {self.hint!r} for " - f"dtype {self.dtype!r}. Pointer annotations only support " - f"alignment={sorted(_VALID_POINTER_HINTS)}." - ) - - def to_tuple(self) -> tuple[str, int]: - """Convert to plain tuple for raw spec format.""" - return (self.dtype, self.hint) - - -# Internal annotation type (after normalization). -Annotation = Union[str, AnnotationHint] - -# User-facing input type (also accepts raw tuples). -AnnotationInput = Union[str, tuple[str, int], AnnotationHint] - - -def _normalize_annotation(ann: AnnotationInput) -> Annotation: - """Convert a raw tuple to AnnotationHint (triggers validation).""" - if isinstance(ann, AnnotationHint): - return ann - if isinstance(ann, tuple): - return AnnotationHint(ann[0], ann[1]) - return ann - - -class SpecCollector(Protocol): - """Callback invoked by TritonAOT.run() to collect kernel specs during AOT compile.""" - - def __call__( - self, - fn: KernelInterface[List[Any]], - annotations: Dict[str, Annotation], - *args: Any, - **kwargs: Any, - ) -> None: ... - - -logger: logging.Logger = logging.getLogger(__name__) - - -class TritonAOTMeta(type): - # TODO consider merge with AOTTCompileState - def __init__(cls, name, bases, attrs): # pyre-ignore [2,3] - super().__init__(name, bases, attrs) - # Initialize an empty list for each new class created - cls._instances: List["TritonAOT"] = [] - - def __call__(cls, *args, **kwargs): # pyre-ignore [2,3] - # Create the instance using the default behavior - instance = super().__call__(*args, **kwargs) - # Store the instance in the class-specific list - cls._instances.append(instance) - return instance - - def get_instances(cls) -> List["TritonAOT"]: - return cls._instances - - -class TritonAOT(KernelInterface[List[Any]], metaclass=TritonAOTMeta): - """Wraps a Triton kernel for ahead-of-time compilation. - - Annotations specify dtype and optional value hints for kernel parameters: - - - Scalar: ``"i32"``, ``"fp32"``, or ``AnnotationHint("i32", 16)`` - where 16 means the runtime value is divisible by 16. - - Pointer: ``AnnotationHint("*fp32", 16)`` for 16-byte aligned tensors. - Only alignment=16 is valid for pointers. - - Tensor: typically inferred from runtime ``torch.Tensor.dtype``. - - Optional tensor: auto-detected when the same kernel is called - with a tensor at one site and ``None`` at another. - """ - - _spec_collector: ClassVar[Optional[SpecCollector]] = None - - def __init__( - self, - fn: KernelInterface[List[Any]], - annotations: Dict[str, AnnotationInput], - ) -> None: - self.fn: KernelInterface[List[Any]] = fn - self.annotations: Dict[str, Annotation] = { - k: _normalize_annotation(v) for k, v in annotations.items() - } - - @classmethod - def set_spec_collector(cls, collector: Optional[SpecCollector]) -> None: - """Register or unregister the spec collection callback. - - When a collector is registered (not None), TritonAOT.run() will call - it to collect kernel specs for AOT compilation. When None, run() - simply delegates to the underlying Triton kernel (normal JIT path). - """ - cls._spec_collector = collector - - # pyrefly: ignore [bad-override] - def run(self, *args: Any, **kwargs: Any) -> Any: - if self._spec_collector is not None: - self._spec_collector(self.fn, self.annotations, *args, **kwargs) - # pyre-ignore[29]: KernelInterface.run is callable at runtime - return self.fn.run(*args, **kwargs) - - -def triton_aot( - annotations: Dict[str, AnnotationInput], -) -> Callable[[KernelInterface[List[Any]]], TritonAOT]: - def decorator(fn: KernelInterface[List[Any]]) -> TritonAOT: - return TritonAOT(fn, annotations) - - return decorator - - -def get_all_triton_aot_instances() -> List[TritonAOT]: - """Return all triton aot function instances (e.g. decorated with @triton_aot).""" - return TritonAOT.get_instances() - - -def reset_all_triton_aot_autotune_cache() -> bool: - """Reset triton autotune cache for all triton aot kernels. - - If triton aot compile is not enabled, this function is no op. Return True if any - kernel's autotune cache is reset. Else return False. - - """ - if TritonAOT._spec_collector is None: - return False - - reset = False - for triton_aot_kernel in get_all_triton_aot_instances(): - if is_autotuner(triton_aot_kernel.fn): - autotune_fn = triton_aot_kernel.fn - autotune_fn.cache.clear() # pyre-ignore [16] - logger.info( - f"Reset autotune cache for triton kernel {autotune_fn.fn.__name__}" # pyre-ignore [16] - ) - reset = True - - return reset diff --git a/recommendation_v4/generative_recommenders/research/data/dataset.py b/recommendation_v4/generative_recommenders/research/data/dataset.py deleted file mode 100644 index 09a18ae01..000000000 --- a/recommendation_v4/generative_recommenders/research/data/dataset.py +++ /dev/null @@ -1,248 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-unsafe - -import csv -import linecache -from typing import Dict, List, Optional, Tuple - -import numpy as np -import pandas as pd -import torch - - -class DatasetV2(torch.utils.data.Dataset): - """In reverse chronological order.""" - - def __init__( - self, - ratings_file: str, - padding_length: int, - ignore_last_n: int, # used for creating train/valid/test sets - shift_id_by: int = 0, - chronological: bool = False, - sample_ratio: float = 1.0, - ) -> None: - """ - Args: - csv_file (string): Path to the csv file. - """ - super().__init__() - - self.ratings_frame: pd.DataFrame = pd.read_csv( - ratings_file, - delimiter=",", - # iterator=True, - ) - self._padding_length: int = padding_length - self._ignore_last_n: int = ignore_last_n - self._cache: Dict[int, Dict[str, torch.Tensor]] = dict() - self._shift_id_by: int = shift_id_by - self._chronological: bool = chronological - self._sample_ratio: float = sample_ratio - - def __len__(self) -> int: - return len(self.ratings_frame) - - def __getitem__(self, idx: int) -> Dict[str, torch.Tensor]: - if idx in self._cache.keys(): - return self._cache[idx] - data = self.ratings_frame.iloc[idx] - sample = self.load_item(data) - self._cache[idx] = sample - return sample - - def load_item(self, data) -> Dict[str, torch.Tensor]: - user_id = data.user_id - - def eval_as_list(x: str, ignore_last_n: int) -> List[int]: - y = eval(x) - y_list = [y] if type(y) == int else list(y) - if ignore_last_n > 0: - # for training data creation - y_list = y_list[:-ignore_last_n] - return y_list - - def eval_int_list( - x: str, - target_len: int, - ignore_last_n: int, - shift_id_by: int, - sampling_kept_mask: Optional[List[bool]], - ) -> Tuple[List[int], int]: - y = eval_as_list(x, ignore_last_n=ignore_last_n) - if sampling_kept_mask is not None: - y = [x for x, kept in zip(y, sampling_kept_mask) if kept] - y_len = len(y) - y.reverse() - if shift_id_by > 0: - y = [x + shift_id_by for x in y] - return y, y_len - - if self._sample_ratio < 1.0: - raw_length = len(eval_as_list(data.sequence_item_ids, self._ignore_last_n)) - sampling_kept_mask = ( - torch.rand((raw_length,), dtype=torch.float32) < self._sample_ratio - ).tolist() - else: - sampling_kept_mask = None - - movie_history, movie_history_len = eval_int_list( - data.sequence_item_ids, - self._padding_length, - self._ignore_last_n, - shift_id_by=self._shift_id_by, - sampling_kept_mask=sampling_kept_mask, - ) - movie_history_ratings, ratings_len = eval_int_list( - data.sequence_ratings, - self._padding_length, - self._ignore_last_n, - 0, - sampling_kept_mask=sampling_kept_mask, - ) - movie_timestamps, timestamps_len = eval_int_list( - data.sequence_timestamps, - self._padding_length, - self._ignore_last_n, - 0, - sampling_kept_mask=sampling_kept_mask, - ) - assert movie_history_len == timestamps_len, ( - f"history len {movie_history_len} differs from timestamp len {timestamps_len}." - ) - assert movie_history_len == ratings_len, ( - f"history len {movie_history_len} differs from ratings len {ratings_len}." - ) - - def _truncate_or_pad_seq( - y: List[int], target_len: int, chronological: bool - ) -> List[int]: - y_len = len(y) - if y_len < target_len: - y = y + [0] * (target_len - y_len) - else: - if not chronological: - y = y[:target_len] - else: - y = y[-target_len:] - assert len(y) == target_len - return y - - historical_ids = movie_history[1:] - historical_ratings = movie_history_ratings[1:] - historical_timestamps = movie_timestamps[1:] - target_ids = movie_history[0] - target_ratings = movie_history_ratings[0] - target_timestamps = movie_timestamps[0] - if self._chronological: - historical_ids.reverse() - historical_ratings.reverse() - historical_timestamps.reverse() - - max_seq_len = self._padding_length - 1 - history_length = min(len(historical_ids), max_seq_len) - historical_ids = _truncate_or_pad_seq( - historical_ids, - max_seq_len, - self._chronological, - ) - historical_ratings = _truncate_or_pad_seq( - historical_ratings, - max_seq_len, - self._chronological, - ) - historical_timestamps = _truncate_or_pad_seq( - historical_timestamps, - max_seq_len, - self._chronological, - ) - # moved to features.py - # if self._chronological: - # historical_ids.append(0) - # historical_ratings.append(0) - # historical_timestamps.append(0) - # print(historical_ids, historical_ratings, historical_timestamps, target_ids, target_ratings, target_timestamps) - ret = { - "user_id": user_id, - "historical_ids": torch.tensor(historical_ids, dtype=torch.int64), - "historical_ratings": torch.tensor(historical_ratings, dtype=torch.int64), - "historical_timestamps": torch.tensor( - historical_timestamps, dtype=torch.int64 - ), - "history_lengths": history_length, - "target_ids": target_ids, - "target_ratings": target_ratings, - "target_timestamps": target_timestamps, - } - return ret - - -class MultiFileDatasetV2(DatasetV2, torch.utils.data.Dataset): - def __init__( - self, - file_prefix: str, - num_files: int, - padding_length: int, - ignore_last_n: int, # used for creating train/valid/test sets - shift_id_by: int = 0, - chronological: bool = False, - sample_ratio: float = 1.0, - ) -> None: - torch.utils.data.Dataset().__init__() - self._file_prefix: str = file_prefix - self._num_files: int = num_files - with open(f"{file_prefix}_users.csv", "r") as file: - reader = csv.reader(file) - self.users_cumsum: List[int] = np.cumsum( - [int(row[1]) for row in reader] - ).tolist() - self._padding_length: int = padding_length - self._ignore_last_n: int = ignore_last_n - self._shift_id_by: int = shift_id_by - self._chronological: bool = chronological - self._sample_ratio: float = sample_ratio - - def __len__(self) -> int: - return self.users_cumsum[-1] - - def _process_line(self, line: str) -> pd.Series: - reader = csv.reader([line]) - parsed_line = next(reader) - user_id = int(parsed_line[0]) - sequence_item_ids = parsed_line[1] - sequence_ratings = parsed_line[2] - return pd.Series( - data={ - "user_id": user_id, - "sequence_item_ids": sequence_item_ids, - "sequence_ratings": sequence_ratings, - "sequence_timestamps": sequence_item_ids, # placeholder - } - ) - - def __getitem__(self, idx) -> Dict[str, torch.Tensor]: - assert idx < self.users_cumsum[-1] - file_idx: int = 0 - while self.users_cumsum[file_idx] <= idx: - file_idx += 1 - if file_idx == 0: - local_idx = idx - else: - local_idx = idx - self.users_cumsum[file_idx - 1] - line = linecache.getline(f"{self._file_prefix}_{file_idx}.csv", local_idx + 1) - data = self._process_line(line) - sample = self.load_item(data) - return sample diff --git a/recommendation_v4/generative_recommenders/research/data/eval.py b/recommendation_v4/generative_recommenders/research/data/eval.py deleted file mode 100644 index 16026e5c7..000000000 --- a/recommendation_v4/generative_recommenders/research/data/eval.py +++ /dev/null @@ -1,263 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-unsafe - -import logging -import sys -from dataclasses import dataclass -from typing import Callable, Dict, List, Optional, Set, Union - -import torch -import torch.distributed as dist -from generative_recommenders.research.indexing.candidate_index import ( - CandidateIndex, - TopKModule, -) -from generative_recommenders.research.modeling.sequential.features import ( - SequentialFeatures, -) -from generative_recommenders.research.rails.similarities.module import SimilarityModule -from torch.utils.tensorboard import SummaryWriter - - -logging.basicConfig(stream=sys.stdout, level=logging.INFO) - - -@dataclass -class EvalState: - all_item_ids: Set[int] - candidate_index: CandidateIndex - top_k_module: TopKModule - - -def get_eval_state( - model: SimilarityModule, - all_item_ids: List[int], # [X] - negatives_sampler: torch.nn.Module, - top_k_module_fn: Callable[[torch.Tensor, torch.Tensor], TopKModule], - device: int, - float_dtype: Optional[torch.dtype] = None, -) -> EvalState: - # Exhaustively eval all items (incl. seen ids). - eval_negatives_ids = torch.as_tensor(all_item_ids).to(device).unsqueeze(0) # [1, X] - # pyre-fixme[29]: `Union[Tensor, Module]` is not a function. - eval_negative_embeddings = negatives_sampler.normalize_embeddings( - # pyre-fixme[29]: `Union[Tensor, Module]` is not a function. - model.get_item_embeddings(eval_negatives_ids) - ) - if float_dtype is not None: - eval_negative_embeddings = eval_negative_embeddings.to(float_dtype) - candidates = CandidateIndex( - ids=eval_negatives_ids, - embeddings=eval_negative_embeddings, - ) - return EvalState( - all_item_ids=set(all_item_ids), - candidate_index=candidates, - top_k_module=top_k_module_fn(eval_negative_embeddings, eval_negatives_ids), - ) - - -@torch.inference_mode # pyre-ignore [56] -def eval_metrics_v2_from_tensors( - eval_state: EvalState, - model: SimilarityModule, - seq_features: SequentialFeatures, - target_ids: torch.Tensor, # [B, 1] - min_positive_rating: int = 4, - target_ratings: Optional[torch.Tensor] = None, # [B, 1] - epoch: Optional[str] = None, - filter_invalid_ids: bool = True, - user_max_batch_size: Optional[int] = None, - dtype: Optional[torch.dtype] = None, -) -> Dict[str, Union[float, torch.Tensor]]: - """ - Args: - eval_negatives_ids: Optional[Tensor]. If not present, defaults to eval over - the entire corpus (`num_items`) excluding all the items that users have - seen in the past (historical_ids, target_ids). This is consistent with - papers like SASRec and TDM but may not be fair in practice as retrieval - modules don't have access to read state during the initial fetch stage. - filter_invalid_ids: bool. If true, filters seen ids by default. - Returns: - keyed metric -> list of values for each example. - """ - B, _ = target_ids.shape - device = target_ids.device - - for target_id in target_ids: - target_id = int(target_id) - if target_id not in eval_state.all_item_ids: - print(f"missing target_id {target_id}") - - # computes ro- part exactly once. - # pyre-fixme[29]: `Union[Tensor, Module]` is not a function. - shared_input_embeddings = model.encode( - past_lengths=seq_features.past_lengths, - past_ids=seq_features.past_ids, - # pyre-fixme[29]: `Union[Tensor, Module]` is not a function. - past_embeddings=model.get_item_embeddings(seq_features.past_ids), - past_payloads=seq_features.past_payloads, - ) - if dtype is not None: - shared_input_embeddings = shared_input_embeddings.to(dtype) - - MAX_K = 2500 - k = min(MAX_K, eval_state.candidate_index.ids.size(1)) - user_max_batch_size = user_max_batch_size or shared_input_embeddings.size(0) - num_batches = ( - shared_input_embeddings.size(0) + user_max_batch_size - 1 - ) // user_max_batch_size - eval_top_k_ids_all = [] - eval_top_k_prs_all = [] - for mb in range(num_batches): - eval_top_k_ids, eval_top_k_prs, _ = ( - eval_state.candidate_index.get_top_k_outputs( - query_embeddings=shared_input_embeddings[ - mb * user_max_batch_size : (mb + 1) * user_max_batch_size, ... - ], - top_k_module=eval_state.top_k_module, - k=k, - invalid_ids=( - seq_features.past_ids[ - mb * user_max_batch_size : (mb + 1) * user_max_batch_size, : - ] - if filter_invalid_ids - else None - ), - return_embeddings=False, - ) - ) - eval_top_k_ids_all.append(eval_top_k_ids) - eval_top_k_prs_all.append(eval_top_k_prs) - - if num_batches == 1: - eval_top_k_ids = eval_top_k_ids_all[0] - eval_top_k_prs = eval_top_k_prs_all[0] - else: - eval_top_k_ids = torch.cat(eval_top_k_ids_all, dim=0) - eval_top_k_prs = torch.cat(eval_top_k_prs_all, dim=0) - - assert eval_top_k_ids.size(1) == k - _, eval_rank_indices = torch.max( - torch.cat( - [eval_top_k_ids, target_ids], - dim=1, - ) - == target_ids, - dim=1, - ) - eval_ranks = torch.where(eval_rank_indices == k, MAX_K + 1, eval_rank_indices + 1) - - output = { - "ndcg@1": torch.where( - eval_ranks <= 1, - torch.div(1.0, torch.log2(eval_ranks + 1)), - torch.zeros(1, dtype=torch.float32, device=device), - ), - "ndcg@10": torch.where( - eval_ranks <= 10, - torch.div(1.0, torch.log2(eval_ranks + 1)), - torch.zeros(1, dtype=torch.float32, device=device), - ), - "ndcg@50": torch.where( - eval_ranks <= 50, - torch.div(1.0, torch.log2(eval_ranks + 1)), - torch.zeros(1, dtype=torch.float32, device=device), - ), - "ndcg@100": torch.where( - eval_ranks <= 100, - torch.div(1.0, torch.log2(eval_ranks + 1)), - torch.zeros(1, dtype=torch.float32, device=device), - ), - "ndcg@200": torch.where( - eval_ranks <= 200, - torch.div(1.0, torch.log2(eval_ranks + 1)), - torch.zeros(1, dtype=torch.float32, device=device), - ), - "hr@1": (eval_ranks <= 1), - "hr@10": (eval_ranks <= 10), - "hr@50": (eval_ranks <= 50), - "hr@100": (eval_ranks <= 100), - "hr@200": (eval_ranks <= 200), - "hr@500": (eval_ranks <= 500), - "hr@1000": (eval_ranks <= 1000), - "mrr": torch.div(1.0, eval_ranks), - } - if target_ratings is not None: - target_ratings = target_ratings.squeeze(1) # [B] - output["ndcg@10_>=4"] = torch.where( - eval_ranks[target_ratings >= 4] <= 10, - torch.div(1.0, torch.log2(eval_ranks[target_ratings >= 4] + 1)), - torch.zeros(1, dtype=torch.float32, device=device), - ) - output[f"hr@10_>={min_positive_rating}"] = ( - eval_ranks[target_ratings >= min_positive_rating] <= 10 - ) - output[f"hr@50_>={min_positive_rating}"] = ( - eval_ranks[target_ratings >= min_positive_rating] <= 50 - ) - output[f"mrr_>={min_positive_rating}"] = torch.div( - 1.0, eval_ranks[target_ratings >= min_positive_rating] - ) - - return output # pyre-ignore [7] - - -def eval_recall_metrics_from_tensors( - eval_state: EvalState, - model: SimilarityModule, - seq_features: SequentialFeatures, - user_max_batch_size: Optional[int] = None, - dtype: Optional[torch.dtype] = None, -) -> Dict[str, torch.Tensor]: - target_ids = seq_features.past_ids[:, -1].unsqueeze(1) - filtered_past_ids = seq_features.past_ids.detach().clone() - filtered_past_ids[:, -1] = torch.zeros_like(target_ids.squeeze(1)) - return eval_metrics_v2_from_tensors( - eval_state=eval_state, - model=model, - seq_features=SequentialFeatures( - past_lengths=seq_features.past_lengths - 1, - past_ids=filtered_past_ids, - past_embeddings=seq_features.past_embeddings, - past_payloads=seq_features.past_payloads, - ), - target_ids=target_ids, - user_max_batch_size=user_max_batch_size, - dtype=dtype, - ) - - -def _avg(x: torch.Tensor, world_size: int) -> torch.Tensor: - _sum_and_numel = torch.tensor( - [x.sum(), x.numel()], dtype=torch.float32, device=x.device - ) - if world_size > 1: - dist.all_reduce(_sum_and_numel, op=dist.ReduceOp.SUM) - return _sum_and_numel[0] / _sum_and_numel[1] - - -def add_to_summary_writer( - writer: Optional[SummaryWriter], - batch_id: int, - metrics: Dict[str, torch.Tensor], - prefix: str, - world_size: int, -) -> None: - for key, values in metrics.items(): - avg_value = _avg(values, world_size) - if writer is not None: - writer.add_scalar(f"{prefix}/{key}", avg_value, batch_id) diff --git a/recommendation_v4/generative_recommenders/research/data/item_features.py b/recommendation_v4/generative_recommenders/research/data/item_features.py deleted file mode 100644 index 8ecb6ea6a..000000000 --- a/recommendation_v4/generative_recommenders/research/data/item_features.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-unsafe - -from dataclasses import dataclass -from typing import List - -import torch - - -@dataclass -class ItemFeatures: - num_items: int - max_jagged_dimension: int - max_ind_range: List[int] # [(,)] x num_features - lengths: List[torch.Tensor] # [(num_items,)] x num_features - values: List[torch.Tensor] # [(num_items, max_jagged_dimension)] x num_features diff --git a/recommendation_v4/generative_recommenders/research/data/preprocessor.py b/recommendation_v4/generative_recommenders/research/data/preprocessor.py deleted file mode 100644 index bf52f41da..000000000 --- a/recommendation_v4/generative_recommenders/research/data/preprocessor.py +++ /dev/null @@ -1,474 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-unsafe - -import abc -import logging -import os -import sys -import tarfile -from typing import Dict, Optional, Union -from urllib.request import urlretrieve -from zipfile import ZipFile - -import numpy as np -import pandas as pd - - -logging.basicConfig(stream=sys.stdout, level=logging.INFO) - - -class DataProcessor: - """ - This preprocessor does not remap item_ids. This is intended so that we can easily join other - side-information based on item_ids later. - """ - - def __init__( - self, - prefix: str, - expected_num_unique_items: Optional[int], - expected_max_item_id: Optional[int], - ) -> None: - self._prefix: str = prefix - self._expected_num_unique_items = expected_num_unique_items - self._expected_max_item_id = expected_max_item_id - - @abc.abstractmethod - def expected_num_unique_items(self) -> Optional[int]: - return self._expected_num_unique_items - - @abc.abstractmethod - def expected_max_item_id(self) -> Optional[int]: - return self._expected_max_item_id - - @abc.abstractmethod - def processed_item_csv(self) -> str: - pass - - def output_format_csv(self) -> str: - return f"tmp/{self._prefix}/sasrec_format.csv" - - def to_seq_data( - self, - ratings_data: pd.DataFrame, - user_data: Optional[pd.DataFrame] = None, - ) -> pd.DataFrame: - if user_data is not None: - ratings_data_transformed = ratings_data.join( - user_data.set_index("user_id"), on="user_id" - ) - else: - ratings_data_transformed = ratings_data - ratings_data_transformed.item_ids = ratings_data_transformed.item_ids.apply( - lambda x: ",".join([str(v) for v in x]) - ) - ratings_data_transformed.ratings = ratings_data_transformed.ratings.apply( - lambda x: ",".join([str(v) for v in x]) - ) - ratings_data_transformed.timestamps = ratings_data_transformed.timestamps.apply( - lambda x: ",".join([str(v) for v in x]) - ) - ratings_data_transformed.rename( - columns={ - "item_ids": "sequence_item_ids", - "ratings": "sequence_ratings", - "timestamps": "sequence_timestamps", - }, - inplace=True, - ) - return ratings_data_transformed - - def file_exists(self, name: str) -> bool: - return os.path.isfile("%s/%s" % (os.getcwd(), name)) - - -class MovielensSyntheticDataProcessor(DataProcessor): - def __init__( - self, - prefix: str, - expected_num_unique_items: Optional[int] = None, - expected_max_item_id: Optional[int] = None, - ) -> None: - super().__init__(prefix, expected_num_unique_items, expected_max_item_id) - - def preprocess_rating(self) -> None: - return - - -class MovielensDataProcessor(DataProcessor): - def __init__( - self, - download_path: str, - saved_name: str, - prefix: str, - convert_timestamp: bool, - expected_num_unique_items: Optional[int] = None, - expected_max_item_id: Optional[int] = None, - ) -> None: - super().__init__(prefix, expected_num_unique_items, expected_max_item_id) - self._download_path = download_path - self._saved_name = saved_name - self._convert_timestamp: bool = convert_timestamp - - def download(self) -> None: - if not self.file_exists(self._saved_name): - urlretrieve(self._download_path, self._saved_name) - if self._saved_name[-4:] == ".zip": - ZipFile(self._saved_name, "r").extractall(path="tmp/") - else: - with tarfile.open(self._saved_name, "r:*") as tar_ref: - tar_ref.extractall("tmp/") - - def processed_item_csv(self) -> str: - return f"tmp/processed/{self._prefix}/movies.csv" - - def sasrec_format_csv_by_user_train(self) -> str: - return f"tmp/{self._prefix}/sasrec_format_by_user_train.csv" - - def sasrec_format_csv_by_user_test(self) -> str: - return f"tmp/{self._prefix}/sasrec_format_by_user_test.csv" - - def preprocess_rating(self) -> int: - self.download() - - if self._prefix == "ml-1m": - users = pd.read_csv( - f"tmp/{self._prefix}/users.dat", - sep="::", - names=["user_id", "sex", "age_group", "occupation", "zip_code"], - ) - ratings = pd.read_csv( - f"tmp/{self._prefix}/ratings.dat", - sep="::", - names=["user_id", "movie_id", "rating", "unix_timestamp"], - ) - movies = pd.read_csv( - f"tmp/{self._prefix}/movies.dat", - sep="::", - names=["movie_id", "title", "genres"], - encoding="iso-8859-1", - ) - elif self._prefix == "ml-20m": - # ml-20m - # ml-20m doesn't have user data. - users = None - # ratings: userId,movieId,rating,timestamp - ratings = pd.read_csv( - f"tmp/{self._prefix}/ratings.csv", - sep=",", - ) - ratings.rename( - columns={ - "userId": "user_id", - "movieId": "movie_id", - "timestamp": "unix_timestamp", - }, - inplace=True, - ) - # movieId,title,genres - # 1,Toy Story (1995),Adventure|Animation|Children|Comedy|Fantasy - # 2,Jumanji (1995),Adventure|Children|Fantasy - movies = pd.read_csv( - f"tmp/{self._prefix}/movies.csv", - sep=",", - encoding="iso-8859-1", - ) - movies.rename(columns={"movieId": "movie_id"}, inplace=True) - else: - assert self._prefix == "ml-20mx16x32" - # ml-1b - user_ids = [] - movie_ids = [] - for i in range(16): - train_file = f"tmp/{self._prefix}/trainx16x32_{i}.npz" - with np.load(train_file) as data: - user_ids.extend([x[0] for x in data["arr_0"]]) - movie_ids.extend([x[1] for x in data["arr_0"]]) - ratings = pd.DataFrame( - data={ - "user_id": user_ids, - "movie_id": movie_ids, - "rating": user_ids, # placeholder - "unix_timestamp": movie_ids, # placeholder - } - ) - users = None - movies = None - - if movies is not None: - # ML-1M and ML-20M only - movies["year"] = movies["title"].apply(lambda x: x[-5:-1]) - movies["cleaned_title"] = movies["title"].apply(lambda x: x[:-7]) - # movies.year = pd.Categorical(movies.year) - # movies["year"] = movies.year.cat.codes - - if users is not None: - ## Users (ml-1m only) - users.sex = pd.Categorical(users.sex) - users["sex"] = users.sex.cat.codes - - users.age_group = pd.Categorical(users.age_group) - users["age_group"] = users.age_group.cat.codes - - users.occupation = pd.Categorical(users.occupation) - users["occupation"] = users.occupation.cat.codes - - users.zip_code = pd.Categorical(users.zip_code) - users["zip_code"] = users.zip_code.cat.codes - - # Normalize movie ids to speed up training - print( - f"{self._prefix} #item before normalize: {len(set(ratings['movie_id'].values))}" - ) - print( - f"{self._prefix} max item id before normalize: {max(set(ratings['movie_id'].values))}" - ) - # print(f"ratings.movie_id.cat.categories={ratings.movie_id.cat.categories}; {type(ratings.movie_id.cat.categories)}") - # print(f"ratings.movie_id.cat.codes={ratings.movie_id.cat.codes}; {type(ratings.movie_id.cat.codes)}") - # print(movie_id_to_cat) - # ratings["movie_id"] = ratings.movie_id.cat.codes - # print(f"{self._prefix} #item after normalize: {len(set(ratings['movie_id'].values))}") - # print(f"{self._prefix} max item id after normalize: {max(set(ratings['movie_id'].values))}") - # movies["remapped_id"] = movies["movie_id"].apply(lambda x: movie_id_to_cat[x]) - - if self._convert_timestamp: - ratings["unix_timestamp"] = pd.to_datetime( - ratings["unix_timestamp"], unit="s" - ) - - # Save primary csv's - if not os.path.exists(f"tmp/processed/{self._prefix}"): - os.makedirs(f"tmp/processed/{self._prefix}") - if users is not None: - users.to_csv(f"tmp/processed/{self._prefix}/users.csv", index=False) - if movies is not None: - movies.to_csv(f"tmp/processed/{self._prefix}/movies.csv", index=False) - ratings.to_csv(f"tmp/processed/{self._prefix}/ratings.csv", index=False) - - num_unique_users = len(set(ratings["user_id"].values)) - num_unique_items = len(set(ratings["movie_id"].values)) - - # SASRec version - ratings_group = ratings.sort_values(by=["unix_timestamp"]).groupby("user_id") - seq_ratings_data = pd.DataFrame( - data={ - "user_id": list(ratings_group.groups.keys()), - "item_ids": list(ratings_group.movie_id.apply(list)), - "ratings": list(ratings_group.rating.apply(list)), - "timestamps": list(ratings_group.unix_timestamp.apply(list)), - } - ) - - result = pd.DataFrame([[]]) - for col in ["item_ids"]: - result[col + "_mean"] = seq_ratings_data[col].apply(len).mean() - result[col + "_min"] = seq_ratings_data[col].apply(len).min() - result[col + "_max"] = seq_ratings_data[col].apply(len).max() - print(self._prefix) - print(result) - - seq_ratings_data = self.to_seq_data(seq_ratings_data, users) - seq_ratings_data.sample(frac=1).reset_index().to_csv( - self.output_format_csv(), index=False, sep="," - ) - - # Split by user ids (not tested yet) - user_id_split = int(num_unique_users * 0.9) - seq_ratings_data_train = seq_ratings_data[ - seq_ratings_data["user_id"] <= user_id_split - ] - seq_ratings_data_train.sample(frac=1).reset_index().to_csv( - self.sasrec_format_csv_by_user_train(), - index=False, - sep=",", - ) - seq_ratings_data_test = seq_ratings_data[ - seq_ratings_data["user_id"] > user_id_split - ] - seq_ratings_data_test.sample(frac=1).reset_index().to_csv( - self.sasrec_format_csv_by_user_test(), index=False, sep="," - ) - print( - f"{self._prefix}: train num user: {len(set(seq_ratings_data_train['user_id'].values))}" - ) - print( - f"{self._prefix}: test num user: {len(set(seq_ratings_data_test['user_id'].values))}" - ) - - # print(seq_ratings_data) - if self.expected_num_unique_items() is not None: - assert self.expected_num_unique_items() == num_unique_items, ( - f"Expected items: {self.expected_num_unique_items()}, got: {num_unique_items}" - ) - - return num_unique_items - - -class AmazonDataProcessor(DataProcessor): - def __init__( - self, - download_path: str, - saved_name: str, - prefix: str, - expected_num_unique_items: Optional[int], - ) -> None: - super().__init__( - prefix, - expected_num_unique_items=expected_num_unique_items, - expected_max_item_id=None, - ) - self._download_path = download_path - self._saved_name = saved_name - self._prefix = prefix - - def download(self) -> None: - if not self.file_exists(self._saved_name): - urlretrieve(self._download_path, self._saved_name) - - def preprocess_rating(self) -> int: - self.download() - - ratings = pd.read_csv( - self._saved_name, - sep=",", - names=["user_id", "item_id", "rating", "timestamp"], - ) - print(f"{self._prefix} #data points before filter: {ratings.shape[0]}") - print( - f"{self._prefix} #user before filter: {len(set(ratings['user_id'].values))}" - ) - print( - f"{self._prefix} #item before filter: {len(set(ratings['item_id'].values))}" - ) - - # filter users and items with presence < 5 - item_id_count = ( - ratings["item_id"] - .value_counts() - .rename_axis("unique_values") - .reset_index(name="item_count") - ) - user_id_count = ( - ratings["user_id"] - .value_counts() - .rename_axis("unique_values") - .reset_index(name="user_count") - ) - ratings = ratings.join(item_id_count.set_index("unique_values"), on="item_id") - ratings = ratings.join(user_id_count.set_index("unique_values"), on="user_id") - ratings = ratings[ratings["item_count"] >= 5] - ratings = ratings[ratings["user_count"] >= 5] - print(f"{self._prefix} #data points after filter: {ratings.shape[0]}") - - # categorize user id and item id - ratings["item_id"] = pd.Categorical(ratings["item_id"]) - ratings["item_id"] = ratings["item_id"].cat.codes - ratings["user_id"] = pd.Categorical(ratings["user_id"]) - ratings["user_id"] = ratings["user_id"].cat.codes - print( - f"{self._prefix} #user after filter: {len(set(ratings['user_id'].values))}" - ) - print( - f"{self._prefix} #item ater filter: {len(set(ratings['item_id'].values))}" - ) - - num_unique_items = len(set(ratings["item_id"].values)) - - # SASRec version - ratings_group = ratings.sort_values(by=["timestamp"]).groupby("user_id") - - seq_ratings_data = pd.DataFrame( - data={ - "user_id": list(ratings_group.groups.keys()), - "item_ids": list(ratings_group.item_id.apply(list)), - "ratings": list(ratings_group.rating.apply(list)), - "timestamps": list(ratings_group.timestamp.apply(list)), - } - ) - - seq_ratings_data = seq_ratings_data[ - seq_ratings_data["item_ids"].apply(len) >= 5 - ] - - result = pd.DataFrame([[]]) - for col in ["item_ids"]: - result[col + "_mean"] = seq_ratings_data[col].apply(len).mean() - result[col + "_min"] = seq_ratings_data[col].apply(len).min() - result[col + "_max"] = seq_ratings_data[col].apply(len).max() - print(self._prefix) - print(result) - - if not os.path.exists(f"tmp/{self._prefix}"): - os.makedirs(f"tmp/{self._prefix}") - - seq_ratings_data = self.to_seq_data(seq_ratings_data) - seq_ratings_data.sample(frac=1).reset_index().to_csv( - self.output_format_csv(), index=False, sep="," - ) - - if self.expected_num_unique_items() is not None: - assert self.expected_num_unique_items() == num_unique_items, ( - f"expected: {self.expected_num_unique_items()}, actual: {num_unique_items}" - ) - logging.info(f"{self.expected_num_unique_items()} unique items.") - - return num_unique_items - - -def get_common_preprocessors() -> Dict[ - str, - Union[AmazonDataProcessor, MovielensDataProcessor, MovielensSyntheticDataProcessor], -]: - ml_1m_dp = MovielensDataProcessor( # pyre-ignore [45] - "http://files.grouplens.org/datasets/movielens/ml-1m.zip", - "tmp/movielens1m.zip", - prefix="ml-1m", - convert_timestamp=False, - expected_num_unique_items=3706, - expected_max_item_id=3952, - ) - ml_20m_dp = MovielensDataProcessor( # pyre-ignore [45] - "http://files.grouplens.org/datasets/movielens/ml-20m.zip", - "tmp/movielens20m.zip", - prefix="ml-20m", - convert_timestamp=False, - expected_num_unique_items=26744, - expected_max_item_id=131262, - ) - ml_1b_dp = MovielensDataProcessor( # pyre-ignore [45] - "https://files.grouplens.org/datasets/movielens/ml-20mx16x32.tar", - "tmp/movielens1b.tar", - prefix="ml-20mx16x32", - convert_timestamp=False, - ) - ml_3b_dp = MovielensSyntheticDataProcessor( # pyre-ignore [45] - prefix="ml-3b", - expected_num_unique_items=26743 * 32, - expected_max_item_id=26743 * 32, - ) - amzn_books_dp = AmazonDataProcessor( # pyre-ignore [45] - "http://snap.stanford.edu/data/amazon/productGraph/categoryFiles/ratings_Books.csv", - "tmp/ratings_Books.csv", - prefix="amzn_books", - expected_num_unique_items=695762, - ) - return { - "ml-1m": ml_1m_dp, - "ml-20m": ml_20m_dp, - "ml-1b": ml_1b_dp, - "ml-3b": ml_3b_dp, - "amzn-books": amzn_books_dp, - } diff --git a/recommendation_v4/generative_recommenders/research/data/reco_dataset.py b/recommendation_v4/generative_recommenders/research/data/reco_dataset.py deleted file mode 100644 index eedcdc08a..000000000 --- a/recommendation_v4/generative_recommenders/research/data/reco_dataset.py +++ /dev/null @@ -1,176 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-unsafe - -from dataclasses import dataclass -from typing import List - -import pandas as pd -import torch -from generative_recommenders.research.data.dataset import DatasetV2, MultiFileDatasetV2 -from generative_recommenders.research.data.item_features import ItemFeatures -from generative_recommenders.research.data.preprocessor import get_common_preprocessors - - -@dataclass -class RecoDataset: - max_sequence_length: int - num_unique_items: int - max_item_id: int - all_item_ids: List[int] - train_dataset: torch.utils.data.Dataset - eval_dataset: torch.utils.data.Dataset - - -def get_reco_dataset( - dataset_name: str, - max_sequence_length: int, - chronological: bool, - positional_sampling_ratio: float = 1.0, -) -> RecoDataset: - if dataset_name == "ml-1m": - dp = get_common_preprocessors()[dataset_name] - train_dataset = DatasetV2( - ratings_file=dp.output_format_csv(), - padding_length=max_sequence_length + 1, # target - ignore_last_n=1, - chronological=chronological, - sample_ratio=positional_sampling_ratio, - ) - eval_dataset = DatasetV2( - ratings_file=dp.output_format_csv(), - padding_length=max_sequence_length + 1, # target - ignore_last_n=0, - chronological=chronological, - sample_ratio=1.0, # do not sample - ) - elif dataset_name == "ml-20m": - dp = get_common_preprocessors()[dataset_name] - train_dataset = DatasetV2( - ratings_file=dp.output_format_csv(), - padding_length=max_sequence_length + 1, # target - ignore_last_n=1, - chronological=chronological, - ) - eval_dataset = DatasetV2( - ratings_file=dp.output_format_csv(), - padding_length=max_sequence_length + 1, # target - ignore_last_n=0, - chronological=chronological, - ) - elif dataset_name == "ml-3b": - dp = get_common_preprocessors()[dataset_name] - train_dataset = MultiFileDatasetV2( - file_prefix="tmp/ml-3b/16x32", - num_files=16, - padding_length=max_sequence_length + 1, # target - ignore_last_n=1, - chronological=chronological, - ) - eval_dataset = MultiFileDatasetV2( - file_prefix="tmp/ml-3b/16x32", - num_files=16, - padding_length=max_sequence_length + 1, # target - ignore_last_n=0, - chronological=chronological, - ) - elif dataset_name == "amzn-books": - dp = get_common_preprocessors()[dataset_name] - train_dataset = DatasetV2( - ratings_file=dp.output_format_csv(), - padding_length=max_sequence_length + 1, # target - ignore_last_n=1, - shift_id_by=1, # [0..n-1] -> [1..n] - chronological=chronological, - ) - eval_dataset = DatasetV2( - ratings_file=dp.output_format_csv(), - padding_length=max_sequence_length + 1, # target - ignore_last_n=0, - shift_id_by=1, # [0..n-1] -> [1..n] - chronological=chronological, - ) - else: - raise ValueError(f"Unknown dataset {dataset_name}") - - if dataset_name == "ml-1m" or dataset_name == "ml-20m": - items = pd.read_csv(dp.processed_item_csv(), delimiter=",") - max_jagged_dimension = 16 - expected_max_item_id = dp.expected_max_item_id() - assert expected_max_item_id is not None - item_features: ItemFeatures = ItemFeatures( - max_ind_range=[63, 16383, 511], - num_items=expected_max_item_id + 1, - max_jagged_dimension=max_jagged_dimension, - lengths=[ - torch.zeros((expected_max_item_id + 1,), dtype=torch.int64), - torch.zeros((expected_max_item_id + 1,), dtype=torch.int64), - torch.zeros((expected_max_item_id + 1,), dtype=torch.int64), - ], - values=[ - torch.zeros( - (expected_max_item_id + 1, max_jagged_dimension), - dtype=torch.int64, - ), - torch.zeros( - (expected_max_item_id + 1, max_jagged_dimension), - dtype=torch.int64, - ), - torch.zeros( - (expected_max_item_id + 1, max_jagged_dimension), - dtype=torch.int64, - ), - ], - ) - all_item_ids = [] - for df_index, row in items.iterrows(): - # print(f"index {df_index}: {row}") - movie_id = int(row["movie_id"]) - genres = row["genres"].split("|") - titles = row["cleaned_title"].split(" ") - # print(f"{index}: genres{genres}, title{titles}") - genres_vector = [hash(x) % item_features.max_ind_range[0] for x in genres] - titles_vector = [hash(x) % item_features.max_ind_range[1] for x in titles] - years_vector = [hash(row["year"]) % item_features.max_ind_range[2]] - item_features.lengths[0][movie_id] = min( - len(genres_vector), max_jagged_dimension - ) - item_features.lengths[1][movie_id] = min( - len(titles_vector), max_jagged_dimension - ) - item_features.lengths[2][movie_id] = min( - len(years_vector), max_jagged_dimension - ) - for f, f_values in enumerate([genres_vector, titles_vector, years_vector]): - for j in range(min(len(f_values), max_jagged_dimension)): - item_features.values[f][movie_id][j] = f_values[j] - all_item_ids.append(movie_id) - max_item_id = dp.expected_max_item_id() - for x in all_item_ids: - assert x > 0, "x in all_item_ids should be positive" - else: - # expected_max_item_id and item_features are not set for Amazon datasets. - item_features = None - max_item_id = dp.expected_num_unique_items() - all_item_ids = [x + 1 for x in range(max_item_id)] # pyre-ignore [6] - - return RecoDataset( - max_sequence_length=max_sequence_length, - num_unique_items=dp.expected_num_unique_items(), # pyre-ignore [6] - max_item_id=max_item_id, # pyre-ignore [6] - all_item_ids=all_item_ids, - train_dataset=train_dataset, - eval_dataset=eval_dataset, - ) diff --git a/recommendation_v4/generative_recommenders/research/indexing/candidate_index.py b/recommendation_v4/generative_recommenders/research/indexing/candidate_index.py deleted file mode 100644 index fee763eaa..000000000 --- a/recommendation_v4/generative_recommenders/research/indexing/candidate_index.py +++ /dev/null @@ -1,179 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-unsafe - -from typing import Optional, Tuple - -import torch -from generative_recommenders.research.rails.indexing.candidate_index import TopKModule - - -class CandidateIndex(object): - def __init__( - self, - ids: torch.Tensor, - embeddings: torch.Tensor, - invalid_ids: Optional[torch.Tensor] = None, - debug_path: Optional[str] = None, - ) -> None: - super().__init__() - - self._ids: torch.Tensor = ids - self._embeddings: torch.Tensor = embeddings - self._invalid_ids: Optional[torch.Tensor] = invalid_ids - self._debug_path: Optional[str] = debug_path - - @property - def ids(self) -> torch.Tensor: - """ - Returns: - (1, X) or (B, X), where valid ids are positive integers. - """ - return self._ids - - @property - def num_objects(self) -> int: - return self._ids.size(1) - - @property - def embeddings(self) -> torch.Tensor: - """ - Returns: - (1, X, D) or (B, X, D) with the same shape as `ids'. - """ - return self._embeddings - - def filter_invalid_ids( - self, - invalid_ids: torch.Tensor, - ) -> "CandidateIndex": - """ - Filters invalid_ids (batch dimension dependent) from the current index. - - Args: - invalid_ids: (B, N) x int64. - - Returns: - CandidateIndex with invalid_ids filtered. - """ - X = self._ids.size(1) - if self._ids.size(0) == 1: - # ((1, X, 1) == (B, 1, N)) -> (B, X) - invalid_mask, _ = (self._ids.unsqueeze(2) == invalid_ids.unsqueeze(1)).max( - dim=2 - ) - lengths = (~invalid_mask).int().sum(-1) # (B,) - valid_1d_mask = (~invalid_mask).view(-1) - B: int = lengths.size(0) - D: int = self._embeddings.size(-1) - jagged_ids = self._ids.expand(B, -1).reshape(-1)[valid_1d_mask] - jagged_embeddings = self._embeddings.expand(B, -1, -1).reshape(-1, D)[ - valid_1d_mask - ] - X_prime: int = lengths.max(-1)[0].item() - jagged_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum(lengths) - return CandidateIndex( - ids=torch.ops.fbgemm.jagged_to_padded_dense( - values=jagged_ids.unsqueeze(-1), - offsets=[jagged_offsets], - max_lengths=[X_prime], - padding_value=0, - ).squeeze(-1), - embeddings=torch.ops.fbgemm.jagged_to_padded_dense( - values=jagged_embeddings, - offsets=[jagged_offsets], - max_lengths=[X_prime], - padding_value=0.0, - ), - debug_path=self._debug_path, - ) - else: - assert self._invalid_ids == None - return CandidateIndex( - ids=self.ids, - embeddings=self.embeddings, - invalid_ids=invalid_ids, - debug_path=self._debug_path, - ) - - def get_top_k_outputs( - self, - query_embeddings: torch.Tensor, - k: int, - top_k_module: TopKModule, - invalid_ids: Optional[torch.Tensor], - r: int = 1, - return_embeddings: bool = False, - ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """ - Gets top-k outputs specified by `policy_fn', while filtering out - invalid ids per row as specified by `invalid_ids'. - - Args: - k: int. top k to return. - policy_fn: lambda that takes in item-side embeddings (B, X, D,) and user-side - embeddings (B * r, ...), and returns predictions (unnormalized logits) - of shape (B * r, X,). - invalid_ids: (B * r, N_0) x int64. The list of ids (if > 0) to filter from - results if present. Expect N_0 to be a small constant. - return_embeddings: bool if we should additionally return embeddings for the - top k results. - - Returns: - A tuple of (top_k_ids, top_k_prs, top_k_embeddings) of shape (B * r, k, ...). - """ - B: int = query_embeddings.size(0) - max_num_invalid_ids = 0 - if invalid_ids is not None: - max_num_invalid_ids = invalid_ids.size(1) - - k_prime = min(k + max_num_invalid_ids, self.num_objects) - top_k_prime_scores, top_k_prime_ids = top_k_module( - query_embeddings=query_embeddings, k=k_prime - ) - # Masks out invalid items rowwise. - if invalid_ids is not None: - id_is_valid = ~( - (top_k_prime_ids.unsqueeze(2) == invalid_ids.unsqueeze(1)).max(2)[0] - ) # [B, K + N_0] - id_is_valid = torch.logical_and( - id_is_valid, torch.cumsum(id_is_valid.int(), dim=1) <= k - ) - # [[1, 0, 1, 0], [0, 1, 1, 1]], k=2 -> [[0, 2], [1, 2]] - top_k_rowwise_offsets = torch.nonzero(id_is_valid, as_tuple=True)[1].view( - -1, k - ) - top_k_scores = torch.gather( - top_k_prime_scores, dim=1, index=top_k_rowwise_offsets - ) - top_k_ids = torch.gather( - top_k_prime_ids, dim=1, index=top_k_rowwise_offsets - ) - else: - top_k_scores = top_k_prime_scores - top_k_ids = top_k_prime_ids - - # TODO: this should be decoupled from candidate_index. - if return_embeddings: - raise ValueError("return_embeddings not supported yet.") - else: - top_k_embeddings = None - return top_k_ids, top_k_scores, top_k_embeddings - - def apply_object_filter(self) -> "CandidateIndex": - """ - Applies general per batch filters. - """ - raise NotImplementedError("not implemented.") diff --git a/recommendation_v4/generative_recommenders/research/indexing/utils.py b/recommendation_v4/generative_recommenders/research/indexing/utils.py deleted file mode 100644 index 972d3c2e7..000000000 --- a/recommendation_v4/generative_recommenders/research/indexing/utils.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-unsafe - -import torch -from generative_recommenders.research.rails.indexing.candidate_index import TopKModule -from generative_recommenders.research.rails.indexing.mips_top_k import ( - MIPSBruteForceTopK, -) -from generative_recommenders.research.rails.indexing.mol_top_k import MoLBruteForceTopK - - -def get_top_k_module( - top_k_method: str, - model: torch.nn.Module, - item_embeddings: torch.Tensor, - item_ids: torch.Tensor, -) -> TopKModule: - if top_k_method == "MIPSBruteForceTopK": - top_k_module = MIPSBruteForceTopK( - item_embeddings=item_embeddings, - item_ids=item_ids, - ) - elif top_k_method == "MoLBruteForceTopK": - top_k_module = MoLBruteForceTopK( # pyre-ignore [20] - item_embeddings=item_embeddings, - item_ids=item_ids, - ) - else: - raise ValueError(f"Invalid top-k method {top_k_method}") - return top_k_module diff --git a/recommendation_v4/generative_recommenders/research/modeling/initialization.py b/recommendation_v4/generative_recommenders/research/modeling/initialization.py deleted file mode 100644 index c80d60075..000000000 --- a/recommendation_v4/generative_recommenders/research/modeling/initialization.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-unsafe - -import torch - - -def truncated_normal(x: torch.Tensor, mean: float, std: float) -> torch.Tensor: - with torch.no_grad(): - size = x.shape - tmp = x.new_empty(size + (4,)).normal_() - valid = (tmp < 2) & (tmp > -2) - ind = valid.max(-1, keepdim=True)[1] - x.data.copy_(tmp.gather(-1, ind).squeeze(-1)) - x.data.mul_(std).add_(mean) - return x - - -def init_mlp_xavier_weights_zero_bias(m: torch.nn.Module) -> None: - if isinstance(m, torch.nn.Linear): - torch.nn.init.xavier_uniform(m.weight) - if getattr(m, "bias", None) is not None: - m.bias.data.fill_(0.0) diff --git a/recommendation_v4/generative_recommenders/research/modeling/sequential/autoregressive_losses.py b/recommendation_v4/generative_recommenders/research/modeling/sequential/autoregressive_losses.py deleted file mode 100644 index c32bedf0e..000000000 --- a/recommendation_v4/generative_recommenders/research/modeling/sequential/autoregressive_losses.py +++ /dev/null @@ -1,477 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-unsafe - -import abc -from collections import OrderedDict -from typing import List, Tuple - -import torch -import torch.nn.functional as F -from generative_recommenders.research.rails.similarities.module import SimilarityModule -from torch.utils.checkpoint import checkpoint - - -class NegativesSampler(torch.nn.Module): - def __init__(self, l2_norm: bool, l2_norm_eps: float) -> None: - super().__init__() - - self._l2_norm: bool = l2_norm - self._l2_norm_eps: float = l2_norm_eps - - def normalize_embeddings(self, x: torch.Tensor) -> torch.Tensor: - return self._maybe_l2_norm(x) - - def _maybe_l2_norm(self, x: torch.Tensor) -> torch.Tensor: - if self._l2_norm: - x = x / torch.clamp( - torch.linalg.norm(x, ord=2, dim=-1, keepdim=True), - min=self._l2_norm_eps, - ) - return x - - @abc.abstractmethod - def debug_str(self) -> str: - pass - - @abc.abstractmethod - def process_batch( - self, - ids: torch.Tensor, - presences: torch.Tensor, - embeddings: torch.Tensor, - ) -> None: - pass - - @abc.abstractmethod - def forward( - self, - positive_ids: torch.Tensor, - num_to_sample: int, - ) -> Tuple[torch.Tensor, torch.Tensor]: - """ - Returns: - A tuple of (sampled_ids, sampled_negative_embeddings). - """ - pass - - -class LocalNegativesSampler(NegativesSampler): - def __init__( - self, - num_items: int, - item_emb: torch.nn.Embedding, - all_item_ids: List[int], - l2_norm: bool, - l2_norm_eps: float, - ) -> None: - super().__init__(l2_norm=l2_norm, l2_norm_eps=l2_norm_eps) - - self._num_items: int = len(all_item_ids) - self._item_emb: torch.nn.Embedding = item_emb - self.register_buffer("_all_item_ids", torch.tensor(all_item_ids)) - - def debug_str(self) -> str: - sampling_debug_str = ( - f"local{f'-l2-eps{self._l2_norm_eps}' if self._l2_norm else ''}" - ) - return sampling_debug_str - - def process_batch( - self, - ids: torch.Tensor, - presences: torch.Tensor, - embeddings: torch.Tensor, - ) -> None: - pass - - def forward( - self, - positive_ids: torch.Tensor, - num_to_sample: int, - ) -> Tuple[torch.Tensor, torch.Tensor]: - """ - Returns: - A tuple of (sampled_ids, sampled_negative_embeddings). - """ - # assert torch.max(torch.abs(self._item_emb(positive_ids) - positive_embeddings)) < 1e-4 - output_shape = positive_ids.size() + (num_to_sample,) - sampled_offsets = torch.randint( - low=0, - high=self._num_items, - size=output_shape, - dtype=positive_ids.dtype, - device=positive_ids.device, - ) - sampled_ids = self._all_item_ids[sampled_offsets.view(-1)].reshape(output_shape) - return sampled_ids, self.normalize_embeddings(self._item_emb(sampled_ids)) - - -class InBatchNegativesSampler(NegativesSampler): - def __init__( - self, - l2_norm: bool, - l2_norm_eps: float, - dedup_embeddings: bool, - ) -> None: - super().__init__(l2_norm=l2_norm, l2_norm_eps=l2_norm_eps) - - self._dedup_embeddings: bool = dedup_embeddings - - def debug_str(self) -> str: - sampling_debug_str = ( - f"in-batch{f'-l2-eps{self._l2_norm_eps}' if self._l2_norm else ''}" - ) - if self._dedup_embeddings: - sampling_debug_str += "-dedup" - return sampling_debug_str - - def process_batch( - self, - ids: torch.Tensor, - presences: torch.Tensor, - embeddings: torch.Tensor, - ) -> None: - """ - Args: - ids: (N') or (B, N) x int64 - presences: (N') or (B, N) x bool - embeddings: (N', D) or (B, N, D) x float - """ - assert ids.size() == presences.size() - assert ids.size() == embeddings.size()[:-1] - if self._dedup_embeddings: - valid_ids = ids[presences] - unique_ids, unique_ids_inverse_indices = torch.unique( - input=valid_ids, sorted=False, return_inverse=True - ) - device = unique_ids.device - unique_embedding_offsets = torch.empty( - (unique_ids.numel(),), - dtype=torch.int64, - device=device, - ) - unique_embedding_offsets[unique_ids_inverse_indices] = torch.arange( - valid_ids.numel(), dtype=torch.int64, device=device - ) - unique_embeddings = embeddings[presences][unique_embedding_offsets, :] - self._cached_embeddings = self._maybe_l2_norm( # pyre-ignore [16] - unique_embeddings - ) - self._cached_ids = unique_ids # pyre-ignore [16] - else: - self._cached_embeddings = self._maybe_l2_norm(embeddings[presences]) - self._cached_ids = ids[presences] - - def get_all_ids_and_embeddings(self) -> Tuple[torch.Tensor, torch.Tensor]: - return self._cached_ids, self._cached_embeddings # pyre-ignore [7] - - def forward( - self, - positive_ids: torch.Tensor, - num_to_sample: int, - ) -> Tuple[torch.Tensor, torch.Tensor]: - """ - Returns: - A tuple of (sampled_ids, sampled_negative_embeddings,). - """ - X = self._cached_ids.size(0) - sampled_offsets = torch.randint( - low=0, - high=X, - size=positive_ids.size() + (num_to_sample,), - dtype=positive_ids.dtype, - device=positive_ids.device, - ) - return ( - self._cached_ids[sampled_offsets], # pyre-ignore [29] - self._cached_embeddings[sampled_offsets], # pyre-ignore [29] - ) - - -class AutoregressiveLoss(torch.nn.Module): - @abc.abstractmethod - def jagged_forward( - self, - output_embeddings: torch.Tensor, - supervision_ids: torch.Tensor, - supervision_embeddings: torch.Tensor, - supervision_weights: torch.Tensor, - negatives_sampler: NegativesSampler, - ) -> torch.Tensor: - """ - Variant of forward() when the tensors are already in jagged format. - - Args: - output_embeddings: [N', D] x float, embeddings for the current - input sequence. - supervision_ids: [N'] x int64, (positive) supervision ids. - supervision_embeddings: [N', D] x float. - supervision_weights: Optional [N'] x float. Optional weights for - masking out invalid positions, or reweighting supervision labels. - negatives_sampler: sampler used to obtain negative examples paired with - positives. - - Returns: - (1), loss for the current engaged sequence. - """ - pass - - @abc.abstractmethod - def forward( - self, - lengths: torch.Tensor, - output_embeddings: torch.Tensor, - supervision_ids: torch.Tensor, - supervision_embeddings: torch.Tensor, - supervision_weights: torch.Tensor, - negatives_sampler: NegativesSampler, - ) -> torch.Tensor: - """ - Args: - lengths: [B] x int32 representing number of non-zero elements per row. - output_embeddings: [B, N, D] x float, embeddings for the current - input sequence. - supervision_ids: [B, N] x int64, (positive) supervision ids. - supervision_embeddings: [B, N, D] x float. - supervision_weights: Optional [B, N] x float. Optional weights for - masking out invalid positions, or reweighting supervision labels. - negatives_sampler: sampler used to obtain negative examples paired with - positives. - - Returns: - (1), loss for the current engaged sequence. - """ - pass - - -class BCELoss(AutoregressiveLoss): - def __init__( - self, - temperature: float, - model: SimilarityModule, - ) -> None: - super().__init__() - self._temperature: float = temperature - self._model = model - - def jagged_forward( - self, - output_embeddings: torch.Tensor, - supervision_ids: torch.Tensor, - supervision_embeddings: torch.Tensor, - supervision_weights: torch.Tensor, - negatives_sampler: NegativesSampler, - ) -> torch.Tensor: - assert output_embeddings.size() == supervision_embeddings.size() - assert supervision_ids.size() == supervision_embeddings.size()[:-1] - assert supervision_ids.size() == supervision_weights.size() - - sampled_ids, sampled_negative_embeddings = negatives_sampler( - positive_ids=supervision_ids, - num_to_sample=1, - ) - - positive_logits = ( - self._model.interaction( # pyre-ignore [29] - input_embeddings=output_embeddings, # [B, D] = [N', D] - target_ids=supervision_ids.unsqueeze(1), # [N', 1] - target_embeddings=supervision_embeddings.unsqueeze( - 1 - ), # [N', D] -> [N', 1, D] - )[0].squeeze(1) - / self._temperature - ) # [N'] - - sampled_negatives_logits = ( - self._model.interaction( # pyre-ignore [29] - input_embeddings=output_embeddings, # [N', D] - target_ids=sampled_ids, # [N', 1] - target_embeddings=sampled_negative_embeddings, # [N', 1, D] - )[0].squeeze(1) - / self._temperature - ) # [N'] - sampled_negatives_valid_mask = ( - supervision_ids != sampled_ids.squeeze(1) - ).float() # [N'] - loss_weights = supervision_weights * sampled_negatives_valid_mask - weighted_losses = ( - ( - F.binary_cross_entropy_with_logits( - input=positive_logits, - target=torch.ones_like(positive_logits), - reduction="none", - ) - + F.binary_cross_entropy_with_logits( - input=sampled_negatives_logits, - target=torch.zeros_like(sampled_negatives_logits), - reduction="none", - ) - ) - * loss_weights - * 0.5 - ) - return weighted_losses.sum() / loss_weights.sum() - - def forward( - self, - lengths: torch.Tensor, - output_embeddings: torch.Tensor, - supervision_ids: torch.Tensor, - supervision_embeddings: torch.Tensor, - supervision_weights: torch.Tensor, - negatives_sampler: NegativesSampler, - ) -> torch.Tensor: - """ - Args: - lengths: [B] x int32 representing number of non-zero elements per row. - output_embeddings: [B, N, D] x float, embeddings for the current - input sequence. - supervision_ids: [B, N] x int64, (positive) supervision ids. - supervision_embeddings: [B, N, D] x float. - supervision_weights: Optional [B, N] x float. Optional weights for - masking out invalid positions, or reweighting supervision labels. - negatives_sampler: sampler used to obtain negative examples paired with - positives. - Returns: - (1), loss for the current engaged sequence. - """ - assert output_embeddings.size() == supervision_embeddings.size() - assert supervision_ids.size() == supervision_embeddings.size()[:-1] - jagged_id_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum(lengths) - jagged_supervision_ids = ( - torch.ops.fbgemm.dense_to_jagged( - supervision_ids.unsqueeze(-1).float(), [jagged_id_offsets] - )[0] - .squeeze(1) - .long() - ) - jagged_supervision_weights = torch.ops.fbgemm.dense_to_jagged( - supervision_weights.unsqueeze(-1), - [jagged_id_offsets], - )[0].squeeze(1) - return self.jagged_forward( - output_embeddings=torch.ops.fbgemm.dense_to_jagged( - output_embeddings, - [jagged_id_offsets], - )[0], - supervision_ids=jagged_supervision_ids, - supervision_embeddings=torch.ops.fbgemm.dense_to_jagged( - supervision_embeddings, - [jagged_id_offsets], - )[0], - supervision_weights=jagged_supervision_weights, - negatives_sampler=negatives_sampler, - ) - - -class BCELossWithRatings(AutoregressiveLoss): - def __init__( - self, - temperature: float, - model: SimilarityModule, - ) -> None: - super().__init__() - self._temperature: float = temperature - self._model = model - - def jagged_forward( - self, - output_embeddings: torch.Tensor, - supervision_ids: torch.Tensor, - supervision_embeddings: torch.Tensor, - supervision_weights: torch.Tensor, - supervision_ratings: torch.Tensor, - negatives_sampler: NegativesSampler, - ) -> torch.Tensor: - assert output_embeddings.size() == supervision_embeddings.size() - assert supervision_ids.size() == supervision_embeddings.size()[:-1] - assert supervision_ids.size() == supervision_weights.size() - - target_logits = ( - self._model.interaction( # pyre-ignore [29] - input_embeddings=output_embeddings, # [B, D] = [N', D] - target_ids=supervision_ids.unsqueeze(1), # [N', 1] - target_embeddings=supervision_embeddings.unsqueeze( - 1 - ), # [N', D] -> [N', 1, D] - )[0].squeeze(1) - / self._temperature - ) # [N', 1] - - weighted_losses = ( - F.binary_cross_entropy_with_logits( - input=target_logits, - target=supervision_ratings.to(dtype=target_logits.dtype), - reduction="none", - ) - ) * supervision_weights - return weighted_losses.sum() / supervision_weights.sum() - - def forward( - self, - lengths: torch.Tensor, - output_embeddings: torch.Tensor, - supervision_ids: torch.Tensor, - supervision_embeddings: torch.Tensor, - supervision_weights: torch.Tensor, - supervision_ratings: torch.Tensor, - negatives_sampler: NegativesSampler, - ) -> torch.Tensor: - """ - Args: - lengths: [B] x int32 representing number of non-zero elements per row. - output_embeddings: [B, N, D] x float, embeddings for the current - input sequence. - supervision_ids: [B, N] x int64, (positive) supervision ids. - supervision_embeddings: [B, N, D] x float. - supervision_weights: Optional [B, N] x float. Optional weights for - masking out invalid positions, or reweighting supervision labels. - negatives_sampler: sampler used to obtain negative examples paired with - positives. - Returns: - (1), loss for the current engaged sequence. - """ - assert output_embeddings.size() == supervision_embeddings.size() - assert supervision_ids.size() == supervision_embeddings.size()[:-1] - jagged_id_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum(lengths) - jagged_supervision_ids = ( - torch.ops.fbgemm.dense_to_jagged( - supervision_ids.unsqueeze(-1).float(), [jagged_id_offsets] - )[0] - .squeeze(1) - .long() - ) - jagged_supervision_weights = torch.ops.fbgemm.dense_to_jagged( - supervision_weights.unsqueeze(-1), - [jagged_id_offsets], - )[0].squeeze(1) - return self.jagged_forward( - output_embeddings=torch.ops.fbgemm.dense_to_jagged( - output_embeddings, - [jagged_id_offsets], - )[0], - supervision_ids=jagged_supervision_ids, - supervision_embeddings=torch.ops.fbgemm.dense_to_jagged( - supervision_embeddings, - [jagged_id_offsets], - )[0], - supervision_weights=jagged_supervision_weights, - supervision_ratings=torch.ops.fbgemm.dense_to_jagged( - supervision_ratings.unsqueeze(-1), - [jagged_id_offsets], - )[0].squeeze(1), - negatives_sampler=negatives_sampler, - ) diff --git a/recommendation_v4/generative_recommenders/research/modeling/sequential/embedding_modules.py b/recommendation_v4/generative_recommenders/research/modeling/sequential/embedding_modules.py deleted file mode 100644 index 6e85a62dd..000000000 --- a/recommendation_v4/generative_recommenders/research/modeling/sequential/embedding_modules.py +++ /dev/null @@ -1,108 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-unsafe - -import abc - -import torch -from generative_recommenders.research.modeling.initialization import truncated_normal - - -class EmbeddingModule(torch.nn.Module): - @abc.abstractmethod - def debug_str(self) -> str: - pass - - @abc.abstractmethod - def get_item_embeddings(self, item_ids: torch.Tensor) -> torch.Tensor: - pass - - @property - @abc.abstractmethod - def item_embedding_dim(self) -> int: - pass - - -class LocalEmbeddingModule(EmbeddingModule): - def __init__( - self, - num_items: int, - item_embedding_dim: int, - ) -> None: - super().__init__() - - self._item_embedding_dim: int = item_embedding_dim - self._item_emb = torch.nn.Embedding( - num_items + 1, item_embedding_dim, padding_idx=0 - ) - self.reset_params() - - def debug_str(self) -> str: - return f"local_emb_d{self._item_embedding_dim}" - - def reset_params(self) -> None: - for name, params in self.named_parameters(): - if "_item_emb" in name: - print( - f"Initialize {name} as truncated normal: {params.data.size()} params" - ) - truncated_normal(params, mean=0.0, std=0.02) - else: - print(f"Skipping initializing params {name} - not configured") - - def get_item_embeddings(self, item_ids: torch.Tensor) -> torch.Tensor: - return self._item_emb(item_ids) - - @property - def item_embedding_dim(self) -> int: - return self._item_embedding_dim - - -class CategoricalEmbeddingModule(EmbeddingModule): - def __init__( - self, - num_items: int, - item_embedding_dim: int, - item_id_to_category_id: torch.Tensor, - ) -> None: - super().__init__() - - self._item_embedding_dim: int = item_embedding_dim - self._item_emb: torch.nn.Embedding = torch.nn.Embedding( - num_items + 1, item_embedding_dim, padding_idx=0 - ) - self.register_buffer("_item_id_to_category_id", item_id_to_category_id) - self.reset_params() - - def debug_str(self) -> str: - return f"cat_emb_d{self._item_embedding_dim}" - - def reset_params(self) -> None: - for name, params in self.named_parameters(): - if "_item_emb" in name: - print( - f"Initialize {name} as truncated normal: {params.data.size()} params" - ) - truncated_normal(params, mean=0.0, std=0.02) - else: - print(f"Skipping initializing params {name} - not configured") - - def get_item_embeddings(self, item_ids: torch.Tensor) -> torch.Tensor: - item_ids = self._item_id_to_category_id[(item_ids - 1).clamp(min=0)] + 1 - return self._item_emb(item_ids) - - @property - def item_embedding_dim(self) -> int: - return self._item_embedding_dim diff --git a/recommendation_v4/generative_recommenders/research/modeling/sequential/encoder_utils.py b/recommendation_v4/generative_recommenders/research/modeling/sequential/encoder_utils.py deleted file mode 100644 index dc64aa2cf..000000000 --- a/recommendation_v4/generative_recommenders/research/modeling/sequential/encoder_utils.py +++ /dev/null @@ -1,150 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-unsafe - -import gin -from generative_recommenders.research.modeling.sequential.embedding_modules import ( - EmbeddingModule, -) -from generative_recommenders.research.modeling.sequential.hstu import HSTU -from generative_recommenders.research.modeling.sequential.input_features_preprocessors import ( - InputFeaturesPreprocessorModule, -) -from generative_recommenders.research.modeling.sequential.output_postprocessors import ( - OutputPostprocessorModule, -) -from generative_recommenders.research.modeling.sequential.sasrec import SASRec -from generative_recommenders.research.modeling.similarity_module import ( - SequentialEncoderWithLearnedSimilarityModule, -) -from generative_recommenders.research.rails.similarities.module import SimilarityModule - - -@gin.configurable -def sasrec_encoder( - max_sequence_length: int, - max_output_length: int, - embedding_module: EmbeddingModule, - similarity_module: SimilarityModule, - input_preproc_module: InputFeaturesPreprocessorModule, - output_postproc_module: OutputPostprocessorModule, - activation_checkpoint: bool, - verbose: bool, - ffn_hidden_dim: int = 64, - ffn_activation_fn: str = "relu", - ffn_dropout_rate: float = 0.2, - num_blocks: int = 2, - num_heads: int = 1, -) -> SequentialEncoderWithLearnedSimilarityModule: - return SASRec( - embedding_module=embedding_module, - max_sequence_len=max_sequence_length, - max_output_len=max_output_length, - embedding_dim=embedding_module.item_embedding_dim, - ffn_hidden_dim=ffn_hidden_dim, - ffn_activation_fn=ffn_activation_fn, - ffn_dropout_rate=ffn_dropout_rate, - num_blocks=num_blocks, - num_heads=num_heads, - similarity_module=similarity_module, # pyre-ignore [6] - input_features_preproc_module=input_preproc_module, - output_postproc_module=output_postproc_module, - activation_checkpoint=activation_checkpoint, - verbose=verbose, - ) - - -@gin.configurable -def hstu_encoder( - max_sequence_length: int, - max_output_length: int, - embedding_module: EmbeddingModule, - similarity_module: SimilarityModule, - input_preproc_module: InputFeaturesPreprocessorModule, - output_postproc_module: OutputPostprocessorModule, - activation_checkpoint: bool, - verbose: bool, - num_blocks: int = 2, - num_heads: int = 1, - dqk: int = 64, - dv: int = 64, - linear_dropout_rate: float = 0.0, - attn_dropout_rate: float = 0.0, - normalization: str = "rel_bias", - linear_config: str = "uvqk", - linear_activation: str = "silu", - concat_ua: bool = False, - enable_relative_attention_bias: bool = True, -) -> SequentialEncoderWithLearnedSimilarityModule: - return HSTU( - embedding_module=embedding_module, - similarity_module=similarity_module, # pyre-ignore [6] - input_features_preproc_module=input_preproc_module, - output_postproc_module=output_postproc_module, - max_sequence_len=max_sequence_length, - max_output_len=max_output_length, - embedding_dim=embedding_module.item_embedding_dim, - num_blocks=num_blocks, - num_heads=num_heads, - attention_dim=dqk, - linear_dim=dv, - linear_dropout_rate=linear_dropout_rate, - attn_dropout_rate=attn_dropout_rate, - linear_config=linear_config, - linear_activation=linear_activation, - normalization=normalization, - concat_ua=concat_ua, - enable_relative_attention_bias=enable_relative_attention_bias, - verbose=verbose, - ) - - -@gin.configurable -def get_sequential_encoder( - module_type: str, - max_sequence_length: int, - max_output_length: int, - embedding_module: EmbeddingModule, - interaction_module: SimilarityModule, - input_preproc_module: InputFeaturesPreprocessorModule, - output_postproc_module: OutputPostprocessorModule, - verbose: bool, - activation_checkpoint: bool = False, -) -> SequentialEncoderWithLearnedSimilarityModule: - if module_type == "SASRec": - model = sasrec_encoder( - max_sequence_length=max_sequence_length, - max_output_length=max_output_length, - embedding_module=embedding_module, - similarity_module=interaction_module, - input_preproc_module=input_preproc_module, - output_postproc_module=output_postproc_module, - activation_checkpoint=activation_checkpoint, - verbose=verbose, - ) - elif module_type == "HSTU": - model = hstu_encoder( - max_sequence_length=max_sequence_length, - max_output_length=max_output_length, - embedding_module=embedding_module, - similarity_module=interaction_module, - input_preproc_module=input_preproc_module, - output_postproc_module=output_postproc_module, - activation_checkpoint=activation_checkpoint, - verbose=verbose, - ) - else: - raise ValueError(f"Unsupported module_type {module_type}") - return model diff --git a/recommendation_v4/generative_recommenders/research/modeling/sequential/features.py b/recommendation_v4/generative_recommenders/research/modeling/sequential/features.py deleted file mode 100644 index 70bf80cc0..000000000 --- a/recommendation_v4/generative_recommenders/research/modeling/sequential/features.py +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-unsafe - -from typing import Dict, NamedTuple, Optional, Tuple - -import torch - - -class SequentialFeatures(NamedTuple): - # (B,) x int64. Requires past_lengths[i] > 0 \forall i. - past_lengths: torch.Tensor - # (B, N,) x int64. 0 denotes valid ids. - past_ids: torch.Tensor - # (B, N, D) x float. - past_embeddings: Optional[torch.Tensor] - # Implementation-specific payloads. - # e.g., past timestamps, past event_types (e.g., clicks, likes), etc. - past_payloads: Dict[str, torch.Tensor] - - -def movielens_seq_features_from_row( - row: Dict[str, torch.Tensor], - device: int, - max_output_length: int, -) -> Tuple[SequentialFeatures, torch.Tensor, torch.Tensor]: - historical_lengths = row["history_lengths"].to(device) # [B] - historical_ids = row["historical_ids"].to(device) # [B, N] - historical_ratings = row["historical_ratings"].to(device) - historical_timestamps = row["historical_timestamps"].to(device) - target_ids = row["target_ids"].to(device).unsqueeze(1) # [B, 1] - target_ratings = row["target_ratings"].to(device).unsqueeze(1) - target_timestamps = row["target_timestamps"].to(device).unsqueeze(1) - if max_output_length > 0: - B = historical_lengths.size(0) - historical_ids = torch.cat( - [ - historical_ids, - torch.zeros( - (B, max_output_length), dtype=historical_ids.dtype, device=device - ), - ], - dim=1, - ) - historical_ratings = torch.cat( - [ - historical_ratings, - torch.zeros( - (B, max_output_length), - dtype=historical_ratings.dtype, - device=device, - ), - ], - dim=1, - ) - historical_timestamps = torch.cat( - [ - historical_timestamps, - torch.zeros( - (B, max_output_length), - dtype=historical_timestamps.dtype, - device=device, - ), - ], - dim=1, - ) - historical_timestamps.scatter_( - dim=1, - index=historical_lengths.view(-1, 1), - src=target_timestamps.view(-1, 1), - ) - # print(f"historical_ids.size()={historical_ids.size()}, historical_timestamps.size()={historical_timestamps.size()}") - features = SequentialFeatures( - past_lengths=historical_lengths, - past_ids=historical_ids, - past_embeddings=None, - past_payloads={ - "timestamps": historical_timestamps, - "ratings": historical_ratings, - }, - ) - return features, target_ids, target_ratings diff --git a/recommendation_v4/generative_recommenders/research/modeling/sequential/hstu.py b/recommendation_v4/generative_recommenders/research/modeling/sequential/hstu.py deleted file mode 100644 index 3c89245a2..000000000 --- a/recommendation_v4/generative_recommenders/research/modeling/sequential/hstu.py +++ /dev/null @@ -1,808 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-unsafe - -""" -Implements HSTU (Hierarchical Sequential Transduction Unit) in -Actions Speak Louder than Words: Trillion-Parameter Sequential Transducers for Generative Recommendations -(https://arxiv.org/abs/2402.17152, ICML'24). -""" - -import abc -import math -from typing import Callable, Dict, List, Optional, Tuple, Union - -import torch -import torch.nn.functional as F -from generative_recommenders.research.modeling.sequential.embedding_modules import ( - EmbeddingModule, -) -from generative_recommenders.research.modeling.sequential.input_features_preprocessors import ( - InputFeaturesPreprocessorModule, -) -from generative_recommenders.research.modeling.sequential.output_postprocessors import ( - OutputPostprocessorModule, -) -from generative_recommenders.research.modeling.sequential.utils import ( - get_current_embeddings, -) -from generative_recommenders.research.modeling.similarity_module import ( - SequentialEncoderWithLearnedSimilarityModule, -) -from generative_recommenders.research.rails.similarities.module import SimilarityModule - - -TIMESTAMPS_KEY = "timestamps" - - -class RelativeAttentionBiasModule(torch.nn.Module): - @abc.abstractmethod - def forward( - self, - all_timestamps: torch.Tensor, - ) -> torch.Tensor: - """ - Args: - all_timestamps: [B, N] x int64 - Returns: - torch.float tensor broadcastable to [B, N, N] - """ - pass - - -class RelativePositionalBias(RelativeAttentionBiasModule): - def __init__(self, max_seq_len: int) -> None: - super().__init__() - - self._max_seq_len: int = max_seq_len - self._w = torch.nn.Parameter( - torch.empty(2 * max_seq_len - 1).normal_(mean=0, std=0.02), - ) - - def forward( - self, - all_timestamps: torch.Tensor, - ) -> torch.Tensor: - del all_timestamps - n: int = self._max_seq_len - t = F.pad(self._w[: 2 * n - 1], [0, n]).repeat(n) - t = t[..., :-n].reshape(1, n, 3 * n - 2) - r = (2 * n - 1) // 2 - return t[..., r:-r] - - -class RelativeBucketedTimeAndPositionBasedBias(RelativeAttentionBiasModule): - """ - Bucketizes timespans based on ts(next-item) - ts(current-item). - """ - - def __init__( - self, - max_seq_len: int, - num_buckets: int, - bucketization_fn: Callable[[torch.Tensor], torch.Tensor], - ) -> None: - super().__init__() - - self._max_seq_len: int = max_seq_len - self._ts_w = torch.nn.Parameter( - torch.empty(num_buckets + 1).normal_(mean=0, std=0.02), - ) - self._pos_w = torch.nn.Parameter( - torch.empty(2 * max_seq_len - 1).normal_(mean=0, std=0.02), - ) - self._num_buckets: int = num_buckets - self._bucketization_fn: Callable[[torch.Tensor], torch.Tensor] = ( - bucketization_fn - ) - - def forward( - self, - all_timestamps: torch.Tensor, - ) -> torch.Tensor: - """ - Args: - all_timestamps: (B, N). - Returns: - (B, N, N). - """ - B = all_timestamps.size(0) - N = self._max_seq_len - t = F.pad(self._pos_w[: 2 * N - 1], [0, N]).repeat(N) - t = t[..., :-N].reshape(1, N, 3 * N - 2) - r = (2 * N - 1) // 2 - - # [B, N + 1] to simplify tensor manipulations. - ext_timestamps = torch.cat( - [all_timestamps, all_timestamps[:, N - 1 : N]], dim=1 - ) - # causal masking. Otherwise [:, :-1] - [:, 1:] works - bucketed_timestamps = torch.clamp( - self._bucketization_fn( - ext_timestamps[:, 1:].unsqueeze(2) - ext_timestamps[:, :-1].unsqueeze(1) - ), - min=0, - max=self._num_buckets, - ).detach() - rel_pos_bias = t[:, :, r:-r] - rel_ts_bias = torch.index_select( - self._ts_w, dim=0, index=bucketed_timestamps.view(-1) - ).view(B, N, N) - return rel_pos_bias + rel_ts_bias - - -HSTUCacheState = Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor] - - -def _hstu_attention_maybe_from_cache( - num_heads: int, - attention_dim: int, - linear_dim: int, - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - cached_q: Optional[torch.Tensor], - cached_k: Optional[torch.Tensor], - delta_x_offsets: Optional[Tuple[torch.Tensor, torch.Tensor]], - x_offsets: torch.Tensor, - all_timestamps: Optional[torch.Tensor], - invalid_attn_mask: torch.Tensor, - rel_attn_bias: RelativeAttentionBiasModule, -) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - B: int = x_offsets.size(0) - 1 - n: int = invalid_attn_mask.size(-1) - if delta_x_offsets is not None: - padded_q, padded_k = cached_q, cached_k - flattened_offsets = delta_x_offsets[1] + torch.arange( - start=0, - end=B * n, - step=n, - device=delta_x_offsets[1].device, - dtype=delta_x_offsets[1].dtype, - ) - assert isinstance(padded_q, torch.Tensor) - assert isinstance(padded_k, torch.Tensor) - padded_q = ( - padded_q.view(B * n, -1) - .index_copy_( - dim=0, - index=flattened_offsets, - source=q, - ) - .view(B, n, -1) - ) - padded_k = ( - padded_k.view(B * n, -1) - .index_copy_( - dim=0, - index=flattened_offsets, - source=k, - ) - .view(B, n, -1) - ) - else: - padded_q = torch.ops.fbgemm.jagged_to_padded_dense( - values=q, offsets=[x_offsets], max_lengths=[n], padding_value=0.0 - ) - padded_k = torch.ops.fbgemm.jagged_to_padded_dense( - values=k, offsets=[x_offsets], max_lengths=[n], padding_value=0.0 - ) - - qk_attn = torch.einsum( - "bnhd,bmhd->bhnm", - padded_q.view(B, n, num_heads, attention_dim), - padded_k.view(B, n, num_heads, attention_dim), - ) - if all_timestamps is not None: - qk_attn = qk_attn + rel_attn_bias(all_timestamps).unsqueeze(1) - qk_attn = F.silu(qk_attn) / n - qk_attn = qk_attn * invalid_attn_mask.unsqueeze(0).unsqueeze(0) - attn_output = torch.ops.fbgemm.dense_to_jagged( - torch.einsum( - "bhnm,bmhd->bnhd", - qk_attn, - torch.ops.fbgemm.jagged_to_padded_dense(v, [x_offsets], [n]).reshape( - B, n, num_heads, linear_dim - ), - ).reshape(B, n, num_heads * linear_dim), - [x_offsets], - )[0] - return attn_output, padded_q, padded_k - - -class SequentialTransductionUnitJagged(torch.nn.Module): - def __init__( - self, - embedding_dim: int, - linear_hidden_dim: int, - attention_dim: int, - dropout_ratio: float, - attn_dropout_ratio: float, - num_heads: int, - linear_activation: str, - relative_attention_bias_module: Optional[RelativeAttentionBiasModule] = None, - normalization: str = "rel_bias", - linear_config: str = "uvqk", - concat_ua: bool = False, - epsilon: float = 1e-6, - max_length: Optional[int] = None, - ) -> None: - super().__init__() - self._embedding_dim: int = embedding_dim - self._linear_dim: int = linear_hidden_dim - self._attention_dim: int = attention_dim - self._dropout_ratio: float = dropout_ratio - self._attn_dropout_ratio: float = attn_dropout_ratio - self._num_heads: int = num_heads - self._rel_attn_bias: Optional[RelativeAttentionBiasModule] = ( - relative_attention_bias_module - ) - self._normalization: str = normalization - self._linear_config: str = linear_config - if self._linear_config == "uvqk": - self._uvqk: torch.nn.Parameter = torch.nn.Parameter( - torch.empty( - ( - embedding_dim, - linear_hidden_dim * 2 * num_heads - + attention_dim * num_heads * 2, - ) - ).normal_(mean=0, std=0.02), - ) - else: - raise ValueError(f"Unknown linear_config {self._linear_config}") - self._linear_activation: str = linear_activation - self._concat_ua: bool = concat_ua - self._o = torch.nn.Linear( - in_features=linear_hidden_dim * num_heads * (3 if concat_ua else 1), - out_features=embedding_dim, - ) - torch.nn.init.xavier_uniform_(self._o.weight) - self._eps: float = epsilon - - def _norm_input(self, x: torch.Tensor) -> torch.Tensor: - return F.layer_norm(x, normalized_shape=[self._embedding_dim], eps=self._eps) - - def _norm_attn_output(self, x: torch.Tensor) -> torch.Tensor: - return F.layer_norm( - x, normalized_shape=[self._linear_dim * self._num_heads], eps=self._eps - ) - - def forward( # pyre-ignore [3] - self, - x: torch.Tensor, - x_offsets: torch.Tensor, - all_timestamps: Optional[torch.Tensor], - invalid_attn_mask: torch.Tensor, - delta_x_offsets: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, - cache: Optional[HSTUCacheState] = None, - return_cache_states: bool = False, - ): - """ - Args: - x: (\sum_i N_i, D) x float. - x_offsets: (B + 1) x int32. - all_timestamps: optional (B, N) x int64. - invalid_attn_mask: (B, N, N) x float, each element in {0, 1}. - delta_x_offsets: optional 2-tuple ((B,) x int32, (B,) x int32). - For the 1st element in the tuple, each element is in [0, x_offsets[-1]). For the - 2nd element in the tuple, each element is in [0, N). - cache: Optional 4-tuple of (v, padded_q, padded_k, output) from prior runs, - where all except padded_q, padded_k are jagged. - Returns: - x' = f(x), (\sum_i N_i, D) x float. - """ - n: int = invalid_attn_mask.size(-1) - cached_q = None - cached_k = None - if delta_x_offsets is not None: - # In this case, for all the following code, x, u, v, q, k become restricted to - # [delta_x_offsets[0], :]. - assert cache is not None - x = x[delta_x_offsets[0], :] - cached_v, cached_q, cached_k, cached_outputs = cache - - normed_x = self._norm_input(x) - - if self._linear_config == "uvqk": - batched_mm_output = torch.mm(normed_x, self._uvqk) - if self._linear_activation == "silu": - batched_mm_output = F.silu(batched_mm_output) - elif self._linear_activation == "none": - batched_mm_output = batched_mm_output - u, v, q, k = torch.split( - batched_mm_output, - [ - self._linear_dim * self._num_heads, - self._linear_dim * self._num_heads, - self._attention_dim * self._num_heads, - self._attention_dim * self._num_heads, - ], - dim=1, - ) - else: - raise ValueError(f"Unknown self._linear_config {self._linear_config}") - - if delta_x_offsets is not None: - v = cached_v.index_copy_(dim=0, index=delta_x_offsets[0], source=v) - - B: int = x_offsets.size(0) - 1 - if self._normalization == "rel_bias" or self._normalization == "hstu_rel_bias": - assert self._rel_attn_bias is not None - attn_output, padded_q, padded_k = _hstu_attention_maybe_from_cache( - num_heads=self._num_heads, - attention_dim=self._attention_dim, - linear_dim=self._linear_dim, - q=q, - k=k, - v=v, - cached_q=cached_q, - cached_k=cached_k, - delta_x_offsets=delta_x_offsets, - x_offsets=x_offsets, - all_timestamps=all_timestamps, - invalid_attn_mask=invalid_attn_mask, - rel_attn_bias=self._rel_attn_bias, - ) - elif self._normalization == "softmax_rel_bias": - if delta_x_offsets is not None: - B = x_offsets.size(0) - 1 - padded_q, padded_k = cached_q, cached_k - flattened_offsets = delta_x_offsets[1] + torch.arange( - start=0, - end=B * n, - step=n, - device=delta_x_offsets[1].device, - dtype=delta_x_offsets[1].dtype, - ) - assert padded_q is not None - assert padded_k is not None - padded_q = ( - padded_q.view(B * n, -1) - .index_copy_( - dim=0, - index=flattened_offsets, - source=q, - ) - .view(B, n, -1) - ) - padded_k = ( - padded_k.view(B * n, -1) - .index_copy_( - dim=0, - index=flattened_offsets, - source=k, - ) - .view(B, n, -1) - ) - else: - padded_q = torch.ops.fbgemm.jagged_to_padded_dense( - values=q, offsets=[x_offsets], max_lengths=[n], padding_value=0.0 - ) - padded_k = torch.ops.fbgemm.jagged_to_padded_dense( - values=k, offsets=[x_offsets], max_lengths=[n], padding_value=0.0 - ) - - qk_attn = torch.einsum("bnd,bmd->bnm", padded_q, padded_k) - if self._rel_attn_bias is not None: - qk_attn = qk_attn + self._rel_attn_bias(all_timestamps) - qk_attn = F.softmax(qk_attn / math.sqrt(self._attention_dim), dim=-1) - qk_attn = qk_attn * invalid_attn_mask - attn_output = torch.ops.fbgemm.dense_to_jagged( - torch.bmm( - qk_attn, - torch.ops.fbgemm.jagged_to_padded_dense(v, [x_offsets], [n]), - ), - [x_offsets], - )[0] - else: - raise ValueError(f"Unknown normalization method {self._normalization}") - - attn_output = ( - attn_output - if delta_x_offsets is None - else attn_output[delta_x_offsets[0], :] - ) - if self._concat_ua: - a = self._norm_attn_output(attn_output) - o_input = torch.cat([u, a, u * a], dim=-1) - else: - o_input = u * self._norm_attn_output(attn_output) - - new_outputs = ( - self._o( - F.dropout( - o_input, - p=self._dropout_ratio, - training=self.training, - ) - ) - + x - ) - - if delta_x_offsets is not None: - new_outputs = cached_outputs.index_copy_( - dim=0, index=delta_x_offsets[0], source=new_outputs - ) - - if return_cache_states and delta_x_offsets is None: - v = v.contiguous() - - return new_outputs, (v, padded_q, padded_k, new_outputs) - - -class HSTUJagged(torch.nn.Module): - def __init__( - self, - modules: List[SequentialTransductionUnitJagged], - autocast_dtype: Optional[torch.dtype], - ) -> None: - super().__init__() - - self._attention_layers: torch.nn.ModuleList = torch.nn.ModuleList( - modules=modules - ) - self._autocast_dtype: Optional[torch.dtype] = autocast_dtype - - def jagged_forward( - self, - x: torch.Tensor, - x_offsets: torch.Tensor, - all_timestamps: Optional[torch.Tensor], - invalid_attn_mask: torch.Tensor, - delta_x_offsets: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, - cache: Optional[List[HSTUCacheState]] = None, - return_cache_states: bool = False, - ) -> Tuple[torch.Tensor, List[HSTUCacheState]]: - """ - Args: - x: (\sum_i N_i, D) x float - x_offsets: (B + 1) x int32 - all_timestamps: (B, 1 + N) x int64 - invalid_attn_mask: (B, N, N) x float, each element in {0, 1} - return_cache_states: bool. True if we should return cache states. - - Returns: - x' = f(x), (\sum_i N_i, D) x float - """ - cache_states: List[HSTUCacheState] = [] - - with torch.autocast( - "cuda", - enabled=self._autocast_dtype is not None, - dtype=self._autocast_dtype or torch.float16, - ): - for i, layer in enumerate(self._attention_layers): - x, cache_states_i = layer( - x=x, - x_offsets=x_offsets, - all_timestamps=all_timestamps, - invalid_attn_mask=invalid_attn_mask, - delta_x_offsets=delta_x_offsets, - cache=cache[i] if cache is not None else None, - return_cache_states=return_cache_states, - ) - if return_cache_states: - cache_states.append(cache_states_i) - - return x, cache_states - - def forward( - self, - x: torch.Tensor, - x_offsets: torch.Tensor, - all_timestamps: Optional[torch.Tensor], - invalid_attn_mask: torch.Tensor, - delta_x_offsets: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, - cache: Optional[List[HSTUCacheState]] = None, - return_cache_states: bool = False, - ) -> Tuple[torch.Tensor, List[HSTUCacheState]]: - """ - Args: - x: (B, N, D) x float. - x_offsets: (B + 1) x int32. - all_timestamps: (B, 1 + N) x int64 - invalid_attn_mask: (B, N, N) x float, each element in {0, 1}. - Returns: - x' = f(x), (B, N, D) x float - """ - if len(x.size()) == 3: - x = torch.ops.fbgemm.dense_to_jagged(x, [x_offsets])[0] - - jagged_x, cache_states = self.jagged_forward( - x=x, - x_offsets=x_offsets, - all_timestamps=all_timestamps, - invalid_attn_mask=invalid_attn_mask, - delta_x_offsets=delta_x_offsets, - cache=cache, - return_cache_states=return_cache_states, - ) - y = torch.ops.fbgemm.jagged_to_padded_dense( - values=jagged_x, - offsets=[x_offsets], - max_lengths=[invalid_attn_mask.size(1)], - padding_value=0.0, - ) - return y, cache_states - - -class HSTU(SequentialEncoderWithLearnedSimilarityModule): - """ - Implements HSTU (Hierarchical Sequential Transduction Unit) in - Actions Speak Louder than Words: Trillion-Parameter Sequential Transducers for Generative Recommendations, - https://arxiv.org/abs/2402.17152. - - Note that this implementation is intended for reproducing experiments in - the traditional sequential recommender setting (Section 4.1.1), and does - not yet use optimized kernels discussed in the paper. - """ - - def __init__( - self, - max_sequence_len: int, - max_output_len: int, - embedding_dim: int, - num_blocks: int, - num_heads: int, - linear_dim: int, - attention_dim: int, - normalization: str, - linear_config: str, - linear_activation: str, - linear_dropout_rate: float, - attn_dropout_rate: float, - embedding_module: EmbeddingModule, - similarity_module: SimilarityModule, - input_features_preproc_module: InputFeaturesPreprocessorModule, - output_postproc_module: OutputPostprocessorModule, - enable_relative_attention_bias: bool = True, - concat_ua: bool = False, - verbose: bool = True, - ) -> None: - super().__init__(ndp_module=similarity_module) - - self._embedding_dim: int = embedding_dim - self._item_embedding_dim: int = embedding_module.item_embedding_dim - self._max_sequence_length: int = max_sequence_len - self._embedding_module: EmbeddingModule = embedding_module - self._input_features_preproc: InputFeaturesPreprocessorModule = ( - input_features_preproc_module - ) - self._output_postproc: OutputPostprocessorModule = output_postproc_module - self._num_blocks: int = num_blocks - self._num_heads: int = num_heads - self._dqk: int = attention_dim - self._dv: int = linear_dim - self._linear_activation: str = linear_activation - self._linear_dropout_rate: float = linear_dropout_rate - self._attn_dropout_rate: float = attn_dropout_rate - self._enable_relative_attention_bias: bool = enable_relative_attention_bias - self._hstu = HSTUJagged( - modules=[ - SequentialTransductionUnitJagged( - embedding_dim=self._embedding_dim, - linear_hidden_dim=linear_dim, - attention_dim=attention_dim, - normalization=normalization, - linear_config=linear_config, - linear_activation=linear_activation, - num_heads=num_heads, - # TODO: change to lambda x. - relative_attention_bias_module=( - RelativeBucketedTimeAndPositionBasedBias( - max_seq_len=max_sequence_len - + max_output_len, # accounts for next item. - num_buckets=128, - bucketization_fn=lambda x: ( - torch.log(torch.abs(x).clamp(min=1)) / 0.301 - ).long(), - ) - if enable_relative_attention_bias - else None - ), - dropout_ratio=linear_dropout_rate, - attn_dropout_ratio=attn_dropout_rate, - concat_ua=concat_ua, - ) - for _ in range(num_blocks) - ], - autocast_dtype=None, - ) - # causal forward, w/ +1 for padding. - self.register_buffer( - "_attn_mask", - torch.triu( - torch.ones( - ( - self._max_sequence_length + max_output_len, - self._max_sequence_length + max_output_len, - ), - dtype=torch.bool, - ), - diagonal=1, - ), - ) - self._verbose: bool = verbose - self.reset_params() - - def reset_params(self) -> None: - for name, params in self.named_parameters(): - if ("_hstu" in name) or ("_embedding_module" in name): - if self._verbose: - print(f"Skipping init for {name}") - continue - try: - torch.nn.init.xavier_normal_(params.data) - if self._verbose: - print( - f"Initialize {name} as xavier normal: {params.data.size()} params" - ) - except: - if self._verbose: - print(f"Failed to initialize {name}: {params.data.size()} params") - - def get_item_embeddings(self, item_ids: torch.Tensor) -> torch.Tensor: - return self._embedding_module.get_item_embeddings(item_ids) - - def debug_str(self) -> str: - debug_str = ( - f"HSTU-b{self._num_blocks}-h{self._num_heads}-dqk{self._dqk}-dv{self._dv}" - + f"-l{self._linear_activation}d{self._linear_dropout_rate}" - + f"-ad{self._attn_dropout_rate}" - ) - if not self._enable_relative_attention_bias: - debug_str += "-norab" - return debug_str - - def generate_user_embeddings( - self, - past_lengths: torch.Tensor, - past_ids: torch.Tensor, - past_embeddings: torch.Tensor, - past_payloads: Dict[str, torch.Tensor], - delta_x_offsets: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, - cache: Optional[List[HSTUCacheState]] = None, - return_cache_states: bool = False, - ) -> Tuple[torch.Tensor, List[HSTUCacheState]]: - """ - [B, N] -> [B, N, D]. - """ - device = past_lengths.device - float_dtype = past_embeddings.dtype - B, N, _ = past_embeddings.size() - - past_lengths, user_embeddings, _ = self._input_features_preproc( - past_lengths=past_lengths, - past_ids=past_ids, - past_embeddings=past_embeddings, - past_payloads=past_payloads, - ) - - float_dtype = user_embeddings.dtype - user_embeddings, cached_states = self._hstu( - x=user_embeddings, - x_offsets=torch.ops.fbgemm.asynchronous_complete_cumsum(past_lengths), - all_timestamps=( - past_payloads[TIMESTAMPS_KEY] - if TIMESTAMPS_KEY in past_payloads - else None - ), - invalid_attn_mask=1.0 - self._attn_mask.to(float_dtype), - delta_x_offsets=delta_x_offsets, - cache=cache, - return_cache_states=return_cache_states, - ) - return self._output_postproc(user_embeddings), cached_states - - def forward( - self, - past_lengths: torch.Tensor, - past_ids: torch.Tensor, - past_embeddings: torch.Tensor, - past_payloads: Dict[str, torch.Tensor], - batch_id: Optional[int] = None, - ) -> torch.Tensor: - """ - Runs the main encoder. - - Args: - past_lengths: (B,) x int64 - past_ids: (B, N,) x int64 where the latest engaged ids come first. In - particular, past_ids[i, past_lengths[i] - 1] should correspond to - the latest engaged values. - past_embeddings: (B, N, D) x float or (\sum_b N_b, D) x float. - past_payloads: implementation-specific keyed tensors of shape (B, N, ...). - - Returns: - encoded_embeddings of [B, N, D]. - """ - encoded_embeddings, _ = self.generate_user_embeddings( - past_lengths=past_lengths, - past_ids=past_ids, - past_embeddings=past_embeddings, - past_payloads=past_payloads, - ) - return encoded_embeddings - - def _encode( - self, - past_lengths: torch.Tensor, - past_ids: torch.Tensor, - past_embeddings: torch.Tensor, - past_payloads: Dict[str, torch.Tensor], - delta_x_offsets: Optional[Tuple[torch.Tensor, torch.Tensor]], - cache: Optional[List[HSTUCacheState]], - return_cache_states: bool, - ) -> Union[torch.Tensor, Tuple[torch.Tensor, List[HSTUCacheState]]]: - """ - Args: - past_lengths: (B,) x int64. - past_ids: (B, N,) x int64. - past_embeddings: (B, N, D,) x float. - past_payloads: implementation-specific keyed tensors of shape (B, N, ...). - return_cache_states: bool. - - Returns: - (B, D) x float, representing embeddings for the current state. - """ - encoded_seq_embeddings, cache_states = self.generate_user_embeddings( - past_lengths=past_lengths, - past_ids=past_ids, - past_embeddings=past_embeddings, - past_payloads=past_payloads, - delta_x_offsets=delta_x_offsets, - cache=cache, - return_cache_states=return_cache_states, - ) # [B, N, D] - current_embeddings = get_current_embeddings( - lengths=past_lengths, encoded_embeddings=encoded_seq_embeddings - ) - if return_cache_states: - return current_embeddings, cache_states - else: - return current_embeddings - - def encode( - self, - past_lengths: torch.Tensor, - past_ids: torch.Tensor, - past_embeddings: torch.Tensor, - past_payloads: Dict[str, torch.Tensor], - delta_x_offsets: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, - cache: Optional[List[HSTUCacheState]] = None, - return_cache_states: bool = False, - ) -> Union[torch.Tensor, Tuple[torch.Tensor, List[HSTUCacheState]]]: - """ - Runs encoder to obtain the current hidden states. - - Args: - past_lengths: (B,) x int. - past_ids: (B, N,) x int. - past_embeddings: (B, N, D) x float. - past_payloads: implementation-specific keyed tensors of shape (B, N, ...). - - Returns: - (B, D,) x float, representing encoded states at the most recent time step. - """ - return self._encode( - past_lengths=past_lengths, - past_ids=past_ids, - past_embeddings=past_embeddings, - past_payloads=past_payloads, - delta_x_offsets=delta_x_offsets, - cache=cache, - return_cache_states=return_cache_states, - ) diff --git a/recommendation_v4/generative_recommenders/research/modeling/sequential/input_features_preprocessors.py b/recommendation_v4/generative_recommenders/research/modeling/sequential/input_features_preprocessors.py deleted file mode 100644 index a461ab879..000000000 --- a/recommendation_v4/generative_recommenders/research/modeling/sequential/input_features_preprocessors.py +++ /dev/null @@ -1,259 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-unsafe - -import abc -import math -from typing import Dict, Tuple - -import torch -from generative_recommenders.research.modeling.initialization import truncated_normal - - -class InputFeaturesPreprocessorModule(torch.nn.Module): - @abc.abstractmethod - def debug_str(self) -> str: - pass - - @abc.abstractmethod - def forward( - self, - past_lengths: torch.Tensor, - past_ids: torch.Tensor, - past_embeddings: torch.Tensor, - past_payloads: Dict[str, torch.Tensor], - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - pass - - -class LearnablePositionalEmbeddingInputFeaturesPreprocessor( - InputFeaturesPreprocessorModule -): - def __init__( - self, - max_sequence_len: int, - embedding_dim: int, - dropout_rate: float, - ) -> None: - super().__init__() - - self._embedding_dim: int = embedding_dim - self._pos_emb: torch.nn.Embedding = torch.nn.Embedding( - max_sequence_len, - self._embedding_dim, - ) - self._dropout_rate: float = dropout_rate - self._emb_dropout = torch.nn.Dropout(p=dropout_rate) - self.reset_state() - - def debug_str(self) -> str: - return f"posi_d{self._dropout_rate}" - - def reset_state(self) -> None: - truncated_normal( - self._pos_emb.weight.data, - mean=0.0, - std=math.sqrt(1.0 / self._embedding_dim), - ) - - def forward( - self, - past_lengths: torch.Tensor, - past_ids: torch.Tensor, - past_embeddings: torch.Tensor, - past_payloads: Dict[str, torch.Tensor], - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - B, N = past_ids.size() - D = past_embeddings.size(-1) - - user_embeddings = past_embeddings * (self._embedding_dim**0.5) + self._pos_emb( - torch.arange(N, device=past_ids.device).unsqueeze(0).repeat(B, 1) - ) - user_embeddings = self._emb_dropout(user_embeddings) - - valid_mask = (past_ids != 0).unsqueeze(-1).float() # [B, N, 1] - user_embeddings *= valid_mask - return past_lengths, user_embeddings, valid_mask - - -class LearnablePositionalEmbeddingRatedInputFeaturesPreprocessor( - InputFeaturesPreprocessorModule -): - def __init__( - self, - max_sequence_len: int, - item_embedding_dim: int, - dropout_rate: float, - rating_embedding_dim: int, - num_ratings: int, - ) -> None: - super().__init__() - - self._embedding_dim: int = item_embedding_dim + rating_embedding_dim - self._pos_emb: torch.nn.Embedding = torch.nn.Embedding( - max_sequence_len, - self._embedding_dim, - ) - self._dropout_rate: float = dropout_rate - self._emb_dropout = torch.nn.Dropout(p=dropout_rate) - self._rating_emb: torch.nn.Embedding = torch.nn.Embedding( - num_ratings, - rating_embedding_dim, - ) - self.reset_state() - - def debug_str(self) -> str: - return f"posir_d{self._dropout_rate}" - - def reset_state(self) -> None: - truncated_normal( - self._pos_emb.weight.data, - mean=0.0, - std=math.sqrt(1.0 / self._embedding_dim), - ) - truncated_normal( - self._rating_emb.weight.data, - mean=0.0, - std=math.sqrt(1.0 / self._embedding_dim), - ) - - def forward( - self, - past_lengths: torch.Tensor, - past_ids: torch.Tensor, - past_embeddings: torch.Tensor, - past_payloads: Dict[str, torch.Tensor], - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - B, N = past_ids.size() - - user_embeddings = torch.cat( - [past_embeddings, self._rating_emb(past_payloads["ratings"].int())], - dim=-1, - ) * (self._embedding_dim**0.5) + self._pos_emb( - torch.arange(N, device=past_ids.device).unsqueeze(0).repeat(B, 1) - ) - user_embeddings = self._emb_dropout(user_embeddings) - - valid_mask = (past_ids != 0).unsqueeze(-1).float() # [B, N, 1] - user_embeddings *= valid_mask - return past_lengths, user_embeddings, valid_mask - - -class CombinedItemAndRatingInputFeaturesPreprocessor(InputFeaturesPreprocessorModule): - def __init__( - self, - max_sequence_len: int, - item_embedding_dim: int, - dropout_rate: float, - num_ratings: int, - ) -> None: - super().__init__() - - self._embedding_dim: int = item_embedding_dim - # Due to [item_0, rating_0, item_1, rating_1, ...] - self._pos_emb: torch.nn.Embedding = torch.nn.Embedding( - max_sequence_len * 2, - self._embedding_dim, - ) - self._dropout_rate: float = dropout_rate - self._emb_dropout = torch.nn.Dropout(p=dropout_rate) - self._rating_emb: torch.nn.Embedding = torch.nn.Embedding( - num_ratings, - item_embedding_dim, - ) - self.reset_state() - - def debug_str(self) -> str: - return f"combir_d{self._dropout_rate}" - - def reset_state(self) -> None: - truncated_normal( - self._pos_emb.weight.data, - mean=0.0, - std=math.sqrt(1.0 / self._embedding_dim), - ) - truncated_normal( - self._rating_emb.weight.data, - mean=0.0, - std=math.sqrt(1.0 / self._embedding_dim), - ) - - def get_preprocessed_ids( - self, - past_lengths: torch.Tensor, - past_ids: torch.Tensor, - past_embeddings: torch.Tensor, - past_payloads: Dict[str, torch.Tensor], - ) -> torch.Tensor: - """ - Returns (B, N * 2,) x int64. - """ - B, N = past_ids.size() - return torch.cat( - [ - past_ids.unsqueeze(2), # (B, N, 1) - past_payloads["ratings"].to(past_ids.dtype).unsqueeze(2), - ], - dim=2, - ).reshape(B, N * 2) - - def get_preprocessed_masks( - self, - past_lengths: torch.Tensor, - past_ids: torch.Tensor, - past_embeddings: torch.Tensor, - past_payloads: Dict[str, torch.Tensor], - ) -> torch.Tensor: - """ - Returns (B, N * 2,) x bool. - """ - B, N = past_ids.size() - return (past_ids != 0).unsqueeze(2).expand(-1, -1, 2).reshape(B, N * 2) - - def forward( - self, - past_lengths: torch.Tensor, - past_ids: torch.Tensor, - past_embeddings: torch.Tensor, - past_payloads: Dict[str, torch.Tensor], - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - B, N = past_ids.size() - D = past_embeddings.size(-1) - - user_embeddings = torch.cat( - [ - past_embeddings, # (B, N, D) - self._rating_emb(past_payloads["ratings"].int()), - ], - dim=2, - ) * (self._embedding_dim**0.5) - user_embeddings = user_embeddings.view(B, N * 2, D) - user_embeddings = user_embeddings + self._pos_emb( - torch.arange(N * 2, device=past_ids.device).unsqueeze(0).repeat(B, 1) - ) - user_embeddings = self._emb_dropout(user_embeddings) - - valid_mask = ( - self.get_preprocessed_masks( - past_lengths, - past_ids, - past_embeddings, - past_payloads, - ) - .unsqueeze(2) - .float() - ) # (B, N * 2, 1,) - user_embeddings *= valid_mask - return past_lengths * 2, user_embeddings, valid_mask diff --git a/recommendation_v4/generative_recommenders/research/modeling/sequential/losses/sampled_softmax.py b/recommendation_v4/generative_recommenders/research/modeling/sequential/losses/sampled_softmax.py deleted file mode 100644 index 8e2195783..000000000 --- a/recommendation_v4/generative_recommenders/research/modeling/sequential/losses/sampled_softmax.py +++ /dev/null @@ -1,193 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-unsafe - -from collections import OrderedDict -from typing import Dict, Optional, Tuple - -import torch -import torch.nn.functional as F -from generative_recommenders.research.modeling.sequential.autoregressive_losses import ( - AutoregressiveLoss, - NegativesSampler, -) -from torch.utils.checkpoint import checkpoint - - -class SampledSoftmaxLoss(AutoregressiveLoss): - def __init__( - self, - num_to_sample: int, - softmax_temperature: float, - model, - activation_checkpoint: bool = False, - ) -> None: - super().__init__() - - self._num_to_sample: int = num_to_sample - self._softmax_temperature: float = softmax_temperature - self._model = model - self._activation_checkpoint: bool = activation_checkpoint - - def jagged_forward( # pyre-ignore [15] - self, - output_embeddings: torch.Tensor, - supervision_ids: torch.Tensor, - supervision_embeddings: torch.Tensor, - supervision_weights: torch.Tensor, - negatives_sampler: NegativesSampler, - **kwargs, - ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: - assert output_embeddings.size() == supervision_embeddings.size() - assert supervision_ids.size() == supervision_embeddings.size()[:-1] - assert supervision_ids.size() == supervision_weights.size() - - sampled_ids, sampled_negative_embeddings = negatives_sampler( - positive_ids=supervision_ids, - num_to_sample=self._num_to_sample, - ) - positive_embeddings = negatives_sampler.normalize_embeddings( - supervision_embeddings - ) - positive_logits, aux_losses = self._model.similarity_fn( - query_embeddings=output_embeddings, # [B, D] = [N', D] - item_ids=supervision_ids.unsqueeze(1), # [N', 1] - item_embeddings=positive_embeddings.unsqueeze(1), # [N', D] -> [N', 1, D] - **kwargs, - ) - positive_logits = positive_logits / self._softmax_temperature # [0] - sampled_negatives_logits, _ = self._model.similarity_fn( - query_embeddings=output_embeddings, # [N', D] - item_ids=sampled_ids, # [N', R] - item_embeddings=sampled_negative_embeddings, # [N', R, D] - **kwargs, - ) # [N', R] # [0] - sampled_negatives_logits = torch.where( - supervision_ids.unsqueeze(1) == sampled_ids, # [N', R] - -5e4, - sampled_negatives_logits / self._softmax_temperature, - ) - jagged_loss = -F.log_softmax( - torch.cat([positive_logits, sampled_negatives_logits], dim=1), dim=1 - )[:, 0] - return ( - jagged_loss * supervision_weights - ).sum() / supervision_weights.sum(), aux_losses - - def forward( # pyre-ignore [15] - self, - lengths: torch.Tensor, - output_embeddings: torch.Tensor, - supervision_ids: torch.Tensor, - supervision_embeddings: torch.Tensor, - supervision_weights: torch.Tensor, - negatives_sampler: NegativesSampler, - **kwargs, - ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: - """ - Args: - lengths: [B] x int32 representing number of non-zero elements per row. - output_embeddings: [B, N, D] x float, embeddings for the current - input sequence. - supervision_ids: [B, N] x int64, (positive) supervision ids. - supervision_embeddings: [B, N, D] x float. - supervision_weights: Optional [B, N] x float. Optional weights for - masking out invalid positions, or reweighting supervision labels. - negatives_sampler: sampler used to obtain negative examples paired with - positives. - - Returns: - Tuple of (loss for the current engaged sequence, str-keyed aux_losses). - """ - torch._assert( - output_embeddings.size() == supervision_embeddings.size(), - "Invalid supervision embeddings size.", - ) - torch._assert( - supervision_ids.size() == supervision_embeddings.size()[:-1], - "Invalid supervision ids size.", - ) - - jagged_id_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum(lengths) - jagged_supervision_ids = ( - torch.ops.fbgemm.dense_to_jagged( - supervision_ids.unsqueeze(-1).float(), [jagged_id_offsets] - )[0] - .squeeze(1) - .long() - ) - if "user_ids" in kwargs: - # expand to jagged. - max_length: int = int(lengths.max()) - kwargs["user_ids"] = torch.ops.fbgemm.dense_to_jagged( - kwargs["user_ids"] - .unsqueeze(1) - .expand(-1, max_length) - .unsqueeze(2), # (B, max_length, 1) - [jagged_id_offsets], - )[0].squeeze(1) - - args = OrderedDict( - [ - ( - "output_embeddings", - torch.ops.fbgemm.dense_to_jagged( - output_embeddings, - [jagged_id_offsets], - )[0], - ), - ("supervision_ids", jagged_supervision_ids), - ( - "supervision_embeddings", - torch.ops.fbgemm.dense_to_jagged( - supervision_embeddings, - [jagged_id_offsets], - )[0], - ), - ( - "supervision_weights", - torch.ops.fbgemm.dense_to_jagged( - supervision_weights.unsqueeze(-1), - [jagged_id_offsets], - )[0].squeeze(1), - ), - ("negatives_sampler", negatives_sampler), - ] - ) - args.update(kwargs) - if self._activation_checkpoint: - return checkpoint( - self.jagged_forward, - *args.values(), - use_reentrant=False, - ) - else: - return self.jagged_forward( - output_embeddings=torch.ops.fbgemm.dense_to_jagged( - output_embeddings, - [jagged_id_offsets], - )[0], - supervision_ids=jagged_supervision_ids, - supervision_embeddings=torch.ops.fbgemm.dense_to_jagged( - supervision_embeddings, - [jagged_id_offsets], - )[0], - supervision_weights=torch.ops.fbgemm.dense_to_jagged( - supervision_weights.unsqueeze(-1), - [jagged_id_offsets], - )[0].squeeze(1), - negatives_sampler=negatives_sampler, - **kwargs, - ) diff --git a/recommendation_v4/generative_recommenders/research/modeling/sequential/output_postprocessors.py b/recommendation_v4/generative_recommenders/research/modeling/sequential/output_postprocessors.py deleted file mode 100644 index 3319dfd93..000000000 --- a/recommendation_v4/generative_recommenders/research/modeling/sequential/output_postprocessors.py +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-unsafe - -import abc - -import torch -import torch.nn.functional as F - - -class OutputPostprocessorModule(torch.nn.Module): - @abc.abstractmethod - def debug_str(self) -> str: - pass - - @abc.abstractmethod - def forward( - self, - output_embeddings: torch.Tensor, - ) -> torch.Tensor: - pass - - -class L2NormEmbeddingPostprocessor(OutputPostprocessorModule): - def __init__( - self, - embedding_dim: int, - eps: float = 1e-6, - ) -> None: - super().__init__() - self._embedding_dim: int = embedding_dim - self._eps: float = eps - - def debug_str(self) -> str: - return "l2" - - def forward( - self, - output_embeddings: torch.Tensor, - ) -> torch.Tensor: - output_embeddings = output_embeddings[..., : self._embedding_dim] - return output_embeddings / torch.clamp( - torch.linalg.norm(output_embeddings, ord=None, dim=-1, keepdim=True), - min=self._eps, - ) - - -class LayerNormEmbeddingPostprocessor(OutputPostprocessorModule): - def __init__( - self, - embedding_dim: int, - eps: float = 1e-6, - ) -> None: - super().__init__() - self._embedding_dim: int = embedding_dim - self._eps: float = eps - - def debug_str(self) -> str: - return "ln" - - def forward( - self, - output_embeddings: torch.Tensor, - ) -> torch.Tensor: - output_embeddings = output_embeddings[..., : self._embedding_dim] - return F.layer_norm( - output_embeddings, - normalized_shape=(self._embedding_dim,), - eps=self._eps, - ) diff --git a/recommendation_v4/generative_recommenders/research/modeling/sequential/sasrec.py b/recommendation_v4/generative_recommenders/research/modeling/sequential/sasrec.py deleted file mode 100644 index 2709ddb08..000000000 --- a/recommendation_v4/generative_recommenders/research/modeling/sequential/sasrec.py +++ /dev/null @@ -1,316 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-unsafe - -""" -Implements SASRec (Self-Attentive Sequential Recommendation, https://arxiv.org/abs/1808.09781, ICDM'18). - -Compared with the original paper which used BCE loss, this implementation is modified so that -we can utilize a Sampled Softmax loss proposed in Revisiting Neural Retrieval on Accelerators -(https://arxiv.org/abs/2306.04039, KDD'23) and Turning Dross Into Gold Loss: is BERT4Rec really -better than SASRec? (https://arxiv.org/abs/2309.07602, RecSys'23), where the authors showed -sampled softmax loss to significantly improved SASRec model quality. -""" - -from typing import Dict, Optional, Tuple - -import torch -import torch.nn.functional as F -from generative_recommenders.research.modeling.sequential.embedding_modules import ( - EmbeddingModule, -) -from generative_recommenders.research.modeling.sequential.input_features_preprocessors import ( - InputFeaturesPreprocessorModule, -) -from generative_recommenders.research.modeling.sequential.output_postprocessors import ( - OutputPostprocessorModule, -) -from generative_recommenders.research.modeling.sequential.utils import ( - get_current_embeddings, -) -from generative_recommenders.research.modeling.similarity_module import ( - SequentialEncoderWithLearnedSimilarityModule, -) -from generative_recommenders.research.rails.similarities.module import SimilarityModule - - -class StandardAttentionFF(torch.nn.Module): - def __init__( - self, - embedding_dim: int, - hidden_dim: int, - activation_fn: str, - dropout_rate: float, - ) -> None: - super().__init__() - - assert activation_fn == "relu" or activation_fn == "gelu", ( - f"Invalid activation_fn {activation_fn}" - ) - - self._conv1d = torch.nn.Sequential( - torch.nn.Conv1d( - in_channels=embedding_dim, - out_channels=hidden_dim, - kernel_size=1, - ), - torch.nn.GELU() if activation_fn == "gelu" else torch.nn.ReLU(), - torch.nn.Dropout(p=dropout_rate), - torch.nn.Conv1d( - in_channels=hidden_dim, - out_channels=embedding_dim, - kernel_size=1, - ), - torch.nn.Dropout(p=dropout_rate), - ) - - def forward(self, inputs: torch.Tensor) -> torch.Tensor: - # Conv1D requires (B, D, N) - return self._conv1d(inputs.transpose(-1, -2)).transpose(-1, -2) + inputs - - -class SASRec(SequentialEncoderWithLearnedSimilarityModule): - """ - Implements SASRec (Self-Attentive Sequential Recommendation, https://arxiv.org/abs/1808.09781, ICDM'18). - - Compared with the original paper which used BCE loss, this implementation is modified so that - we can utilize a Sampled Softmax loss proposed in Revisiting Neural Retrieval on Accelerators - (https://arxiv.org/abs/2306.04039, KDD'23) and Turning Dross Into Gold Loss: is BERT4Rec really - better than SASRec? (https://arxiv.org/abs/2309.07602, RecSys'23), where the authors showed - sampled softmax loss to significantly improved SASRec model quality. - """ - - def __init__( - self, - max_sequence_len: int, - max_output_len: int, - embedding_dim: int, - num_blocks: int, - num_heads: int, - ffn_hidden_dim: int, - ffn_activation_fn: str, - ffn_dropout_rate: float, - embedding_module: EmbeddingModule, - similarity_module: SimilarityModule, - input_features_preproc_module: InputFeaturesPreprocessorModule, - output_postproc_module: OutputPostprocessorModule, - activation_checkpoint: bool = False, - verbose: bool = False, - ) -> None: - super().__init__(ndp_module=similarity_module) - - self._embedding_module: EmbeddingModule = embedding_module - self._embedding_dim: int = embedding_dim - self._item_embedding_dim: int = embedding_module.item_embedding_dim - self._max_sequence_length: int = max_sequence_len + max_output_len - self._input_features_preproc: InputFeaturesPreprocessorModule = ( - input_features_preproc_module - ) - self._output_postproc: OutputPostprocessorModule = output_postproc_module - self._activation_checkpoint: bool = activation_checkpoint - self._verbose: bool = verbose - - self.attention_layers = torch.nn.ModuleList() - self.forward_layers = torch.nn.ModuleList() - self._num_blocks: int = num_blocks - self._num_heads: int = num_heads - self._ffn_hidden_dim: int = ffn_hidden_dim - self._ffn_activation_fn: str = ffn_activation_fn - self._ffn_dropout_rate: float = ffn_dropout_rate - - for _ in range(num_blocks): - self.attention_layers.append( - torch.nn.MultiheadAttention( - embed_dim=self._embedding_dim, - num_heads=num_heads, - dropout=ffn_dropout_rate, - batch_first=True, - ) - ) - self.forward_layers.append( - StandardAttentionFF( - embedding_dim=self._embedding_dim, - hidden_dim=ffn_hidden_dim, - activation_fn=ffn_activation_fn, - dropout_rate=self._ffn_dropout_rate, - ) - ) - - self.register_buffer( - "_attn_mask", - torch.triu( - torch.ones( - (self._max_sequence_length, self._max_sequence_length), - dtype=torch.bool, - ), - diagonal=1, - ), - ) - self.reset_state() - - def reset_state(self) -> None: - for name, params in self.named_parameters(): - if ( - "_input_features_preproc" in name - or "_embedding_module" in name - or "_output_postproc" in name - ): - if self._verbose: - print(f"Skipping initialization for {name}") - continue - try: - torch.nn.init.xavier_normal_(params.data) - if self._verbose: - print( - f"Initialize {name} as xavier normal: {params.data.size()} params" - ) - except: - if self._verbose: - print(f"Failed to initialize {name}: {params.data.size()} params") - - def get_item_embeddings(self, item_ids: torch.Tensor) -> torch.Tensor: - return self._embedding_module.get_item_embeddings(item_ids) - - def debug_str(self) -> str: - return ( - f"SASRec-d{self._item_embedding_dim}-b{self._num_blocks}-h{self._num_heads}" - + "-" - + self._input_features_preproc.debug_str() - + "-" - + self._output_postproc.debug_str() - + f"-ffn{self._ffn_hidden_dim}-{self._ffn_activation_fn}-d{self._ffn_dropout_rate}" - + f"{'-ac' if self._activation_checkpoint else ''}" - ) - - def _run_one_layer( - self, - i: int, - user_embeddings: torch.Tensor, - valid_mask: torch.Tensor, - ) -> torch.Tensor: - Q = F.layer_norm( - user_embeddings, - normalized_shape=(self._embedding_dim,), - eps=1e-8, - ) - mha_outputs, _ = self.attention_layers[i]( - query=Q, - key=user_embeddings, - value=user_embeddings, - attn_mask=self._attn_mask, - ) - user_embeddings = self.forward_layers[i]( - F.layer_norm( - Q + mha_outputs, - normalized_shape=(self._embedding_dim,), - eps=1e-8, - ) - ) - user_embeddings *= valid_mask - return user_embeddings - - def generate_user_embeddings( - self, - past_lengths: torch.Tensor, - past_ids: torch.Tensor, - past_embeddings: torch.Tensor, - past_payloads: Dict[str, torch.Tensor], - ) -> torch.Tensor: - """ - Args: - past_ids: (B, N,) x int - - Returns: - (B, N, D,) x float - """ - past_lengths, user_embeddings, valid_mask = self._input_features_preproc( - past_lengths=past_lengths, - past_ids=past_ids, - past_embeddings=past_embeddings, - past_payloads=past_payloads, - ) - - for i in range(len(self.attention_layers)): - if self._activation_checkpoint: - user_embeddings = torch.utils.checkpoint.checkpoint( - self._run_one_layer, - i, - user_embeddings, - valid_mask, - use_reentrant=False, - ) - else: - user_embeddings = self._run_one_layer(i, user_embeddings, valid_mask) - - return self._output_postproc(user_embeddings) - - def forward( - self, - past_lengths: torch.Tensor, - past_ids: torch.Tensor, - past_embeddings: torch.Tensor, - past_payloads: Dict[str, torch.Tensor], - batch_id: Optional[int] = None, - ) -> torch.Tensor: - """ - Args: - past_ids: [B, N] x int64 where the latest engaged ids come first. In - particular, [:, 0] should correspond to the last engaged values. - past_ratings: [B, N] x int64. - past_timestamps: [B, N] x int64. - - Returns: - encoded_embeddings of [B, N, D]. - """ - encoded_embeddings = self.generate_user_embeddings( - past_lengths, - past_ids, - past_embeddings, - past_payloads, - ) - return encoded_embeddings - - def encode( - self, - past_lengths: torch.Tensor, - past_ids: torch.Tensor, # [B, N] x int64 - past_embeddings: torch.Tensor, - past_payloads: Dict[str, torch.Tensor], - ) -> torch.Tensor: - encoded_seq_embeddings = self.generate_user_embeddings( - past_lengths, past_ids, past_embeddings, past_payloads - ) # [B, N, D] - return get_current_embeddings( - lengths=past_lengths, encoded_embeddings=encoded_seq_embeddings - ) - - def predict( - self, - past_ids: torch.Tensor, - past_ratings: torch.Tensor, - past_timestamps: torch.Tensor, - next_timestamps: torch.Tensor, - target_ids: torch.Tensor, - batch_id: Optional[int] = None, - ) -> torch.Tensor: - return self.interaction( # pyre-ignore [29] - self.encode( - past_ids, - past_ratings, - past_timestamps, - next_timestamps, # pyre-ignore [6] - ), - target_ids, - ) # [B, X] diff --git a/recommendation_v4/generative_recommenders/research/modeling/sequential/utils.py b/recommendation_v4/generative_recommenders/research/modeling/sequential/utils.py deleted file mode 100644 index 60dfb8e44..000000000 --- a/recommendation_v4/generative_recommenders/research/modeling/sequential/utils.py +++ /dev/null @@ -1,129 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-unsafe - -import torch - - -def batch_gather_embeddings( - rowwise_indices: torch.Tensor, - embeddings: torch.Tensor, -) -> torch.Tensor: - """ - Args: - rowwise_indices: (B, N) x int, where each entry is in [0, X). - embeddings: (B, X, D,) x float. - - Returns: - (B, N, D,) x float, embeddings corresponding to rowwise_indices. - """ - _, N = rowwise_indices.size() - B, X, D = embeddings.size() - flattened_indices = ( - rowwise_indices - + torch.arange( - start=0, - end=B, - step=1, - dtype=rowwise_indices.dtype, - device=rowwise_indices.device, - ) - .unsqueeze(1) - .expand(-1, N) - * X - ) - return embeddings.view(-1, D)[flattened_indices, :].reshape( - rowwise_indices.size() + (D,) - ) - - -def batch_scatter_embeddings( - dst_embeddings: torch.Tensor, - rowwise_indices: torch.Tensor, - src_embeddings: torch.Tensor, -) -> None: - """ - Args: - dst_embeddings: (B, N, D,) x float. - rowwise_indices: (B,) x int, where each entry is in [0, N - 1). - source_embeddings: (B, D,) x float. - """ - B, N, D = dst_embeddings.size() - flattened_indices = rowwise_indices + torch.arange( - start=0, - end=B * N, - step=N, - dtype=rowwise_indices.dtype, - device=rowwise_indices.device, - ) - dst_embeddings.view(B * N, D)[flattened_indices, :] = src_embeddings - - -def get_current_embeddings( - lengths: torch.Tensor, - encoded_embeddings: torch.Tensor, -) -> torch.Tensor: - """ - Args: - lengths: (B,) x int - seq_embeddings: (B, N, D,) x float - - Returns: - (B, D,) x float, where [i, :] == encoded_embeddings[i, lengths[i] - 1, :] - """ - B, N, D = encoded_embeddings.size() - flattened_offsets = (lengths - 1) + torch.arange( - start=0, end=B, step=1, dtype=lengths.dtype, device=lengths.device - ) * N - return encoded_embeddings.reshape(-1, D)[flattened_offsets, :].reshape(B, D) - - -def jagged_or_dense_repeat_interleave_dim0( - x: torch.Tensor, lengths: torch.Tensor, repeats: int -) -> torch.Tensor: - if len(x.size()) == 3: - return x.repeat_interleave(repeats, dim=0) - else: - assert len(x.size()) == 2, f"x.size() = {x.size()}" - padded_x = torch.ops.fbgemm.jagged_to_padded_dense( - values=x, - offsets=[torch.ops.fbgemm.asynchronous_complete_cumsum(lengths)], - max_lengths=[lengths.max()], - padding_value=0.0, - ) - lengths = lengths.repeat_interleave(repeats, dim=0) - return torch.ops.fbgemm.dense_to_jagged( - padded_x.repeat_interleave(repeats, dim=0), - [torch.ops.fbgemm.asynchronous_complete_cumsum(lengths)], - )[0] - - -def jagged_or_dense_index_select_dim0( - x: torch.Tensor, lengths: torch.Tensor, indices: torch.Tensor -) -> torch.Tensor: - if len(x.size()) == 3: - return x[indices, :, :] - else: - assert len(x.size()) == 2, f"x.size() = {x.size()}" - padded_x = torch.ops.fbgemm.jagged_to_padded_dense( - values=x, - offsets=[torch.ops.fbgemm.asynchronous_complete_cumsum(lengths)], - max_lengths=[lengths.max()], - padding_value=0.0, - ) - return torch.ops.fbgemm.dense_to_jagged( - padded_x[indices, :], - [torch.ops.fbgemm.asynchronous_complete_cumsum(lengths[indices])], - )[0] diff --git a/recommendation_v4/generative_recommenders/research/modeling/similarity_module.py b/recommendation_v4/generative_recommenders/research/modeling/similarity_module.py deleted file mode 100644 index 3ba32d239..000000000 --- a/recommendation_v4/generative_recommenders/research/modeling/similarity_module.py +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-unsafe - -import abc -from typing import Optional - -import torch -from generative_recommenders.research.rails.similarities.module import SimilarityModule - - -class SequentialEncoderWithLearnedSimilarityModule(torch.nn.Module): - """ - Interface enabling using various similarity functions (besides inner products) - as part of a sequential encoder/decoder. - - See rails/ for more details. - """ - - def __init__( - self, - ndp_module: SimilarityModule, - ) -> None: - super().__init__() - - self._ndp_module: SimilarityModule = ndp_module - - @abc.abstractmethod - def debug_str( - self, - ) -> str: - pass - - def similarity_fn( - self, - query_embeddings: torch.Tensor, - item_ids: torch.Tensor, - item_embeddings: Optional[torch.Tensor] = None, - **kwargs, - ) -> torch.Tensor: - torch._assert( - len(query_embeddings.size()) == 2, "len(query_embeddings.size()) must be 2" - ) - torch._assert(len(item_ids.size()) == 2, "len(item_ids.size()) must be 2") - if item_embeddings is None: - item_embeddings = self.get_item_embeddings(item_ids) # pyre-ignore [29] - torch._assert( - len(item_embeddings.size()) == 3, "len(item_embeddings.size()) must be 3" - ) - - return self._ndp_module( - query_embeddings=query_embeddings, # (B, query_embedding_dim) - item_embeddings=item_embeddings, # (1/B, X, item_embedding_dim) - item_ids=item_ids, - **kwargs, - ) diff --git a/recommendation_v4/generative_recommenders/research/modeling/similarity_utils.py b/recommendation_v4/generative_recommenders/research/modeling/similarity_utils.py deleted file mode 100644 index 7fd870b4b..000000000 --- a/recommendation_v4/generative_recommenders/research/modeling/similarity_utils.py +++ /dev/null @@ -1,222 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-unsafe - -from typing import List, Optional, Tuple - -import gin -import torch -from generative_recommenders.research.rails.similarities.dot_product_similarity_fn import ( - DotProductSimilarity, -) -from generative_recommenders.research.rails.similarities.layers import SwiGLU -from generative_recommenders.research.rails.similarities.mol.item_embeddings_fn import ( - RecoMoLItemEmbeddingsFn, -) -from generative_recommenders.research.rails.similarities.mol.query_embeddings_fn import ( - RecoMoLQueryEmbeddingsFn, -) -from generative_recommenders.research.rails.similarities.mol.similarity_fn import ( - MoLSimilarity, - SoftmaxDropoutCombiner, -) - - -def init_mlp_xavier_weights_zero_bias(m) -> None: - if isinstance(m, torch.nn.Linear): - torch.nn.init.xavier_uniform(m.weight) - if getattr(m, "bias", None) is not None: - m.bias.data.fill_(0.0) - - -@gin.configurable -def create_mol_interaction_module( - query_embedding_dim: int, - item_embedding_dim: int, - dot_product_dimension: int, - query_dot_product_groups: int, - item_dot_product_groups: int, - temperature: float, - query_dropout_rate: float, - query_hidden_dim: int, - item_dropout_rate: float, - item_hidden_dim: int, - gating_query_hidden_dim: int, - gating_qi_hidden_dim: int, - gating_item_hidden_dim: int, - softmax_dropout_rate: float, - bf16_training: bool, - gating_query_fn: bool = True, - gating_item_fn: bool = True, - dot_product_l2_norm: bool = True, - query_nonlinearity: str = "geglu", - item_nonlinearity: str = "geglu", - uid_dropout_rate: float = 0.5, - uid_embedding_hash_sizes: Optional[List[int]] = None, - uid_embedding_level_dropout: bool = False, - gating_combination_type: str = "glu_silu", - gating_item_dropout_rate: float = 0.0, - gating_qi_dropout_rate: float = 0.0, - eps: float = 1e-6, -) -> Tuple[MoLSimilarity, str]: - """ - Gin wrapper for creating MoL learned similarity. - """ - mol_module = MoLSimilarity( - query_embedding_dim=query_embedding_dim, - item_embedding_dim=item_embedding_dim, - dot_product_dimension=dot_product_dimension, - query_dot_product_groups=query_dot_product_groups, - item_dot_product_groups=item_dot_product_groups, - temperature=temperature, - dot_product_l2_norm=dot_product_l2_norm, - query_embeddings_fn=RecoMoLQueryEmbeddingsFn( - query_embedding_dim=query_embedding_dim, - query_dot_product_groups=query_dot_product_groups, - dot_product_dimension=dot_product_dimension, - dot_product_l2_norm=dot_product_l2_norm, - proj_fn=lambda input_dim, output_dim: ( - torch.nn.Sequential( - torch.nn.Dropout(p=query_dropout_rate), - SwiGLU( - in_features=input_dim, - out_features=query_hidden_dim, - ), - torch.nn.Linear( - in_features=query_hidden_dim, - out_features=output_dim, - ), - ).apply(init_mlp_xavier_weights_zero_bias) - ), - eps=eps, - ), - item_embeddings_fn=RecoMoLItemEmbeddingsFn( - item_embedding_dim=item_embedding_dim, - item_dot_product_groups=item_dot_product_groups, - dot_product_dimension=dot_product_dimension, - dot_product_l2_norm=dot_product_l2_norm, - proj_fn=lambda input_dim, output_dim: ( - torch.nn.Sequential( - torch.nn.Dropout(p=item_dropout_rate), - SwiGLU(in_features=input_dim, out_features=item_hidden_dim), - torch.nn.Linear( - in_features=item_hidden_dim, - out_features=output_dim, - ), - ).apply(init_mlp_xavier_weights_zero_bias) - ), - eps=eps, - ), - gating_query_only_partial_fn=lambda input_dim, output_dim: ( # pyre-ignore [6] - torch.nn.Sequential( - torch.nn.Linear( - in_features=input_dim, - out_features=gating_query_hidden_dim, - ), - torch.nn.SiLU(), - torch.nn.Linear( - in_features=gating_query_hidden_dim, - out_features=output_dim, - bias=False, - ), - ).apply(init_mlp_xavier_weights_zero_bias) - if gating_query_fn - else None - ), - gating_item_only_partial_fn=lambda input_dim, output_dim: ( # pyre-ignore [6] - torch.nn.Sequential( - torch.nn.Dropout(p=gating_item_dropout_rate), - torch.nn.Linear( - in_features=input_dim, - out_features=gating_item_hidden_dim, - ), - torch.nn.SiLU(), - torch.nn.Linear( - in_features=gating_item_hidden_dim, - out_features=output_dim, - bias=False, - ), - ).apply(init_mlp_xavier_weights_zero_bias) - if gating_item_fn - else None - ), - gating_qi_partial_fn=lambda input_dim, output_dim: ( # pyre-ignore [6] - torch.nn.Sequential( - torch.nn.Dropout(p=gating_qi_dropout_rate), - torch.nn.Linear( - in_features=input_dim, - out_features=gating_qi_hidden_dim, - ), - torch.nn.SiLU(), - torch.nn.Linear( - in_features=gating_qi_hidden_dim, - out_features=output_dim, - ), - ).apply(init_mlp_xavier_weights_zero_bias) - if gating_qi_hidden_dim > 0 - else torch.nn.Sequential( - torch.nn.Dropout(p=gating_qi_dropout_rate), - torch.nn.Linear( - in_features=input_dim, - out_features=output_dim, - ), - ).apply(init_mlp_xavier_weights_zero_bias) - ), - gating_combination_type=gating_combination_type, - gating_normalization_fn=lambda _: SoftmaxDropoutCombiner( - dropout_rate=softmax_dropout_rate, eps=1e-6 - ), - eps=eps, - autocast_bf16=bf16_training, - ) - interaction_module_debug_str = ( - f"MoL-{query_dot_product_groups}x{item_dot_product_groups}x{dot_product_dimension}" - + f"-t{temperature}-d{softmax_dropout_rate}" - + f"{'-l2' if dot_product_l2_norm else ''}" - + f"-q{query_hidden_dim}d{query_dropout_rate}{query_nonlinearity}" - + f"-i{item_hidden_dim}d{item_dropout_rate}{item_nonlinearity}" - + (f"-gq{gating_query_hidden_dim}" if gating_query_fn else "") - + ( - f"-gi{gating_item_hidden_dim}d{gating_item_dropout_rate}" - if gating_item_fn - else "" - ) - + f"-gqi{gating_qi_hidden_dim}d{gating_qi_dropout_rate}-x-{gating_combination_type}" - ) - return mol_module, interaction_module_debug_str - - -@gin.configurable -def get_similarity_function( - module_type: str, - query_embedding_dim: int, - item_embedding_dim: int, - bf16_training: bool = False, - activation_checkpoint: bool = False, -) -> Tuple[torch.nn.Module, str]: - if module_type == "DotProduct": - interaction_module = DotProductSimilarity() - interaction_module_debug_str = "DotProduct" - elif module_type == "MoL": - interaction_module, interaction_module_debug_str = ( - create_mol_interaction_module( - query_embedding_dim=query_embedding_dim, - item_embedding_dim=item_embedding_dim, - bf16_training=bf16_training, - ) - ) - else: - raise ValueError(f"Unknown interaction_module_type {module_type}") - return interaction_module, interaction_module_debug_str diff --git a/recommendation_v4/generative_recommenders/research/rails/indexing/candidate_index.py b/recommendation_v4/generative_recommenders/research/rails/indexing/candidate_index.py deleted file mode 100644 index f628468ce..000000000 --- a/recommendation_v4/generative_recommenders/research/rails/indexing/candidate_index.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-unsafe - -import abc -from typing import Tuple - -import torch - - -class TopKModule(torch.nn.Module): - @abc.abstractmethod - def forward( - self, - query_embeddings: torch.Tensor, - k: int, - sorted: bool = True, - **kwargs, - ) -> Tuple[torch.Tensor, torch.Tensor]: - """ - Args: - query_embeddings: (B, X, ...). Implementation-specific. - k: int. top k to return. - sorted: bool. - - Returns: - Tuple of (top_k_scores, top_k_ids), both of shape (B, K,) - """ - pass diff --git a/recommendation_v4/generative_recommenders/research/rails/indexing/mips_top_k.py b/recommendation_v4/generative_recommenders/research/rails/indexing/mips_top_k.py deleted file mode 100644 index 810b24c42..000000000 --- a/recommendation_v4/generative_recommenders/research/rails/indexing/mips_top_k.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-unsafe - -from typing import Tuple - -import torch -from generative_recommenders.research.rails.indexing.candidate_index import TopKModule - - -class MIPSTopKModule(TopKModule): - def __init__( - self, - item_embeddings: torch.Tensor, - item_ids: torch.Tensor, - ) -> None: - """ - Args: - item_embeddings: (1, X, D) - item_ids: (1, X,) - """ - super().__init__() - - self._item_embeddings: torch.Tensor = item_embeddings - self._item_ids: torch.Tensor = item_ids - - -class MIPSBruteForceTopK(MIPSTopKModule): - def __init__( - self, - item_embeddings: torch.Tensor, - item_ids: torch.Tensor, - ) -> None: - super().__init__( - item_embeddings=item_embeddings, - item_ids=item_ids, - ) - del self._item_embeddings - self._item_embeddings_t: torch.Tensor = item_embeddings.permute( - 2, 1, 0 - ).squeeze(2) - - def forward( - self, - query_embeddings: torch.Tensor, - k: int, - sorted: bool = True, - **kwargs, - ) -> Tuple[torch.Tensor, torch.Tensor]: - """ - Args: - query_embeddings: (B, ...). Implementation-specific. - k: int. final top-k to return. - sorted: bool. whether to sort final top-k results or not. - - Returns: - Tuple of (top_k_scores x float, top_k_ids x int), both of shape (B, K,) - """ - # (B, X,) - all_logits = torch.mm(query_embeddings, self._item_embeddings_t) - top_k_logits, top_k_indices = torch.topk( - all_logits, - dim=1, - k=k, - sorted=sorted, - largest=True, - ) # (B, k,) - return top_k_logits, self._item_ids.squeeze(0)[top_k_indices] diff --git a/recommendation_v4/generative_recommenders/research/rails/indexing/mol_top_k.py b/recommendation_v4/generative_recommenders/research/rails/indexing/mol_top_k.py deleted file mode 100644 index fe88ca919..000000000 --- a/recommendation_v4/generative_recommenders/research/rails/indexing/mol_top_k.py +++ /dev/null @@ -1,132 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-unsafe - -""" -Defines exact- and approximate- Top-K modules for Mixture-of-Logits (MoL), -discussed in Retrieval with Learned Similarities (https://arxiv.org/abs/2407.15462). - -Forked from bailuding/rails @ 664fdb9. -""" - -from typing import Tuple - -import torch -from generative_recommenders.research.rails.indexing.candidate_index import TopKModule -from generative_recommenders.research.rails.similarities.mol.similarity_fn import ( - MoLSimilarity, -) - - -class MoLTopKModule(TopKModule): - def __init__( - self, - mol_module: MoLSimilarity, - item_embeddings: torch.Tensor, - item_ids: torch.Tensor, - flatten_item_ids_and_embeddings: bool, - keep_component_level_item_embeddings: bool, - component_level_item_embeddings_dtype: torch.dtype = torch.bfloat16, - ) -> None: - """ - Args: - mol_module: MoLSimilarity. - item_embeddings: (1, X, D) if mol_module._apply_item_embeddings_fn is True, - (1, X, P_X, D_P) otherwise. - item_ids: (1, X,) representing the item ids. - flatten_item_ids_and_embeddings: bool. If true, do not keep the extra (1,) - dimension at size(0). - keep_component_level_item_embeddings: bool. If true, keep P_x component-level - embeddings in `self._mol_item_embeddings` for downstream applications. - component_level_item_embeddings_dtype: torch.dtype. If set, the dtype - to keep component-level item embeddings in. By default we use bfloat16. - """ - super().__init__() - - self._mol_module: MoLSimilarity = mol_module - self._item_embeddings: torch.Tensor = ( - item_embeddings - if not flatten_item_ids_and_embeddings - else item_embeddings.squeeze(0) - ) - - if keep_component_level_item_embeddings: - self._mol_item_embeddings: torch.Tensor = ( - mol_module.get_item_component_embeddings( - ( - self._item_embeddings.squeeze(0) - if not flatten_item_ids_and_embeddings - else self._item_embeddings - ), - decoupled_inference=True, - )[0] # (X, D) -> (X, P_X, D_P) - ).to(component_level_item_embeddings_dtype) - - self._item_ids: torch.Tensor = ( - item_ids if not flatten_item_ids_and_embeddings else item_ids.squeeze(0) - ) - - @property - def mol_module(self) -> MoLSimilarity: - return self._mol_module - - -class MoLBruteForceTopK(MoLTopKModule): - def __init__( - self, - mol_module: MoLSimilarity, - item_embeddings: torch.Tensor, - item_ids: torch.Tensor, - ) -> None: - super().__init__( - mol_module=mol_module, - item_embeddings=item_embeddings, - item_ids=item_ids, - flatten_item_ids_and_embeddings=False, - keep_component_level_item_embeddings=False, - ) - - def forward( - self, - query_embeddings: torch.Tensor, - k: int, - sorted: bool = True, - **kwargs, - ) -> Tuple[torch.Tensor, torch.Tensor]: - """ - Args: - query_embeddings: (B, X, D) if mol_module._apply_query_embeddings_fn is True, - (B, X, P_Q, D_P) otherwise. - k: int. final top-k to return. - sorted: bool. whether to sort final top-k results or not. - **kwargs: Implementation-specific keys/values. - - Returns: - Tuple of (top_k_scores x float, top_k_ids x int), both of shape (B, K,) - """ - # (B, X,) - all_logits, _ = self.mol_module( - query_embeddings, - self._item_embeddings, - **kwargs, - ) - top_k_logits, top_k_indices = torch.topk( - all_logits, - dim=1, - k=k, - sorted=sorted, - largest=True, - ) # (B, k,) - return top_k_logits, self._item_ids.squeeze(0)[top_k_indices] diff --git a/recommendation_v4/generative_recommenders/research/rails/similarities/dot_product_similarity_fn.py b/recommendation_v4/generative_recommenders/research/rails/similarities/dot_product_similarity_fn.py deleted file mode 100644 index 9357fd0e4..000000000 --- a/recommendation_v4/generative_recommenders/research/rails/similarities/dot_product_similarity_fn.py +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-unsafe - -from typing import Dict, Tuple - -import torch -from generative_recommenders.research.rails.similarities.module import SimilarityModule - - -class DotProductSimilarity(SimilarityModule): - def __init__( - self, - ) -> None: - super().__init__() - - def debug_str(self) -> str: - return "dp" - - def forward( - self, - query_embeddings: torch.Tensor, - item_embeddings: torch.Tensor, - **kwargs, - ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: - """ - Args: - query_embeddings: (B, D,) or (B * r, D) x float. - item_embeddings: (1, X, D) or (B, X, D) x float. - - Returns: - (B, X) x float. - """ - - B_I, X, D = item_embeddings.size() - if B_I == 1: - # [B, D] x ([1, X, D] -> [D, X]) => [B, X] - return ( - torch.mm(query_embeddings, item_embeddings.squeeze(0).t()), - {}, - ) # [B, X] - elif query_embeddings.size(0) != B_I: - # (B * r, D) x (B, X, D). - return ( - torch.bmm( - query_embeddings.view(B_I, -1, D), - item_embeddings.permute(0, 2, 1), - ).view(-1, X), - {}, - ) - else: - # [B, X, D] x ([B, D] -> [B, D, 1]) => [B, X, 1] -> [B, X] - return ( - torch.bmm(item_embeddings, query_embeddings.unsqueeze(2)).squeeze(2), - {}, - ) diff --git a/recommendation_v4/generative_recommenders/research/rails/similarities/layers.py b/recommendation_v4/generative_recommenders/research/rails/similarities/layers.py deleted file mode 100644 index 3f838bc48..000000000 --- a/recommendation_v4/generative_recommenders/research/rails/similarities/layers.py +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-unsafe - -""" -Defines network architectures used in constructing various learned similarities. - -Forked from bailuding/rails @ 664fdb9. -""" - -import torch -import torch.nn.functional as F - - -class GeGLU(torch.nn.Module): - def __init__( - self, - in_features: int, - out_features: int, - ) -> None: - super().__init__() - - self._in_features = in_features - self._out_features = out_features - self._w = torch.nn.Parameter( - torch.empty((in_features, out_features * 2)).normal_(mean=0, std=0.02), - ) - self._b = torch.nn.Parameter( - torch.zeros((1, out_features * 2)), - ) - - def forward(self, x: torch.Tensor) -> torch.Tensor: - bs = x.size()[:-1] - lhs, rhs = torch.split( - torch.mm(x.reshape(-1, self._in_features), self._w) + self._b, - [self._out_features, self._out_features], - dim=-1, - ) - return (F.gelu(lhs) * rhs).reshape(bs + (self._out_features,)) - - -class SwiGLU(torch.nn.Module): - """ - SwiGLU from https://arxiv.org/abs/2002.05202. - """ - - def __init__( - self, - in_features: int, - out_features: int, - ) -> None: - super().__init__() - - self._in_features = in_features - self._out_features = out_features - self._w = torch.nn.Parameter( - torch.empty((in_features, out_features * 2)).normal_(mean=0, std=0.02), - ) - self._b = torch.nn.Parameter( - torch.zeros((1, out_features * 2)), - ) - - def forward(self, x: torch.Tensor) -> torch.Tensor: - bs = x.size()[:-1] - lhs, rhs = torch.split( - torch.mm(x.reshape(-1, self._in_features), self._w) + self._b, - [self._out_features, self._out_features], - dim=-1, - ) - return (F.silu(lhs) * rhs).reshape(bs + (self._out_features,)) diff --git a/recommendation_v4/generative_recommenders/research/rails/similarities/module.py b/recommendation_v4/generative_recommenders/research/rails/similarities/module.py deleted file mode 100644 index e4061fa74..000000000 --- a/recommendation_v4/generative_recommenders/research/rails/similarities/module.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-unsafe - -import abc -from typing import Dict, Tuple - -import torch - - -class SimilarityModule(torch.nn.Module): - """ - Interface enabling interfacing with various similarity functions. - - While the discussions in our initial ICML'24 paper are based on inner products - for simplicity, we provide this interface (SimilarityModule) to support various - learned similarities at the retrieval stage, such as MLPs, Factorization Machines - (FMs), and Mixture-of-Logits (MoL), which we discussed in - - Revisiting Neural Retrieval on Accelerators (KDD'23), and - - Retrieval with Learned Similarities (https://arxiv.org/abs/2407.15462). - """ - - @abc.abstractmethod - def forward( - self, - query_embeddings: torch.Tensor, - item_embeddings: torch.Tensor, - **kwargs, - ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: - """ - Args: - query_embeddings: (B, input_embedding_dim) x float. - item_embeddings: (1/B, X, item_embedding_dim) x float. - **kwargs: Implementation-specific keys/values (e.g., - item ids / sideinfo, etc.) - - Returns: - A tuple of ( - (B, X,) similarity values, - keyed outputs representing auxiliary losses at training time. - ). - """ - pass diff --git a/recommendation_v4/generative_recommenders/research/rails/similarities/mol/embeddings_fn.py b/recommendation_v4/generative_recommenders/research/rails/similarities/mol/embeddings_fn.py deleted file mode 100644 index fd94e6f22..000000000 --- a/recommendation_v4/generative_recommenders/research/rails/similarities/mol/embeddings_fn.py +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-unsafe - -""" -Defines interface for generating query- and item-side embeddings for MoL. - -Forked from bailuding/rails @ 664fdb9. -""" - -import abc -from typing import Dict, Tuple - -import torch - - -class MoLEmbeddingsFn(torch.nn.Module): - """ - Generates K_Q query-side (K_I item-side) embeddings for MoL based on - input embeddings and other optional implementation-specific tensors. - """ - - @abc.abstractmethod - def forward( - self, - input_embeddings: torch.Tensor, - **kwargs, - ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: - """ - Args: - input_embeddings: (B, ...) x float where B is the batch size. - kwargs: implementation-specific. - - Returns: - Tuple of ( - (B, query_dot_product_groups/item_dot_product_groups, dot_product_embedding_dim) x float, - str-keyed auxiliary losses. - ). - """ - pass diff --git a/recommendation_v4/generative_recommenders/research/rails/similarities/mol/item_embeddings_fn.py b/recommendation_v4/generative_recommenders/research/rails/similarities/mol/item_embeddings_fn.py deleted file mode 100644 index 237cd8942..000000000 --- a/recommendation_v4/generative_recommenders/research/rails/similarities/mol/item_embeddings_fn.py +++ /dev/null @@ -1,99 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-unsafe - -""" -Defines functions to generate item-side embeddings for MoL. - -Forked from bailuding/rails @ 664fdb9. -""" - -from typing import Callable, Dict, Tuple - -import torch -from generative_recommenders.research.rails.similarities.mol.embeddings_fn import ( - MoLEmbeddingsFn, -) - - -def init_mlp_xavier_weights_zero_bias(m) -> None: - if isinstance(m, torch.nn.Linear): - torch.nn.init.xavier_uniform_(m.weight) - if getattr(m, "bias", None) is not None: - m.bias.data.fill_(0.0) - - -class RecoMoLItemEmbeddingsFn(MoLEmbeddingsFn): - """ - Generates P_X query-side embeddings for MoL based on input embeddings and other - optional tensors for recommendation models. Tested for sequential retrieval - scenarios. - """ - - def __init__( - self, - item_embedding_dim: int, - item_dot_product_groups: int, - dot_product_dimension: int, - dot_product_l2_norm: bool, - proj_fn: Callable[[int, int], torch.nn.Module], - eps: float, - ) -> None: - super().__init__() - - self._item_emb_based_dot_product_groups: int = item_dot_product_groups - self._item_emb_proj_module: torch.nn.Module = proj_fn( - item_embedding_dim, - dot_product_dimension * self._item_emb_based_dot_product_groups, - ) - self._dot_product_dimension: int = dot_product_dimension - self._dot_product_l2_norm: bool = dot_product_l2_norm - self._eps: float = eps - - def forward( - self, - input_embeddings: torch.Tensor, - **kwargs, - ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: - """ - Args: - input_embeddings: (B, item_embedding_dim,) x float where B is the batch size. - kwargs: str-keyed tensors. Implementation-specific. - - Returns: - Tuple of ( - (B, item_dot_product_groups, dot_product_embedding_dim) x float, - str-keyed aux_losses, - ). - """ - split_item_embeddings = self._item_emb_proj_module(input_embeddings).reshape( - input_embeddings.size()[:-1] - + ( - self._item_emb_based_dot_product_groups, - self._dot_product_dimension, - ) - ) - - if self._dot_product_l2_norm: - split_item_embeddings = split_item_embeddings / torch.clamp( - torch.linalg.norm( - split_item_embeddings, - ord=None, - dim=-1, - keepdim=True, - ), - min=self._eps, - ) - return split_item_embeddings, {} diff --git a/recommendation_v4/generative_recommenders/research/rails/similarities/mol/query_embeddings_fn.py b/recommendation_v4/generative_recommenders/research/rails/similarities/mol/query_embeddings_fn.py deleted file mode 100644 index 8fe28ee11..000000000 --- a/recommendation_v4/generative_recommenders/research/rails/similarities/mol/query_embeddings_fn.py +++ /dev/null @@ -1,164 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-unsafe - -""" -Defines functions to generate query-side embeddings for MoL. - -Forked from bailuding/rails @ 664fdb9. -""" - -from typing import Callable, Dict, List, Optional, Tuple - -import torch -import torch.nn.functional as F -from generative_recommenders.research.rails.similarities.mol.embeddings_fn import ( - MoLEmbeddingsFn, -) - - -def init_mlp_xavier_weights_zero_bias(m) -> None: - if isinstance(m, torch.nn.Linear): - torch.nn.init.xavier_uniform_(m.weight) - if getattr(m, "bias", None) is not None: - m.bias.data.fill_(0.0) - - -class RecoMoLQueryEmbeddingsFn(MoLEmbeddingsFn): - """ - Generates P_Q query-side embeddings for MoL based on input embeddings and other - optional tensors for recommendation models. Tested for sequential retrieval - scenarios. - - The current implementation accesses user_ids associated with the query from - `user_ids' in kwargs. - """ - - def __init__( - self, - query_embedding_dim: int, - query_dot_product_groups: int, - dot_product_dimension: int, - dot_product_l2_norm: bool, - proj_fn: Callable[[int, int], torch.nn.Module], - eps: float, - uid_embedding_hash_sizes: Optional[List[int]] = None, - uid_dropout_rate: float = 0.0, - uid_embedding_level_dropout: bool = False, - ) -> None: - super().__init__() - self._uid_embedding_hash_sizes: List[int] = uid_embedding_hash_sizes or [] - self._query_emb_based_dot_product_groups: int = query_dot_product_groups - len( - self._uid_embedding_hash_sizes - ) - self._query_emb_proj_module: torch.nn.Module = proj_fn( - query_embedding_dim, - dot_product_dimension * self._query_emb_based_dot_product_groups, - ) - self._dot_product_dimension: int = dot_product_dimension - self._dot_product_l2_norm: bool = dot_product_l2_norm - if len(self._uid_embedding_hash_sizes) > 0: - for i, hash_size in enumerate(self._uid_embedding_hash_sizes): - setattr( - self, - f"_uid_embeddings_{i}", - torch.nn.Embedding( - hash_size + 1, dot_product_dimension, padding_idx=0 - ), - ) - self._uid_dropout_rate: float = uid_dropout_rate - self._uid_embedding_level_dropout: bool = uid_embedding_level_dropout - self._eps: float = eps - - def forward( - self, - input_embeddings: torch.Tensor, - **kwargs, - ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: - """ - Args: - input_embeddings: (B, query_embedding_dim,) x float where B is the batch size. - kwargs: str-keyed tensors. Implementation-specific. - - Returns: - Tuple of ( - (B, query_dot_product_groups, dot_product_embedding_dim) x float, - str-keyed aux_losses, - ). - """ - split_query_embeddings = self._query_emb_proj_module(input_embeddings).reshape( - ( - input_embeddings.size(0), - self._query_emb_based_dot_product_groups, - self._dot_product_dimension, - ) - ) - - aux_losses: Dict[str, torch.Tensor] = {} - - if len(self._uid_embedding_hash_sizes) > 0: - all_uid_embeddings = [] - for i, hash_size in enumerate(self._uid_embedding_hash_sizes): - # TODO: decouple this from MoLQueryEmbeddingFn. - uid_embeddings = getattr(self, f"_uid_embeddings_{i}")( - (kwargs["user_ids"] % hash_size) + 1 - ) - if self.training: - l2_norm = (uid_embeddings * uid_embeddings).sum(-1).mean() - if i == 0: - aux_losses["uid_embedding_l2_norm"] = l2_norm - else: - aux_losses["uid_embedding_l2_norm"] = ( - aux_losses["uid_embedding_l2_norm"] + l2_norm - ) - - if self._uid_dropout_rate > 0.0: - if self._uid_embedding_level_dropout: - # conditionally dropout the entire embedding. - if self.training: - uid_dropout_mask = ( - torch.rand( - uid_embeddings.size()[:-1], - device=uid_embeddings.device, - ) - > self._uid_dropout_rate - ) - uid_embeddings = ( - uid_embeddings - * uid_dropout_mask.unsqueeze(-1) - / (1.0 - self._uid_dropout_rate) - ) - else: - uid_embeddings = F.dropout( - uid_embeddings, - p=self._uid_dropout_rate, - training=self.training, - ) - all_uid_embeddings.append(uid_embeddings.unsqueeze(1)) - split_query_embeddings = torch.cat( - [split_query_embeddings] + all_uid_embeddings, dim=1 - ) - - if self._dot_product_l2_norm: - split_query_embeddings = split_query_embeddings / torch.clamp( - torch.linalg.norm( - split_query_embeddings, - ord=None, - dim=-1, - keepdim=True, - ), - min=self._eps, - ) - return split_query_embeddings, aux_losses diff --git a/recommendation_v4/generative_recommenders/research/rails/similarities/mol/similarity_fn.py b/recommendation_v4/generative_recommenders/research/rails/similarities/mol/similarity_fn.py deleted file mode 100644 index 34e4c4a23..000000000 --- a/recommendation_v4/generative_recommenders/research/rails/similarities/mol/similarity_fn.py +++ /dev/null @@ -1,388 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-unsafe - -""" -Implements MoL (Mixture-of-Logits) with load balancing regularization loss, as discussed in: -- Revisiting Neural Retrieval on Accelerators (https://arxiv.org/abs/2306.04039, KDD'23). -- Retrieval with Learned Similarities (https://arxiv.org/abs/2407.15462). - -Forked from bailuding/rails @ 664fdb9. -""" - -from typing import Callable, Dict, Optional, Tuple - -import torch -import torch.nn.functional as F -from generative_recommenders.research.rails.similarities.module import SimilarityModule -from generative_recommenders.research.rails.similarities.mol.embeddings_fn import ( - MoLEmbeddingsFn, -) - - -@torch.compile(dynamic=True) -def _softmax_dropout_combiner_fn( - x: torch.Tensor, - y: torch.Tensor, - dropout_pr: float, - eps: float, - training: bool, -) -> Tuple[torch.Tensor, torch.Tensor]: - """ - Computes (_softmax_dropout_fn(x) * y).sum(-1). - """ - x = F.softmax(x, dim=-1) - if dropout_pr > 0.0: - x = F.dropout(x, p=dropout_pr, training=training) - x = x / torch.clamp(x.sum(-1, keepdims=True), min=eps) # pyre-ignore [19] - return x, (x * y).sum(-1) - - -@torch.compile -def _load_balancing_mi_loss_fn( - gating_prs: torch.Tensor, - eps: float, -) -> torch.Tensor: - """ - See Retrieval with Learned Similarities (RAILS, https://arxiv.org/abs/2407.15462) for discussions. - """ - B, X, E = gating_prs.size() - expert_util_prs = gating_prs.view(B * X, E).sum(0, keepdim=False) / (1.0 * B * X) - expert_util_entropy = -(expert_util_prs * torch.log(expert_util_prs + eps)).sum() - per_example_expert_entropy = -(gating_prs * torch.log(gating_prs + eps)).sum() / ( - 1.0 * B * X - ) - return -expert_util_entropy + per_example_expert_entropy - - -class SoftmaxDropoutCombiner(torch.nn.Module): - def __init__( - self, - dropout_rate: float, - eps: float, - ) -> None: - super().__init__() - - self._dropout_rate: float = dropout_rate - self._eps: float = eps - - def forward( - self, - gating_weights: torch.Tensor, - x: torch.Tensor, - ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: - gating_prs, combined_logits = _softmax_dropout_combiner_fn( - x=gating_weights, - y=x, - dropout_pr=self._dropout_rate, - eps=self._eps, - training=self.training, - ) - - aux_losses = {} - if self.training: - aux_losses["mi_loss"] = _load_balancing_mi_loss_fn( - gating_prs, eps=self._eps - ) - - return combined_logits, aux_losses - - -class MoLGatingFn(torch.nn.Module): - """ - Implements the gating function for MoL, used to compute pi_p(q, x) for a given (p, x) pair. - """ - - def __init__( - self, - num_logits: int, - query_embedding_dim: int, - item_embedding_dim: int, - query_only_partial_fn: Optional[Callable[[int, int], torch.nn.Module]], - item_only_partial_fn: Optional[Callable[[int, int], torch.nn.Module]], - qi_partial_fn: Optional[Callable[[int, int], torch.nn.Module]], - combination_type: str, - normalization_fn: Callable[[int], torch.nn.Module], - ) -> None: - super().__init__() - - self._query_only_partial_module: Optional[torch.nn.Module] = ( - query_only_partial_fn(query_embedding_dim, num_logits) - if query_only_partial_fn - else None - ) - self._item_only_partial_module: Optional[torch.nn.Module] = ( - item_only_partial_fn(item_embedding_dim, num_logits) - if item_only_partial_fn - else None - ) - self._qi_partial_module: Optional[torch.nn.Module] = ( - qi_partial_fn( - num_logits, - num_logits, - ) - if qi_partial_fn is not None - else None - ) - if ( - self._query_only_partial_module is None - and self._item_only_partial_module is None - and self._qi_partial_module is None - ): - raise ValueError( - "At least one of query_only_partial_fn, item_only_partial_fn, " - "and qi_partial_fn must not be None." - ) - self._num_logits: int = num_logits - self._combination_type: str = combination_type - self._normalization_fn: torch.nn.Module = normalization_fn(num_logits) - - def forward( - self, - logits: torch.Tensor, - query_embeddings: torch.Tensor, - item_embeddings: torch.Tensor, - ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: - """ - Args: - logits: (B, X, P_Q * P_X) x float; - query_embeddings: (B, D) x float; - item_embeddings: (1/B, X, D') x float; - - Returns: - (B, X) x float, Dict[str, Tensor] representing auxiliary losses. - """ - B, X, _ = logits.size() - # [B, 1, F], [1/B, X, F], [B, X, F] - query_partial_inputs, item_partial_inputs, qi_partial_inputs = None, None, None - if self._query_only_partial_module is not None: - query_partial_inputs = self._query_only_partial_module( - query_embeddings - ).unsqueeze(1) - if self._item_only_partial_module is not None: - item_partial_inputs = self._item_only_partial_module(item_embeddings) - if self._qi_partial_module is not None: - qi_partial_inputs = self._qi_partial_module(logits) - - if self._combination_type == "glu_silu": - gating_inputs = ( - query_partial_inputs * item_partial_inputs + qi_partial_inputs - ) - gating_weights = gating_inputs * F.sigmoid(gating_inputs) - elif self._combination_type == "glu_silu_ln": - gating_inputs = ( - query_partial_inputs * item_partial_inputs + qi_partial_inputs - ) - gating_weights = gating_inputs * F.sigmoid( - F.layer_norm(gating_inputs, normalized_shape=[self._num_logits]) - ) - elif self._combination_type == "none": - gating_inputs = query_partial_inputs - if gating_inputs is None: - gating_inputs = item_partial_inputs - elif item_partial_inputs is not None: - gating_inputs += item_partial_inputs - if gating_inputs is None: - gating_inputs = qi_partial_inputs - elif qi_partial_inputs is not None: - gating_inputs += qi_partial_inputs - gating_weights = gating_inputs - else: - raise ValueError(f"Unknown combination_type {self._combination_type}") - - return self._normalization_fn(gating_weights, logits) - - -class MoLSimilarity(SimilarityModule): - def __init__( - self, - query_embedding_dim: int, - item_embedding_dim: int, - dot_product_dimension: int, - query_dot_product_groups: int, - item_dot_product_groups: int, - temperature: float, - dot_product_l2_norm: bool, - query_embeddings_fn: MoLEmbeddingsFn, - item_embeddings_fn: Optional[MoLEmbeddingsFn], - gating_query_only_partial_fn: Optional[Callable[[int, int], torch.nn.Module]], - gating_item_only_partial_fn: Optional[Callable[[int, int], torch.nn.Module]], - gating_qi_partial_fn: Optional[Callable[[int], torch.nn.Module]], - gating_combination_type: str, - gating_normalization_fn: Callable[[int], torch.nn.Module], - eps: float, - apply_query_embeddings_fn: bool = True, - apply_item_embeddings_fn: bool = True, - autocast_bf16: bool = False, - ) -> None: - """ - Args: - apply_query_embeddings_fn: bool. If true, compute query_embeddings_fn - to input during forward(). Otherwise, we assume the caller will - invoke get_query_component_embeddings() separately before - calling forward(). - apply_item_embeddings_fn: bool. If true, compute item_embeddings_fn - to input during forward(). Otherwise, we assume the caller will - invoke get_item_component_embeddings() separately before - calling forward(). - """ - super().__init__() - - self._gating_fn: MoLGatingFn = MoLGatingFn( - num_logits=query_dot_product_groups * item_dot_product_groups, - query_embedding_dim=query_embedding_dim, - item_embedding_dim=item_embedding_dim, - query_only_partial_fn=gating_query_only_partial_fn, - item_only_partial_fn=gating_item_only_partial_fn, - qi_partial_fn=gating_qi_partial_fn, # pyre-ignore [6] - combination_type=gating_combination_type, - normalization_fn=gating_normalization_fn, - ) - self._query_embeddings_fn: MoLEmbeddingsFn = query_embeddings_fn - self._item_embeddings_fn: MoLEmbeddingsFn = ( # pyre-ignore [8] - item_embeddings_fn - ) - self._apply_query_embeddings_fn: bool = apply_query_embeddings_fn - self._apply_item_embeddings_fn: bool = apply_item_embeddings_fn - self._dot_product_l2_norm: bool = dot_product_l2_norm - self._query_dot_product_groups: int = query_dot_product_groups - self._item_dot_product_groups: int = item_dot_product_groups - self._dot_product_dimension: int = dot_product_dimension - self._temperature: float = temperature - self._eps: float = eps - self._autocast_bf16: bool = autocast_bf16 - - def get_query_component_embeddings( - self, - input_embeddings: torch.Tensor, - decoupled_inference: bool = False, - **kwargs, - ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: - """ - Args: - input_embeddings: (B, self._input_embedding_dim,) x float - or (B, P_Q, self._dot_product_dimension) x float. - decoupled_inference: bool. If true, the call represents an attempt to run - forward() in decoupled mode at inference time (e.g., to pre-compute - component-level query embeddings for filtering, etc.). We simulate - the logic in forward() in this case (e.g., if forward() doesn't apply - query_embeddings_fn, then this call won't either). - kwargs: additional implementation-specific arguments. - - Returns: - (B, query_dot_product_groups, dot_product_embedding_dim) x float. - """ - if decoupled_inference and not self._apply_query_embeddings_fn: - return input_embeddings, {} - return self._query_embeddings_fn(input_embeddings, **kwargs) - - def get_item_component_embeddings( - self, - input_embeddings: torch.Tensor, - decoupled_inference: bool = False, - **kwargs, - ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: - """ - Args: - input_embeddings: (..., self._input_embedding_dim,) x float - or (..., P_X, self._dot_product_dimension) x float. - decoupled_inference: bool. If true, the call represents an attempt to run - forward() in decoupled mode at inference time (e.g., to pre-compute - component-level item embeddings for filtering, etc.). We simulate - the logic in forward() in this case (e.g., if forward() doesn't apply - item_embeddings_fn, then this call won't either). - kwargs: additional implementation-specific arguments. - - Returns: - (..., item_dot_product_groups, dot_product_embedding_dim) x float. - """ - if decoupled_inference and not self._apply_item_embeddings_fn: - return input_embeddings, {} - - return self._item_embeddings_fn(input_embeddings, **kwargs) - - def forward( - self, - query_embeddings: torch.Tensor, - item_embeddings: torch.Tensor, - **kwargs, - ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: - """ - Args: - query_embeddings: (B, self._input_embedding_dim) x float or - (B, P_Q, self._dot_product_dimension) x float (when query_embeddings_fn - is applied externally). - item_embeddings: (1/B, X, self._item_embedding_dim) x float or - (1/B, X, P_X, self._dot_product_dimension) x float (when item_embeddings_fn - is applied externally). - kwargs: additional implementation-specific arguments. - - Returns: - (B, X) x float, Dict[str, Tensor] representing auxiliary losses. - """ - with torch.autocast( - enabled=self._autocast_bf16, dtype=torch.bfloat16, device_type="cuda" - ): - B = query_embeddings.size(0) - B_prime = item_embeddings.shape[0] # 1 or B - X = item_embeddings.shape[1] - - if self._apply_query_embeddings_fn: - ( - split_query_embeddings, - query_aux_losses, - ) = self.get_query_component_embeddings( - query_embeddings, - **kwargs, - ) - else: - split_query_embeddings, query_aux_losses = query_embeddings, {} - - if self._apply_item_embeddings_fn: - ( - split_item_embeddings, - item_aux_losses, - ) = self.get_item_component_embeddings( - input_embeddings=item_embeddings, - **kwargs, - ) - else: - split_item_embeddings, item_aux_losses = item_embeddings, {} - - if B_prime == 1: - logits = torch.einsum( - "bnd,xmd->bxnm", - split_query_embeddings, - split_item_embeddings.squeeze(0), - ).reshape( - B, X, self._query_dot_product_groups * self._item_dot_product_groups - ) - else: - logits = torch.einsum( - "bnd,bxmd->bxnm", split_query_embeddings, split_item_embeddings - ).reshape( - B, X, self._query_dot_product_groups * self._item_dot_product_groups - ) - - gated_outputs, gating_aux_losses = self._gating_fn( - logits=logits / self._temperature, # [B, X, L] - query_embeddings=query_embeddings, # [B, D] - item_embeddings=item_embeddings, # [1/B, X, D'] - ) - return gated_outputs, { - **gating_aux_losses, - **query_aux_losses, - **item_aux_losses, - } diff --git a/recommendation_v4/generative_recommenders/research/trainer/data_loader.py b/recommendation_v4/generative_recommenders/research/trainer/data_loader.py deleted file mode 100644 index 390b04bdb..000000000 --- a/recommendation_v4/generative_recommenders/research/trainer/data_loader.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-unsafe - -import os -from typing import Optional, Tuple - -import gin -import torch - - -@gin.configurable -def create_data_loader( - dataset: torch.utils.data.Dataset, - batch_size: int, - world_size: int, - rank: int, - shuffle: bool, - prefetch_factor: int = 128, - num_workers: Optional[int] = os.cpu_count(), - drop_last: bool = False, -) -> Tuple[ - Optional[torch.utils.data.distributed.DistributedSampler[torch.utils.data.Dataset]], - torch.utils.data.DataLoader, -]: - if shuffle: - sampler = torch.utils.data.distributed.DistributedSampler( - dataset, - num_replicas=world_size, - rank=rank, - shuffle=True, - seed=0, - drop_last=drop_last, - ) - else: - sampler = None - data_loader = torch.utils.data.DataLoader( - dataset, - batch_size=batch_size, - # shuffle=True, cannot use with sampler - num_workers=num_workers or 0, - sampler=sampler, - prefetch_factor=prefetch_factor, - ) - return sampler, data_loader diff --git a/recommendation_v4/generative_recommenders/research/trainer/train.py b/recommendation_v4/generative_recommenders/research/trainer/train.py deleted file mode 100644 index 6d2da5be7..000000000 --- a/recommendation_v4/generative_recommenders/research/trainer/train.py +++ /dev/null @@ -1,532 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-unsafe - -import logging -import os -import random -import time -from datetime import date -from typing import Dict, Optional - -import gin -import torch -import torch.distributed as dist -from generative_recommenders.research.data.eval import ( - _avg, - add_to_summary_writer, - eval_metrics_v2_from_tensors, - get_eval_state, -) -from generative_recommenders.research.data.reco_dataset import get_reco_dataset -from generative_recommenders.research.indexing.utils import get_top_k_module -from generative_recommenders.research.modeling.sequential.autoregressive_losses import ( - BCELoss, - InBatchNegativesSampler, - LocalNegativesSampler, -) -from generative_recommenders.research.modeling.sequential.embedding_modules import ( - EmbeddingModule, - LocalEmbeddingModule, -) -from generative_recommenders.research.modeling.sequential.encoder_utils import ( - get_sequential_encoder, -) -from generative_recommenders.research.modeling.sequential.features import ( - movielens_seq_features_from_row, -) -from generative_recommenders.research.modeling.sequential.input_features_preprocessors import ( - LearnablePositionalEmbeddingInputFeaturesPreprocessor, -) -from generative_recommenders.research.modeling.sequential.losses.sampled_softmax import ( - SampledSoftmaxLoss, -) -from generative_recommenders.research.modeling.sequential.output_postprocessors import ( - L2NormEmbeddingPostprocessor, - LayerNormEmbeddingPostprocessor, -) -from generative_recommenders.research.modeling.similarity_utils import ( - get_similarity_function, -) -from generative_recommenders.research.trainer.data_loader import create_data_loader -from torch.nn.parallel import DistributedDataParallel as DDP -from torch.utils.tensorboard import SummaryWriter - - -def setup(rank: int, world_size: int, master_port: int) -> None: - os.environ["MASTER_ADDR"] = "localhost" - os.environ["MASTER_PORT"] = str(master_port) - - # initialize the process group - dist.init_process_group("nccl", rank=rank, world_size=world_size) - - -def cleanup() -> None: - dist.destroy_process_group() - - -@gin.configurable -def get_weighted_loss( - main_loss: torch.Tensor, - aux_losses: Dict[str, torch.Tensor], - weights: Dict[str, float], -) -> torch.Tensor: - weighted_loss = main_loss - for key, weight in weights.items(): - cur_weighted_loss = aux_losses[key] * weight - weighted_loss = weighted_loss + cur_weighted_loss - return weighted_loss - - -@gin.configurable -def train_fn( - rank: int, - world_size: int, - master_port: int, - dataset_name: str = "ml-20m", - max_sequence_length: int = 200, - positional_sampling_ratio: float = 1.0, - local_batch_size: int = 128, - eval_batch_size: int = 128, - eval_user_max_batch_size: Optional[int] = None, - main_module: str = "SASRec", - main_module_bf16: bool = False, - dropout_rate: float = 0.2, - user_embedding_norm: str = "l2_norm", - sampling_strategy: str = "in-batch", - loss_module: str = "SampledSoftmaxLoss", - loss_weights: Optional[Dict[str, float]] = {}, - num_negatives: int = 1, - loss_activation_checkpoint: bool = False, - item_l2_norm: bool = False, - temperature: float = 0.05, - num_epochs: int = 101, - learning_rate: float = 1e-3, - num_warmup_steps: int = 0, - weight_decay: float = 1e-3, - top_k_method: str = "MIPSBruteForceTopK", - eval_interval: int = 100, - full_eval_every_n: int = 1, - save_ckpt_every_n: int = 1000, - partial_eval_num_iters: int = 32, - embedding_module_type: str = "local", - item_embedding_dim: int = 240, - interaction_module_type: str = "", - gr_output_length: int = 10, - l2_norm_eps: float = 1e-6, - enable_tf32: bool = False, - random_seed: int = 42, -) -> None: - # to enable more deterministic results. - random.seed(random_seed) - torch.backends.cuda.matmul.allow_tf32 = enable_tf32 - torch.backends.cudnn.allow_tf32 = enable_tf32 - logging.info(f"cuda.matmul.allow_tf32: {enable_tf32}") - logging.info(f"cudnn.allow_tf32: {enable_tf32}") - logging.info(f"Training model on rank {rank}.") - setup(rank, world_size, master_port) - - dataset = get_reco_dataset( - dataset_name=dataset_name, - max_sequence_length=max_sequence_length, - chronological=True, - positional_sampling_ratio=positional_sampling_ratio, - ) - - train_data_sampler, train_data_loader = create_data_loader( - dataset.train_dataset, - batch_size=local_batch_size, - world_size=world_size, - rank=rank, - shuffle=True, - drop_last=world_size > 1, - ) - eval_data_sampler, eval_data_loader = create_data_loader( - dataset.eval_dataset, - batch_size=eval_batch_size, - world_size=world_size, - rank=rank, - shuffle=True, # needed for partial eval - drop_last=world_size > 1, - ) - - model_debug_str = main_module - if embedding_module_type == "local": - embedding_module: EmbeddingModule = LocalEmbeddingModule( - num_items=dataset.max_item_id, - item_embedding_dim=item_embedding_dim, - ) - else: - raise ValueError(f"Unknown embedding_module_type {embedding_module_type}") - model_debug_str += f"-{embedding_module.debug_str()}" - - interaction_module, interaction_module_debug_str = get_similarity_function( - module_type=interaction_module_type, - query_embedding_dim=item_embedding_dim, - item_embedding_dim=item_embedding_dim, - ) - - assert user_embedding_norm == "l2_norm" or user_embedding_norm == "layer_norm", ( - f"Not implemented for {user_embedding_norm}" - ) - output_postproc_module = ( - L2NormEmbeddingPostprocessor( - embedding_dim=item_embedding_dim, - eps=1e-6, - ) - if user_embedding_norm == "l2_norm" - else LayerNormEmbeddingPostprocessor( - embedding_dim=item_embedding_dim, - eps=1e-6, - ) - ) - input_preproc_module = LearnablePositionalEmbeddingInputFeaturesPreprocessor( - max_sequence_len=dataset.max_sequence_length + gr_output_length + 1, - embedding_dim=item_embedding_dim, - dropout_rate=dropout_rate, - ) - - model = get_sequential_encoder( - module_type=main_module, - max_sequence_length=dataset.max_sequence_length, - max_output_length=gr_output_length + 1, - embedding_module=embedding_module, - interaction_module=interaction_module, - input_preproc_module=input_preproc_module, - output_postproc_module=output_postproc_module, - verbose=True, - ) - model_debug_str = model.debug_str() - - # loss - loss_debug_str = loss_module - if loss_module == "BCELoss": - loss_debug_str = loss_debug_str[:-4] - assert temperature == 1.0 - ar_loss = BCELoss(temperature=temperature, model=model) - elif loss_module == "SampledSoftmaxLoss": - loss_debug_str = "ssl" - if temperature != 1.0: - loss_debug_str += f"-t{temperature}" - ar_loss = SampledSoftmaxLoss( - num_to_sample=num_negatives, - softmax_temperature=temperature, - model=model, - activation_checkpoint=loss_activation_checkpoint, - ) - loss_debug_str += ( - f"-n{num_negatives}{'-ac' if loss_activation_checkpoint else ''}" - ) - else: - raise ValueError(f"Unrecognized loss module {loss_module}.") - - # sampling - if sampling_strategy == "in-batch": - negatives_sampler = InBatchNegativesSampler( - l2_norm=item_l2_norm, - l2_norm_eps=l2_norm_eps, - dedup_embeddings=True, - ) - sampling_debug_str = ( - f"in-batch{f'-l2-eps{l2_norm_eps}' if item_l2_norm else ''}-dedup" - ) - elif sampling_strategy == "local": - negatives_sampler = LocalNegativesSampler( - num_items=dataset.max_item_id, - item_emb=model._embedding_module._item_emb, - all_item_ids=dataset.all_item_ids, - l2_norm=item_l2_norm, - l2_norm_eps=l2_norm_eps, - ) - else: - raise ValueError(f"Unrecognized sampling strategy {sampling_strategy}.") - sampling_debug_str = negatives_sampler.debug_str() - - # Creates model and moves it to GPU with id rank - device = rank - if main_module_bf16: - model = model.to(torch.bfloat16) - model = model.to(device) - ar_loss = ar_loss.to(device) - negatives_sampler = negatives_sampler.to(device) - model = DDP(model, device_ids=[rank], broadcast_buffers=False) - - # TODO: wrap in create_optimizer. - opt = torch.optim.AdamW( - model.parameters(), - lr=learning_rate, - betas=(0.9, 0.98), - weight_decay=weight_decay, - ) - - date_str = date.today().strftime("%Y-%m-%d") - model_subfolder = f"{dataset_name}-l{max_sequence_length}" - model_desc = ( - f"{model_subfolder}" - + f"/{model_debug_str}_{interaction_module_debug_str}_{sampling_debug_str}_{loss_debug_str}" - + f"{f'-ddp{world_size}' if world_size > 1 else ''}-b{local_batch_size}-lr{learning_rate}-wu{num_warmup_steps}-wd{weight_decay}{'' if enable_tf32 else '-notf32'}-{date_str}" - ) - if full_eval_every_n > 1: - model_desc += f"-fe{full_eval_every_n}" - if positional_sampling_ratio is not None and positional_sampling_ratio < 1: - model_desc += f"-d{positional_sampling_ratio}" - # creates subfolders. - os.makedirs(f"./exps/{model_subfolder}", exist_ok=True) - os.makedirs(f"./ckpts/{model_subfolder}", exist_ok=True) - log_dir = f"./exps/{model_desc}" - if rank == 0: - writer = SummaryWriter(log_dir=log_dir) - logging.info(f"Rank {rank}: writing logs to {log_dir}") - else: - writer = None - logging.info(f"Rank {rank}: disabling summary writer") - - last_training_time = time.time() - torch.autograd.set_detect_anomaly(True) - - batch_id = 0 - epoch = 0 - for epoch in range(num_epochs): - if train_data_sampler is not None: - train_data_sampler.set_epoch(epoch) - if eval_data_sampler is not None: - eval_data_sampler.set_epoch(epoch) - model.train() - for row in iter(train_data_loader): - seq_features, target_ids, target_ratings = movielens_seq_features_from_row( - row, - device=device, - max_output_length=gr_output_length + 1, - ) - - if (batch_id % eval_interval) == 0: - model.eval() - - eval_state = get_eval_state( - model=model.module, - all_item_ids=dataset.all_item_ids, - negatives_sampler=negatives_sampler, - top_k_module_fn=lambda item_embeddings, item_ids: get_top_k_module( - top_k_method=top_k_method, - model=model.module, - item_embeddings=item_embeddings, - item_ids=item_ids, - ), - device=device, - float_dtype=torch.bfloat16 if main_module_bf16 else None, - ) - eval_dict = eval_metrics_v2_from_tensors( - eval_state, - model.module, - seq_features, - target_ids=target_ids, - target_ratings=target_ratings, - user_max_batch_size=eval_user_max_batch_size, - dtype=torch.bfloat16 if main_module_bf16 else None, - ) - add_to_summary_writer( - writer, batch_id, eval_dict, prefix="eval", world_size=world_size - ) - logging.info( - f"rank {rank}: batch-stat (eval): iter {batch_id} (epoch {epoch}): " - + f"NDCG@10 {_avg(eval_dict['ndcg@10'], world_size):.4f}, " - f"HR@10 {_avg(eval_dict['hr@10'], world_size):.4f}, " - f"HR@50 {_avg(eval_dict['hr@50'], world_size):.4f}, " - + f"MRR {_avg(eval_dict['mrr'], world_size):.4f} " - ) - model.train() - - # TODO: consider separating this out? - B, N = seq_features.past_ids.shape - seq_features.past_ids.scatter_( - dim=1, - index=seq_features.past_lengths.view(-1, 1), - src=target_ids.view(-1, 1), - ) - - opt.zero_grad() - input_embeddings = model.module.get_item_embeddings(seq_features.past_ids) - seq_embeddings = model( - past_lengths=seq_features.past_lengths, - past_ids=seq_features.past_ids, - past_embeddings=input_embeddings, - past_payloads=seq_features.past_payloads, - ) # [B, X] - - supervision_ids = seq_features.past_ids - - if sampling_strategy == "in-batch": - # get_item_embeddings currently assume 1-d tensor. - in_batch_ids = supervision_ids.view(-1) - negatives_sampler.process_batch( - ids=in_batch_ids, - presences=(in_batch_ids != 0), - embeddings=model.module.get_item_embeddings(in_batch_ids), - ) - else: - # pyre-fixme[16]: `InBatchNegativesSampler` has no attribute - # `_item_emb`. - negatives_sampler._item_emb = model.module._embedding_module._item_emb - - ar_mask = supervision_ids[:, 1:] != 0 - loss, aux_losses = ar_loss( - lengths=seq_features.past_lengths, # [B], - output_embeddings=seq_embeddings[:, :-1, :], # [B, N-1, D] - supervision_ids=supervision_ids[:, 1:], # [B, N-1] - supervision_embeddings=input_embeddings[:, 1:, :], # [B, N - 1, D] - supervision_weights=ar_mask.float(), - negatives_sampler=negatives_sampler, - **seq_features.past_payloads, - ) # [B, N] - - main_loss = loss.detach().clone() - loss = get_weighted_loss(loss, aux_losses, weights=loss_weights or {}) - - if rank == 0: - assert writer is not None - writer.add_scalar("losses/ar_loss", loss, batch_id) - writer.add_scalar("losses/main_loss", main_loss, batch_id) - - loss.backward() - - # Optional linear warmup. - if batch_id < num_warmup_steps: - lr_scalar = min(1.0, float(batch_id + 1) / num_warmup_steps) - for pg in opt.param_groups: - pg["lr"] = lr_scalar * learning_rate - lr = lr_scalar * learning_rate - else: - lr = learning_rate - - if (batch_id % eval_interval) == 0: - logging.info( - f" rank: {rank}, batch-stat (train): step {batch_id} " - f"(epoch {epoch} in {time.time() - last_training_time:.2f}s): {loss:.6f}" - ) - last_training_time = time.time() - if rank == 0: - assert writer is not None - writer.add_scalar("loss/train", loss, batch_id) - writer.add_scalar("lr", lr, batch_id) - - opt.step() - - batch_id += 1 - - def is_full_eval(epoch: int) -> bool: - return (epoch % full_eval_every_n) == 0 - - # eval per epoch - eval_dict_all = None - eval_start_time = time.time() - model.eval() - eval_state = get_eval_state( - model=model.module, - all_item_ids=dataset.all_item_ids, - negatives_sampler=negatives_sampler, - top_k_module_fn=lambda item_embeddings, item_ids: get_top_k_module( - top_k_method=top_k_method, - model=model.module, - item_embeddings=item_embeddings, - item_ids=item_ids, - ), - device=device, - float_dtype=torch.bfloat16 if main_module_bf16 else None, - ) - for eval_iter, row in enumerate(iter(eval_data_loader)): - seq_features, target_ids, target_ratings = movielens_seq_features_from_row( - row, device=device, max_output_length=gr_output_length + 1 - ) - eval_dict = eval_metrics_v2_from_tensors( - eval_state, - model.module, - seq_features, - target_ids=target_ids, - target_ratings=target_ratings, - user_max_batch_size=eval_user_max_batch_size, - dtype=torch.bfloat16 if main_module_bf16 else None, - ) - - if eval_dict_all is None: - eval_dict_all = {} - for k, v in eval_dict.items(): - eval_dict_all[k] = [] - - for k, v in eval_dict.items(): - eval_dict_all[k] = eval_dict_all[k] + [v] - del eval_dict - - if (eval_iter + 1 >= partial_eval_num_iters) and (not is_full_eval(epoch)): - logging.info( - f"Truncating epoch {epoch} eval to {eval_iter + 1} iters to save cost.." - ) - break - - assert eval_dict_all is not None - for k, v in eval_dict_all.items(): - eval_dict_all[k] = torch.cat(v, dim=-1) - - ndcg_10 = _avg(eval_dict_all["ndcg@10"], world_size=world_size) - ndcg_50 = _avg(eval_dict_all["ndcg@50"], world_size=world_size) - hr_10 = _avg(eval_dict_all["hr@10"], world_size=world_size) - hr_50 = _avg(eval_dict_all["hr@50"], world_size=world_size) - mrr = _avg(eval_dict_all["mrr"], world_size=world_size) - - add_to_summary_writer( - writer, - batch_id=epoch, - metrics=eval_dict_all, - prefix="eval_epoch", - world_size=world_size, - ) - if full_eval_every_n > 1 and is_full_eval(epoch): - add_to_summary_writer( - writer, - batch_id=epoch, - metrics=eval_dict_all, - prefix="eval_epoch_full", - world_size=world_size, - ) - if rank == 0 and epoch > 0 and (epoch % save_ckpt_every_n) == 0: - torch.save( - { - "epoch": epoch, - "model_state_dict": model.state_dict(), - "optimizer_state_dict": opt.state_dict(), - }, - f"./ckpts/{model_desc}_ep{epoch}", - ) - - logging.info( - f"rank {rank}: eval @ epoch {epoch} in {time.time() - eval_start_time:.2f}s: " - f"NDCG@10 {ndcg_10:.4f}, NDCG@50 {ndcg_50:.4f}, HR@10 {hr_10:.4f}, HR@50 {hr_50:.4f}, MRR {mrr:.4f}" - ) - last_training_time = time.time() - - if rank == 0: - if writer is not None: - writer.flush() - writer.close() - - torch.save( - { - "epoch": epoch, - "model_state_dict": model.state_dict(), - "optimizer_state_dict": opt.state_dict(), - }, - f"./ckpts/{model_desc}_ep{epoch}", - ) - - cleanup() diff --git a/recommendation_v4/main.py b/recommendation_v4/main.py deleted file mode 100644 index 445f25820..000000000 --- a/recommendation_v4/main.py +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-unsafe - -""" -Main entry point for model training. Please refer to README.md for usage instructions. -""" - -import logging -import os -from typing import List, Optional - -os.environ["TF_CPP_MIN_LOG_LEVEL"] = "1" # Hide excessive tensorflow debug messages -import sys - -import fbgemm_gpu # noqa: F401, E402 -import gin -import torch -import torch.multiprocessing as mp -from absl import app, flags -from generative_recommenders.research.trainer.train import train_fn - -logging.basicConfig(stream=sys.stdout, level=logging.INFO) - - -def delete_flags(FLAGS, keys_to_delete: List[str]) -> None: # pyre-ignore [2] - keys = [key for key in FLAGS._flags()] - for key in keys: - if key in keys_to_delete: - delattr(FLAGS, key) - - -delete_flags(flags.FLAGS, ["gin_config_file", "master_port"]) -flags.DEFINE_string("gin_config_file", None, "Path to the config file.") -flags.DEFINE_integer("master_port", 12355, "Master port.") -FLAGS = flags.FLAGS # pyre-ignore [5] - - -def mp_train_fn( - rank: int, - world_size: int, - master_port: int, - gin_config_file: Optional[str], -) -> None: - if gin_config_file is not None: - # Hack as absl doesn't support flag parsing inside multiprocessing. - logging.info(f"Rank {rank}: loading gin config from {gin_config_file}") - gin.parse_config_file(gin_config_file) - - train_fn(rank, world_size, master_port) - - -def _main(argv) -> None: # pyre-ignore [2] - world_size = torch.cuda.device_count() - - mp.set_start_method("forkserver") - mp.spawn( - mp_train_fn, - args=(world_size, FLAGS.master_port, FLAGS.gin_config_file), - nprocs=world_size, - join=True, - ) - - -def main() -> None: - app.run(_main) - - -if __name__ == "__main__": - main() diff --git a/recommendation_v4/preprocess_public_data.py b/recommendation_v4/preprocess_public_data.py deleted file mode 100644 index 927ccf4c6..000000000 --- a/recommendation_v4/preprocess_public_data.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-unsafe - -""" -Usage: mkdir -p tmp/ && python3 preprocess_public_data.py -""" - -from generative_recommenders.research.data.preprocessor import get_common_preprocessors - - -def main() -> None: - get_common_preprocessors()["ml-1m"].preprocess_rating() - get_common_preprocessors()["ml-20m"].preprocess_rating() - # get_common_preprocessors()["ml-1b"].preprocess_rating() - get_common_preprocessors()["amzn-books"].preprocess_rating() - - -if __name__ == "__main__": - main() diff --git a/recommendation_v4/run_fractal_expansion.py b/recommendation_v4/run_fractal_expansion.py deleted file mode 100644 index 308eadea2..000000000 --- a/recommendation_v4/run_fractal_expansion.py +++ /dev/null @@ -1,588 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pyre-unsafe - -""" -Run fractal expansion introduced in https://arxiv.org/abs/1901.08910. -Implementation adapted from the scripts used to generate MovieLens-1B -(https://grouplens.org/datasets/movielens/movielens-1b/). -""" - -# Generate a 3B dataset (takes around 50 minutes): -# python run_fractal_expansion.py --input-csv-file ~/data/ml-20m/ratings.csv --write-dataset True --output-prefix ~/data/ml-3b/ -# Generate a 13B dataset with 440M item size: -# python run_fractal_expansion.py --input-csv-file ~/data/ml-20m/ratings.csv --write-dataset True --output-prefix ~/data/ml-13b/ --num-row-multiplier 16 --num-col-multiplier 16384 --element-sample-rate 0.2 --block-sample-rate 0.05 -# Generate a 18B dataset with 1B item size: -# python run_fractal_expansion.py --input-csv-file ~/data/ml-20m/ratings.csv --write-dataset True --output-prefix ~/data/ml-18b/ --num-row-multiplier 20 --num-col-multiplier 36864 --element-sample-rate 0.08 --block-sample-rate 0.05 - -import csv -import linecache -import logging -import os -import pickle -from dataclasses import dataclass - -import click -import matplotlib.pyplot as plt -import numpy as np -import pandas as pd -import scipy.linalg -import skimage.transform as transform -from scipy import sparse -from scipy.sparse import linalg -from sklearn.utils import shuffle -from tqdm import tqdm - - -logging.basicConfig() -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) - - -@dataclass -class SparseMatrixMetadata: - num_interactions: int = 0 - num_rows: int = 0 - num_cols: int = 0 - - -def _dropout_sparse_coo_matrix( - sparse_matrix, rate, min_dropout_rate=0.005, max_dropout_rate=0.999 -): - assert min_dropout_rate <= max_dropout_rate - sampling_rate = 1.0 - rate - - sampled_fraction = min( - max(sampling_rate, 1.0 - max_dropout_rate), 1.0 - min_dropout_rate - ) - if sampled_fraction != sampling_rate: - logger.warning( - f"Desired sampling rate {sampling_rate} clipped to {sampled_fraction}." - ) - num_sampled = min( - max(int(sparse_matrix.nnz * sampled_fraction), 1), sparse_matrix.nnz - ) - sampled_indices = np.random.choice( - sparse_matrix.nnz, size=num_sampled, replace=False - ) - return sparse.coo_matrix( - ( - sparse_matrix.data[sampled_indices], - (sparse_matrix.row[sampled_indices], sparse_matrix.col[sampled_indices]), - ), - shape=sparse_matrix.shape, - ) - - -def shuffle_sparse_matrix( - sparse_matrix, dropout_rate=0.0, min_dropout_rate=0.005, max_dropout_rate=0.999 -): - """ - Shuffle sparse matrix encoded as a SciPy csr matrix. - """ - - assert dropout_rate >= 0.0 and dropout_rate <= 1.0 - (num_rows, num_cols) = sparse_matrix.shape - shuffled_rows = shuffle(np.arange(num_rows)) - shuffled_cols = shuffle(np.arange(num_cols)) - sparse_matrix = _dropout_sparse_coo_matrix( - sparse_matrix, dropout_rate, min_dropout_rate, max_dropout_rate - ) - new_row = np.take(shuffled_rows, sparse_matrix.row) - new_col = np.take(shuffled_cols, sparse_matrix.col) - return sparse.csr_matrix( - (sparse_matrix.data, (new_row, new_col)), shape=(num_rows, num_cols) - ) - - -def graph_reduce(usv, num_rows, num_cols): - """Apply algorithm 2 in https://arxiv.org/pdf/1901.08910.pdf.""" - - def _closest_column_orthogonal_matrix(matrix): - return np.matmul( - matrix, np.linalg.inv(scipy.linalg.sqrtm(np.matmul(matrix.T, matrix))) - ) - - u, s, v = usv - k = min(num_rows, num_cols) - u_random_proj = transform.resize(u[:, :k], (num_rows, k)) - v_random_proj = transform.resize(v[:k, :], (k, num_cols)) - u_random_proj_orth = _closest_column_orthogonal_matrix(u_random_proj) - v_random_proj_orth = _closest_column_orthogonal_matrix(v_random_proj.T).T - return np.matmul(u_random_proj_orth, np.matmul(np.diag(s[:k]), v_random_proj_orth)) - - -def rescale(matrix, rescale_w_abs=False, element_sample_rate=1.0): - """Rescale all values of the matrix into [0, 1].""" - if rescale_w_abs: - abs_matrix = np.abs(matrix.copy()) - out = abs_matrix / abs_matrix.max() - else: - out = (matrix - matrix.min()) / (matrix.max() - matrix.min()) - assert out.min() >= 0 and out.max() <= 1 - return out * element_sample_rate - - -def _compute_row_block( - i, left_matrix, right_matrix, block_sample_rate, indices_out_path, remove_empty_rows -): - """Compute row block of expansion for row i of the left_matrix.""" - - kron_blocks = [] - num_rows = 0 - num_removed_rows = 0 - num_interactions = 0 - - for j in range(left_matrix.shape[1]): - if np.random.random() <= block_sample_rate: - dropout_rate = 1.0 - left_matrix[i, j] - kron_block = shuffle_sparse_matrix(right_matrix, dropout_rate).tocsr() - num_interactions += kron_block.nnz - kron_blocks.append(kron_block) - logger.info(f"Kronecker block ({i}, {j}) processed.") - else: - kron_blocks.append(sparse.csr_matrix(right_matrix.shape)) - logger.info(f"Kronecker block ({i}, {j}) skipped.") - - rows_to_write = sparse.hstack(kron_blocks).tocsr() - logger.info("Writing dataset row by row.") - - # Write Kronecker product line per line. - filepath = f"{indices_out_path}_{i}.csv" - os.makedirs(os.path.dirname(filepath), exist_ok=True) - with open(filepath, "w", newline="") as file: - writer = csv.writer(file) - for k in range(right_matrix.shape[0]): - items_to_write = rows_to_write.getrow(k).indices - ratings_to_write = rows_to_write.getrow(k).data - num = items_to_write.shape[0] - if remove_empty_rows and (not num): - logger.info(f"Removed empty output row {i * left_matrix.shape[0] + k}.") - num_removed_rows += 1 - continue - num_rows += 1 - writer.writerow( - [ - i * right_matrix.shape[0] + k, - ",".join([str(x) for x in items_to_write]), - ",".join([str(x) for x in ratings_to_write]), - ] - ) - if k % 100000 == 0: - logger.info(f"Done producing data set row {k}.") - - num_cols = rows_to_write.shape[1] - metadata = SparseMatrixMetadata( - num_interactions=num_interactions, num_rows=num_rows, num_cols=num_cols - ) - logger.info( - f"Done with left matrix row {i}, {num_interactions} interactions written in shard, {num_removed_rows} rows removed in shard." - ) - return (num_removed_rows, metadata) - - -def visualize_samples( - right_matrix, - visualize_num_samples, - expanded_file_name, - output_prefix, -): - # Note: only the rows of the first Kronecker block are visualized. - logger.info("visualize dataset row by row.") - fig, axs = plt.subplots(1, 2, figsize=(12, 5)) - axs[0].set_title("Original data Histogram") - axs[0].set_xlabel("Value") - axs[0].set_ylabel("Frequency") - axs[1].set_title("Expended Row Histogram") - axs[1].set_xlabel("Value") - axs[1].set_ylabel("Frequency") - for k in range(visualize_num_samples): - original_row = right_matrix.getrow(k).data - line = linecache.getline(expanded_file_name, k + 1) - reader = csv.reader([line]) - parsed_line = next(reader) - expended_row = eval(parsed_line[2]) - original_hist_counts, original_bin_edges = np.histogram(original_row, bins=9) - expended_hist_counts, expended_bin_edges = np.histogram(expended_row, bins=9) - axs[0].plot(original_bin_edges[:-1], original_hist_counts, alpha=0.2) - axs[1].plot(expended_bin_edges[:-1], expended_hist_counts, alpha=0.2) - axs[0].fill_between(original_bin_edges[:-1], original_hist_counts, alpha=0.2) - axs[1].fill_between(expended_bin_edges[:-1], expended_hist_counts, alpha=0.2) - plt.tight_layout() - plt.savefig(f"{output_prefix}_sample_distribution.png") - logger.info("Sample visualization finished.") - - -def build_randomized_kronecker( - left_matrix, - right_matrix, - block_sample_rate, - indices_out_path, - metadata_out_path=None, - remove_empty_rows=True, -): - """Compute randomized Kronecker product and dump it on the fly based on https://arxiv.org/pdf/1901.08910.pdf.""" - logger.info(f"Writing item sequences to pickle files {metadata_out_path}.") - - num_rows = 0 - num_removed_rows = 0 - num_cols = left_matrix.shape[1] * right_matrix.shape[1] - num_interactions = 0 - - filepath = f"{indices_out_path}_users.csv" - os.makedirs(os.path.dirname(filepath), exist_ok=True) - with open(filepath, "w", newline="") as file: - writer = csv.writer(file) - for i in tqdm(range(left_matrix.shape[0])): - (shard_num_removed_rows, shard_metadata) = _compute_row_block( - i, - left_matrix, - right_matrix, - block_sample_rate, - indices_out_path, - remove_empty_rows, - ) - writer.writerow([i, shard_metadata.num_rows]) - file.flush() - num_rows += shard_metadata.num_rows - num_removed_rows += shard_num_removed_rows - num_interactions += shard_metadata.num_interactions - - logger.info(f"{num_interactions / num_rows} average sequence length") - logger.info(f"{num_interactions} total interactions written.") - logger.info(f"{num_removed_rows} total rows removed.") - - metadata = SparseMatrixMetadata( - num_interactions=num_interactions, num_rows=num_rows, num_cols=num_cols - ) - if metadata_out_path is not None: - logger.info(f"Writing metadata file to {metadata_out_path}") - with open(metadata_out_path, "wb") as output_file: - pickle.dump(metadata, output_file) - return metadata - - -def _preprocess_movie_lens(ratings_df, binary=False): - """ - Filters out users with less than three distinct timestamps. - """ - - def _create_index(df, colname): - value_set = sorted(set(df[colname].values)) - num_unique = len(value_set) - return dict(zip(value_set, range(num_unique))) - - if not binary: - ratings_df["data"] = ratings_df["rating"] - else: - ratings_df["data"] = 1.0 - ratings_df["binary_data"] = 1.0 - num_timestamps = ratings_df[["userId", "timestamp"]].groupby("userId").nunique() - ratings_df["numberOfTimestamps"] = ratings_df["userId"].apply( - lambda x: num_timestamps["timestamp"][x] - ) - ratings_df = ratings_df[ratings_df["numberOfTimestamps"] > 2] - user_id_to_user_idx = _create_index(ratings_df, "userId") - item_id_to_item_idx = _create_index(ratings_df, "movieId") - ratings_df["row"] = ratings_df["userId"].apply(lambda x: user_id_to_user_idx[x]) - ratings_df["col"] = ratings_df["movieId"].apply(lambda x: item_id_to_item_idx[x]) - return ratings_df - - -def normalize(matrix): - norm_matrix = matrix.copy() - if isinstance(norm_matrix, np.ndarray): - norm_matrix -= norm_matrix.mean() - else: - norm_matrix.data -= norm_matrix.mean() - max_val = norm_matrix.max() - min_val = norm_matrix.min() - if isinstance(norm_matrix, np.ndarray): - norm_matrix /= max(abs(max_val), abs(min_val)) - else: - norm_matrix.data /= max(abs(max_val), abs(min_val)) - return norm_matrix - - -def plot_distribution(user_wise_sum, item_wise_sum, s, title_prefix, normalized=False): - y_label = "rating sums" if normalized else "number of ratings" - fig, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize=(15, 5)) - ax1.loglog( - np.arange(len(user_wise_sum)) + 1, - np.sort(user_wise_sum)[::-1], - linestyle="-", - color="blue", - marker="", - ) - ax1.set_title(f"{title_prefix} matrix user-wise rating sums") - ax1.set_xlabel("User rank") - ax1.set_ylabel(y_label) - ax1.grid(True) - ax2.loglog( - np.arange(len(item_wise_sum)) + 1, - np.sort(item_wise_sum)[::-1], - linestyle="-", - color="green", - marker="", - ) - ax2.set_title(f"{title_prefix} matrix item-wise rating sums") - ax2.set_xlabel("Item rank") - ax2.set_ylabel(y_label) - ax2.grid(True) - ax3.loglog( - np.arange(len(s)) + 1, np.sort(s)[::-1], linestyle="-", color="red", marker="" - ) - ax3.set_title(f"{title_prefix} matrix singular values") - ax3.set_xlabel("Singular value Rank") - ax3.set_ylabel("Magnitude") - ax3.grid(True) - plt.tight_layout() - plt.savefig(f"{title_prefix}_distribution.png") - - -def visualize_distribution(mat, reduced_mat, s, reduced_s, normalized=False, title=""): - user_wise_sum = np.asarray(mat.sum(axis=1)).flatten() - item_wise_sum = np.asarray(mat.sum(axis=0)).flatten() - assert len(user_wise_sum) == mat.shape[0] - assert len(item_wise_sum) == mat.shape[1] - plot_distribution( - user_wise_sum, - item_wise_sum, - s, - title_prefix=f"{title}_Original", - normalized=normalized, - ) - - reduced_user_wise_sum = np.asarray(reduced_mat.sum(axis=1)).flatten() - reduced_item_wise_sum = np.asarray(reduced_mat.sum(axis=0)).flatten() - assert len(reduced_user_wise_sum) == reduced_mat.shape[0] - assert len(reduced_item_wise_sum) == reduced_mat.shape[1] - plot_distribution( - reduced_user_wise_sum, - reduced_item_wise_sum, - reduced_s, - title_prefix=f"{title}_Reduced", - normalized=normalized, - ) - - expanded_s = np.einsum("i,j->ij", reduced_s, s).flatten() - expanded_user_wise_sum = np.einsum("ij,k->ik", reduced_mat, user_wise_sum).flatten() - expanded_item_wise_sum = np.einsum("ij,k->jk", reduced_mat, item_wise_sum).flatten() - assert len(expanded_user_wise_sum) == reduced_mat.shape[0] * mat.shape[0] - assert len(expanded_item_wise_sum) == reduced_mat.shape[1] * mat.shape[1] - plot_distribution( - expanded_user_wise_sum, - expanded_item_wise_sum, - expanded_s, - title_prefix=f"{title}_Expanded", - normalized=normalized, - ) - - -def expand_dataset( - ratings_matrix, - binary_ratings_matrix, - num_users, - num_items, - reduced_num_rows, - reduced_num_cols, - rescale_w_abs, - element_sample_rate, - block_sample_rate, - visualize, - write_dataset, - output_prefix, -): - k = min(reduced_num_rows, reduced_num_cols) - norm_rating_matrix = normalize(ratings_matrix) - (u, s, v) = linalg.svds( - norm_rating_matrix, k=k, maxiter=None, return_singular_vectors=True - ) - - logger.info( - f"Creating reduced rating matrix (size {reduced_num_rows}, {reduced_num_cols})" - ) - reduced_matrix = graph_reduce((u, s, v), reduced_num_rows, reduced_num_cols) - norm_reduced_matrix = normalize(reduced_matrix) - (_, s_reduce, _) = linalg.svds( - norm_reduced_matrix, k=k - 1, maxiter=None, return_singular_vectors=True - ) - reduced_matrix = rescale( - reduced_matrix, - rescale_w_abs=rescale_w_abs, - element_sample_rate=element_sample_rate, - ) - logger.info(f"largest singular value of the reduced matrix is {s_reduce[-1]}") - logger.info( - f"Sampling rate mean is {reduced_matrix.mean()}, var is {reduced_matrix.var()}, min is {reduced_matrix.min()}, max is {reduced_matrix.max()}" - ) - samples = reduced_matrix.sum() * ratings_matrix.nnz * block_sample_rate - logger.info( - f"Expected number of synthetic samples: {samples}, sparsity is {samples / (num_users * num_items * reduced_num_rows * reduced_num_cols)}, average seqlen is {samples / (num_users * reduced_num_rows)}" - ) - - if visualize: - s = linalg.svds( - norm_rating_matrix, k=20 * k, maxiter=None, return_singular_vectors=False - ) - visualize_distribution( - norm_rating_matrix, - norm_reduced_matrix, - s, - s_reduce, - normalized=True, - title="Normalized", - ) - visualize_distribution( - binary_ratings_matrix, - reduced_matrix, - s, - s_reduce, - normalized=False, - title="Binary", - ) - if write_dataset: - output_file = ( - output_prefix + str(reduced_num_rows) + "x" + str(reduced_num_cols) - ) - output_file_metadata = None - - logger.info(f"Creating synthetic dataset and dumping to {output_file}.") - build_randomized_kronecker( - left_matrix=reduced_matrix, - right_matrix=ratings_matrix.tocoo(), - block_sample_rate=block_sample_rate, - indices_out_path=output_file, - metadata_out_path=output_file_metadata, - ) - - -@click.command() -@click.option( - "--random-seed", - type=int, - default=0, -) -@click.option( - "--input-csv-file", - type=str, - default="ratings.csv", -) -@click.option( - "--output-prefix", - type=str, - default="", -) -@click.option( - "--num-row-multiplier", - type=int, - default=16, -) -@click.option( - "--num-col-multiplier", - type=int, - default=32, -) -@click.option( - "--element-sample-rate", - type=float, - default=1.0, -) -@click.option( - "--block-sample-rate", - type=float, - default=1.0, -) -@click.option( - "--visualize", - type=bool, - default=False, -) -@click.option( - "--write-dataset", - type=bool, - default=False, -) -@click.option( - "--visualize-num-samples", - type=int, - default=0, -) -def main( - random_seed: int, - input_csv_file: str, - output_prefix: str, - num_row_multiplier: int, - num_col_multiplier: int, - element_sample_rate: float, - block_sample_rate: float, - visualize: bool, - write_dataset: bool, - visualize_num_samples: int, -): - np.random.seed(random_seed) - - logger.info(f"Loading and preprocessing MovieLens-20m from {input_csv_file}") - with open(input_csv_file, "r") as infile: - ratings_df = pd.read_csv(infile, sep=",", header=0) - ratings_df = _preprocess_movie_lens(ratings_df, binary=False) - num_ratings = len(ratings_df) - num_users = len(set(ratings_df["row"].values)) - num_items = len(set(ratings_df["col"].values)) - logger.info( - f"number of ratings of input dataset is {num_ratings}, number of users is {num_users}, number of items is {num_items}, sparsity is {num_ratings / (num_users * num_items)}, average seqlen is {num_ratings / num_users}" - ) - - ratings_matrix = sparse.csr_matrix( - ( - ratings_df["data"].values, - (ratings_df["row"].values, ratings_df["col"].values), - ), - shape=(num_users, num_items), - ) - binary_ratings_matrix = sparse.csr_matrix( - ( - ratings_df["binary_data"].values, - (ratings_df["row"].values, ratings_df["col"].values), - ), - shape=(num_users, num_items), - ) - if write_dataset or visualize: - expand_dataset( - ratings_matrix=ratings_matrix, - binary_ratings_matrix=binary_ratings_matrix, - num_users=num_users, - num_items=num_items, - reduced_num_rows=num_row_multiplier, - reduced_num_cols=num_col_multiplier, - rescale_w_abs=False, - element_sample_rate=element_sample_rate, - block_sample_rate=block_sample_rate, - visualize=visualize, - write_dataset=write_dataset, - output_prefix=output_prefix, - ) - if visualize_num_samples > 0: - logger.info(f"Visualizing {visualize_num_samples} samples.") - visualize_samples( - right_matrix=ratings_matrix.tocoo(), - visualize_num_samples=visualize_num_samples, - expanded_file_name=f"{output_prefix}{num_row_multiplier}x{num_col_multiplier}_0.csv", - output_prefix="Sample_Histogram", - ) - - -if __name__ == "__main__": - main() From b68bfb711130372781c8abbde3ca2d566fe02413 Mon Sep 17 00:00:00 2001 From: Chris Cai Date: Fri, 26 Jun 2026 05:05:31 +0000 Subject: [PATCH 103/127] Make data-fraction eval cadence the default Change the default eval cadence from per-window (EVAL_EVERY_N_WINDOWS=1) to data-fraction every 0.5% of data (EVAL_EVERY_N_WINDOWS=0, EVAL_EVERY_DATA_PCT=0.005). Per-window spacing is uneven in data volume since each daily window holds a different number of samples; the data-fraction cadence yields ~200 evenly-spaced-by-compute eval points. Updates the gin defaults and the launch_slurm.sh / launch_local.sh fallbacks together so the two cadences are never both >0 (which raises a ValueError at startup), and corrects the corresponding comments. Co-authored-by: Cursor --- .../dlrm_v3/train/gin/yambda_5b.gin | 29 +++++++++++-------- recommendation_v4/scripts/launch_local.sh | 5 +++- recommendation_v4/scripts/launch_slurm.sh | 5 +++- 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index dd645f06a..14bc9d106 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -358,21 +358,24 @@ pl/env_int.default = 1 # (EVAL_EVERY_DATA_PCT>0) requires EVAL_EVERY_N_WINDOWS=0; setting both >0 raises # a ValueError at startup. The final end-of-run eval always runs in either mode. # -# (1) PER-WINDOW cadence (EVAL_EVERY_N_WINDOWS, the default). +# (1) PER-WINDOW cadence (EVAL_EVERY_N_WINDOWS). # Full-holdout eval cadence (single knob; replaces the old EVAL_EACH_WINDOW -# on/off switch). 0 = eval disabled (train-only, e.g. perf benchmarking or the -# resume test; the eval dataloader isn't even built). 1 (default) = eval after -# every window. N>1 (e.g. 5 via $EVAL_EVERY_N_WINDOWS) = eval every Nth window -# (and always the final one) to amortize the cost of consuming the full next-day -# eval window. The cadence is anchored to the absolute ts grid so eval points -# stay stable across a mid-run resume. +# on/off switch). 0 (default) = per-window cadence OFF -> defer to the +# data-fraction cadence below (EVAL_EVERY_DATA_PCT); if that is also 0, eval is +# disabled entirely (train-only, e.g. perf benchmarking or the resume test; the +# eval dataloader isn't even built). 1 = eval after every window. N>1 (e.g. 5 +# via $EVAL_EVERY_N_WINDOWS) = eval every Nth window (and always the final one) +# to amortize the cost of consuming the full next-day eval window. The cadence is +# anchored to the absolute ts grid so eval points stay stable across a mid-run +# resume. # NOTE: each daily window has a DIFFERENT number of training samples, so a # per-window cadence produces eval points that are UNEVENLY spaced in terms of -# how much data was trained between them. Use the data-fraction cadence below if -# you want evenly-spaced-by-data eval points instead. +# how much data was trained between them. This is why the data-fraction cadence +# below is now the default; enable this per-window knob only if you specifically +# want eval anchored to the daily window grid. streaming_train_eval_loop.eval_every_n_windows = @evn/env_int() evn/env_int.key = "EVAL_EVERY_N_WINDOWS" -evn/env_int.default = 1 +evn/env_int.default = 0 # # (2) DATA-FRACTION cadence (EVAL_EVERY_DATA_PCT). # Run the full-holdout eval every time the run has trained this FRACTION of the @@ -380,7 +383,9 @@ evn/env_int.default = 1 # (compute), independent of how many samples each daily window happens to hold. # This is the fix for the per-window cadence's uneven spacing noted above. # value semantics (it is a fraction in (0, 1], NOT a percent number): -# 0.0 (default) = OFF -> fall back to the per-window EVAL_EVERY_N_WINDOWS. +# 0.0 = OFF -> fall back to the per-window EVAL_EVERY_N_WINDOWS; +# if that is also 0, eval is disabled entirely (train-only). +# 0.005 (default)= eval every 0.5% of the data -> ~200 eval points total. # 0.01 = eval every 1% of the data -> ~100 eval points total. # 0.05 = eval every 5% of the data -> ~20 eval points total. # 0.10 = eval every 10% of the data -> ~10 eval points total. @@ -397,7 +402,7 @@ evn/env_int.default = 1 # trajectory can be plotted against data volume. Override via $EVAL_EVERY_DATA_PCT. streaming_train_eval_loop.eval_every_data_pct = @edp/env_float() edp/env_float.key = "EVAL_EVERY_DATA_PCT" -edp/env_float.default = 0.0 +edp/env_float.default = 0.005 # Double-buffer windows: prepare the next window (index mask + first-batch # prefetch) in a background thread during the current window's compute, hiding # the per-window reset. Needs persistent_loader=1. Override via env. diff --git a/recommendation_v4/scripts/launch_local.sh b/recommendation_v4/scripts/launch_local.sh index 9a69a36e5..45f4bb6f2 100755 --- a/recommendation_v4/scripts/launch_local.sh +++ b/recommendation_v4/scripts/launch_local.sh @@ -89,7 +89,10 @@ if [ "$SMOKE" = "1" ]; then export NUM_TRAIN_TS=${NUM_TRAIN_TS:-1} export NUM_TRAIN_BATCHES=${NUM_TRAIN_BATCHES:-20} export NUM_EVAL_BATCHES=${NUM_EVAL_BATCHES:-10} - export EVAL_EVERY_N_WINDOWS=${EVAL_EVERY_N_WINDOWS:-1} + # Default eval cadence: per-window OFF (0), data-fraction every 0.5% of data + # (0.005). Mutually exclusive (both >0 raises a ValueError at startup). + export EVAL_EVERY_N_WINDOWS=${EVAL_EVERY_N_WINDOWS:-0} + export EVAL_EVERY_DATA_PCT=${EVAL_EVERY_DATA_PCT:-0.005} export METRIC_LOG_FREQ=${METRIC_LOG_FREQ:-5} # Smaller per-sample shape keeps the smoke run light; drop these to use the # gin defaults (4086/4096). Reuse an existing hstu_cache_L/ if present. diff --git a/recommendation_v4/scripts/launch_slurm.sh b/recommendation_v4/scripts/launch_slurm.sh index a1b6334e3..af9ff3907 100755 --- a/recommendation_v4/scripts/launch_slurm.sh +++ b/recommendation_v4/scripts/launch_slurm.sh @@ -142,7 +142,10 @@ orchestrate() { NUM_TRAIN_BATCHES=${NUM_TRAIN_BATCHES:-20} NUM_EVAL_BATCHES=${NUM_EVAL_BATCHES:-10} EVAL_EACH_WINDOW=${EVAL_EACH_WINDOW:-1} - EVAL_EVERY_N_WINDOWS=${EVAL_EVERY_N_WINDOWS:-1} + # Default eval cadence: per-window OFF (0), data-fraction every 0.5% of data + # (0.005). The two are mutually exclusive (both >0 raises a ValueError). + EVAL_EVERY_N_WINDOWS=${EVAL_EVERY_N_WINDOWS:-0} + EVAL_EVERY_DATA_PCT=${EVAL_EVERY_DATA_PCT:-0.005} METRIC_LOG_FREQ=${METRIC_LOG_FREQ:-5} FORCE_PROVISION=${FORCE_PROVISION:-0} From 7e8de35d1ff10130c8cf5345a239a216945a4467 Mon Sep 17 00:00:00 2001 From: chris Date: Fri, 26 Jun 2026 09:26:05 +0000 Subject: [PATCH 104/127] dlrmv3 streaming: fix distributed sync + generalize checkpoint/resume e2e test - broadcast total_train_anchors from rank-0 (avoid redundant mmap-gather + UID-hash recompute on every rank) and add a window-boundary dist.barrier() to prevent NCCL collective deadlock on skewed per-rank data prep. - generalize streaming_resume_test.sh with --platform auto-detect for both NVIDIA B200 and AMD MI350/355 (container names, dataset paths, ckpt roots, node-local data staging), adding midwindow + multiwindow scenarios. - extend streaming_resume_test.py with a `summarize` subcommand that parses logs for anchor-broadcast, window-barrier, eval-trigger and resume signals. - env-gated barrier debug log in utils.py for test observability. Validated end-to-end on B200: both midwindow and multiwindow scenarios PASS. Co-authored-by: Cursor --- .../dlrm_v3/train/_env_bootstrap.py | 12 +- .../train/tests/streaming_resume_test.py | 125 +++- .../train/tests/streaming_resume_test.sh | 561 ++++++++++++------ .../dlrm_v3/train/utils.py | 106 +++- recommendation_v4/scripts/launch_slurm.sh | 5 +- 5 files changed, 619 insertions(+), 190 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/_env_bootstrap.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/_env_bootstrap.py index 2890851de..5470d4e39 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/_env_bootstrap.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/_env_bootstrap.py @@ -23,6 +23,16 @@ def apply_env_bootstrap( TRITON_FULL_AUTOTUNE: Optional[bool] = None, ) -> None: - if TRITON_FULL_AUTOTUNE is not None: + # A pre-set environment variable wins over the gin binding. The pinned + # triton configs are MI350X-specific, so a different GPU arch (e.g. B200 + # sm_100) sets TRITON_FULL_AUTOTUNE=1 in the launcher environment to + # re-enable the full autotune search WITHOUT editing this (AMD-default) + # gin file. Cross-cluster launchers thus stay config-as-code via env. + if "TRITON_FULL_AUTOTUNE" in os.environ: + logger.info( + "env bootstrap: honoring pre-set TRITON_FULL_AUTOTUNE=%s (overrides gin binding)", + os.environ["TRITON_FULL_AUTOTUNE"], + ) + elif TRITON_FULL_AUTOTUNE is not None: os.environ["TRITON_FULL_AUTOTUNE"] = "1" if TRITON_FULL_AUTOTUNE else "0" logger.info("env bootstrap: TRITON_FULL_AUTOTUNE=%s", os.environ["TRITON_FULL_AUTOTUNE"]) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.py index b46da936b..cfe9b2e84 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.py @@ -14,12 +14,27 @@ """End-to-end failure-injection test for streaming resume. -Validates the four resume features end-to-end on the yambda-5b stack: +Two scenarios, driven by the sibling `streaming_resume_test.sh` (see its header +for the full B200 launch wiring). This module is the shared log parser + a CLI +the driver shells out to. + +SCENARIO `midwindow` — exact-once mid-window resume. Validates the four +single-window resume features end-to-end on the yambda-5b stack: 1. Mid-window save (in_window_checkpoint_frequency) 2. Within-window exact-once skip (StreamingWindowSampler.set_window skip) 3. Auto-detect-latest checkpoint subdir 4. keep_last_n retention (default 1) +SCENARIO `multiwindow` — distributed-sync regression guard for the two fixes the +mid-window test cannot reach (it runs ONE window with per-window eval off): + A. total_train_anchors() computed once on rank 0 + broadcast (not world_size×). + B. window-boundary dist.barrier() before the first forward of each window. +Both only matter across >=2 windows with the data-fraction eval cadence +(EVAL_EVERY_DATA_PCT>0) active, and the deadlock they fix originally struck at a +window boundary mid-run — so the scenario trains multiple windows AND resumes +across a completed-window boundary. The signals are extracted by `summarize` +(see `summarize_run`) and asserted in the shell driver. + Test flow (driven by the sibling `streaming_resume_test.sh`): Phase 1 (baseline): Run streaming-train-eval for N=2 train_ts × K batches/window with die_at_step=-1. Capture per-batch window_ne / window_auc into traj_baseline.json. @@ -49,7 +64,7 @@ import json import re import sys -from typing import Dict, Tuple +from typing import Dict, List, Optional, Tuple # Per-step metrics from MetricsLogger.compute_and_log are emitted like: # "train - Step 51 metrics: {'metric/lifetime_ne/listen_plus': tensor(1.0954, ...) @@ -61,6 +76,97 @@ _WACC_RE = re.compile(r"window_accuracy/listen_plus.*?tensor\(([0-9.]+)") +# --- multi-window / data-pct-eval regression signals ------------------------- +# These cover the two distributed-sync fixes that the single-window mid-window +# test above does NOT exercise (it runs one window with per-window eval off): +# +# (A) total_train_anchors() rank-0 broadcast. The data-fraction eval cadence +# needs total_train_anchors — a multi-minute, single-threaded O(N) gather +# + uid-hash over the mmap'd anchor array. Run on EVERY rank it both wastes +# 8x CPU and desyncs the NCCL stream (a fast rank races into the first +# embedding all-to-all while slow ranks still hash) → deadlock. The fix +# computes it ONCE on rank 0 and broadcasts the scalar. yambda logs exactly +# one `total_train_anchors(start_ts=…)` line per call, so the regression +# guard is: that line appears EXACTLY ONCE per launch (was world_size×). +# +# (B) window-boundary barrier. Per-window data prep (`window_indices`, an O(N) +# mask over the ~18GB mmap) finishes at very different times across ranks; +# without a sync before the first forward the collective stream desyncs and +# the job hangs at the boundary. The fix adds a dist.barrier() at each +# window boundary. It is silent on the healthy path, so the trainer emits a +# `[window-barrier] … rendezvous complete` line (rank 0) per crossed window +# ONLY under WINDOW_BARRIER_DEBUG=1 — the guard counts those == #windows. +_TTA_RE = re.compile(r"total_train_anchors\(start_ts=(\d+),\s*num_ts=(\d+)\):") +_BARRIER_RE = re.compile(r"\[window-barrier\] train_ts=(\d+) rendezvous complete") +_DATA_PCT_SETUP_RE = re.compile( + r"\[data-pct-eval\] eval_every_data_pct=.*?eval_interval_steps=(\d+)" +) +_DATA_PCT_TRIGGER_RE = re.compile(r"\[data-pct-eval\] trigger eval train_ts=(\d+)") +_RESUME_COMPLETED_RE = re.compile(r"Resuming from completed train_ts=(\d+)") +_RESUME_MIDWINDOW_RE = re.compile( + r"Resuming mid-window at train_ts=(\d+) batch_idx_in_window=(\d+)" +) +# Test driver appends this sentinel after the trainer returns (clean OR crash); +# code 0 == the run finished all requested windows + final eval without hanging. +_PHASE_EXIT_RE = re.compile(r"PHASE_EXIT=(-?\d+)") + + +def summarize_run(log_path: str) -> Dict[str, object]: + """Extract the multi-window / data-pct-eval regression signals from a run log. + + Returns a JSON-able dict the shell driver asserts on. All counts are over the + WHOLE log (one launch's worth — the driver uses a fresh per-phase log).""" + tta_calls: List[Tuple[int, int]] = [] + barrier_windows: List[int] = [] + data_pct_eval_setup: bool = False + data_pct_eval_interval: Optional[int] = None + data_pct_eval_triggers: List[int] = [] + resume_completed_ts: Optional[int] = None + resume_midwindow: Optional[Tuple[int, int]] = None + phase_exit: Optional[int] = None + with open(log_path, "r", errors="replace") as f: + for line in f: + m = _TTA_RE.search(line) + if m: + tta_calls.append((int(m.group(1)), int(m.group(2)))) + m = _BARRIER_RE.search(line) + if m: + barrier_windows.append(int(m.group(1))) + m = _DATA_PCT_SETUP_RE.search(line) + if m: + data_pct_eval_setup = True + data_pct_eval_interval = int(m.group(1)) + m = _DATA_PCT_TRIGGER_RE.search(line) + if m: + data_pct_eval_triggers.append(int(m.group(1))) + m = _RESUME_COMPLETED_RE.search(line) + if m: + resume_completed_ts = int(m.group(1)) + m = _RESUME_MIDWINDOW_RE.search(line) + if m: + resume_midwindow = (int(m.group(1)), int(m.group(2))) + m = _PHASE_EXIT_RE.search(line) + if m: + phase_exit = int(m.group(1)) + return { + # (A) rank-0 broadcast: must be exactly 1 (was world_size× before the fix) + "total_train_anchors_calls": len(tta_calls), + "total_train_anchors_args": tta_calls, + # (B) barrier executed once per crossed window (rank 0, debug-gated) + "window_barrier_count": len(barrier_windows), + "windows_trained": sorted(set(barrier_windows)), + # data-fraction eval cadence active + actually fired + "data_pct_eval_setup": data_pct_eval_setup, + "data_pct_eval_interval_steps": data_pct_eval_interval, + "data_pct_eval_trigger_count": len(data_pct_eval_triggers), + # resume classification + "resume_completed_ts": resume_completed_ts, + "resume_midwindow": resume_midwindow, + # terminal status (None => still running / killed without sentinel) + "phase_exit": phase_exit, + } + + def parse_trajectory(log_path: str) -> Dict[int, Dict[str, float]]: """Extract a {step: {window_ne, window_auc, window_accuracy}} dict from a train.log. The grep is loose on the metric line itself — we accept the @@ -142,6 +248,13 @@ def main() -> int: p_cmp.add_argument("--min-resume-step", type=int, required=True) p_cmp.add_argument("--atol", type=float, default=0.15) + p_sum = sub.add_parser( + "summarize", + help="Emit multi-window / data-pct-eval regression signals from a run log", + ) + p_sum.add_argument("log") + p_sum.add_argument("out", nargs="?", help="optional JSON output path") + args = ap.parse_args() if args.cmd == "parse": traj = parse_trajectory(args.log) @@ -159,6 +272,14 @@ def main() -> int: ) print(msg) return 0 if ok else 1 + if args.cmd == "summarize": + summary = summarize_run(args.log) + out = json.dumps(summary, indent=2) + if args.out: + with open(args.out, "w") as f: + f.write(out) + print(out) + return 0 return 0 diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh b/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh index e14e557e8..afdc65805 100755 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh @@ -1,242 +1,443 @@ #!/bin/bash # End-to-end failure-injection + resume test for streaming-train-eval. # -# Validates exact-once mid-window resume on the yambda-5b stack: -# Phase 1 (baseline): uninterrupted run for N=2 train_ts × K batches/window -# Phase 2 (interrupted): same config but die_at_step=M → exits at step M -# after the in-window checkpoint lands -# Phase 3 (resume): re-launch with same CKPT_PATH → auto-latest picks -# the in-window save → finishes the partial window -# and the rest of the requested train_ts list -# Assertion: traj_resumed[step].window_ne / window_auc / window_accuracy match -# traj_baseline bit-equal (np.allclose atol=1e-4) for all step > die_at_step. +# PLATFORM-GENERAL: runs on both NVIDIA B200 and AMD MI350/MI355 (ROCm/meta64). +# The only hardware-specific bits are picked by --platform (auto-detected from the +# running container if omitted): the container name, the dataset path, and the +# checkpoint root. Everything else — the worker entrypoint (scripts/launch_slurm.sh, +# which is the shared launcher both clusters' supervisors use), the env-driven gin +# knobs, and all assertions — is identical across platforms. # -# Driven entirely via env-driven gin knobs defined in yambda_5b.gin: -# NUM_TRAIN_TS / NUM_TRAIN_BATCHES / IN_WINDOW_CKPT_FREQ / DIE_AT_STEP / -# CKPT_PATH / KEEP_LAST_N / EVAL_EVERY_N_WINDOWS +# Two scenarios (select with --scenario; default runs both): +# +# midwindow — exact-once MID-WINDOW resume (single window). +# P1 baseline: uninterrupted 1 train_ts × K batches. +# P2 interrupted: same + die_at_step=M → exits AFTER the in-window ckpt at M. +# P3 resume: relaunch w/ same CKPT_PATH → auto-latest picks the in-window +# save, skips the M already-trained batches, finishes. +# Gates: re-entered at batch_idx_in_window=M, per-rank RNG restored, first +# resumed step == M+1, atomic save + keep_last_n, trajectory within --atol. +# +# multiwindow — distributed-sync REGRESSION guard for the two fixes the +# mid-window test cannot reach (it runs ONE window with per-window eval off): +# (A) total_train_anchors() computed ONCE on rank 0 + broadcast (the +# data-fraction eval cadence needs it; running the multi-minute O(N) +# mmap gather + uid-hash on every rank desynced NCCL → boundary hang). +# (B) a dist.barrier() at every window boundary before the first forward +# (per-rank data-prep skew otherwise desyncs the collective stream). +# Both only bite across >=2 windows with EVAL_EVERY_DATA_PCT>0, and the +# deadlock struck at a boundary mid-run, so: +# P1 mw_baseline: cold run over MW_TS windows w/ data-pct eval. Asserts +# total_train_anchors logged EXACTLY ONCE (computed at setup + broadcast +# from rank 0), the barrier fired on EVERY window, the data-pct cadence +# was set up, and the run COMPLETED (no boundary hang). +# P2 mw_seed: 1 window → clean end-of-window (WINDOW_COMPLETE) ckpt. +# P3 mw_resume: relaunch over MW_TS windows w/ same CKPT_PATH → resumes +# past the completed window and CROSSES the boundary into the next +# windows. Asserts "Resuming from completed", barrier fired on each +# remaining window, anchors broadcast once, and the run COMPLETED — +# i.e. the exact boundary-crossing-on-resume case that used to hang. +# +# Driven entirely via env-driven gin knobs (yambda_5b.gin) through the SAME B200 +# worker entrypoint the production supervisor uses: `bash scripts/launch_slurm.sh` +# (worker phase, auto-detected inside the container). WINDOW_BARRIER_DEBUG=1 makes +# the otherwise-silent barrier emit one rank-0 line per crossed window. +# +# CHECKPOINT/DATASET PLACEMENT (the one real platform difference): +# * B200: virtiofs/NFS WEDGES under the trainer's concurrent mmap LOAD, so the +# checkpoint root AND the mmap'd dataset cache MUST be node-local (defaults +# /tmp/...). The dataset must already be staged node-local at --data-path +# (the e2e supervisor's stage_data_in does this); the test fails fast if not. +# * MI350/MI355 (meta64): NFS mmap is fine, so the checkpoint root + dataset +# read directly from shared NFS (defaults /apps/chcai/...), as the original +# test did. No staging needed. +# Logs always use read()/write() only, so they live on shared /apps/chcai and +# are grep-able from the head node on both platforms. # # Usage: -# bash generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh --jobid -# [--container yambda_primus] -# [--num-train-batches 200] -# [--die-at-step 350] -# [--keep] # retain LOG_DIR + CKPT after run for inspection +# bash generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh \ +# --jobid [--platform b200|mi350] [--scenario all] +# [--container ] [--data-path ] [--ckpt-root ] [--start-ts 150] +# [--num-train-batches 200] [--die-at-step 100] # midwindow knobs +# [--mw-num-train-ts 3] [--mw-num-train-batches 20] # multiwindow knobs +# [--mw-eval-pct 0.34] [--keep] +# --platform is auto-detected from the running container when omitted. Any of +# --container/--data-path/--ckpt-root override the platform default. set -uo pipefail JOBID="" -CONTAINER="yambda_primus" +REPO=/home/chcai/training/recommendation_v4 +DATASET_SUBDIR=processed_5b/hstu_cache_L4086 +SCENARIO=all # midwindow | multiwindow | all +START_TS=150 +KEEP=0 +LOG_DIR=/apps/chcai/streaming_resume_test # shared NFS (read()/write() only) +# Platform + the three platform-specific paths. Empty sentinels here; filled by +# apply_platform_defaults() AFTER platform detection unless the user overrode +# them on the command line. (DATA_PATH uses a distinct sentinel because an +# explicit empty value is meaningful: "do not inject DLRM_DATA_PATH; let the gin +# default apply".) +PLATFORM="" # b200 | mi350 | mi355 ; auto if empty +CONTAINER="" # default: per-platform +DATA_PATH="__AUTO__" # default: per-platform +CKPT_ROOT="" # default: per-platform (node-local on B200) + +# --- midwindow knobs --- NUM_TRAIN_BATCHES=200 -DIE_AT_STEP=350 +NUM_EVAL_BATCHES=5 # cap the per-phase FINAL eval (0 = full holdout, very slow) +DIE_AT_STEP=100 IN_WINDOW_FREQ=50 -KEEP=0 -# Trajectory closeness bound — NOT a bit-equality check. The ROCm training stack -# is nondeterministic across runs (non-deterministic atomic scatter-add in the -# embedding/attention backward): two independent *cold* runs already drift -# ~7e-4 in window_ne over 20 steps, and early-training chaos (AUC~0.5) amplifies -# any seed difference. So resume-vs-baseline can legitimately differ by a few -# percent. This bound just catches GROSS divergence (wrong data skip, totally -# unrestored state) while tolerating nondeterministic drift. The HARD resume -# correctness gates are the functional-invariant checks below (RNG restored, -# resumed-at-correct-step, atomic/keep_last_n), not this number. -ATOL=0.15 -CKPT_ROOT=/apps/chcai/ckpts_resume_test -LOG_DIR=/apps/chcai/streaming_resume_test -REPO=/home/chcai/training/recommendation_v4 +ATOL=0.15 # trajectory closeness bound (NOT bit-equality; see py module) +MW_TIMEOUT=1800 + +# --- multiwindow knobs --- +MW_TS=3 # windows to train (>=2 to cross a boundary) +MW_BATCHES=20 # train batches per window (small = fast) +MW_EVAL_BATCHES=5 # holdout eval batches per fired eval +MW_EVAL_PCT=0.34 # data-fraction eval cadence (>0 enables the anchors path) +MW_SPLIT=0.90 # train split (<1 => holdout exists => uid-hash anchor path) +MW_HOLDOUT_TS=200 # PINNED holdout window (must match across seed→resume) +MW_RUN_TIMEOUT=3600 # generous: init + planner + anchors gather can take min while [[ $# -gt 0 ]]; do case $1 in --jobid) JOBID="$2"; shift 2;; + --platform) PLATFORM="$2"; shift 2;; --container) CONTAINER="$2"; shift 2;; + --repo) REPO="$2"; shift 2;; + --data-path) DATA_PATH="$2"; shift 2;; + --dataset-subdir) DATASET_SUBDIR="$2"; shift 2;; + --scenario) SCENARIO="$2"; shift 2;; + --start-ts) START_TS="$2"; shift 2;; + --ckpt-root) CKPT_ROOT="$2"; shift 2;; + --log-dir) LOG_DIR="$2"; shift 2;; --num-train-batches) NUM_TRAIN_BATCHES="$2"; shift 2;; + --num-eval-batches) NUM_EVAL_BATCHES="$2"; shift 2;; --die-at-step) DIE_AT_STEP="$2"; shift 2;; --in-window-freq) IN_WINDOW_FREQ="$2"; shift 2;; --atol) ATOL="$2"; shift 2;; + --mw-num-train-ts) MW_TS="$2"; shift 2;; + --mw-num-train-batches) MW_BATCHES="$2"; shift 2;; + --mw-num-eval-batches) MW_EVAL_BATCHES="$2"; shift 2;; + --mw-eval-pct) MW_EVAL_PCT="$2"; shift 2;; + --mw-split) MW_SPLIT="$2"; shift 2;; + --mw-holdout-ts) MW_HOLDOUT_TS="$2"; shift 2;; --keep) KEEP=1; shift;; *) echo "Unknown arg: $1"; exit 1;; esac done [[ -z "$JOBID" ]] && { echo "Error: --jobid required"; exit 1; } +case "$SCENARIO" in midwindow|multiwindow|all) ;; *) echo "Error: --scenario must be midwindow|multiwindow|all"; exit 1;; esac +(( MW_TS < 2 )) && { echo "Error: --mw-num-train-ts must be >=2 to cross a boundary"; exit 1; } +[[ -n "$PLATFORM" ]] && case "$PLATFORM" in b200|mi350|mi355) ;; *) echo "Error: --platform must be b200|mi350|mi355"; exit 1;; esac + +# --- resolve platform + its three hardware-specific paths -------------------- +# Precedence: explicit --platform > inferred from explicit --container > probe +# the allocation's docker for a known training container > default b200. +if [[ -z "$PLATFORM" ]]; then + if [[ "$CONTAINER" == "yambda_b200" ]]; then PLATFORM=b200 + elif [[ "$CONTAINER" == "yambda_primus" ]]; then PLATFORM=mi350 + else + _names=$(srun --jobid="$JOBID" --overlap docker ps -a --format '{{.Names}}' 2>/dev/null) + if grep -qx yambda_b200 <<<"$_names"; then PLATFORM=b200 + elif grep -qx yambda_primus <<<"$_names"; then PLATFORM=mi350 + else PLATFORM=b200; echo "Warning: could not auto-detect platform (no known container on job $JOBID) — defaulting to b200"; fi + fi + echo "[$(date)] auto-detected platform: $PLATFORM" +fi +case "$PLATFORM" in + b200) + : "${CONTAINER:=yambda_b200}" + # B200: mmap (ckpt LOAD + dataset cache) must NOT touch virtiofs/NFS. + [[ "$DATA_PATH" == "__AUTO__" ]] && DATA_PATH=/tmp/yambda_data + : "${CKPT_ROOT:=/tmp/yambda_resume_test/ckpts}" + ;; + mi350|mi355) + : "${CONTAINER:=yambda_primus}" + # meta64: NFS mmap is fine — read dataset + write ckpt directly on NFS + # (matches the original MI350 test). /apps/chcai/dlrm_data is the gin default. + [[ "$DATA_PATH" == "__AUTO__" ]] && DATA_PATH=/apps/chcai/dlrm_data + : "${CKPT_ROOT:=/apps/chcai/ckpts_resume_test}" + ;; +esac +echo "[$(date)] platform=$PLATFORM container=$CONTAINER data_path=${DATA_PATH:-} ckpt_root=$CKPT_ROOT" mkdir -p "$LOG_DIR" +PYHELPER="$REPO/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.py" -# Single-window mid-window resume: NUM_TRAIN_TS=1, so the whole test runs inside -# train_ts=START_TS. die_at_step must land strictly inside that window, AT a -# multiple of IN_WINDOW_FREQ so an in-window checkpoint is saved right before -# the crash (resume then skips exactly DIE_AT_STEP already-trained batches). -if (( DIE_AT_STEP <= 0 || DIE_AT_STEP >= NUM_TRAIN_BATCHES )); then - echo "Warning: die_at_step=$DIE_AT_STEP not strictly inside window (0, $NUM_TRAIN_BATCHES)" >&2 -fi -if (( DIE_AT_STEP % IN_WINDOW_FREQ != 0 )); then - echo "Warning: die_at_step=$DIE_AT_STEP not a multiple of in_window_freq=$IN_WINDOW_FREQ; no save lands exactly at crash" >&2 -fi +# --- container helpers (inspect CKPT/dataset via docker exec — works whether the +# path is node-local on B200 or shared NFS on MI350) --- +sx() { srun --jobid="$JOBID" --overlap docker exec "$CONTAINER" bash -lc "$1" 2>/dev/null; } cleanup_workers() { - srun --jobid="$JOBID" --overlap docker exec "$CONTAINER" bash -lc \ - "pkill -9 -f generative_recommenders 2>/dev/null; sleep 2; \ - pkill -9 -f spawn_main 2>/dev/null; sleep 3; true" 2>/dev/null || true + sx "pkill -9 -f train_ranker 2>/dev/null; pkill -9 -f generative_recommenders 2>/dev/null; \ + pkill -9 -f multiprocessing 2>/dev/null; sleep 2; pkill -9 -f spawn_main 2>/dev/null; sleep 3; true" || true } -clean_ckpt() { - srun --jobid="$JOBID" --overlap docker exec "$CONTAINER" rm -rf "$CKPT_ROOT" 2>/dev/null || true +clean_ckpt() { sx "rm -rf '$1'" || true; } + +# Precheck: the dataset cache must be readable at $DATA_PATH. On B200 it must be +# staged node-local (the supervisor's stage_data_in does this) since mmap from +# virtiofs/NFS wedges; on MI350 it reads directly from NFS. Skipped when DATA_PATH +# is empty (the trainer falls back to its gin default and we don't know the path). +precheck_data() { + [[ -z "$DATA_PATH" ]] && { echo "[$(date)] data path unset — trainer will use its gin default; skipping precheck"; return 0; } + local ok + ok=$(sx "[ -d '$DATA_PATH/$DATASET_SUBDIR' ] && echo yes || echo no") + if [[ "$ok" != "yes" ]]; then + echo "FAIL: dataset cache not found at $DATA_PATH/$DATASET_SUBDIR inside '$CONTAINER' (platform=$PLATFORM)." + if [[ "$PLATFORM" == "b200" ]]; then + echo " B200: stage it node-local first (the e2e supervisor does this via stage_data_in)," + echo " or pass --data-path to an already-staged local mirror. mmap from virtiofs/NFS wedges." + else + echo " MI350/MI355: pass --data-path to the NFS dataset root (gin default is /apps/chcai/dlrm_data)." + fi + exit 1 + fi } -# Wait for a log line to appear OR a crash sentinel. Returns 0 if target found, -# 1 if crash sentinel found first. +# Wait (host-side grep on the shared-NFS log) for a target regex OR a crash +# sentinel. 0=target found, 1=crash first, 2=timeout. wait_for_log() { - local log="$1"; local target_re="$2"; local timeout_s="${3:-1500}" + local log="$LOG_DIR/$1.log"; local target_re="$2"; local timeout_s="${3:-1800}" local elapsed=0 while (( elapsed < timeout_s )); do - if grep -qE "$target_re" "$log" 2>/dev/null; then - return 0 - fi - if grep -qE "Traceback|RuntimeError|OutOfMemoryError" "$log" 2>/dev/null; then - return 1 - fi - sleep 5 - elapsed=$((elapsed + 5)) + grep -qE "$target_re" "$log" 2>/dev/null && return 0 + grep -qE "Traceback|RuntimeError|OutOfMemoryError|CUDA error" "$log" 2>/dev/null && return 1 + sleep 5; elapsed=$((elapsed + 5)) done return 2 } -# Single train window of NUM_TRAIN_BATCHES steps → last train step == NUM_TRAIN_BATCHES. -LAST_STEP=$NUM_TRAIN_BATCHES - +# Launch one trainer phase (detached), appending a PHASE_EXIT sentinel after the +# trainer returns (clean OR crash) — exactly like the production supervisor. The +# common env (data path, mode, start_ts, barrier debug) is fixed; per-phase knobs +# are passed as additional "K=V" words. run_phase() { local name="$1"; shift local log="$LOG_DIR/${name}.log" - # Join the per-phase env overrides into ONE word. Using `$*` (not `$@`) is - # essential: `$@` embedded mid-string in the double-quoted `bash -lc "..."` - # expands to *multiple* arguments, so bash -lc would only run up to the - # first override and treat the rest as positional params — launch_smoke - # would never execute (silent 0-byte log). + # `$*` (joined into ONE word), NOT `$@`: embedded mid-string in the + # double-quoted `bash -lc "..."`, `$@` would expand to multiple args and + # bash -lc would stop after the first override (silent 0-byte log). local env_overrides="$*" + # Inject DLRM_DATA_PATH only when a path is set; an empty DATA_PATH means + # "use the trainer's gin default" (the meta64 NFS root). + local data_env="" + [[ -n "$DATA_PATH" ]] && data_env="DLRM_DATA_PATH=$DATA_PATH" : > "$log" echo "[$(date)] === phase '$name' ===" cleanup_workers srun --jobid="$JOBID" --overlap docker exec -d "$CONTAINER" bash -lc " cd $REPO && + $data_env \ HSTU_HAMMER_KERNEL=TRITON \ - $env_overrides \ + MODE=streaming-train-eval \ + START_TS=$START_TS \ + WINDOW_BARRIER_DEBUG=1 \ RUN_NAME=resume_test_$name \ LOG=$log \ - bash scripts/launch_smoke_8gpu.sh + $env_overrides \ + bash scripts/launch_slurm.sh; + echo \"PHASE_EXIT=\$? \$(date '+%F %T')\" >> $log " } -# === Phase 1: baseline === -clean_ckpt -run_phase baseline \ - "NUM_TRAIN_TS=1" \ - "EVAL_EVERY_N_WINDOWS=0" \ - "METRIC_LOG_FREQ=1" \ - "NUM_TRAIN_BATCHES=$NUM_TRAIN_BATCHES" \ - "DIE_AT_STEP=-1" -wait_for_log "$LOG_DIR/baseline.log" "train - Step $LAST_STEP metrics" 1500 -rc=$? -cleanup_workers -[[ $rc -ne 0 ]] && { echo "FAIL: baseline didn't finish"; tail -20 "$LOG_DIR/baseline.log"; exit 1; } - -# === Phase 2: interrupted === -clean_ckpt -run_phase interrupt \ - "NUM_TRAIN_TS=1" \ - "EVAL_EVERY_N_WINDOWS=0" \ - "METRIC_LOG_FREQ=1" \ - "NUM_TRAIN_BATCHES=$NUM_TRAIN_BATCHES" \ - "IN_WINDOW_CKPT_FREQ=$IN_WINDOW_FREQ" \ - "KEEP_LAST_N=1" \ - "DIE_AT_STEP=$DIE_AT_STEP" \ - "CKPT_PATH=$CKPT_ROOT" -wait_for_log "$LOG_DIR/interrupt.log" "die_at_step=$DIE_AT_STEP hit" 1500 -rc=$? -cleanup_workers -[[ $rc -ne 0 ]] && { echo "FAIL: interrupt didn't hit die_at_step"; tail -20 "$LOG_DIR/interrupt.log"; exit 1; } - -SAVED=$(srun --jobid="$JOBID" --overlap docker exec "$CONTAINER" ls "$CKPT_ROOT" 2>/dev/null | tr '\n' ' ') -echo "Saved checkpoints after interrupt: $SAVED" - -# === Phase 3: resume === -run_phase resume \ - "NUM_TRAIN_TS=1" \ - "EVAL_EVERY_N_WINDOWS=0" \ - "METRIC_LOG_FREQ=1" \ - "NUM_TRAIN_BATCHES=$NUM_TRAIN_BATCHES" \ - "IN_WINDOW_CKPT_FREQ=$IN_WINDOW_FREQ" \ - "KEEP_LAST_N=1" \ - "DIE_AT_STEP=-1" \ - "CKPT_PATH=$CKPT_ROOT" -wait_for_log "$LOG_DIR/resume.log" "train - Step $LAST_STEP metrics" 1500 -rc=$? -[[ $rc -ne 0 ]] && { cleanup_workers; echo "FAIL: resume didn't finish"; tail -20 "$LOG_DIR/resume.log"; exit 1; } -# The resume run performs an end-of-window checkpoint save AFTER the final -# step's metric line. That save (hundreds of GB) writes .tmp and then -# atomically renames it onto , logging "checkpoint successfully saved" only -# once the rename completes. If we kill workers right after the step line we'd -# orphan a half-written .tmp and trip the stale-dir gate below — a harness -# race, not a resume bug. Wait for the save to finish before tearing down. -wait_for_log "$LOG_DIR/resume.log" "checkpoint successfully saved" 1500 -save_rc=$? -cleanup_workers -[[ $save_rc -ne 0 ]] && { echo "FAIL: resume end-of-window checkpoint save did not complete"; tail -20 "$LOG_DIR/resume.log"; exit 1; } - -# === HARD resume-correctness gates (functional invariants) === -# These — not the trajectory closeness check below — are the authoritative -# proof the resume path is correct, because they're deterministic and immune -# to the GPU nondeterminism that perturbs the metric trajectory. - -# (1) Re-entered the partial window at exactly the saved batch_idx_in_window. -if ! grep -qE "Resuming mid-window at train_ts=[0-9]+ batch_idx_in_window=$DIE_AT_STEP\b" "$LOG_DIR/resume.log" 2>/dev/null; then - echo "FAIL: resume did not re-enter mid-window at batch_idx_in_window=$DIE_AT_STEP" - grep -E "Resuming" "$LOG_DIR/resume.log" 2>/dev/null | head -2 - exit 1 -fi -# (2) Per-rank RNG state was actually restored (dropout determinism path). -RNG_RESTORED=$(grep -c "RNG state restored from" "$LOG_DIR/resume.log" 2>/dev/null || echo 0) -echo "RNG state restored on $RNG_RESTORED ranks" -[[ "$RNG_RESTORED" -lt 1 ]] && { echo "FAIL: no RNG state restored on resume"; exit 1; } -# (3) The FIRST training step after resume is exactly die_at_step+1, i.e. the -# skip-already-trained-batches logic emitted the next unseen batch (not a -# restart from step 1, and not a gap). -FIRST_RESUMED=$(grep -oE 'train - Step [0-9]+ metrics: \{.metric' "$LOG_DIR/resume.log" 2>/dev/null \ - | grep -oE 'Step [0-9]+' | awk '{print $2}' | sort -n | head -1) -echo "First resumed train step: $FIRST_RESUMED (expect $((DIE_AT_STEP + 1)))" -[[ "$FIRST_RESUMED" != "$((DIE_AT_STEP + 1))" ]] && { - echo "FAIL: resume did not continue at step $((DIE_AT_STEP + 1)) (got $FIRST_RESUMED)"; exit 1; } - -# === Final on-disk state checks (atomic save + retention) === -NUM_CKPT=$(srun --jobid="$JOBID" --overlap docker exec "$CONTAINER" bash -lc \ - "ls $CKPT_ROOT 2>/dev/null | grep -E '^[0-9]+$' | wc -l" | tr -d ' ') -# Both .tmp (interrupted write) and .old (interrupted atomic-overwrite swap) -# must be absent — their presence means a save crashed without clean recovery. -STALE_CKPT=$(srun --jobid="$JOBID" --overlap docker exec "$CONTAINER" bash -lc \ - "ls $CKPT_ROOT 2>/dev/null | grep -E '\\.(tmp|old)$' | wc -l" | tr -d ' ') -echo "Final: $NUM_CKPT numeric ckpt subdirs, $STALE_CKPT stale (.tmp/.old) dirs (expect 1, 0)" -[[ "$NUM_CKPT" != "1" ]] && { echo "FAIL: keep_last_n=1 violated"; exit 1; } -[[ "$STALE_CKPT" != "0" ]] && { echo "FAIL: stale .tmp/.old dirs left behind"; exit 1; } -echo "=== Resume functional invariants: ALL PASS ===" - -# === Trajectory closeness (sanity bound, NOT bit-equality) === -# Catches gross resume bugs (wrong data slice, unrestored model) that throw the -# metric trajectory far off. Small drift is expected & tolerated (see ATOL note -# at top). The functional invariants above are the real correctness proof. -python3 $REPO/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.py parse \ - "$LOG_DIR/baseline.log" "$LOG_DIR/traj_baseline.json" -python3 $REPO/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.py parse \ - "$LOG_DIR/resume.log" "$LOG_DIR/traj_resumed.json" - -python3 $REPO/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.py compare \ - "$LOG_DIR/traj_baseline.json" "$LOG_DIR/traj_resumed.json" \ - --min-resume-step $((DIE_AT_STEP + 1)) --atol $ATOL -RC=$? +# Read a scalar field from a summarize-JSON. +jget() { python3 -c "import json,sys;print(json.load(open(sys.argv[1])).get(sys.argv[2]))" "$1" "$2"; } + +FAIL=0 +fail() { echo "FAIL: $*"; FAIL=1; } + +precheck_data + +# ============================================================================= +# SCENARIO: midwindow +# ============================================================================= +run_midwindow() { + echo "########## scenario: midwindow ##########" + local LAST_STEP=$NUM_TRAIN_BATCHES + if (( DIE_AT_STEP <= 0 || DIE_AT_STEP >= NUM_TRAIN_BATCHES )); then + echo "Warning: die_at_step=$DIE_AT_STEP not strictly inside window (0, $NUM_TRAIN_BATCHES)" >&2 + fi + if (( DIE_AT_STEP % IN_WINDOW_FREQ != 0 )); then + echo "Warning: die_at_step=$DIE_AT_STEP not a multiple of in_window_freq=$IN_WINDOW_FREQ; no save lands exactly at crash" >&2 + fi + + # P1 baseline + clean_ckpt "$CKPT_ROOT" + run_phase baseline \ + "NUM_TRAIN_TS=1" "EVAL_EVERY_N_WINDOWS=0" "METRIC_LOG_FREQ=1" \ + "NUM_TRAIN_BATCHES=$NUM_TRAIN_BATCHES" "NUM_EVAL_BATCHES=$NUM_EVAL_BATCHES" \ + "TRAIN_SPLIT_PERCENTAGE=1.0" "DIE_AT_STEP=-1" + wait_for_log baseline "PHASE_EXIT=0" "$MW_TIMEOUT"; local rc=$? + cleanup_workers + (( rc != 0 )) && { echo "FAIL: midwindow baseline didn't finish (rc=$rc)"; tail -20 "$LOG_DIR/baseline.log"; return 1; } + + # P2 interrupted + clean_ckpt "$CKPT_ROOT" + run_phase interrupt \ + "NUM_TRAIN_TS=1" "EVAL_EVERY_N_WINDOWS=0" "METRIC_LOG_FREQ=1" \ + "NUM_TRAIN_BATCHES=$NUM_TRAIN_BATCHES" "NUM_EVAL_BATCHES=$NUM_EVAL_BATCHES" \ + "TRAIN_SPLIT_PERCENTAGE=1.0" \ + "IN_WINDOW_CKPT_FREQ=$IN_WINDOW_FREQ" "KEEP_LAST_N=1" \ + "DIE_AT_STEP=$DIE_AT_STEP" "CKPT_PATH=$CKPT_ROOT" + wait_for_log interrupt "die_at_step=$DIE_AT_STEP hit" "$MW_TIMEOUT"; rc=$? + cleanup_workers + (( rc != 0 )) && { echo "FAIL: interrupt didn't hit die_at_step (rc=$rc)"; tail -20 "$LOG_DIR/interrupt.log"; return 1; } + echo "Saved checkpoints after interrupt: $(sx "ls '$CKPT_ROOT' 2>/dev/null | tr '\n' ' '")" + + # P3 resume + run_phase resume \ + "NUM_TRAIN_TS=1" "EVAL_EVERY_N_WINDOWS=0" "METRIC_LOG_FREQ=1" \ + "NUM_TRAIN_BATCHES=$NUM_TRAIN_BATCHES" "NUM_EVAL_BATCHES=$NUM_EVAL_BATCHES" \ + "TRAIN_SPLIT_PERCENTAGE=1.0" \ + "IN_WINDOW_CKPT_FREQ=$IN_WINDOW_FREQ" "KEEP_LAST_N=1" \ + "DIE_AT_STEP=-1" "CKPT_PATH=$CKPT_ROOT" + # PHASE_EXIT=0 only after the (blocking) end-of-window save renames cleanly, + # so this also confirms the final atomic save completed. + wait_for_log resume "PHASE_EXIT=0" "$MW_TIMEOUT"; rc=$? + cleanup_workers + (( rc != 0 )) && { echo "FAIL: resume didn't finish (rc=$rc)"; tail -20 "$LOG_DIR/resume.log"; return 1; } + + # HARD functional invariants (deterministic; the real correctness proof). + if ! grep -qE "Resuming mid-window at train_ts=[0-9]+ batch_idx_in_window=$DIE_AT_STEP\b" "$LOG_DIR/resume.log" 2>/dev/null; then + fail "resume did not re-enter mid-window at batch_idx_in_window=$DIE_AT_STEP" + grep -E "Resuming" "$LOG_DIR/resume.log" 2>/dev/null | head -2 + fi + local rng_restored + rng_restored=$(grep -c "RNG state restored from" "$LOG_DIR/resume.log" 2>/dev/null || echo 0) + echo "RNG state restored on $rng_restored ranks" + (( rng_restored < 1 )) && fail "no RNG state restored on resume" + local first_resumed + first_resumed=$(grep -oE 'train - Step [0-9]+ metrics: \{.metric' "$LOG_DIR/resume.log" 2>/dev/null \ + | grep -oE 'Step [0-9]+' | awk '{print $2}' | sort -n | head -1) + echo "First resumed train step: $first_resumed (expect $((DIE_AT_STEP + 1)))" + [[ "$first_resumed" != "$((DIE_AT_STEP + 1))" ]] && fail "resume did not continue at step $((DIE_AT_STEP + 1)) (got $first_resumed)" + + # On-disk: atomic save + retention. + local num_ckpt stale_ckpt + num_ckpt=$(sx "ls '$CKPT_ROOT' 2>/dev/null | grep -E '^[0-9]+$' | wc -l" | tr -d ' ') + stale_ckpt=$(sx "ls '$CKPT_ROOT' 2>/dev/null | grep -E '\\.(tmp|old|staging)$' | wc -l" | tr -d ' ') + echo "Final: $num_ckpt numeric ckpt subdirs, $stale_ckpt stale dirs (expect 1, 0)" + [[ "$num_ckpt" != "1" ]] && fail "keep_last_n=1 violated (got $num_ckpt)" + [[ "$stale_ckpt" != "0" ]] && fail "stale .tmp/.old/.staging dirs left behind ($stale_ckpt)" + + # Trajectory closeness (loose sanity bound, NOT bit-equality). + python3 "$PYHELPER" parse "$LOG_DIR/baseline.log" "$LOG_DIR/traj_baseline.json" + python3 "$PYHELPER" parse "$LOG_DIR/resume.log" "$LOG_DIR/traj_resumed.json" + if ! python3 "$PYHELPER" compare "$LOG_DIR/traj_baseline.json" "$LOG_DIR/traj_resumed.json" \ + --min-resume-step $((DIE_AT_STEP + 1)) --atol "$ATOL"; then + fail "trajectory diverged beyond $ATOL (likely wrong data slice / unrestored state)" + fi + (( FAIL == 0 )) && echo "=== midwindow: PASS ===" || echo "=== midwindow: FAIL ===" +} + +# ============================================================================= +# SCENARIO: multiwindow (regression guard for the broadcast + barrier fixes) +# ============================================================================= +# Common split contract — MUST be byte-identical between mw_seed and mw_resume, +# else the resume aborts on a split-contract mismatch (the holdout_ts default of +# start_ts+num_train_ts differs between a 1-window seed and an MW_TS resume, so +# it is PINNED here). +MW_SPLIT_ENV=( "TRAIN_SPLIT_PERCENTAGE=$MW_SPLIT" "SPLIT_SALT=0" + "EVAL_HOLDOUT_TS=$MW_HOLDOUT_TS" "EVAL_HOLDOUT_NUM_WINDOWS=1" ) + +run_multiwindow() { + echo "########## scenario: multiwindow ##########" + local sum + + # P1 mw_baseline — cold multi-window run with data-pct eval. + clean_ckpt "$CKPT_ROOT" + run_phase mw_baseline \ + "NUM_TRAIN_TS=$MW_TS" "NUM_TRAIN_BATCHES=$MW_BATCHES" "NUM_EVAL_BATCHES=$MW_EVAL_BATCHES" \ + "EVAL_EVERY_N_WINDOWS=0" "EVAL_EVERY_DATA_PCT=$MW_EVAL_PCT" "METRIC_LOG_FREQ=1" \ + "${MW_SPLIT_ENV[@]}" + wait_for_log mw_baseline "PHASE_EXIT=" "$MW_RUN_TIMEOUT"; local rc=$? + cleanup_workers + (( rc == 1 )) && { echo "FAIL: mw_baseline crashed"; tail -30 "$LOG_DIR/mw_baseline.log"; return 1; } + (( rc == 2 )) && { echo "FAIL: mw_baseline timed out (possible boundary deadlock)"; tail -30 "$LOG_DIR/mw_baseline.log"; return 1; } + + sum="$LOG_DIR/mw_baseline.summary.json" + python3 "$PYHELPER" summarize "$LOG_DIR/mw_baseline.log" "$sum" >/dev/null + echo "--- mw_baseline summary ---"; cat "$sum" + local exit_code anchors barriers dpct_setup dpct_trig + exit_code=$(jget "$sum" phase_exit) + anchors=$(jget "$sum" total_train_anchors_calls) + barriers=$(jget "$sum" window_barrier_count) + dpct_setup=$(jget "$sum" data_pct_eval_setup) + dpct_trig=$(jget "$sum" data_pct_eval_trigger_count) + # (barrier B) ran through ALL windows and exited 0 — no boundary deadlock. + [[ "$exit_code" != "0" ]] && fail "mw_baseline did not complete cleanly (phase_exit=$exit_code)" + [[ "$barriers" != "$MW_TS" ]] && fail "window barrier fired $barriers times, expected $MW_TS (one per window; need world_size>=2)" + # (broadcast A) total_train_anchors computed exactly once (rank 0), not Nx. + # It is computed at loop SETUP (before any training), so this exercises the + # broadcast regardless of whether an eval later fires. + [[ "$anchors" != "1" ]] && fail "total_train_anchors computed $anchors times, expected 1 (rank-0 broadcast regressed)" + # data-fraction eval cadence set up (the path that needs total_train_anchors). + [[ "$dpct_setup" != "True" ]] && fail "data-pct eval cadence not set up (total_train_anchors path not reached)" + # Trigger firing depends on (full-window) anchor count vs the few test steps, + # so it is informational — not required to exercise the broadcast fix. + echo "data-pct eval triggers fired: $dpct_trig (informational)" + + # P2 mw_seed — 1 window → clean WINDOW_COMPLETE checkpoint. + clean_ckpt "$CKPT_ROOT" + run_phase mw_seed \ + "NUM_TRAIN_TS=1" "NUM_TRAIN_BATCHES=$MW_BATCHES" "NUM_EVAL_BATCHES=$MW_EVAL_BATCHES" \ + "EVAL_EVERY_N_WINDOWS=0" "EVAL_EVERY_DATA_PCT=$MW_EVAL_PCT" "METRIC_LOG_FREQ=1" \ + "KEEP_LAST_N=1" "CKPT_PATH=$CKPT_ROOT" "${MW_SPLIT_ENV[@]}" + wait_for_log mw_seed "PHASE_EXIT=0" "$MW_RUN_TIMEOUT"; rc=$? + cleanup_workers + (( rc != 0 )) && { echo "FAIL: mw_seed didn't finish/checkpoint (rc=$rc)"; tail -30 "$LOG_DIR/mw_seed.log"; return 1; } + local seed_ckpt + seed_ckpt=$(sx "ls '$CKPT_ROOT' 2>/dev/null | grep -E '^[0-9]+$' | sort -n | tail -1" | tr -d ' ') + echo "mw_seed end-of-window checkpoint: ${seed_ckpt:-} (expect $START_TS)" + [[ "$seed_ckpt" != "$START_TS" ]] && { fail "mw_seed did not save end-of-window ckpt $START_TS (got '$seed_ckpt')"; return 1; } + + # P3 mw_resume — relaunch over MW_TS windows; resume past the completed + # window and CROSS the boundary into the remaining windows (the exact case + # that used to deadlock). The full split contract matches the seed. + run_phase mw_resume \ + "NUM_TRAIN_TS=$MW_TS" "NUM_TRAIN_BATCHES=$MW_BATCHES" "NUM_EVAL_BATCHES=$MW_EVAL_BATCHES" \ + "EVAL_EVERY_N_WINDOWS=0" "EVAL_EVERY_DATA_PCT=$MW_EVAL_PCT" "METRIC_LOG_FREQ=1" \ + "KEEP_LAST_N=1" "CKPT_PATH=$CKPT_ROOT" "${MW_SPLIT_ENV[@]}" + wait_for_log mw_resume "PHASE_EXIT=" "$MW_RUN_TIMEOUT"; rc=$? + cleanup_workers + (( rc == 1 )) && { echo "FAIL: mw_resume crashed"; tail -30 "$LOG_DIR/mw_resume.log"; return 1; } + (( rc == 2 )) && { echo "FAIL: mw_resume timed out (possible boundary deadlock on resume)"; tail -30 "$LOG_DIR/mw_resume.log"; return 1; } + + sum="$LOG_DIR/mw_resume.summary.json" + python3 "$PYHELPER" summarize "$LOG_DIR/mw_resume.log" "$sum" >/dev/null + echo "--- mw_resume summary ---"; cat "$sum" + local r_exit r_resume_ts r_anchors r_barriers r_dpct + r_exit=$(jget "$sum" phase_exit) + r_resume_ts=$(jget "$sum" resume_completed_ts) + r_anchors=$(jget "$sum" total_train_anchors_calls) + r_barriers=$(jget "$sum" window_barrier_count) + r_dpct=$(jget "$sum" data_pct_eval_setup) + # Resumed from the completed seed window (advanced past the boundary). + [[ "$r_resume_ts" != "$START_TS" ]] && fail "mw_resume did not resume from completed train_ts=$START_TS (got $r_resume_ts)" + # Crossed the boundary into the remaining MW_TS-1 windows and exited 0. + [[ "$r_exit" != "0" ]] && fail "mw_resume did not complete cleanly (phase_exit=$r_exit) — boundary deadlock on resume?" + [[ "$r_barriers" != "$((MW_TS - 1))" ]] && fail "mw_resume barrier fired $r_barriers times, expected $((MW_TS - 1)) (windows after the resumed one)" + # Broadcast still once on the resume path; data-pct cadence rebuilt. + [[ "$r_anchors" != "1" ]] && fail "mw_resume total_train_anchors computed $r_anchors times, expected 1" + [[ "$r_dpct" != "True" ]] && fail "mw_resume data-pct eval cadence not set up" + + (( FAIL == 0 )) && echo "=== multiwindow: PASS ===" || echo "=== multiwindow: FAIL ===" +} + +# ============================================================================= +[[ "$SCENARIO" == "midwindow" || "$SCENARIO" == "all" ]] && run_midwindow +[[ "$SCENARIO" == "multiwindow" || "$SCENARIO" == "all" ]] && run_multiwindow if [[ "$KEEP" != "1" ]]; then rm -rf "$LOG_DIR" - clean_ckpt + clean_ckpt "$CKPT_ROOT" fi -if [[ $RC -eq 0 ]]; then - echo "=== PASS: resume validated (functional invariants + trajectory within $ATOL of baseline) ===" -else - echo "=== FAIL: trajectory diverged beyond $ATOL — likely a real resume bug (wrong data slice / unrestored state), not nondeterminism ===" +if (( FAIL == 0 )); then + echo "=== PASS: all selected scenarios validated ===" + exit 0 fi -exit $RC +echo "=== FAIL: one or more scenarios failed (see above) ===" +exit 1 diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py index 526a6f89d..9c422c161 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -154,7 +154,11 @@ def setup( os.environ["MASTER_PORT"] = str(master_port) BACKEND = dist.Backend.NCCL - TIMEOUT = 1800 + # Process-group / NCCL watchdog timeout (seconds). Env-overridable so a + # diagnostic run can use a short, finite timeout that trips the NCCL flight + # recorder dump (TORCH_NCCL_TRACE_BUFFER_SIZE + TORCH_NCCL_DUMP_ON_TIMEOUT) + # on a collective desync instead of hanging for the full default. + TIMEOUT = int(os.environ.get("PG_TIMEOUT_S", "1800")) # set device BEFORE init_process_group so NCCL binds this rank to its # own GPU; otherwise every rank's first CUDA context lands on GPU 0, @@ -196,6 +200,55 @@ def cleanup() -> None: dist.destroy_process_group() +def _window_boundary_barrier( + device: torch.device, world_size: int, train_ts: int +) -> None: + """Collective rendezvous at a streaming window boundary. + + The per-window data prep (``window_indices``: an O(N) mask over the ~18 GB + mmap'd ``anchor_ts`` array) can complete at very different times across + ranks. The embedding input-dist all-to-all that follows is a collective, so + if a fast rank reaches it while a slow rank is still in prep, the NCCL + stream desyncs and the job deadlocks (one rank a collective behind the + rest). Synchronizing here makes prep-time skew harmless: every rank waits + until all ranks have a ready window before any issues the first forward. + + Cost is one near-zero-payload barrier per window (299 total over a full + run). In the healthy case prep already overlapped the previous window's + compute via the prefetcher, so the barrier returns immediately; it only + blocks for the real prep skew it is there to absorb. + """ + if not (dist.is_available() and dist.is_initialized()) or world_size <= 1: + return + t0 = time.time() + if device.type == "cuda": + dist.barrier(device_ids=[device.index]) + else: + dist.barrier() + waited = time.time() - t0 + # Surface non-trivial skew (the thing this barrier exists to absorb) so a + # node with a slow rank is visible without trawling the flight recorder. + if waited > 5.0: + logger.warning( + "[window-barrier] train_ts=%d: waited %.1fs at boundary " + "rendezvous (per-rank data-prep skew)", + train_ts, + waited, + ) + # Test/debug observability: the healthy-path barrier is otherwise SILENT + # (the skew warning above only fires on >5s waits), so the resume e2e test + # has no signal that the boundary rendezvous actually executed. When + # WINDOW_BARRIER_DEBUG=1, rank 0 emits exactly one line per crossed window + # so the test can assert the barrier ran at EVERY boundary (regression guard + # for the desync deadlock the barrier fixes). Off by default — zero prod cost. + if os.environ.get("WINDOW_BARRIER_DEBUG") == "1" and dist.get_rank() == 0: + logger.info( + "[window-barrier] train_ts=%d rendezvous complete (waited %.3fs)", + train_ts, + waited, + ) + + class HammerToTorchDataset(TorchDataset): def __init__( self, @@ -534,6 +587,14 @@ def make_optimizer_and_shard( # local_world_size = GPUs per node so the planner respects the intra-node # (xGMI/NVLink) vs inter-node hierarchy when placing shards. Defaults to # world_size for the single-node case (no behavior change). + logger.info( + "[hbm-cap] make_optimizer_and_shard: hbm_cap_gb=%s (planner Topology hbm_cap=%d bytes), " + "world_size=%s local_world_size=%s", + hbm_cap_gb, + hbm_cap_gb * 1024 * 1024 * 1024, + world_size, + local_world_size or world_size, + ) planner = EmbeddingShardingPlanner( topology=Topology( local_world_size=local_world_size or world_size, @@ -1728,9 +1789,30 @@ def streaming_train_eval_loop( else int(os.environ.get("BATCH_SIZE", "1024")) ) if hasattr(dataset.dataset, "total_train_anchors"): - total_train_anchors = dataset.dataset.total_train_anchors( # pyre-ignore[16] - eval_anchor_ts, requested_end_ts - eval_anchor_ts - ) + # total_train_anchors does a full-range gather over the mmap'd uid + # array for ~billions of positions + a uid hash. It is slow + # (minutes, single-threaded) AND, run on every rank independently, + # a large per-rank skew source: a fast rank finishes and races into + # the first embedding all-to-all while slow ranks are still hashing, + # desyncing the NCCL collective stream and hanging the job. The + # result is a pure function of the (identical) dataset + split, so + # compute it ONCE on rank 0 and broadcast the scalar; ranks 1..N + # skip the gather entirely (no 8x mmap/CPU contention, no skew). + if world_size > 1 and torch.distributed.is_initialized(): + _tta = ( + dataset.dataset.total_train_anchors( # pyre-ignore[16] + eval_anchor_ts, requested_end_ts - eval_anchor_ts + ) + if rank == 0 + else 0 + ) + _tta_t = torch.tensor([_tta], dtype=torch.int64, device=device) + torch.distributed.broadcast(_tta_t, src=0) + total_train_anchors = int(_tta_t.item()) + else: + total_train_anchors = dataset.dataset.total_train_anchors( # pyre-ignore[16] + eval_anchor_ts, requested_end_ts - eval_anchor_ts + ) total_train_steps = total_train_anchors // max(1, bs * world_size) eval_interval_steps = max( 1, round(eval_every_data_pct * total_train_steps) @@ -2401,6 +2483,18 @@ def _do_eval_db(train_ts: int, gstep: int) -> None: if i == 0 and resume_batch_idx_in_window > 0 else 0 ) + # Rendezvous all ranks at the window boundary BEFORE the first + # forward of this window. The prefetcher has already handed back a + # ready iterator (this window's window_indices mmap scan is done), + # but that O(N) scan over the ~18 GB anchor_ts array can finish at + # very different times across ranks. Without this barrier a fast + # rank issues the first embedding all-to-all while a slow rank is + # still in prep, desyncing the NCCL collective stream and hanging + # the job (observed at a window boundary via the flight recorder: + # ranks split across consecutive collective seq ids). This only + # absorbs prep skew (one near-zero sync per window); it does not + # serialize the background prefetch of future windows. + _window_boundary_barrier(device, world_size, train_ts) if _per_window_blocks: mlt.block_start() _run_train_window( @@ -2489,6 +2583,10 @@ def _do_eval_nb(train_ts: int, gstep: int) -> None: if i == 0 and resume_batch_idx_in_window > 0 else 0 ) + # See the double-buffer path: rendezvous all ranks at the window + # boundary before the first forward so per-rank data-prep skew + # cannot desync the NCCL collective stream and hang the job. + _window_boundary_barrier(device, world_size, train_ts) if _per_window_blocks: mlt.block_start() _run_train_window( diff --git a/recommendation_v4/scripts/launch_slurm.sh b/recommendation_v4/scripts/launch_slurm.sh index af9ff3907..2177f446c 100755 --- a/recommendation_v4/scripts/launch_slurm.sh +++ b/recommendation_v4/scripts/launch_slurm.sh @@ -35,10 +35,10 @@ # what run_streaming_e2e.sh invokes per relaunch) # Perf pair: # LOG=/apps/chcai/perf_1node.log NUM_TRAIN_BATCHES=200 NUM_EVAL_BATCHES=0 \ -# EVAL_EACH_WINDOW=0 METRIC_LOG_FREQ=20 \ +# EVAL_EVERY_N_WINDOWS=0 METRIC_LOG_FREQ=20 \ # sbatch --nodes=1 --job-name=y1 scripts/launch_slurm.sh # LOG=/apps/chcai/perf_2node.log NUM_TRAIN_BATCHES=200 NUM_EVAL_BATCHES=0 \ -# EVAL_EACH_WINDOW=0 METRIC_LOG_FREQ=20 \ +# EVAL_EVERY_N_WINDOWS=0 METRIC_LOG_FREQ=20 \ # sbatch --nodes=2 --job-name=y2 scripts/launch_slurm.sh # # then: bash scripts/compare_node_perf.sh /apps/chcai/perf_1node.log /apps/chcai/perf_2node.log # @@ -285,7 +285,6 @@ orchestrate() { -e MODE=$MODE \ -e START_TS=$START_TS \ -e NUM_TRAIN_TS=$NUM_TRAIN_TS \ - -e EVAL_EACH_WINDOW=$EVAL_EACH_WINDOW \ -e EVAL_EVERY_N_WINDOWS=$EVAL_EVERY_N_WINDOWS \ ${EVAL_EVERY_DATA_PCT:+-e EVAL_EVERY_DATA_PCT=$EVAL_EVERY_DATA_PCT} \ -e NUM_TRAIN_BATCHES=$NUM_TRAIN_BATCHES \ From f8807ca280409f10eadb6978c8db60b3c2975a1f Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Fri, 26 Jun 2026 22:16:43 +0000 Subject: [PATCH 105/127] dlrmv3 streaming: make resume e2e test pass on MI350/NFS Platform-aware per-phase timeouts (meta64 NFS full-model checkpoints take ~9 min each vs B200 node-local NVMe), exposed via new --phase-timeout / --mw-run-timeout overrides. Fix cleanup_workers self-kill: a plain `pkill -f generative_recommenders` matched its own shell and SIGKILLed cleanup mid-run, leaking trainer VRAM so the next phase OOM'd; now uses bracketed patterns and blocks until trainers exit and VRAM drains. Validated PASS end-to-end on MI350 (midwindow + multiwindow). Co-authored-by: Cursor --- .../train/tests/streaming_resume_test.py | 4 +- .../train/tests/streaming_resume_test.sh | 76 ++++++++++++++++--- 2 files changed, 68 insertions(+), 12 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.py index cfe9b2e84..e3f78886c 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.py @@ -15,8 +15,8 @@ """End-to-end failure-injection test for streaming resume. Two scenarios, driven by the sibling `streaming_resume_test.sh` (see its header -for the full B200 launch wiring). This module is the shared log parser + a CLI -the driver shells out to. +for the full, platform-general launch wiring — NVIDIA B200 and AMD MI350/MI355). +This module is the shared log parser + a CLI the driver shells out to. SCENARIO `midwindow` — exact-once mid-window resume. Validates the four single-window resume features end-to-end on the yambda-5b stack: diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh b/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh index afdc65805..8acbf8269 100755 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh @@ -38,8 +38,8 @@ # remaining window, anchors broadcast once, and the run COMPLETED — # i.e. the exact boundary-crossing-on-resume case that used to hang. # -# Driven entirely via env-driven gin knobs (yambda_5b.gin) through the SAME B200 -# worker entrypoint the production supervisor uses: `bash scripts/launch_slurm.sh` +# Driven entirely via env-driven gin knobs (yambda_5b.gin) through the SAME worker +# entrypoint both platforms' production supervisors use: `bash scripts/launch_slurm.sh` # (worker phase, auto-detected inside the container). WINDOW_BARRIER_DEBUG=1 makes # the otherwise-silent barrier emit one rank-0 line per crossed window. # @@ -60,14 +60,22 @@ # [--container ] [--data-path ] [--ckpt-root ] [--start-ts 150] # [--num-train-batches 200] [--die-at-step 100] # midwindow knobs # [--mw-num-train-ts 3] [--mw-num-train-batches 20] # multiwindow knobs -# [--mw-eval-pct 0.34] [--keep] +# [--mw-eval-pct 0.34] [--phase-timeout S] [--mw-run-timeout S] [--keep] # --platform is auto-detected from the running container when omitted. Any of # --container/--data-path/--ckpt-root override the platform default. +# Per-phase wait budgets default per-platform (B200 node-local NVMe: 1800/3600s; +# MI350/MI355 shared-NFS full-model ckpts ~9 min each: 5400/5400s) and can be +# overridden with --phase-timeout (midwindow) / --mw-run-timeout (multiwindow). set -uo pipefail JOBID="" -REPO=/home/chcai/training/recommendation_v4 +# Repo root is derived from THIS script's location +# (/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh — +# four levels up), so the test is not pinned to any one user's home. Override with +# --repo if the repo is mounted at a different path inside the container. +_SELF_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +REPO=$(cd "$_SELF_DIR/../../../.." && pwd) DATASET_SUBDIR=processed_5b/hstu_cache_L4086 SCENARIO=all # midwindow | multiwindow | all START_TS=150 @@ -89,7 +97,12 @@ NUM_EVAL_BATCHES=5 # cap the per-phase FINAL eval (0 = full holdout, very s DIE_AT_STEP=100 IN_WINDOW_FREQ=50 ATOL=0.15 # trajectory closeness bound (NOT bit-equality; see py module) -MW_TIMEOUT=1800 +# Per-phase wait budget. Left empty here and filled per-platform below (a B200 +# ckpt save/load hits node-local NVMe and is fast; on meta64 each full-model DCP +# save/load lands on shared NFS and takes ~9 min, and the resume phase does a +# LOAD + several in-window saves + an end-of-window save, so it needs far longer). +# Override explicitly with --phase-timeout. +MW_TIMEOUT="" # --- multiwindow knobs --- MW_TS=3 # windows to train (>=2 to cross a boundary) @@ -98,7 +111,9 @@ MW_EVAL_BATCHES=5 # holdout eval batches per fired eval MW_EVAL_PCT=0.34 # data-fraction eval cadence (>0 enables the anchors path) MW_SPLIT=0.90 # train split (<1 => holdout exists => uid-hash anchor path) MW_HOLDOUT_TS=200 # PINNED holdout window (must match across seed→resume) -MW_RUN_TIMEOUT=3600 # generous: init + planner + anchors gather can take min +# generous: init + planner + anchors gather can take min; on NFS add ckpt save/load. +# Empty => filled per-platform below. Override with --mw-run-timeout. +MW_RUN_TIMEOUT="" while [[ $# -gt 0 ]]; do case $1 in @@ -117,6 +132,8 @@ while [[ $# -gt 0 ]]; do --die-at-step) DIE_AT_STEP="$2"; shift 2;; --in-window-freq) IN_WINDOW_FREQ="$2"; shift 2;; --atol) ATOL="$2"; shift 2;; + --phase-timeout) MW_TIMEOUT="$2"; shift 2;; + --mw-run-timeout) MW_RUN_TIMEOUT="$2"; shift 2;; --mw-num-train-ts) MW_TS="$2"; shift 2;; --mw-num-train-batches) MW_BATCHES="$2"; shift 2;; --mw-num-eval-batches) MW_EVAL_BATCHES="$2"; shift 2;; @@ -142,7 +159,20 @@ if [[ -z "$PLATFORM" ]]; then _names=$(srun --jobid="$JOBID" --overlap docker ps -a --format '{{.Names}}' 2>/dev/null) if grep -qx yambda_b200 <<<"$_names"; then PLATFORM=b200 elif grep -qx yambda_primus <<<"$_names"; then PLATFORM=mi350 - else PLATFORM=b200; echo "Warning: could not auto-detect platform (no known container on job $JOBID) — defaulting to b200"; fi + else + # No known training container yet (e.g. container not provisioned). + # Fall back to probing the allocation's GPU vendor on the host so we + # do NOT silently assume a platform. + _vendor=$(srun --jobid="$JOBID" --overlap bash -lc \ + 'if command -v rocm-smi >/dev/null 2>&1; then echo amd; \ + elif command -v nvidia-smi >/dev/null 2>&1; then echo nvidia; \ + else echo unknown; fi' 2>/dev/null | head -1) + case "$_vendor" in + amd) PLATFORM=mi350; echo "[$(date)] no known container — detected AMD GPU host (rocm-smi) → mi350";; + nvidia) PLATFORM=b200; echo "[$(date)] no known container — detected NVIDIA GPU host (nvidia-smi) → b200";; + *) echo "Error: could not auto-detect platform on job $JOBID (no yambda_b200/yambda_primus container and no rocm-smi/nvidia-smi). Pass --platform b200|mi350|mi355."; exit 1;; + esac + fi fi echo "[$(date)] auto-detected platform: $PLATFORM" fi @@ -152,6 +182,9 @@ case "$PLATFORM" in # B200: mmap (ckpt LOAD + dataset cache) must NOT touch virtiofs/NFS. [[ "$DATA_PATH" == "__AUTO__" ]] && DATA_PATH=/tmp/yambda_data : "${CKPT_ROOT:=/tmp/yambda_resume_test/ckpts}" + # Node-local NVMe: full-model save/load is fast. + : "${MW_TIMEOUT:=1800}" + : "${MW_RUN_TIMEOUT:=3600}" ;; mi350|mi355) : "${CONTAINER:=yambda_primus}" @@ -159,9 +192,15 @@ case "$PLATFORM" in # (matches the original MI350 test). /apps/chcai/dlrm_data is the gin default. [[ "$DATA_PATH" == "__AUTO__" ]] && DATA_PATH=/apps/chcai/dlrm_data : "${CKPT_ROOT:=/apps/chcai/ckpts_resume_test}" + # Shared NFS: each full-model DCP save/load is ~9 min. The midwindow resume + # phase chains a LOAD + multiple in-window saves + an end-of-window save + # (>2000s observed), so the B200 budgets are far too tight — abandoning a + # still-running trainer leaks GPU VRAM and OOMs the next phase. Be generous. + : "${MW_TIMEOUT:=5400}" + : "${MW_RUN_TIMEOUT:=5400}" ;; esac -echo "[$(date)] platform=$PLATFORM container=$CONTAINER data_path=${DATA_PATH:-} ckpt_root=$CKPT_ROOT" +echo "[$(date)] platform=$PLATFORM container=$CONTAINER data_path=${DATA_PATH:-} ckpt_root=$CKPT_ROOT phase_timeout=${MW_TIMEOUT}s mw_run_timeout=${MW_RUN_TIMEOUT}s" mkdir -p "$LOG_DIR" PYHELPER="$REPO/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.py" @@ -170,9 +209,26 @@ PYHELPER="$REPO/generative_recommenders/dlrm_v3/train/tests/streaming_resume_tes # path is node-local on B200 or shared NFS on MI350) --- sx() { srun --jobid="$JOBID" --overlap docker exec "$CONTAINER" bash -lc "$1" 2>/dev/null; } +# Kill any lingering trainer procs from a prior phase AND block until they are +# really gone, so the freed GPU VRAM is reclaimed before the next phase shards +# its embedding tables (otherwise it OOMs on the leaked memory). +# * Bracketed patterns ([t]rain_ranker, …) are REQUIRED: a plain `pkill -f +# train_ranker` issued inside `bash -lc "...train_ranker..."` matches its OWN +# command line and SIGKILLs this very shell (docker exec returns 137), which +# silently aborted the rest of the old cleanup and leaked trainers/VRAM. +# * After signalling, poll until no trainer remains (bounded), then a short +# settle so the driver finishes reclaiming device memory. cleanup_workers() { - sx "pkill -9 -f train_ranker 2>/dev/null; pkill -9 -f generative_recommenders 2>/dev/null; \ - pkill -9 -f multiprocessing 2>/dev/null; sleep 2; pkill -9 -f spawn_main 2>/dev/null; sleep 3; true" || true + sx ' + for pat in "[t]rain_ranker" "[g]enerative_recommenders" "[s]pawn_main" "[m]ultiprocessing"; do + pkill -9 -f "$pat" 2>/dev/null + done + for _ in $(seq 1 30); do + pgrep -f "[t]rain_ranker" >/dev/null 2>&1 || \ + pgrep -f "[g]enerative_recommenders" >/dev/null 2>&1 || break + sleep 2 + done + sleep 3; true' || true } clean_ckpt() { sx "rm -rf '$1'" || true; } From ea370649de4d424bc85206327b993c1dc5add8b7 Mon Sep 17 00:00:00 2001 From: chris Date: Fri, 26 Jun 2026 23:17:05 +0000 Subject: [PATCH 106/127] dlrmv3 streaming: document midwindow vs multiwindow resume test purpose Add a plain-language header section to streaming_resume_test.sh explaining why there are two scenarios and how they differ: midwindow guards resume CORRECTNESS (land on the right batch/RNG/checkpoint within one window), while multiwindow guards LIVENESS at window seams (all ranks cross a window boundary in lockstep without an NCCL desync deadlock). Includes before/after timelines of the boundary hang vs the rank-0-broadcast + dist.barrier fixes, why a single-window test structurally cannot catch those bugs, and a comparison table. Comments only; no behavior change. Co-authored-by: Cursor --- .../train/tests/streaming_resume_test.sh | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh b/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh index 8acbf8269..e093fb31a 100755 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh @@ -1,6 +1,73 @@ #!/bin/bash # End-to-end failure-injection + resume test for streaming-train-eval. # +# ============================================================================ +# WHY TWO TESTS (the intuition, in plain language) +# ============================================================================ +# Training runs over consecutive time WINDOWS (window 0, then 1, then 2, ...). +# All N GPUs must march from one window to the next IN LOCKSTEP: they constantly +# do "everybody-talk-at-once" group ops (NCCL collectives — sharing embeddings +# across GPUs), and every GPU must enter each group-op at the same time. If one +# GPU is late, the rest wait for it forever and the whole job FREEZES (deadlock). +# +# The two scenarios check DIFFERENT kinds of failure — not bigger vs smaller: +# +# midwindow = CORRECTNESS. "When I crash and resume, do I land on the exact +# right batch with the right RNG state and produce the same +# numbers?" (stays inside ONE window; never crosses a seam.) +# +# multiwindow = LIVENESS / NO-DEADLOCK. "Can all N GPUs hand off across a +# window SEAM together without one falling out of step and +# hanging the job?" (needs >=2 windows so a seam actually exists.) +# +# The dangerous spot is the SEAM between two windows: there, each GPU does solo +# prep work (load next window's data; count anchors for the eval cadence) and +# they DON'T all finish at the same speed. Two bugs lived exactly there, and BOTH +# are invisible to a single-window test: +# (A) every GPU separately ran a slow O(N) "count all the data" pass -> they +# finished at different times -> fast GPU barged into the next group-op +# while others were still counting -> freeze. +# FIX: only rank 0 counts, then broadcasts the number to everyone else. +# (B) no rendezvous at the seam -> uneven data-prep -> same desync -> freeze. +# FIX: a dist.barrier() at every window boundary (all GPUs wait, then cross +# together). WINDOW_BARRIER_DEBUG=1 makes rank 0 log one line per seam. +# +# TIMELINE — without the fixes (each GPU on its own clock at the seam): +# win0 train | solo prep (varies) | next group-op +# GPU0 ########|=====| >> waiting.......... +# GPU1 ########|========| >> waiting....... +# GPU2 ########|===========| >> waiting.... +# GPU3 ########|==============| >> never lines up -> HANG +# +# TIMELINE — with the fixes (rank 0 counts + a barrier gate at the seam): +# win0 train | [== BARRIER: all wait ==] | win1 train +# GPU0 ########| count | wait # |######## +# GPU1 ########| | wait # |######## +# GPU2 ########| | wait # |######## +# GPU3 ########| | wait # |######## +# rank0 shares count ^ all cross together ^ -> OK +# +# Why midwindow can NOT catch (A)/(B): it runs a SINGLE window with per-window +# eval off (NUM_TRAIN_TS=1, EVAL_EVERY_N_WINDOWS=0, split=1.0), so it never +# reaches a seam and never turns on the data-fraction-eval/anchor-count path. +# A broken barrier or broken broadcast passes midwindow silently. +# +# Why the multiwindow RESUME phase (P3 below) is the meanest case: restarting +# from a checkpoint loads the saved window and then IMMEDIATELY steps across a +# seam into the next window — landing right on the spot that used to freeze, AND +# re-running all that slow setup on the resume path. If (A)/(B) regressed, P3 +# hangs and the test fails by timing out. +# +# | midwindow | multiwindow +# --------------+----------------------+----------------------------- +# proves | resume to RIGHT spot | cross seam WITHOUT freezing +# windows | 1 (no seam) | >=2 (crosses >=1 seam) +# data-pct eval | off | on (exercises the anchor count) +# catches | wrong batch/RNG/ckpt | missing barrier/broadcast -> HANG +# failure mode | wrong NUMBERS | job FREEZES forever +# They are complementary: you need BOTH. +# ============================================================================ +# # PLATFORM-GENERAL: runs on both NVIDIA B200 and AMD MI350/MI355 (ROCm/meta64). # The only hardware-specific bits are picked by --platform (auto-detected from the # running container if omitted): the container name, the dataset path, and the From 8db66f3dfc7ca2a1de925f34315890d0677a4305 Mon Sep 17 00:00:00 2001 From: chris Date: Sat, 27 Jun 2026 00:54:12 +0000 Subject: [PATCH 107/127] dlrmv3: gin/env-configurable embedding table placement (HBM/UVM) Add per-table and global control over embedding table placement via gin, overridable by env var. make_optimizer_and_shard now translates an EMB_PLACEMENT global default plus per-table EMB_PLACEMENT_OVERRIDES (hbm|uvm|uvm_caching|auto) into torchrec ParameterConstraints fed to the EmbeddingShardingPlanner; "auto" leaves the table to the planner so the default is byte-identical to the prior behavior (constraints=None). A new env_str_map gin helper parses "name=val,name=val" with opt-in per-key merge so a launch-time env tweak layers over the gin default. Also logs the planner's ACTUAL per-table compute kernel ([emb-placement] plan: ...). Validated end-to-end on 8x B200: force-HBM put every table on fused, and a per-table override put uid on fused_uvm_caching while the rest stayed fused; both trained cleanly. Co-authored-by: Cursor --- .../dlrm_v3/train/gin/yambda_5b.gin | 21 +++ .../dlrm_v3/train/train_ranker.py | 1 + .../dlrm_v3/train/utils.py | 131 +++++++++++++++++- .../generative_recommenders/dlrm_v3/utils.py | 39 ++++++ 4 files changed, 191 insertions(+), 1 deletion(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index 14bc9d106..23cd7886b 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -158,6 +158,27 @@ make_optimizer_and_shard.hbm_cap_gb = @env_int() env_int.key = "HBM_CAP_GB" env_int.default = 260 +# Embedding table placement. Global default applied to EVERY table: +# auto -> no constraint; planner decides from the HBM cap (default) +# hbm -> FUSED (resident in GPU HBM) +# uvm -> FUSED_UVM (host DDR via UVM, no HBM cache) +# uvm_caching -> FUSED_UVM_CACHING (host DDR + HBM cache) +# "force HBM" for all tables = set this to "hbm" (or export EMB_PLACEMENT=hbm). +make_optimizer_and_shard.embedding_placement = @emp/env_str() +emp/env_str.key = "EMB_PLACEMENT" +emp/env_str.default = "auto" + +# Per-table placement overrides (win over the global default above). Configure +# each table independently here as a gin dict; allowed values per table: +# "hbm" | "uvm" | "uvm_caching" | "auto" (unlisted tables use the global +# default above). merge=True means a per-run env override LAYERS ON TOP of this +# dict per key (tweak one table at launch, keep the rest), e.g.: +# EMB_PLACEMENT_OVERRIDES="uid=hbm" # only retargets uid; others stay as below +make_optimizer_and_shard.embedding_placement_overrides = @env_str_map() +env_str_map.key = "EMB_PLACEMENT_OVERRIDES" +env_str_map.merge = True +env_str_map.default = {} + # Sparse embedding all-to-all wire precision. The embedding shuffle is the # dominant, bandwidth-bound (esp. multi-node) collective; quantizing it via # TorchRec QCommsConfig halves (bf16/fp16, both 2 bytes) the wire volume. diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py index d5797697b..a57153f60 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py @@ -150,6 +150,7 @@ def _main_func( device=device, world_size=world_size, local_world_size=gpus_per_node, + embedding_table_configs=embedding_table_configs, ) # Decorrelate forward-time stochasticity (HSTU dropout) per data-parallel # rank. MUST run after make_model() + make_optimizer_and_shard() so the diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py index 9c422c161..8d52a055b 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -55,8 +55,10 @@ from torch.optim.optimizer import Optimizer from torch.utils.data import DataLoader, Dataset as TorchDataset from torch.utils.data.distributed import _T_co, DistributedSampler +from torchrec.distributed.embedding_types import EmbeddingComputeKernel from torchrec.distributed.model_parallel import DistributedModelParallel from torchrec.distributed.planner import EmbeddingShardingPlanner, Topology +from torchrec.distributed.planner.types import ParameterConstraints from torchrec.distributed.sharding_plan import get_default_sharders from torchrec.distributed.types import ShardedTensor, ShardingEnv from torchrec.modules.embedding_configs import EmbeddingConfig @@ -75,6 +77,16 @@ EmbeddingCollection, } +# Embedding placement vocabulary -> torchrec compute kernel. Used by +# make_optimizer_and_shard to translate the gin/env placement strings +# ("hbm"/"uvm"/"uvm_caching") into ParameterConstraints. "auto" (or anything not +# in this map) means "no constraint": the planner decides from the HBM cap. +_PLACEMENT_TO_KERNEL: Dict[str, EmbeddingComputeKernel] = { + "hbm": EmbeddingComputeKernel.FUSED, + "uvm": EmbeddingComputeKernel.FUSED_UVM, + "uvm_caching": EmbeddingComputeKernel.FUSED_UVM_CACHING, +} + @gin.configurable def seed_everything(seed: int = -1, rank: int = 0) -> None: @@ -552,6 +564,89 @@ def _maybe_apply_qcomm_a2a( return sharders +def _embedding_table_names( + model: torch.nn.Module, + embedding_table_configs: Optional[Dict[str, EmbeddingConfig]], +) -> List[str]: + """All embedding table names the planner will place. + + Prefers the authoritative `embedding_table_configs` (keys == table names == + planner parameter names). Falls back to walking the model's EBC/EC modules + when the configs are not passed in. + """ + if embedding_table_configs: + return list(embedding_table_configs.keys()) + names: List[str] = [] + for _, module in model.named_modules(): + if type(module) in TORCHREC_TYPES: + if isinstance(module, EmbeddingBagCollection): + names.extend(c.name for c in module.embedding_bag_configs()) + elif isinstance(module, EmbeddingCollection): + names.extend(c.name for c in module.embedding_configs()) + return names + + +def _build_placement_constraints( + model: torch.nn.Module, + embedding_placement: str, + embedding_placement_overrides: Dict[str, str], + embedding_table_configs: Optional[Dict[str, EmbeddingConfig]], +) -> Dict[str, ParameterConstraints]: + """Translate gin/env placement strings into torchrec ParameterConstraints. + + Resolution per table: ``overrides.get(name, embedding_placement)``. A value + of ``auto`` (or empty) means "no constraint" and the table is omitted so the + planner keeps deciding from the HBM cap. Unknown values raise ValueError. + """ + valid = set(_PLACEMENT_TO_KERNEL) | {"auto", ""} + for where, val in [ + ("embedding_placement", embedding_placement), + *[ + (f"embedding_placement_overrides[{k}]", v) + for k, v in embedding_placement_overrides.items() + ], + ]: + if val not in valid: + raise ValueError( + f"Invalid embedding placement {val!r} for {where}; " + f"expected one of {sorted(valid - {''})}." + ) + + names = _embedding_table_names(model, embedding_table_configs) + unknown = set(embedding_placement_overrides) - set(names) + if unknown: + logger.warning( + "[emb-placement] override(s) for unknown table(s) %s ignored; " + "known tables: %s", + sorted(unknown), + sorted(names), + ) + + constraints: Dict[str, ParameterConstraints] = {} + resolved: Dict[str, str] = {} + for name in names: + placement = embedding_placement_overrides.get(name, embedding_placement) + resolved[name] = placement or "auto" + kernel = _PLACEMENT_TO_KERNEL.get(placement) + if kernel is not None: + constraints[name] = ParameterConstraints( + compute_kernels=[kernel.value] + ) + + rank = dist.get_rank() if dist.is_initialized() else 0 + if rank == 0: + logger.info( + "[emb-placement] global=%r overrides=%s -> resolved=%s " + "(constrained=%d/%d tables; the rest are planner-auto)", + embedding_placement, + embedding_placement_overrides or {}, + resolved, + len(constraints), + len(names), + ) + return constraints + + @gin.configurable def make_optimizer_and_shard( model: torch.nn.Module, @@ -561,6 +656,9 @@ def make_optimizer_and_shard( hbm_cap_gb: int = 260, sparse_a2a_forward_precision: str = "fp32", sparse_a2a_backward_precision: str = "fp32", + embedding_placement: str = "auto", + embedding_placement_overrides: Optional[Dict[str, str]] = None, + embedding_table_configs: Optional[Dict[str, EmbeddingConfig]] = None, ) -> Tuple[DistributedModelParallel, torch.optim.Optimizer]: dense_opt_cls, dense_opt_args, dense_opt_factory = ( dense_optimizer_factory_and_class() @@ -595,6 +693,17 @@ def make_optimizer_and_shard( world_size, local_world_size or world_size, ) + # Resolve per-table embedding placement (gin/env-driven). Global default + # `embedding_placement` applies to every table; `embedding_placement_overrides` + # (table name -> placement) wins per table. Tables resolving to "auto" carry + # no constraint (planner decides from hbm_cap). When nothing is constrained we + # pass constraints=None so the plan is byte-identical to the legacy path. + constraints = _build_placement_constraints( + model=model, + embedding_placement=embedding_placement, + embedding_placement_overrides=embedding_placement_overrides or {}, + embedding_table_configs=embedding_table_configs, + ) planner = EmbeddingShardingPlanner( topology=Topology( local_world_size=local_world_size or world_size, @@ -602,7 +711,8 @@ def make_optimizer_and_shard( compute_device="cuda", hbm_cap=hbm_cap_gb * 1024 * 1024 * 1024, ddr_cap=0, - ) + ), + constraints=constraints or None, ) pg = dist.GroupMember.WORLD env = ShardingEnv.from_process_group(pg) # pyre-ignore [6] @@ -610,6 +720,25 @@ def make_optimizer_and_shard( plan = planner.collective_plan(model, sharders, pg) + # Authoritative placement log: report the compute kernel the planner ACTUALLY + # assigned to each table (vs the [emb-placement] line above, which reports what + # was requested). "fused" = HBM; "fused_uvm"/"fused_uvm_caching" = UVM-backed. + # Rank 0 only, best-effort (never break the build over a logging shape change). + if (dist.get_rank() if dist.is_initialized() else 0) == 0: + try: + for module_path, param_plans in plan.plan.items(): + for param_name, ps in param_plans.items(): + logger.info( + "[emb-placement] plan: %s.%s -> compute_kernel=%s " + "sharding_type=%s", + module_path, + param_name, + getattr(ps, "compute_kernel", "?"), + getattr(ps, "sharding_type", "?"), + ) + except Exception as e: # logging only; must never fail the build + logger.warning("[emb-placement] could not dump plan kernels: %s", e) + # Re-seed right before DMP materializes/inits the sharded embedding tables. # The per-table seeded init_fn (configs.get_embedding_table_config) handles # the eager path, but the fused FBGEMM TBE path inits weights on-device and diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py index ed456bde6..1e5d79993 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py @@ -1500,6 +1500,45 @@ def env_str(key: str = "", default: str = "") -> str: return raw if raw else default +@gin.configurable +def env_str_map( + key: str = "", + default: Optional[Dict[str, str]] = None, + merge: bool = False, +) -> Dict[str, str]: + """Parse os.environ[key] as 'name=value,name=value' into a dict. + + Falls back to `default` (gin) when the env var is unset/empty. Companion to + `env_str` for map-valued overrides (e.g. per-table embedding placement). + Example gin usage: + + make_optimizer_and_shard.embedding_placement_overrides = @env_str_map() + env_str_map.key = "EMB_PLACEMENT_OVERRIDES" + env_str_map.default = {} + + Example env override: EMB_PLACEMENT_OVERRIDES="uid=uvm_caching,item_id=hbm". + + `merge` controls how the parsed env entries combine with `default`: + * False (default): the env var REPLACES `default` wholesale (whole-dict + override) — set the env var and you fully define the map. + * True: the parsed env entries are LAYERED ON TOP of `default` (per-key + override) — gin `default` defines the base map and the env var tweaks + only the named keys, leaving the rest of `default` intact. + """ + base = dict(default or {}) + raw = os.environ.get(key) if key else None + if not raw: + return base + parsed: Dict[str, str] = {} + for pair in raw.split(","): + pair = pair.strip() + if not pair: + continue + k, _, v = pair.partition("=") + parsed[k.strip()] = v.strip() + return {**base, **parsed} if merge else parsed + + @gin.configurable def env_int(key: str = "", default: int = 0) -> int: """Resolve an int from os.environ[key], falling back to `default`. From d0ce11e2fa1919482b7c91a4961eb32788767085 Mon Sep 17 00:00:00 2001 From: Chris Cai Date: Mon, 29 Jun 2026 06:40:15 +0000 Subject: [PATCH 108/127] dlrmv3 qcomm: hard-fail instead of silently falling back to fp32 a2a MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When embedding all-to-all quantization is configured (SPARSE_A2A_FWD/BWD) but cannot actually be enabled, _maybe_apply_qcomm_a2a previously logged a warning and returned the unquantized sharders, silently running fp32. That hides real misconfiguration and, worse, could leave some ranks on fp32 while others run fp16 — desyncing the embedding collectives. Now every "configured but not enabled" path raises (on all ranks, so the job aborts consistently): - unknown precision string -> ValueError - codec registry build failure -> RuntimeError (chained from cause) - codec built but no EmbeddingCollectionSharder to bind it to -> RuntimeError The legitimate no-quant default (forward=backward=fp32) still returns the sharders untouched. --- .../dlrm_v3/train/utils.py | 78 +++++++++++-------- 1 file changed, 44 insertions(+), 34 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py index 8d52a055b..19451078a 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -512,14 +512,12 @@ def _maybe_apply_qcomm_a2a( rank0 = (not dist.is_initialized()) or dist.get_rank() == 0 for name, p in (("forward", fwd), ("backward", bwd)): if p not in _COMM: - if rank0: - logger.warning( - "DLRMV4 qcomm a2a: unknown %s precision %r (want " - "fp32|bf16|fp16); using fp32 a2a", - name, - p, - ) - return sharders + # Misconfigured precision: fail loudly rather than silently running + # fp32. A typo in SPARSE_A2A_{FWD,BWD} must not pass as "no quant". + raise ValueError( + f"DLRMV4 qcomm a2a: unknown {name} precision {p!r} " + f"(want one of fp32|bf16|fp16)" + ) if fwd == "fp32" and bwd == "fp32": return sharders try: @@ -535,33 +533,45 @@ def _maybe_apply_qcomm_a2a( backward_precision=getattr(CommType, _COMM[bwd]), ) registry = get_qcomm_codecs_registry(qcfg, device=device) - new_sharders = [] - replaced = False - for s in sharders: - if type(s).__name__ == "EmbeddingCollectionSharder" and not replaced: - new_sharders.append( - EmbeddingCollectionSharder(qcomm_codecs_registry=registry) - ) - replaced = True - else: - new_sharders.append(s) - if rank0: - logger.info( - "DLRMV4 qcomm a2a ENABLED: forward=%s backward=%s " - "replaced_ec_sharder=%s", - fwd, - bwd, - replaced, + except Exception as e: # noqa: BLE001 + # A configured quantized a2a that fails to build is a hard error. Silently + # downgrading to fp32 would change numerics/throughput with no signal, and + # a partial failure (one rank fp32, others fp16) would also desync the + # collectives. Raise on every rank so the whole job aborts consistently. + raise RuntimeError( + f"DLRMV4 qcomm a2a: failed to enable configured quantization " + f"(forward={fwd} backward={bwd}): {type(e).__name__}: {e}" + ) from e + + new_sharders = [] + replaced = False + for s in sharders: + if type(s).__name__ == "EmbeddingCollectionSharder" and not replaced: + new_sharders.append( + EmbeddingCollectionSharder(qcomm_codecs_registry=registry) ) - return new_sharders - except Exception as e: # noqa: BLE001 — fall back to fp32 a2a on any failure - if rank0: - logger.warning( - "DLRMV4 qcomm a2a: failed to enable (%s: %s); using fp32 a2a", - type(e).__name__, - e, - ) - return sharders + replaced = True + else: + new_sharders.append(s) + if not replaced: + # Codec registry built fine, but there was no EmbeddingCollectionSharder to + # bind it to, so the quantized a2a would be silently inert. Treat this as a + # hard failure too — "configured but not applied" is the bug we want caught. + raise RuntimeError( + f"DLRMV4 qcomm a2a: quantization configured (forward={fwd} " + f"backward={bwd}) but no EmbeddingCollectionSharder was found to attach " + f"the qcomm codec registry to; refusing to run with quantization " + f"silently disabled" + ) + if rank0: + logger.info( + "DLRMV4 qcomm a2a ENABLED: forward=%s backward=%s " + "replaced_ec_sharder=%s", + fwd, + bwd, + replaced, + ) + return new_sharders def _embedding_table_names( From 584cb66f3902a9a2c46bc647d4226c02674efa20 Mon Sep 17 00:00:00 2001 From: Chris Cai Date: Mon, 29 Jun 2026 18:57:16 +0000 Subject: [PATCH 109/127] dlrmv3 yambda-5b: default embedding a2a quantization to fp16/fp16 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flip SPARSE_A2A_FWD/BWD gin defaults from fp32 to fp16. An A/B run vs the fp32 interleaved baseline matched window AUC to within fixed-seed noise (mean Δ≈-1e-6, max|Δ|≈5e-5, Pearson r=1.0 over the 0-54.5% data overlap) while halving the embedding all-to-all wire volume (~6% end-to-end speedup). Grads on yambda-5b stay well inside fp16 range (grad-clip=1.0, lr=1e-7), so fp16 backward is convergence-neutral here; set both to "fp32" to restore the unquantized path. --- .../dlrm_v3/train/gin/yambda_5b.gin | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index 23cd7886b..008fe6d0b 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -183,17 +183,20 @@ env_str_map.default = {} # dominant, bandwidth-bound (esp. multi-node) collective; quantizing it via # TorchRec QCommsConfig halves (bf16/fp16, both 2 bytes) the wire volume. # Forward and backward are set independently (each: "fp32" | "bf16" | "fp16"). -# Both "fp32" = off (default; numerically identical to baseline trunk). -# Per TorchRec golden_training, fwd=fp16 / bwd=bf16 is the recommended quantized -# mix: fp16's mantissa suits bounded forward activations, while bf16's wider -# exponent range avoids overflow on gradients. -# Override via $SPARSE_A2A_FWD / $SPARSE_A2A_BWD. +# Both "fp32" = off (numerically identical to baseline trunk). +# Default is now fp16/fp16: an A/B run vs the fp32 baseline matched window AUC to +# within fixed-seed noise (mean Δ≈-1e-6, max|Δ|≈5e-5, r=1.0 over 0-54.5% data) +# while cutting the embedding-a2a wire volume in half (~6% end-to-end speedup). +# TorchRec golden_training suggests fwd=fp16 / bwd=bf16 (bf16's wider exponent +# hedges gradient overflow), but on yambda-5b the grads stay well in fp16 range +# (grad-clip=1.0, lr=1e-7), so fp16/fp16 is convergence-neutral here. +# Override via $SPARSE_A2A_FWD / $SPARSE_A2A_BWD (e.g. set both "fp32" to disable). make_optimizer_and_shard.sparse_a2a_forward_precision = @saaf/env_str() saaf/env_str.key = "SPARSE_A2A_FWD" -saaf/env_str.default = "fp32" +saaf/env_str.default = "fp16" make_optimizer_and_shard.sparse_a2a_backward_precision = @saab/env_str() saab/env_str.key = "SPARSE_A2A_BWD" -saab/env_str.default = "fp32" +saab/env_str.default = "fp16" get_dataset.name = %dataset get_dataset.new_path_prefix = %DATA_PATH From 3a695608d471cb2a37a632bbb582944301449985 Mon Sep 17 00:00:00 2001 From: Chris Cai Date: Mon, 29 Jun 2026 19:04:18 +0000 Subject: [PATCH 110/127] dlrmv3 yambda-5b: default gin to the canonical fp16 full-corpus run Align the gin defaults so a no-override streaming-train-eval launch reproduces the validated fp16 run: - START_TS 150 -> 0 and NUM_TRAIN_TS 149 -> 299 (sweep the full ts=0..298 corpus instead of the dense ts=150..298 sub-range) - EVAL_HOLDOUT_TS -1 -> 299 (the window just past training; equivalent to the prior runtime-resolved value under the new sweep, now explicit) - CKPT_TIME_INTERVAL_S 0.0 -> 3600 (hourly saves) Comments updated; each remains env-overridable to restore the old behavior. --- .../dlrm_v3/train/gin/yambda_5b.gin | 31 ++++++++++--------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index 008fe6d0b..2834f52ee 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -335,15 +335,17 @@ make_persistent_streaming_dataloader.num_workers = %num_workers make_persistent_streaming_dataloader.prefetch_factor = %prefetch_factor streaming_train_eval_loop.num_train_ts = @nts/env_int() nts/env_int.key = "NUM_TRAIN_TS" -# 149 daily windows -> with start_ts=150 the run sweeps ts=150..298, the full -# dense range of the corpus (matches the long e2e runs). Clamped to the -# dataset's available window count at runtime. Override via $NUM_TRAIN_TS. -nts/env_int.default = 149 +# 299 daily windows -> with start_ts=0 the run sweeps ts=0..298, the full corpus +# (matches the long fp16 e2e run). Clamped to the dataset's available window +# count at runtime. Override via $NUM_TRAIN_TS. +nts/env_int.default = 299 # Anchors need >= history_length prior events, so the first ~130 daily windows -# are near-empty warm-up; start at a dense window. Override via $START_TS. +# are near-empty warm-up. Default start_ts=0 trains the full corpus from the +# start (matches the canonical fp16 run); set $START_TS=150 to skip warm-up and +# begin at a dense window. Override via $START_TS. streaming_train_eval_loop.start_ts = @sts/env_int() sts/env_int.key = "START_TS" -sts/env_int.default = 150 +sts/env_int.default = 0 # Per-step metric logging cadence. Default 50 (one compute_and_log GPU->CPU # sync per 50 batches). The streaming-resume test sets METRIC_LOG_FREQ=1 so # every step emits a parseable "Step N metrics" line for trajectory comparison. @@ -434,12 +436,13 @@ streaming_train_eval_loop.double_buffer = @db/env_int() db/env_int.key = "DOUBLE_BUFFER" db/env_int.default = 1 # Fixed eval-holdout window range (held-out users' anchors over these windows -# form the eval set evaluated at EVERY eval step). EVAL_HOLDOUT_TS<0 (default) -# resolves at runtime to start_ts+num_train_ts (the window just past training), -# which is stable across resume. EVAL_HOLDOUT_NUM_WINDOWS widens the eval span. +# form the eval set evaluated at EVERY eval step). Default 299 = the window just +# past the ts=0..298 training sweep (matches the canonical fp16 run). Set +# EVAL_HOLDOUT_TS<0 to instead resolve at runtime to start_ts+num_train_ts (also +# stable across resume). EVAL_HOLDOUT_NUM_WINDOWS widens the eval span. streaming_train_eval_loop.eval_holdout_ts = @eht/env_int() eht/env_int.key = "EVAL_HOLDOUT_TS" -eht/env_int.default = -1 +eht/env_int.default = 299 streaming_train_eval_loop.eval_holdout_num_windows = @ehnw/env_int() ehnw/env_int.key = "EVAL_HOLDOUT_NUM_WINDOWS" ehnw/env_int.default = 1 @@ -545,12 +548,12 @@ streaming_train_eval_loop.checkpoint_step_frequency = @csf/env_int() csf/env_int.key = "CKPT_STEP_FREQ" csf/env_int.default = 0 # Wall-clock checkpoint cadence in seconds: save when >= this many seconds have -# elapsed since the last save (e.g. 3600 for hourly). Rank 0 owns the clock and -# broadcasts the decision so all ranks save together. 0.0 (default) = off. -# Override via $CKPT_TIME_INTERVAL_S. +# elapsed since the last save. Rank 0 owns the clock and broadcasts the decision +# so all ranks save together. Default 3600 = hourly saves (matches the canonical +# fp16 run); set 0.0 to disable. Override via $CKPT_TIME_INTERVAL_S. streaming_train_eval_loop.checkpoint_time_interval_s = @ctis/env_float() ctis/env_float.key = "CKPT_TIME_INTERVAL_S" -ctis/env_float.default = 0.0 +ctis/env_float.default = 3600.0 # Cap each train_ts window's batch count (mostly for the resume test driver). # Unset / 0 = use the full window. streaming_train_eval_loop.num_train_batches = @ntb/env_int() From 4f987c1f6c6f83cf69da3ad301a858c0635208fc Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Tue, 30 Jun 2026 21:05:53 +0000 Subject: [PATCH 111/127] dlrmv3 qcomm: low-memory fbgemm fp16/bf16 a2a codec to fix skewed-batch OOM hang The embedding all-to-all is row-wise sharded, so a data-skewed batch routes a few extremely hot IDs to a single owner rank, ballooning its a2a input tensor. fbgemm's quant codec packs the payload via `torch.clamp(t, MIN, MAX).half()`, where clamp() allocates a full-size fp32 temp before the cast (~2.5x input peak). On the hottest shard that temp reached ~81.5 GiB and OOM'd the rank, which then dropped out of the collective while peers blocked in the a2a -> a deterministic ~30-min NCCL-watchdog hang (yambda-5b 4-node fp16, window 235 / global step 43621, every run). Fix: monkeypatch the codec to cast first then clamp in place (`t.half().clamp_(MIN, MAX)`), dropping the full-size fp32 temp. Bit-for-bit identical output (values above HALF_MAX cast to +inf which clamp_ maps back to HALF_MAX; NaNs unchanged) and no throughput regression (strictly less memory traffic). Validated: the patched run trains through step 43621 with 0 OOM / 0 watchdog timeouts and an unperturbed window-AUC trajectory. Gin-configurable via make_optimizer_and_shard.qcomm_lowmem_clamp_cast ($QCOMM_LOWMEM_CODEC), ON by default, under a new RUNTIME PATCHES section in yambda_5b.gin documenting the rationale; a no-op when the a2a is unquantized. launch_slurm.sh: forward PG_TIMEOUT_S + TORCH_NCCL_* flight-recorder env into the container (the instrumentation used to root-cause this hang). Co-authored-by: Cursor --- .../dlrm_v3/train/gin/yambda_5b.gin | 58 ++++++++++++ .../dlrm_v3/train/utils.py | 90 +++++++++++++++++++ recommendation_v4/scripts/launch_slurm.sh | 5 ++ 3 files changed, 153 insertions(+) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index 2834f52ee..2eb92b30f 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -198,6 +198,64 @@ make_optimizer_and_shard.sparse_a2a_backward_precision = @saab/env_str() saab/env_str.key = "SPARSE_A2A_BWD" saab/env_str.default = "fp16" +# ============================================================================= +# RUNTIME PATCHES / MONKEYPATCHES +# ----------------------------------------------------------------------------- +# Third-party kernels we override AT RUNTIME (no fork of the dependency). Each +# knob here is a kill switch for one such patch: ON by default (it fixes a real +# bug we hit), but flippable per-run so we can A/B the patch, reproduce the +# original failure, or fall back to stock if a future dependency version makes +# the patch unnecessary or incorrect. Patches are applied during model +# build/shard (make_optimizer_and_shard), before any training step. +# ============================================================================= +# +# --- qcomm_lowmem_clamp_cast: low-memory fbgemm fp16/bf16 quant codec --------- +# WHAT IT PATCHES +# fbgemm_gpu's embedding-a2a quantizer, fp32_to_fp16_with_clamp (and the bf16 +# variant), which the TorchRec qcomm codec calls to pack the sparse +# embedding all-to-all payload onto the wire. Stock implementation is: +# torch.clamp(tensor, HALF_MIN, HALF_MAX).half() +# torch.clamp() materializes a SECOND full-size fp32 tensor (same numel as the +# input) BEFORE the cast, so the transient peak is +# input(fp32) + clamp_temp(fp32) + output(fp16) ~= 2.5x the input. +# The patch reorders to an in-place, allocation-free-equivalent: +# tensor.half().clamp_(HALF_MIN, HALF_MAX) +# i.e. cast FIRST (only the fp16 output is allocated), then clamp IN PLACE — +# dropping the full-size fp32 clamp temp and cutting the transient peak by the +# size of the input tensor. +# +# WHY IT'S NEEDED (the bug it fixes) +# Embeddings are ROW-WISE sharded: every lookup of row r routes to the single +# rank that owns r. On a data-skewed batch a few extremely hot IDs send a +# disproportionate share of the global lookups to ONE owner rank, so that +# rank's a2a input tensor balloons. With the stock codec the extra fp32 clamp +# temp on that rank reached ~81.5 GiB, which OOM'd the rank mid-forward. The +# OOM'd rank then dropped out of the collective while its peers blocked forever +# in the embedding all-to-all -> ~30-min NCCL watchdog timeout -> the whole job +# SIGABRTs. This was a DETERMINISTIC hang (same window/step every run; e.g. +# yambda-5b 4-node fp16 a2a hit it at window 235 / global step 43621). +# See generative_recommenders/dlrm_v3/train/utils.py +# (_patch_fbgemm_lowmem_clamp_cast) for the full diagnosis + flight-recorder +# evidence. +# +# CORRECTNESS / PERF +# Numerically IDENTICAL (bit-for-bit) to stock: the single fp32->fp16 cast is +# the only rounding step in both orders; an fp32 value above HALF_MAX casts to +# +inf which clamp_ maps back to HALF_MAX, and NaNs pass through unchanged. +# In-place is safe because the codec encode() runs inside the qcomm autograd +# Function.forward with grad disabled (no graph to corrupt). No throughput +# regression (measured step time within run-to-run noise; it does strictly +# LESS memory traffic + allocation than stock). +# +# WHEN TO TURN OFF (set 0): only to reproduce the original OOM/hang for +# debugging, or to revalidate against stock after an fbgemm_gpu upgrade. +# Override via $QCOMM_LOWMEM_CODEC. Only takes effect when the embedding a2a is +# actually quantized (SPARSE_A2A_FWD/BWD not both fp32); with fp32 a2a there is +# no codec to patch and this knob is a no-op. +make_optimizer_and_shard.qcomm_lowmem_clamp_cast = @qlcc/env_int() +qlcc/env_int.key = "QCOMM_LOWMEM_CODEC" +qlcc/env_int.default = 1 + get_dataset.name = %dataset get_dataset.new_path_prefix = %DATA_PATH # Total user-interaction-history (UIH) budget per sample, distributed evenly diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py index 19451078a..dae3d92e7 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -480,11 +480,94 @@ def sparse_optimizer_factory_and_class( return optimizer_cls, kwargs, optimizer_factory +_FBGEMM_LOWMEM_PATCHED = False + + +def _patch_fbgemm_lowmem_clamp_cast(enabled: bool = True, rank0: bool = False) -> None: + """Replace fbgemm's quant clamp+cast with a memory-frugal equivalent. + + ``enabled`` is the gin/env-driven kill switch (see + ``make_optimizer_and_shard.qcomm_lowmem_clamp_cast`` / + ``$QCOMM_LOWMEM_CODEC``). Default ON; pass ``enabled=False`` to fall back to + stock fbgemm (e.g. to reproduce the pre-patch OOM, or if a future fbgemm + version changes the codec and the patch needs revalidation). + + fbgemm's ``fp32_to_fp16_with_clamp`` (and the bf16 variant) does + ``torch.clamp(tensor, MIN, MAX).half()``. ``torch.clamp(...)`` allocates a + SECOND full-size fp32 tensor (same numel as the input) *before* the cast, so + the transient peak is input(fp32) + clamp_temp(fp32) + output(fp16) ~= 2.5x + the input. On a skewed row-wise-sharded batch the hottest shard's embedding + tensor is huge (observed 81.5 GiB clamp temp), and that extra fp32 copy is + exactly the allocation that OOMs the rank — which then exits the train loop + while peers block forever in the a2a (a 30-min NCCL-watchdog hang). See + HANG_ROOTCAUSE.md / flight-recorder dump for the full diagnosis. + + Cast FIRST then clamp IN PLACE: ``tensor.half().clamp_(MIN, MAX)``. This + allocates only the fp16 output (no full-size fp32 temp), cutting the peak by + the size of the input tensor, while being numerically identical: an fp32 + value above HALF_MAX casts to +inf, which clamp_ maps back to HALF_MAX (and + NaNs pass through unchanged), matching clamp-then-cast bit for bit. Safe to + do in place because the codec ``encode()`` runs inside the qcomm autograd + ``Function.forward`` (grad disabled), so there is no graph to corrupt. + """ + global _FBGEMM_LOWMEM_PATCHED + if not enabled: + if rank0: + logger.warning( + "[qcomm-lowmem] DISABLED (qcomm_lowmem_clamp_cast=False / " + "QCOMM_LOWMEM_CODEC=0) — running stock fbgemm clamp+cast, which " + "allocates a full-size fp32 clamp temp and can OOM->hang the " + "hottest row-wise embedding shard on skewed batches." + ) + return + if _FBGEMM_LOWMEM_PATCHED: + return + try: + from fbgemm_gpu import quantize_comm, quantize_utils + + _HMIN = quantize_utils.TORCH_HALF_MIN + _HMAX = quantize_utils.TORCH_HALF_MAX + _BMIN = quantize_utils.TORCH_BFLOAT16_MIN + _BMAX = quantize_utils.TORCH_BFLOAT16_MAX + + def _lowmem_fp16(tensor: torch.Tensor) -> torch.Tensor: + return tensor.half().clamp_(_HMIN, _HMAX) + + def _lowmem_bf16(tensor: torch.Tensor) -> torch.Tensor: + return tensor.bfloat16().clamp_(_BMIN, _BMAX) + + # Patch BOTH the definition module and quantize_comm, which imported the + # names directly (``from .quantize_utils import fp32_to_fp16_with_clamp``) + # so its module-level reference must be overridden too. + for _mod in (quantize_utils, quantize_comm): + if hasattr(_mod, "fp32_to_fp16_with_clamp"): + _mod.fp32_to_fp16_with_clamp = _lowmem_fp16 + if hasattr(_mod, "fp32_to_bf16_with_clamp"): + _mod.fp32_to_bf16_with_clamp = _lowmem_bf16 + + _FBGEMM_LOWMEM_PATCHED = True + if rank0: + logger.info( + "[qcomm-lowmem] patched fbgemm fp32->fp16/bf16 clamp+cast to " + "cast-then-clamp_ (drops the full-size fp32 clamp temp; avoids " + "OOM on skewed row-wise embedding a2a)" + ) + except Exception as e: # noqa: BLE001 — patch is best-effort, never fatal + if rank0: + logger.warning( + "[qcomm-lowmem] could not patch fbgemm clamp+cast (%s: %s); " + "running with stock (higher-peak) quantizer", + type(e).__name__, + e, + ) + + def _maybe_apply_qcomm_a2a( sharders: List[Any], device: torch.device, forward_precision: str = "fp32", backward_precision: str = "fp32", + lowmem_clamp_cast: bool = True, ) -> List[Any]: """Optionally quantize the embedding all-to-all payload via TorchRec qcomm. @@ -520,6 +603,11 @@ def _maybe_apply_qcomm_a2a( ) if fwd == "fp32" and bwd == "fp32": return sharders + # Before building the codec, swap fbgemm's clamp+cast for a memory-frugal + # equivalent — see _patch_fbgemm_lowmem_clamp_cast for why (avoids a full + # extra fp32 temp that OOMs the hottest row-wise shard on skewed batches). + # Gated by `lowmem_clamp_cast` (gin/env); ON by default. + _patch_fbgemm_lowmem_clamp_cast(enabled=lowmem_clamp_cast, rank0=rank0) try: from torchrec.distributed.embedding import EmbeddingCollectionSharder from torchrec.distributed.fbgemm_qcomm_codec import ( @@ -666,6 +754,7 @@ def make_optimizer_and_shard( hbm_cap_gb: int = 260, sparse_a2a_forward_precision: str = "fp32", sparse_a2a_backward_precision: str = "fp32", + qcomm_lowmem_clamp_cast: bool = True, embedding_placement: str = "auto", embedding_placement_overrides: Optional[Dict[str, str]] = None, embedding_table_configs: Optional[Dict[str, EmbeddingConfig]] = None, @@ -691,6 +780,7 @@ def make_optimizer_and_shard( device, forward_precision=sparse_a2a_forward_precision, backward_precision=sparse_a2a_backward_precision, + lowmem_clamp_cast=qcomm_lowmem_clamp_cast, ) # local_world_size = GPUs per node so the planner respects the intra-node # (xGMI/NVLink) vs inter-node hierarchy when placing shards. Defaults to diff --git a/recommendation_v4/scripts/launch_slurm.sh b/recommendation_v4/scripts/launch_slurm.sh index 2177f446c..2994b11dd 100755 --- a/recommendation_v4/scripts/launch_slurm.sh +++ b/recommendation_v4/scripts/launch_slurm.sh @@ -325,6 +325,11 @@ orchestrate() { ${CKPT_PATH:+-e CKPT_PATH=$CKPT_PATH} \ ${SPARSE_A2A_FWD:+-e SPARSE_A2A_FWD=$SPARSE_A2A_FWD} \ ${SPARSE_A2A_BWD:+-e SPARSE_A2A_BWD=$SPARSE_A2A_BWD} \ + ${PG_TIMEOUT_S:+-e PG_TIMEOUT_S=$PG_TIMEOUT_S} \ + ${TORCH_NCCL_TRACE_BUFFER_SIZE:+-e TORCH_NCCL_TRACE_BUFFER_SIZE=$TORCH_NCCL_TRACE_BUFFER_SIZE} \ + ${TORCH_NCCL_DUMP_ON_TIMEOUT:+-e TORCH_NCCL_DUMP_ON_TIMEOUT=$TORCH_NCCL_DUMP_ON_TIMEOUT} \ + ${TORCH_NCCL_TRACE_CPP_STACK:+-e TORCH_NCCL_TRACE_CPP_STACK=$TORCH_NCCL_TRACE_CPP_STACK} \ + ${TORCH_NCCL_DEBUG_INFO_TEMP_FILE:+-e TORCH_NCCL_DEBUG_INFO_TEMP_FILE=$TORCH_NCCL_DEBUG_INFO_TEMP_FILE} \ -e LOG=$LOG \ $NCCL_ENV_ARGS \ $CONTAINER bash -lc 'cd $REPO && LAUNCH_SLURM_PHASE=worker bash scripts/launch_slurm.sh' From a856229c7575a4c945c927e4e286db1084199d3e Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Tue, 30 Jun 2026 21:29:10 +0000 Subject: [PATCH 112/127] dlrmv3 launch_slurm: forward QCOMM_LOWMEM_CODEC env into the container Completes the qcomm low-memory codec knob: without this the gin qlcc/env_int($QCOMM_LOWMEM_CODEC) binding could never see an env override inside the container (it only saw the gin default). Now $QCOMM_LOWMEM_CODEC set at submit time reaches the trainer, so the patch can be toggled per-run. Co-authored-by: Cursor --- recommendation_v4/scripts/launch_slurm.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/recommendation_v4/scripts/launch_slurm.sh b/recommendation_v4/scripts/launch_slurm.sh index 2994b11dd..80bb3643c 100755 --- a/recommendation_v4/scripts/launch_slurm.sh +++ b/recommendation_v4/scripts/launch_slurm.sh @@ -325,6 +325,7 @@ orchestrate() { ${CKPT_PATH:+-e CKPT_PATH=$CKPT_PATH} \ ${SPARSE_A2A_FWD:+-e SPARSE_A2A_FWD=$SPARSE_A2A_FWD} \ ${SPARSE_A2A_BWD:+-e SPARSE_A2A_BWD=$SPARSE_A2A_BWD} \ + ${QCOMM_LOWMEM_CODEC:+-e QCOMM_LOWMEM_CODEC=$QCOMM_LOWMEM_CODEC} \ ${PG_TIMEOUT_S:+-e PG_TIMEOUT_S=$PG_TIMEOUT_S} \ ${TORCH_NCCL_TRACE_BUFFER_SIZE:+-e TORCH_NCCL_TRACE_BUFFER_SIZE=$TORCH_NCCL_TRACE_BUFFER_SIZE} \ ${TORCH_NCCL_DUMP_ON_TIMEOUT:+-e TORCH_NCCL_DUMP_ON_TIMEOUT=$TORCH_NCCL_DUMP_ON_TIMEOUT} \ From b8f82985b9121d4c071073337fe2cc0f7379031f Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Wed, 1 Jul 2026 03:21:53 +0000 Subject: [PATCH 113/127] dlrmv3 sharding: gin/env-configurable per-table embedding sharding-type overrides Add EMB_SHARDING_OVERRIDES (gin/env) so individual embedding tables can be pinned to a sharding type, orthogonal to the existing placement override. Default OFF -> plan is byte-identical to the legacy all-ROW_WISE path. Motivation: ROW_WISE routes every lookup of a hot ID to its single owner rank, so a few popular albums/artists concentrate the embedding all-to-all onto one rank; the burst scales ~linearly with global batch size and OOM'd the hot rank (~208-238 GiB / 288) at window ~248 on the yambda-5b 4-node run. Moving album_id/artist_id to COLUMN_WISE balances the a2a by rank regardless of hot-ID skew. Validated by a reshard smoke: DCP loaded the ROW_WISE ckpt into the CW plan cleanly, window_auc stayed ~0.78-0.80, and the hot rank sat at ~58% (~120 GiB free) through the previously-OOM window. - utils: _build_placement_constraints/make_optimizer_and_shard accept embedding_sharding_overrides and merge them into ParameterConstraints. - gin: EMB_SHARDING_OVERRIDES env_str_map binding + rationale/example comments. - launch_slurm: forward EMB_SHARDING_OVERRIDES (+ placement) into the container. Co-authored-by: Cursor --- .../dlrm_v3/train/gin/yambda_5b.gin | 38 ++++++++ .../dlrm_v3/train/utils.py | 88 +++++++++++++++---- recommendation_v4/scripts/launch_slurm.sh | 4 + 3 files changed, 113 insertions(+), 17 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index 2eb92b30f..5da1d31c7 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -179,6 +179,44 @@ env_str_map.key = "EMB_PLACEMENT_OVERRIDES" env_str_map.merge = True env_str_map.default = {} +# Per-table SHARDING-TYPE overrides (shard layout, orthogonal to placement above). +# Absent/"auto" tables keep the planner's choice (ROW_WISE for the large yambda +# tables). Allowed per table: "row_wise"|"column_wise"|"table_wise"| +# "table_row_wise"|"table_column_wise"|"data_parallel"|"auto" (aliases rw/cw/tw/ +# twrw). DEFAULT OFF ({}) -> plan is byte-identical to the legacy all-ROW_WISE +# path. Opt in per run via env, e.g.: +# EMB_SHARDING_OVERRIDES="album_id=column_wise,artist_id=column_wise" +# WHY: ROW_WISE routes every lookup of row r to the single owner rank, so a few +# hot IDs concentrate the embedding all-to-all onto one rank and OOM it (the +# yambda-5b skew hang; album_id ~2.8x, artist_id ~1.3x per-rank load). COLUMN_WISE +# splits the table by embedding dim (every rank holds all rows, dim/world cols), +# so the a2a load is balanced by RANK regardless of which IDs are hot — removing +# the value-skew OOM — at identical per-rank table bytes. Convert only the skewed +# high-volume tables (album_id, artist_id); leave the balanced, highest-volume +# item_id and the tiny length-1 contextual/cross tables on ROW_WISE. +# WHY THIS GETS WORSE AT LARGER GLOBAL BATCH: the ROW_WISE a2a input buffer on the +# owner rank is sized by how many times its hot IDs appear across the WHOLE global +# batch, so that transient scales ~linearly with global batch size (here 32 ranks +# x 1024 = 32768, each carrying ~4096-token UIH sequences -> tens of millions of +# lookups/step, heavily re-hitting the same few popular albums/artists). Doubling +# the global batch ~doubles the hot-rank burst while every other rank stays idle, +# which is exactly what tipped GPU5/GPU3 from a saturated steady state (~208-238 +# GiB) over 288 GiB at window ~248. COLUMN_WISE makes each rank receive dim/world +# of EVERY lookup, so the per-rank a2a volume is ~global_batch/world (balanced) +# and grows with world size, not with which IDs are hot -> it scales cleanly as +# you push global batch / add ranks, instead of piling the growth onto one shard. +# Example (the yambda-5b 4-node run, global batch 32768, that hit the skew OOM): +# EMB_SHARDING_OVERRIDES="album_id=column_wise,artist_id=column_wise" +# validated by the CW smoke: reshard-loaded the ROW_WISE ckpt cleanly, window_auc +# stayed ~0.78-0.80, and the hot rank sat at ~58% (~120 GiB free) THROUGH the old +# OOM window (vs 16 GiB free right before the ROW_WISE crash). +# NOTE: changing the sharding plan changes the on-disk shard layout; a checkpoint +# written under a different plan must be resharded/validated on load. +make_optimizer_and_shard.embedding_sharding_overrides = @esh/env_str_map() +esh/env_str_map.key = "EMB_SHARDING_OVERRIDES" +esh/env_str_map.merge = True +esh/env_str_map.default = {} + # Sparse embedding all-to-all wire precision. The embedding shuffle is the # dominant, bandwidth-bound (esp. multi-node) collective; quantizing it via # TorchRec QCommsConfig halves (bf16/fp16, both 2 bytes) the wire volume. diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py index dae3d92e7..f226595a8 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -60,7 +60,7 @@ from torchrec.distributed.planner import EmbeddingShardingPlanner, Topology from torchrec.distributed.planner.types import ParameterConstraints from torchrec.distributed.sharding_plan import get_default_sharders -from torchrec.distributed.types import ShardedTensor, ShardingEnv +from torchrec.distributed.types import ShardedTensor, ShardingEnv, ShardingType from torchrec.modules.embedding_configs import EmbeddingConfig from torchrec.modules.embedding_modules import ( EmbeddingBagCollection, @@ -87,6 +87,25 @@ "uvm_caching": EmbeddingComputeKernel.FUSED_UVM_CACHING, } +# Per-table sharding-type vocabulary -> torchrec ShardingType. Used by +# make_optimizer_and_shard to pin a table's shard layout via ParameterConstraints +# (e.g. move a hot, data-skewed table off ROW_WISE to COLUMN_WISE so its +# embedding all-to-all load is balanced by rank instead of routed by row/value). +# "auto" (or anything not in this map) means "no constraint": the planner decides. +# Short aliases (rw/cw/tw/twrw) are accepted alongside the canonical names. +_SHARDING_TO_TYPE: Dict[str, ShardingType] = { + "row_wise": ShardingType.ROW_WISE, + "column_wise": ShardingType.COLUMN_WISE, + "table_wise": ShardingType.TABLE_WISE, + "table_row_wise": ShardingType.TABLE_ROW_WISE, + "table_column_wise": ShardingType.TABLE_COLUMN_WISE, + "data_parallel": ShardingType.DATA_PARALLEL, + "rw": ShardingType.ROW_WISE, + "cw": ShardingType.COLUMN_WISE, + "tw": ShardingType.TABLE_WISE, + "twrw": ShardingType.TABLE_ROW_WISE, +} + @gin.configurable def seed_everything(seed: int = -1, rank: int = 0) -> None: @@ -689,14 +708,27 @@ def _build_placement_constraints( embedding_placement: str, embedding_placement_overrides: Dict[str, str], embedding_table_configs: Optional[Dict[str, EmbeddingConfig]], + embedding_sharding_overrides: Optional[Dict[str, str]] = None, ) -> Dict[str, ParameterConstraints]: - """Translate gin/env placement strings into torchrec ParameterConstraints. - - Resolution per table: ``overrides.get(name, embedding_placement)``. A value - of ``auto`` (or empty) means "no constraint" and the table is omitted so the - planner keeps deciding from the HBM cap. Unknown values raise ValueError. + """Translate gin/env placement + sharding strings into ParameterConstraints. + + Two orthogonal per-table knobs are merged into one constraint per table: + + * Placement (compute kernel / memory tier): + ``embedding_placement_overrides.get(name, embedding_placement)``. + ``auto``/empty -> no compute-kernel constraint (planner decides from HBM). + * Sharding type (shard layout): ``embedding_sharding_overrides.get(name)``. + ``auto``/empty (or absent) -> no sharding-type constraint (planner decides, + which is ROW_WISE for the large yambda tables). Use e.g. ``column_wise`` + to move a hot, data-skewed table off ROW_WISE so its embedding all-to-all + is balanced by rank instead of routed by (hot) row. + + A table is added to the returned dict only if at least one knob is set for it + (so with everything ``auto`` we return {} and the plan is byte-identical to + the legacy path). Unknown values raise ValueError. """ - valid = set(_PLACEMENT_TO_KERNEL) | {"auto", ""} + embedding_sharding_overrides = embedding_sharding_overrides or {} + valid_place = set(_PLACEMENT_TO_KERNEL) | {"auto", ""} for where, val in [ ("embedding_placement", embedding_placement), *[ @@ -704,14 +736,24 @@ def _build_placement_constraints( for k, v in embedding_placement_overrides.items() ], ]: - if val not in valid: + if val not in valid_place: raise ValueError( f"Invalid embedding placement {val!r} for {where}; " - f"expected one of {sorted(valid - {''})}." + f"expected one of {sorted(valid_place - {''})}." + ) + valid_shard = set(_SHARDING_TO_TYPE) | {"auto", ""} + for k, v in embedding_sharding_overrides.items(): + if v not in valid_shard: + raise ValueError( + f"Invalid embedding sharding {v!r} for " + f"embedding_sharding_overrides[{k}]; " + f"expected one of {sorted(valid_shard - {''})}." ) names = _embedding_table_names(model, embedding_table_configs) - unknown = set(embedding_placement_overrides) - set(names) + unknown = ( + set(embedding_placement_overrides) | set(embedding_sharding_overrides) + ) - set(names) if unknown: logger.warning( "[emb-placement] override(s) for unknown table(s) %s ignored; " @@ -721,24 +763,34 @@ def _build_placement_constraints( ) constraints: Dict[str, ParameterConstraints] = {} - resolved: Dict[str, str] = {} + resolved_place: Dict[str, str] = {} + resolved_shard: Dict[str, str] = {} for name in names: placement = embedding_placement_overrides.get(name, embedding_placement) - resolved[name] = placement or "auto" + sharding = embedding_sharding_overrides.get(name, "auto") + resolved_place[name] = placement or "auto" + resolved_shard[name] = sharding or "auto" kernel = _PLACEMENT_TO_KERNEL.get(placement) + stype = _SHARDING_TO_TYPE.get(sharding) + kwargs: Dict[str, Any] = {} if kernel is not None: - constraints[name] = ParameterConstraints( - compute_kernels=[kernel.value] - ) + kwargs["compute_kernels"] = [kernel.value] + if stype is not None: + kwargs["sharding_types"] = [stype.value] + if kwargs: + constraints[name] = ParameterConstraints(**kwargs) rank = dist.get_rank() if dist.is_initialized() else 0 if rank == 0: logger.info( - "[emb-placement] global=%r overrides=%s -> resolved=%s " + "[emb-placement] placement(global=%r overrides=%s) sharding(overrides=%s) " + "-> resolved_placement=%s resolved_sharding=%s " "(constrained=%d/%d tables; the rest are planner-auto)", embedding_placement, embedding_placement_overrides or {}, - resolved, + embedding_sharding_overrides or {}, + resolved_place, + resolved_shard, len(constraints), len(names), ) @@ -757,6 +809,7 @@ def make_optimizer_and_shard( qcomm_lowmem_clamp_cast: bool = True, embedding_placement: str = "auto", embedding_placement_overrides: Optional[Dict[str, str]] = None, + embedding_sharding_overrides: Optional[Dict[str, str]] = None, embedding_table_configs: Optional[Dict[str, EmbeddingConfig]] = None, ) -> Tuple[DistributedModelParallel, torch.optim.Optimizer]: dense_opt_cls, dense_opt_args, dense_opt_factory = ( @@ -802,6 +855,7 @@ def make_optimizer_and_shard( model=model, embedding_placement=embedding_placement, embedding_placement_overrides=embedding_placement_overrides or {}, + embedding_sharding_overrides=embedding_sharding_overrides or {}, embedding_table_configs=embedding_table_configs, ) planner = EmbeddingShardingPlanner( diff --git a/recommendation_v4/scripts/launch_slurm.sh b/recommendation_v4/scripts/launch_slurm.sh index 80bb3643c..cb171593c 100755 --- a/recommendation_v4/scripts/launch_slurm.sh +++ b/recommendation_v4/scripts/launch_slurm.sh @@ -289,6 +289,7 @@ orchestrate() { ${EVAL_EVERY_DATA_PCT:+-e EVAL_EVERY_DATA_PCT=$EVAL_EVERY_DATA_PCT} \ -e NUM_TRAIN_BATCHES=$NUM_TRAIN_BATCHES \ -e NUM_EVAL_BATCHES=$NUM_EVAL_BATCHES \ + ${DIE_AT_STEP:+-e DIE_AT_STEP=$DIE_AT_STEP} \ -e METRIC_LOG_FREQ=$METRIC_LOG_FREQ \ ${MLPERF_LOGGING:+-e MLPERF_LOGGING=$MLPERF_LOGGING} \ ${MLPERF_TRAIN_LOSS_LOG_FREQ:+-e MLPERF_TRAIN_LOSS_LOG_FREQ=$MLPERF_TRAIN_LOSS_LOG_FREQ} \ @@ -326,6 +327,9 @@ orchestrate() { ${SPARSE_A2A_FWD:+-e SPARSE_A2A_FWD=$SPARSE_A2A_FWD} \ ${SPARSE_A2A_BWD:+-e SPARSE_A2A_BWD=$SPARSE_A2A_BWD} \ ${QCOMM_LOWMEM_CODEC:+-e QCOMM_LOWMEM_CODEC=$QCOMM_LOWMEM_CODEC} \ + ${EMB_SHARDING_OVERRIDES:+-e EMB_SHARDING_OVERRIDES=$EMB_SHARDING_OVERRIDES} \ + ${EMB_PLACEMENT_OVERRIDES:+-e EMB_PLACEMENT_OVERRIDES=$EMB_PLACEMENT_OVERRIDES} \ + ${EMB_PLACEMENT:+-e EMB_PLACEMENT=$EMB_PLACEMENT} \ ${PG_TIMEOUT_S:+-e PG_TIMEOUT_S=$PG_TIMEOUT_S} \ ${TORCH_NCCL_TRACE_BUFFER_SIZE:+-e TORCH_NCCL_TRACE_BUFFER_SIZE=$TORCH_NCCL_TRACE_BUFFER_SIZE} \ ${TORCH_NCCL_DUMP_ON_TIMEOUT:+-e TORCH_NCCL_DUMP_ON_TIMEOUT=$TORCH_NCCL_DUMP_ON_TIMEOUT} \ From 8163b4eadb748b5cb265c8b9180c81e85fb277a9 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Tue, 14 Jul 2026 01:31:39 +0000 Subject: [PATCH 114/127] dlrmv3 qcomm: gin/env-configurable EC in-batch index dedup (default on) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add EC_INDEX_DEDUP knob that enables TorchRec use_index_dedup on the sequence EmbeddingCollectionSharder, collapsing duplicate ids within a batch to unique rows before the sparse input all-to-all (lookup once, scatter back via reverse_indices). Applied as a final pass over the sharder list so it covers BOTH construction paths — the qcomm-rebuilt sharder (fp16 default) and the bare get_default_sharders() one (SPARSE_A2A_*=fp32) — preserving the qcomm codec registry so dedup composes with the quantized a2a. Lossless for a non-pooled EmbeddingCollection; fixed-seed A/B on this config gave IDENTICAL metrics (1-node train_loss/NE/AUC, 2-node train_loss) with a large, node-count-scaling speedup (-14.4% step_ms at 1 node, -26.2% at 2 nodes, since it also shrinks the cross-node embedding a2a). Dedup buffers are persistent=False, so the toggle is checkpoint-safe (no reshard). Default on; set EC_INDEX_DEDUP=0 for high-shuffle runs. Also forward EC_INDEX_DEDUP into the container in launch_slurm.sh. Co-authored-by: Cursor --- .../dlrm_v3/train/gin/yambda_5b.gin | 53 +++++++++++++ .../dlrm_v3/train/utils.py | 74 +++++++++++++++++++ recommendation_v4/scripts/launch_slurm.sh | 1 + 3 files changed, 128 insertions(+) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index 5da1d31c7..ef127ef7a 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -294,6 +294,59 @@ make_optimizer_and_shard.qcomm_lowmem_clamp_cast = @qlcc/env_int() qlcc/env_int.key = "QCOMM_LOWMEM_CODEC" qlcc/env_int.default = 1 +# --- ec_index_dedup: in-batch embedding index deduplication ------------------ +# Enables TorchRec's use_index_dedup on the sequence EmbeddingCollectionSharder. +# Before the sparse input all-to-all, duplicate ids within a batch are collapsed +# to unique rows (torch.ops.fbgemm.jagged_unique_indices), the embedding lookup +# runs once per unique id, then results scatter back to their original positions +# via reverse_indices. For a NON-POOLED EmbeddingCollection this reconstruction +# is exact, so it is numerically LOSSLESS (unlike the fp16 a2a it does not move +# AUC/NE) — it only cuts (a) the sparse input all-to-all element COUNT and (b) +# the number of rows the fused TBE kernel gathers per step. +# +# WHY IT HELPS HERE: the default streaming order is user-major +# (STREAMING_SHUFFLE_FRACTION=0), so consecutive anchors are the same user's UIH +# re-reading the same item/artist/album ids -> very high in-batch duplicate rate. +# Measured via DIAG_UNIQUE_EMB on this config: the big UIH sequence tables are +# ~98% duplicate WITHIN a batch (uih.item_id ~2.4% unique over 2.69M ids/batch, +# uih.album_id ~1.5%, uih.artist_id ~0.6%), so collapsing to unique rows removes +# the bulk of the lookup + a2a volume. Dedup COMPOSES with the fp16 qcomm a2a +# (dedup cuts element count, qcomm cuts bytes/element) and the CW skew fix. +# +# MEASURED A/B (this exact config; fixed SEED=1; qcomm fp16 fwd+bwd; 200-step +# train-only; c-series MI350 nodes). Steady-state (steps>=60) mean step time and +# global throughput, dedup OFF -> ON: +# * 1 node (WORLD_SIZE=8): step_ms 941.6 -> 805.7 (-14.4%, +16.9% sps), +# MFU 21.3% -> 24.7% +# * 2 node (WORLD_SIZE=16): step_ms 1108.0 -> 818.0 (-26.2%, +37.6% sps), +# MFU 18.1% -> 24.9% +# The win GROWS with node count because at >1 node dedup also shrinks the +# cross-node embedding all-to-all — the dominant bandwidth-bound multi-node +# collective. It nearly erases the multi-node comm penalty: OFF weak-scales +# 21.3%->18.1% MFU going 1->2 node, ON holds ~24.7%->24.9%. +# +# NUMERICAL NEUTRALITY (why default-on is safe): with a fixed seed the OFF and +# ON runs produced IDENTICAL metrics, confirming the reconstruction is exact and +# training is bit-for-bit unaffected: +# * 1 node @ step41: train_loss 0.13778 == 0.13778; window_ne/lifetime_ne +# 1.0338 == 1.0338; window_auc 0.4947 == 0.4947 +# * 2 node @ step41: train_loss 0.13811 == 0.13811 +# (Contrast the fp16 a2a, which is lossy-but-tolerable; this knob moves nothing.) +# +# The dedup buffers are persistent=False -> NO change to the on-disk shard layout +# and safe to toggle on an existing checkpoint (no reshard). Applied as a final +# pass over the sharder list so it covers BOTH the qcomm-on (default) and +# qcomm-off (SPARSE_A2A_*=fp32) construction paths. +# +# DEFAULT ON (1): lossless + a large, node-count-scaling speedup on this config. +# The payoff shrinks toward neutral/slightly-negative only as batch diversity +# rises (high STREAMING_SHUFFLE_FRACTION -> fewer in-batch dups, so the +# jagged_unique_indices + scatter overhead stops paying for itself); set +# EC_INDEX_DEDUP=0 for such a high-shuffle run. Override per-run via $EC_INDEX_DEDUP. +make_optimizer_and_shard.ec_index_dedup = @ecid/env_int() +ecid/env_int.key = "EC_INDEX_DEDUP" +ecid/env_int.default = 1 + get_dataset.name = %dataset get_dataset.new_path_prefix = %DATA_PATH # Total user-interaction-history (UIH) budget per sample, distributed evenly diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py index f226595a8..762afc101 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -681,6 +681,75 @@ def _maybe_apply_qcomm_a2a( return new_sharders +def _maybe_apply_ec_index_dedup( + sharders: List[Any], + enabled: bool, +) -> List[Any]: + """Optionally enable TorchRec's in-batch index dedup on the EC sharder. + + Runs as a FINAL pass AFTER get_default_sharders() and _maybe_apply_qcomm_a2a, + so it covers BOTH sharder construction paths with a single knob: + * qcomm ON (SPARSE_A2A_* != fp32, the default): the EC sharder was already + rebuilt with a qcomm codec registry inside _maybe_apply_qcomm_a2a. + * qcomm OFF (SPARSE_A2A_* == fp32): that function early-returns the bare + get_default_sharders() list untouched. + In either case we rebuild the one EmbeddingCollectionSharder in place, + PRESERVING its existing fused_params + qcomm codec registry, and only flip + use_index_dedup. That ordering guarantees dedup composes with (never clobbers) + the fp16/bf16 a2a instead of depending on which qcomm branch ran. + + use_index_dedup collapses duplicate ids within a batch to unique rows before + the input all-to-all (torch.ops.fbgemm.jagged_unique_indices), does the + embedding lookup once, then scatters back via reverse_indices. For a + non-pooled EmbeddingCollection (this sequence model) that reconstruction is + exact, so the forward is numerically lossless — unlike the fp16 a2a it does + not perturb AUC/NE. The buffers it registers (_hash_size_cumsum_tensor_*, + _hash_size_offset_tensor_*) are persistent=False, so it does NOT change the + on-disk shard layout and can be toggled on an existing checkpoint with no + reshard. Payoff scales with the in-batch duplicate rate: large under + user-major batching (STREAMING_SHUFFLE_FRACTION=0, the default, where one + user's UIH re-reads the same item/artist/album ids), shrinking toward + neutral/negative as batch diversity rises. + + The knob only exists on EmbeddingCollectionSharder, which is the sole default + sharder this model actually binds (a pure sequence EmbeddingCollection), so + no other sharder needs touching. Configured via gin on + ``make_optimizer_and_shard`` (env-overridable $EC_INDEX_DEDUP). + """ + if not enabled: + return sharders + + from torchrec.distributed.embedding import EmbeddingCollectionSharder + + rank0 = (not dist.is_initialized()) or dist.get_rank() == 0 + new_sharders = [] + replaced = False + for s in sharders: + if type(s).__name__ == "EmbeddingCollectionSharder" and not replaced: + new_sharders.append( + EmbeddingCollectionSharder( + fused_params=s.fused_params, + qcomm_codecs_registry=s.qcomm_codecs_registry, + use_index_dedup=True, + ) + ) + replaced = True + else: + new_sharders.append(s) + if not replaced: + # Dedup configured but no EC sharder to bind it to would mean it is + # silently inert — the same "configured but not applied" bug class the + # qcomm path guards against. Fail loudly on every rank instead. + raise RuntimeError( + "DLRMV4 EC index dedup configured (use_index_dedup=True) but no " + "EmbeddingCollectionSharder was found to enable it on; refusing to " + "run with dedup silently disabled" + ) + if rank0: + logger.info("DLRMV4 EC index dedup ENABLED (use_index_dedup=True)") + return new_sharders + + def _embedding_table_names( model: torch.nn.Module, embedding_table_configs: Optional[Dict[str, EmbeddingConfig]], @@ -807,6 +876,7 @@ def make_optimizer_and_shard( sparse_a2a_forward_precision: str = "fp32", sparse_a2a_backward_precision: str = "fp32", qcomm_lowmem_clamp_cast: bool = True, + ec_index_dedup: bool = False, embedding_placement: str = "auto", embedding_placement_overrides: Optional[Dict[str, str]] = None, embedding_sharding_overrides: Optional[Dict[str, str]] = None, @@ -835,6 +905,10 @@ def make_optimizer_and_shard( backward_precision=sparse_a2a_backward_precision, lowmem_clamp_cast=qcomm_lowmem_clamp_cast, ) + # Final pass: enable in-batch index dedup on the EC sharder regardless of + # whether the qcomm branch above rebuilt it (fp16 default) or returned the + # bare get_default_sharders() list (fp32). Preserves the qcomm registry. + sharders = _maybe_apply_ec_index_dedup(sharders, enabled=ec_index_dedup) # local_world_size = GPUs per node so the planner respects the intra-node # (xGMI/NVLink) vs inter-node hierarchy when placing shards. Defaults to # world_size for the single-node case (no behavior change). diff --git a/recommendation_v4/scripts/launch_slurm.sh b/recommendation_v4/scripts/launch_slurm.sh index cb171593c..d3bc4e188 100755 --- a/recommendation_v4/scripts/launch_slurm.sh +++ b/recommendation_v4/scripts/launch_slurm.sh @@ -327,6 +327,7 @@ orchestrate() { ${SPARSE_A2A_FWD:+-e SPARSE_A2A_FWD=$SPARSE_A2A_FWD} \ ${SPARSE_A2A_BWD:+-e SPARSE_A2A_BWD=$SPARSE_A2A_BWD} \ ${QCOMM_LOWMEM_CODEC:+-e QCOMM_LOWMEM_CODEC=$QCOMM_LOWMEM_CODEC} \ + ${EC_INDEX_DEDUP:+-e EC_INDEX_DEDUP=$EC_INDEX_DEDUP} \ ${EMB_SHARDING_OVERRIDES:+-e EMB_SHARDING_OVERRIDES=$EMB_SHARDING_OVERRIDES} \ ${EMB_PLACEMENT_OVERRIDES:+-e EMB_PLACEMENT_OVERRIDES=$EMB_PLACEMENT_OVERRIDES} \ ${EMB_PLACEMENT:+-e EMB_PLACEMENT=$EMB_PLACEMENT} \ From 273475acd423f3ac15e3d2b55a0ac63e8731cd26 Mon Sep 17 00:00:00 2001 From: suachong Date: Tue, 14 Jul 2026 03:14:49 +0000 Subject: [PATCH 115/127] Add steps-to-target CV script for run-to-run convergence analysis. Provides a CLI to compute CV of samples-to-reach-target eval-window-AUC from a folder of MLPerf logs, with configurable single targets or AUC ranges. Co-authored-by: Cursor --- recommendation_v4/cv/steps_to_target_cv.py | 174 +++++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100644 recommendation_v4/cv/steps_to_target_cv.py diff --git a/recommendation_v4/cv/steps_to_target_cv.py b/recommendation_v4/cv/steps_to_target_cv.py new file mode 100644 index 000000000..9b6c8accf --- /dev/null +++ b/recommendation_v4/cv/steps_to_target_cv.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +"""Run-to-run convergence CV via the steps-to-target method. + +For each target eval-window AUC t, for each run find the first samples_count +where AUC >= t (first-crossing, no interpolation). Report mean/std/CV/min/max +across the runs that reached t. + +Reads MLPerf ``:::MLLOG`` logs (``eval_accuracy`` == window_auc/listen_plus). + +Examples:: + + # auto sweep 0.60 .. max-AUC @ step 0.005 + python3 cv/steps_to_target_cv.py --log-dir /path/to/logs -o cv/table.csv + + # single target + python3 cv/steps_to_target_cv.py --log-dir /path/to/logs --target 0.75 + + # explicit list + python3 cv/steps_to_target_cv.py --log-dir /path/to/logs --targets 0.70,0.75,0.78 + + # range with step + python3 cv/steps_to_target_cv.py --log-dir /path/to/logs --target-range 0.60:0.78 --step 0.005 +""" +import argparse +import glob +import json +import os +import re +import sys + +import numpy as np +import pandas as pd + +_MLLOG = re.compile(r":::MLLOG (\{.*\})\s*$") + + +def parse_log(path): + """MLPerf log -> (samples_sorted, auc_sorted). Empty arrays if no evals.""" + pts = {} + with open(path, "r", errors="replace") as f: + for line in f: + m = _MLLOG.search(line) + if not m: + continue + try: + rec = json.loads(m.group(1)) + except json.JSONDecodeError: + continue + if rec.get("key") != "eval_accuracy" or rec.get("value") is None: + continue + sc = rec.get("metadata", {}).get("samples_count") + if sc is not None: + pts[int(sc)] = float(rec["value"]) + if not pts: + return np.array([]), np.array([]) + samples = np.array(sorted(pts)) + auc = np.array([pts[s] for s in samples]) + return samples, auc + + +def load_runs(log_dir): + """All *.log files in log_dir -> {basename: (samples, auc)}.""" + paths = sorted(glob.glob(os.path.join(log_dir, "*.log"))) + if not paths: + sys.exit(f"no *.log files in {log_dir}") + runs = {} + for path in paths: + name = os.path.splitext(os.path.basename(path))[0] + samples, auc = parse_log(path) + if len(auc) >= 1: + runs[name] = (samples, auc) + print(f" {name}: {len(auc)} eval points", file=sys.stderr) + return runs + + +def first_crossing(samples, auc, target): + for s, a in zip(samples, auc): + if a >= target: + return s + return None + + +def resolve_targets(args, runs): + """Build target AUC list from CLI flags.""" + if args.target is not None: + return [float(args.target)] + if args.targets is not None: + return [float(t.strip()) for t in args.targets.split(",") if t.strip()] + + lo, hi = args.target_range + top = max(a.max() for _, a in runs.values()) + hi = min(hi, top) + if lo > hi: + sys.exit(f"target-range lo={lo} > hi={hi} (max AUC across runs = {top:.4f})") + return list(np.arange(lo, hi + 1e-9, args.step)) + + +def cv_table(runs, targets, min_runs): + rows = [] + for t in targets: + hits = [s for samples, auc in runs.values() + if (s := first_crossing(samples, auc, t)) is not None] + n = len(hits) + if n < min_runs: + continue + arr = np.array(hits, dtype=float) + mean = arr.mean() + std = arr.std(ddof=1) + rows.append({ + "target_eval_auc": round(float(t), 4), + "runs_reached": n, + "samples_mean": round(mean, 1), + "samples_std": round(std, 1), + "cv_pct": round(std / mean * 100.0, 3) if mean > 0 else 0.0, + "cv_fraction": round(std / mean, 5) if mean > 0 else 0.0, + "samples_min": int(arr.min()), + "samples_max": int(arr.max()), + }) + return pd.DataFrame(rows) + + +def parse_target_range(s): + if ":" not in s: + raise argparse.ArgumentTypeError("expected LO:HI, e.g. 0.60:0.78") + lo, hi = s.split(":", 1) + return float(lo), float(hi) + + +def main(): + p = argparse.ArgumentParser( + description="Steps-to-target CV table from a folder of MLPerf logs.") + p.add_argument("--log-dir", required=True, + help="directory containing one *.log per seed run") + p.add_argument("-o", "--output", + help="output CSV path (default: /steps_to_target_cv.csv)") + p.add_argument("--min-runs", type=int, default=2, + help="min runs that must reach a target to include it (default: 2)") + + g = p.add_mutually_exclusive_group() + g.add_argument("--target", type=float, + help="single target eval-window AUC, e.g. 0.75") + g.add_argument("--targets", metavar="T1,T2,...", + help="comma-separated target AUCs, e.g. 0.70,0.75,0.78") + g.add_argument("--target-range", type=parse_target_range, metavar="LO:HI", + help="inclusive AUC range, e.g. 0.60:0.78 (default when unset)") + + p.add_argument("--step", type=float, default=0.005, + help="step for --target-range sweep (default: 0.005)") + p.add_argument("--range-lo", type=float, default=0.60, + help="lower bound when no target flag given (default: 0.60)") + args = p.parse_args() + + if args.target_range is None and args.target is None and args.targets is None: + args.target_range = (args.range_lo, 1.0) # hi clipped to max AUC in resolve_targets + + print(f"loading logs from {args.log_dir}", file=sys.stderr) + runs = load_runs(args.log_dir) + if len(runs) < args.min_runs: + sys.exit(f"only {len(runs)} usable run(s) in {args.log_dir} (need >= {args.min_runs})") + + targets = resolve_targets(args, runs) + df = cv_table(runs, targets, args.min_runs) + if df.empty: + sys.exit("no targets had enough runs — check --target / --target-range / --min-runs") + + out = args.output or os.path.join(args.log_dir, "steps_to_target_cv.csv") + os.makedirs(os.path.dirname(os.path.abspath(out)) or ".", exist_ok=True) + df.to_csv(out, index=False) + print(f"wrote {out} ({len(runs)} runs, {len(df)} targets)", file=sys.stderr) + print(df.to_string(index=False)) + + +if __name__ == "__main__": + main() From 795eccd66c5d7263f1713fea7f8b91dd54b9ee3f Mon Sep 17 00:00:00 2001 From: suachong Date: Tue, 14 Jul 2026 03:33:13 +0000 Subject: [PATCH 116/127] Move steps-to-target CV script under scripts/cv/. Relocates the convergence CV tool to recommendation_v4/scripts/cv/ and updates usage examples in the docstring. Co-authored-by: Cursor --- recommendation_v4/{ => scripts}/cv/steps_to_target_cv.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) rename recommendation_v4/{ => scripts}/cv/steps_to_target_cv.py (94%) diff --git a/recommendation_v4/cv/steps_to_target_cv.py b/recommendation_v4/scripts/cv/steps_to_target_cv.py similarity index 94% rename from recommendation_v4/cv/steps_to_target_cv.py rename to recommendation_v4/scripts/cv/steps_to_target_cv.py index 9b6c8accf..465c48fd4 100644 --- a/recommendation_v4/cv/steps_to_target_cv.py +++ b/recommendation_v4/scripts/cv/steps_to_target_cv.py @@ -10,16 +10,16 @@ Examples:: # auto sweep 0.60 .. max-AUC @ step 0.005 - python3 cv/steps_to_target_cv.py --log-dir /path/to/logs -o cv/table.csv + python3 scripts/cv/steps_to_target_cv.py --log-dir /path/to/logs -o cv/table.csv # single target - python3 cv/steps_to_target_cv.py --log-dir /path/to/logs --target 0.75 + python3 scripts/cv/steps_to_target_cv.py --log-dir /path/to/logs --target 0.75 # explicit list - python3 cv/steps_to_target_cv.py --log-dir /path/to/logs --targets 0.70,0.75,0.78 + python3 scripts/cv/steps_to_target_cv.py --log-dir /path/to/logs --targets 0.70,0.75,0.78 # range with step - python3 cv/steps_to_target_cv.py --log-dir /path/to/logs --target-range 0.60:0.78 --step 0.005 + python3 scripts/cv/steps_to_target_cv.py --log-dir /path/to/logs --target-range 0.60:0.78 --step 0.005 """ import argparse import glob From 2ef046508046615b59aa9b209f8ce7154ec68b18 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Wed, 15 Jul 2026 00:51:58 +0000 Subject: [PATCH 117/127] dlrmv3 train: gin/env-configurable LR warmup (default OFF) Linearly ramp LR from 0 -> base over the first N global steps, then hold at base. Scales both LR channels: the dense/non-fused optimizer and the in-backward fused sparse-embedding optimizer, pushing the warmed LR into the FBGEMM TBE backward kernel so the embedding backward sees the ramped value in the same step. Keyed off the checkpoint-restored global_step, so the ramp is resume-safe (mid-warmup resumes at the correct multiplier; past-warmup is a no-op). Controlled by $LR_WARMUP_STEPS -> gin lr_warmup_steps; default 0 = OFF, byte-identical to the pre-warmup baseline (base LRs are never mutated). Adds LR_WARMUP_STEPS passthrough in launch_slurm.sh and a detailed config-doc block in yambda_5b.gin. Co-authored-by: Cursor --- .../dlrm_v3/train/gin/yambda_5b.gin | 48 ++++++++++ .../dlrm_v3/train/utils.py | 95 +++++++++++++++++++ recommendation_v4/scripts/launch_slurm.sh | 1 + 3 files changed, 144 insertions(+) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index ef127ef7a..6c2ce6c71 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -121,6 +121,54 @@ streaming_train_eval_loop.grad_clip_norm = @gcn/env_float() gcn/env_float.key = "GRAD_CLIP_NORM" gcn/env_float.default = 1.0 +# ============================================================================ +# LR WARMUP (streaming train/eval path) +# ============================================================================ +# WHAT IT DOES +# Linearly ramps the learning rate from 0 up to the configured base LR over +# the first N GLOBAL train steps, then holds flat at the base LR for the rest +# of the run: +# +# effective_lr(gstep) = base_lr * min(1.0, gstep / N) +# +# The ramp scales BOTH learning-rate channels in lockstep: +# 1. the dense / non-fused optimizer (the `optimizer` passed to the loop), and +# 2. the in-backward FUSED sparse-embedding optimizer (model.fused_optimizer), +# i.e. the per-table embedding LR that DOMINATES convergence for this +# model. The warmed LR is also pushed into the FBGEMM TBE backward kernel +# (via set_learning_rate), so the embedding backward actually sees the +# ramped value in the SAME step (not one step late). +# +# WHY IT EXISTS +# Warmup tames the early-step trajectory divergence that a large LR causes on +# freshly-initialized weights/embeddings. That early divergence is a major +# source of run-to-run variance (CV). Warmup lets us use a higher base LR (or +# stretch the gentle early-rise region) without paying the early-instability +# penalty. It does NOT reduce plateau-region CV (that is set by data ordering). +# +# RESUME SAFETY +# The ramp is keyed off the monotonic, checkpoint-restored global_step, so it +# is identical on a cold start and on every relaunch: +# - a run that crashes mid-warmup resumes at the correct multiplier; +# - a run already past step N is a pure no-op (LRs already hold base). +# +# HOW TO ENABLE / DISABLE +# Controlled by the $LR_WARMUP_STEPS environment variable (read at startup): +# - DISABLED (default): unset $LR_WARMUP_STEPS, or set it to 0. +# => warmup code early-returns; base LRs are NEVER mutated; behavior is +# byte-identical to the pre-warmup baseline. This is the default so +# existing/baseline runs are unaffected. +# - ENABLED: export LR_WARMUP_STEPS= (N = number of global steps to ramp) +# e.g. export LR_WARMUP_STEPS=1500 # short warmup (~0.5% of data @ GB=8192) +# export LR_WARMUP_STEPS=16000 # long warmup (~5.7% of data @ GB=8192) +# Rule of thumb: data fraction spent ramping = N * global_batch / total_train. +# +# NOTE: this only changes the LR SCHEDULE. The base LR itself is set separately +# via DENSE_LR / SPARSE_LR; warmup never overrides those base values. +streaming_train_eval_loop.lr_warmup_steps = @lrw/env_int() +lrw/env_int.key = "LR_WARMUP_STEPS" +lrw/env_int.default = 0 # 0 = OFF (disabled by default; baseline-safe) + # Data root: resolved at runtime from $DLRM_DATA_PATH if set, else the literal # below. Used by both make_train_test_dataloaders and get_dataset. # Scoped (`data/env_path`) so this binding doesn't collide with the RUN_NAME diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py index 762afc101..124b196c9 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -2007,6 +2007,11 @@ def streaming_train_eval_loop( # --- gradient clipping (streaming path, dense params). 0.0 = OFF, which # preserves legacy streaming behavior. Wired to $GRAD_CLIP_NORM via gin. --- grad_clip_norm: float = 0.0, + # --- LR warmup (streaming path). Linearly ramp LR 0 -> base over the first + # N GLOBAL train steps, then hold. Scales BOTH the dense optimizer and + # the in-backward fused sparse-embedding optimizer. 0 = OFF (byte- + # identical). Wired to $LR_WARMUP_STEPS via gin. --- + lr_warmup_steps: int = 0, # --- diagnostic: log per-batch unique/total embedding-id counts --- streaming_diag_unique_emb: bool = False, # --- test-only failure injection knob --- @@ -2340,6 +2345,90 @@ def _save_mid_window(train_ts: int, batch_idx_in_window: int) -> None: split_contract=live_split_contract, ) + # --- LR warmup (streaming path) ------------------------------------------ + # Linearly ramp LR from 0 -> base over the first `lr_warmup_steps` GLOBAL + # train steps, then hold at base. Keyed off the monotonic, checkpoint- + # restored global_step so the ramp is identical on cold start and on every + # resume (a run that crashes mid-warmup resumes at the correct multiplier; + # a run already past warmup is a no-op). Scales BOTH LR channels: the + # dense/non-fused optimizer passed in as `optimizer`, AND the in-backward + # fused sparse-embedding optimizer (model.fused_optimizer), whose per-table + # LR dominates this model. `lr_warmup_steps=0` (default) => OFF and the base + # LRs are never touched (byte-identical to the pre-warmup path). + _warmup_groups = list(optimizer.param_groups) + _warmup_fused_opt = getattr(model, "fused_optimizer", None) + if _warmup_fused_opt is not None: + _warmup_groups += list(_warmup_fused_opt.param_groups) + _warmup_base_lrs = [pg["lr"] for pg in _warmup_groups] + if lr_warmup_steps and lr_warmup_steps > 0 and rank == 0: + logger.info( + "[lr-warmup] linear warmup over %d global steps across %d param " + "group(s); base LRs: %s", + lr_warmup_steps, + len(_warmup_groups), + ", ".join(f"{b:.3e}" for b in _warmup_base_lrs), + ) + + def _apply_lr_warmup(gstep: int) -> None: + # OFF, or ramp already finished (kernel + param_groups already hold the + # base LR from the last sync at gstep==lr_warmup_steps): nothing to do. + if not (lr_warmup_steps and lr_warmup_steps > 0) or gstep > lr_warmup_steps: + return + mult = min(1.0, float(gstep) / float(lr_warmup_steps)) + for pg, base in zip(_warmup_groups, _warmup_base_lrs): + pg["lr"] = base * mult + # Push the warmed LR into the FBGEMM TBE backward kernel. The in-backward + # fused embedding optimizer only calls + # `emb_module.set_learning_rate(param_groups[0]["lr"])` from its + # step()/zero_grad() (torchrec/distributed/batched_embedding_kernel.py), + # and this loop steps the dense `optimizer` — NOT model.fused_optimizer — + # so WITHOUT this explicit call the embedding tables would keep training + # at the construction-time base LR and warmup would silently apply to the + # dense params only. `.step()` on the fused (in-backward) optimizer does + # no parameter update; it only re-syncs the LR to the kernel. + if _warmup_fused_opt is not None: + _warmup_fused_opt.step() + if rank == 0 and gstep % max(1, metric_log_frequency) == 0: + _fused_lr = "" + if _warmup_fused_opt is not None: + _fused_lr = ( + f" fused_pg_lr={_warmup_fused_opt.param_groups[0]['lr']:.3e}" + ) + # Best-effort readback of the ACTUAL kernel LR to confirm + # set_learning_rate() propagated. model.fused_optimizer is a + # (possibly nested) CombinedOptimizer whose leaves carry the TBE + # module as `_emb_module`, so recurse to the first leaf. Purely + # diagnostic; wrapped so it can never break training. + def _first_tbe_kernel_lr(opt, depth=0): + if depth > 6 or opt is None: + return None + _em = getattr(opt, "_emb_module", None) + if _em is not None and hasattr(_em, "get_learning_rate"): + return float(_em.get_learning_rate()) + for _sub in getattr(opt, "_optims", []): + _child = _sub[1] if isinstance(_sub, tuple) else _sub + _lr = _first_tbe_kernel_lr(_child, depth + 1) + if _lr is not None: + return _lr + _wrapped = getattr(opt, "_optimizer", None) + if _wrapped is not None: + return _first_tbe_kernel_lr(_wrapped, depth + 1) + return None + + try: + _klr = _first_tbe_kernel_lr(_warmup_fused_opt) + if _klr is not None: + _fused_lr += f" tbe_kernel_lr={_klr:.3e}" + except Exception: + pass + logger.info( + "[lr-warmup] gstep=%d mult=%.4f dense_lr=%.3e%s", + gstep, + mult, + _warmup_groups[0]["lr"], + _fused_lr, + ) + def _run_train_window( train_data_iterator, train_ts: int, @@ -2373,6 +2462,12 @@ def _run_train_window( max_steps=int(os.environ.get("DIAG_EMB_STEPS", "100")), log_every=metric_log_frequency, ) + # LR warmup: set the scaled LR on BOTH channels (dense param_groups + # + the fused embedding TBE via set_learning_rate) BEFORE the + # forward/backward, so the in-backward embedding update THIS step + # uses the warmed LR (the dense optimizer reads it at .step() below). + # No-op when lr_warmup_steps=0. + _apply_lr_warmup(metric_logger.global_step["train"]) optimizer.zero_grad() sample.to(device) ( diff --git a/recommendation_v4/scripts/launch_slurm.sh b/recommendation_v4/scripts/launch_slurm.sh index d3bc4e188..5659c3014 100755 --- a/recommendation_v4/scripts/launch_slurm.sh +++ b/recommendation_v4/scripts/launch_slurm.sh @@ -305,6 +305,7 @@ orchestrate() { ${DENSE_LR:+-e DENSE_LR=$DENSE_LR} \ ${SPARSE_LR:+-e SPARSE_LR=$SPARSE_LR} \ ${GRAD_CLIP_NORM:+-e GRAD_CLIP_NORM=$GRAD_CLIP_NORM} \ + ${LR_WARMUP_STEPS:+-e LR_WARMUP_STEPS=$LR_WARMUP_STEPS} \ ${HSTU_NUM_LAYERS:+-e HSTU_NUM_LAYERS=$HSTU_NUM_LAYERS} \ ${MAX_SEQ_LEN:+-e MAX_SEQ_LEN=$MAX_SEQ_LEN} \ ${HISTORY_LENGTH:+-e HISTORY_LENGTH=$HISTORY_LENGTH} \ From 4baaa838528e32d25b1dd981a54590151506285c Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Thu, 16 Jul 2026 04:42:54 +0000 Subject: [PATCH 118/127] dlrmv3 train: configurable LR-warmup starting LR (default 0) Add $LR_WARMUP_START_LR (gin: streaming_train_eval_loop.lr_warmup_start_lr) so the warmup ramp can start from a nonzero floor instead of 0, i.e. ramp start -> base over LR_WARMUP_STEPS: effective_lr(gstep) = start + (base - start) * min(1, gstep / N) start is applied per param group, clamped to <= that group's base LR, and propagates to the dense optimizer, the fused sparse-embedding optimizer, and the FBGEMM TBE kernel (same path as the existing ramp). Default 0.0 collapses to the original 0 -> base schedule and is byte-identical to the pre-change behavior; only meaningful when LR_WARMUP_STEPS > 0. Motivation: a long warmup spends its first several thousand steps at a near-zero LR that barely trains; a small floor reclaims that dead head of the ramp to reach a target AUC faster, at the cost of some early seed-divergence damping (keep start << base to stay in the low-CV regime). Validated on a single open-pool node (warmup=500, start=2.5e-8=base/4): ramp starts at 2.5e-8 at gstep 0 and rises linearly to base, confirmed on dense, fused, and TBE kernel LR readbacks. Co-authored-by: Cursor --- .../dlrm_v3/train/gin/yambda_5b.gin | 35 +++++++++++++++++++ .../dlrm_v3/train/utils.py | 30 ++++++++++++---- recommendation_v4/scripts/launch_slurm.sh | 1 + 3 files changed, 60 insertions(+), 6 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index 6c2ce6c71..deacd7e49 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -169,6 +169,41 @@ streaming_train_eval_loop.lr_warmup_steps = @lrw/env_int() lrw/env_int.key = "LR_WARMUP_STEPS" lrw/env_int.default = 0 # 0 = OFF (disabled by default; baseline-safe) +# ---------------------------------------------------------------------------- +# LR WARMUP FLOOR / STARTING LR (only meaningful when $LR_WARMUP_STEPS > 0) +# ---------------------------------------------------------------------------- +# WHAT IT DOES +# Sets the LR the ramp STARTS from at global step 0 instead of starting from +# zero, turning the schedule into a "partial warmup" that ramps from `start` +# up to the base LR over $LR_WARMUP_STEPS, then holds flat: +# +# effective_lr(gstep) = start + (base_lr - start) * min(1.0, gstep / N) +# +# `start` is applied PER param group and clamped to <= that group's base LR +# (a floor >= base means "no ramp" for that group -> it starts at base). Like +# the main warmup, the floor scales both the dense optimizer and the fused +# sparse-embedding optimizer, and is pushed into the FBGEMM TBE kernel. +# +# WHY IT EXISTS +# With a long warmup the first several thousand steps run at a near-zero LR +# (e.g. at gstep 1k of a 24k ramp, LR ~= base/24 ~= 4e-9) and barely train. +# A small nonzero floor reclaims that near-dead head of the ramp, banking +# extra effective learning (integral rises from 1/2*N*base to +# 1/2*N*(base+start)) to reach a target AUC in fewer steps. The trade-off is +# that a higher floor reintroduces some early seed-divergence, so it costs +# some of warmup's CV-reduction benefit; keep `start` << base to stay in the +# low-CV regime. +# +# HOW TO ENABLE / DISABLE +# Controlled by the $LR_WARMUP_START_LR environment variable (read at startup): +# - DEFAULT 0.0: classic 0 -> base ramp; byte-identical to plain warmup. +# - NONZERO: export LR_WARMUP_START_LR= (must be < base LR) +# e.g. at base=1e-7: 1.25e-8 (= base/8) 2.5e-8 (= base/4) +# No effect unless $LR_WARMUP_STEPS > 0 (there is no ramp to lift a floor on). +streaming_train_eval_loop.lr_warmup_start_lr = @lrws/env_float() +lrws/env_float.key = "LR_WARMUP_START_LR" +lrws/env_float.default = 0.0 # 0.0 = classic 0 -> base ramp (baseline-safe) + # Data root: resolved at runtime from $DLRM_DATA_PATH if set, else the literal # below. Used by both make_train_test_dataloaders and get_dataset. # Scoped (`data/env_path`) so this binding doesn't collide with the RUN_NAME diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py index 124b196c9..9679a30f0 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -2012,6 +2012,15 @@ def streaming_train_eval_loop( # the in-backward fused sparse-embedding optimizer. 0 = OFF (byte- # identical). Wired to $LR_WARMUP_STEPS via gin. --- lr_warmup_steps: int = 0, + # --- LR warmup floor (streaming path). Absolute LR the ramp STARTS from at + # gstep 0 (per group, clamped to <= that group's base LR). The ramp then + # goes start -> base over `lr_warmup_steps`. 0.0 (default) reproduces the + # 0 -> base ramp exactly (byte-identical). A small nonzero floor turns the + # warmup into a "partial warmup" that reclaims the near-dead head of the + # ramp (trading a little seed-divergence damping for faster time-to-target). + # Only has any effect when lr_warmup_steps > 0. Wired to + # $LR_WARMUP_START_LR via gin. --- + lr_warmup_start_lr: float = 0.0, # --- diagnostic: log per-batch unique/total embedding-id counts --- streaming_diag_unique_emb: bool = False, # --- test-only failure injection knob --- @@ -2360,12 +2369,17 @@ def _save_mid_window(train_ts: int, batch_idx_in_window: int) -> None: if _warmup_fused_opt is not None: _warmup_groups += list(_warmup_fused_opt.param_groups) _warmup_base_lrs = [pg["lr"] for pg in _warmup_groups] + # Per-group start LR for the ramp: the configured floor, clamped so it can + # never exceed a group's base (a floor >= base would mean "no ramp" for that + # group, which we express by starting it AT base). 0.0 => classic 0 -> base. + _warmup_start_lrs = [min(float(lr_warmup_start_lr), b) for b in _warmup_base_lrs] if lr_warmup_steps and lr_warmup_steps > 0 and rank == 0: logger.info( "[lr-warmup] linear warmup over %d global steps across %d param " - "group(s); base LRs: %s", + "group(s); start LRs: %s -> base LRs: %s", lr_warmup_steps, len(_warmup_groups), + ", ".join(f"{s:.3e}" for s in _warmup_start_lrs), ", ".join(f"{b:.3e}" for b in _warmup_base_lrs), ) @@ -2374,9 +2388,13 @@ def _apply_lr_warmup(gstep: int) -> None: # base LR from the last sync at gstep==lr_warmup_steps): nothing to do. if not (lr_warmup_steps and lr_warmup_steps > 0) or gstep > lr_warmup_steps: return - mult = min(1.0, float(gstep) / float(lr_warmup_steps)) - for pg, base in zip(_warmup_groups, _warmup_base_lrs): - pg["lr"] = base * mult + progress = min(1.0, float(gstep) / float(lr_warmup_steps)) + # Ramp each group from its (clamped) start LR up to its base LR. With + # start=0 this collapses to `base * progress` (the original schedule). + for pg, base, start in zip( + _warmup_groups, _warmup_base_lrs, _warmup_start_lrs + ): + pg["lr"] = start + (base - start) * progress # Push the warmed LR into the FBGEMM TBE backward kernel. The in-backward # fused embedding optimizer only calls # `emb_module.set_learning_rate(param_groups[0]["lr"])` from its @@ -2422,9 +2440,9 @@ def _first_tbe_kernel_lr(opt, depth=0): except Exception: pass logger.info( - "[lr-warmup] gstep=%d mult=%.4f dense_lr=%.3e%s", + "[lr-warmup] gstep=%d progress=%.4f dense_lr=%.3e%s", gstep, - mult, + progress, _warmup_groups[0]["lr"], _fused_lr, ) diff --git a/recommendation_v4/scripts/launch_slurm.sh b/recommendation_v4/scripts/launch_slurm.sh index 5659c3014..c75f5356a 100755 --- a/recommendation_v4/scripts/launch_slurm.sh +++ b/recommendation_v4/scripts/launch_slurm.sh @@ -306,6 +306,7 @@ orchestrate() { ${SPARSE_LR:+-e SPARSE_LR=$SPARSE_LR} \ ${GRAD_CLIP_NORM:+-e GRAD_CLIP_NORM=$GRAD_CLIP_NORM} \ ${LR_WARMUP_STEPS:+-e LR_WARMUP_STEPS=$LR_WARMUP_STEPS} \ + ${LR_WARMUP_START_LR:+-e LR_WARMUP_START_LR=$LR_WARMUP_START_LR} \ ${HSTU_NUM_LAYERS:+-e HSTU_NUM_LAYERS=$HSTU_NUM_LAYERS} \ ${MAX_SEQ_LEN:+-e MAX_SEQ_LEN=$MAX_SEQ_LEN} \ ${HISTORY_LENGTH:+-e HISTORY_LENGTH=$HISTORY_LENGTH} \ From a40f3fd8b3218565c8e9bac677ead289a76b8e08 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Fri, 24 Jul 2026 14:22:27 +0000 Subject: [PATCH 119/127] dlrmv3 train: gate interim eval metric logging (default OFF, AUC-neutral) During each streaming eval pass, _run_eval_window called compute_and_log(mode= "eval") every METRIC_LOG_FREQ batches, producing partial-holdout AUC/NE numbers that were only logged to TB/console. The durable metrics.jsonl, early-stop, and MLPerf EVAL_ACCURACY all use only the single end-of-pass compute() over the whole holdout, so those interim calls were redundant BinnedCumulativeAUC histogram all-reduces + GPU->CPU syncs (x window_ and lifetime_ sets), repeated ~200 eval passes/run. Gate them behind a new $EVAL_INTERIM_LOG env (streaming_train_eval_loop. eval_interim_metrics), default 0 = OFF. Wired through launch_slurm.sh's worker docker-exec passthrough. AUC-metric neutral by construction: per-batch update() runs unconditionally, and the reported AUC is the end-of-pass compute() (a pure function of the accumulated histogram that compute() does not mutate); the gated interim calls only READ that state. Verified e2e (SEED=1, same node, interim actually firing when ON): OFF-vs-ON max window/lifetime AUC |diff| = 2.4e-6, below the OFF-vs-OFF control noise floor of 3.3e-6 (run-to-run GPU FP nondeterminism). Co-authored-by: Cursor --- .../dlrm_v3/train/gin/yambda_5b.gin | 27 +++++++++++++++++++ .../dlrm_v3/train/utils.py | 9 ++++++- recommendation_v4/scripts/launch_slurm.sh | 1 + 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index deacd7e49..7ecd5d823 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -584,6 +584,33 @@ sts/env_int.default = 0 streaming_train_eval_loop.metric_log_frequency = @mlf/env_int() mlf/env_int.key = "METRIC_LOG_FREQ" mlf/env_int.default = 50 +# Interim eval logging: when >0, log partial-holdout metrics every +# METRIC_LOG_FREQ batches DURING an eval pass (debug/progress only). The durable +# .metrics.jsonl, early-stop, and MLPerf EVAL_ACCURACY all use ONLY the +# end-of-pass whole-holdout compute(), so these interim calls are redundant AUC +# reductions (a BinnedCumulativeAUC histogram all-reduce + GPU->CPU sync per +# call, x window_ and lifetime_ sets). 0 (default) = OFF -> skip them. +# +# *** AUC-METRIC NEUTRAL — toggling this NEVER changes the reported eval AUC. *** +# WHY (by construction): every eval batch's predictions/labels are folded into +# the metric state via metric_logger.update() UNCONDITIONALLY, once per batch, +# regardless of this flag. The reported eval AUC is the SINGLE end-of-pass +# metric_logger.compute(mode="eval") over the whole holdout, which is a pure +# function of that accumulated state (BinnedCumulativeAUC integrates its score +# histogram; compute() does NOT mutate the histogram). The interim +# compute_and_log() calls this flag gates only READ that same state to print a +# running partial-holdout number — they cannot alter what update() accumulated, +# so the final AUC is identical whether they fire 0 or N times. This is why the +# flag is safe to leave OFF by default (pure logging removal, not a metric change). +# VERIFIED e2e (SEED=1, same node, 2 windows, interim actually firing when ON): +# OFF-vs-ON max window/lifetime AUC |diff| = 2.4e-6 +# OFF-vs-OFF control (identical runs) max AUC |diff| = 3.3e-6 +# i.e. the flag's effect on AUC is BELOW the run-to-run GPU FP nondeterminism +# floor (the ~1e-6 residual is nondeterministic training kernels, not this flag). +# Override via $EVAL_INTERIM_LOG. +streaming_train_eval_loop.eval_interim_metrics = @eil/env_int() +eil/env_int.key = "EVAL_INTERIM_LOG" +eil/env_int.default = 0 # MLPerf train_loss event cadence (global train steps), INDEPENDENT of # METRIC_LOG_FREQ above. 0 (default) = fall back to METRIC_LOG_FREQ, preserving # the prior coupled behavior. Set $MLPERF_TRAIN_LOSS_LOG_FREQ>0 to log the MLPerf diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py index 9679a30f0..2b85722a7 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -1969,6 +1969,13 @@ def streaming_train_eval_loop( num_eval_batches: Optional[int] = None, output_trace: bool = False, metric_log_frequency: int = 1, + # Interim eval logging cadence gate. 0 (default) = OFF: skip the per-batch + # compute_and_log DURING an eval pass. Those interim calls only log + # partial-holdout metrics (progress/debug); the value we keep -- the durable + # metrics.jsonl, early-stop, and MLPerf EVAL_ACCURACY -- is the end-of-pass + # compute() over the whole holdout, so OFF is numerically neutral for the + # recorded eval AUC. >0 restores the old interim logging. $EVAL_INTERIM_LOG. + eval_interim_metrics: int = 0, # MLPerf `train_loss` event cadence, in global train steps, INDEPENDENT of # metric_log_frequency (the console/TB cadence). 0 = fall back to # metric_log_frequency (preserves prior coupled behavior). Wired to @@ -2691,7 +2698,7 @@ def _run_eval_window( if output_trace: assert profiler is not None profiler.step() - if eval_batch_idx % metric_log_frequency == 0: + if eval_interim_metrics and eval_batch_idx % metric_log_frequency == 0: metric_logger.compute_and_log(mode="eval") if num_eval_batches is not None and eval_batch_idx >= num_eval_batches: break diff --git a/recommendation_v4/scripts/launch_slurm.sh b/recommendation_v4/scripts/launch_slurm.sh index c75f5356a..8fd21a519 100755 --- a/recommendation_v4/scripts/launch_slurm.sh +++ b/recommendation_v4/scripts/launch_slurm.sh @@ -291,6 +291,7 @@ orchestrate() { -e NUM_EVAL_BATCHES=$NUM_EVAL_BATCHES \ ${DIE_AT_STEP:+-e DIE_AT_STEP=$DIE_AT_STEP} \ -e METRIC_LOG_FREQ=$METRIC_LOG_FREQ \ + ${EVAL_INTERIM_LOG:+-e EVAL_INTERIM_LOG=$EVAL_INTERIM_LOG} \ ${MLPERF_LOGGING:+-e MLPERF_LOGGING=$MLPERF_LOGGING} \ ${MLPERF_TRAIN_LOSS_LOG_FREQ:+-e MLPERF_TRAIN_LOSS_LOG_FREQ=$MLPERF_TRAIN_LOSS_LOG_FREQ} \ ${STREAMING_SHUFFLE_FRACTION:+-e STREAMING_SHUFFLE_FRACTION=$STREAMING_SHUFFLE_FRACTION} \ From ac89f824bce5013e42dee92d0f13e46edc100a67 Mon Sep 17 00:00:00 2001 From: suachong Date: Fri, 24 Jul 2026 15:32:52 -0500 Subject: [PATCH 120/127] dlrmv4: add RCP reference logs by global batch size Add MLPerf convergence logs under recommendation_v4/rcp_logs, grouped by global batch size: gbs_8192 (1 node), gbs_16384 (2 node), gbs_32768 (4 node), 10 seeds each. Negation .gitignore keeps them tracked despite the *.log ignore. Co-authored-by: Cursor --- recommendation_v4/rcp_logs/.gitignore | 2 + .../rcp_logs/gbs_16384/seed149983452.log | 820 +++++++ .../rcp_logs/gbs_16384/seed151156919.log | 799 +++++++ .../rcp_logs/gbs_16384/seed169450858.log | 1378 ++++++++++++ .../rcp_logs/gbs_16384/seed295346581.log | 1388 ++++++++++++ .../rcp_logs/gbs_16384/seed462193368.log | 1380 ++++++++++++ .../rcp_logs/gbs_16384/seed534232611.log | 825 +++++++ .../rcp_logs/gbs_16384/seed574397838.log | 1333 ++++++++++++ .../rcp_logs/gbs_16384/seed803889169.log | 1353 ++++++++++++ .../rcp_logs/gbs_16384/seed869903124.log | 800 +++++++ .../rcp_logs/gbs_16384/seed951784539.log | 813 +++++++ .../rcp_logs/gbs_32768/seed1056795887.log | 670 ++++++ .../rcp_logs/gbs_32768/seed147670635.log | 662 ++++++ .../rcp_logs/gbs_32768/seed162052595.log | 685 ++++++ .../rcp_logs/gbs_32768/seed182111600.log | 1410 ++++++++++++ .../rcp_logs/gbs_32768/seed412282151.log | 1427 ++++++++++++ .../rcp_logs/gbs_32768/seed430583489.log | 1333 ++++++++++++ .../rcp_logs/gbs_32768/seed451547632.log | 888 ++++++++ .../rcp_logs/gbs_32768/seed480431706.log | 1362 ++++++++++++ .../rcp_logs/gbs_32768/seed651579272.log | 691 ++++++ .../rcp_logs/gbs_32768/seed813474716.log | 649 ++++++ .../rcp_logs/gbs_8192/seed380229711.log | 1869 ++++++++++++++++ .../rcp_logs/gbs_8192/seed402905276.log | 1853 ++++++++++++++++ .../rcp_logs/gbs_8192/seed404908753.log | 1855 ++++++++++++++++ .../rcp_logs/gbs_8192/seed445449777.log | 1830 ++++++++++++++++ .../rcp_logs/gbs_8192/seed537267118.log | 1870 ++++++++++++++++ .../rcp_logs/gbs_8192/seed552965065.log | 1815 ++++++++++++++++ .../rcp_logs/gbs_8192/seed608892790.log | 1862 ++++++++++++++++ .../rcp_logs/gbs_8192/seed666410936.log | 1818 ++++++++++++++++ .../rcp_logs/gbs_8192/seed681082816.log | 1916 +++++++++++++++++ .../rcp_logs/gbs_8192/seed914924190.log | 1840 ++++++++++++++++ 31 files changed, 39196 insertions(+) create mode 100644 recommendation_v4/rcp_logs/.gitignore create mode 100644 recommendation_v4/rcp_logs/gbs_16384/seed149983452.log create mode 100644 recommendation_v4/rcp_logs/gbs_16384/seed151156919.log create mode 100644 recommendation_v4/rcp_logs/gbs_16384/seed169450858.log create mode 100644 recommendation_v4/rcp_logs/gbs_16384/seed295346581.log create mode 100644 recommendation_v4/rcp_logs/gbs_16384/seed462193368.log create mode 100644 recommendation_v4/rcp_logs/gbs_16384/seed534232611.log create mode 100644 recommendation_v4/rcp_logs/gbs_16384/seed574397838.log create mode 100644 recommendation_v4/rcp_logs/gbs_16384/seed803889169.log create mode 100644 recommendation_v4/rcp_logs/gbs_16384/seed869903124.log create mode 100644 recommendation_v4/rcp_logs/gbs_16384/seed951784539.log create mode 100644 recommendation_v4/rcp_logs/gbs_32768/seed1056795887.log create mode 100644 recommendation_v4/rcp_logs/gbs_32768/seed147670635.log create mode 100644 recommendation_v4/rcp_logs/gbs_32768/seed162052595.log create mode 100644 recommendation_v4/rcp_logs/gbs_32768/seed182111600.log create mode 100644 recommendation_v4/rcp_logs/gbs_32768/seed412282151.log create mode 100644 recommendation_v4/rcp_logs/gbs_32768/seed430583489.log create mode 100644 recommendation_v4/rcp_logs/gbs_32768/seed451547632.log create mode 100644 recommendation_v4/rcp_logs/gbs_32768/seed480431706.log create mode 100644 recommendation_v4/rcp_logs/gbs_32768/seed651579272.log create mode 100644 recommendation_v4/rcp_logs/gbs_32768/seed813474716.log create mode 100644 recommendation_v4/rcp_logs/gbs_8192/seed380229711.log create mode 100644 recommendation_v4/rcp_logs/gbs_8192/seed402905276.log create mode 100644 recommendation_v4/rcp_logs/gbs_8192/seed404908753.log create mode 100644 recommendation_v4/rcp_logs/gbs_8192/seed445449777.log create mode 100644 recommendation_v4/rcp_logs/gbs_8192/seed537267118.log create mode 100644 recommendation_v4/rcp_logs/gbs_8192/seed552965065.log create mode 100644 recommendation_v4/rcp_logs/gbs_8192/seed608892790.log create mode 100644 recommendation_v4/rcp_logs/gbs_8192/seed666410936.log create mode 100644 recommendation_v4/rcp_logs/gbs_8192/seed681082816.log create mode 100644 recommendation_v4/rcp_logs/gbs_8192/seed914924190.log diff --git a/recommendation_v4/rcp_logs/.gitignore b/recommendation_v4/rcp_logs/.gitignore new file mode 100644 index 000000000..348996cfc --- /dev/null +++ b/recommendation_v4/rcp_logs/.gitignore @@ -0,0 +1,2 @@ +# RCP reference logs are intentionally tracked despite the parent *.log ignore +!*.log diff --git a/recommendation_v4/rcp_logs/gbs_16384/seed149983452.log b/recommendation_v4/rcp_logs/gbs_16384/seed149983452.log new file mode 100644 index 000000000..9ebb43ac9 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_16384/seed149983452.log @@ -0,0 +1,820 @@ +:::MLLOG {"namespace": "", "time_ms": 1784867221561, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784867221562, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784867241789, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784867241799, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784867241799, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784867241799, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784867241799, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784867241799, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 16384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784867241799, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784867241799, "event_type": "POINT_IN_TIME", "key": "seed", "value": 149983452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784867241799, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784867241799, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784867241799, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784867241799, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784867241812, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784867241812, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784868083329, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784868083330, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784868083648, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784868260319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13772815465927124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.5833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784868273993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13815942406654358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2499999999999997e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784868287657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13840842247009277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784868301553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386566162109375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.583333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784868315530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387680172920227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.25e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784868329366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13854938745498657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784868343125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13837754726409912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868343125, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784868343126, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784868388379, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5001096725463867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784868388380, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784868388380, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784868402017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381252110004425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868415677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381990760564804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868429607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382634937763214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868443628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13821136951446533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868457397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382584571838379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9916666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868471683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13782069087028503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868486160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13727018237113953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868486161, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784868486161, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784868529410, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5002773404121399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784868529411, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784868529411, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784868543340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13810822367668152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868557589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13767564296722412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868571490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13777995109558105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868585824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13760356605052948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.9916666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868599498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379493921995163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868613837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380046308040619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868627967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13732922077178955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868627967, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784868627968, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784868670621, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5004666447639465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784868670621, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784868670622, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784868684629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13829371333122253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868699219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13753271102905273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.8249999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868713259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368006467819214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868727463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13794606924057007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868742323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13732090592384338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3249999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868756414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13691341876983643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868770881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13711164891719818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868770882, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784868770882, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784868812764, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5007104277610779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784868812765, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784868812765, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784868826379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13634498417377472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868840238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13738033175468445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868854551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13714037835597992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1583333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868868395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366560161113739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868882159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351199448108673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868896121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13666412234306335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868910218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13538694381713867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868910219, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784868910219, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784868953291, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5010009407997131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784868953292, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784868953292, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784868966990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362731158733368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868980753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13598227500915527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868994947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13727161288261414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869009156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.136247456073761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869022952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13448652625083923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.658333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869036706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13500705361366272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869050655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13745510578155518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869050655, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784869050656, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784869094686, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5013360381126404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784869094687, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784869094687, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784869108984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13668543100357056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869123165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352468729019165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869136780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13553249835968018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869150689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13454163074493408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869164788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13462460041046143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869179469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13426440954208374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869193884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13465335965156555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869193884, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784869193884, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784869236578, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5017980337142944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784869236578, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784869236579, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784869251147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336870938539505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869265704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363849937915802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869280592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13813598453998566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869294751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13360777497291565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.824999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869309198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13585391640663147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869323367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346030831336975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.158333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869338114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13655516505241394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.324999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869338115, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784869338115, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784869382642, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5024309754371643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784869382642, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784869382642, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784869397331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13633349537849426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869412000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366102695465088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869426577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13467179238796234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869441081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12979018688201904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869455536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13233448565006256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869469609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334487497806549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869483523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13486891984939575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869483577, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784869483578, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784869526963, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5031209588050842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784869526963, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784869526963, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784869541146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13501936197280884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869555274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12894396483898163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869570133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1309923678636551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869584341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13687478005886078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.1158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869598727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322268843650818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869612869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13117389380931854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869627518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324489414691925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869627518, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784869627519, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784869670861, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5038923621177673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784869670861, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784869670862, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784869684831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1316559910774231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869699636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13410931825637817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869713575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12776678800582886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869727600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12948453426361084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869741894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337880790233612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869756495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12821874022483826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869770527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297735869884491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869770528, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784869770528, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784869814656, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5049933791160583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784869814656, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784869814657, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784869829852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13469770550727844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869843805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330922544002533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.3158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869858192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13447514176368713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869872523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13492733240127563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869887325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13625238835811615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.3658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869901686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13364478945732117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869916268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13695815205574036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869916269, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784869916269, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784869960066, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5064151287078857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784869960066, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784869960066, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784869974731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13432876765727997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.4158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869989404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1305205523967743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870004077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338917464017868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.4491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870019011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14038683474063873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.4658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870033872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13039377331733704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870048858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13235123455524445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870063561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13820785284042358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870063562, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784870063562, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784870105457, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.507748007774353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784870105458, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784870105458, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784870120485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12402431666851044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870135269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311700940132141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870150277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13704606890678406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870165132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354977786540985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870180268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13239121437072754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870194643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12879443168640137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.6158333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870209043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12803059816360474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870209044, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784870209044, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784870251309, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5091164708137512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784870251309, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784870251309, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784870265820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301184892654419, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870280297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347193717956543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.6658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870295023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14224904775619507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6825000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870309432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13430684804916382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870324231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131514310836792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.7158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870338428, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13373112678527832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870352798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13598880171775818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870352799, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784870352799, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784870396007, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5107304453849792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784870396007, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784870396007, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784870410227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13470184803009033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870424440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13639642298221588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870438987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135719895362854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870453573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331528276205063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870468195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13431167602539062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870482971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13294729590415955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870497412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13246354460716248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8658333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870497413, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784870497414, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784870540706, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5124543905258179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784870540707, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784870540707, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784870555068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13177615404129028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870569490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344207525253296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8991666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870584122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1419132947921753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870598954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13436853885650635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870613620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13427215814590454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870628277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13228777050971985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870642589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312450051307678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870642590, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784870642590, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784870686825, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5145044922828674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784870686825, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784870686825, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784870700829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13844549655914307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870715065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13300520181655884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870729105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12877511978149414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870743261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13002096116542816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870757692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13793738186359406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870772318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13159775733947754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870786967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13604898750782013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870786967, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784870786968, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784870832213, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5168299674987793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784870832213, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784870832213, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784870846693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13672715425491333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.1158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870860945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363033652305603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870875481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13126802444458008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1491666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870889729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12996044754981995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870904345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1314932107925415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870919169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13532155752182007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870934042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326977014541626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870934042, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784870934043, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784870977418, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5195149779319763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784870977418, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784870977418, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784870991867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13604511320590973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871006909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13234862685203552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871022010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13791608810424805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871036659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13639384508132935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871051645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13855662941932678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871066427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13435132801532745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871081607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13917629420757294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871081608, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784871081608, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784871125590, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5223249197006226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784871125590, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784871125590, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784871140684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13494080305099487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871155757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14973768591880798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871171080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13510572910308838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871186117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330367624759674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871201469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13428595662117004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871216568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13315477967262268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871231839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365164965391159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871231839, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784871231840, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784871274811, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5255382061004639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784871274811, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784871274811, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784871289904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13624602556228638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871304958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351504623889923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871320095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13375455141067505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871334689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336391270160675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871349237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332518756389618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871363787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12945963442325592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871378203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12555891275405884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871378204, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784871378204, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784871421008, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5293525457382202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784871421009, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784871421009, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784871435507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13273680210113525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.5824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871450245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13252326846122742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871464960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13511598110198975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871479579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354249268770218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871494226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13672424852848053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871508945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12888166308403015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871523615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12985894083976746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871523616, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784871523616, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784871566554, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5349465012550354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784871566555, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784871566555, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784871581260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13249850273132324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.699166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871596079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12706145644187927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871610897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13315142691135406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871625909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13203096389770508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871641155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12697166204452515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871656578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13216593861579895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871671331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13422314822673798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871671331, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784871671332, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784871713717, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.54236900806427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784871713717, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784871713717, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784871728556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294289380311966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871743064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13034021854400635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871757345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13068878650665283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871771941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12990888953208923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.8658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871786168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12770181894302368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871800663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12336447089910507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871815446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12714539468288422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871815447, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784871815447, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784871857465, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5525143146514893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784871857465, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784871857466, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784871872230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13573843240737915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871887054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13331949710845947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871901289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1391291618347168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871916548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13364344835281372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871931616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12785980105400085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871946631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12964238226413727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871961841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12713074684143066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871961842, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784871961842, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784872004367, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5651109218597412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784872004368, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784872004368, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784872019272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289408802986145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872034294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12751975655555725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872049712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325790286064148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872065252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13282522559165955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872080799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13142193853855133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872095819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13382871448993683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872111131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13053035736083984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872111132, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784872111133, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784872153383, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5805420875549316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784872153384, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784872153384, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784872168178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297227293252945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872182941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12986285984516144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872197712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12978605926036835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872212501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132462739944458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872226909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13368993997573853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872241189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12948079407215118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872256247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13042142987251282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872256248, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784872256248, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784872298899, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5944309234619141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784872298900, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784872298900, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784872313707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1284046769142151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872328869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13082963228225708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872343322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12869814038276672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872358955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12781338393688202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872374529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12751121819019318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872390083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13185951113700867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872405750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13254693150520325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872405750, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784872405751, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784872448312, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6092762351036072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784872448312, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784872448312, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784872463484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13016945123672485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872478963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12808242440223694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872494856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304662823677063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872510695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13209816813468933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872526452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13339239358901978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872541714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12923569977283478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872557392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12887969613075256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.4991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872557393, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784872557393, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784872600796, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6244020462036133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784872600796, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784872600797, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784872615726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12976551055908203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.5158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872630833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12823563814163208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872646041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12930944561958313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.5491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872660927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13536153733730316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872675945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312510073184967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872691260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12624222040176392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872706785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12984615564346313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872706786, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784872706786, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784872751177, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6381945013999939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784872751178, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784872751178, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784872766215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12668848037719727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6325000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872781800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12811246514320374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872797025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12778359651565552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872812090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1274714171886444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872827029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1256883144378662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872842023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12291957437992096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872856922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12433032691478729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872856923, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784872856923, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784872899371, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6545944213867188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784872899372, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784872899372, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784872915036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12722235918045044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872930523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12354118376970291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872945655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1172906756401062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.7824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872961014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12324897199869156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872976045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11887331306934357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872990884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12011583894491196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8324999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873005833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11772780865430832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.8491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873005834, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784873005834, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784873048946, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6724461317062378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784873048946, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784873048947, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784873063942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1227400004863739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873078557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11845839023590088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873093196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11540408432483673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.8991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873107975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12178345024585724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873123177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11978889256715775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873138447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11919917166233063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.9491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873153419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11026335507631302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.9658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873153420, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784873153420, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784873197402, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7056285738945007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784873197403, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784873197403, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784873211999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11685685813426971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873227413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11701890826225281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873242529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12067495286464691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.0158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873257302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1152673214673996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873272231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11358959227800369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873286912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11546924710273743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0658333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873301897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11077293753623962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873301949, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784873301949, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784873345179, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7284626960754395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784873345180, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784873345180, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784873359881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11633269488811493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873375208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1139206513762474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.1158333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873390263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11834435909986496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873405781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11373282223939896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873420569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11334877461194992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1658333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873436037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12116408348083496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873450954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10846187174320221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873450955, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784873450955, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784873494357, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7436291575431824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784873494358, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784873494358, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784873509326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11665308475494385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873524307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11491361260414124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873539488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11215978860855103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873554423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11173047125339508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873569359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10565708577632904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873584157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1188698559999466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873599679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11491502821445465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873599679, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784873599680, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784873643380, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7527515888214111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784873643381, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784873643381, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784873658639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11722313612699509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873674266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1142822876572609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.3491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873689246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11739121377468109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873704656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10365428030490875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873719508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11486963927745819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.3991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873734353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1089627593755722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.4158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873749223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10780847072601318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873749224, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784873749224, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784873791943, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7588794231414795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784873791944, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784873791944, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784873806762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10941693931818008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.4491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873821633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10932140052318573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873836166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11457014828920364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.4824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873851050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10932914167642593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873866306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11406384408473969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873881529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10383208096027374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873896866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11209554970264435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873896867, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784873896867, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784873939054, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7624707221984863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784873939055, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784873939055, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784873954098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10784801840782166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873968842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10967233031988144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.5824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873984350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12115511298179626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873999726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12226130068302155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874014765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11518111824989319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874029978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1110188364982605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874045125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11640200763940811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874045125, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784874045126, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784874087045, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7644136548042297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784874087046, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784874087046, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784874102058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11487240344285965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874117277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10799841582775116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.6991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874132783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11743854731321335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874148097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1106678694486618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874163679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11399441212415695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874178887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11398621648550034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874194608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10447630286216736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874194609, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784874194609, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784874237756, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7661281824111938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784874237757, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784874237757, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784874253235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11849859356880188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874268645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10712753236293793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874283988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10469253361225128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874299314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11358416080474854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874314591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11764125525951385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874329965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10625972598791122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874345751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1133931577205658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874345751, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784874345752, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784874389270, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7679239511489868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784874389270, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784874389271, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784874404705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11468034237623215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874420453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1067500188946724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874435667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10814451426267624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874451339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11340861022472382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874466497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10683618485927582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874481460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10354581475257874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874496302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11618898808956146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.015833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874496302, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784874496303, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784874539614, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7690886855125427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784874539614, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784874539615, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784874554611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10949137806892395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874569608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11487124860286713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874584553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10597743093967438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874599445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10536263883113861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874614598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10205460339784622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874629879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09598654508590698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874645452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10544203221797943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874645453, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784874645453, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784874688675, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7701836228370667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784874688676, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784874688676, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784874703876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10508546233177185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874718663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10986009240150452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874734499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1177596002817154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874749932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1096261665225029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874765009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10614313930273056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874779983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10048998892307281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874795186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10617236793041229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874795187, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784874795187, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784874837744, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7711830735206604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784874837744, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784874837745, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784874852918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10507175326347351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874867686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10522839426994324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874882752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10943177342414856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874898010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072753444314003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874913426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10564559698104858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874928775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11117061227560043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874944330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10950988531112671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874944330, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784874944331, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784874988365, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7716242671012878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784874988366, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784874988366, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784875003255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10498504340648651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875018681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1016923114657402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875034047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080247238278389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875049489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10858781635761261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875064753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10471656173467636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875079843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10037096589803696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875095073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1047038733959198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875095073, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784875095074, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784875137503, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7722070813179016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784875137503, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784875137504, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784875152594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10469478368759155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875167653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09971795231103897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875182663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10420377552509308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875198330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10992108285427094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875213858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10584680736064911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875229655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10315334796905518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875245302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1095694750547409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875245302, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784875245302, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784875287750, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7726502418518066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784875287750, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784875287750, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784875302685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09953103959560394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875318664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10269433259963989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875333583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10699674487113953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875348553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1031702384352684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875363483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10225556790828705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875378470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10660998523235321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875393530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10841883718967438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875393531, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784875393531, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784875436173, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7729765772819519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784875436174, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784875436174, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784875451139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1054895669221878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875466199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10322723537683487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875480971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10779187083244324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875496185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10571154952049255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875511449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10956151783466339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875526849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10728314518928528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875542035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11067120730876923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875542035, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784875542036, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784875584941, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7733228206634521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784875584943, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784875584943, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784875599860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10684366524219513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875615558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10118263214826584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875630817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10425108671188354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875646290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10173295438289642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875661626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081111878156662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875676932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11331483721733093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875692249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10800141096115112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875692249, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784875692249, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784875735509, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7736426591873169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784875735510, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784875735510, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784875750712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10335630178451538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875765959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11083666980266571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875781116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10733458399772644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875796470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09961307048797607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875811980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11155813932418823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875827449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11225941777229309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875843271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1052287220954895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875843271, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784875843272, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784875885968, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7737842202186584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784875885969, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784875885969, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784875901249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10785843431949615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875916088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09951336681842804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875932086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11546574532985687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875947614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11504833400249481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875962860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11456966400146484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875978206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10358546674251556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875993641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10612562298774719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875993642, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784875993642, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784876036723, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7736989855766296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784876036724, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784876036724, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784876052156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11121830344200134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876067309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11332354694604874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876082775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10971621423959732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876098520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11016611754894257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876114301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11393864452838898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876129948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11196573078632355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876145919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11246471852064133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876145919, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784876145920, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784876188846, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7740312218666077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784876188847, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784876188847, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784876203895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10881702601909637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876219685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10721959173679352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876235359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10651014000177383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876251304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11418122798204422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876266701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11764384806156158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876282174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10930554568767548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876298001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11525054275989532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.415833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876298002, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784876298003, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784876341270, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742974758148193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784876341271, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784876341271, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784876356440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11334221065044403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876371958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11304493993520737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876387605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11554333567619324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876403236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11334138363599777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876418955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10866191238164902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876434527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10871867835521698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.515833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876449689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11129793524742126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876449690, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784876449690, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784876494686, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7744693756103516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784876494687, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784876494687, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784876510156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.108326755464077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876525639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10460807383060455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876540392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10416696965694427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876555315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10422013700008392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876570163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10649580508470535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.615833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876585335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1026889830827713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876600277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10804010927677155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876600278, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784876600278, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784876643597, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7747345566749573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784876643598, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784876643598, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784876658554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.109056755900383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876673323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10182122886180878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876688701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725954174995422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876703903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10300560295581818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876719246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10171197354793549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876734795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10528149455785751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876749629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09943249821662903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876749630, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784876749630, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784876793383, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7748974561691284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784876793383, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784876793384, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784876807981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772033035755157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876823668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10997729003429413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876838951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10660609602928162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876854109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10596560686826706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876869118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071426272392273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876883902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10334858298301697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876898929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10747091472148895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876898930, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784876898931, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784876941630, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7751222252845764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784876941631, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784876941631, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784876956705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10794425755739212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876971550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10502054542303085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876986792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10858356952667236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877001893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10253095626831055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877017327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10847947746515274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877032805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10957629978656769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877048510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10478397458791733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877048511, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784877048511, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784877091320, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.775201141834259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784877091321, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784877091321, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784877106870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10735461115837097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877121786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10175085812807083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877137265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11208415776491165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877152061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10219833999872208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877167721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09989409148693085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877182878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10237956047058105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877197914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1053721159696579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877197914, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784877197915, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784877241608, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7753732204437256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784877241609, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784877241609, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784877256426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.102993443608284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877271828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10002323985099792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877287380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10841314494609833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877302784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10942204296588898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877317911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1042664498090744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877333272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10809394717216492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877348779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1091051697731018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877348780, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784877348780, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784877391602, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7756937742233276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784877391603, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784877391603, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784877406997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1108466237783432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877422814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10406004637479782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877438354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1043575182557106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877453411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10537479817867279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877468775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09991897642612457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877484136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10716941207647324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877499942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10371799767017365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877499943, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784877499943, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784877543867, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7757908701896667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784877543867, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784877543868, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784877559226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1062736064195633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877574543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10755100846290588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877590004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10000943392515182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877605673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1088067889213562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877621196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09664613008499146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877636809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10174892842769623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877652023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10381093621253967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877652024, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784877652024, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784877696893, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7759956121444702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784877696893, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784877696894, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784877712517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10213927924633026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877728232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10771737992763519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877743852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10828986763954163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877759761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1104770302772522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877775477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10760228335857391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877790668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10908931493759155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.565833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877806313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10133074223995209, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877806314, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784877806314, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784877851807, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.77634197473526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784877851808, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784877851808, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784877866917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10943952202796936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877882308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10381582379341125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149749760, "lr": 7.615833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877897343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10668186843395233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877912410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11139500141143799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150405120, "lr": 7.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877927525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10754232108592987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877942635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11225109547376633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151060480, "lr": 7.682499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877957582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10351458191871643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877957582, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784877957583, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784878002338, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7765970230102539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784878002339, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784878002339, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784878017117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1060328558087349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151715840, "lr": 7.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878032079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09777149558067322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878047070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11009465903043747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152371200, "lr": 7.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878062209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10435119271278381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878077597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11125467717647552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153026560, "lr": 7.782499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878092977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11212539672851562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878108876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10349702090024948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153681920, "lr": 7.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878108877, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784878108877, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} diff --git a/recommendation_v4/rcp_logs/gbs_16384/seed151156919.log b/recommendation_v4/rcp_logs/gbs_16384/seed151156919.log new file mode 100644 index 000000000..8f4fa5bb1 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_16384/seed151156919.log @@ -0,0 +1,799 @@ +:::MLLOG {"namespace": "", "time_ms": 1784846779600, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784846779600, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784846806661, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784846806663, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784846806663, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784846806663, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784846806663, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784846806663, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 16384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784846806663, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784846806663, "event_type": "POINT_IN_TIME", "key": "seed", "value": 151156919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784846806663, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784846806663, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784846806663, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784846806663, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784846807048, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784846807048, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784847733125, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784847733126, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784847733411, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784847862224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13882054388523102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.5833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784847876372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388241946697235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2499999999999997e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784847890057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13869300484657288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784847903933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13871368765830994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.583333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784847917861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13874506950378418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.25e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784847931753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386300027370453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784847945559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138418048620224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784847945560, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784847945560, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784847988754, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49982750415802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784847988755, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784847988755, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784848002616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13874763250350952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848016385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13869933784008026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848030442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13869796693325043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848044440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386922150850296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848058476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13865910470485687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9916666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848072840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384579986333847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848087321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13885018229484558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848087322, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784848087323, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784848130925, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49998733401298523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784848130925, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784848130926, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784848145013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384497433900833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848159586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13827064633369446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848173560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382831335067749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848188192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382591724395752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.9916666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848202170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13827215135097504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848216759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384887844324112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848230639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380145400762558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848230639, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784848230640, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784848272923, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5001637935638428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784848272924, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784848272924, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784848287006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13833743333816528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848301871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13800093531608582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.8249999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848316130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13741961121559143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848330491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382007896900177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848345715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13785354793071747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3249999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848359932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13766677677631378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848374691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374267339706421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848374692, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784848374692, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784848416800, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5003942251205444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784848416801, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784848416801, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784848430564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13708099722862244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848444678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13767597079277039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848459263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13756930828094482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1583333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848473313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371050775051117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848487568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359502673149109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848501811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371367871761322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848516152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360795497894287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848516153, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784848516154, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784848559407, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5006735324859619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784848559408, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784848559408, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784848573471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13694435358047485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848587453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13646936416625977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848601920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13757139444351196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848616569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13664746284484863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848630847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13514992594718933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.658333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848645205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355534791946411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848659687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13748618960380554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848659688, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784848659689, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784848704803, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5009748935699463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784848704803, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784848704803, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784848719544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13685576617717743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848734222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357807070016861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848748423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13600876927375793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848762994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351243555545807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848777671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13518376648426056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848792959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13476210832595825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848807845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13513348996639252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848807845, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784848807846, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784848850798, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5013629794120789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784848850799, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784848850799, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784848865638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13425996899604797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848880544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363610476255417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848895991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138233482837677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848910739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13416802883148193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.824999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848925714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13612619042396545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848940435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13491898775100708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.158333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848955669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366947591304779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.324999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848955670, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784848955670, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784848999645, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5019301176071167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784848999646, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784848999646, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784849014733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366378664970398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784849029755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366163194179535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784849044609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13477492332458496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784849059411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13050410151481628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784849074393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13265976309776306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849088748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337883174419403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849103050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348382532596588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849103103, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784849103103, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784849146667, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5025563836097717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784849146668, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784849146668, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784849161162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349460482597351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849175483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12938405573368073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849190796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1314333826303482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849205128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368253380060196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.1158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849219634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13229815661907196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849234091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13157176971435547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849249323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13253915309906006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849249324, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784849249324, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784849293509, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5032495260238647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784849293510, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784849293510, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784849307673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317821741104126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849322985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13425028324127197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849337128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1279902160167694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849351222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12965713441371918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849365565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13377517461776733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849380055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12819096446037292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849394126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12978589534759521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849394127, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784849394127, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784849438364, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5041555166244507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784849438364, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784849438364, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784849453527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13466337323188782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849467861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13308776915073395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.3158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849482531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345287263393402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849497107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349264681339264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849512195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362331211566925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.3658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849526770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13368327915668488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849541612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13697633147239685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849541613, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784849541613, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784849585132, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5055156350135803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784849585133, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784849585133, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784849599948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13432535529136658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.4158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849614902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1306743025779724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849629730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13383376598358154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.4491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849644854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1403389871120453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.4658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849659770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304193139076233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849675100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324896663427353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849690301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13818489015102386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849690301, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784849690302, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784849733240, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5070247650146484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784849733241, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784849733241, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784849748533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1263432502746582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849763635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1313425898551941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849778943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13705787062644958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849793996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13535504043102264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849809377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13244761526584625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849823789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12867212295532227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.6158333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849838396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12803372740745544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849838397, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784849838397, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784849881689, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5085452198982239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784849881689, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784849881690, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784849896373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13003361225128174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849911066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13489823043346405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.6658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849925968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1422210931777954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6825000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849940597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13405194878578186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849955962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1314431130886078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.7158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849970855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13363401591777802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849985733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13600751757621765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849985733, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784849985734, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784850029356, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5101237893104553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784850029357, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784850029358, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784850044055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346210092306137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850058961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13629218935966492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850074164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13564729690551758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850089170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13314875960350037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850104393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343318074941635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850119384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329774558544159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850134051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13255181908607483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8658333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850134052, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784850134052, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784850178689, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5119063258171082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784850178690, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784850178690, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784850193342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13196393847465515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850208174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345001608133316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8991666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850223154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14192607998847961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850238334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13363781571388245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850253365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134255051612854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850268624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13234907388687134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850283567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331523209810257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850283568, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784850283568, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784850327370, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5139719843864441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784850327370, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784850327371, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784850342164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383851021528244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850356979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13314425945281982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850371724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1287626326084137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850386567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300586611032486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850401790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13793018460273743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850417028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13163718581199646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850432148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13621094822883606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850432149, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784850432149, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784850477299, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5163280367851257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784850477299, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784850477300, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784850492062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13683071732521057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.1158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850506541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13643181324005127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850521180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13139963150024414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1491666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850535602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13013362884521484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850550255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13157643377780914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850565160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354098618030548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850580113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326732635498047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850580113, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784850580114, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784850623105, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5189617872238159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784850623106, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784850623106, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784850637813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13592734932899475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850653494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13231578469276428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850669038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13782858848571777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850684344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362910270690918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850699773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13859358429908752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850715140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13444578647613525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850730834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1391860395669937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850730835, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784850730835, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784850774938, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.521634578704834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784850774939, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784850774939, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784850790451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13501930236816406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850805982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14851126074790955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850821824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13521283864974976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850837017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13311710953712463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850852267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13427919149398804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850867271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332881599664688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850882552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365194320678711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850882552, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784850882553, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784850925433, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5246269106864929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784850925434, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784850925434, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784850940728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13606123626232147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850955838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13526341319084167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850971290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13389819860458374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850986213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336824595928192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851001205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333129107952118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851016122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295630782842636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851030768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12569110095500946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851030769, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784851030769, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784851073463, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5280777215957642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784851073464, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784851073464, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784851088160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13275770843029022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.5824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851103069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13257047533988953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851118035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13504862785339355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851132982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353694051504135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851148272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13678652048110962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851163142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12887413799762726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851177821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12977521121501923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851177822, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784851177823, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784851221279, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5330978035926819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784851221280, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784851221280, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784851235815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13256460428237915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.699166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851250311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12706884741783142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851264940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331857144832611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851279651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13218916952610016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851294611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1271083652973175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851309807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13236582279205322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851324424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13444076478481293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851324424, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784851324425, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784851368533, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5396130681037903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784851368533, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784851368533, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784851384212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294931173324585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851399352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13052934408187866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851414222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13070666790008545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851429177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12993022799491882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.8658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851443907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12793771922588348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851458773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12336001545190811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851474043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1273115575313568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851474044, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784851474044, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784851517330, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5484501719474792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784851517330, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784851517330, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784851532512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13588634133338928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851547661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13335800170898438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851562310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13922393321990967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851577879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13373517990112305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851592724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12794984877109528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851607513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298736333847046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851622494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.127152681350708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851622494, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784851622495, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784851666607, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5593397617340088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784851666608, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784851666608, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784851681083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12908676266670227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851695493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12775126099586487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851710247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13280647993087769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851725114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13283148407936096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851740230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1316329389810562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851755005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1340041309595108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851770360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13068073987960815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851770361, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784851770361, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784851813972, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5720669627189636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784851813973, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784851813973, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784851829216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12996506690979004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851844224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12992912530899048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851859230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301020085811615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851874353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13299424946308136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851889217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13400772213935852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851903813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300612986087799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851919176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13081389665603638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851919177, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784851919177, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784851962097, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5847154259681702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784851962097, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784851962097, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784851977150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12870632112026215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851992665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13146652281284332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852007491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12911540269851685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852023200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1282547414302826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852038434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12799251079559326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852053540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13242407143115997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852068785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13324198126792908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852068786, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784852068786, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784852112188, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5967762470245361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784852112188, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784852112188, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784852127040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13071003556251526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852142418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288137435913086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852158153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13127401471138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852173948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13248059153556824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852189610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13398823142051697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852204849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13010163605213165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852220868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12944281101226807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.4991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852220869, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784852220869, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784852264888, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6088135838508606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784852264889, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784852264889, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784852280176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13072431087493896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.5158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852295481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12861061096191406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852310841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13022005558013916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.5491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852326042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362059861421585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852341462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13150455057621002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852356999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12708717584609985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852373470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13096663355827332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852373471, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784852373471, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784852417312, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6180189251899719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784852417312, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784852417312, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784852432421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12745684385299683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6325000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852448380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1290859878063202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852463666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.128651961684227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852478733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1290985643863678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852493794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12663903832435608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852508901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12398587912321091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852523920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12526993453502655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852523921, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784852523921, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784852567398, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6279645562171936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784852567399, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784852567399, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784852583037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12855257093906403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852598515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12444411218166351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852613845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11909816414117813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.7824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852629075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12448503077030182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852644290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12055952101945877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852659423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12134136259555817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8324999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852674606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11894641816616058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.8491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852674607, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784852674608, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784852718131, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6375717520713806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784852718132, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784852718132, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784852733408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1236836239695549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852748285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1199759691953659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852763306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11727391183376312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.8991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852778374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12392610311508179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852793838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12194432318210602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852809331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12043236196041107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.9491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852824410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1121465265750885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.9658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852824411, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784852824411, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784852868306, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.65995192527771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784852868306, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784852868307, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784852883253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11822301149368286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852898856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11791133135557175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852914082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12175138294696808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.0158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852929099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11661405861377716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852945122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1150299459695816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852960101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11652299016714096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0658333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852975256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1123342365026474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852975317, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784852975318, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784853019462, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6770018935203552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784853019463, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784853019463, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784853034380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11769773066043854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853049985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11552567780017853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.1158333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853065466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11977910250425339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853081362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.115785151720047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853096489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11573126912117004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1658333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853112184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12330823391675949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853127201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1102057546377182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853127202, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784853127203, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784853170667, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7059394121170044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784853170668, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784853170668, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784853185703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11792708933353424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853200788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1159525066614151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853216079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11380937695503235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853231152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11235920339822769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853246207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10769937932491302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853261089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12022453546524048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853276691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11644601076841354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853276692, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784853276692, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784853320740, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7365131974220276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784853320740, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784853320740, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784853336123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11910886317491531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853351851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11595627665519714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.3491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853367058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1178789958357811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853382699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10490047186613083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853398216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11624341458082199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.3991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853413800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11112409830093384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.4158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853429129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10965251922607422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853429129, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784853429130, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784853472481, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.755134642124176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784853472481, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784853472481, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784853487579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111704982817173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.4491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853502869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11114948987960815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853518030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11526723951101303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.4824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853533525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11090824007987976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853549224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11539767682552338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853564768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10549770295619965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853580218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11281201988458633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853580219, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784853580219, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784853624312, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.762135922908783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784853624312, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784853624313, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784853639825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10867486894130707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853654933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11093977093696594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.5824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853671083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12095609307289124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853687000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12323690950870514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853702604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11714081466197968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853718331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11188685148954391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853734055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11665312200784683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853734056, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784853734056, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784853778154, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.766153872013092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784853778155, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784853778155, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784853793781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11630232632160187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853809611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1091659814119339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.6991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853825642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11831356585025787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853841488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11137460172176361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853857651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11488417536020279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853873293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11444784700870514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853889720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10535600781440735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853889721, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784853889721, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784853935012, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7679151892662048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784853935013, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784853935013, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784853950601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1189681813120842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853966241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10803516209125519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853981860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1056821271777153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853997333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11414182186126709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854012718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11906518042087555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854028128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10744224488735199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854043951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11470820009708405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854043952, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784854043953, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784854089104, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7687287926673889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784854089105, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784854089105, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784854104744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11554098129272461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854120691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10791954398155212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854136010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10902760922908783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854152142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11377592384815216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854167610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10771903395652771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854182952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10449446737766266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854198213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11720092594623566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.015833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854198214, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784854198214, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784854242675, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.768823504447937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784854242676, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784854242676, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784854258026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10945896804332733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854273392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11498135328292847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854288709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10681477189064026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854303838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10503461956977844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854319483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10255461931228638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854335018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09621278196573257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854350731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10574954748153687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854350732, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784854350732, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784854394711, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.769791305065155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784854394712, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784854394712, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784854410040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10559000074863434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854425082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11070096492767334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854441114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11885470151901245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854456482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10952217876911163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854471669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10643278062343597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854486602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1008431613445282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854501975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10625860095024109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854501975, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784854501976, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784854545738, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7705718278884888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784854545739, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784854545739, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784854561001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10534784942865372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854575958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10583368688821793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854591098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10943309962749481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854606376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10739423334598541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854621851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10632839798927307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854637238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11196541786193848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854652808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10968375205993652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854652809, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784854652809, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784854697048, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7713091373443604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784854697049, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784854697049, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784854712218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10467244684696198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854727783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10241571813821793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854743349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10881274938583374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854758929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10882112383842468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854774299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1049659475684166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854789692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10064209997653961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854805343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10516241937875748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854805344, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784854805344, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784854849231, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7718011140823364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784854849232, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784854849232, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784854864751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10450093448162079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854880350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10017445683479309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854895886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10483947396278381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854911817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11029823124408722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854927656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10596564412117004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854943756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10348650813102722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854959594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10980327427387238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854959594, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784854959595, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784855003259, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7722395062446594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784855003260, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784855003260, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784855018402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10022804141044617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855034397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1030786782503128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855049787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10684189200401306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855065178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10357688367366791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855080427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10281796753406525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855095716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10665173083543777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855111126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10973069071769714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855111127, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784855111127, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784855154207, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.772571325302124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784855154208, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784855154208, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784855169413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10534993559122086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855184842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10338887572288513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855200065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10820692032575607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855215673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10577557235956192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855231391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10992462933063507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855247219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10757985711097717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855262833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11083436012268066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855262834, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784855262834, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784855306858, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7727365493774414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784855306858, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784855306859, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784855322006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10693249106407166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855338105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10163913667201996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855353712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10458188503980637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855369650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10211275517940521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855385376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10848505795001984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855401129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11352227628231049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855416861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10826694220304489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855416861, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784855416862, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784855460142, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730767130851746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784855460143, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784855460143, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784855475791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10375536978244781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855491257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11135101318359375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855506836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10773605108261108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855522598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10004182159900665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855538541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11177179217338562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855554414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11284411698579788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855570610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10500927269458771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855570610, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784855570611, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784855613657, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773326575756073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784855613658, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784855613658, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784855629273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082344651222229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855644543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10016219317913055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855660723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11595264077186584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855676616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11557774990797043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855692363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11482610553503036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855708103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10390865802764893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855724036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10657839477062225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855724037, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784855724037, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784855766585, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7733494639396667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784855766586, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784855766586, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784855782506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11105512082576752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855798235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11399313807487488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855814031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10995682328939438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855830030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11021818220615387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855846011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11463446915149689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855861868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11198095232248306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855877864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11307837069034576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855877865, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784855877865, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784855920679, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773645281791687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784855920679, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784855920680, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784855935912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10929356515407562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855952057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10710210353136063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855968030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732079297304153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855984201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1146358847618103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856000150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11858002841472626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856016245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10986648499965668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856032591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11544212698936462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.415833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856032592, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784856032592, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784856077589, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738677263259888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784856077590, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784856077590, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784856093493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11335670202970505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856109548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1135437935590744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856125791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11580827832221985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856142061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1135413646697998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856158295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10943345725536346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856174376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10889366269111633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.515833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856190026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11247888952493668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856190026, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784856190027, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784856235695, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7739756107330322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784856235696, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784856235696, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784856251719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10894247144460678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856267548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10495366156101227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856282666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10423135757446289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856297958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10431760549545288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856313168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10667216777801514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.615833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856328729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10294046252965927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856344088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10820260643959045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856344089, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784856344089, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784856388384, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7740434408187866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784856388384, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784856388384, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784856403684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1094837486743927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856418818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10242711007595062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856434336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10767830908298492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856449813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10335372388362885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856465464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10213589668273926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856481258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10598452389240265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856496586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09993871301412582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856496587, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784856496587, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784856542188, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742499709129333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784856542188, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784856542189, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784856557093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10787840187549591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856573231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11019475758075714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856588877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1065472662448883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856604400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10632923245429993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856619972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10734912753105164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856635263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1032852753996849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856650819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792672634124756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856650820, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784856650820, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784856694841, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7743613123893738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784856694841, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784856694842, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784856710204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10789279639720917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856725485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10586035996675491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856741082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10874854773283005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856756557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10280495136976242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856772484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10881105065345764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856788410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10954567790031433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856804314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1050684005022049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856804315, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784856804315, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784856848568, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7744684815406799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784856848569, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784856848569, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784856864503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792002081871033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856879733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10201144963502884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856895515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11214528232812881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856910380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10272318124771118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856926023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10054314881563187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856941299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10298210382461548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856956440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1054534912109375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856956441, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784856956442, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784857000403, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7745795845985413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784857000403, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784857000404, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784857015610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10302111506462097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857031252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10049617290496826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857046831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10821403563022614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857062238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10961610078811646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857077571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10445409268140793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857093248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10797997564077377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857108936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10922190546989441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857108936, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784857108937, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784857151728, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7745900750160217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784857151729, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784857151729, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784857167421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1105247288942337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857183224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10458701103925705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857198655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10446542501449585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857213662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10578341037034988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857229427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10011807829141617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857245240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10758892446756363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857261127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10436390340328217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857261127, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784857261128, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784857304449, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746885418891907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784857304449, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784857304449, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784857319829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10704543441534042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857335099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10829165577888489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857350801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10000364482402802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857366272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10940422862768173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857381680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09671854972839355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857397204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10253430902957916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857412375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10385517030954361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857412375, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784857412376, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784857456258, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7748563885688782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784857456259, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784857456259, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784857471912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.102349191904068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857487490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10852664709091187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857503074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10901114344596863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857518824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11108499765396118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857534407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10799803584814072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857549602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10909239947795868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.565833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857565447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10148080438375473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857565448, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784857565448, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784857609333, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7748023271560669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784857609334, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784857609334, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} diff --git a/recommendation_v4/rcp_logs/gbs_16384/seed169450858.log b/recommendation_v4/rcp_logs/gbs_16384/seed169450858.log new file mode 100644 index 000000000..c92b747d8 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_16384/seed169450858.log @@ -0,0 +1,1378 @@ +:::MLLOG {"namespace": "", "time_ms": 1784704997579, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784704997579, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784705022955, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784705022957, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784705022957, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784705022957, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784705022957, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784705022957, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 16384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784705022957, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784705022957, "event_type": "POINT_IN_TIME", "key": "seed", "value": 169450858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784705022957, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784705022957, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784705022957, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784705022957, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784705028098, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784705028099, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784705672887, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784705672888, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784705673138, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784706124390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13934564590454102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.5833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784706137940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13909479975700378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2499999999999997e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784706151384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13881759345531464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784706164984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13876499235630035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.583333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784706178846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13864529132843018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.25e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784706192588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13866840302944183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784706206248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1390020251274109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706206248, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784706206249, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784706293520, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5012962222099304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784706293521, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784706293521, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784706307025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13899950683116913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706320603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13885724544525146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706334313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13877752423286438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706348206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13881084322929382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706362038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13874900341033936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9916666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706376367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389700472354889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706390758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388654112815857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706390759, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784706390759, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784706434053, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5014854073524475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784706434054, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784706434054, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784706447926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13870179653167725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706462076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13848313689231873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706475794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13849811255931854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706490100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13856196403503418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.9916666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706503663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13855358958244324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706517913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385328471660614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706531840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13837596774101257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706531841, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784706531841, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784706575410, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5017085075378418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784706575411, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784706575411, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784706589334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13851556181907654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706603699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384570300579071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.8249999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706617524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13781607151031494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706631615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13829545676708221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706646063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380615532398224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3249999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706659917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13771703839302063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706674352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13785254955291748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706674353, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784706674353, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784706718015, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5020002126693726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784706718016, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784706718016, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784706731544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13741199672222137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706745303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13776597380638123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706759274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13768726587295532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1583333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706773027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13745611906051636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706786967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13633498549461365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706800897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13720612227916718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706814894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364779770374298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706814895, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784706814896, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784706859747, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5023314356803894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784706859748, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784706859748, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784706873385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370897889137268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706886943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13678543269634247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706900978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13760198652744293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706915119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13696140050888062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706928866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13547061383724213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.658333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706942653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359313726425171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706956593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137655571103096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706956593, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784706956594, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784707001910, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5026870965957642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784707001911, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784707001911, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784707015878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13705486059188843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707029899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359308660030365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707043699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362440586090088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707057656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13541041314601898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707071734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13550639152526855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707086169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13496294617652893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707100265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13521766662597656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707100266, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784707100266, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784707143986, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5031247735023499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784707143986, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784707143986, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784707158121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13453572988510132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707172414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13660217821598053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707187024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13811524212360382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707201378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13443204760551453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.824999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707215974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13611042499542236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707230301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13515549898147583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.158333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707244967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13652117550373077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.324999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707244968, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784707244968, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784707289003, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5037915110588074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784707289003, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784707289003, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784707303244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13651007413864136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707317453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13663959503173828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707331659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13498686254024506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707345925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13073977828025818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707360382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13294528424739838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707374568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13388067483901978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707388720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13493485748767853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707388771, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784707388771, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784707432006, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5045122504234314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784707432007, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784707432007, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784707446351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350564956665039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707460565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12960085272789001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707475287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13152998685836792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707489024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13671524822711945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.1158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707502972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325109899044037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707516729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317097544670105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707531096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13264797627925873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707531097, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784707531097, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784707573809, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5053156018257141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784707573809, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784707573810, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784707587501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13197173178195953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707602200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13426119089126587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707616112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12829650938510895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707630223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298401653766632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707644567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13375365734100342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707659149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12844517827033997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707673161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13001719117164612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707673161, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784707673162, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784707717168, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5064457058906555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784707717168, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784707717169, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784707731808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13462510704994202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707745724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330670565366745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.3158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707759892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13438847661018372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707773945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348436176776886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707788547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13599160313606262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.3658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707802776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13363826274871826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707817149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13692057132720947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707817149, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784707817150, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784707860810, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5079631209373474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784707860811, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784707860811, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784707875347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1342317909002304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.4158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707890059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13052499294281006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707904577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1339837908744812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.4491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707919475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14027121663093567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.4658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707934111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.130463108420372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707948794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13246062397956848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707963049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381942182779312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707963050, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784707963050, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784708004955, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5094265341758728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784708004956, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784708004956, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784708019472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1257256269454956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708033830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13125458359718323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708048664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13700002431869507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708063216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13541248440742493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708078082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13236276805400848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708092230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288347840309143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.6158333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708106551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12796840071678162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708106551, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784708106552, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784708149231, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5108858942985535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784708149231, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784708149231, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784708163310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13013429939746857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708177705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13474637269973755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.6658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708192240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14225800335407257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6825000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708206593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13415345549583435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708221508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13145878911018372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.7158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708235870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336643099784851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708250503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13605180382728577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708250503, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784708250504, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784708294031, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5125228762626648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784708294032, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784708294032, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784708308417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13462844491004944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708322790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362040638923645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708337500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13577982783317566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708352105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13324254751205444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708366699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13435284793376923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708381219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330452263355255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708395574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13257843255996704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8658333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708395574, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784708395575, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784708438185, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.514238715171814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784708438185, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784708438186, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784708452523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13196463882923126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708466940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345776915550232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8991666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708481449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14172695577144623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708496402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13410049676895142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708511165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343328058719635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708525891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13242995738983154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708540353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312357664108276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708540354, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784708540354, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784708583010, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5161473751068115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784708583011, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784708583011, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784708597095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384182870388031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708611259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13298577070236206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708625305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288609504699707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708639435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12990142405033112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708653845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379750669002533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708668452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13168470561504364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708682707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13624517619609833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708682708, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784708682709, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784708727487, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5182508230209351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784708727487, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784708727487, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784708742010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.136775940656662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.1158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708756174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364579200744629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708770670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1314641833305359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1491666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708784854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13006696105003357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708799290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13144081830978394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708813948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13534171879291534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708828703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13274559378623962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708828703, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784708828704, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784708872585, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5205264091491699, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784708872585, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784708872585, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784708887047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13600826263427734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708902030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13228470087051392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708916831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13781657814979553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708931481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13637033104896545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708946172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13852858543395996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708960645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13450099527835846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708975623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1391758918762207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708975624, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784708975624, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784709018913, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5227922797203064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784709018914, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784709018914, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784709033831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13497819006443024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709048738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14756529033184052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709063904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13514935970306396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709078808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13299930095672607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709093606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1342543661594391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709108164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13301649689674377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709122818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13665775954723358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709122819, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784709122819, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784709165957, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5252751708030701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784709165958, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784709165958, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784709180783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361621916294098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709195674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13535308837890625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709210573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13377340137958527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709224981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13367971777915955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709239371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13329064846038818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709253780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294342577457428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709267972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12586551904678345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709267973, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784709267973, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784709311580, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5281782150268555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784709311580, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784709311580, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784709325946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13271589577198029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.5824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709340588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13266083598136902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709355348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350477933883667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709370092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13541118800640106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709384794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1369103193283081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709399217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289854645729065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709413465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13001489639282227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709413466, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784709413467, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784709457135, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5322018265724182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784709457136, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784709457136, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784709471393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13276158273220062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.699166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709485561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1273319125175476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709499842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13318373262882233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709514344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13237938284873962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709529161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12726779282093048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709544198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13242024183273315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709558549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345301866531372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709558550, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784709558550, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784709602167, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5373883247375488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784709602168, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784709602168, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784709617115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12967930734157562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709631570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13048714399337769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709645882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13091596961021423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709660352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13009649515151978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.8658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709674468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12811893224716187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709688721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12348876893520355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709703489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12750403583049774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709703489, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784709703490, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784709746223, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5443886518478394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784709746224, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784709746224, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784709760786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13598410785198212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709775440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13379690051078796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709789543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13955292105674744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709804626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13397543132305145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709819508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1281648576259613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709834417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13017219305038452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709849498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12759943306446075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709849498, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784709849499, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784709892137, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5534690618515015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784709892138, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784709892139, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784709906679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12964753806591034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709921382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1281525045633316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709936347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330098658800125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709951330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13334733247756958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709966318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13203084468841553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709981008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344614177942276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709995883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13107417523860931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709995883, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784709995884, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784710038461, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5655330419540405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784710038461, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784710038461, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784710053164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13017848134040833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710067870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13048714399337769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710082643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13074448704719543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710097334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13357073068618774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710111899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13463521003723145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710126165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.130702942609787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710141299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13138309121131897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710141299, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784710141300, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784710184491, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5791877508163452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784710184492, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784710184492, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784710199295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.129245787858963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710214471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13225261867046356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710228903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12995046377182007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710244183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12911981344223022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710259770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12874147295951843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710275106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330917328596115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710290546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338033676147461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710290546, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784710290547, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784710333581, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5925002098083496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784710333581, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784710333581, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784710348489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13152767717838287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710363583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295604407787323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710379073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13198989629745483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710394462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13309034705162048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710409950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13465771079063416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710424983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13073796033859253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710440393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300548017024994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.4991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710440393, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784710440394, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784710483451, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6048631072044373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784710483451, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784710483451, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784710498241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13127043843269348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.5158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710513328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12942911684513092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710528544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13098296523094177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.5491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710543383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13663187623023987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710558611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13205131888389587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710573906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12753908336162567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710589217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13170988857746124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710589217, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784710589218, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784710632867, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6142460107803345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784710632867, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784710632867, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784710647652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1282360702753067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6325000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710663279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12958259880542755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710678376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12888579070568085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710693391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12929807603359222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710708162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12710097432136536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710723112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12467522919178009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710738018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12523584067821503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710738019, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784710738019, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784710780385, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.622561514377594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784710780386, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784710780386, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784710795965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12893664836883545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710811248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12470994889736176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710826393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11906374990940094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.7824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710841440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12448423355817795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710856381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1207442432641983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710871062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12135551869869232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8324999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710885916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1189020574092865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.8491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710885917, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784710885917, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784710928793, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6318137049674988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784710928794, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784710928794, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784710943824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12405118346214294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710958431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.119718998670578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710973045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11617644876241684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.8991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710987834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12368768453598022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711002893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12102761119604111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711018157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1201084554195404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.9491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711033035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1112934798002243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.9658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711033036, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784711033037, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784711076911, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6459285020828247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784711076912, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784711076912, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784711091511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11756554245948792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711106641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11745916306972504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711121566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12033573538064957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.0158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711136213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11576130986213684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711151076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1137530729174614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711165764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11535854637622833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0658333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711180290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1117687001824379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711180333, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784711180334, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784711224096, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6601696610450745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784711224097, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784711224097, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784711238561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11647741496562958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711253832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11438675969839096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.1158333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711268831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11870233714580536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711284199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11457877606153488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711298902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11468038707971573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1658333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711314190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12129223346710205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711328897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10904015600681305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711328897, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784711328898, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784711373232, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6828247904777527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784711373233, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784711373233, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784711387936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11649450659751892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711402659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11604668200016022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711417526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11266785860061646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711432048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11212901771068573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711446617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10650745034217834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711460900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1206890195608139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711476207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11618584394454956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711476208, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784711476208, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784711519745, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7120975255966187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784711519746, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784711519746, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784711534546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11884923279285431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711549730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11623626947402954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.3491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711564452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11831817030906677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711579586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10449458658695221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711594621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11593669652938843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.3991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711609289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11040298640727997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.4158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711624006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10937277972698212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711624007, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784711624007, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784711668242, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7349262237548828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784711668243, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784711668243, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784711682901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11137908697128296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.4491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711697583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11116205155849457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711711987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11653327941894531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.4824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711726653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11148111522197723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711741747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11638261377811432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711756621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10688932240009308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711771539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1148272454738617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711771540, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784711771540, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784711815375, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7484737038612366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784711815375, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784711815375, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784711830258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10949403047561646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711844787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1118779182434082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.5824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711860087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12250057607889175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711875216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1234634667634964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711890026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11788441985845566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711905094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11237897723913193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711920199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11814898252487183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711920199, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784711920200, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784711963263, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7553998827934265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784711963263, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784711963264, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784711978122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11700116097927094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711993262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10979719460010529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.6991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712008769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11862444877624512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712024076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11272574216127396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712039702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.115536168217659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712054825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11524207144975662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712070181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10534532368183136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712070181, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784712070182, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784712114233, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7599262595176697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784712114234, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784712114235, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784712129376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11889444291591644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712144463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10857857018709183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712159472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10687479376792908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712174329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11513744294643402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712189202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11889898031949997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712204235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10739222168922424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712219673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11465433239936829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712219673, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784712219674, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784712262020, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7624871134757996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784712262020, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784712262020, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784712277150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11560891568660736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712292640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10767362266778946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712307569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10912342369556427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712322931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11424178630113602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712338171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10769177973270416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712353115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10403800010681152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712367963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11604955792427063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.015833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712367964, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784712367964, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784712410463, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7642265558242798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784712410464, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784712410464, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784712425305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11065420508384705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712440190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11621294915676117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712454944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10652686655521393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712469574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10552902519702911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712484678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1031557098031044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712499745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09597709774971008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712515258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10581743717193604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712515259, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784712515259, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784712559691, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7644188404083252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784712559691, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784712559691, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784712574511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10517622530460358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712589131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10994886606931686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712604635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11806486546993256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712619834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10960260033607483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712634691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10652033984661102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712649406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1008707731962204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712664382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10609720647335052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712664383, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784712664383, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784712707729, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7656647562980652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784712707730, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784712707730, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784712722731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1054212898015976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712737412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10569630563259125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712752306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10996456444263458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712767421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10738703608512878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712782594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10632451623678207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712797761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11153200268745422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712813136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11025360226631165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712813136, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784712813137, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784712857279, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.766525387763977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784712857279, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784712857280, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784712872160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10499867051839828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712887279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10244914889335632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712902292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10864590108394623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712917444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10833898931741714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712932368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1050087958574295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712947311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10078812390565872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712962212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10523165762424469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712962212, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784712962213, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784713005771, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7670781016349792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784713005772, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784713005772, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784713020627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10521787405014038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713035404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09964337944984436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713050067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10517880320549011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713065318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11106695979833603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713080446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10625691711902618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713095913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10354968160390854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713111221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10902467370033264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713111221, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784713111222, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784713155239, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7680963277816772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784713155239, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784713155240, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784713169881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09940485656261444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713185327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10285056382417679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713200522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10765117406845093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713215629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10380882024765015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713230616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10298453271389008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713245685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10745249688625336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713260958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10921622067689896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713260958, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784713260959, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784713303771, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7690072059631348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784713303772, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784713303772, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784713318780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10565415024757385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713333765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10341772437095642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713348692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10842376202344894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713364190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10620873421430588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713379742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10984630137681961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713395317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10786747932434082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713410762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11072956025600433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713410763, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784713410763, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784713453832, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7699215412139893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784713453833, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784713453833, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784713468743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10690926015377045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713484202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1017439067363739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713499459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10504136234521866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713514779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10204260051250458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713529876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1089305728673935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713544953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11338037252426147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713560062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1086040586233139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713560062, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784713560063, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784713602755, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.770125150680542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784713602755, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784713602755, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784713617876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10379593819379807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713632910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11135870218276978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713648020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725864768028259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713663249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0997251495718956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713678477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11207271367311478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713693754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11274390667676926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713709526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10611146688461304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713709527, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784713709527, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784713752367, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7702265977859497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784713752368, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784713752368, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784713767562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10822077095508575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713782523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10013392567634583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713798420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11581400036811829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713814219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11523501574993134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713829696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11491701006889343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713845147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10401125252246857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713860807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10659560561180115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713860807, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784713860808, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784713903690, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.770200252532959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784713903691, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784713903691, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784713919275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11146437376737595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713934626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11398723721504211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713950273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11008527874946594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713966096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11035259068012238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713981884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1146092563867569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713997509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11242333054542542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714013403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1126490980386734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714013404, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784714013405, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784714057295, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.77055823802948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784714057295, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784714057296, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784714072542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10918173938989639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714088268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10737434774637222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714103704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10654455423355103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714119557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11508001387119293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714135027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11840542405843735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714150535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10968900471925735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714166529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11554568260908127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.415833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714166529, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784714166530, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784714210278, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7711919546127319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784714210278, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784714210279, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784714225729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11364011466503143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714241293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11364781111478806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714256915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11551962047815323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714272558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11355239897966385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714288334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10905277729034424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714303775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10896290838718414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.515833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714318831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11190667748451233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714318832, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784714318832, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784714363049, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7715903520584106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784714363050, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784714363050, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784714378599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10880498588085175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714394239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1046888679265976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714409142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10506308078765869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714424409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10464328527450562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714439490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10672253370285034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.615833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714454825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10362933576107025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714469961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10831527411937714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714469962, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784714469962, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784714514237, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7721980214118958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784714514237, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784714514237, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784714529380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10918289422988892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714544352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10227397084236145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714559648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10773412883281708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714574991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10335443168878555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714590625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10184946656227112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714606276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10557729750871658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714621524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0997723862528801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714621524, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784714621525, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784714664950, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7722623944282532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784714664950, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784714664950, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784714679972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10779687762260437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714695592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1105005294084549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714711221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10659851878881454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714726269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10599905252456665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714741284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10782286524772644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714755988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10379375517368317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714771037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792653262615204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714771037, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784714771038, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784714814697, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7726943492889404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784714814698, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784714814698, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784714829772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10781963169574738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714844594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10582665354013443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714859853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10896560549736023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714874984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10263598710298538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714890370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10900495201349258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714905860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10970762372016907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714921421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10512566566467285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714921421, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784714921422, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784714963801, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.772916316986084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784714963801, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784714963802, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784714979244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10791385173797607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714994245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10236972570419312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715009539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11206050217151642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715024478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10342784225940704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715040061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10014945268630981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715055167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10246367752552032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715070018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10628064721822739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715070018, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784715070019, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784715111978, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7732444405555725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784715111978, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784715111979, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784715126679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10331086814403534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715141866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10067000985145569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715157210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10847043991088867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715172265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10966414213180542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715187250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10442395508289337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715202510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10831081867218018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715217970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10954949259757996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715217971, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784715217971, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784715261313, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7735998630523682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784715261314, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784715261314, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784715276816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11100797355175018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715292561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10486466437578201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715307885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10435252636671066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715322760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10589651763439178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715337986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10038968920707703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715353067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10724781453609467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715368391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10439062118530273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715368392, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784715368392, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784715411910, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7737373113632202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784715411911, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784715411911, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784715426887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1067752093076706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715441719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10790053009986877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715456666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09991295635700226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715471933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1092996746301651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715486940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09668903052806854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715502016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10249090194702148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715516802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1038886308670044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715516802, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784715516803, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784715560752, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7737910151481628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784715560753, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784715560753, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784715576054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10225589573383331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715591241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10880200564861298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715606561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10880531370639801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715622059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11159330606460571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715637449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10795804858207703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715652321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10937802493572235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.565833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715667651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10168161988258362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715667651, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784715667651, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784715711632, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742061018943787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784715711632, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784715711633, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784715726582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10936897993087769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715742157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.104078508913517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149749760, "lr": 7.615833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715757311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10677896440029144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715772565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11173360794782639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150405120, "lr": 7.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715787778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10779348760843277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715803284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11301404237747192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151060480, "lr": 7.682499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715818677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1040080338716507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715818678, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784715818678, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784715863718, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7741485834121704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784715863719, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784715863719, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784715879005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10678361356258392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151715840, "lr": 7.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715894327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09820735454559326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715909702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11096575856208801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152371200, "lr": 7.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715925278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10439030826091766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715940986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11184456944465637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153026560, "lr": 7.782499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715956834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.112368144094944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715972816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10397449135780334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153681920, "lr": 7.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715972817, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784715972817, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784716016461, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774474024772644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784716016462, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784716016462, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784716032019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10277226567268372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716047046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10708660632371902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154337280, "lr": 7.849166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716062928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10250790417194366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716078442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11033092439174652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154992640, "lr": 7.882499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716094465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1099572479724884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716110054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11854058504104614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155648000, "lr": 7.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716125444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11771951615810394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716125444, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784716125445, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784716169161, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746531367301941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784716169161, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784716169161, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784716184689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11839602887630463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156303360, "lr": 7.949166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716200319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10976830869913101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716215730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1150747761130333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156958720, "lr": 7.982499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716231204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11191844195127487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716246856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11473256349563599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157614080, "lr": 8.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716262685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1165887862443924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716278423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10891944169998169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158269440, "lr": 8.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716278424, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784716278424, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784716322988, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7744470834732056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784716322989, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784716322989, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784716338702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11149198561906815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716354487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1046934574842453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158924800, "lr": 8.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716369847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11245617270469666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.099166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716385994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11179967224597931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159580160, "lr": 8.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716401997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11030451953411102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716417633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10784877836704254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160235520, "lr": 8.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716433201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11017562448978424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716433202, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784716433202, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784716476165, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774411678314209, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784716476166, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784716476166, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784716491527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11355479061603546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160890880, "lr": 8.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716507035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10520391911268234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.199166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716522832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10821182280778885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161546240, "lr": 8.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716538305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10944776237010956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716553892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11484190821647644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162201600, "lr": 8.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716570103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11216242611408234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.265833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716585740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11587244272232056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162856960, "lr": 8.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716585740, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784716585741, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784716628989, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746368050575256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784716628989, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784716628990, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784716645177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11588144302368164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.299166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716660978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11487625539302826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163512320, "lr": 8.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716676302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11684540659189224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716691977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10424716770648956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164167680, "lr": 8.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716707402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10153334587812424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716722701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10640478879213333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164823040, "lr": 8.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716737875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10628598928451538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716737875, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784716737876, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784716780619, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7748202681541443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784716780620, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784716780620, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784716795629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11043190956115723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165478400, "lr": 8.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716810892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1075211763381958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716826124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1015079915523529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166133760, "lr": 8.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716841451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1047772541642189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716857121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10440938174724579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166789120, "lr": 8.482499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716872247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0978470891714096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.499166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716887726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1055726483464241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167444480, "lr": 8.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716887726, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784716887727, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784716930427, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7749550342559814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784716930428, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784716930428, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784716945997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10623591393232346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.532499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716961829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09950584173202515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168099840, "lr": 8.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716977586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09436272084712982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716993151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10044125467538834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168755200, "lr": 8.582499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717008318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11285276710987091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717023702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10562553256750107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169410560, "lr": 8.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717039056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10985551029443741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717039057, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784717039057, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784717081422, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.775263786315918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784717081423, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784717081423, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784717097385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10610409080982208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170065920, "lr": 8.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717112788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10304738581180573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717127997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11596254259347916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170721280, "lr": 8.682499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717143337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09791189432144165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717158826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071079820394516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171376640, "lr": 8.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717174138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10428725183010101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717189518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1007135808467865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172032000, "lr": 8.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717189519, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784717189519, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784717232888, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7754648923873901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784717232889, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784717232889, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784717248277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10036969184875488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717263729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.105600506067276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172687360, "lr": 8.782499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717279532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10598866641521454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717295242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10752294957637787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173342720, "lr": 8.815833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717311082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10285156965255737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717326843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10326234996318817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173998080, "lr": 8.849166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717342441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10563100129365921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.865833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717342442, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784717342442, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784717386607, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7755644917488098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784717386608, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784717386608, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784717401580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10250881314277649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174653440, "lr": 8.882499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717417182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1058211401104927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717432385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10285107791423798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175308800, "lr": 8.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717447851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11117731034755707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717463087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10706163942813873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175964160, "lr": 8.949166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717478244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10294287651777267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.965833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717493643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11348843574523926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176619520, "lr": 8.982499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717493644, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784717493644, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784717536589, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7758268713951111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784717536590, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784717536590, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784717551924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10251526534557343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717567203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10297829657793045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177274880, "lr": 9.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717582381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10024412721395493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717597480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10295794904232025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177930240, "lr": 9.049166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717612986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10597911477088928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.065833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717628653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11014921218156815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178585600, "lr": 9.082499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717644225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09897997975349426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717644225, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784717644226, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784717687606, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7758817076683044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784717687607, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784717687607, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784717703141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10118260979652405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179240960, "lr": 9.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717718704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09902419149875641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717734017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10052886605262756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179896320, "lr": 9.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717748862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09835644066333771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.165833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717764620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10619896650314331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180551680, "lr": 9.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717779847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10482576489448547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717795226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10709483176469803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181207040, "lr": 9.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717795227, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784717795227, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784717838318, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7762871384620667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784717838319, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784717838319, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784717853527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10586218535900116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717868592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09903058409690857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181862400, "lr": 9.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717883704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10503502190113068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.265833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717899080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09971314668655396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182517760, "lr": 9.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717914242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09966272115707397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.299166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717929435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10404357314109802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183173120, "lr": 9.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717944448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10430090874433517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717944448, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784717944449, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784717987838, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7764149308204651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784717987839, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784717987839, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784718003122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09829320013523102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183828480, "lr": 9.349166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718018606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10700777918100357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718033909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10237913578748703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184483840, "lr": 9.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718049715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11075074225664139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718065336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10794331878423691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185139200, "lr": 9.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718080815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11114858090877533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718095983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10371427237987518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185794560, "lr": 9.449166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718095983, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784718095983, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784718140903, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.776760458946228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784718140903, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784718140903, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784718156335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10119932889938354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718171620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10998442769050598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186449920, "lr": 9.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718187560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10105492174625397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.499166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718203030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10564713925123215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187105280, "lr": 9.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718218473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10037228465080261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718233795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1111309677362442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187760640, "lr": 9.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718249168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09961611032485962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.565833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718249168, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784718249169, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784718293175, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7769425511360168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784718293175, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784718293175, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784718308764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10532509535551071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188416000, "lr": 9.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718324200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10224469751119614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718339398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10727985203266144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189071360, "lr": 9.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718354809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10369716584682465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718370344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10715292394161224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189726720, "lr": 9.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718386140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1128745898604393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718401734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10237938165664673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190382080, "lr": 9.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718401734, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784718401735, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784718444633, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.777364194393158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784718444634, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784718444634, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784718460415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10655859112739563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.699166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718476205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10085964947938919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191037440, "lr": 9.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718491693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10258321464061737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718507094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10659569501876831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191692800, "lr": 9.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718522633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10776382684707642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.765833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718538244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1077551543712616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192348160, "lr": 9.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718554016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11553133279085159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718554016, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784718554017, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784718598156, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7778531312942505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784718598157, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784718598157, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784718613760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10600222647190094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193003520, "lr": 9.815833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718629271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10817550122737885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.832499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718644930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11268860101699829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193658880, "lr": 9.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718660645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11439554393291473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718676342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10901325941085815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194314240, "lr": 9.882499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718691996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10614500939846039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.899166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718707633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10687576979398727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194969600, "lr": 9.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718707634, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784718707634, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784718752205, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7779173254966736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784718752205, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784718752206, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784718767863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11117085069417953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718783607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11112609505653381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195624960, "lr": 9.949166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718799491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11052393913269043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.965833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718815704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11241906881332397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196280320, "lr": 9.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718831759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10913659632205963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718847236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11420922726392746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196935680, "lr": 1.0015833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718862821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10346678644418716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1.0032499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718862822, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784718862822, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784718906666, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.778179407119751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784718906667, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784718906667, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784718922384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10808008909225464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197591040, "lr": 1.0049166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718938749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11111049354076385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1.0065833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718954575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10688400268554688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198246400, "lr": 1.00825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718970371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11284898221492767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1.0099166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718986106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11407589912414551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198901760, "lr": 1.0115833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719002145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10905386507511139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1.01325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719017917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11355520784854889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199557120, "lr": 1.0149166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719017918, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784719017918, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784719060708, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.77876877784729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784719060709, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784719060709, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784719076306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11437833309173584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1.0165833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719092289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10866573452949524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200212480, "lr": 1.01825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719108297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11305172741413116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1.0199166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719124260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11123690009117126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200867840, "lr": 1.0215833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719140358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10719306766986847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1.02325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719156302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10512807220220566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201523200, "lr": 1.0249166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719171837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11035403609275818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1.0265833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719171837, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784719171837, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784719215917, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7790215611457825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784719215917, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784719215917, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784719231665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10021120309829712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202178560, "lr": 1.02825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719246916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1065647304058075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1.0299166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719262565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10286694020032883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202833920, "lr": 1.0315833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719277686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1056973934173584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1.03325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719292819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10467492043972015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203489280, "lr": 1.0349166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719307900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11144206672906876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1.0365833333333335e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719323199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09815308451652527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204144640, "lr": 1.0382499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719323200, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784719323200, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784719366672, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7795535326004028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784719366673, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784719366673, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784719381850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10016518086194992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1.0399166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719397021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10411946475505829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204800000, "lr": 1.0415833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719412120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10137036442756653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1.04325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719427232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09724283963441849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205455360, "lr": 1.0449166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719442503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10112623870372772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1.0465833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719457675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1018562912940979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206110720, "lr": 1.0482499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719473120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10478146374225616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1.0499166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719473121, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784719473121, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784719516827, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7795337438583374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784719516827, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784719516828, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784719532392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10829724371433258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206766080, "lr": 1.0515833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719547752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10236680507659912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1.05325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719563126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09312150627374649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207421440, "lr": 1.0549166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719578218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10415370017290115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1.0565833333333335e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719593914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10166840255260468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208076800, "lr": 1.0582499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719609238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10194624960422516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1.0599166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719624944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10584545135498047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208732160, "lr": 1.0615833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719624945, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784719624945, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784719668787, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7802691459655762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784719668788, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784719668788, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784719684207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09762443602085114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1.06325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719699602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10538434982299805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209387520, "lr": 1.0649166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719714686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10848619043827057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1.0665833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719730119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09904220700263977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210042880, "lr": 1.06825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719745700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10159961879253387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1.0699166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719760826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1021275445818901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210698240, "lr": 1.0715833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719776108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1037774384021759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1.07325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719776109, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784719776109, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784719820693, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7804332375526428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784719820694, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784719820694, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784719835794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10188956558704376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211353600, "lr": 1.0749166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719851481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10896667093038559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1.076583333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719866993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09611663222312927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212008960, "lr": 1.0782499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719882202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10328197479248047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1.0799166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719897899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10297082364559174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212664320, "lr": 1.0815833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719913767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09834529459476471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1.08325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719929232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10598879307508469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213319680, "lr": 1.0849166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719929232, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784719929233, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784719973572, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7810023427009583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784719973573, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784719973573, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784719988951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10181226581335068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1.0865833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720004046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10462918877601624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213975040, "lr": 1.08825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720019637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10589256137609482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1.0899166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720035280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10012894868850708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214630400, "lr": 1.0915833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720050955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10591012239456177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1.09325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720066524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1075468584895134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215285760, "lr": 1.0949166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720081697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1025223582983017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1.096583333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720081698, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784720081698, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784720125432, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7810301184654236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784720125433, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784720125433, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784720140913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10393649339675903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215941120, "lr": 1.0982499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720156433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10110801458358765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1.0999166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720172191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10115319490432739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216596480, "lr": 1.1015833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720187907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09922226518392563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1.10325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720203492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10481630265712738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217251840, "lr": 1.1049166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720218930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11289288103580475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217579520, "lr": 1.1065833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720234754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10132540762424469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217907200, "lr": 1.10825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720234755, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784720234755, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784720278658, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7814329862594604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784720278658, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784720278658, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784720294246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10542774945497513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218234880, "lr": 1.1099166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720309787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10502569377422333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218562560, "lr": 1.1115833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720325640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1023498922586441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218890240, "lr": 1.11325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720341445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10788467526435852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219217920, "lr": 1.1149166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720357016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10428573191165924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219545600, "lr": 1.1165833333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720372155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09826651215553284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219873280, "lr": 1.1182499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720387553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09789703041315079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220200960, "lr": 1.1199166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720387553, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784720387554, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784720432158, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7815671563148499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784720432159, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784720432159, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784720447151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10324009507894516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220528640, "lr": 1.1215833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720462837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09697344154119492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220856320, "lr": 1.12325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720478291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09772045910358429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221184000, "lr": 1.1249166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720493580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082935631275177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221511680, "lr": 1.1265833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720508978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10205840319395065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221839360, "lr": 1.12825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720524309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1052875965833664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222167040, "lr": 1.1299166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720540045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11350563168525696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222494720, "lr": 1.1315833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720540045, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784720540046, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784720584015, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7819533348083496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784720584015, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784720584016, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784720599258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11024938523769379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222822400, "lr": 1.1332500000000001e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720614587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10428037494421005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223150080, "lr": 1.1349166666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720629973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10129426419734955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223477760, "lr": 1.1365833333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720645428, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10965394228696823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223805440, "lr": 1.1382499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720661413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10539788752794266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224133120, "lr": 1.1399166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720677262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793767869472504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224460800, "lr": 1.1415833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720693065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09144231677055359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224788480, "lr": 1.14325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720693066, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784720693066, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784720736684, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7818242907524109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784720736685, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784720736686, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784720752720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1023976281285286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225116160, "lr": 1.1449166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720768216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10218344628810883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225443840, "lr": 1.1465833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720783704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10842398554086685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225771520, "lr": 1.14825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720799022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11023109406232834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226099200, "lr": 1.1499166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720814352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10639545321464539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226426880, "lr": 1.1515833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720830053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10470446199178696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226754560, "lr": 1.1532500000000001e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720845862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09962939471006393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227082240, "lr": 1.1549166666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720845862, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784720845862, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784720889059, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7823770642280579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784720889059, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784720889060, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784720905020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1154165118932724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227409920, "lr": 1.1565833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720920738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10640987753868103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227737600, "lr": 1.1582499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720936417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1016315370798111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228065280, "lr": 1.1599166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720952052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11071199178695679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228392960, "lr": 1.1615833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720967622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10643474757671356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228720640, "lr": 1.1632499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720983403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10863149166107178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229048320, "lr": 1.1649166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720999053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09885184466838837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229376000, "lr": 1.1665833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720999054, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784720999054, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784721043142, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7826308608055115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784721043142, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784721043142, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784721058882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11331970244646072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229703680, "lr": 1.16825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721074819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10421782732009888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230031360, "lr": 1.1699166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721090365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10217311978340149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230359040, "lr": 1.1715833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721106449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10415861010551453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230686720, "lr": 1.17325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721122273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1062001883983612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231014400, "lr": 1.1749166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721138022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1020582839846611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231342080, "lr": 1.1765833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721154064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10961039364337921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231669760, "lr": 1.17825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721154065, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784721154065, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784721197870, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7825408577919006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784721197870, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784721197870, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784721213668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1142808198928833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231997440, "lr": 1.1799166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721229352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10781680047512054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 232325120, "lr": 1.1815833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721245060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10755473375320435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 232652800, "lr": 1.1832499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721260334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10767115652561188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 232980480, "lr": 1.1849166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721276544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1030198335647583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233308160, "lr": 1.1865833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721292632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10404408723115921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233635840, "lr": 1.18825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721308576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11879545450210571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233963520, "lr": 1.1899166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721308576, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784721308577, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784721352543, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7828908562660217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784721352543, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784721352544, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784721368396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1146412193775177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 234291200, "lr": 1.1915833333333335e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721384211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10888591408729553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 234618880, "lr": 1.19325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721399955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11376173049211502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 234946560, "lr": 1.1949166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721415985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10440954566001892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235274240, "lr": 1.1965833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721431867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12130096554756165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235601920, "lr": 1.19825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721447880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11379151046276093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235929600, "lr": 1.1999166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721463615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11419744789600372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 236257280, "lr": 1.2015833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721463616, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784721463616, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784721506232, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.782987654209137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784721506233, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784721506233, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784721522006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1161126047372818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 236584960, "lr": 1.2032499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721538161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1135096549987793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 236912640, "lr": 1.2049166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721554249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11356890201568604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237240320, "lr": 1.2065833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721570417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11020569503307343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237568000, "lr": 1.20825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721586661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10974390059709549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237895680, "lr": 1.2099166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721602841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11534513533115387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 238223360, "lr": 1.2115833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721618487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11048781126737595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 238551040, "lr": 1.21325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721618487, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784721618488, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784721661313, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7831225395202637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784721661314, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784721661314, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784721676838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11557736992835999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 238878720, "lr": 1.2149166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721693459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11446021497249603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239206400, "lr": 1.2165833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721709826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11065036058425903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239534080, "lr": 1.21825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721725770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11160239577293396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239861760, "lr": 1.2199166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721742150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1151488646864891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 240189440, "lr": 1.2215833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721758111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11316564679145813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 240517120, "lr": 1.2232499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721774353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11733804643154144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 240844800, "lr": 1.2249166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721774354, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784721774354, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784721817101, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.783629834651947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784721817102, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784721817102, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784721833057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11577410250902176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241172480, "lr": 1.2265833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721849271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11609680950641632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241500160, "lr": 1.22825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721865163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11042168736457825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241827840, "lr": 1.2299166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721881614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1127186045050621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 242155520, "lr": 1.2315833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721897681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11411838233470917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 242483200, "lr": 1.2332499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721913991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1152319610118866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 242810880, "lr": 1.2349166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721930328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10985612124204636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243138560, "lr": 1.2365833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721930329, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784721930329, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784721974270, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7831918597221375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784721974270, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784721974271, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784721990169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11520175635814667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243466240, "lr": 1.23825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722006072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11519154906272888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243793920, "lr": 1.2399166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722022171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10703061521053314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 244121600, "lr": 1.2415833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722037937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10450810939073563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 244449280, "lr": 1.24325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722054044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10666705667972565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 244776960, "lr": 1.2449166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722069960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10809028148651123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245104640, "lr": 1.2465833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722085864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11005546897649765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245432320, "lr": 1.24825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722085865, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784722085865, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784722128978, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7837228178977966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784722128978, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784722128979, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784722144732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10587616264820099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245760000, "lr": 1.2499166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722160899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10478836297988892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 246087680, "lr": 1.2515833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722176759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1132969930768013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 246415360, "lr": 1.2532499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722192872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11247602105140686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 246743040, "lr": 1.2549166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722208659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10403748601675034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247070720, "lr": 1.2565833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722224512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725639760494232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247398400, "lr": 1.25825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722240334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10945013165473938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247726080, "lr": 1.2599166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722240335, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784722240335, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784722283461, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7842088341712952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784722283461, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784722283462, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784722299488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11283378303050995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 248053760, "lr": 1.2615833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722315582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1156386062502861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 248381440, "lr": 1.26325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722331856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10131308436393738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 248709120, "lr": 1.2649166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722347785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11143805831670761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249036800, "lr": 1.2665833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722363683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1042846143245697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249364480, "lr": 1.26825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722379744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10927733778953552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249692160, "lr": 1.2699166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722395559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10525558888912201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 250019840, "lr": 1.2715833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722395613, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784722395614, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784722438222, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7839191555976868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784722438222, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784722438223, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784722453914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10083188116550446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 250347520, "lr": 1.2732499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722469427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1008932888507843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 250675200, "lr": 1.2749166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722484589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10084040462970734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251002880, "lr": 1.2765833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722499923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09945324808359146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251330560, "lr": 1.27825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722515799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10101647675037384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251658240, "lr": 1.2799166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722531132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10187111049890518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251985920, "lr": 1.2815833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722546694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10900698602199554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 252313600, "lr": 1.28325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722546740, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784722546740, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784722590212, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7841613292694092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784722590213, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784722590213, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784722605855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08406013250350952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 252641280, "lr": 1.2849166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722621356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10473915934562683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 252968960, "lr": 1.2865833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722637373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10877791047096252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 253296640, "lr": 1.28825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722653067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11594905704259872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 253624320, "lr": 1.2899166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722668753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11483582854270935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 253952000, "lr": 1.2915833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722684552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10424450784921646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 254279680, "lr": 1.2932499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722700501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10365892201662064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 254607360, "lr": 1.2949166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722700546, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784722700546, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784722743898, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7840080857276917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784722743899, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784722743899, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784722759255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10216554999351501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 254935040, "lr": 1.2965833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722774528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10316088795661926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 255262720, "lr": 1.2982499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722790563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1074608787894249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 255590400, "lr": 1.2999166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722806631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10874128341674805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 255918080, "lr": 1.3015833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722822715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1078343540430069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 256245760, "lr": 1.30325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722838772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09872718155384064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 256573440, "lr": 1.3049166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722854872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10679888725280762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 256901120, "lr": 1.3065833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722854872, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784722854873, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784722897974, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.784669816493988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784722897975, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784722897975, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784722913876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10316096246242523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 257228800, "lr": 1.30825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722930276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10199180990457535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 257556480, "lr": 1.3099166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722946520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1064729392528534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 257884160, "lr": 1.3115833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722962688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10840563476085663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 258211840, "lr": 1.3132499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722978886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10562644898891449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 258539520, "lr": 1.3149166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722994994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10651440173387527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 258867200, "lr": 1.3165833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784723011191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10584577918052673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 259194880, "lr": 1.3182499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784723011192, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784723011192, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784723053926, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7839418053627014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784723053926, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784723053927, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784723070237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10985317081212997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 259522560, "lr": 1.3199166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784723086583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10859616100788116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 259850240, "lr": 1.3215833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784723102761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10675647854804993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 260177920, "lr": 1.32325e-06}} diff --git a/recommendation_v4/rcp_logs/gbs_16384/seed295346581.log b/recommendation_v4/rcp_logs/gbs_16384/seed295346581.log new file mode 100644 index 000000000..b5e7d0062 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_16384/seed295346581.log @@ -0,0 +1,1388 @@ +:::MLLOG {"namespace": "", "time_ms": 1784700452147, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784700452147, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784700473175, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784700473177, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784700473177, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784700473177, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784700473177, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784700473177, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 16384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784700473177, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784700473177, "event_type": "POINT_IN_TIME", "key": "seed", "value": 295346581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784700473177, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784700473177, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784700473177, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784700473177, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784700473178, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784700473178, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784701370258, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784701370259, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784701370572, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784701498242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13957616686820984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.5833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784701511732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13927045464515686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2499999999999997e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784701525002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388877034187317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784701538673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387902796268463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.583333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784701552133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13857632875442505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.25e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784701565699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386290192604065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784701579149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389015018939972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784701579150, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784701579150, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784701622853, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4989592432975769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784701622853, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784701622854, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784701636415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13915544748306274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784701649916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13903583586215973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784701663546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13891249895095825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784701677286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389027237892151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784701690838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388002336025238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9916666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784701704978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13897597789764404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784701719189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13867418467998505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784701719190, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784701719190, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784701761587, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4991030693054199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784701761587, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784701761588, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784701775319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13871900737285614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784701789454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13871459662914276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784701803071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386757493019104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784701817347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860046863555908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.9916666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784701831036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13852231204509735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784701845400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388411819934845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784701859104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13838902115821838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784701859105, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784701859105, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784701901792, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4992532730102539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784701901792, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784701901793, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784701915594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13864558935165405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784701929983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13833904266357422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.8249999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784701943952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13798651099205017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784701958036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13848334550857544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784701972957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13804680109024048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3249999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784701987065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378539502620697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702001539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13792461156845093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702001540, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784702001540, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784702044482, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4994494915008545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784702044482, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784702044482, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784702058080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13745862245559692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702071864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13808664679527283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702086130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377759873867035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1583333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702100124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13739851117134094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702114050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365707814693451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702128077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13729120790958405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702142244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365933120250702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702142244, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784702142245, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784702184583, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49968764185905457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784702184583, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784702184584, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784702198426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13714709877967834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702212227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13680338859558105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702226354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13763514161109924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702240702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13696303963661194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702254621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13566283881664276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.658333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702268496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13589628040790558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702282573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13774576783180237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702282574, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784702282574, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784702326506, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49995237588882446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784702326506, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784702326506, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784702340873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13697797060012817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702355188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13594967126846313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702369046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362307369709015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702383028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354161649942398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702397205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13551735877990723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702411763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350596845149994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702425842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13536252081394196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702425843, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784702425843, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784702468658, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5003459453582764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784702468658, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784702468658, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784702482855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345982849597931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702497171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13660603761672974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702512029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380530595779419, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702526446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13437619805335999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.824999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702541104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13626176118850708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702555415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13506832718849182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.158333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702570022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13685756921768188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.324999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702570023, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784702570024, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784702614068, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5008941888809204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784702614069, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784702614069, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784702628356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13679492473602295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702642643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13674396276474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702656983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13508644700050354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702671351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1307932436466217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702685705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327751725912094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784702699721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338176280260086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784702713522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350933462381363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784702713574, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784702713575, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784702756477, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5015209913253784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784702756478, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784702756478, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784702770564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13509628176689148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784702784631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294676959514618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784702799400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13150933384895325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784702813426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13681040704250336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.1158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784702827520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13245896995067596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784702841433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1316336691379547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784702855904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13244032859802246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784702855904, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784702855905, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784702899015, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5022422671318054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784702899016, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784702899016, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784702912866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13192984461784363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784702927512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343478262424469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784702941326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12811052799224854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784702955321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12978525459766388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784702969525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13375459611415863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784702983961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12832079827785492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784702997901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300479620695114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784702997902, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784702997902, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784703040902, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5032747983932495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784703040903, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784703040903, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784703055735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13453617691993713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703069529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13308075070381165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.3158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703083653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344427466392517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703097691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347837746143341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703112085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359872817993164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.3658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703126230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13343545794487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703140734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13692359626293182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703140734, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784703140735, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784703184326, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5045849680900574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784703184327, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784703184327, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784703198846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13426348567008972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.4158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703213554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13053633272647858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703228045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13399618864059448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.4491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703242954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14035740494728088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.4658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703257639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13037024438381195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703272756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323278248310089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703287212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13828232884407043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703287213, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784703287213, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784703330715, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5058749914169312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784703330716, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784703330716, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784703345369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1264919489622116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703359905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13108479976654053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703374709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370495855808258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703389324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13539549708366394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703404330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13231725990772247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703418526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12883611023426056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.6158333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703432966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12801657617092133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703432967, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784703432967, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784703476442, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5072599053382874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784703476442, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784703476442, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784703490618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13012802600860596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703505008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13477855920791626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.6658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703519629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.142243891954422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6825000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703533981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13391552865505219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703549134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13145312666893005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.7158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703563462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13374894857406616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703577820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13591666519641876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703577821, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784703577822, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784703622557, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5088711977005005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784703622558, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784703622558, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784703636695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346304714679718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703651000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13632752001285553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703665690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13579228520393372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703680375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13322588801383972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703695101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13430361449718475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703709808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13293227553367615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703724104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13261570036411285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8658333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703724105, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784703724105, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784703768545, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5106270909309387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784703768545, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784703768546, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784703782800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13197611272335052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703797152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13447172939777374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8991666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703811740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14167185127735138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703826666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335456222295761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703841257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341942995786667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703855966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13220030069351196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703870862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332024335861206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703870862, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784703870863, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784703914468, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5126577615737915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784703914469, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784703914469, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784703928827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13851480185985565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703943396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13298574090003967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703957904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288197934627533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703972474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12996622920036316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703987457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13799403607845306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704002590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13153943419456482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704017470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13611677289009094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704017471, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784704017471, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784704060880, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5149666666984558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784704060880, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784704060881, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784704075585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368284374475479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.1158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704090000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13627567887306213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704104606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13145941495895386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1491666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704119045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.130088210105896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704133672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13154831528663635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704148641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13516822457313538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704163601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13278286159038544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704163602, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784704163603, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784704206748, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5176932215690613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784704206750, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784704206750, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784704221266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358933299779892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704236650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322949081659317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704252182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13775517046451569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704266990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13629800081253052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704281925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13855420053005219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704296784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13418352603912354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704312062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389394998550415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704312062, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784704312063, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784704355344, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5205807089805603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784704355357, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784704355357, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784704370429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13503681123256683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704385610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.15163502097129822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704400860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13526108860969543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704415564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329825520515442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704430499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341647058725357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704445181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332026571035385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704460047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13649754226207733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704460047, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784704460048, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784704504128, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.523981511592865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784704504129, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784704504129, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784704519018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13606154918670654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704534017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13528087735176086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704549384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13363173604011536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704564017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335419863462448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704578599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13320505619049072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704593121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1292557567358017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704607464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12555238604545593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704607464, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784704607465, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784704651622, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5280427932739258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784704651622, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784704651623, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784704665935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13242147862911224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.5824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704680674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13248448073863983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704695664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348724663257599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704710405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352178454399109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704725207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13656961917877197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704739836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1287117749452591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704754273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12964081764221191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704754274, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784704754274, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784704798593, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5343165993690491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784704798594, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784704798594, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784704812854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13253383338451385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.699166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704827191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12692014873027802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704841605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13317278027534485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704856047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320868730545044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704870803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12692447006702423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704885737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322203278541565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704900122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13409411907196045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704900122, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784704900123, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784704945500, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5431707501411438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784704945500, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784704945500, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784704960728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12921807169914246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704975438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13047336041927338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704989980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304897665977478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705004667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12972179055213928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.8658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705019016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12768200039863586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705033729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12297208607196808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705048735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1271069049835205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705048736, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784705048736, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784705092507, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.555158257484436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784705092507, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784705092507, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784705107257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353226602077484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705122021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13287204504013062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705136298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860034942626953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705151318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13318899273872375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705165834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12733492255210876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705180176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.129330575466156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705194682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12651756405830383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705194683, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784705194683, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784705239414, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5695282220840454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784705239415, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784705239415, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784705253670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12865950167179108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705267815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12697109580039978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705282383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13204234838485718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705297093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13211140036582947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705311958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13090316951274872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705326443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331961750984192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705341456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12994639575481415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705341457, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784705341457, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784705385297, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5853544473648071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784705385297, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784705385298, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784705400005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12916463613510132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705414596, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12907521426677704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705429454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12862977385520935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705444378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13185931742191315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705459041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323195993900299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705473404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12857605516910553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705488485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12939102947711945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705488485, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784705488486, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784705531964, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.598621129989624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784705531965, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784705531965, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784705546706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1270987093448639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705561796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12971951067447662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705576128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12680824100971222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705591648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12651732563972473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705606858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12619341909885406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705621649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300029158592224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705636569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13085468113422394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705636570, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784705636571, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784705679775, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6128823757171631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784705679776, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784705679776, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784705694222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1282716542482376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705709072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12607577443122864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705724240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289796233177185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705739427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13047687709331512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705754626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318102777004242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705769467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12758594751358032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705784870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12777704000473022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.4991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705784871, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784705784871, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784705827300, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6289750933647156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784705827301, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784705827301, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784705842183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12766043841838837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.5158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705857210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1256454885005951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705872187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1265164017677307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.5491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705886941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332460641860962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705901960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12870945036411285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705917116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12394080311059952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705932339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12632769346237183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705932339, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784705932340, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784705975616, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6437177062034607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784705975616, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784705975616, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784705990374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12348443269729614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6325000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706006240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1257573366165161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706021292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12492460012435913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706036275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12386658042669296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706051130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1225750744342804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706066276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11919476836919785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706081377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12186513841152191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706081377, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784706081378, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784706124274, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6610406041145325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784706124274, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784706124274, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784706139896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1238168403506279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706155271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12060323357582092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706170683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11341191828250885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.7824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706185900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1200990229845047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706200611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11521220952272415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706215267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11630280315876007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8324999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706229998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11356912553310394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.8491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706229999, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784706229999, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784706272019, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6857621669769287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784706272019, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784706272020, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784706286777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1193329244852066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706301500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11525056511163712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706316369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11081122606992722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.8991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706331465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11823108792304993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706346813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11717721074819565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706362298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11603929102420807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.9491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706377345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10767126828432083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.9658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706377345, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784706377346, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784706421059, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.722382128238678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784706421060, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784706421060, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784706435789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11515598744153976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706451379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11472255736589432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706466693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1182146817445755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.0158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706481689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11250481009483337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706496887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11116039752960205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706511850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11218788474798203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0658333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706526775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10819882154464722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706526842, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784706526842, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784706570413, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.743534505367279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784706570413, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784706570414, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784706585217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11294214427471161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706600672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11135341227054596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.1158333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706615977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11754946410655975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706631493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11122632026672363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706646335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11204834282398224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1658333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706662048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11783528327941895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706676974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10560405254364014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706676975, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784706676975, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784706720854, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.75544273853302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784706720855, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784706720856, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784706735747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11429207772016525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706750692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11224336177110672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706765681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10995312035083771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706780432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10973592102527618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706795099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10391714423894882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706809510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11804880201816559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706824765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11340317130088806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706824765, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784706824766, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784706868076, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7619572281837463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784706868077, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784706868077, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784706882891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11639315634965897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706898147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1120193749666214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.3491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706912897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11545673757791519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706927958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10085255652666092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706942964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1149802878499031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.3991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706957981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10794450342655182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.4158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706972858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10620200634002686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706972858, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784706972859, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784707016877, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7647040486335754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784707016878, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784707016878, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784707031617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10880229622125626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.4491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707046475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10818974673748016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707061111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11384358257055283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.4824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707076036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1092352494597435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707091258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1127295047044754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707106404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10305044054985046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707121697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11156109720468521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707121698, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784707121698, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784707164043, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7664219737052917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784707164044, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784707164044, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784707179129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1069960743188858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707193978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1086893305182457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.5824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707209436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11946246027946472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707224751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12302196770906448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707239771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11624555289745331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707255003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11079861223697662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707270127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11565650999546051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707270128, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784707270128, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784707312430, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7679110169410706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784707312431, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784707312431, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784707327371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11523889005184174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707342623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10795799642801285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.6991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707358087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1175260990858078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707373289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11070951819419861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707388887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11403083056211472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707404000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11381706595420837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707419632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10421766340732574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707419632, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784707419633, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784707463189, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7688267230987549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784707463189, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784707463189, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784707478292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11820750683546066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707493552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10677085816860199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707508933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10448351502418518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707524136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11397172510623932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707539352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11776423454284668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707554764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10660265386104584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707570515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11395548284053802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707570515, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784707570516, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784707614094, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7695017457008362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784707614095, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784707614095, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784707629386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11479946970939636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707645139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10712074488401413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707660281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792982578277588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707675958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1133309155702591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707691366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10713309049606323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707706254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10314524173736572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707721134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1166490763425827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.015833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707721135, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784707721135, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784707764964, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7697186470031738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784707764965, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784707764965, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784707779879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10998595505952835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707794759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11428645998239517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707809450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10728143155574799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707824095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10474465042352676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707839439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10182788968086243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707854720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09593044966459274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707870307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10593542456626892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707870307, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784707870308, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784707913030, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7705028057098389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784707913031, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784707913031, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784707928068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10517742484807968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707942831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10959457606077194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707958638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11871393769979477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707973943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087842583656311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707988827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10609767585992813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708003733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10039588809013367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708018792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.107060007750988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708018793, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784708018793, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784708061664, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7709845900535583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784708061665, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784708061665, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784708076709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10460880398750305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708091444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10496154427528381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708106469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10916922986507416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708121706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10710449516773224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708136982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10586023330688477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708152378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11162842810153961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708167970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10929155349731445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708167971, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784708167971, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784708211296, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7716566920280457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784708211296, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784708211297, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784708226334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10484659671783447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708241668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10267823934555054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708256842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1102062538266182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708272154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084715723991394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708287301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1048530787229538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708302489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.100511834025383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708317771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10438425838947296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708317771, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784708317772, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784708360785, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7719554901123047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784708360785, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784708360786, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784708375971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10434667766094208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708391308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09978929162025452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708406402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10446156561374664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708421830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1104075163602829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708437072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1063050776720047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708452510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1033535823225975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708467908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10982035100460052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708467908, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784708467909, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784708510908, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7723216414451599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784708510908, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784708510909, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784708525588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09964022040367126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708541185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1032966822385788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708556334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10679009556770325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708571378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10369756817817688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708586374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.102259062230587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708601432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732024908065796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708616744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10915398597717285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708616744, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784708616745, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784708660324, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.772561252117157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784708660325, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784708660325, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784708675279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10564585030078888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708690331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10292419791221619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708705340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10844852030277252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708720745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10611052811145782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708736147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11004140228033066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708751588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10761827230453491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708766806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1108442023396492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708766807, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784708766807, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784708810868, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7729083299636841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784708810869, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784708810869, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784708825746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10666005313396454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708841557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10103051364421844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708856518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10438816249370575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708871708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10148504376411438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708886726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10798054933547974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708901703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11312831193208694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708916692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.108194500207901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708916692, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784708916693, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784708961063, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730993628501892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784708961063, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784708961063, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784708975851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10473780333995819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708990661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.110733263194561, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709005459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725847631692886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709020551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10001984238624573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709035779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11183114349842072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709051064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11243492364883423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709066752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10478958487510681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709066752, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784709066753, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784709111464, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773082435131073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784709111465, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784709111465, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784709126606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10810476541519165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709141349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09996126592159271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709157299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1161646917462349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709172958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.116140216588974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709188337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11433209478855133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709203832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10406321287155151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709219402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1069929450750351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709219403, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784709219403, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784709264199, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730380296707153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784709264200, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784709264200, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784709279703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11081506311893463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709294910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11385469138622284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709310479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11007240414619446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709326335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11051034182310104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709342174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11452402174472809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709357780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11219951510429382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709373745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11317367851734161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709373745, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784709373745, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784709417188, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7733471989631653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784709417189, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784709417189, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784709432435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10920851677656174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709448309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10707633197307587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709463566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10673075914382935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709479259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11478951573371887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709494370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11833846569061279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709509379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10954742133617401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709524744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11561432480812073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.415833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709524745, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784709524745, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784709569334, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7735484838485718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784709569335, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784709569335, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784709584386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11344634741544724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709599643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11366361379623413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709614919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11554424464702606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709630308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11366993188858032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709645708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10932931303977966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709660994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10898171365261078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.515833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709676081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11217661947011948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709676082, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784709676082, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784709721314, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7736967206001282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784709721314, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784709721315, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784709736767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10868397355079651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709752335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10477279871702194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709767340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10459859669208527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709782614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1038246899843216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709797628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1065840944647789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.615833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709813069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10313042998313904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709828104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10846316814422607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709828105, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784709828105, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784709870904, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773931086063385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784709870904, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784709870905, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784709885906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10948474705219269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709900874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10239019989967346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709916533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10780103504657745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709932026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10356497764587402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709947572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10209286212921143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709963329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10567861050367355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709978479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09968734532594681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709978479, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784709978480, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784710022999, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7739701867103577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784710023000, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784710023000, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784710037794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10817234218120575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710053569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11046759784221649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710068733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10646992176771164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710083788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10588032752275467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710098765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10746665298938751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710113422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10302180796861649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710128585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10770076513290405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710128585, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784710128586, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784710172180, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742652893066406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784710172181, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784710172181, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784710187158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10796566307544708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710201963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10559845715761185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710217152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10827981680631638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710232239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10325493663549423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710247446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10875147581100464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710262810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10962513089179993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710278366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10451638698577881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710278367, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784710278367, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784710322268, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7743571400642395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784710322268, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784710322269, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784710337709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10791508853435516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710352560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10210170596837997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710367927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1123097687959671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710382767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1028391569852829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710398259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10010194778442383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710413243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10230256617069244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710428011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10578222572803497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710428011, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784710428012, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784710472348, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7744569182395935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784710472348, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784710472349, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784710487019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1030750647187233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710502238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10028361529111862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710517513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081901490688324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710532670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10994838178157806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710547432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10472879558801651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710562541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085418164730072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710577679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10925007611513138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710577680, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784710577680, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784710622117, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7745069265365601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784710622118, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784710622118, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784710637367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11164618283510208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710652786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10460987687110901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710667964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10404352843761444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710682815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1060253232717514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710698197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10062375664710999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710713250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10717925429344177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710728747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10460489243268967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710728747, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784710728748, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784710772349, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746050357818604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784710772349, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784710772350, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784710787350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10673008114099503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710802434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10802753269672394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710817665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10030718147754669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710832890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10959891974925995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710848312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09698538482189178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710863642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10196927934885025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710878556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10298049449920654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710878557, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784710878557, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784710922576, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7747551798820496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784710922577, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784710922577, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784710937892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10223405063152313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710953185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10850746929645538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710968389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10898345708847046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710983945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11112113296985626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710999224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1079784482717514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711014038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10963107645511627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.565833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711029585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10198260843753815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711029586, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784711029586, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784711073818, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7749716639518738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784711073819, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784711073819, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784711088805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10931524634361267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711104174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10404764115810394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149749760, "lr": 7.615833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711119152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1067451685667038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711134319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11151644587516785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150405120, "lr": 7.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711149247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080920472741127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711164623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11236126720905304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151060480, "lr": 7.682499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711179689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10401678830385208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711179690, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784711179690, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784711223712, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7747899889945984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784711223713, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784711223713, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784711239024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10667860507965088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151715840, "lr": 7.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711254254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09822062402963638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711269391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11049732565879822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152371200, "lr": 7.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711284778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10395275056362152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711300458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11161624640226364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153026560, "lr": 7.782499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711316238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11257307231426239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711332010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1037968099117279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153681920, "lr": 7.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711332011, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784711332011, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784711377371, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7751120924949646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784711377371, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784711377371, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784711392545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10269594937562943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711407589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10696907341480255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154337280, "lr": 7.849166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711423413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10266377031803131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711438752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11085072159767151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154992640, "lr": 7.882499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711454435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10993054509162903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711469752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11869849264621735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155648000, "lr": 7.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711485084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1179451271891594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711485085, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784711485085, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784711529987, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7751756906509399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784711529988, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784711529988, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784711545420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11780475080013275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156303360, "lr": 7.949166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711561184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10997919738292694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711576416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11505343019962311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156958720, "lr": 7.982499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711591774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11250555515289307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711607343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.115058034658432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157614080, "lr": 8.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711623254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11659406870603561, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711638988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10843934118747711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158269440, "lr": 8.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711638988, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784711638989, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784711682338, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752023339271545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784711682338, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784711682338, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784711697994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11136235296726227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711713638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10465019941329956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158924800, "lr": 8.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711728927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11203014105558395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.099166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711744905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11216053366661072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159580160, "lr": 8.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711760647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11010628938674927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711776347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10774393379688263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160235520, "lr": 8.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711791980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11030219495296478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711791981, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784711791981, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784711835435, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752821445465088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784711835435, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784711835435, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784711850706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11322090774774551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160890880, "lr": 8.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711866222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10491353273391724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.199166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711881954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10818074643611908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161546240, "lr": 8.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711897355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10967309772968292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711912911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11531258374452591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162201600, "lr": 8.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711928898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11190153658390045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.265833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711944450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11567486822605133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162856960, "lr": 8.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711944451, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784711944451, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784711988889, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.775458037853241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784711988890, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784711988890, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784712005016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11618630588054657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.299166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712020575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11461242288351059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163512320, "lr": 8.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712035818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11656659841537476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712051612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10409681499004364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164167680, "lr": 8.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712066873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10149030387401581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712082081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10663650929927826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164823040, "lr": 8.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712097103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10604012757539749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712097103, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784712097104, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784712139566, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7758594155311584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784712139566, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784712139566, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784712154386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11059018224477768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165478400, "lr": 8.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712169426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10761161148548126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712184412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10190902650356293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166133760, "lr": 8.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712199372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10432031750679016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712214390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10439032316207886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166789120, "lr": 8.482499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712229296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09793457388877869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.499166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712244621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10541936755180359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167444480, "lr": 8.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712244622, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784712244622, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784712287234, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7757909893989563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784712287235, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784712287235, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784712302347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10596324503421783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.532499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712317581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09990203380584717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168099840, "lr": 8.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712333080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0942482054233551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712348235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09991670399904251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168755200, "lr": 8.582499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712363163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1125955879688263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712378571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10532797873020172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169410560, "lr": 8.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712393592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11000612378120422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712393593, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784712393593, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784712436317, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7758732438087463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784712436317, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784712436318, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784712451751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10609817504882812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170065920, "lr": 8.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712466797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10300657153129578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712481679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1155489832162857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170721280, "lr": 8.682499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712496552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09769146144390106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712511690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732591152191162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171376640, "lr": 8.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712526687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039753407239914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712541774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10083383321762085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172032000, "lr": 8.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712541775, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784712541775, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784712584101, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7760679721832275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784712584102, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784712584102, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784712599011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10002179443836212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712614095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10570557415485382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172687360, "lr": 8.782499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712629543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10625478625297546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712644998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10737244039773941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173342720, "lr": 8.815833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712660660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10271624475717545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712675961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10333675146102905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173998080, "lr": 8.849166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712691151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10567821562290192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.865833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712691151, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784712691151, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784712734756, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.776301383972168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784712734756, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784712734756, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784712749692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1021130383014679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174653440, "lr": 8.882499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712765499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10609882324934006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712780770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10277725756168365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175308800, "lr": 8.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712796084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11160688102245331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712811147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10682842135429382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175964160, "lr": 8.949166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712826196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.102628692984581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.965833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712841538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1128888949751854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176619520, "lr": 8.982499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712841538, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784712841539, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784712883499, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7766081690788269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784712883499, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784712883500, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784712898769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10274626314640045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712913844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10283750295639038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177274880, "lr": 9.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712928770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10031703114509583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712943649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10308007895946503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177930240, "lr": 9.049166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712958896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10603135824203491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.065833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712974408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10969556868076324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178585600, "lr": 9.082499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712989743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09886236488819122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712989744, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784712989744, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784713033151, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7767153978347778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784713033151, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784713033152, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784713048804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1012096032500267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179240960, "lr": 9.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713064250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09878285229206085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713079378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10067147761583328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179896320, "lr": 9.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713094178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09812184423208237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.165833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713110211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10630433261394501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180551680, "lr": 9.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713125539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.105019211769104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713141006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723890364170074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181207040, "lr": 9.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713141007, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784713141007, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784713184007, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.776864230632782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784713184008, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784713184008, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784713199232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10589642822742462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713214387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09916868805885315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181862400, "lr": 9.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713229432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10483500361442566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.265833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713244700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09973035752773285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182517760, "lr": 9.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713259761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09967764467000961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.299166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713275088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1038578599691391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183173120, "lr": 9.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713290112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10480546951293945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713290112, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784713290113, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784713333252, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7772340178489685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784713333253, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784713333253, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784713348645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09862935543060303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183828480, "lr": 9.349166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713364127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10726273059844971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713379560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10290852189064026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184483840, "lr": 9.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713395406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11027908325195312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713411161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10800963640213013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185139200, "lr": 9.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713426580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11177048832178116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713441758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1034964993596077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185794560, "lr": 9.449166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713441759, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784713441760, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784713486320, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7774514555931091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784713486321, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784713486321, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784713502022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1011776253581047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713517247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10954532027244568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186449920, "lr": 9.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713532965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10112223774194717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.499166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713548077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10495088994503021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187105280, "lr": 9.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713563181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10002858936786652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713578400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11143483966588974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187760640, "lr": 9.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713593500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09970352053642273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.565833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713593501, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784713593502, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784713637247, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7777019739151001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784713637248, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784713637248, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784713652524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10504929721355438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188416000, "lr": 9.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713667575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10204079747200012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713682409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10697085410356522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189071360, "lr": 9.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713697548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10398974269628525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713712775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10700753331184387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189726720, "lr": 9.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713728129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11313782632350922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713743618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10214406251907349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190382080, "lr": 9.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713743618, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784713743619, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784713787083, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7780258655548096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784713787084, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784713787084, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784713802608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10632907599210739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.699166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713818226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10075646638870239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191037440, "lr": 9.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713833485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1028570681810379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713848815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10620227456092834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191692800, "lr": 9.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713864584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10766138136386871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.765833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713880480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10813899338245392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192348160, "lr": 9.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713896451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1152288019657135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713896451, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784713896452, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784713940349, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7778012752532959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784713940350, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784713940350, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784713956066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10577422380447388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193003520, "lr": 9.815833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713971720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10818919539451599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.832499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713987464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1124553233385086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193658880, "lr": 9.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714003315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11495541781187057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714019193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10871459543704987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194314240, "lr": 9.882499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714035125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10615584254264832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.899166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714050892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10670152306556702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194969600, "lr": 9.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714050892, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784714050893, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784714094485, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7784315347671509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784714094486, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784714094486, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784714110233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11128683388233185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714126256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11105357110500336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195624960, "lr": 9.949166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714142256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11075497418642044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.965833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714158891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11209480464458466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196280320, "lr": 9.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714174848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10924965143203735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714190516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11430509388446808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196935680, "lr": 1.0015833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714206254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10320131480693817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1.0032499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714206254, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784714206255, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784714250495, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7785643935203552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784714250495, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784714250496, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784714266159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10802292823791504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197591040, "lr": 1.0049166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714282226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11142293363809586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1.0065833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714297669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10654892772436142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198246400, "lr": 1.00825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714313118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11282733827829361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1.0099166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714328447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11397533118724823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198901760, "lr": 1.0115833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714344238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10864043980836868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1.01325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714359918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11300921440124512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199557120, "lr": 1.0149166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714359918, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784714359919, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784714403991, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7791324853897095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784714403992, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784714403992, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784714419504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11447970569133759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1.0165833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714435179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10909083485603333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200212480, "lr": 1.01825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714450915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11283069849014282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1.0199166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714466699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1109427958726883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200867840, "lr": 1.0215833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714482765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10698486119508743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1.02325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714498557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10476842522621155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201523200, "lr": 1.0249166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714513933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11101936548948288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1.0265833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714513934, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784714513934, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784714559275, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7794379591941833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784714559276, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784714559276, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784714574867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10032030194997787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202178560, "lr": 1.02825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714590120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10651960223913193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1.0299166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714605824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10336887836456299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202833920, "lr": 1.0315833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714621184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10597541183233261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1.03325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714636388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10425235331058502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203489280, "lr": 1.0349166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714651527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11166437715291977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1.0365833333333335e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714666814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09826329350471497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204144640, "lr": 1.0382499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714666815, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784714666816, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784714709837, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7796248197555542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784714709838, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784714709838, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784714725277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1002817153930664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1.0399166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714740885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10422342270612717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204800000, "lr": 1.0415833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714756172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10056360065937042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1.04325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714771295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.097478486597538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205455360, "lr": 1.0449166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714786633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10135414451360703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1.0465833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714801891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10140049457550049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206110720, "lr": 1.0482499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714817410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10446792840957642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1.0499166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714817410, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784714817411, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784714861404, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7796503901481628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784714861404, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784714861404, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784714877046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1079564169049263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206766080, "lr": 1.0515833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714892438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.102310411632061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1.05325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714907747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09276972711086273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207421440, "lr": 1.0549166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714922981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10403728485107422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1.0565833333333335e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714938603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10182388126850128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208076800, "lr": 1.0582499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714953840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1019408106803894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1.0599166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714969391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10545764863491058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208732160, "lr": 1.0615833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714969391, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784714969392, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784715012817, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7800098061561584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784715012817, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784715012817, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784715028148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09756751358509064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1.06325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715043544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10536845028400421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209387520, "lr": 1.0649166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715058614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085633933544159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1.0665833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715073995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09863869100809097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210042880, "lr": 1.06825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715089465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10210210084915161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1.0699166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715104773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1023004949092865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210698240, "lr": 1.0715833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715119991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10400596261024475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1.07325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715119991, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784715119992, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784715164137, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.780346691608429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784715164138, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784715164138, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784715179184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10249769687652588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211353600, "lr": 1.0749166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715194770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10884077101945877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1.076583333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715210165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0965743213891983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212008960, "lr": 1.0782499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715225280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10317334532737732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1.0799166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715241066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10269191116094589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212664320, "lr": 1.0815833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715256921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09829481691122055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1.08325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715272499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10600118339061737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213319680, "lr": 1.0849166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715272500, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784715272501, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784715315524, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7808949947357178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784715315525, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784715315525, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784715331082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1016143411397934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1.0865833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715346332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10476227849721909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213975040, "lr": 1.08825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715362212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10582304000854492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1.0899166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715377571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10003211349248886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214630400, "lr": 1.0915833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715393071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10543353855609894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1.09325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715408434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10790842771530151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215285760, "lr": 1.0949166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715423384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10273230820894241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1.096583333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715423385, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784715423385, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784715467102, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7808226943016052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784715467103, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784715467103, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784715482212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10352812707424164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215941120, "lr": 1.0982499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715497420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10097678005695343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1.0999166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715512865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10142894834280014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216596480, "lr": 1.1015833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715528168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09868713468313217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1.10325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715543593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10474147647619247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217251840, "lr": 1.1049166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715558848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11287657916545868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217579520, "lr": 1.1065833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715574406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10075506567955017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217907200, "lr": 1.10825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715574406, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784715574407, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784715617922, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7809662222862244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784715617923, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784715617923, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784715633458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10573391616344452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218234880, "lr": 1.1099166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715648893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10444199293851852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218562560, "lr": 1.1115833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715664659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10225283354520798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218890240, "lr": 1.11325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715680338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10785022377967834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219217920, "lr": 1.1149166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715695560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10405472666025162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219545600, "lr": 1.1165833333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715710737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09791626036167145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219873280, "lr": 1.1182499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715726317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09805934876203537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220200960, "lr": 1.1199166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715726317, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784715726318, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784715770807, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7811769247055054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784715770807, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784715770808, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784715785769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10341829061508179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220528640, "lr": 1.1215833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715801407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09687105566263199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220856320, "lr": 1.12325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715816597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09773723036050797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221184000, "lr": 1.1249166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715831870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10799866914749146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221511680, "lr": 1.1265833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715847250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10168245434761047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221839360, "lr": 1.12825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715862553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10505443811416626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222167040, "lr": 1.1299166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715878222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1131686121225357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222494720, "lr": 1.1315833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715878223, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784715878223, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784715921288, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7816923260688782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784715921289, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784715921289, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784715936539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11018398404121399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222822400, "lr": 1.1332500000000001e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715952094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10372035205364227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223150080, "lr": 1.1349166666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715967617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10097894817590714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223477760, "lr": 1.1365833333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715982806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10914048552513123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223805440, "lr": 1.1382499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715998438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10458207130432129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224133120, "lr": 1.1399166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716013811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10835739225149155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224460800, "lr": 1.1415833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716029473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09103193879127502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224788480, "lr": 1.14325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716029474, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784716029474, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784716072807, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7818500995635986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784716072808, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784716072808, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784716088899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10249608755111694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225116160, "lr": 1.1449166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716104420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1010972261428833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225443840, "lr": 1.1465833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716120173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793283581733704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225771520, "lr": 1.14825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716135840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11021257936954498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226099200, "lr": 1.1499166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716151826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10591515898704529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226426880, "lr": 1.1515833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716167475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10468196868896484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226754560, "lr": 1.1532500000000001e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716183259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09903691709041595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227082240, "lr": 1.1549166666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716183260, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784716183260, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784716227203, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7819221019744873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784716227204, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784716227204, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784716242969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11523161828517914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227409920, "lr": 1.1565833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716258693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10619346052408218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227737600, "lr": 1.1582499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716274278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10157469660043716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228065280, "lr": 1.1599166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716289791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11025890707969666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228392960, "lr": 1.1615833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716305447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10672996193170547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228720640, "lr": 1.1632499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716321235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082482561469078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229048320, "lr": 1.1649166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716336838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09903305023908615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229376000, "lr": 1.1665833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716336839, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784716336839, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784716379875, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7818952798843384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784716379875, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784716379876, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784716395655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11327756941318512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229703680, "lr": 1.16825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716411530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10365606844425201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230031360, "lr": 1.1699166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716427021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10230477154254913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230359040, "lr": 1.1715833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716442882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10398493707180023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230686720, "lr": 1.17325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716458325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10610069334506989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231014400, "lr": 1.1749166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716474062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1018802598118782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231342080, "lr": 1.1765833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716489941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10939688980579376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231669760, "lr": 1.17825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716489942, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784716489942, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784716532952, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7817438244819641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784716532953, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784716532953, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784716548831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11399243772029877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231997440, "lr": 1.1799166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716564602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10808024555444717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 232325120, "lr": 1.1815833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716580333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10715091973543167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 232652800, "lr": 1.1832499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716595527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10702367126941681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 232980480, "lr": 1.1849166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716611939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10295712947845459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233308160, "lr": 1.1865833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716628108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10412858426570892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233635840, "lr": 1.18825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716644293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11826032400131226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233963520, "lr": 1.1899166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716644294, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784716644295, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784716686878, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.78209388256073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784716686879, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784716686879, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784716702758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11450108885765076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 234291200, "lr": 1.1915833333333335e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716718787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10832184553146362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 234618880, "lr": 1.19325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716734644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11416899412870407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 234946560, "lr": 1.1949166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716750850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10442917793989182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235274240, "lr": 1.1965833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716767013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12015669792890549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235601920, "lr": 1.19825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716783226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11344447731971741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235929600, "lr": 1.1999166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716799275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11417818069458008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 236257280, "lr": 1.2015833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716799276, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784716799277, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784716843256, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7820115685462952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784716843257, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784716843257, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784716859158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1160070151090622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 236584960, "lr": 1.2032499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716875602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11353110522031784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 236912640, "lr": 1.2049166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716891748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11229313910007477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237240320, "lr": 1.2065833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716907964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11014613509178162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237568000, "lr": 1.20825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716924282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10993581265211105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237895680, "lr": 1.2099166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716940568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11504900455474854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 238223360, "lr": 1.2115833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716956504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1106758862733841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 238551040, "lr": 1.21325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716956504, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784716956505, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784717000886, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7820450067520142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784717000887, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784717000887, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784717016713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11542415618896484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 238878720, "lr": 1.2149166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717033154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11424928903579712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239206400, "lr": 1.2165833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717049292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11058169603347778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239534080, "lr": 1.21825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717065185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11175692826509476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239861760, "lr": 1.2199166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717081313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11506173014640808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 240189440, "lr": 1.2215833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717097064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11321870982646942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 240517120, "lr": 1.2232499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717113073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11754293739795685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 240844800, "lr": 1.2249166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717113073, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784717113074, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784717157206, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7829115390777588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784717157207, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784717157207, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784717172938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11521229147911072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241172480, "lr": 1.2265833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717189003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11600616574287415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241500160, "lr": 1.22825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717204506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11014876514673233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241827840, "lr": 1.2299166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717220831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1125507801771164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 242155520, "lr": 1.2315833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717236611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11433758586645126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 242483200, "lr": 1.2332499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717252641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11538734287023544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 242810880, "lr": 1.2349166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717268874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10960458964109421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243138560, "lr": 1.2365833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717268874, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784717268875, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784717311937, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7805144190788269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784717311938, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784717311938, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784717327571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1150488406419754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243466240, "lr": 1.23825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717343386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11477524787187576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243793920, "lr": 1.2399166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717359887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10731863230466843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 244121600, "lr": 1.2415833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717375976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10531021654605865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 244449280, "lr": 1.24325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717392443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1066674143075943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 244776960, "lr": 1.2449166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717408495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10795517265796661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245104640, "lr": 1.2465833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717424527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11112558841705322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245432320, "lr": 1.24825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717424528, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784717424528, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784717468206, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7825460433959961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784717468207, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784717468207, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784717484172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10579445958137512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245760000, "lr": 1.2499166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717500455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10413683205842972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 246087680, "lr": 1.2515833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717516405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11334630846977234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 246415360, "lr": 1.2532499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717532791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1121726706624031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 246743040, "lr": 1.2549166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717548809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10444684326648712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247070720, "lr": 1.2565833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717564869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10695438086986542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247398400, "lr": 1.25825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717581087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10873205959796906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247726080, "lr": 1.2599166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717581087, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784717581088, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784717623901, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.783027708530426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784717623901, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784717623902, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784717640188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11269940435886383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 248053760, "lr": 1.2615833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717656596, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11546231806278229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 248381440, "lr": 1.26325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717672857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10124969482421875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 248709120, "lr": 1.2649166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717688705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10925869643688202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249036800, "lr": 1.2665833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717704580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10384128987789154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249364480, "lr": 1.26825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717720962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10944317281246185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249692160, "lr": 1.2699166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717736873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1047864705324173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 250019840, "lr": 1.2715833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717736922, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784717736922, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784717781036, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7825866937637329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784717781037, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784717781037, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784717796774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1005425751209259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 250347520, "lr": 1.2732499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717812350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10118546336889267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 250675200, "lr": 1.2749166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717827466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10084110498428345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251002880, "lr": 1.2765833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717842807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09930329024791718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251330560, "lr": 1.27825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717858769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10094643384218216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251658240, "lr": 1.2799166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717874000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10166378319263458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251985920, "lr": 1.2815833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717889688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10966642946004868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 252313600, "lr": 1.28325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717889736, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784717889737, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784717933603, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.782227098941803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784717933603, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784717933604, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784717949061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08273527026176453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 252641280, "lr": 1.2849166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717964498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10501809418201447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 252968960, "lr": 1.2865833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717980565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10934704542160034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 253296640, "lr": 1.28825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717996524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11578688770532608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 253624320, "lr": 1.2899166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718012372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11452081799507141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 253952000, "lr": 1.2915833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718028265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10457257926464081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 254279680, "lr": 1.2932499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718044249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10367301106452942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 254607360, "lr": 1.2949166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718044295, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784718044296, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784718088783, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7826183438301086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784718088784, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784718088784, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784718104084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10226867347955704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 254935040, "lr": 1.2965833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718119367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1029936820268631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 255262720, "lr": 1.2982499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718135702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10811120271682739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 255590400, "lr": 1.2999166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718151413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10833308100700378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 255918080, "lr": 1.3015833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718167184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1073065996170044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 256245760, "lr": 1.30325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718182833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09855342656373978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 256573440, "lr": 1.3049166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718198279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1066572442650795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 256901120, "lr": 1.3065833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718198279, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784718198279, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784718242581, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7819464206695557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784718242582, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784718242582, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784718258162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10296022146940231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 257228800, "lr": 1.30825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718274061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10190264880657196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 257556480, "lr": 1.3099166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718289591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10654173046350479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 257884160, "lr": 1.3115833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718305029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10834093391895294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 258211840, "lr": 1.3132499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718320666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10573046654462814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 258539520, "lr": 1.3149166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718336135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10643549263477325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 258867200, "lr": 1.3165833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718351895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10566719621419907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 259194880, "lr": 1.3182499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718351896, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784718351896, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784718395495, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7822108268737793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784718395495, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784718395495, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784718411478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10963568091392517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 259522560, "lr": 1.3199166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718427558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10777470469474792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 259850240, "lr": 1.3215833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718443415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10602708160877228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 260177920, "lr": 1.32325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718459425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1097845509648323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 260505600, "lr": 1.3249166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718475296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10197171568870544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 260833280, "lr": 1.3265833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718491167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10104045271873474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 261160960, "lr": 1.32825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718507347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10802723467350006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 261488640, "lr": 1.3299166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718507348, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784718507348, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784718551885, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7829669117927551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784718551886, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784718551886, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784718567295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10598933696746826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 261816320, "lr": 1.3315833333333332e-06}} diff --git a/recommendation_v4/rcp_logs/gbs_16384/seed462193368.log b/recommendation_v4/rcp_logs/gbs_16384/seed462193368.log new file mode 100644 index 000000000..e72ee56bb --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_16384/seed462193368.log @@ -0,0 +1,1380 @@ +:::MLLOG {"namespace": "", "time_ms": 1784719270658, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784719270658, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784719291040, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784719291041, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784719291041, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784719291041, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784719291041, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784719291041, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 16384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784719291041, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784719291042, "event_type": "POINT_IN_TIME", "key": "seed", "value": 462193368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784719291042, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784719291042, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784719291042, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784719291042, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784719291091, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784719291092, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784719876257, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784719876258, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784719876555, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784720357924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13893669843673706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.5833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784720371866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13896781206130981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2499999999999997e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784720385271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388426572084427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784720399131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138778418302536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.583333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784720412804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13870219886302948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.25e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784720426662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860449194908142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784720440126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13908934593200684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720440127, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784720440127, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784720526866, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4975312352180481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784720526867, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784720526867, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784720540471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13892878592014313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720554134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388140767812729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720568027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13861927390098572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720581861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386427879333496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720595716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13866238296031952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9916666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720609983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13871459662914276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720624398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13808685541152954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720624399, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784720624399, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784720667895, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4977196753025055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784720667896, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784720667896, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784720681810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13851550221443176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720696162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13847818970680237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720710122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385708600282669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720724374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383846402168274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.9916666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720738050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13840961456298828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720752444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1392565220594406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720766486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13824033737182617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720766487, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784720766487, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784720809273, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4979620575904846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784720809274, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784720809274, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784720823417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13853728771209717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720837823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13830606639385223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.8249999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720851652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13778305053710938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720865724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13828372955322266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720880447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13787047564983368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3249999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720894449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13777440786361694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720908886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13773104548454285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720908887, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784720908887, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784720951396, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4982835054397583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784720951396, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784720951396, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784720964934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13731138408184052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720978717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378984898328781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720992917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13771380484104156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1583333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721006863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373315155506134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721020991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364738792181015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721035088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13734737038612366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721049239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364866942167282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721049240, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784721049240, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784721091998, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4986475110054016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784721091999, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784721091999, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784721105619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13718420267105103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721119136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13669535517692566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721133085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13765046000480652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721147357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13696137070655823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721161213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13564667105674744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.658333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721175040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359473615884781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721189013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137473464012146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721189014, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784721189014, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784721232643, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49907079339027405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784721232643, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784721232644, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784721247000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13716834783554077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721260917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13618701696395874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721274656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13617223501205444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721288690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353398561477661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721302859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13554197549819946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721317492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13514447212219238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721331800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13544785976409912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721331801, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784721331801, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784721374058, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49958303570747375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784721374059, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784721374059, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784721388478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347178965806961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721403022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366688758134842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721417656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13807165622711182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721431890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13444191217422485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.824999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721446306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13624411821365356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721460470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13516557216644287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.158333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721475060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367613971233368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.324999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721475061, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784721475062, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784721518073, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5003060698509216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784721518074, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784721518074, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784721532598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13657723367214203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721547127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367053985595703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721561753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350792646408081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721576340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310189962387085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721590658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13303568959236145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721604714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13403113186359406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721618614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351870596408844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721618665, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784721618665, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784721661495, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5011168122291565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784721661495, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784721661495, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784721675621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350448727607727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721689585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.130008727312088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721704682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13181743025779724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721718736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13678358495235443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.1158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721732884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13265156745910645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721746970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13188216090202332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721761520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13270971179008484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721761520, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784721761521, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784721805148, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5019113421440125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784721805149, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784721805149, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784721818996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13203981518745422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721833597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13437283039093018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721847326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12853457033634186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721861217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12997624278068542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721875328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13385823369026184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721889696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12868638336658478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721903570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13014987111091614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721903571, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784721903571, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784721948285, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5030502676963806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784721948286, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784721948286, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784721962816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344757378101349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721976792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13308599591255188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.3158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721990990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13448700308799744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722005066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13503821194171906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722019637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359674334526062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.3658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722033868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13352365791797638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722048336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13690617680549622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722048337, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784722048337, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784722091257, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5046063661575317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784722091258, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784722091258, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784722105822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13429683446884155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.4158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722120633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13058871030807495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722135365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13394570350646973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.4491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722150466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14039936661720276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.4658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722165185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13055747747421265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722180080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13249847292900085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722194536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13813847303390503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722194537, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784722194537, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784722238212, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5061318278312683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784722238213, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784722238213, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784722252958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12393048405647278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722267402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13125941157341003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722282293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13715136051177979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722296924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13542446494102478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722311912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323881447315216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722326331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12883734703063965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.6158333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722340812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12807968258857727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722340812, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784722340813, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784722383145, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5077716112136841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784722383146, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784722383147, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784722397573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13011398911476135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722412095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13480791449546814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.6658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722426951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14227595925331116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6825000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722441480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13387781381607056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722456065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131576806306839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.7158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722470340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13375312089920044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722484878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13604183495044708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722484878, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784722484879, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784722527446, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5096477270126343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784722527447, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784722527447, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784722541893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346476972103119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722556466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13641908764839172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722571245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356702595949173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722585912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13323161005973816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722600556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13444699347019196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722615084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13297362625598907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722629417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13256192207336426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8658333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722629418, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784722629418, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784722672534, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5116674900054932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784722672535, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784722672535, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784722686741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13202986121177673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722701232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13435685634613037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8991666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722715853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.141645610332489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722730941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13347189128398895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722745778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13422992825508118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722760484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324262022972107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722775005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331416666507721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722775006, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784722775006, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784722818420, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5139864683151245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784722818421, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784722818421, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784722832767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13871824741363525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722847287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13315482437610626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722861585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12878021597862244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722876117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13002726435661316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722890874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13791847229003906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722905793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315857172012329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722920249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363862305879593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722920250, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784722920250, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784722963919, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5166791081428528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784722963919, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784722963919, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784722978492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13674697279930115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.1158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722992696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13646166026592255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723007224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13147017359733582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1491666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723021542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300285905599594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723036037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131544291973114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723050856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13530004024505615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723065671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13265833258628845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723065672, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784723065672, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784723109393, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5197435021400452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784723109394, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784723109394, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784723123928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13595983386039734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723139235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13234946131706238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723154474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13795632123947144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723169437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13634918630123138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723184608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13856303691864014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723199582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344960331916809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723214891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13916543126106262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723214891, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784723214892, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784723258242, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5228663086891174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784723258242, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784723258242, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784723273414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13497188687324524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723288579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.15076710283756256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723303918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13515877723693848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723318813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13298112154006958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723333965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13426166772842407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723348786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331261694431305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723363754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365644931793213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723363755, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784723363755, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784723406214, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5266435146331787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784723406214, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784723406215, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784723421285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13609597086906433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723436376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135352224111557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723451901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13377362489700317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723466742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13355298340320587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723481565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332196593284607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723496435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294664442539215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723511202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1256832629442215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723511202, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784723511215, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784723554249, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5311552882194519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784723554274, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784723554275, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784723569012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13269229233264923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.5824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723584064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13245698809623718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723598906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349279135465622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723613940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13523846864700317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723629403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13669750094413757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723644514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288905143737793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723659430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12984086573123932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723659431, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784723659443, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784723702642, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5371183156967163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784723702666, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784723702666, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784723717431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13262006640434265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.699166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723732126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1270495355129242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723746942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13322393596172333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723761870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13222065567970276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723777117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12705254554748535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723792641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13233700394630432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723807261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13426288962364197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723807262, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784723807262, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784723851476, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5451311469078064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784723851477, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784723851477, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784723866458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294240951538086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723881067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304645836353302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723895596, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13064908981323242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723910290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297428011894226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.8658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723924523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12789687514305115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723939090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12324057519435883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723954129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12725773453712463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723954130, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784723954131, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784723997248, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5555940866470337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784723997248, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784723997249, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784724012097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358340084552765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724027037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333588808774948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724041453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13916289806365967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724056701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13345454633235931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724071620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12780356407165527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724086372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12967276573181152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724101435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12692376971244812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724101436, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784724101436, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784724144684, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5680530667304993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784724144684, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784724144685, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784724159154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12907150387763977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724173958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12757593393325806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724189103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13253918290138245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724204343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13266819715499878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724219737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13147008419036865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724234711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13362927734851837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724249833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13060015439987183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724249834, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784724249834, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784724293407, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5810766220092773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784724293407, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784724293408, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784724308306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12949863076210022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724323120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297198086977005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724337948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12988050282001495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724352774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13279208540916443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724367501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13362397253513336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724382061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12953057885169983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724397348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13044844567775726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724397349, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784724397349, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784724440477, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5932912230491638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784724440477, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784724440477, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784724455381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12837481498718262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724470574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13109003007411957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724485024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12858253717422485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724500797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12794435024261475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724516254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12747445702552795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724531582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13191258907318115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724546970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327086240053177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724546971, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784724546971, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784724590388, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6071704626083374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784724590389, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784724590389, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784724605126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12987375259399414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724620279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12825949490070343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724635701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13038456439971924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724651127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13208064436912537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724666577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13332387804985046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724681544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12944380939006805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724697100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12901416420936584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.4991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724697101, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784724697102, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784724740645, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6233689188957214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784724740646, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784724740646, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784724755474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12970036268234253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.5158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724770446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1278052031993866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724785640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12927353382110596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.5491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724800662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13537994027137756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724815865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13060256838798523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724831281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12607181072235107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724846870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294553279876709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724846870, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784724846871, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784724891749, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6358329653739929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784724891750, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784724891750, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784724906773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12616634368896484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6325000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724922333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12818750739097595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724937362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12717963755130768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724952151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12721920013427734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724966882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12541484832763672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724981664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12246383726596832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724996467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12382277846336365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724996468, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784724996468, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784725039847, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.651576042175293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784725039847, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784725039847, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784725055274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12696433067321777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725070502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12304140627384186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725085609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1165928766131401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.7824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725100599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12262792885303497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725115782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11833040416240692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725130604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11940717697143555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8324999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725145623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11700749397277832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.8491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725145623, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784725145624, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784725189348, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6671977043151855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784725189349, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784725189349, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784725204456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12235084176063538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725219247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11795192956924438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725233961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11461281031370163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.8991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725248724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12096667289733887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725263931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1193600594997406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725279245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1181427612900734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.9491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725294193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10924862325191498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.9658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725294193, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784725294194, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784725338153, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7128434777259827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784725338153, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784725338154, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784725352848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11604639887809753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725367939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11600340902805328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725382653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11969386041164398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.0158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725397346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11478599905967712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725412115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11331582814455032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725426708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11416161805391312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0658333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725441551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10969021916389465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725441597, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784725441598, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784725484549, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7390022277832031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784725484550, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784725484550, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784725499183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11466140300035477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725514520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1127132847905159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.1158333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725529627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11796179413795471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725545124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11259748041629791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725559941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11219245195388794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1658333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725575547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11931085586547852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725590448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10730534046888351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725590448, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784725590449, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784725633873, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7554304599761963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784725633873, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784725633874, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784725648721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11500144004821777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725663400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.113107830286026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725678431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11079579591751099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725693179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10985603928565979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725708094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10425853729248047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725722787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11695531755685806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725738287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11277531087398529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725738287, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784725738288, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784725781298, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7614619135856628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784725781299, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784725781299, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784725796420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11589442193508148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725811868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11296522617340088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.3491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725826684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11516264081001282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725841873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10125316679477692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725856668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11403702199459076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.3991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725871364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10733340680599213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.4158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725886020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10650570690631866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725886020, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784725886021, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784725929682, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7639240026473999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784725929683, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784725929683, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784725944231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10851176828145981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.4491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725958785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10760509967803955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725973092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11270982027053833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.4824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725987762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10858064889907837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726002784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11191538721323013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726017629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10194587707519531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726032594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11031801253557205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726032595, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784726032596, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784726077720, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7652116417884827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784726077720, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784726077720, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784726092578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10651849210262299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726107171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10842617601156235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.5824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726122683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11920344829559326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726138033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1214369386434555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726153063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11404015868902206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726168226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11083677411079407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726183312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11496612429618835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726183312, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784726183313, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784726227192, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7666016221046448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784726227193, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784726227193, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784726242014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11458340287208557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726257237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10774167627096176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.6991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726272672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11747124791145325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726288062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11016599833965302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726303714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11381636559963226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726318976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11403126269578934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726334429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10440704226493835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726334429, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784726334430, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784726377076, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7677991390228271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784726377077, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784726377077, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784726392096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11834365129470825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726407078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10652053356170654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726422124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10515066981315613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726437012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11292680352926254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726451818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11846679449081421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726466749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10726063698530197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726482147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11430469155311584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726482147, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784726482148, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784726525041, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7690052390098572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784726525041, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784726525042, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784726540034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11494681984186172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726555537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10713953524827957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726570266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10853086411952972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726585600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11336154490709305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726600548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.107160285115242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726615263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10399685800075531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726629976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1167742982506752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.015833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726629977, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784726629977, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784726672443, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7696030735969543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784726672444, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784726672444, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784726687182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10935831815004349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726702001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1146104708313942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726716664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10595807433128357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726731264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10469216108322144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726746346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10164809226989746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726761346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09619437158107758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726776602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10640332102775574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726776603, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784726776603, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784726819291, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7702373266220093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784726819292, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784726819292, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784726836184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10473702847957611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726850759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11028195917606354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726867496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11799097061157227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726883438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10970813035964966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726901224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10574804246425629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726918763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10090966522693634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726939534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10649309307336807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726939535, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784726939535, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784726981940, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7708130478858948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784726981941, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784726981941, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784727003769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10527485609054565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727028385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10580509901046753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727048700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10933275520801544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727063847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10678771883249283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727079013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10611891746520996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727094163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11158455908298492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727109572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1092674732208252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727109573, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784727109573, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784727152788, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7714694738388062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784727152788, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784727152788, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784727167647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10437706857919693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727182885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10235366225242615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727198069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10832292586565018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727213341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10899942368268967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727228350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1047234982252121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727243455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10086062550544739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727258575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10460534691810608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727258575, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784727258576, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784727300567, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7717311382293701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784727300567, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784727300567, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784727315471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10445437580347061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727330558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10021249949932098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727345317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10490051656961441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727360521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11008711904287338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727375614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10552039742469788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727391020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10266974568367004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727406432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10964898020029068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727406433, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784727406447, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784727449840, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7723022103309631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784727449866, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784727449866, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784727464769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10027772188186646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727480406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10338522493839264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727495709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10640020668506622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727511095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1035483330488205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727526324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10232290625572205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727541526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10633393377065659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727556786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10897031426429749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727556786, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784727556799, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784727600468, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.772521436214447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784727600494, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784727600495, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784727615708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10571062564849854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727631013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10418636351823807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727646010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10774342715740204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727661517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10584823787212372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727677124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10924822092056274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727692801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10750848054885864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727708433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11075285077095032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727708434, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784727708447, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784727751463, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730774879455566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784727751489, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784727751489, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784727766576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10668856650590897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727781935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10163629055023193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727797026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10422785580158234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727812157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10186198353767395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727827111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10828734934329987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727841972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11328025907278061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727856864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10798776149749756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727856864, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784727856865, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784727900516, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7732545733451843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784727900517, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784727900517, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784727915494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10313259810209274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727930438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11139428615570068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727945480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10767336189746857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727960725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10034355521202087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727976027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1116548478603363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727991271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11241618543863297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728006994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10463912785053253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728006995, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784728006995, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784728050870, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7733749747276306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784728050870, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784728050870, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784728066104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10780662298202515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728081046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09984919428825378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728097188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11602994054555893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728113005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11504407227039337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728128496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1144031286239624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728144001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039624810218811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728159669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10660772770643234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728159669, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784728159670, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784728203750, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773219108581543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784728203751, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784728203751, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784728219470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11076157540082932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728234799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1137712299823761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728250377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10980388522148132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728266099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10983070731163025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728281697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11419390141963959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728297230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11192235350608826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728313089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11298248171806335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728313090, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784728313090, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784728356724, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7735974192619324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784728356725, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784728356725, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784728371863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1093788668513298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728387499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10712501406669617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728403223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723394900560379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728419156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11428209394216537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728434585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11844397336244583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728450172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1095581203699112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728466033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11558160930871964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.415833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728466034, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784728466034, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784728510431, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7740015387535095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784728510431, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784728510431, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784728525853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11359748244285583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728541404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11344676464796066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728557091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11566121876239777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728572772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11367519199848175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728588619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10926980525255203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728604193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10897397249937057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.515833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728619381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11195182800292969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728619382, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784728619382, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784728663727, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7741607427597046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784728663727, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784728663727, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784728679357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10867741703987122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728694965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10465399920940399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728709993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10442653298377991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728725325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10400320589542389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728740427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1065400093793869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.615833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728756022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10259495675563812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728771257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10788720101118088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728771257, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784728771258, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784728814370, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746317386627197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784728814370, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784728814371, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784728829734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10916749387979507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728844782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10245875269174576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728860381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772121697664261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728875948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10330729931592941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728891648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10202420502901077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728907517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10548358410596848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728922907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09969821572303772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728922907, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784728922908, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784728967826, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7747117280960083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784728967827, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784728967827, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784728982813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10788844525814056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728998415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1101015955209732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729013723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10629718750715256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729028856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10617353022098541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729043830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10698670148849487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729058594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10353559255599976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729073597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1073928028345108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729073597, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784729073598, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784729117639, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7750006914138794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784729117640, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784729117640, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784729132662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772106051445007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729147534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1052151769399643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729162672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10849309712648392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729177766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10293030738830566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729193146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10899125039577484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729208531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10936196148395538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729224111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10529421269893646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729224111, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784729224112, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784729269106, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7751185297966003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784729269107, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784729269107, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784729284549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10761916637420654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729299506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10171954333782196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729314815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11166489124298096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729329882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10243584215641022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729345510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10017955303192139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729360866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10245376825332642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729376084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10518445074558258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729376085, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784729376085, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784729420308, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7753562331199646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784729420308, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784729420308, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784729435388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10327232629060745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729450830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10043381154537201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729466394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10823230445384979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729481628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1094655692577362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729496656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10471397638320923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729512058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10771641135215759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729527458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10915465652942657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729527458, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784729527459, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784729571925, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7755560874938965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784729571925, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784729571925, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784729587383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11057352274656296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729603002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10432210564613342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729618431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10408469289541245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729633364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10543924570083618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729648866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09980843961238861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729664030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10731787979602814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729679715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10442028939723969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729679716, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784729679716, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784729723420, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7756752371788025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784729723421, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784729723422, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784729738687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10656488686800003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729753946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10809750109910965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729769367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10004767775535583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729784822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10903456807136536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729800309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09654764086008072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729815964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10214927792549133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729831043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10398636758327484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729831044, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784729831044, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784729874408, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7758289575576782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784729874408, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784729874409, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784729889905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1021241620182991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729905587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10806003212928772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729921211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10854384303092957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729937167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11031462252140045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729952915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10790829360485077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729968094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10877801477909088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.565833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729983676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1014413982629776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729983677, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784729983678, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784730027229, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7761046886444092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784730027229, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784730027229, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784730042319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10986441373825073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730058014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10407352447509766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149749760, "lr": 7.615833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730073232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10669203102588654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730088440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11145322024822235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150405120, "lr": 7.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730103427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10806909948587418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730118769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11233828961849213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151060480, "lr": 7.682499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730133945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10425819456577301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730133945, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784730133946, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784730177553, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7760875225067139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784730177554, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784730177554, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784730192849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10647537559270859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151715840, "lr": 7.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730208263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.098326176404953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730223593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11001647263765335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152371200, "lr": 7.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730238931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10427667200565338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730254526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11142121255397797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153026560, "lr": 7.782499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730270098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1123659536242485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730286061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10411910712718964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153681920, "lr": 7.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730286061, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784730286062, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784730330518, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7764241099357605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784730330518, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784730330518, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784730345931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10268926620483398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730361028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1063736081123352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154337280, "lr": 7.849166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730377167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10200667381286621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730392832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11045487225055695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154992640, "lr": 7.882499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730408709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10964588820934296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730424359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1185777336359024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155648000, "lr": 7.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730440013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11743141710758209, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730440014, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784730440014, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784730484016, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7766559720039368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784730484016, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784730484016, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784730499441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11783412843942642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156303360, "lr": 7.949166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730515101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10979273915290833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730530503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11455871164798737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156958720, "lr": 7.982499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730546077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11173561215400696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730561685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11458539962768555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157614080, "lr": 8.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730577828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11688542366027832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730593633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10838210582733154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158269440, "lr": 8.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730593633, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784730593634, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784730637491, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7767434120178223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784730637491, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784730637492, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784730653261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11091819405555725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730669285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10443833470344543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158924800, "lr": 8.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730684703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1119924858212471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.099166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730700699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11189533770084381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159580160, "lr": 8.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730716637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10967712849378586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730732784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10711801052093506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160235520, "lr": 8.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730748696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1099385991692543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730748696, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784730748697, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784730792913, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7768189311027527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784730792913, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784730792913, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784730808455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11341968178749084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160890880, "lr": 8.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730824137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10470253974199295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.199166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730840128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082114726305008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161546240, "lr": 8.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730856020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10938552767038345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730871914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.115289106965065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162201600, "lr": 8.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730888294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11134612560272217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.265833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730904387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11574308574199677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162856960, "lr": 8.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730904387, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784730904388, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784730947685, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.777043342590332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784730947685, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784730947685, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784730964062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11603327095508575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.299166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730980203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11454711854457855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163512320, "lr": 8.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730995715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11673632264137268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731011761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1038413941860199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164167680, "lr": 8.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731027431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10145619511604309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731043001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10611574351787567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164823040, "lr": 8.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731058535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1058407723903656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731058535, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784731058536, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784731103512, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7773395776748657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784731103513, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784731103513, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784731118784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10958661139011383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165478400, "lr": 8.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731134221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10770374536514282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731149680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10126145929098129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166133760, "lr": 8.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731165237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10472145676612854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731180517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10395751893520355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166789120, "lr": 8.482499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731195707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0983285903930664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.499166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731211403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10527652502059937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167444480, "lr": 8.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731211404, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784731211404, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784731255483, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7774782180786133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784731255484, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784731255484, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784731270865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10588622093200684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.532499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731286549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09952152520418167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168099840, "lr": 8.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731302252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09394059330224991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731317702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09963355958461761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168755200, "lr": 8.582499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731332851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11210288107395172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731348319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10495726019144058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169410560, "lr": 8.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731363984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11003443598747253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731363984, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784731363985, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784731406920, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7778375744819641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784731406920, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784731406920, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784731422762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10558995604515076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170065920, "lr": 8.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731438260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1028808206319809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731453668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11477480828762054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170721280, "lr": 8.682499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731468984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09771712124347687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731484611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10637752711772919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171376640, "lr": 8.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731500078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10384131968021393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731515740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10045450925827026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172032000, "lr": 8.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731515740, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784731515741, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784731558682, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7780590057373047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784731558682, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784731558682, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784731574151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09988564252853394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731589641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10553757846355438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172687360, "lr": 8.782499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731605573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10593345761299133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731621323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10713745653629303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173342720, "lr": 8.815833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731637314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10246780514717102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731653090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10314980149269104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173998080, "lr": 8.849166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731668761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10500646382570267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.865833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731668761, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784731668762, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784731711707, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7783434987068176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784731711709, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784731711709, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784731726941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10200902074575424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174653440, "lr": 8.882499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731742391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10553476214408875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731757558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10226352512836456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175308800, "lr": 8.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731772875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11073602735996246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731788099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1064261719584465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175964160, "lr": 8.949166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731803118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10203736275434494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.965833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731818359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11312112957239151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176619520, "lr": 8.982499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731818360, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784731818360, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784731861104, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7788947224617004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784731861105, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784731861105, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784731876304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10202586650848389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731891408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10295695066452026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177274880, "lr": 9.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731906541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0998462438583374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731921633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10262124240398407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177930240, "lr": 9.049166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731937108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10562968254089355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.065833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731952664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10979302227497101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178585600, "lr": 9.082499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731968143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09899887442588806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731968143, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784731968148, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784732010883, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.779147207736969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784732010884, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784732010884, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784732026900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10109525918960571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179240960, "lr": 9.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732042648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09853309392929077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732057936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10031123459339142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179896320, "lr": 9.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732072858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09779283404350281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.165833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732090742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10587599873542786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180551680, "lr": 9.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732106177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1034076139330864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732121612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10708893835544586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181207040, "lr": 9.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732121613, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784732121614, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784732164557, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7794066667556763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784732164558, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784732164559, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784732179673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10548287630081177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732194605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09766482561826706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181862400, "lr": 9.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732209616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10431067645549774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.265833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732227004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09863699972629547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182517760, "lr": 9.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732241996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09912759810686111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.299166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732257109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10374850034713745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183173120, "lr": 9.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732272094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10353760421276093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732272094, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784732272096, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784732315478, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7796381711959839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784732315480, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784732315480, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784732330764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09808877110481262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183828480, "lr": 9.349166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732346304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10627345740795135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732361697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10275260359048843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184483840, "lr": 9.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732377318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10955418646335602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732392805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10757777839899063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185139200, "lr": 9.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732408110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11022800207138062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732423142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10248042643070221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185794560, "lr": 9.449166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732423142, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784732423143, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784732466546, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7799597382545471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784732466547, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784732466548, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784732482055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10116896033287048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732497234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10965381562709808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186449920, "lr": 9.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732512977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10053612291812897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.499166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732528333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1044645607471466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187105280, "lr": 9.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732543693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09934377670288086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732559086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10965455323457718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187760640, "lr": 9.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732574343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09953175485134125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.565833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732574344, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784732574344, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784732617628, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7801960706710815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784732617629, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784732617629, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784732632978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1049535870552063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188416000, "lr": 9.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732648397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10175870358943939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732663666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10629284381866455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189071360, "lr": 9.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732679111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10320675373077393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732694687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10654442012310028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189726720, "lr": 9.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732710315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11273766309022903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732726088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1018814742565155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190382080, "lr": 9.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732726089, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784732726089, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784732768452, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.780381977558136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784732768453, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784732768453, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784732784333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10607306659221649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.699166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732800285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10110554099082947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191037440, "lr": 9.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732815963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10235553234815598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732831558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10638972371816635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191692800, "lr": 9.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732847276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10708479583263397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.765833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732862796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10779450088739395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192348160, "lr": 9.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732878427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1136423796415329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732878428, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784732878428, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784732921571, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7807738184928894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784732921572, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784732921572, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784732937193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10456795990467072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193003520, "lr": 9.815833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732952678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10840998589992523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.832499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732968225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1110549196600914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193658880, "lr": 9.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732983902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11367947608232498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732999612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084848940372467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194314240, "lr": 9.882499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784733015344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10535597801208496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.899166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784733030943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10610745847225189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194969600, "lr": 9.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784733030944, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784733030945, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784733073994, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7809861302375793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784733073995, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784733073995, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784733089851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11073896288871765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784733105661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11055272817611694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195624960, "lr": 9.949166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784733121519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10965836048126221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.965833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784733137733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11174846440553665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196280320, "lr": 9.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784733153639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10840727388858795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784733169153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11394806206226349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196935680, "lr": 1.0015833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733184663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10287575423717499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1.0032499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733184663, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784733184664, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784733228157, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7813759446144104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784733228158, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784733228158, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784733244045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10765742510557175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197591040, "lr": 1.0049166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733260455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11114536225795746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1.0065833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733276192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10581330955028534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198246400, "lr": 1.00825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733291833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1130990982055664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1.0099166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733307362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.113710418343544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198901760, "lr": 1.0115833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733323170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10828092694282532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1.01325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733338806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11294449865818024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199557120, "lr": 1.0149166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733338806, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784733338806, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784733383184, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7820587754249573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784733383185, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784733383185, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784733398724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11473743617534637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1.0165833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733414593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10835939645767212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200212480, "lr": 1.01825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733430528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11286681890487671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1.0199166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733446426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11069492995738983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200867840, "lr": 1.0215833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733462564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10683862864971161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1.02325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733478458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10448743402957916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201523200, "lr": 1.0249166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733493875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11069124937057495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1.0265833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733493876, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784733493876, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784733537048, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7820477485656738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784733537049, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784733537050, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784733552751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10001949965953827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202178560, "lr": 1.02825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733567717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1061709076166153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1.0299166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733583330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1019812524318695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202833920, "lr": 1.0315833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733598538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10486561805009842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1.03325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733613621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10386441648006439, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203489280, "lr": 1.0349166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733628506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11110839247703552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1.0365833333333335e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733643560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09766088426113129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204144640, "lr": 1.0382499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733643561, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784733643561, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784733686757, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.781900942325592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784733686758, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784733686758, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784733701998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09990837424993515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1.0399166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733717357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10372111201286316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204800000, "lr": 1.0415833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733732606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10070177167654037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1.04325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733747869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09766922891139984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205455360, "lr": 1.0449166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733763289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10110069811344147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1.0465833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733778731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10156048834323883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206110720, "lr": 1.0482499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733794193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10405881702899933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1.0499166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733794193, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784733794194, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784733837830, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.782390832901001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784733837830, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784733837830, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784733853532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10790228098630905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206766080, "lr": 1.0515833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733869142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10149426758289337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1.05325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733884555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09235085546970367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207421440, "lr": 1.0549166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733899696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10384237766265869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1.0565833333333335e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733915510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10180173069238663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208076800, "lr": 1.0582499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733930768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10155151784420013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1.0599166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733946554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1046365350484848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208732160, "lr": 1.0615833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733946555, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784733946555, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784733990429, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7823275923728943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784733990430, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784733990430, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784734005920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0954965353012085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1.06325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734021423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1053522527217865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209387520, "lr": 1.0649166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734036622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10818882286548615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1.0665833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734052009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09888565540313721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210042880, "lr": 1.06825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734067567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10147744417190552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1.0699166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734082833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10184995085000992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210698240, "lr": 1.0715833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734097948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10384805500507355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1.07325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734097948, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784734097949, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784734140854, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7825010418891907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784734140855, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784734140855, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784734155895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1020069420337677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211353600, "lr": 1.0749166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734171499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10877172648906708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1.076583333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734186844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09560234844684601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212008960, "lr": 1.0782499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734201967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10304555296897888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1.0799166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734217708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10216248035430908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212664320, "lr": 1.0815833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734233474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0982958972454071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1.08325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734248903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10568936169147491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213319680, "lr": 1.0849166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734248903, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784734248904, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784734292192, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7827907204627991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784734292193, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784734292193, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784734307571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10182933509349823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1.0865833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734322664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10362203419208527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213975040, "lr": 1.08825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734338184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10622374713420868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1.0899166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734353807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0993339866399765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214630400, "lr": 1.0915833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734369386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10555823147296906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1.09325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734384827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10728155076503754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215285760, "lr": 1.0949166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734400011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10263123363256454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1.096583333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734400011, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784734400012, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784734442765, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7828752994537354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784734442765, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784734442765, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784734457991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10375629365444183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215941120, "lr": 1.0982499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734473425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10100289434194565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1.0999166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734488861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10115846991539001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216596480, "lr": 1.1015833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734504170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09910757839679718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1.10325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734519546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10462293028831482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217251840, "lr": 1.1049166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734534626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11273322999477386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217579520, "lr": 1.1065833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734550008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10095466673374176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217907200, "lr": 1.10825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734550008, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784734550009, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784734592370, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7827110886573792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784734592371, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784734592371, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784734607760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10541138052940369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218234880, "lr": 1.1099166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734623168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10464529693126678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218562560, "lr": 1.1115833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734638883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10208207368850708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218890240, "lr": 1.11325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734654590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10764418542385101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219217920, "lr": 1.1149166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734669919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10399505496025085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219545600, "lr": 1.1165833333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734684973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09769628942012787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219873280, "lr": 1.1182499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734700384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09809520840644836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220200960, "lr": 1.1199166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734700384, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784734700385, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784734744671, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.783117413520813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784734744672, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784734744672, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784734759628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10293124616146088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220528640, "lr": 1.1215833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734775402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09693505614995956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220856320, "lr": 1.12325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734790687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09725377708673477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221184000, "lr": 1.1249166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734805950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792208462953568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221511680, "lr": 1.1265833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734821275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10158513486385345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221839360, "lr": 1.12825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734836524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10487066209316254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222167040, "lr": 1.1299166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734852175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11285562813282013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222494720, "lr": 1.1315833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734852176, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784734852177, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784734895728, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7834545373916626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784734895729, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784734895729, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784734910974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11042782664299011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222822400, "lr": 1.1332500000000001e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734926395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10354583710432053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223150080, "lr": 1.1349166666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734941720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10081326961517334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223477760, "lr": 1.1365833333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734956999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087978333234787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223805440, "lr": 1.1382499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734972737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10450433194637299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224133120, "lr": 1.1399166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734988443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10800415277481079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224460800, "lr": 1.1415833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735004250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09056134521961212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224788480, "lr": 1.14325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735004250, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784735004251, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784735048753, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7837228775024414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784735048753, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784735048753, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784735064694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10273706912994385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225116160, "lr": 1.1449166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735080227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10146333277225494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225443840, "lr": 1.1465833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735095748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10864144563674927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225771520, "lr": 1.14825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735111290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10987287759780884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226099200, "lr": 1.1499166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735126833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.105409637093544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226426880, "lr": 1.1515833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735142631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10432331264019012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226754560, "lr": 1.1532500000000001e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735158400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0991472452878952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227082240, "lr": 1.1549166666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735158400, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784735158401, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784735202501, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7839759588241577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784735202502, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784735202502, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784735218121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11505156755447388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227409920, "lr": 1.1565833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735233721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10613352060317993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227737600, "lr": 1.1582499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735249327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10126958787441254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228065280, "lr": 1.1599166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735264954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10961239039897919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228392960, "lr": 1.1615833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735280448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10631899535655975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228720640, "lr": 1.1632499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735296079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10863429307937622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229048320, "lr": 1.1649166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735311407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09883023798465729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229376000, "lr": 1.1665833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735311408, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784735311408, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784735356462, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7840574979782104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784735356463, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784735356463, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784735371944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1127198189496994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229703680, "lr": 1.16825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735387477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10379203408956528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230031360, "lr": 1.1699166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735402681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10215102136135101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230359040, "lr": 1.1715833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735418433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10417039692401886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230686720, "lr": 1.17325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735433846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10600510239601135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231014400, "lr": 1.1749166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735449388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10207997262477875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231342080, "lr": 1.1765833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735465244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10938268899917603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231669760, "lr": 1.17825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735465245, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784735465245, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784735509525, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7840730547904968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784735509526, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784735509526, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784735525172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11381416022777557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231997440, "lr": 1.1799166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735540709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10802052170038223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 232325120, "lr": 1.1815833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735556435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1070374995470047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 232652800, "lr": 1.1832499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735571647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.107216015458107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 232980480, "lr": 1.1849166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735587500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10255666077136993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233308160, "lr": 1.1865833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735603463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10359667241573334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233635840, "lr": 1.18825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735619323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11737973988056183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233963520, "lr": 1.1899166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735619324, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784735619324, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784735663529, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.784687876701355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784735663530, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784735663530, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784735679280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1145341545343399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 234291200, "lr": 1.1915833333333335e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735694990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10826212167739868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 234618880, "lr": 1.19325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735710606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11363540589809418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 234946560, "lr": 1.1949166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735726683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10451896488666534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235274240, "lr": 1.1965833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735742555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12105835974216461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235601920, "lr": 1.19825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735758554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11337463557720184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235929600, "lr": 1.1999166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735774238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11337663978338242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 236257280, "lr": 1.2015833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735774239, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784735774239, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784735818458, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7846050262451172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784735818458, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784735818458, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784735834028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11679750680923462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 236584960, "lr": 1.2032499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735849976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11328455805778503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 236912640, "lr": 1.2049166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735865666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11281918734312057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237240320, "lr": 1.2065833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735881249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11013953387737274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237568000, "lr": 1.20825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735897143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10964155197143555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237895680, "lr": 1.2099166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735913093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1148909330368042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 238223360, "lr": 1.2115833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735928636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10988613963127136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 238551040, "lr": 1.21325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735928637, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784735928637, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784735973497, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7846099734306335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784735973498, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784735973498, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784735988948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11556240916252136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 238878720, "lr": 1.2149166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736005115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11456804722547531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239206400, "lr": 1.2165833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736021243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11038945615291595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239534080, "lr": 1.21825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736037025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11122534424066544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239861760, "lr": 1.2199166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736053140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11474299430847168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 240189440, "lr": 1.2215833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736068675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11328279972076416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 240517120, "lr": 1.2232499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736084520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11727095395326614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 240844800, "lr": 1.2249166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736084521, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784736084522, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784736129505, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7849793434143066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784736129507, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784736129507, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784736145132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11550366878509521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241172480, "lr": 1.2265833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736161277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1159433126449585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241500160, "lr": 1.22825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736176998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11070811003446579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241827840, "lr": 1.2299166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736193323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11286839842796326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 242155520, "lr": 1.2315833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736209417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1150108054280281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 242483200, "lr": 1.2332499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736225358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11620332300662994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 242810880, "lr": 1.2349166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736241533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10923641920089722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243138560, "lr": 1.2365833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736241534, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784736241534, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784736285830, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.784963846206665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784736285831, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784736285831, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784736301555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11467503011226654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243466240, "lr": 1.23825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736317508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11470887809991837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243793920, "lr": 1.2399166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736333637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10740408301353455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 244121600, "lr": 1.2415833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736349430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10500656068325043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 244449280, "lr": 1.24325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736365685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10630430281162262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 244776960, "lr": 1.2449166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736381631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10819120705127716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245104640, "lr": 1.2465833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736397299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11098582297563553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245432320, "lr": 1.24825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736397300, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784736397302, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784736440292, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7848413586616516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784736440293, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784736440293, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784736455953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10611863434314728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245760000, "lr": 1.2499166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736471987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10369881987571716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 246087680, "lr": 1.2515833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736487800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11318433284759521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 246415360, "lr": 1.2532499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736503944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11235027015209198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 246743040, "lr": 1.2549166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736519839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10458268225193024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247070720, "lr": 1.2565833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736535846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10671311616897583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247398400, "lr": 1.25825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736551763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10943952202796936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247726080, "lr": 1.2599166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736551764, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784736551766, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784736595419, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7852113246917725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784736595420, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784736595420, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784736611629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11290726810693741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 248053760, "lr": 1.2615833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736627941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11447322368621826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 248381440, "lr": 1.26325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736644143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10123225301504135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 248709120, "lr": 1.2649166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736659910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1093830019235611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249036800, "lr": 1.2665833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736675678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10336621850728989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249364480, "lr": 1.26825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736691780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10887601971626282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249692160, "lr": 1.2699166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736707659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10500390082597733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 250019840, "lr": 1.2715833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736707733, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784736707733, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784736752221, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7847932577133179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784736752222, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784736752222, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784736768017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10086378455162048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 250347520, "lr": 1.2732499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736783912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10102714598178864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 250675200, "lr": 1.2749166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736799334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10098101943731308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251002880, "lr": 1.2765833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736815028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09872197359800339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251330560, "lr": 1.27825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736831150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10046815872192383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251658240, "lr": 1.2799166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736846890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10241229087114334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251985920, "lr": 1.2815833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736862918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10939529538154602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 252313600, "lr": 1.28325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736862973, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784736862974, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784736906765, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.784920871257782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784736906765, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784736906765, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784736922572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08410438895225525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 252641280, "lr": 1.2849166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736938033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10469726473093033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 252968960, "lr": 1.2865833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736954086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10900053381919861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 253296640, "lr": 1.28825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736969897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11606618762016296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 253624320, "lr": 1.2899166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736985508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11469963192939758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 253952000, "lr": 1.2915833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784737001165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.104792021214962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 254279680, "lr": 1.2932499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784737017201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10305969417095184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 254607360, "lr": 1.2949166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784737017257, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784737017257, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784737060839, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7856467962265015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784737060840, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784737060840, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784737076177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10205554962158203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 254935040, "lr": 1.2965833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784737091479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10314513742923737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 255262720, "lr": 1.2982499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784737107561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10758110880851746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 255590400, "lr": 1.2999166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784737123285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10828036814928055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 255918080, "lr": 1.3015833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784737139105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10743861645460129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 256245760, "lr": 1.30325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784737154800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0990375280380249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 256573440, "lr": 1.3049166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784737170566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1070919930934906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 256901120, "lr": 1.3065833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784737170567, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784737170567, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784737213357, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7854977250099182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784737213357, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784737213358, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784737228868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10325668752193451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 257228800, "lr": 1.30825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784737244958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10166488587856293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 257556480, "lr": 1.3099166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784737260429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1059676855802536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 257884160, "lr": 1.3115833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784737276064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10848291218280792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 258211840, "lr": 1.3132499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784737291511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10512186586856842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 258539520, "lr": 1.3149166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784737306815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1064056009054184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 258867200, "lr": 1.3165833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784737322410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1061730682849884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 259194880, "lr": 1.3182499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784737322411, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784737322412, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784737365847, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7852309942245483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784737365847, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784737365848, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784737381728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10961703211069107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 259522560, "lr": 1.3199166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784737397707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1077723503112793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 259850240, "lr": 1.3215833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784737413641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10624031722545624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 260177920, "lr": 1.32325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784737429659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11056247353553772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 260505600, "lr": 1.3249166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784737445435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10212203115224838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 260833280, "lr": 1.3265833333333332e-06}} diff --git a/recommendation_v4/rcp_logs/gbs_16384/seed534232611.log b/recommendation_v4/rcp_logs/gbs_16384/seed534232611.log new file mode 100644 index 000000000..5f8f2092a --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_16384/seed534232611.log @@ -0,0 +1,825 @@ +:::MLLOG {"namespace": "", "time_ms": 1784867651473, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784867651474, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784867672186, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784867672188, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784867672188, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784867672188, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784867672188, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784867672188, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 16384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784867672188, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784867672188, "event_type": "POINT_IN_TIME", "key": "seed", "value": 534232611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784867672188, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784867672189, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784867672189, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784867672189, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784867672203, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784867672203, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784868550467, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784868550469, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784868550812, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784868695683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13831129670143127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.5833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784868709143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13829046487808228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2499999999999997e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784868722443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13861596584320068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784868735746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386701464653015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.583333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784868749264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138719379901886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.25e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784868762716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387530267238617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784868776034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13855043053627014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868776035, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784868776035, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784868821115, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5002787709236145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784868821116, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784868821116, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784868834449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13816913962364197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868847924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381126344203949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868861518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13856345415115356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868875078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13848696649074554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868888534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382846087217331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9916666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868902346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13782641291618347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868916374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13845005631446838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868916375, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784868916375, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784868958959, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.50047367811203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784868958959, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784868958960, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784868972552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383132040500641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868986486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13792482018470764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868999891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13790208101272583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869014104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13791029155254364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.9916666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869027543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379641443490982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869041596, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13790130615234375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869055073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13767024874687195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869055074, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784869055074, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784869097697, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5006957054138184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784869097697, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784869097697, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784869111321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13837409019470215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869125436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13781332969665527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.8249999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869139119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13703078031539917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869153026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13802888989448547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869167549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13771840929985046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3249999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869181351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137057363986969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869195636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13692200183868408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869195637, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784869195637, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784869237395, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5009660124778748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784869237396, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784869237396, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784869250854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13664832711219788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869264599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13745582103729248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869278784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13728252053260803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1583333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869292493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13659481704235077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869306283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352694034576416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869320104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13662637770175934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869334254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13565444946289062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869334254, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784869334255, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784869377252, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5013074278831482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784869377252, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784869377253, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784869390924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13658525049686432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869404540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13593320548534393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869418468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13737134635448456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869432586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13624969124794006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869446108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13469094038009644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.658333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869459667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13512760400772095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869473430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373823881149292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869473430, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784869473431, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784869516899, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5016821622848511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784869516899, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784869516899, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784869531145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13667824864387512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869545193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353776752948761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869558890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13552415370941162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869572891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13467246294021606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869587051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13479028642177582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869601598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13420803844928741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869615736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13473792374134064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869615736, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784869615737, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784869658136, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5021560192108154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784869658137, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784869658137, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784869672415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13396133482456207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869686819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13636848330497742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869701615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13821198046207428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869715852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13383223116397858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.824999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869730452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360228806734085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869744720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13468918204307556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.158333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869759551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13674212992191315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.324999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869759552, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784869759553, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784869802921, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5028121471405029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784869802922, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784869802922, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784869817377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13661150634288788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869831988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13662999868392944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869846437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13475696742534637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869860880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13002508878707886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869875481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322983205318451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869889734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13363084197044373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869903776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13497835397720337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869903826, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784869903827, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784869948043, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5035316348075867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784869948043, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784869948043, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784869962233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13490664958953857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869976320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12893325090408325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869991180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312253624200821, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870004994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368437111377716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.1158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870019121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13216906785964966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870033022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312730461359024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870047507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13248851895332336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870047507, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784870047507, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784870090829, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.50435471534729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784870090829, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784870090829, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784870104673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13162660598754883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870119427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343628168106079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870133166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1278204768896103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870146975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12951980531215668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870161053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13387258350849152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870175475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12812435626983643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870189444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12976954877376556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870189444, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784870189445, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784870233581, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5053808689117432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784870233581, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784870233581, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784870248551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13467857241630554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870262512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13298210501670837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.3158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870276688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13448388874530792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870290816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13487520813941956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870305401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13627316057682037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.3658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870319624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13353262841701508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870334064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13687735795974731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870334064, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784870334065, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784870377357, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5067327618598938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784870377358, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784870377358, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784870391979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13451287150382996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.4158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870406660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13067777454853058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870421256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338222175836563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.4491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870436217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14044302701950073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.4658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870450994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13037657737731934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870466161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13259266316890717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870480622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381818950176239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870480622, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784870480623, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784870522910, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.508118212223053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784870522910, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784870522910, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784870537584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1265324354171753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870552171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13104583323001862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870567172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370180994272232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870581865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13544104993343353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870597095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324378401041031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870611339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12870019674301147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.6158333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870625623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12795382738113403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870625624, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784870625624, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784870668941, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5095651149749756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784870668941, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784870668941, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784870683229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13016265630722046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870697470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347208172082901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.6658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870712131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14227090775966644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6825000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870726534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1340966373682022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870741617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13141459226608276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.7158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870756195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336064636707306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870770735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361127495765686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870770736, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784870770736, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784870814100, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.511222243309021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784870814100, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784870814100, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784870828331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13460510969161987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870842649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13628923892974854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870857149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13579177856445312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870871764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13317230343818665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870886540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134371817111969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870901051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329118013381958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870915203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13249127566814423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8658333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870915203, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784870915204, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784870959017, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5129993557929993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784870959018, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784870959018, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784870973267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13198944926261902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870987478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345224678516388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8991666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871001888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14200475811958313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871016491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13311918079853058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871031127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343311220407486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871046009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13230592012405396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871060606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13309752941131592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871060606, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784871060607, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784871105124, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5150312185287476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784871105124, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784871105125, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784871119486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384512186050415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871133883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13306714594364166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871148161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12880659103393555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871162575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12999999523162842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871177283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13789905607700348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871192155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13159886002540588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871206704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13636717200279236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871206705, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784871206705, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784871252537, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5173589587211609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784871252538, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784871252538, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784871267193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13684043288230896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.1158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871281758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13633577525615692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871296683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13138574361801147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1491666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871311295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13009317219257355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871325924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1316499412059784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871340788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352316290140152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871355768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13263779878616333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871355768, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784871355769, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784871400111, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5200806856155396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784871400111, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784871400112, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784871414702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13599750399589539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871429996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13229453563690186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871444888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378875970840454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871459643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13633385300636292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871474674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385122537612915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871489565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13454076647758484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871504780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13908183574676514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871504781, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784871504781, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784871548784, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5228760838508606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784871548785, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784871548785, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784871563796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13498273491859436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871578869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.15005724132061005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871594238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13509711623191833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871609236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13288716971874237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871624488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13426116108894348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871639378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13290467858314514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871654430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13655880093574524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871654431, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784871654431, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784871698973, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5260833501815796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784871698974, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784871698974, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784871714033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13598588109016418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871729065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13518673181533813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871744159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13388904929161072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871758678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13356491923332214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871773140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312998414039612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871787615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295195370912552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871801843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1255398988723755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871801844, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784871801844, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784871845643, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.529862105846405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784871845643, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784871845644, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784871860079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326170265674591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.5824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871874658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13248085975646973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871889320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13490593433380127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871903957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13536624610424042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871918852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13666260242462158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871933714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288367658853531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871948319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12988066673278809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871948319, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784871948320, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784871991685, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5357736945152283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784871991686, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784871991686, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784872006154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13271330296993256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.699166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872020538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12714263796806335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872034999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13320419192314148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872049508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132139652967453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872064305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12692728638648987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872079307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13218209147453308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872093775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13423976302146912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872093776, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784872093776, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784872137597, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5439373850822449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784872137598, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784872137598, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784872153028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12945930659770966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872167753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13037633895874023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872182130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13060522079467773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872196543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12979668378829956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.8658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872210776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12774425745010376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872225071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12309624254703522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872239812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12717992067337036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872239812, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784872239813, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784872282216, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5549163818359375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784872282217, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784872282217, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784872296798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13560032844543457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872311464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13324019312858582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872325734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13895997405052185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872340691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13356754183769226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872355233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12763342261314392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872369701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12975981831550598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872384396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1268594115972519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872384396, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784872384397, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784872426291, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5685737729072571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784872426291, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784872426292, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784872440659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1291247010231018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872455119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1275143027305603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872470037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13262897729873657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872484999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13273945450782776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872499972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13133704662322998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872514645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13354523479938507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872529865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13046039640903473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872529866, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784872529866, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784872573856, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5849584341049194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784872573856, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784872573856, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784872588761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295124888420105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872603584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12973034381866455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872618587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295442134141922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872633630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324356645345688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872648296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13326729834079742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872662986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12946389615535736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872678328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13010287284851074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872678329, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784872678329, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784872721388, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5990827679634094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784872721389, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784872721389, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784872736316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12796804308891296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872751588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1305992305278778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872766071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12820696830749512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872781466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12745344638824463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872796633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1271393597126007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872811671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13128598034381866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872826870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318860650062561, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872826871, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784872826871, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784872869669, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6118651032447815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784872869669, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784872869670, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784872884341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12972694635391235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872899338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12760606408119202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872914623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301276981830597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872929954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13196443021297455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872945316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13283665478229523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872960304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12867116928100586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872975850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12857729196548462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.4991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872975850, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784872975850, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784873019004, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6231918334960938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784873019005, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784873019005, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784873033872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12883318960666656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.5158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873048664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1270534247159958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873063416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1282009482383728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.5491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873078080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347387135028839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873092987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13031813502311707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873108066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12536409497261047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873123356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.128271222114563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873123356, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784873123356, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784873167401, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6341937780380249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784873167402, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784873167402, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784873182150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12495394051074982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6325000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873197766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1273140013217926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873212706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12641362845897675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873227400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12582455575466156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873242028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12403364479541779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873256850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12109233438968658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873271539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12288492918014526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873271540, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784873271540, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784873314252, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6470412611961365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784873314252, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784873314252, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784873329653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12572011351585388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873344797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12206205725669861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873359836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11521092802286148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.7824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873374946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1217304915189743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873389639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11684360355138779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873404202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11771039664745331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8324999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873418837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1150534451007843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.8491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873418838, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784873418838, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784873461037, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.66670161485672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784873461038, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784873461038, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784873475781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12070204317569733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873490187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11623561382293701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873504604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11198745667934418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.8991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873519297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12018879503011703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873534332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11804008483886719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873549513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11729928851127625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.9491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873564270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10870730131864548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.9658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873564271, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784873564271, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784873608046, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7062681913375854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784873608046, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784873608046, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784873622583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11517713218927383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873637997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11593617498874664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873652985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11792518198490143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.0158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873667841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11309096962213516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873682872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111665740609169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873697672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11299652606248856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0658333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873712493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10834610462188721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873712548, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784873712548, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784873755609, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7432421445846558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784873755609, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784873755609, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784873770073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11349162459373474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873785368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11211257427930832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.1158333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873800380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11759836226701736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873815663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11126935482025146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873830276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11288833618164062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1658333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873845863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11800827831029892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873860600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10655629634857178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873860601, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784873860601, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784873903613, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7534071803092957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784873903613, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784873903613, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784873918387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11514326930046082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873932991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11278489977121353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873947793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11013766378164291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873962355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.110187828540802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873976914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10412570834159851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873991294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11813747137784958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874006487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11366793513298035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874006488, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784874006488, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784874048595, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7560184597969055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784874048595, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784874048596, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784874063524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1171932965517044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874078905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11224673688411713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.3491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874093798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11614972352981567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874109317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10140113532543182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874124286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11528624594211578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.3991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874139133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10814052075147629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.4158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874153905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10726726055145264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874153905, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784874153905, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784874197273, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7572418451309204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784874197273, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784874197274, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784874212082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10921473801136017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.4491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874226830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1078261286020279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874241325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11415769159793854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.4824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874256198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10890506207942963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874271260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11312130093574524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874286280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10327710211277008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874301475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1116342693567276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874301476, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784874301476, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784874345145, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7587093114852905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784874345145, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784874345146, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784874360055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793809592723846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874374827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10947825014591217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.5824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874390575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11962705850601196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874406040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12237070500850677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874421078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11646337807178497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874436355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11161745339632034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874451533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11665044724941254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874451534, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784874451534, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784874494824, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7572910189628601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784874494825, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784874494825, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784874509884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11499284207820892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874525242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10820601880550385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.6991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874540814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11711960285902023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874556286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11074027419090271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874572163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11458276957273483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874587456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11425580084323883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874603404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10479997098445892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874603405, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784874603405, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784874646162, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7575445771217346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784874646162, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784874646162, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784874661389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11888640373945236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874676677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10727981477975845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874691949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1049422174692154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874707159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11451928317546844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874722226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11786940693855286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874737417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1068190336227417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874753020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1140018105506897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874753021, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784874753021, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784874796245, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7627907991409302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784874796246, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784874796246, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784874811446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11537505686283112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874827065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10777853429317474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874841958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10897491872310638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874857359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11399312317371368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874872628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072881892323494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874887516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10386516153812408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874902347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11760631203651428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.015833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874902348, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784874902348, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784874943964, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7636346220970154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784874943965, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784874943965, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784874958808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11033906042575836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874973559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11608783900737762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874988262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10727870464324951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875002843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10496845096349716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875018005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10264793038368225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875033090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09652960300445557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875048533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10687272995710373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875048534, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784875048534, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784875091273, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7647917866706848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784875091274, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784875091274, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784875106312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1057821661233902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875121079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1099134087562561, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875136819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11827477812767029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875151865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10941098630428314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875166668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10609988868236542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875181306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10085061192512512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875196300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10702045261859894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875196300, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784875196301, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784875239207, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7655775547027588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784875239207, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784875239207, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784875254097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10501977801322937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875268784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10582669079303741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875283554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10977998375892639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875298547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1073780357837677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875313659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10696415603160858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875328664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11229658126831055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875344058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11018851399421692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875344059, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784875344059, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784875386739, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7678825855255127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784875386739, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784875386739, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784875401506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1049475148320198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875416723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10258585214614868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875431759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1104254424571991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875447063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1092381626367569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875462224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10500769317150116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875477334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10081113874912262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875492594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10508377850055695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875492595, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784875492595, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784875536049, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7681392431259155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784875536049, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784875536049, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784875551163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10473819077014923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875566507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09997733682394028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875581645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10482582449913025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875597330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11065603792667389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875612849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10642364621162415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875628609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10358202457427979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875644000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10963509231805801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875644000, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784875644000, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784875688013, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7683038115501404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784875688014, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784875688014, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784875702622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10012021660804749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875718174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10405413806438446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875733240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1074783056974411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875748323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10458119958639145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875763260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10306283831596375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875778230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10785192251205444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875793181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10974135994911194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875793182, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784875793182, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784875835666, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7681777477264404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784875835666, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784875835666, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784875850615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10556681454181671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875865481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1026572659611702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875880079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10863825678825378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875895230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10656335204839706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875910534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11028216034173965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875925889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10810880362987518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875941070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11126358807086945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875941071, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784875941071, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784875984493, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.769487202167511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784875984494, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784875984494, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784875999145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10681439191102982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876014630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10141846537590027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876029450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10518169403076172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876044383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1026071161031723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876059129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10885406285524368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876073722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11308582872152328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876088339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10905987024307251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876088339, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784876088340, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784876132686, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7695403695106506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784876132687, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784876132687, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784876147517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1052633747458458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876162276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11102969944477081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876176907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10788805782794952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876191733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10042519122362137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876206575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11216172575950623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876221453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11279698461294174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876236886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10515595972537994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876236887, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784876236887, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784876280333, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.770579993724823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784876280334, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784876280334, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784876295162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080983430147171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876309718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10041041672229767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876325641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11628971993923187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876341156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11663931608200073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876356364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11525185406208038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876371632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10465206205844879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876387067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10690028965473175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876387072, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784876387073, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784876430377, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7702311873435974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784876430377, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784876430378, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784876445650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11191637814044952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876460640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11431531608104706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876475888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11036457121372223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876491257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11096756160259247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876506537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11458972841501236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876521784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11220055818557739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876537269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11326867341995239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876537270, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784876537270, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784876579995, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7705276608467102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784876579996, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784876579996, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784876594858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10935087502002716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876610562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10736905038356781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876625899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10685007274150848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876641411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11541114747524261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876656497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11842210590839386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876671668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10989197343587875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876687220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11559885740280151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.415833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876687221, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784876687221, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784876730638, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7703103423118591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784876730639, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784876730639, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784876745665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11353950202465057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876760873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11387099325656891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876776145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1158018484711647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876791714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11422774195671082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876807303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10905022919178009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876822591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10919202864170074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.515833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876837528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11261864006519318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876837528, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784876837528, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784876882716, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7712194919586182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784876882717, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784876882717, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784876898400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10881486535072327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876913897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10526171326637268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876928697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10457506775856018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876943674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10420073568820953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876958536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071244329214096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.615833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876973828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10324344038963318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876988939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10873030126094818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876988940, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784876988940, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784877034018, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7719352841377258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784877034019, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784877034019, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784877049103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10980759561061859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877063951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10247887670993805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877079233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10806737840175629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877094663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10388046503067017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877110130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10257390141487122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877125992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10563518851995468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877141230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09948348999023438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877141231, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784877141231, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784877186635, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7719484567642212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784877186636, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784877186636, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784877201565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10791778564453125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877217260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.110660620033741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877232496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10686101019382477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877247572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10644641518592834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877262471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10808980464935303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877277192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10311993956565857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877292189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10848309099674225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877292190, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784877292190, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784877338057, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7722658514976501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784877338058, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784877338058, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784877352981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10811396688222885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877367656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10583990812301636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877382638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10896550863981247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877397399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10320387780666351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877412681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10932959616184235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877428341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10939851403236389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877444037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10541057586669922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877444037, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784877444050, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784877488268, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7727868556976318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784877488269, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784877488269, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784877503791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10818426311016083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877518604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10234349220991135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877534222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11314310878515244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877549323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10312984883785248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877564986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10009586811065674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877580201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10269467532634735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877595296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10627450048923492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877595297, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784877595297, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784877638990, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.772662341594696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784877638991, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784877638991, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784877653988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10313217341899872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877669356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10066875070333481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877685013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1083517074584961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877700289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10992932319641113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877715351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10472334921360016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877730875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10928407311439514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877746320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10957943648099899, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877746321, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784877746321, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784877789945, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7732727527618408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784877789945, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784877789945, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784877805600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1120997816324234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877821625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10529768466949463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877837142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10451212525367737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877852130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10595577210187912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877867604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10096967220306396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877882943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1076056957244873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877898470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10474605858325958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877898471, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784877898471, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784877941784, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7729840278625488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784877941785, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784877941785, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784877956911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10686597228050232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877971974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10866574943065643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877987115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10062025487422943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878002387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11008056998252869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878017739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09696544706821442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878033206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10263118147850037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878048387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10382784903049469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878048387, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784878048388, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784878093041, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7732160091400146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784878093041, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784878093041, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784878108607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10260915011167526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878124175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10816407203674316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878139797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10919177532196045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878155526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11106003820896149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878171205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10811763256788254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878186208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10904116183519363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.565833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878201976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10235576331615448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878201976, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784878201977, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784878245994, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7735798954963684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784878245994, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784878245994, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784878261227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10920596122741699, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878277036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10420659184455872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149749760, "lr": 7.615833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878292283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10674382746219635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878307356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11143476516008377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150405120, "lr": 7.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878322474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10800670087337494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878337821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11300487816333771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151060480, "lr": 7.682499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878352858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1045161560177803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878352859, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784878352859, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784878396649, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7737305164337158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784878396649, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784878396650, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784878411570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10666278004646301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151715840, "lr": 7.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878426674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09833501279354095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878441961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11088569462299347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152371200, "lr": 7.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878457295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10446596890687943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878472790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11152861267328262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153026560, "lr": 7.782499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878488387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11261607706546783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878504339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10403723269701004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153681920, "lr": 7.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878504340, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784878504340, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784878546872, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773900032043457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784878546873, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784878546873, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784878562373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10297709703445435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878577500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10837668180465698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154337280, "lr": 7.849166666666667e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_16384/seed574397838.log b/recommendation_v4/rcp_logs/gbs_16384/seed574397838.log new file mode 100644 index 000000000..7a8bd3742 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_16384/seed574397838.log @@ -0,0 +1,1333 @@ +:::MLLOG {"namespace": "", "time_ms": 1784704984542, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784704984542, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784705004477, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784705004478, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784705004478, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784705004479, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784705004479, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784705004479, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 16384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784705004479, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784705004479, "event_type": "POINT_IN_TIME", "key": "seed", "value": 574397838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784705004479, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784705004479, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784705004479, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784705004479, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784705004479, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784705004480, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784705917226, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784705917227, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784705917461, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784706049170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13678115606307983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.5833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784706063022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13729268312454224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2499999999999997e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784706076655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13811150193214417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784706090492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13855701684951782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.583333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784706104384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13887956738471985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.25e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784706118352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13869912922382355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784706132282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13809779286384583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706132282, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784706132283, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784706218512, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5021247267723083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784706218513, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784706218513, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784706232258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13734427094459534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706246325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374339461326599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706260481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13788153231143951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706274552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13771745562553406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706288853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13791358470916748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9916666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706303403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13688242435455322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706318175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377115249633789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706318175, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784706318175, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784706363614, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.502267062664032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784706363615, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784706363615, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784706377866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13770084083080292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706392455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368705928325653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706406768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13723787665367126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706421406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13699859380722046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.9916666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706435424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13733619451522827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706450125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13701067864894867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706464711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1369401216506958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706464712, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784706464712, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784706509395, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5024484992027283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784706509396, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784706509396, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784706523922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380326747894287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706538621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13735926151275635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.8249999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706553055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13575127720832825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706567684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13773183524608612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706582900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368788331747055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3249999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706597447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13623544573783875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706612492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362687349319458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706612493, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784706612494, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784706656696, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5026519894599915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784706656696, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784706656696, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784706670839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13574501872062683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706685157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13703618943691254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706699786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13672637939453125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1583333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706714280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13598471879959106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706728786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13405023515224457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706743289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13603150844573975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706757778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13470838963985443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706757779, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784706757779, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784706801039, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5029142498970032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784706801040, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784706801040, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784706815206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13600413501262665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706829400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353408396244049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706844085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13707087934017181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706858704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358252763748169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706873040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13364127278327942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.658333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706887561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1342800110578537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706902169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13717931509017944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706902169, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784706902170, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784706945957, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5032214522361755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784706945958, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784706945958, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784706960732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13638333976268768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706975643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13488508760929108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706990416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13507366180419922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707005206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13417547941207886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707020123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13415563106536865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707035289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13368839025497437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707050078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13413108885288239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707050079, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784707050080, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784707092199, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5036430358886719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784707092200, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784707092200, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784707107111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13313929736614227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707122111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361522674560547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707137477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13849252462387085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707152521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13327787816524506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.824999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707167613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13585487008094788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707182565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344502568244934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.158333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707197799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13666526973247528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.324999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707197800, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784707197800, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784707241596, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5042558312416077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784707241596, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784707241596, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784707256651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13644909858703613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707271682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13655266165733337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707286766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348104476928711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707301726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12943804264068604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707316797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320440173149109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707331626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13344067335128784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707346397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13482198119163513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707346444, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784707346445, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784707390003, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5049930214881897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784707390003, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784707390003, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784707404866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13494408130645752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707419602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12849479913711548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707435003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308819055557251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707449395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367919147014618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.1158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707464062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1319478303194046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707478592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311306655406952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707493586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13230809569358826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707493586, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784707493587, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784707538573, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5058194994926453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784707538574, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784707538574, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784707552831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13156861066818237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707568249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343226134777069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707582987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12753626704216003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707597772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12939167022705078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707612854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13387072086334229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707628006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1279432475566864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707642774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12971800565719604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707642775, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784707642775, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784707688588, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5069074034690857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784707688589, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784707688589, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784707704284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13473762571811676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707718850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329372525215149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.3158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707733723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13435810804367065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707748617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135039821267128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707763928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361311674118042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.3658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707778745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13367685675621033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707793814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13700881600379944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707793815, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784707793815, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784707837528, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5081657767295837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784707837528, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784707837528, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784707852754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13447213172912598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.4158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707868137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13049235939979553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707883245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341848075389862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.4491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707898592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14056363701820374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.4658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707913877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13049763441085815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707929294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13239121437072754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707944508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381777822971344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707944509, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784707944509, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784707987588, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5093967318534851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784707987588, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784707987589, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784708003047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12491695582866669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708018165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13119672238826752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708033492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13695210218429565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708048533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135238915681839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708064031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13239309191703796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708078905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12877142429351807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.6158333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708093984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12801429629325867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708093985, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784708093985, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784708136567, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5107027888298035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784708136568, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784708136568, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784708151373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13012459874153137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708166238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13466057181358337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.6658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708181492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14231783151626587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6825000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708196473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13388407230377197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708211739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13148511946201324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.7158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708226347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13371621072292328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708241090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13600169122219086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708241091, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784708241091, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784708284287, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5122944116592407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784708284287, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784708284288, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784708298850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13463608920574188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708313612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13630685210227966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708328695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13562335073947906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708343660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332632303237915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708358771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13444316387176514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708374067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13294827938079834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708389032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323569416999817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8658333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708389033, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784708389033, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784708434014, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5140208601951599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784708434015, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784708434015, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784708449012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13191458582878113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708464043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345173567533493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8991666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708479318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14189203083515167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708494799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13404875993728638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708510102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13418328762054443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708525325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321803629398346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708540237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13310536742210388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708540238, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784708540238, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784708583293, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5160511136054993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784708583294, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784708583294, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784708598002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13854734599590302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708612845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13289617002010345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708627675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1287558674812317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708642603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1299595683813095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708657859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377996802330017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708673129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13154929876327515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708688210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13606473803520203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708688211, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784708688211, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784708732283, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5183751583099365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784708732284, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784708732284, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784708747542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13683921098709106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.1158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708762667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13643240928649902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708778070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13132993876934052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1491666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708793236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13007484376430511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708808588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13149333000183105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708824005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13535988330841064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708839542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13259997963905334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708839542, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784708839542, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784708883126, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5210455060005188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784708883127, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784708883127, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784708898159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359223872423172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708913731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13215667009353638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708929753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13784730434417725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708945346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13642770051956177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708960904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387103796005249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708976400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13445347547531128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708992262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13900992274284363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708992262, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784708992263, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784709035182, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5238339900970459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784709035183, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784709035183, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784709050828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13496795296669006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709066397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14830750226974487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709082202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13497358560562134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709097765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13314715027809143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709113645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13426057994365692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709129284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330876350402832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709145116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13649897277355194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709145116, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784709145117, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784709188675, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5270437002182007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784709188675, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784709188676, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784709204557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361445188522339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709220388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13510899245738983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709236298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13367614150047302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709251466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133608877658844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709266728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331852376461029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709281843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12929673492908478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709296855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12563146650791168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709296856, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784709296856, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784709341324, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5310468673706055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784709341324, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784709341325, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784709356407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13253550231456757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.5824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709371885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13241031765937805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709387457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349334716796875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709402740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352764219045639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709418185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13675656914710999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709433916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12876278162002563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709449476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12968118488788605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709449477, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784709449477, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784709492459, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5370678305625916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784709492459, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784709492459, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784709507781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13255378603935242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.699166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709522939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1269911229610443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709538054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13311585783958435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709553163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13217848539352417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709568634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12697190046310425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709584229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13224545121192932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709599232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341181993484497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709599232, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784709599233, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784709643362, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5454789400100708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784709643362, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784709643362, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784709659089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1292891800403595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709674522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13039928674697876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709689776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1306607723236084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709705083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12975266575813293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.8658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709720295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12776601314544678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709735588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12307661771774292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709751279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12699627876281738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709751280, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784709751281, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784709794974, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5566381812095642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784709794975, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784709794975, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784709810599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13560721278190613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709826117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13328197598457336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709841144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13869233429431915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709856909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13339653611183167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709872358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12758395075798035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709887601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295308619737625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709903058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1266912817955017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709903059, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784709903059, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784709947124, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5701677203178406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784709947124, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784709947125, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784709962231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288178563117981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709977493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1271423101425171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709993151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13234663009643555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710008870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13255274295806885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710024590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311180591583252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710039818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13359948992729187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710055722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1302267163991928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710055723, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784710055723, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784710099561, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5860069990158081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784710099562, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784710099562, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784710115445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12938192486763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710131083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12952160835266113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710146800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12940382957458496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710162506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13238075375556946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710178020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13299626111984253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710193283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12907801568508148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710209281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12993918359279633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710209281, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784710209281, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784710253438, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5995160341262817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784710253438, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784710253439, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784710269179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12783825397491455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710285072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13072824478149414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710300235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12774929404258728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710316268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12727266550064087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710332200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12692758440971375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710347916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13107740879058838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710363781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13198339939117432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710363782, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784710363782, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784710407164, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6116767525672913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784710407164, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784710407165, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784710422354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1293150782585144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710437973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12734246253967285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710453863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12991347908973694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710469761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13133150339126587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710485668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327056884765625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710501189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1286361813545227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710517556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1283879280090332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.4991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710517556, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784710517557, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784710561112, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.622648298740387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784710561112, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784710561112, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784710576828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1287226378917694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.5158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710592591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12678758800029755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710608461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12823013961315155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.5491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710623993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13485483825206757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710639764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13015206158161163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710655910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12509675323963165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710671856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.128437340259552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710671857, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784710671857, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784710715873, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6338461637496948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784710715873, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784710715874, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784710731331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12487220764160156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6325000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710747408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1268545240163803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710762853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12622877955436707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710778141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12567636370658875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710793302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12427359819412231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710808579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12109366059303284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710823855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12313582748174667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710823856, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784710823856, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784710867045, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6502136588096619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784710867045, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784710867046, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784710882925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1258542239665985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710898698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12217087298631668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710914335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1152331605553627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.7824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710930012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12206404656171799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710945668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1169990673661232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710961166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11784873902797699, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8324999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710976779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11531135439872742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.8491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710976780, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784710976780, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784711020731, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6716891527175903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784711020732, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784711020732, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784711036583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12109453231096268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711052090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11673419177532196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711067526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11289585381746292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.8991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711083161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12006983906030655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711099078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11831097304821014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711115146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11769858002662659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.9491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711130671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10881951451301575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.9658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711130671, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784711130672, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784711174853, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.718448281288147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784711174854, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784711174854, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784711190009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11560289561748505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711205688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11596754193305969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711221210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1189248114824295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.0158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711236561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11427098512649536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711252186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11241769790649414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711267529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11350549757480621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0658333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711283043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10947905480861664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711283089, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784711283090, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784711327411, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7447913885116577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784711327411, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784711327411, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784711342641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11429397761821747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711358554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11237627267837524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.1158333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711374270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11785563081502914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711390166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11237527430057526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711405483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11228816956281662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1658333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711421135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11935360729694366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711436463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10695824027061462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711436464, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784711436464, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784711481741, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7511307001113892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784711481741, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784711481741, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784711497044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11551590263843536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711512466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11336953938007355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711528091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11111672222614288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711543488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11058039963245392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711558959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10467646270990372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711574078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1184108778834343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711590038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11339399963617325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711590038, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784711590039, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784711635249, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7508862614631653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784711635250, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784711635250, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784711650904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11681544780731201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711666790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11295398324728012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.3491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711682275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11633673310279846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711698138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10237440466880798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711713702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1141490787267685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.3991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711729290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1079946756362915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.4158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711744857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10692372918128967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711744858, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784711744858, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784711789654, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7482230067253113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784711789655, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784711789655, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784711805144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1091657280921936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.4491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711820688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10824458301067352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711836067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11382021009922028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.4824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711851714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10911980271339417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711867670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11309777200222015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711883652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1031579077243805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711899774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11155291646718979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711899775, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784711899775, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784711945291, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7463878393173218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784711945291, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784711945292, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784711961134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723450779914856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711976562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1090511828660965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.5824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711992712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11975656449794769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712008660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12248923629522324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712024282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11577610671520233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712040111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11120578646659851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712055822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11671298742294312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712055823, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784712055823, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784712100626, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7418042421340942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784712100627, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784712100627, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784712116218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11473843455314636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712131992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080293282866478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.6991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712148091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11746110022068024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712164074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11082018911838531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712180246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11405116319656372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712195880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11381456255912781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712212051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10427669435739517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712212052, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784712212052, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784712257654, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.745593786239624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784712257655, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784712257655, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784712273658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11862176656723022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712289607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10719950497150421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712305513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10508612543344498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712321395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11402912437915802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712337226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11785299330949783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712353236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10688035190105438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712369572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1137847974896431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712369573, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784712369573, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784712414955, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7529391646385193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784712414955, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784712414955, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784712431070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11504510045051575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712447436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10744623839855194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712463050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084267646074295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712479092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1138271689414978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712495020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10728257894515991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712510701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10387499630451202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712526292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1161072850227356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.015833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712526292, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784712526293, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784712571873, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7559993267059326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784712571873, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784712571873, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784712587588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10980044305324554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712603294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11531826853752136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712618941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10645739734172821, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712634423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1047268956899643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712650317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10220031440258026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712666153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09626621007919312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712682246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10584858059883118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712682246, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784712682246, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784712727313, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7604027986526489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784712727314, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784712727314, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784712743032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1052447035908699, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712758512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11003037542104721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712774782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11823403835296631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712790663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10929524898529053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712806253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1058986634016037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712821776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10076198726892471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712837563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10683576762676239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712837563, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784712837564, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784712881557, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7630046606063843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784712881558, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784712881558, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784712897268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1050235778093338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712912777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10546886175870895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712928307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10946950316429138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712943988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10711351037025452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712959979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10613347589969635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712975823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11169468611478806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712991942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10982415080070496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712991943, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784712991943, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784713036318, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7657108902931213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784713036319, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784713036319, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784713051865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10471461713314056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713067847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10246274620294571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713083558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10919727385044098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713099404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10918228328227997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713115219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10469284653663635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713130984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10012458264827728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713146858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1043616384267807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713146858, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784713146858, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784713191852, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7657580375671387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784713191853, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784713191853, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784713207563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10434417426586151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713223274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09997686743736267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713239024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10477480292320251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713255280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11016078293323517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713271422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10576694458723068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713287784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10372260212898254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713303903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1099407970905304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713303903, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784713303904, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784713349958, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7671083211898804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784713349959, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784713349959, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784713365343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09976474940776825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713381598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10291826725006104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713397410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10703868418931961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713413188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10340824723243713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713428849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10268912464380264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713444472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10694345831871033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713460252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10914334654808044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713460253, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784713460253, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784713505209, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7670337557792664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784713505210, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784713505210, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784713520840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10573878884315491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713536558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10330233722925186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713552009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10831698775291443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713567874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10574829578399658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713583701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11024101078510284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713599527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10788169503211975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713615200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11062557995319366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713615200, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784713615200, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784713659781, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7688913941383362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784713659781, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784713659782, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784713675056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10647168755531311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713691410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10158318281173706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713707354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10427691042423248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713723330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10245370864868164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713739148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793409496545792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713754909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11303690075874329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713770691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10786759853363037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713770691, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784713770691, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784713815436, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7692140936851501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784713815437, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784713815437, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784713831352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10396213829517365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713847316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111285001039505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713863282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10747972130775452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713879396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10002534836530685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713895526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11184526979923248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713911502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11267533898353577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713927954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10496722161769867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713927954, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784713927955, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784713972626, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7704147100448608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784713972627, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784713972627, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784713988600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10831139981746674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714004128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09967784583568573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714020693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11587793380022049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714036987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11561179161071777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714053134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11439947038888931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714069167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10378529131412506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714085422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1066395491361618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714085423, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784714085423, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784714130052, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7705183029174805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784714130052, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784714130053, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784714146185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11138412356376648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714162098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11412566155195236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714178296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10988733917474747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714194794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11026187241077423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714211229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11429246515035629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714227410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11209089308977127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714243893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11311142146587372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714243893, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784714243893, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784714287169, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7706372141838074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784714287170, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784714287170, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784714302850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10950420796871185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714319183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10719717293977737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714335435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10700345784425735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714351926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11440586298704147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714368014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11830472946166992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714384242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1097198873758316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714400703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11598764359951019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.415833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714400704, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784714400704, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784714443906, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7713290452957153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784714443907, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784714443907, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784714459809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11342689394950867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714475911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11379707604646683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714492300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11568190157413483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714508721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11348772048950195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714525327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10935530066490173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714541663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10915760695934296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.515833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714557522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11192916333675385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714557522, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784714557523, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784714601965, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7717194557189941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784714601966, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784714601966, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784714618347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10863660275936127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714634516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10449492931365967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714650114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10433351993560791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714665909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1042913943529129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714681666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10668540000915527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.615833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714697825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10300249606370926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714713719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10841414332389832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714713719, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784714713720, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784714758168, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.772498607635498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784714758169, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784714758169, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784714773913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10966675728559494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714789654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10264299809932709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714805845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10770389437675476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714821937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1032090112566948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714838250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10189671069383621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714854669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10572059452533722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714870666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09968750923871994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714870666, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784714870667, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784714915655, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.772510826587677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784714915656, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784714915656, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784714931218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10757178068161011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714947619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11059461534023285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714963561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1061808243393898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714979330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10597725957632065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714995133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10734632611274719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715010702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10305216908454895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715026504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10781186074018478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715026505, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784715026505, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784715070421, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730041742324829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784715070422, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784715070422, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784715086141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10763998329639435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715101716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10550148040056229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715117648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085672602057457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715133387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10272519290447235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715149458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1089555025100708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715165574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10947328805923462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715181946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10489004850387573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715181947, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784715181948, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784715226976, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.77330082654953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784715226977, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784715226977, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784715243017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10769224166870117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715258501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10205085575580597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715274236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11210760474205017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715289596, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10280647873878479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715305716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1002216637134552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715321404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10239684581756592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715337164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1054937094449997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715337164, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784715337164, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784715381384, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773595929145813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784715381385, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784715381385, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784715396868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10314000397920609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715412788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10023093223571777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715428778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10833650827407837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715444527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10962367057800293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715459958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10467614233493805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715475699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10806091129779816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715491656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10944290459156036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715491656, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784715491657, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784715536352, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7739872336387634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784715536353, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784715536353, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784715552235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11130280792713165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715568429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10444895923137665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715584287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10398076474666595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715599766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10551093518733978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715615781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09999346733093262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715631996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10745185613632202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715648496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10445526242256165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715648497, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784715648497, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784715692931, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7741057276725769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784715692931, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784715692932, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784715708871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10658799111843109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715724749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10856886208057404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715740660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10016454011201859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715756526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10935620963573456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715772421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09672191739082336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715788544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10230182111263275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715804389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10364185273647308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715804390, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784715804390, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784715847829, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7740169167518616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784715847830, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784715847830, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784715864024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10238377749919891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715880230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10827438533306122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715896397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087898388504982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715912687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11109986901283264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715928926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10795487463474274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715944563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10903607308864594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.565833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715960735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10181647539138794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715960736, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784715960737, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784716005960, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746319770812988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784716005960, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784716005960, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784716021708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10956057906150818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716038013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1041705533862114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149749760, "lr": 7.615833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716053954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10661749541759491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716069944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11177147924900055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150405120, "lr": 7.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716085986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10775599628686905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716102220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11231070756912231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151060480, "lr": 7.682499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716118341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10415603220462799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716118341, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784716118341, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784716161573, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774536669254303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784716161573, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784716161574, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784716177494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10662136971950531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151715840, "lr": 7.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716193589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09821304678916931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716209669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.110697440803051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152371200, "lr": 7.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716225864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10426183044910431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716242283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11164264380931854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153026560, "lr": 7.782499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716258739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11237410455942154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716275391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10386109352111816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153681920, "lr": 7.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716275392, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784716275393, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784716319390, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7748371362686157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784716319390, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784716319390, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784716335399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10296620428562164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716351089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071704626083374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154337280, "lr": 7.849166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716367514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10252465307712555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716383642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11067979037761688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154992640, "lr": 7.882499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716400163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10976903140544891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716416677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11869074404239655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155648000, "lr": 7.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716433088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11720414459705353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716433089, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784716433089, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784716476565, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.775113046169281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784716476566, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784716476566, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784716492913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11813318729400635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156303360, "lr": 7.949166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716509313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10990417003631592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716525434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11484763026237488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156958720, "lr": 7.982499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716541793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11212357133626938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716558194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1146206259727478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157614080, "lr": 8.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716574852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11670105159282684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716591418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10853183269500732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158269440, "lr": 8.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716591419, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784716591419, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784716635008, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7750730514526367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784716635009, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784716635009, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784716651387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11059731990098953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716667828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.104692742228508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158924800, "lr": 8.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716683690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11229506134986877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.099166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716700299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11172997951507568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159580160, "lr": 8.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716716904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11020301282405853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716733250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10780760645866394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160235520, "lr": 8.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716749490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1104561984539032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716749490, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784716749491, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784716793212, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752073407173157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784716793213, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784716793213, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784716809272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11368927359580994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160890880, "lr": 8.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716825476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10520327091217041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.199166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716841876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10843460261821747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161546240, "lr": 8.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716858030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10958436131477356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716874188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11529944837093353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162201600, "lr": 8.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716890656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11153604090213776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.265833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716907069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1162586659193039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162856960, "lr": 8.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716907069, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784716907070, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784716950591, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7754827737808228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784716950592, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784716950592, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784716967005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1160585880279541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.299166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716983174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11500181257724762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163512320, "lr": 8.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716999058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1164824366569519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717015319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10392438620328903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164167680, "lr": 8.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717031351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10152419656515121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717047557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10665567219257355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164823040, "lr": 8.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717063511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10598799586296082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717063512, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784717063512, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784717106960, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7755924463272095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784717106960, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784717106961, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784717122576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10993970930576324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165478400, "lr": 8.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717138545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10786277800798416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717154440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10170702636241913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166133760, "lr": 8.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717170554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10459049046039581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717186671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10421943664550781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166789120, "lr": 8.482499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717202588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09818124771118164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.499166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717218958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10521373152732849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167444480, "lr": 8.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717218959, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784717218959, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784717263094, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7754903435707092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784717263094, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784717263094, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784717279224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10582372546195984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.532499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717295524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09982378780841827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168099840, "lr": 8.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717311837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09436718374490738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717327804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0999535620212555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168755200, "lr": 8.582499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717343445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11259014904499054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717359761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10530219972133636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169410560, "lr": 8.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717375882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11002316325902939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717375883, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784717375883, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784717420181, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7757834792137146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784717420182, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784717420182, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784717436695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1060321033000946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170065920, "lr": 8.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717452825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1028168797492981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717469056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1155717670917511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170721280, "lr": 8.682499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717485308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09791266918182373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717501750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10703369975090027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171376640, "lr": 8.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717518075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10387107729911804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717534521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10077771544456482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172032000, "lr": 8.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717534521, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784717534521, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784717579095, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7759135961532593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784717579096, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784717579096, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784717595368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10051266849040985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717611565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10568572580814362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172687360, "lr": 8.782499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717628065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1058696061372757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717644574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10726279765367508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173342720, "lr": 8.815833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717661337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10260441899299622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717677860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10333190113306046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173998080, "lr": 8.849166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717694153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10579352080821991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.865833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717694153, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784717694154, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784717740005, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7762230038642883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784717740005, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784717740005, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784717755964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10209223628044128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174653440, "lr": 8.882499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717772272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10591535270214081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717788131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10268545150756836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175308800, "lr": 8.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717804293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11117199063301086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717820127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10678252577781677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175964160, "lr": 8.949166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717835861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1027451679110527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.965833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717851850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11338189244270325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176619520, "lr": 8.982499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717851851, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784717851851, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784717897157, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7765662670135498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784717897158, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784717897158, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784717912987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10254387557506561, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717928815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10315075516700745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177274880, "lr": 9.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717944859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10037703812122345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717960858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10316871851682663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177930240, "lr": 9.049166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717977211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10619707405567169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.065833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717993625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10999047756195068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178585600, "lr": 9.082499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718009825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09932806342840195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718009826, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784718009826, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784718054733, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7766432166099548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784718054733, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784718054733, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784718071200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10129788517951965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179240960, "lr": 9.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718087486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09885728359222412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718103521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10069602727890015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179896320, "lr": 9.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718119084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09821663796901703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.165833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718136393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10620729625225067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180551680, "lr": 9.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718152834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10498849302530289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718169507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10733412206172943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181207040, "lr": 9.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718169508, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784718169508, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784718214127, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7768873572349548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784718214128, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784718214128, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784718230595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10622720420360565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718246936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.098812535405159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181862400, "lr": 9.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718263245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10490953922271729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.265833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718279687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09975077211856842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182517760, "lr": 9.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718295943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0995636060833931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.299166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718312330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10401482880115509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183173120, "lr": 9.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718328701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1044561117887497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718328702, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784718328702, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784718373844, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7770395278930664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784718373844, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784718373845, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784718390401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09887871146202087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183828480, "lr": 9.349166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718406807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10690142214298248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718423115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10246307402849197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184483840, "lr": 9.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718439761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11032740771770477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718456212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10813398659229279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185139200, "lr": 9.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718472441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11172866076231003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718488381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10399800539016724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185794560, "lr": 9.449166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718488382, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784718488382, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784718533687, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7772161364555359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784718533687, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784718533688, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784718550234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10115639865398407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718566469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10955877602100372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186449920, "lr": 9.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718583080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10165698826313019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.499166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718599334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10507823526859283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187105280, "lr": 9.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718615483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09994281828403473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718631711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11128522455692291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187760640, "lr": 9.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718647709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09986527264118195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.565833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718647710, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784718647711, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784718693456, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7773630023002625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784718693457, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784718693457, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784718709695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1053665280342102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188416000, "lr": 9.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718725808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10201014578342438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718741718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10672169923782349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189071360, "lr": 9.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718757859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10401943325996399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718774008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10706591606140137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189726720, "lr": 9.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718790275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11319051682949066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718806531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10226558148860931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190382080, "lr": 9.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718806531, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784718806532, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784718851784, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7777611017227173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784718851784, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784718851784, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784718868116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10630898177623749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.699166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718884525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10117488354444504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191037440, "lr": 9.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718900720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10277942568063736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718916784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1062966138124466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191692800, "lr": 9.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718933182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10764588415622711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.765833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718949745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10803830623626709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192348160, "lr": 9.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718966573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11574406921863556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718966573, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784718966574, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784719010852, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7778844833374023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784719010853, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784719010853, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784719027424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10587279498577118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193003520, "lr": 9.815833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784719043795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10882639139890671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.832499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784719060303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1125369518995285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193658880, "lr": 9.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784719076812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11473090946674347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784719093341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10887718200683594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194314240, "lr": 9.882499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784719109908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10639022290706635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.899166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784719126362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10671916604042053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194969600, "lr": 9.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784719126362, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784719126363, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784719171801, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7783492207527161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784719171802, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784719171802, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784719188366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11107456684112549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784719205065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11099162697792053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195624960, "lr": 9.949166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784719221779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1106681376695633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.965833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784719238817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11214964091777802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196280320, "lr": 9.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784719255517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10932677984237671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784719271837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1144658625125885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196935680, "lr": 1.0015833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719288201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10341060161590576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1.0032499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719288201, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784719288202, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784719333469, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7783010005950928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784719333469, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784719333469, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784719350328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10830318182706833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197591040, "lr": 1.0049166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719367077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11158190667629242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1.0065833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719383275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10705292224884033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198246400, "lr": 1.00825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719399634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11280624568462372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1.0099166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719415861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11447066068649292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198901760, "lr": 1.0115833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719432298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10902142524719238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1.01325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719448708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11339870095252991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199557120, "lr": 1.0149166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719448708, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784719448708, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784719493996, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7786862254142761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784719493996, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784719493996, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784719510127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11494646966457367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1.0165833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719526569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10888584703207016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200212480, "lr": 1.01825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719543299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11352908611297607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1.0199166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719560116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1113106831908226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200867840, "lr": 1.0215833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719577036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1073007583618164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1.02325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719593776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10536563396453857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201523200, "lr": 1.0249166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719610154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11091163754463196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1.0265833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719610154, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784719610154, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784719655260, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7789868116378784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784719655261, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784719655261, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784719671894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10031377524137497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202178560, "lr": 1.02825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719688014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10645779967308044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1.0299166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719704608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10326876491308212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202833920, "lr": 1.0315833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719720926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10553986579179764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1.03325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719737116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1040838286280632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203489280, "lr": 1.0349166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719753223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11181595176458359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1.0365833333333335e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719769497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09834714233875275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204144640, "lr": 1.0382499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719769497, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784719769498, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784719814087, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7787212133407593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784719814088, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784719814088, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784719830419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10028916597366333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1.0399166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719846688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10399628430604935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204800000, "lr": 1.0415833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719862827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10187791287899017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1.04325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719878937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09789930284023285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205455360, "lr": 1.0449166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719895202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10124550759792328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1.0465833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719911362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10175026208162308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206110720, "lr": 1.0482499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719927676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1043524295091629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1.0499166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719927677, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784719927678, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784719972919, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7791268229484558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784719972919, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784719972920, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784719989445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10814207047224045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206766080, "lr": 1.0515833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720005844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10278531908988953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1.05325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720022109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09349173307418823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207421440, "lr": 1.0549166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720038010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10424105077981949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1.0565833333333335e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720054558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10224610567092896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208076800, "lr": 1.0582499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720070763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10235967487096786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1.0599166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720087326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10545244812965393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208732160, "lr": 1.0615833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720087326, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784720087327, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784720132360, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7792197465896606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784720132361, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784720132361, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784720148737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09769806265830994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1.06325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720165040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10561825335025787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209387520, "lr": 1.0649166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720181083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10869885981082916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1.0665833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720197356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09922777116298676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210042880, "lr": 1.06825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720213706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10201633721590042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1.0699166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720229694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10258819907903671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210698240, "lr": 1.0715833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720245788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10421102494001389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1.07325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720245789, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784720245789, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784720290092, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7794784903526306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784720290092, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784720290093, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784720306031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10274870693683624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211353600, "lr": 1.0749166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720322403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10906291007995605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1.076583333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720338650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09648561477661133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212008960, "lr": 1.0782499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720354670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10386062413454056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1.0799166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720371142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10309275984764099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212664320, "lr": 1.0815833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720387800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09828941524028778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1.08325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720404050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10647307336330414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213319680, "lr": 1.0849166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720404051, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784720404051, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784720447714, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7798039317131042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784720447715, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784720447715, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784720463955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1023845300078392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1.0865833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720480051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10449281334877014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213975040, "lr": 1.08825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720496720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10582105815410614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1.0899166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720513053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09980867803096771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214630400, "lr": 1.0915833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720529653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10592561215162277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1.09325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720546154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10781943798065186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215285760, "lr": 1.0949166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720562372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10304506123065948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1.096583333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720562373, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784720562373, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784720607409, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7798337936401367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784720607409, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784720607409, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784720623720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10407781600952148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215941120, "lr": 1.0982499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720640018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10161325335502625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1.0999166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720656308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1015048623085022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216596480, "lr": 1.1015833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720672546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0991407260298729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1.10325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720688920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10509677976369858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217251840, "lr": 1.1049166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720705086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11267267912626266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217579520, "lr": 1.1065833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720721485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1019439846277237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217907200, "lr": 1.10825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720721485, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784720721486, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784720766184, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7800476551055908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784720766184, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784720766184, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784720782298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10630983114242554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218234880, "lr": 1.1099166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720798484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1053580641746521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218562560, "lr": 1.1115833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720814937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1027563214302063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218890240, "lr": 1.11325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720831378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10887555778026581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219217920, "lr": 1.1149166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720847458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10434967279434204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219545600, "lr": 1.1165833333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720863063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09846536815166473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219873280, "lr": 1.1182499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720879264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09820818901062012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220200960, "lr": 1.1199166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720879265, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784720879265, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784720923452, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7803586721420288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784720923452, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784720923452, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784720939302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10387272387742996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220528640, "lr": 1.1215833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720955755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0972662940621376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220856320, "lr": 1.12325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720972012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09777513891458511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221184000, "lr": 1.1249166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720988267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10820606350898743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221511680, "lr": 1.1265833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721004701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10245595872402191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221839360, "lr": 1.12825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721020875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1056051105260849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222167040, "lr": 1.1299166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721037393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11348232626914978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222494720, "lr": 1.1315833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721037394, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784721037394, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784721082548, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7806719541549683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784721082549, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784721082549, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784721098698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11114560067653656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222822400, "lr": 1.1332500000000001e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721114992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10455549508333206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223150080, "lr": 1.1349166666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721131173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10168935358524323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223477760, "lr": 1.1365833333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721147225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10995468497276306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223805440, "lr": 1.1382499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721163652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10539674758911133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224133120, "lr": 1.1399166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721179971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10849665105342865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224460800, "lr": 1.1415833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721196428, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09198581427335739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224788480, "lr": 1.14325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721196429, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784721196429, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784721241160, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7808188199996948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784721241160, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784721241161, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784721257862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10232578217983246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225116160, "lr": 1.1449166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721274203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10204920172691345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225443840, "lr": 1.1465833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721290692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10876034200191498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225771520, "lr": 1.14825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721306778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11049613356590271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226099200, "lr": 1.1499166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721323009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10587653517723083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226426880, "lr": 1.1515833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721339271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1045297309756279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226754560, "lr": 1.1532500000000001e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721355693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09935158491134644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227082240, "lr": 1.1549166666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721355693, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784721355694, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784721400378, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7812017798423767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784721400378, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784721400379, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784721416769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11559431254863739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227409920, "lr": 1.1565833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721432954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10642871260643005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227737600, "lr": 1.1582499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721449179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10182337462902069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228065280, "lr": 1.1599166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721465444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11053429543972015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228392960, "lr": 1.1615833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721481799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10736563801765442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228720640, "lr": 1.1632499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721498205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10840614140033722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229048320, "lr": 1.1649166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721514542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09961773455142975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229376000, "lr": 1.1665833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721514543, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784721514543, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784721558974, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7814204096794128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784721558974, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784721558974, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784721575309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11279487609863281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229703680, "lr": 1.16825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721591797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1038765013217926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230031360, "lr": 1.1699166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721608075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10230474919080734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230359040, "lr": 1.1715833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721624838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10426436364650726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230686720, "lr": 1.17325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721641251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10633537173271179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231014400, "lr": 1.1749166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721657672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10237658768892288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231342080, "lr": 1.1765833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721674278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11021578311920166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231669760, "lr": 1.17825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721674279, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784721674279, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784721717982, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7814531922340393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784721717983, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784721717983, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784721734557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11529261618852615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231997440, "lr": 1.1799166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721750924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10850804299116135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 232325120, "lr": 1.1815833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721767398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10748592019081116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 232652800, "lr": 1.1832499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721783411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10760371387004852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 232980480, "lr": 1.1849166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721800257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10381923615932465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233308160, "lr": 1.1865833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721816967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10471384227275848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233635840, "lr": 1.18825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721833675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11847542971372604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233963520, "lr": 1.1899166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721833676, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784721833676, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784721877165, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7820051908493042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784721877165, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784721877166, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784721893604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11528337001800537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 234291200, "lr": 1.1915833333333335e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721910068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080276370048523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 234618880, "lr": 1.19325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721926457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11410458385944366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 234946560, "lr": 1.1949166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721943103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10464365780353546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235274240, "lr": 1.1965833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721959631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12182991951704025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235601920, "lr": 1.19825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721976195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11422177404165268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235929600, "lr": 1.1999166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721992527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11403868347406387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 236257280, "lr": 1.2015833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721992527, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784721992528, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784722035962, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7820935845375061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784722035963, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784722035963, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784722052470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11641409993171692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 236584960, "lr": 1.2032499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722069234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11435239017009735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 236912640, "lr": 1.2049166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722085899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11425276100635529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237240320, "lr": 1.2065833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722102817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11115696281194687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237568000, "lr": 1.20825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722119691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10982206463813782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237895680, "lr": 1.2099166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722136631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1152428686618805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 238223360, "lr": 1.2115833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722153061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11032983660697937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 238551040, "lr": 1.21325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722153062, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784722153062, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784722197184, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7822748422622681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784722197185, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784722197185, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784722213391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11646227538585663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 238878720, "lr": 1.2149166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722230465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11479295790195465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239206400, "lr": 1.2165833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722247317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11073316633701324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239534080, "lr": 1.21825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722263772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11156004667282104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239861760, "lr": 1.2199166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722280586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11504267156124115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 240189440, "lr": 1.2215833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722297028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11433908343315125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 240517120, "lr": 1.2232499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722313786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11755968630313873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 240844800, "lr": 1.2249166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722313787, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784722313787, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784722358261, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7825287580490112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784722358262, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784722358262, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784722374811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11632236838340759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241172480, "lr": 1.2265833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722391584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11647965013980865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241500160, "lr": 1.22825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722408240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11138010025024414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241827840, "lr": 1.2299166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722425499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11300849914550781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 242155520, "lr": 1.2315833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722442465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11482895165681839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 242483200, "lr": 1.2332499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722459500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11598774790763855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 242810880, "lr": 1.2349166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722476585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11004891991615295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243138560, "lr": 1.2365833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722476586, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784722476586, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784722520353, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.783000111579895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784722520353, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784722520353, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784722537113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1150289922952652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243466240, "lr": 1.23825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722553644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11547352373600006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243793920, "lr": 1.2399166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722570629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10785512626171112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 244121600, "lr": 1.2415833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722587031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10593065619468689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 244449280, "lr": 1.24325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722603790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10698173940181732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 244776960, "lr": 1.2449166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722620205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10911960154771805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245104640, "lr": 1.2465833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722636748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11137877404689789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245432320, "lr": 1.24825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722636749, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784722636749, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784722681499, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7830920219421387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784722681499, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784722681500, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784722697963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10644946992397308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245760000, "lr": 1.2499166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722714597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10488349944353104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 246087680, "lr": 1.2515833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722730964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11371200531721115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 246415360, "lr": 1.2532499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722747588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11261331290006638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 246743040, "lr": 1.2549166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722764020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10462117940187454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247070720, "lr": 1.2565833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722780683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10716857016086578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247398400, "lr": 1.25825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722797335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1090645045042038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247726080, "lr": 1.2599166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722797336, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784722797336, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784722842214, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7832556366920471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784722842215, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784722842215, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784722858749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11340990662574768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 248053760, "lr": 1.2615833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722875489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11492715775966644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 248381440, "lr": 1.26325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722892137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10123977810144424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 248709120, "lr": 1.2649166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722908344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11139471083879471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249036800, "lr": 1.2665833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722924665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10388767719268799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249364480, "lr": 1.26825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722941620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10979323089122772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249692160, "lr": 1.2699166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722958016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10549436509609222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 250019840, "lr": 1.2715833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722958061, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784722958062, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784723003062, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7834515571594238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784723003063, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784723003063, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784723019382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10098406672477722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 250347520, "lr": 1.2732499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784723035605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10161283612251282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 250675200, "lr": 1.2749166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784723051687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1006360650062561, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251002880, "lr": 1.2765833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784723068060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09875153750181198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251330560, "lr": 1.27825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784723084854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.101140096783638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251658240, "lr": 1.2799166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784723101061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10259610414505005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251985920, "lr": 1.2815833333333332e-06}} diff --git a/recommendation_v4/rcp_logs/gbs_16384/seed803889169.log b/recommendation_v4/rcp_logs/gbs_16384/seed803889169.log new file mode 100644 index 000000000..0fafe5d5b --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_16384/seed803889169.log @@ -0,0 +1,1353 @@ +:::MLLOG {"namespace": "", "time_ms": 1784718508646, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784718508646, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784718527972, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784718527973, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784718527973, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784718527973, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784718527973, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784718527973, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 16384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784718527973, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784718527973, "event_type": "POINT_IN_TIME", "key": "seed", "value": 803889169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784718527973, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784718527973, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784718527973, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784718527973, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784718528026, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784718528027, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784719166965, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784719166966, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784719167171, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784719304337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1390523612499237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.5833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784719318058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388501524925232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2499999999999997e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784719331909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13873723149299622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784719345904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13870353996753693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.583333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784719360031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13864946365356445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.25e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784719374392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13876891136169434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784719388585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387384533882141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719388586, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784719388586, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784719433130, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5022972226142883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784719433130, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784719433131, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784719447101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388857662677765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719461349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386702060699463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719475640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13868966698646545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719489942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13867343962192535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719504341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386623978614807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9916666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719518975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385720670223236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719533674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384349763393402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719533674, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784719533675, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784719577892, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5024787187576294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784719577892, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784719577892, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784719592407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860026001930237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719607001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13834749162197113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719621315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13834577798843384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719636176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13824811577796936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.9916666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719650552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384546458721161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719665380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382480263710022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719679829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13808271288871765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719679829, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784719679830, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784719724460, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5027028918266296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784719724460, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784719724461, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784719738878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13851448893547058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719754043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13810987770557404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.8249999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719768876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13760298490524292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719783799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13818247616291046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719799047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13785387575626373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3249999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719813597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377972960472107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719828656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13743987679481506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719828657, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784719828657, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784719872911, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5029817819595337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784719872912, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784719872912, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784719887668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13722531497478485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719902487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378052532672882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719917304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13762414455413818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1583333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719931919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13724291324615479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719946552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13607898354530334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719961122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13712239265441895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719975722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13619504868984222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719975723, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784719975723, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784720020482, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5033083558082581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784720020483, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784720020483, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784720035214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13694316148757935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720049940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366569995880127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720064888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375298947095871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720079605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367347091436386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720094133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13540756702423096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.658333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720108621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13575372099876404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720123173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13759252429008484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720123174, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784720123174, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784720168272, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5036553144454956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784720168273, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784720168273, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784720183062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370520144701004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720198123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13575688004493713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720213050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360742449760437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720228067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351906955242157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720243086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13537010550498962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720258252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13486450910568237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720273300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13520705699920654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720273301, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784720273301, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784720316920, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5041154026985168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784720316920, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784720316921, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784720331849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13437822461128235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720346767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13662150502204895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720362016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13821586966514587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720377177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13410572707653046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.824999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720392461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13622058928012848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720407564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13498471677303314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.158333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720422642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13659608364105225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.324999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720422643, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784720422644, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784720467335, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5047582983970642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784720467336, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784720467336, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784720482433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13660430908203125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720497643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13663652539253235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720512822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13507090508937836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720527900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13055455684661865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720542892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13281726837158203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784720557743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13384531438350677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784720572352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13503021001815796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784720572405, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784720572405, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784720616596, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5054590106010437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784720616597, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784720616597, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784720631455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350078582763672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784720646131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295129954814911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784720661178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13138344883918762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784720675707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13671058416366577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.1158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784720690500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13239499926567078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784720705252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13157698512077332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784720720374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13253431022167206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784720720374, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784720720375, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784720764323, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5062825679779053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784720764324, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784720764324, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784720778708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13181641697883606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784720793875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13420993089675903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784720808474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12824687361717224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784720823125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12957406044006348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784720837852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13375240564346313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784720852917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.128381609916687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784720867433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12992101907730103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784720867433, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784720867434, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784720911477, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5074487328529358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784720911477, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784720911478, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784720926538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345025897026062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784720941114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13307929039001465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.3158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784720955989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344081163406372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784720970816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13489562273025513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784720986081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361265778541565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.3658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721000817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13346286118030548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721015590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13682737946510315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721015590, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784721015591, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784721060086, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5090480446815491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784721060087, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784721060087, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784721075221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13432525098323822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.4158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721090696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13036221265792847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721106177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13393200933933258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.4491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721121715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14038397371768951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.4658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721137021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13042956590652466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721152330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13240262866020203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721167524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13810104131698608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721167524, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784721167524, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784721211594, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5106326341629028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784721211595, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784721211595, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784721227151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12769201397895813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721242549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310761570930481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721258000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13703018426895142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721273202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13534967601299286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721288588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323331594467163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721303655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12871646881103516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.6158333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721319095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12793110311031342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721319096, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784721319096, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784721363035, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5122409462928772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784721363036, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784721363036, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784721378288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301274299621582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721393665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347433477640152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.6658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721409178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1421850621700287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6825000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721424348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134195476770401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721439593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13139688968658447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.7158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721454416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13369929790496826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721469513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359732449054718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721469513, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784721469514, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784721513901, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5140767693519592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784721513901, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784721513902, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784721528745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134589284658432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721543836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363328993320465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721559338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355687379837036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721574755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133085697889328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721589838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13419881463050842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721605297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13296031951904297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721620553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13256165385246277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8658333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721620553, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784721620554, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784721663848, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5160124897956848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784721663848, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784721663848, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784721678898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13200627267360687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721693974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343829333782196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8991666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721709276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14185689389705658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721724871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13476096093654633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721740374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1340646743774414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721755623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13242128491401672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721770917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13310250639915466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721770918, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784721770918, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784721814188, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5183067917823792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784721814189, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784721814189, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784721829560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13795071840286255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721844935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330115795135498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721860261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288823038339615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721875688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13005316257476807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721891334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13784128427505493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721907002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13151703774929047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721922184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362605094909668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721922185, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784721922185, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784721966982, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5208142399787903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784721966983, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784721966983, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784721982482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13686370849609375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.1158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721997912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13643664121627808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722013450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13128966093063354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1491666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722028768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13000941276550293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722044257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13139887154102325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722059957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13523942232131958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722075781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326705515384674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722075782, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784722075782, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784722119907, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5238367915153503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784722119908, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784722119908, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784722135256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13590773940086365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722150966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13222824037075043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722166736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13789910078048706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722182456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362209916114807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722198351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384010761976242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722214073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13435612618923187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722230161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13909050822257996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722230162, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784722230162, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784722275357, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5269607901573181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784722275358, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784722275358, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784722291264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13505956530570984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722307161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14679580926895142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722323092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13508281111717224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722339012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13293758034706116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722355074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341703236103058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722370825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13284799456596375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722386748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13641753792762756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722386749, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784722386749, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784722431201, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5305635929107666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784722431201, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784722431201, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784722447195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360931098461151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722462984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13533049821853638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722478723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13370807468891144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722494117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13350558280944824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722509473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13316738605499268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722524789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12948790192604065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722540020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12553173303604126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722540021, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784722540021, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784722584927, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5347399115562439, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784722584928, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784722584928, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784722600371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13242758810520172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.5824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722615970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13239943981170654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722631544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13490024209022522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722647037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13532647490501404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722662348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13656187057495117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722678083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1287839114665985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722693576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297197937965393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722693577, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784722693577, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784722738161, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5410400629043579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784722738161, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784722738161, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784722753588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13249143958091736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.699166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722769040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1271311342716217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722784554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13302499055862427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722800169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13210885226726532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722816044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1269637644290924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722831968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13212203979492188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722847331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13415095210075378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722847332, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784722847332, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784722892803, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5499864816665649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784722892804, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784722892804, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784722908439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1293686032295227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722923961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13019992411136627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722939428, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13050153851509094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722954915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12972044944763184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.8658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722970150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1276628077030182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722985516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12315213680267334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723001274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12700039148330688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723001275, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784723001275, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784723045381, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5615397691726685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784723045382, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784723045382, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784723060797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13551929593086243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723076350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13307034969329834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723091478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387125551700592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723107284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13328012824058533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723122948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12759727239608765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723138444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294931173324585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723154105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12659883499145508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723154106, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784723154106, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784723198658, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5750980973243713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784723198659, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784723198659, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784723214037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12883900105953217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723229532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12733806669712067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723245362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13219070434570312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723261389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13250693678855896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723277360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311759203672409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723292880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13339129090309143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723308402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13014408946037292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723308402, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784723308403, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784723352065, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5899373292922974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784723352066, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784723352066, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784723367709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12923134863376617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723383201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12950006127357483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723398660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12902358174324036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723414141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322569102048874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723429513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327083259820938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723444740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1291511356830597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723460696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13008032739162445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723460696, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784723460697, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784723505328, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6016857028007507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784723505329, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784723505329, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784723520935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1275852918624878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723536782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.130485400557518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723551899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12753818929195404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723567965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1273450255393982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723584347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12686514854431152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723600412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13087710738182068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723616598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13167059421539307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723616599, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784723616599, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784723659605, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6119763851165771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784723659605, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784723659606, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784723675306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12934130430221558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723691423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12711289525032043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723707881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12977400422096252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723724159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13168638944625854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723740301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325710117816925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723756052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12841099500656128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723772059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12836244702339172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.4991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723772060, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784723772060, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784723815623, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6216235160827637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784723815624, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784723815624, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784723831375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289774775505066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.5158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723847265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12705320119857788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723863126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12773168087005615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.5491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723878702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13443449139595032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723894476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301208734512329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723910432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1250869333744049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723926375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12796461582183838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723926375, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784723926376, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784723971064, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6298657655715942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784723971064, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784723971065, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784723986546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12487674504518509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6325000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724002784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12758871912956238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724018504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12606820464134216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724034220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12569333612918854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724049649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12442421913146973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724065090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12114951014518738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724080508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12325599044561386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724080509, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784724080510, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784724123772, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6392695307731628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784724123773, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784724123773, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784724139702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1259945183992386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724155565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12241348624229431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724171230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11552789062261581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.7824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724186647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12225696444511414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724202319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11785619705915451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724217713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11848830431699753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8324999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724233080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11576742678880692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.8491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724233080, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784724233081, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784724276544, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.651831328868866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784724276545, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784724276545, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784724292165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12142974138259888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724307523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1176995262503624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724322743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11322980374097824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.8991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724338235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1226675808429718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724354049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1199483796954155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724369811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11902622878551483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.9491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724385260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11081910133361816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.9658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724385260, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784724385261, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784724429266, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6663236618041992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784724429266, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784724429266, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784724444482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11709339171648026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724460513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11700358986854553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724476225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12005665898323059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.0158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724491659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11522425711154938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724507085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11353608965873718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724522321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11464214324951172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0658333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724537634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11170372366905212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724537684, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784724537684, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784724580390, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6843106150627136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784724580390, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784724580390, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784724595500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11614332348108292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724611378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11431874334812164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.1158333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724627197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12045003473758698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724643249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11463892459869385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724658579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11601287871599197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1658333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724674294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12053036689758301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724689527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1102297455072403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724689527, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784724689528, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784724733331, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7019452452659607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784724733332, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784724733332, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784724748687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11793700605630875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724764058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11531615257263184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724779699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1129547655582428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724795093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11266811192035675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724810586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10804346203804016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724825769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12087342143058777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724841781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11781731992959976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724841781, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784724841782, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784724885217, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.71845543384552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784724885218, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784724885218, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784724900903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12122289836406708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724916887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1153930202126503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.3491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724932336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11900125443935394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724947939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1063040941953659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724963649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11768296360969543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.3991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724979330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11282731592655182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.4158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724994973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11057417094707489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724994974, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784724994974, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784725037999, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7311462759971619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784725037999, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784725037999, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784725053409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11316868662834167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.4491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725068960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11203798651695251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725084220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11708870530128479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.4824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725099810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11325110495090485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725115599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11697155237197876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725131300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10784853249788284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725147213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11657627671957016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725147214, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784725147214, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784725189462, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7400932312011719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784725189462, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784725189462, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784725205135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11140729486942291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725220548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11376457661390305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.5824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725236610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12451612204313278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725252657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12673529982566833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725268492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12120756506919861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725284527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11539237201213837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725300433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11977110803127289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725300433, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784725300434, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784725343517, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7441847920417786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784725343518, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784725343518, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784725359230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11965851485729218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725375252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1120695024728775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.6991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725391474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12210661917924881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725407632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11458833515644073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725423908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11975344270467758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725439575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11710778623819351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725455407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10880248248577118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725455408, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784725455408, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784725499002, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7476539015769958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784725499003, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784725499003, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784725514831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12086735665798187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725530823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11133447289466858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725546693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10670046508312225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725562506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11746825277805328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725578382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12124906480312347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725594375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11110112071037292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725610841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11738571524620056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725610841, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784725610842, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784725653921, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7559205293655396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784725653921, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784725653921, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784725669988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11769688874483109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725686292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11162959039211273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725701919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11129388213157654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725717596, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11609230935573578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725733290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10968497395515442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725748788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10583549737930298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725764156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11650621145963669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.015833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725764157, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784725764157, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784725807340, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7577565312385559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784725807341, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784725807341, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784725822957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11166330426931381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725838559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11629930138587952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725854060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11227557808160782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725869411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1070525199174881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725885285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1048966646194458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725901141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09892280399799347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725917247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10827575623989105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725917248, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784725917248, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784725960674, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7606504559516907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784725960674, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784725960675, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784725976332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732267796993256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725991637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11065878719091415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726007994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12159538269042969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726024009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10977181792259216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726039704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732895135879517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726055049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10282759368419647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726070697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10872595012187958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726070697, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784726070698, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784726114448, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7610953450202942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784726114448, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784726114449, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784726130025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10547864437103271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726145411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10770297050476074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726161086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11041879653930664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726176663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10850059986114502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726192521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1074829176068306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726208314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11263226717710495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726224305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11148113012313843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726224305, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784726224305, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784726268639, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7617540955543518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784726268639, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784726268640, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784726284170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10651957988739014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726299786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10274983942508698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726315560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11037150025367737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726331401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10945439338684082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726346969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10532931238412857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726362482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1007247269153595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726378205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10472682118415833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726378205, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784726378206, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784726421303, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7613145709037781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784726421304, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784726421304, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784726436783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10425564646720886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726452509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10023986548185349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726468020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10531425476074219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726483976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11106280982494354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726499791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.106552854180336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726515792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10399836301803589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726531645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11033733189105988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726531645, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784726531646, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784726574954, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7627428770065308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784726574954, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784726574954, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784726590130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0998762920498848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726605845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10379237681627274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726621518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10749106854200363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726637098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1048876941204071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726652622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1021476536989212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726667993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10815134644508362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726683575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10903064906597137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726683575, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784726683576, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784726726355, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7628162503242493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784726726356, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784726726356, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784726741834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10614549368619919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726757358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1029953807592392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726772744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10947231948375702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726788540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10675263404846191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726812613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11030014604330063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726834473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10803112387657166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726857460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11122530698776245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726857465, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784726857466, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784726901151, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7651209235191345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784726901151, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784726901151, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784726916378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10623662918806076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726935182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10103116929531097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726951036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10486719012260437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726968928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10187637060880661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726984580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10834425687789917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727001301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11300167441368103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727016982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10861711204051971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727016983, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784727021749, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784727063756, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7649539113044739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784727063757, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784727063757, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784727079557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10485458374023438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727095315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11053668707609177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727110976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10794192552566528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727126869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.100522980093956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727142819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11272501945495605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727158742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1130276620388031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727174981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10530925542116165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727174981, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784727174982, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784727217634, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7672056555747986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784727217635, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784727217636, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784727233470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10838152468204498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727248938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0996268168091774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727265116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11560660600662231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727281542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1155121847987175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727297577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11480718851089478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727313528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10463976860046387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727329722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10689745098352432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727329722, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784727329722, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784727371880, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7677083611488342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784727371881, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784727371881, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784727387983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1111941710114479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727404042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11392922699451447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727420298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11018303036689758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727436787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11094126105308533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727453154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11404999345541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727469398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11217768490314484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727485888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11278559267520905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727485889, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784727485889, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784727530227, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7679795026779175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784727530228, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784727530228, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784727546027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10879379510879517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727562074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1069520115852356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727578277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10687307268381119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727594805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11442161351442337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727610818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11775188893079758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727626866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11012467741966248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727643287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11526647955179214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.415833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727643288, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784727643289, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784727686665, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7685266733169556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784727686666, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784727686666, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784727702713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11420920491218567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727718851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11282798647880554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727735187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11558730900287628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727751541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11398781836032867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727768002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10895553231239319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727784266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10878886282444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.515833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727800107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11089980602264404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727800107, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784727800108, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784727844168, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.769229531288147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784727844168, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784727844169, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784727860038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087651327252388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727876171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10491664707660675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727891747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10486388206481934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727907447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10457822680473328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727923141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10692471265792847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.615833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727939252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1029764711856842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727955058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10839636623859406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727955059, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784727955059, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784727998312, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7713470458984375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784727998313, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784727998313, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784728014104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10952776670455933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728029732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10218502581119537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728045784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10745237022638321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728061797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10284270346164703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728077921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10221205651760101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728094230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10571612417697906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728110059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09943531453609467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728110060, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784728110060, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784728154302, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.771411657333374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784728154302, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784728154302, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784728169783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1074303388595581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728185981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1105521097779274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728202162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10622557252645493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728218224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10613907873630524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728234264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10729673504829407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728250071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10332653671503067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728266060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10760782659053802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728266060, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784728266061, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784728309305, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7726250290870667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784728309306, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784728309306, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784728325291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10791368782520294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728341202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10515071451663971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728357328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10832835733890533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728373261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10297199338674545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728389490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084987074136734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728405810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10917229950428009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728422279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10451546311378479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728422280, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784728422280, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784728466344, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7728912234306335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784728466344, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784728466345, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784728482650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10743865370750427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728498311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10188774764537811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728514233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11178989708423615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728529804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10281847417354584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728546081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0998542532324791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728561906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10193581879138947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728577700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10566350817680359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728577700, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784728577701, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784728622607, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7731733918190002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784728622608, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784728622608, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784728638206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1030646562576294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728654294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1000518873333931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728670511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10847225785255432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728686568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10928893089294434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728702225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1042746976017952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728718369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081262081861496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728734454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10910243541002274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728734455, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784728734455, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784728777452, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738591432571411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784728777453, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784728777453, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784728793717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11054211109876633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728810320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10420765727758408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728826527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10393781214952469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728842226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10548366606235504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728858266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10036160051822662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728874183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10720208287239075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728890644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10405413806438446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728890644, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784728890645, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784728935217, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742528915405273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784728935218, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784728935218, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784728951202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10695576667785645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728967088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10768810659646988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728983023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10034225881099701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728999021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10892686247825623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729015052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09661240875720978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729031257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10202871263027191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729046851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10320933163166046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729046851, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784729046852, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784729089928, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7737529873847961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784729089929, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784729089929, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784729106184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10234060138463974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729122359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10822752118110657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729138423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084246039390564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729154766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1100640520453453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729171103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10766290128231049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729186872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10934347659349442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.565833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729202845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10149212181568146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729202845, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784729202846, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784729245959, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7750225067138672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784729245959, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784729245959, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784729262126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10921581089496613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729278526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10414236783981323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149749760, "lr": 7.615833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729294563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10655064880847931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729310519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11123634874820709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150405120, "lr": 7.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729326424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10743696987628937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729342579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11240220069885254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151060480, "lr": 7.682499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729358573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10371364653110504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729358574, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784729358574, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784729403248, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7749373912811279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784729403248, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784729403249, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784729419171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10611330717802048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151715840, "lr": 7.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729435212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09792156517505646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729451257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11036402732133865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152371200, "lr": 7.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729467353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10410649329423904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729483689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1114671379327774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153026560, "lr": 7.782499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729500008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11271827667951584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729516545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10361406952142715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153681920, "lr": 7.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729516546, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784729516546, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784729561185, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752889394760132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784729561186, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784729561186, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784729577200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10233916342258453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729592920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10686949640512466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154337280, "lr": 7.849166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729609395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10195275396108627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729625774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11009397357702255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154992640, "lr": 7.882499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729642483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1103205680847168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729659034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11846470832824707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155648000, "lr": 7.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729675503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11766298115253448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729675503, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784729675504, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784729719895, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7757650017738342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784729719896, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784729719896, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784729736332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11789388209581375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156303360, "lr": 7.949166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729752860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10958980023860931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729769057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11453508585691452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156958720, "lr": 7.982499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729785412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11140819638967514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729801957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11448876559734344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157614080, "lr": 8.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729818676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11637459695339203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729835407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10811048746109009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158269440, "lr": 8.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729835407, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784729835408, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784729879595, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7750682830810547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784729879596, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784729879596, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784729896399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11065700650215149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729912987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10460060834884644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158924800, "lr": 8.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729929044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11137206852436066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.099166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729945449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11191797256469727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159580160, "lr": 8.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729961881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10945475846529007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729978157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10719314962625504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160235520, "lr": 8.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729994544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10965609550476074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729994544, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784729994544, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784730038242, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774948000907898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784730038243, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784730038243, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784730054449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11280408501625061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160890880, "lr": 8.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730070834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10480460524559021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.199166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730087545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10788847506046295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161546240, "lr": 8.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730104154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10910025238990784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730120671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11489630490541458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162201600, "lr": 8.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730137448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11111407727003098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.265833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730153975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11540211737155914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162856960, "lr": 8.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730153976, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784730153976, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784730197066, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7758489847183228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784730197066, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784730197066, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784730213823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11598198860883713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.299166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730230244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11404958367347717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163512320, "lr": 8.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730246382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1164322942495346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730262795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10385856032371521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164167680, "lr": 8.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730279236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10153228044509888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730295622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10634193569421768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164823040, "lr": 8.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730311970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10585927963256836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730311970, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784730311971, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784730354722, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7758554220199585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784730354722, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784730354722, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784730370858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11026285588741302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165478400, "lr": 8.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730387074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10741571336984634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730403567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10134261846542358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166133760, "lr": 8.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730419919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10488282144069672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730436168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10423272848129272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166789120, "lr": 8.482499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730452164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09793966263532639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.499166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730468597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10543755441904068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167444480, "lr": 8.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730468598, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784730468598, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784730512119, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7762970924377441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784730512120, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784730512120, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784730528426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1057351753115654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.532499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730545142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09967593848705292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168099840, "lr": 8.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730561871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09407258033752441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730578369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0998416543006897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168755200, "lr": 8.582499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730594456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11171022802591324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730610600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1043933853507042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169410560, "lr": 8.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730626578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11005619168281555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730626579, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784730626579, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784730669733, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7766962647438049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784730669734, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784730669734, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784730686341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1053791269659996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170065920, "lr": 8.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730702529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10240234434604645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730718515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11486268043518066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170721280, "lr": 8.682499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730734502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0975942611694336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730750736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10718424618244171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171376640, "lr": 8.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730766842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10380083322525024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730782981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10034935176372528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172032000, "lr": 8.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730782981, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784730782982, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784730825684, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7766362428665161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784730825685, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784730825685, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784730841661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1002180427312851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730857776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10537664592266083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172687360, "lr": 8.782499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730874225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10599758476018906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730890693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1076703816652298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173342720, "lr": 8.815833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730907317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10234256088733673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730923698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10311084240674973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173998080, "lr": 8.849166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730939951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10521750152111053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.865833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730939951, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784730939952, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784730982859, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7767080068588257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784730982859, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784730982860, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784730998607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10157783329486847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174653440, "lr": 8.882499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731014939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10564866662025452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731031252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10270634293556213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175308800, "lr": 8.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731047578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11123847961425781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731063772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10678718984127045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175964160, "lr": 8.949166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731079617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10304736346006393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.965833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731095826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11288927495479584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176619520, "lr": 8.982499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731095826, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784731095827, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784731138971, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7770130634307861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784731138971, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784731138972, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784731155049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10224159061908722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731170977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10287649929523468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177274880, "lr": 9.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731186884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0999222993850708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731202633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10263115167617798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177930240, "lr": 9.049166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731218715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10579152405261993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.065833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731235028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10949715226888657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178585600, "lr": 9.082499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731251241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0987078845500946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731251241, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784731251242, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784731295843, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.777108907699585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784731295844, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784731295844, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784731312164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10096532106399536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179240960, "lr": 9.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731328508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0984276831150055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731344553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10014566779136658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179896320, "lr": 9.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731360136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09784939140081406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.165833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731376463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10590891540050507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180551680, "lr": 9.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731392512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10419508814811707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731408654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10692042112350464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181207040, "lr": 9.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731408654, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784731408655, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784731453415, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7773790955543518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784731453415, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784731453416, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784731469378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10563509166240692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731485269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09837815165519714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181862400, "lr": 9.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731500989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10459919273853302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.265833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731516977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09919635951519012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182517760, "lr": 9.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731532798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0996703952550888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.299166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731548701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10403575003147125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183173120, "lr": 9.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731564680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10372591018676758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731564680, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784731564681, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784731608141, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.77744060754776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784731608142, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784731608142, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784731624140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09777683019638062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183828480, "lr": 9.349166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731640254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10738794505596161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731656373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10249939560890198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184483840, "lr": 9.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731672784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.110219806432724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731689041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10760065913200378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185139200, "lr": 9.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731705013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1120840460062027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731720753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10304437577724457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185794560, "lr": 9.449166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731720753, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784731720754, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784731765237, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7775139212608337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784731765239, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784731765239, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784731781303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10109181702136993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731797353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10929428040981293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186449920, "lr": 9.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731813834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10025452077388763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.499166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731829936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10474307835102081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187105280, "lr": 9.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731845856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09931440651416779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731861799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1098860576748848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187760640, "lr": 9.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731877591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09947013854980469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.565833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731877592, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784731877593, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784731921292, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7776626944541931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784731921293, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784731921293, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784731937509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10485856980085373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188416000, "lr": 9.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731954655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10178034007549286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731970428, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10628293454647064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189071360, "lr": 9.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731986439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10371878743171692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732002481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10677438974380493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189726720, "lr": 9.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732018809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1130642294883728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732035005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1021890789270401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190382080, "lr": 9.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732035005, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784732035006, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784732077831, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7777687311172485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784732077832, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784732077832, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784732096836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1066158264875412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.699166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732113164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10105064511299133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191037440, "lr": 9.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732129316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1027524322271347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732145342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10624788701534271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191692800, "lr": 9.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732161349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10726308822631836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.765833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732177683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772638022899628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192348160, "lr": 9.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732194260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11433997005224228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732194261, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784732194261, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784732238364, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7782884240150452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784732238365, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784732238365, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784732254848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10532569885253906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193003520, "lr": 9.815833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732271359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10796800255775452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.832499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732287994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11194942891597748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193658880, "lr": 9.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732304498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11423560976982117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732321202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10888529568910599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194314240, "lr": 9.882499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732338009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10580956935882568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.899166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732354583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10653418302536011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194969600, "lr": 9.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732354583, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784732354584, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784732399639, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7781241536140442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784732399640, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784732399640, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784732416445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11110199242830276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732433389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11029176414012909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195624960, "lr": 9.949166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732450187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10988758504390717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.965833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732467298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11196576803922653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196280320, "lr": 9.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732484283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10891179740428925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732500802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11396321654319763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196935680, "lr": 1.0015833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732517205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10322140157222748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1.0032499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732517205, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784732517206, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784732561950, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7782864570617676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784732561951, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784732561951, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784732578904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10776640474796295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197591040, "lr": 1.0049166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732595898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11115226149559021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1.0065833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732612438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1061759814620018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198246400, "lr": 1.00825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732628971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11278054863214493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1.0099166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732645492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11384168267250061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198901760, "lr": 1.0115833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732662268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10862746834754944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1.01325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732678958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11347614973783493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199557120, "lr": 1.0149166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732678958, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784732678959, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784732722998, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7784984111785889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784732722998, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784732722998, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784732739344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11532709002494812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1.0165833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732755953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10860724002122879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200212480, "lr": 1.01825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732772728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11362588405609131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1.0199166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732789448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11091751605272293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200867840, "lr": 1.0215833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732806243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10697639733552933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1.02325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732822791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10478414595127106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201523200, "lr": 1.0249166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732838804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11046688258647919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1.0265833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732838805, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784732838805, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784732883901, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7781712412834167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784732883902, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784732883902, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784732900346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1001100093126297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202178560, "lr": 1.02825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732916637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10626315325498581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1.0299166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732933382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10371638834476471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202833920, "lr": 1.0315833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732949856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10555684566497803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1.03325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732966246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10388609021902084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203489280, "lr": 1.0349166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732982413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11072899401187897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1.0365833333333335e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732998760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09834247827529907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204144640, "lr": 1.0382499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732998760, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784732998761, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784733042628, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7784311175346375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784733042628, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784733042628, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784733058878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09986448287963867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1.0399166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733075182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10424336791038513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204800000, "lr": 1.0415833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733091274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10088085383176804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1.04325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733107398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09753648936748505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205455360, "lr": 1.0449166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733123735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10089342296123505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1.0465833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733139964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10138905048370361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206110720, "lr": 1.0482499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733156152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10389858484268188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1.0499166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733156152, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784733156153, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784733201128, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7782261967658997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784733201128, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784733201128, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784733217780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10857610404491425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206766080, "lr": 1.0515833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733234088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10271921753883362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1.05325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733250320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09216845780611038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207421440, "lr": 1.0549166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733266081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10402827709913254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1.0565833333333335e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733282193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10206253826618195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208076800, "lr": 1.0582499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733298167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1019136905670166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1.0599166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733314431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10491164028644562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208732160, "lr": 1.0615833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733314432, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784733314432, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784733360105, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.77864009141922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784733360106, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784733360106, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784733376141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09776505827903748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1.06325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733392229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1052480936050415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209387520, "lr": 1.0649166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733408081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1083431988954544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1.0665833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733424251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09885672479867935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210042880, "lr": 1.06825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733440514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10184574127197266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1.0699166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733456546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10207248479127884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210698240, "lr": 1.0715833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733472543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10451863706111908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1.07325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733472544, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784733472544, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784733517236, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.778857946395874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784733517237, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784733517237, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784733533071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10248737037181854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211353600, "lr": 1.0749166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733549591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10962264239788055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1.076583333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733565914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09647436439990997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212008960, "lr": 1.0782499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733581914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1036107987165451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1.0799166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733598390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10297323763370514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212664320, "lr": 1.0815833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733615014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09844411909580231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1.08325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733631296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10600442439317703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213319680, "lr": 1.0849166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733631297, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784733631297, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784733674875, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7791138291358948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784733674876, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784733674876, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784733691040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10167054831981659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1.0865833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733706895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10367664694786072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213975040, "lr": 1.08825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733723178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10568075627088547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1.0899166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733739514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09975217282772064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214630400, "lr": 1.0915833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733755890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10463470965623856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1.09325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733772146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10771060734987259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215285760, "lr": 1.0949166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733787981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10343889892101288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1.096583333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733787981, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784733787982, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784733831791, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7793381214141846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784733831792, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784733831792, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784733847830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10369203984737396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215941120, "lr": 1.0982499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733864025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10120244324207306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1.0999166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733880231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10121618211269379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216596480, "lr": 1.1015833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733896356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09908410161733627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1.10325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733912352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10478067398071289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217251840, "lr": 1.1049166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733928259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11257591843605042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217579520, "lr": 1.1065833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733944476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10214415192604065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217907200, "lr": 1.10825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733944477, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784733944477, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784733988354, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7794325351715088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784733988354, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784733988355, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784734004529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10583549737930298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218234880, "lr": 1.1099166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734020718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10491068661212921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218562560, "lr": 1.1115833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734037192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.102378711104393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218890240, "lr": 1.11325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734053767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10782058537006378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219217920, "lr": 1.1149166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734069963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10413624346256256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219545600, "lr": 1.1165833333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734085922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09849926829338074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219873280, "lr": 1.1182499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734102092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09803474694490433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220200960, "lr": 1.1199166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734102092, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784734102093, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784734145748, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7793163657188416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784734145748, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784734145749, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784734161788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10406550765037537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220528640, "lr": 1.1215833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734178612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0973050594329834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220856320, "lr": 1.12325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734195087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09759830683469772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221184000, "lr": 1.1249166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734211376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10814547538757324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221511680, "lr": 1.1265833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734227926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10227794945240021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221839360, "lr": 1.12825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734244262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10554040968418121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222167040, "lr": 1.1299166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734260993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1133560836315155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222494720, "lr": 1.1315833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734260993, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784734260994, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784734304253, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7795529961585999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784734304253, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784734304254, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784734320760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11095888912677765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222822400, "lr": 1.1332500000000001e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734337133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10435716062784195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223150080, "lr": 1.1349166666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734353459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10097211599349976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223477760, "lr": 1.1365833333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734369548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1100517064332962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223805440, "lr": 1.1382499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734386130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10504601150751114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224133120, "lr": 1.1399166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734402476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10799048840999603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224460800, "lr": 1.1415833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734418903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09116865694522858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224788480, "lr": 1.14325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734418904, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784734418904, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784734463123, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7793929576873779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784734463124, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784734463124, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784734479869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10214358568191528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225116160, "lr": 1.1449166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734496318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10173705220222473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225443840, "lr": 1.1465833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734512755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10866668820381165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225771520, "lr": 1.14825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734528891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11040407419204712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226099200, "lr": 1.1499166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734544950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10686729848384857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226426880, "lr": 1.1515833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734561395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10413295030593872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226754560, "lr": 1.1532500000000001e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734578143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09904969483613968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227082240, "lr": 1.1549166666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734578144, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784734578144, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784734621834, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7798340916633606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784734621835, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784734621835, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784734638457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11622126400470734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227409920, "lr": 1.1565833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734654947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10661369562149048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227737600, "lr": 1.1582499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734671369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10230319201946259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228065280, "lr": 1.1599166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734687783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1106029748916626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228392960, "lr": 1.1615833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734704225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10707777738571167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228720640, "lr": 1.1632499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734720949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10888388752937317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229048320, "lr": 1.1649166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734737551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0992068275809288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229376000, "lr": 1.1665833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734737551, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784734737552, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784734782177, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7799350023269653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784734782178, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784734782178, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784734799014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11319194734096527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229703680, "lr": 1.16825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734815857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1032496839761734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230031360, "lr": 1.1699166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734832395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10262706130743027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230359040, "lr": 1.1715833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734849399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10430803149938583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230686720, "lr": 1.17325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734865961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10618540644645691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231014400, "lr": 1.1749166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734882572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10243329405784607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231342080, "lr": 1.1765833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734899502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1099577397108078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231669760, "lr": 1.17825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734899503, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784734899503, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784734945318, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7798921465873718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784734945319, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784734945319, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784734961967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11538331210613251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231997440, "lr": 1.1799166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734978624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082068681716919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 232325120, "lr": 1.1815833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734995408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10755395144224167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 232652800, "lr": 1.1832499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735011586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10734444856643677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 232980480, "lr": 1.1849166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735028368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10325433313846588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233308160, "lr": 1.1865833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735045335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10473500937223434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233635840, "lr": 1.18825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735062395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11826424300670624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233963520, "lr": 1.1899166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735062395, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784735062396, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784735107695, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7802773714065552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784735107695, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784735107696, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784735124571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11472838371992111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 234291200, "lr": 1.1915833333333335e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735141462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10838501155376434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 234618880, "lr": 1.19325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735158189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11438743770122528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 234946560, "lr": 1.1949166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735175237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1046493873000145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235274240, "lr": 1.1965833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735192085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12161421775817871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235601920, "lr": 1.19825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735209077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11403752863407135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235929600, "lr": 1.1999166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735225866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11403299123048782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 236257280, "lr": 1.2015833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735225867, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784735225867, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784735270286, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7804437875747681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784735270287, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784735270287, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784735287140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11619822680950165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 236584960, "lr": 1.2032499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735304125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11396980285644531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 236912640, "lr": 1.2049166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735320952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11359626054763794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237240320, "lr": 1.2065833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735337846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11143413931131363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237568000, "lr": 1.20825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735355017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11059422791004181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237895680, "lr": 1.2099166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735372216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11538569629192352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 238223360, "lr": 1.2115833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735388903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11005118489265442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 238551040, "lr": 1.21325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735388903, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784735388903, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784735434989, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7805465459823608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784735434990, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784735434990, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784735451473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11589397490024567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 238878720, "lr": 1.2149166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735468539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.114352285861969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239206400, "lr": 1.2165833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735485566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1104709729552269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239534080, "lr": 1.21825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735502343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1118701696395874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239861760, "lr": 1.2199166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735519279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11492198705673218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 240189440, "lr": 1.2215833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735535835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11399495601654053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 240517120, "lr": 1.2232499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735552627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11759863793849945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 240844800, "lr": 1.2249166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735552627, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784735552628, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784735597569, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7806946039199829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784735597570, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784735597571, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784735614402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11643226444721222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241172480, "lr": 1.2265833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735631345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11645737290382385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241500160, "lr": 1.22825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735647931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11130307614803314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241827840, "lr": 1.2299166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735665146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11298063397407532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 242155520, "lr": 1.2315833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735681996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11581367999315262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 242483200, "lr": 1.2332499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735699007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1162201315164566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 242810880, "lr": 1.2349166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735716143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10952402651309967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243138560, "lr": 1.2365833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735716143, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784735716144, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784735760677, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7805962562561035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784735760678, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784735760678, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784735777558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11422137171030045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243466240, "lr": 1.23825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735794553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11554843932390213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243793920, "lr": 1.2399166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735811633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10724194347858429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 244121600, "lr": 1.2415833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735828349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10533390939235687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 244449280, "lr": 1.24325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735845248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10724429786205292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 244776960, "lr": 1.2449166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735861859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10914549231529236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245104640, "lr": 1.2465833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735878455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11081696301698685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245432320, "lr": 1.24825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735878456, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784735878456, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784735921733, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7804448008537292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784735921734, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784735921734, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784735938155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10604891180992126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245760000, "lr": 1.2499166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735954882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10528513044118881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 246087680, "lr": 1.2515833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735971417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11391547322273254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 246415360, "lr": 1.2532499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735988335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11231239140033722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 246743040, "lr": 1.2549166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736005016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10443706065416336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247070720, "lr": 1.2565833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736021726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10727232694625854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247398400, "lr": 1.25825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736038533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10979050397872925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247726080, "lr": 1.2599166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736038533, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784736038534, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784736083626, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7806668877601624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784736083627, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784736083627, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784736100398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11330112814903259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 248053760, "lr": 1.2615833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736117330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1154743880033493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 248381440, "lr": 1.26325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736134158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1015772894024849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 248709120, "lr": 1.2649166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736150502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10994978249073029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249036800, "lr": 1.2665833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736166875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.104033462703228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249364480, "lr": 1.26825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736183846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10986549407243729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249692160, "lr": 1.2699166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736200634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10561409592628479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 250019840, "lr": 1.2715833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736200732, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784736200733, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784736244336, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7803975939750671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784736244337, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784736244337, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784736261167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10055485367774963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 250347520, "lr": 1.2732499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736277899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.101424939930439, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 250675200, "lr": 1.2749166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736294186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10134632885456085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251002880, "lr": 1.2765833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736310651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09912469238042831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251330560, "lr": 1.27825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736327463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10073123872280121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251658240, "lr": 1.2799166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736343975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10266347229480743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251985920, "lr": 1.2815833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736360617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10923762619495392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 252313600, "lr": 1.28325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736360722, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784736360723, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784736405532, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7808695435523987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784736405533, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784736405533, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784736421865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08319468796253204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 252641280, "lr": 1.2849166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736438295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10439413785934448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 252968960, "lr": 1.2865833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736455346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10947485268115997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 253296640, "lr": 1.28825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736472027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11633162945508957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 253624320, "lr": 1.2899166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736488637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11579447239637375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 253952000, "lr": 1.2915833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736505464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1051582396030426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 254279680, "lr": 1.2932499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736522442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10363534837961197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 254607360, "lr": 1.2949166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736522559, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784736522560, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784736566599, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7809459567070007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784736566600, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784736566600, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784736582993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10193581134080887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 254935040, "lr": 1.2965833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736599306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10305796563625336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 255262720, "lr": 1.2982499999999998e-06}} diff --git a/recommendation_v4/rcp_logs/gbs_16384/seed869903124.log b/recommendation_v4/rcp_logs/gbs_16384/seed869903124.log new file mode 100644 index 000000000..2853d5fd4 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_16384/seed869903124.log @@ -0,0 +1,800 @@ +:::MLLOG {"namespace": "", "time_ms": 1784878421935, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784878421936, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784878442425, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784878442427, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784878442427, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784878442427, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784878442427, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784878442427, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 16384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784878442427, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784878442427, "event_type": "POINT_IN_TIME", "key": "seed", "value": 869903124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784878442427, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784878442427, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784878442427, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784878442427, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784878442459, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784878442460, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784879337356, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784879337358, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784879337684, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784879483719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13902723789215088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.5833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784879498004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388220191001892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2499999999999997e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784879511997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388496607542038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784879526129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13869400322437286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.583333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784879540300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13869109749794006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.25e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784879554490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386108249425888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784879568537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13870951533317566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784879568538, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784879568538, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784879613004, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49948135018348694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784879613004, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784879613005, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784879627154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13871479034423828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784879641067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388287991285324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784879655272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13862305879592896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784879669543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385851800441742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784879683825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13870257139205933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9916666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784879698290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13838618993759155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784879713012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13813239336013794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784879713013, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784879713014, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784879757729, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4996693432331085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784879757729, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784879757730, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784879772233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13857746124267578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784879786808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383836269378662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784879800895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13838255405426025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784879815849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13813820481300354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.9916666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784879830279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13833756744861603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784879845381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13784964382648468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784879859731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13819152116775513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784879859731, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784879859732, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784879903825, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49989718198776245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784879903826, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784879903826, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784879918120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13850408792495728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784879933407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13797779381275177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.8249999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784879948159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13760192692279816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784879962969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13843271136283875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784879978330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13783353567123413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3249999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784879992728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13771922886371613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880007750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13751110434532166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880007750, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784880007751, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784880052683, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5001887679100037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784880052684, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784880052684, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784880067215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371850073337555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880081863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13778796792030334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880096909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13757671415805817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1583333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880111332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13713082671165466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880125857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360544115304947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880140307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371259093284607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880154992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13614502549171448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880154992, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784880154993, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784880199250, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5005394816398621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784880199250, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784880199251, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784880213873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13692352175712585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880228520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13652163743972778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880243286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375226378440857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880258065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367616057395935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880272435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13523969054222107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.658333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880286734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13555759191513062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880301297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13741439580917358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880301298, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784880301298, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784880346063, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5009147524833679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784880346063, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784880346063, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784880361072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368352472782135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880376005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13584862649440765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880390513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359136998653412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880405190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13513784110546112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880419989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352650225162506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880435318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13473102450370789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880450237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351621001958847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880450238, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784880450238, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784880494196, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5013654232025146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784880494197, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784880494197, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784880509199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13426682353019714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880524174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365453004837036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880539581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380923092365265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880554574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13421359658241272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.824999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880569784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360633373260498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880584555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13498283922672272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.158333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880599780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13652828335762024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.324999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880599781, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784880599782, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784880644403, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.501996636390686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784880644403, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784880644404, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784880659449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364193558692932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880674649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13672363758087158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880689663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13480229675769806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880704539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13046202063560486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880719561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326901614665985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880734330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13379594683647156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880749027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13503769040107727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880749073, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784880749073, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784880792899, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5027307868003845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784880792900, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784880792900, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784880807724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13523536920547485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880822415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294761449098587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880838021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13150382041931152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880852535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368393450975418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.1158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880867221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13229474425315857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880881820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13134543597698212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880896893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13255684077739716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880896893, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784880896894, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784880941105, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.503585159778595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784880941106, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784880941106, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784880955458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13179387152194977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880970802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13424454629421234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880985398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12805727124214172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881000039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12978601455688477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881014755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13375438749790192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881029760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12843844294548035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881044288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12991228699684143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881044288, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784881044289, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784881088837, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5047359466552734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784881088838, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784881088838, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784881104607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13452830910682678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881119235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13304005563259125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.3158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881134431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13441838324069977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881149592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13495436310768127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881165107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360294669866562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.3658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881180008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335815191268921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881195122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13696858286857605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881195123, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784881195124, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784881239389, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5062566995620728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784881239389, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784881239389, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784881254514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13438254594802856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.4158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881269745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13060498237609863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881284976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13397793471813202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.4491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881300374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1402631402015686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.4658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881315534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13036981225013733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881331110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324351727962494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881346676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13828888535499573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881346677, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784881346677, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784881391479, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5077824592590332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784881391480, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784881391480, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784881407116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12666308879852295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881422587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311078816652298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881438139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13696922361850739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881453485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13548436760902405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881468879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323229968547821, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881483542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12890875339508057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.6158333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881498423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1280996948480606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881498424, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784881498424, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784881542411, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5093724131584167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784881542412, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784881542412, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784881557198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300535947084427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881572025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13485436141490936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.6658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881587203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14211055636405945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6825000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881602043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133957639336586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881617841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1314096301794052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.7158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881632798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13379377126693726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881647926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13605107367038727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881647926, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784881647927, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784881691714, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5112033486366272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784881691714, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784881691714, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784881706623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13471680879592896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881721617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13633757829666138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881736926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13567891716957092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881752049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13328413665294647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881767436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13430365920066833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881782572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13285613059997559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881797431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13253292441368103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8658333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881797432, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784881797432, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784881841596, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5131807923316956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784881841597, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784881841597, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784881856315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13187536597251892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881871236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345425546169281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8991666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881886380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14173981547355652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881901735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338273286819458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881916855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1342877745628357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881932139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323838233947754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881947398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330711543560028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881947398, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784881947399, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784881991671, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5155107975006104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784881991672, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784881991672, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784882006775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13810886442661285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882021903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329961121082306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882036978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12879669666290283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882052119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13000085949897766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882067562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13800230622291565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882082922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13156217336654663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882098003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13630540668964386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882098004, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784882098004, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784882142121, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5181513428688049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784882142122, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784882142122, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784882157195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1369515061378479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.1158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882172002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13636893033981323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882187079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1316046267747879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1491666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882201916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13008445501327515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882216973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13153241574764252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882232210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13539931178092957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882247467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13270655274391174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882247468, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784882247468, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784882291335, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5213719606399536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784882291336, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784882291336, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784882306132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13594774901866913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882321686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13243332505226135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882337470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13783761858940125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882352729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363075077533722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882368338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13833120465278625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882383682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13429144024848938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882399424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13901996612548828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882399424, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784882399425, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784882444104, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5248342752456665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784882444104, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784882444104, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784882459703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13507553935050964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882475310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.15003672242164612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882491014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350673884153366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882506306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13297200202941895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882521676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13413770496845245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882536856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331249177455902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882552359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13652577996253967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882552359, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784882552359, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784882597022, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5288404822349548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784882597022, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784882597023, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784882612517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362319141626358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882628004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352708786725998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882643637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13374681770801544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882658962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13358551263809204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882674215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13313594460487366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882689301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12954360246658325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882704299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1254587322473526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882704299, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784882704300, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784882748579, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5335767865180969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784882748579, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784882748579, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784882763564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325266808271408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.5824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882778862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323854774236679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882794176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13499264419078827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882809390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351875364780426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882824709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365991234779358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882839821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1287517547607422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882855025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12981286644935608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882855026, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784882855026, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784882899726, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5407536625862122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784882899726, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784882899726, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784882914622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132473424077034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.699166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882929581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12693768739700317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882944574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13300460577011108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882959511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13205119967460632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882974898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12701192498207092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882990482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13204741477966309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883005411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13391397893428802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883005412, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784883005412, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784883049971, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5507375001907349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784883049971, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784883049972, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784883065638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1293145716190338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883080688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13031598925590515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883095748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13043394684791565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883110917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12970122694969177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.8658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883125916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12773172557353973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883141213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12299135327339172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883156882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12705975770950317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883156882, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784883156883, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784883201026, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5633708834648132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784883201027, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784883201027, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784883216588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13547179102897644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883232055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329178363084793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883246955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13852578401565552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883262542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13292622566223145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883277735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12734726071357727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883293010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12916061282157898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883308305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12633724510669708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883308306, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784883308306, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784883353814, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5779109597206116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784883353815, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784883353815, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784883368812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12861087918281555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883383780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12696023285388947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883399325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13185226917266846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883415016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13210925459861755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883430696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13081499934196472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883445845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13323232531547546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883461543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12987250089645386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883461544, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784883461545, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784883506116, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5923579335212708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784883506116, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784883506116, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784883521453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12907058000564575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883536845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12889514863491058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883552165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12865829467773438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883567392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13161945343017578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883582387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320609450340271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883597313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12852944433689117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883612992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1293717324733734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883612992, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784883612992, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784883655595, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6036451458930969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784883655596, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784883655596, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784883670795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12726420164108276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883686414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12979458272457123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883701372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12704327702522278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883717290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12657874822616577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883733275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12619024515151978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883749103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13000261783599854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883765141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308237910270691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883765141, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784883765142, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784883807965, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6143229603767395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784883807966, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784883807966, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784883823314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12811371684074402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883839143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12624892592430115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883855231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12890909612178802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883871270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1306958794593811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883887172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13210442662239075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883902685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12727555632591248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883919039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12766143679618835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.4991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883919039, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784883919040, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784883962634, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6242018938064575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784883962635, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784883962635, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784883978264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12818753719329834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.5158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883994044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1259239763021469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884009732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12692531943321228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.5491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884025164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13346745073795319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884040978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12931716442108154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884056859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12428872287273407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884072771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12670081853866577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884072772, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784884072772, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784884117025, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6322973370552063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784884117025, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784884117025, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784884133220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12341340631246567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6325000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884150170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1262967735528946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884166001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12494393438100815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884181437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12448546290397644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884196728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12288210541009903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884212236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11963628232479095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884227726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.122336745262146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884227727, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784884227727, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784884270982, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6421008706092834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784884270983, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784884270983, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784884287026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12490452080965042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884302877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12152063101530075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884318684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1140320897102356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.7824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884334697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1208811104297638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884350262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11634863913059235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884365668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11660975217819214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8324999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884381027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11424744129180908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.8491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884381028, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784884381028, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784884424704, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6541092395782471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784884424705, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784884424705, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784884440342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12034326791763306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884455498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11608635634183884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884470699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1114271953701973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.8991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884486109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11954939365386963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884501908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11830929666757584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884517765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11737175285816193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.9491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884533178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10850750654935837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.9658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884533179, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784884533179, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784884577534, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6686692833900452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784884577534, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784884577534, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784884592577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11603310704231262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884608757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1159333884716034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884624487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1189829558134079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.0158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884639808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11367775499820709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884655233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11201710999011993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884670562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11359912902116776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0658333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884685969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11030973494052887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884686017, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784884686017, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784884730798, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.686305820941925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784884730799, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784884730799, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784884745923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11484992504119873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884761790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11325390636920929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.1158333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884777353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11897536367177963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884793255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11301518976688385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884808396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11422395706176758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1658333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884824467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11995816230773926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884839913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10813666880130768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884839913, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784884839914, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784884884582, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7019391059875488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784884884582, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784884884582, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784884900036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11674494296312332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884915397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11472740024328232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884931047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11219489574432373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884946403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11210930347442627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884961621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10648125410079956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884976691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1201966404914856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884992457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11633791029453278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884992458, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784884992458, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784885036798, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7186846733093262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784885036799, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784885036799, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784885052376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11887113749980927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885068318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11452525109052658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.3491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885083730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11837680637836456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885099388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10459060966968536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885114779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11651688814163208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.3991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885130279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11167623102664948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.4158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885145712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10946372151374817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885145713, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784885145713, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784885189923, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7306598424911499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784885189924, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784885189924, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784885205122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11218371987342834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.4491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885220508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11125865578651428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885235721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11636927723884583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.4824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885251233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11230327934026718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885266917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11638173460960388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885282562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.106814906001091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885298420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11531634628772736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885298421, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784885298421, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784885341574, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7388095259666443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784885341574, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784885341575, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784885357226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10982125252485275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885372410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11270520836114883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.5824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885388680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12351125478744507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885404533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12614545226097107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885420088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12052823603153229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885435800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11347337067127228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885451498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11888405680656433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885451499, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784885451499, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784885495833, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7411333322525024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784885495834, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784885495834, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784885511429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1185310110449791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885527298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11132147908210754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.6991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885543449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12059813737869263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885559464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11352504789829254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885575666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11809920519590378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885591448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11577446758747101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885607630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10710237920284271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885607631, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784885607631, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784885651043, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7431813478469849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784885651044, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784885651044, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784885666898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11998830735683441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885682753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10978075861930847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885698600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10635251551866531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885714320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11613015830516815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885730024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11983199417591095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885745843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1090974509716034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885762049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11564761400222778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885762050, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784885762050, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784885806598, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7473428249359131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784885806599, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784885806599, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784885822381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11663596332073212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885838553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.109881691634655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885854029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11005009710788727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885870310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1156681627035141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885886160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10877615958452225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885901730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1051831990480423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885917150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11642705649137497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.015833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885917151, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784885917151, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784885961727, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7491852045059204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784885961728, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784885961728, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784885977248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11150628328323364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885992765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11527034640312195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886008275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10948403179645538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886023674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10618782043457031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886039544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10369804501533508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886055377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09754347801208496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886071452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1063532829284668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886071453, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784886071453, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784886116010, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7507283687591553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784886116011, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784886116011, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784886131601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10609328001737595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886146942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11000120639801025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886163026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11983487755060196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886178649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10948535054922104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886194060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1070617288351059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886209276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10168589651584625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886224780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.107625812292099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886224780, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784886224781, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784886269713, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7523094415664673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784886269714, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784886269714, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784886285282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10520683228969574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886300619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10639762878417969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886316157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11030781269073486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886331711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793404281139374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886347474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10654143989086151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886363311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11236825585365295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886379342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11047788709402084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886379342, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784886379343, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784886424057, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7528225779533386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784886424058, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784886424058, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784886439512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10627776384353638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886455371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1025695651769638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886470964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11096490919589996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886486799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10968382656574249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886502371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10509342700242996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886518006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09995420277118683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886533740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10486344993114471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886533741, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784886533741, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784886578447, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7533925771713257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784886578448, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784886578448, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784886594022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10490253567695618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886609701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10107666254043579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886625161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10521187633275986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886641120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11067203432321548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886657033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10675753653049469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886673230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10400919616222382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886689102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11030931770801544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886689103, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784886689103, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784886734263, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7548802495002747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784886734264, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784886734264, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784886749443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09999969601631165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886765667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10393793135881424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886781335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10747300088405609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886796817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10454963147640228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886812252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10274485498666763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886827688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10828486829996109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886843198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10917554795742035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886843199, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784886843199, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784886888443, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.755776584148407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784886888443, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784886888444, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784886903889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10637389868497849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886919343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10329841822385788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886934748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10920552909374237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886950443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10662408918142319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886966228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11031889170408249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886982206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10829818993806839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886998057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11123397946357727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886998058, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784886998058, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784887042873, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7583844065666199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784887042874, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784887042874, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784887058182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10688069462776184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887074232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10142569243907928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887089906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10526770353317261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887105642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10241837799549103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887121295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10863736271858215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887136918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11323200911283493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887152628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10897528380155563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887152629, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784887152629, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784887197967, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7578659653663635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784887197968, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784887197968, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784887213762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10561420768499374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887229488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11148138344287872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887245253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10805236548185349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887261181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10011404752731323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887277271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11318198591470718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887293352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11344785243272781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887309837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10635009407997131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887309837, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784887309838, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784887355417, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7597779631614685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784887355418, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784887355418, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784887371240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10875151306390762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887386789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09980037808418274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887403583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11622019857168198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887419796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11738927662372589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887435793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11509484052658081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887451735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10419213771820068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887467723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10698643326759338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887467724, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784887467724, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784887513506, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7594877481460571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784887513506, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784887513506, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784887529503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11188510060310364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887545420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11472668498754501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887561500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11107189953327179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887577794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11073224991559982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887594005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11500691622495651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887610102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11303189396858215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887626342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11349945515394211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887626343, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784887626343, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784887670665, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7606602311134338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784887670666, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784887670666, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784887686211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11007742583751678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887702838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10765440762042999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887719194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10719819366931915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887735675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11508440971374512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887751952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11806926131248474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887768258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11040113866329193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887784782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11563509702682495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.415833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887784782, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784887784783, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784887830744, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7615730166435242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784887830745, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784887830745, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784887846837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11409120261669159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887863143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11383748054504395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887879412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11626587808132172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887895816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11362634599208832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887912269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10929984599351883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887928506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10975191742181778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.515833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887944506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11239226162433624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887944507, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784887944507, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784887989699, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7619365453720093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784887989700, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784887989700, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784888005684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10952140390872955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888021684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10542389750480652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888037197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10526394844055176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888052850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10467644035816193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888068375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10697239637374878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.615833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888084344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10385341942310333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888100199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10905382037162781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888100199, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784888100200, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784888145098, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7640102505683899, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784888145098, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784888145099, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784888160830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10999567806720734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888176368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10322894901037216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888192322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10850055515766144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888208148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10371177643537521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888224238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10278913378715515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888240431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10591438412666321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888256147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09963692724704742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888256147, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784888256148, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784888301247, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7643990516662598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784888301248, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784888301248, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784888316598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10828794538974762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888333319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11073216795921326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888349601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10654619336128235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888365639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10616447031497955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888381393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10804960131645203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888397026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1034943163394928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888412771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10855419933795929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888412771, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784888412771, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784888457644, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7659206390380859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784888457645, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784888457645, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784888473546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10825395584106445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888489197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10587707161903381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888505214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10856816172599792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888521150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10351316630840302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888537299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10960492491722107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888553422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10959599167108536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888569749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10490000247955322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888569750, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784888569751, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784888615088, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7663514614105225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784888615088, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784888615088, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784888631389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10841026902198792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888647076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10360360145568848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888663330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11340406537055969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888678593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10369633883237839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888694727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10066264867782593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888710540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1027437150478363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888726125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10619745403528214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888726125, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784888726126, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784888770694, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7671012282371521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784888770694, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784888770694, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784888786124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10334204137325287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888802080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10084840655326843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888818139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10931108891963959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888834261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11066321283578873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888849993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10471417009830475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888866024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10977821797132492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888882269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10982029139995575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888882270, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784888882271, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784888926921, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7680212259292603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784888926922, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784888926922, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784888943104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11174552142620087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888959546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10516181588172913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888975560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1046343743801117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888991086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10675226151943207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889007216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10143038630485535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889023283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10797788202762604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889039665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1049303188920021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889039665, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784889039665, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784889084013, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7684004902839661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784889084013, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784889084014, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784889100002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10764630138874054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889115693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10868345946073532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889131445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10069389641284943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889147262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11026312410831451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889163096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09741430729627609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889179072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10261938720941544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889194847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10366559773683548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889194848, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784889194848, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784889239946, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7678852677345276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784889239947, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784889239947, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784889256089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10315199196338654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889272026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10913345962762833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889288075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10921309888362885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889304341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111593097448349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889320469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10843849927186966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889336150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11074727028608322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.565833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889352646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1026947945356369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889352647, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784889352648, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784889398312, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7692918181419373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784889398313, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784889398313, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784889414321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10948358476161957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599166666666666e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_16384/seed951784539.log b/recommendation_v4/rcp_logs/gbs_16384/seed951784539.log new file mode 100644 index 000000000..44f2341ba --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_16384/seed951784539.log @@ -0,0 +1,813 @@ +:::MLLOG {"namespace": "", "time_ms": 1784851975239, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784851975239, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784851996535, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784851996537, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784851996537, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784851996537, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784851996537, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784851996537, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 16384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784851996537, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784851996537, "event_type": "POINT_IN_TIME", "key": "seed", "value": 951784539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784851996537, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784851996537, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784851996537, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784851996537, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784851996537, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784851996538, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784852868526, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784852868528, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784852868845, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784853035087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13977870345115662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.5833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784853049455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1395249366760254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2499999999999997e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784853062967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13908663392066956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784853076590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13882756233215332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.583333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784853090469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385909914970398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.25e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784853104404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13887521624565125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784853118092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13906452059745789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853118093, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784853118094, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784853162033, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49850887060165405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784853162034, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784853162034, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784853175564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1394306719303131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853189398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1392485499382019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853203226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389772742986679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853217038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13914638757705688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853230848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13890136778354645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9916666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853245135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1390654444694519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853259629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13873830437660217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853259630, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784853259631, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784853302818, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49868327379226685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784853302818, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784853302819, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784853316660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13895556330680847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853331042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389259696006775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853344925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13892263174057007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853359637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13877283036708832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.9916666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853373443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860848546028137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853387886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13830067217350006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853402000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13847076892852783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853402000, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784853402001, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784853444966, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4989106059074402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784853444966, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784853444966, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784853459069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13868898153305054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853473824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13865353167057037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.8249999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853488102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13824690878391266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853502472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13836650550365448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853517515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13829484581947327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3249999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853531732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13817493617534637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853546417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382242739200592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853546418, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784853546419, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784853590202, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49920740723609924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784853590203, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784853590203, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784853604196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13783115148544312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853618286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381637006998062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853632821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13790035247802734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1583333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853646943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13742251694202423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853661019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13672737777233124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853675165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13746941089630127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853689581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.136905699968338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853689582, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784853689583, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784853733757, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49959617853164673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784853733757, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784853733757, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784853747706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13727951049804688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853761543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13690972328186035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853775770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13783463835716248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853790389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13696962594985962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853804326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13579711318016052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.658333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853818338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13604602217674255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853832445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376153826713562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853832446, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784853832446, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784853876659, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5000578165054321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784853876660, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784853876660, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784853891130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13708563148975372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853905500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13611625134944916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853919365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363174021244049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853933612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13553360104560852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853947958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13560551404953003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853962952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13517211377620697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853977677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13533194363117218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853977678, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784853977678, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784854021214, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5006815195083618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784854021214, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784854021214, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784854035707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13462725281715393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784854050073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365996152162552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784854065034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13813352584838867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784854079761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13450849056243896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.824999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784854094623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362687051296234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784854109199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13522803783416748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.158333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784854124188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368955671787262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.324999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784854124189, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784854124190, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784854168217, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5014812350273132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784854168217, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784854168217, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784854182780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367320716381073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784854197414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13676312565803528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784854212031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13509845733642578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784854226591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1307719349861145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784854241255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1328062117099762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854255702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13396000862121582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854270050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13507068157196045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854270102, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784854270102, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784854313806, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5023983716964722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784854313806, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784854313806, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784854328384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13504526019096375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854342726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12966354191303253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854357594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131600022315979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854371742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13671740889549255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.1158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854385944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13244110345840454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854400069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13168656826019287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854414701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325618326663971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854414702, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784854414703, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784854457862, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5034013390541077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784854457863, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784854457863, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784854471739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13200458884239197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854486500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13413919508457184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854500457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1281592696905136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854514476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1299247443675995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854528732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13391238451004028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854543262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1284601092338562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854557275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12988990545272827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854557276, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784854557276, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784854600655, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5046472549438477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784854600655, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784854600655, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784854615653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13456977903842926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854629704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13297349214553833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.3158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854644012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13446299731731415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854658161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13502052426338196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854672858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13595229387283325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.3658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854687161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13355115056037903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854701813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13702735304832458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854701814, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784854701815, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784854746096, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5060774087905884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784854746096, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784854746097, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784854760551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13440021872520447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.4158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854775117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1305731236934662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854789620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13395574688911438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.4491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854804423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14039865136146545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.4658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854819024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13042327761650085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854834286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13256768882274628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854849325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13827170431613922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854849326, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784854849326, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784854894325, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5073773860931396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784854894326, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784854894326, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784854909403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1254481077194214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854924245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310766339302063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854939436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13699129223823547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854954400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353837251663208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854969570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13230520486831665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854983962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1287952959537506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.6158333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854998494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12812493741512299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854998494, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784854998495, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784855041995, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5087918639183044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784855041995, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784855041996, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784855056404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301097422838211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855070983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347505748271942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.6658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855085747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14208923280239105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6825000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855100209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13415130972862244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855115548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1314110904932022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.7158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855129989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13367614150047302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855144811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13601581752300262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855144812, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784855144812, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784855188322, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5104959011077881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784855188323, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784855188323, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784855202883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13471533358097076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855217607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13625389337539673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855232680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13563483953475952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855247575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13322389125823975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855262584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13440245389938354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855277499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132987380027771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855292001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132482647895813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8658333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855292001, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784855292002, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784855335946, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5122043490409851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784855335946, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784855335946, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784855350430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13183578848838806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855365081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13448020815849304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8991666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855379812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.141771137714386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855394798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338716447353363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855409630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13434725999832153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855424479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13227760791778564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855439199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13322272896766663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855439200, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784855439201, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784855482021, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5142053961753845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784855482021, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784855482021, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784855496470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13831019401550293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855510967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330241858959198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855525385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289616823196411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855539892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13004809617996216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855554636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13791781663894653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855569483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13156160712242126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855584322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13620129227638245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855584323, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784855584323, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784855627749, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5165714621543884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784855627750, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784855627750, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784855642404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13672053813934326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.1158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855656887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13642993569374084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855671574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13151815533638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1491666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855685952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13017573952674866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855700639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13156163692474365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855715584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13549986481666565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855730556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13266444206237793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855730556, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784855730557, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784855773325, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5192568898200989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784855773326, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784855773326, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784855787904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13601410388946533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855803186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13232021033763885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855818801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378486156463623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855834082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13645252585411072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855849580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13854020833969116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855865038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343478560447693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855880936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13914990425109863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855880936, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784855880937, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784855923824, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5220048427581787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784855923825, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784855923825, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784855939360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351572573184967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855954833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1474159061908722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855970259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13516643643379211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855985166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133060023188591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856000270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13423460721969604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856015052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13313522934913635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856030080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13662061095237732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856030080, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784856030081, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784856073148, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5253992080688477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784856073149, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784856073149, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784856088182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13610520958900452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856103297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13518142700195312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856118806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13376283645629883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856133926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13370278477668762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856149152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133277028799057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856164284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12938231229782104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856179294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1255214512348175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856179295, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784856179295, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784856221845, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5294685959815979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784856221845, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784856221845, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784856236775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13234540820121765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.5824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856252069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13259878754615784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856267412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349216252565384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856282583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13536718487739563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856297668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367337703704834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856312644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1287517249584198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856327625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298348754644394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856327626, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784856327626, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784856371199, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5351846814155579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784856371199, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784856371200, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784856385750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323830485343933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.699166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856400386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12709780037403107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856414977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13323813676834106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856429654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321672797203064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856444846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12706798315048218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856460189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13223931193351746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856474829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13421255350112915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856474830, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784856474830, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784856517893, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5429272055625916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784856517894, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784856517894, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784856533413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294001042842865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856548340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13042163848876953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856563107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13052666187286377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856577945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12976936995983124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.8658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856592475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12781012058258057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856607123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12334531545639038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856622050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12721571326255798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856622051, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784856622051, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784856665170, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5531517863273621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784856665171, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784856665171, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784856680125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13572606444358826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856694914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13321761786937714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856709353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13898617029190063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856724620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13361455500125885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856739480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1276819109916687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856754287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12964972853660583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856769303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.126911461353302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856769304, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784856769304, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784856811980, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5657010078430176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784856811981, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784856811981, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784856826629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.128914475440979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856841414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12761792540550232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856856482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13255253434181213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856871610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13265672326087952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856886797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13145321607589722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856901700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13368065655231476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856917043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304277777671814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856917044, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784856917044, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784856959515, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5795171856880188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784856959516, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784856959516, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784856974473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1296239048242569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856989253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12982077896595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857004055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12977895140647888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857018897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326110064983368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857033490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13359391689300537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857047726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12963911890983582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857062735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13033534586429596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857062735, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784857062736, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784857106128, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5917503833770752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784857106129, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784857106129, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784857120814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12829571962356567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857135891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13092881441116333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857150335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12839409708976746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857166104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1278197169303894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857181756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12747280299663544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857197354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13194936513900757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857212901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13244834542274475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857212901, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784857212902, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784857256065, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6049648523330688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784857256066, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784857256066, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784857271295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300986260175705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857286977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1281396448612213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857302902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13028928637504578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857318719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320466697216034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857334476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13347317278385162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857349843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12922336161136627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857365820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12906329333782196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.4991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857365821, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784857365821, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784857408851, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6199292540550232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784857408852, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784857408852, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784857423861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1296074241399765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.5158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857439150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1278473436832428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857454495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1291283369064331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.5491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857469595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352851837873459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857484918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13077189028263092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857500571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1259632706642151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857516189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12958770990371704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857516190, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784857516190, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784857559945, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6332611441612244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784857559945, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784857559945, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784857574896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12606002390384674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6325000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857590585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1280054748058319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857605885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12711311876773834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857621067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12715399265289307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857636108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1252662092447281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857651164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12256310880184174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857666220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12359119951725006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857666221, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784857666221, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784857710623, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6493738889694214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784857710624, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784857710624, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784857726213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1264951080083847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857741520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12323768436908722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857756714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11673526465892792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.7824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857772101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12266702950000763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857787291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11833084374666214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857802262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11889462918043137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8324999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857817244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1166396513581276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.8491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857817244, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784857817244, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784857862159, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6687652468681335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784857862160, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784857862160, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784857877203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12178266048431396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857891964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1180567592382431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857906708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11430901288986206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.8991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857921599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12064269930124283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857936765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11912749707698822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857952083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11843380331993103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.9491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857967003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10911882668733597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.9658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857967004, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784857967004, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784858010952, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7190183997154236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784858010952, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784858010952, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784858025508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11568094789981842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858040947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11622567474842072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858055997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11925844103097916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.0158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858070738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1135980412364006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858085485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11292877793312073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858100198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11391014605760574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0658333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858114932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10956189036369324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858114990, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784858114990, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784858158927, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7450937032699585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784858158928, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784858158928, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784858173372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11398570239543915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858188557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11217360198497772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.1158333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858203620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1174994707107544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858219045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11183597147464752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858233886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11189030110836029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1658333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858249123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11923982203006744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858263891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10621768236160278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858263891, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784858263892, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784858309052, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7599461674690247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784858309053, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784858309053, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784858324006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11437100172042847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858338881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11275321990251541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858353866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11042416095733643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858368613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10979124903678894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858383369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10403941571712494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858397957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11682511866092682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858413301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11229068040847778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858413302, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784858413302, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784858457543, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.765038251876831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784858457544, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784858457544, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784858472577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11537792533636093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858488006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11255918443202972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.3491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858502921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1150406002998352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858518350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10097943246364594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858533415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11377923935651779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.3991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858548647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10654614120721817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.4158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858563915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10540561378002167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858563916, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784858563916, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784858608268, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7676844596862793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784858608269, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784858608269, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784858623265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10802226513624191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.4491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858638400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10741771757602692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858653358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11240646988153458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.4824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858668477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10791085660457611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858683887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11152725666761398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858699223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10154286026954651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858714706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1096731424331665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858714706, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784858714707, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784858758188, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7694432735443115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784858758188, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784858758188, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784858773490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10659456253051758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858788464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10890606045722961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.5824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858804206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11860384047031403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858819737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12121735513210297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858835318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11349359154701233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858850929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1105959564447403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858866545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11507739126682281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858866546, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784858866546, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784858908855, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7706847190856934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784858908856, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784858908856, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784858924318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11379161477088928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858939920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10729272663593292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.6991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858955760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11662890017032623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858971404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11002273857593536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858987411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1131453663110733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859002877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11332535743713379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859019077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10382398962974548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859019078, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784859019078, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784859062380, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7715259194374084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784859062381, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784859062381, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784859077936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11810429394245148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859093425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10594377666711807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859108964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10463782399892807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859124439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11320088803768158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859139845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11753396689891815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859155470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10606750845909119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859171492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11365388333797455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859171493, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784859171493, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784859215175, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7722527384757996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784859215175, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784859215175, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784859230713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11480388045310974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859246586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10697754472494125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859261856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10773228108882904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859277831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11292294412851334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859293062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072423905134201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859308116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10400329530239105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859323112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11637115478515625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.015833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859323113, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784859323113, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784859366255, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7723494172096252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784859366256, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784859366256, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784859381331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10958072543144226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859396435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11386846005916595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859411457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10594604909420013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859426511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10477620363235474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859442000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10169149935245514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859457423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09581378102302551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859473070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10557590425014496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859473070, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784859473071, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784859517174, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7731118202209473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784859517175, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784859517175, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784859532340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.104611337184906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859547314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11004723608493805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859563353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11727826297283173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859579043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10925933718681335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859594397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10554684698581696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859609622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10054537653923035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859625139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10609760880470276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859625140, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784859625140, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784859669313, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7735000252723694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784859669314, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784859669314, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784859684765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10479633510112762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859699768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10530713200569153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859715059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10910331457853317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859730548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10666949301958084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859746045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10591058433055878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859761494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11159217357635498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859777237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10894042253494263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859777238, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784859777238, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784859822266, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773842990398407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784859822267, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784859822267, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784859837338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1044512391090393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859852713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10219275951385498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859868171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10832234472036362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859883665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10885071754455566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859899046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10474218428134918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859914290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10079826414585114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859929706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10489971935749054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859929706, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784859929707, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784859974151, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7740800976753235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784859974151, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784859974152, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784859989305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10430637747049332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860004438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0996665358543396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860019136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10418014228343964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860034425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10987190902233124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860049917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1057700514793396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860065517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10324975103139877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860080761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10962473601102829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860080761, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784860080762, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784860124786, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742171287536621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784860124786, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784860124787, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784860139600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09957945346832275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860155481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10282144695520401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860170835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10675491392612457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860186282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1032351702451706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860201762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10242350399494171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860217245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10586179047822952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860232810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10917812585830688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860232810, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784860232811, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784860277259, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7745112180709839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784860277260, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784860277260, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784860292797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10477420687675476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860308330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10325432568788528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860323794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10758255422115326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860339605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10566399991512299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860355365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10947977006435394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860371270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.107231006026268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860386914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1108068972826004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860386914, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784860386915, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784860430056, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7748541831970215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784860430056, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784860430057, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784860445143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10655468702316284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860460790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1016412228345871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860475957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10410970449447632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860491201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10194375365972519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860506335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10822223126888275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860521386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11305699497461319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860536347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10796105861663818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860536348, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784860536348, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784860580359, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7747754454612732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784860580360, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784860580360, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784860595588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10356120765209198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860610585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11071601510047913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860625619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10734091699123383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860640913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09973365068435669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860656224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11137358099222183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860671498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11232250928878784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860687304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10467194020748138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860687305, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784860687305, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784860730537, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746170163154602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784860730538, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784860730538, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784860745666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10731877386569977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860760532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09969064593315125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860776668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11561498045921326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860792597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11505136638879776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860808100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11444780230522156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860823770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10409136116504669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860839481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10599321126937866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860839482, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784860839482, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784860883130, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742201089859009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784860883131, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784860883131, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784860898812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11098089814186096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860914436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1135120540857315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860930427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10940209031105042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860946525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10994853079319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860962792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1141912117600441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860978783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11205662786960602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860995110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11293764412403107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860995111, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784860995111, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784861038992, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7747712135314941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784861038993, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784861038993, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784861054266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1090703010559082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861070344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10709428787231445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861085959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1067553162574768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861101963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11423467099666595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861117338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11830250918865204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861132876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10937145352363586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861148736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11570192128419876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.415833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861148737, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784861148737, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784861192548, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7750566601753235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784861192549, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784861192549, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784861207860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11349136382341385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861223502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11365240812301636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861239214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11547505855560303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861255092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11345110833644867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861270959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10927315056324005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861286544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10880571603775024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.515833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861301840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11147642135620117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861301841, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784861301841, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784861345754, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752222418785095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784861345755, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784861345755, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784861361695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10826878249645233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861377361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10431358963251114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861392540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10359922051429749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861407799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1038605272769928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861422974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10646893829107285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.615833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861438562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10256658494472504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861453829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080443263053894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861453829, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784861453830, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784861497113, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7755559682846069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784861497114, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784861497114, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784861512286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10915820300579071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861527246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1020842045545578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861542692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10774146020412445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861558025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10315996408462524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861573553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10203651338815689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861589367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10545563697814941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861604710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09948772192001343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861604711, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784861604711, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784861648987, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7753822207450867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784861648988, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784861648988, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784861663971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10744132846593857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861679987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11009790748357773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861695478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10651333630084991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861710829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1058780625462532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861726014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10714523494243622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861740866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10315479338169098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861756153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10746743530035019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861756154, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784861756154, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784861799866, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7756831049919128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784861799867, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784861799867, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784861815008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10757589340209961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861829972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10554182529449463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861845064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10854367911815643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861860144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10244768857955933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861875559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10852764546871185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861891059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10930828005075455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861906709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10490131378173828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861906709, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784861906710, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784861949699, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7755860686302185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784861949699, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784861949699, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784861965058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10747458040714264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861980003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10178473591804504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861995577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1116437017917633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862010680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10262079536914825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862026295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10003829747438431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862041513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10246284306049347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862056773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10531807690858841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862056774, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784862056774, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784862099853, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7758181095123291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784862099853, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784862099854, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784862114964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10292338579893112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862130514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10016191005706787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862146278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10820866376161575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862161891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10933057963848114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862177006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10431256890296936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862192446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10753269493579865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862208082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10895559936761856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862208083, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784862208083, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784862251417, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7759561538696289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784862251417, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784862251418, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784862267117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11082680523395538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862282945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10414381325244904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862298539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10450305044651031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862313659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1054667979478836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862329268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09996563196182251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862344784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10724349319934845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862360594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10401810705661774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862360595, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784862360595, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784862403968, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7760851979255676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784862403968, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784862403968, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784862419297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10677347332239151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862434484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10775505751371384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862449891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09969399124383926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862465523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10915930569171906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862481053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09664444625377655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862496552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1018233448266983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862511615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10378125309944153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862511616, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784862511616, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784862555001, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7761543989181519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784862555002, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784862555002, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784862570739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10228225588798523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862586274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1079125851392746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862601631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.108666330575943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862617217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11045290529727936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862632876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1076684296131134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862647918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10908976197242737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.565833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862663706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10176437348127365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862663706, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784862663707, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784862708026, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7767019271850586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784862708026, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784862708026, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784862723460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10944220423698425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862739380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10393519699573517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149749760, "lr": 7.615833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862754932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10680383443832397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862770377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11155407130718231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150405120, "lr": 7.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862786112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793647915124893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862801812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11245062947273254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151060480, "lr": 7.682499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862817162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10413508862257004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862817163, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784862817163, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784862861254, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7761220932006836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784862861255, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784862861255, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784862876321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10608251392841339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151715840, "lr": 7.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862891604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09811774641275406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.7325e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_32768/seed1056795887.log b/recommendation_v4/rcp_logs/gbs_32768/seed1056795887.log new file mode 100644 index 000000000..0c3a56ba3 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_32768/seed1056795887.log @@ -0,0 +1,670 @@ +:::MLLOG {"namespace": "", "time_ms": 1784867506376, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784867506376, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784867530027, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784867530030, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784867530030, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784867530030, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784867530031, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784867530031, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784867530031, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784867530031, "event_type": "POINT_IN_TIME", "key": "seed", "value": 1056795887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784867530031, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784867530031, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784867530031, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784867530031, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784867530075, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784867530076, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784868319820, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784868319821, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784868320146, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784868583874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1398722231388092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.1666666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784868599682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1394955962896347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.4999999999999995e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784868614697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13954627513885498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784868622121, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784868622122, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784868674016, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49981313943862915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784868674017, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784868674017, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784868681432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13959470391273499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3166666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868696362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13964825868606567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868711968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1393793225288391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9833333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868727358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13929669559001923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868727358, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784868727358, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784868750818, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.499986857175827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784868750819, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784868750819, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784868765873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13973769545555115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868781182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13950994610786438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868796495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13942719995975494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868803890, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784868803891, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784868827488, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5001888275146484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784868827488, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784868827488, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784868835404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13899779319763184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868850776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13906079530715942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868866387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138914555311203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868881826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13893060386180878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868881827, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784868881827, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784868904824, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5004494190216064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784868904825, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784868904825, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784868920044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13876968622207642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868935110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387263685464859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868950488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13856622576713562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868958079, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784868958080, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784868981134, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5007497668266296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784868981135, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784868981135, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784868988492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13828685879707336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869004227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382460594177246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869019178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380765438079834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869034626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379339098930359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869034627, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784869034627, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784869059084, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5010827779769897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784869059085, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784869059085, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784869074133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137836754322052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.316666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869089532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13787750899791718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869105569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13725730776786804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869113196, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784869113197, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784869136826, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5014649033546448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784869136827, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784869136827, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784869144461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13772696256637573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869160804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368989199399948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869176286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13702628016471863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869192514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13735176622867584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869192514, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784869192515, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784869216837, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5019446611404419, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784869216838, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784869216838, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784869232075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13684672117233276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869248009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357397437095642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.983333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869263182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13686548173427582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869270798, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784869270799, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784869294861, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5024238228797913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784869294862, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784869294862, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784869302473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1340218484401703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869318258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13512614369392395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869333255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353791058063507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869348269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13549679517745972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869348347, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784869348348, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784869372748, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5029433965682983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784869372749, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784869372749, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784869387968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1342841237783432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869403068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13486988842487335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869418171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13547426462173462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869426472, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784869426472, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784869450035, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5035143494606018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784869450035, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784869450035, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784869457412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135753333568573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869472552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13511493802070618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869487922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322389841079712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869503916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134807288646698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869503917, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784869503917, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784869527355, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5042659640312195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784869527356, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784869527356, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784869542695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13388673961162567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869558298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351604461669922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869574422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13502998650074005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869582311, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784869582311, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784869606323, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5053830146789551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784869606324, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784869606324, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784869614044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13640692830085754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869629755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13352081179618835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869646125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343008428812027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869661547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13217121362686157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869661614, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784869661615, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784869685574, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5065687894821167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784869685575, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784869685575, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784869700838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13283199071884155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869717047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13126058876514435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869732672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330745816230774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869740501, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784869740502, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784869764297, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5078709125518799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784869764297, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784869764297, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784869771973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384873241186142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869787507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13107624650001526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869803510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12986254692077637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869818876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1302585005760193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869818877, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784869818877, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784869842778, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.509249746799469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784869842779, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784869842779, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784869858292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13271939754486084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869873902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12976901233196259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869890131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13058239221572876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869897852, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784869897853, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784869921726, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5107477307319641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784869921727, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784869921727, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784869929530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315629929304123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869945013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13508418202400208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869960766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331859529018402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869977082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13559654355049133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869977082, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784869977083, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784870000908, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5123155117034912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784870000909, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784870000909, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784870016401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13512252271175385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870031942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13254289329051971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1649999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870047568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332014948129654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870055280, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784870055280, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784870079625, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5138659477233887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784870079626, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784870079626, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784870088318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363433599472046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870104495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13517031073570251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870120483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13621464371681213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870136732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13633191585540771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870136733, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784870136733, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784870160838, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5153893232345581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784870160839, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784870160839, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784870177531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13472765684127808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870193575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13079649209976196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870209505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13626107573509216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870217672, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784870217672, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784870241509, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5170062780380249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784870241509, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784870241509, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784870249413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13586583733558655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870265971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13246802985668182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870281688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325032114982605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870297358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13080202043056488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870297358, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784870297359, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784870320875, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5187673568725586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784870320875, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784870320875, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784870336680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13097792863845825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870353118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13830988109111786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870368920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12935273349285126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870376733, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784870376734, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784870399877, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5207361578941345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784870399877, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784870399878, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784870407609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12931860983371735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870423315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13469098508358002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870439034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344577670097351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870455745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13033756613731384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870455746, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784870455746, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784870479872, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5230439901351929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784870479873, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784870479873, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784870495555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1280662715435028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870511352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318562626838684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870527224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357206404209137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870535292, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784870535293, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784870559553, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5256800055503845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784870559553, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784870559553, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784870567549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13651970028877258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870584011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334550529718399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870599895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13187682628631592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870615478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1307375431060791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870615479, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784870615479, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784870639434, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5287814140319824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784870639435, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784870639435, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784870655238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12960055470466614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870671041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13238534331321716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870687499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13527284562587738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870695637, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784870695638, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784870718978, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5322568416595459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784870718978, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784870718978, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784870726906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13174831867218018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870743075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332702934741974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870759024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13486693799495697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870775198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13432256877422333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870775198, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784870775198, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784870799354, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5364710092544556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784870799355, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784870799355, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784870815595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13576388359069824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870832258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337500810623169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870848388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330413818359375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3650000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870856331, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784870856331, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784870879922, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5412055253982544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784870879922, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784870879923, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784870888131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1400018185377121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870904485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351039707660675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870921008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13431549072265625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4649999999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870937988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13505688309669495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870937989, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784870937989, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784870961535, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.546598494052887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784870961536, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784870961536, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784870977780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349261850118637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870994206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13294580578804016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.5649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871010657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13205139338970184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871019441, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784871019442, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784871043518, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5526149868965149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784871043518, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784871043518, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784871051670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13230308890342712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871067748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13051572442054749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871083568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13562801480293274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871099713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13197320699691772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871099713, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784871099714, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784871123506, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5593355894088745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784871123507, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784871123507, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784871139672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13411077857017517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871156373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12963411211967468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871172729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13137181103229523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871180746, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784871180746, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784871204365, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5676172971725464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784871204366, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784871204366, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784871212228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13162961602210999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871228246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1299332082271576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871244433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13236156105995178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871260917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131648987531662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871260918, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784871260918, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784871284503, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5768386721611023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784871284504, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784871284504, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784871300574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310787796974182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871316585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12695489823818207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871332533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13296812772750854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871340699, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784871340699, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784871363858, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5856000185012817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784871363858, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784871363859, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784871372030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13349094986915588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871387957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12568312883377075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871404416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12546953558921814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871420743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13133439421653748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871420743, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784871420744, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784871443761, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5953964591026306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784871443761, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784871443762, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784871459626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12787990272045135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871475625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1269734799861908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871491941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1261996477842331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871499840, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784871499840, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784871523480, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6013913154602051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784871523481, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784871523481, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784871531549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12856020033359528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871548212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1254032403230667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871564307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12821558117866516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871580359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12643548846244812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871580360, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784871580360, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784871603727, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6106206774711609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784871603728, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784871603728, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784871619541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12463903427124023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871635792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12786027789115906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4983333333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871651831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13057392835617065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871659855, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784871659856, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784871683541, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6196419596672058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784871683542, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784871683542, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784871691906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12884750962257385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871708581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12399516999721527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871725186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12891492247581482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871741408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12350916862487793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871741409, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784871741409, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784871765023, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6298641562461853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784871765023, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784871765023, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784871781619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1264992356300354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871797918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12775784730911255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871814763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12541553378105164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871823208, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784871823208, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784871846142, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6428177356719971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784871846142, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784871846143, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784871854334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12492509186267853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871870490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12481343001127243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871886915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12466108053922653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871903342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12412206083536148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.898333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871903343, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784871903343, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784871926951, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6552293300628662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784871926951, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784871926951, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784871943514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11726808547973633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871959827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11778176575899124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871975920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11563322693109512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871983979, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784871983979, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784872006942, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6620043516159058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784872006943, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784872006943, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784872014852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11753008514642715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872031144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11455973982810974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872047355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1194625273346901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872063390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11648675054311752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872063391, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784872063391, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784872087354, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6889047622680664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784872087355, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784872087355, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784872104339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11845828592777252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872120362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10932497680187225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.198333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872136439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11548402905464172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872144347, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784872144348, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784872168084, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7180413007736206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784872168084, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784872168084, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784872175920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11636011302471161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872192279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11633143573999405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.298333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872208645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11107916384935379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872224588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11446774750947952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872224661, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784872224662, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784872249428, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7412678003311157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784872249428, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784872249428, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784872265646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11314014345407486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.398333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872281674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11301463842391968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872298004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10603693127632141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872306111, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784872306112, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784872329576, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.75250244140625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784872329577, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784872329577, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784872337686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11042304337024689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.498333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872354179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11135181784629822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.531666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872370459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10833243280649185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872387034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11114495247602463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.598333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872387035, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784872387035, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784872410532, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7620359063148499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784872410533, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784872410533, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784872426855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10977653414011002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872443131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10904458165168762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872459549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11302250623703003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872467679, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784872467679, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784872491517, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7648057341575623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784872491517, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784872491518, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784872499525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10226615518331528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872516099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11115804314613342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872532469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10516249388456345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872548860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10855589807033539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872548861, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784872548861, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784872573547, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7666770219802856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784872573548, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784872573548, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784872590089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11235357820987701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872606426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10899011790752411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872622788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10727697610855103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872630834, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784872630834, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784872654333, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.767882227897644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784872654334, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784872654334, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784872662502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10815460979938507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872679232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11019998043775558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872696030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11105614900588989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872712062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10402505844831467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872712062, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784872712063, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784872736843, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.768605649471283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784872736844, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784872736844, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784872754626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11389748752117157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872771356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11809904128313065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872788155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11488141119480133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872796441, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784872796441, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784872819227, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7680383920669556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784872819228, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784872819228, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784872827390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11397811025381088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872844193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11413004249334335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872860778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11151266098022461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872877133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11308296769857407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872877133, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784872877134, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784872901798, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7691541314125061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784872901799, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784872901799, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784872918524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1173432469367981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872934973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11746994405984879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872951697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10999585688114166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872959980, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784872959980, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784872983264, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7694912552833557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784872983265, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784872983265, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784872991605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11441735923290253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873008365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11514928191900253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873025067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1107366532087326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873042173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10387398302555084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873042173, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784873042174, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784873065615, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7705880999565125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784873065616, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784873065616, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784873081712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10610837489366531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873098240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10539531707763672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873114503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10575135797262192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873122715, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784873122716, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784873146943, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.770862340927124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784873146944, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784873146944, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784873155142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10354553908109665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873171883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11105423420667648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873188306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071772500872612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873205215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11242979764938354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873205216, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784873205216, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784873228963, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7712953090667725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784873228964, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784873228964, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784873245622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10641881823539734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873261931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10611210018396378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873278454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10143525898456573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873286587, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784873286587, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784873310092, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7716472148895264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784873310092, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784873310092, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784873318380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10253874957561493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873335010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10099146515130997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873351847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11075311154127121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873368151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10389666259288788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873368151, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784873368151, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784873393083, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7724156379699707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784873393083, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784873393083, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784873409894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1059865653514862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873426384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10610627382993698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873442325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1064484566450119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873450645, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784873450646, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784873474378, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7729170918464661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784873474378, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784873474378, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784873482513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10674449801445007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873498650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09943196177482605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873515194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10758432745933533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873531746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10412043333053589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873531746, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784873531746, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784873555741, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730358242988586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784873555742, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784873555742, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784873571806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10764424502849579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873589020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10763980448246002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873605708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10423363000154495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873613908, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784873613908, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784873637150, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730283737182617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784873637151, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784873637151, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784873645205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10904991626739502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873661968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10726580023765564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873678405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11145062744617462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873695179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10778208822011948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873695180, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784873695180, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784873718818, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7734581828117371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784873718819, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784873718819, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784873735511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10501052439212799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873751721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10177071392536163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873768449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10765264928340912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.564999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873776882, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784873776882, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784873800969, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738660573959351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784873800969, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784873800969, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784873809036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10378532856702805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873825067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09889679402112961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873841478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1141795739531517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.664999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873857748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11037707328796387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873857749, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784873857749, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784873881930, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738699913024902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784873881931, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784873881931, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784873898210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1063421368598938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873914914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10840026289224625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873931328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10784465074539185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873939361, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784873939362, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784873963040, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742166519165039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784873963041, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784873963041, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784873971827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10958980768918991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873988783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11416039615869522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874005732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11206736415624619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874022611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11256551742553711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874022611, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784874022611, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784874046561, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7739840745925903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784874046562, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784874046562, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784874063300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10971637815237045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874080527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11351051926612854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.998333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874097572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1101815328001976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874105934, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784874105934, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784874129300, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774037778377533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784874129300, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784874129300, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784874137632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11057361215353012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874155224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10910207033157349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874171937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11251810938119888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.131666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874188641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11066124588251114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874188642, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784874188642, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784874212054, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774360179901123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784874212055, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784874212055, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784874228805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1099185049533844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874245889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11251439154148102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874262877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11659518629312515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874271370, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784874271370, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784874296147, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7747328877449036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784874296148, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784874296148, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784874304287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11445218324661255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874321635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10752061009407043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.331666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874338131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10109102725982666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874354750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10675425082445145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874354751, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784874354752, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784874378724, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7754207849502563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784874378725, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784874378725, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784874395361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10469962656497955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874411940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1061108261346817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874428920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10504664480686188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874437514, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784874437514, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784874461764, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7756708264350891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784874461765, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784874461765, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784874470163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.108424112200737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874486548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10305130481719971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874503919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10496752709150314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874520580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10716409981250763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.631666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874520580, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784874520581, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784874543815, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7757408022880554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784874543816, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784874543816, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784874560058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1002260074019432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874576731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10355134308338165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874593240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10299888253211975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874601475, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784874601476, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784874624624, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7759867310523987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784874624624, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784874624625, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784874632991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10373096168041229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874650163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10248906165361404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874666660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1135580986738205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874683431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10268759727478027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874683431, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784874683432, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784874707218, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7759820818901062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784874707219, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784874707219, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784874724120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10676568746566772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874740770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10714593529701233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874757550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10144971311092377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874765937, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784874765938, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} diff --git a/recommendation_v4/rcp_logs/gbs_32768/seed147670635.log b/recommendation_v4/rcp_logs/gbs_32768/seed147670635.log new file mode 100644 index 000000000..8a4d6ff6d --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_32768/seed147670635.log @@ -0,0 +1,662 @@ +:::MLLOG {"namespace": "", "time_ms": 1784893931704, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784893931705, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784893959819, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784893959821, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784893959821, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784893959821, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784893959821, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784893959821, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784893959821, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784893959821, "event_type": "POINT_IN_TIME", "key": "seed", "value": 147670635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784893959821, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784893959821, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784893959822, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784893959822, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784893959849, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784893959849, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784894870277, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784894870277, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784894870625, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784895097409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13852256536483765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.1666666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784895113030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138487309217453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.4999999999999995e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784895128224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384914517402649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784895135869, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784895135869, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784895213256, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.500963032245636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784895213257, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784895213257, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784895220719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13843511044979095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3166666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895235564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13835781812667847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895250883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13852310180664062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9833333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895266115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13836170732975006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895266115, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784895266115, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784895289413, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5010941624641418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784895289414, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784895289415, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784895304588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138079434633255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895319802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13823965191841125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895335198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13800527155399323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895342602, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784895342603, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784895366436, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5012567043304443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784895366437, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784895366437, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784895374300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382782906293869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895389459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13821950554847717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895404976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13823376595973969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895420322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13774409890174866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895420322, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784895420322, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784895443798, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5014377236366272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784895443799, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784895443799, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784895458897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13820523023605347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895473941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13747933506965637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895489241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13783982396125793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895496842, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784895496842, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784895520432, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5016515851020813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784895520433, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784895520434, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784895527799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13700108230113983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895543178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13722437620162964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895557962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367916762828827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895573290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13701564073562622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895573291, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784895573291, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784895597757, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5018600821495056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784895597758, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784895597758, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784895612760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1369762420654297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.316666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895628023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371479481458664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895643728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362568438053131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895651251, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784895651251, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784895675019, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5020810961723328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784895675019, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784895675020, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784895682656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13702823221683502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895698870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361314207315445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895714359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364269256591797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895730536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13696961104869843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895730537, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784895730537, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784895754122, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5023696422576904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784895754123, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784895754123, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784895769305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360345184803009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895785307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13483795523643494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.983333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895800434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13651731610298157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784895808027, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784895808027, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784895832030, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5026381015777588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784895832031, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784895832031, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784895839645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13293305039405823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784895855386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13446690142154694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784895870444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13486433029174805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784895885558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13504204154014587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784895885630, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784895885630, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784895909974, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5029171109199524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784895909974, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784895909974, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784895925167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336742788553238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784895940288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13440456986427307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784895955457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13510270416736603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784895963838, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784895963838, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784895987427, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5032342076301575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784895987427, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784895987428, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784895995041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13578671216964722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896010418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347689926624298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896025880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318744719028473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896041792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13456213474273682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896041792, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784896041792, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784896064435, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5036888718605042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784896064435, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784896064436, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784896079910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336183398962021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896095598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13526363670825958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896111982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13505816459655762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896119890, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784896119890, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784896143110, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5045623183250427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784896143111, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784896143111, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784896150805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365327090024948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896166478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13345322012901306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896182654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343473643064499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896198070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320655196905136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896198140, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784896198140, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784896220820, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5055482387542725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784896220821, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784896220821, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784896236165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13279137015342712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896252185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13119813799858093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896267534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13309703767299652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896275266, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784896275267, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784896297760, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5065887570381165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784896297761, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784896297761, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784896305417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13843144476413727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896320929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13103938102722168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896336910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298784762620926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896352317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13039827346801758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896352318, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784896352318, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784896375469, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.507820188999176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784896375470, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784896375470, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784896390835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13285544514656067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896406357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297694444656372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896422655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13066092133522034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896430330, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784896430330, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784896453035, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5091142058372498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784896453036, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784896453036, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784896460753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13150924444198608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896476036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350899338722229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896491686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13321739435195923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896507833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13557414710521698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896507834, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784896507834, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784896530922, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5105797648429871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784896530923, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784896530923, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784896546365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350288689136505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896561859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13259193301200867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1649999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896577384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13315552473068237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896585071, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784896585071, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784896608484, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.512100338935852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784896608485, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784896608485, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784896617223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13639819622039795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896633164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13524311780929565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896648987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363169252872467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896665050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364056020975113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896665051, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784896665051, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784896688923, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5136758089065552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784896688923, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784896688924, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784896705530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346978098154068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896721376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13093256950378418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896737056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362392008304596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896745083, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784896745083, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784896768831, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5152435898780823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784896768832, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784896768832, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784896776650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13604652881622314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896793099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13255569338798523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896808822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13264665007591248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896824336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13075962662696838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896824337, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784896824337, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784896847886, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5167389512062073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784896847887, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784896847887, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784896863539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310567706823349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896879810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13816577196121216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896895623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12962199747562408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896903324, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784896903324, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784896925914, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5184480547904968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784896925915, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784896925915, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784896933505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1293485164642334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896949359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13477648794651031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896965076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345919370651245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896981425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13027849793434143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896981426, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784896981426, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784897004343, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5205942392349243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784897004344, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784897004344, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784897019832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1281328797340393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897035474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13183604180812836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897051391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13551270961761475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897059397, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784897059397, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784897082764, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.522999107837677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784897082764, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784897082765, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784897090618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13650383055210114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897106835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335427463054657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897122482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13206198811531067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897138009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13091826438903809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897138010, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784897138010, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784897161771, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5259330868721008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784897161771, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784897161772, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784897177751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12954136729240417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897193599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13239799439907074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897209970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13534800708293915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897218055, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784897218055, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784897240666, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5292792916297913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784897240667, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784897240667, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784897248506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13186874985694885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897264652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13340707123279572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897280622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13493117690086365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897296665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345319151878357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897296665, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784897296665, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784897320534, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.533307671546936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784897320535, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784897320535, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784897336672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358770728111267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897353356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13397452235221863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897369561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13303044438362122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3650000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897377513, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784897377514, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784897401184, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5381539463996887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784897401185, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784897401185, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784897409467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14022080600261688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897425906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352008432149887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897442540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13432572782039642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4649999999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897459331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135033518075943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897459331, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784897459332, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784897482791, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.543677031993866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784897482791, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784897482792, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784897498877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13504838943481445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897515136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330607533454895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.5649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897531396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13230332732200623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897540024, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784897540024, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784897564642, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5496487617492676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784897564642, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784897564643, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784897572838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323358416557312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897588978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13064241409301758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897604923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356230229139328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897621173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13214121758937836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897621174, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784897621174, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784897644786, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5568435192108154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784897644787, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784897644787, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784897661000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13437968492507935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897677561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12969988584518433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897693721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13130205869674683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897701651, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784897701651, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784897725888, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.565775990486145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784897725888, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784897725889, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784897733610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13183937966823578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897749634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13006800413131714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897765718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324712038040161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897782213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13185124099254608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897782214, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784897782214, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784897806097, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5753545761108398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784897806098, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784897806098, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784897822248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312178075313568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897838176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1272422820329666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897853812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330271065235138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897861857, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784897861857, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784897885077, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5846254825592041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784897885078, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784897885078, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784897893171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13346035778522491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897909117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12567263841629028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897925519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12546154856681824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897941928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13131502270698547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897941929, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784897941929, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784897965530, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5975090265274048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784897965531, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784897965531, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784897981461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12783822417259216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897997402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12697544693946838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898013733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12649433314800262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898021631, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784898021631, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784898045443, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6037235260009766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784898045444, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784898045444, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784898053531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12856747210025787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898070120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1251889020204544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898086091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1280025690793991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898102198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12641173601150513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898102199, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784898102199, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784898125364, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6158297657966614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784898125365, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784898125365, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784898141210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12431211769580841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898157503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12747898697853088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4983333333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898173482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13015219569206238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898181448, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784898181448, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784898205802, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6292466521263123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784898205803, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784898205803, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784898214373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12847375869750977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898231136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12376555055379868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898247874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12864044308662415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898264205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12311524152755737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898264206, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784898264206, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784898288112, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.645656943321228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784898288113, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784898288113, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784898304884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1257796436548233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898321332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12709352374076843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898338034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12527740001678467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898346368, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784898346368, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784898369661, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6691115498542786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784898369662, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784898369662, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784898377740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12418829649686813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898393718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12438581883907318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898409995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12402839213609695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898426256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.123397096991539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.898333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898426257, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784898426257, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784898450176, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6920463442802429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784898450177, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784898450177, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784898466913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11687647551298141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898483085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11629381775856018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898499006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11469613760709763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898507024, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784898507024, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784898530086, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7032926082611084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784898530087, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784898530087, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784898537905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11603941023349762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898554177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11318821460008621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898570397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11847894638776779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898586212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11541688442230225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898586212, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784898586212, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784898610455, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7327128052711487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784898610455, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784898610455, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784898627225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11702260375022888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898643017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10767310857772827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.198333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898658851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11355894804000854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898666674, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784898666674, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784898690040, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7458020448684692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784898690041, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784898690041, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784898697832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11491900682449341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898714045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11420338600873947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.298333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898730201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10962509363889694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898745949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11239200085401535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898746024, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784898746024, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784898770926, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7510687112808228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784898770927, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784898770927, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784898787208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11074003577232361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.398333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898803251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11140166223049164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898819534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1043093129992485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898827552, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784898827552, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784898851451, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7528185844421387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784898851452, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784898851452, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784898859490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087883710861206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.498333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898875898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10846098512411118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.531666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898892243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10580871254205704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898908941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10888613015413284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.598333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898908942, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784898908942, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784898932511, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7587505578994751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784898932512, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784898932512, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784898948601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10718685388565063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898964533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10658130794763565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898980784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11190325021743774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898988795, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784898988795, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784899012234, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.759536862373352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784899012235, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784899012235, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784899020130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10168871283531189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899036619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11028319597244263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899052903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1044854149222374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899069399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10730867832899094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899069400, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784899069400, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784899093280, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7612096667289734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784899093281, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784899093281, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784899109559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11149907857179642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899125722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10883177071809769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899141741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10736341774463654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899149586, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784899149586, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784899172908, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.764346182346344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784899172909, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784899172909, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784899181058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10814838856458664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899197332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10950808972120285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899213831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11098454892635345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899229688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10457506030797958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899229688, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784899229688, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784899253767, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7640153169631958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784899253768, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784899253768, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784899271083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11437021195888519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899287683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11826812475919724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899304550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1149819865822792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899312916, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784899312916, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784899336179, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7681159377098083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784899336180, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784899336180, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784899344407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11470068991184235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899361246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11415497958660126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899377831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11187861114740372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899394245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11321951448917389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899394246, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784899394246, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784899418992, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7679373025894165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784899418993, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784899418993, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784899435555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11698412895202637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899451948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1169692650437355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899468723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11011853814125061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899477079, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784899477079, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784899500883, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.767989993095398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784899500884, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784899500884, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784899509257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11503450572490692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899526127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11480952054262161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899542816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11055590212345123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899560056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1040646880865097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899560057, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784899560057, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784899583799, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7679269313812256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784899583799, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784899583799, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784899599752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10659617185592651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899616187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1054217740893364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899632403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10546448081731796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899640520, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784899640520, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784899663990, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7680603861808777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784899663990, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784899663991, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784899672149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10395914316177368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899688770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11140336841344833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899705272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10736186057329178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899721995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11233330518007278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899721995, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784899721996, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784899746501, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7688477635383606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784899746502, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784899746503, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784899763219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10669861733913422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899779480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10640697181224823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899795893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10160395503044128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899803882, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784899803883, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784899827938, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7695965766906738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784899827938, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784899827939, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784899836198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10273171961307526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899852745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10119348764419556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899869454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11089996248483658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899885502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10378745943307877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899885503, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784899885503, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784899910022, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7692160606384277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784899910022, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784899910022, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784899926608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10572392493486404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899943187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10630445182323456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899959314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10625782608985901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899967742, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784899967742, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784899991120, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7696937918663025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784899991121, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784899991121, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784899999330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723436623811722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900015503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0996604636311531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900032064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10763821750879288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900048748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10450759530067444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900048748, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784900048748, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784900072360, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7706308960914612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784900072361, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784900072361, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784900088574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10796477645635605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900105717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10785935819149017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900122397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10411620140075684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900130639, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784900130639, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784900154171, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7719217538833618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784900154172, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784900154172, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784900162291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1091800406575203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900178946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10715372860431671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900195274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11172036081552505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900211862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10789933055639267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900211862, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784900211862, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784900235209, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7720355987548828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784900235210, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784900235210, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784900251802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10524765402078629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900267940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10210694372653961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900284936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10775960981845856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.564999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900293371, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784900293371, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784900316753, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7716939449310303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784900316754, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784900316754, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784900324845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1044904962182045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900340856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09926146268844604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900357318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11450066417455673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.664999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900373652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1102755069732666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900373653, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784900373653, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784900397188, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7724789381027222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784900397188, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784900397189, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784900413582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10617899894714355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900430405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10843904316425323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900447007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080745980143547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900455122, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784900455123, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784900479271, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7725208401679993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784900479272, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784900479272, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784900488073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10987064242362976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900504837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11465753614902496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900521607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11199462413787842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900538393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11328709125518799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900538394, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784900538394, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784900562121, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7735397219657898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784900562121, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784900562122, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784900578782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10961014777421951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900595817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11376698315143585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.998333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900612691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11032286286354065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900621084, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784900621084, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784900644551, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774141788482666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784900644553, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784900644553, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784900652834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11041593551635742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900670340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10884043574333191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900687105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11251145601272583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.131666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900703942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11085303127765656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900703943, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784900703943, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784900726948, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7736579179763794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784900726949, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784900726949, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784900743699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11013428121805191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900760903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11248345673084259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900777910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11676716804504395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900786419, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784900786419, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784900809727, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7743808627128601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784900809727, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784900809727, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784900817892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11452975869178772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900835410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10751271992921829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.331666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900851962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10104171186685562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900868634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10696075856685638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900868634, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784900868634, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784900892216, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7723627090454102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784900892217, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784900892217, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784900908856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10446551442146301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900925411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10647419095039368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900942320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10540623962879181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900950874, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784900950874, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784900973844, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7739356756210327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784900973845, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784900973845, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784900982201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10848148912191391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900998504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10315167903900146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784901015939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10477735102176666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784901032858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10736958682537079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.631666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784901032859, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784901032859, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784901056310, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7741047143936157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784901056311, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784901056311, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784901072635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1001630499958992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784901089331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1037607491016388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784901105990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10295327007770538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784901114275, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784901114275, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784901138458, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7743462920188904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784901138459, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784901138459, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784901146818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10380604863166809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784901164067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10249249637126923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784901180653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11331678181886673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784901197802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10316306352615356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784901197803, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784901197803, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} diff --git a/recommendation_v4/rcp_logs/gbs_32768/seed162052595.log b/recommendation_v4/rcp_logs/gbs_32768/seed162052595.log new file mode 100644 index 000000000..84ee6571d --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_32768/seed162052595.log @@ -0,0 +1,685 @@ +:::MLLOG {"namespace": "", "time_ms": 1784874656304, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784874656304, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784874679946, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784874679950, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784874679950, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784874679950, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784874679950, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784874679950, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784874679950, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784874679950, "event_type": "POINT_IN_TIME", "key": "seed", "value": 162052595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784874679950, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784874679950, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784874679950, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784874679951, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784874680074, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784874680074, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784875572346, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784875572347, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784875572674, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784875720888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.139136403799057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.1666666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784875736276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13897855579853058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.4999999999999995e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784875751281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1390974223613739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784875758721, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784875758721, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784875809938, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49946528673171997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784875809939, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784875809939, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784875817312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13905450701713562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3166666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784875831991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1390204131603241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784875847093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13890133798122406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9833333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784875862159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13885723054409027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784875862159, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784875862160, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784875885147, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4996291995048523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784875885148, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784875885148, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784875900270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13884124159812927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784875915334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13883893191814423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784875930494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13872233033180237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784875937806, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784875937806, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784875961435, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49983853101730347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784875961436, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784875961436, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784875969438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387232393026352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784875984470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860827684402466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784875999844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13857673108577728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784876015231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13839475810527802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784876015231, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784876015232, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784876038219, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5000891089439392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784876038220, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784876038220, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784876053071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13858966529369354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784876068144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13812458515167236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784876083499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13822151720523834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784876090900, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784876090901, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784876114054, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5004041790962219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784876114055, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784876114055, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784876121285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13766759634017944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784876136646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377570778131485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784876151501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13745611906051636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784876166686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13747118413448334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784876166687, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784876166687, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784876190722, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5007785558700562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784876190723, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784876190723, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784876205362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13760077953338623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.316666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784876220355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13747134804725647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784876236090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13680291175842285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784876243598, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784876243598, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784876266884, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5011768937110901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784876266885, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784876266885, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784876274471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13742958009243011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784876290707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13662174344062805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784876305869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13683035969734192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784876321944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13733302056789398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784876321945, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784876321945, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784876344981, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5016996264457703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784876344982, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784876344982, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784876360116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365136206150055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784876375791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13519632816314697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.983333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784876390806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368206888437271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876398354, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784876398355, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784876422816, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5022421479225159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784876422817, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784876422817, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784876430410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336698830127716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876445988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13501881062984467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876460839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13522186875343323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876475794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354401707649231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876475869, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784876475870, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784876500058, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5028268694877625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784876500059, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784876500059, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784876515190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1339917927980423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876530176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13485319912433624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876545191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353011429309845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876553620, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784876553621, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784876577295, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5034870505332947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784876577296, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784876577296, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784876584586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357787400484085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876599500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13498300313949585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876614688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13219957053661346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876630782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346915066242218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876630783, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784876630783, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784876653656, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5042855739593506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784876653656, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784876653657, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784876668984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13384021818637848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876684608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13534656167030334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876700569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13498808443546295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876708297, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784876708298, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784876731966, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5052836537361145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784876731969, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784876731969, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784876739556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13649600744247437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876754985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13346731662750244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876771137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343257874250412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876786521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13202841579914093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876786590, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784876786591, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784876809624, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5062920451164246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784876809624, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784876809624, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784876824839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1328946202993393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876840837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312018781900406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876855954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13299345970153809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876863545, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784876863546, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784876886681, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5074412226676941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784876886681, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784876886681, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784876894232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13842402398586273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876909541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13110655546188354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876925391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298253834247589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876940778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13048326969146729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876940779, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784876940779, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784876963995, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5086959600448608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784876963996, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784876963996, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784876979405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13275139033794403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876995004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12969368696212769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877011265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13065853714942932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877018773, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784877018774, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784877041843, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5100130438804626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784877041843, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784877041843, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784877049527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315067857503891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877064700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13517636060714722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877080258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13323339819908142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877096207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356988400220871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877096207, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784877096208, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784877119834, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5114462375640869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784877119835, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784877119835, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784877135360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13526105880737305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877150871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13258680701255798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1649999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877166470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13323065638542175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877174171, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784877174172, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784877198155, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5127866864204407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784877198156, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784877198156, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784877206943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13637754321098328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877222909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13528980314731598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877238869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362985074520111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877254998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13647182285785675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877254998, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784877254999, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784877278968, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5140691995620728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784877278968, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784877278969, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784877295652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13482147455215454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877311734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13085411489009857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877327590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361377090215683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877335591, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784877335591, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784877359852, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5154426097869873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784877359853, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784877359853, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784877367656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359340250492096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877384221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323823481798172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877400185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326555758714676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877415888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1309511810541153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877415888, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784877415889, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784877439295, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5169503092765808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784877439295, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784877439295, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784877454928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311296820640564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877471142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382627934217453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877486767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12952865660190582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877494537, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784877494537, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784877517649, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5186318159103394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784877517650, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784877517651, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784877525353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12943795323371887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877541149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13479670882225037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877556778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13455377519130707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877573467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13044559955596924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877573468, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784877573468, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784877596721, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5204984545707703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784877596722, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784877596722, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784877612287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1281389445066452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877627873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13183456659317017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877643467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353229284286499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877651407, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784877651408, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784877675034, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5225229859352112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784877675035, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784877675035, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784877682863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366700828075409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877699007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13358429074287415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877714448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321735978126526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877729798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13096463680267334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877729798, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784877729799, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784877752916, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.524826169013977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784877752916, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784877752917, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784877768459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12961654365062714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877784038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13243579864501953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877800383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13535983860492706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877808464, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784877808465, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784877831309, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.527394711971283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784877831309, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784877831310, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784877839074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13197562098503113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877854864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13348931074142456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877870688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13505880534648895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877886562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13465414941310883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877886563, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784877886563, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784877910913, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5303020477294922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784877910914, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784877910914, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784877926883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360570788383484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877943353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13398416340351105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877959399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312938809394836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3650000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877967332, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784877967332, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784877990950, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5335052609443665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784877990951, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784877990951, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784877999144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1403648853302002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878015564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13529512286186218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878032179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13443440198898315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4649999999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878048847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352185606956482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878048847, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784878048848, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784878071673, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5369900465011597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784878071673, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784878071674, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784878087481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13519394397735596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878103446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13324449956417084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.5649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878119373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13242632150650024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878127934, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784878127934, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784878151918, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.540977954864502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784878151919, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784878151919, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784878159947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13253335654735565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878175816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13083170354366302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878191559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13588334619998932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878207686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13234944641590118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878207686, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784878207686, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784878231012, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5458800792694092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784878231013, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784878231013, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784878247115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13481681048870087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878263550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13031920790672302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878279310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13193704187870026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878287101, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784878287101, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784878309786, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5522220730781555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784878309787, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784878309787, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784878317465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13231487572193146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878333183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13084672391414642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878349177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13316617906093597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878365653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13269564509391785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878365654, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784878365655, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784878389469, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.560624897480011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784878389470, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784878389470, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784878405605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13214902579784393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878421717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12820085883140564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878437387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13415174186229706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878445497, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784878445498, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784878468864, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5701613426208496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784878468865, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784878468865, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784878476985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13467752933502197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878492920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1267929971218109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878509301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12692049145698547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878525232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13305294513702393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878525233, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784878525233, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784878548148, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.580833911895752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784878548149, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784878548149, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784878563915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12945358455181122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878579813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12861892580986023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878595980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12803718447685242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878603763, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784878603764, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784878626744, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5907667279243469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784878626745, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784878626745, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784878634820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301960051059723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878651204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.127642422914505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878667173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13019591569900513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878683058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12845537066459656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878683059, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784878683059, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784878706330, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6009939908981323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784878706330, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784878706330, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784878722070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1270119547843933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878738378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12985949218273163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4983333333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878754337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13321232795715332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878762292, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784878762292, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784878786123, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6097543835639954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784878786123, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784878786123, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784878794679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131648987531662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878811294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12670856714248657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878827559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13133926689624786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878843729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12666799128055573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878843729, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784878843729, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784878867130, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6181161999702454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784878867130, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784878867130, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784878883757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12895919382572174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878900063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301700472831726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878916975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1281658560037613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878925188, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784878925189, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784878947975, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6264981627464294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784878947975, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784878947975, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784878955995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12763278186321259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878972085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12747254967689514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878988364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12809044122695923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879004645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12735523283481598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.898333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879004646, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784879004646, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784879027750, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6338135600090027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784879027751, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784879027751, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784879044244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12173520773649216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879060483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12080155313014984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879076377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1192319318652153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879084373, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784879084374, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784879106877, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6373912692070007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784879106878, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784879106878, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784879114685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12123046070337296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879130783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11787310242652893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879146905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12247320264577866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879162788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11956541240215302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879162788, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784879162789, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784879186585, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6524298191070557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784879186586, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784879186586, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784879203218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12100406736135483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879218994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11224818974733353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.198333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879234915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11818795651197433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879242809, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784879242810, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784879266619, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6693538427352905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784879266619, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784879266620, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784879274364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11878838390111923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879290478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11891284584999084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.298333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879306660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11355441063642502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879322449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11727797985076904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879322523, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784879322524, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784879347564, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6964111924171448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784879347564, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784879347565, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784879363720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11616040766239166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.398333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879379716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11631593853235245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879395955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10925589501857758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879403971, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784879403972, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784879428265, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7233104705810547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784879428265, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784879428266, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784879436353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11295989900827408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.498333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879452914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11456789821386337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.531666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879469200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1123288944363594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879485569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11466862261295319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.598333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879485570, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784879485570, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784879509469, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7477177381515503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784879509469, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784879509470, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784879525470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11341744661331177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879541375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11278784275054932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879557434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11687988042831421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879565364, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784879565365, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784879589118, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7550884485244751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784879589119, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784879589119, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784879596975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10695717483758926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879613075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11430208384990692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879628944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080881655216217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879645072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11270922422409058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879645072, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784879645073, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784879669830, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7600334286689758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784879669830, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784879669830, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784879686320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11509367823600769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879702564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1130475252866745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879718679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111073337495327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879726702, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784879726702, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784879750893, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7628577351570129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784879750894, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784879750894, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784879759003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11134184151887894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879775446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11283501982688904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879792049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11287983506917953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879807981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10751734673976898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879807982, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784879807982, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784879831595, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7646251916885376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784879831596, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784879831596, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784879848757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11568867415189743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879864859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11994699388742447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879881112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11763225495815277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879889217, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784879889218, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784879912406, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7653693556785583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784879912406, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784879912407, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784879920348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11592942476272583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879936817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11562953889369965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879953024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11292555183172226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879969105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11377976834774017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879969105, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784879969106, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784879993430, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7660014033317566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784879993431, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784879993431, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784880010168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11853061616420746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880026710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11773885786533356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880043471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11099152266979218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880051855, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784880051855, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784880075221, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7665657997131348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784880075222, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784880075222, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784880083665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11578910052776337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880100499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1155736893415451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880117065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11120844632387161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880134096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10460948199033737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880134096, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784880134097, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784880156875, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7672369480133057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784880156876, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784880156876, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784880172765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10672132670879364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880189037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10604295134544373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880205292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10621634125709534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880213448, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784880213449, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784880236991, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7677485346794128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784880236991, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784880236992, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784880245205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10419361293315887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880261949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111544169485569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880278377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10761044919490814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880295395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11285081505775452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880295395, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784880295396, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784880318947, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7685725092887878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784880318948, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784880318948, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784880335527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10694804042577744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880351675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10631028562784195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880367962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1021115630865097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880375953, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784880375953, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784880400201, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7692181468009949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784880400202, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784880400202, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784880408383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1024765595793724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880424860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10133492946624756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880441389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11165677756071091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880457409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039206013083458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880457410, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784880457410, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784880481946, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7699382305145264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784880481946, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784880481946, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784880498437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10663795471191406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880514913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10712285339832306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880530866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10720464587211609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880539193, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784880539194, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784880563178, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7704401016235352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784880563179, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784880563179, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784880571396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10768880695104599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880587560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09955427050590515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880604268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1078442633152008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880620930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10473290085792542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880620930, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784880620931, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784880645630, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7710413932800293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784880645631, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784880645631, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784880661816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1076197698712349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880678849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10801464319229126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880695398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1044226884841919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880703585, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784880703586, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784880727020, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7713555097579956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784880727021, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784880727021, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784880734963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10871867090463638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880751378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1076890230178833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880767648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11172975599765778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880784166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10804307460784912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880784167, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784880784167, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784880807745, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7717275023460388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784880807746, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784880807746, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784880824296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10519541800022125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880840359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10188403725624084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880857352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10813146084547043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.564999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880865852, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784880865852, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784880889768, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7721058130264282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784880889768, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784880889768, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784880897880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10448190569877625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880914130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09934572875499725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880930690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11410385370254517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.664999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880947184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11135473102331161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880947185, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784880947185, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784880971622, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7724032998085022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784880971623, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784880971623, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784880988239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10647058486938477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881005199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10862186551094055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881021796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10850712656974792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881029950, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784881029951, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784881054457, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7726014256477356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784881054457, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784881054457, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784881063272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1101488247513771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881080220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11467408388853073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881097049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1123494952917099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881113936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11316981166601181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881113936, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784881113937, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784881138141, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7725335359573364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784881138141, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784881138141, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784881154878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11001693457365036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881172037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11351300776004791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.998333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881188979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11064265668392181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881197388, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784881197388, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784881221612, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7728598117828369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784881221612, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784881221612, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784881229818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11063958704471588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881247154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10942143946886063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881263803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11240100860595703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.131666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881280467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11124078184366226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881280467, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784881280468, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784881303813, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7732689380645752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784881303813, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784881303814, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784881320488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11008694022893906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881337530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11285825818777084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881354308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11707936972379684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881362726, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784881362726, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784881387576, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7732497453689575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784881387577, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784881387577, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784881395674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11484773457050323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881412864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10761266946792603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.331666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881429287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10130003839731216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881445648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10698056221008301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881445648, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784881445649, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784881469293, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773747980594635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784881469294, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784881469294, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784881485725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10485746711492538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881502024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10611388087272644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881518764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10533851385116577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881527253, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784881527254, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784881550623, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738913893699646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784881550623, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784881550624, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784881558966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1086914986371994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881575225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10305783897638321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881592473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10475026071071625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881609206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10715780407190323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.631666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881609207, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784881609207, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784881632656, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7739665508270264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784881632657, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784881632657, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784881648654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10071577876806259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881665058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10376092791557312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881681376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10344579070806503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881689453, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784881689453, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784881712649, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774172306060791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784881712650, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784881712650, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784881720868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10401216894388199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881737834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1029793992638588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881754344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11406856775283813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881771257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10318776965141296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881771258, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784881771258, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784881794883, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742573022842407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784881794883, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784881794883, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784881811548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10716130584478378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881828084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1075611412525177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881844541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10174486041069031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881852723, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784881852723, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784881876374, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7743743658065796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784881876375, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784881876375, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784881884557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10137377679347992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881901123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10038921236991882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881917934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10604315251111984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881934808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10288899391889572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881934808, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784881934809, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784881958614, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7745136618614197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784881958615, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784881958615, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784881974822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10770188271999359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881992267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1035032868385315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882008879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10598792135715485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.198333333333333e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_32768/seed182111600.log b/recommendation_v4/rcp_logs/gbs_32768/seed182111600.log new file mode 100644 index 000000000..5e4d3e155 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_32768/seed182111600.log @@ -0,0 +1,1410 @@ +:::MLLOG {"namespace": "", "time_ms": 1784846531479, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784846531480, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784846554730, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784846554732, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784846554732, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784846554732, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784846554732, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784846554732, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784846554732, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784846554732, "event_type": "POINT_IN_TIME", "key": "seed", "value": 182111600, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784846554732, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784846554732, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784846554732, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784846554732, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784846555027, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784846555027, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784847421337, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784847421339, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784847421628, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784847591917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13874059915542603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.1666666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784847607593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387520283460617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.4999999999999995e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784847622192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13866497576236725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784847629473, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784847629474, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784847653204, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4952045679092407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784847653205, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784847653205, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784847660504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13862104713916779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3166666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784847675213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13879913091659546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784847690417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386469453573227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9833333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784847705583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13865526020526886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784847705584, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784847705584, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784847728541, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4953469932079315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784847728542, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784847728542, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784847743422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13858963549137115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784847758215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13852186501026154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784847773338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13836008310317993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784847780698, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784847780699, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784847804452, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4955340027809143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784847804453, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784847804453, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784847812141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13852718472480774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784847827195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384502798318863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784847842610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13843536376953125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784847857762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13814565539360046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784847857763, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784847857763, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784847880778, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4957517385482788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784847880779, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784847880779, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784847895692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138319194316864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784847910566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13792499899864197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784847925630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13812458515167236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784847933057, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784847933058, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784847956373, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49601995944976807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784847956374, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784847956374, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784847963598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13753008842468262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784847978806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13744856417179108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784847993488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13723765313625336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848008591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371459662914276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848008592, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784848008593, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784848032278, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.496308296918869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784848032280, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784848032280, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784848046896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374565213918686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.316666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848061816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13732826709747314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848077649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367386132478714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848085146, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784848085146, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784848108811, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49664002656936646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784848108811, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784848108811, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784848116342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13729068636894226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848132498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13650794327259064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848147587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.136808842420578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848163583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13709215819835663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848163584, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784848163584, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784848186531, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49704262614250183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784848186532, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784848186532, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784848201670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364547312259674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848217395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13543549180030823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.983333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848232363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13678905367851257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848239941, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784848239942, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784848262910, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.497429221868515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784848262912, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784848262912, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784848270455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133612260222435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848285921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350184828042984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848300815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13525837659835815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848315720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13536915183067322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848315798, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784848315799, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784848339998, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49785351753234863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784848339998, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784848339999, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784848355081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13414955139160156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848370174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13487350940704346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848385169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13536527752876282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848393401, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784848393402, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784848416738, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4983193576335907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784848416738, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784848416738, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784848424065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358182430267334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848438988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13508230447769165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848454158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324046105146408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848470042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346844583749771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848470043, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784848470043, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784848492520, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4989452660083771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784848492520, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784848492520, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784848507917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13397303223609924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848523517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13531820476055145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848539413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13513554632663727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848547298, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784848547299, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784848570683, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49988052248954773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784848570683, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784848570683, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784848578326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13641014695167542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848593913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335371434688568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848610347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343303769826889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848625838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13219481706619263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848625910, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784848625911, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784848649701, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5009028911590576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784848649702, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784848649702, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784848665037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13297244906425476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848681031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1313360035419464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848696228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331281065940857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848703887, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784848703888, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784848727245, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5020161867141724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784848727246, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784848727246, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784848734849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383158266544342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848750228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131073459982872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848766123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12997221946716309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848781610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304538995027542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848781610, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784848781611, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784848805041, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5032609701156616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784848805042, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784848805042, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784848820519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13279670476913452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848836198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298374980688095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848852574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13066096603870392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848860170, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784848860171, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784848883380, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5046176910400391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784848883381, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784848883381, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784848891016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13170096278190613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848906222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13515371084213257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848921879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13314726948738098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848937756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13561691343784332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848937757, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784848937757, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784848960839, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.506084144115448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784848960839, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784848960840, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784848976356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13522502779960632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848992041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13260427117347717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1649999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849007700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13327214121818542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849015455, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784849015456, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784849039246, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5075395107269287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784849039247, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784849039247, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784849047701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13643869757652283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849063630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353423297405243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849079592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13622204959392548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849095685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364784687757492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849095685, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784849095686, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784849119767, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5089837312698364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784849119767, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784849119768, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784849136420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348285973072052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849152667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.130825936794281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849168557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13626834750175476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849176661, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784849176661, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784849200570, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5104460120201111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784849200571, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784849200571, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784849208417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13593967258930206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849224473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325288563966751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849239883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13267068564891815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849255247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13093504309654236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849255247, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784849255248, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784849278982, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5120187401771545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784849278983, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784849278983, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784849294540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311337649822235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849310712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382880061864853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849326305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12963393330574036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849334069, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784849334070, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784849357521, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5138807892799377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784849357522, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784849357522, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784849365153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12941116094589233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849380810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13476358354091644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849396549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345067173242569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849412834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13038606941699982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849412835, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784849412835, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784849436367, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5162106156349182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784849436367, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784849436368, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784849451647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12804071605205536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849467100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13196001946926117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849482717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13516877591609955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849490651, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784849490652, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784849514788, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5187477469444275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784849514788, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784849514789, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784849522595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13659456372261047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849538880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13355419039726257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849554669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13209617137908936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849570332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13096198439598083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849570332, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784849570333, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784849593759, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5216954350471497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784849593759, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784849593759, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784849609626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12960633635520935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849625489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13253825902938843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849641695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13533185422420502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849649694, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784849649695, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784849673066, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5251766443252563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784849673066, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784849673066, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784849680848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13197404146194458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849696787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13337263464927673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849712644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13494770228862762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849728522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13458283245563507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849728522, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784849728523, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784849752576, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.529434084892273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784849752577, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784849752577, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784849768576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135977104306221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849785278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1339210718870163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849801631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331060230731964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3650000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849809630, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784849809631, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784849832841, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5341885685920715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784849832842, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784849832842, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784849841076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14020708203315735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849857481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352551430463791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849874106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13432583212852478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4649999999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849890694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351906955242157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849890694, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784849890695, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784849913584, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5395292043685913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784849913584, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784849913584, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784849929704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351621448993683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849945871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332181692123413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.5649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849962022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13221514225006104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849970674, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784849970675, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784849994377, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5453697443008423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784849994378, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784849994378, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784850002535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132341206073761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850018461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13072775304317474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850034179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13570399582386017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850050247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13214674592018127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850050248, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784850050248, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784850074100, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5523521900177002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784850074100, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784850074101, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784850090089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13437247276306152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850106729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12997815012931824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850122641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13165324926376343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850130523, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784850130524, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784850154743, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5610808730125427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784850154744, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784850154744, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784850162572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13202570378780365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850178621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13025768101215363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850194799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326046586036682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850211245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13212110102176666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850211246, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784850211246, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784850235531, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5715835690498352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784850235532, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784850235532, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784850251590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1316252499818802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850267549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1274266093969345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850283126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333703249692917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850291090, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784850291091, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784850314819, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.581810712814331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784850314820, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784850314820, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784850322901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1339533030986786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850338806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12606078386306763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850354844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12592929601669312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850370987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13199320435523987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850370988, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784850370988, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784850395004, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.595130980014801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784850395005, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784850395006, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784850410862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12836328148841858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850426852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.127407506108284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850443075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12682726979255676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850450919, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784850450919, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784850475024, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6025228500366211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784850475025, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784850475025, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784850483093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12889370322227478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850499605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1260717660188675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850515477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12880700826644897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850531292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1270032525062561, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850531292, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784850531293, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784850554652, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6162679195404053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784850554653, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784850554653, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784850570492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12523774802684784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850586781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1281438171863556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4983333333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850602751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13069632649421692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850610720, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784850610721, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784850634632, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.630220890045166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784850634633, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784850634633, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784850643103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12928304076194763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850659757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12454427778720856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850676102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12923333048820496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850692111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12412504106760025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850692112, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784850692112, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784850715284, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6470739245414734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784850715285, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784850715285, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784850731757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1268034279346466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850747912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12774905562400818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850764324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1257985383272171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850772581, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784850772582, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784850796785, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6673780679702759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784850796786, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784850796786, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784850805098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12494467198848724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850821191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12476576864719391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850837549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12483946233987808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850853829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12397348880767822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.898333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850853830, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784850853830, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784850877710, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6874874234199524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784850877710, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784850877710, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784850894252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11780256032943726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850910425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11792366206645966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850926175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11568206548690796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850934158, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784850934159, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784850957331, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6948780417442322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784850957331, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784850957332, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784850965152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1174389198422432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850981144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11432179808616638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850997188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11931685358285904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851012972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11631713062524796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851012972, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784851012973, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784851036244, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7241867184638977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784851036245, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784851036245, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784851053010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1180490031838417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851069179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10900969803333282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.198333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851085370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11498980224132538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851093412, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784851093412, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784851116619, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7437043190002441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784851116620, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784851116620, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784851124482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11641532927751541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851140912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11560536921024323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.298333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851157267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1114698201417923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851173169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1138516217470169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851173246, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784851173246, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784851197926, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7570252418518066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784851197927, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784851197927, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784851214116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11208604276180267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.398333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851229980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11223212629556656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851246102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10570789873600006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851254082, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784851254083, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784851277775, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7622617483139038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784851277776, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784851277776, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784851285771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10981258749961853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.498333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851302068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1102246642112732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.531666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851318182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10704228281974792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851334511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11013675481081009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.598333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851334512, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784851334512, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784851357809, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7667514681816101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784851357810, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784851357810, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784851373746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1083611324429512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851389619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10702814161777496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851405557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11203691363334656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851413452, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784851413453, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784851436764, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7680715322494507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784851436765, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784851436765, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784851444600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10155724734067917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851460770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1109599769115448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851476832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10473621636629105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851493286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792173445224762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851493286, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784851493287, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784851516645, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7692736387252808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784851516646, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784851516646, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784851532916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11180353164672852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851549037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1090700775384903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851565160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10771831125020981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851573063, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784851573063, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784851595612, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7699450850486755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784851595613, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784851595613, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784851603783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10816849023103714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851620272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1099638044834137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851636831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11117807775735855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851652667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10407275706529617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851652668, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784851652668, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784851675859, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7705444693565369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784851675859, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784851675859, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784851692898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11377395689487457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851709267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11853807419538498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851725960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11476903408765793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851734200, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784851734201, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784851757063, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7713190913200378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784851757063, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784851757063, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784851765227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11417792737483978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851782048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11356823891401291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851798418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1114412397146225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851814660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11281315982341766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851814661, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784851814661, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784851838833, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7715410590171814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784851838834, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784851838834, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784851855523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11696930974721909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851871947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11747737973928452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851888732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11004626750946045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851897099, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784851897099, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784851920021, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7718138098716736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784851920021, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784851920022, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784851928421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11476525664329529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851945197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11472690105438232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851961818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11088638752698898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851978992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10357476770877838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851978993, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784851978993, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784852002299, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7718023061752319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784852002299, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784852002300, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784852018306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10583936423063278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852034698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10557520389556885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852050930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10528592765331268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852059065, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784852059065, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784852082182, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7721022963523865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784852082183, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784852082183, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784852090323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10391482710838318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852106852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11099706590175629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852123127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10746991634368896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852140011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11174505949020386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852140011, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784852140012, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784852163419, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7723265886306763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784852163420, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784852163420, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784852180102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10646963119506836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852196096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10612708330154419, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852212389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10121628642082214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852220431, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784852220432, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784852243628, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7725151777267456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784852243628, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784852243629, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784852251798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10242553055286407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852268179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1010938361287117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852284735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11066654324531555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852300802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10377956926822662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852300802, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784852300803, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784852324861, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7725733518600464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784852324862, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784852324862, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784852341266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10565954446792603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852357678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10564178228378296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852373502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10647234320640564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852381736, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784852381737, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784852405357, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7726770639419556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784852405358, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784852405358, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784852413611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10686194151639938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852429621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09934306144714355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852446116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.107803575694561, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852462763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10408821702003479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852462764, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784852462764, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784852486083, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7728015184402466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784852486084, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784852486084, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784852502041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10734564065933228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852519023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792483389377594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852535557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10393762588500977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852543695, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784852543695, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784852566051, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7729513645172119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784852566052, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784852566052, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784852574175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10878214240074158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852590776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10705874860286713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852607066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11139542609453201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852623623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10767091065645218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852623624, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784852623624, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784852646094, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730157971382141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784852646094, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784852646094, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784852662851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10480092465877533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852678942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10175109654664993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852695696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772254317998886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.564999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852704185, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784852704186, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784852726842, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730743288993835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784852726843, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784852726843, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784852735009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10404272377490997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852751319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09936217963695526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852767922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11410480737686157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.664999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852784356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11048762500286102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852784356, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784852784357, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784852807881, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773175835609436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784852807882, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784852807882, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784852824414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10601241886615753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852841209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10830448567867279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852857691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10808895528316498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852865785, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784852865785, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784852890248, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7732925415039062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784852890249, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784852890249, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784852899178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10956227779388428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852916040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11437904834747314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852932780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11191356182098389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852949587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11248272657394409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852949587, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784852949588, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784852973042, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7733628749847412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784852973043, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784852973043, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784852989611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10992423444986343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853006509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11353226006031036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.998333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853023511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11022711545228958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853031864, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784853031865, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784853055720, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7734570503234863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784853055721, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784853055721, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784853063998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1104089766740799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853081335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10923752188682556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853098038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11237768828868866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.131666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853114758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11081299185752869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853114758, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784853114759, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784853137531, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7735739350318909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784853137532, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784853137532, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784853154287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10968382656574249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853171542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11250720173120499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853188587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11690311133861542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853197042, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784853197043, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784853221459, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7736304402351379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784853221459, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784853221460, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784853229589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11478319764137268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853246917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10771667212247849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.331666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853263543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10108467191457748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853280030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10669853538274765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853280030, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784853280031, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784853303821, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7735432386398315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784853303822, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784853303822, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784853320223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1045977771282196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853336534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10587982088327408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853353146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10496162623167038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853361542, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784853361543, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784853385209, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7736853361129761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784853385210, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784853385210, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784853393489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10855565965175629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853409624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10329199582338333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853426759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10471451282501221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853443516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10709378123283386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.631666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853443517, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784853443518, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784853466664, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7736452221870422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784853466665, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784853466665, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784853482783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10049986839294434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853499202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10384801775217056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853515487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10315150022506714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853523639, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784853523640, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784853547168, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7737537026405334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784853547169, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784853547169, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784853555499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10393078625202179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853572541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10291284322738647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853589019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11331601440906525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853605882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10294747352600098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853605883, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784853605884, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784853629087, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7737916111946106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784853629087, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784853629088, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784853645623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10704577714204788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853662122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10729119181632996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853678673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10156388580799103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853686915, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784853686915, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784853710640, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7739288806915283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784853710641, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784853710642, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784853718823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1011691614985466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853735244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10040774941444397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853751788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10582031309604645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853768574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10305297374725342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853768574, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784853768575, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784853792339, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738819122314453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784853792340, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784853792340, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784853808710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10733947157859802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853826129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10374440252780914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853842698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10566699504852295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853850734, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784853850734, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784853873952, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774043083190918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784853873953, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784853873953, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784853882080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10307753831148148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853898602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10808684676885605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853914969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1023399829864502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853931619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10682377219200134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853931619, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784853931619, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784853955396, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7740324139595032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784853955397, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784853955397, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784853972223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10173565149307251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853988518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10645587742328644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854005288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10921825468540192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854013649, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784854013650, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784854037538, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7740351557731628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784854037539, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784854037539, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784854046097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11136828362941742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854062509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10721053928136826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854078996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10261761397123337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854095599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10261096060276031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854095600, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784854095600, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784854119534, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7740909457206726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784854119535, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784854119535, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784854135801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10506537556648254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854152532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1049593836069107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.631666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854169385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11197260022163391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854177749, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784854177749, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784854201576, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7740944623947144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784854201577, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784854201577, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784854209694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10693272203207016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854226567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11255978792905807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854243376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10907749086618423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854259949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11258698999881744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854259949, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784854259949, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784854284043, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742562890052795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784854284043, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784854284044, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784854300771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10887175798416138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.831666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854317486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11377819627523422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854334299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11186163127422333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854342656, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784854342656, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784854366975, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7743983864784241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784854366975, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784854366976, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784854375368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11227315664291382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854392125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11177199333906174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854409202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10730700194835663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854426313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11056552082300186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1.0031666666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854426314, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784854426314, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784854450589, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7743541598320007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784854450590, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784854450590, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784854467104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11397042125463486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1.0064999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854483629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10835488140583038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1.0098333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854500175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1113186925649643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1.0131666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854508609, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784854508610, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784854532855, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7745516300201416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784854532855, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784854532856, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784854541340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11147874593734741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1.0165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854558234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10403944551944733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1.0198333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854574767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1106557622551918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1.0231666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854591885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10867097228765488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1.0264999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854591886, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784854591886, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784854615980, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7745243310928345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784854615981, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784854615981, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784854632669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10319972783327103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1.0298333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854649162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1002250462770462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1.0331666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854665845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10189524292945862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1.0365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854674212, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784854674213, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784854697989, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7744430303573608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784854697989, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784854697989, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784854706194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09970983862876892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1.0398333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854722727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10060033202171326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1.0431666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854739361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10245925188064575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1.0464999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854756206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10579092800617218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1.0498333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854756207, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784854756207, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784854780067, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7744647264480591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784854780068, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784854780068, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784854796225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1043790876865387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1.0531666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854813352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10999961942434311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1.0565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854829855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10613450407981873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1.0598333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854838034, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784854838034, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784854862052, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7744717001914978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784854862053, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784854862053, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784854870098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10162927210330963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1.0631666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854886651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10634391754865646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1.0664999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854902870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10059225559234619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1.0698333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854919151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10188908874988556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1.0731666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854919152, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784854919152, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784854943306, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774426281452179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784854943307, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784854943308, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784854959803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10198353976011276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1.0765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854976684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10342279076576233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1.0798333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854993003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10159864276647568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1.0831666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855001742, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784855001743, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784855025867, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7745714783668518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784855025868, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784855025868, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784855034210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1041301041841507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1.0864999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855050821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1069408431649208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1.0898333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855067244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10244019329547882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1.0931666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855083863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10018271952867508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1.0965e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855083863, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784855083863, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784855107635, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7745604515075684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784855107635, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784855107635, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784855124187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10494521260261536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1.0998333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855140664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10155381262302399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1.1031666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855157384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10200262814760208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217579520, "lr": 1.1065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855165874, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784855165875, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784855189630, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7745813131332397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784855189631, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784855189631, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784855197995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11206164956092834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218234880, "lr": 1.1098333333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855214356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10943593084812164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218890240, "lr": 1.1131666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855231200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10765033960342407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219545600, "lr": 1.1165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855248006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10822014510631561, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220200960, "lr": 1.1198333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855248006, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784855248007, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784855271896, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746951580047607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784855271897, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784855271897, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784855288561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10254458338022232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220856320, "lr": 1.1231666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855305208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10471378266811371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221511680, "lr": 1.1265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855321800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10065443068742752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222167040, "lr": 1.1298333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855330060, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784855330061, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784855353177, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774821400642395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784855353177, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784855353178, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784855361606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10505133867263794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222822400, "lr": 1.1331666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855378562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10417428612709045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223477760, "lr": 1.1365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855395196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10728173702955246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224133120, "lr": 1.1398333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855411968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1088341549038887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224788480, "lr": 1.1431666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855411969, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784855411969, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784855436081, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774836003780365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784855436082, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784855436082, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784855452396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10844895243644714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225443840, "lr": 1.1465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855469576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1088062971830368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226099200, "lr": 1.1498333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855486622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1075727567076683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226754560, "lr": 1.1531666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855495007, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784855495007, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784855518276, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7750075459480286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784855518276, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784855518277, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784855526541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11260562390089035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227409920, "lr": 1.1565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855543372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11032036691904068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228065280, "lr": 1.1598333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855560296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072482168674469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228720640, "lr": 1.1631666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855577122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10657977312803268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229376000, "lr": 1.1665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855577123, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784855577123, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784855600482, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7750773429870605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784855600483, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784855600483, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784855617509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10657744109630585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230031360, "lr": 1.1698333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855634355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10506314784288406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230686720, "lr": 1.1731666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855651555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11086967587471008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231342080, "lr": 1.1765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855659670, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784855659670, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784855684424, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7750322818756104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784855684425, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784855684425, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784855692695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082935631275177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231997440, "lr": 1.1798333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855709818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11012326925992966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 232652800, "lr": 1.1831666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855726978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11079300194978714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233308160, "lr": 1.1865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855743988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11153607070446014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233963520, "lr": 1.1898333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855743988, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784855743989, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784855767273, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7751931548118591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784855767275, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784855767275, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784855784374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11457380652427673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 234618880, "lr": 1.1931666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855801483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11345470696687698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235274240, "lr": 1.1965e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855818471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11130247265100479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235929600, "lr": 1.1998333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855827085, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784855827086, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784855850299, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7753354907035828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784855850299, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784855850300, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784855858692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10871616750955582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 236584960, "lr": 1.2031666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855875991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11301910877227783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237240320, "lr": 1.2064999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855892699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11385111510753632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237895680, "lr": 1.2098333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855910047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11856640875339508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 238551040, "lr": 1.2131666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855910048, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784855910048, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784855933217, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7753516435623169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784855933218, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784855933218, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784855950493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1177339032292366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239206400, "lr": 1.2165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855967525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11356871575117111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239861760, "lr": 1.2198333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855984617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11375580728054047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 240517120, "lr": 1.2231666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855993256, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784855993257, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784856016565, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7755023837089539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784856016565, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784856016565, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784856024953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11460378766059875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241172480, "lr": 1.2264999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856042241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11515093594789505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241827840, "lr": 1.2298333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856059497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11460176110267639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 242483200, "lr": 1.2331666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856076491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1183830127120018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243138560, "lr": 1.2365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856076491, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784856076492, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784856100877, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7757167220115662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784856100877, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784856100877, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784856118788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10938702523708344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243793920, "lr": 1.2398333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856136192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11541733145713806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 244449280, "lr": 1.2431666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856153290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11411824822425842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245104640, "lr": 1.2464999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856162029, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784856162029, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784856185351, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7757127285003662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784856185351, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784856185352, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784856193956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11063022911548615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245760000, "lr": 1.2498333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856211023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11267948150634766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 246415360, "lr": 1.2531666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856228202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792471468448639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247070720, "lr": 1.2565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856245508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1078297346830368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247726080, "lr": 1.2598333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856245509, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784856245509, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784856269346, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7758269906044006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784856269346, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784856269346, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784856286274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11377263069152832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 248381440, "lr": 1.2631666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856303590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10984829813241959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249036800, "lr": 1.2664999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856320521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10842043161392212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249692160, "lr": 1.2698333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856328957, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784856328958, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784856352296, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7759113907814026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784856352297, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784856352297, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784856360476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10623124241828918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 250347520, "lr": 1.2731666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856377566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10510675609111786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251002880, "lr": 1.2765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856394349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10458006709814072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251658240, "lr": 1.2798333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856411256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10889117419719696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 252313600, "lr": 1.2831666666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856411256, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784856411257, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784856434060, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.775875985622406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784856434060, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784856434060, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784856451107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11142009496688843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 252968960, "lr": 1.2864999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856468100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10581836104393005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 253624320, "lr": 1.2898333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856484704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793590545654297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 254279680, "lr": 1.2931666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856493368, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784856493368, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784856517830, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7759811282157898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784856517830, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784856517831, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784856526129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10699032992124557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 254935040, "lr": 1.2965e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856543075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1067790761590004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 255590400, "lr": 1.2998333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856559652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1096699982881546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 256245760, "lr": 1.3031666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856576557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10826516151428223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 256901120, "lr": 1.3065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856576557, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784856576558, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784856599515, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7761326432228088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784856599515, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784856599516, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784856616155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1068604588508606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 257556480, "lr": 1.3098333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856632849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10528860986232758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 258211840, "lr": 1.3131666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856649956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10421723872423172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 258867200, "lr": 1.3165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856658494, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784856658495, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784856681310, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7760447263717651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784856681310, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784856681310, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784856689833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11185679584741592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 259522560, "lr": 1.3198333333333335e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856706709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10634652525186539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 260177920, "lr": 1.3231666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856723862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10563578456640244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 260833280, "lr": 1.3265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856741041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10572417825460434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 261488640, "lr": 1.3298333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856741041, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784856741042, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784856765300, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7761743068695068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784856765301, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784856765301, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784856782439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10310821235179901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 262144000, "lr": 1.3331666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856799390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10112926363945007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 262799360, "lr": 1.3365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856816364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10504800081253052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 263454720, "lr": 1.3398333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856824829, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 263782400, "epoch_num": 0.11514681384425231}} +:::MLLOG {"namespace": "", "time_ms": 1784856824830, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 263782400, "epoch_num": 0.11514681384425231}} +:::MLLOG {"namespace": "", "time_ms": 1784856848631, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7761879563331604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 263782400, "epoch_num": 0.11514681384425231}} +:::MLLOG {"namespace": "", "time_ms": 1784856848632, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 263782400, "epoch_num": 0.11514681384425231}} +:::MLLOG {"namespace": "", "time_ms": 1784856848632, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 263782400, "epoch_num": 0.11514681384425231}} +:::MLLOG {"namespace": "", "time_ms": 1784856857267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10944179445505142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 264110080, "lr": 1.3431666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856874557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1078421100974083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 264765440, "lr": 1.3465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856891721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10461512207984924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 265420800, "lr": 1.3498333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856908837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1037779226899147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 266076160, "lr": 1.3531666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856908838, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 266076160, "epoch_num": 0.11614809048637624}} +:::MLLOG {"namespace": "", "time_ms": 1784856908838, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 266076160, "epoch_num": 0.11614809048637624}} +:::MLLOG {"namespace": "", "time_ms": 1784856932411, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7764398455619812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 266076160, "epoch_num": 0.11614809048637624}} +:::MLLOG {"namespace": "", "time_ms": 1784856932411, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 266076160, "epoch_num": 0.11614809048637624}} +:::MLLOG {"namespace": "", "time_ms": 1784856932411, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 266076160, "epoch_num": 0.11614809048637624}} +:::MLLOG {"namespace": "", "time_ms": 1784856948889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10402368009090424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 266731520, "lr": 1.3565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856966405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10709254443645477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 267386880, "lr": 1.359833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856983240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10881530493497849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 268042240, "lr": 1.3631666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856991808, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 268369920, "epoch_num": 0.11714936712850017}} +:::MLLOG {"namespace": "", "time_ms": 1784856991808, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 268369920, "epoch_num": 0.11714936712850017}} +:::MLLOG {"namespace": "", "time_ms": 1784857014975, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7763392925262451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 268369920, "epoch_num": 0.11714936712850017}} +:::MLLOG {"namespace": "", "time_ms": 1784857014976, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 268369920, "epoch_num": 0.11714936712850017}} +:::MLLOG {"namespace": "", "time_ms": 1784857014976, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 268369920, "epoch_num": 0.11714936712850017}} +:::MLLOG {"namespace": "", "time_ms": 1784857023212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11114377528429031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 268697600, "lr": 1.3665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857040188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10547736287117004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 269352960, "lr": 1.3698333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857056978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10373465716838837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 270008320, "lr": 1.3731666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857073813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1096193790435791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 270663680, "lr": 1.3765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857073814, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 270663680, "epoch_num": 0.1181506437706241}} +:::MLLOG {"namespace": "", "time_ms": 1784857073814, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 270663680, "epoch_num": 0.1181506437706241}} +:::MLLOG {"namespace": "", "time_ms": 1784857097143, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7765759825706482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 270663680, "epoch_num": 0.1181506437706241}} +:::MLLOG {"namespace": "", "time_ms": 1784857097143, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 270663680, "epoch_num": 0.1181506437706241}} +:::MLLOG {"namespace": "", "time_ms": 1784857097143, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 270663680, "epoch_num": 0.1181506437706241}} +:::MLLOG {"namespace": "", "time_ms": 1784857114217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11077713966369629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 271319040, "lr": 1.379833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857131371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11123096942901611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 271974400, "lr": 1.3831666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857148193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1090632826089859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 272629760, "lr": 1.3865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857156620, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 272957440, "epoch_num": 0.11915192041274804}} +:::MLLOG {"namespace": "", "time_ms": 1784857156621, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 272957440, "epoch_num": 0.11915192041274804}} +:::MLLOG {"namespace": "", "time_ms": 1784857180898, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7767032980918884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 272957440, "epoch_num": 0.11915192041274804}} +:::MLLOG {"namespace": "", "time_ms": 1784857180898, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 272957440, "epoch_num": 0.11915192041274804}} +:::MLLOG {"namespace": "", "time_ms": 1784857180898, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 272957440, "epoch_num": 0.11915192041274804}} +:::MLLOG {"namespace": "", "time_ms": 1784857189329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11178439855575562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 273285120, "lr": 1.3898333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857207071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11095317453145981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 273940480, "lr": 1.3931666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857224047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11810500174760818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 274595840, "lr": 1.3965e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857240748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11609608680009842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 275251200, "lr": 1.3998333333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857240749, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 275251200, "epoch_num": 0.12015319705487197}} +:::MLLOG {"namespace": "", "time_ms": 1784857240749, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 275251200, "epoch_num": 0.12015319705487197}} +:::MLLOG {"namespace": "", "time_ms": 1784857264668, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7768868803977966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 275251200, "epoch_num": 0.12015319705487197}} +:::MLLOG {"namespace": "", "time_ms": 1784857264668, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 275251200, "epoch_num": 0.12015319705487197}} +:::MLLOG {"namespace": "", "time_ms": 1784857264669, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 275251200, "epoch_num": 0.12015319705487197}} +:::MLLOG {"namespace": "", "time_ms": 1784857281722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11160030961036682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 275906560, "lr": 1.4031666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857298755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11024048924446106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 276561920, "lr": 1.4065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857316102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11898262798786163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 277217280, "lr": 1.4098333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857324692, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 277544960, "epoch_num": 0.1211544736969959}} +:::MLLOG {"namespace": "", "time_ms": 1784857324693, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 277544960, "epoch_num": 0.1211544736969959}} +:::MLLOG {"namespace": "", "time_ms": 1784857348885, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7770905494689941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 277544960, "epoch_num": 0.1211544736969959}} +:::MLLOG {"namespace": "", "time_ms": 1784857348886, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 277544960, "epoch_num": 0.1211544736969959}} +:::MLLOG {"namespace": "", "time_ms": 1784857348886, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 277544960, "epoch_num": 0.1211544736969959}} +:::MLLOG {"namespace": "", "time_ms": 1784857357600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11310559511184692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 277872640, "lr": 1.4131666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857374561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10993674397468567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 278528000, "lr": 1.4165000000000001e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857392329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11769270151853561, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 279183360, "lr": 1.4198333333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857409548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10982421785593033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 279838720, "lr": 1.4231666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857409549, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 279838720, "epoch_num": 0.12215575033911984}} +:::MLLOG {"namespace": "", "time_ms": 1784857409549, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 279838720, "epoch_num": 0.12215575033911984}} +:::MLLOG {"namespace": "", "time_ms": 1784857433094, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7771967053413391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 279838720, "epoch_num": 0.12215575033911984}} +:::MLLOG {"namespace": "", "time_ms": 1784857433095, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 279838720, "epoch_num": 0.12215575033911984}} +:::MLLOG {"namespace": "", "time_ms": 1784857433095, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 279838720, "epoch_num": 0.12215575033911984}} +:::MLLOG {"namespace": "", "time_ms": 1784857449790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11112210899591446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 280494080, "lr": 1.4265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857466819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11192137748003006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 281149440, "lr": 1.4298333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857483840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11591127514839172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 281804800, "lr": 1.4331666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857492359, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 282132480, "epoch_num": 0.12315702698124378}} +:::MLLOG {"namespace": "", "time_ms": 1784857492359, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 282132480, "epoch_num": 0.12315702698124378}} +:::MLLOG {"namespace": "", "time_ms": 1784857515916, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7772505283355713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 282132480, "epoch_num": 0.12315702698124378}} +:::MLLOG {"namespace": "", "time_ms": 1784857515916, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 282132480, "epoch_num": 0.12315702698124378}} +:::MLLOG {"namespace": "", "time_ms": 1784857515916, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 282132480, "epoch_num": 0.12315702698124378}} +:::MLLOG {"namespace": "", "time_ms": 1784857524709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725043714046478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 282460160, "lr": 1.4365000000000001e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857542046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11376588046550751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 283115520, "lr": 1.4398333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857559281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11152923107147217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 283770880, "lr": 1.4431666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857576467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1113654226064682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 284426240, "lr": 1.4464999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857576468, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 284426240, "epoch_num": 0.12415830362336772}} +:::MLLOG {"namespace": "", "time_ms": 1784857576468, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 284426240, "epoch_num": 0.12415830362336772}} +:::MLLOG {"namespace": "", "time_ms": 1784857600384, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7772983908653259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 284426240, "epoch_num": 0.12415830362336772}} +:::MLLOG {"namespace": "", "time_ms": 1784857600384, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 284426240, "epoch_num": 0.12415830362336772}} +:::MLLOG {"namespace": "", "time_ms": 1784857600384, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 284426240, "epoch_num": 0.12415830362336772}} +:::MLLOG {"namespace": "", "time_ms": 1784857617781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10871952772140503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 285081600, "lr": 1.4498333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857634851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11355166137218475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 285736960, "lr": 1.4531666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857651834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11432386189699173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 286392320, "lr": 1.4564999999999997e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857660362, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 286720000, "epoch_num": 0.12515958026549165}} +:::MLLOG {"namespace": "", "time_ms": 1784857660363, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 286720000, "epoch_num": 0.12515958026549165}} +:::MLLOG {"namespace": "", "time_ms": 1784857683214, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7772395014762878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 286720000, "epoch_num": 0.12515958026549165}} +:::MLLOG {"namespace": "", "time_ms": 1784857683214, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 286720000, "epoch_num": 0.12515958026549165}} +:::MLLOG {"namespace": "", "time_ms": 1784857683214, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 286720000, "epoch_num": 0.12515958026549165}} +:::MLLOG {"namespace": "", "time_ms": 1784857691539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11481152474880219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 287047680, "lr": 1.4598333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857708558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11103422194719315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 287703040, "lr": 1.4631666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857725912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1131301298737526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 288358400, "lr": 1.4664999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857742861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11430374532938004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 289013760, "lr": 1.4698333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857742862, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 289013760, "epoch_num": 0.12616085690761558}} +:::MLLOG {"namespace": "", "time_ms": 1784857742862, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 289013760, "epoch_num": 0.12616085690761558}} +:::MLLOG {"namespace": "", "time_ms": 1784857766671, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7774875164031982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 289013760, "epoch_num": 0.12616085690761558}} +:::MLLOG {"namespace": "", "time_ms": 1784857766672, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 289013760, "epoch_num": 0.12616085690761558}} +:::MLLOG {"namespace": "", "time_ms": 1784857766672, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 289013760, "epoch_num": 0.12616085690761558}} +:::MLLOG {"namespace": "", "time_ms": 1784857783534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10769511014223099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 289669120, "lr": 1.4731666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857800407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10571912676095963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 290324480, "lr": 1.4764999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857817977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10120947659015656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 290979840, "lr": 1.4798333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857826444, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 291307520, "epoch_num": 0.12716213354973951}} +:::MLLOG {"namespace": "", "time_ms": 1784857826445, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 291307520, "epoch_num": 0.12716213354973951}} +:::MLLOG {"namespace": "", "time_ms": 1784857849954, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7775743007659912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 291307520, "epoch_num": 0.12716213354973951}} +:::MLLOG {"namespace": "", "time_ms": 1784857849955, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 291307520, "epoch_num": 0.12716213354973951}} +:::MLLOG {"namespace": "", "time_ms": 1784857849955, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 291307520, "epoch_num": 0.12716213354973951}} +:::MLLOG {"namespace": "", "time_ms": 1784857858493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1059933453798294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 291635200, "lr": 1.4831666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857875348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10906000435352325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 292290560, "lr": 1.4864999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857892307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10506992042064667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 292945920, "lr": 1.4898333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857908953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10564552992582321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 293601280, "lr": 1.4931666666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857908954, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 293601280, "epoch_num": 0.12816341019186345}} +:::MLLOG {"namespace": "", "time_ms": 1784857908954, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 293601280, "epoch_num": 0.12816341019186345}} +:::MLLOG {"namespace": "", "time_ms": 1784857932578, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7772654294967651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 293601280, "epoch_num": 0.12816341019186345}} +:::MLLOG {"namespace": "", "time_ms": 1784857932578, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 293601280, "epoch_num": 0.12816341019186345}} +:::MLLOG {"namespace": "", "time_ms": 1784857932579, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 293601280, "epoch_num": 0.12816341019186345}} +:::MLLOG {"namespace": "", "time_ms": 1784857949101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10821204632520676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 294256640, "lr": 1.4964999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857966130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10389973223209381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 294912000, "lr": 1.4998333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857983418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10783262550830841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 295567360, "lr": 1.5031666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857992090, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 295895040, "epoch_num": 0.12916468683398738}} +:::MLLOG {"namespace": "", "time_ms": 1784857992090, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 295895040, "epoch_num": 0.12916468683398738}} +:::MLLOG {"namespace": "", "time_ms": 1784858015707, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7773218154907227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 295895040, "epoch_num": 0.12916468683398738}} +:::MLLOG {"namespace": "", "time_ms": 1784858015708, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 295895040, "epoch_num": 0.12916468683398738}} +:::MLLOG {"namespace": "", "time_ms": 1784858015708, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 295895040, "epoch_num": 0.12916468683398738}} +:::MLLOG {"namespace": "", "time_ms": 1784858024177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10413098335266113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 296222720, "lr": 1.5064999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858040743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10504911094903946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 296878080, "lr": 1.5098333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858058003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10865291953086853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 297533440, "lr": 1.5131666666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858074918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10949075222015381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 298188800, "lr": 1.5164999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858074919, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 298188800, "epoch_num": 0.1301659634761113}} +:::MLLOG {"namespace": "", "time_ms": 1784858074919, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 298188800, "epoch_num": 0.1301659634761113}} +:::MLLOG {"namespace": "", "time_ms": 1784858098415, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7773584723472595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 298188800, "epoch_num": 0.1301659634761113}} +:::MLLOG {"namespace": "", "time_ms": 1784858098415, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 298188800, "epoch_num": 0.1301659634761113}} +:::MLLOG {"namespace": "", "time_ms": 1784858098415, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 298188800, "epoch_num": 0.1301659634761113}} +:::MLLOG {"namespace": "", "time_ms": 1784858115297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10498364269733429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 298844160, "lr": 1.5198333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858132282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10402709245681763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 299499520, "lr": 1.5231666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858149233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10379621386528015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 300154880, "lr": 1.5265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858157865, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 300482560, "epoch_num": 0.13116724011823525}} +:::MLLOG {"namespace": "", "time_ms": 1784858157865, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 300482560, "epoch_num": 0.13116724011823525}} +:::MLLOG {"namespace": "", "time_ms": 1784858181881, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7772647738456726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 300482560, "epoch_num": 0.13116724011823525}} +:::MLLOG {"namespace": "", "time_ms": 1784858181882, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 300482560, "epoch_num": 0.13116724011823525}} +:::MLLOG {"namespace": "", "time_ms": 1784858181882, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 300482560, "epoch_num": 0.13116724011823525}} +:::MLLOG {"namespace": "", "time_ms": 1784858190426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11057762056589127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 300810240, "lr": 1.5298333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858207458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10012814402580261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 301465600, "lr": 1.5331666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858224585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10167626291513443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 302120960, "lr": 1.5364999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858242076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10736967623233795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 302776320, "lr": 1.5398333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858242077, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 302776320, "epoch_num": 0.13216851676035918}} +:::MLLOG {"namespace": "", "time_ms": 1784858242077, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 302776320, "epoch_num": 0.13216851676035918}} +:::MLLOG {"namespace": "", "time_ms": 1784858266414, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7772900462150574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 302776320, "epoch_num": 0.13216851676035918}} +:::MLLOG {"namespace": "", "time_ms": 1784858266414, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 302776320, "epoch_num": 0.13216851676035918}} +:::MLLOG {"namespace": "", "time_ms": 1784858266414, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 302776320, "epoch_num": 0.13216851676035918}} +:::MLLOG {"namespace": "", "time_ms": 1784858283082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11007805168628693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 303431680, "lr": 1.5431666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858300639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10813884437084198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 304087040, "lr": 1.5465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858317878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10412456840276718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 304742400, "lr": 1.5498333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858326481, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 305070080, "epoch_num": 0.1331697934024831}} +:::MLLOG {"namespace": "", "time_ms": 1784858326482, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 305070080, "epoch_num": 0.1331697934024831}} +:::MLLOG {"namespace": "", "time_ms": 1784858350385, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7775171399116516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 305070080, "epoch_num": 0.1331697934024831}} +:::MLLOG {"namespace": "", "time_ms": 1784858350385, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 305070080, "epoch_num": 0.1331697934024831}} +:::MLLOG {"namespace": "", "time_ms": 1784858350385, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 305070080, "epoch_num": 0.1331697934024831}} +:::MLLOG {"namespace": "", "time_ms": 1784858358904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10254718363285065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 305397760, "lr": 1.5531666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858375724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10770008713006973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 306053120, "lr": 1.5564999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858392576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10551701486110687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 306708480, "lr": 1.5598333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858409300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10581379383802414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 307363840, "lr": 1.5631666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858409301, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 307363840, "epoch_num": 0.13417107004460704}} +:::MLLOG {"namespace": "", "time_ms": 1784858409301, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 307363840, "epoch_num": 0.13417107004460704}} +:::MLLOG {"namespace": "", "time_ms": 1784858432589, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7774600386619568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 307363840, "epoch_num": 0.13417107004460704}} +:::MLLOG {"namespace": "", "time_ms": 1784858432589, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 307363840, "epoch_num": 0.13417107004460704}} +:::MLLOG {"namespace": "", "time_ms": 1784858432589, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 307363840, "epoch_num": 0.13417107004460704}} +:::MLLOG {"namespace": "", "time_ms": 1784858449508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10691876709461212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 308019200, "lr": 1.5665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858466570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09893545508384705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 308674560, "lr": 1.5698333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858483568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10470032691955566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 309329920, "lr": 1.5731666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858492174, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 309657600, "epoch_num": 0.13517234668673098}} +:::MLLOG {"namespace": "", "time_ms": 1784858492175, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 309657600, "epoch_num": 0.13517234668673098}} +:::MLLOG {"namespace": "", "time_ms": 1784858515945, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7774308323860168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 309657600, "epoch_num": 0.13517234668673098}} +:::MLLOG {"namespace": "", "time_ms": 1784858515946, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 309657600, "epoch_num": 0.13517234668673098}} +:::MLLOG {"namespace": "", "time_ms": 1784858515946, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 309657600, "epoch_num": 0.13517234668673098}} +:::MLLOG {"namespace": "", "time_ms": 1784858524589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10804032534360886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 309985280, "lr": 1.5764999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858541216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10250061750411987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 310640640, "lr": 1.5798333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858558925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10975907742977142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 311296000, "lr": 1.5831666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858576172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1083860769867897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 311951360, "lr": 1.5865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858576172, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 311951360, "epoch_num": 0.1361736233288549}} +:::MLLOG {"namespace": "", "time_ms": 1784858576173, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 311951360, "epoch_num": 0.1361736233288549}} +:::MLLOG {"namespace": "", "time_ms": 1784858599290, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7775616645812988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 311951360, "epoch_num": 0.1361736233288549}} +:::MLLOG {"namespace": "", "time_ms": 1784858599290, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 311951360, "epoch_num": 0.1361736233288549}} +:::MLLOG {"namespace": "", "time_ms": 1784858599290, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 311951360, "epoch_num": 0.1361736233288549}} +:::MLLOG {"namespace": "", "time_ms": 1784858616318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10665212571620941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 312606720, "lr": 1.5898333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858633100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10696311295032501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 313262080, "lr": 1.5931666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858650184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1054968610405922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 313917440, "lr": 1.5964999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858658601, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 314245120, "epoch_num": 0.13717489997097884}} +:::MLLOG {"namespace": "", "time_ms": 1784858658601, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 314245120, "epoch_num": 0.13717489997097884}} +:::MLLOG {"namespace": "", "time_ms": 1784858681556, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7778539657592773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 314245120, "epoch_num": 0.13717489997097884}} +:::MLLOG {"namespace": "", "time_ms": 1784858681557, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 314245120, "epoch_num": 0.13717489997097884}} +:::MLLOG {"namespace": "", "time_ms": 1784858681557, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 314245120, "epoch_num": 0.13717489997097884}} +:::MLLOG {"namespace": "", "time_ms": 1784858690034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10927486419677734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 314572800, "lr": 1.5998333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858707046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1045970544219017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 315228160, "lr": 1.6031666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858724261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10377046465873718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 315883520, "lr": 1.6065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858741244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10280053317546844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 316538880, "lr": 1.6098333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858741244, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 316538880, "epoch_num": 0.13817617661310277}} +:::MLLOG {"namespace": "", "time_ms": 1784858741245, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 316538880, "epoch_num": 0.13817617661310277}} +:::MLLOG {"namespace": "", "time_ms": 1784858765131, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7778280377388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 316538880, "epoch_num": 0.13817617661310277}} +:::MLLOG {"namespace": "", "time_ms": 1784858765132, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 316538880, "epoch_num": 0.13817617661310277}} +:::MLLOG {"namespace": "", "time_ms": 1784858765132, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 316538880, "epoch_num": 0.13817617661310277}} +:::MLLOG {"namespace": "", "time_ms": 1784858782525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10808867961168289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 317194240, "lr": 1.6131666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858799362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10975947976112366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 317849600, "lr": 1.6165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858816654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11359553039073944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 318504960, "lr": 1.6198333333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858825357, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 318832640, "epoch_num": 0.1391774532552267}} +:::MLLOG {"namespace": "", "time_ms": 1784858825358, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 318832640, "epoch_num": 0.1391774532552267}} +:::MLLOG {"namespace": "", "time_ms": 1784858848916, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7779173851013184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 318832640, "epoch_num": 0.1391774532552267}} +:::MLLOG {"namespace": "", "time_ms": 1784858848916, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 318832640, "epoch_num": 0.1391774532552267}} +:::MLLOG {"namespace": "", "time_ms": 1784858848916, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 318832640, "epoch_num": 0.1391774532552267}} +:::MLLOG {"namespace": "", "time_ms": 1784858857681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11554653197526932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 319160320, "lr": 1.6231666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858874814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10502948611974716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 319815680, "lr": 1.6265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858892024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11336571723222733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 320471040, "lr": 1.6298333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858909252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1078476682305336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 321126400, "lr": 1.6331666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858909252, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 321126400, "epoch_num": 0.14017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784858909253, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 321126400, "epoch_num": 0.14017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784858933129, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7781810760498047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 321126400, "epoch_num": 0.14017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784858933129, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 321126400, "epoch_num": 0.14017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784858933129, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 321126400, "epoch_num": 0.14017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784858950196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11103670299053192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 321781760, "lr": 1.6365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858967346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10964231193065643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 322437120, "lr": 1.6398333333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858984642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1133769154548645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 323092480, "lr": 1.6431666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858993274, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 323420160, "epoch_num": 0.14118000653947457}} +:::MLLOG {"namespace": "", "time_ms": 1784858993274, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 323420160, "epoch_num": 0.14118000653947457}} +:::MLLOG {"namespace": "", "time_ms": 1784859016659, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7783515453338623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 323420160, "epoch_num": 0.14118000653947457}} +:::MLLOG {"namespace": "", "time_ms": 1784859016660, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 323420160, "epoch_num": 0.14118000653947457}} +:::MLLOG {"namespace": "", "time_ms": 1784859016660, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 323420160, "epoch_num": 0.14118000653947457}} +:::MLLOG {"namespace": "", "time_ms": 1784859025368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10990244150161743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 323747840, "lr": 1.6465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859042782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10531838983297348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 324403200, "lr": 1.6498333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859059639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10677000880241394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 325058560, "lr": 1.6531666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859077643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11453349888324738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 325713920, "lr": 1.6565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859077644, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 325713920, "epoch_num": 0.1421812831815985}} +:::MLLOG {"namespace": "", "time_ms": 1784859077644, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 325713920, "epoch_num": 0.1421812831815985}} +:::MLLOG {"namespace": "", "time_ms": 1784859100757, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7784065008163452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 325713920, "epoch_num": 0.1421812831815985}} +:::MLLOG {"namespace": "", "time_ms": 1784859100757, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 325713920, "epoch_num": 0.1421812831815985}} +:::MLLOG {"namespace": "", "time_ms": 1784859100757, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 325713920, "epoch_num": 0.1421812831815985}} +:::MLLOG {"namespace": "", "time_ms": 1784859118167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1101144403219223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 326369280, "lr": 1.6598333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859135560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11272601783275604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327024640, "lr": 1.6631666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859153174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1086471825838089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680000, "lr": 1.6665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859161855, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 328007680, "epoch_num": 0.14318255982372244}} +:::MLLOG {"namespace": "", "time_ms": 1784859161856, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 328007680, "epoch_num": 0.14318255982372244}} +:::MLLOG {"namespace": "", "time_ms": 1784859186683, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7786264419555664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 328007680, "epoch_num": 0.14318255982372244}} +:::MLLOG {"namespace": "", "time_ms": 1784859186684, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 328007680, "epoch_num": 0.14318255982372244}} +:::MLLOG {"namespace": "", "time_ms": 1784859186684, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 328007680, "epoch_num": 0.14318255982372244}} +:::MLLOG {"namespace": "", "time_ms": 1784859195119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11473233997821808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 328335360, "lr": 1.6698333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859212532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10861527919769287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 328990720, "lr": 1.6731666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859229848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11374745517969131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 329646080, "lr": 1.6765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859247322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10903310030698776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 330301440, "lr": 1.6798333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859247323, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 330301440, "epoch_num": 0.14418383646584637}} +:::MLLOG {"namespace": "", "time_ms": 1784859247324, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 330301440, "epoch_num": 0.14418383646584637}} +:::MLLOG {"namespace": "", "time_ms": 1784859270985, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7786377668380737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 330301440, "epoch_num": 0.14418383646584637}} +:::MLLOG {"namespace": "", "time_ms": 1784859270985, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 330301440, "epoch_num": 0.14418383646584637}} +:::MLLOG {"namespace": "", "time_ms": 1784859270985, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 330301440, "epoch_num": 0.14418383646584637}} +:::MLLOG {"namespace": "", "time_ms": 1784859288037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11542695760726929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 330956800, "lr": 1.6831666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859305751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11274464428424835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 331612160, "lr": 1.6865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859322844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10371083766222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 332267520, "lr": 1.6898333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859331319, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 332595200, "epoch_num": 0.1451851131079703}} +:::MLLOG {"namespace": "", "time_ms": 1784859331320, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 332595200, "epoch_num": 0.1451851131079703}} +:::MLLOG {"namespace": "", "time_ms": 1784859353979, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7785068154335022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 332595200, "epoch_num": 0.1451851131079703}} +:::MLLOG {"namespace": "", "time_ms": 1784859353979, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 332595200, "epoch_num": 0.1451851131079703}} +:::MLLOG {"namespace": "", "time_ms": 1784859353979, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 332595200, "epoch_num": 0.1451851131079703}} +:::MLLOG {"namespace": "", "time_ms": 1784859362535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10764964669942856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 332922880, "lr": 1.6931666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859379349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10453618317842484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 333578240, "lr": 1.6964999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859396587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10841890424489975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 334233600, "lr": 1.6998333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859413635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10225097090005875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 334888960, "lr": 1.7031666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859413635, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 334888960, "epoch_num": 0.14618638975009424}} +:::MLLOG {"namespace": "", "time_ms": 1784859413636, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 334888960, "epoch_num": 0.14618638975009424}} +:::MLLOG {"namespace": "", "time_ms": 1784859436364, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7784369587898254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 334888960, "epoch_num": 0.14618638975009424}} +:::MLLOG {"namespace": "", "time_ms": 1784859436364, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 334888960, "epoch_num": 0.14618638975009424}} +:::MLLOG {"namespace": "", "time_ms": 1784859436364, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 334888960, "epoch_num": 0.14618638975009424}} +:::MLLOG {"namespace": "", "time_ms": 1784859453388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10612834244966507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 335544320, "lr": 1.7064999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859470796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10674174129962921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 336199680, "lr": 1.7098333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859488040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10928407311439514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 336855040, "lr": 1.7131666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859496771, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 337182720, "epoch_num": 0.14718766639221817}} +:::MLLOG {"namespace": "", "time_ms": 1784859496772, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 337182720, "epoch_num": 0.14718766639221817}} +:::MLLOG {"namespace": "", "time_ms": 1784859520048, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7783263921737671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 337182720, "epoch_num": 0.14718766639221817}} +:::MLLOG {"namespace": "", "time_ms": 1784859520049, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 337182720, "epoch_num": 0.14718766639221817}} +:::MLLOG {"namespace": "", "time_ms": 1784859520049, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 337182720, "epoch_num": 0.14718766639221817}} +:::MLLOG {"namespace": "", "time_ms": 1784859528857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10081770271062851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 337510400, "lr": 1.7164999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859545742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10237932950258255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 338165760, "lr": 1.7198333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859563545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10786841064691544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 338821120, "lr": 1.7231666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859580830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10310006886720657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 339476480, "lr": 1.7265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859580831, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 339476480, "epoch_num": 0.1481889430343421}} +:::MLLOG {"namespace": "", "time_ms": 1784859580832, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 339476480, "epoch_num": 0.1481889430343421}} +:::MLLOG {"namespace": "", "time_ms": 1784859604583, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7784701585769653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 339476480, "epoch_num": 0.1481889430343421}} +:::MLLOG {"namespace": "", "time_ms": 1784859604583, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 339476480, "epoch_num": 0.1481889430343421}} +:::MLLOG {"namespace": "", "time_ms": 1784859604584, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 339476480, "epoch_num": 0.1481889430343421}} +:::MLLOG {"namespace": "", "time_ms": 1784859621784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10250856727361679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 340131840, "lr": 1.7298333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859638863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.104228176176548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 340787200, "lr": 1.7331666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859656105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10621675103902817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 341442560, "lr": 1.7364999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859664736, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 341770240, "epoch_num": 0.14919021967646603}} +:::MLLOG {"namespace": "", "time_ms": 1784859664736, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 341770240, "epoch_num": 0.14919021967646603}} +:::MLLOG {"namespace": "", "time_ms": 1784859688268, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7786815762519836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 341770240, "epoch_num": 0.14919021967646603}} +:::MLLOG {"namespace": "", "time_ms": 1784859688269, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 341770240, "epoch_num": 0.14919021967646603}} +:::MLLOG {"namespace": "", "time_ms": 1784859688269, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 341770240, "epoch_num": 0.14919021967646603}} +:::MLLOG {"namespace": "", "time_ms": 1784859696877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10379502177238464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 342097920, "lr": 1.7398333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859713748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1019432544708252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 342753280, "lr": 1.7431666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859731092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10690438747406006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 343408640, "lr": 1.7465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859748633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10899271816015244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 344064000, "lr": 1.7498333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859748634, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 344064000, "epoch_num": 0.15019149631858997}} +:::MLLOG {"namespace": "", "time_ms": 1784859748634, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 344064000, "epoch_num": 0.15019149631858997}} +:::MLLOG {"namespace": "", "time_ms": 1784859772450, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7785674333572388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 344064000, "epoch_num": 0.15019149631858997}} +:::MLLOG {"namespace": "", "time_ms": 1784859772450, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 344064000, "epoch_num": 0.15019149631858997}} +:::MLLOG {"namespace": "", "time_ms": 1784859772450, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 344064000, "epoch_num": 0.15019149631858997}} +:::MLLOG {"namespace": "", "time_ms": 1784859789966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10620206594467163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 344719360, "lr": 1.7531666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859807031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10851576924324036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 345374720, "lr": 1.7564999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859824159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10110019147396088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 346030080, "lr": 1.7598333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859833382, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 346357760, "epoch_num": 0.1511927729607139}} +:::MLLOG {"namespace": "", "time_ms": 1784859833383, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 346357760, "epoch_num": 0.1511927729607139}} +:::MLLOG {"namespace": "", "time_ms": 1784859857104, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7786456942558289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 346357760, "epoch_num": 0.1511927729607139}} +:::MLLOG {"namespace": "", "time_ms": 1784859857104, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 346357760, "epoch_num": 0.1511927729607139}} +:::MLLOG {"namespace": "", "time_ms": 1784859857104, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 346357760, "epoch_num": 0.1511927729607139}} +:::MLLOG {"namespace": "", "time_ms": 1784859865861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10579589009284973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 346685440, "lr": 1.7631666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859883386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10349348932504654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 347340800, "lr": 1.7665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859900653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10765940696001053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 347996160, "lr": 1.7698333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859917657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10155075043439865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 348651520, "lr": 1.7731666666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859917726, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 348651520, "epoch_num": 0.15219404960283783}} +:::MLLOG {"namespace": "", "time_ms": 1784859917726, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 348651520, "epoch_num": 0.15219404960283783}} +:::MLLOG {"namespace": "", "time_ms": 1784859941931, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7787408828735352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 348651520, "epoch_num": 0.15219404960283783}} +:::MLLOG {"namespace": "", "time_ms": 1784859941931, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 348651520, "epoch_num": 0.15219404960283783}} +:::MLLOG {"namespace": "", "time_ms": 1784859941931, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 348651520, "epoch_num": 0.15219404960283783}} +:::MLLOG {"namespace": "", "time_ms": 1784859959026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10983335971832275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 349306880, "lr": 1.7764999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859976134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10794498026371002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 349962240, "lr": 1.7798333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859993097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10472961515188217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 350617600, "lr": 1.7831666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860001626, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 350945280, "epoch_num": 0.15319532624496177}} +:::MLLOG {"namespace": "", "time_ms": 1784860001627, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 350945280, "epoch_num": 0.15319532624496177}} +:::MLLOG {"namespace": "", "time_ms": 1784860025253, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7787332534790039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 350945280, "epoch_num": 0.15319532624496177}} +:::MLLOG {"namespace": "", "time_ms": 1784860025254, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 350945280, "epoch_num": 0.15319532624496177}} +:::MLLOG {"namespace": "", "time_ms": 1784860025254, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 350945280, "epoch_num": 0.15319532624496177}} +:::MLLOG {"namespace": "", "time_ms": 1784860033753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10525750368833542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 351272960, "lr": 1.7865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860050861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10128888487815857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 351928320, "lr": 1.7898333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860068095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1012791320681572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 352583680, "lr": 1.7931666666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860084613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10542941838502884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 353239040, "lr": 1.7964999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860084683, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 353239040, "epoch_num": 0.1541966028870857}} +:::MLLOG {"namespace": "", "time_ms": 1784860084684, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 353239040, "epoch_num": 0.1541966028870857}} +:::MLLOG {"namespace": "", "time_ms": 1784860108448, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7789169549942017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 353239040, "epoch_num": 0.1541966028870857}} +:::MLLOG {"namespace": "", "time_ms": 1784860108448, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 353239040, "epoch_num": 0.1541966028870857}} +:::MLLOG {"namespace": "", "time_ms": 1784860108448, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 353239040, "epoch_num": 0.1541966028870857}} +:::MLLOG {"namespace": "", "time_ms": 1784860126344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10735345631837845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 353894400, "lr": 1.7998333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860144004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10624371469020844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 354549760, "lr": 1.8031666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860161202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10677920281887054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 355205120, "lr": 1.8065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860169684, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 355532800, "epoch_num": 0.15519787952920963}} +:::MLLOG {"namespace": "", "time_ms": 1784860169685, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 355532800, "epoch_num": 0.15519787952920963}} +:::MLLOG {"namespace": "", "time_ms": 1784860193680, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7791123986244202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 355532800, "epoch_num": 0.15519787952920963}} +:::MLLOG {"namespace": "", "time_ms": 1784860193680, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 355532800, "epoch_num": 0.15519787952920963}} +:::MLLOG {"namespace": "", "time_ms": 1784860193681, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 355532800, "epoch_num": 0.15519787952920963}} +:::MLLOG {"namespace": "", "time_ms": 1784860202039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10379913449287415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 355860480, "lr": 1.8098333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860219294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10220405459403992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 356515840, "lr": 1.8131666666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860236662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10308822244405746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 357171200, "lr": 1.8164999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860253815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10846558213233948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 357826560, "lr": 1.8198333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860253815, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 357826560, "epoch_num": 0.15619915617133356}} +:::MLLOG {"namespace": "", "time_ms": 1784860253816, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 357826560, "epoch_num": 0.15619915617133356}} +:::MLLOG {"namespace": "", "time_ms": 1784860277647, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7791405320167542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 357826560, "epoch_num": 0.15619915617133356}} +:::MLLOG {"namespace": "", "time_ms": 1784860277647, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 357826560, "epoch_num": 0.15619915617133356}} +:::MLLOG {"namespace": "", "time_ms": 1784860277647, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 357826560, "epoch_num": 0.15619915617133356}} +:::MLLOG {"namespace": "", "time_ms": 1784860294984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10192917287349701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 358481920, "lr": 1.8231666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860312319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10290251672267914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 359137280, "lr": 1.8265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860329846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10347205400466919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 359792640, "lr": 1.8298333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860338541, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 360120320, "epoch_num": 0.1572004328134575}} +:::MLLOG {"namespace": "", "time_ms": 1784860338542, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 360120320, "epoch_num": 0.1572004328134575}} +:::MLLOG {"namespace": "", "time_ms": 1784860362403, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7791821956634521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 360120320, "epoch_num": 0.1572004328134575}} +:::MLLOG {"namespace": "", "time_ms": 1784860362403, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 360120320, "epoch_num": 0.1572004328134575}} +:::MLLOG {"namespace": "", "time_ms": 1784860362403, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 360120320, "epoch_num": 0.1572004328134575}} +:::MLLOG {"namespace": "", "time_ms": 1784860370674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10623236745595932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 360448000, "lr": 1.8331666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860388402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10879791527986526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 361103360, "lr": 1.8365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860405942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10917603224515915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 361758720, "lr": 1.8398333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860423196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.110903799533844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 362414080, "lr": 1.8431666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860423196, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 362414080, "epoch_num": 0.15820170945558143}} +:::MLLOG {"namespace": "", "time_ms": 1784860423197, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 362414080, "epoch_num": 0.15820170945558143}} +:::MLLOG {"namespace": "", "time_ms": 1784860447356, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7791388630867004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 362414080, "epoch_num": 0.15820170945558143}} +:::MLLOG {"namespace": "", "time_ms": 1784860447357, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 362414080, "epoch_num": 0.15820170945558143}} +:::MLLOG {"namespace": "", "time_ms": 1784860447357, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 362414080, "epoch_num": 0.15820170945558143}} +:::MLLOG {"namespace": "", "time_ms": 1784860464584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10448715090751648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 363069440, "lr": 1.8465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860481761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11000551283359528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 363724800, "lr": 1.8498333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860499033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10844696313142776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 364380160, "lr": 1.8531666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860507594, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 364707840, "epoch_num": 0.15920298609770536}} +:::MLLOG {"namespace": "", "time_ms": 1784860507594, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 364707840, "epoch_num": 0.15920298609770536}} +:::MLLOG {"namespace": "", "time_ms": 1784860531205, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7791720628738403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 364707840, "epoch_num": 0.15920298609770536}} +:::MLLOG {"namespace": "", "time_ms": 1784860531205, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 364707840, "epoch_num": 0.15920298609770536}} +:::MLLOG {"namespace": "", "time_ms": 1784860531205, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 364707840, "epoch_num": 0.15920298609770536}} +:::MLLOG {"namespace": "", "time_ms": 1784860539830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10908392071723938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 365035520, "lr": 1.8565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860556943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11388638615608215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 365690880, "lr": 1.8598333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860574166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10484784096479416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 366346240, "lr": 1.8631666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860591459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10637783259153366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 367001600, "lr": 1.8665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860591460, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 367001600, "epoch_num": 0.1602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784860591460, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 367001600, "epoch_num": 0.1602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784860614860, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7793101668357849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 367001600, "epoch_num": 0.1602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784860614860, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 367001600, "epoch_num": 0.1602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784860614860, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 367001600, "epoch_num": 0.1602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784860632306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10714592039585114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 367656960, "lr": 1.869833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860649210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10403244197368622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 368312320, "lr": 1.8731666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860667443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1117076575756073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 368967680, "lr": 1.8765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860676475, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 369295360, "epoch_num": 0.16120553938195323}} +:::MLLOG {"namespace": "", "time_ms": 1784860676476, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 369295360, "epoch_num": 0.16120553938195323}} +:::MLLOG {"namespace": "", "time_ms": 1784860699524, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7795265316963196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 369295360, "epoch_num": 0.16120553938195323}} +:::MLLOG {"namespace": "", "time_ms": 1784860699525, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 369295360, "epoch_num": 0.16120553938195323}} +:::MLLOG {"namespace": "", "time_ms": 1784860699525, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 369295360, "epoch_num": 0.16120553938195323}} +:::MLLOG {"namespace": "", "time_ms": 1784860708257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11494135856628418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 369623040, "lr": 1.8798333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860725637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1097816526889801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 370278400, "lr": 1.8831666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860742939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11395252496004105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 370933760, "lr": 1.8865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860760490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11151443421840668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 371589120, "lr": 1.889833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860760491, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 371589120, "epoch_num": 0.16220681602407716}} +:::MLLOG {"namespace": "", "time_ms": 1784860760491, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 371589120, "epoch_num": 0.16220681602407716}} +:::MLLOG {"namespace": "", "time_ms": 1784860784601, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7795580625534058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 371589120, "epoch_num": 0.16220681602407716}} +:::MLLOG {"namespace": "", "time_ms": 1784860784602, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 371589120, "epoch_num": 0.16220681602407716}} +:::MLLOG {"namespace": "", "time_ms": 1784860784602, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 371589120, "epoch_num": 0.16220681602407716}} +:::MLLOG {"namespace": "", "time_ms": 1784860801969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10923460125923157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 372244480, "lr": 1.8931666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860819455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11102035641670227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 372899840, "lr": 1.8965e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860837179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11411987990140915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 373555200, "lr": 1.8998333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860846092, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 373882880, "epoch_num": 0.1632080926662011}} +:::MLLOG {"namespace": "", "time_ms": 1784860846093, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 373882880, "epoch_num": 0.1632080926662011}} +:::MLLOG {"namespace": "", "time_ms": 1784860869575, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7795700430870056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 373882880, "epoch_num": 0.1632080926662011}} +:::MLLOG {"namespace": "", "time_ms": 1784860869575, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 373882880, "epoch_num": 0.1632080926662011}} +:::MLLOG {"namespace": "", "time_ms": 1784860869575, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 373882880, "epoch_num": 0.1632080926662011}} +:::MLLOG {"namespace": "", "time_ms": 1784860878486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11377333104610443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 374210560, "lr": 1.9031666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860895986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10457989573478699, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 374865920, "lr": 1.9065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860913115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11016236245632172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 375521280, "lr": 1.9098333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860931569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11318803578615189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 376176640, "lr": 1.9131666666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860931570, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 376176640, "epoch_num": 0.16420936930832503}} +:::MLLOG {"namespace": "", "time_ms": 1784860931570, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 376176640, "epoch_num": 0.16420936930832503}} diff --git a/recommendation_v4/rcp_logs/gbs_32768/seed412282151.log b/recommendation_v4/rcp_logs/gbs_32768/seed412282151.log new file mode 100644 index 000000000..0a704689a --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_32768/seed412282151.log @@ -0,0 +1,1427 @@ +:::MLLOG {"namespace": "", "time_ms": 1784881836574, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784881836574, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784881860034, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784881860035, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784881860035, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784881860035, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784881860036, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784881860036, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784881860036, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784881860036, "event_type": "POINT_IN_TIME", "key": "seed", "value": 412282151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784881860036, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784881860036, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784881860036, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784881860036, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784881860036, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784881860037, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784882770650, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784882770650, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784882770973, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784882908876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13807998597621918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.1666666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784882924232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13828760385513306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.4999999999999995e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784882938449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13823862373828888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784882945519, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784882945520, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784882994828, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5009365081787109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784882994829, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784882994829, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784883001986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13807447254657745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3166666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883015986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380128264427185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883030796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381179839372635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9833333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883045116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381169855594635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883045116, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784883045117, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784883068237, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.501096785068512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784883068237, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784883068237, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784883082511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375601887702942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883096826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377730369567871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883111261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13758930563926697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883118171, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784883118171, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784883141781, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5012893080711365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784883141781, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784883141782, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784883149195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13798409700393677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883163557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13796892762184143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883178275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13801202178001404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883192748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13748383522033691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883192749, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784883192749, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784883215949, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5015255212783813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784883215949, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784883215949, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784883230105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13796773552894592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883244287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370914876461029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883258838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13749809563159943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883265883, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784883265884, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784883289002, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5017967820167542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784883289003, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784883289003, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784883295920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13651180267333984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883310823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13682329654693604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883324870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13640326261520386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883339294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365259736776352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883339295, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784883339295, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784883363148, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5020853281021118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784883363148, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784883363148, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784883377127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13673245906829834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.316666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883391567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13675953447818756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883406616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13589933514595032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883413863, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784883413864, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784883437620, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5024215579032898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784883437621, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784883437621, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784883444916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13691216707229614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883460568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13585227727890015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883475389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362978219985962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883490822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13678207993507385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883490822, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784883490823, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784883514424, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5028579235076904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784883514424, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784883514424, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784883529026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13591539859771729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883544305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13446034491062164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.983333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883558612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13638511300086975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883565904, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784883565905, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784883589439, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5033006072044373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784883589440, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784883589440, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784883596708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325722336769104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883611827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13428348302841187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883626054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13473141193389893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883640390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13503065705299377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883640461, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784883640462, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784883664245, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5037540793418884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784883664246, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784883664246, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784883678475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13350646197795868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883692772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13445255160331726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883707099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350371241569519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883715027, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784883715028, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784883738754, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5042587518692017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784883738755, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784883738755, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784883745710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13561716675758362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883759972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13471955060958862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883774583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13181357085704803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883790198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13450121879577637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883790199, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784883790199, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784883813849, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5049453377723694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784883813850, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784883813850, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784883828619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133632093667984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883843756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352507472038269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883859216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13488447666168213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883866726, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784883866727, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784883890637, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5059807896614075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784883890638, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784883890638, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784883897926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363917589187622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883912858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13346333801746368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883928147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13429191708564758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883942918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320817619562149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883942980, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784883942980, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784883967031, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5070451498031616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784883967032, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784883967032, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784883981710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13293536007404327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883997239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13122685253620148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884011810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13321855664253235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884019119, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784884019120, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784884042619, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5081766843795776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784884042620, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784884042620, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784884049916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13849496841430664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884064716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13112814724445343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884080133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12993282079696655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884094780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304386854171753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884094781, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784884094782, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784884118328, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5094400644302368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784884118329, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784884118329, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784884132971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13277746737003326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884148771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12980109453201294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884165108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13065706193447113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884172504, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784884172505, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784884195640, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.510814368724823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784884195641, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784884195642, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784884203021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13163718581199646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884217719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351456344127655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884232880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332039088010788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884248327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356140375137329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884248327, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784884248328, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784884271366, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5122842192649841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784884271367, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784884271367, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784884286176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13522332906723022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884300912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13257700204849243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1649999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884315904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13318942487239838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884323316, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784884323317, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784884346802, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5137537121772766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784884346802, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784884346803, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784884355312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13641522824764252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884370751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13527192175388336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884385950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13617873191833496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884401409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13647881150245667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884401410, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784884401410, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784884424861, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.515275776386261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784884424861, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784884424862, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784884440765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13474561274051666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884456116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13096243143081665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884471199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13626818358898163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884478949, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784884478949, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784884502468, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5167999267578125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784884502468, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784884502469, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784884510058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13593316078186035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884525719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13259334862232208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884540481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13261431455612183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884555086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13091184198856354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884555087, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784884555087, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784884578249, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5183252692222595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784884578249, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784884578249, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784884593174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13103771209716797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884608687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13832567632198334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884623573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295783817768097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884630872, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784884630873, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784884653478, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.520051121711731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784884653479, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784884653479, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784884660655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12931504845619202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884675467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346992701292038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884690490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345359981060028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884706038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304713934659958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884706039, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784884706039, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784884728825, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5220866799354553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784884728826, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784884728826, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784884743611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12810611724853516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884758414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13182762265205383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884773446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354852318763733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884781035, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784884781036, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784884804589, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.524408221244812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784884804590, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784884804590, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784884812139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13663586974143982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884827644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336010843515396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884842510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13206075131893158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884857118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13099178671836853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884857118, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784884857119, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784884880037, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5271634459495544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784884880037, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784884880038, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784884895108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12958845496177673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884910113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13251513242721558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884925723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354474425315857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884933191, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784884933191, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784884956066, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5302358865737915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784884956067, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784884956067, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784884963453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13195481896400452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884978682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13331034779548645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884993870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349671334028244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885009128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13459256291389465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885009129, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784885009129, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784885032219, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5338658690452576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784885032220, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784885032220, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784885047466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13603444397449493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885063484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13388536870479584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885079087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13310499489307404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3650000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885086616, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784885086617, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784885109784, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5380253195762634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784885109785, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784885109785, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784885117673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1403258740901947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885133331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352105736732483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885148968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13446646928787231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4649999999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885164807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13507713377475739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885164808, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784885164809, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784885187407, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5424267649650574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784885187408, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784885187408, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784885202595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13523012399673462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885218015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332034170627594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.5649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885233520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13218024373054504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885241849, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784885241850, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784885265743, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5473649501800537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784885265744, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784885265744, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784885273529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13249912858009338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885288816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13073000311851501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885303981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135860413312912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885319439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13221409916877747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885319440, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784885319440, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784885342651, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5533077716827393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784885342652, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784885342652, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784885358175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13453739881515503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885374106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13004320859909058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885389195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13169467449188232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885396671, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784885396671, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784885419726, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5610353946685791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784885419726, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784885419727, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784885427122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321503072977066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885442358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13046523928642273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885457881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13287129998207092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885473830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13242560625076294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885473831, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784885473831, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784885497242, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5708088278770447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784885497243, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784885497243, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784885512574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13173486292362213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885527827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12780660390853882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885542866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337345540523529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885550790, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784885550791, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784885574488, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5811532735824585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784885574488, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784885574488, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784885582246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13439635932445526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885597641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1264544427394867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885613505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12648701667785645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885628902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324400007724762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885628902, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784885628903, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784885651843, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5924150347709656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784885651844, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784885651844, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784885666973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12871401011943817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885682073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12798070907592773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885697764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12736761569976807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885705344, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784885705344, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784885728238, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6008750200271606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784885728239, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784885728239, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784885736025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12963394820690155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885751743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12685397267341614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885766960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12948161363601685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885782203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12753652036190033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885782204, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784885782204, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784885805197, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6110828518867493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784885805198, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784885805198, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784885820226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1260138601064682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885835767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289222687482834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4983333333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885851061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13188913464546204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885858731, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784885858732, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784885881960, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6200058460235596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784885881961, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784885881961, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784885889996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13015779852867126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885905873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12530440092086792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885921650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13034780323505402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885937173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12510843575000763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885937173, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784885937174, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784885960867, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.629524827003479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784885960868, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784885960868, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784885976807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12760215997695923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885992585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288541704416275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886008760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12691180408000946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886016630, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784886016630, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784886039837, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6391634345054626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784886039838, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784886039838, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784886047769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12607929110527039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886063073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12577877938747406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886078647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12622126936912537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886094249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1252909004688263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.898333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886094250, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784886094250, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784886118028, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.646886944770813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784886118029, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784886118029, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784886133949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11920290440320969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886149503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11874064058065414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886164710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11666086316108704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886172343, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784886172344, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784886196321, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6496631503105164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784886196322, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784886196322, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784886203808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11870196461677551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886219226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11580918729305267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886234737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12047069519758224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886250047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11777995526790619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886250047, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784886250047, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784886274474, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6729874610900879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784886274474, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784886274475, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784886290556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11940652877092361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886305900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11047067493200302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.198333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886321310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11647825688123703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886328874, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784886328875, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784886352724, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6904646754264832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784886352725, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784886352725, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784886360216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11731501668691635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886375933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11719302088022232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.298333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886391692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11204637587070465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886406958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11569606512784958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886407027, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784886407027, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784886431100, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7187668681144714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784886431101, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784886431101, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784886446681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11451415717601776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.398333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886462007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11461204290390015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886477597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10838174819946289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886485241, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784886485242, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784886508251, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7376513481140137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784886508252, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784886508252, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784886515991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11200104653835297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.498333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886531720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11328410357236862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.531666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886547332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1112077534198761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886563168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11381107568740845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.598333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886563168, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784886563169, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784886586776, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7541411519050598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784886586776, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784886586777, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784886602277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11276717483997345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886617534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11182033270597458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886632922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11556382477283478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886640493, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784886640494, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784886664140, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7577847838401794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784886664140, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784886664141, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784886671695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1057494729757309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886687385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1135985404253006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886702924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10756121575832367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886718563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11232875287532806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886718564, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784886718565, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784886742807, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.761486291885376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784886742808, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784886742808, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784886758383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11483544856309891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886773784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11261174082756042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886789171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11072814464569092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886796629, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784886796630, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784886819923, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7631018757820129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784886819924, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784886819924, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784886827701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1106744110584259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886843385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11278600990772247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886859375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11238733679056168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886874810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10728377103805542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886874811, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784886874811, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784886898512, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7646206021308899, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784886898512, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784886898513, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784886915399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11565229296684265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886931040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1194465160369873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886946742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11721863597631454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886954502, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784886954503, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784886977854, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7652996182441711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784886977854, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784886977854, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784886985558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11605584621429443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887001503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11522561311721802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887017261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11281131207942963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887032837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11350332200527191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887032838, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784887032838, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784887056276, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7656848430633545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784887056277, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784887056277, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784887072381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11834351718425751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887088134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1175394132733345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887104235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11069384962320328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887112301, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784887112303, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784887135144, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7660859227180481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784887135145, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784887135145, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784887143176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11623311787843704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887159389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11538176983594894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887175399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1111554279923439, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887191833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10459825396537781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887191833, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784887191834, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784887214570, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7662698030471802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784887214570, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784887214570, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784887229779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10683605074882507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887245382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10601920634508133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887260790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10600610077381134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887268651, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784887268652, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784887291904, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7663646936416626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784887291905, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784887291905, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784887299745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10400446504354477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887315781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11147905141115189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887331618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10775072127580643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887347726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1129511147737503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887347727, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784887347727, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784887371622, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.766907811164856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784887371623, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784887371623, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784887387764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723884403705597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887403306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10649074614048004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887419059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10242845118045807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887426750, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784887426750, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784887449883, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7671055793762207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784887449884, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784887449884, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784887457809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1027032881975174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887473615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10130460560321808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887489824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11181332170963287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887505507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10480780899524689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887505508, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784887505508, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784887529178, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7678101062774658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784887529179, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784887529179, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784887545338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10658878833055496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887561206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10749409347772598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887576710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10695990920066833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887584796, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784887584797, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784887608133, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.768152117729187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784887608134, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784887608134, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784887616062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10724715888500214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887631583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1000676229596138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887647358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10787332057952881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887663171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10470045357942581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887663172, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784887663172, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784887687333, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7691392302513123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784887687334, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784887687334, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784887702754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10770837962627411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887719105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080397367477417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887734988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10476988554000854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887742786, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784887742786, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784887765794, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7692858576774597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784887765794, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784887765794, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784887773554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.109261155128479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887789413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10724271088838577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887805095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1119142398238182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887821142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10790212452411652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887821142, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784887821143, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784887844164, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7697048783302307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784887844164, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784887844165, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784887860356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10544456541538239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887876064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10269701480865479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887892398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085181012749672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.564999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887900528, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784887900528, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784887923227, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7699686288833618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784887923228, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784887923228, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784887930944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10440301895141602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887946263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0996471643447876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887961915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11429009586572647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.664999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887977319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11113195866346359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887977319, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784887977320, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784888000245, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7703400254249573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784888000246, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784888000246, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784888015840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10675999522209167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888031895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10888203233480453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888047697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10815788060426712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888055446, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784888055446, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784888079062, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7708685398101807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784888079063, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784888079063, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784888087737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11026258021593094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888104055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11482395231723785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888120268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11256112903356552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888136530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11387111246585846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888136531, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784888136531, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784888159396, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7714524865150452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784888159397, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784888159397, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784888175350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11051137000322342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888191738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11383944749832153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.998333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888208084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11094431579113007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888216163, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784888216163, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784888239996, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7715817093849182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784888239996, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784888239997, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784888247941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1110675185918808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888264501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10952961444854736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888280592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11302266269922256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.131666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888296741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11151298880577087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888296741, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784888296742, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784888319548, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7717637419700623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784888319549, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784888319549, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784888335765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11046922206878662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888352209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.112847238779068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888368522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11714545637369156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888376611, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784888376612, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784888400135, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7723289728164673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784888400136, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784888400136, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784888407945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11468864977359772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888424372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10795850306749344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.331666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888440307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10158606618642807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888455974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10731766372919083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888455974, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784888455975, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784888479120, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7714527249336243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784888479121, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784888479121, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784888494714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10515391081571579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888510219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10674387216567993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888526259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10589493811130524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888534374, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784888534374, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784888558213, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7721667289733887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784888558214, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784888558214, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784888566165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1086978018283844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888581805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10337284207344055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888598289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10471700131893158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888614567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10748215764760971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.631666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888614568, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784888614568, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784888637206, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7729381918907166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784888637206, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784888637206, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784888652684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1006227359175682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888668595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10405139625072479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888684234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10346541553735733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888692000, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784888692000, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784888715249, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7729799747467041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784888715250, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784888715250, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784888723212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10432588309049606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888739502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10318921506404877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888755297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11437897384166718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888771447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10358002036809921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888771448, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784888771449, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784888795740, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7733559608459473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784888795741, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784888795741, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784888811528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10719603300094604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888827149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10773928463459015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888842752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10171644389629364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888850519, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784888850519, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784888875097, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773585319519043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784888875097, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784888875097, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784888882813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10142611712217331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888898368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10042697936296463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888914276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10647735744714737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888930398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10317666828632355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888930398, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784888930399, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784888954316, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738311290740967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784888954317, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784888954317, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784888969837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10767389088869095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888986921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10407574474811554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889002691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1062961220741272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889010398, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784889010398, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784889034276, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7741561532020569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784889034277, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784889034277, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784889042098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10396243631839752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889057884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10864794254302979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889073781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10333497077226639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889090021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10691781342029572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889090022, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784889090022, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784889113431, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7739772200584412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784889113432, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784889113432, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784889129703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10249138623476028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889145454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10708852112293243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889161702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1099308431148529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889169642, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784889169642, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784889193743, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7745820879936218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784889193744, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784889193744, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784889201937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11218194663524628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889217518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10745362937450409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889233343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1030845195055008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889249294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1029382199048996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889249294, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784889249294, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784889273113, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7748270034790039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784889273114, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784889273114, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784889288575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10542712360620499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889304798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10553093254566193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.631666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889320905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11196473240852356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889328931, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784889328931, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784889352320, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746824622154236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784889352320, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784889352321, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784889360105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10701341927051544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889376384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11258228123188019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889393081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10984642803668976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889409357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11281634867191315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889409358, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784889409358, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784889432830, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752846479415894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784889432831, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784889432831, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784889449160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10888948291540146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.831666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889465466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11408251523971558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889481746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11205873638391495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889489903, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784889489904, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784889513835, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752264738082886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784889513836, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784889513836, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784889522057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11272973567247391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889538419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11211259663105011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889555021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772246867418289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889571461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11093437671661377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1.0031666666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784889571461, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784889571461, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784889594877, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.775435209274292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784889594878, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784889594878, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784889610746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11426657438278198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1.0064999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784889626830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10900958627462387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1.0098333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784889642977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11160405725240707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1.0131666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784889651218, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784889651219, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784889675034, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7758574485778809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784889675035, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784889675035, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784889683443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11142663657665253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1.0165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784889700116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10403945297002792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1.0198333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784889716284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11027885228395462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1.0231666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784889732864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10821132361888885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1.0264999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784889732865, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784889732865, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784889756019, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7755417823791504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784889756020, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784889756020, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784889771842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10392211377620697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1.0298333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784889787279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09993409365415573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1.0331666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784889803071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1019909679889679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1.0365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784889810962, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784889810963, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784889833999, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7758267521858215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784889834000, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784889834000, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784889841864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0997801423072815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1.0398333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784889857438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10138490051031113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1.0431666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784889873226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1034000962972641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1.0464999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784889889322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1056935042142868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1.0498333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784889889323, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784889889323, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784889912475, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7759043574333191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784889912475, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784889912475, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784889928026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10461531579494476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1.0531666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784889944453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10991238802671432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1.0565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784889960558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10648208856582642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1.0598333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784889968551, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784889968551, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784889991584, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7760001420974731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784889991584, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784889991584, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784889999369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10180509835481644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1.0631666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890015446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10575634241104126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1.0664999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890031368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1007687896490097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1.0698333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890047203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10166770219802856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1.0731666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890047203, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784890047204, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784890070904, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7761595249176025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784890070905, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784890070905, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784890087140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10224998742341995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1.0765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890103716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10349644720554352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1.0798333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890119704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10181119292974472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1.0831666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890128487, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784890128488, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784890153319, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7766220569610596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784890153319, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784890153320, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784890161242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10438612848520279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1.0864999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890177329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772015154361725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1.0898333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890193203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10294751822948456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1.0931666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890209192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10070635378360748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1.0965e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890209193, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784890209193, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784890233077, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7762840390205383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784890233078, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784890233078, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784890249007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10617813467979431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1.0998333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890264804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10205268859863281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1.1031666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890280910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10202352702617645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217579520, "lr": 1.1065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890289085, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784890289085, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784890312587, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7766115665435791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784890312588, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784890312588, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784890320642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11169992387294769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218234880, "lr": 1.1098333333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890336499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10978461802005768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218890240, "lr": 1.1131666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890352922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10802322626113892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219545600, "lr": 1.1165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890369307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087338849902153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220200960, "lr": 1.1198333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890369308, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784890369308, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784890393078, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7767050862312317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784890393079, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784890393079, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784890409241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10323735326528549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220856320, "lr": 1.1231666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890432420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1049700453877449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221511680, "lr": 1.1265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890448573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10085123032331467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222167040, "lr": 1.1298333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890456728, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784890457895, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784890481386, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7770466804504395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784890481386, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784890481387, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784890492652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10558268427848816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222822400, "lr": 1.1331666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890509062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10445626825094223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223477760, "lr": 1.1365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890525401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10702858120203018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224133120, "lr": 1.1398333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890541982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10957591235637665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224788480, "lr": 1.1431666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890541982, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784890541982, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784890565744, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7767850160598755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784890565745, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784890565745, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784890581807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087769865989685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225443840, "lr": 1.1465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890598583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10924778878688812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226099200, "lr": 1.1498333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890615275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792893916368484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226754560, "lr": 1.1531666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890623357, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784890623357, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784890646972, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7775548100471497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784890646972, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784890646972, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784890654867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11261694878339767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227409920, "lr": 1.1565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890670978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10995608568191528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228065280, "lr": 1.1598333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890693324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072986051440239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228720640, "lr": 1.1631666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890709377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10688146948814392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229376000, "lr": 1.1665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890709378, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784890709379, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784890732744, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.777417778968811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784890732744, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784890732745, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784890750525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10705672204494476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230031360, "lr": 1.1698333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890766511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10548437386751175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230686720, "lr": 1.1731666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890790210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11085185408592224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231342080, "lr": 1.1765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890797945, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784890797946, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784890822312, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7773594260215759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784890822312, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784890822312, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784890830296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10878045856952667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231997440, "lr": 1.1798333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890847283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10959143936634064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 232652800, "lr": 1.1831666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890864101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11109735071659088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233308160, "lr": 1.1865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890880755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11164698004722595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233963520, "lr": 1.1898333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890880756, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784890880756, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784890904773, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7778894305229187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784890904774, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784890904774, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784890921555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11401958763599396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 234618880, "lr": 1.1931666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890940764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11351203173398972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235274240, "lr": 1.1965e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890957253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11116446554660797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235929600, "lr": 1.1998333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890965610, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784890965611, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784890990109, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7778226137161255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784890990109, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784890990109, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784890998264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10877678543329239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 236584960, "lr": 1.2031666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891015262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11281182616949081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237240320, "lr": 1.2064999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891031583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1136709451675415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237895680, "lr": 1.2098333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891048546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1194048821926117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 238551040, "lr": 1.2131666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891048546, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784891050528, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784891073547, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.778149425983429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784891073547, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784891073547, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784891089936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11748066544532776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239206400, "lr": 1.2165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891106115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11387547850608826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239861760, "lr": 1.2198333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891129173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11380765587091446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 240517120, "lr": 1.2231666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891137423, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784891137423, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784891160969, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7782598733901978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784891160969, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784891160970, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784891168887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1148531436920166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241172480, "lr": 1.2264999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891194614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1150275245308876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241827840, "lr": 1.2298333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891211188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11443700641393661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 242483200, "lr": 1.2331666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891227385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11791713535785675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243138560, "lr": 1.2365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891227386, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784891227386, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784891251367, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7785623669624329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784891251367, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784891251367, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784891268527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10973426699638367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243793920, "lr": 1.2398333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891294798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1155354306101799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 244449280, "lr": 1.2431666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891315300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1144709661602974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245104640, "lr": 1.2464999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891323850, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784891323850, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784891347579, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7784359455108643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784891347579, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784891347580, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784891355728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11052628606557846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245760000, "lr": 1.2498333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891380032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1123395711183548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 246415360, "lr": 1.2531666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891396553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10779352486133575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247070720, "lr": 1.2565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891414725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10790731757879257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247726080, "lr": 1.2598333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891414725, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784891414726, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784891438166, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7787181735038757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784891438167, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784891438168, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784891454478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1133267804980278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 248381440, "lr": 1.2631666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891474945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10972362756729126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249036800, "lr": 1.2664999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891491473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10825686901807785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249692160, "lr": 1.2698333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891499586, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784891499587, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784891522987, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7782536745071411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784891522987, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784891522987, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784891530774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10615513473749161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 250347520, "lr": 1.2731666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891553868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10496032983064651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251002880, "lr": 1.2765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891571239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10424234718084335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251658240, "lr": 1.2798333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891587568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10842503607273102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 252313600, "lr": 1.2831666666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891587568, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784891587569, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784891611158, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7787237167358398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784891611158, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784891611158, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784891627833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11064860224723816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 252968960, "lr": 1.2864999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891648772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10539144277572632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 253624320, "lr": 1.2898333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891664893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10778208076953888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 254279680, "lr": 1.2931666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891673448, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784891673449, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784891697110, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7790219187736511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784891697111, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784891697111, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784891705209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10714731365442276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 254935040, "lr": 1.2965e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891721517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10707167536020279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 255590400, "lr": 1.2998333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891744177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10914561152458191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 256245760, "lr": 1.3031666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891760662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10668477416038513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 256901120, "lr": 1.3065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891760662, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784891760662, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784891784336, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7791630625724792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784891784337, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784891784337, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784891805377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10662214457988739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 257556480, "lr": 1.3098333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891821369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10520495474338531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 258211840, "lr": 1.3131666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891839377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10398474335670471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 258867200, "lr": 1.3165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891853974, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784891853975, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784891877759, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7792909741401672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784891877760, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784891877760, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784891893718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11158297955989838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 259522560, "lr": 1.3198333333333335e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891909974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10542181134223938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 260177920, "lr": 1.3231666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891926540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10575206577777863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 260833280, "lr": 1.3265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891942921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10583299398422241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 261488640, "lr": 1.3298333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891942922, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784891942922, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784891966487, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.779312014579773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784891966487, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784891966487, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784891988425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10351482033729553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 262144000, "lr": 1.3331666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892004494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10145476460456848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 262799360, "lr": 1.3365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892025278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10555120557546616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 263454720, "lr": 1.3398333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892047222, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 263782400, "epoch_num": 0.11514681384425231}} +:::MLLOG {"namespace": "", "time_ms": 1784892047223, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 263782400, "epoch_num": 0.11514681384425231}} +:::MLLOG {"namespace": "", "time_ms": 1784892070552, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7795237898826599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 263782400, "epoch_num": 0.11514681384425231}} +:::MLLOG {"namespace": "", "time_ms": 1784892070553, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 263782400, "epoch_num": 0.11514681384425231}} +:::MLLOG {"namespace": "", "time_ms": 1784892070554, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 263782400, "epoch_num": 0.11514681384425231}} +:::MLLOG {"namespace": "", "time_ms": 1784892083479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10941708832979202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 264110080, "lr": 1.3431666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892100066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082533597946167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 264765440, "lr": 1.3465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892120083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10494057089090347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 265420800, "lr": 1.3498333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892143277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10395629703998566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 266076160, "lr": 1.3531666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892143278, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 266076160, "epoch_num": 0.11614809048637624}} +:::MLLOG {"namespace": "", "time_ms": 1784892143278, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 266076160, "epoch_num": 0.11614809048637624}} +:::MLLOG {"namespace": "", "time_ms": 1784892167260, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7797556519508362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 266076160, "epoch_num": 0.11614809048637624}} +:::MLLOG {"namespace": "", "time_ms": 1784892167260, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 266076160, "epoch_num": 0.11614809048637624}} +:::MLLOG {"namespace": "", "time_ms": 1784892167261, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 266076160, "epoch_num": 0.11614809048637624}} +:::MLLOG {"namespace": "", "time_ms": 1784892183269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10440080612897873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 266731520, "lr": 1.3565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892204871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10722469538450241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 267386880, "lr": 1.359833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892221189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10904078930616379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 268042240, "lr": 1.3631666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892229494, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 268369920, "epoch_num": 0.11714936712850017}} +:::MLLOG {"namespace": "", "time_ms": 1784892229495, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 268369920, "epoch_num": 0.11714936712850017}} +:::MLLOG {"namespace": "", "time_ms": 1784892252868, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7795620560646057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 268369920, "epoch_num": 0.11714936712850017}} +:::MLLOG {"namespace": "", "time_ms": 1784892252868, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 268369920, "epoch_num": 0.11714936712850017}} +:::MLLOG {"namespace": "", "time_ms": 1784892252869, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 268369920, "epoch_num": 0.11714936712850017}} +:::MLLOG {"namespace": "", "time_ms": 1784892260768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11116981506347656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 268697600, "lr": 1.3665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892284226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10530374199151993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 269352960, "lr": 1.3698333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892301685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10369445383548737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 270008320, "lr": 1.3731666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892318014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10945510864257812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 270663680, "lr": 1.3765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892318015, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 270663680, "epoch_num": 0.1181506437706241}} +:::MLLOG {"namespace": "", "time_ms": 1784892318015, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 270663680, "epoch_num": 0.1181506437706241}} +:::MLLOG {"namespace": "", "time_ms": 1784892342946, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7801367044448853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 270663680, "epoch_num": 0.1181506437706241}} +:::MLLOG {"namespace": "", "time_ms": 1784892342946, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 270663680, "epoch_num": 0.1181506437706241}} +:::MLLOG {"namespace": "", "time_ms": 1784892342947, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 270663680, "epoch_num": 0.1181506437706241}} +:::MLLOG {"namespace": "", "time_ms": 1784892359486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11088544130325317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 271319040, "lr": 1.379833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892376064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11114522069692612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 271974400, "lr": 1.3831666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892392302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1092117503285408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 272629760, "lr": 1.3865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892400486, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 272957440, "epoch_num": 0.11915192041274804}} +:::MLLOG {"namespace": "", "time_ms": 1784892400486, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 272957440, "epoch_num": 0.11915192041274804}} +:::MLLOG {"namespace": "", "time_ms": 1784892425621, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7798935770988464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 272957440, "epoch_num": 0.11915192041274804}} +:::MLLOG {"namespace": "", "time_ms": 1784892425622, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 272957440, "epoch_num": 0.11915192041274804}} +:::MLLOG {"namespace": "", "time_ms": 1784892425622, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 272957440, "epoch_num": 0.11915192041274804}} +:::MLLOG {"namespace": "", "time_ms": 1784892433749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11127183586359024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 273285120, "lr": 1.3898333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892450859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11063509434461594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 273940480, "lr": 1.3931666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892467122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11858926713466644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 274595840, "lr": 1.3965e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892483089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11748874932527542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 275251200, "lr": 1.3998333333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892483090, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 275251200, "epoch_num": 0.12015319705487197}} +:::MLLOG {"namespace": "", "time_ms": 1784892483090, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 275251200, "epoch_num": 0.12015319705487197}} +:::MLLOG {"namespace": "", "time_ms": 1784892506621, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7805247902870178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 275251200, "epoch_num": 0.12015319705487197}} +:::MLLOG {"namespace": "", "time_ms": 1784892506622, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 275251200, "epoch_num": 0.12015319705487197}} +:::MLLOG {"namespace": "", "time_ms": 1784892506622, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 275251200, "epoch_num": 0.12015319705487197}} +:::MLLOG {"namespace": "", "time_ms": 1784892523124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11163142323493958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 275906560, "lr": 1.4031666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892539519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11092260479927063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 276561920, "lr": 1.4065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892556283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1183691918849945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 277217280, "lr": 1.4098333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892564542, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 277544960, "epoch_num": 0.1211544736969959}} +:::MLLOG {"namespace": "", "time_ms": 1784892564543, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 277544960, "epoch_num": 0.1211544736969959}} +:::MLLOG {"namespace": "", "time_ms": 1784892587672, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7806130051612854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 277544960, "epoch_num": 0.1211544736969959}} +:::MLLOG {"namespace": "", "time_ms": 1784892587672, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 277544960, "epoch_num": 0.1211544736969959}} +:::MLLOG {"namespace": "", "time_ms": 1784892587672, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 277544960, "epoch_num": 0.1211544736969959}} +:::MLLOG {"namespace": "", "time_ms": 1784892596134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11328991502523422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 277872640, "lr": 1.4131666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892612631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10968223214149475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 278528000, "lr": 1.4165000000000001e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892629874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11832235753536224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 279183360, "lr": 1.4198333333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892646515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10964210331439972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 279838720, "lr": 1.4231666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892646515, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 279838720, "epoch_num": 0.12215575033911984}} +:::MLLOG {"namespace": "", "time_ms": 1784892646516, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 279838720, "epoch_num": 0.12215575033911984}} +:::MLLOG {"namespace": "", "time_ms": 1784892669931, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7808818221092224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 279838720, "epoch_num": 0.12215575033911984}} +:::MLLOG {"namespace": "", "time_ms": 1784892669931, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 279838720, "epoch_num": 0.12215575033911984}} +:::MLLOG {"namespace": "", "time_ms": 1784892669932, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 279838720, "epoch_num": 0.12215575033911984}} +:::MLLOG {"namespace": "", "time_ms": 1784892686084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11131740361452103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 280494080, "lr": 1.4265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892702481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11222444474697113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 281149440, "lr": 1.4298333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892718782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11529958248138428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 281804800, "lr": 1.4331666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892726904, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 282132480, "epoch_num": 0.12315702698124378}} +:::MLLOG {"namespace": "", "time_ms": 1784892726904, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 282132480, "epoch_num": 0.12315702698124378}} +:::MLLOG {"namespace": "", "time_ms": 1784892750145, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7807187438011169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 282132480, "epoch_num": 0.12315702698124378}} +:::MLLOG {"namespace": "", "time_ms": 1784892750146, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 282132480, "epoch_num": 0.12315702698124378}} +:::MLLOG {"namespace": "", "time_ms": 1784892750146, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 282132480, "epoch_num": 0.12315702698124378}} +:::MLLOG {"namespace": "", "time_ms": 1784892758731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10731026530265808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 282460160, "lr": 1.4365000000000001e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892775385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11370497941970825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 283115520, "lr": 1.4398333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892792117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1116315945982933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 283770880, "lr": 1.4431666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892808677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11120506376028061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 284426240, "lr": 1.4464999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892808678, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 284426240, "epoch_num": 0.12415830362336772}} +:::MLLOG {"namespace": "", "time_ms": 1784892808678, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 284426240, "epoch_num": 0.12415830362336772}} +:::MLLOG {"namespace": "", "time_ms": 1784892833832, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.78078693151474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 284426240, "epoch_num": 0.12415830362336772}} +:::MLLOG {"namespace": "", "time_ms": 1784892833833, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 284426240, "epoch_num": 0.12415830362336772}} +:::MLLOG {"namespace": "", "time_ms": 1784892833833, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 284426240, "epoch_num": 0.12415830362336772}} +:::MLLOG {"namespace": "", "time_ms": 1784892850547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10855686664581299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 285081600, "lr": 1.4498333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892867079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11371850967407227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 285736960, "lr": 1.4531666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892883563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11424928903579712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 286392320, "lr": 1.4564999999999997e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892891861, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 286720000, "epoch_num": 0.12515958026549165}} +:::MLLOG {"namespace": "", "time_ms": 1784892891862, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 286720000, "epoch_num": 0.12515958026549165}} +:::MLLOG {"namespace": "", "time_ms": 1784892915509, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.78091961145401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 286720000, "epoch_num": 0.12515958026549165}} +:::MLLOG {"namespace": "", "time_ms": 1784892915510, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 286720000, "epoch_num": 0.12515958026549165}} +:::MLLOG {"namespace": "", "time_ms": 1784892915510, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 286720000, "epoch_num": 0.12515958026549165}} +:::MLLOG {"namespace": "", "time_ms": 1784892923596, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11450488865375519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 287047680, "lr": 1.4598333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892940089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11068125069141388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 287703040, "lr": 1.4631666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892957067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11296287178993225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 288358400, "lr": 1.4664999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892973949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11344553530216217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 289013760, "lr": 1.4698333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892973949, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 289013760, "epoch_num": 0.12616085690761558}} +:::MLLOG {"namespace": "", "time_ms": 1784892973949, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 289013760, "epoch_num": 0.12616085690761558}} +:::MLLOG {"namespace": "", "time_ms": 1784892997627, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7812180519104004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 289013760, "epoch_num": 0.12616085690761558}} +:::MLLOG {"namespace": "", "time_ms": 1784892997628, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 289013760, "epoch_num": 0.12616085690761558}} +:::MLLOG {"namespace": "", "time_ms": 1784892997628, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 289013760, "epoch_num": 0.12616085690761558}} +:::MLLOG {"namespace": "", "time_ms": 1784893014130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10764484107494354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 289669120, "lr": 1.4731666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893030675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10561063140630722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 290324480, "lr": 1.4764999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893047816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10119298100471497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 290979840, "lr": 1.4798333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893055975, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 291307520, "epoch_num": 0.12716213354973951}} +:::MLLOG {"namespace": "", "time_ms": 1784893055975, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 291307520, "epoch_num": 0.12716213354973951}} +:::MLLOG {"namespace": "", "time_ms": 1784893079533, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7813050746917725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 291307520, "epoch_num": 0.12716213354973951}} +:::MLLOG {"namespace": "", "time_ms": 1784893079534, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 291307520, "epoch_num": 0.12716213354973951}} +:::MLLOG {"namespace": "", "time_ms": 1784893079534, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 291307520, "epoch_num": 0.12716213354973951}} +:::MLLOG {"namespace": "", "time_ms": 1784893087801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10596254467964172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 291635200, "lr": 1.4831666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893104161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10911301523447037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 292290560, "lr": 1.4864999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893120689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10537146776914597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 292945920, "lr": 1.4898333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893136979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10552927851676941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 293601280, "lr": 1.4931666666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893136980, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 293601280, "epoch_num": 0.12816341019186345}} +:::MLLOG {"namespace": "", "time_ms": 1784893136980, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 293601280, "epoch_num": 0.12816341019186345}} +:::MLLOG {"namespace": "", "time_ms": 1784893160928, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7812530994415283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 293601280, "epoch_num": 0.12816341019186345}} +:::MLLOG {"namespace": "", "time_ms": 1784893160928, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 293601280, "epoch_num": 0.12816341019186345}} +:::MLLOG {"namespace": "", "time_ms": 1784893160929, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 293601280, "epoch_num": 0.12816341019186345}} +:::MLLOG {"namespace": "", "time_ms": 1784893177265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1083410456776619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 294256640, "lr": 1.4964999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893193995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10415935516357422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 294912000, "lr": 1.4998333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893211003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10831423103809357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 295567360, "lr": 1.5031666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893219479, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 295895040, "epoch_num": 0.12916468683398738}} +:::MLLOG {"namespace": "", "time_ms": 1784893219480, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 295895040, "epoch_num": 0.12916468683398738}} +:::MLLOG {"namespace": "", "time_ms": 1784893243883, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7816811800003052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 295895040, "epoch_num": 0.12916468683398738}} +:::MLLOG {"namespace": "", "time_ms": 1784893243883, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 295895040, "epoch_num": 0.12916468683398738}} +:::MLLOG {"namespace": "", "time_ms": 1784893243884, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 295895040, "epoch_num": 0.12916468683398738}} +:::MLLOG {"namespace": "", "time_ms": 1784893252101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10419677942991257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 296222720, "lr": 1.5064999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893268264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10508085042238235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 296878080, "lr": 1.5098333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893285461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10849079489707947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 297533440, "lr": 1.5131666666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893302026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10993081331253052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 298188800, "lr": 1.5164999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893302027, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 298188800, "epoch_num": 0.1301659634761113}} +:::MLLOG {"namespace": "", "time_ms": 1784893302028, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 298188800, "epoch_num": 0.1301659634761113}} +:::MLLOG {"namespace": "", "time_ms": 1784893326083, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7818453907966614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 298188800, "epoch_num": 0.1301659634761113}} +:::MLLOG {"namespace": "", "time_ms": 1784893326084, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 298188800, "epoch_num": 0.1301659634761113}} +:::MLLOG {"namespace": "", "time_ms": 1784893326084, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 298188800, "epoch_num": 0.1301659634761113}} +:::MLLOG {"namespace": "", "time_ms": 1784893342620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10521847009658813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 298844160, "lr": 1.5198333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893366340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10350354015827179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 299499520, "lr": 1.5231666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893383535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10401301831007004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 300154880, "lr": 1.5265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893391747, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 300482560, "epoch_num": 0.13116724011823525}} +:::MLLOG {"namespace": "", "time_ms": 1784893391748, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 300482560, "epoch_num": 0.13116724011823525}} +:::MLLOG {"namespace": "", "time_ms": 1784893415720, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7815906405448914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 300482560, "epoch_num": 0.13116724011823525}} +:::MLLOG {"namespace": "", "time_ms": 1784893415721, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 300482560, "epoch_num": 0.13116724011823525}} +:::MLLOG {"namespace": "", "time_ms": 1784893415721, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 300482560, "epoch_num": 0.13116724011823525}} +:::MLLOG {"namespace": "", "time_ms": 1784893423825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11037202924489975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 300810240, "lr": 1.5298333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893446814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10011391341686249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 301465600, "lr": 1.5331666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893463165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10127050429582596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 302120960, "lr": 1.5364999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893489184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10707957297563553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 302776320, "lr": 1.5398333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893489185, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 302776320, "epoch_num": 0.13216851676035918}} +:::MLLOG {"namespace": "", "time_ms": 1784893489187, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 302776320, "epoch_num": 0.13216851676035918}} +:::MLLOG {"namespace": "", "time_ms": 1784893513434, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7814986109733582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 302776320, "epoch_num": 0.13216851676035918}} +:::MLLOG {"namespace": "", "time_ms": 1784893513434, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 302776320, "epoch_num": 0.13216851676035918}} +:::MLLOG {"namespace": "", "time_ms": 1784893513434, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 302776320, "epoch_num": 0.13216851676035918}} +:::MLLOG {"namespace": "", "time_ms": 1784893529478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10999277234077454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 303431680, "lr": 1.5431666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893546275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10750923305749893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 304087040, "lr": 1.5465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893562986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10381879657506943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 304742400, "lr": 1.5498333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893572078, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 305070080, "epoch_num": 0.1331697934024831}} +:::MLLOG {"namespace": "", "time_ms": 1784893572079, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 305070080, "epoch_num": 0.1331697934024831}} +:::MLLOG {"namespace": "", "time_ms": 1784893595753, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7821285724639893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 305070080, "epoch_num": 0.1331697934024831}} +:::MLLOG {"namespace": "", "time_ms": 1784893595753, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 305070080, "epoch_num": 0.1331697934024831}} +:::MLLOG {"namespace": "", "time_ms": 1784893595753, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 305070080, "epoch_num": 0.1331697934024831}} +:::MLLOG {"namespace": "", "time_ms": 1784893616245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10237764567136765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 305397760, "lr": 1.5531666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893637602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10762208700180054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 306053120, "lr": 1.5564999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893653878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10559121519327164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 306708480, "lr": 1.5598333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893676858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1059480607509613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 307363840, "lr": 1.5631666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893676859, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 307363840, "epoch_num": 0.13417107004460704}} +:::MLLOG {"namespace": "", "time_ms": 1784893676859, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 307363840, "epoch_num": 0.13417107004460704}} +:::MLLOG {"namespace": "", "time_ms": 1784893700608, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7820316553115845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 307363840, "epoch_num": 0.13417107004460704}} +:::MLLOG {"namespace": "", "time_ms": 1784893700608, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 307363840, "epoch_num": 0.13417107004460704}} +:::MLLOG {"namespace": "", "time_ms": 1784893700609, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 307363840, "epoch_num": 0.13417107004460704}} +:::MLLOG {"namespace": "", "time_ms": 1784893716801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10679879039525986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 308019200, "lr": 1.5665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893739921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09924949705600739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 308674560, "lr": 1.5698333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893757319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10427863895893097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 309329920, "lr": 1.5731666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893771515, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 309657600, "epoch_num": 0.13517234668673098}} +:::MLLOG {"namespace": "", "time_ms": 1784893771516, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 309657600, "epoch_num": 0.13517234668673098}} +:::MLLOG {"namespace": "", "time_ms": 1784893795184, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7820866107940674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 309657600, "epoch_num": 0.13517234668673098}} +:::MLLOG {"namespace": "", "time_ms": 1784893795184, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 309657600, "epoch_num": 0.13517234668673098}} +:::MLLOG {"namespace": "", "time_ms": 1784893795184, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 309657600, "epoch_num": 0.13517234668673098}} +:::MLLOG {"namespace": "", "time_ms": 1784893803383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10791271924972534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 309985280, "lr": 1.5764999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893819281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10196676105260849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 310640640, "lr": 1.5798333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893836453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1091044545173645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 311296000, "lr": 1.5831666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893853142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10756444931030273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 311951360, "lr": 1.5865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893853143, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 311951360, "epoch_num": 0.1361736233288549}} +:::MLLOG {"namespace": "", "time_ms": 1784893855932, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 311951360, "epoch_num": 0.1361736233288549}} +:::MLLOG {"namespace": "", "time_ms": 1784893878695, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7827751636505127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 311951360, "epoch_num": 0.1361736233288549}} +:::MLLOG {"namespace": "", "time_ms": 1784893878696, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 311951360, "epoch_num": 0.1361736233288549}} +:::MLLOG {"namespace": "", "time_ms": 1784893878696, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 311951360, "epoch_num": 0.1361736233288549}} +:::MLLOG {"namespace": "", "time_ms": 1784893898787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10556105524301529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 312606720, "lr": 1.5898333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893919982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10632667690515518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 313262080, "lr": 1.5931666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893936447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10531498491764069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 313917440, "lr": 1.5964999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893952515, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 314245120, "epoch_num": 0.13717489997097884}} +:::MLLOG {"namespace": "", "time_ms": 1784893952516, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 314245120, "epoch_num": 0.13717489997097884}} +:::MLLOG {"namespace": "", "time_ms": 1784893976436, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7830345034599304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 314245120, "epoch_num": 0.13717489997097884}} +:::MLLOG {"namespace": "", "time_ms": 1784893976436, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 314245120, "epoch_num": 0.13717489997097884}} +:::MLLOG {"namespace": "", "time_ms": 1784893976436, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 314245120, "epoch_num": 0.13717489997097884}} +:::MLLOG {"namespace": "", "time_ms": 1784893984595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10918641090393066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 314572800, "lr": 1.5998333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894000850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10450717806816101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 315228160, "lr": 1.6031666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894028627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10411831736564636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 315883520, "lr": 1.6065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894045003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10365171730518341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 316538880, "lr": 1.6098333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894045004, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 316538880, "epoch_num": 0.13817617661310277}} +:::MLLOG {"namespace": "", "time_ms": 1784894045005, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 316538880, "epoch_num": 0.13817617661310277}} +:::MLLOG {"namespace": "", "time_ms": 1784894068781, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7829300761222839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 316538880, "epoch_num": 0.13817617661310277}} +:::MLLOG {"namespace": "", "time_ms": 1784894068781, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 316538880, "epoch_num": 0.13817617661310277}} +:::MLLOG {"namespace": "", "time_ms": 1784894068781, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 316538880, "epoch_num": 0.13817617661310277}} +:::MLLOG {"namespace": "", "time_ms": 1784894085677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10770942270755768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 317194240, "lr": 1.6131666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894102035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1091964840888977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 317849600, "lr": 1.6165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894119175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11342395097017288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 318504960, "lr": 1.6198333333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894127543, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 318832640, "epoch_num": 0.1391774532552267}} +:::MLLOG {"namespace": "", "time_ms": 1784894127544, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 318832640, "epoch_num": 0.1391774532552267}} +:::MLLOG {"namespace": "", "time_ms": 1784894151346, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7831977605819702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 318832640, "epoch_num": 0.1391774532552267}} +:::MLLOG {"namespace": "", "time_ms": 1784894151346, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 318832640, "epoch_num": 0.1391774532552267}} +:::MLLOG {"namespace": "", "time_ms": 1784894151346, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 318832640, "epoch_num": 0.1391774532552267}} +:::MLLOG {"namespace": "", "time_ms": 1784894159750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11460433155298233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 319160320, "lr": 1.6231666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894176170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10474401712417603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 319815680, "lr": 1.6265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894192701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11296282708644867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 320471040, "lr": 1.6298333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894210419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10769301652908325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 321126400, "lr": 1.6331666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894210420, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 321126400, "epoch_num": 0.14017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784894210420, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 321126400, "epoch_num": 0.14017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784894234573, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7831026315689087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 321126400, "epoch_num": 0.14017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784894234573, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 321126400, "epoch_num": 0.14017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784894234574, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 321126400, "epoch_num": 0.14017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784894251068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11067172884941101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 321781760, "lr": 1.6365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894267752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10913459211587906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 322437120, "lr": 1.6398333333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894284538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11297105997800827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 323092480, "lr": 1.6431666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894292847, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 323420160, "epoch_num": 0.14118000653947457}} +:::MLLOG {"namespace": "", "time_ms": 1784894292848, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 323420160, "epoch_num": 0.14118000653947457}} +:::MLLOG {"namespace": "", "time_ms": 1784894316682, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7838637828826904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 323420160, "epoch_num": 0.14118000653947457}} +:::MLLOG {"namespace": "", "time_ms": 1784894316683, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 323420160, "epoch_num": 0.14118000653947457}} +:::MLLOG {"namespace": "", "time_ms": 1784894316683, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 323420160, "epoch_num": 0.14118000653947457}} +:::MLLOG {"namespace": "", "time_ms": 1784894325137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10910353064537048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 323747840, "lr": 1.6465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894342045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10494019091129303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 324403200, "lr": 1.6498333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894358424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1055947095155716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 325058560, "lr": 1.6531666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894375834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11418972909450531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 325713920, "lr": 1.6565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894375835, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 325713920, "epoch_num": 0.1421812831815985}} +:::MLLOG {"namespace": "", "time_ms": 1784894375835, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 325713920, "epoch_num": 0.1421812831815985}} +:::MLLOG {"namespace": "", "time_ms": 1784894399436, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7837482690811157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 325713920, "epoch_num": 0.1421812831815985}} +:::MLLOG {"namespace": "", "time_ms": 1784894399436, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 325713920, "epoch_num": 0.1421812831815985}} +:::MLLOG {"namespace": "", "time_ms": 1784894399436, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 325713920, "epoch_num": 0.1421812831815985}} +:::MLLOG {"namespace": "", "time_ms": 1784894416274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1097169816493988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 326369280, "lr": 1.6598333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894432783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11206197738647461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327024640, "lr": 1.6631666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894449819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10756146162748337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680000, "lr": 1.6665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894458198, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 328007680, "epoch_num": 0.14318255982372244}} +:::MLLOG {"namespace": "", "time_ms": 1784894458198, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 328007680, "epoch_num": 0.14318255982372244}} +:::MLLOG {"namespace": "", "time_ms": 1784894481896, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7839120626449585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 328007680, "epoch_num": 0.14318255982372244}} +:::MLLOG {"namespace": "", "time_ms": 1784894481897, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 328007680, "epoch_num": 0.14318255982372244}} +:::MLLOG {"namespace": "", "time_ms": 1784894481897, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 328007680, "epoch_num": 0.14318255982372244}} +:::MLLOG {"namespace": "", "time_ms": 1784894490135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11348499357700348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 328335360, "lr": 1.6698333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894506896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10823830962181091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 328990720, "lr": 1.6731666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894523864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11373060941696167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 329646080, "lr": 1.6765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894540860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10907566547393799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 330301440, "lr": 1.6798333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894540860, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 330301440, "epoch_num": 0.14418383646584637}} +:::MLLOG {"namespace": "", "time_ms": 1784894540861, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 330301440, "epoch_num": 0.14418383646584637}} +:::MLLOG {"namespace": "", "time_ms": 1784894564170, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7845208644866943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 330301440, "epoch_num": 0.14418383646584637}} +:::MLLOG {"namespace": "", "time_ms": 1784894564170, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 330301440, "epoch_num": 0.14418383646584637}} +:::MLLOG {"namespace": "", "time_ms": 1784894564170, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 330301440, "epoch_num": 0.14418383646584637}} +:::MLLOG {"namespace": "", "time_ms": 1784894580803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11450023204088211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 330956800, "lr": 1.6831666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894598155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11260047554969788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 331612160, "lr": 1.6865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894614654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10361038893461227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 332267520, "lr": 1.6898333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894622925, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 332595200, "epoch_num": 0.1451851131079703}} +:::MLLOG {"namespace": "", "time_ms": 1784894622926, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 332595200, "epoch_num": 0.1451851131079703}} +:::MLLOG {"namespace": "", "time_ms": 1784894647066, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7840016484260559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 332595200, "epoch_num": 0.1451851131079703}} +:::MLLOG {"namespace": "", "time_ms": 1784894647066, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 332595200, "epoch_num": 0.1451851131079703}} +:::MLLOG {"namespace": "", "time_ms": 1784894647066, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 332595200, "epoch_num": 0.1451851131079703}} +:::MLLOG {"namespace": "", "time_ms": 1784894655429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10751772671937943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 332922880, "lr": 1.6931666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894671684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1047401949763298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 333578240, "lr": 1.6964999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894694274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10799053311347961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 334233600, "lr": 1.6998333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894710727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10173635929822922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 334888960, "lr": 1.7031666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894710728, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 334888960, "epoch_num": 0.14618638975009424}} +:::MLLOG {"namespace": "", "time_ms": 1784894710728, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 334888960, "epoch_num": 0.14618638975009424}} +:::MLLOG {"namespace": "", "time_ms": 1784894734153, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7845655679702759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 334888960, "epoch_num": 0.14618638975009424}} +:::MLLOG {"namespace": "", "time_ms": 1784894734153, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 334888960, "epoch_num": 0.14618638975009424}} +:::MLLOG {"namespace": "", "time_ms": 1784894734153, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 334888960, "epoch_num": 0.14618638975009424}} +:::MLLOG {"namespace": "", "time_ms": 1784894750289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10583449900150299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 335544320, "lr": 1.7064999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894771788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1067437157034874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 336199680, "lr": 1.7098333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894788277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10887971520423889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 336855040, "lr": 1.7131666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894796656, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 337182720, "epoch_num": 0.14718766639221817}} +:::MLLOG {"namespace": "", "time_ms": 1784894796657, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 337182720, "epoch_num": 0.14718766639221817}} +:::MLLOG {"namespace": "", "time_ms": 1784894820733, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7840965390205383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 337182720, "epoch_num": 0.14718766639221817}} +:::MLLOG {"namespace": "", "time_ms": 1784894820733, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 337182720, "epoch_num": 0.14718766639221817}} +:::MLLOG {"namespace": "", "time_ms": 1784894820733, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 337182720, "epoch_num": 0.14718766639221817}} +:::MLLOG {"namespace": "", "time_ms": 1784894829122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10034190863370895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 337510400, "lr": 1.7164999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894845317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10226625949144363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 338165760, "lr": 1.7198333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894862490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10781531780958176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 338821120, "lr": 1.7231666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894886494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10244381427764893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 339476480, "lr": 1.7265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894886495, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 339476480, "epoch_num": 0.1481889430343421}} +:::MLLOG {"namespace": "", "time_ms": 1784894886495, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 339476480, "epoch_num": 0.1481889430343421}} +:::MLLOG {"namespace": "", "time_ms": 1784894909050, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7843897938728333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 339476480, "epoch_num": 0.1481889430343421}} +:::MLLOG {"namespace": "", "time_ms": 1784894909051, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 339476480, "epoch_num": 0.1481889430343421}} +:::MLLOG {"namespace": "", "time_ms": 1784894909051, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 339476480, "epoch_num": 0.1481889430343421}} +:::MLLOG {"namespace": "", "time_ms": 1784894925476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10224098712205887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 340131840, "lr": 1.7298333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894951904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10383173078298569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 340787200, "lr": 1.7331666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894968334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10637040436267853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 341442560, "lr": 1.7364999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894979345, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 341770240, "epoch_num": 0.14919021967646603}} +:::MLLOG {"namespace": "", "time_ms": 1784894979346, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 341770240, "epoch_num": 0.14919021967646603}} +:::MLLOG {"namespace": "", "time_ms": 1784895002977, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7848254442214966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 341770240, "epoch_num": 0.14919021967646603}} +:::MLLOG {"namespace": "", "time_ms": 1784895002977, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 341770240, "epoch_num": 0.14919021967646603}} +:::MLLOG {"namespace": "", "time_ms": 1784895002978, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 341770240, "epoch_num": 0.14919021967646603}} +:::MLLOG {"namespace": "", "time_ms": 1784895011384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10392780601978302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 342097920, "lr": 1.7398333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895027675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10137007385492325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 342753280, "lr": 1.7431666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895052992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10645703971385956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 343408640, "lr": 1.7465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895076821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085062026977539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 344064000, "lr": 1.7498333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895076822, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 344064000, "epoch_num": 0.15019149631858997}} +:::MLLOG {"namespace": "", "time_ms": 1784895076822, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 344064000, "epoch_num": 0.15019149631858997}} +:::MLLOG {"namespace": "", "time_ms": 1784895100486, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7848402857780457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 344064000, "epoch_num": 0.15019149631858997}} +:::MLLOG {"namespace": "", "time_ms": 1784895100486, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 344064000, "epoch_num": 0.15019149631858997}} +:::MLLOG {"namespace": "", "time_ms": 1784895100486, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 344064000, "epoch_num": 0.15019149631858997}} +:::MLLOG {"namespace": "", "time_ms": 1784895117383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10616272687911987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 344719360, "lr": 1.7531666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895136003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10741312801837921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 345374720, "lr": 1.7564999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895152358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.100556880235672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 346030080, "lr": 1.7598333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895161137, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 346357760, "epoch_num": 0.1511927729607139}} +:::MLLOG {"namespace": "", "time_ms": 1784895161137, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 346357760, "epoch_num": 0.1511927729607139}} +:::MLLOG {"namespace": "", "time_ms": 1784895184555, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7852123379707336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 346357760, "epoch_num": 0.1511927729607139}} +:::MLLOG {"namespace": "", "time_ms": 1784895184555, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 346357760, "epoch_num": 0.1511927729607139}} +:::MLLOG {"namespace": "", "time_ms": 1784895184556, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 346357760, "epoch_num": 0.1511927729607139}} +:::MLLOG {"namespace": "", "time_ms": 1784895196088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10533897578716278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 346685440, "lr": 1.7631666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895213085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10342331975698471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 347340800, "lr": 1.7665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895237115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10680028051137924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 347996160, "lr": 1.7698333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895253681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10101582854986191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 348651520, "lr": 1.7731666666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895260087, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 348651520, "epoch_num": 0.15219404960283783}} +:::MLLOG {"namespace": "", "time_ms": 1784895260087, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 348651520, "epoch_num": 0.15219404960283783}} +:::MLLOG {"namespace": "", "time_ms": 1784895282886, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7849647998809814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 348651520, "epoch_num": 0.15219404960283783}} +:::MLLOG {"namespace": "", "time_ms": 1784895282886, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 348651520, "epoch_num": 0.15219404960283783}} +:::MLLOG {"namespace": "", "time_ms": 1784895282886, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 348651520, "epoch_num": 0.15219404960283783}} +:::MLLOG {"namespace": "", "time_ms": 1784895299399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10905459523200989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 349306880, "lr": 1.7764999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895316052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10746116191148758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 349962240, "lr": 1.7798333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895332587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10431288182735443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 350617600, "lr": 1.7831666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895340962, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 350945280, "epoch_num": 0.15319532624496177}} +:::MLLOG {"namespace": "", "time_ms": 1784895340964, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 350945280, "epoch_num": 0.15319532624496177}} +:::MLLOG {"namespace": "", "time_ms": 1784895364659, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7851539850234985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 350945280, "epoch_num": 0.15319532624496177}} +:::MLLOG {"namespace": "", "time_ms": 1784895364659, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 350945280, "epoch_num": 0.15319532624496177}} +:::MLLOG {"namespace": "", "time_ms": 1784895364659, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 350945280, "epoch_num": 0.15319532624496177}} +:::MLLOG {"namespace": "", "time_ms": 1784895372993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10486442595720291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 351272960, "lr": 1.7865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895389690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10076399892568588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 351928320, "lr": 1.7898333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895406512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10032033920288086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 352583680, "lr": 1.7931666666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895422632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10564962029457092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 353239040, "lr": 1.7964999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895422708, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 353239040, "epoch_num": 0.1541966028870857}} +:::MLLOG {"namespace": "", "time_ms": 1784895422709, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 353239040, "epoch_num": 0.1541966028870857}} +:::MLLOG {"namespace": "", "time_ms": 1784895446994, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7852594256401062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 353239040, "epoch_num": 0.1541966028870857}} +:::MLLOG {"namespace": "", "time_ms": 1784895446994, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 353239040, "epoch_num": 0.1541966028870857}} +:::MLLOG {"namespace": "", "time_ms": 1784895446994, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 353239040, "epoch_num": 0.1541966028870857}} +:::MLLOG {"namespace": "", "time_ms": 1784895464592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10632521659135818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 353894400, "lr": 1.7998333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895481701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10607919096946716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 354549760, "lr": 1.8031666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895498238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10602600872516632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 355205120, "lr": 1.8065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895506415, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 355532800, "epoch_num": 0.15519787952920963}} +:::MLLOG {"namespace": "", "time_ms": 1784895506415, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 355532800, "epoch_num": 0.15519787952920963}} +:::MLLOG {"namespace": "", "time_ms": 1784895530083, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7856638431549072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 355532800, "epoch_num": 0.15519787952920963}} +:::MLLOG {"namespace": "", "time_ms": 1784895530084, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 355532800, "epoch_num": 0.15519787952920963}} +:::MLLOG {"namespace": "", "time_ms": 1784895530084, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 355532800, "epoch_num": 0.15519787952920963}} +:::MLLOG {"namespace": "", "time_ms": 1784895538135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10370985418558121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 355860480, "lr": 1.8098333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895554716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10173111408948898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 356515840, "lr": 1.8131666666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895571084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1026175320148468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 357171200, "lr": 1.8164999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895594843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10742859542369843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 357826560, "lr": 1.8198333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895594844, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 357826560, "epoch_num": 0.15619915617133356}} +:::MLLOG {"namespace": "", "time_ms": 1784895594844, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 357826560, "epoch_num": 0.15619915617133356}} +:::MLLOG {"namespace": "", "time_ms": 1784895617619, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7857645153999329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 357826560, "epoch_num": 0.15619915617133356}} +:::MLLOG {"namespace": "", "time_ms": 1784895617619, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 357826560, "epoch_num": 0.15619915617133356}} +:::MLLOG {"namespace": "", "time_ms": 1784895617620, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 357826560, "epoch_num": 0.15619915617133356}} +:::MLLOG {"namespace": "", "time_ms": 1784895634381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10120522975921631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 358481920, "lr": 1.8231666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895651118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10234185308218002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 359137280, "lr": 1.8265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895668008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10315757989883423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 359792640, "lr": 1.8298333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895676390, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 360120320, "epoch_num": 0.1572004328134575}} +:::MLLOG {"namespace": "", "time_ms": 1784895676391, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 360120320, "epoch_num": 0.1572004328134575}} +:::MLLOG {"namespace": "", "time_ms": 1784895700249, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7861076593399048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 360120320, "epoch_num": 0.1572004328134575}} +:::MLLOG {"namespace": "", "time_ms": 1784895700249, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 360120320, "epoch_num": 0.1572004328134575}} +:::MLLOG {"namespace": "", "time_ms": 1784895700250, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 360120320, "epoch_num": 0.1572004328134575}} +:::MLLOG {"namespace": "", "time_ms": 1784895708190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10542351007461548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 360448000, "lr": 1.8331666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895725119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084589809179306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 361103360, "lr": 1.8365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895742095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10876689851284027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 361758720, "lr": 1.8398333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895758764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11057984828948975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 362414080, "lr": 1.8431666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895758764, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 362414080, "epoch_num": 0.15820170945558143}} +:::MLLOG {"namespace": "", "time_ms": 1784895758765, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 362414080, "epoch_num": 0.15820170945558143}} +:::MLLOG {"namespace": "", "time_ms": 1784895782371, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7862517237663269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 362414080, "epoch_num": 0.15820170945558143}} +:::MLLOG {"namespace": "", "time_ms": 1784895782371, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 362414080, "epoch_num": 0.15820170945558143}} +:::MLLOG {"namespace": "", "time_ms": 1784895782372, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 362414080, "epoch_num": 0.15820170945558143}} +:::MLLOG {"namespace": "", "time_ms": 1784895798943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10429901629686356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 363069440, "lr": 1.8465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895815500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10907542705535889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 363724800, "lr": 1.8498333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895832093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10790199786424637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 364380160, "lr": 1.8531666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895840249, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 364707840, "epoch_num": 0.15920298609770536}} +:::MLLOG {"namespace": "", "time_ms": 1784895840249, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 364707840, "epoch_num": 0.15920298609770536}} +:::MLLOG {"namespace": "", "time_ms": 1784895863719, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7864620685577393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 364707840, "epoch_num": 0.15920298609770536}} +:::MLLOG {"namespace": "", "time_ms": 1784895863719, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 364707840, "epoch_num": 0.15920298609770536}} +:::MLLOG {"namespace": "", "time_ms": 1784895863719, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 364707840, "epoch_num": 0.15920298609770536}} +:::MLLOG {"namespace": "", "time_ms": 1784895872060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10829105973243713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 365035520, "lr": 1.8565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895888779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11081413179636002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 365690880, "lr": 1.8598333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895905708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10451792180538177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 366346240, "lr": 1.8631666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895922603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10537559539079666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 367001600, "lr": 1.8665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895922603, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 367001600, "epoch_num": 0.1602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784895922604, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 367001600, "epoch_num": 0.1602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784895946715, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7865036129951477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 367001600, "epoch_num": 0.1602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784895946715, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 367001600, "epoch_num": 0.1602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784895946715, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 367001600, "epoch_num": 0.1602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784895963750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10692057758569717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 367656960, "lr": 1.869833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895980268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1038617417216301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 368312320, "lr": 1.8731666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895998072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11203882098197937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 368967680, "lr": 1.8765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784896006715, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 369295360, "epoch_num": 0.16120553938195323}} +:::MLLOG {"namespace": "", "time_ms": 1784896006716, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 369295360, "epoch_num": 0.16120553938195323}} +:::MLLOG {"namespace": "", "time_ms": 1784896030632, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7866503596305847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 369295360, "epoch_num": 0.16120553938195323}} +:::MLLOG {"namespace": "", "time_ms": 1784896030633, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 369295360, "epoch_num": 0.16120553938195323}} +:::MLLOG {"namespace": "", "time_ms": 1784896030633, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 369295360, "epoch_num": 0.16120553938195323}} +:::MLLOG {"namespace": "", "time_ms": 1784896039097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11553560197353363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 369623040, "lr": 1.8798333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784896055810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10953158885240555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 370278400, "lr": 1.8831666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784896072447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11304080486297607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 370933760, "lr": 1.8865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784896089339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11092185229063034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 371589120, "lr": 1.889833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784896089340, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 371589120, "epoch_num": 0.16220681602407716}} +:::MLLOG {"namespace": "", "time_ms": 1784896089340, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 371589120, "epoch_num": 0.16220681602407716}} +:::MLLOG {"namespace": "", "time_ms": 1784896113116, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.786654531955719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 371589120, "epoch_num": 0.16220681602407716}} +:::MLLOG {"namespace": "", "time_ms": 1784896113117, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 371589120, "epoch_num": 0.16220681602407716}} +:::MLLOG {"namespace": "", "time_ms": 1784896113117, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 371589120, "epoch_num": 0.16220681602407716}} +:::MLLOG {"namespace": "", "time_ms": 1784896129885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10873869061470032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 372244480, "lr": 1.8931666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784896146654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11083774268627167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 372899840, "lr": 1.8965e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784896163473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11291994899511337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 373555200, "lr": 1.8998333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784896171878, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 373882880, "epoch_num": 0.1632080926662011}} +:::MLLOG {"namespace": "", "time_ms": 1784896171878, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 373882880, "epoch_num": 0.1632080926662011}} +:::MLLOG {"namespace": "", "time_ms": 1784896195678, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7868607044219971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 373882880, "epoch_num": 0.1632080926662011}} +:::MLLOG {"namespace": "", "time_ms": 1784896195678, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 373882880, "epoch_num": 0.1632080926662011}} +:::MLLOG {"namespace": "", "time_ms": 1784896195679, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 373882880, "epoch_num": 0.1632080926662011}} +:::MLLOG {"namespace": "", "time_ms": 1784896204149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1132204607129097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 374210560, "lr": 1.9031666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784896220997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10438196361064911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 374865920, "lr": 1.9065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784896237640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10985841602087021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 375521280, "lr": 1.9098333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784896255470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1123722568154335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 376176640, "lr": 1.9131666666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784896255470, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 376176640, "epoch_num": 0.16420936930832503}} +:::MLLOG {"namespace": "", "time_ms": 1784896255471, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 376176640, "epoch_num": 0.16420936930832503}} +:::MLLOG {"namespace": "", "time_ms": 1784896279291, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7867296934127808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 376176640, "epoch_num": 0.16420936930832503}} +:::MLLOG {"namespace": "", "time_ms": 1784896279292, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 376176640, "epoch_num": 0.16420936930832503}} +:::MLLOG {"namespace": "", "time_ms": 1784896279292, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 376176640, "epoch_num": 0.16420936930832503}} +:::MLLOG {"namespace": "", "time_ms": 1784896296208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11039707064628601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 376832000, "lr": 1.9165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784896313252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10585296899080276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 377487360, "lr": 1.9198333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784896330128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10862220078706741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 378142720, "lr": 1.9231666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784896338698, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 378470400, "epoch_num": 0.16521064595044896}} +:::MLLOG {"namespace": "", "time_ms": 1784896338699, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 378470400, "epoch_num": 0.16521064595044896}} +:::MLLOG {"namespace": "", "time_ms": 1784896362530, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7875293493270874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 378470400, "epoch_num": 0.16521064595044896}} +:::MLLOG {"namespace": "", "time_ms": 1784896362531, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 378470400, "epoch_num": 0.16521064595044896}} +:::MLLOG {"namespace": "", "time_ms": 1784896362531, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 378470400, "epoch_num": 0.16521064595044896}} +:::MLLOG {"namespace": "", "time_ms": 1784896370867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10947298258543015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 378798080, "lr": 1.9265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784896387801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11659389734268188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 379453440, "lr": 1.929833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784896405201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11182478070259094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 380108800, "lr": 1.9331666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784896422358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11455091834068298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 380764160, "lr": 1.9365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784896422359, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 380764160, "epoch_num": 0.1662119225925729}} +:::MLLOG {"namespace": "", "time_ms": 1784896422359, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 380764160, "epoch_num": 0.1662119225925729}} diff --git a/recommendation_v4/rcp_logs/gbs_32768/seed430583489.log b/recommendation_v4/rcp_logs/gbs_32768/seed430583489.log new file mode 100644 index 000000000..8aa8ebfa9 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_32768/seed430583489.log @@ -0,0 +1,1333 @@ +:::MLLOG {"namespace": "", "time_ms": 1784872301635, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784872301636, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784872325188, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784872325189, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784872325189, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784872325189, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784872325189, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784872325189, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784872325189, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784872325189, "event_type": "POINT_IN_TIME", "key": "seed", "value": 430583489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784872325189, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784872325189, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784872325189, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784872325190, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784872325190, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784872325191, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784873099877, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784873099878, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784873100180, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784873381678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13909201323986053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.1666666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784873397519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13892775774002075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.4999999999999995e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784873412470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13894310593605042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784873419914, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784873419914, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784873472090, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5016211867332458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784873472091, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784873472091, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784873479617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13894794881343842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3166666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873494196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13903319835662842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873509455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387622058391571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9833333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873524358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13880331814289093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873524358, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784873524359, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784873550549, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5017668008804321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784873550549, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784873550549, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784873565483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13882702589035034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873580477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387886255979538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873595929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13869570195674896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873603097, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784873603097, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784873629206, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5019413232803345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784873629207, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784873629207, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784873637115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385650932788849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873652183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13865023851394653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873667542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13857436180114746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873682984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13836699724197388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873682985, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784873682985, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784873709291, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5021404027938843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784873709292, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784873709292, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784873724386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13842740654945374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873739710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379508376121521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873755395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13824157416820526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873762995, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784873762996, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784873789491, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5023781061172485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784873789492, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784873789492, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784873796759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13759472966194153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873812706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13766053318977356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873827683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13741396367549896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873843161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13733963668346405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873843162, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784873843163, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784873871246, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5026318430900574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784873871247, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784873871247, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784873886181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13737431168556213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.316666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873901308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374024599790573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873917732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13677534461021423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873925304, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784873925304, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784873952360, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5028992891311646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784873952361, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784873952361, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784873960045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373516321182251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873976658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13646158576011658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873992178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13682430982589722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784874009073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13719674944877625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784874009074, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784874009074, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784874035858, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5032258033752441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784874035859, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784874035859, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784874051159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.136397123336792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784874068002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13519124686717987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.983333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784874083337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13669180870056152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874090977, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784874090977, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784874118185, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5035300850868225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784874118186, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784874118186, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784874125767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13347268104553223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874142046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13495463132858276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874157220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350741684436798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874172297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13534510135650635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874172369, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784874172369, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784874201807, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5038523077964783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784874201808, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784874201808, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784874217518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13393738865852356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874232793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13472744822502136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874248035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13532942533493042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874256855, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784874256856, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784874284366, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5041912198066711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784874284367, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784874284367, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784874292020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13576596975326538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874307296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13494935631752014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874322814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322031319141388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874339797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13467763364315033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874339797, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784874339798, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784874365531, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5046853423118591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784874365531, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784874365531, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784874381315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13384094834327698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874397020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135206937789917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874413655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350545585155487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874421668, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784874421669, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784874448105, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5055559277534485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784874448106, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784874448106, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784874455711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13631513714790344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874471306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13350194692611694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874488005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13435465097427368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874503827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321951150894165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874503894, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784874503894, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784874530972, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5064919590950012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784874530972, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784874530972, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784874546272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13284419476985931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874563180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13114963471889496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874579260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312779366970062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874587339, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784874587340, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784874613817, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5074946284294128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784874613818, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784874613818, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784874621557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13836759328842163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874636995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13103565573692322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874653717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12998874485492706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874669510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13041874766349792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874669511, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784874669511, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784874695403, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5087280869483948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784874695404, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784874695404, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784874711038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327371597290039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874726532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12974074482917786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874743301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13072185218334198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874751059, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784874751060, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784874776830, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5100922584533691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784874776830, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784874776831, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784874784707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13152679800987244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874800485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351388543844223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874816193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13307194411754608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874833250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13563579320907593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874833251, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784874833251, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784874859516, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5116199851036072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784874859516, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784874859517, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784874875490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13516424596309662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874891289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325656920671463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1649999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874906893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13323219120502472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874914658, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784874914658, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784874941674, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.513173520565033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784874941675, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784874941675, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784874950629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363818198442459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874967218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13523449003696442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874983696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362694650888443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874999897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13640615344047546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874999898, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784874999898, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784875027393, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5146539211273193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784875027394, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784875027394, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784875044835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13481596112251282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875061430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13084128499031067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875077516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13618192076683044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875085614, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784875085615, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784875112993, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.51615309715271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784875112994, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784875112994, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784875120955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13596248626708984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875138349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13246837258338928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875154592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13261884450912476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875170785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1307477355003357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875170786, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784875170786, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784875197116, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5175656676292419, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784875197116, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784875197117, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784875212834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13103370368480682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875229662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13815760612487793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875246283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12957757711410522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875254246, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784875254246, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784875280443, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5191511511802673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784875280444, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784875280444, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784875288181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12937575578689575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875303905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13473454117774963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875319606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13457587361335754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875336588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304568499326706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875336588, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784875336589, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784875363376, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5210553407669067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784875363376, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784875363377, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784875379492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12812873721122742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875395602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13195113837718964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875411766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13556939363479614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875419728, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784875419729, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784875446190, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5231814980506897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784875446190, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784875446190, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784875454060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13652411103248596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875471086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335722953081131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875487190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13214372098445892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875503133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1309097707271576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875503133, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784875503134, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784875529715, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.525613009929657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784875529715, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784875529715, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784875545779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12960535287857056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875561468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324344426393509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875578335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353551149368286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875586711, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784875586712, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784875612337, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.528427004814148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784875612338, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784875612338, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784875620286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13194602727890015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875636762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334417760372162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875653003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13498324155807495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875668940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13451890647411346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875668941, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784875668941, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784875695923, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5316962003707886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784875695924, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784875695924, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784875712085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359575092792511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875729312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13395200669765472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875746043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13313175737857819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3650000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875754112, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784875754113, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784875780866, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5352805256843567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784875780866, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784875780867, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784875789289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14031895995140076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875805462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351775974035263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875822116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343766748905182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4649999999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875838811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351994276046753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875838811, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784875838812, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784875865314, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5390453934669495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784875865314, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784875865315, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784875881851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352241188287735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875898352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13323475420475006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.5649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875914489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13242769241333008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875923685, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784875923686, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784875951273, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5433268547058105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784875951274, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784875951274, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784875959490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13241755962371826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875976045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13080596923828125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875992399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135768324136734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876008787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13225997984409332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876008788, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784876008788, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784876035002, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5484570860862732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784876035003, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784876035003, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784876051222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13463056087493896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876068458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301889717578888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876085032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318211853504181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876093161, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784876093161, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784876119487, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5553075075149536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784876119487, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784876119487, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784876127589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13213148713111877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876144111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13059677183628082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876160318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13295882940292358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876177098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13250571489334106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876177099, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784876177099, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784876203696, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5642054677009583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784876203696, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784876203697, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784876220025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13198570907115936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876236322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1282147765159607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876252287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1339353770017624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876260496, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784876260497, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784876287005, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5738388299942017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784876287005, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784876287005, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784876295313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13451720774173737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876311391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12665829062461853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876328432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1266464740037918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876344910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13279074430465698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876344911, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784876344911, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784876370412, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5861564874649048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784876370413, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784876370413, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784876386651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12907230854034424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876402918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12835894525051117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876419515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12784025073051453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876427262, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784876427262, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784876453591, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5952786207199097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784876453592, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784876453592, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784876461633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12975049018859863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876478755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1272677481174469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876495310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12982437014579773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876511780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12810683250427246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876511780, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784876511781, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784876537901, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6071932315826416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784876537901, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784876537901, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784876554108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12639012932777405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876570924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1293286681175232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4983333333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876586949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13241958618164062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876594924, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784876594924, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784876621499, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6194169521331787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784876621499, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784876621500, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784876630477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13077670335769653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876647555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1261427402496338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876664476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.130892813205719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876681169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12603875994682312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876681169, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784876681170, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784876707326, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6325674653053284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784876707326, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784876707327, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784876724291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12845271825790405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876740662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12954001128673553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876758213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12751400470733643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876766764, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784876766765, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784876793464, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.647455632686615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784876793465, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784876793465, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784876801916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12670348584651947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876818721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12666654586791992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876835867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12688620388507843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876852337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12609434127807617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.898333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876852338, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784876852338, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784876879542, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6624293923377991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784876879542, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784876879542, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784876897034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12025047093629837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876913979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11976930499076843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876930363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1178336888551712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876938623, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784876938623, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784876964796, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.669293224811554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784876964796, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784876964797, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784876972911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11956385523080826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876989628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11664720624685287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877006159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12111291289329529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877022155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11840757727622986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877022155, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784877022155, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784877050288, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6982882618904114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784877050289, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784877050289, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784877068097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11975301802158356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877084754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11084087193012238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.198333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877101339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11664748936891556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877109578, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784877109579, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784877135992, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7255589365959167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784877135993, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784877135993, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784877144129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11746588349342346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877161100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11718057841062546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.298333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877177650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11223042011260986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877193671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11557135730981827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877193764, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784877193764, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784877223195, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7465112805366516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784877223195, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784877223195, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784877240034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11420091986656189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.398333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877256565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11379703879356384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877273172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10681147873401642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877281394, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784877281395, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784877307199, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7559648752212524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784877307200, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784877307200, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784877315401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11123735457658768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.498333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877332206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11146212369203568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.531666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877348570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10887308418750763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877365633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11122060567140579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.598333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877365634, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784877365634, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784877393063, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7643150687217712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784877393064, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784877393064, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784877409618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10974457114934921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877426171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10870744287967682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877442808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11330638080835342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877450991, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784877450991, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784877477055, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7670736908912659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784877477055, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784877477056, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784877485127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10240675508975983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877501792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11127492785453796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877517897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10474755614995956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877534237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10821051150560379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877534238, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784877534238, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784877562426, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7690516114234924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784877562427, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784877562427, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784877579097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1119849756360054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877595643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10849833488464355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877612233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10750812292098999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877620222, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784877620223, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784877646541, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7701050639152527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784877646542, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784877646542, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784877654842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10833859443664551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877671785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10993903130292892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877688542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11104696989059448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877704495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10385443270206451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877704496, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784877704496, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784877731842, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.770939826965332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784877731843, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784877731843, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784877750094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11355269700288773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877766888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11799377202987671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877783918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11464808881282806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877792360, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784877792361, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784877818883, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7713145017623901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784877818883, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784877818883, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784877827192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11407429724931717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877844206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11376967281103134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877860716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11093425005674362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877876946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11268207430839539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877876947, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784877876947, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784877905569, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7717317342758179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784877905569, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784877905569, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784877922767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11706589162349701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877939704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11688587814569473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877956905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11009058356285095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877965467, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784877965467, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784877992045, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7720302939414978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784877992046, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784877992046, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784878000656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11448546499013901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878017806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11504314839839935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878034630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11041316390037537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878052703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10364827513694763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878052704, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784878052705, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784878079309, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7721851468086243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784878079309, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784878079310, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784878096033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10606614500284195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878113029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10557246953248978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878129875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10577499121427536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878138374, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784878138375, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784878165045, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7725644111633301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784878165046, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784878165046, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784878173455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10355091094970703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878190754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1108306348323822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878207555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10703044384717941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878224906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11225683987140656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878224907, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784878224908, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784878251697, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7728180885314941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784878251697, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784878251697, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784878268622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10630032420158386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878285318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10596829652786255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878302151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10112152248620987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878310494, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784878310494, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784878337371, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730467319488525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784878337371, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784878337371, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784878345916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10198060423135757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878362873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10090090334415436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878380025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11094141006469727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878396445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10341700911521912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878396446, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784878396446, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784878423985, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730304002761841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784878423985, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784878423985, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784878441712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10583388805389404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878458857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1058521568775177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878475470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10636318475008011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878484044, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784878484045, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784878510948, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7731840014457703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784878510948, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784878510948, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784878519515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1066451221704483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878536244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09935830533504486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878553322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10763504356145859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878570277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10384012013673782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878570278, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784878570278, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784878597138, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7731637954711914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784878597139, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784878597139, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784878613273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10706353187561035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878630999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10797331482172012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878647927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1038292944431305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878656294, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784878656294, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784878682394, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7733307480812073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784878682394, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784878682394, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784878690618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085740178823471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878707511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1070186197757721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878724196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1115579903125763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878741125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10780944675207138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878741126, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784878741126, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784878767443, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7734217643737793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784878767443, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784878767444, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784878784490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10496828705072403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878800802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10168004781007767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878818347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772636532783508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.564999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878827032, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784878827033, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784878853239, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773438572883606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784878853240, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784878853240, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784878861544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10412139445543289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878878431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09897157549858093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878895597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11386825144290924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.664999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878912461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11046722531318665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878912462, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784878912462, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784878938797, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7734869718551636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784878938798, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784878938798, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784878955623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10606662929058075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878972854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10797479748725891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878989453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1079455018043518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878997547, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784878997547, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784879025021, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7735653519630432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784879025021, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784879025021, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784879034101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.109782874584198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879051667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11437786370515823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879069077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11182302236557007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879086404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11264774203300476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879086404, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784879086405, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784879113376, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7736684679985046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784879113376, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784879113376, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784879130498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10956050455570221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879147970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11354139447212219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.998333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879165211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11017386615276337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879173541, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784879173541, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784879200574, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7737393379211426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784879200574, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784879200574, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784879208937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11037389934062958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879227447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10927202552556992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879244814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11239590495824814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.131666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879262119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11082178354263306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879262120, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784879262120, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784879289135, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738435864448547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784879289136, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784879289136, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784879306413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10991131514310837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879323986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11235972493886948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879341257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1169000193476677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879349745, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784879349746, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784879377010, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738432884216309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784879377010, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784879377010, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784879385236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11463431268930435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879403372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10744787007570267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.331666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879420241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10094116628170013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879437241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10686801373958588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879437242, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784879437242, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784879464266, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7737768292427063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784879464266, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784879464266, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784879481209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10467156022787094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879498137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10596537590026855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879515339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10502307116985321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879524039, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784879524040, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784879550788, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738203406333923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784879550789, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784879550789, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784879559090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10833795368671417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879575375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10309673845767975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879593553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.104722760617733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879610567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725998133420944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.631666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879610568, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784879610568, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784879637527, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7736971378326416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784879637528, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784879637528, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784879654400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10080114752054214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879671453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10385146737098694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879688087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10329112410545349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879696470, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784879696470, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784879723452, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773865818977356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784879723453, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784879723453, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784879732008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10380733013153076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879749272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10296894609928131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879765765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1133434921503067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879783309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10294903814792633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879783310, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784879783310, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784879810767, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7739827036857605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784879810767, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784879810768, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784879827864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1068851575255394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879844728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071871668100357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879861465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10158921033143997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879869725, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784879869726, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784879896217, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7741191983222961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784879896218, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784879896218, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784879904441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10118478536605835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879921202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10026373714208603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879938277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10581312328577042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879955228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10292205959558487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879955228, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784879955229, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784879982248, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7740507125854492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784879982248, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784879982248, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784879998682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10734647512435913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880017183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1035865917801857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880034284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10546319931745529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880042565, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784880042566, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784880069428, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742621302604675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784880069428, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784880069429, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784880077908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10328023880720139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880094910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10801050066947937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880111769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10230499505996704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880129157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10654216259717941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880129158, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784880129158, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784880156270, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742007970809937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784880156271, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784880156271, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784880173660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1019757091999054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880189957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10636185854673386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880207003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10920056700706482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880215747, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784880215748, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784880242507, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742610573768616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784880242508, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784880242508, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784880251134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1114501804113388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880267957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10710757225751877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880284828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10261788964271545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880301821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10220258682966232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880301822, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784880301824, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784880328160, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742795944213867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784880328160, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784880328161, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784880344908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10511620342731476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880362121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10520771145820618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.631666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880379245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1119186282157898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880387675, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784880387675, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784880414655, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7744277715682983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784880414656, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784880414656, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784880422726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10702938586473465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880440267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11252247542142868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880457928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10887999832630157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880475066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11273589730262756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880475067, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784880475069, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784880501376, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746526002883911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784880501376, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784880501377, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784880518606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10873370617628098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.831666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880536015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1137348860502243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880553434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11181853711605072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880562157, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784880562157, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784880588312, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774747371673584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784880588312, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784880588312, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784880597060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11212936043739319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880614064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11174830049276352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880631524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071571335196495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880649613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11055875569581985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1.0031666666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880649614, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784880649614, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784880676253, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7749077081680298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784880676253, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784880676253, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784880693308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11375707387924194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1.0064999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880710585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10850358009338379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1.0098333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880727883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111175537109375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1.0131666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880736626, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784880736627, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784880764073, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752801179885864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784880764073, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784880764074, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784880772885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11132730543613434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1.0165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880790315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10369984805583954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1.0198333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880807133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11025257408618927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1.0231666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880825427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10843939334154129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1.0264999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880825428, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784880825428, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784880851671, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7753132581710815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784880851671, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784880851671, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784880868954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10287339985370636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1.0298333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880886042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1000678688287735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1.0331666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880903277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1017855629324913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1.0365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880911927, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784880911928, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784880938404, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7750768661499023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784880938405, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784880938405, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784880946901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0997554287314415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1.0398333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880963844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10061616450548172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1.0431666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880980965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10210435837507248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1.0464999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880998101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10562942922115326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1.0498333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880998102, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784880998102, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784881025340, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.775119423866272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784881025341, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784881025341, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784881041383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10428306460380554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1.0531666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881059165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10997132211923599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1.0565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881076185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1058502048254013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1.0598333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881084571, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784881084572, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784881111713, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.775197446346283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784881111713, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784881111713, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784881119959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10140497237443924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1.0631666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881136900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10606551915407181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1.0664999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881153705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1003783643245697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1.0698333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881170456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10164888948202133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1.0731666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881170456, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784881170457, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784881197448, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752478122711182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784881197449, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784881197449, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784881214537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1020207479596138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1.0765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881231924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10320638120174408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1.0798333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881248263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10149478167295456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1.0831666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881257509, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784881257509, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784881285632, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7756108045578003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784881285632, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784881285633, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784881294083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10404146462678909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1.0864999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881311075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1067739948630333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1.0898333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881327961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10221640020608902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1.0931666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881345061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10027649998664856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1.0965e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881345062, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784881345062, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784881371765, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7756236791610718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784881371765, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784881371765, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784881388955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10522972792387009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1.0998333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881406053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10139770805835724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1.1031666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881423240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10233891010284424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217579520, "lr": 1.1065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881431951, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784881431951, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784881458109, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.77572101354599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784881458110, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784881458110, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784881466706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11159884184598923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218234880, "lr": 1.1098333333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881483125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10948771238327026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218890240, "lr": 1.1131666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881500330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10742851346731186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219545600, "lr": 1.1165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881517475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10794633626937866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220200960, "lr": 1.1198333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881517475, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784881517476, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784881544309, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7759625911712646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784881544310, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784881544310, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784881561571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10251496732234955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220856320, "lr": 1.1231666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881578972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10442511737346649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221511680, "lr": 1.1265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881596095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1006203293800354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222167040, "lr": 1.1298333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881604802, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784881604803, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784881631611, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7764884233474731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784881631612, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784881631612, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784881640392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10487370193004608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222822400, "lr": 1.1331666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881657733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10407453775405884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223477760, "lr": 1.1365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881674909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10689110308885574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224133120, "lr": 1.1398333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881692266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087162122130394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224788480, "lr": 1.1431666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881692267, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784881692267, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784881719225, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7764785289764404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784881719225, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784881719225, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784881735630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10843661427497864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225443840, "lr": 1.1465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881754034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10833916068077087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226099200, "lr": 1.1498333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881771601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10737835615873337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226754560, "lr": 1.1531666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881780242, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784881780243, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784881806787, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7768930196762085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784881806788, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784881806788, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784881815224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11229072511196136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227409920, "lr": 1.1565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881832367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10977190732955933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228065280, "lr": 1.1598333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881849420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10680942237377167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228720640, "lr": 1.1631666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881866602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1063469797372818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229376000, "lr": 1.1665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881866603, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784881866604, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784881893546, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7772257328033447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784881893546, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784881893547, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784881911064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10642879456281662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230031360, "lr": 1.1698333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881928347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10493206977844238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230686720, "lr": 1.1731666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881945743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11032192409038544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231342080, "lr": 1.1765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881953922, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784881953923, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784881981358, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7770702242851257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784881981358, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784881981358, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784881989843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10796862095594406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231997440, "lr": 1.1798333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882008314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10963757336139679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 232652800, "lr": 1.1831666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882026242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11005446314811707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233308160, "lr": 1.1865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882043989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11169590055942535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233963520, "lr": 1.1898333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882043989, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784882043990, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784882070839, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7779525518417358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784882070839, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784882070840, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784882088406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11403624713420868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 234618880, "lr": 1.1931666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882105890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11278814822435379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235274240, "lr": 1.1965e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882123327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11101000010967255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235929600, "lr": 1.1998333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882132178, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784882132178, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784882158515, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7781747579574585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784882158515, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784882158515, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784882167103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081387847661972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 236584960, "lr": 1.2031666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882184580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11237704753875732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237240320, "lr": 1.2064999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882201318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11379911750555038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237895680, "lr": 1.2098333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882219671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11854514479637146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 238551040, "lr": 1.2131666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882219672, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784882219672, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784882246627, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7781659364700317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784882246627, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784882246627, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784882264368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11686407774686813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239206400, "lr": 1.2165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882282133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11318659782409668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239861760, "lr": 1.2198333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882299941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11312167346477509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 240517120, "lr": 1.2231666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882308794, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784882308795, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784882335210, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.778802752494812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784882335210, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784882335210, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784882343890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1143522635102272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241172480, "lr": 1.2264999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882361530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11457681655883789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241827840, "lr": 1.2298333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882379001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11425282061100006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 242483200, "lr": 1.2331666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882395833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11731891334056854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243138560, "lr": 1.2365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882395834, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784882395834, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784882423104, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.779157280921936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784882423105, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784882423105, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784882441716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10908744484186172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243793920, "lr": 1.2398333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882459091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1146826297044754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 244449280, "lr": 1.2431666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882476395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11342257261276245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245104640, "lr": 1.2464999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882485498, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784882485499, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784882511331, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7786187529563904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784882511331, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784882511331, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784882519968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10992269963026047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245760000, "lr": 1.2498333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882537344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11193664371967316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 246415360, "lr": 1.2531666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882555039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10719948261976242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247070720, "lr": 1.2565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882572658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10701747983694077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247726080, "lr": 1.2598333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882572658, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784882572659, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784882600028, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7791222333908081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784882600028, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784882600028, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784882616959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11362102627754211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 248381440, "lr": 1.2631666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882635741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10865205526351929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249036800, "lr": 1.2664999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882653276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10759884864091873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249692160, "lr": 1.2698333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882661980, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784882661981, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784882689117, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7789472341537476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784882689118, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784882689118, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784882697482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10543449223041534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 250347520, "lr": 1.2731666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882715037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10442384332418442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251002880, "lr": 1.2765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882732521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10383749008178711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251658240, "lr": 1.2798333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882749898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10873693227767944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 252313600, "lr": 1.2831666666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882749899, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784882749900, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784882776612, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7791144847869873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784882776613, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784882776613, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784882794410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11039336025714874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 252968960, "lr": 1.2864999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882812013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10517846792936325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 253624320, "lr": 1.2898333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882828890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10753601789474487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 254279680, "lr": 1.2931666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882838075, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784882838076, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784882866174, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7797883152961731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784882866174, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784882866174, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784882874794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10642007738351822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 254935040, "lr": 1.2965e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882892237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10593937337398529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 255590400, "lr": 1.2998333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882909454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10864928364753723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 256245760, "lr": 1.3031666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882926957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1070941612124443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 256901120, "lr": 1.3065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882926958, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784882926958, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784882953712, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7803550958633423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784882953713, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784882953713, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784882970749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10669394582509995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 257556480, "lr": 1.3098333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882987830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10427002608776093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 258211840, "lr": 1.3131666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883005299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1028098538517952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 258867200, "lr": 1.3165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883014017, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784883014017, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784883040600, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7797175049781799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784883040600, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784883040601, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784883049410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11121314764022827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 259522560, "lr": 1.3198333333333335e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883066244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10581167042255402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 260177920, "lr": 1.3231666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883084106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10480302572250366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 260833280, "lr": 1.3265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883101606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1050860807299614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 261488640, "lr": 1.3298333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883101606, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784883101607, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784883128369, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7802764177322388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784883128369, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784883128370, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784883145872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10296987742185593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 262144000, "lr": 1.3331666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883163326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10099201649427414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 262799360, "lr": 1.3365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883180787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10465098917484283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 263454720, "lr": 1.3398333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883189491, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 263782400, "epoch_num": 0.11514681384425231}} +:::MLLOG {"namespace": "", "time_ms": 1784883189491, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 263782400, "epoch_num": 0.11514681384425231}} +:::MLLOG {"namespace": "", "time_ms": 1784883216007, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7800474762916565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 263782400, "epoch_num": 0.11514681384425231}} +:::MLLOG {"namespace": "", "time_ms": 1784883216007, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 263782400, "epoch_num": 0.11514681384425231}} +:::MLLOG {"namespace": "", "time_ms": 1784883216007, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 263782400, "epoch_num": 0.11514681384425231}} +:::MLLOG {"namespace": "", "time_ms": 1784883224735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10853356122970581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 264110080, "lr": 1.3431666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883242503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10728973150253296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 264765440, "lr": 1.3465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883260364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10451675951480865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 265420800, "lr": 1.3498333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883277710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10300679504871368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 266076160, "lr": 1.3531666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883277711, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 266076160, "epoch_num": 0.11614809048637624}} +:::MLLOG {"namespace": "", "time_ms": 1784883277711, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 266076160, "epoch_num": 0.11614809048637624}} +:::MLLOG {"namespace": "", "time_ms": 1784883305375, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7808418869972229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 266076160, "epoch_num": 0.11614809048637624}} +:::MLLOG {"namespace": "", "time_ms": 1784883305376, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 266076160, "epoch_num": 0.11614809048637624}} +:::MLLOG {"namespace": "", "time_ms": 1784883305376, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 266076160, "epoch_num": 0.11614809048637624}} +:::MLLOG {"namespace": "", "time_ms": 1784883322181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1028703898191452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 266731520, "lr": 1.3565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883341087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10622532665729523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 267386880, "lr": 1.359833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883358538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1077176034450531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 268042240, "lr": 1.3631666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883367386, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 268369920, "epoch_num": 0.11714936712850017}} +:::MLLOG {"namespace": "", "time_ms": 1784883367386, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 268369920, "epoch_num": 0.11714936712850017}} +:::MLLOG {"namespace": "", "time_ms": 1784883394587, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7802652716636658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 268369920, "epoch_num": 0.11714936712850017}} +:::MLLOG {"namespace": "", "time_ms": 1784883394587, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 268369920, "epoch_num": 0.11714936712850017}} +:::MLLOG {"namespace": "", "time_ms": 1784883394587, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 268369920, "epoch_num": 0.11714936712850017}} +:::MLLOG {"namespace": "", "time_ms": 1784883403171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11054477095603943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 268697600, "lr": 1.3665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883420674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10468289256095886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 269352960, "lr": 1.3698333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883438125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1032881811261177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 270008320, "lr": 1.3731666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883455616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10878737270832062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 270663680, "lr": 1.3765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883455617, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 270663680, "epoch_num": 0.1181506437706241}} +:::MLLOG {"namespace": "", "time_ms": 1784883455617, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 270663680, "epoch_num": 0.1181506437706241}} +:::MLLOG {"namespace": "", "time_ms": 1784883482320, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.781043529510498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 270663680, "epoch_num": 0.1181506437706241}} +:::MLLOG {"namespace": "", "time_ms": 1784883482321, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 270663680, "epoch_num": 0.1181506437706241}} +:::MLLOG {"namespace": "", "time_ms": 1784883482321, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 270663680, "epoch_num": 0.1181506437706241}} +:::MLLOG {"namespace": "", "time_ms": 1784883500211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1104065477848053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 271319040, "lr": 1.379833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883517993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11092062294483185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 271974400, "lr": 1.3831666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883535176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10827282071113586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 272629760, "lr": 1.3865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883543708, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 272957440, "epoch_num": 0.11915192041274804}} +:::MLLOG {"namespace": "", "time_ms": 1784883543708, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 272957440, "epoch_num": 0.11915192041274804}} +:::MLLOG {"namespace": "", "time_ms": 1784883571304, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7808769941329956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 272957440, "epoch_num": 0.11915192041274804}} +:::MLLOG {"namespace": "", "time_ms": 1784883571305, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 272957440, "epoch_num": 0.11915192041274804}} +:::MLLOG {"namespace": "", "time_ms": 1784883571305, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 272957440, "epoch_num": 0.11915192041274804}} +:::MLLOG {"namespace": "", "time_ms": 1784883579900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11149346083402634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 273285120, "lr": 1.3898333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883598359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.109781414270401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 273940480, "lr": 1.3931666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883615946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11698111146688461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 274595840, "lr": 1.3965e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883633435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1152462363243103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 275251200, "lr": 1.3998333333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883633435, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 275251200, "epoch_num": 0.12015319705487197}} +:::MLLOG {"namespace": "", "time_ms": 1784883633435, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 275251200, "epoch_num": 0.12015319705487197}} +:::MLLOG {"namespace": "", "time_ms": 1784883659940, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.780853807926178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 275251200, "epoch_num": 0.12015319705487197}} +:::MLLOG {"namespace": "", "time_ms": 1784883659940, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 275251200, "epoch_num": 0.12015319705487197}} +:::MLLOG {"namespace": "", "time_ms": 1784883659940, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 275251200, "epoch_num": 0.12015319705487197}} +:::MLLOG {"namespace": "", "time_ms": 1784883677707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11112719029188156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 275906560, "lr": 1.4031666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883695256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10986240208148956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 276561920, "lr": 1.4065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883713214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11704541742801666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 277217280, "lr": 1.4098333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883722075, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 277544960, "epoch_num": 0.1211544736969959}} +:::MLLOG {"namespace": "", "time_ms": 1784883722075, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 277544960, "epoch_num": 0.1211544736969959}} +:::MLLOG {"namespace": "", "time_ms": 1784883748596, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.781636655330658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 277544960, "epoch_num": 0.1211544736969959}} +:::MLLOG {"namespace": "", "time_ms": 1784883748597, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 277544960, "epoch_num": 0.1211544736969959}} +:::MLLOG {"namespace": "", "time_ms": 1784883748597, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 277544960, "epoch_num": 0.1211544736969959}} +:::MLLOG {"namespace": "", "time_ms": 1784883757549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11222244054079056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 277872640, "lr": 1.4131666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883774716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10855716466903687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 278528000, "lr": 1.4165000000000001e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883793132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11667680740356445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 279183360, "lr": 1.4198333333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883811148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10864514112472534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 279838720, "lr": 1.4231666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883811149, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 279838720, "epoch_num": 0.12215575033911984}} +:::MLLOG {"namespace": "", "time_ms": 1784883811149, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 279838720, "epoch_num": 0.12215575033911984}} +:::MLLOG {"namespace": "", "time_ms": 1784883837931, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7816686630249023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 279838720, "epoch_num": 0.12215575033911984}} +:::MLLOG {"namespace": "", "time_ms": 1784883837931, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 279838720, "epoch_num": 0.12215575033911984}} +:::MLLOG {"namespace": "", "time_ms": 1784883837931, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 279838720, "epoch_num": 0.12215575033911984}} +:::MLLOG {"namespace": "", "time_ms": 1784883855336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11099278181791306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 280494080, "lr": 1.4265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883873087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11088529229164124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 281149440, "lr": 1.4298333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883890778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11520281434059143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 281804800, "lr": 1.4331666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883899551, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 282132480, "epoch_num": 0.12315702698124378}} +:::MLLOG {"namespace": "", "time_ms": 1784883899552, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 282132480, "epoch_num": 0.12315702698124378}} +:::MLLOG {"namespace": "", "time_ms": 1784883925543, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7813778519630432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 282132480, "epoch_num": 0.12315702698124378}} +:::MLLOG {"namespace": "", "time_ms": 1784883925543, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 282132480, "epoch_num": 0.12315702698124378}} +:::MLLOG {"namespace": "", "time_ms": 1784883925543, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 282132480, "epoch_num": 0.12315702698124378}} +:::MLLOG {"namespace": "", "time_ms": 1784883934592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10578251630067825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 282460160, "lr": 1.4365000000000001e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883952469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11305507272481918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 283115520, "lr": 1.4398333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883969955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11061988770961761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 283770880, "lr": 1.4431666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883987160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1111774891614914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 284426240, "lr": 1.4464999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883987161, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 284426240, "epoch_num": 0.12415830362336772}} +:::MLLOG {"namespace": "", "time_ms": 1784883987162, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 284426240, "epoch_num": 0.12415830362336772}} +:::MLLOG {"namespace": "", "time_ms": 1784884016680, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7816402912139893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 284426240, "epoch_num": 0.12415830362336772}} +:::MLLOG {"namespace": "", "time_ms": 1784884016680, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 284426240, "epoch_num": 0.12415830362336772}} +:::MLLOG {"namespace": "", "time_ms": 1784884016680, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 284426240, "epoch_num": 0.12415830362336772}} +:::MLLOG {"namespace": "", "time_ms": 1784884034604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10766953229904175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 285081600, "lr": 1.4498333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884052226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11220499128103256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 285736960, "lr": 1.4531666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884069952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11374041438102722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 286392320, "lr": 1.4564999999999997e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884078870, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 286720000, "epoch_num": 0.12515958026549165}} +:::MLLOG {"namespace": "", "time_ms": 1784884078871, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 286720000, "epoch_num": 0.12515958026549165}} +:::MLLOG {"namespace": "", "time_ms": 1784884105647, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7814568281173706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 286720000, "epoch_num": 0.12515958026549165}} +:::MLLOG {"namespace": "", "time_ms": 1784884105648, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 286720000, "epoch_num": 0.12515958026549165}} +:::MLLOG {"namespace": "", "time_ms": 1784884105648, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 286720000, "epoch_num": 0.12515958026549165}} +:::MLLOG {"namespace": "", "time_ms": 1784884114406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11381067335605621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 287047680, "lr": 1.4598333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884132523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11051641404628754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 287703040, "lr": 1.4631666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884150494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11242207884788513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 288358400, "lr": 1.4664999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884167635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11255357414484024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 289013760, "lr": 1.4698333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884167636, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 289013760, "epoch_num": 0.12616085690761558}} +:::MLLOG {"namespace": "", "time_ms": 1784884167636, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 289013760, "epoch_num": 0.12616085690761558}} +:::MLLOG {"namespace": "", "time_ms": 1784884194339, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7820760011672974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 289013760, "epoch_num": 0.12616085690761558}} +:::MLLOG {"namespace": "", "time_ms": 1784884194339, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 289013760, "epoch_num": 0.12616085690761558}} +:::MLLOG {"namespace": "", "time_ms": 1784884194340, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 289013760, "epoch_num": 0.12616085690761558}} +:::MLLOG {"namespace": "", "time_ms": 1784884211647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10707518458366394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 289669120, "lr": 1.4731666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884228841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10487651824951172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 290324480, "lr": 1.4764999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884247214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10053040832281113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 290979840, "lr": 1.4798333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884255942, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 291307520, "epoch_num": 0.12716213354973951}} +:::MLLOG {"namespace": "", "time_ms": 1784884255943, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 291307520, "epoch_num": 0.12716213354973951}} +:::MLLOG {"namespace": "", "time_ms": 1784884282414, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7816370725631714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 291307520, "epoch_num": 0.12716213354973951}} +:::MLLOG {"namespace": "", "time_ms": 1784884282415, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 291307520, "epoch_num": 0.12716213354973951}} +:::MLLOG {"namespace": "", "time_ms": 1784884282415, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 291307520, "epoch_num": 0.12716213354973951}} +:::MLLOG {"namespace": "", "time_ms": 1784884291228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10478147119283676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 291635200, "lr": 1.4831666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884308628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10847786068916321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 292290560, "lr": 1.4864999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884326034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10464003682136536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 292945920, "lr": 1.4898333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884343433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1052820086479187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 293601280, "lr": 1.4931666666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884343434, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 293601280, "epoch_num": 0.12816341019186345}} +:::MLLOG {"namespace": "", "time_ms": 1784884343434, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 293601280, "epoch_num": 0.12816341019186345}} +:::MLLOG {"namespace": "", "time_ms": 1784884369928, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7815587520599365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 293601280, "epoch_num": 0.12816341019186345}} +:::MLLOG {"namespace": "", "time_ms": 1784884369929, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 293601280, "epoch_num": 0.12816341019186345}} +:::MLLOG {"namespace": "", "time_ms": 1784884369929, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 293601280, "epoch_num": 0.12816341019186345}} +:::MLLOG {"namespace": "", "time_ms": 1784884387040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10738224536180496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 294256640, "lr": 1.4964999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884404599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10345128178596497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 294912000, "lr": 1.4998333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884422398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10768543183803558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 295567360, "lr": 1.5031666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884431247, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 295895040, "epoch_num": 0.12916468683398738}} +:::MLLOG {"namespace": "", "time_ms": 1784884431248, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 295895040, "epoch_num": 0.12916468683398738}} +:::MLLOG {"namespace": "", "time_ms": 1784884457971, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7820292711257935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 295895040, "epoch_num": 0.12916468683398738}} +:::MLLOG {"namespace": "", "time_ms": 1784884457971, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 295895040, "epoch_num": 0.12916468683398738}} +:::MLLOG {"namespace": "", "time_ms": 1784884457971, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 295895040, "epoch_num": 0.12916468683398738}} +:::MLLOG {"namespace": "", "time_ms": 1784884466559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10391265153884888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 296222720, "lr": 1.5064999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884483383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10419527441263199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 296878080, "lr": 1.5098333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884502011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10822319984436035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 297533440, "lr": 1.5131666666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884519649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10928736627101898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 298188800, "lr": 1.5164999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884519649, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 298188800, "epoch_num": 0.1301659634761113}} +:::MLLOG {"namespace": "", "time_ms": 1784884519650, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 298188800, "epoch_num": 0.1301659634761113}} +:::MLLOG {"namespace": "", "time_ms": 1784884546145, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7818660140037537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 298188800, "epoch_num": 0.1301659634761113}} +:::MLLOG {"namespace": "", "time_ms": 1784884546145, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 298188800, "epoch_num": 0.1301659634761113}} +:::MLLOG {"namespace": "", "time_ms": 1784884546145, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 298188800, "epoch_num": 0.1301659634761113}} +:::MLLOG {"namespace": "", "time_ms": 1784884563727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10512958467006683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 298844160, "lr": 1.5198333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884581296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1027807667851448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 299499520, "lr": 1.5231666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884598711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10381969064474106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 300154880, "lr": 1.5265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884607595, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 300482560, "epoch_num": 0.13116724011823525}} +:::MLLOG {"namespace": "", "time_ms": 1784884607596, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 300482560, "epoch_num": 0.13116724011823525}} +:::MLLOG {"namespace": "", "time_ms": 1784884634310, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7819225192070007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 300482560, "epoch_num": 0.13116724011823525}} +:::MLLOG {"namespace": "", "time_ms": 1784884634310, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 300482560, "epoch_num": 0.13116724011823525}} +:::MLLOG {"namespace": "", "time_ms": 1784884634311, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 300482560, "epoch_num": 0.13116724011823525}} +:::MLLOG {"namespace": "", "time_ms": 1784884643122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11020181328058243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 300810240, "lr": 1.5298333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884660493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09947437793016434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 301465600, "lr": 1.5331666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884678021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10074611008167267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 302120960, "lr": 1.5364999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884695773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1067749634385109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 302776320, "lr": 1.5398333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884695774, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 302776320, "epoch_num": 0.13216851676035918}} +:::MLLOG {"namespace": "", "time_ms": 1784884695774, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 302776320, "epoch_num": 0.13216851676035918}} +:::MLLOG {"namespace": "", "time_ms": 1784884722632, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7820176482200623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 302776320, "epoch_num": 0.13216851676035918}} +:::MLLOG {"namespace": "", "time_ms": 1784884722632, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 302776320, "epoch_num": 0.13216851676035918}} +:::MLLOG {"namespace": "", "time_ms": 1784884722632, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 302776320, "epoch_num": 0.13216851676035918}} +:::MLLOG {"namespace": "", "time_ms": 1784884739338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1092187762260437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 303431680, "lr": 1.5431666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884757239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10603762418031693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 304087040, "lr": 1.5465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884774867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10271081328392029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 304742400, "lr": 1.5498333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884783569, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 305070080, "epoch_num": 0.1331697934024831}} +:::MLLOG {"namespace": "", "time_ms": 1784884783569, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 305070080, "epoch_num": 0.1331697934024831}} +:::MLLOG {"namespace": "", "time_ms": 1784884810469, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7823720574378967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 305070080, "epoch_num": 0.1331697934024831}} +:::MLLOG {"namespace": "", "time_ms": 1784884810470, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 305070080, "epoch_num": 0.1331697934024831}} +:::MLLOG {"namespace": "", "time_ms": 1784884810470, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 305070080, "epoch_num": 0.1331697934024831}} +:::MLLOG {"namespace": "", "time_ms": 1784884819247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10143186151981354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 305397760, "lr": 1.5531666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884836571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10721389204263687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 306053120, "lr": 1.5564999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884853968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10487998276948929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 306708480, "lr": 1.5598333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884871175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10511496663093567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 307363840, "lr": 1.5631666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884871175, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 307363840, "epoch_num": 0.13417107004460704}} +:::MLLOG {"namespace": "", "time_ms": 1784884871176, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 307363840, "epoch_num": 0.13417107004460704}} +:::MLLOG {"namespace": "", "time_ms": 1784884897624, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7821642160415649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 307363840, "epoch_num": 0.13417107004460704}} +:::MLLOG {"namespace": "", "time_ms": 1784884897624, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 307363840, "epoch_num": 0.13417107004460704}} +:::MLLOG {"namespace": "", "time_ms": 1784884897625, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 307363840, "epoch_num": 0.13417107004460704}} +:::MLLOG {"namespace": "", "time_ms": 1784884915099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10639236122369766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 308019200, "lr": 1.5665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884932587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0981251671910286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 308674560, "lr": 1.5698333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884950038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10373543947935104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 309329920, "lr": 1.5731666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884958717, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 309657600, "epoch_num": 0.13517234668673098}} +:::MLLOG {"namespace": "", "time_ms": 1784884958717, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 309657600, "epoch_num": 0.13517234668673098}} +:::MLLOG {"namespace": "", "time_ms": 1784884984864, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7820995450019836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 309657600, "epoch_num": 0.13517234668673098}} +:::MLLOG {"namespace": "", "time_ms": 1784884984865, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 309657600, "epoch_num": 0.13517234668673098}} +:::MLLOG {"namespace": "", "time_ms": 1784884984865, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 309657600, "epoch_num": 0.13517234668673098}} +:::MLLOG {"namespace": "", "time_ms": 1784884993538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10774700343608856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 309985280, "lr": 1.5764999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885009912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10133875161409378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 310640640, "lr": 1.5798333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885028277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10858563333749771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 311296000, "lr": 1.5831666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885046143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723941773176193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 311951360, "lr": 1.5865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885046144, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 311951360, "epoch_num": 0.1361736233288549}} +:::MLLOG {"namespace": "", "time_ms": 1784885046144, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 311951360, "epoch_num": 0.1361736233288549}} +:::MLLOG {"namespace": "", "time_ms": 1784885072804, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7826921939849854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 311951360, "epoch_num": 0.1361736233288549}} +:::MLLOG {"namespace": "", "time_ms": 1784885072804, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 311951360, "epoch_num": 0.1361736233288549}} +:::MLLOG {"namespace": "", "time_ms": 1784885072805, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 311951360, "epoch_num": 0.1361736233288549}} +:::MLLOG {"namespace": "", "time_ms": 1784885090404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10498051345348358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 312606720, "lr": 1.5898333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885107960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10607510805130005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 313262080, "lr": 1.5931666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885125583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10513058304786682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 313917440, "lr": 1.5964999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885134245, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 314245120, "epoch_num": 0.13717489997097884}} +:::MLLOG {"namespace": "", "time_ms": 1784885134246, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 314245120, "epoch_num": 0.13717489997097884}} +:::MLLOG {"namespace": "", "time_ms": 1784885161102, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7826122641563416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 314245120, "epoch_num": 0.13717489997097884}} +:::MLLOG {"namespace": "", "time_ms": 1784885161102, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 314245120, "epoch_num": 0.13717489997097884}} +:::MLLOG {"namespace": "", "time_ms": 1784885161102, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 314245120, "epoch_num": 0.13717489997097884}} +:::MLLOG {"namespace": "", "time_ms": 1784885169844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10795781761407852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 314572800, "lr": 1.5998333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885187271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1040109246969223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 315228160, "lr": 1.6031666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885205106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10326971113681793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 315883520, "lr": 1.6065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885222673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10239951312541962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 316538880, "lr": 1.6098333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885222674, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 316538880, "epoch_num": 0.13817617661310277}} +:::MLLOG {"namespace": "", "time_ms": 1784885222674, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 316538880, "epoch_num": 0.13817617661310277}} +:::MLLOG {"namespace": "", "time_ms": 1784885249367, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7828235626220703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 316538880, "epoch_num": 0.13817617661310277}} +:::MLLOG {"namespace": "", "time_ms": 1784885249367, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 316538880, "epoch_num": 0.13817617661310277}} +:::MLLOG {"namespace": "", "time_ms": 1784885249367, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 316538880, "epoch_num": 0.13817617661310277}} +:::MLLOG {"namespace": "", "time_ms": 1784885267230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10710550844669342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 317194240, "lr": 1.6131666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885284127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10854456573724747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 317849600, "lr": 1.6165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885302306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11203959584236145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 318504960, "lr": 1.6198333333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885311373, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 318832640, "epoch_num": 0.1391774532552267}} +:::MLLOG {"namespace": "", "time_ms": 1784885311373, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 318832640, "epoch_num": 0.1391774532552267}} +:::MLLOG {"namespace": "", "time_ms": 1784885337559, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7829596996307373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 318832640, "epoch_num": 0.1391774532552267}} +:::MLLOG {"namespace": "", "time_ms": 1784885337560, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 318832640, "epoch_num": 0.1391774532552267}} +:::MLLOG {"namespace": "", "time_ms": 1784885337560, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 318832640, "epoch_num": 0.1391774532552267}} +:::MLLOG {"namespace": "", "time_ms": 1784885346477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11328011751174927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 319160320, "lr": 1.6231666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885363991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10393938422203064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 319815680, "lr": 1.6265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885381495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11281699687242508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 320471040, "lr": 1.6298333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885399083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10704928636550903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 321126400, "lr": 1.6331666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885399084, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 321126400, "epoch_num": 0.14017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784885399085, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 321126400, "epoch_num": 0.14017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784885425520, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7825653553009033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 321126400, "epoch_num": 0.14017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784885425520, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 321126400, "epoch_num": 0.14017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784885425520, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 321126400, "epoch_num": 0.14017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784885442837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11030997335910797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 321781760, "lr": 1.6365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885460506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10835695266723633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 322437120, "lr": 1.6398333333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885478402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11175896972417831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 323092480, "lr": 1.6431666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885487364, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 323420160, "epoch_num": 0.14118000653947457}} +:::MLLOG {"namespace": "", "time_ms": 1784885487365, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 323420160, "epoch_num": 0.14118000653947457}} +:::MLLOG {"namespace": "", "time_ms": 1784885514116, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7833569049835205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 323420160, "epoch_num": 0.14118000653947457}} +:::MLLOG {"namespace": "", "time_ms": 1784885514116, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 323420160, "epoch_num": 0.14118000653947457}} +:::MLLOG {"namespace": "", "time_ms": 1784885514116, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 323420160, "epoch_num": 0.14118000653947457}} +:::MLLOG {"namespace": "", "time_ms": 1784885523111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10886958241462708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 323747840, "lr": 1.6465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885540817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10411982238292694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 324403200, "lr": 1.6498333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885557825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10521664470434189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 325058560, "lr": 1.6531666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885576898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11272744834423065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 325713920, "lr": 1.6565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885576898, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 325713920, "epoch_num": 0.1421812831815985}} +:::MLLOG {"namespace": "", "time_ms": 1784885576899, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 325713920, "epoch_num": 0.1421812831815985}} +:::MLLOG {"namespace": "", "time_ms": 1784885603068, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.783111572265625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 325713920, "epoch_num": 0.1421812831815985}} +:::MLLOG {"namespace": "", "time_ms": 1784885603068, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 325713920, "epoch_num": 0.1421812831815985}} +:::MLLOG {"namespace": "", "time_ms": 1784885603068, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 325713920, "epoch_num": 0.1421812831815985}} +:::MLLOG {"namespace": "", "time_ms": 1784885621014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10938086360692978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 326369280, "lr": 1.6598333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885638659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11157510429620743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327024640, "lr": 1.6631666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885656738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10702474415302277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680000, "lr": 1.6665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885665628, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 328007680, "epoch_num": 0.14318255982372244}} +:::MLLOG {"namespace": "", "time_ms": 1784885665628, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 328007680, "epoch_num": 0.14318255982372244}} +:::MLLOG {"namespace": "", "time_ms": 1784885692014, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7829775810241699, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 328007680, "epoch_num": 0.14318255982372244}} +:::MLLOG {"namespace": "", "time_ms": 1784885692014, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 328007680, "epoch_num": 0.14318255982372244}} +:::MLLOG {"namespace": "", "time_ms": 1784885692015, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 328007680, "epoch_num": 0.14318255982372244}} +:::MLLOG {"namespace": "", "time_ms": 1784885700791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11359578371047974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 328335360, "lr": 1.6698333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885718620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1077975332736969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 328990720, "lr": 1.6731666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885736710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11246925592422485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 329646080, "lr": 1.6765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885754608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10854213684797287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 330301440, "lr": 1.6798333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885754609, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 330301440, "epoch_num": 0.14418383646584637}} +:::MLLOG {"namespace": "", "time_ms": 1784885754609, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 330301440, "epoch_num": 0.14418383646584637}} +:::MLLOG {"namespace": "", "time_ms": 1784885781449, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7834774255752563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 330301440, "epoch_num": 0.14418383646584637}} +:::MLLOG {"namespace": "", "time_ms": 1784885781449, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 330301440, "epoch_num": 0.14418383646584637}} +:::MLLOG {"namespace": "", "time_ms": 1784885781449, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 330301440, "epoch_num": 0.14418383646584637}} +:::MLLOG {"namespace": "", "time_ms": 1784885798436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11477530747652054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 330956800, "lr": 1.6831666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885817143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11142729222774506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 331612160, "lr": 1.6865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885834951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10296297818422318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 332267520, "lr": 1.6898333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885843829, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 332595200, "epoch_num": 0.1451851131079703}} +:::MLLOG {"namespace": "", "time_ms": 1784885843830, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 332595200, "epoch_num": 0.1451851131079703}} +:::MLLOG {"namespace": "", "time_ms": 1784885870843, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7832448482513428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 332595200, "epoch_num": 0.1451851131079703}} +:::MLLOG {"namespace": "", "time_ms": 1784885870844, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 332595200, "epoch_num": 0.1451851131079703}} +:::MLLOG {"namespace": "", "time_ms": 1784885870844, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 332595200, "epoch_num": 0.1451851131079703}} +:::MLLOG {"namespace": "", "time_ms": 1784885879641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10660481452941895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 332922880, "lr": 1.6931666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885896974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10434991866350174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 333578240, "lr": 1.6964999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885914550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072196289896965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 334233600, "lr": 1.6998333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885931976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10094915330410004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 334888960, "lr": 1.7031666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885931977, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 334888960, "epoch_num": 0.14618638975009424}} +:::MLLOG {"namespace": "", "time_ms": 1784885931977, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 334888960, "epoch_num": 0.14618638975009424}} +:::MLLOG {"namespace": "", "time_ms": 1784885958407, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7836584448814392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 334888960, "epoch_num": 0.14618638975009424}} +:::MLLOG {"namespace": "", "time_ms": 1784885958407, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 334888960, "epoch_num": 0.14618638975009424}} +:::MLLOG {"namespace": "", "time_ms": 1784885958407, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 334888960, "epoch_num": 0.14618638975009424}} +:::MLLOG {"namespace": "", "time_ms": 1784885975842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10468810051679611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 335544320, "lr": 1.7064999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885993547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10637696087360382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 336199680, "lr": 1.7098333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886011126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10800019651651382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 336855040, "lr": 1.7131666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886020016, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 337182720, "epoch_num": 0.14718766639221817}} +:::MLLOG {"namespace": "", "time_ms": 1784886020017, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 337182720, "epoch_num": 0.14718766639221817}} +:::MLLOG {"namespace": "", "time_ms": 1784886046443, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7835452556610107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 337182720, "epoch_num": 0.14718766639221817}} +:::MLLOG {"namespace": "", "time_ms": 1784886046444, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 337182720, "epoch_num": 0.14718766639221817}} +:::MLLOG {"namespace": "", "time_ms": 1784886046444, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 337182720, "epoch_num": 0.14718766639221817}} +:::MLLOG {"namespace": "", "time_ms": 1784886055385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09968975186347961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 337510400, "lr": 1.7164999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886072222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10133862495422363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 338165760, "lr": 1.7198333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886090492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10700932145118713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 338821120, "lr": 1.7231666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886108358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10191167891025543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 339476480, "lr": 1.7265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886108358, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 339476480, "epoch_num": 0.1481889430343421}} +:::MLLOG {"namespace": "", "time_ms": 1784886108359, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 339476480, "epoch_num": 0.1481889430343421}} +:::MLLOG {"namespace": "", "time_ms": 1784886135415, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7831670641899109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 339476480, "epoch_num": 0.1481889430343421}} +:::MLLOG {"namespace": "", "time_ms": 1784886135415, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 339476480, "epoch_num": 0.1481889430343421}} +:::MLLOG {"namespace": "", "time_ms": 1784886135416, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 339476480, "epoch_num": 0.1481889430343421}} +:::MLLOG {"namespace": "", "time_ms": 1784886153114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10120445489883423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 340131840, "lr": 1.7298333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886170765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10352165997028351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 340787200, "lr": 1.7331666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886188341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10605931282043457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 341442560, "lr": 1.7364999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886197081, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 341770240, "epoch_num": 0.14919021967646603}} +:::MLLOG {"namespace": "", "time_ms": 1784886197082, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 341770240, "epoch_num": 0.14919021967646603}} +:::MLLOG {"namespace": "", "time_ms": 1784886224304, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.784066915512085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 341770240, "epoch_num": 0.14919021967646603}} +:::MLLOG {"namespace": "", "time_ms": 1784886224305, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 341770240, "epoch_num": 0.14919021967646603}} +:::MLLOG {"namespace": "", "time_ms": 1784886224305, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 341770240, "epoch_num": 0.14919021967646603}} +:::MLLOG {"namespace": "", "time_ms": 1784886233076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10288579761981964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 342097920, "lr": 1.7398333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886250530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10118129104375839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 342753280, "lr": 1.7431666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886268232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10591751337051392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 343408640, "lr": 1.7465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886285926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10832994431257248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 344064000, "lr": 1.7498333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886285926, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 344064000, "epoch_num": 0.15019149631858997}} +:::MLLOG {"namespace": "", "time_ms": 1784886285927, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 344064000, "epoch_num": 0.15019149631858997}} +:::MLLOG {"namespace": "", "time_ms": 1784886312696, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7839410305023193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 344064000, "epoch_num": 0.15019149631858997}} +:::MLLOG {"namespace": "", "time_ms": 1784886312696, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 344064000, "epoch_num": 0.15019149631858997}} +:::MLLOG {"namespace": "", "time_ms": 1784886312696, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 344064000, "epoch_num": 0.15019149631858997}} +:::MLLOG {"namespace": "", "time_ms": 1784886330436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10591191053390503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 344719360, "lr": 1.7531666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886347347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10773947834968567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 345374720, "lr": 1.7564999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886364252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09953959286212921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 346030080, "lr": 1.7598333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886374233, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 346357760, "epoch_num": 0.1511927729607139}} +:::MLLOG {"namespace": "", "time_ms": 1784886374233, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 346357760, "epoch_num": 0.1511927729607139}} +:::MLLOG {"namespace": "", "time_ms": 1784886401327, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7836059927940369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 346357760, "epoch_num": 0.1511927729607139}} +:::MLLOG {"namespace": "", "time_ms": 1784886401328, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 346357760, "epoch_num": 0.1511927729607139}} +:::MLLOG {"namespace": "", "time_ms": 1784886401328, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 346357760, "epoch_num": 0.1511927729607139}} +:::MLLOG {"namespace": "", "time_ms": 1784886410418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1040981262922287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 346685440, "lr": 1.7631666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886428030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1027936339378357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 347340800, "lr": 1.7665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886445684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10634923726320267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 347996160, "lr": 1.7698333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886463053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10113748908042908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 348651520, "lr": 1.7731666666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886463124, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 348651520, "epoch_num": 0.15219404960283783}} +:::MLLOG {"namespace": "", "time_ms": 1784886463124, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 348651520, "epoch_num": 0.15219404960283783}} +:::MLLOG {"namespace": "", "time_ms": 1784886489762, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7840479612350464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 348651520, "epoch_num": 0.15219404960283783}} +:::MLLOG {"namespace": "", "time_ms": 1784886489762, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 348651520, "epoch_num": 0.15219404960283783}} +:::MLLOG {"namespace": "", "time_ms": 1784886489762, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 348651520, "epoch_num": 0.15219404960283783}} +:::MLLOG {"namespace": "", "time_ms": 1784886507311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10862861573696136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 349306880, "lr": 1.7764999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886524936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10735245794057846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 349962240, "lr": 1.7798333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886542495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039494127035141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 350617600, "lr": 1.7831666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886551317, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 350945280, "epoch_num": 0.15319532624496177}} +:::MLLOG {"namespace": "", "time_ms": 1784886551318, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 350945280, "epoch_num": 0.15319532624496177}} +:::MLLOG {"namespace": "", "time_ms": 1784886578175, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7839494943618774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 350945280, "epoch_num": 0.15319532624496177}} +:::MLLOG {"namespace": "", "time_ms": 1784886578175, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 350945280, "epoch_num": 0.15319532624496177}} +:::MLLOG {"namespace": "", "time_ms": 1784886578176, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 350945280, "epoch_num": 0.15319532624496177}} +:::MLLOG {"namespace": "", "time_ms": 1784886586951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10457011312246323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 351272960, "lr": 1.7865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886604598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10028287768363953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 351928320, "lr": 1.7898333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886622155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09993876516819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 352583680, "lr": 1.7931666666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886638738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10535379499197006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 353239040, "lr": 1.7964999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886638801, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 353239040, "epoch_num": 0.1541966028870857}} +:::MLLOG {"namespace": "", "time_ms": 1784886638801, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 353239040, "epoch_num": 0.1541966028870857}} +:::MLLOG {"namespace": "", "time_ms": 1784886665627, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7842042446136475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 353239040, "epoch_num": 0.1541966028870857}} +:::MLLOG {"namespace": "", "time_ms": 1784886665628, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 353239040, "epoch_num": 0.1541966028870857}} +:::MLLOG {"namespace": "", "time_ms": 1784886665628, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 353239040, "epoch_num": 0.1541966028870857}} +:::MLLOG {"namespace": "", "time_ms": 1784886684027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10617722570896149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 353894400, "lr": 1.7998333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886701677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1053471639752388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 354549760, "lr": 1.8031666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886718778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10617402195930481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 355205120, "lr": 1.8065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886727276, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 355532800, "epoch_num": 0.15519787952920963}} +:::MLLOG {"namespace": "", "time_ms": 1784886727276, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 355532800, "epoch_num": 0.15519787952920963}} diff --git a/recommendation_v4/rcp_logs/gbs_32768/seed451547632.log b/recommendation_v4/rcp_logs/gbs_32768/seed451547632.log new file mode 100644 index 000000000..12d0940ea --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_32768/seed451547632.log @@ -0,0 +1,888 @@ +:::MLLOG {"namespace": "", "time_ms": 1784881834498, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784881834498, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784881863128, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784881863130, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784881863130, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784881863130, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784881863130, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784881863130, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784881863130, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784881863130, "event_type": "POINT_IN_TIME", "key": "seed", "value": 451547632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784881863130, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784881863130, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784881863130, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784881863131, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784881863134, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784881863134, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784882734844, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784882734845, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784882735147, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784882909560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384854018688202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.1666666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784882925031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13846956193447113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.4999999999999995e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784882939455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13855421543121338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784882946729, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784882946729, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784882970865, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4985979497432709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784882970866, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784882970866, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784882978098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13844791054725647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3166666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784882992444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13849245011806488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883007409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13838844001293182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9833333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883022132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13844826817512512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883022133, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784883022133, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784883045050, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49874135851860046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784883045051, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784883045051, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784883059813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13815146684646606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883074516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382545530796051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883089513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13810575008392334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883096738, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784883096739, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784883120184, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4989156424999237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784883120185, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784883120185, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784883127903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13820409774780273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883142667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13817428052425385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883157868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13825726509094238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883172748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13780304789543152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883172749, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784883172749, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784883196001, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4991208612918854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784883196001, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784883196001, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784883210588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13811808824539185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883225501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13751903176307678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883240556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378563493490219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883247859, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784883247860, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784883270871, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4993593990802765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784883270871, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784883270871, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784883277988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13710342347621918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883293116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13719689846038818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883307808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13689610362052917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883322862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13693523406982422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883322863, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784883322863, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784883346966, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49963119626045227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784883346967, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784883346967, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784883361496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371329128742218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.316666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883376238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13700729608535767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883391913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13637614250183105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883399468, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784883399469, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784883423179, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49990007281303406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784883423180, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784883423180, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784883430807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13718068599700928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883446747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13614732027053833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883461787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13649728894233704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883477869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13685129582881927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883477869, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784883477870, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784883501696, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5002606511116028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784883501696, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784883501696, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784883516819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13617272675037384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883532605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13482511043548584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.983333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883547509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13657855987548828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883554968, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784883554968, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784883579237, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5006150007247925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784883579237, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784883579237, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784883586648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13309909403324127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883602182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345905065536499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883616946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13502156734466553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883631777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13517574965953827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883631844, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784883631844, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784883655846, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5009852051734924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784883655846, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784883655846, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784883670722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13379567861557007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883685576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13462378084659576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883700356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13514432311058044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883708501, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784883708502, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784883731935, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5014122724533081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784883731936, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784883731936, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784883739247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13569791615009308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883753978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13490717113018036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883769108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321820318698883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883784736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344946324825287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883784737, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784883784737, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784883807330, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5019556283950806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784883807330, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784883807331, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784883822544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336936503648758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883837980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13520751893520355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883853923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350414752960205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883861632, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784883861632, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784883885091, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5028138160705566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784883885091, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784883885092, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784883892541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13632610440254211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883907758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335156410932541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883923628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343071460723877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883938712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321495920419693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883938779, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784883938779, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784883962088, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5037333965301514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784883962089, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784883962089, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784883977024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329214870929718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883992733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13123804330825806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884007622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13311107456684113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884015204, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784884015205, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784884038387, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5047145485877991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784884038387, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784884038387, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784884045919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382797509431839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884061046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311229020357132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884076784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12994074821472168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884091793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13037902116775513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884091793, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784884091794, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784884114997, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5058517456054688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784884114998, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784884114998, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784884129955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327379047870636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884145903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12976643443107605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884162805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1306706815958023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884170614, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784884170614, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784884193699, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5070832371711731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784884193699, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784884193699, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784884201379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315518170595169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884216401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13520045578479767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884231826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331876516342163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884247538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356295794248581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884247538, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784884247539, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784884270504, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5084412693977356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784884270505, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784884270505, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784884285765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352328509092331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884300947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13262133300304413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1649999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884316227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332818865776062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884323794, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784884323794, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784884347227, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5098029375076294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784884347228, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784884347228, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784884355710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13637012243270874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884371371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353350430727005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884386783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13620281219482422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884402495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13649961352348328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884402496, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784884402496, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784884426204, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5112141370773315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784884426205, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784884426205, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784884442617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13482657074928284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884458324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308828443288803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884473875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13632959127426147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884481785, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784884481786, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784884505157, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5125888586044312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784884505158, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784884505158, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784884512946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13602668046951294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884528761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13257287442684174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884543973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326364427804947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884559147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13082876801490784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884559148, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784884559148, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784884582002, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5138918161392212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784884582002, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784884582002, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784884597240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13101013004779816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884613295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13819396495819092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884628827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295868456363678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884636510, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784884636510, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784884659320, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.515356719493866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784884659321, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784884659321, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784884666980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12929759919643402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884682543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13475418090820312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884698010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13466393947601318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884714230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13046078383922577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884714231, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784884714231, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784884737066, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5171405076980591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784884737067, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784884737067, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784884752191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12818178534507751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884767420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13180766999721527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884782948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13555274903774261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884790733, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784884790734, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784884814622, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5190075039863586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784884814622, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784884814623, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784884822256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13662786781787872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884838150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13355585932731628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884853382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13209594786167145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884868619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13100934028625488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884868620, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784884868620, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784884891781, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5212580561637878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784884891782, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784884891782, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784884907348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12957559525966644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884922866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13253560662269592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884938873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13546618819236755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884946661, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784884946662, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784884969709, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5238773226737976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784884969710, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784884969710, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784884977300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13198041915893555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884992992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335175782442093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885008634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13497677445411682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885024366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13461826741695404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885024367, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784885024367, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784885048206, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5268421769142151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784885048207, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784885048207, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784885063979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13598506152629852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885080349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1339123398065567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885096264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13317953050136566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3650000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885104041, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784885104042, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784885127513, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.530098021030426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784885127513, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784885127514, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784885135589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1403830647468567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885151709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13526974618434906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885168168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13451947271823883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4649999999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885184326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13538914918899536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885184327, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784885184327, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784885207460, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.533525288105011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784885207461, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784885207461, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784885223218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13520905375480652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885239092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13329973816871643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.5649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885254925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13231799006462097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885263392, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784885263393, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784885287163, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5373422503471375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784885287163, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784885287163, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784885295154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325460523366928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885310744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13097858428955078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885326201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13595107197761536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885341997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324041187763214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885341998, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784885341998, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784885366253, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5419042706489563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784885366253, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784885366253, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784885382041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13489094376564026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885398374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1302386224269867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885414075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1319689303636551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885421856, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784885421856, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784885444524, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5480696558952332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784885444525, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784885444525, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784885452183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13234473764896393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885467953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308414489030838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885483809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13315989077091217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885500086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326913982629776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885500087, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784885500087, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784885523835, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5564262270927429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784885523835, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784885523835, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784885539368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321081519126892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885554760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1283462643623352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885569898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341208815574646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885577749, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784885577750, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784885600496, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5660231709480286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784885600497, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784885600497, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784885608409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13487836718559265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885624003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1269482672214508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885640179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12718407809734344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885656042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133131742477417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885656043, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784885656043, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784885679400, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5772469639778137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784885679401, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784885679401, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784885694855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12947459518909454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885710388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289048194885254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885726312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12820348143577576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885733989, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784885733989, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784885756858, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5883010029792786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784885756859, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784885756859, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784885764825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13033714890480042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885781142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1279478669166565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885796869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13037243485450745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885812523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1287526786327362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885812523, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784885812524, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784885834858, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6000393033027649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784885834858, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784885834859, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784885850314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12716662883758545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885866203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13007861375808716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4983333333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885881961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13331890106201172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885889838, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784885889838, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784885912824, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6104844212532043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784885912825, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784885912825, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784885921287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13178789615631104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885937657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12730303406715393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885953947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318204253911972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885970062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1271214783191681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885970063, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784885970063, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784885992626, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6203960180282593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784885992627, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784885992627, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784886009018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12962038815021515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886025196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13059818744659424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886041915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12838490307331085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886050296, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784886050297, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784886073154, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6303582787513733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784886073155, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784886073155, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784886081319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12769338488578796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886097518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12754881381988525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886113863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1285361498594284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886130253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1275898516178131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.898333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886130253, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784886130253, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784886154088, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6381576657295227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784886154088, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784886154089, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784886170663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12207625806331635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886186669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12101994454860687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886202405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11956822872161865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886210363, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784886210364, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784886233305, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6430808305740356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784886233306, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784886233306, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784886241043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12113345414400101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886257089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1181231290102005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886273045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12238481640815735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886288634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11998529732227325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886288635, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784886288635, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784886311878, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6583248376846313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784886311879, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784886311879, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784886328600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12113495171070099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886344432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11231429874897003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.198333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886360233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11802317947149277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886368019, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784886368019, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784886390891, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.675504744052887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784886390892, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784886390892, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784886398691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11864148080348969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886414693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11875653266906738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.298333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886430659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11339452862739563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886446221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11708558350801468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886446293, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784886446294, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784886471170, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6990647912025452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784886471171, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784886471171, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784886487267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11571413278579712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.398333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886503029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11592039465904236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886519131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10897403210401535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886527132, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784886527132, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784886550440, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7209038734436035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784886550441, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784886550441, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784886558413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11281952261924744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.498333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886574736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1141560897231102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.531666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886590920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11247013509273529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886607321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11404088139533997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.598333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886607322, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784886607322, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784886631311, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7425894737243652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784886631311, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784886631312, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784886647047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11283911764621735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886662751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1120222806930542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886678620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11622168123722076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886686426, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784886686427, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784886710149, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7532435655593872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784886710150, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784886710150, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784886717952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10578270256519318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886734179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11315152049064636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886750119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1073826402425766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886766377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11156688630580902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886766377, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784886766378, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784886790796, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7575405240058899, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784886790796, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784886790796, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784886807047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11411601305007935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886822874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11149115860462189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886838665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10993040353059769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886846367, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784886846368, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784886870596, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7607637643814087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784886870597, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784886870597, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784886878474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10981893539428711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886894482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11144974827766418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886910679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11196261644363403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886926255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10590637475252151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886926255, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784886926256, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784886950316, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7619407176971436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784886950317, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784886950317, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784886967521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11511506140232086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886983698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11873918771743774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886999945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11585532873868942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887007996, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784887007997, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784887031732, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7631088495254517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784887031733, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784887031733, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784887039694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11489902436733246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887056134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11458874493837357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887072292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1118176057934761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887088309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11288157105445862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887088310, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784887088310, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784887112108, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7641599774360657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784887112108, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784887112109, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784887128477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11750884354114532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887144808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11722282320261002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887161268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11035475134849548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887169483, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784887169483, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784887193455, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7652466297149658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784887193456, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784887193456, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784887201699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11494617909193039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887218288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11494610458612442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887234571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11056286096572876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887251414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039777398109436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887251415, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784887251415, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784887274915, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7667369842529297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784887274916, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784887274916, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784887290484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10612562298774719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887306484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10539884120225906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887322373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10544685274362564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887330303, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784887330303, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784887354123, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7677466869354248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784887354124, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784887354124, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784887362051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10378299653530121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887378370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11120280623435974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887394510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10708394646644592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887411145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11254532635211945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887411145, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784887411146, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784887434178, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.768333911895752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784887434178, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784887434178, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784887450158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10654276609420776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887465764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10604039579629898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887481859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10149441659450531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887489724, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784887489724, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784887512933, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7691887021064758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784887512934, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784887512934, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784887520958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1021139919757843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887537110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10121025145053864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887553491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11119026690721512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887569335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10373054444789886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887569336, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784887569336, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784887593153, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7699817419052124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784887593153, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784887593154, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784887609564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1059487983584404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887625742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1061072126030922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887641354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1064687967300415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887649566, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784887649566, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784887673086, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7705470323562622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784887673086, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784887673087, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784887681190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10719600319862366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887697101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09936170279979706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887713366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10750710964202881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887729725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10406427085399628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887729725, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784887729726, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784887753955, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7710296511650085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784887753955, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784887753956, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784887769779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10745806246995926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887786522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10785879194736481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887802656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10401144623756409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887810593, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784887810594, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784887833721, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.771191418170929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784887833722, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784887833722, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784887841606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10878447443246841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887857781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10713283717632294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887873768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11170347779989243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887889962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10786720365285873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887889963, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784887889963, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784887913108, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7715926766395569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784887913109, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784887913110, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784887929567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10502666234970093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887945444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10169744491577148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887962256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10761388391256332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.564999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887970654, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784887970654, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784887993837, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7719863653182983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784887993837, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784887993838, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784888001978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10453122854232788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888018081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09901329129934311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888034469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11382275819778442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.664999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888050693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11072532832622528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888050694, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784888050694, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784888074574, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7721026539802551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784888074575, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784888074575, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784888090797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10622035712003708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888107469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10846894979476929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888123761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10799203813076019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888131729, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784888131730, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784888156079, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7723450660705566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784888156079, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784888156079, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784888164836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10995133221149445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888181393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11437161266803741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888197839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11200709640979767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888214288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11302826553583145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888214288, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784888214288, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784888238225, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7725673913955688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784888238225, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784888238226, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784888254331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10963433980941772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888270975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11375337094068527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.998333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888287527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11048053205013275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888295697, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784888295698, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784888320328, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7726718187332153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784888320328, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784888320328, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784888328408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11038023978471756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888345538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1091945469379425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888362118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11258844286203384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.131666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888378688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11076980829238892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888378689, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784888378689, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784888402544, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7729315161705017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784888402544, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784888402545, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784888419161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10992637276649475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888436119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1126999482512474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888452909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11698555946350098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888461303, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784888461303, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784888485153, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730462551116943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784888485154, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784888485154, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784888493193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11476169526576996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888510201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10750459134578705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.331666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888526601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10135578364133835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888542828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10673581808805466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888542829, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784888542829, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784888567054, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730514407157898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784888567055, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784888567055, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784888583219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10490082204341888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888599159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10618437826633453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888615634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10515834391117096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888623920, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784888623920, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784888648430, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730526924133301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784888648430, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784888648431, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784888656584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10851341485977173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888672485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10297362506389618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888689460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1045868843793869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888705716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723283886909485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.631666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888705716, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784888705717, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784888729235, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7732975482940674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784888729236, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784888729236, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784888745090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1004183366894722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888761306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10356836766004562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888777437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10315417498350143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888785451, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784888785451, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784888808667, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773424506187439, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784888808668, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784888808668, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784888816777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1038614884018898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888833586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10288657248020172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888849803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11389464139938354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888866382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10312244296073914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888866383, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784888866383, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784888890352, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7736362814903259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784888890352, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784888890353, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784888906644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10704539716243744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888922761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1074783131480217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888938917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10137490928173065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888946991, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784888946991, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784888971261, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738037705421448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784888971262, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784888971262, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784888979222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10129030048847198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888995433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10046500712633133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889011864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10608832538127899, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889028317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10321349650621414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889028318, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784889028318, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784889052643, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773750901222229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784889052644, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784889052644, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784889068461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1073622927069664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889085447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10357165336608887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889101709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10582283139228821, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889109672, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784889109673, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784889133172, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7740002274513245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784889133173, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784889133173, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784889141307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10348125547170639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889157679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10815732926130295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889173981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10257329791784286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889190585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10650776326656342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889190586, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784889190586, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784889219317, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7740384340286255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784889222279, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784889222279, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784889238926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10207125544548035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889259540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10636487603187561, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889285781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10962066054344177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889316805, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784889321105, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784889344811, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742623090744019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784889344811, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784889344811, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784889366940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11177186667919159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889407893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10716742277145386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889460758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10285426676273346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889506852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10232706367969513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889506853, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784889506854, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784889530822, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7743484377861023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784889530823, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784889530823, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784889579256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10487593710422516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889653645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10540784895420074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.631666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889716789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11143623292446136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889733778, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784889733779, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784889757962, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746222615242004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784889757962, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784889757962, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784889769404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10702411830425262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889786514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11245780438184738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889803576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10873749852180481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889820113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11266264319419861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889820114, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784889820114, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784889843417, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7745409607887268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784889843418, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784889843418, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784889860008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10864926874637604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.831666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889876713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11401481181383133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889893461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11180715262889862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889901906, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784889901907, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784889925497, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7747890949249268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784889925497, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784889925498, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784889933972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1123509630560875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889950870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11189191043376923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889968282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10708567500114441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889985750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11070600152015686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1.0031666666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784889985751, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784889985752, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784890009159, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7749075889587402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784890009159, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784890009159, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784890025755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11395107209682465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1.0064999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890042528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10857507586479187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1.0098333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890059403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11130531132221222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1.0131666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890067993, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784890067994, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784890091695, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7749379873275757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784890091696, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784890091696, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784890100223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1114620715379715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1.0165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890117327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039004921913147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1.0198333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890133899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10987570881843567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1.0231666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890151062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10803408175706863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1.0264999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890151063, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784890151063, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784890174737, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7753698825836182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784890174738, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784890174738, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784890191118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1034744456410408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1.0298333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890207407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09988760203123093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1.0331666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890223846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10200152546167374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1.0365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890232062, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784890232062, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784890255799, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7749087810516357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784890255799, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784890255800, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784890263881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09954259544610977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1.0398333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890280053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10059131681919098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1.0431666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890296346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10226759314537048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1.0464999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890313015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1054251492023468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1.0498333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890313016, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784890313016, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784890337060, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7750999331474304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784890337060, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784890337060, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784890353008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10401719063520432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1.0531666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890370209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10979333519935608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1.0565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890386951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10600004345178604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1.0598333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890395160, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784890395161, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784890418781, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7753608226776123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784890418782, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784890418782, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784890426905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10143493115901947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1.0631666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890443720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10558691620826721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1.0664999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890460245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1003618910908699, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1.0698333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890476674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10171882808208466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1.0731666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890476675, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784890476675, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784890499908, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7755463123321533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784890499909, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784890499909, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784890516481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10192404687404633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1.0765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890534663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10308690369129181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1.0798333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890550947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10167673975229263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1.0831666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890559850, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784890559850, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784890583714, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7754968404769897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784890583714, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784890583714, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784890591801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10389401763677597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1.0864999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890608161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10697809606790543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1.0898333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890625619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10247914493083954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1.0931666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890642029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10033334791660309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1.0965e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890642029, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784890642030, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784890666028, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7756677865982056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784890666029, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784890666029, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784890682270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10560767352581024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1.0998333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890698498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10177922248840332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1.1031666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890715040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.101817287504673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217579520, "lr": 1.1065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890723352, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784890723353, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784890746115, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7755540013313293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784890746116, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784890746116, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784890754422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11161994189023972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218234880, "lr": 1.1098333333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890770586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10933602601289749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218890240, "lr": 1.1131666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890787040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10765137523412704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219545600, "lr": 1.1165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890803683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10814084857702255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220200960, "lr": 1.1198333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890803684, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784890803684, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784890826253, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.775669515132904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784890826253, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784890826253, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784890842571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10288897901773453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220856320, "lr": 1.1231666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890858920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1045256182551384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221511680, "lr": 1.1265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890875326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10063591599464417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222167040, "lr": 1.1298333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890883615, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784890883616, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784890906861, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7762965559959412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784890906862, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784890906862, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784890915236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1053745299577713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222822400, "lr": 1.1331666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890932344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10407590121030807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223477760, "lr": 1.1365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890948848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10699744522571564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224133120, "lr": 1.1398333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890965676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10887517035007477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224788480, "lr": 1.1431666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890965676, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784890965676, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784890989136, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7762446999549866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784890989137, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784890989137, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784891005349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10853388905525208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225443840, "lr": 1.1465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891022349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10867057740688324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226099200, "lr": 1.1498333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891039103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10745660960674286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226754560, "lr": 1.1531666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891047307, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784891047307, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784891070253, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7767040133476257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784891070254, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784891070254, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784891078176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11238779127597809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227409920, "lr": 1.1565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891094455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10983629524707794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228065280, "lr": 1.1598333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891110842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10680873692035675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228720640, "lr": 1.1631666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891130844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10654565691947937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229376000, "lr": 1.1665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891130847, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784891130848, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784891154789, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7767565250396729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784891154789, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784891154789, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784891171288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.106717050075531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230031360, "lr": 1.1698333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891188931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1053004115819931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230686720, "lr": 1.1731666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891205529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.110427625477314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231342080, "lr": 1.1765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891213551, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784891213551, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784891237856, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7766452431678772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784891237856, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784891237856, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784891246078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.108339823782444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231997440, "lr": 1.1798333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891263451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10983248054981232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 232652800, "lr": 1.1831666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891280436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11051743477582932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233308160, "lr": 1.1865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891297384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11183653026819229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233963520, "lr": 1.1898333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891297384, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784891297385, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784891320620, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7771741151809692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784891320621, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784891320621, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784891337659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11423642188310623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 234618880, "lr": 1.1931666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891354502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11336721479892731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235274240, "lr": 1.1965e-06}} diff --git a/recommendation_v4/rcp_logs/gbs_32768/seed480431706.log b/recommendation_v4/rcp_logs/gbs_32768/seed480431706.log new file mode 100644 index 000000000..a9febc79c --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_32768/seed480431706.log @@ -0,0 +1,1362 @@ +:::MLLOG {"namespace": "", "time_ms": 1784867444848, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784867444848, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784867468581, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784867468582, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784867468582, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784867468582, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784867468582, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784867468583, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784867468583, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784867468583, "event_type": "POINT_IN_TIME", "key": "seed", "value": 480431706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784867468583, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784867468583, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784867468583, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784867468583, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784867468772, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784867468773, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784868253139, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784868253140, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784868253439, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784868541256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13960187137126923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.1666666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784868556394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13951970636844635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.4999999999999995e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784868570922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13930314779281616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784868578040, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784868578041, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784868602242, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5024071335792542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784868602243, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784868602243, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784868609454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1394234299659729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3166666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868623561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13938814401626587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868638344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13923543691635132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9833333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868652806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13914842903614044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868652807, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784868652807, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784868675907, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5025042295455933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784868675907, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784868675908, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784868690444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13950300216674805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868704855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13922733068466187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868719572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13919144868850708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868726649, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784868726649, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784868750666, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5025946497917175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784868750666, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784868750667, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784868758151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389952003955841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868772703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389254629611969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868787621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388626992702484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868802294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13882331550121307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868802294, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784868802295, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784868827557, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5027042627334595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784868827558, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784868827558, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784868841950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13871249556541443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868856543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13866952061653137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868871308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13859710097312927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868878667, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784868878668, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784868904777, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5028249621391296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784868904778, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784868904778, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784868911865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13823392987251282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868927200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381521075963974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868941418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380544900894165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868956166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13789710402488708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868956167, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784868956168, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784868983256, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.502922534942627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784868983256, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784868983256, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784868997322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379317194223404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.316666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869016026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13780824840068817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869031328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13735631108283997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869038847, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784869038847, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784869064119, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.503044068813324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784869064120, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784869064120, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784869071614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376889944076538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869087276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13713300228118896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869102196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13727882504463196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869117903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13750839233398438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869117904, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784869117904, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784869144110, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5031846761703491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784869144111, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784869144111, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784869159170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13694360852241516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869175104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13600842654705048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.983333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869190230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371055245399475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869197605, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784869197605, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784869222098, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5033061504364014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784869222099, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784869222099, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784869229617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13467338681221008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869244770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355947107076645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869259599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13577140867710114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869274296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359337419271469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869274364, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784869274365, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784869302045, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.503423810005188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784869302046, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784869302046, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784869316696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13475078344345093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869331540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13527940213680267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869347374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356479823589325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869355388, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784869355389, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784869382062, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5035634636878967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784869382063, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784869382063, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784869389534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13601453602313995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869404787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135273277759552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869420058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13289812207221985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869435723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349869668483734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869435723, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784869435724, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784869461325, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5038191080093384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784869461325, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784869461326, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784869476867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13422535359859467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869492392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13533399999141693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869508589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13514146208763123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869516409, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784869516410, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784869542159, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5044566988945007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784869542160, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784869542160, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784869549690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13627605140209198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869564951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13372358679771423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869580991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13440744578838348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869596307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13236114382743835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869596373, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784869596373, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784869622550, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5052478313446045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784869622551, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784869622551, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784869637564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13299736380577087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869653123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13141265511512756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869668508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13305820524692535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869676055, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784869676055, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784869702348, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5061846971511841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784869702349, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784869702349, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784869709946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382550597190857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869725159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13108554482460022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869753738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12996336817741394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869769221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1305082142353058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869769222, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784869769222, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784869795935, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5074151754379272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784869795935, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784869795935, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784869811139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13281163573265076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869826391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12984523177146912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869842539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1306927651166916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869850305, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784869850306, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784869876502, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5087503790855408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784869876502, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784869876502, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784869884325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13153862953186035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869899628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351822316646576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869915187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13313345611095428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869931321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355888843536377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869931322, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784869931322, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784869957443, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5102331042289734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784869957443, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784869957443, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784869972940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13514763116836548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869988489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13245455920696259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1649999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870003836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13321825861930847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870011530, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784870011530, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784870035248, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5117356181144714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784870035248, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784870035248, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784870043604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13641242682933807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870059818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13536536693572998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870075655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13620659708976746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870091507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13634614646434784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870091508, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784870091508, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784870115588, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5132945775985718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784870115588, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784870115588, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784870132033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13479682803153992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870148273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13088038563728333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870163908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362115442752838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870171947, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784870171948, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784870196996, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5149398446083069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784870196996, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784870196997, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784870204703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13593539595603943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870221039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13244375586509705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870236813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13261418044567108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870252633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13072596490383148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870252633, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784870252633, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784870279091, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5163095593452454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784870279092, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784870279092, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784870294595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13100048899650574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870310818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381828635931015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870326444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12963172793388367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870334175, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784870334175, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784870360222, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.517895519733429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784870360222, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784870360222, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784870367814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12941692769527435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870383616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13469967246055603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870399102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13457255065441132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870415700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13043828308582306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870415700, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784870415701, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784870441899, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5198949575424194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784870441899, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784870441899, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784870457581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12799577414989471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870473113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13187435269355774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870489023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13546621799468994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870496822, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784870496823, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784870521429, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5221729278564453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784870521429, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784870521430, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784870529278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13654735684394836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870545392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13358831405639648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870561313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320589780807495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870576701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13090161979198456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870576701, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784870576701, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784870602597, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5248616933822632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784870602598, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784870602598, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784870618058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294836848974228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870633552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13241124153137207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870664333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13532134890556335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870672356, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784870672357, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784870698502, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5279560089111328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784870698502, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784870698502, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784870706510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1319103240966797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870722407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13332870602607727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870738564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13494209945201874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870754363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13455268740653992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870754363, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784870754364, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784870780569, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5316870212554932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784870780569, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784870780569, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784870796295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13590353727340698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870813127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13392561674118042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870829341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13309741020202637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3650000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870837335, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784870837336, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784870863066, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5359281301498413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784870863066, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784870863066, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784870871390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1402277946472168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870887470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352296769618988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870903767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13432475924491882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4649999999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870920651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13512077927589417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870920652, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784870920653, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784870946536, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5404513478279114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784870946536, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784870946536, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784870962498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13509441912174225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870978663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13320186734199524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.5649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870994691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13217371702194214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871002856, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784871002856, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784871030117, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5453173518180847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784871030118, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784871030118, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784871038073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13241620361804962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871054450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13073284924030304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871069970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13577063381671906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871085783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13206464052200317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871085784, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784871085784, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784871109498, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5512090921401978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784871109499, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784871109499, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784871125404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13443222641944885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871141908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12998931109905243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871157814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317145824432373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871165678, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784871165679, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784871192047, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5588834881782532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784871192047, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784871192048, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784871199976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13203224539756775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871215787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13036257028579712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871231783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13272826373577118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871247704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13223569095134735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871247705, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784871247705, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784871275502, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5679917931556702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784871275502, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784871275502, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784871291401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13169607520103455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871307181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12770648300647736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871322909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13355958461761475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871330762, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784871330763, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784871357143, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5770664215087891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784871357143, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784871357143, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784871365215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13416656851768494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871380994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12638649344444275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871397324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12626144289970398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871413742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324056088924408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871413742, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784871413743, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784871439480, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5879344940185547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784871439481, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784871439481, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784871455270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1285756230354309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871471179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1278831660747528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871487451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1274101734161377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871495172, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784871495172, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784871519611, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5939127802848816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784871519611, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784871519611, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784871527530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12935884296894073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871554202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1267962008714676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871570612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12928639352321625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871586898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12763184309005737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871586899, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784871586900, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784871613067, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6026132106781006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784871613067, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784871613068, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784871629203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12602275609970093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871645854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12911652028560638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4983333333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871661911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1319112330675125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871669977, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784871669977, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784871694672, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6098581552505493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784871694672, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784871694672, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784871703466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13022561371326447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871720003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12552836537361145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871736574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13037095963954926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871752942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12553226947784424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871752943, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784871752943, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784871779548, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6164672374725342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784871779548, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784871779549, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784871796022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12795931100845337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871812156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12930437922477722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871828931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12728019058704376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871837263, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784871837264, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784871863043, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.622691810131073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784871863043, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784871863043, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784871871250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12660594284534454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871887529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1262943148612976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871904147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1266767829656601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871920533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12579941749572754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.898333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871920533, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784871920534, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784871946101, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6265881657600403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784871946101, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784871946101, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784871962949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12023770064115524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871979143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11959978938102722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871994943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11763253808021545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872002939, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784872002939, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784872029166, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6279382705688477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784872029166, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784872029167, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784872036982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11963594704866409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872052550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11677604913711548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872068509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12153321504592896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872084180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11853953450918198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872084181, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784872084181, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784872109089, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6378780603408813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784872109090, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784872109090, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784872125878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12015789747238159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872142018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11148414015769958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.198333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872158117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1174096167087555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872166229, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784872166230, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784872192338, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6482210755348206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784872192338, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784872192339, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784872200175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11885418742895126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872216816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11863496899604797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.298333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872233020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1131073608994484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872248737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11698250472545624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872248812, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784872248812, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784872276747, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6679155230522156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784872276748, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784872276748, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784872292580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11610491573810577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.398333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872308506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11613431572914124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872324644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10923086106777191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872332560, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784872332561, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784872358805, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6924450993537903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784872358806, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784872358806, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784872366759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11354181915521622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.498333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872383373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11482144892215729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.531666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872399403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11321911215782166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872416373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11503211408853531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.598333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872416374, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784872416374, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784872443267, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7239167094230652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784872443267, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784872443267, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784872459484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11391657590866089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872475359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11335363239049911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872491672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11755858361721039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872499293, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784872499294, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784872525739, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7435150146484375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784872525740, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784872525740, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784872533798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10759831964969635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872550140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11503562331199646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872566259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10865181684494019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872582563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11289098113775253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872582564, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784872582564, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784872609186, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7529715895652771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784872609187, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784872609187, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784872625315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1160656288266182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872641295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11339071393013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872657561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11164426803588867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872665088, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784872665089, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784872691195, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7576790452003479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784872691196, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784872691196, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784872699524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11168144643306732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872715810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1132410392165184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872732452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11326690763235092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872748006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10769503563642502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872748007, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784872748007, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784872773078, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7592757940292358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784872773079, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784872773079, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784872790514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11586843430995941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872807142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11998851597309113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872823872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1166994497179985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872832137, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784872832138, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784872857968, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7606172561645508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784872857969, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784872857969, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784872865939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1163027212023735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872882863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11587199568748474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872899154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11297629773616791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872915140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11374258249998093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872915141, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784872915141, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784872942715, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7617396116256714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784872942716, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784872942716, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784872959326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11830100417137146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872975593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11777079105377197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872992644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11115650087594986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873000693, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784873000694, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784873026701, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7615295052528381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784873026701, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784873026701, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784873035105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11601066589355469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873051808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11578478664159775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873068220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1117202416062355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873085156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10462786257266998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873085156, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784873085157, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784873111212, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7637561559677124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784873111212, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784873111213, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784873127378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1067371815443039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873144027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1063401997089386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873160307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10635355114936829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873168518, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784873168519, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784873194499, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7642830610275269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784873194500, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784873194500, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784873202756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10425573587417603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873219577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11139756441116333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873235824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10787396132946014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873252586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11303438991308212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873252586, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784873252587, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784873279658, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7664561867713928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784873279659, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784873279659, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784873296303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1069658100605011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873312357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10635150969028473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873328988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10213606059551239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873336986, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784873336986, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784873363484, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7670254111289978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784873363485, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784873363485, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784873371850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10305430740118027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873388345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10174938291311264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873405313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11162330210208893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873421267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10425014048814774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873421268, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784873421268, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784873445143, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7686237096786499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784873445144, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784873445144, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784873461569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10659242421388626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873478140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10634402930736542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873493943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10711205005645752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873502288, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784873502288, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784873528126, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7692049145698547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784873528127, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784873528127, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784873536302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10699662566184998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873552730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1000220775604248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873569247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10783572494983673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873585796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10437917709350586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873585796, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784873585797, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784873610093, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7692315578460693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784873610094, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784873610094, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784873626114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10789170116186142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873643222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10821069777011871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873660013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10443820059299469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873668073, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784873668074, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784873693404, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7689886689186096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784873693404, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784873693404, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784873701525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10912206768989563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873718013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10803283005952835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873734680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11195133626461029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873751393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10835914313793182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873751394, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784873751394, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784873777601, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7700977921485901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784873777601, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784873777602, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784873794333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10566505044698715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873810475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10200470685958862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873827472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10803614556789398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.564999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873836161, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784873836162, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784873862628, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7706372737884521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784873862629, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784873862629, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784873870930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10440228879451752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873887318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09965522587299347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873904023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11450385302305222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.664999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873920895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11096329987049103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873920896, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784873920896, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784873947683, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7707976698875427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784873947684, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784873947684, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784873964264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10654191672801971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873981345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10904353857040405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873997851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10815221071243286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874005920, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784874005920, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784874030936, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7710488438606262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784874030937, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784874030937, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784874039789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10983755439519882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874057068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11480110138654709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874073974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11290328204631805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874090988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11294040083885193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874090989, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784874090990, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784874117845, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7705087661743164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784874117845, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784874117845, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784874134280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10997430235147476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874151297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11405563354492188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.998333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874168433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11079305410385132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874176681, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784874176682, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784874202240, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7710055112838745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784874202240, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784874202240, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784874210396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11095263063907623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874227936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10977377742528915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874244586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11282762140035629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.131666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874261318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11123505979776382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874261319, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784874261319, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784874287693, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7714809775352478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784874287693, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784874287693, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784874304375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11028919368982315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874321591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11301977932453156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874338357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11692669987678528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874346790, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784874346790, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784874372140, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.771756649017334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784874372141, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784874372141, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784874380072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1149681881070137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874397251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793116688728333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.331666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874413539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1014094352722168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874430245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10705330967903137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874430245, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784874430246, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784874456269, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7723060250282288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784874456269, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784874456269, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784874472725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10515276342630386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874489393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.106197290122509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874506312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1051708459854126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874514839, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784874514840, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784874540992, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7723135352134705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784874540992, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784874540993, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784874549127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10861322283744812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874565689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10356128215789795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874582474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10488907247781754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874599268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10764247179031372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.631666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874599268, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784874599269, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784874624791, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7725885510444641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784874624792, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784874624792, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784874640745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.100814089179039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874657201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10401462763547897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874673687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10362040996551514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874681738, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784874681739, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784874708129, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7726647853851318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784874708129, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784874708129, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784874716525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10408464074134827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874733532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10321643948554993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874749804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11388707906007767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874766553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10347025096416473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874766553, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784874766553, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784874793587, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7727397084236145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784874793587, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784874793588, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784874810455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1073504164814949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874826882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1073383018374443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874843464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10163310170173645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874851721, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784874851721, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784874877942, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7728522419929504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784874877942, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784874877943, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784874886294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10191021114587784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874902815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10063903033733368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874919762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1061922237277031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874936476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1033170223236084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874936477, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784874936477, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784874962037, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773128867149353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784874962038, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784874962038, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784874978203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10758358985185623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874995850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10372114181518555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875012531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10587470233440399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875020758, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784875020759, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784875047035, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7732409238815308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784875047036, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784875047036, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784875055404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10351286083459854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875071754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10826461017131805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875088199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1026310995221138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875105092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10695387423038483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875105092, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784875105092, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784875131068, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7734061479568481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784875131068, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784875131069, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784875148010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10209468007087708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875164330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10664685070514679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875181119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10954679548740387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875189626, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784875189627, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784875215972, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7735728621482849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784875215973, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784875215973, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784875224598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11174821853637695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875240966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10717251151800156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875257848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10295665264129639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875274726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10287126898765564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875274726, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784875274727, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784875301164, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773654580116272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784875301165, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784875301165, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784875317627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10529692471027374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875334946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10536757111549377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.631666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875351940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11221002042293549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875360452, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784875360453, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784875387365, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738010883331299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784875387365, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784875387366, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784875395386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1073450818657875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875412576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11271215230226517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875429939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10911401361227036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875446878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11275266855955124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875446879, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784875446879, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784875473116, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738651037216187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784875473117, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784875473117, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784875490207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10929903388023376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.831666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875507213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11402737349271774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875524237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11219710856676102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875532864, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784875532865, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784875558454, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7740628123283386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784875558455, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784875558455, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784875567121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11236479133367538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875583839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11181817948818207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875601221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1075657457113266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875618639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11078696697950363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1.0031666666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784875618640, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784875618640, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784875645406, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774256706237793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784875645406, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784875645406, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784875662105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11391526460647583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1.0064999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784875679442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10874009132385254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1.0098333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784875696403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11155892163515091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1.0131666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784875705253, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784875705254, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784875731648, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742708921432495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784875731648, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784875731648, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784875740120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1114581748843193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1.0165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784875757486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10381974279880524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1.0198333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784875773820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11052607744932175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1.0231666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784875791233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10853676497936249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1.0264999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784875791233, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784875791234, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784875817241, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774607241153717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784875817241, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784875817242, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784875833902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10347690433263779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1.0298333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784875850297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09992682188749313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1.0331666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784875866664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1020459532737732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1.0365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784875875027, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784875875027, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784875900240, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7747066020965576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784875900241, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784875900241, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784875908543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09985481202602386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1.0398333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784875925540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10087360441684723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1.0431666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784875941982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10275264084339142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1.0464999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784875958943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10574311017990112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1.0498333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784875958944, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784875958944, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784875985233, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7747799158096313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784875985234, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784875985234, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784876001194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10453285276889801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1.0531666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876018495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11029452085494995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1.0565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876035394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10615034401416779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1.0598333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876043703, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784876043704, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784876070451, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774861752986908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784876070451, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784876070452, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784876078927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1015370637178421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1.0631666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876095796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10646311193704605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1.0664999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876112657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1005607470870018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1.0698333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876129224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10189248621463776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1.0731666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876129225, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784876129225, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784876156018, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7749244570732117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784876156019, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784876156019, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784876172804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10196132957935333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1.0765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876190193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1037810891866684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1.0798333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876206424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10158762335777283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1.0831666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876221957, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784876221959, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784876248455, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.775108277797699, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784876248455, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784876248455, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784876256577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10416729003190994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1.0864999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876273580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732139647006989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1.0898333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876289850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10241137444972992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1.0931666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876306686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1006992906332016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1.0965e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876306686, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784876306687, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784876332461, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752127647399902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784876332461, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784876332461, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784876349233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10528814047574997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1.0998333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876365462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1017683744430542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1.1031666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876382383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10212449729442596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217579520, "lr": 1.1065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876390691, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784876390692, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784876416805, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752569317817688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784876416805, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784876416805, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784876425320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11202415078878403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218234880, "lr": 1.1098333333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876441522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1095343679189682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218890240, "lr": 1.1131666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876458102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1077665388584137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219545600, "lr": 1.1165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876475253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10833108425140381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220200960, "lr": 1.1198333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876475254, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784876475254, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784876501610, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7754195332527161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784876501611, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784876501611, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784876518211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10268713533878326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220856320, "lr": 1.1231666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876535068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10478586703538895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221511680, "lr": 1.1265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876551731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10077724605798721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222167040, "lr": 1.1298333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876560126, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784876560126, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784876586316, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.775762677192688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784876586317, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784876586317, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784876594752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10516747087240219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222822400, "lr": 1.1331666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876612033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10425721108913422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223477760, "lr": 1.1365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876628921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10712283104658127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224133120, "lr": 1.1398333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876646027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10914191603660583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224788480, "lr": 1.1431666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876646028, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784876646028, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784876671843, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7757933735847473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784876671844, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784876671844, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784876688372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10891011357307434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225443840, "lr": 1.1465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876705616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10883310437202454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226099200, "lr": 1.1498333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876723329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10767901688814163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226754560, "lr": 1.1531666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876731540, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784876731541, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784876757956, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7759814858436584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784876757957, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784876757957, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784876766396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1125701442360878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227409920, "lr": 1.1565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876783381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11014118045568466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228065280, "lr": 1.1598333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876800640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10735329985618591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228720640, "lr": 1.1631666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876817525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10669127851724625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229376000, "lr": 1.1665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876817526, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784876817526, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784876843551, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.776190996170044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784876843552, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784876843552, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784876860819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10663922131061554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230031360, "lr": 1.1698333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876877938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10510078817605972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230686720, "lr": 1.1731666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876894883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11113302409648895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231342080, "lr": 1.1765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876903037, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784876903037, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784876929708, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7761644721031189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784876929709, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784876929709, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784876937993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10852935165166855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231997440, "lr": 1.1798333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876955608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1103091612458229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 232652800, "lr": 1.1831666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876973478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11081992089748383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233308160, "lr": 1.1865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876990733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1117633655667305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233963520, "lr": 1.1898333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876990734, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784876990735, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784877017147, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.776522159576416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784877017147, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784877017147, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784877034418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11444389820098877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 234618880, "lr": 1.1931666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877051930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11344210803508759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235274240, "lr": 1.1965e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877069031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11159084737300873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235929600, "lr": 1.1998333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877077827, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784877077828, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784877104135, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7765971422195435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784877104135, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784877104136, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784877112499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10889574885368347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 236584960, "lr": 1.2031666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877130111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11282279342412949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237240320, "lr": 1.2064999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877146629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11385565996170044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237895680, "lr": 1.2098333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877164416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1188768595457077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 238551040, "lr": 1.2131666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877164416, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784877164417, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784877190947, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7767658829689026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784877190948, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784877190948, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784877208321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11765376478433609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239206400, "lr": 1.2165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877225568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11369530856609344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239861760, "lr": 1.2198333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877242921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11364969611167908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 240517120, "lr": 1.2231666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877251771, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784877251772, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784877277340, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7770736217498779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784877277340, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784877277340, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784877285901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1145663931965828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241172480, "lr": 1.2264999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877303232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11527255922555923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241827840, "lr": 1.2298333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877320473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11450725048780441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 242483200, "lr": 1.2331666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877337390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11855706572532654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243138560, "lr": 1.2365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877337391, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784877337392, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784877362317, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.77735435962677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784877362318, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784877362318, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784877380326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10944036394357681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243793920, "lr": 1.2398333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877397515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11549168080091476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 244449280, "lr": 1.2431666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877414868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1143721491098404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245104640, "lr": 1.2464999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877423531, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784877423531, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784877449005, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.777315080165863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784877449005, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784877449005, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784877457594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11085381358861923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245760000, "lr": 1.2498333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877474883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1125624030828476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 246415360, "lr": 1.2531666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877492260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080082580447197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247070720, "lr": 1.2565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877509507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10776368528604507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247726080, "lr": 1.2598333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877509507, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784877509508, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784877535612, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7775195240974426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784877535613, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784877535613, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784877552378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11380261927843094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 248381440, "lr": 1.2631666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877569781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10983745008707047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249036800, "lr": 1.2664999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877587198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10837839543819427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249692160, "lr": 1.2698333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877595668, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784877595668, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784877621718, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7776509523391724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784877621718, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784877621719, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784877630206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10612668097019196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 250347520, "lr": 1.2731666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877647487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10492366552352905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251002880, "lr": 1.2765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877664867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10454561561346054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251658240, "lr": 1.2798333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877681796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10886861383914948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 252313600, "lr": 1.2831666666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877681796, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784877681797, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784877707802, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7777895331382751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784877707803, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784877707803, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784877725088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11121919751167297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 252968960, "lr": 1.2864999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877742346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10549067705869675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 253624320, "lr": 1.2898333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877758818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10815805196762085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 254279680, "lr": 1.2931666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877767810, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784877767810, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784877794154, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7780740261077881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784877794155, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784877794155, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784877802408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1068798303604126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 254935040, "lr": 1.2965e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877819570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1067953035235405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 255590400, "lr": 1.2998333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877836203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10952312499284744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 256245760, "lr": 1.3031666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877853219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1076192557811737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 256901120, "lr": 1.3065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877853220, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784877853220, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784877879068, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7782573103904724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784877879068, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784877879068, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784877895926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10677085071802139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 257556480, "lr": 1.3098333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877912616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10528752207756042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 258211840, "lr": 1.3131666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877929836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10414434969425201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 258867200, "lr": 1.3165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877938453, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784877938453, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784877965256, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.778245747089386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784877965256, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784877965256, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784877974004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11186493933200836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 259522560, "lr": 1.3198333333333335e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877990612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10650398582220078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 260177920, "lr": 1.3231666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878008028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10539013892412186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 260833280, "lr": 1.3265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878025407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10577202588319778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 261488640, "lr": 1.3298333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878025408, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784878025408, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784878051655, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7785159945487976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784878051656, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784878051656, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784878068522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1031506359577179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 262144000, "lr": 1.3331666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878085478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10132347047328949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 262799360, "lr": 1.3365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878102368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10498636215925217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 263454720, "lr": 1.3398333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878110804, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 263782400, "epoch_num": 0.11514681384425231}} +:::MLLOG {"namespace": "", "time_ms": 1784878110804, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 263782400, "epoch_num": 0.11514681384425231}} +:::MLLOG {"namespace": "", "time_ms": 1784878136281, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.778559684753418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 263782400, "epoch_num": 0.11514681384425231}} +:::MLLOG {"namespace": "", "time_ms": 1784878136281, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 263782400, "epoch_num": 0.11514681384425231}} +:::MLLOG {"namespace": "", "time_ms": 1784878136281, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 263782400, "epoch_num": 0.11514681384425231}} +:::MLLOG {"namespace": "", "time_ms": 1784878144704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10908696055412292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 264110080, "lr": 1.3431666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878162005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10813461244106293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 264765440, "lr": 1.3465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878178868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10484569519758224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 265420800, "lr": 1.3498333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878195843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10393667966127396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 266076160, "lr": 1.3531666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878195843, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 266076160, "epoch_num": 0.11614809048637624}} +:::MLLOG {"namespace": "", "time_ms": 1784878195843, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 266076160, "epoch_num": 0.11614809048637624}} +:::MLLOG {"namespace": "", "time_ms": 1784878219834, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7789802551269531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 266076160, "epoch_num": 0.11614809048637624}} +:::MLLOG {"namespace": "", "time_ms": 1784878219834, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 266076160, "epoch_num": 0.11614809048637624}} +:::MLLOG {"namespace": "", "time_ms": 1784878219835, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 266076160, "epoch_num": 0.11614809048637624}} +:::MLLOG {"namespace": "", "time_ms": 1784878236258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10451085865497589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 266731520, "lr": 1.3565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878254300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071353480219841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 267386880, "lr": 1.359833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878271674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10895939916372299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 268042240, "lr": 1.3631666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878280532, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 268369920, "epoch_num": 0.11714936712850017}} +:::MLLOG {"namespace": "", "time_ms": 1784878280533, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 268369920, "epoch_num": 0.11714936712850017}} +:::MLLOG {"namespace": "", "time_ms": 1784878306866, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7790228724479675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 268369920, "epoch_num": 0.11714936712850017}} +:::MLLOG {"namespace": "", "time_ms": 1784878306867, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 268369920, "epoch_num": 0.11714936712850017}} +:::MLLOG {"namespace": "", "time_ms": 1784878306867, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 268369920, "epoch_num": 0.11714936712850017}} +:::MLLOG {"namespace": "", "time_ms": 1784878315317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11114048212766647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 268697600, "lr": 1.3665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878332417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10535639524459839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 269352960, "lr": 1.3698333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878349489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10365013778209686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 270008320, "lr": 1.3731666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878366597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10961153358221054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 270663680, "lr": 1.3765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878366598, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 270663680, "epoch_num": 0.1181506437706241}} +:::MLLOG {"namespace": "", "time_ms": 1784878366598, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 270663680, "epoch_num": 0.1181506437706241}} +:::MLLOG {"namespace": "", "time_ms": 1784878392899, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7793654799461365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 270663680, "epoch_num": 0.1181506437706241}} +:::MLLOG {"namespace": "", "time_ms": 1784878392900, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 270663680, "epoch_num": 0.1181506437706241}} +:::MLLOG {"namespace": "", "time_ms": 1784878392900, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 270663680, "epoch_num": 0.1181506437706241}} +:::MLLOG {"namespace": "", "time_ms": 1784878410236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.110635906457901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 271319040, "lr": 1.379833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878427782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11084513366222382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 271974400, "lr": 1.3831666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878444605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10903294384479523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 272629760, "lr": 1.3865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878453024, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 272957440, "epoch_num": 0.11915192041274804}} +:::MLLOG {"namespace": "", "time_ms": 1784878453025, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 272957440, "epoch_num": 0.11915192041274804}} +:::MLLOG {"namespace": "", "time_ms": 1784878479048, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7795668244361877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 272957440, "epoch_num": 0.11915192041274804}} +:::MLLOG {"namespace": "", "time_ms": 1784878479048, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 272957440, "epoch_num": 0.11915192041274804}} +:::MLLOG {"namespace": "", "time_ms": 1784878479048, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 272957440, "epoch_num": 0.11915192041274804}} +:::MLLOG {"namespace": "", "time_ms": 1784878487372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11149115860462189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 273285120, "lr": 1.3898333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878505552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11069142818450928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 273940480, "lr": 1.3931666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878522647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11795582622289658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 274595840, "lr": 1.3965e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878539842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11608877032995224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 275251200, "lr": 1.3998333333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878539842, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 275251200, "epoch_num": 0.12015319705487197}} +:::MLLOG {"namespace": "", "time_ms": 1784878539843, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 275251200, "epoch_num": 0.12015319705487197}} +:::MLLOG {"namespace": "", "time_ms": 1784878566408, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7796778082847595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 275251200, "epoch_num": 0.12015319705487197}} +:::MLLOG {"namespace": "", "time_ms": 1784878566408, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 275251200, "epoch_num": 0.12015319705487197}} +:::MLLOG {"namespace": "", "time_ms": 1784878566408, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 275251200, "epoch_num": 0.12015319705487197}} +:::MLLOG {"namespace": "", "time_ms": 1784878583631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111283078789711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 275906560, "lr": 1.4031666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878600862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11022337526082993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 276561920, "lr": 1.4065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878618721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11875507235527039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 277217280, "lr": 1.4098333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878627390, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 277544960, "epoch_num": 0.1211544736969959}} +:::MLLOG {"namespace": "", "time_ms": 1784878627391, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 277544960, "epoch_num": 0.1211544736969959}} +:::MLLOG {"namespace": "", "time_ms": 1784878653464, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7800881266593933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 277544960, "epoch_num": 0.1211544736969959}} +:::MLLOG {"namespace": "", "time_ms": 1784878653465, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 277544960, "epoch_num": 0.1211544736969959}} +:::MLLOG {"namespace": "", "time_ms": 1784878653465, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 277544960, "epoch_num": 0.1211544736969959}} +:::MLLOG {"namespace": "", "time_ms": 1784878662156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11289004236459732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 277872640, "lr": 1.4131666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878679158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10954069346189499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 278528000, "lr": 1.4165000000000001e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878696945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11747020483016968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 279183360, "lr": 1.4198333333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878714419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10951782017946243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 279838720, "lr": 1.4231666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878714420, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 279838720, "epoch_num": 0.12215575033911984}} +:::MLLOG {"namespace": "", "time_ms": 1784878714421, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 279838720, "epoch_num": 0.12215575033911984}} +:::MLLOG {"namespace": "", "time_ms": 1784878741254, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7802106142044067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 279838720, "epoch_num": 0.12215575033911984}} +:::MLLOG {"namespace": "", "time_ms": 1784878741254, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 279838720, "epoch_num": 0.12215575033911984}} +:::MLLOG {"namespace": "", "time_ms": 1784878741254, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 279838720, "epoch_num": 0.12215575033911984}} +:::MLLOG {"namespace": "", "time_ms": 1784878758283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11112889647483826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 280494080, "lr": 1.4265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878775584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11212456226348877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 281149440, "lr": 1.4298333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878792913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11575989425182343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 281804800, "lr": 1.4331666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878801610, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 282132480, "epoch_num": 0.12315702698124378}} +:::MLLOG {"namespace": "", "time_ms": 1784878801611, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 282132480, "epoch_num": 0.12315702698124378}} +:::MLLOG {"namespace": "", "time_ms": 1784878827995, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7803984880447388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 282132480, "epoch_num": 0.12315702698124378}} +:::MLLOG {"namespace": "", "time_ms": 1784878827995, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 282132480, "epoch_num": 0.12315702698124378}} +:::MLLOG {"namespace": "", "time_ms": 1784878827995, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 282132480, "epoch_num": 0.12315702698124378}} +:::MLLOG {"namespace": "", "time_ms": 1784878836816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10680616647005081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 282460160, "lr": 1.4365000000000001e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878854711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11350098252296448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 283115520, "lr": 1.4398333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878871989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11177115142345428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 283770880, "lr": 1.4431666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878889004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11131538450717926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 284426240, "lr": 1.4464999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878889005, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 284426240, "epoch_num": 0.12415830362336772}} +:::MLLOG {"namespace": "", "time_ms": 1784878889005, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 284426240, "epoch_num": 0.12415830362336772}} +:::MLLOG {"namespace": "", "time_ms": 1784878916006, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.780606746673584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 284426240, "epoch_num": 0.12415830362336772}} +:::MLLOG {"namespace": "", "time_ms": 1784878916007, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 284426240, "epoch_num": 0.12415830362336772}} +:::MLLOG {"namespace": "", "time_ms": 1784878916007, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 284426240, "epoch_num": 0.12415830362336772}} +:::MLLOG {"namespace": "", "time_ms": 1784878933218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10864861309528351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 285081600, "lr": 1.4498333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878950482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11331866681575775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 285736960, "lr": 1.4531666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878967887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1138535737991333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 286392320, "lr": 1.4564999999999997e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878976298, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 286720000, "epoch_num": 0.12515958026549165}} +:::MLLOG {"namespace": "", "time_ms": 1784878976299, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 286720000, "epoch_num": 0.12515958026549165}} +:::MLLOG {"namespace": "", "time_ms": 1784879001716, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7806305289268494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 286720000, "epoch_num": 0.12515958026549165}} +:::MLLOG {"namespace": "", "time_ms": 1784879001717, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 286720000, "epoch_num": 0.12515958026549165}} +:::MLLOG {"namespace": "", "time_ms": 1784879001717, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 286720000, "epoch_num": 0.12515958026549165}} +:::MLLOG {"namespace": "", "time_ms": 1784879010343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11447151005268097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 287047680, "lr": 1.4598333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879027620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11074212193489075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 287703040, "lr": 1.4631666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879045382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11306057870388031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 288358400, "lr": 1.4664999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879062679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11345760524272919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 289013760, "lr": 1.4698333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879062679, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 289013760, "epoch_num": 0.12616085690761558}} +:::MLLOG {"namespace": "", "time_ms": 1784879062680, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 289013760, "epoch_num": 0.12616085690761558}} +:::MLLOG {"namespace": "", "time_ms": 1784879089182, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7809916734695435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 289013760, "epoch_num": 0.12616085690761558}} +:::MLLOG {"namespace": "", "time_ms": 1784879089182, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 289013760, "epoch_num": 0.12616085690761558}} +:::MLLOG {"namespace": "", "time_ms": 1784879089182, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 289013760, "epoch_num": 0.12616085690761558}} +:::MLLOG {"namespace": "", "time_ms": 1784879106286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772402584552765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 289669120, "lr": 1.4731666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879123172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10544408857822418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 290324480, "lr": 1.4764999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879141365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10091269016265869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 290979840, "lr": 1.4798333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879150053, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 291307520, "epoch_num": 0.12716213354973951}} +:::MLLOG {"namespace": "", "time_ms": 1784879150053, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 291307520, "epoch_num": 0.12716213354973951}} +:::MLLOG {"namespace": "", "time_ms": 1784879176014, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7807896137237549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 291307520, "epoch_num": 0.12716213354973951}} +:::MLLOG {"namespace": "", "time_ms": 1784879176015, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 291307520, "epoch_num": 0.12716213354973951}} +:::MLLOG {"namespace": "", "time_ms": 1784879176015, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 291307520, "epoch_num": 0.12716213354973951}} +:::MLLOG {"namespace": "", "time_ms": 1784879184593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10580084472894669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 291635200, "lr": 1.4831666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879201809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10873007774353027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 292290560, "lr": 1.4864999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879219042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10485218465328217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 292945920, "lr": 1.4898333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879236178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10538893193006516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 293601280, "lr": 1.4931666666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879236179, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 293601280, "epoch_num": 0.12816341019186345}} +:::MLLOG {"namespace": "", "time_ms": 1784879236180, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 293601280, "epoch_num": 0.12816341019186345}} +:::MLLOG {"namespace": "", "time_ms": 1784879262241, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7805507779121399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 293601280, "epoch_num": 0.12816341019186345}} +:::MLLOG {"namespace": "", "time_ms": 1784879262241, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 293601280, "epoch_num": 0.12816341019186345}} +:::MLLOG {"namespace": "", "time_ms": 1784879262242, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 293601280, "epoch_num": 0.12816341019186345}} +:::MLLOG {"namespace": "", "time_ms": 1784879279452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10802070796489716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 294256640, "lr": 1.4964999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879296813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10394807159900665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 294912000, "lr": 1.4998333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879314628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10781040787696838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 295567360, "lr": 1.5031666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879323449, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 295895040, "epoch_num": 0.12916468683398738}} +:::MLLOG {"namespace": "", "time_ms": 1784879323449, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 295895040, "epoch_num": 0.12916468683398738}} +:::MLLOG {"namespace": "", "time_ms": 1784879350842, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7808706164360046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 295895040, "epoch_num": 0.12916468683398738}} +:::MLLOG {"namespace": "", "time_ms": 1784879350843, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 295895040, "epoch_num": 0.12916468683398738}} +:::MLLOG {"namespace": "", "time_ms": 1784879350843, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 295895040, "epoch_num": 0.12916468683398738}} +:::MLLOG {"namespace": "", "time_ms": 1784879359400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1041443794965744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 296222720, "lr": 1.5064999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879375915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1051386296749115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 296878080, "lr": 1.5098333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879393681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10876233875751495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 297533440, "lr": 1.5131666666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879410996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10965332388877869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 298188800, "lr": 1.5164999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879410996, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 298188800, "epoch_num": 0.1301659634761113}} +:::MLLOG {"namespace": "", "time_ms": 1784879410997, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 298188800, "epoch_num": 0.1301659634761113}} +:::MLLOG {"namespace": "", "time_ms": 1784879437607, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7808049917221069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 298188800, "epoch_num": 0.1301659634761113}} +:::MLLOG {"namespace": "", "time_ms": 1784879437607, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 298188800, "epoch_num": 0.1301659634761113}} +:::MLLOG {"namespace": "", "time_ms": 1784879437607, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 298188800, "epoch_num": 0.1301659634761113}} +:::MLLOG {"namespace": "", "time_ms": 1784879454977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10519455373287201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 298844160, "lr": 1.5198333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879472062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10322828590869904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 299499520, "lr": 1.5231666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879489246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10389071702957153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 300154880, "lr": 1.5265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879497741, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 300482560, "epoch_num": 0.13116724011823525}} +:::MLLOG {"namespace": "", "time_ms": 1784879497742, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 300482560, "epoch_num": 0.13116724011823525}} +:::MLLOG {"namespace": "", "time_ms": 1784879524243, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7810112237930298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 300482560, "epoch_num": 0.13116724011823525}} +:::MLLOG {"namespace": "", "time_ms": 1784879524243, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 300482560, "epoch_num": 0.13116724011823525}} +:::MLLOG {"namespace": "", "time_ms": 1784879524243, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 300482560, "epoch_num": 0.13116724011823525}} +:::MLLOG {"namespace": "", "time_ms": 1784879532917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11041580885648727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 300810240, "lr": 1.5298333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879549837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1000039279460907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 301465600, "lr": 1.5331666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879567324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10109160840511322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 302120960, "lr": 1.5364999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879584754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10718171298503876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 302776320, "lr": 1.5398333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879584755, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 302776320, "epoch_num": 0.13216851676035918}} +:::MLLOG {"namespace": "", "time_ms": 1784879584755, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 302776320, "epoch_num": 0.13216851676035918}} +:::MLLOG {"namespace": "", "time_ms": 1784879611101, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.780792772769928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 302776320, "epoch_num": 0.13216851676035918}} +:::MLLOG {"namespace": "", "time_ms": 1784879611101, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 302776320, "epoch_num": 0.13216851676035918}} +:::MLLOG {"namespace": "", "time_ms": 1784879611101, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 302776320, "epoch_num": 0.13216851676035918}} +:::MLLOG {"namespace": "", "time_ms": 1784879627786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10989843308925629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 303431680, "lr": 1.5431666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879645255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1079757958650589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 304087040, "lr": 1.5465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879662728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10371625423431396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 304742400, "lr": 1.5498333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879671243, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 305070080, "epoch_num": 0.1331697934024831}} +:::MLLOG {"namespace": "", "time_ms": 1784879671244, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 305070080, "epoch_num": 0.1331697934024831}} +:::MLLOG {"namespace": "", "time_ms": 1784879698118, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7811894416809082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 305070080, "epoch_num": 0.1331697934024831}} +:::MLLOG {"namespace": "", "time_ms": 1784879698118, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 305070080, "epoch_num": 0.1331697934024831}} +:::MLLOG {"namespace": "", "time_ms": 1784879698118, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 305070080, "epoch_num": 0.1331697934024831}} +:::MLLOG {"namespace": "", "time_ms": 1784879706742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10205487161874771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 305397760, "lr": 1.5531666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879723959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10760756582021713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 306053120, "lr": 1.5564999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879740961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10541243851184845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 306708480, "lr": 1.5598333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879757877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10543207824230194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 307363840, "lr": 1.5631666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879757877, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 307363840, "epoch_num": 0.13417107004460704}} +:::MLLOG {"namespace": "", "time_ms": 1784879757878, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 307363840, "epoch_num": 0.13417107004460704}} +:::MLLOG {"namespace": "", "time_ms": 1784879784204, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7813134789466858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 307363840, "epoch_num": 0.13417107004460704}} +:::MLLOG {"namespace": "", "time_ms": 1784879784204, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 307363840, "epoch_num": 0.13417107004460704}} +:::MLLOG {"namespace": "", "time_ms": 1784879784205, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 307363840, "epoch_num": 0.13417107004460704}} +:::MLLOG {"namespace": "", "time_ms": 1784879801471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10665886104106903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 308019200, "lr": 1.5665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879818636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.098711296916008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 308674560, "lr": 1.5698333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879835595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10422119498252869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 309329920, "lr": 1.5731666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879844223, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 309657600, "epoch_num": 0.13517234668673098}} +:::MLLOG {"namespace": "", "time_ms": 1784879844224, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 309657600, "epoch_num": 0.13517234668673098}} +:::MLLOG {"namespace": "", "time_ms": 1784879869918, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7812719345092773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 309657600, "epoch_num": 0.13517234668673098}} +:::MLLOG {"namespace": "", "time_ms": 1784879869919, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 309657600, "epoch_num": 0.13517234668673098}} +:::MLLOG {"namespace": "", "time_ms": 1784879869919, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 309657600, "epoch_num": 0.13517234668673098}} +:::MLLOG {"namespace": "", "time_ms": 1784879878662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10782726854085922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 309985280, "lr": 1.5764999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879894942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10205455869436264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 310640640, "lr": 1.5798333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879912651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10922025889158249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 311296000, "lr": 1.5831666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879930285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10817717015743256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 311951360, "lr": 1.5865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879930286, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 311951360, "epoch_num": 0.1361736233288549}} +:::MLLOG {"namespace": "", "time_ms": 1784879930286, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 311951360, "epoch_num": 0.1361736233288549}} +:::MLLOG {"namespace": "", "time_ms": 1784879956640, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7817267775535583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 311951360, "epoch_num": 0.1361736233288549}} +:::MLLOG {"namespace": "", "time_ms": 1784879956641, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 311951360, "epoch_num": 0.1361736233288549}} +:::MLLOG {"namespace": "", "time_ms": 1784879956641, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 311951360, "epoch_num": 0.1361736233288549}} +:::MLLOG {"namespace": "", "time_ms": 1784879973810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10596449673175812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 312606720, "lr": 1.5898333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879990873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10644502937793732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 313262080, "lr": 1.5931666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880008335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10489962995052338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 313917440, "lr": 1.5964999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880016431, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 314245120, "epoch_num": 0.13717489997097884}} +:::MLLOG {"namespace": "", "time_ms": 1784880016432, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 314245120, "epoch_num": 0.13717489997097884}} +:::MLLOG {"namespace": "", "time_ms": 1784880043190, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7820276618003845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 314245120, "epoch_num": 0.13717489997097884}} +:::MLLOG {"namespace": "", "time_ms": 1784880043191, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 314245120, "epoch_num": 0.13717489997097884}} +:::MLLOG {"namespace": "", "time_ms": 1784880043191, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 314245120, "epoch_num": 0.13717489997097884}} +:::MLLOG {"namespace": "", "time_ms": 1784880051944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10890327394008636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 314572800, "lr": 1.5998333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880068996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10426285862922668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 315228160, "lr": 1.6031666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880086367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10358267277479172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 315883520, "lr": 1.6065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880103353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10264913737773895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 316538880, "lr": 1.6098333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880103354, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 316538880, "epoch_num": 0.13817617661310277}} +:::MLLOG {"namespace": "", "time_ms": 1784880103354, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 316538880, "epoch_num": 0.13817617661310277}} +:::MLLOG {"namespace": "", "time_ms": 1784880129711, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7818729877471924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 316538880, "epoch_num": 0.13817617661310277}} +:::MLLOG {"namespace": "", "time_ms": 1784880129711, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 316538880, "epoch_num": 0.13817617661310277}} +:::MLLOG {"namespace": "", "time_ms": 1784880129711, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 316538880, "epoch_num": 0.13817617661310277}} +:::MLLOG {"namespace": "", "time_ms": 1784880147477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10759764909744263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 317194240, "lr": 1.6131666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880164278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1093674898147583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 317849600, "lr": 1.6165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880181880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11265158653259277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 318504960, "lr": 1.6198333333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880190711, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 318832640, "epoch_num": 0.1391774532552267}} +:::MLLOG {"namespace": "", "time_ms": 1784880190712, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 318832640, "epoch_num": 0.1391774532552267}} +:::MLLOG {"namespace": "", "time_ms": 1784880217487, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7825738191604614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 318832640, "epoch_num": 0.1391774532552267}} +:::MLLOG {"namespace": "", "time_ms": 1784880217487, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 318832640, "epoch_num": 0.1391774532552267}} +:::MLLOG {"namespace": "", "time_ms": 1784880217487, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 318832640, "epoch_num": 0.1391774532552267}} +:::MLLOG {"namespace": "", "time_ms": 1784880226461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1147259995341301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 319160320, "lr": 1.6231666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880243685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10432347655296326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 319815680, "lr": 1.6265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880261231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11272400617599487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 320471040, "lr": 1.6298333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880278734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081160306930542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 321126400, "lr": 1.6331666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880278735, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 321126400, "epoch_num": 0.14017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784880278735, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 321126400, "epoch_num": 0.14017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784880304539, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7824981212615967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 321126400, "epoch_num": 0.14017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784880304539, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 321126400, "epoch_num": 0.14017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784880304540, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 321126400, "epoch_num": 0.14017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784880321708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1110304519534111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 321781760, "lr": 1.6365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880339404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10878812521696091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 322437120, "lr": 1.6398333333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880356900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11272354423999786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 323092480, "lr": 1.6431666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880365752, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 323420160, "epoch_num": 0.14118000653947457}} +:::MLLOG {"namespace": "", "time_ms": 1784880365753, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 323420160, "epoch_num": 0.14118000653947457}} +:::MLLOG {"namespace": "", "time_ms": 1784880391595, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7824715375900269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 323420160, "epoch_num": 0.14118000653947457}} +:::MLLOG {"namespace": "", "time_ms": 1784880391596, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 323420160, "epoch_num": 0.14118000653947457}} +:::MLLOG {"namespace": "", "time_ms": 1784880391596, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 323420160, "epoch_num": 0.14118000653947457}} +:::MLLOG {"namespace": "", "time_ms": 1784880400191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10932572185993195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 323747840, "lr": 1.6465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880417742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10488694906234741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 324403200, "lr": 1.6498333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880434518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10581450164318085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 325058560, "lr": 1.6531666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880452609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11382389068603516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 325713920, "lr": 1.6565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880452610, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 325713920, "epoch_num": 0.1421812831815985}} +:::MLLOG {"namespace": "", "time_ms": 1784880452610, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 325713920, "epoch_num": 0.1421812831815985}} +:::MLLOG {"namespace": "", "time_ms": 1784880478826, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7831377983093262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 325713920, "epoch_num": 0.1421812831815985}} +:::MLLOG {"namespace": "", "time_ms": 1784880478826, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 325713920, "epoch_num": 0.1421812831815985}} +:::MLLOG {"namespace": "", "time_ms": 1784880478826, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 325713920, "epoch_num": 0.1421812831815985}} +:::MLLOG {"namespace": "", "time_ms": 1784880496200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10993654280900955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 326369280, "lr": 1.6598333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880513561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1119922548532486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327024640, "lr": 1.6631666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880531010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10788845270872116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680000, "lr": 1.6665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880539815, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 328007680, "epoch_num": 0.14318255982372244}} +:::MLLOG {"namespace": "", "time_ms": 1784880539815, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 328007680, "epoch_num": 0.14318255982372244}} +:::MLLOG {"namespace": "", "time_ms": 1784880566864, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7831099629402161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 328007680, "epoch_num": 0.14318255982372244}} +:::MLLOG {"namespace": "", "time_ms": 1784880566864, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 328007680, "epoch_num": 0.14318255982372244}} +:::MLLOG {"namespace": "", "time_ms": 1784880566864, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 328007680, "epoch_num": 0.14318255982372244}} +:::MLLOG {"namespace": "", "time_ms": 1784880575271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11398866027593613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 328335360, "lr": 1.6698333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880592647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081230565905571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 328990720, "lr": 1.6731666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880610079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1127447560429573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 329646080, "lr": 1.6765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880627860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10886513441801071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 330301440, "lr": 1.6798333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880627862, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 330301440, "epoch_num": 0.14418383646584637}} +:::MLLOG {"namespace": "", "time_ms": 1784880627863, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 330301440, "epoch_num": 0.14418383646584637}} +:::MLLOG {"namespace": "", "time_ms": 1784880654587, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7834937572479248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 330301440, "epoch_num": 0.14418383646584637}} +:::MLLOG {"namespace": "", "time_ms": 1784880654587, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 330301440, "epoch_num": 0.14418383646584637}} +:::MLLOG {"namespace": "", "time_ms": 1784880654588, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 330301440, "epoch_num": 0.14418383646584637}} +:::MLLOG {"namespace": "", "time_ms": 1784880671390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11487838625907898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 330956800, "lr": 1.6831666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880689465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1120140478014946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 331612160, "lr": 1.6865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880707067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10319690406322479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 332267520, "lr": 1.6898333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880715569, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 332595200, "epoch_num": 0.1451851131079703}} +:::MLLOG {"namespace": "", "time_ms": 1784880715570, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 332595200, "epoch_num": 0.1451851131079703}} +:::MLLOG {"namespace": "", "time_ms": 1784880741602, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7828986048698425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 332595200, "epoch_num": 0.1451851131079703}} +:::MLLOG {"namespace": "", "time_ms": 1784880741602, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 332595200, "epoch_num": 0.1451851131079703}} +:::MLLOG {"namespace": "", "time_ms": 1784880741602, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 332595200, "epoch_num": 0.1451851131079703}} +:::MLLOG {"namespace": "", "time_ms": 1784880750391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10714592039585114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 332922880, "lr": 1.6931666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880767532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10414561629295349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 333578240, "lr": 1.6964999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880785047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080765575170517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 334233600, "lr": 1.6998333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880802143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10169827938079834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 334888960, "lr": 1.7031666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880802143, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 334888960, "epoch_num": 0.14618638975009424}} +:::MLLOG {"namespace": "", "time_ms": 1784880802144, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 334888960, "epoch_num": 0.14618638975009424}} +:::MLLOG {"namespace": "", "time_ms": 1784880828355, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7832822799682617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 334888960, "epoch_num": 0.14618638975009424}} +:::MLLOG {"namespace": "", "time_ms": 1784880828356, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 334888960, "epoch_num": 0.14618638975009424}} +:::MLLOG {"namespace": "", "time_ms": 1784880828356, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 334888960, "epoch_num": 0.14618638975009424}} +:::MLLOG {"namespace": "", "time_ms": 1784880845532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1052163764834404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 335544320, "lr": 1.7064999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880862906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10674464702606201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 336199680, "lr": 1.7098333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880879856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10845504701137543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 336855040, "lr": 1.7131666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880888538, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 337182720, "epoch_num": 0.14718766639221817}} +:::MLLOG {"namespace": "", "time_ms": 1784880888539, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 337182720, "epoch_num": 0.14718766639221817}} +:::MLLOG {"namespace": "", "time_ms": 1784880914893, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7839930057525635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 337182720, "epoch_num": 0.14718766639221817}} +:::MLLOG {"namespace": "", "time_ms": 1784880914894, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 337182720, "epoch_num": 0.14718766639221817}} +:::MLLOG {"namespace": "", "time_ms": 1784880914894, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 337182720, "epoch_num": 0.14718766639221817}} +:::MLLOG {"namespace": "", "time_ms": 1784880923832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09997718781232834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 337510400, "lr": 1.7164999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880940564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10220523923635483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 338165760, "lr": 1.7198333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880958151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10721482336521149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 338821120, "lr": 1.7231666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880975986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10241539031267166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 339476480, "lr": 1.7265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880975987, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 339476480, "epoch_num": 0.1481889430343421}} +:::MLLOG {"namespace": "", "time_ms": 1784880975988, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 339476480, "epoch_num": 0.1481889430343421}} +:::MLLOG {"namespace": "", "time_ms": 1784881002090, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7838149070739746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 339476480, "epoch_num": 0.1481889430343421}} +:::MLLOG {"namespace": "", "time_ms": 1784881002091, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 339476480, "epoch_num": 0.1481889430343421}} +:::MLLOG {"namespace": "", "time_ms": 1784881002091, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 339476480, "epoch_num": 0.1481889430343421}} +:::MLLOG {"namespace": "", "time_ms": 1784881019251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1017722561955452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 340131840, "lr": 1.7298333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881036587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10357986390590668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 340787200, "lr": 1.7331666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881053855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10623490065336227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 341442560, "lr": 1.7364999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881062526, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 341770240, "epoch_num": 0.14919021967646603}} +:::MLLOG {"namespace": "", "time_ms": 1784881062527, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 341770240, "epoch_num": 0.14919021967646603}} +:::MLLOG {"namespace": "", "time_ms": 1784881089007, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7844852209091187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 341770240, "epoch_num": 0.14919021967646603}} +:::MLLOG {"namespace": "", "time_ms": 1784881089007, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 341770240, "epoch_num": 0.14919021967646603}} +:::MLLOG {"namespace": "", "time_ms": 1784881089007, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 341770240, "epoch_num": 0.14919021967646603}} +:::MLLOG {"namespace": "", "time_ms": 1784881097331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10348066687583923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 342097920, "lr": 1.7398333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881114317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1013488695025444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 342753280, "lr": 1.7431666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881131527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10627192258834839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 343408640, "lr": 1.7465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881148751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10817524790763855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 344064000, "lr": 1.7498333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881148751, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 344064000, "epoch_num": 0.15019149631858997}} +:::MLLOG {"namespace": "", "time_ms": 1784881148752, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 344064000, "epoch_num": 0.15019149631858997}} +:::MLLOG {"namespace": "", "time_ms": 1784881175306, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7840684652328491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 344064000, "epoch_num": 0.15019149631858997}} +:::MLLOG {"namespace": "", "time_ms": 1784881175306, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 344064000, "epoch_num": 0.15019149631858997}} +:::MLLOG {"namespace": "", "time_ms": 1784881175307, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 344064000, "epoch_num": 0.15019149631858997}} +:::MLLOG {"namespace": "", "time_ms": 1784881192869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10608062148094177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 344719360, "lr": 1.7531666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881209495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792084038257599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 345374720, "lr": 1.7564999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881226202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0998866930603981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 346030080, "lr": 1.7598333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881235339, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 346357760, "epoch_num": 0.1511927729607139}} +:::MLLOG {"namespace": "", "time_ms": 1784881235339, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 346357760, "epoch_num": 0.1511927729607139}} +:::MLLOG {"namespace": "", "time_ms": 1784881261942, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7832533121109009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 346357760, "epoch_num": 0.1511927729607139}} +:::MLLOG {"namespace": "", "time_ms": 1784881261942, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 346357760, "epoch_num": 0.1511927729607139}} +:::MLLOG {"namespace": "", "time_ms": 1784881261942, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 346357760, "epoch_num": 0.1511927729607139}} +:::MLLOG {"namespace": "", "time_ms": 1784881270719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10408343374729156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 346685440, "lr": 1.7631666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881288014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10297829657793045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 347340800, "lr": 1.7665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881305626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1067451760172844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 347996160, "lr": 1.7698333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881322648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10133975744247437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 348651520, "lr": 1.7731666666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881322715, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 348651520, "epoch_num": 0.15219404960283783}} +:::MLLOG {"namespace": "", "time_ms": 1784881322716, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 348651520, "epoch_num": 0.15219404960283783}} +:::MLLOG {"namespace": "", "time_ms": 1784881349248, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7845158576965332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 348651520, "epoch_num": 0.15219404960283783}} +:::MLLOG {"namespace": "", "time_ms": 1784881349248, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 348651520, "epoch_num": 0.15219404960283783}} +:::MLLOG {"namespace": "", "time_ms": 1784881349249, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 348651520, "epoch_num": 0.15219404960283783}} +:::MLLOG {"namespace": "", "time_ms": 1784881366468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10860717296600342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 349306880, "lr": 1.7764999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881383848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10718584060668945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 349962240, "lr": 1.7798333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881400820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10421346873044968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 350617600, "lr": 1.7831666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881409630, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 350945280, "epoch_num": 0.15319532624496177}} +:::MLLOG {"namespace": "", "time_ms": 1784881409631, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 350945280, "epoch_num": 0.15319532624496177}} +:::MLLOG {"namespace": "", "time_ms": 1784881436001, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7842366099357605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 350945280, "epoch_num": 0.15319532624496177}} +:::MLLOG {"namespace": "", "time_ms": 1784881436001, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 350945280, "epoch_num": 0.15319532624496177}} +:::MLLOG {"namespace": "", "time_ms": 1784881436001, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 350945280, "epoch_num": 0.15319532624496177}} +:::MLLOG {"namespace": "", "time_ms": 1784881444717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1047084778547287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 351272960, "lr": 1.7865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881462126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10063154250383377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 351928320, "lr": 1.7898333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881479442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10015848278999329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 352583680, "lr": 1.7931666666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881495929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10551470518112183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 353239040, "lr": 1.7964999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881495995, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 353239040, "epoch_num": 0.1541966028870857}} +:::MLLOG {"namespace": "", "time_ms": 1784881495996, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 353239040, "epoch_num": 0.1541966028870857}} +:::MLLOG {"namespace": "", "time_ms": 1784881522896, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7844433188438416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 353239040, "epoch_num": 0.1541966028870857}} +:::MLLOG {"namespace": "", "time_ms": 1784881522897, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 353239040, "epoch_num": 0.1541966028870857}} +:::MLLOG {"namespace": "", "time_ms": 1784881522897, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 353239040, "epoch_num": 0.1541966028870857}} +:::MLLOG {"namespace": "", "time_ms": 1784881540965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10625308752059937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 353894400, "lr": 1.7998333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881558599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10574117302894592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 354549760, "lr": 1.8031666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881575664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10595713555812836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 355205120, "lr": 1.8065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881584311, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 355532800, "epoch_num": 0.15519787952920963}} +:::MLLOG {"namespace": "", "time_ms": 1784881584312, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 355532800, "epoch_num": 0.15519787952920963}} +:::MLLOG {"namespace": "", "time_ms": 1784881610972, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7853514552116394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 355532800, "epoch_num": 0.15519787952920963}} +:::MLLOG {"namespace": "", "time_ms": 1784881610972, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 355532800, "epoch_num": 0.15519787952920963}} +:::MLLOG {"namespace": "", "time_ms": 1784881610972, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 355532800, "epoch_num": 0.15519787952920963}} +:::MLLOG {"namespace": "", "time_ms": 1784881619154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10340135544538498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 355860480, "lr": 1.8098333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881636598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10204973816871643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 356515840, "lr": 1.8131666666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881653798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10213686525821686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 357171200, "lr": 1.8164999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881670892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10727822035551071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 357826560, "lr": 1.8198333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881670893, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 357826560, "epoch_num": 0.15619915617133356}} +:::MLLOG {"namespace": "", "time_ms": 1784881670893, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 357826560, "epoch_num": 0.15619915617133356}} +:::MLLOG {"namespace": "", "time_ms": 1784881697143, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.783963143825531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 357826560, "epoch_num": 0.15619915617133356}} +:::MLLOG {"namespace": "", "time_ms": 1784881697144, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 357826560, "epoch_num": 0.15619915617133356}} +:::MLLOG {"namespace": "", "time_ms": 1784881697144, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 357826560, "epoch_num": 0.15619915617133356}} +:::MLLOG {"namespace": "", "time_ms": 1784881714591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10117511451244354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 358481920, "lr": 1.8231666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881731898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10207109153270721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 359137280, "lr": 1.8265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881749495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10299451649188995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 359792640, "lr": 1.8298333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881758044, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 360120320, "epoch_num": 0.1572004328134575}} +:::MLLOG {"namespace": "", "time_ms": 1784881758045, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 360120320, "epoch_num": 0.1572004328134575}} +:::MLLOG {"namespace": "", "time_ms": 1784881784190, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7837881445884705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 360120320, "epoch_num": 0.1572004328134575}} +:::MLLOG {"namespace": "", "time_ms": 1784881784191, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 360120320, "epoch_num": 0.1572004328134575}} +:::MLLOG {"namespace": "", "time_ms": 1784881784191, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 360120320, "epoch_num": 0.1572004328134575}} +:::MLLOG {"namespace": "", "time_ms": 1784881792379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10535009205341339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 360448000, "lr": 1.8331666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881809809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10834812372922897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 361103360, "lr": 1.8365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881827797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10832902044057846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 361758720, "lr": 1.8398333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881845205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11049330979585648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 362414080, "lr": 1.8431666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881845206, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 362414080, "epoch_num": 0.15820170945558143}} +:::MLLOG {"namespace": "", "time_ms": 1784881845206, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 362414080, "epoch_num": 0.15820170945558143}} +:::MLLOG {"namespace": "", "time_ms": 1784881871879, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7849831581115723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 362414080, "epoch_num": 0.15820170945558143}} +:::MLLOG {"namespace": "", "time_ms": 1784881871879, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 362414080, "epoch_num": 0.15820170945558143}} +:::MLLOG {"namespace": "", "time_ms": 1784881871879, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 362414080, "epoch_num": 0.15820170945558143}} diff --git a/recommendation_v4/rcp_logs/gbs_32768/seed651579272.log b/recommendation_v4/rcp_logs/gbs_32768/seed651579272.log new file mode 100644 index 000000000..a37bfab8b --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_32768/seed651579272.log @@ -0,0 +1,691 @@ +:::MLLOG {"namespace": "", "time_ms": 1784886606980, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784886606980, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784886631051, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784886631052, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784886631052, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784886631052, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784886631052, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784886631052, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784886631052, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784886631052, "event_type": "POINT_IN_TIME", "key": "seed", "value": 651579272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784886631052, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784886631052, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784886631052, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784886631052, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784886631104, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784886631105, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784887531522, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784887531523, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784887531873, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784887711877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13906842470169067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.1666666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784887727487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387837529182434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.4999999999999995e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784887742186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13899624347686768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784887749558, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784887749559, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784887773835, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5016107559204102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784887773835, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784887773835, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784887781419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13899101316928864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3166666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784887795850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13896268606185913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784887811100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13886314630508423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9833333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784887826220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13880038261413574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784887826221, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784887826221, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784887849569, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5017505884170532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784887849569, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784887849569, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784887864291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13885259628295898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784887879402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387483924627304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784887894676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386648416519165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784887901905, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784887901905, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784887925179, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5019131898880005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784887925179, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784887925180, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784887932850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13872335851192474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784887947855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385679394006729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784887962999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385728120803833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784887977938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13835546374320984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784887977938, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784887977939, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784888000835, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5021071434020996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784888000835, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784888000835, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784888015493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384839415550232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784888030255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138114333152771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784888045122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382291316986084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784888052512, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784888052512, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784888075752, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5023513436317444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784888075753, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784888075753, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784888082820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13764053583145142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784888098056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13761968910694122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784888112543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13739246129989624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784888127451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13731086254119873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784888127452, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784888127452, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784888151388, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5026282072067261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784888151388, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784888151388, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784888165755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13745886087417603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.316666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784888180449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13741086423397064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784888196065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13675087690353394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784888203601, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784888203601, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784888226973, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5029541850090027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784888226973, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784888226973, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784888234538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373930275440216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784888250576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13651636242866516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784888265609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13684141635894775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784888281735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13722771406173706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784888281736, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784888281736, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784888304742, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5033634305000305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784888304743, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784888304743, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784888322785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13644744455814362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784888338622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13522303104400635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.983333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784888353496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13678282499313354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888360877, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784888360878, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784888384368, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.503792941570282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784888384368, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784888384368, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784888391835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13359138369560242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888407262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13493895530700684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888422048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13518017530441284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888436906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13545306026935577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888436973, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784888436974, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784888461383, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5042440891265869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784888461384, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784888461384, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784888476223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1340002566576004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888491433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347191482782364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888508532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13527646660804749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888516792, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784888516793, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784888540681, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5047375559806824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784888540681, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784888540681, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784888548068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13565899431705475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888563006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13494490087032318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888578151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322707235813141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888593974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13459856808185577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888593974, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784888593975, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784888616819, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5053887963294983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784888616820, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784888616820, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784888631966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13376331329345703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888649264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13529177010059357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888665356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349320113658905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888673137, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784888673138, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784888696069, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5064141154289246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784888696070, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784888696070, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784888703592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13644276559352875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888720478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13348844647407532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888736266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13427196443080902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888751498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13207237422466278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888751570, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784888751571, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784888774670, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5074495673179626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784888774671, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784888774671, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784888789531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13291579484939575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888805445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13108178973197937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888820712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13314732909202576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888828343, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784888828344, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784888851075, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5085789561271667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784888851075, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784888851076, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784888858597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13846680521965027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888873780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310528814792633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888889513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12986384332180023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888904548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13039179146289825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888904548, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784888904549, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784888927262, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5098863840103149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784888927263, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784888927263, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784888942308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13277080655097961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888957560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297703981399536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888973476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13060742616653442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888980977, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784888980977, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784889003804, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5112761855125427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784889003805, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784889003805, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784889011369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13155606389045715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889026300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351446956396103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889041736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13310132920742035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889057758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13559769093990326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889057759, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784889057759, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784889080529, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5128117203712463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784889080529, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784889080530, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784889096017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13524091243743896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889111456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326037496328354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1649999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889126883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13318994641304016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889134521, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784889134521, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784889156994, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5143459439277649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784889156995, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784889156995, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784889165855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13641202449798584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889181725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13511522114276886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889197420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13627469539642334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889213167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13637815415859222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889213167, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784889213168, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784889236248, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5158920884132385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784889236249, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784889236249, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784889252853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347636580467224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889268821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13093572854995728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889284587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362290382385254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889292626, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784889292626, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784889315982, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5173915028572083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784889315983, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784889315983, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784889323803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358761489391327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889339739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13248589634895325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889355034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326661854982376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889370130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13088509440422058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889370131, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784889370131, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784889392837, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5188457369804382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784889392837, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784889392838, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784889408190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1309865117073059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889424088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13837938010692596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889439611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12951168417930603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889447309, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784889447309, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784889470252, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.520500898361206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784889470252, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784889470252, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784889477767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12927600741386414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889493282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13476714491844177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889508776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345004141330719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889525285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303626298904419, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889525286, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784889525287, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784889547974, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5224828720092773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784889547974, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784889547975, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784889563285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12801901996135712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889578624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13180002570152283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889594131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13556404411792755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889602106, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784889602107, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784889625691, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.524665117263794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784889625691, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784889625692, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784889633503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13657613098621368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889649481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335998773574829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889665171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321249008178711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889680685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13089843094348907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889680685, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784889680686, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784889703338, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.527126669883728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784889703339, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784889703339, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784889719124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1296313852071762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889734737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13248883187770844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889750748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13550090789794922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889758573, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784889758573, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784889781682, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5300031900405884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784889781683, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784889781683, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784889789282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13199348747730255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889804919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334475874900818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889820453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13492831587791443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889835980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346670687198639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889835981, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784889835981, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784889860129, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.533117413520813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784889860129, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784889860129, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784889883148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13605768978595734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889899934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338774561882019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889916001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13315415382385254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3650000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889923959, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784889923960, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784889947220, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5365585684776306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784889947220, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784889947220, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784889955371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14028900861740112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889971522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13534507155418396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889987805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13442233204841614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4649999999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890003943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13523830473423004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890003944, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784890003944, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784890027162, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5400183200836182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784890027162, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784890027162, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784890042811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13519418239593506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890058556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331949681043625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.5649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890076683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13231433928012848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890085371, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784890085372, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784890109234, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5441867113113403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784890109235, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784890109235, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784890117227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13254261016845703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890132826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13076838850975037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890148344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13583028316497803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890164139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322653442621231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890164139, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784890164139, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784890187570, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.549422025680542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784890187570, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784890187570, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784890203323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13473527133464813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890219492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1302829086780548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890235049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13179311156272888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890242703, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784890242704, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784890266457, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5562306642532349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784890266458, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784890266458, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784890274031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323864758014679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890289600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1306699514389038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890305290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13308139145374298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890321493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13261987268924713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890321494, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784890321495, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784890345542, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5654029250144958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784890345543, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784890345543, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784890361234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13199448585510254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890376806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12822920083999634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890392121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134017676115036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890400004, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784890400007, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784890422715, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5757771134376526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784890422716, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784890422716, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784890430601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13479064404964447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890446248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12676578760147095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890462402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1268426924943924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890478188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13287129998207092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890478189, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784890478189, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784890501625, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5880189538002014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784890501626, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784890501626, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784890517179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12929780781269073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890532695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1285778284072876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890548739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12798969447612762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890556455, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784890556456, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784890580478, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5997128486633301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784890580479, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784890580479, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784890588408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13004116714000702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890608733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12749849259853363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890624258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12979714572429657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890639751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12810684740543365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890639751, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784890639752, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784890663590, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6127049326896667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784890663590, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784890663590, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784890678873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12654559314250946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890694834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12948423624038696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4983333333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890710538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13256146013736725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890718316, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784890718316, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784890743096, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6249397993087769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784890743097, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784890743097, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784890751585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13089877367019653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890767871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12607181072235107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890784030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13064022362232208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890799871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1259201467037201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890799871, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784890799872, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784890824532, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6376864910125732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784890824533, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784890824533, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784890840712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12842094898223877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890856736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12938256561756134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890873320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12749700248241425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890881333, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784890881334, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784890905554, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6515070199966431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784890905554, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784890905554, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784890913410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12649285793304443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890929175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12652578949928284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890945225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12676461040973663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890961214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1259365975856781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.898333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890961215, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784890961215, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784890986507, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6658112406730652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784890986508, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784890986508, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784891002798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11957870423793793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891018948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11934898793697357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891034770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11721716821193695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891042678, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784891042679, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784891067085, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.677969753742218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784891067086, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784891067086, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784891074878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11937661468982697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891090884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11615047603845596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891107049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12081261724233627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891122905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11782926321029663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891122905, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784891122906, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784891147706, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6982152462005615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784891147706, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784891147707, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784891164411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11966632306575775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891180145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11082813143730164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.198333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891195856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11664722859859467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891203545, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784891203546, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784891227701, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7150721549987793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784891227702, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784891227702, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784891235445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11743553727865219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891251435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1173960492014885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.298333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891267299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11230888962745667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891282873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1160268634557724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891282950, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784891282950, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784891308136, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7298416495323181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784891308137, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784891308137, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784891324330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11463218182325363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.398333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891340333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11456344276666641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891356622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10838775336742401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891364654, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784891364654, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784891388600, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7397327423095703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784891388600, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784891388600, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784891396700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11172173917293549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.498333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891413118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11274328082799911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.531666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891429395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11041340976953506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891445920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11327363550662994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.598333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891445921, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784891445921, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784891471063, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7469193339347839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784891471064, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784891471064, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784891486784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11160502582788467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891502429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11107081919908524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891518239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11535638570785522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891526073, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784891526073, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784891549608, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7512174248695374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784891549608, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784891549608, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784891557323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10448231548070908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891573372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11308379471302032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891589119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10662493854761124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891605269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11093631386756897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891605269, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784891605270, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784891629146, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7536928653717041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784891629147, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784891629147, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784891645275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1141892522573471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891661296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11083892732858658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891677103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10999979078769684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891684835, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784891684836, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784891708448, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7560389637947083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784891708449, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784891708449, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784891716346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10999810695648193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891732410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11194638907909393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891748733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11230692267417908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891764714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10596001893281937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891764714, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784891764715, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784891789041, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7566849589347839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784891789042, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784891789042, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784891806266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11480924487113953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891822503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11906598508358002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891838909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11715337634086609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891846998, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784891846998, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784891870979, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.758203387260437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784891870980, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784891870980, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784891879047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11422158777713776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891895671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11477014422416687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891911927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11243162304162979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891927907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1136232316493988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891927907, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784891927908, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784891952795, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7597623467445374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784891952795, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784891952796, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784891969130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11814562231302261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891985110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11735286563634872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892001359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11099065840244293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892009489, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784892009489, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784892033171, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7606346011161804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784892033171, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784892033171, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784892041387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11649218946695328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892057791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11589295417070389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892073996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11125297099351883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892090987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10478861629962921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892090987, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784892090988, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784892113944, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7632928490638733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784892113945, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784892113945, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784892129717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1070249080657959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892145823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10598494112491608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892161696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10613615810871124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892169793, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784892169793, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784892193710, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7650867104530334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784892193710, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784892193710, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784892201778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1045650765299797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892218211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11180512607097626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892234370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1079343929886818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892250985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1127995103597641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892250986, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784892250986, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784892274233, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7651737928390503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784892274233, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784892274234, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784892290412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10721711814403534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892306325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10612587630748749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892322429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10237470269203186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892330187, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784892330187, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784892354178, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7665369510650635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784892354178, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784892354178, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784892362213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10256901383399963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892378351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1017247810959816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892394807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11199380457401276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892410646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10435178130865097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892410646, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784892410646, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784892434178, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7683365345001221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784892434179, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784892434179, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784892450657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10653980821371078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892466941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1069025844335556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892482722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10698974132537842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892490921, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784892490922, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784892514622, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.769115149974823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784892514623, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784892514623, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784892522749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1073540672659874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892538592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0995936170220375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892554825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10796601325273514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892571079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10403429716825485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892571079, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784892571079, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784892594100, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7695165872573853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784892594101, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784892594101, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784892609910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10766206681728363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892626797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793063044548035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892643102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10429377108812332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892651073, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784892651074, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784892673966, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7683959007263184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784892673967, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784892673967, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784892681837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1086614802479744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892697998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10765066742897034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892713821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11175756901502609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892729960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792073607444763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892729961, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784892729961, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784892753087, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7708179354667664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784892753088, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784892753089, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784892769384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10536094009876251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892785292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10192438215017319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892802129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10861892998218536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.564999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892810462, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784892810463, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784892834086, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7714892029762268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784892834087, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784892834087, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784892842171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10465173423290253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892857996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0995938628911972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892874089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11402788758277893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.664999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892890055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11108420044183731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892890055, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784892890056, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784892913767, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7714943885803223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784892913768, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784892913768, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784892929815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10645310580730438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892946244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10845470428466797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892962384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10845200717449188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892970297, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784892970298, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784892993940, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7711619138717651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784892993941, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784892993941, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784893002536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11049939692020416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893019198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11482617259025574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893035648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11245420575141907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893052128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11297064274549484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893052128, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784893052129, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784893076328, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7713772654533386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784893076328, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784893076329, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784893092693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11022461950778961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893109300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11377108097076416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.998333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893125900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11078467220067978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893134110, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784893134110, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784893157688, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7717871069908142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784893157689, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784893157689, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784893165748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11045512557029724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893182893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10924027115106583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893199406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.112511545419693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.131666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893215981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11101587861776352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893215981, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784893215982, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784893239854, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7724250555038452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784893239855, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784893239855, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784893256393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10992748290300369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893273278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11269301176071167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893289925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11683948338031769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893298175, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784893298176, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784893322055, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7728994488716125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784893322056, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784893322056, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784893330097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11467509716749191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893347161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10796239972114563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.331666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893363216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10151535272598267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893379120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10690972208976746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893379120, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784893379120, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784893402342, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730903625488281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784893402343, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784893402343, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784893418451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10465559363365173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893434429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10664968192577362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893450788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10542459785938263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893459095, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784893459096, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784893481535, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7732721567153931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784893481536, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784893481536, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784893489662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085653230547905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893505545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10306200385093689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893522503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10460115224123001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893538907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10728271305561066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.631666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893538908, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784893538908, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784893561171, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7736385464668274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784893561172, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784893561172, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784893577125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10069732367992401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893593420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10396308451890945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893609618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10367564857006073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893617738, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784893617738, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784893641159, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773847222328186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784893641160, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784893641160, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784893649367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10411949455738068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893666257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10317803919315338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893682663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11369638890028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893699589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1031976044178009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893699590, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784893699590, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784893722590, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7741410732269287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784893722591, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784893722591, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784893738838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1070108413696289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893754970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080286055803299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893771084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10163038223981857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893779080, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784893779080, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784893802190, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7735962271690369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784893802191, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784893802191, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784893810209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10135368257761002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893826434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10048162937164307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893842931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10599635541439056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893859542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10323463380336761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893859542, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784893859543, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784893882778, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7744584083557129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784893882778, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784893882779, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784893898764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10750461369752884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893916185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10386943817138672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893932506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10591638088226318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893940489, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784893940490, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784893966765, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7747220396995544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784893966766, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784893966766, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784893974794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10346657782793045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.231666666666666e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_32768/seed813474716.log b/recommendation_v4/rcp_logs/gbs_32768/seed813474716.log new file mode 100644 index 000000000..9bb285f7b --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_32768/seed813474716.log @@ -0,0 +1,649 @@ +:::MLLOG {"namespace": "", "time_ms": 1784865317254, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784865317254, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784865343759, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784865343760, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784865343760, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784865343760, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784865343760, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784865343760, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784865343761, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784865343761, "event_type": "POINT_IN_TIME", "key": "seed", "value": 813474716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784865343761, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784865343761, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784865343761, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784865343761, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784865343784, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784865343784, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784866238014, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784866238015, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784866238325, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784866386407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13889388740062714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.1666666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784866401786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13889822363853455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.4999999999999995e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784866416614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13886550068855286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784866423996, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784866423997, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784866473498, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49992015957832336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784866473499, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784866473499, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784866480984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13879427313804626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3166666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866495633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138699010014534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866510990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13867506384849548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9833333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866525890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13865134119987488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866525891, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784866525891, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784866549071, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5000882148742676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784866549071, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784866549071, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784866563921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13854649662971497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866578861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13847501575946808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866593981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382513791322708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866601229, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784866601230, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784866624355, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5002747178077698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784866624356, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784866624356, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784866632131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13832880556583405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866647189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13838960230350494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866662420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13838106393814087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866677681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13806653022766113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866677681, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784866677682, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784866700811, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5005096793174744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784866700811, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784866700812, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784866715895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13824012875556946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866730737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378125548362732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866745907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13802699744701385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866753396, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784866753397, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784866776776, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5007984042167664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784866776777, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784866776777, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784866784153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13738197088241577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866799551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13745559751987457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866814135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13718438148498535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866829115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13722258806228638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866829116, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784866829116, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784866853042, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5011144876480103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784866853043, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784866853043, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784866867994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13740065693855286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.316666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866883196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373768448829651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866898982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13659091293811798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866906525, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784866906526, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784866929724, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5015040040016174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784866929724, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784866929725, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784866937405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13738906383514404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866953481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13652417063713074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866968715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13676974177360535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866984639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13714928925037384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866984640, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784866984641, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784867007700, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5019540786743164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784867007701, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784867007701, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784867022974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364782154560089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784867038859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13537099957466125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.983333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784867053901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13664430379867554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867061414, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784867061414, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784867084825, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5024121403694153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784867084825, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784867084826, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784867092347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13343486189842224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867107953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349656581878662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867122880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13523045182228088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867137964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13534927368164062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867138039, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784867138039, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784867162139, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.502917468547821, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784867162139, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784867162139, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784867177222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13404159247875214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867192253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13476285338401794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867207426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13537245988845825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867215645, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784867215645, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784867239454, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5034622550010681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784867239455, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784867239455, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784867247023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358211785554886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867262243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13508576154708862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867277726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13224801421165466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867293705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347406804561615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867293706, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784867293706, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784867316764, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5041109919548035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784867316764, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784867316765, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784867332275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13390648365020752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867347990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352284848690033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867364120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349969208240509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867372047, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784867372048, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784867395060, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5050241351127625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784867395061, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784867395061, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784867402814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13636213541030884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867418579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13351479172706604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867434883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13435077667236328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867450577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321287900209427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867450642, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784867450643, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784867474150, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5060195326805115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784867474150, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784867474151, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784867489613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13294489681720734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867505856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13125818967819214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867521415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330741047859192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867529209, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784867529209, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784867552109, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5071291327476501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784867552110, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784867552110, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784867559784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13837289810180664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867575371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311246007680893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867591459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12998881936073303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867607109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13047222793102264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867607110, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784867607110, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784867630289, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5083502531051636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784867630290, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784867630290, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784867645801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13273750245571136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867661545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12984631955623627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867677782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13069316744804382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867685494, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784867685494, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784867708516, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5096793174743652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784867708517, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784867708517, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784867716263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13159923255443573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867731620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13515201210975647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867747439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13317684829235077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867763486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356806457042694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867763487, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784867763488, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784867786531, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5109959840774536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784867786532, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784867786532, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784867802159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351565569639206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867817928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13248607516288757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1649999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867833742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13316947221755981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867841591, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784867841592, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784867865299, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5122719407081604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784867865299, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784867865299, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784867873932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13636627793312073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867889828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13516679406166077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867905602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13632236421108246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867921619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13646163046360016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867921619, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784867921620, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784867945271, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5134730935096741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784867945271, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784867945272, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784867961900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13483776152133942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867978070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1307704597711563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867994047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362009346485138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868002202, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784868002203, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784868026125, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5147641897201538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784868026126, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784868026126, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784868034059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13592682778835297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868050071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13252314925193787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868065764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13274820148944855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868081193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13080215454101562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868081194, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784868081194, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784868104294, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.516205370426178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784868104295, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784868104295, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784868119882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13109523057937622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868135895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13824208080768585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868151991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12956905364990234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868159924, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784868159925, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784868183154, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5177932381629944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784868183154, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784868183154, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784868190978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1293424516916275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868206940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13470570743083954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868222840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344418227672577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868239330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13032689690589905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868239331, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784868239331, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784868262595, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5194911360740662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784868262595, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784868262595, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784868278120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12800902128219604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868293666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13198813796043396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868309290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13569359481334686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868317256, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784868317256, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784868341263, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5213505029678345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784868341263, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784868341264, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784868349138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13653047382831573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868365340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13355834782123566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868381385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13200271129608154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868397189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13098497688770294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868397190, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784868397190, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784868420903, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5234540700912476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784868420904, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784868420904, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784868436925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12961238622665405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868452826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324542760848999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868469012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354411542415619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868476931, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784868476931, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784868500430, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5256763100624084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784868500431, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784868500431, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784868508160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13190442323684692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868523960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13345566391944885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868539745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13501973450183868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868555552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13469095528125763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868555552, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784868555553, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784868579605, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5281840562820435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784868579606, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784868579606, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784868595512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13608399033546448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868612242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13401179015636444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868628422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13314710557460785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3650000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868636368, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784868636369, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784868660404, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5308187007904053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784868660405, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784868660405, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784868668538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14038047194480896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868684795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13532966375350952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868701246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13441476225852966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4649999999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868717570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13527311384677887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868717571, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784868717571, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784868741286, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5339008569717407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784868741286, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784868741286, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784868757209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352696567773819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868773441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13327954709529877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.5649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868789595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13237273693084717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868798153, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784868798153, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784868822989, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5375076532363892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784868822989, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784868822990, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784868830999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13256919384002686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868846799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13100332021713257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868862360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13588321208953857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868878446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13233965635299683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868878447, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784868878447, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784868902153, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.541955292224884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784868902153, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784868902154, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784868918192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347806751728058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868934623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13029298186302185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868950630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1319665014743805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868958561, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784868958562, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784868982110, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5474483966827393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784868982111, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784868982111, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784868989918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13240768015384674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869006063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13086046278476715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869022311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13315799832344055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869038899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13277721405029297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869038900, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784869038900, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784869062815, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5541642308235168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784869062815, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784869062816, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784869078872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13220542669296265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869094861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1284438967704773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869110855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13433212041854858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869119067, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784869119067, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784869142250, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5625165700912476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784869142250, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784869142250, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784869150492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13489949703216553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869166664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12683990597724915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869182742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12723568081855774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869198871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13317354023456573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869198871, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784869198872, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784869222185, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5717131495475769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784869222185, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784869222185, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784869237983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1296549290418625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869253908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12893636524677277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869270206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12840279936790466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869278122, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784869278122, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784869301429, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5807201862335205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784869301429, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784869301429, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784869309542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304447501897812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869326143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1280917376279831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869342297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.130548894405365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869358331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12892493605613708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869358332, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784869358332, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784869381288, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5919283628463745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784869381289, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784869381289, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784869397210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12758556008338928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869413646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303061544895172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4983333333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869429726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337101012468338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869437715, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784869437716, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784869461245, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6020538806915283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784869461245, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784869461246, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784869469755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321471631526947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869486158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12734384834766388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869502396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13204357028007507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869518327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12758326530456543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869518327, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784869518327, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784869541412, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6135897040367126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784869541413, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784869541413, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784869557678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12994195520877838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869573743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13094595074653625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869590510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12889507412910461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869598917, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784869598917, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784869622360, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6252811551094055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784869622361, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784869622361, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784869630541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1281415820121765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869646785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1282893568277359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869663396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12890596687793732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869679872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12817367911338806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.898333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869679873, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784869679873, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784869703484, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6339766979217529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784869703485, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784869703485, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784869720179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12283225357532501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869736204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12186761200428009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869751840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12014015018939972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869759744, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784869759745, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784869782145, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6379403471946716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784869782145, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784869782145, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784869789871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12186644971370697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869805872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11871105432510376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869821913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12296853214502335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869837705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12075923383235931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869837705, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784869837706, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784869861310, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6574773788452148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784869861311, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784869861311, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784869878184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12137813866138458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869894369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1130509003996849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.198333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869910577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11892740428447723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869918584, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784869918585, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784869942392, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6787935495376587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784869942392, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784869942393, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784869950300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1192016750574112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869966566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11925527453422546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.298333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869982934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11364169418811798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869998828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1172865778207779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869998920, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784869998921, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784870023005, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7068197131156921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784870023005, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784870023005, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784870039266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11590640246868134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.398333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870055252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11621499806642532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870071402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10903821885585785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870079309, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784870079310, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784870101957, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7287963032722473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784870101958, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784870101958, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784870110024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11302515864372253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.498333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870126603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1138143241405487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.531666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870142982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11225545406341553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870159570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11345705389976501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.598333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870159570, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784870159571, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784870182881, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7536516189575195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784870182881, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784870182881, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784870199187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11255127936601639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870215388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11129208654165268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870231758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11543719470500946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870239889, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784870239890, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784870262916, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7615731954574585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784870262916, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784870262917, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784870270876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10521388053894043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870287529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11244302988052368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870303818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1062399223446846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870320032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10990236699581146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870320032, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784870320033, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784870344418, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.766869843006134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784870344419, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784870344419, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784870360928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11345174908638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870377098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11027105897665024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870393201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10867815464735031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870401093, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784870401093, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784870424286, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7685171961784363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784870424287, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784870424287, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784870432445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10924504697322845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870448728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11034150421619415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870465199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11133092641830444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870481145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10492929816246033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870481146, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784870481146, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784870505723, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7699735164642334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784870505723, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784870505723, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784870523317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11435538530349731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870540144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11844556033611298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870556877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11553098261356354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870565198, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784870565199, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784870587663, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7689064145088196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784870587664, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784870587664, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784870595867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11447051167488098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870612877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11394639313220978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870629506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11145921051502228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870645934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11279840767383575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870645934, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784870645935, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784870670178, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7698440551757812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784870670178, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784870670178, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784870687049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11717913299798965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870703861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11732777953147888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870720690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11010465025901794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870729056, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784870729057, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784870751586, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7703806757926941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784870751586, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784870751587, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784870760003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11478739976882935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870776738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11481395363807678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870793355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11053420603275299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870810471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10400791466236115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870810472, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784870810473, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784870833463, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7715024352073669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784870833464, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784870833464, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784870849457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10648521780967712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870865901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10529173165559769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870882056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10546055436134338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870890183, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784870890184, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784870913238, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7719013094902039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784870913238, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784870913238, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784870921380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10347109287977219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870938023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11097723990678787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870954472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10698778927326202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870971148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1121751219034195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870971149, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784870971149, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784870993876, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7722285985946655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784870993876, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784870993876, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784871010285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10628720372915268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871026390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1061728447675705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871042670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10152549296617508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871050641, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784871050641, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784871073461, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7725415825843811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784871073462, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784871073462, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784871081701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10204286128282547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871098239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10106198489665985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871115001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11092683672904968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871131032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10366864502429962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871131032, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784871131033, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784871155178, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730610370635986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784871155179, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784871155179, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784871171696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10571674257516861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871188289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10607373714447021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871204559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10652877390384674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871212977, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784871212977, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784871236192, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7733955383300781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784871236193, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784871236193, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784871244495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10678546130657196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871260819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09931115806102753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871277546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10751073062419891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871294264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1042754054069519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871294264, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784871294265, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784871317704, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7734543085098267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784871317705, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784871317705, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784871334029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10768401622772217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871350952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792233794927597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871367324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10385309159755707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871375451, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784871375452, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784871398771, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7731227278709412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784871398772, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784871398772, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784871406837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10882290452718735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871423320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10702510178089142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871439693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11151189357042313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871456268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10794492065906525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871456269, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784871456269, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784871479336, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7734448909759521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784871479337, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784871479337, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784871495863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1052134782075882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871511945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10156769305467606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871529036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10798138380050659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.564999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871537529, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784871537529, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784871561306, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7739500999450684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784871561307, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784871561307, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784871569493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10398925840854645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871585726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09904269874095917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871602266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11385751515626907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.664999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871618636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11014744639396667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871618637, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784871618637, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784871642780, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738193273544312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784871642780, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784871642781, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784871659217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10616227239370346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871676087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082400530576706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871692699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10777496546506882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871700837, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784871700837, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784871724794, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742146849632263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784871724795, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784871724795, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784871733682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10965999960899353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871750608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11435089260339737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871767439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11172356456518173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871784219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11256837099790573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871784220, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784871784220, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784871807956, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7734901309013367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784871807957, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784871807957, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784871824533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10966978967189789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871841560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11340569704771042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.998333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871858417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11017830669879913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871866810, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784871866811, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784871891746, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738568186759949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784871891746, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784871891747, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784871900056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11044783145189285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871917396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10885351896286011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871934089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11259768158197403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.131666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871950927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11080705374479294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871950928, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784871950928, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784871974307, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742443084716797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784871974307, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784871974307, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784871991152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11015733331441879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872008237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11224915087223053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872025224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11663895100355148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872033714, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784872033715, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784872057454, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7745472192764282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784872057454, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784872057455, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784872065626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1143219918012619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872082831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732045024633408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.331666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872099350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10087919980287552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872115787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10663136839866638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872115788, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784872115788, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784872139007, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7751693725585938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784872139007, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784872139008, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784872155433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.104435034096241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872171802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1063511073589325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872188509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10491480678319931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872196934, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784872196935, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784872220288, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7753638029098511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784872220288, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784872220289, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784872228635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10828186571598053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872244915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10302504152059555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872261981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10468199104070663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872278634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10687203705310822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.631666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872278635, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784872278635, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784872301789, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.775496780872345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784872301790, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784872301790, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784872317997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10002575814723969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_8192/seed380229711.log b/recommendation_v4/rcp_logs/gbs_8192/seed380229711.log new file mode 100644 index 000000000..5dbbc2f0e --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_8192/seed380229711.log @@ -0,0 +1,1869 @@ +:::MLLOG {"namespace": "", "time_ms": 1784651373092, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784651373092, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784651394632, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784651394635, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784651394636, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784651394636, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784651394636, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784651394636, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 8192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784651394636, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784651394636, "event_type": "POINT_IN_TIME", "key": "seed", "value": 380229711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784651394636, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784651394636, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784651394636, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784651394636, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784651394636, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784651394637, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784652219685, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784652219686, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784652220086, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784652336018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13829340040683746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840, "lr": 7.916666666666666e-10}} +:::MLLOG {"namespace": "", "time_ms": 1784652348462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13841071724891663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.6249999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652361275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13854172825813293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 491520, "lr": 2.458333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652374408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385321021080017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652387165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13865114748477936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 819200, "lr": 4.125e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652400195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386178433895111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652413786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13844262063503265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1146880, "lr": 5.7916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652426846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13894973695278168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.625e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652439808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13859781622886658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1474560, "lr": 7.458333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652452599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13844317197799683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.291666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652465989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385466754436493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1802240, "lr": 9.124999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652479345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13869792222976685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652492326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13824759423732758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2129920, "lr": 1.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652505221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13966311514377594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652505222, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652505223, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652588451, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.501117467880249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652588452, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652588452, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652601325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13839909434318542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2457600, "lr": 1.2458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652614399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385101079940796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652627745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138491690158844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2785280, "lr": 1.4124999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652640408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13825608789920807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4958333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652653484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13820061087608337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3112960, "lr": 1.5791666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652667060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13820622861385345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652680033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382625252008438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3440640, "lr": 1.7458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652693232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13804283738136292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652706676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379861831665039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3768320, "lr": 1.9124999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652719707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13803361356258392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9958333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652733120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13835781812667847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4096000, "lr": 2.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652746464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13841018080711365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.1624999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652759691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13794080913066864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4423680, "lr": 2.2458333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652773366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13820689916610718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3291666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652773367, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652773367, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652855274, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5012875199317932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652855275, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652855275, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652868575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13767823576927185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4751360, "lr": 2.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652881912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381848305463791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652895307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13746176660060883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5079040, "lr": 2.5791666666666668e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652908870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13632436096668243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652922235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378561407327652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5406720, "lr": 2.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652935501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13792404532432556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652948856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13739514350891113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5734400, "lr": 2.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652962651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1369534134864807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652976038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13673639297485352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6062080, "lr": 3.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652989222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13758163154125214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653002703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13747048377990723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6389760, "lr": 3.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653015946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13764236867427826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653029989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377759873867035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6717440, "lr": 3.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653043366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13778544962406158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653043367, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784653043367, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784653125662, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5014969110488892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784653125664, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784653125664, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784653138818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13661929965019226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7045120, "lr": 3.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653152299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373259574174881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653165701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13673388957977295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7372800, "lr": 3.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653179691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13711419701576233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.829166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653193067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13731049001216888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7700480, "lr": 3.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653206799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13772282004356384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653220409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13662183284759521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8028160, "lr": 4.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653233919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13726992905139923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653247918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13670524954795837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8355840, "lr": 4.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653261692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13858027756214142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3291666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653275306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362139731645584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8683520, "lr": 4.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653288859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13770025968551636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.4958333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653302627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351468414068222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9011200, "lr": 4.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653316607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13665056228637695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.6624999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653316607, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653316608, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653399325, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5018054246902466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653399325, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653399326, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653412694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13610577583312988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9338880, "lr": 4.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653426266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354997754096985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653439512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13754920661449432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9666560, "lr": 4.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653453052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346389204263687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653466704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13652677834033966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9994240, "lr": 5.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653480056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13339805603027344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1624999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653493762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13550393283367157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10321920, "lr": 5.2458333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653506941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13530461490154266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653520337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378270387649536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10649600, "lr": 5.4124999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653533580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13506457209587097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653546869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14742782711982727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10977280, "lr": 5.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653560556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355466991662979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653573827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13415521383285522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11304960, "lr": 5.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653587305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13788515329360962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653587306, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653587306, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653669437, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5022485852241516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653669438, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653669438, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653682772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13523557782173157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11632640, "lr": 5.9124999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653696072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13974608480930328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653709461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13576072454452515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11960320, "lr": 6.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653722769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330488920211792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653736232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13483205437660217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12288000, "lr": 6.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653749866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13916133344173431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.329166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653763402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332724392414093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12615680, "lr": 6.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653777099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13292570412158966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653790467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13310086727142334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12943360, "lr": 6.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653803569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13163936138153076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653817091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13118129968643188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13271040, "lr": 6.745833333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653830256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134862020611763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653843591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13152752816677094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13598720, "lr": 6.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653857302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388949751853943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653857302, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784653857303, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784653938982, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5028337836265564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784653938983, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784653938983, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784653952504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336965560913086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13926400, "lr": 7.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653965929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13328483700752258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653979615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13570626080036163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14254080, "lr": 7.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653992951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13492083549499512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654006219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376088410615921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14581760, "lr": 7.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654019556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1279987394809723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654032525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13532200455665588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14909440, "lr": 7.579166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654045752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12934422492980957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654059219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13329501450061798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15237120, "lr": 7.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654072886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326330006122589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.829166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654086352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13338404893875122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15564800, "lr": 7.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654100153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13440586626529694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.995833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654114119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13519714772701263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15892480, "lr": 8.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654127838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333647519350052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654127838, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654127839, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654210733, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5037636160850525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654210734, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654210734, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654224520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323649138212204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16220160, "lr": 8.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654238115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337590217590332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654251852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354834884405136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16547840, "lr": 8.412500000000001e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654265968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13541211187839508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654279782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13954435288906097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16875520, "lr": 8.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654293421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13339203596115112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.662499999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654307793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13545319437980652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17203200, "lr": 8.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654321530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332833468914032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654335387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13315187394618988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17530880, "lr": 8.912499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654349376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13161520659923553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654363074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13571426272392273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17858560, "lr": 9.079166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654377358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13355480134487152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.162499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654391416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13318656384944916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18186240, "lr": 9.245833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654405165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13077718019485474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654405166, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654405167, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654487734, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5049558281898499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654487735, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654487735, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654502071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13781844079494476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18513920, "lr": 9.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654515912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13507965207099915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.495833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654529803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13537418842315674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18841600, "lr": 9.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654543755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312287449836731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.662499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654557467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335454136133194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19169280, "lr": 9.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654571407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13419362902641296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654585567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370365023612976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19496960, "lr": 9.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654599628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134102925658226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654613692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13283273577690125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19824640, "lr": 1.0079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654627451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12971584498882294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0162499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654640719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12763750553131104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20152320, "lr": 1.0245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654654247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13281188905239105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654667736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295786201953888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20480000, "lr": 1.0412499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654681067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12460252642631531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654681067, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654681067, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654763235, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5062140822410583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654763236, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654763236, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654776956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13111639022827148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20807680, "lr": 1.0579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654790519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13595892488956451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654804344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13325944542884827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21135360, "lr": 1.0745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654818262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13026030361652374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654831772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12811905145645142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21463040, "lr": 1.0912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654845567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12646374106407166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654859300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12889397144317627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21790720, "lr": 1.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654872586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13230128586292267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654886022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13567900657653809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22118400, "lr": 1.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654899489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12482313066720963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654912749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12866170704364777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22446080, "lr": 1.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654926109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13429470360279083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654939589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1309855729341507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22773760, "lr": 1.1579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654953329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1292484998703003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654953330, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784654953330, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784655034915, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.507777988910675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784655034916, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784655034916, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784655048703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13063903152942657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23101440, "lr": 1.1745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655062159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1291147619485855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655075266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13290391862392426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23429120, "lr": 1.1912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655089471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12374183535575867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655103385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295236349105835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23756800, "lr": 1.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655116845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12577804923057556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655130277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300514191389084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24084480, "lr": 1.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655143969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1216576024889946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655157333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357887089252472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24412160, "lr": 1.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655171214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371704787015915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655184872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1235177144408226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24739840, "lr": 1.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655198993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12314169108867645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655212990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12556329369544983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25067520, "lr": 1.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655226780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12904168665409088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655226781, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655226781, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655308430, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5095562934875488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655308430, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655308431, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655321920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13665804266929626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25395200, "lr": 1.2912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655336195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13645566999912262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655349649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356658935546875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25722880, "lr": 1.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655363308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1399223655462265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655377063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13673624396324158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26050560, "lr": 1.3245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655390862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138333261013031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655404494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12752462923526764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26378240, "lr": 1.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655418107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12604793906211853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655431654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14219488203525543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26705920, "lr": 1.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655445692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14102703332901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655459646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1273341029882431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27033600, "lr": 1.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655473466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12840241193771362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655487288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13264304399490356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27361280, "lr": 1.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655500582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13153164088726044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655500582, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655500583, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655583065, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5115115642547607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655583066, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655583066, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655597766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14134420454502106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27688960, "lr": 1.4079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655612007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14244362711906433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655625862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135404571890831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28016640, "lr": 1.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655639896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13946174085140228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655654147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14488255977630615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28344320, "lr": 1.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655668133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14103546738624573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655682372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12569870054721832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28672000, "lr": 1.4579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655696645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13773049414157867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655710902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13182565569877625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28999680, "lr": 1.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655725114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13625067472457886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655739262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13769058883190155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29327360, "lr": 1.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655753228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330302357673645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4995833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655767808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13448892533779144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29655040, "lr": 1.5079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655782097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13638092577457428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655782097, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784655782098, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784655864645, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5135753154754639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784655864646, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784655864646, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784655878472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13862839341163635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29982720, "lr": 1.5245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655892708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137920543551445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655907093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13173285126686096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30310400, "lr": 1.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655921417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13306105136871338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655935631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337752342224121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30638080, "lr": 1.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655950247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13968920707702637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655964727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13280341029167175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30965760, "lr": 1.5745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655979430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14145448803901672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655993473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14072348177433014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31293440, "lr": 1.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656007701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13505767285823822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656021477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1276986002922058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31621120, "lr": 1.6079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656035090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13005103170871735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656048660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341242492198944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31948800, "lr": 1.6245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656062298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12747906148433685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656062298, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784656062299, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784656146506, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5160952210426331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784656146507, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784656146507, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784656160067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323164999485016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32276480, "lr": 1.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656173690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13693760335445404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656187359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13378456234931946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32604160, "lr": 1.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656200934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1319260597229004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656214736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13390298187732697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32931840, "lr": 1.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656228480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335354745388031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656242664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13369110226631165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33259520, "lr": 1.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656256471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1328757256269455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656270029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14000087976455688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33587200, "lr": 1.7079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656284490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13193389773368835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656298477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12873482704162598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33914880, "lr": 1.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656312117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13105955719947815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656325792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13216131925582886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34242560, "lr": 1.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656339636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13580787181854248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7495833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656339637, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656339637, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656423616, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.519439160823822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656423617, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656423617, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656437489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337026059627533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34570240, "lr": 1.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656451376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1230003610253334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656465383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13062839210033417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34897920, "lr": 1.7745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656479611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13046808540821075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656493584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13519425690174103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35225600, "lr": 1.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656508183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13522779941558838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656522379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13522547483444214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35553280, "lr": 1.8079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656537064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12389795482158661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8162500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656551475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13092230260372162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35880960, "lr": 1.8245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656565514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13108700513839722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656580103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13037510216236115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36208640, "lr": 1.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656593938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312530517578125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656607844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13088789582252502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36536320, "lr": 1.8579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656621586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301867812871933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656621586, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656621587, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656703970, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5237941741943359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656703971, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656703971, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656717758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13782340288162231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36864000, "lr": 1.8745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656731699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13229811191558838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656745486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336624026298523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37191680, "lr": 1.8912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656759186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13037289679050446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656772841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1272742748260498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37519360, "lr": 1.9079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656786429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323188841342926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656800507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13567706942558289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37847040, "lr": 1.9245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656814558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13283924758434296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656828498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311909258365631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38174720, "lr": 1.9412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656842572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388363391160965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656856623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13371853530406952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38502400, "lr": 1.9579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656870164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375824213027954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656884635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13202527165412903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38830080, "lr": 1.9745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656898780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13334956765174866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656898781, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784656898782, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784656981020, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5296569466590881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784656981021, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784656981022, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784656994985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13127672672271729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39157760, "lr": 1.9912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657008912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349802315235138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657022904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378108412027359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39485440, "lr": 2.0079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657036873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13306479156017303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657050674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12724143266677856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39813120, "lr": 2.0245833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657064721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12935931980609894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0329166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657078792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12585681676864624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40140800, "lr": 2.0412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657092752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12395190447568893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657107023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13597726821899414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40468480, "lr": 2.0579166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657121448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13717271387577057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657135760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13473482429981232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40796160, "lr": 2.0745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657150735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12712106108665466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0829166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657164985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13059158623218536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41123840, "lr": 2.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657178721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12354076653718948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657178722, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657178722, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657261561, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5380802750587463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657261562, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657261562, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657275729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13240675628185272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41451520, "lr": 2.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657289542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13265280425548553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657303372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338113248348236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41779200, "lr": 2.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657317259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379699409008026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657331047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363019496202469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42106880, "lr": 2.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657345146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1313190460205078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657359175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1272008717060089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42434560, "lr": 2.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657373291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13506227731704712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657387112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1245088130235672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42762240, "lr": 2.1745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657401349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13035312294960022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657415475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323871612548828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43089920, "lr": 2.1912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657429756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13407666981220245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657443980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14030081033706665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43417600, "lr": 2.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657458184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361282467842102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657458184, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657458185, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657541249, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5505567193031311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657541250, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657541250, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657556008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13631132245063782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43745280, "lr": 2.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657570250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13159041106700897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657584129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13250501453876495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44072960, "lr": 2.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657598240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13882331550121307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657612992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13002412021160126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44400640, "lr": 2.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657627882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13840016722679138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657642315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13482104241847992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44728320, "lr": 2.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657656880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13693153858184814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657671442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13485115766525269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45056000, "lr": 2.2912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657686048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13069741427898407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657700628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14041316509246826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45383680, "lr": 2.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657715321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13479438424110413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657729883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12885114550590515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45711360, "lr": 2.3245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657744579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333363652229309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657744579, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657744579, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657828553, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5647387504577637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657828554, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657828555, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657843653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14193421602249146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46039040, "lr": 2.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657858252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13241635262966156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657873407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13345593214035034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46366720, "lr": 2.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657888109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304544061422348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657903155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13533078134059906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46694400, "lr": 2.3745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657917959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13350869715213776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657932259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337253302335739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47022080, "lr": 2.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657946671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13557466864585876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657960905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.130796417593956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47349760, "lr": 2.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657975458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13248062133789062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657989879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13305050134658813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47677440, "lr": 2.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658004344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13698026537895203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658018436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13622698187828064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48005120, "lr": 2.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658032825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13712841272354126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.4495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658032826, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784658032826, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784658115633, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5804768800735474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784658115635, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784658115635, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784658129893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13148318231105804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48332800, "lr": 2.4579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658144466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13537278771400452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658158823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13402807712554932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48660480, "lr": 2.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658173654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13217994570732117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658187851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13688704371452332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48988160, "lr": 2.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658202069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12949129939079285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658217014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13345573842525482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49315840, "lr": 2.507916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658231325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.123517245054245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.5162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658245602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13047495484352112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49643520, "lr": 2.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658259772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13122744858264923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658273940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13103258609771729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49971200, "lr": 2.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658288030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13115911185741425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.5495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658302371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12843884527683258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50298880, "lr": 2.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658316566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13060179352760315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.5662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658316566, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658316567, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658401394, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5932435989379883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658401395, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658401395, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658415352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1216440200805664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50626560, "lr": 2.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658429604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1267280876636505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658443666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12881459295749664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50954240, "lr": 2.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658457912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13309982419013977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658472520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315208524465561, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51281920, "lr": 2.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658486782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12149475514888763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658501468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1299738585948944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51609600, "lr": 2.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658515968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317678689956665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658530375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13656359910964966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51937280, "lr": 2.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658544423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12792307138442993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.6495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658559129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13254986703395844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52264960, "lr": 2.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658573158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11760612577199936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.6662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658587409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12106890231370926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52592640, "lr": 2.6745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658601698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1249297708272934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658601699, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658601699, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658685092, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6059479117393494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658685093, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658685093, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658699164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12090276181697845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52920320, "lr": 2.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658713166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12745165824890137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.6995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658727529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.128008171916008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53248000, "lr": 2.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658741728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13148799538612366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658755888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12110266834497452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53575680, "lr": 2.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658770105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13432499766349792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658784276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12870335578918457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53903360, "lr": 2.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658798285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12124145776033401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.7495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658812613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12680023908615112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54231040, "lr": 2.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658827115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12703518569469452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658841565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11775732785463333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54558720, "lr": 2.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658856329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13084383308887482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658870551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1267264038324356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54886400, "lr": 2.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658885009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.128393292427063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658885010, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784658885010, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784658969457, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6175912618637085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784658969458, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784658969458, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784658983300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12751346826553345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55214080, "lr": 2.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658997977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11841747164726257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659012254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.121140256524086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55541760, "lr": 2.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659026733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12104526162147522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659040828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1233300194144249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55869440, "lr": 2.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659054858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12560363113880157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659068796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11566450446844101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56197120, "lr": 2.8579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659083021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12248659133911133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659097178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12464210391044617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56524800, "lr": 2.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659111130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12339713424444199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659124823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11511978507041931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56852480, "lr": 2.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659138731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12328635901212692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659152640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10919079929590225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57180160, "lr": 2.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659166870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1243671402335167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659166871, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659166871, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659251258, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6279959678649902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659251259, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659251259, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659265719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11644700169563293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57507840, "lr": 2.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659280088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11399606615304947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659294230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12831485271453857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57835520, "lr": 2.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659308865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12139315903186798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659323399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12365765869617462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58163200, "lr": 2.9579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659337679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11577653139829636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659351507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11953535676002502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58490880, "lr": 2.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659366020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12149475514888763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659380363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1209372878074646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58818560, "lr": 2.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659394788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11249811202287674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659409268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12007308006286621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59146240, "lr": 3.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659423531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12104984372854233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.0162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659437748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12418225407600403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59473920, "lr": 3.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659452246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11681370437145233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659452247, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659452248, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659535482, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6421580910682678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659535483, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659535483, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659549842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11869238317012787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59801600, "lr": 3.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659564206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11134573817253113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659578504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10963888466358185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60129280, "lr": 3.0579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659592586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11282549053430557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.0662499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659606757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10841432213783264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60456960, "lr": 3.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659620945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12247999012470245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659635369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11573781073093414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60784640, "lr": 3.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659649987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12028391659259796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659664389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10206672549247742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61112320, "lr": 3.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659679108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1194593608379364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.1162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659693624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11222472041845322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61440000, "lr": 3.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659707967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11287707835435867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659722172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10855568200349808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61767680, "lr": 3.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659736572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11581901460886002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659736573, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659736573, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659819616, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6618416905403137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659819617, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659819617, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659834095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12027944624423981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62095360, "lr": 3.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659848784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11959521472454071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.1662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659863506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12367334961891174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62423040, "lr": 3.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659878209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10564717650413513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659892913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11447127163410187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62750720, "lr": 3.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659907307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11782326549291611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659921895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11950752884149551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63078400, "lr": 3.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659936440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11823312193155289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659950833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085125058889389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63406080, "lr": 3.2245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659965165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11781273782253265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659979528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11551768332719803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63733760, "lr": 3.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659993550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11627687513828278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660007697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11310724914073944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64061440, "lr": 3.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660022348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12504491209983826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660022348, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784660022349, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784660104886, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6829200387001038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784660104887, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784660104887, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784660119600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12393999099731445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64389120, "lr": 3.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660134152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11676191538572311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660148593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11656732857227325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64716800, "lr": 3.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660163548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12193623185157776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660178287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10985244810581207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65044480, "lr": 3.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660192921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11163397133350372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660206978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11310764402151108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65372160, "lr": 3.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660221640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10906417667865753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660236176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111731618642807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65699840, "lr": 3.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660251249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11594659835100174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660265998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.123603954911232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66027520, "lr": 3.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660280604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11539462953805923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660295482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1272854208946228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66355200, "lr": 3.3745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660310259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11196786165237427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660310259, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660310260, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660395039, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7308523654937744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660395040, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660395040, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660409961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12601223587989807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66682880, "lr": 3.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660424544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1135559156537056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.3995833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660439145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11725805699825287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67010560, "lr": 3.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660453972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11294300109148026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660468623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1190137043595314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67338240, "lr": 3.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660483414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1181623786687851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660498550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12083406001329422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67665920, "lr": 3.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660513567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12056971341371536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660528368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11509037017822266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67993600, "lr": 3.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660543741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11283241212368011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660558600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11557026207447052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68321280, "lr": 3.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660573250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11603665351867676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660587835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12466657906770706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68648960, "lr": 3.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660602865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11617149412631989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660602866, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660602866, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660687418, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7491692304611206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660687419, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660687419, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660702020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12206432968378067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68976640, "lr": 3.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660716810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11982233077287674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660731440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11344163864850998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69304320, "lr": 3.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660746274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11449598520994186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660760916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10875038802623749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69632000, "lr": 3.5412500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660775583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11564753949642181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660790343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11837427318096161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69959680, "lr": 3.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660804780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11097416281700134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660819479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10908593237400055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70287360, "lr": 3.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660834170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10807961970567703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660849223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12310436367988586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70615040, "lr": 3.5912500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660864307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10719393938779831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660879087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1201542317867279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70942720, "lr": 3.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660894353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12001089006662369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660894354, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784660894354, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784660977426, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7533538341522217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784660977427, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784660977427, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784660992437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1193796768784523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71270400, "lr": 3.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661007074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1187913790345192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661021768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11515774577856064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71598080, "lr": 3.6412499999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661036863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11561524868011475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661051585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10322309285402298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71925760, "lr": 3.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661066237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1033594161272049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661080773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11947090178728104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72253440, "lr": 3.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661095283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11537747085094452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661110011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10459444671869278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72581120, "lr": 3.6912499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661124539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10294073075056076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661138945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10668264329433441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72908800, "lr": 3.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661153546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11283215135335922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661168208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11098276823759079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73236480, "lr": 3.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661182827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11500217765569687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661182868, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661182868, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661266432, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7561971545219421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661266434, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661266434, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661281054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10416693240404129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73564160, "lr": 3.7412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661296003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10657726973295212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661312332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11506704986095428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73891840, "lr": 3.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661326630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11082287132740021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661341303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10746338218450546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74219520, "lr": 3.7745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661356145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11513837426900864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661370688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11293015629053116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74547200, "lr": 3.7912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661385178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11375156790018082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661399947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1161278560757637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74874880, "lr": 3.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661414572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111349917948246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661429041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10072828084230423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75202560, "lr": 3.8245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661443367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11323763430118561, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661457604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10630354285240173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75530240, "lr": 3.8412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661472069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10487168282270432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661472071, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661472071, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661555575, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.759394109249115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661555576, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661555576, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661569903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10640595853328705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75857920, "lr": 3.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661584295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1134401485323906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661598684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09646406769752502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76185600, "lr": 3.8745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661613120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10509346425533295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661627592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11196281760931015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76513280, "lr": 3.8912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661641934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10483013838529587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661656365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08917921036481857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76840960, "lr": 3.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661670744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1086266040802002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661685351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10077331215143204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77168640, "lr": 3.9245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661700148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11782227456569672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661715019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10692702233791351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77496320, "lr": 3.9412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661729391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10001412034034729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661744336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10004107654094696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77824000, "lr": 3.9579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661759291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.107075534760952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661759292, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784661759292, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784661842570, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7607152462005615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784661842572, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784661842572, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784661857309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11485830694437027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78151680, "lr": 3.9745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661872020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11230829358100891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661886132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10488305240869522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78479360, "lr": 3.9912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661901010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10570260137319565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661915146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09977969527244568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78807040, "lr": 4.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661929560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10219839215278625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661943655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10649001598358154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79134720, "lr": 4.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661957829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10845445096492767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661971784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09778342396020889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79462400, "lr": 4.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661986105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10890550911426544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662000288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10502446442842484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79790080, "lr": 4.0579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662014406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10672957450151443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662028480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10878538340330124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80117760, "lr": 4.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662042671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11299736052751541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662042671, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784662042672, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784662125234, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.762943685054779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784662125234, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784662125234, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784662139584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11044498533010483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80445440, "lr": 4.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662153698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09857896715402603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662168139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10321275144815445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80773120, "lr": 4.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662182657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10662467032670975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662197367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11090771853923798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81100800, "lr": 4.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662212154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11320265382528305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662226812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10029469430446625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81428480, "lr": 4.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662241590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10586708784103394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662256560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10733276605606079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81756160, "lr": 4.1579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662270951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10572990775108337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662285638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11344972997903824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82083840, "lr": 4.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662299786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09533485770225525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662314757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09834638983011246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82411520, "lr": 4.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662328937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10736040025949478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662328937, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662328937, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662411848, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7638337016105652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662411849, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662411849, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662426109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09244950115680695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82739200, "lr": 4.2079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662440749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10431644320487976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662455205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10947254300117493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83066880, "lr": 4.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662470003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09873170405626297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662484432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10968795418739319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83394560, "lr": 4.2412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662499183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11131758242845535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662513816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09756426513195038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83722240, "lr": 4.2579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662528174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11116018891334534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662542791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10473248362541199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84049920, "lr": 4.2745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662557276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10433872044086456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662571812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10760371387004852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84377600, "lr": 4.2912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662585989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10474761575460434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662600360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09752921760082245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84705280, "lr": 4.3079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662615034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10608375817537308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662615034, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662615035, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662698638, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.76540607213974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662698639, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662698639, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662713567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10444135963916779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85032960, "lr": 4.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662728303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10388924181461334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662742678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10531295090913773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85360640, "lr": 4.3412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662757219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10373640805482864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662771819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09929756075143814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85688320, "lr": 4.3579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662786329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09957404434680939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662801185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10509589314460754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86016000, "lr": 4.3745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662815373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09703259915113449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662830235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10774241387844086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86343680, "lr": 4.3912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662844987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10828062146902084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662859613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11725950241088867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86671360, "lr": 4.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662874497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11502257734537125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662889235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11630140244960785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86999040, "lr": 4.4245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662903899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11501485854387283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662903900, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784662903901, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784662987633, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7664561867713928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784662987634, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784662987634, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784663002227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1005549505352974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87326720, "lr": 4.4412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663016911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11609774082899094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663031199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10803630948066711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87654400, "lr": 4.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663045767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10208433121442795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663060090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09120471775531769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87982080, "lr": 4.4745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663074727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09777296334505081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663088900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10348690301179886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88309760, "lr": 4.4912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663103285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10713324695825577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663117713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10852944850921631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88637440, "lr": 4.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663132077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09968733042478561, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663146929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11323514580726624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88965120, "lr": 4.5245833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663161695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10001974552869797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663176399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10943035781383514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89292800, "lr": 4.5412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663190732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11215957999229431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.5495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663190733, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663190733, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663275145, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7673767805099487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663275146, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663275146, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663289715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10934483259916306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89620480, "lr": 4.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663304167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10165052115917206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663318506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11021225154399872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89948160, "lr": 4.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663333119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09120970964431763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663347358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1144377738237381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90275840, "lr": 4.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663361581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10656873881816864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663376818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11090672761201859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90603520, "lr": 4.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663391460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11197463423013687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663406492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1220879927277565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90931200, "lr": 4.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663421171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10518863052129745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663435917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11140458285808563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91258880, "lr": 4.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663450871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12279821932315826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663465669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10005267709493637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91586560, "lr": 4.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663480394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11114545911550522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663480394, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663480395, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663565002, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7687292098999023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663565003, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663565004, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663579754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10420405119657516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91914240, "lr": 4.6745833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663594567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11859744042158127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663609055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10921478271484375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92241920, "lr": 4.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663624068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10391142964363098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663638823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09649574756622314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92569600, "lr": 4.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663653662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11289657652378082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663668677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10642848908901215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92897280, "lr": 4.7245833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663683644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10684917122125626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663698460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11569753289222717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93224960, "lr": 4.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663712946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09986719489097595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663727916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12494774907827377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93552640, "lr": 4.7579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663742790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10787850618362427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663757538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11531612277030945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93880320, "lr": 4.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663772201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10763051360845566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663772201, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784663772202, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784663859175, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7698436379432678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784663859176, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784663859176, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784663873926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11431890726089478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94208000, "lr": 4.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663888549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11469495296478271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663903079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11084362119436264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94535680, "lr": 4.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663917796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1175481304526329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663932336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.114722840487957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94863360, "lr": 4.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663946954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11921758204698563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663961764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12736876308918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95191040, "lr": 4.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663976281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12209886312484741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663990823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11620990931987762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95518720, "lr": 4.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664005728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11014153808355331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664020302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081041544675827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95846400, "lr": 4.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664034911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10887960344552994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664049406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11530867218971252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96174080, "lr": 4.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664064290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11461014300584793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664064291, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664064291, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664148312, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7704995274543762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664148313, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664148314, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664163394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12025100737810135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96501760, "lr": 4.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664178364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10731568932533264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664192910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10851902514696121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96829440, "lr": 4.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664208018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10757756233215332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664222871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11046609282493591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97157120, "lr": 4.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664237760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1099366545677185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664252541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11354880779981613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97484800, "lr": 4.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664267214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10804223269224167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664281966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10604243725538254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97812480, "lr": 4.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664296244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10039248317480087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664311057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10440068691968918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98140160, "lr": 4.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664325477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09674947708845139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664340058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10417348146438599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98467840, "lr": 5.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664354483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10114539414644241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664354484, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664354484, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664439853, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7697924375534058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664439853, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664439854, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664454382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10992652922868729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98795520, "lr": 5.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664468964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10132226347923279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664483551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09622229635715485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99123200, "lr": 5.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664498174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10847452282905579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664512768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11565735936164856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99450880, "lr": 5.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664527476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10446178168058395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664542180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10452935099601746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99778560, "lr": 5.074583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664556957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10475940257310867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664571581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10156097263097763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100106240, "lr": 5.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664586501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12043803930282593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664601511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11696745455265045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100433920, "lr": 5.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664616430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10134115815162659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664631686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10063081234693527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100761600, "lr": 5.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664646585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11999626457691193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664646586, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664646586, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664730769, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7705788016319275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664730770, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664730770, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664746128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11058726161718369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101089280, "lr": 5.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664761183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09675180166959763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664775979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10500451922416687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101416960, "lr": 5.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664790955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10316867381334305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664805508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10858456790447235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101744640, "lr": 5.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664820231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10466189682483673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.182916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664834619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1062064990401268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102072320, "lr": 5.191249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664849092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11229003220796585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664863761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10451152175664902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102400000, "lr": 5.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664878363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10801547765731812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664892816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10545183718204498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102727680, "lr": 5.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664907368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11486061662435532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664921679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10666225850582123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103055360, "lr": 5.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664936120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10230589658021927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664936121, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784664936121, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784665020702, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.770980954170227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784665020703, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784665020704, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784665035115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09974232316017151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103383040, "lr": 5.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665049666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09374117106199265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665064111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09575874358415604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103710720, "lr": 5.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665078540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0970461443066597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.282916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665092988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09693384170532227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104038400, "lr": 5.291249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665107741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09537187218666077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665122412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10056593269109726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104366080, "lr": 5.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665136928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09672918915748596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665151673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10297764837741852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104693760, "lr": 5.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665166290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10841596871614456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665181213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10542818158864975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105021440, "lr": 5.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665196109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10246261209249496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665210710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11464779824018478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105349120, "lr": 5.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665226231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11120420694351196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665226232, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665226232, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665311225, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7714695930480957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665311226, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665311227, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665326008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09842654317617416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105676800, "lr": 5.374583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665340892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1069764792919159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.382916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665356156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1100892499089241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106004480, "lr": 5.391249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665370801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10099475085735321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665385709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10041195899248123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106332160, "lr": 5.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665400355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10846994072198868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665415383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11087469756603241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106659840, "lr": 5.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665430360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10660597681999207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665445635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10287773609161377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106987520, "lr": 5.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665460449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10164184123277664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665475396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10046999901533127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107315200, "lr": 5.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665490220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10244188457727432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665505074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10882644355297089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107642880, "lr": 5.474583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665519850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11454056948423386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.482916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665519851, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665519851, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665603701, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7717136144638062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665603702, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665603703, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665618400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10980725288391113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107970560, "lr": 5.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665633240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723134130239487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665648002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09088263660669327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108298240, "lr": 5.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665662869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10277314484119415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665677950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1065533384680748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108625920, "lr": 5.524583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665692700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10835027694702148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665707623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11045555025339127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108953600, "lr": 5.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665722606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1029876172542572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665737646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10035520792007446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109281280, "lr": 5.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665752628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1043168231844902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665767687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10272707790136337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109608960, "lr": 5.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665782458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10759923607110977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665797797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10529007017612457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109936640, "lr": 5.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665813136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10911634564399719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665813137, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784665813138, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784665896874, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7721043825149536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784665896875, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784665896875, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784665911769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10374972969293594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110264320, "lr": 5.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665926679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10289891809225082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665941432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10387485474348068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110592000, "lr": 5.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665955708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10281788557767868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665970562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10757305473089218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110919680, "lr": 5.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665984681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10155656933784485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665999016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09846553206443787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111247360, "lr": 5.657916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666013845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10210607945919037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.666250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666028633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1117379367351532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111575040, "lr": 5.674583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666043407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10463415831327438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666057913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10394962131977081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111902720, "lr": 5.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666072420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10241670906543732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666087069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10366848111152649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112230400, "lr": 5.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666101490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10643181949853897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666101490, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666101491, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666186627, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.772265613079071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666186628, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666186628, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666201032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10833737254142761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112558080, "lr": 5.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666215504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10183437168598175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666229660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10401518642902374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112885760, "lr": 5.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666243982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10120435804128647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666258562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10513056814670563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113213440, "lr": 5.757916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666272716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1126818135380745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.766250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666287236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10155731439590454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113541120, "lr": 5.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666301543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11051828414201736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666316122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11340554803609848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113868800, "lr": 5.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666330865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10679840296506882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666345276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.103239044547081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114196480, "lr": 5.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666359781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1050005704164505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666374637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10515744984149933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114524160, "lr": 5.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666389304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09904467314481735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666389305, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666389305, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666472846, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7722175121307373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666472847, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666472847, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666487204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10194999724626541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114851840, "lr": 5.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666501505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09972742199897766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666515828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10591227561235428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115179520, "lr": 5.857916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666529802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10063723474740982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666545199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10819052904844284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115507200, "lr": 5.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666560003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10917039215564728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666575112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10791371762752533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115834880, "lr": 5.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666590461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11577954143285751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666605376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11423498392105103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116162560, "lr": 5.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666620141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10076266527175903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666635035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10416564345359802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116490240, "lr": 5.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666649920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10284483432769775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666664702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10512460768222809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116817920, "lr": 5.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666679561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10713998973369598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666679562, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666679563, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666761308, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7726176977157593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666761309, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666761309, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666776273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10183430463075638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117145600, "lr": 5.957916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666791198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10064389556646347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666806317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10862462967634201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117473280, "lr": 5.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666821222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10519406944513321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666835861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09421591460704803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117800960, "lr": 5.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666850988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11257577687501907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666865644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10132579505443573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118128640, "lr": 6.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666880342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10770556330680847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666895537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10392492264509201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118456320, "lr": 6.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666910802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10357531905174255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666925710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10938117653131485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118784000, "lr": 6.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666940884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11407408118247986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666956073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10579852014780045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119111680, "lr": 6.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666971288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10310456156730652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666971289, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784666971290, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784667052054, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730396389961243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784667052055, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784667052055, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784667067199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11389942467212677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119439360, "lr": 6.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667081889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10891831666231155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667096484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10962451994419098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119767040, "lr": 6.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667111446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10722047835588455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667125872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10170675814151764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120094720, "lr": 6.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667139937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10825762152671814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667155376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.118283711373806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120422400, "lr": 6.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667170118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10924483090639114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667184781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11643178761005402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120750080, "lr": 6.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667199759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12265549600124359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667214298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10709595680236816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121077760, "lr": 6.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667228827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11061666160821915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667243433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11460398882627487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121405440, "lr": 6.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667257963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11017925292253494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667257964, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667257964, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667339559, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7733845114707947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667339560, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667339560, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667354184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11346598714590073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121733120, "lr": 6.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667369187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1096586212515831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667383908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11077022552490234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122060800, "lr": 6.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667398782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10669764131307602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667413420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10410945117473602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122388480, "lr": 6.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667428130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11372289806604385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667442791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1069139838218689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122716160, "lr": 6.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667457528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11483991891145706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667472436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11283750832080841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123043840, "lr": 6.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667487467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11947200447320938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667502480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10679788887500763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123371520, "lr": 6.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667517129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1181703731417656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667531854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732700675725937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123699200, "lr": 6.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667546813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1196020096540451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667546813, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667546814, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667629594, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.77339768409729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667629595, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667629595, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667644514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11951906234025955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124026880, "lr": 6.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667659237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10794027894735336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667673755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12008480727672577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124354560, "lr": 6.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667688038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11429235339164734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667702562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10785841941833496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124682240, "lr": 6.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667717847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11535007506608963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667732595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1117597222328186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125009920, "lr": 6.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667747703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11669755727052689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667762788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11212971806526184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125337600, "lr": 6.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667777517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10618355870246887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667792350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11390302330255508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125665280, "lr": 6.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667807154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10663886368274689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667821942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11707159876823425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125992960, "lr": 6.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667836985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1048993244767189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667836986, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784667836986, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784667920455, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7736804485321045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784667920456, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784667920456, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784667935247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10986807197332382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126320640, "lr": 6.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667950401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12042950838804245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667965193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11037833243608475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126648320, "lr": 6.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667979927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11356047540903091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667994500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11592894792556763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126976000, "lr": 6.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668009197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10954279452562332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668024106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10852224379777908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127303680, "lr": 6.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668039070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11007335036993027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.482916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668054055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.112454853951931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127631360, "lr": 6.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668068871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11259818077087402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668083914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10997245460748672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127959040, "lr": 6.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668099082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11068709194660187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668113988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1116097941994667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128286720, "lr": 6.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668128824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10905879735946655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668128825, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668128825, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668211674, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773500919342041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668211675, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668211675, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668226269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11774756759405136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128614400, "lr": 6.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668241205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11079639196395874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668255950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11124616861343384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128942080, "lr": 6.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668270550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10351888090372086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668285271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10567089915275574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129269760, "lr": 6.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668300151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11216442286968231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668314755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10850219428539276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129597440, "lr": 6.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668329327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10768789052963257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668343953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09824030846357346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129925120, "lr": 6.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668358514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10231804847717285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668373235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10587631911039352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130252800, "lr": 6.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668387656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1047225221991539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668402323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11003842204809189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130580480, "lr": 6.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668417183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11598428338766098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668417184, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668417184, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668499417, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773408055305481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668499418, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668499419, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668513893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11490625143051147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130908160, "lr": 6.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668528503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10553254932165146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668542811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10095150023698807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131235840, "lr": 6.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668557320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09262016415596008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668571731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11422418057918549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131563520, "lr": 6.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668585865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10428491979837418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668600463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10222814232110977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131891200, "lr": 6.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668615299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11237892508506775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668629799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10721446573734283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132218880, "lr": 6.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668644552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10379359126091003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668659182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10927070677280426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132546560, "lr": 6.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668673986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11101113259792328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668688761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10753492265939713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132874240, "lr": 6.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668703586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10753771662712097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668703586, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668703587, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668785122, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738193869590759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668785122, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668785123, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668799695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11034836620092392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133201920, "lr": 6.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668814618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10313718020915985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668829209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11035464704036713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133529600, "lr": 6.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668843768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09959962218999863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668858947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11611591279506683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133857280, "lr": 6.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668873508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11030875891447067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668888012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10352560132741928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134184960, "lr": 6.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668902952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10817839950323105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668917653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793309658765793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134512640, "lr": 6.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668931989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10926184803247452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668946527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11088120937347412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134840320, "lr": 6.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668961049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09926919639110565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668975306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10704761743545532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135168000, "lr": 6.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668989639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08692336082458496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668989639, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784668989640, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669072137, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7740043997764587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669072138, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669072138, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669086709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09562891721725464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135495680, "lr": 6.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669101232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11081914603710175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669115732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10756561160087585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135823360, "lr": 6.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669130184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10161614418029785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669144725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10204233974218369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136151040, "lr": 6.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669159449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09064833074808121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669173985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10339982062578201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136478720, "lr": 6.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669188505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09696300327777863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669202980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1037474051117897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136806400, "lr": 6.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669217753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10640872269868851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669232486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1104414165019989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137134080, "lr": 6.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669247188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10835058987140656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669261700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10643113404512405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137461760, "lr": 6.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669276349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10332643240690231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669276350, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669276350, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669357566, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7743675112724304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669357567, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669357567, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669372356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09608607739210129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137789440, "lr": 7.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669387306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10569863766431808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669402075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0948735773563385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138117120, "lr": 7.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669416679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10271468758583069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669431351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09856298565864563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138444800, "lr": 7.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669445596, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10085918754339218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669459866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1095188707113266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138772480, "lr": 7.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669474964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10162882506847382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.066249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669489299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10900818556547165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139100160, "lr": 7.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669503881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10985112190246582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669518964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10686750710010529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139427840, "lr": 7.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669533820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1046820655465126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669548075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10973281413316727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139755520, "lr": 7.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669562393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10878913104534149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669562394, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669562394, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669644535, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7743968367576599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669644535, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669644536, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669658773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10642104595899582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140083200, "lr": 7.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669673044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10451556742191315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669687343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10386507958173752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140410880, "lr": 7.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669701702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10025257617235184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669716146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1075466126203537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140738560, "lr": 7.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669730552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09613707661628723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669745492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10556589066982269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141066240, "lr": 7.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669759877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1031847596168518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669774166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11086196452379227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141393920, "lr": 7.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669788719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09874508529901505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669802831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09768599271774292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141721600, "lr": 7.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669817219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10241729766130447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669831838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09120237827301025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142049280, "lr": 7.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669846441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10726030170917511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669846442, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784669846442, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784669930236, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.77467942237854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784669930237, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784669930237, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784669944925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09951434284448624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142376960, "lr": 7.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669959844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10117271542549133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669974315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09940160810947418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142704640, "lr": 7.257916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669989108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11006274074316025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670003818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10070127993822098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143032320, "lr": 7.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670018631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10214022547006607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670033221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10279761999845505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143360000, "lr": 7.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670047746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10566247254610062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670062447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10178618133068085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143687680, "lr": 7.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670076772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11270712316036224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670091600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10145214200019836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144015360, "lr": 7.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670106038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10827276855707169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670120427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10616626590490341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144343040, "lr": 7.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670135059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10674960911273956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670135060, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670135060, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670220111, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7749477028846741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670220112, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670220112, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670234811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10690705478191376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144670720, "lr": 7.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670249436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09863538295030594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670263873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1070742979645729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144998400, "lr": 7.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670278626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1070944219827652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670293354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10256865620613098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145326080, "lr": 7.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670307685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10754718631505966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670322182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.097910575568676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145653760, "lr": 7.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670337052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09862445294857025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670351688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10522571951150894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145981440, "lr": 7.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670366133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.104313425719738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670380779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10324126482009888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146309120, "lr": 7.441250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670395442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1104024350643158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670410161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10687801986932755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146636800, "lr": 7.457916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670424898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10030284523963928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670424899, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670424899, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670509146, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7750598788261414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670509147, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670509147, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670523503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1034000962972641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146964480, "lr": 7.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670537874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10573933273553848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670552474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09497185796499252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147292160, "lr": 7.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670567398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09885570406913757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670581928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10237310081720352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147619840, "lr": 7.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670596890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10023505240678787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670611486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10695992410182953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147947520, "lr": 7.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670626252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10964440554380417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670641064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1089567020535469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148275200, "lr": 7.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670655819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10023632645606995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670670702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10271842777729034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148602880, "lr": 7.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670685146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09924427419900894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670699933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11250100284814835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148930560, "lr": 7.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670714345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11111122369766235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670714345, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670714346, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670799444, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7751511931419373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670799446, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670799446, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670813723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1056293249130249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149258240, "lr": 7.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670828791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10510219633579254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670843194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10649427026510239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149585920, "lr": 7.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670857898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11111558973789215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149749760, "lr": 7.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670872987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10641635954380035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149913600, "lr": 7.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670887812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10926701128482819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670902279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1076853945851326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150241280, "lr": 7.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670916876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09738931059837341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150405120, "lr": 7.649583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670931306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10748692601919174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150568960, "lr": 7.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670945620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10360351204872131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670960098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10822132229804993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150896640, "lr": 7.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670974325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10358644276857376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151060480, "lr": 7.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670989024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10653557628393173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151224320, "lr": 7.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671003394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09534375369548798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671003442, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671003443, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671084957, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7756553888320923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671084958, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671084958, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671099396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0997256189584732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151552000, "lr": 7.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671113781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10536875575780869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151715840, "lr": 7.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671128258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10293678194284439, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151879680, "lr": 7.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671142845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1043795794248581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671157433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1083914265036583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152207360, "lr": 7.741249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671171903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1234198734164238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152371200, "lr": 7.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671186388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10823573917150497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152535040, "lr": 7.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671200960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09883608669042587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671215727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1122567355632782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152862720, "lr": 7.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671230204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11211643368005753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153026560, "lr": 7.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671244944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11475453525781631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153190400, "lr": 7.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671259745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09914160519838333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671274650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10662434995174408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153518080, "lr": 7.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671289445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10459683835506439, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153681920, "lr": 7.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671289495, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671289495, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671375371, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.775739848613739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671375372, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671375372, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671390127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09802483022212982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153845760, "lr": 7.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671404881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09855715185403824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.832916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671419378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11313686519861221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154173440, "lr": 7.841249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671434369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11771392077207565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154337280, "lr": 7.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671449068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11426731199026108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154501120, "lr": 7.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671463363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11105821281671524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671478434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10790861397981644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154828800, "lr": 7.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671493052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12056625634431839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154992640, "lr": 7.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671507721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10496387630701065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155156480, "lr": 7.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671522787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10835476219654083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671537816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11109901964664459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155484160, "lr": 7.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671552576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11084402352571487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155648000, "lr": 7.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671567394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11280660331249237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155811840, "lr": 7.924583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671582311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11211161315441132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.932916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671582311, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671582312, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671667050, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7760637998580933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671667051, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671667051, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671681902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10773083567619324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156139520, "lr": 7.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671696767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11036955565214157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156303360, "lr": 7.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671711834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11461295187473297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156467200, "lr": 7.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671726771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11026841402053833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671741748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10831058770418167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156794880, "lr": 7.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671756674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10864215344190598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156958720, "lr": 7.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671771488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10197678208351135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157122560, "lr": 7.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671786223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11302895843982697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671800962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12099143117666245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157450240, "lr": 8.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671815887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10958773642778397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157614080, "lr": 8.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671830602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10842838138341904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157777920, "lr": 8.024583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671845799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1047552302479744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.032916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671860944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.103840172290802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158105600, "lr": 8.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671875872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10870501399040222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158269440, "lr": 8.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671875873, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784671875873, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784671961201, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7763190865516663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784671961202, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784671961202, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784671975957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11413625627756119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158433280, "lr": 8.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671990853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11940961331129074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672005840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11425187438726425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158760960, "lr": 8.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672021009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11219896376132965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158924800, "lr": 8.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672036051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11147034913301468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159088640, "lr": 8.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672050691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11844337731599808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672065564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1123284175992012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159416320, "lr": 8.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672079936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10934049636125565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159580160, "lr": 8.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672095137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11062411218881607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159744000, "lr": 8.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672110137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09907621145248413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672124981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10989834368228912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160071680, "lr": 8.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672140146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10965242236852646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160235520, "lr": 8.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672155109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10443272441625595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160399360, "lr": 8.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672169818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11049510538578033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672169818, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672169820, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672252757, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7766001224517822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672252758, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672252759, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672267349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11304599791765213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160727040, "lr": 8.174583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672282199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1286531537771225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160890880, "lr": 8.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672296931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10808128863573074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161054720, "lr": 8.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672311633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11241109669208527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672326214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11293796449899673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161382400, "lr": 8.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672341071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11091360449790955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161546240, "lr": 8.216250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672356018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11406953632831573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161710080, "lr": 8.224583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672371058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11298517137765884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672385656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10707268863916397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162037760, "lr": 8.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672400252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10860412567853928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162201600, "lr": 8.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672415316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12969017028808594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162365440, "lr": 8.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672430160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11317678540945053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672445361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11325285583734512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162693120, "lr": 8.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672460394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11630431562662125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162856960, "lr": 8.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672460395, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672460395, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672543832, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7767265439033508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672543833, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672543833, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672558515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09566988050937653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163020800, "lr": 8.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672573662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11149223893880844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672588793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12232346087694168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163348480, "lr": 8.307916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672604012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10694972425699234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163512320, "lr": 8.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672619192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1092289462685585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163676160, "lr": 8.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672634123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10782106220722198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672649153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11228401958942413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164003840, "lr": 8.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672664038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11293987184762955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164167680, "lr": 8.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672679217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10339007526636124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164331520, "lr": 8.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672693837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11772049218416214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672708593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10841769725084305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164659200, "lr": 8.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672723513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11138839274644852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164823040, "lr": 8.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672738688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09259173274040222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164986880, "lr": 8.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672753382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10372539609670639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672753382, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784672753383, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784672835087, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7768469452857971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784672835088, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784672835088, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784672849880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10178933292627335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165314560, "lr": 8.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672864405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09725296497344971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165478400, "lr": 8.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672878769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11476772278547287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165642240, "lr": 8.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672893187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10826599597930908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672907558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10977867245674133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165969920, "lr": 8.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672922408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10395795106887817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166133760, "lr": 8.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672936974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10770581662654877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166297600, "lr": 8.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672951647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11322201043367386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672966350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09989649057388306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166625280, "lr": 8.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672981234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0950709655880928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166789120, "lr": 8.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672995969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09742160886526108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166952960, "lr": 8.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673010541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09703372418880463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673024770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10261232405900955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167280640, "lr": 8.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673039204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10102732479572296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167444480, "lr": 8.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673039205, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673039205, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673120962, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7768898606300354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673120963, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673120964, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673135477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10340649634599686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167608320, "lr": 8.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673150142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10309004038572311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673164676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10425867885351181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167936000, "lr": 8.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673179195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10700611025094986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168099840, "lr": 8.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673193791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10225681960582733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168263680, "lr": 8.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673208639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09977797418832779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673223649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10943516343832016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168591360, "lr": 8.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673238437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10521243512630463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168755200, "lr": 8.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673253190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10714945197105408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168919040, "lr": 8.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673267710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10625135898590088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673282536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10444574803113937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169246720, "lr": 8.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673296901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10813320428133011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169410560, "lr": 8.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673311201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10628252476453781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169574400, "lr": 8.624583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673326276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11306241154670715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673326320, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673326321, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673407100, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7770264744758606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673407101, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673407102, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673421720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085599809885025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169902080, "lr": 8.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673436564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1116977110505104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170065920, "lr": 8.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673451731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10706361383199692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170229760, "lr": 8.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673466978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725348442792892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673481791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10020777583122253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170557440, "lr": 8.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673496843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10251869261264801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170721280, "lr": 8.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673511815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09619446843862534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170885120, "lr": 8.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673526590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10844972729682922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673541285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10023615509271622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171212800, "lr": 8.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673556041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10279840975999832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171376640, "lr": 8.716249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673571132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10158883035182953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171540480, "lr": 8.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673585873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10363361239433289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673600722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09954883903265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171868160, "lr": 8.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673615618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10452954471111298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172032000, "lr": 8.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673615668, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673615668, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673697519, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7771875858306885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673697520, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673697521, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673712387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09807215631008148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172195840, "lr": 8.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673727268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1079208180308342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673741983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09975045919418335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172523520, "lr": 8.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673756601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10291707515716553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172687360, "lr": 8.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673771076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09453010559082031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172851200, "lr": 8.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673785939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10668230801820755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673801094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10411141812801361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173178880, "lr": 8.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673816206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10398287326097488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173342720, "lr": 8.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673831518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10478890687227249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173506560, "lr": 8.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673846516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11183065176010132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673861564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09627819061279297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173834240, "lr": 8.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673876754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11170240491628647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173998080, "lr": 8.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673892078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11234752833843231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174161920, "lr": 8.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673907033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10185886174440384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673907080, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784673907080, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784673990266, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7773709893226624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784673990267, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784673990267, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784674004923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09837927669286728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174489600, "lr": 8.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674020000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10743149369955063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174653440, "lr": 8.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674035109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10183029621839523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174817280, "lr": 8.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674049549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10610140860080719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674064545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1006036177277565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175144960, "lr": 8.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674079023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10912833362817764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175308800, "lr": 8.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674093526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10029467195272446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175472640, "lr": 8.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674108284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09871814399957657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674122975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10871776938438416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175800320, "lr": 8.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674137567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10900858789682388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175964160, "lr": 8.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674152183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10015781968832016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176128000, "lr": 8.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674166805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09472009539604187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674181148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09551286697387695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176455680, "lr": 8.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674195472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10555104911327362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176619520, "lr": 8.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674195472, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674195473, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674277412, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7778046131134033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674277413, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674277413, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674292044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1031537875533104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176783360, "lr": 8.991249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674306643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1138470247387886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674321210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10108646750450134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177111040, "lr": 9.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674335876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1055607795715332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177274880, "lr": 9.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674350485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11031918227672577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177438720, "lr": 9.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674364864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10098724067211151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.032916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674379131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10986310988664627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177766400, "lr": 9.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674393429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10164827853441238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177930240, "lr": 9.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674407791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10785217583179474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178094080, "lr": 9.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674422097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10168984532356262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674436444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09800645709037781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178421760, "lr": 9.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674451160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11160330474376678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178585600, "lr": 9.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674465842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11917757242918015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178749440, "lr": 9.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674480489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10914945602416992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674480490, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674480490, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674562854, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7780547142028809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674562855, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674562855, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674577704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11569374799728394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179077120, "lr": 9.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674592266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10078112035989761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179240960, "lr": 9.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674607278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10009168833494186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179404800, "lr": 9.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674622004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10087382048368454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.132916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674636959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10539665073156357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179732480, "lr": 9.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674651767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09911918640136719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179896320, "lr": 9.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674666316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10049830377101898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180060160, "lr": 9.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674681259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1059483215212822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674695795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10564155131578445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180387840, "lr": 9.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674710156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09415332973003387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180551680, "lr": 9.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674725657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10373508185148239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180715520, "lr": 9.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674740379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10170957446098328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674755194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09382753819227219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181043200, "lr": 9.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674770171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10022344440221786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181207040, "lr": 9.216249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674770172, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784674770172, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784674853117, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7782136797904968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784674853118, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784674853118, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784674868239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11151231080293655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181370880, "lr": 9.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674883249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11346184462308884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674897881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1005650982260704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181698560, "lr": 9.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674912662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10722338408231735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181862400, "lr": 9.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674927441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10322137176990509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182026240, "lr": 9.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674941932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10303706675767899, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674956337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10642563551664352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182353920, "lr": 9.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674970689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10845844447612762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182517760, "lr": 9.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674985189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09795655310153961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182681600, "lr": 9.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674999550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0972965732216835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675014048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11170579493045807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183009280, "lr": 9.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675028340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09812899678945541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183173120, "lr": 9.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675043000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10232963413000107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183336960, "lr": 9.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675057284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10273466259241104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675057284, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675057285, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675138414, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7785486578941345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675138415, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675138416, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675152897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10200421512126923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183664640, "lr": 9.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675167283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10845832526683807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183828480, "lr": 9.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675181722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1120946928858757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183992320, "lr": 9.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675196594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09405931830406189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675211381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10345330089330673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184320000, "lr": 9.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675226085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10767491906881332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184483840, "lr": 9.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675240857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1164625957608223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184647680, "lr": 9.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675255368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10109949111938477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675270071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10544735938310623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184975360, "lr": 9.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675285019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1101650670170784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185139200, "lr": 9.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675299882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09775616228580475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185303040, "lr": 9.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675314566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1079145222902298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675329357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09884167462587357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185630720, "lr": 9.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675343814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10461627691984177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185794560, "lr": 9.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675343815, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675343815, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675426403, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.778867244720459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675426404, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675426404, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675441366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11021801829338074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185958400, "lr": 9.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675455813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10432850569486618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675470049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09197249263525009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186286080, "lr": 9.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675485204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1004711389541626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186449920, "lr": 9.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675499873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1086505874991417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186613760, "lr": 9.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675514547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11100473254919052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675529598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09622912108898163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186941440, "lr": 9.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675544582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10388513654470444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187105280, "lr": 9.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675559429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10613022744655609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187269120, "lr": 9.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675574002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09903017431497574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675588495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1043291762471199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187596800, "lr": 9.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675603020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10498496890068054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187760640, "lr": 9.549583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675617462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11049318313598633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187924480, "lr": 9.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675632288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11387167125940323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675632289, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675632289, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675714599, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7790331840515137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675714600, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675714600, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675729192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10969353467226028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188252160, "lr": 9.574583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675744110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10531973838806152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188416000, "lr": 9.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675768618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10431502759456635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188579840, "lr": 9.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675783434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10833665728569031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675801587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10212124139070511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188907520, "lr": 9.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675816448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09732409566640854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189071360, "lr": 9.616249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675831737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10279352962970734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189235200, "lr": 9.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675846214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11197809875011444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675861048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10132557153701782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189562880, "lr": 9.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675875923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09514189511537552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189726720, "lr": 9.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675890744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1049627959728241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189890560, "lr": 9.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675905930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1126299500465393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.666249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675921049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10195273160934448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190218240, "lr": 9.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675936089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11835530400276184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190382080, "lr": 9.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675936089, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784675936090, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676019818, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7793129086494446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676019819, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676019819, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676035038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10039306432008743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190545920, "lr": 9.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676050181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09702438861131668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676065130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1057538390159607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190873600, "lr": 9.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676080465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11203397065401077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191037440, "lr": 9.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676096048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11815813183784485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191201280, "lr": 9.724583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676111372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09657103568315506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676126719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10608436912298203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191528960, "lr": 9.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676141896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10078692436218262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191692800, "lr": 9.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676157397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10105298459529877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191856640, "lr": 9.757916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676172481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10478632897138596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676187013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10666177421808243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192184320, "lr": 9.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676202390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11044885963201523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192348160, "lr": 9.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676217413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09952165186405182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192512000, "lr": 9.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676232739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1145361140370369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.799583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676232739, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676232740, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676316471, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7798336148262024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676316472, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676316472, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676331904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11244729161262512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192839680, "lr": 9.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676347441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11101854592561722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193003520, "lr": 9.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676362869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10639326274394989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193167360, "lr": 9.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676378180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12034976482391357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676393496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11603467166423798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193495040, "lr": 9.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676408929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10122153908014297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193658880, "lr": 9.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676424435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11838071048259735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193822720, "lr": 9.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676439526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11405821889638901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.866249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676454862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10582417249679565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194150400, "lr": 9.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676470288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10715403407812119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194314240, "lr": 9.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676485794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10720597207546234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194478080, "lr": 9.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676501011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10674495995044708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676516341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1068098321557045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194805760, "lr": 9.907916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676531166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11594592034816742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194969600, "lr": 9.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676531167, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676531167, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676617632, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7798252701759338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676617633, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676617633, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676632948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11189926415681839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195133440, "lr": 9.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676647993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1038990318775177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676663063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10292354226112366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195461120, "lr": 9.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676678111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11577247083187103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195624960, "lr": 9.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676693224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082901656627655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195788800, "lr": 9.957916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676708097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1055714562535286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676723246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10485219955444336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196116480, "lr": 9.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676738238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1138983964920044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196280320, "lr": 9.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676753752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1179085299372673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196444160, "lr": 9.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676769049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10535586625337601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.999583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676784261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11700118333101273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196771840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676799457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1120888888835907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196935680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676814070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10795577615499496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197099520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676829408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11597539484500885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676829409, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784676829409, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784676911510, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.780177116394043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784676911511, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784676911511, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784676926316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10127025097608566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197427200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676941719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11120437830686569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197591040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676956792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11515446752309799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197754880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676971866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10164916515350342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676987233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09998716413974762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198082560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677002427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10784757882356644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198246400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677017644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09861156344413757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198410240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677032804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10853639990091324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677048123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1005110889673233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198737920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677063161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10521341115236282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198901760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677078032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11453376710414886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199065600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677093318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11008785665035248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677108420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11760544776916504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199393280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677123788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11297906935214996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199557120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677123789, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677123789, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677205810, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7804377675056458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677205811, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677205811, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677220995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1121070608496666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199720960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677236369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10711140930652618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677251381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11129752546548843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200048640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677266597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11665990203619003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200212480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677281790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11211231350898743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200376320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677297146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11673084646463394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677312497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09937578439712524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200704000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677327628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10882090032100677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200867840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677342840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10856369137763977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201031680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677358126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10836820304393768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677373470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11848177015781403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201359360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677388713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11131976544857025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201523200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677403882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10458554327487946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201687040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677419221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11146172881126404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677419222, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677419223, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677501800, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7801834940910339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677501801, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677501801, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677516542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10552278161048889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202014720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677531578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11124267429113388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202178560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677546489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11923377215862274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202342400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677561610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10637649148702621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677576233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10535627603530884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202670080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677591009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10123763978481293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202833920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677606322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10588178038597107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202997760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677621058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10973680019378662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677635831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11304163932800293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203325440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677650396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09333436191082001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203489280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677665273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1176779642701149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203653120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677680121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10103669762611389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677694656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0979141891002655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203980800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677709244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10803405940532684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204144640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677709245, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784677709246, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784677790773, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7804396152496338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784677790774, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784677790774, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784677805668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10140839964151382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204308480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677820821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09241409599781036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677835965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09676700830459595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204636160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677850845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10940131545066833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204800000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677865552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10810177028179169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204963840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677880304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11227597296237946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677895003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09494082629680634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205291520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677910299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10594635456800461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205455360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677924892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10582750290632248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205619200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677939654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11030212789773941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677954544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09861717373132706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205946880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677969684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0994294136762619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206110720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677984544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11151576787233353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206274560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677999144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09851646423339844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677999145, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784677999145, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678080563, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7807923555374146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678080564, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678080564, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678095663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10618717223405838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206602240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678110257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09566718339920044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206766080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678125383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10377930104732513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206929920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678140910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09554250538349152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678156386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10189348459243774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207257600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678171523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11087577790021896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207421440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678186673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10750734061002731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207585280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678201647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.095061294734478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678216772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1145651638507843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207912960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678231808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10397583991289139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208076800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678246476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10355723649263382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208240640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678261818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10249118506908417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678276337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10340496897697449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208568320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678291074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10142682492733002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208732160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678291075, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678291076, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678375791, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7805388569831848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678375792, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678375792, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678390760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10060816258192062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208896000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678405625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09915928542613983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678420603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10213182866573334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209223680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678435203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10258708149194717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209387520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678450358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09926508367061615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209551360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678465134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10112959146499634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678479758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09692111611366272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209879040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678494287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10021396726369858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210042880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678509130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10316716879606247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210206720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678523896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10815519839525223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678538706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10413432121276855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210534400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678553484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10794682055711746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210698240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678568379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10172726213932037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210862080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678583092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09596018493175507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678583092, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678583093, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678667013, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7805784344673157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678667014, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678667014, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678682093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09411449730396271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211189760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678696742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.099203921854496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211353600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678711578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09690281748771667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211517440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678726090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10020432621240616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678740756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10422751307487488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211845120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678756120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09561479091644287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212008960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678770799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10258792340755463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212172800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678785736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09796527773141861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678800413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0982784628868103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212500480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678815251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10456255823373795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212664320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678829910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09355214238166809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212828160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678845011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10744638741016388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678860129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10520530492067337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213155840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678875322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11053648591041565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213319680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678875323, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784678875323, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784678958110, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7804100513458252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784678958111, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784678958111, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784678972851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10545706003904343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213483520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678987716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10105311125516891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679002582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10534685850143433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213811200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679017908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10261138528585434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213975040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679032562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09620721638202667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214138880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679047215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10474959015846252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679062364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09592415392398834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214466560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679077308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11414738744497299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214630400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679092350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09826737642288208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214794240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679107959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08993493020534515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679123206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10158028453588486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215121920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679138271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10531985759735107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215285760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679153408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10455114394426346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215449600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679168610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1088247075676918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679168611, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679168611, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679250808, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7807700037956238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679250809, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679250809, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679265703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10084936022758484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215777280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679280362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10527419298887253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215941120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679295138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09807007759809494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216104960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679310000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09940826147794724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679324953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0999804362654686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216432640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679339748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09462970495223999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216596480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679354566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10436948388814926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216760320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679369659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10266221314668655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679384454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10050204396247864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217088000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679399408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11060192435979843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217251840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679413988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09768807142972946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217415680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679428943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09999164938926697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217579520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679443635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09960659593343735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217743360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679458471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10598740726709366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217907200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679458471, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679458472, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679540605, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7803234457969666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679540605, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679540605, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679555296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10176502168178558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218071040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679570090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09501972049474716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218234880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679585111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0981292575597763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218398720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679599598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10771171748638153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218562560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679614418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10334828495979309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218726400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679628937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08742990344762802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218890240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679643761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10575804114341736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219054080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679658680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09307288378477097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219217920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679673596, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10489991307258606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219381760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679688410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09579215198755264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219545600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679703208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09497947990894318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219709440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679717626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10250106453895569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219873280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679732421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11195578426122665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220037120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679747289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0983903557062149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220200960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679747290, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784679747290, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784679830861, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7811052203178406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784679830862, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784679830862, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784679845145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09507735073566437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220364800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679860393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09555069357156754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220528640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679875149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10175060480833054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220692480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679890058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10296501964330673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220856320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679905317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10357441753149033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221020160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679920580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10589603334665298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221184000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679935856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080113872885704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221347840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679950721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11563023924827576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221511680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679965598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10834398865699768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221675520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679980626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09875458478927612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221839360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679995588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11085623502731323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222003200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680010666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09608373790979385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222167040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680025782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09400222450494766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222330880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680040877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085687205195427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222494720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680040878, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784680040878, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784680123341, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7806923985481262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784680123342, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784680123342, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784680138487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09893912822008133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222658560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680153660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11486727744340897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222822400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680168503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10609079152345657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222986240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680183592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09309050440788269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223150080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680198897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10426244884729385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223313920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680214225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0981103926897049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223477760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680229409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09503877907991409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223641600, "lr": 1e-06}} diff --git a/recommendation_v4/rcp_logs/gbs_8192/seed402905276.log b/recommendation_v4/rcp_logs/gbs_8192/seed402905276.log new file mode 100644 index 000000000..40426e8c7 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_8192/seed402905276.log @@ -0,0 +1,1853 @@ +:::MLLOG {"namespace": "", "time_ms": 1784651471631, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784651471632, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784651491798, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784651491802, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784651491802, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784651491802, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784651491802, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784651491802, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 8192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784651491802, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784651491802, "event_type": "POINT_IN_TIME", "key": "seed", "value": 402905276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784651491802, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784651491802, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784651491802, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784651491802, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784651491805, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784651491805, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784652175033, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784652175034, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784652175251, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784652346394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13819746673107147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840, "lr": 7.916666666666666e-10}} +:::MLLOG {"namespace": "", "time_ms": 1784652358194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13950763642787933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.6249999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652370534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13881269097328186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 491520, "lr": 2.458333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652383019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13904573023319244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652395749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389351636171341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 819200, "lr": 4.125e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652408616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13884398341178894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652421483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13907524943351746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1146880, "lr": 5.7916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652434330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13924768567085266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.625e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652447278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13884134590625763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1474560, "lr": 7.458333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652460172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13898244500160217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.291666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652473283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13887031376361847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1802240, "lr": 9.124999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652486452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13890177011489868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652499633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13907665014266968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2129920, "lr": 1.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652512573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13859662413597107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652512574, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652512575, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652712534, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4998720586299896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652712535, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652712535, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652725388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13881732523441315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2457600, "lr": 1.2458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652738463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13890108466148376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652751533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13883653283119202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2785280, "lr": 1.4124999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652764206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389695554971695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4958333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652777191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13872775435447693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3112960, "lr": 1.5791666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652790547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13877524435520172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652803510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388952136039734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3440640, "lr": 1.7458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652816679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13868148624897003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652830072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13852308690547943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3768320, "lr": 1.9124999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652843267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138639897108078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9958333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652856649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13841751217842102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4096000, "lr": 2.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652870026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13843059539794922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.1624999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652883342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13851898908615112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4423680, "lr": 2.2458333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652896998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13871067762374878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3291666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652896999, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652896999, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652981062, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5001035928726196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652981064, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652981065, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652994443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13828761875629425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4751360, "lr": 2.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653007976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384304314851761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653021414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13788056373596191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5079040, "lr": 2.5791666666666668e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653034799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137782022356987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653047943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13823029398918152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5406720, "lr": 2.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653061246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13844412565231323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653074605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13772529363632202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5734400, "lr": 2.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653088303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13766610622406006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653101702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13740934431552887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6062080, "lr": 3.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653114893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382511705160141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653128233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13783077895641327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6389760, "lr": 3.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653141345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379188597202301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653155189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13796213269233704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6717440, "lr": 3.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653168672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379639208316803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653168672, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784653168673, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784653252286, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5004276633262634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784653252287, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784653252287, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784653265578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13708610832691193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7045120, "lr": 3.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653279184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13772368431091309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653292698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13697530329227448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7372800, "lr": 3.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653306612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13769014179706573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.829166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653320071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13768187165260315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7700480, "lr": 3.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653333823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13784800469875336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653347472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370435655117035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8028160, "lr": 4.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653361114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13765472173690796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653375078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371624618768692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8355840, "lr": 4.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653389039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387886106967926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3291666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653402942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365799903869629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8683520, "lr": 4.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653416595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13785314559936523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.4958333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653430396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13601510226726532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9011200, "lr": 4.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653444181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13698939979076385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.6624999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653444182, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653444182, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653526759, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5008711218833923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653526763, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653526763, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653540242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13654515147209167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9338880, "lr": 4.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653553829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361364722251892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653567202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382061243057251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9666560, "lr": 4.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653580886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13479453325271606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653594756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13694606721401215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9994240, "lr": 5.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653608398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13389073312282562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1624999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653621948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13586211204528809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10321920, "lr": 5.2458333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653635197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13551540672779083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653648748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13769683241844177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10649600, "lr": 5.4124999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653662141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13548630475997925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653675591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14583265781402588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10977280, "lr": 5.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653689357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13555815815925598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653702856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13438664376735687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11304960, "lr": 5.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653716252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378909796476364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653716253, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653716253, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653798585, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5014587044715881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653798586, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653798586, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653812049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13539038598537445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11632640, "lr": 5.9124999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653825475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13949958980083466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653838933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13617518544197083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11960320, "lr": 6.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653852350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336449533700943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653865867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13524192571640015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12288000, "lr": 6.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653879588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13918328285217285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.329166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653893244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13358157873153687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12615680, "lr": 6.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653906793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13335493206977844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653920395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13352203369140625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12943360, "lr": 6.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653933699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13162244856357574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653947182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13143546879291534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13271040, "lr": 6.745833333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653960415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13496491312980652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653973860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324758529663086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13598720, "lr": 6.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653987515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13876773416996002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653987515, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784653987516, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784654069614, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5021482706069946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784654069615, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784654069615, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784654083292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338728368282318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13926400, "lr": 7.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654096918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13358229398727417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654110529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13572999835014343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14254080, "lr": 7.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654124101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13515248894691467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654137627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13758859038352966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14581760, "lr": 7.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654151353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12829500436782837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654164727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13516052067279816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14909440, "lr": 7.579166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654178295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12951162457466125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654192069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13354657590389252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15237120, "lr": 7.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654206068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13258597254753113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.829166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654219844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13356329500675201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15564800, "lr": 7.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654233652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344124674797058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.995833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654247820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13544605672359467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15892480, "lr": 8.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654261696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13368918001651764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654261697, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654261697, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654343980, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5031433701515198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654343981, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654343982, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654358008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326826959848404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16220160, "lr": 8.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654371964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337706744670868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654386026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13556906580924988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16547840, "lr": 8.412500000000001e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654400442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13537903130054474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654414466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13947182893753052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16875520, "lr": 8.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654428330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13319000601768494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.662499999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654442830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13530756533145905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17203200, "lr": 8.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654456708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13362222909927368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654470665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13322514295578003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17530880, "lr": 8.912499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654484910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13196660578250885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654499043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13562069833278656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17858560, "lr": 9.079166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654513757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13366413116455078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.162499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654527965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13331632316112518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18186240, "lr": 9.245833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654542016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13060864806175232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654542017, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654542017, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654625455, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5043404698371887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654625456, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654625456, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654640190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13793738186359406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18513920, "lr": 9.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654654280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13466113805770874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.495833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654668420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13545167446136475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18841600, "lr": 9.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654682650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13116931915283203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.662499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654696554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13361607491970062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19169280, "lr": 9.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654710542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341937631368637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654724809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13681691884994507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19496960, "lr": 9.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654739073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13396196067333221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654753150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13276150822639465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19824640, "lr": 1.0079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654767423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297440528869629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0162499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654781389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1275618076324463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20152320, "lr": 1.0245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654795615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13289308547973633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654809822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12978452444076538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20480000, "lr": 1.0412499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654823648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12469957768917084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654823649, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654823649, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654904855, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5056301355361938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654904856, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654904856, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654919143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13114610314369202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20807680, "lr": 1.0579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654933449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13601070642471313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654947941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13317933678627014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21135360, "lr": 1.0745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654962423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13029444217681885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654976287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12840522825717926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21463040, "lr": 1.0912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654990086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12640337646007538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655004104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1290772706270218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21790720, "lr": 1.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655017709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13244619965553284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655031452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353566199541092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22118400, "lr": 1.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655045231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12494295090436935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655058706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12871040403842926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22446080, "lr": 1.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655072242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343991905450821, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655086133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13100531697273254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22773760, "lr": 1.1579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655100294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1291925311088562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655100295, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784655100296, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784655180511, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5072245597839355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784655180512, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784655180512, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784655194618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13053689897060394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23101440, "lr": 1.1745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655208371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12909294664859772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655221821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13314363360404968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23429120, "lr": 1.1912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655236057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12382461130619049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655250399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1293984055519104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23756800, "lr": 1.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655264023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12577994167804718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655277709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12998716533184052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24084480, "lr": 1.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655291619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1216912791132927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655305163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13576748967170715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24412160, "lr": 1.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655319258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13719062507152557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655333163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12360979616641998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24739840, "lr": 1.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655347648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1232110932469368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655361776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12561459839344025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25067520, "lr": 1.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655375832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12900619208812714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655375832, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655375833, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655458798, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5091200470924377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655458799, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655458799, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655472522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13635362684726715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25395200, "lr": 1.2912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655486617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13658486306667328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655500439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13583150506019592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25722880, "lr": 1.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655514244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14007160067558289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655528103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13661162555217743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26050560, "lr": 1.3245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655542016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13813841342926025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655555895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1276405155658722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26378240, "lr": 1.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655569676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12613284587860107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655583389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14223848283290863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26705920, "lr": 1.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655597614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1409912109375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655611831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12721505761146545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27033600, "lr": 1.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655625977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1282535344362259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655640096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13263185322284698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27361280, "lr": 1.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655653542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13145509362220764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655653542, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655653543, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655735727, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5111812353134155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655735728, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655735728, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655750028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1414056122303009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27688960, "lr": 1.4079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655764536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14250516891479492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655778475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13533349335193634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28016640, "lr": 1.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655792521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1393638402223587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655806904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14502838253974915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28344320, "lr": 1.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655820859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1409822404384613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655835126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12583588063716888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28672000, "lr": 1.4579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655849350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375603824853897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655863654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13189005851745605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28999680, "lr": 1.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655878144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.136182501912117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655892364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13779854774475098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29327360, "lr": 1.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655906370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13299599289894104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4995833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655920993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13453057408332825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29655040, "lr": 1.5079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655935547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13647028803825378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655935547, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784655935548, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784656017689, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5132744312286377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784656017690, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784656017691, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784656031737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13879700005054474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29982720, "lr": 1.5245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656046293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13811707496643066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656060977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13177116215229034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30310400, "lr": 1.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656075422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13288623094558716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656089867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13374565541744232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30638080, "lr": 1.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656104424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13986912369728088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656118837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13297416269779205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30965760, "lr": 1.5745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656133509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1415056735277176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656147564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1408229023218155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31293440, "lr": 1.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656161618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13499988615512848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656175540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12769140303134918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31621120, "lr": 1.6079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656189267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12980493903160095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656203150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13430774211883545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31948800, "lr": 1.6245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656217162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12759913504123688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656217164, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784656217164, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784656302018, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5159565210342407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784656302019, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784656302020, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784656315884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322515308856964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32276480, "lr": 1.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656329627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13692016899585724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656343381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13338574767112732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32604160, "lr": 1.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656356988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320623904466629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656370832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13404756784439087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32931840, "lr": 1.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656384597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332983374595642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656398863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336558312177658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33259520, "lr": 1.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656412905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13276593387126923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656426766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13970325887203217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33587200, "lr": 1.7079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656441206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13177499175071716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656455495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288599669933319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33914880, "lr": 1.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656469575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13129357993602753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656483624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13220496475696564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34242560, "lr": 1.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656497861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356482058763504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7495833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656497862, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656497862, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656581070, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5197709798812866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656581072, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656581072, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656595399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13370835781097412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34570240, "lr": 1.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656609498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12298467010259628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656623681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1305800825357437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34897920, "lr": 1.7745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656637899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13032317161560059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656651948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13517138361930847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35225600, "lr": 1.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656666484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13542111217975616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656680634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352979987859726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35553280, "lr": 1.8079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656695183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1239435002207756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8162500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656709521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13100023567676544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35880960, "lr": 1.8245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656723418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13112732768058777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656737662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13020358979701996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36208640, "lr": 1.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656751408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312837481498718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656765383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13088209927082062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36536320, "lr": 1.8579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656779139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301681250333786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656779140, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656779140, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656862712, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5247397422790527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656862713, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656862713, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656876575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13787689805030823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36864000, "lr": 1.8745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656890481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13223469257354736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656904359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13343797624111176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37191680, "lr": 1.8912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656918122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13021492958068848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656931929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12705978751182556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37519360, "lr": 1.9079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656945674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13190700113773346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656959860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13550794124603271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37847040, "lr": 1.9245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656973927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13281068205833435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656987961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13116483390331268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38174720, "lr": 1.9412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657002223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13888293504714966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657016392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13383561372756958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38502400, "lr": 1.9579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657029986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375105232000351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657044356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13177569210529327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38830080, "lr": 1.9745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657058561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13317392766475677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657058562, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784657058563, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784657142056, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5315152406692505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784657142057, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784657142058, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784657156074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13099122047424316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39157760, "lr": 1.9912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657170149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13501997292041779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657184217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13770999014377594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39485440, "lr": 2.0079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657198368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133268803358078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657212433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12726271152496338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39813120, "lr": 2.0245833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657226599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12911739945411682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0329166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657240789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12584739923477173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40140800, "lr": 2.0412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657254876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12393850088119507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657269273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13598975539207458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40468480, "lr": 2.0579166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657283754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13728754222393036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657298075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13450075685977936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40796160, "lr": 2.0745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657313053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12678854167461395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0829166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657327401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13088752329349518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41123840, "lr": 2.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657341262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12363197654485703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657341263, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657341263, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657428207, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5408981442451477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657428208, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657428208, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657442378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13241338729858398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41451520, "lr": 2.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657456294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13243310153484344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657470243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13360995054244995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41779200, "lr": 2.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657484184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376221477985382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657497978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13604146242141724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42106880, "lr": 2.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657511897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13117219507694244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657525687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12716004252433777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42434560, "lr": 2.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657539624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13502638041973114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657553343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1245623528957367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42762240, "lr": 2.1745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657567481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13018497824668884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657581514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13239744305610657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43089920, "lr": 2.1912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657595704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13374949991703033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657610020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1402326226234436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43417600, "lr": 2.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657624078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13596586883068085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657624079, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657624079, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657709917, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5549882650375366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657709918, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657709918, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657724581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13643577694892883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43745280, "lr": 2.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657738811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1314111202955246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657752525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13238941133022308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44072960, "lr": 2.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657766422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13885250687599182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657780972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12984861433506012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44400640, "lr": 2.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657795681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13822031021118164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657809992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13456641137599945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44728320, "lr": 2.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657824467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13689850270748138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657838816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13448373973369598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45056000, "lr": 2.2912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657853377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13039416074752808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657867842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14006178081035614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45383680, "lr": 2.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657882232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13444191217422485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657896665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1287049800157547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45711360, "lr": 2.3245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657911127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332336664199829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657911128, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657911128, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657998125, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.569856584072113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657998126, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657998126, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784658012809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1415831595659256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46039040, "lr": 2.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658027227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325182318687439, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658042237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13320700824260712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46366720, "lr": 2.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658056751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13046975433826447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658071705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13497895002365112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46694400, "lr": 2.3745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658086089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333393007516861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658100391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13359671831130981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47022080, "lr": 2.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658114871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353781372308731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658129050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13075894117355347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47349760, "lr": 2.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658143466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325116604566574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658157783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13284635543823242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47677440, "lr": 2.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658172090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13673491775989532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658186075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13594141602516174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48005120, "lr": 2.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658200334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13699877262115479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.4495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658200335, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784658200335, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784658284650, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5848451852798462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784658284651, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784658284651, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784658298837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13125668466091156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48332800, "lr": 2.4579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658313222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13494935631752014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658327359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13391411304473877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48660480, "lr": 2.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658342132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13171307742595673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658356325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13669037818908691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48988160, "lr": 2.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658370427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12918339669704437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658384767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13320118188858032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49315840, "lr": 2.507916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658398798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1233672946691513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.5162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658413030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13028642535209656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49643520, "lr": 2.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658427138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310386061668396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658441300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1306069791316986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49971200, "lr": 2.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658455285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1309659481048584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.5495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658469342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12847399711608887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50298880, "lr": 2.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658483525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13030056655406952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.5662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658483526, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658483526, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658567947, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5972340703010559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658567948, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658567948, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658581750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12143416702747345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50626560, "lr": 2.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658595819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1263360232114792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658609850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12825143337249756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50954240, "lr": 2.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658623894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13301631808280945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658638318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13115853071212769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51281920, "lr": 2.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658652346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12146598845720291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658666734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1296994835138321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51609600, "lr": 2.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658681078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13159050047397614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658695424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13636577129364014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51937280, "lr": 2.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658709255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1275504231452942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.6495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658723566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324080377817154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52264960, "lr": 2.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658737399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1173725351691246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.6662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658751353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12089701741933823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52592640, "lr": 2.6745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658765470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12487062066793442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658765470, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658765471, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658851128, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.608054518699646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658851129, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658851129, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658865030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12085188180208206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52920320, "lr": 2.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658878758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12698253989219666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.6995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658892656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1278155893087387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53248000, "lr": 2.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658906468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131222665309906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658920266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12093706429004669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53575680, "lr": 2.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658934160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13446144759655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658947919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12859424948692322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53903360, "lr": 2.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658962039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12093382328748703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.7495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658975970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12644872069358826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54231040, "lr": 2.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658990325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12646286189556122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659004424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11786336451768875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54558720, "lr": 2.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659019128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13051143288612366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659033500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12679621577262878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54886400, "lr": 2.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659047822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12829768657684326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659047823, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784659047823, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784659133403, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6174548268318176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784659133404, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784659133404, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784659147201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12666384875774384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55214080, "lr": 2.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659161577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11775150895118713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659175567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12130794674158096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55541760, "lr": 2.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659189678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12063918262720108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659203572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12339562177658081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55869440, "lr": 2.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659217413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12526148557662964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659231592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1158250942826271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56197120, "lr": 2.8579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659245527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12253908067941666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659259328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12420675158500671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56524800, "lr": 2.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659273087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12291256338357925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659286736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1147550493478775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56852480, "lr": 2.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659300550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12235027551651001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659314318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10889632254838943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57180160, "lr": 2.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659328274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12471466511487961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659328274, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659328275, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659413617, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6245580315589905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659413618, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659413618, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659427761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11599375307559967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57507840, "lr": 2.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659442128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11411761492490768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659456225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12897831201553345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57835520, "lr": 2.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659470860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12095210701227188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659485057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12257914990186691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58163200, "lr": 2.9579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659499298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11590501666069031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659513099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11904366314411163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58490880, "lr": 2.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659527539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12100479006767273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659541846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12025170773267746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58818560, "lr": 2.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659556198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11222049593925476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659570476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11943966150283813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59146240, "lr": 3.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659584539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12240869551897049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.0162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659598676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12408193200826645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59473920, "lr": 3.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659613008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11690074950456619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659613009, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659613010, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659696179, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6322518587112427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659696179, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659696180, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659710285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11744926869869232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59801600, "lr": 3.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659724433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1110648363828659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659738383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10878518223762512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60129280, "lr": 3.0579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659752432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11296313256025314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.0662499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659766495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10852327197790146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60456960, "lr": 3.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659780690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12239103019237518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659795108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11537310481071472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60784640, "lr": 3.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659809770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.119859978556633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659823954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10157602280378342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61112320, "lr": 3.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659838545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11923729628324509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.1162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659853015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11220603436231613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61440000, "lr": 3.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659867562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11294619739055634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659881844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10904913395643234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61767680, "lr": 3.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659896132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11613071709871292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659896133, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659896133, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659980859, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6410361528396606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659980860, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659980860, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659994894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12040553241968155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62095360, "lr": 3.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660009270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11967069655656815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.1662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660023863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12406820058822632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62423040, "lr": 3.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660038179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.104983851313591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660052678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11401994526386261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62750720, "lr": 3.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660066819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11802013963460922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660081353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12010172754526138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63078400, "lr": 3.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660095798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11922513693571091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660110427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10881360620260239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63406080, "lr": 3.2245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660124766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11775386333465576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660139041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11659163236618042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63733760, "lr": 3.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660153046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11694411188364029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660167249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11343822628259659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64061440, "lr": 3.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660181881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1262878179550171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660181881, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784660181881, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784660266373, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6506502032279968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784660266374, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784660266374, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784660280935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12453753501176834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64389120, "lr": 3.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660295466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11678850650787354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660309831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11688544601202011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64716800, "lr": 3.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660324680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1227162703871727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660339143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11071524769067764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65044480, "lr": 3.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660353609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11207699030637741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660367712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11274111270904541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65372160, "lr": 3.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660382264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10906227678060532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660396946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11164949089288712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65699840, "lr": 3.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660412060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11712441593408585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660426921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12462543696165085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66027520, "lr": 3.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660441593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1161370798945427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660456451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12771467864513397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66355200, "lr": 3.3745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660471228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11271289736032486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660471229, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660471229, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660556124, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.661865770816803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660556124, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660556125, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660571001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1277013123035431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66682880, "lr": 3.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660585615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11518143117427826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.3995833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660600196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11863718926906586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67010560, "lr": 3.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660615116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11397581547498703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660629840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12042784690856934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67338240, "lr": 3.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660644739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1197768896818161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660659825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12242291122674942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67665920, "lr": 3.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660674839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12173338979482651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660689626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11599969118833542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67993600, "lr": 3.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660705054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11445175111293793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660719779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11669301986694336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68321280, "lr": 3.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660734411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11892326176166534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660749007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1271360218524933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68648960, "lr": 3.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660763904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11695407330989838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660763905, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660763906, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660846825, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6749802827835083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660846825, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660846826, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660861494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12379258126020432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68976640, "lr": 3.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660876385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12043952196836472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660890982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11516378074884415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69304320, "lr": 3.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660905826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1167357787489891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660920343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11060626804828644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69632000, "lr": 3.5412500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660934834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11787156760692596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660949202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.119374580681324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69959680, "lr": 3.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660963360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11368997395038605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660977827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1102914810180664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70287360, "lr": 3.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660992165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10994520783424377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661006915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12365061044692993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70615040, "lr": 3.5912500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661021839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10823149979114532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661036511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12278957664966583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70942720, "lr": 3.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661051661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12069886177778244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661051661, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784661051662, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784661134243, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6894315481185913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784661134244, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784661134245, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784661149152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12059152871370316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71270400, "lr": 3.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661163653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12069932371377945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661178268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11657549440860748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71598080, "lr": 3.6412499999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661192796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11779584735631943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661207365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10447993129491806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71925760, "lr": 3.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661221984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072642058134079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661236371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12248382717370987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72253440, "lr": 3.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661250688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11542472243309021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661264999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10658194869756699, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72581120, "lr": 3.6912499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661279284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10506518930196762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661305471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10911008715629578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72908800, "lr": 3.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661319813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1128365620970726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661334019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11652246862649918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73236480, "lr": 3.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661348224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11848966777324677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661348277, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661348277, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661429961, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7075953483581543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661429964, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661429964, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661444263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10639489442110062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73564160, "lr": 3.7412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661458914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10989250242710114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661473759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11822960525751114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73891840, "lr": 3.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661488233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11330640316009521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661502788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1099964901804924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74219520, "lr": 3.7745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661517610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11919862031936646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661532044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11440346390008926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74547200, "lr": 3.7912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661546436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11641742289066315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661560784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11806654930114746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74874880, "lr": 3.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661575368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11414056271314621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661589662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1030544638633728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75202560, "lr": 3.8245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661603945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11615146696567535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661618024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10962222516536713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75530240, "lr": 3.8412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661632297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10910722613334656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661632298, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661632298, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661715524, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7282242178916931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661715524, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661715525, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661729648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11004821211099625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75857920, "lr": 3.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661744039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11595746129751205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661758315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09918224066495895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76185600, "lr": 3.8745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661772599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10516500473022461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661786734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11565835028886795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76513280, "lr": 3.8912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661800859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10774337500333786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661815133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09035241603851318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76840960, "lr": 3.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661829362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11113575100898743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661843735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10582055896520615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77168640, "lr": 3.9245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661858210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11801189929246902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661872853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10869535058736801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77496320, "lr": 3.9412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661887294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10169269144535065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661902106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1046934425830841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77824000, "lr": 3.9579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661916797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10909556597471237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661916798, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784661916798, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784662000893, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7468910217285156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784662000894, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784662000894, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784662015503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11823681741952896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78151680, "lr": 3.9745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662030091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11294310539960861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662044010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10593727231025696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78479360, "lr": 3.9912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662058571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11050170660018921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662072706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10144224762916565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78807040, "lr": 4.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662087061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10782667994499207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662101162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10992449522018433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79134720, "lr": 4.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662115449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11109036952257156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662129478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09937143325805664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79462400, "lr": 4.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662143863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1110287606716156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662158034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10643333941698074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79790080, "lr": 4.0579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662172168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11057796329259872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662186321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10893215984106064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80117760, "lr": 4.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662200538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.113469198346138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662200538, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784662200539, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784662284056, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7602785229682922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784662284057, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784662284057, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784662298377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1107206642627716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80445440, "lr": 4.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662312451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.100597083568573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662326870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1046474501490593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80773120, "lr": 4.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662341508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10768430680036545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662356427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11239002645015717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81100800, "lr": 4.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662371282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11347556114196777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662385968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10132960975170135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81428480, "lr": 4.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662400811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10560515522956848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662415831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10810061544179916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81756160, "lr": 4.1579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662430463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10657986253499985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662445044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11291825026273727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82083840, "lr": 4.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662459076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09498506039381027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662473786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09903867542743683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82411520, "lr": 4.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662488002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10800617933273315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662488003, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662488003, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662571478, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7612440586090088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662571478, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662571479, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662585633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09196124225854874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82739200, "lr": 4.2079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662600267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10468873381614685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662614695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10973555594682693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83066880, "lr": 4.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662629241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09902395308017731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662643621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11111479997634888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83394560, "lr": 4.2412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662658143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11112132668495178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662672588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09789379686117172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83722240, "lr": 4.2579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662686796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11108769476413727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662701160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10540840774774551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84049920, "lr": 4.2745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662715314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10519007593393326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662729707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10917702317237854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84377600, "lr": 4.2912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662743763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10479922592639923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662757820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09843868762254715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84705280, "lr": 4.3079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662772226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10631629079580307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662772227, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662772227, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662853049, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7605648636817932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662853050, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662853051, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662867809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10453695058822632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85032960, "lr": 4.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662882620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10458430647850037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662896986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10596440732479095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85360640, "lr": 4.3412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662911578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10423330962657928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662926230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09844663739204407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85688320, "lr": 4.3579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662940761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10006583482027054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662955529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10717222094535828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86016000, "lr": 4.3745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662969653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09656419605016708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662984381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10887797921895981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86343680, "lr": 4.3912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662999127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10935727506875992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663013835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11685173958539963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86671360, "lr": 4.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663028893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1146412119269371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663043613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1168329194188118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86999040, "lr": 4.4245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663058451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11541256308555603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663058451, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784663058452, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784663141028, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7628855109214783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784663141029, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784663141029, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784663155548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1006113663315773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87326720, "lr": 4.4412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663170229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1157359778881073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663184508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10813833773136139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87654400, "lr": 4.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663199209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10211723297834396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663213860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09118825197219849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87982080, "lr": 4.4745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663228701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09834829717874527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663243068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10449545085430145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88309760, "lr": 4.4912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663257750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10727156698703766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663272064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10826419293880463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88637440, "lr": 4.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663286408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10038845241069794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663301225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1130651980638504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88965120, "lr": 4.5245833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663316058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09949540346860886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663331027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1092979907989502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89292800, "lr": 4.5412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663345951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1117076426744461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.5495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663345952, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663345953, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663427504, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7627204060554504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663427505, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663427506, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663442252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10784751176834106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89620480, "lr": 4.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663456995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10173382610082626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663471768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11130397021770477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89948160, "lr": 4.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663486524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09191540628671646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663500943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1150563582777977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90275840, "lr": 4.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663515429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10799773037433624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663530577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11110162734985352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90603520, "lr": 4.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663545374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11129823327064514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663560526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1217532753944397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90931200, "lr": 4.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663575318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1054505705833435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663590107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11192307621240616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91258880, "lr": 4.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663605173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12343083322048187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663620203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09922470152378082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91586560, "lr": 4.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663635154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11093252897262573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663635155, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663635155, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663716701, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.763086199760437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663716702, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663716702, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663731563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10452700406312943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91914240, "lr": 4.6745833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663746453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11835993081331253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663761181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10943161696195602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92241920, "lr": 4.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663776389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10439400374889374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663791361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09734898805618286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92569600, "lr": 4.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663806379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11279719322919846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663821662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10614794492721558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92897280, "lr": 4.7245833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663836756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10656874626874924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663851948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11660734564065933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93224960, "lr": 4.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663866867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09899064153432846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663882185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12458959966897964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93552640, "lr": 4.7579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663897445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10739947110414505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663912397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11720562726259232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93880320, "lr": 4.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663927239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1076003909111023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663927240, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784663927240, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784664010079, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7634991407394409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784664010079, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784664010080, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784664025050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1138763427734375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94208000, "lr": 4.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664039847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11554230004549026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664054536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11065344512462616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94535680, "lr": 4.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664069496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11790971457958221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664084387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11494779586791992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94863360, "lr": 4.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664099322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11930850148200989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664114421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12722738087177277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95191040, "lr": 4.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664129072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12184260040521622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664143806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11621535569429398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95518720, "lr": 4.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664158866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11105328053236008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664173459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10838357359170914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95846400, "lr": 4.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664188250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10822886973619461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664202972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11494293808937073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96174080, "lr": 4.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664217947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11446838825941086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664217948, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664217948, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664301221, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7660683393478394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664301221, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664301222, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664316391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11976563930511475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96501760, "lr": 4.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664331510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723067820072174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664346257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10867631435394287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96829440, "lr": 4.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664361648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10806574672460556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664376706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11097189784049988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97157120, "lr": 4.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664391748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10920815169811249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664406638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11403042823076248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97484800, "lr": 4.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664421352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084703579545021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664436253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10602352768182755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97812480, "lr": 4.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664451029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10017441213130951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664466219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10438129305839539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98140160, "lr": 4.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664481088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09691792726516724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664495978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10474026203155518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98467840, "lr": 5.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664510857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10086853802204132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664510858, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664510858, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664595448, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.767128586769104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664595449, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664595449, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664610380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10951074957847595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98795520, "lr": 5.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664625501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10127587616443634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664640499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09667656570672989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99123200, "lr": 5.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664655262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10858919471502304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664670097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11527600139379501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99450880, "lr": 5.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664684895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10502174496650696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664699797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10541025549173355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99778560, "lr": 5.074583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664714683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1050359383225441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664729544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10217798501253128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100106240, "lr": 5.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664744499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12009777128696442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664759428, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1172926053404808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100433920, "lr": 5.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664774351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10162971913814545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664789608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10063369572162628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100761600, "lr": 5.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664804435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12065791338682175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664804435, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664804436, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664888577, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7679428458213806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664888578, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664888578, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664903808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11056744307279587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101089280, "lr": 5.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664918796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09686340391635895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664933799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10490384697914124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101416960, "lr": 5.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664948789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.103971466422081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664963390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10834019631147385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101744640, "lr": 5.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664978103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10589830577373505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.182916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664992729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10689693689346313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102072320, "lr": 5.191249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665007232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1122908815741539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665022053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10484613478183746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102400000, "lr": 5.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665036789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10754530131816864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665051363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10536693036556244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102727680, "lr": 5.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665066133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11505033075809479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665080499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10655016452074051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103055360, "lr": 5.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665095017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10245692729949951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665095018, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784665095018, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784665179318, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7687231302261353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784665179319, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784665179319, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784665193943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09991183131933212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103383040, "lr": 5.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665208782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09460242837667465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665223197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09629514068365097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103710720, "lr": 5.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665237828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09675031155347824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.282916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665252294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09738292545080185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104038400, "lr": 5.291249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665267048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09517312794923782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665281606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10025905072689056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104366080, "lr": 5.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665296320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09699917584657669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665311126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10336016118526459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104693760, "lr": 5.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665325828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10850762575864792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665340955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10484777390956879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105021440, "lr": 5.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665355814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10297292470932007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665370660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11428475379943848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105349120, "lr": 5.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665386049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11114716529846191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665386051, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665386051, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665469353, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7702964544296265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665469354, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665469354, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665484206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09832131862640381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105676800, "lr": 5.374583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665498766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10774210095405579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.382916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665513508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11011657118797302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106004480, "lr": 5.391249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665527938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10115676373243332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665542400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10100670158863068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106332160, "lr": 5.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665557233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10757340490818024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665571640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11105553805828094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106659840, "lr": 5.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665586270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10643874108791351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665601240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10284318774938583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106987520, "lr": 5.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665615735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10151012986898422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665630346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10002276301383972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107315200, "lr": 5.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665644911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10282547026872635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665659540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11003781855106354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107642880, "lr": 5.474583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665673917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11496444791555405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.482916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665673917, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665673918, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665757475, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7697357535362244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665757476, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665757476, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665772110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11008140444755554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107970560, "lr": 5.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665786642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10694951564073563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665801210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09141485393047333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108298240, "lr": 5.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665815684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10335971415042877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665830468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10670991986989975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108625920, "lr": 5.524583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665845061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1079997643828392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665859587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11039476841688156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108953600, "lr": 5.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665874368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10432478785514832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665889213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1001417338848114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109281280, "lr": 5.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665904151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10430619865655899, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665919115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10314653813838959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109608960, "lr": 5.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665933772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10839755833148956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665948709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10596249252557755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109936640, "lr": 5.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665963681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10868170112371445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665963682, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784665963682, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784666047362, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7708514928817749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784666047363, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784666047363, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784666062151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10298645496368408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110264320, "lr": 5.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666076869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10416081547737122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666091287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10455246269702911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110592000, "lr": 5.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666105420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10286732017993927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666120142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10737911611795425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110919680, "lr": 5.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666134467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10128167271614075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666148886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09885524958372116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111247360, "lr": 5.657916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666163497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10238613933324814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.666250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666177907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11231443285942078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111575040, "lr": 5.674583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666192458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10529325902462006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666206835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10377984493970871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111902720, "lr": 5.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666221046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10271511226892471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666235539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10448960214853287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112230400, "lr": 5.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666249819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10699984431266785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666249820, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666249820, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666335143, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7712752819061279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666335144, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666335144, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666349649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10846805572509766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112558080, "lr": 5.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666364336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10216074436903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666378674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10432850569486618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112885760, "lr": 5.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666393166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10138452053070068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666407712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10527149587869644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113213440, "lr": 5.757916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666421764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11265778541564941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.766250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666436058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10155604034662247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113541120, "lr": 5.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666450201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11043548583984375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666464510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11317460983991623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113868800, "lr": 5.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666479243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10630780458450317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666493592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10336211323738098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114196480, "lr": 5.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666507909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10465807467699051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666522631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1051015555858612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114524160, "lr": 5.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666537194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09949947893619537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666537195, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666537195, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666620421, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7719082236289978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666620421, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666620422, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666635137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1024491935968399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114851840, "lr": 5.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666649689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10005109757184982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666664202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1065589040517807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115179520, "lr": 5.857916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666678316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10089564323425293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666693248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10835602134466171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115507200, "lr": 5.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666707571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10961975902318954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666722270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10713335871696472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115834880, "lr": 5.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666737194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11553364992141724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666751799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11391910165548325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116162560, "lr": 5.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666766378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10102958232164383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666780822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10386902093887329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116490240, "lr": 5.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666795294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10323270410299301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666809798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10534048825502396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116817920, "lr": 5.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666824253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10750764608383179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666824254, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666824254, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666907270, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.77168208360672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666907270, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666907271, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666921809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10267499834299088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117145600, "lr": 5.957916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666936234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10076992213726044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666950947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10902368277311325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117473280, "lr": 5.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666965408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10553031414747238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666979772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09474509954452515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117800960, "lr": 5.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666994603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1131652370095253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667008878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10157237201929092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118128640, "lr": 6.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667023230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10791938006877899, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667037877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10328307747840881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118456320, "lr": 6.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667052897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1041509360074997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667067361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10973578691482544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118784000, "lr": 6.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667082111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11438381671905518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667096821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10665946453809738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119111680, "lr": 6.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667111461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10359269380569458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667111462, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784667111462, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784667194706, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.772057831287384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784667194707, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784667194707, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784667209835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11343999952077866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119439360, "lr": 6.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667224329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10933522880077362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667238756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10963496565818787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119767040, "lr": 6.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667253387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072702407836914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667267477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10115323215723038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120094720, "lr": 6.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667281554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10783040523529053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667296524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11803048104047775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120422400, "lr": 6.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667311298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10916643589735031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667326093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11696617305278778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120750080, "lr": 6.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667341187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1236400306224823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667356002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10803011059761047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121077760, "lr": 6.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667370809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11052369326353073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667385708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11536417156457901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121405440, "lr": 6.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667400330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11039696633815765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667400331, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667400331, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667486055, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7714381814002991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667486056, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667486056, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667500896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11371860653162003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121733120, "lr": 6.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667515954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10998880863189697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667530757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11151552200317383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122060800, "lr": 6.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667545499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10674116015434265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667560098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10392677038908005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122388480, "lr": 6.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667574809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11409730464220047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667589518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732229799032211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122716160, "lr": 6.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667604263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11502863466739655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667619161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11232750117778778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123043840, "lr": 6.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667634079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12063301354646683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667649224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10593245923519135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123371520, "lr": 6.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667663940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11757101118564606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667678858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10733812302350998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123699200, "lr": 6.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667693946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11935953795909882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667693947, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667693947, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667777577, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7721333503723145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667777578, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667777579, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667792713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1200442761182785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124026880, "lr": 6.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667807901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10831836611032486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667822770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11956508457660675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124354560, "lr": 6.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667837449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11393530666828156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667852145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1074434146285057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124682240, "lr": 6.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667867147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11531837284564972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667881673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11189184337854385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125009920, "lr": 6.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667896734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11706958711147308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667911717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11359743773937225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125337600, "lr": 6.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667926321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10592935234308243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667941077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11363410949707031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125665280, "lr": 6.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667955777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10577132552862167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667970324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11693986505270004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125992960, "lr": 6.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667985245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10549811273813248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667985246, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784667985246, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784668068849, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7729451656341553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784668068850, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784668068850, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784668083729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1105070412158966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126320640, "lr": 6.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668098830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12152128666639328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668113511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11071529984474182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126648320, "lr": 6.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668128199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11361563950777054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668142972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11609168350696564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126976000, "lr": 6.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668157789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1096084713935852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668172817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10830987244844437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127303680, "lr": 6.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668188017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1102236658334732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.482916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668203333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11244923621416092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127631360, "lr": 6.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668218372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11300552636384964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668233522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11047206819057465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127959040, "lr": 6.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668248831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11098240315914154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668263786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11213255673646927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128286720, "lr": 6.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668278629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10957322269678116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668278629, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668278630, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668361201, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7733933925628662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668361202, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668361202, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668375989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11830538511276245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128614400, "lr": 6.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668390623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11099719256162643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668405170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11089000105857849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128942080, "lr": 6.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668419856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10295223444700241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668434665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10620418936014175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129269760, "lr": 6.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668449612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.112242192029953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668464165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10866345465183258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129597440, "lr": 6.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668478819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10868684202432632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668493521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09863147884607315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129925120, "lr": 6.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668508237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10194140672683716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668522870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1057443618774414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130252800, "lr": 6.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668537068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10504019260406494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668551580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11001214385032654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130580480, "lr": 6.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668566233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11645406484603882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668566234, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668566235, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668649443, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7736559510231018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668649444, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668649444, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668663864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11493359506130219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130908160, "lr": 6.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668678448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10572365671396255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668693175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10075582563877106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131235840, "lr": 6.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668708013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09219758957624435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668722590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11382222920656204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131563520, "lr": 6.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668737127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10600171238183975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668752083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10292893648147583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131891200, "lr": 6.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668767353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11266669631004333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668782242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10740269720554352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132218880, "lr": 6.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668797411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10351830720901489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668812602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10961853712797165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132546560, "lr": 6.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668827811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11301130056381226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668842954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10769873112440109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132874240, "lr": 6.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668857885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10745930671691895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668857887, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668857887, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668940313, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773674726486206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668940314, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668940314, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668955080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11009936779737473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133201920, "lr": 6.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668970051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10356488078832626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668984750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11080040037631989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133529600, "lr": 6.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668999397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09968709200620651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669014195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11589410156011581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133857280, "lr": 6.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669029117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11024806648492813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669044005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10391764342784882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134184960, "lr": 6.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669059168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10815531015396118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669074176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10826365649700165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134512640, "lr": 6.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669088812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10877732932567596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669103742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1103912964463234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134840320, "lr": 6.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669118593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09951891750097275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669133376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10713057965040207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135168000, "lr": 6.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669148143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0875130146741867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669148144, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669148144, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669229830, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738063335418701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669229831, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669229831, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669244761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09587721526622772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135495680, "lr": 6.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669259644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11106683313846588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669274573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10858206450939178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135823360, "lr": 6.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669289552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10158545523881912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669304495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10237496346235275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136151040, "lr": 6.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669319492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09066607803106308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669334275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10335546731948853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136478720, "lr": 6.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669348801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09772074967622757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669363306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10412737727165222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136806400, "lr": 6.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669378168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10692570358514786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669392981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11075430363416672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137134080, "lr": 6.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669407805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10817232728004456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669422315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10640104115009308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137461760, "lr": 6.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669437026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10404159128665924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669437026, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669437027, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669520224, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7739784121513367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669520225, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669520225, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669535116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09678374230861664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137789440, "lr": 7.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669550221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10574622452259064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669565109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09523353725671768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138117120, "lr": 7.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669580024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10268575698137283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669594782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09807257354259491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138444800, "lr": 7.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669609182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10036353766918182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669623515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10939355194568634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138772480, "lr": 7.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669638382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10192069411277771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.066249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669652814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1090758666396141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139100160, "lr": 7.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669667326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1098741963505745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669682364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10716075450181961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139427840, "lr": 7.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669697325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10504095256328583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669712044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10994888097047806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139755520, "lr": 7.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669726869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10882140696048737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669726870, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669726870, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669807943, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774314284324646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669807944, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669807944, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669822481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10738036781549454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140083200, "lr": 7.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669837074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10478634387254715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669851669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10399822890758514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140410880, "lr": 7.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669866207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10063517838716507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669881009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10736851394176483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140738560, "lr": 7.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669895966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09601115435361862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669911297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10588589310646057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141066240, "lr": 7.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669925998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10411686450242996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669940805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11072895675897598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141393920, "lr": 7.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669955771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09908285737037659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669970381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09750610589981079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141721600, "lr": 7.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669985065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10283509641885757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669999901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09141449630260468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142049280, "lr": 7.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670014912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10786201059818268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670014913, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784670014914, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784670096085, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7745040655136108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784670096086, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784670096086, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784670110831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10015322268009186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142376960, "lr": 7.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670125797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10142719000577927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670140492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09894315898418427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142704640, "lr": 7.257916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670155577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11060698330402374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670170505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10175041854381561, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143032320, "lr": 7.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670185519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10281802713871002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670200226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10295681655406952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143360000, "lr": 7.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670214810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10584192723035812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670229721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10145244747400284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143687680, "lr": 7.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670244222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11464563757181168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670258986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10176695883274078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144015360, "lr": 7.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670273599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10877106338739395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670288115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10593461245298386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144343040, "lr": 7.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670302858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10686276108026505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670302859, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670302859, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670384653, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7745293974876404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670384654, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670384654, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670399689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10717935860157013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144670720, "lr": 7.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670414572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09847749024629593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670429060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10739550739526749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144998400, "lr": 7.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670443770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1074003353714943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670458445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10333920270204544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145326080, "lr": 7.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670472734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10737534612417221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670487151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0974242091178894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145653760, "lr": 7.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670501810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09844139218330383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670516283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10559909045696259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145981440, "lr": 7.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670530969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10488833487033844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670545788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10324802249670029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146309120, "lr": 7.441250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670560392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11007905751466751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670574890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10704905539751053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146636800, "lr": 7.457916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670589300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10029508173465729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670589301, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670589301, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670672591, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7747678756713867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670672592, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670672593, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670687056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1041945293545723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146964480, "lr": 7.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670701488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10551738739013672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670715968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09473259747028351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147292160, "lr": 7.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670731046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09924458712339401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670745739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10220786929130554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147619840, "lr": 7.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670760863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10060194879770279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670775510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10693423449993134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147947520, "lr": 7.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670790333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10980036854743958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670805290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085733100771904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148275200, "lr": 7.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670820211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1002650186419487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670835118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10282011330127716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148602880, "lr": 7.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670849540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09919870644807816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670864525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11207056790590286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148930560, "lr": 7.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670879004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1119183823466301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670879004, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670879005, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670963184, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7751191258430481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670963185, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670963185, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670977548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10581377148628235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149258240, "lr": 7.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670992656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10482316464185715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671007290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10614874958992004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149585920, "lr": 7.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671022032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11146502196788788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149749760, "lr": 7.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671037143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10672386735677719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149913600, "lr": 7.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671052126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10859683156013489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671066784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10840944200754166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150241280, "lr": 7.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671081661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09819262474775314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150405120, "lr": 7.649583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671096395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10740560293197632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150568960, "lr": 7.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671111230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10352522134780884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671126088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10844647884368896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150896640, "lr": 7.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671140587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10343514382839203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151060480, "lr": 7.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671155713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10693550109863281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151224320, "lr": 7.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671170585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09653159230947495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671170635, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671170635, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671252464, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7750065326690674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671252465, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671252465, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671267246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0994252935051918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151552000, "lr": 7.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671281912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10557056218385696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151715840, "lr": 7.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671296601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10242240875959396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151879680, "lr": 7.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671311329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10477522015571594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671326053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10821282863616943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152207360, "lr": 7.741249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671340593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12420887500047684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152371200, "lr": 7.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671355435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10921511054039001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152535040, "lr": 7.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671370275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09934014081954956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671385228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11308761686086655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152862720, "lr": 7.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671400116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1123194471001625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153026560, "lr": 7.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671415146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11516707390546799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153190400, "lr": 7.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671430036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09989200532436371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671445101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1066519021987915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153518080, "lr": 7.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671460036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10486659407615662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153681920, "lr": 7.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671460082, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671460083, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671541830, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7753995656967163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671541831, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671541831, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671556880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0993204116821289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153845760, "lr": 7.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671571771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09818881005048752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.832916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671586459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11227034032344818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154173440, "lr": 7.841249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671601733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11799252033233643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154337280, "lr": 7.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671616597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11420032382011414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154501120, "lr": 7.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671631120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11159062385559082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671646256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10756858438253403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154828800, "lr": 7.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671661072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12008000910282135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154992640, "lr": 7.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671675898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10579030215740204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155156480, "lr": 7.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671691140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10913442820310593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671706367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11261257529258728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155484160, "lr": 7.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671721234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11069950461387634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155648000, "lr": 7.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671736169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11349726468324661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155811840, "lr": 7.924583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671751055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11156562715768814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.932916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671751056, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671751056, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671833950, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7753637433052063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671833951, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671833951, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671848950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10716667771339417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156139520, "lr": 7.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671864000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11073228716850281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156303360, "lr": 7.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671879062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1148456409573555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156467200, "lr": 7.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671894292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11047735810279846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671909465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10892483592033386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156794880, "lr": 7.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671924706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10822834819555283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156958720, "lr": 7.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671939720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1018039807677269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157122560, "lr": 7.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671954758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11251349747180939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671969873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12091082334518433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157450240, "lr": 8.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671985076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11008087545633316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157614080, "lr": 8.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672000043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10885624587535858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157777920, "lr": 8.024583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672015493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10511958599090576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.032916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672031053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10421549528837204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158105600, "lr": 8.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672046422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10956082493066788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158269440, "lr": 8.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672046422, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784672046423, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784672128528, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.775506317615509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784672128529, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784672128529, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784672143780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11439725011587143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158433280, "lr": 8.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672158981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11915957182645798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672174249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11443984508514404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158760960, "lr": 8.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672189683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11292374134063721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158924800, "lr": 8.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672205208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11109115928411484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159088640, "lr": 8.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672220307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12158956378698349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672235691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11175887286663055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159416320, "lr": 8.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672250454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10940655320882797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159580160, "lr": 8.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672265698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11068594455718994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159744000, "lr": 8.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672280830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09890744090080261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672295901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10982246696949005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160071680, "lr": 8.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672311513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10951825976371765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160235520, "lr": 8.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672326880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10470032691955566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160399360, "lr": 8.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672341981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11091388016939163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672341981, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672341982, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672424804, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7757564783096313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672424805, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672424805, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672440172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11297285556793213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160727040, "lr": 8.174583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672455483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12945306301116943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160890880, "lr": 8.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672470752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10840769112110138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161054720, "lr": 8.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672485807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11266057193279266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672500954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11371589452028275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161382400, "lr": 8.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672516458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11205576360225677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161546240, "lr": 8.216250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672531877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11415136605501175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161710080, "lr": 8.224583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672547391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11339927464723587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672562617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10605619102716446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162037760, "lr": 8.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672577658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10849667340517044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162201600, "lr": 8.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672593057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12911207973957062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162365440, "lr": 8.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672608157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11374388635158539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672623822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11327707022428513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162693120, "lr": 8.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672639473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11600472033023834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162856960, "lr": 8.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672639473, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672639474, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672723184, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7760140895843506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672723186, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672723186, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672738245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09635359048843384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163020800, "lr": 8.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672753816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11157336086034775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672769391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12232904136180878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163348480, "lr": 8.307916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672785001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1067429855465889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163512320, "lr": 8.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672800791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11006677150726318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163676160, "lr": 8.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672816305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10743748396635056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672831643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11229138821363449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164003840, "lr": 8.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672846676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11223474144935608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164167680, "lr": 8.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672861677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10350295901298523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164331520, "lr": 8.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672876516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11771882325410843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672891262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10753951966762543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164659200, "lr": 8.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672906309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11173088103532791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164823040, "lr": 8.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672921457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09264107048511505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164986880, "lr": 8.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672936195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10345718264579773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672936196, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784672936196, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784673018527, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.776203453540802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784673018528, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784673018528, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784673033735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10143986344337463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165314560, "lr": 8.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673048595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09696684777736664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165478400, "lr": 8.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673063362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11533608287572861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165642240, "lr": 8.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673078034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10791213810443878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673092745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10937260091304779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165969920, "lr": 8.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673108087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10499086230993271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166133760, "lr": 8.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673123381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10833192616701126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166297600, "lr": 8.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673138509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1137584000825882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673153638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10037632286548615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166625280, "lr": 8.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673168914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0969451367855072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166789120, "lr": 8.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673183954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0971379354596138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166952960, "lr": 8.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673199039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09738202393054962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673214000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10274812579154968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167280640, "lr": 8.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673229063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10090919584035873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167444480, "lr": 8.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673229064, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673229064, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673312729, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7761097550392151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673312730, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673312730, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673327818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10336118936538696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167608320, "lr": 8.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673343056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10399220138788223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673358255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10434095561504364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167936000, "lr": 8.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673373504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1075262501835823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168099840, "lr": 8.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673388719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10345729440450668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168263680, "lr": 8.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673404126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10016651451587677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673419680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10999985784292221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168591360, "lr": 8.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673435091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10569489747285843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168755200, "lr": 8.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673450165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10712305456399918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168919040, "lr": 8.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673464919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10603657364845276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673479988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10469590127468109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169246720, "lr": 8.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673494794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.109395831823349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169410560, "lr": 8.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673509360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10583586990833282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169574400, "lr": 8.624583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673524117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11416136473417282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673524166, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673524166, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673606867, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7763838768005371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673606868, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673606868, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673621478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10888488590717316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169902080, "lr": 8.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673636253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11170461773872375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170065920, "lr": 8.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673651211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10738073289394379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170229760, "lr": 8.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673666398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10724639892578125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673681213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10068723559379578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170557440, "lr": 8.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673696351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10255015641450882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170721280, "lr": 8.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673711311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09616556763648987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170885120, "lr": 8.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673725935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10837341099977493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673740562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0993192121386528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171212800, "lr": 8.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673755335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10250266641378403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171376640, "lr": 8.716249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673770403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10166031122207642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171540480, "lr": 8.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673785240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10365241020917892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673800007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09995357692241669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171868160, "lr": 8.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673814930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10457131266593933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172032000, "lr": 8.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673814973, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673814974, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673900255, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7764325737953186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673900256, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673900256, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673915105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09813448786735535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172195840, "lr": 8.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673929914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10750970989465714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673944797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09980528056621552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172523520, "lr": 8.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673959598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10390094667673111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172687360, "lr": 8.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673974135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09407398104667664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172851200, "lr": 8.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673989170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10746144503355026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674004306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10403516888618469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173178880, "lr": 8.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674019159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10419614613056183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173342720, "lr": 8.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674034270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10542979091405869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173506560, "lr": 8.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674049269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11257931590080261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674064435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09714223444461823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173834240, "lr": 8.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674079522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11144015938043594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173998080, "lr": 8.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674094437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11197132617235184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174161920, "lr": 8.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674108964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10182566195726395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674109014, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784674109014, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784674193190, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.776740312576294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784674193191, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784674193191, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784674208171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09828910231590271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174489600, "lr": 8.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674223408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10749667137861252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174653440, "lr": 8.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674238412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10190513730049133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174817280, "lr": 8.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674252914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10657233744859695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674267855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10086538642644882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175144960, "lr": 8.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674282726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085042804479599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175308800, "lr": 8.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674297616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10043999552726746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175472640, "lr": 8.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674312604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09899303317070007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674327468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10918073356151581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175800320, "lr": 8.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674342366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1094970703125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175964160, "lr": 8.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674357056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09990138560533524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176128000, "lr": 8.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674371749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09499051421880722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674386176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09631041437387466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176455680, "lr": 8.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674400569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10669966787099838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176619520, "lr": 8.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674400570, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674400570, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674486472, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7768571376800537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674486473, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674486473, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674500993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10330145806074142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176783360, "lr": 8.991249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674515408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11377554386854172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674529952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10219596326351166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177111040, "lr": 9.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674544526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10655273497104645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177274880, "lr": 9.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674559046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11158392578363419, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177438720, "lr": 9.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674573584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10075905919075012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.032916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674588004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10963762551546097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177766400, "lr": 9.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674602358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10202150791883469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177930240, "lr": 9.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674616797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10708962380886078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178094080, "lr": 9.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674631154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10221695899963379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674645611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09832601249217987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178421760, "lr": 9.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674660437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1124105155467987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178585600, "lr": 9.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674675334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11955831199884415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178749440, "lr": 9.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674690252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10939908772706985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674690253, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674690253, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674776288, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7771997451782227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674776289, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674776290, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674791237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11638860404491425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179077120, "lr": 9.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674805917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10144786536693573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179240960, "lr": 9.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674821023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10027829557657242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179404800, "lr": 9.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674836022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10109374672174454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.132916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674850946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10635987669229507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179732480, "lr": 9.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674865921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09999415278434753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179896320, "lr": 9.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674880760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10092324018478394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180060160, "lr": 9.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674895842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10630727559328079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674910396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10620313882827759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180387840, "lr": 9.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674924824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09481685608625412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180551680, "lr": 9.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674940019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10474253445863724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180715520, "lr": 9.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674954710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10214683413505554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674969485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09388476610183716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181043200, "lr": 9.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674984488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10031144320964813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181207040, "lr": 9.216249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674984489, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784674984489, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784675070329, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7771161198616028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784675070330, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784675070331, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784675085313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11124923825263977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181370880, "lr": 9.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675100163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11351790279150009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675114884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09995310008525848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181698560, "lr": 9.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675129597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725530236959457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181862400, "lr": 9.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675144378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10351251810789108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182026240, "lr": 9.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675159094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1035999283194542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675173656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10653866827487946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182353920, "lr": 9.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675188187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084151566028595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182517760, "lr": 9.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675203004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09903000295162201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182681600, "lr": 9.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675217617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09779775142669678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675232390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11176199465990067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183009280, "lr": 9.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675246944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09769050031900406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183173120, "lr": 9.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675262050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10211338847875595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183336960, "lr": 9.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675276595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10295592248439789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675276596, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675276596, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675363505, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.777419924736023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675363506, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675363506, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675378302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10242239385843277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183664640, "lr": 9.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675392947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10963595658540726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183828480, "lr": 9.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675407443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11253992468118668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183992320, "lr": 9.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675422589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09453985840082169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675437691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1047186404466629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184320000, "lr": 9.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675452680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10741548985242844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184483840, "lr": 9.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675467529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11701241135597229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184647680, "lr": 9.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675482066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10100889950990677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675496926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10576742142438889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184975360, "lr": 9.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675512055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11159158498048782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185139200, "lr": 9.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675526863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09871482849121094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185303040, "lr": 9.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675541502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10921669751405716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675556158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09907015413045883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185630720, "lr": 9.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675570426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10262705385684967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185794560, "lr": 9.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675570427, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675570427, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675655569, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7775924801826477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675655570, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675655570, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675670712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11045119166374207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185958400, "lr": 9.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675685378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10467255860567093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675699789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09209749102592468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186286080, "lr": 9.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675714661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1004200354218483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186449920, "lr": 9.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675729412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10950727015733719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186613760, "lr": 9.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675744335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11099233478307724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675765505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0967506691813469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186941440, "lr": 9.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675780514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10520383715629578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187105280, "lr": 9.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675796209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10633336752653122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187269120, "lr": 9.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675810826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09994109719991684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675825367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10475575178861618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187596800, "lr": 9.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675839967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10496099293231964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187760640, "lr": 9.549583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675854438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11148913949728012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187924480, "lr": 9.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675869387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11567454785108566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675869388, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675869389, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675954093, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7774476408958435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675954094, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675954094, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675968636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10968538373708725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188252160, "lr": 9.574583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675983548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1047464981675148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188416000, "lr": 9.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675998123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1044725850224495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188579840, "lr": 9.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676013039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10888775438070297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676027738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1028684675693512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188907520, "lr": 9.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676042689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09767556190490723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189071360, "lr": 9.616249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676057506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1038031280040741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189235200, "lr": 9.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676071984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11282400786876678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676086524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10106676071882248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189562880, "lr": 9.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676101362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09612448513507843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189726720, "lr": 9.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676116038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10489282011985779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189890560, "lr": 9.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676131245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11236950010061264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.666249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676146386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1023569107055664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190218240, "lr": 9.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676161282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11849050223827362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190382080, "lr": 9.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676161283, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676161283, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676245652, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7780337333679199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676245654, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676245654, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676260628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10138971358537674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190545920, "lr": 9.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676275304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09788148105144501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676290031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1055399477481842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190873600, "lr": 9.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676305069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11238902062177658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191037440, "lr": 9.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676320134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11890029162168503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191201280, "lr": 9.724583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676334980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09579908102750778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676349947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10558819025754929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191528960, "lr": 9.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676364538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10171858221292496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191692800, "lr": 9.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676379585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10181591659784317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191856640, "lr": 9.757916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676394359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10445522516965866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676408718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10642440617084503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192184320, "lr": 9.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676423979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1096419095993042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192348160, "lr": 9.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676439007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10024363547563553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192512000, "lr": 9.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676454273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11606930196285248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.799583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676454274, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676454274, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676538987, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7778628468513489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676538987, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676538988, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676554383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1121923178434372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192839680, "lr": 9.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676569731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11199335008859634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193003520, "lr": 9.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676584817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723741352558136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193167360, "lr": 9.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676600099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12139029800891876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676615195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11620738357305527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193495040, "lr": 9.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676630500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10266728699207306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193658880, "lr": 9.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676645809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11942452937364578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193822720, "lr": 9.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676660780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11451617628335953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.866249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676675980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10586967319250107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194150400, "lr": 9.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676691140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081598699092865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194314240, "lr": 9.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676706425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10741422325372696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194478080, "lr": 9.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676721348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10626745223999023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676736629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10753922909498215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194805760, "lr": 9.907916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676751478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11551567167043686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194969600, "lr": 9.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676751479, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676751479, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676835328, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7784557342529297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676835329, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676835329, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676850754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11212132126092911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195133440, "lr": 9.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676865723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10507866740226746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676880832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10282167792320251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195461120, "lr": 9.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676896260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11594627052545547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195624960, "lr": 9.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676911709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10904514789581299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195788800, "lr": 9.957916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676927045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10630740225315094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676942527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10629831254482269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196116480, "lr": 9.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676957789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11414361000061035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196280320, "lr": 9.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676973544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11805244535207748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196444160, "lr": 9.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676989148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10621708631515503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.999583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677004819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11746064573526382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196771840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677020333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11244921386241913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196935680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677035352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10862593352794647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197099520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677050941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11541257798671722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677050942, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784677050942, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784677133366, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7784021496772766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784677133367, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784677133367, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784677148435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10111035406589508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197427200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677163661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11125673353672028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197591040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677178916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11498051136732101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197754880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677193849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10147064179182053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677209101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09993498772382736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198082560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677224117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10857061296701431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198246400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677239124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09875458478927612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198410240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677254038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10686930269002914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677269088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10235090553760529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198737920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677283889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10568232834339142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198901760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677298787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11520799249410629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199065600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677313653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11106706410646439, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677328456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11755956709384918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199393280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677343539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11391210556030273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199557120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677343540, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677343541, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677427384, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7786070704460144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677427385, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677427385, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677442351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11307752877473831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199720960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677457353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10813210904598236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677472174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1112925335764885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200048640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677487184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11713911592960358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200212480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677502323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11181667447090149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200376320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677517274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11749300360679626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677532497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09966171532869339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200704000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677547515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10760229080915451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200867840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677562699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10994407534599304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201031680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677577900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10983673483133316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677593370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11971066147089005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201359360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677608867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11277391016483307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201523200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677624252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10532436519861221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201687040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677639804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11201044172048569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677639805, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677639805, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677725013, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7788670659065247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677725015, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677725015, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677739835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10562842339277267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202014720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677754949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11115812510251999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202178560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677769744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1188526600599289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202342400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677785032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10673261433839798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677799975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10627102851867676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202670080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677815077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10203764587640762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202833920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677830765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10687017440795898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202997760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677845765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11071613430976868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677860914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11429107189178467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203325440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677876064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09468884766101837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203489280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677891355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11704372614622116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203653120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677906707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10069263726472855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677921601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09925527125597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203980800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677936705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10839702188968658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204144640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677936705, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784677936706, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784678022469, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7787046432495117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784678022470, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784678022470, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784678037615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10333960503339767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204308480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678052956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09368181228637695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678068430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09722970426082611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204636160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678083854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10860507935285568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204800000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678099147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10895679146051407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204963840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678114643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11341352760791779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678129875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09628899395465851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205291520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678145439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10689026862382889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205455360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678160920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10741821676492691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205619200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678176340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11094111204147339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678191712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09897607564926147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205946880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678207435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10004715621471405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206110720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678222894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11142079532146454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206274560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678238185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09862854331731796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678238186, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678238187, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678321359, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7788223624229431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678321360, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678321360, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678337281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10628765821456909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206602240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678352604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09633145481348038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206766080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678367948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10520705580711365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206929920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678383429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09679976105690002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678399125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10313232988119125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207257600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678414237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11199750751256943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207421440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678429587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772772878408432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207585280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678444356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0961853414773941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678459240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11545320600271225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207912960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678474182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10429949313402176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208076800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678488748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10385017096996307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208240640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678503862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10170061886310577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678518686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10279906541109085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208568320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678533689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10160376131534576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208732160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678533690, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678533690, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678615662, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7791153788566589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678615663, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678615663, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678630974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10202021896839142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208896000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678646064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09969518333673477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678660980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10269606113433838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209223680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678675774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10273982584476471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209387520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678690874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09874886274337769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209551360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678705805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10159865021705627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678720555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0966375395655632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209879040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678735398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09886843711137772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210042880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678750497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10385508835315704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210206720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678765463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10976136475801468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678780383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10474221408367157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210534400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678795108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10786890238523483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210698240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678810148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10179410129785538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210862080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678824851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09627371281385422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678824852, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678824852, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678908316, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7791991829872131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678908317, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678908317, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678923293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0946439579129219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211189760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678938161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0992782711982727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211353600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678952741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09746194630861282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211517440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678967450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09969807416200638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678982237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10385003685951233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211845120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678997627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0959002748131752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212008960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679012516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10347907245159149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212172800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679027683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09826178848743439, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679042541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.098362997174263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212500480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679057534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10540945082902908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212664320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679072557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09291791170835495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212828160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679087823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10787548869848251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679103128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10546589642763138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213155840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679118394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1113038882613182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213319680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679118394, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784679118395, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784679202321, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7795173525810242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784679202321, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784679202322, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784679217374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10495059192180634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213483520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679232396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10256247222423553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679247245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10648944973945618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213811200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679262597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10348733514547348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213975040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679277210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09680640697479248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214138880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679291872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1053454577922821, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679307047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09527099877595901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214466560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679321788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1144523099064827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214630400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679336578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09859102964401245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214794240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679351844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09043776988983154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679367002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09951069205999374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215121920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679382064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10545547306537628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215285760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679396980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10298407822847366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215449600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679411817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10852208733558655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679411817, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679411818, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679494400, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7793664336204529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679494401, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679494401, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679509414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10099931061267853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215777280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679524253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10603466629981995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215941120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679539058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09758089482784271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216104960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679554023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10033950954675674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679569085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10060872882604599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216432640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679583934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0955408439040184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216596480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679599022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1038282960653305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216760320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679614295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10283731669187546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679629194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10169358551502228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217088000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679644564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10904139280319214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217251840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679659400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09817789494991302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217415680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679674600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10061819851398468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217579520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679689704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1000683382153511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217743360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679704763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10563643276691437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217907200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679704764, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679704764, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679787778, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7796338796615601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679787779, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679787779, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679802913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10074558109045029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218071040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679818083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09511298686265945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218234880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679833246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09845756739377975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218398720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679848295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10808788239955902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218562560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679863632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10309040546417236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218726400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679878871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08788947761058807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218890240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679894272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10490954667329788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219054080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679909790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09209781885147095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219217920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679925155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10377095639705658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219381760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679940455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09581772238016129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219545600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679955570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09623252600431442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219709440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679970626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10290946811437607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219873280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679985953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11184053868055344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220037120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680001065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0997270867228508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220200960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680001065, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784680001065, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784680085086, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.78006911277771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784680085087, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784680085087, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784680099633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09594202041625977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220364800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680114755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09558748453855515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220528640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680129852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10225993394851685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220692480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680144911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10294283181428909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220856320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680160264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10478690266609192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221020160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680175662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723139345645905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221184000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680191059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10782039910554886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221347840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680206126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11585014313459396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221511680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680221119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10916212201118469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221675520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680236503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09968430548906326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221839360, "lr": 1e-06}} diff --git a/recommendation_v4/rcp_logs/gbs_8192/seed404908753.log b/recommendation_v4/rcp_logs/gbs_8192/seed404908753.log new file mode 100644 index 000000000..8c8be0daa --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_8192/seed404908753.log @@ -0,0 +1,1855 @@ +:::MLLOG {"namespace": "", "time_ms": 1784658279987, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784658279987, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784658301684, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784658301686, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784658301687, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784658301687, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784658301687, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784658301687, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 8192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784658301687, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784658301687, "event_type": "POINT_IN_TIME", "key": "seed", "value": 404908753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784658301687, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784658301687, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784658301687, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784658301687, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784658301699, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784658301700, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784659224563, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784659224565, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784659225038, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784659354829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13872191309928894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840, "lr": 7.916666666666666e-10}} +:::MLLOG {"namespace": "", "time_ms": 1784659367028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13792134821414948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.6249999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659379683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13875937461853027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 491520, "lr": 2.458333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659392477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384267359972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659405350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385139673948288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 819200, "lr": 4.125e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659418144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13882684707641602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659431116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13831759989261627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1146880, "lr": 5.7916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659444268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13911612331867218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.625e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659457610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13870635628700256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1474560, "lr": 7.458333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659470720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383618265390396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.291666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659484016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13852068781852722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1802240, "lr": 9.124999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659497269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385715752840042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659510833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383802890777588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2129920, "lr": 1.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659524112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13994906842708588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659524113, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784659524114, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784659606840, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.503982663154602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784659606841, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784659606841, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784659619630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382921189069748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2457600, "lr": 1.2458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659632626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13838481903076172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659646245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13840237259864807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2785280, "lr": 1.4124999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659659368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13794779777526855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4958333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659672718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13780073821544647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3112960, "lr": 1.5791666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659686430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13791024684906006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659699412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379682868719101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3440640, "lr": 1.7458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659712788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13778206706047058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659726840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13777776062488556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3768320, "lr": 1.9124999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659740133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13791829347610474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9958333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659754061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13809239864349365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4096000, "lr": 2.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659768031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13848333060741425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.1624999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659781598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13775137066841125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4423680, "lr": 2.2458333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659795715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13803647458553314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3291666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659795715, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784659795716, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784659877587, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5042181611061096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784659877588, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784659877589, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784659891139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13736827671527863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4751360, "lr": 2.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659904818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382444053888321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659918455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371767371892929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5079040, "lr": 2.5791666666666668e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659932043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359771192073822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659945122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13779909908771515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5406720, "lr": 2.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659958443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378798484802246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659971809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13691961765289307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5734400, "lr": 2.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659985562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13674500584602356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659999000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13632827997207642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6062080, "lr": 3.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660012326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1372164636850357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660025894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13685451447963715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6389760, "lr": 3.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660039130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373167335987091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660053119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13741059601306915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6717440, "lr": 3.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660066387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1372901201248169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660066388, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784660066388, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784660148225, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5045158863067627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784660148226, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784660148226, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784660161358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13629257678985596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7045120, "lr": 3.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660174832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13732671737670898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660188244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13624390959739685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7372800, "lr": 3.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660202147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13675348460674286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.829166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660215610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13719294965267181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7700480, "lr": 3.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660229308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13746456801891327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660242969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13610924780368805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8028160, "lr": 4.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660256549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13707108795642853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660270673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13642749190330505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8355840, "lr": 4.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660284448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13876603543758392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3291666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660298188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13588422536849976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8683520, "lr": 4.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660311734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13750389218330383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.4958333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660325433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347351372241974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9011200, "lr": 4.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660339174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.136424258351326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.6624999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660339175, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784660339176, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784660422681, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5049307942390442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784660422682, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784660422682, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784660435908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13586808741092682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9338880, "lr": 4.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660449236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350637525320053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660462356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13754047453403473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9666560, "lr": 4.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660475789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13410231471061707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660489482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13640016317367554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9994240, "lr": 5.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660502877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327681541442871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1624999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660516517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13511745631694794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10321920, "lr": 5.2458333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660529912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13510721921920776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660543371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13780146837234497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10649600, "lr": 5.4124999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660556717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13485507667064667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660570178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1490192711353302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10977280, "lr": 5.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660584002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13505882024765015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660597447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13324595987796783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11304960, "lr": 5.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660611013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13755583763122559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660611014, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784660611014, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784660693797, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5054449439048767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784660693798, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784660693799, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784660707043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348438560962677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11632640, "lr": 5.9124999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660720362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14014002680778503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660733760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13590778410434723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11960320, "lr": 6.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660747086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13260875642299652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660760600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344234198331833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12288000, "lr": 6.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660774333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1391633301973343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.329166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660787891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13289353251457214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12615680, "lr": 6.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660801523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320856511592865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660814966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13258136808872223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12943360, "lr": 6.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660828088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13072340190410614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660841413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303355097770691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13271040, "lr": 6.745833333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660854330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13465899229049683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660867578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1313757598400116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13598720, "lr": 6.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660881144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13896510004997253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660881144, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784660881145, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784660962857, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5061421990394592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784660962857, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784660962858, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784660976189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133186936378479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13926400, "lr": 7.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660989535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13293100893497467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661003220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13560815155506134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14254080, "lr": 7.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661016630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13501405715942383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661030059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378655731678009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14581760, "lr": 7.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661043529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12655267119407654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661056640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13533490896224976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14909440, "lr": 7.579166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661070046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12842705845832825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661083659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13303227722644806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15237120, "lr": 7.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661097481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13235242664813995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.829166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661111107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331627070903778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15564800, "lr": 7.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661124866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13422371447086334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.995833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661138823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13531669974327087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15892480, "lr": 8.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661152502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13315647840499878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661152503, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784661152503, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784661234541, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5071825385093689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784661234542, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784661234542, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784661248424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13240845501422882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16220160, "lr": 8.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661262171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13348960876464844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661276148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355404108762741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16547840, "lr": 8.412500000000001e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661290430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13555969297885895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661304433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1399197280406952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16875520, "lr": 8.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661318239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13283292949199677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.662499999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661332906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13511496782302856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17203200, "lr": 8.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661346922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333910971879959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661360817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325853168964386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17530880, "lr": 8.912499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661374800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315787136554718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661388617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13568434119224548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17858560, "lr": 9.079166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661402918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13354401290416718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.162499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661416853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13333576917648315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18186240, "lr": 9.245833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661430659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13023127615451813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661430659, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784661430660, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784661512478, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5083341002464294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784661512479, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784661512480, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784661527218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378193199634552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18513920, "lr": 9.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661541078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13470424711704254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.495833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661554964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353430449962616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18841600, "lr": 9.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661569016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.130997434258461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.662499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661582842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13365639746189117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19169280, "lr": 9.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661596807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13409505784511566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661610913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370925009250641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19496960, "lr": 9.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661625011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13387656211853027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661638897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13277670741081238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19824640, "lr": 1.0079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661652849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1296180784702301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0162499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661666360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1272665113210678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20152320, "lr": 1.0245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661680147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13297715783119202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661693967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12946730852127075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20480000, "lr": 1.0412499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661707367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12439506500959396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661707367, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784661707368, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784661787800, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5096920728683472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784661787801, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784661787801, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784661801640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310318559408188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20807680, "lr": 1.0579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661815463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13599659502506256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661829665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13309383392333984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21135360, "lr": 1.0745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661843889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13019941747188568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661857637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12811873853206635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21463040, "lr": 1.0912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661871803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12618333101272583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661885896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12885881960391998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21790720, "lr": 1.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661899370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13224554061889648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661913076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13553747534751892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22118400, "lr": 1.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661926850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12470822781324387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661940370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12859947979450226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22446080, "lr": 1.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661953997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13424187898635864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661967718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310272067785263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22773760, "lr": 1.1579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661981778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12911245226860046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661981779, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784661981779, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784662063197, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5114042162895203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784662063198, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784662063198, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784662077345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13032324612140656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23101440, "lr": 1.1745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662091315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12877674400806427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662104822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13288480043411255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23429120, "lr": 1.1912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662119180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12364430725574493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662133362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12946265935897827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23756800, "lr": 1.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662146748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12581516802310944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662160281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12978492677211761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24084480, "lr": 1.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662174039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12164569646120071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662187398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357671171426773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24412160, "lr": 1.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662201302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13727900385856628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662214958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12345343083143234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24739840, "lr": 1.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662229069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12299516797065735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662243008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1256469190120697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25067520, "lr": 1.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662256841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12885668873786926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662256842, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784662256842, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784662340423, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5133918523788452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784662340424, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784662340424, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784662353794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366121917963028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25395200, "lr": 1.2912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662368280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13650792837142944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662381965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13576924800872803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25722880, "lr": 1.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662395570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13994424045085907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662409253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13654452562332153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26050560, "lr": 1.3245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662423067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13825061917304993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662436676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1273626834154129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26378240, "lr": 1.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662450479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1258450448513031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662464163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.142131507396698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26705920, "lr": 1.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662478465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14091677963733673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662492762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12718595564365387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27033600, "lr": 1.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662506986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12852437794208527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662521216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13229210674762726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27361280, "lr": 1.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662534748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13174015283584595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662534749, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784662534749, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784662618868, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5155647993087769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784662618869, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784662618869, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784662633313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14127767086029053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27688960, "lr": 1.4079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662647500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14246965944766998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662661447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13533765077590942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28016640, "lr": 1.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662675438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13944897055625916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662689761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14498162269592285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28344320, "lr": 1.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662703610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14095790684223175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662717755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12575286626815796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28672000, "lr": 1.4579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662731947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13750717043876648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662746214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13188594579696655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28999680, "lr": 1.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662760688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13604722917079926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662774999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13776856660842896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29327360, "lr": 1.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662788886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1328531801700592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4995833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662803215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13423846662044525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29655040, "lr": 1.5079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662817497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13629576563835144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662817498, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784662817498, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784662901396, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5179113745689392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784662901396, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784662901397, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784662915317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13856752216815948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29982720, "lr": 1.5245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662929631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13798515498638153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662943926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13171716034412384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30310400, "lr": 1.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662958144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13295279443264008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662972367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13361647725105286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30638080, "lr": 1.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662986883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13964518904685974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663001280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13277076184749603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30965760, "lr": 1.5745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663015755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14118692278862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663029586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14059560000896454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31293440, "lr": 1.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663044030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350722312927246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663058136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1275438815355301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31621120, "lr": 1.6079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663072037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12975084781646729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663086056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341870129108429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31948800, "lr": 1.6245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663100179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12750360369682312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663100180, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784663100180, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784663182747, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5210840106010437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784663182748, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784663182748, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784663196721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320725828409195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32276480, "lr": 1.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663210652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13678203523159027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663224823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13346800208091736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32604160, "lr": 1.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663238911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13190612196922302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663253206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1340359002351761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32931840, "lr": 1.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663267500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13323649764060974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663282214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13350513577461243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33259520, "lr": 1.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663296696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13271337747573853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663310714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13994616270065308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33587200, "lr": 1.7079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663325208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1316613256931305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663339063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12874913215637207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33914880, "lr": 1.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663352720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13096857070922852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663366372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13218221068382263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34242560, "lr": 1.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663380130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13575440645217896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7495833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663380130, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784663380131, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784663462282, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5254508256912231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784663462283, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784663462283, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784663476112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13317549228668213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34570240, "lr": 1.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663489777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1228833794593811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663503416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13032668828964233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34897920, "lr": 1.7745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663517105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13052722811698914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663530780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349017471075058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35225600, "lr": 1.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663545007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13502797484397888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663558730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13534682989120483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35553280, "lr": 1.8079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663573016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12382978945970535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8162500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663587056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1309158205986023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35880960, "lr": 1.8245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663600730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13092853128910065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663615283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1299973577260971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36208640, "lr": 1.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663629234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13292357325553894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663643317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13056698441505432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36536320, "lr": 1.8579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663657150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12983548641204834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663657150, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784663657151, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784663739615, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5319961309432983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784663739616, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784663739616, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784663753550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376085877418518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36864000, "lr": 1.8745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663767621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13208214938640594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663781535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332598775625229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37191680, "lr": 1.8912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663795500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1302037239074707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663809314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12660741806030273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37519360, "lr": 1.9079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663823202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323586404323578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663837564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13539256155490875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37847040, "lr": 1.9245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663851908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325187236070633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663866210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13103589415550232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38174720, "lr": 1.9412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663880614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13852651417255402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663894844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13342168927192688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38502400, "lr": 1.9579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663908458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373627483844757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663922820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315673142671585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38830080, "lr": 1.9745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663936613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13287213444709778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663936614, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784663936614, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784664018157, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5409263968467712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784664018158, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784664018158, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784664032131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13102388381958008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39157760, "lr": 1.9912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664046109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344655305147171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664060187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137370303273201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39485440, "lr": 2.0079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664074326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13273020088672638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664088267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12687426805496216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39813120, "lr": 2.0245833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664102211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12884823977947235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0329166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664116135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12574587762355804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40140800, "lr": 2.0412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664130025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1234988421201706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664144317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13580213487148285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40468480, "lr": 2.0579166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664158769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13684715330600739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664173153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13422074913978577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40796160, "lr": 2.0745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664187888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12637455761432648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0829166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664202311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13069403171539307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41123840, "lr": 2.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664216138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12331299483776093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664216138, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784664216139, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784664300757, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5538215637207031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784664300758, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784664300758, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784664315494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13168612122535706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41451520, "lr": 2.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664329680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320989429950714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664343846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332508623600006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41779200, "lr": 2.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664357903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13704612851142883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664372043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13539302349090576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42106880, "lr": 2.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664386143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1307644546031952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664400180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12650282680988312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42434560, "lr": 2.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664414271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13415507972240448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664428163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12374832481145859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42762240, "lr": 2.1745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664442593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12948307394981384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664456784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13155274093151093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43089920, "lr": 2.1912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664471135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330888718366623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664485423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1394653469324112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43417600, "lr": 2.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664499497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13541299104690552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664499498, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784664499499, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784664583689, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5712177753448486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784664583690, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784664583690, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784664598508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13575008511543274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43745280, "lr": 2.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664613015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13046343624591827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664627119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13152122497558594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44072960, "lr": 2.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664641356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13788340985774994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664656472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12900832295417786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44400640, "lr": 2.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664671482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374169886112213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664685816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13404397666454315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44728320, "lr": 2.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664700284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13581062853336334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664714763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13335882127285004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45056000, "lr": 2.2912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664729357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12970055639743805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664743872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13915173709392548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45383680, "lr": 2.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664758390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335444152355194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664772752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12725117802619934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45711360, "lr": 2.3245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664787355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329387128353119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664787355, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784664787356, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784664872485, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5897519588470459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784664872486, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784664872486, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784664887237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14040637016296387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46039040, "lr": 2.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664901761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312401443719864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664916736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323026865720749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46366720, "lr": 2.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664931240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12861275672912598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664946115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13421005010604858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46694400, "lr": 2.3745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664960984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317714899778366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664975572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13227106630802155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47022080, "lr": 2.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664990202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341848522424698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665004811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12895804643630981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47349760, "lr": 2.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665019694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13040904700756073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665034473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1313854604959488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47677440, "lr": 2.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665049219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13536468148231506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665063563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347709447145462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48005120, "lr": 2.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665078297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354997754096985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.4495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665078298, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784665078298, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784665163277, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6072400212287903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784665163277, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784665163278, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784665177920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12952592968940735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48332800, "lr": 2.4579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665192956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13367144763469696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665207663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323917806148529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48660480, "lr": 2.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665222663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13017809391021729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665237157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13515880703926086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48988160, "lr": 2.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665251658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1270749568939209, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665266594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1314704567193985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49315840, "lr": 2.507916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665280666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1199415996670723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.5162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665294933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1276252567768097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49643520, "lr": 2.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665309176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12809023261070251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665323526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12765783071517944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49971200, "lr": 2.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665337488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12760548293590546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.5495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665351748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12584078311920166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50298880, "lr": 2.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665366074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12716840207576752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.5662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665366075, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784665366075, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784665451183, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6233605146408081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784665451184, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784665451184, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784665465271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11839921027421951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50626560, "lr": 2.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665479619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12305781990289688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665493819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1244233176112175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50954240, "lr": 2.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665508200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12999087572097778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665522989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12812231481075287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51281920, "lr": 2.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665537488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11725426465272903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665552158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12521764636039734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51609600, "lr": 2.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665566619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12825195491313934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665581071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323302537202835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51937280, "lr": 2.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665595001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12409287691116333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.6495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665610123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12804998457431793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52264960, "lr": 2.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665624457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11414527148008347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.6662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665638723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11548877507448196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52592640, "lr": 2.6745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665653052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12060507386922836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665653052, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784665653053, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784665738467, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6491081714630127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784665738468, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784665738469, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784665752719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1168200746178627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52920320, "lr": 2.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665766854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12250177562236786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.6995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665781111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12271711230278015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53248000, "lr": 2.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665795067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1256849467754364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665808996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11604680866003036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53575680, "lr": 2.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665823008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12928512692451477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665836927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12366705387830734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53903360, "lr": 2.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665850805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11614133417606354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.7495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665864980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12201929092407227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54231040, "lr": 2.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665879377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12083500623703003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665893600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11247116327285767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54558720, "lr": 2.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665908376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12671560049057007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665922706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12224479764699936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54886400, "lr": 2.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665936916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12508846819400787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665936916, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784665936917, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784666021769, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6842385530471802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784666021770, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784666021770, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784666035647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12053387612104416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55214080, "lr": 2.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666050382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11358692497015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666064750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11689387261867523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55541760, "lr": 2.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666079077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11578401178121567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666093227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11970062553882599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55869440, "lr": 2.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666107412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12093625962734222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666121471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10989678651094437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56197120, "lr": 2.8579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666135710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11909569799900055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666149663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11946966499090195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56524800, "lr": 2.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666163654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11800671368837357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666177348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10977841168642044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56852480, "lr": 2.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666191334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11814434826374054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666205407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10302707552909851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57180160, "lr": 2.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666219547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11861368268728256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666219547, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784666219548, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784666304332, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7208166718482971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784666304333, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784666304333, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784666318696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11100917309522629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57507840, "lr": 2.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666332902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10756219923496246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666346973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11980361491441727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57835520, "lr": 2.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666361485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11679232865571976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666375667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11854974925518036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58163200, "lr": 2.9579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666389888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1110692024230957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666403623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11283967643976212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58490880, "lr": 2.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666418371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11912018805742264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666432492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11497130990028381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58818560, "lr": 2.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666446584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10526479035615921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666460622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11557655036449432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59146240, "lr": 3.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666474520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11714500188827515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.0162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666488603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12218542397022247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59473920, "lr": 3.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666502880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11346852779388428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666502881, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784666502881, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784666588566, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.74155193567276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784666588567, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784666588567, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784666602960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11406029760837555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59801600, "lr": 3.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666616883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10634948313236237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666630744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10585951060056686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60129280, "lr": 3.0579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666644715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1100049540400505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.0662499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666658887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1044350117444992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60456960, "lr": 3.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666672789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11781815439462662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666687313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11155009269714355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60784640, "lr": 3.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666701931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11629744619131088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666716160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09566514939069748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61112320, "lr": 3.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666730729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11769804358482361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.1162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666745211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11183296144008636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61440000, "lr": 3.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666759400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10898750275373459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666773466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10433172434568405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61767680, "lr": 3.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666788058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11265647411346436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666788058, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784666788059, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784666874305, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7526703476905823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784666874306, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784666874306, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784666888328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11324287205934525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62095360, "lr": 3.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666902679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1155402660369873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.1662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666917137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11850082129240036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62423040, "lr": 3.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666931530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10098177194595337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666945921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11109302192926407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62750720, "lr": 3.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666960075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11528945714235306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666974592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11628304421901703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63078400, "lr": 3.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666988959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1134243905544281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667003068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10433486849069595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63406080, "lr": 3.2245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667017235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11484570056200027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667031494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10989979654550552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63733760, "lr": 3.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667045608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11406011879444122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667059651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10796429216861725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64061440, "lr": 3.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667074114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1204761490225792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667074114, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784667074115, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784667158506, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7559629082679749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784667158507, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784667158507, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784667172964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11889028549194336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64389120, "lr": 3.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667187319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11496753245592117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667201527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11200999468564987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64716800, "lr": 3.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667216332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11546646058559418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667230756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10171603411436081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65044480, "lr": 3.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667245313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072123646736145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667259292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10937671363353729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65372160, "lr": 3.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667273627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10633508116006851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667287913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10901352018117905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65699840, "lr": 3.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667302715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11421369016170502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667317207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12021950632333755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66027520, "lr": 3.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667331544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11199455708265305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667346082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12371375411748886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66355200, "lr": 3.3745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667360686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10956120491027832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667360686, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784667360687, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784667442881, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7618590593338013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784667442882, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784667442882, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784667457525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12292811274528503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66682880, "lr": 3.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667471809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11058665812015533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.3995833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667486082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11266599595546722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67010560, "lr": 3.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667500738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11300022155046463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667514975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11406271904706955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67338240, "lr": 3.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667529432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11380574852228165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667544153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11998632550239563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67665920, "lr": 3.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667558970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1191389411687851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667573520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11396753787994385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67993600, "lr": 3.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667588579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11048728972673416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667603215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1131676658987999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68321280, "lr": 3.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667617736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10975128412246704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667632189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1215638816356659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68648960, "lr": 3.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667647259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11207817494869232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667647260, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784667647261, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784667729013, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7633564472198486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784667729014, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784667729015, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784667743652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11696646362543106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68976640, "lr": 3.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667758588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11605062335729599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667773470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111247718334198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69304320, "lr": 3.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667788514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11232543736696243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667803284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10656879097223282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69632000, "lr": 3.5412500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667817970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11256082355976105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667832813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11518719792366028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69959680, "lr": 3.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667847434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082940474152565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667862422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087239533662796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70287360, "lr": 3.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667877189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10754744708538055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667892082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12198149412870407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70615040, "lr": 3.5912500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667907086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10446488857269287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667921903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1186998263001442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70942720, "lr": 3.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667937141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11882232129573822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667937141, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784667937141, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784668018745, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.764214813709259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784668018746, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784668018746, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784668033670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11701370030641556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71270400, "lr": 3.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668048206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11714618653059006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668062801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11424907296895981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71598080, "lr": 3.6412499999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668077782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11223562061786652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668092332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10078959912061691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71925760, "lr": 3.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668106902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10057037323713303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668121346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11676743626594543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72253440, "lr": 3.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668135803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11583195626735687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668150248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10320685803890228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72581120, "lr": 3.6912499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668164708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10149935632944107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668178981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10546234250068665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72908800, "lr": 3.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668193508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11216865479946136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668207858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10759148001670837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73236480, "lr": 3.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668222174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11365044116973877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668222217, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784668222217, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784668306152, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7647486925125122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784668306153, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784668306153, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784668320705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10403795540332794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73564160, "lr": 3.7412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668335450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10626691579818726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668350114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11355811357498169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73891840, "lr": 3.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668364478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10913395136594772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668379105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10639641433954239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74219520, "lr": 3.7745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668393999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11350499838590622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668408507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11267802864313126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74547200, "lr": 3.7912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668422862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1108456701040268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668437636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1156584694981575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74874880, "lr": 3.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668451983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11065227538347244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668466376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09967102855443954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75202560, "lr": 3.8245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668480863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1125451847910881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668495055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10569482296705246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75530240, "lr": 3.8412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668509472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10479975491762161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668509473, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784668509473, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784668593951, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7662606835365295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784668593953, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784668593953, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784668608181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10510695725679398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75857920, "lr": 3.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668622579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11323992162942886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668636880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09616216272115707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76185600, "lr": 3.8745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668651041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1041678711771965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668665386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11195676773786545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76513280, "lr": 3.8912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668679527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10503135621547699, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668693908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08913657069206238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76840960, "lr": 3.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668708211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10839783400297165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668722385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10079697519540787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77168640, "lr": 3.9245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668736828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11783495545387268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668751437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10744921118021011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77496320, "lr": 3.9412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668765723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10115483403205872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668780643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09986159950494766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77824000, "lr": 3.9579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668795476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1074487715959549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668795477, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784668795477, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784668881352, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7676895260810852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784668881353, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784668881353, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784668896050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1144772619009018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78151680, "lr": 3.9745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668910701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11283175647258759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668924848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10545306652784348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78479360, "lr": 3.9912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668939764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10531309247016907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668953842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09990929067134857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78807040, "lr": 4.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668968267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10092645138502121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668982467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10679590702056885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79134720, "lr": 4.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668996895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10783237963914871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669011113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09695351868867874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79462400, "lr": 4.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669025497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1094551607966423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669039648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10472668707370758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79790080, "lr": 4.0579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669053785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10669580101966858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669067830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10858882963657379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80117760, "lr": 4.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669082095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11300211399793625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669082096, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784669082096, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784669168330, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7692927718162537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784669168331, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784669168331, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784669182613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1098208874464035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80445440, "lr": 4.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669196716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09873262792825699, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669211212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10293906927108765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80773120, "lr": 4.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669225760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10530391335487366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669240374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11073584109544754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81100800, "lr": 4.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669255113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1124073714017868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669269727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10041084885597229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81428480, "lr": 4.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669284529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10538040846586227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669299303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10733634233474731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81756160, "lr": 4.1579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669313607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10510848462581635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669328204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11311589181423187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82083840, "lr": 4.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669342186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09500420093536377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669357151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09775573760271072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82411520, "lr": 4.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669371616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10776984691619873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669371617, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784669371617, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784669457594, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7707017660140991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784669457595, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784669457595, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784669471938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09186312556266785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82739200, "lr": 4.2079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669486680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10443904250860214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669501253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11019538342952728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83066880, "lr": 4.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669515996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09800581634044647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669530471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11097890883684158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83394560, "lr": 4.2412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669545301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1118568405508995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669560078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09766083210706711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83722240, "lr": 4.2579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669574579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11107911169528961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669589135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10522343218326569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84049920, "lr": 4.2745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669603472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10411130636930466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669618079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10698511451482773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84377600, "lr": 4.2912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669632335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10551409423351288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669646641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09689651429653168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84705280, "lr": 4.3079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669661388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10615560412406921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669661388, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784669661389, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784669745903, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.771099328994751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784669745904, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784669745904, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784669760734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10409845411777496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85032960, "lr": 4.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669775613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.104132279753685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669790107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10582209378480911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85360640, "lr": 4.3412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669804822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10431825369596481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669819705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09889298677444458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85688320, "lr": 4.3579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669834198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10033106803894043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669849101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1049722209572792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86016000, "lr": 4.3745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669863215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09736523777246475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669878003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10846318304538727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86343680, "lr": 4.3912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669892391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10900215804576874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669906653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11696688830852509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86671360, "lr": 4.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669921174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11539065092802048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669935486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11696126312017441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86999040, "lr": 4.4245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669949916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11514466255903244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669949917, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784669949917, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784670033930, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7716795206069946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784670033931, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784670033932, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784670048144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10127688944339752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87326720, "lr": 4.4412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670062577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11594422161579132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670076892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10662350803613663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87654400, "lr": 4.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670091606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10214688628911972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670106042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.092311792075634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87982080, "lr": 4.4745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670120665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09846533834934235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670135012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10318780690431595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88309760, "lr": 4.4912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670149559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10779423266649246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670163795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10802856087684631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88637440, "lr": 4.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670178110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10126125812530518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670192869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11318206042051315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88965120, "lr": 4.5245833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670207483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10046210139989853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670222361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11032683402299881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89292800, "lr": 4.5412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670236956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11154815554618835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.5495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670236956, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784670236956, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784670319362, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7719902992248535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784670319363, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784670319363, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784670333966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10903002321720123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89620480, "lr": 4.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670348567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10300253331661224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670363003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11063723266124725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89948160, "lr": 4.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670377729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0918099656701088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670392104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11498288810253143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90275840, "lr": 4.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670406524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10747072845697403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670421939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1105593740940094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90603520, "lr": 4.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670436621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11195390671491623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670451727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12240701913833618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90931200, "lr": 4.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670466522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10596273094415665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670481371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11076009273529053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91258880, "lr": 4.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670496513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12202608585357666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670511376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10082592070102692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91586560, "lr": 4.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670526196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11140176653862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670526197, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784670526197, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784670608592, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7726359367370605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784670608593, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784670608593, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784670623321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10519932955503464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91914240, "lr": 4.6745833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670638129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11828391253948212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670652801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10905776917934418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92241920, "lr": 4.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670667939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10528876632452011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670682611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09640546888113022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92569600, "lr": 4.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670697379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11342189460992813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670712405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10650155693292618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92897280, "lr": 4.7245833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670727327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10626474767923355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670742411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11573249101638794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93224960, "lr": 4.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670757167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1004483699798584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670772321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1242324560880661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93552640, "lr": 4.7579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670787340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10704244673252106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670802302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11420438438653946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93880320, "lr": 4.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670817128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10780567675828934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670817129, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784670817129, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784670899731, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730265259742737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784670899732, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784670899733, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784670914803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1147032156586647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94208000, "lr": 4.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670929494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11529593914747238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670944106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11121894419193268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94535680, "lr": 4.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670958990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1177494153380394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670973699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11464901268482208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94863360, "lr": 4.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670988330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11878790706396103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671003112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12691837549209595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95191040, "lr": 4.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671017529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12089917063713074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671032017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11711347848176956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95518720, "lr": 4.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671046773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11077819764614105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671061193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10843328386545181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95846400, "lr": 4.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671075834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085876002907753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671090348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11518041044473648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96174080, "lr": 4.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671105094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1150398924946785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671105095, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784671105095, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784671186170, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7731436491012573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784671186171, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784671186171, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784671201123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11910033226013184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96501760, "lr": 4.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671215916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10780452191829681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671230379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10835769772529602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96829440, "lr": 4.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671245598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10913559794425964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671260448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11074277758598328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97157120, "lr": 4.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671275312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11060437560081482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671290045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1138368472456932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97484800, "lr": 4.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671304712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10840590298175812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671319516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10622086375951767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97812480, "lr": 4.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671334085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10069446265697479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671349142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.104729063808918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98140160, "lr": 4.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671363813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.096420057117939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671378377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10439933836460114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98467840, "lr": 5.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671393000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1012272834777832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671393000, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784671393001, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784671474654, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7729725241661072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784671474655, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784671474655, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784671489237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11082924902439117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98795520, "lr": 5.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671503825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1019272729754448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671518267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09736814349889755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99123200, "lr": 5.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671532637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1088876873254776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671547083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1152411699295044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99450880, "lr": 5.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671561414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1044747456908226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671575763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1040901467204094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99778560, "lr": 5.074583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671590201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10477704554796219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671604571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10234639793634415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100106240, "lr": 5.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671619231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12101320177316666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671633819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1175750121474266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100433920, "lr": 5.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671648398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10168971121311188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671663100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10066631436347961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100761600, "lr": 5.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671677352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12050683796405792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671677352, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784671677353, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784671760769, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773280143737793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784671760770, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784671760770, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784671775702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1113334372639656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101089280, "lr": 5.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671790494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0964871495962143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671805123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10537530481815338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101416960, "lr": 5.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671819802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10400043427944183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671834113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10799982398748398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101744640, "lr": 5.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671848778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10477922111749649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.182916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671863256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1059030070900917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102072320, "lr": 5.191249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671877630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11229583621025085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671892384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10490991175174713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102400000, "lr": 5.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671907033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10892148315906525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671921517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10554353892803192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102727680, "lr": 5.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671936020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11503773927688599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671950261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732710361480713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103055360, "lr": 5.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671964644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10254962742328644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671964645, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784671964645, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784672047181, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7732754349708557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784672047182, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784672047182, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784672061586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10010216385126114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103383040, "lr": 5.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672076142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0948290005326271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672090502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0967169776558876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103710720, "lr": 5.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672104941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09718084335327148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.282916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672119084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09712252765893936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104038400, "lr": 5.291249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672133468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09551500529050827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672147613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10062872618436813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104366080, "lr": 5.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672161826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09790609031915665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672176301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10345061868429184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104693760, "lr": 5.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672190848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10832735896110535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672205594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10574306547641754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105021440, "lr": 5.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672220009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10353170335292816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672234435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11516150832176208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105349120, "lr": 5.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672249463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11072667688131332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672249464, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784672249464, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784672331904, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773652970790863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784672331905, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784672331906, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784672346546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09886396676301956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105676800, "lr": 5.374583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672360942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725046694278717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.382916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672375677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1101347953081131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106004480, "lr": 5.391249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672390063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10226182639598846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672404509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09999532252550125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106332160, "lr": 5.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672419385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10865913331508636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672433855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11005833745002747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106659840, "lr": 5.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672448331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10660487413406372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672463068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10288863629102707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106987520, "lr": 5.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672477503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1014714315533638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672491979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10017254948616028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107315200, "lr": 5.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672506432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10285357385873795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672520863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1095564141869545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107642880, "lr": 5.474583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672535137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.115663081407547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.482916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672535137, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784672535138, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784672615832, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7739480137825012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784672615833, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784672615833, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784672630326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11010105907917023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107970560, "lr": 5.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672644852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10696738958358765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672659331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09171672910451889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108298240, "lr": 5.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672673899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10208073258399963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672688462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10678946226835251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108625920, "lr": 5.524583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672702803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084529235959053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672717077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11055485159158707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108953600, "lr": 5.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672731588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10232897102832794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672746117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10115064680576324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109281280, "lr": 5.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672760642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10447774082422256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672775355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10269561409950256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109608960, "lr": 5.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672789790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10757773369550705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672804631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1050054058432579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109936640, "lr": 5.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672819517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10888843983411789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672819518, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784672819518, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784672901964, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7739936113357544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784672901966, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784672901966, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784672916599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10367866605520248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110264320, "lr": 5.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672931116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10276588052511215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672945404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10417564958333969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110592000, "lr": 5.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672959568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10281248390674591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672974783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10750377178192139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110919680, "lr": 5.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672989246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10143844783306122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673003821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09785077720880508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111247360, "lr": 5.657916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673018743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10212890803813934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.666250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673033488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11179200559854507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111575040, "lr": 5.674583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673048207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10526502132415771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673062797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10433641821146011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111902720, "lr": 5.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673077249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1028667539358139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673091864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10413458943367004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112230400, "lr": 5.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673106307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10694941133260727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673106308, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784673106309, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784673190007, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7741411328315735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784673190008, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784673190008, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784673204656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10857594758272171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112558080, "lr": 5.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673219432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1018170714378357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673233769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10401420295238495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112885760, "lr": 5.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673248346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10121431946754456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673263053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1058625802397728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113213440, "lr": 5.757916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673277165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11315269768238068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.766250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673291721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10196031630039215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113541120, "lr": 5.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673306042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11094789952039719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673320752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11294246464967728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113868800, "lr": 5.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673335698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10709269344806671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673350233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10321040451526642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114196480, "lr": 5.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673364805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10577686876058578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673379528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10598510503768921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114524160, "lr": 5.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673394239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0985577404499054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673394240, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784673394241, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784673476355, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7740490436553955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784673476356, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784673476356, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784673491082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10137558728456497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114851840, "lr": 5.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673505809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09927301108837128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673520474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10603984445333481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115179520, "lr": 5.857916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673534553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10083628445863724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673549566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10831145197153091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115507200, "lr": 5.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673564091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10920422524213791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673578867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10770034044981003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115834880, "lr": 5.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673593974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1157749593257904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673608656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11374292522668839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116162560, "lr": 5.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673623336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10055181384086609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673637905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10389499366283417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116490240, "lr": 5.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673652524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10303709656000137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673667087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10565704852342606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116817920, "lr": 5.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673681397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1067570149898529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673681398, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784673681398, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784673765686, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742127180099487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784673765687, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784673765687, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784673780187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10171112418174744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117145600, "lr": 5.957916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673794694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10099811851978302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673809324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10924949496984482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117473280, "lr": 5.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673823782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1044216975569725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673838104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09444723278284073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117800960, "lr": 5.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673852904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11245757341384888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673867173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10170836001634598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118128640, "lr": 6.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673881467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10738861560821533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673896046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10336083918809891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118456320, "lr": 6.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673910959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10364249348640442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673925523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10973650962114334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118784000, "lr": 6.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673940284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11417762190103531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673954984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10568512976169586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119111680, "lr": 6.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673969736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1030217707157135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673969736, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784673969737, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784674056479, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7744092345237732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784674056480, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784674056480, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784674071487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11325210332870483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119439360, "lr": 6.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674086207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10900008678436279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674100750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1098882257938385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119767040, "lr": 6.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674115497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10765545070171356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674129731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.101941779255867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120094720, "lr": 6.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674143928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080082356929779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674159517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11847283691167831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120422400, "lr": 6.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674174701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10969369113445282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674189757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11565899103879929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120750080, "lr": 6.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674205089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1216958537697792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674220035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10720092803239822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121077760, "lr": 6.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674235117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11003612726926804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674250227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11537057906389236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121405440, "lr": 6.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674265300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11062610149383545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674265301, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784674265301, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784674350417, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7744579911231995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784674350418, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784674350419, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784674365384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11361860483884811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121733120, "lr": 6.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674380411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10977119952440262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674395211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11056604236364365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122060800, "lr": 6.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674409982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10671469569206238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674424741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10457728803157806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122388480, "lr": 6.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674439528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11342015117406845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674454334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10726781189441681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122716160, "lr": 6.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674469198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11481605470180511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674484336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11294239014387131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123043840, "lr": 6.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674499634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11924897134304047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674514980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10641536861658096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123371520, "lr": 6.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674530088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1186642050743103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674545031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732463002204895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123699200, "lr": 6.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674560331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11926592141389847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674560332, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784674560333, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784674645227, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746560573577881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784674645228, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784674645228, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784674660137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11926387995481491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124026880, "lr": 6.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674675275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10790906846523285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674690196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11953384429216385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124354560, "lr": 6.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674704764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11448189616203308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674719513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10840631276369095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124682240, "lr": 6.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674734963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11549966037273407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674750049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11157293617725372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125009920, "lr": 6.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674765255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11658242344856262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674780426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11150003969669342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125337600, "lr": 6.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674795235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10610838979482651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674810032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11279892176389694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125665280, "lr": 6.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674824819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10633190721273422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674839525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11731670051813126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125992960, "lr": 6.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674854435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10517553985118866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674854437, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784674854437, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784674941315, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746797800064087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784674941316, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784674941316, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784674956235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10985300689935684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126320640, "lr": 6.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674971297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.119237519800663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674986178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11097937822341919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126648320, "lr": 6.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675000935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11358553171157837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675015705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11616391688585281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126976000, "lr": 6.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675030418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10956526547670364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675045243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10881289839744568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127303680, "lr": 6.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675060130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10987008363008499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.482916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675075211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11300243437290192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127631360, "lr": 6.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675089968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11256417632102966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675105010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11017627269029617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127959040, "lr": 6.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675120124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11093887686729431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675134991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11169843375682831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128286720, "lr": 6.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675150024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10906955599784851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675150025, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784675150026, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784675236227, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7747929096221924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784675236228, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784675236228, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784675250818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11736836284399033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128614400, "lr": 6.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675265851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11047876626253128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675280730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111203134059906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128942080, "lr": 6.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675295393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10354144871234894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675310199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10581883788108826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129269760, "lr": 6.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675325006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11154655367136002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675339420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10814142227172852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129597440, "lr": 6.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675353928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10799571871757507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675368535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09770813584327698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129925120, "lr": 6.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675383052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10172649472951889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675397978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10598032921552658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130252800, "lr": 6.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675412416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10513197630643845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675427214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11040043085813522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130580480, "lr": 6.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675442111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11609349399805069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675442111, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784675442112, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784675526952, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7748189568519592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784675526953, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784675526953, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784675541673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11454084515571594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130908160, "lr": 6.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675556409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10561888664960861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675570996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10073310881853104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131235840, "lr": 6.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675585777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09248678386211395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675600530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11451594531536102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131563520, "lr": 6.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675615044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10327611118555069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675629720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10239776968955994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131891200, "lr": 6.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675644792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11253315210342407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675659753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1076105535030365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132218880, "lr": 6.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675674930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10329253226518631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675689929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10923326015472412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132546560, "lr": 6.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675705153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1107889860868454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675720323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723020881414413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132874240, "lr": 6.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675735267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10680096596479416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675735268, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784675735269, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784675818506, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7750232815742493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784675818507, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784675818507, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784675833313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10964151471853256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133201920, "lr": 6.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675848243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10315757244825363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675862719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11034443229436874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133529600, "lr": 6.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675877167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09847546368837357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675892016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11618465185165405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133857280, "lr": 6.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675906654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11008898168802261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675921194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10383674502372742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134184960, "lr": 6.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675936051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10833372920751572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675950684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10785295814275742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134512640, "lr": 6.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675965015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10923995077610016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675979438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1103135421872139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134840320, "lr": 6.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675994082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09957253187894821, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676008422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10687325149774551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135168000, "lr": 6.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676022747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08696789294481277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676022747, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784676022748, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784676106463, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752275466918945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784676106464, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784676106464, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784676120953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0964154377579689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135495680, "lr": 6.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676135623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11151071637868881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676150236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10703825205564499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135823360, "lr": 6.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676164886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10124871879816055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676179543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10238882154226303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136151040, "lr": 6.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676194227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09081850200891495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676208702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1033264622092247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136478720, "lr": 6.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676223247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0971212387084961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676237912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.103432297706604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136806400, "lr": 6.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676252621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10629823058843613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676267249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10987204313278198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137134080, "lr": 6.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676281980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10860294103622437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676296636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10702292621135712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137461760, "lr": 6.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676311531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1032353937625885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676311532, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784676311532, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784676393281, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7751433849334717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784676393282, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784676393283, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784676408206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09603924304246902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137789440, "lr": 7.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676423336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10554690659046173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676438168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0950503796339035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138117120, "lr": 7.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676453175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10257495939731598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676468126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09822263568639755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138444800, "lr": 7.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676482661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10138359665870667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676497229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10947489738464355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138772480, "lr": 7.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676512209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10145499557256699, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.066249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676526689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10900191962718964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139100160, "lr": 7.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676541278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1098347082734108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676556234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10674287378787994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139427840, "lr": 7.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676571338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1041121631860733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676585908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10958507657051086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139755520, "lr": 7.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676600551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10895469039678574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676600552, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784676600552, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784676682658, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.77522873878479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784676682659, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784676682659, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784676697304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10638441145420074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140083200, "lr": 7.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676711999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10438171774148941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676726480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10351470112800598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140410880, "lr": 7.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676741079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10006443411111832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676755878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10706165432929993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140738560, "lr": 7.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676770724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0960162803530693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676785896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10521396994590759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141066240, "lr": 7.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676800482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.102692611515522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676814856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11107123643159866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141393920, "lr": 7.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676829488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09869901835918427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676843657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09840186685323715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141721600, "lr": 7.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676858021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10240103304386139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676872786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09036347270011902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142049280, "lr": 7.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676887402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1070370227098465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676887402, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784676887403, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784676970771, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7753761410713196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784676970772, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784676970772, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784676985457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09923937171697617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142376960, "lr": 7.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677000332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1014372780919075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677014820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0994548425078392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142704640, "lr": 7.257916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677029682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10954989492893219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677044584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10072864592075348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143032320, "lr": 7.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677059339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10227768123149872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677073981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10209447890520096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143360000, "lr": 7.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677088366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10529769957065582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677103107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10148883610963821, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143687680, "lr": 7.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677117415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11180774867534637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677132541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10142150521278381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144015360, "lr": 7.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677147310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1075846329331398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677162184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10560718178749084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144343040, "lr": 7.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677177177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10631829500198364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677177178, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784677177178, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784677258831, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.775485634803772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784677258832, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784677258832, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784677273853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10670337080955505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144670720, "lr": 7.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677288723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09846008569002151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677303479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10681327432394028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144998400, "lr": 7.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677318541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10614781826734543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677333415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10275444388389587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145326080, "lr": 7.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677347953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10759209096431732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677362716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09760276228189468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145653760, "lr": 7.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677377791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09815816581249237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677392740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1048969030380249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145981440, "lr": 7.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677407625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1042083278298378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677422635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10231301188468933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146309120, "lr": 7.441250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677437661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11021484434604645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677452647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10659989714622498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146636800, "lr": 7.457916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677467631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10012307018041611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677467632, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784677467633, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784677548552, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7757453918457031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784677548553, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784677548554, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784677563356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10294932872056961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146964480, "lr": 7.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677578033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10596641153097153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677592886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09528881311416626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147292160, "lr": 7.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677608194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09834437072277069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677623119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10194918513298035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147619840, "lr": 7.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677638444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10028335452079773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677653559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10765327513217926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147947520, "lr": 7.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677668711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10979016125202179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677683770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10953936725854874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148275200, "lr": 7.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677698853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09941855072975159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677714014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1027035340666771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148602880, "lr": 7.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677728731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09865130484104156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677743859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11271131038665771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148930560, "lr": 7.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677758567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11108668148517609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677758568, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784677758568, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784677841955, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7758045196533203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784677841956, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784677841957, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784677856499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1058601588010788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149258240, "lr": 7.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677871857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1053425595164299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677886424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1064748466014862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149585920, "lr": 7.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677901186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11136309057474136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149749760, "lr": 7.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677916172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10678496211767197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149913600, "lr": 7.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677931100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10880690813064575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677945666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772278159856796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150241280, "lr": 7.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677960439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09745806455612183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150405120, "lr": 7.649583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677975114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10670184344053268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150568960, "lr": 7.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677989526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10343519598245621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678004030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10805102437734604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150896640, "lr": 7.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678018229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1033305749297142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151060480, "lr": 7.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678033024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10647996515035629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151224320, "lr": 7.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678047685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09536559879779816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678047731, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784678047731, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784678130459, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7757459282875061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784678130460, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784678130460, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784678144984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09995338320732117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151552000, "lr": 7.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678159605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10597242414951324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151715840, "lr": 7.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678174297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10290095955133438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151879680, "lr": 7.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678189266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10443823039531708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678204032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10840485990047455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152207360, "lr": 7.741249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678218665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12289216369390488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152371200, "lr": 7.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678233551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10788550972938538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152535040, "lr": 7.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678248372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09920508414506912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678263388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11188022792339325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152862720, "lr": 7.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678278210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11139551550149918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153026560, "lr": 7.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678293341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11499494314193726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153190400, "lr": 7.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678308381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09828504174947739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678323487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1066369041800499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153518080, "lr": 7.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678338439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10413217544555664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153681920, "lr": 7.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678338483, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784678338484, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784678422183, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.776190459728241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784678422184, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784678422184, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784678437277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09714909642934799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153845760, "lr": 7.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678452293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09811784327030182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.832916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678466953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11283499002456665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154173440, "lr": 7.841249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678481798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11811192333698273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154337280, "lr": 7.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678496331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11430195719003677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154501120, "lr": 7.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678510773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1113375723361969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678526185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1074720099568367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154828800, "lr": 7.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678541296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12032898515462875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154992640, "lr": 7.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678556393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10470280796289444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155156480, "lr": 7.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678571870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10824974626302719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678587272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11084116250276566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155484160, "lr": 7.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678602430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11079145222902298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155648000, "lr": 7.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678617680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11292116343975067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155811840, "lr": 7.924583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678632854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11150255799293518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.932916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678632855, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784678632856, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784678713547, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7763019800186157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784678713549, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784678713549, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784678728757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10781121999025345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156139520, "lr": 7.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678743790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11042304337024689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156303360, "lr": 7.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678759179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11422759294509888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156467200, "lr": 7.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678774483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10999657213687897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678789773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10796412080526352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156794880, "lr": 7.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678805140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10863330215215683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156958720, "lr": 7.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678820391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10180072486400604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157122560, "lr": 7.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678835465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11265945434570312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678850675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12045430392026901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157450240, "lr": 8.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678865946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10938068479299545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157614080, "lr": 8.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678881156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10872292518615723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157777920, "lr": 8.024583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678896619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10476361215114594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.032916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678912286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10351309180259705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158105600, "lr": 8.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678927636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10905911773443222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158269440, "lr": 8.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678927637, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784678927637, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784679010367, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7764267921447754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784679010367, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784679010368, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784679025832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11444143950939178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158433280, "lr": 8.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679041382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11914826929569244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679056867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11431998759508133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158760960, "lr": 8.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679072450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11251617223024368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158924800, "lr": 8.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679087995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11082107573747635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159088640, "lr": 8.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679103193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11925379186868668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679118649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11154226958751678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159416320, "lr": 8.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679133486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10920005291700363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159580160, "lr": 8.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679149130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1107422485947609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159744000, "lr": 8.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679163898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09917393326759338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679178851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1099138930439949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160071680, "lr": 8.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679194325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10976127535104752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160235520, "lr": 8.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679209506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10373005270957947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160399360, "lr": 8.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679224283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11036645621061325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679224283, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784679224284, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784679305654, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7766224145889282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784679305655, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784679305655, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784679320583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11327454447746277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160727040, "lr": 8.174583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679335666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1276201754808426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160890880, "lr": 8.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679350629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10846871882677078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161054720, "lr": 8.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679365521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11250832676887512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679380394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11223548650741577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161382400, "lr": 8.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679395466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11158936470746994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161546240, "lr": 8.216250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679410648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11409647762775421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161710080, "lr": 8.224583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679425874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11276110261678696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679440789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10658910870552063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162037760, "lr": 8.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679455695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10856159776449203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162201600, "lr": 8.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679470900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12966232001781464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162365440, "lr": 8.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679485807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11320150643587112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679501222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11281570792198181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162693120, "lr": 8.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679516483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11655424535274506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162856960, "lr": 8.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679516484, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784679516484, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784679598706, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7766678929328918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784679598708, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784679598708, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784679613736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09538178890943527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163020800, "lr": 8.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679628926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11155154556035995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679644165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12193825095891953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163348480, "lr": 8.307916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679659411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071677878499031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163512320, "lr": 8.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679674726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1097772866487503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163676160, "lr": 8.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679689931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10775579512119293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679705008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11211900413036346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164003840, "lr": 8.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679719833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11341337859630585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164167680, "lr": 8.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679735141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10313203930854797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164331520, "lr": 8.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679749899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11807423830032349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679764977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10810986906290054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164659200, "lr": 8.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679780156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11141656339168549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164823040, "lr": 8.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679795386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09225845336914062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164986880, "lr": 8.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679810191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10349839925765991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679810191, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784679810192, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784679892051, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7769864797592163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784679892053, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784679892053, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784679907249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1011153906583786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165314560, "lr": 8.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679922235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09686257690191269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165478400, "lr": 8.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679937016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11406899988651276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165642240, "lr": 8.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679951971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1076369434595108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679966859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10996238887310028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165969920, "lr": 8.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679982078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10357410460710526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166133760, "lr": 8.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679997098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10773316025733948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166297600, "lr": 8.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680012128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11301535367965698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680027192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09965254366397858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166625280, "lr": 8.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680042406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09525133669376373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166789120, "lr": 8.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680057573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09714441001415253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166952960, "lr": 8.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680072669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09674493223428726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680087630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10250845551490784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167280640, "lr": 8.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680102768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10121256113052368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167444480, "lr": 8.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680102769, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784680102769, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784680186446, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7771611213684082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784680186447, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784680186447, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784680201683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10359865427017212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167608320, "lr": 8.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680217129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1028703823685646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680232289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039377897977829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167936000, "lr": 8.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680247464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10705272853374481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168099840, "lr": 8.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680262732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10230817645788193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168263680, "lr": 8.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680278133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09911313652992249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680293700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085686981678009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168591360, "lr": 8.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680309062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10526680946350098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168755200, "lr": 8.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680324289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10691183805465698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168919040, "lr": 8.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680339476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10606282949447632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680354915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10440484434366226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169246720, "lr": 8.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680369791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10788781940937042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169410560, "lr": 8.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680384489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10593172162771225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169574400, "lr": 8.624583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680399851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1130446195602417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680399890, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784680399890, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784680483211, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7773980498313904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784680483212, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784680483212, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784680497885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10821207612752914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169902080, "lr": 8.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680512865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11190617084503174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170065920, "lr": 8.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680528141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725502669811249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170229760, "lr": 8.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680543694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10695639252662659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680558739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10072077810764313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170557440, "lr": 8.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680573953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10204949975013733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170721280, "lr": 8.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680589280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09587429463863373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170885120, "lr": 8.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680604029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10778123885393143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680618739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09962281584739685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171212800, "lr": 8.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680633592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10239101946353912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171376640, "lr": 8.716249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680648792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1011098101735115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171540480, "lr": 8.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680663620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10284099727869034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680678509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09926744550466537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171868160, "lr": 8.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680693598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10394667088985443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172032000, "lr": 8.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680693650, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784680693650, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784680776455, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7775459289550781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784680776456, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784680776456, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784680791398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0981733500957489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172195840, "lr": 8.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680806190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.107809878885746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680820763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0996847003698349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172523520, "lr": 8.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680835432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10265941172838211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172687360, "lr": 8.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680849971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09381452202796936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172851200, "lr": 8.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680864826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10632255673408508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680879739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10362081974744797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173178880, "lr": 8.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680894529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10383623093366623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173342720, "lr": 8.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680909606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10488373041152954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173506560, "lr": 8.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680924290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11131029576063156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680939207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09540805220603943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173834240, "lr": 8.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680954222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11148938536643982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173998080, "lr": 8.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680969178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11175429821014404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174161920, "lr": 8.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680983736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10183583945035934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680983784, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784680983785, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784681066979, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7777686715126038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784681066979, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784681066979, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784681081583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09861286729574203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174489600, "lr": 8.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681096570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10763608664274216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174653440, "lr": 8.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681111431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10209841281175613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174817280, "lr": 8.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681125818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10555166751146317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681141054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10082439333200455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175144960, "lr": 8.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681156123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10888291150331497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175308800, "lr": 8.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681170966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10069537162780762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175472640, "lr": 8.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681186113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09818210452795029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681201277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10918281972408295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175800320, "lr": 8.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681216288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10931375622749329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175964160, "lr": 8.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681231187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1000932976603508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176128000, "lr": 8.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681246239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09395729750394821, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681261028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09448432177305222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176455680, "lr": 8.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681275901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10547591000795364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176619520, "lr": 8.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681275902, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784681275903, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784681359137, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7780077457427979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784681359137, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784681359138, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784681374125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10273171961307526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176783360, "lr": 8.991249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681389158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11394757777452469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681404329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1003136932849884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177111040, "lr": 9.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681419473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10538869351148605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177274880, "lr": 9.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681434624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1101079136133194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177438720, "lr": 9.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681449559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10051004588603973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.032916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681464555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11016517132520676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177766400, "lr": 9.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681479342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10167046636343002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177930240, "lr": 9.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681494163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10777716338634491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178094080, "lr": 9.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681509028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1007218062877655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681523997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09705885499715805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178421760, "lr": 9.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681539267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11157678812742233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178585600, "lr": 9.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681554662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11892847716808319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178749440, "lr": 9.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681569909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10901457816362381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681569910, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784681569910, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784681654735, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7782136797904968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784681654736, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784681654736, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784681670115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11538846045732498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179077120, "lr": 9.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681685279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1006985604763031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179240960, "lr": 9.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681700933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09990853816270828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179404800, "lr": 9.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681716123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10038463771343231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.132916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681731441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10503783077001572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179732480, "lr": 9.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681746759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09887580573558807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179896320, "lr": 9.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681761863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09994149208068848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180060160, "lr": 9.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681777337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1063770055770874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681792204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10583483427762985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180387840, "lr": 9.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681806917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09355238080024719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180551680, "lr": 9.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681822488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10384513437747955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180715520, "lr": 9.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681837237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10151863843202591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681852171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09393402934074402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181043200, "lr": 9.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681867168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10030394792556763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181207040, "lr": 9.216249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681867169, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784681867170, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784681951850, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.778218686580658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784681951851, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784681951851, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784681966917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11100616306066513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181370880, "lr": 9.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681981892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1137327030301094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681996802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10105447471141815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181698560, "lr": 9.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682011849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10675612837076187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181862400, "lr": 9.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682026971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10313858836889267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182026240, "lr": 9.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682041925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10222437232732773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682056868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10687699913978577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182353920, "lr": 9.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682071816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10826552659273148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182517760, "lr": 9.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682086799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0976090133190155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182681600, "lr": 9.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682101590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09711572527885437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682116608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11156520992517471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183009280, "lr": 9.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682131383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09742366522550583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183173120, "lr": 9.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682146493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10180482268333435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183336960, "lr": 9.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682161236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10298536717891693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682161237, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784682161237, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784682243903, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7784460783004761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784682243903, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784682243904, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784682258993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10212673991918564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183664640, "lr": 9.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682273734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10844377428293228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183828480, "lr": 9.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682288554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11172603070735931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183992320, "lr": 9.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682304027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09359940886497498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682319317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10293970257043839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184320000, "lr": 9.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682334540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10785417258739471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184483840, "lr": 9.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682349918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11584458500146866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184647680, "lr": 9.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682365006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10089842975139618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682380382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10579598695039749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184975360, "lr": 9.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682396061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11033281683921814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185139200, "lr": 9.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682411497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09727093577384949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185303040, "lr": 9.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682426738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10735788196325302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682442051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09823369979858398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185630720, "lr": 9.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682456992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10380303859710693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185794560, "lr": 9.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682456993, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784682456993, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784682541783, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7785574197769165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784682541784, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784682541784, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784682557188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11034020036458969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185958400, "lr": 9.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682572150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10412675887346268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682586900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09207472205162048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186286080, "lr": 9.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682602441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10072078555822372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186449920, "lr": 9.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682617353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10866384953260422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186613760, "lr": 9.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682632494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1109641045331955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682648068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09674691408872604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186941440, "lr": 9.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682663526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10363379865884781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187105280, "lr": 9.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682678663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10508956015110016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187269120, "lr": 9.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682693727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09940829128026962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682708784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039276272058487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187596800, "lr": 9.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682723948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10445026308298111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187760640, "lr": 9.549583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682738901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1106603667140007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187924480, "lr": 9.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682754113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11364229768514633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682754114, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784682754114, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784682839998, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7787365317344666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784682839999, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784682839999, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784682854859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10953126847743988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188252160, "lr": 9.574583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682870064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1049618124961853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188416000, "lr": 9.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682885024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10425511002540588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188579840, "lr": 9.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682900158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10881608724594116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682915053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1022578775882721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188907520, "lr": 9.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682930226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09722264856100082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189071360, "lr": 9.616249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682945116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10267947614192963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189235200, "lr": 9.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682959688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11169193685054779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682974456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10102775692939758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189562880, "lr": 9.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682989368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09478280693292618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189726720, "lr": 9.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683004261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1040741428732872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189890560, "lr": 9.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683019414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11211119592189789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.666249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683034769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10159993171691895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190218240, "lr": 9.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683049742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11758559197187424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190382080, "lr": 9.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683049743, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784683049743, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784683134792, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7788498401641846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784683134792, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784683134792, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784683149928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1003885343670845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190545920, "lr": 9.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683164807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09708064794540405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683180025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10577516257762909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190873600, "lr": 9.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683195660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11239761114120483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191037440, "lr": 9.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683211074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11771836131811142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191201280, "lr": 9.724583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683226344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09625890105962753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683241603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1060340479016304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191528960, "lr": 9.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683256731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1011858880519867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191692800, "lr": 9.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683272017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10126181691884995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191856640, "lr": 9.757916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683287025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10453146696090698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683301579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10647908598184586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192184320, "lr": 9.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683317371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11013389378786087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192348160, "lr": 9.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683332553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0991099625825882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192512000, "lr": 9.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683347877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11420777440071106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.799583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683347878, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784683347878, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784683431461, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7791565656661987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784683431462, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784683431462, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784683446884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11200584471225739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192839680, "lr": 9.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683462455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11079196631908417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193003520, "lr": 9.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683477972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10650143772363663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193167360, "lr": 9.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683493375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12041407823562622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683508638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1162724494934082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193495040, "lr": 9.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683524050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10085723549127579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193658880, "lr": 9.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683539503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11795628815889359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193822720, "lr": 9.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683554664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11381060630083084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.866249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683569836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10569417476654053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194150400, "lr": 9.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683585030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10699986666440964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194314240, "lr": 9.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683600387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10714977979660034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194478080, "lr": 9.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683615420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10610151290893555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683630769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10682041943073273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194805760, "lr": 9.907916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683645603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11586028337478638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194969600, "lr": 9.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683645604, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784683645605, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784683729364, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.779501736164093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784683729365, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784683729365, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784683744722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11149322986602783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195133440, "lr": 9.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683759848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10399642586708069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683775169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10249096900224686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195461120, "lr": 9.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683790718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11604654043912888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195624960, "lr": 9.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683806269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10841739177703857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195788800, "lr": 9.957916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683821444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10538666695356369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683836823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10470211505889893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196116480, "lr": 9.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683852052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11378294974565506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196280320, "lr": 9.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683867687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11808822304010391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196444160, "lr": 9.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683883190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10548596829175949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.999583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683898702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11644231528043747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196771840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784683914078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11214455962181091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196935680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784683928807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10788727551698685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197099520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784683944264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1160818487405777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784683944264, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784683944265, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784684027838, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7797027826309204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784684027839, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784684027839, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784684042757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10115838050842285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197427200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684058249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111550472676754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197591040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684073711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11535286158323288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197754880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684089184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10152465850114822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684104772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0993957668542862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198082560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684120170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10810161381959915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198246400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684135484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0985700935125351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198410240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684150681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10824607312679291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684166092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10037233680486679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198737920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684181179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1052270233631134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198901760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684196470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11526396870613098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199065600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684211735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11025506258010864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684226886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11784093827009201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199393280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684242386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11263345181941986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199557120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684242386, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784684242387, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784684324194, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7799981236457825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784684324195, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784684324195, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784684339639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11193428188562393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199720960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684371101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10648433119058609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684386456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11120548844337463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200048640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684409605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11658936738967896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200212480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684424958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11262182891368866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200376320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684454239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11686275899410248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684469907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09934070706367493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200704000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684486084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10826950520277023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200867840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684501824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10807077586650848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201031680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684521995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080017164349556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684537779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11824564635753632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201359360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684559916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11082987487316132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201523200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684575545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10429419577121735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201687040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684593974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11158625781536102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684593976, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784684593977, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784684675680, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7800657749176025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784684675681, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784684675681, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784684692575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10547368973493576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202014720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684707986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11117233335971832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202178560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684723151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11889274418354034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202342400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684738666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10567892342805862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684753374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10526358336210251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202670080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684768307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10117437690496445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202833920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684783871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10615232586860657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202997760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684798882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10948555171489716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684813973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11300502717494965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203325440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684828768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09311604499816895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203489280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684843788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11693289130926132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203653120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684858710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10071970522403717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684873205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09796854108572006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203980800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684887851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10742899775505066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204144640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684887852, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784684887852, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784684968671, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7799180150032043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784684968672, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784684968673, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784684983554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10097391158342361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204308480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684998548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09184364229440689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685013590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0963878482580185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204636160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685028616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10924379527568817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204800000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685043670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10813269764184952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204963840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685058629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11205624788999557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685073329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09495174139738083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205291520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685088218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1063651368021965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205455360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685103063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10637501627206802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205619200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685117845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11016860604286194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685132621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09832217544317245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205946880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685147913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09884101152420044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206110720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685162882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11130787432193756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206274560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685177615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09853550046682358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685177615, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784685177616, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784685259564, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7802433371543884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784685259565, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784685259565, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784685274706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1060217097401619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206602240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685289643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09556010365486145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206766080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685304759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10334471613168716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206929920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685320040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09597670286893845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685335195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1021464467048645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207257600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685350114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11148891597986221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207421440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685365152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10694924741983414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207585280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685380017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09534692019224167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685395004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11481105536222458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207912960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685410117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10404258221387863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208076800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685424787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10294898599386215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208240640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685440402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10188038647174835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685455326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10309437662363052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208568320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685470399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10140959918498993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208732160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685470400, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784685470401, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784685553822, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7802426815032959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784685553823, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784685553824, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784685569225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10023621469736099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208896000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685584363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09900522232055664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685599615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10133899748325348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209223680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685614483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10265591740608215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209387520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685629876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09884487837553024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209551360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685645008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1007550060749054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685659889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09689755737781525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209879040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685674822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10029678046703339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210042880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685689984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1035233810544014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210206720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685704975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10838811844587326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685719997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1043303832411766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210534400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685734992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10755548626184464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210698240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685750119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10206685960292816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210862080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685764830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09600643813610077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685764831, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784685764831, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784685849072, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7803417444229126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784685849073, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784685849073, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784685864110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09409871697425842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211189760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685878750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0994352474808693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211353600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685893476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09688679873943329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211517440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685908036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10022024810314178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685922836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10425805300474167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211845120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685938144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09541871398687363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212008960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685952993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10254562646150589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212172800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685968006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09814518690109253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685982835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09815218299627304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212500480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685997746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.104596808552742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212664320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686012539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09333734959363937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212828160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686027591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10676295310258865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686042977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10497158765792847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213155840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686058424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10976212471723557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213319680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686058425, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784686058426, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784686143501, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7804283499717712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784686143502, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784686143502, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784686158463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10562677681446075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213483520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686173376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10108567029237747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686188428, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10541152954101562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213811200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686203774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10290229320526123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213975040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686218217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0964650809764862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214138880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686232943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10445129126310349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686248197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09566338360309601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214466560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686263195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11410368233919144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214630400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686278264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09832635521888733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214794240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686293859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09015043824911118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686309048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10200516134500504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215121920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686324146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10538216680288315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215285760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686339110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10431114584207535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215449600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686354187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10895597189664841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686354188, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784686354188, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784686437895, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7806271910667419, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784686437896, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784686437896, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784686452859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1003699079155922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215777280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686467594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10487233847379684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215941120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686482612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09817977994680405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216104960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686497777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09943222999572754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686513067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10040724277496338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216432640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686527934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09505974501371384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216596480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686542927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10461347550153732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216760320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686558225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10294365137815475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686573206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.100703164935112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217088000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686588649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11014919728040695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217251840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686603517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09751194715499878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217415680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686618889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09978830814361572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217579520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686633942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09958916902542114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217743360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686648947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10534138232469559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217907200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686648948, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784686648948, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784686732065, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7805741429328918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784686732066, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784686732066, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784686747311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1016494557261467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218071040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686762607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09549698233604431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218234880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686778010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09779186546802521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218398720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686793206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10794572532176971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218562560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686808782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10285071283578873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218726400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686824282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08722840249538422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218890240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686839882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10583741962909698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219054080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686855432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09293760359287262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219217920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686870877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1048056036233902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219381760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686886314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09583890438079834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219545600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686901647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09526937454938889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219709440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686916552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10244292765855789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219873280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686931835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11188504099845886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220037120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686947037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0978153645992279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220200960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686947037, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784686947038, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784687032225, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7809751033782959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784687032227, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784687032227, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784687046966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09515947848558426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220364800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784687062436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09551017731428146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220528640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784687077362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10120581835508347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220692480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784687092394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10294757038354874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220856320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784687107686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10335270315408707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221020160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784687122985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10568339377641678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221184000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784687138366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10824423283338547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221347840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784687153318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11589077860116959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221511680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784687168288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10887610912322998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221675520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784687183366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09861230850219727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221839360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784687198419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11096364259719849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222003200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784687213575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0962110385298729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222167040, "lr": 1e-06}} diff --git a/recommendation_v4/rcp_logs/gbs_8192/seed445449777.log b/recommendation_v4/rcp_logs/gbs_8192/seed445449777.log new file mode 100644 index 000000000..52afb47d2 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_8192/seed445449777.log @@ -0,0 +1,1830 @@ +:::MLLOG {"namespace": "", "time_ms": 1784658272109, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784658272109, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784658293692, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784658293694, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784658293694, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784658293694, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784658293694, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784658293694, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 8192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784658293694, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784658293694, "event_type": "POINT_IN_TIME", "key": "seed", "value": 445449777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784658293694, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784658293694, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784658293694, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784658293694, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784658293695, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784658293695, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784659177290, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784659177290, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784659177644, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784659294532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385040134191513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840, "lr": 7.916666666666666e-10}} +:::MLLOG {"namespace": "", "time_ms": 1784659306651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1391722410917282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.6249999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659319030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860109448432922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 491520, "lr": 2.458333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659331918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13903427124023438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659344634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13902097940444946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 819200, "lr": 4.125e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659357454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387871503829956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659370334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13904257118701935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1146880, "lr": 5.7916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659383353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13868319988250732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.625e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659396287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13895158469676971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1474560, "lr": 7.458333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659409134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388815939426422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.291666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659422473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385752409696579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1802240, "lr": 9.124999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659435659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860474526882172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659448790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13898178935050964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2129920, "lr": 1.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659461871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13872160017490387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659461872, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784659461872, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784659546525, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5003085136413574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784659546525, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784659546526, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784659559475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13879281282424927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2457600, "lr": 1.2458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659572738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13875100016593933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659586422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860586285591125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2785280, "lr": 1.4124999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659599425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388510912656784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4958333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659612763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13879604637622833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3112960, "lr": 1.5791666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659626666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138803631067276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659639849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13870464265346527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3440640, "lr": 1.7458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659653252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386973261833191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659666949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13849572837352753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3768320, "lr": 1.9124999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659680224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13871446251869202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9958333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659694102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13865667581558228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4096000, "lr": 2.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659707900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13858886063098907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.1624999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659721511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13855072855949402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4423680, "lr": 2.2458333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659735542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13854500651359558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3291666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659735543, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784659735543, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784659818968, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.500533938407898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784659818969, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784659818970, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784659832266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383405327796936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4751360, "lr": 2.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659845711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13843666017055511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659859139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13789646327495575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5079040, "lr": 2.5791666666666668e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659873054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13788044452667236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659886640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13816708326339722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5406720, "lr": 2.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659900326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383134126663208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659914015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13788366317749023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5734400, "lr": 2.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659928054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13761326670646667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659941614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13736452162265778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6062080, "lr": 3.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659954971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13786755502223969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659968529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13747118413448334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6389760, "lr": 3.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659981836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13795284926891327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659996068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13806922733783722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6717440, "lr": 3.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660009881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13790225982666016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660009882, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784660009882, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784660092682, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5008573532104492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784660092683, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784660092683, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784660106127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13716761767864227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7045120, "lr": 3.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660119927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13769954442977905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660133599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13731221854686737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7372800, "lr": 3.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660148014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137658953666687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.829166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660161721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376846581697464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7700480, "lr": 3.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660175598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376277506351471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660189355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13694307208061218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8028160, "lr": 4.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660203024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13775019347667694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660217084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13709893822669983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8355840, "lr": 4.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660230935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385861486196518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3291666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660244765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13653744757175446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8683520, "lr": 4.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660258493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13805636763572693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.4958333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660272311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13581249117851257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9011200, "lr": 4.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660286500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13685853779315948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.6624999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660286500, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784660286501, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784660371704, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5013286471366882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784660371705, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784660371705, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784660385257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366022229194641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9338880, "lr": 4.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660398924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13577473163604736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660412331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375255584716797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9666560, "lr": 4.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660426000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349341720342636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660439895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13689658045768738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9994240, "lr": 5.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660453532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338019222021103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1624999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660467688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13563168048858643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10321920, "lr": 5.2458333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660481425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13558539748191833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660495050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377965211868286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10649600, "lr": 5.4124999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660508494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13531076908111572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660522013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14599379897117615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10977280, "lr": 5.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660535918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13563737273216248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660549517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13421490788459778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11304960, "lr": 5.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660563337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13789433240890503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660563338, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784660563338, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784660648086, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5019841194152832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784660648087, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784660648087, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784660661727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355731338262558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11632640, "lr": 5.9124999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660675423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13943177461624146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660689055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13597846031188965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11960320, "lr": 6.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660702764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332540512084961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660716646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135092094540596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12288000, "lr": 6.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660730731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13907773792743683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.329166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660744531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13362634181976318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12615680, "lr": 6.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660758413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312415778636932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660772171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13330930471420288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12943360, "lr": 6.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660785797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13175931572914124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660799667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13118524849414825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13271040, "lr": 6.745833333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660813266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13521181046962738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660827047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321125030517578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13598720, "lr": 6.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660841154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13896715641021729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660841154, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784660841155, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784660926618, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.502794086933136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784660926619, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784660926619, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784660940549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13382405042648315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13926400, "lr": 7.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660954419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333058625459671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660968361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13596372306346893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14254080, "lr": 7.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660982063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134818434715271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660995760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13783268630504608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14581760, "lr": 7.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661009534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1278054565191269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661023005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13542217016220093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14909440, "lr": 7.579166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661036647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12920928001403809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661050504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13354243338108063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15237120, "lr": 7.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661064673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13262854516506195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.829166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661078496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13366079330444336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15564800, "lr": 7.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661092700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345110982656479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.995833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661107373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13554905354976654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15892480, "lr": 8.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661121465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13356848061084747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661121465, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784661121466, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784661205393, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.503851592540741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784661205394, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784661205394, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784661219570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13256527483463287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16220160, "lr": 8.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661233594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13361167907714844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661247693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356119066476822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16547840, "lr": 8.412500000000001e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661262094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13538873195648193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661276292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1396019607782364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16875520, "lr": 8.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661290273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13324999809265137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.662499999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661305216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13533097505569458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17203200, "lr": 8.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661319563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13347502052783966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661333747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13277536630630493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17530880, "lr": 8.912499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661348036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13199873268604279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661362008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13566499948501587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17858560, "lr": 9.079166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661376420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13374212384223938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.162499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661390415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13331852853298187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18186240, "lr": 9.245833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661404289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13060279190540314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661404290, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784661404291, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784661490405, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5050032734870911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784661490406, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784661490406, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784661505321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13791999220848083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18513920, "lr": 9.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661519844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350167691707611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.495833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661534341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13540539145469666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18841600, "lr": 9.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661548857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13124504685401917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.662499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661563164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13375398516654968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19169280, "lr": 9.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661577556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13399969041347504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661592070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13705812394618988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19496960, "lr": 9.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661606577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13387799263000488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661620965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13301748037338257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19824640, "lr": 1.0079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661635043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12965385615825653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0162499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661648876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12771254777908325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20152320, "lr": 1.0245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661662928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13298986852169037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661677130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297127902507782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20480000, "lr": 1.0412499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661690926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12475553900003433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661690926, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784661690927, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784661776481, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5062720775604248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784661776482, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784661776482, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784661790587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13114920258522034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20807680, "lr": 1.0579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661804591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13605161011219025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661819026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13347499072551727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21135360, "lr": 1.0745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661833415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13031218945980072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661847419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1283774971961975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21463040, "lr": 1.0912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661861765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12669014930725098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661876196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289454698562622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21790720, "lr": 1.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661890195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13236573338508606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661904257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13562940061092377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22118400, "lr": 1.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661918430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12484090030193329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661932370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1287190467119217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22446080, "lr": 1.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661946391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13419446349143982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661960578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13107553124427795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22773760, "lr": 1.1579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661975040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12926720082759857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661975041, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784661975041, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784662059671, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5078045725822449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784662059672, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784662059672, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784662073933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304478943347931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23101440, "lr": 1.1745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662087972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12899984419345856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662101633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331644505262375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23429120, "lr": 1.1912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662116176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12384305894374847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662130476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12952300906181335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23756800, "lr": 1.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662144262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1259061098098755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662158101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13010390102863312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24084480, "lr": 1.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662172063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1218615472316742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662185627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13611231744289398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24412160, "lr": 1.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662199673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13703440129756927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662213488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12356889992952347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24739840, "lr": 1.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662227939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12334201484918594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662242152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12568338215351105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25067520, "lr": 1.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662256251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12908585369586945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662256252, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784662256252, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784662341223, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5095950365066528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784662341224, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784662341224, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784662355012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13637247681617737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25395200, "lr": 1.2912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662369253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13665024936199188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662382966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13567212224006653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25722880, "lr": 1.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662396600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1399870067834854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662410366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13675029575824738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26050560, "lr": 1.3245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662424249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13833028078079224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662438048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12740832567214966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26378240, "lr": 1.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662451826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12614795565605164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662465557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14223341643810272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26705920, "lr": 1.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662479709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14121288061141968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662493932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12714490294456482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27033600, "lr": 1.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662508062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12856099009513855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662522184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13304686546325684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27361280, "lr": 1.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662535747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13139653205871582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662535747, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784662535748, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784662621101, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5114534497261047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784662621102, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784662621102, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784662635862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14146853983402252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27688960, "lr": 1.4079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662650277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14241482317447662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662664342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13555076718330383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28016640, "lr": 1.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662678570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13955754041671753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662693003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14497242867946625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28344320, "lr": 1.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662706992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14090517163276672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662721333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1258312612771988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28672000, "lr": 1.4579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662735723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13756626844406128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662750185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13183952867984772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28999680, "lr": 1.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662764702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13614732027053833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662779158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13774849474430084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29327360, "lr": 1.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662793382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330164521932602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4995833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662808188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13433340191841125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29655040, "lr": 1.5079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662822810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363224983215332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662822811, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784662822811, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784662906114, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5131489038467407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784662906116, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784662906116, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784662920218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13867628574371338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29982720, "lr": 1.5245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662934698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13806012272834778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662949322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13170631229877472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30310400, "lr": 1.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662963799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13282501697540283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662978203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1340297907590866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30638080, "lr": 1.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662992887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13993069529533386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663007424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1328442394733429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30965760, "lr": 1.5745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663022105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1410883218050003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663036165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14092837274074554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31293440, "lr": 1.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663050826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13517029583454132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663065062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1278383731842041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31621120, "lr": 1.6079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663079276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13003265857696533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663093431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1340280920267105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31948800, "lr": 1.6245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663107753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12783387303352356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663107754, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784663107754, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784663190436, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5153077840805054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784663190437, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784663190437, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784663204637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13201507925987244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32276480, "lr": 1.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663218802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13681241869926453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663233063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13363030552864075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32604160, "lr": 1.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663247340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318785548210144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663261630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13406118750572205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32931840, "lr": 1.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663275737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13353371620178223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663290321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13370196521282196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33259520, "lr": 1.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663304769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13253095746040344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663319027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13996823132038116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33587200, "lr": 1.7079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663333940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13196982443332672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663348143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289006769657135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33914880, "lr": 1.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663362115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311865597963333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663376232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325758993625641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34242560, "lr": 1.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663390437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13598774373531342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7495833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663390437, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784663390438, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784663473385, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5182751417160034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784663473386, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784663473386, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784663487579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336842179298401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34570240, "lr": 1.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663501681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12280520796775818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663515856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13049151003360748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34897920, "lr": 1.7745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663530083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303102970123291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663544243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13534431159496307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35225600, "lr": 1.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663558886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351184844970703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663573090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13553860783576965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35553280, "lr": 1.8079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663587733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12404460459947586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8162500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663602032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13110758364200592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35880960, "lr": 1.8245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663615993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311349719762802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663630776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13012734055519104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36208640, "lr": 1.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663645055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13330265879631042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663659617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308409422636032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36536320, "lr": 1.8579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663673807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301446557044983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663673808, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784663673808, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784663756762, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5219566226005554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784663756763, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784663756764, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784663771178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378519982099533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36864000, "lr": 1.8745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663785619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13239964842796326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663799754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13370469212532043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37191680, "lr": 1.8912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663813933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13044136762619019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663828063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12718911468982697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37519360, "lr": 1.9079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663842109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13188238441944122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663856727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355810910463333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37847040, "lr": 1.9245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663871329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13295339047908783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663885734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13145066797733307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38174720, "lr": 1.9412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663900357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13916756212711334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663914821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13393810391426086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38502400, "lr": 1.9579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663928662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13771072030067444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663943313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13223223388195038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38830080, "lr": 1.9745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663957560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13353055715560913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663957560, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784663957561, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784664040919, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5269417762756348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784664040919, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784664040920, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784664055124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13109833002090454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39157760, "lr": 1.9912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664069356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13501331210136414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664083444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13783860206604004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39485440, "lr": 2.0079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664097703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13325060904026031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664111801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12740154564380646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39813120, "lr": 2.0245833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664125848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12934662401676178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0329166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664139886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1259523332118988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40140800, "lr": 2.0412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664153873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12417270243167877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664168137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13624532520771027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40468480, "lr": 2.0579166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664182542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373194456100464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664196935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13469500839710236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40796160, "lr": 2.0745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664211858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1271541714668274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0829166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664226217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13090214133262634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41123840, "lr": 2.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664240110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12371795624494553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664240111, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784664240111, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784664324554, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5336902141571045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784664324555, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784664324556, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784664339048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13250072300434113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41451520, "lr": 2.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664353556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13261140882968903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664368045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338394433259964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41779200, "lr": 2.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664382527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380421370267868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664396823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13632096350193024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42106880, "lr": 2.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664411310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131511390209198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664425553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12731605768203735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42434560, "lr": 2.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664439881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13497331738471985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664454025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12488988786935806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42762240, "lr": 2.1745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664468408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1307215690612793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664482774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325303465127945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43089920, "lr": 2.1912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664497235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13447019457817078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664511739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14028140902519226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43417600, "lr": 2.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664525956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363411396741867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664525957, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784664525957, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784664611431, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5437871813774109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784664611432, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784664611432, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784664626159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13679808378219604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43745280, "lr": 2.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664640468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13159193098545074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664654507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13270489871501923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44072960, "lr": 2.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664668787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1391608864068985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664683610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300201117992401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44400640, "lr": 2.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664698529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13856743276119232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664712862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13516876101493835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44728320, "lr": 2.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664727449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13743507862091064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664741792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13522303104400635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45056000, "lr": 2.2912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664756426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13061872124671936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664770966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14082814753055573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45383680, "lr": 2.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664785512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13488323986530304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664800068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12922334671020508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45711360, "lr": 2.3245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664814734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133436381816864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664814735, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784664814736, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784664899963, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5562683939933777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784664899964, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784664899965, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784664914891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14219091832637787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46039040, "lr": 2.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664929366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13293559849262238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664944462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13366325199604034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46366720, "lr": 2.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664959108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13086801767349243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664974185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13540861010551453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46694400, "lr": 2.3745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664989066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13373404741287231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665003718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13432857394218445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47022080, "lr": 2.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665018391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13577519357204437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665032936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131750226020813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47349760, "lr": 2.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665047865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13297758996486664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665062599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13289733231067657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47677440, "lr": 2.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665077620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13719181716442108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665092220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13620123267173767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48005120, "lr": 2.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665106963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13765430450439453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.4495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665106964, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784665106964, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784665190783, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5718265175819397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784665190784, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784665190784, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784665205417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13194261491298676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48332800, "lr": 2.4579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665220334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13541091978549957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665234977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346476525068283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48660480, "lr": 2.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665250205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13216978311538696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665264766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374480426311493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48988160, "lr": 2.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665279358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13002115488052368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665294189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13387461006641388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49315840, "lr": 2.507916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665308423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12462623417377472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.5162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665322894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315898597240448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49643520, "lr": 2.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665337157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13197967410087585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665351506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13142399489879608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49971200, "lr": 2.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665365639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13209949433803558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.5495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665379787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12913814187049866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50298880, "lr": 2.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665394121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13135449588298798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.5662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665394121, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784665394122, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784665475606, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5872239470481873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784665475607, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784665475607, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784665489710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1231083795428276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50626560, "lr": 2.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665504179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12747760117053986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665518508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13000479340553284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50954240, "lr": 2.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665532928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341046541929245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665547704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13237379491329193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51281920, "lr": 2.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665562114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12269998341798782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665576848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13128764927387238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51609600, "lr": 2.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665591462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13255879282951355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665606092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13778647780418396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51937280, "lr": 2.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665620246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12908224761486053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.6495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665635159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13419172167778015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52264960, "lr": 2.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665649549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.118312306702137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.6662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665663800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12278629094362259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52592640, "lr": 2.6745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665678238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12610751390457153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665678238, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784665678239, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784665759727, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6029505133628845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784665759728, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784665759728, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784665773991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1222175881266594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52920320, "lr": 2.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665788046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12834540009498596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.6995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665802254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12943239510059357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53248000, "lr": 2.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665816332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13285161554813385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665830409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1226353645324707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53575680, "lr": 2.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665844776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356327086687088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665858853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13023331761360168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53903360, "lr": 2.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665872927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12240096926689148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.7495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665887290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12800364196300507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54231040, "lr": 2.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665902002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12878865003585815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665916368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11954698711633682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54558720, "lr": 2.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665931324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13233055174350739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665946107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12799927592277527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54886400, "lr": 2.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665960750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12910166382789612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665960751, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784665960751, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784666043845, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6165855526924133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784666043846, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784666043846, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784666057999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12899810075759888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55214080, "lr": 2.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666073048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11958838999271393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666087610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12287405133247375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55541760, "lr": 2.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666102371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12231510877609253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666117051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1242271214723587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55869440, "lr": 2.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666131551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1267382651567459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666146181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11767017096281052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56197120, "lr": 2.8579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666160943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12362334132194519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666175827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.126207172870636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56524800, "lr": 2.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666190506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12434735149145126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666204990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11645860970020294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56852480, "lr": 2.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666219581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12434536218643188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666233920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1109306663274765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57180160, "lr": 2.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666248572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12544524669647217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666248573, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784666248573, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784666331132, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6281856894493103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784666331133, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784666331133, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784666345942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1175813302397728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57507840, "lr": 2.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666360790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11597001552581787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666375471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13036756217479706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57835520, "lr": 2.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666390467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12248865514993668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666405159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1241280660033226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58163200, "lr": 2.9579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666419750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11752587556838989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666433927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12046768516302109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58490880, "lr": 2.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666449048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12169107794761658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666463695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12163486331701279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58818560, "lr": 2.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666478546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11401055008172989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666493651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12078645080327988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59146240, "lr": 3.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666508393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12187737226486206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.0162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666523209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1241844892501831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59473920, "lr": 3.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666538291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11763212829828262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666538291, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784666538292, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784666621592, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6436138153076172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784666621593, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784666621594, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784666636479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11968015879392624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59801600, "lr": 3.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666651263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11195351928472519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666665923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11045076698064804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60129280, "lr": 3.0579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666680592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11325675249099731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.0662499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666695117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10887802392244339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60456960, "lr": 3.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666709828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12303686887025833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666724644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11633007973432541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60784640, "lr": 3.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666739817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12056350708007812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666754714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10309607535600662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61112320, "lr": 3.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666770117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11949978768825531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.1162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666785310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1125161349773407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61440000, "lr": 3.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666800433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11341500282287598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666815177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10968302190303802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61767680, "lr": 3.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666830004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11636824160814285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666830005, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784666830005, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784666912291, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6684707999229431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784666912292, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784666912292, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784666926651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11990544945001602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62095360, "lr": 3.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666941268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12004435807466507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.1662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666956033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1236724704504013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62423040, "lr": 3.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666970686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10591008514165878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666985290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11441670358181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62750720, "lr": 3.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666999766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11722967773675919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667014552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1193791851401329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63078400, "lr": 3.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667029276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11855858564376831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667043841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10891126096248627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63406080, "lr": 3.2245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667058401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11707554012537003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667072946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11651825904846191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63733760, "lr": 3.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667087064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11674211174249649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667101401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11279110610485077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64061440, "lr": 3.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667115980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12598268687725067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667115980, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784667115981, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784667197336, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6964637041091919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784667197337, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784667197337, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784667212066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12429320812225342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64389120, "lr": 3.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667226709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11657688766717911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667241255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11682773381471634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64716800, "lr": 3.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667256232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1227530911564827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667271005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1097065880894661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65044480, "lr": 3.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667285798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11137907952070236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667299937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11310146003961563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65372160, "lr": 3.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667314821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10829786211252213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667329864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11126122623682022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65699840, "lr": 3.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667345342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11614967882633209, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667360696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12282561510801315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66027520, "lr": 3.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667375718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11565254628658295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667391044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12656131386756897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66355200, "lr": 3.3745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667406051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11175753176212311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667406052, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784667406052, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784667488906, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7423039078712463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784667488907, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784667488907, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784667504084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12671229243278503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66682880, "lr": 3.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667518845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11370538175106049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.3995833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667533487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11704907566308975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67010560, "lr": 3.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667548488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11263636499643326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667563259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11872576922178268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67338240, "lr": 3.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667578281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1188865453004837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667593440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1214253306388855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67665920, "lr": 3.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667608570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12022525817155838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667623386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11640334874391556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67993600, "lr": 3.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667638961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11370821297168732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667653927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11649208515882492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68321280, "lr": 3.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667668699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11602320522069931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667683403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12517361342906952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68648960, "lr": 3.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667698478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11489777266979218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667698479, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784667698479, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784667783587, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7547053694725037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784667783588, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784667783588, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784667798334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12158678472042084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68976640, "lr": 3.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667813266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12049844115972519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667828159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11373749375343323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69304320, "lr": 3.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667843220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11460613459348679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667857997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10830501466989517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69632000, "lr": 3.5412500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667872943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11510661244392395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667887863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11830668896436691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69959680, "lr": 3.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667902444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11134132742881775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667917313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10956967622041702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70287360, "lr": 3.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667932136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10803871601819992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667947321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12329375743865967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70615040, "lr": 3.5912500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667962483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10657834261655807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667977370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1210867315530777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70942720, "lr": 3.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667992619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12068542838096619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667992621, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784667992621, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784668076814, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7593855857849121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784668076815, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784668076815, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784668091805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11957179754972458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71270400, "lr": 3.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668106505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11944931000471115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668121342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11537792533636093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71598080, "lr": 3.6412499999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668136619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11585044860839844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668151496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10338321328163147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71925760, "lr": 3.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668166313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1036469042301178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668181105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11889645457267761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72253440, "lr": 3.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668195845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11512161791324615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668210493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1037197932600975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72581120, "lr": 3.6912499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668225119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10322792828083038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668239464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10662948340177536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72908800, "lr": 3.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668254137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11211103945970535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668268838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11151934415102005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73236480, "lr": 3.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668283447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11480628699064255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668283487, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784668283487, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784668366774, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7612780332565308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784668366775, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784668366775, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784668381249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1042030081152916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73564160, "lr": 3.7412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668396116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10709629207849503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668411105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11499354988336563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73891840, "lr": 3.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668425854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11082373559474945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668440644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10753297060728073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74219520, "lr": 3.7745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668455860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11463724821805954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668470590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11304540187120438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74547200, "lr": 3.7912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668485193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11314819008111954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668499907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11601400375366211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74874880, "lr": 3.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668514463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11061184853315353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668528854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10223202407360077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75202560, "lr": 3.8245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668543432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11329970508813858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668557855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10728388279676437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75530240, "lr": 3.8412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668572216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10652810335159302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668572217, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784668572217, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784668654027, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7634466290473938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784668654028, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784668654028, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784668668496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1063060387969017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75857920, "lr": 3.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668683156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11356443166732788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668697742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0959916040301323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76185600, "lr": 3.8745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668712336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10372047126293182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668726857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11300912499427795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76513280, "lr": 3.8912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668741189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10489965230226517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668755683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08879072964191437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76840960, "lr": 3.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668770121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10907050967216492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668784659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10166241228580475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77168640, "lr": 3.9245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668799384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11808981746435165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668814200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10752058029174805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77496320, "lr": 3.9412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668828574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10072484612464905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668843527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10056548565626144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77824000, "lr": 3.9579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668858362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10740729421377182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668858362, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784668858363, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784668940878, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7640739679336548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784668940878, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784668940879, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784668955452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11603651195764542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78151680, "lr": 3.9745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668970199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11248176544904709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668984423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10324057936668396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78479360, "lr": 3.9912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668999792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10543646663427353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669014350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09887168556451797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78807040, "lr": 4.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669029188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10208022594451904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669043979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10591927170753479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79134720, "lr": 4.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669058753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10864003002643585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669073554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09735401719808578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79462400, "lr": 4.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669088548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10939139127731323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669103096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10501661151647568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79790080, "lr": 4.0579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669117778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10715468972921371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669132477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10835100710391998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80117760, "lr": 4.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669147308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11274756491184235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669147309, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784669147309, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784669230655, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7652957439422607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784669230656, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784669230656, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784669245250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11158257722854614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80445440, "lr": 4.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669259483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0993644967675209, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669274174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10296499729156494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80773120, "lr": 4.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669288995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10595682263374329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669303970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11075560003519058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81100800, "lr": 4.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669319081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11388204991817474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669333914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1005367636680603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81428480, "lr": 4.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669348859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10605353862047195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669363786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10718484222888947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81756160, "lr": 4.1579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669378511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10642769932746887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669393435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11253490298986435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82083840, "lr": 4.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669407630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09423387795686722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669422749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09836533665657043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82411520, "lr": 4.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669437514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072072833776474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669437515, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784669437515, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784669520563, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7670378088951111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784669520564, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784669520564, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784669534845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09248369932174683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82739200, "lr": 4.2079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669549464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10529439896345139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669563890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11054576188325882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83066880, "lr": 4.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669578534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09811338037252426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669592982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11023346334695816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83394560, "lr": 4.2412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669607605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11097199469804764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669622189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09711474925279617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83722240, "lr": 4.2579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669636688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11115062236785889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669651253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10540974140167236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84049920, "lr": 4.2745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669665678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10368084907531738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669680349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10809589177370071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84377600, "lr": 4.2912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669694623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10546012222766876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669709032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09706605225801468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84705280, "lr": 4.3079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669723729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10598652809858322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669723729, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784669723730, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784669806308, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7678406238555908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784669806309, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784669806309, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784669821192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10421591252088547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85032960, "lr": 4.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669836216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10477129369974136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669850957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10556849092245102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85360640, "lr": 4.3412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669865944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10440590232610703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669880877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09829280525445938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85688320, "lr": 4.3579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669895768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10007564723491669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669911015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10645541548728943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86016000, "lr": 4.3745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669925471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09659729897975922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669940402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10811535269021988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86343680, "lr": 4.3912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669955141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10868263989686966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669969787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11699599027633667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86671360, "lr": 4.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669984667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11491916328668594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669999348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11626185476779938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86999040, "lr": 4.4245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670013932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11506538093090057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670013933, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784670013933, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784670095528, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7688217759132385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784670095529, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784670095529, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784670109889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10056958347558975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87326720, "lr": 4.4412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670124690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11556973308324814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670139080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1073557510972023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87654400, "lr": 4.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670153801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10152551531791687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670168409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09061499685049057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87982080, "lr": 4.4745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670183136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09802178293466568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670197571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10440539568662643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88309760, "lr": 4.4912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670212310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10742942243814468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670226653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10768548399209976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88637440, "lr": 4.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670241205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10115746408700943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670256309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11328820139169693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88965120, "lr": 4.5245833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670271355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10057849436998367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670286293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10962004959583282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89292800, "lr": 4.5412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670300860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11145118623971939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.5495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670300861, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784670300861, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784670385081, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7691139578819275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784670385082, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784670385083, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784670399772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10810726881027222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89620480, "lr": 4.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670414575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10179270058870316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670429132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11044004559516907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89948160, "lr": 4.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670444004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09199096262454987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670458338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11475147306919098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90275840, "lr": 4.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670472665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10637982189655304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670488029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11076415330171585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90603520, "lr": 4.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670502961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11153299361467361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670518077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12168435752391815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90931200, "lr": 4.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670533055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10472878813743591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670547942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11139868199825287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91258880, "lr": 4.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670563128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12310938537120819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670578073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09968694299459457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91586560, "lr": 4.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670592960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11144192516803741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670592961, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784670592961, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784670674026, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7694743871688843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784670674027, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784670674027, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784670688919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10461154580116272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91914240, "lr": 4.6745833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670703883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11844455450773239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670718687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10921897739171982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92241920, "lr": 4.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670733791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10620278865098953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670748702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09662234038114548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92569600, "lr": 4.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670763604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1131836548447609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670778942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10618959367275238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92897280, "lr": 4.7245833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670794082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10602917522192001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670809360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11693654209375381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93224960, "lr": 4.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670824221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09949507564306259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670839443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12520316243171692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93552640, "lr": 4.7579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670854614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10756687819957733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670869639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1167549416422844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93880320, "lr": 4.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670884599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10780352354049683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670884599, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784670884600, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784670967915, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7695014476776123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784670967916, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784670967916, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784670983065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11440563946962357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94208000, "lr": 4.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670998165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11496679484844208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671013271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11147721856832504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94535680, "lr": 4.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671028603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11776152998209, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671043864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1142815351486206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94863360, "lr": 4.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671059079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11852502077817917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671074233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12731370329856873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95191040, "lr": 4.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671089120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12109313160181046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671104207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1163223385810852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95518720, "lr": 4.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671119708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11060924828052521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671134787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10829655081033707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95846400, "lr": 4.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671150083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10794391483068466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671165229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11530078947544098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96174080, "lr": 4.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671180608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1146334633231163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671180609, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784671180609, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784671263253, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7705512046813965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784671263254, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784671263254, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784671278647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11870265752077103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96501760, "lr": 4.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671293997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10753200203180313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671308978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1088855043053627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96829440, "lr": 4.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671324603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10817350447177887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671340010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1105554923415184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97157120, "lr": 4.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671355334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10940834879875183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671370580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1137402206659317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97484800, "lr": 4.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671385783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081015020608902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671400535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10614008456468582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97812480, "lr": 4.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671415134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10015848278999329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671430157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10425149649381638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98140160, "lr": 4.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671444929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09686174243688583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671459690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10431080311536789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98467840, "lr": 5.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671474297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10053648054599762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671474298, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784671474299, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784671555517, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7714986801147461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784671555518, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784671555518, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784671570019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10965543240308762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98795520, "lr": 5.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671584690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10135402530431747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671599427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09624281525611877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99123200, "lr": 5.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671614149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.108899787068367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671628952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1154002919793129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99450880, "lr": 5.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671643671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10438907146453857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671658434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1042509377002716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99778560, "lr": 5.074583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671673232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10457712411880493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671687917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1020629033446312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100106240, "lr": 5.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671702874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.120290108025074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671717803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11682896316051483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100433920, "lr": 5.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671732647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10139837861061096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671747836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10023221373558044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100761600, "lr": 5.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671762598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12038003653287888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671762599, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784671762600, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784671843581, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7718601822853088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784671843582, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784671843582, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784671858852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11046811938285828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101089280, "lr": 5.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671873715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09648442268371582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671888378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10467661172151566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101416960, "lr": 5.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671903082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10286858677864075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671917587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10853414237499237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101744640, "lr": 5.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671932348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10454616695642471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.182916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671947235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10606733709573746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102072320, "lr": 5.191249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671962030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11239464581012726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671977121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10467716306447983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102400000, "lr": 5.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671992075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10801473259925842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672006905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10542131215333939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102727680, "lr": 5.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672021615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11506766080856323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672036121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10694839060306549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103055360, "lr": 5.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672050893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10216545313596725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672050893, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784672050893, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784672132830, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7722415328025818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784672132832, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784672132832, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784672147713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09983208775520325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103383040, "lr": 5.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672162774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0942969098687172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672177423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0959373489022255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103710720, "lr": 5.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672192405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09634906053543091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.282916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672207114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09659279882907867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104038400, "lr": 5.291249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672222093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0943305641412735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672236847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10039058327674866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104366080, "lr": 5.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672251672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09708424657583237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672266960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10355392098426819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104693760, "lr": 5.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672282136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10830309987068176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672297668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10488438606262207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105021440, "lr": 5.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672312921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10255158692598343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672328122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11410194635391235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105349120, "lr": 5.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672343679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11099975556135178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672343680, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784672343680, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784672425169, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7725469470024109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784672425170, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784672425170, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784672440323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09853262454271317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105676800, "lr": 5.374583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672455337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723048448562622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.382916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672470590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1093062162399292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106004480, "lr": 5.391249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672485236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10144847631454468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672499948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10011178255081177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106332160, "lr": 5.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672514877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1077674925327301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672529709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11075367778539658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106659840, "lr": 5.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672544571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1064181923866272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672559752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10243790596723557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106987520, "lr": 5.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672574544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10129999369382858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672589423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10001054406166077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107315200, "lr": 5.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672604093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10247248411178589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672619037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10931456089019775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107642880, "lr": 5.474583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672633745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11418963223695755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.482916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672633747, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784672633747, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784672715442, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7726956605911255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784672715444, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784672715444, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784672730247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11019807308912277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107970560, "lr": 5.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672745051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10644898563623428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672759820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09108684957027435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108298240, "lr": 5.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672774388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10261962562799454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672789322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10661717504262924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108625920, "lr": 5.524583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672803917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10802407562732697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672818642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11020546406507492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108953600, "lr": 5.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672833598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1029590368270874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672848645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09975637495517731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109281280, "lr": 5.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672863722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1041017696261406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672878865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10294979810714722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109608960, "lr": 5.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672893691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10811974108219147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672908978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10513974726200104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109936640, "lr": 5.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672924126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10905253142118454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672924127, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784672924127, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784673006418, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7729923725128174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784673006419, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784673006419, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784673021419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10360193252563477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110264320, "lr": 5.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673036278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1027137041091919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673051012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10422030091285706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110592000, "lr": 5.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673065387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10288931429386139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673080530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10720425099134445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110919680, "lr": 5.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673095089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10065748542547226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673109906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09831822663545609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111247360, "lr": 5.657916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673125009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10228090733289719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.666250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673139955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11204880475997925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111575040, "lr": 5.674583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673154706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10503429919481277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673169294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10368529707193375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111902720, "lr": 5.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673184062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10243048518896103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673199135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10407949239015579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112230400, "lr": 5.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673214065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10649300366640091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673214066, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784673214066, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784673296957, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7732399106025696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784673296959, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784673296959, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784673311964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10851900279521942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112558080, "lr": 5.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673327111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10147605091333389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673342128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10409355908632278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112885760, "lr": 5.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673357192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10110916197299957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673372308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10461244732141495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113213440, "lr": 5.757916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673387039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11255928874015808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.766250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673401648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10104639828205109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113541120, "lr": 5.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673416119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11001124233007431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673430986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11312329769134521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113868800, "lr": 5.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673446432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10655687004327774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673461459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1032642051577568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114196480, "lr": 5.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673476665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10473445057868958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673492116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1051735207438469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114524160, "lr": 5.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673507390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09851009398698807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673507390, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784673507391, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784673590461, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773639976978302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784673590462, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784673590462, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784673605676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10224621742963791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114851840, "lr": 5.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673620628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09954150766134262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673635475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10643717646598816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115179520, "lr": 5.857916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673649877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10064958781003952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673665034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10787218809127808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115507200, "lr": 5.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673679685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10937006771564484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673694496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10717037320137024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115834880, "lr": 5.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673709432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11546512693166733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673724276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11394838243722916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116162560, "lr": 5.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673739011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10022850334644318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673753748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10410064458847046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116490240, "lr": 5.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673768517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10271741449832916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673783202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1050671711564064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116817920, "lr": 5.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673797824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10719837248325348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673797824, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784673797825, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784673881151, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7735855579376221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784673881152, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784673881152, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784673895802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10202959179878235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117145600, "lr": 5.957916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673910490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10041125863790512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673925526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10857228189706802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117473280, "lr": 5.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673940283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1050824448466301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673955003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09375472366809845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117800960, "lr": 5.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673970187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11273176968097687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673984945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10117126256227493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118128640, "lr": 6.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673999786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10779134184122086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674014761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1033160388469696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118456320, "lr": 6.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674029987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10398197174072266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674044917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10901445895433426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118784000, "lr": 6.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674060014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11414525657892227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674075200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10616274178028107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119111680, "lr": 6.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674090479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10294228047132492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674090480, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784674090481, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784674172362, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773617684841156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784674172363, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784674172363, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784674187717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11273746192455292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119439360, "lr": 6.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674202814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085951030254364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674217795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10924539715051651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119767040, "lr": 6.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674232805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10682352632284164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674247507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10121100395917892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120094720, "lr": 6.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674262128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10721360146999359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674277571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11847499758005142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120422400, "lr": 6.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674292638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10908778756856918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674307857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11632179468870163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120750080, "lr": 6.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674323250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12299729138612747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674338450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10734564810991287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121077760, "lr": 6.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674353579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11058859527111053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674368778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11474892497062683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121405440, "lr": 6.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674383752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11010472476482391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674383754, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784674383754, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784674465512, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7732471823692322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784674465513, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784674465513, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784674480709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11348290741443634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121733120, "lr": 6.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674496010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10937212407588959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674511055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11017338186502457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122060800, "lr": 6.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674526210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10686174035072327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674541238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1041201576590538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122388480, "lr": 6.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674556438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11361793428659439, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674571514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10733374953269958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122716160, "lr": 6.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674586747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11473297327756882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674602139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11260194331407547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123043840, "lr": 6.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674617576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12019200623035431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674633077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1057654544711113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123371520, "lr": 6.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674648433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11758798360824585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674663589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10736441612243652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123699200, "lr": 6.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674679079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11927633732557297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674679079, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784674679080, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784674762041, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7736504077911377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784674762042, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784674762042, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784674777272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1192304790019989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124026880, "lr": 6.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674792600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792165249586105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674807701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1194903627038002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124354560, "lr": 6.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674822433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1136130839586258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674837348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10736623406410217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124682240, "lr": 6.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674852887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11526527255773544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674867866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11154192686080933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125009920, "lr": 6.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674883293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11698611825704575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674898697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11277275532484055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125337600, "lr": 6.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674913997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10606300830841064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674929201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11257407814264297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125665280, "lr": 6.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674944573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10583767294883728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674959914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11678359657526016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125992960, "lr": 6.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674975581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10515813529491425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674975581, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784674975582, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784675059730, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7740634083747864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784675059731, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784675059731, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784675074961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1097182035446167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126320640, "lr": 6.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675090512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1204419806599617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675105958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10975834727287292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126648320, "lr": 6.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675121768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11349661648273468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675137189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11624829471111298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126976000, "lr": 6.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675152458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10927244275808334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675167785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10818178951740265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127303680, "lr": 6.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675183228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10977223515510559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.482916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675198899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11297088116407394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127631360, "lr": 6.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675214219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11182568967342377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675229586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1103486642241478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127959040, "lr": 6.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675245021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11090102791786194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675260491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11104629188776016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128286720, "lr": 6.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675275923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10883808881044388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675275924, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784675275924, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784675361262, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7744095325469971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784675361263, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784675361263, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784675376373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11791641265153885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128614400, "lr": 6.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675391683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11013548076152802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675406691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11073222756385803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128942080, "lr": 6.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675421434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10278283804655075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675436241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10585616528987885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129269760, "lr": 6.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675451465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11151833087205887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675466065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10789678245782852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129597440, "lr": 6.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675480710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725130885839462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675495287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09857775270938873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129925120, "lr": 6.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675509898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10173670947551727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675524433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10521489381790161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130252800, "lr": 6.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675538863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1048695519566536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675553597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10977935045957565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130580480, "lr": 6.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675568508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11537409573793411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675568509, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784675568510, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784675652270, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746921181678772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784675652270, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784675652271, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784675666904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11451999843120575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130908160, "lr": 6.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675681607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10513345152139664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675696508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1005859375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131235840, "lr": 6.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675711456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09214384108781815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675726230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11428915709257126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131563520, "lr": 6.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675741093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10381657630205154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675755935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10235056281089783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131891200, "lr": 6.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675770976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11278576403856277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675785802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10673633217811584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132218880, "lr": 6.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675800847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10341820120811462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675815753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10935336351394653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132546560, "lr": 6.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675830769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11064570397138596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675845894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1074945405125618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132874240, "lr": 6.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675860845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1069459468126297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675860846, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784675860846, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784675943955, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746965289115906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784675943956, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784675943956, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784675958786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10978227853775024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133201920, "lr": 6.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675973692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10315960645675659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675988300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11056045442819595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133529600, "lr": 6.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676002993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09896985441446304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676018441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11589457839727402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133857280, "lr": 6.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676033640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11002589017152786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676048653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10343604534864426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134184960, "lr": 6.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676064018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10795431584119797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676079375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10853412747383118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134512640, "lr": 6.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676094260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10830919444561005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676109422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11023123562335968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134840320, "lr": 6.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676124568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0994846373796463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676139460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10669802129268646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135168000, "lr": 6.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676154333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08741606026887894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676154334, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784676154335, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784676238227, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7749086022377014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784676238228, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784676238229, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784676253300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0953153744339943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135495680, "lr": 6.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676268128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1109287366271019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676283098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10712539404630661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135823360, "lr": 6.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676297925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10112056136131287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676312959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10215352475643158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136151040, "lr": 6.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676327923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09023141860961914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676342821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10361332446336746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136478720, "lr": 6.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676357603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09746335446834564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676372507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10415627062320709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136806400, "lr": 6.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676387796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10638593137264252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676403085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11023956537246704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137134080, "lr": 6.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676418343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10811512917280197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676433259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10638315975666046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137461760, "lr": 6.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676448516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10349105298519135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676448517, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784676448518, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784676530496, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7749032378196716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784676530496, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784676530497, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784676545543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0962458997964859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137789440, "lr": 7.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676560754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10515347123146057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676575758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09469259530305862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138117120, "lr": 7.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676590816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10254253447055817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676605920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09776675701141357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138444800, "lr": 7.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676620504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10037637501955032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676635061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10971267521381378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138772480, "lr": 7.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676650598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10162622481584549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.066249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676665748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10901549458503723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139100160, "lr": 7.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676681028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10980881005525589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676696711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10647410899400711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139427840, "lr": 7.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676712426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10422652214765549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676727827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10919458419084549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139755520, "lr": 7.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676743313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10877827554941177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676743314, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784676743314, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784676825376, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.775143563747406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784676825377, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784676825378, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784676840612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10621056705713272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140083200, "lr": 7.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676856064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10444435477256775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676871426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10352024435997009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140410880, "lr": 7.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676886802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10025523602962494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676902090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10717789083719254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140738560, "lr": 7.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676917595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09599078446626663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676933624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10522492974996567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141066240, "lr": 7.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676948942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10301094502210617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676964153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11040294915437698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141393920, "lr": 7.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676979571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09890804439783096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676994685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09766446799039841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141721600, "lr": 7.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677009725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10201621800661087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677024972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09080013632774353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142049280, "lr": 7.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677040298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10708419233560562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677040298, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784677040299, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784677124150, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.775304913520813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784677124151, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784677124152, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784677139376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09954556077718735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142376960, "lr": 7.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677154833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10124719887971878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677170194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09938570857048035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142704640, "lr": 7.257916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677185881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10976489633321762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677201652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10039476305246353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143032320, "lr": 7.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677217564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10231221467256546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677233156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10276664793491364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143360000, "lr": 7.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677248755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1052609235048294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677264480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10147903114557266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143687680, "lr": 7.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677279655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11177193373441696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677295095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10104475915431976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144015360, "lr": 7.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677310054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10817451030015945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677325038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10559087246656418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144343040, "lr": 7.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677340346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10621187835931778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677340347, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784677340347, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784677423718, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7753314971923828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784677423719, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784677423719, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784677438916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1065097525715828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144670720, "lr": 7.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677454009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09819330275058746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677468940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10680083930492401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144998400, "lr": 7.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677484149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10659655928611755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677499314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.102410688996315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145326080, "lr": 7.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677514100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10710478574037552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677529049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09765323996543884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145653760, "lr": 7.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677544242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09798505902290344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677559272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10557659715414047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145981440, "lr": 7.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677574325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10427972674369812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677589511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10264332592487335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146309120, "lr": 7.441250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677604792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10991820693016052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677620044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10672436654567719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146636800, "lr": 7.457916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677635343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09976138919591904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677635344, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784677635344, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784677716917, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7755942940711975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784677716918, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784677716918, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784677731948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10307260602712631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146964480, "lr": 7.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677747104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10531698167324066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677762319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09444250911474228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147292160, "lr": 7.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677777822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09853300452232361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677792774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10186491161584854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147619840, "lr": 7.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677808427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10024400800466537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677823742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723703354597092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147947520, "lr": 7.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677839214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10975366085767746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677854648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10889792442321777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148275200, "lr": 7.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677870222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09912896156311035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677885642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10250487923622131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148602880, "lr": 7.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677900605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09898407012224197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677916206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11199508607387543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148930560, "lr": 7.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677931319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11050741374492645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677931319, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784677931320, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784678015042, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7757527232170105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784678015043, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784678015043, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784678029884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10561197996139526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149258240, "lr": 7.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678045459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10495955497026443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678060483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10634572058916092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149585920, "lr": 7.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678075538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11103934049606323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149749760, "lr": 7.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678091066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10648325085639954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149913600, "lr": 7.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678106653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10857702046632767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678121943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10758531838655472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150241280, "lr": 7.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678137242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09703832119703293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150405120, "lr": 7.649583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678152560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10684499889612198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150568960, "lr": 7.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678167714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10363306850194931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678182941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10847792774438858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150896640, "lr": 7.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678197815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10364038497209549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151060480, "lr": 7.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678213254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10667382925748825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151224320, "lr": 7.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678228402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09583313763141632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678228449, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784678228450, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784678310928, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7757769227027893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784678310929, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784678310929, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784678326055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09963405877351761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151552000, "lr": 7.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678341153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10568317025899887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151715840, "lr": 7.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678356309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1026768758893013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151879680, "lr": 7.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678371438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10423463582992554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678386366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10845762491226196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152207360, "lr": 7.741249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678401120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12268467992544174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152371200, "lr": 7.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678415975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10805606096982956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152535040, "lr": 7.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678431006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09912183880805969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678446406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11126422882080078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152862720, "lr": 7.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678461562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11117323487997055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153026560, "lr": 7.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678476976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1148165613412857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153190400, "lr": 7.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678492337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09866204112768173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678507876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10655929148197174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153518080, "lr": 7.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678523352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10412930697202682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153681920, "lr": 7.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678523397, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784678523397, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784678605595, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7761412262916565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784678605597, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784678605597, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784678620996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09763433039188385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153845760, "lr": 7.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678636426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09793169796466827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.832916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678651745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1125013455748558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154173440, "lr": 7.841249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678667141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11754307895898819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154337280, "lr": 7.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678682300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11391586065292358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154501120, "lr": 7.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678697211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11075018346309662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678712858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072455644607544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154828800, "lr": 7.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678728169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12043075263500214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154992640, "lr": 7.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678743411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10489626973867416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155156480, "lr": 7.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678759042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10830383002758026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678774522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11078112572431564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155484160, "lr": 7.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678789813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11024985462427139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155648000, "lr": 7.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678805282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11307384073734283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155811840, "lr": 7.924583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678820859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11103109270334244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.932916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678820859, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784678820860, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784678905278, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7762284278869629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784678905279, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784678905279, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784678920850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10720402747392654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156139520, "lr": 7.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678936345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11021386831998825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156303360, "lr": 7.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678951959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11427932977676392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156467200, "lr": 7.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678967300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11022694408893585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678982627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10821555554866791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156794880, "lr": 7.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678997950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10788659751415253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156958720, "lr": 7.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679013131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1014896109700203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157122560, "lr": 7.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679028088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11270152032375336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679043342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12082923948764801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157450240, "lr": 8.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679058595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1092170849442482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157614080, "lr": 8.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679073659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10843342542648315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157777920, "lr": 8.024583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679089150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10411018133163452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.032916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679104512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10388119518756866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158105600, "lr": 8.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679119808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10880380868911743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158269440, "lr": 8.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679119809, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784679119809, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784679203215, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7763116359710693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784679203216, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784679203216, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784679218682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11401378363370895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158433280, "lr": 8.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679234254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11880327016115189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679249610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11382234841585159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158760960, "lr": 8.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679265151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11236588656902313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158924800, "lr": 8.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679280664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11115028709173203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159088640, "lr": 8.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679295913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11911268532276154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679311354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11187660694122314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159416320, "lr": 8.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679326309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10883311182260513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159580160, "lr": 8.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679342030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11048910766839981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159744000, "lr": 8.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679357589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0989348292350769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679373081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11035005003213882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160071680, "lr": 8.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679388979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10917095839977264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160235520, "lr": 8.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679404640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1041174903512001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160399360, "lr": 8.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679419824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11034511774778366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679419824, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784679419824, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784679503458, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7765874862670898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784679503458, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784679503458, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784679518845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11279422789812088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160727040, "lr": 8.174583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679534523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12851126492023468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160890880, "lr": 8.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679550111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10785603523254395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161054720, "lr": 8.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679565464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11212355643510818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679580776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11270568519830704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161382400, "lr": 8.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679596356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11163929849863052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161546240, "lr": 8.216250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679611910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11419732123613358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161710080, "lr": 8.224583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679627411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11242497712373734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679642773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10601702332496643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162037760, "lr": 8.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679658215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10812132060527802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162201600, "lr": 8.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679673930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12960118055343628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162365440, "lr": 8.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679689429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11293582618236542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679705323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11252341419458389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162693120, "lr": 8.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679721087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11593203246593475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162856960, "lr": 8.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679721088, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784679721088, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784679805805, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7768086791038513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784679805806, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784679805806, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784679821109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09503095597028732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163020800, "lr": 8.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679836716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11143868416547775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679852493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1220734715461731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163348480, "lr": 8.307916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679868349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10689578205347061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163512320, "lr": 8.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679884212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10924811661243439, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163676160, "lr": 8.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679899976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10748913884162903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679915602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11170747876167297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164003840, "lr": 8.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679930850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11309115588665009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164167680, "lr": 8.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679946180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10334143787622452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164331520, "lr": 8.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679961221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11789017915725708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679976465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071835458278656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164659200, "lr": 8.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679991924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11154791712760925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164823040, "lr": 8.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680007453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09195210039615631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164986880, "lr": 8.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680022579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10329702496528625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680022580, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784680022580, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784680106513, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7772626876831055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784680106514, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784680106514, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784680121847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10152880102396011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165314560, "lr": 8.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680137108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09651099145412445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165478400, "lr": 8.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680152184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11447761952877045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165642240, "lr": 8.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680167279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10771961510181427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680182341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10951607674360275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165969920, "lr": 8.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680197745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10393239557743073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166133760, "lr": 8.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680213117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10801416635513306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166297600, "lr": 8.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680228647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11332305520772934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680243975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09924273192882538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166625280, "lr": 8.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680259478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09526634216308594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166789120, "lr": 8.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680274839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09677182883024216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166952960, "lr": 8.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680290186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09682827442884445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680305331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10239659249782562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167280640, "lr": 8.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680320699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10042516142129898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167444480, "lr": 8.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680320699, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784680320700, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784680405288, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7774080038070679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784680405288, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784680405288, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784680420840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10343273729085922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167608320, "lr": 8.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680436464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10255222022533417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680451903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10411445796489716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167936000, "lr": 8.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680467434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10670411586761475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168099840, "lr": 8.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680482917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10204163938760757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168263680, "lr": 8.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680498498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0992836132645607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680514084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10929471254348755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168591360, "lr": 8.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680529274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10531695932149887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168755200, "lr": 8.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680544573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10690239071846008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168919040, "lr": 8.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680559884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10564370453357697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680575341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10427649319171906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169246720, "lr": 8.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680590251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10790415853261948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169410560, "lr": 8.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680605033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10589414089918137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169574400, "lr": 8.624583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680620394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1129705086350441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680620433, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784680620433, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784680704708, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7776393890380859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784680704709, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784680704709, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784680719690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10803785175085068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169902080, "lr": 8.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680734819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11147508770227432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170065920, "lr": 8.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680750257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10699281841516495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170229760, "lr": 8.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680765755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10739461332559586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680780824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1000911146402359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170557440, "lr": 8.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680796119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1021203100681305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170721280, "lr": 8.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680811622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09574586153030396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170885120, "lr": 8.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680826905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10733816027641296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680842068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09935653209686279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171212800, "lr": 8.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680857483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1024320125579834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171376640, "lr": 8.716249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680872991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10142025351524353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171540480, "lr": 8.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680887995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10294774174690247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680902881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09936762601137161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171868160, "lr": 8.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680918128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1037883311510086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172032000, "lr": 8.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680918172, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784680918173, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784681002042, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7779293060302734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784681002043, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784681002043, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784681017169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09783514589071274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172195840, "lr": 8.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681032193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10709939152002335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681047131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09923037141561508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172523520, "lr": 8.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681062108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10263077914714813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172687360, "lr": 8.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681076903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09431903064250946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172851200, "lr": 8.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681091909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10648668557405472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681107052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10346520692110062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173178880, "lr": 8.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681122001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10339540243148804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173342720, "lr": 8.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681137283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10496792942285538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173506560, "lr": 8.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681152347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11204688251018524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681167774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09604917466640472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173834240, "lr": 8.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681182971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111388199031353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173998080, "lr": 8.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681198156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11220068484544754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174161920, "lr": 8.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681213070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10160427540540695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681213118, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784681213118, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784681296458, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7782747745513916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784681296459, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784681296459, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784681311437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09766505658626556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174489600, "lr": 8.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681326652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10694130510091782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174653440, "lr": 8.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681341918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10076536983251572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174817280, "lr": 8.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681356530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10592563450336456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681371759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10037766396999359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175144960, "lr": 8.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681386841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10890448093414307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175308800, "lr": 8.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681401825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10011798143386841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175472640, "lr": 8.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681417091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09886784851551056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681432231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10863588750362396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175800320, "lr": 8.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681447324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10879096388816833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175964160, "lr": 8.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681462184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09990335255861282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176128000, "lr": 8.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681477283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09396615624427795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681492135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.094964899122715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176455680, "lr": 8.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681506986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10560403019189835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176619520, "lr": 8.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681506987, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784681506987, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784681591095, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7788073420524597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784681591096, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784681591097, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784681606126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10316324979066849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176783360, "lr": 8.991249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681621074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11352656781673431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681636219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10107781738042831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177111040, "lr": 9.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681651264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1051739752292633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177274880, "lr": 9.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681666145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11057046800851822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177438720, "lr": 9.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681680929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10067886114120483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.032916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681695650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10916092246770859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177766400, "lr": 9.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681710359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10080328583717346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177930240, "lr": 9.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681725228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10689371079206467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178094080, "lr": 9.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681739960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10121692717075348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681754774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09728371351957321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178421760, "lr": 9.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681769902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11156788468360901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178585600, "lr": 9.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681785128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11934922635555267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178749440, "lr": 9.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681800174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10894174128770828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681800175, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784681800175, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784681884044, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7792158722877502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784681884045, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784681884045, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784681899190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11561653017997742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179077120, "lr": 9.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681914173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10069450736045837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179240960, "lr": 9.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681929498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09992402046918869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179404800, "lr": 9.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681944701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10044783353805542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.132916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681959877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10488776117563248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179732480, "lr": 9.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681974926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0989568904042244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179896320, "lr": 9.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681989842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1002737358212471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180060160, "lr": 9.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682005014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10610240697860718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682019660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10545233637094498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180387840, "lr": 9.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682034181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09394663572311401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180551680, "lr": 9.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682049703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10442201048135757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180715520, "lr": 9.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682064352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10183911770582199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682079207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09321044385433197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181043200, "lr": 9.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682094201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1003878191113472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181207040, "lr": 9.216249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682094202, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784682094202, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784682178396, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7795929908752441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784682178397, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784682178397, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784682193421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11068933457136154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181370880, "lr": 9.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682208375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1133204847574234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682223027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10060106962919235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181698560, "lr": 9.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682237926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10647847503423691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181862400, "lr": 9.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682252812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10315808653831482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182026240, "lr": 9.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682267478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10255585610866547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682282167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10667102783918381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182353920, "lr": 9.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682296871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10837776958942413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182517760, "lr": 9.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682311604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09762067347764969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182681600, "lr": 9.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682326255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09731391072273254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682341108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11086210608482361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183009280, "lr": 9.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682355561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09824257344007492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183173120, "lr": 9.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682370524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10210739076137543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183336960, "lr": 9.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682385083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1027718037366867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682385084, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784682385084, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784682467364, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7800740599632263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784682467365, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784682467365, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784682482139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1015990749001503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183664640, "lr": 9.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682496793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10785435885190964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183828480, "lr": 9.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682511603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11173856258392334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183992320, "lr": 9.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682526932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09399579465389252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682542103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10329471528530121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184320000, "lr": 9.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682557235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10761809349060059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184483840, "lr": 9.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682572499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11576589941978455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184647680, "lr": 9.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682587434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10107108950614929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682602649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10560904443264008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184975360, "lr": 9.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682618115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10969234257936478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185139200, "lr": 9.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682633399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09699670970439911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185303040, "lr": 9.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682648632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10736818611621857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682663983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09823979437351227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185630720, "lr": 9.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682678911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10403389483690262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185794560, "lr": 9.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682678912, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784682678912, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784682763181, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7804328799247742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784682763182, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784682763182, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784682778625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11002206057310104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185958400, "lr": 9.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682793521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10431156307458878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682807939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09204374253749847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186286080, "lr": 9.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682823472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10003985464572906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186449920, "lr": 9.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682838476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084296777844429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186613760, "lr": 9.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682853526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11071561276912689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682868826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09570522606372833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186941440, "lr": 9.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682883855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10405900329351425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187105280, "lr": 9.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682898950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10565535724163055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187269120, "lr": 9.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682913792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09919320046901703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682928558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10401365906000137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187596800, "lr": 9.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682943465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10425493866205215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187760640, "lr": 9.549583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682958204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11085358262062073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187924480, "lr": 9.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682973145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11432752758264542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682973146, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784682973147, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784683058382, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7805263996124268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784683058383, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784683058383, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784683073154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1092694029211998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188252160, "lr": 9.574583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683088361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1045776978135109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188416000, "lr": 9.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683103193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10457926988601685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188579840, "lr": 9.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683118305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10843704640865326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683133087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10196291655302048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188907520, "lr": 9.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683148143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09715113788843155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189071360, "lr": 9.616249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683163127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10260411351919174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189235200, "lr": 9.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683177898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11221269518136978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683192898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1009652242064476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189562880, "lr": 9.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683207807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09509485960006714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189726720, "lr": 9.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683222801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10484038293361664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189890560, "lr": 9.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683238113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11214220523834229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.666249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683253452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10161492228507996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190218240, "lr": 9.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683268456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11829638481140137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190382080, "lr": 9.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683268457, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784683268457, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784683353642, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7810918092727661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784683353643, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784683353643, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784683368946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10011789202690125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190545920, "lr": 9.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683383871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09712893515825272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683398923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1057550236582756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190873600, "lr": 9.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683414175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1123063862323761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191037440, "lr": 9.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683429420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11755997687578201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191201280, "lr": 9.724583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683444732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09612757712602615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683459823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10550062358379364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191528960, "lr": 9.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683474650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10121006518602371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191692800, "lr": 9.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683489832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10099315643310547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191856640, "lr": 9.757916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683504796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10415304452180862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683519434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10611975938081741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192184320, "lr": 9.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683535064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10938828438520432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192348160, "lr": 9.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683550152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0993092805147171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192512000, "lr": 9.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683565396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11497005075216293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.799583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683565397, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784683565397, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784683649664, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7811068892478943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784683649665, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784683649665, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784683665065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11183054000139236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192839680, "lr": 9.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683680573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11075765639543533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193003520, "lr": 9.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683695900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10612843930721283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193167360, "lr": 9.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683711308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12034787237644196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683726624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1159222424030304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193495040, "lr": 9.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683742119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10198116302490234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193658880, "lr": 9.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683757584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11811933666467667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193822720, "lr": 9.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683772711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11363101750612259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.866249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683788131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10510976612567902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194150400, "lr": 9.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683803708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10692610591650009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194314240, "lr": 9.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683819371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10712676495313644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194478080, "lr": 9.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683834793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10615044832229614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683850488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10715014487504959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194805760, "lr": 9.907916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683865582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11581508070230484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194969600, "lr": 9.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683865582, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784683865583, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784683947677, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.78179532289505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784683947678, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784683947678, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784683963191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11202171444892883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195133440, "lr": 9.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683978404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10343202203512192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683993840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10241885483264923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195461120, "lr": 9.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784684009483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11674439162015915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195624960, "lr": 9.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784684025137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10817829519510269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195788800, "lr": 9.957916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784684040805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10598837584257126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784684056623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10463307052850723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196116480, "lr": 9.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784684072425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11353691667318344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196280320, "lr": 9.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784684088520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11831803619861603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196444160, "lr": 9.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784684104358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10456200689077377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.999583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784684120017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.116880401968956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196771840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684135537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1120956763625145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196935680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684150679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10783299803733826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197099520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684166277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11544512212276459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684166277, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784684166277, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784684248365, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7818892002105713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784684248366, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784684248366, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784684263627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10039455443620682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197427200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684279191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11077302694320679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197591040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684294784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11435970664024353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197754880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684310258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10087068378925323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684326444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09957294166088104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198082560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684342053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10812903195619583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198246400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684357556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09831515699625015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198410240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684372799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10788055509328842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684388270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10078393667936325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198737920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684403414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10532651841640472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198901760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684418729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11434761434793472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199065600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684433994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10971762984991074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684449058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11690615117549896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199393280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684464513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1128171756863594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199557120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684464513, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784684464514, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784684546423, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.782130777835846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784684546427, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784684546427, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784684561702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1119459718465805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199720960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684577134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10697105526924133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684592603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11076167970895767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200048640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684608079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11665022373199463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200212480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684623456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11178120225667953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200376320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684639117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11721538007259369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684655304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09928639978170395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200704000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684670993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10822805017232895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200867840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684686980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1069592610001564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201031680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684702675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10864470899105072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684718515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1181265339255333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201359360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684734227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11239174008369446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201523200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684749582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10335464030504227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201687040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684765179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11128195375204086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684765180, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784684765181, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784684848251, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7820969820022583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784684848252, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784684848252, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784684863560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10538104176521301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202014720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684879304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1103857234120369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202178560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684894536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11958566308021545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202342400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684910085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10610470920801163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684925047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10548275709152222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202670080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684940227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10143610835075378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202833920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684955929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1064373031258583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202997760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684971146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10962782055139542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684986600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11319278180599213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203325440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685001735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09416196495294571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203489280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685017164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11671292036771774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203653120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685032395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1007932648062706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685047318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09857542812824249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203980800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685062265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10781706124544144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204144640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685062266, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784685062267, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784685146098, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7822530269622803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784685146099, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784685146099, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784685161113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1023843064904213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204308480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685176378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09373998641967773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685191620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09667449444532394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204636160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685206786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10839026421308517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204800000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685221952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10827963799238205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204963840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685236984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11202222853899002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685251905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09572867304086685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205291520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685266972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10642511397600174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205455360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685282073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10572035610675812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205619200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685297029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11061465740203857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685312061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09853202849626541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205946880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685327458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09879030287265778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206110720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685342634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11000529676675797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206274560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685357650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09834957122802734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685357651, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784685357652, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784685441418, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7825146317481995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784685441419, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784685441419, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784685456830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10540813207626343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206602240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685472007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09572780877351761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206766080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685487403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10388844460248947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206929920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685502866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09606749564409256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685518437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10235074162483215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207257600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685533802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11141018569469452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207421440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685549247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10672280937433243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207585280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685564328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0954878032207489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685579761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1141490638256073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207912960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685595281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10395628958940506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208076800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685610192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1025371253490448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208240640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685626053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10084431618452072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685640984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10234826058149338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208568320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685656163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10125065594911575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208732160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685656163, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784685656164, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784685741406, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7828105688095093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784685741408, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784685741408, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784685756756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10069417953491211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208896000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685772145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09909168630838394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685787406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10224876552820206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209223680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685802512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10217224806547165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209387520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685817810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09870462119579315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209551360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685833064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10109435766935349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685848148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0961841568350792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209879040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685863233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09903848171234131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210042880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685878543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10319586843252182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210206720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685893716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10827502608299255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685908733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10448689013719559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210534400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685923838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10755237191915512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210698240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685939159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10178631544113159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210862080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685954140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09564744681119919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685954141, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784685954141, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784686038246, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7830361723899841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784686038247, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784686038247, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784686053691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09401482343673706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211189760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686068828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09889029711484909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211353600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686083994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09711940586566925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211517440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686098932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09965526312589645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686113935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1032312661409378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211845120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686129487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09571346640586853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212008960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686144535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10243865102529526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212172800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686159600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09794268012046814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686174527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09822866320610046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212500480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686189494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10398364812135696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212664320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686204537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0928955003619194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212828160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686219877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10720173269510269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686235263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1043834462761879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213155840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686250489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10818883031606674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213319680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686250490, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784686250490, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784686332950, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7834908962249756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784686332950, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784686332951, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784686347924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10419876128435135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213483520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686362987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10144037753343582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686378015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1051105260848999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213811200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686393560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10270301252603531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213975040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686408377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0964488685131073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214138880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686423229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1044936329126358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686438649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09517260640859604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214466560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686453871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11367480456829071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214630400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686469111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09831000864505768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214794240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686484841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0897345021367073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686500484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09996048361063004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215121920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686516119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10454338043928146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215285760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686531452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1034594178199768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215449600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686546620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10945401340723038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686546621, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784686546622, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784686626840, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7835420370101929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784686626840, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784686626841, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784686641903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10071173310279846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215777280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686656839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10554233193397522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215941120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686671781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09723325818777084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216104960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686686977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09968752413988113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686702198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09987471997737885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216432640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686717160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09418851137161255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216596480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686732356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10431268066167831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216760320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686747741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10175753384828568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686762673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10019813477993011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217088000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686778012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1096058040857315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217251840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686792800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09763608127832413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217415680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686808173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09986037015914917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217579520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686823501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09915543347597122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217743360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686838856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1045573279261589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217907200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686838856, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784686838857, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784686920703, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7837992906570435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784686920704, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784686920704, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784686935937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10048116743564606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218071040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686951270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09457116574048996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218234880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686966688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09790826588869095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218398720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686981887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1079886183142662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218562560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686997391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1025884598493576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218726400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784687012884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08729563653469086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218890240, "lr": 1e-06}} diff --git a/recommendation_v4/rcp_logs/gbs_8192/seed537267118.log b/recommendation_v4/rcp_logs/gbs_8192/seed537267118.log new file mode 100644 index 000000000..1d0d9f721 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_8192/seed537267118.log @@ -0,0 +1,1870 @@ +:::MLLOG {"namespace": "", "time_ms": 1784651273394, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784651273395, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784651294763, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784651294767, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784651294767, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784651294767, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784651294768, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784651294768, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 8192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784651294768, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784651294768, "event_type": "POINT_IN_TIME", "key": "seed", "value": 537267118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784651294768, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784651294768, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784651294768, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784651294768, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784651294771, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784651294771, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784651893648, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784651893649, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784651894026, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784652001264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13905291259288788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840, "lr": 7.916666666666666e-10}} +:::MLLOG {"namespace": "", "time_ms": 1784652013373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383819878101349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.6249999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652025915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13865436613559723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 491520, "lr": 2.458333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652038522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13811571896076202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652051532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13861262798309326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 819200, "lr": 4.125e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652064378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13852612674236298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652077556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13826708495616913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1146880, "lr": 5.7916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652090575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13886496424674988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.625e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652103837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13849292695522308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1474560, "lr": 7.458333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652117135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384684443473816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.291666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652130203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387362778186798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1802240, "lr": 9.124999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652143386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13861554861068726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652156898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13842080533504486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2129920, "lr": 1.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652170071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1397927850484848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652170072, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652170073, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652255806, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5013150572776794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652255808, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652255808, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652268605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13825581967830658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2457600, "lr": 1.2458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652281685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13812769949436188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652295135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13838350772857666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2785280, "lr": 1.4124999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652308356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378079205751419, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4958333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652321756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13786821067333221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3112960, "lr": 1.5791666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652335137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13765624165534973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652348165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13790643215179443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3440640, "lr": 1.7458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652361388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13778048753738403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652374929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13740061223506927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3768320, "lr": 1.9124999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652388315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13795824348926544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9958333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652401839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13833366334438324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4096000, "lr": 2.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652415440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13835908472537994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.1624999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652428807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13782469928264618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4423680, "lr": 2.2458333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652442560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13833220303058624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3291666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652442561, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652442562, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652526733, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5015504360198975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652526734, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652526734, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652540300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13749249279499054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4751360, "lr": 2.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652553973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138078972697258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652567665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13668853044509888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5079040, "lr": 2.5791666666666668e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652581400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361338049173355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652594836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13777515292167664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5406720, "lr": 2.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652608285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13801129162311554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652621758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13712751865386963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5734400, "lr": 2.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652635648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13654927909374237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652649317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13630004227161407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6062080, "lr": 3.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652662794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13747747242450714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652676518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13692618906497955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6389760, "lr": 3.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652689980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13749808073043823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652703985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373593658208847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6717440, "lr": 3.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652717592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13750667870044708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652717594, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784652717595, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784652799485, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5018365383148193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784652799486, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784652799487, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784652812723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13640303909778595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7045120, "lr": 3.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652826311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371050924062729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652839730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13645507395267487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7372800, "lr": 3.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652853828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13687530159950256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.829166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652867550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13715152442455292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7700480, "lr": 3.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652881450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13759849965572357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652895259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13605187833309174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8028160, "lr": 4.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652908951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13726723194122314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652922923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13626965880393982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8355840, "lr": 4.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652936820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13845643401145935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3291666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652950543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359703689813614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8683520, "lr": 4.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652963983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13761208951473236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.4958333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652977584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349330097436905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9011200, "lr": 4.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652991375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13637356460094452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.6624999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652991375, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784652991376, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653072319, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5022042393684387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653072320, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653072320, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653085832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13582539558410645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9338880, "lr": 4.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653099446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351640224456787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653112701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13794422149658203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9666560, "lr": 4.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653126357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13403582572937012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653140266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13644272089004517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9994240, "lr": 5.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653153837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13271409273147583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1624999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653167392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13520798087120056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10321920, "lr": 5.2458333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653180946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13502292335033417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653194606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13786080479621887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10649600, "lr": 5.4124999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653208051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13477793335914612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653221541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14784717559814453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10977280, "lr": 5.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653235437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13524438440799713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653248895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334683895111084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11304960, "lr": 5.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653262307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13766011595726013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653262308, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653262309, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653343538, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5025769472122192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653343539, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653343539, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653357242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13493965566158295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11632640, "lr": 5.9124999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653370867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13988468050956726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653384579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13556444644927979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11960320, "lr": 6.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653398297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13271233439445496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653412119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13458006083965302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12288000, "lr": 6.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653426196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1394202560186386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.329166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653440014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13308864831924438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12615680, "lr": 6.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653453707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13265173137187958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653467258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13299515843391418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12943360, "lr": 6.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653480521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311027854681015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653494096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13058887422084808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13271040, "lr": 6.745833333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653507276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347285658121109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653520774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13159601390361786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13598720, "lr": 6.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653534515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13905005156993866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653534516, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784653534516, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784653616707, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5030032396316528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784653616708, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784653616709, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784653630383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335202157497406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13926400, "lr": 7.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653643926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13313370943069458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653657768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13585060834884644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14254080, "lr": 7.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653671522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348785161972046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653685321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137860506772995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14581760, "lr": 7.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653699113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1272876113653183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653712595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13521234691143036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14909440, "lr": 7.579166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653726413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12891434133052826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653740637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13324779272079468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15237120, "lr": 7.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653754981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13229750096797943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.829166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653768905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13341619074344635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15564800, "lr": 7.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653782562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13443754613399506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.995833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653796677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13542264699935913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15892480, "lr": 8.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653810602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13336294889450073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653810602, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784653810603, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784653892732, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5037084221839905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784653892733, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784653892733, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784653906693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13216347992420197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16220160, "lr": 8.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653920579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337888240814209, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653934611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13548748195171356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16547840, "lr": 8.412500000000001e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653948984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13552838563919067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653963156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13980352878570557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16875520, "lr": 8.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653977048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330888271331787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.662499999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653991674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13521912693977356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17203200, "lr": 8.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654005426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13342374563217163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654019226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13308016955852509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17530880, "lr": 8.912499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654033307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13165520131587982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654047082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13573472201824188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17858560, "lr": 9.079166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654061399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13366791605949402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.162499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654075244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312064111232758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18186240, "lr": 9.245833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654089007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13049447536468506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654089008, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654089008, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654173626, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5046871900558472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654173627, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654173627, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654187989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13774473965168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18513920, "lr": 9.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654202143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13487617671489716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.495833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654216263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355314552783966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18841600, "lr": 9.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654230357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13109689950942993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.662499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654244362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13365577161312103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19169280, "lr": 9.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654258525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338740587234497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654272976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13718882203102112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19496960, "lr": 9.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654287296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13403812050819397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654301386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13281309604644775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19824640, "lr": 1.0079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654315261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12966188788414001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0162499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654328828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12761668860912323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20152320, "lr": 1.0245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654342483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330430805683136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654356237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1292903572320938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20480000, "lr": 1.0412499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654369800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.124602772295475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654369800, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654369800, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654453627, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5057211518287659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654453628, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654453628, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654467549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13117800652980804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20807680, "lr": 1.0579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654481446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360829472541809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654495756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13310366868972778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21135360, "lr": 1.0745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654510091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13030661642551422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654523924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12812383472919464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21463040, "lr": 1.0912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654537698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1265401542186737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654551976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12890461087226868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21790720, "lr": 1.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654565931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13226529955863953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654579962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13561370968818665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22118400, "lr": 1.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654594050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12479511648416519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654607995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12877395749092102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22446080, "lr": 1.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654621820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134392648935318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654635977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13088801503181458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22773760, "lr": 1.1579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654650463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12928347289562225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654650464, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784654650465, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784654733573, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5070603489875793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784654733574, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784654733575, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784654747931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304352730512619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23101440, "lr": 1.1745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654761934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12905968725681305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654775518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13326068222522736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23429120, "lr": 1.1912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654789414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12384738773107529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654803414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12939174473285675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23756800, "lr": 1.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654816911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1255599856376648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654830432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13010071218013763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24084480, "lr": 1.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654844153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12183447182178497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654857514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13567152619361877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24412160, "lr": 1.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654871437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13719011843204498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654885149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12381488084793091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24739840, "lr": 1.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654899407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12324212491512299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654913510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1256517916917801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25067520, "lr": 1.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654927391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1290084719657898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654927392, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784654927392, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655010741, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5086907744407654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655010742, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655010742, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655024143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365588903427124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25395200, "lr": 1.2912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655038480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365814507007599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655052506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358100026845932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25722880, "lr": 1.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655066452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14007684588432312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655080616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13670232892036438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26050560, "lr": 1.3245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655094619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13834303617477417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655108556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12762440741062164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26378240, "lr": 1.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655122461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12627626955509186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655136469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1421617865562439, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26705920, "lr": 1.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655150945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14100994169712067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655165283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12731561064720154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27033600, "lr": 1.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655179540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12837907671928406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655193645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13276323676109314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27361280, "lr": 1.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655207231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1313244104385376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655207231, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655207232, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655290781, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5105935335159302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655290782, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655290782, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655305114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1414000689983368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27688960, "lr": 1.4079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655319343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14230719208717346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655333153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13551203906536102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28016640, "lr": 1.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655347167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13926614820957184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655361564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14475174248218536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28344320, "lr": 1.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655375584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14106833934783936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655389830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12587971985340118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28672000, "lr": 1.4579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655404035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375739872455597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655418338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13180091977119446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28999680, "lr": 1.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655432648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362384855747223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655446949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13793636858463287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29327360, "lr": 1.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655460929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13304756581783295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4995833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655475728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13443680107593536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29655040, "lr": 1.5079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655490452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13662664592266083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655490452, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784655490453, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784655573903, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5126215815544128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784655573904, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784655573904, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784655587886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387251317501068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29982720, "lr": 1.5245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655602465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383458375930786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655617160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13182488083839417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30310400, "lr": 1.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655631669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13309255242347717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655646186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13372772932052612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30638080, "lr": 1.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655660935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13967831432819366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655675629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327541470527649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30965760, "lr": 1.5745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655690480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14136695861816406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655704579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1408742070198059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31293440, "lr": 1.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655718967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13511493802070618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655733195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1277177333831787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31621120, "lr": 1.6079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655747060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13006910681724548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655761127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13419099152088165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31948800, "lr": 1.6245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655775199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12776371836662292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655775200, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784655775201, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784655856564, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5150377750396729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784655856565, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784655856565, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784655870514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13209843635559082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32276480, "lr": 1.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655884448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13693247735500336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655898401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333790421485901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32604160, "lr": 1.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655912331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13187964260578156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655926491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13390159606933594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32931840, "lr": 1.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655940526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13341177999973297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655954965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13394001126289368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33259520, "lr": 1.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655969128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1328214854001999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655983088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1399206668138504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33587200, "lr": 1.7079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655997631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13204075396060944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656012025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12894731760025024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33914880, "lr": 1.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656025958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13117629289627075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656040026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324034035205841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34242560, "lr": 1.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656054101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13604873418807983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7495833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656054102, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656054102, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656137096, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5182050466537476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656137096, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656137097, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656151276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334814578294754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34570240, "lr": 1.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656165328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12303140759468079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656179338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13055171072483063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34897920, "lr": 1.7745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656193403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304926872253418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656207379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13514043390750885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35225600, "lr": 1.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656221905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13536594808101654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656236072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13536381721496582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35553280, "lr": 1.8079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656250728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12416739016771317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8162500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656264958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312824934720993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35880960, "lr": 1.8245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656278707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312505602836609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656292978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13021351397037506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36208640, "lr": 1.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656306963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13328921794891357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656321105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13094472885131836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36536320, "lr": 1.8579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656334891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13024933636188507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656334892, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656334892, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656417161, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5225784778594971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656417164, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656417164, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656431252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13775016367435455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36864000, "lr": 1.8745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656445360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324005424976349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656459371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335562914609909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37191680, "lr": 1.8912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656473431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.130352184176445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656487435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1272180825471878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37519360, "lr": 1.9079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656501355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1319679617881775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656515769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357017457485199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37847040, "lr": 1.9245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656530097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13279806077480316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656544402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13144567608833313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38174720, "lr": 1.9412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656558780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1391756534576416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656573157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13370825350284576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38502400, "lr": 1.9579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656587023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375160664319992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656601364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318248212337494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38830080, "lr": 1.9745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656615590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13339447975158691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656615591, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784656615591, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784656696734, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5287758111953735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784656696735, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784656696736, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784656710936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312011182308197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39157760, "lr": 1.9912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656725088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13504961133003235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656739439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13785243034362793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39485440, "lr": 2.0079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656753788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13330303132534027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656767920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12736664712429047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39813120, "lr": 2.0245833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656782104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12922877073287964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0329166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656796096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12596295773983002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40140800, "lr": 2.0412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656810004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12402292340993881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656824406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13631075620651245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40468480, "lr": 2.0579166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656838946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13735300302505493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656853239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346355825662613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40796160, "lr": 2.0745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656868117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1269901990890503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0829166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656882668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308128982782364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41123840, "lr": 2.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656896684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12367988377809525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656896685, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784656896685, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784656980876, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5378608107566833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784656980878, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784656980878, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784656995240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13238655030727386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41451520, "lr": 2.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657009492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13260112702846527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657023854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338382512331009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41779200, "lr": 2.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657038086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13777533173561096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657052434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13638554513454437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42106880, "lr": 2.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657066876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13142633438110352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657081195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12717366218566895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42434560, "lr": 2.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657095602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13503398001194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657109799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12451492249965668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42762240, "lr": 2.1745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657124364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304546594619751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657138864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13232453167438507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43089920, "lr": 2.1912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657153520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13391444087028503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657168220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14027461409568787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43417600, "lr": 2.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657182681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13613636791706085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657182682, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657182683, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657264825, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5516666173934937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657264827, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657264827, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657279852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13653331995010376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43745280, "lr": 2.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657294480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13144652545452118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657308736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323525607585907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44072960, "lr": 2.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657322998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1390550434589386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657337808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298941820859909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44400640, "lr": 2.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657352974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384761780500412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657367465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13502159714698792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44728320, "lr": 2.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657382017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13723310828208923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657396460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134572833776474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45056000, "lr": 2.2912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657411134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13038048148155212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657425698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1403605043888092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45383680, "lr": 2.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657440268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347608119249344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657454847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289958357810974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45711360, "lr": 2.3245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657469467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13316909968852997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657469468, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657469469, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657551653, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5675350427627563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657551654, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657551654, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657566546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14188151061534882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46039040, "lr": 2.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657581101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13235323131084442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657596326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133346289396286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46366720, "lr": 2.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657610972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13026738166809082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657625960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13530735671520233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46694400, "lr": 2.3745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657640623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334136426448822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657655465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13390342891216278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47022080, "lr": 2.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657670512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13558954000473022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657685411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13090038299560547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47349760, "lr": 2.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657700576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13242791593074799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657715499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327843964099884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47677440, "lr": 2.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657730492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13697364926338196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657745190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361313909292221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48005120, "lr": 2.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657760140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13715185225009918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.4495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657760141, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784657760141, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784657843041, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5848743319511414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784657843042, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784657843042, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784657857775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13142886757850647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48332800, "lr": 2.4579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657872794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13522754609584808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657887526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13409875333309174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48660480, "lr": 2.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657902773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13192200660705566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657917598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371985375881195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48988160, "lr": 2.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657932319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12931403517723083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657947303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133403941988945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49315840, "lr": 2.507916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657961681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12341707944869995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.5162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657976279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13044756650924683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49643520, "lr": 2.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657991023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13111522793769836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658005722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1309221088886261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49971200, "lr": 2.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658020375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13082937896251678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.5495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658035069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12877953052520752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50298880, "lr": 2.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658049695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13015419244766235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.5662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658049695, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658049696, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658130604, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5996133685112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658130605, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658130605, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658145056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12181005626916885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50626560, "lr": 2.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658159674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1266557276248932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658174166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12856416404247284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50954240, "lr": 2.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658188813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325235664844513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658203841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13150878250598907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51281920, "lr": 2.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658218456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12118998169898987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658233476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12994740903377533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51609600, "lr": 2.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658248403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13148283958435059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658263266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13671283423900604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51937280, "lr": 2.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658277529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12732014060020447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.6495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658292290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13250099122524261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52264960, "lr": 2.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658306803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1170162633061409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.6662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658321365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12050482630729675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52592640, "lr": 2.6745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658336128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12465347349643707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658336128, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658336129, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658417330, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6135889291763306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658417330, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658417331, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658431976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12068549543619156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52920320, "lr": 2.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658446457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12670598924160004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.6995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658461072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1279887855052948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53248000, "lr": 2.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658475732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13151425123214722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658490199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12058936804533005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53575680, "lr": 2.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658504794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341048777103424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658519138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12890098989009857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53903360, "lr": 2.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658533474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12033837288618088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.7495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658547917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12649273872375488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54231040, "lr": 2.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658562543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.126371830701828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658576948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11750021576881409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54558720, "lr": 2.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658592008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13062484562397003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658606621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1261119544506073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54886400, "lr": 2.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658621021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12764962017536163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658621022, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784658621022, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784658703418, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6267098784446716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784658703419, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784658703419, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784658717399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12651419639587402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55214080, "lr": 2.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658732027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1176154688000679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658746614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12102960050106049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55541760, "lr": 2.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658761479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11995167285203934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658775945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12321807444095612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55869440, "lr": 2.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658790480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12511321902275085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658805047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11470798403024673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56197120, "lr": 2.8579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658819701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12150394916534424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658834220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12408538907766342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56524800, "lr": 2.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658848743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12227829545736313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658863139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11433691531419754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56852480, "lr": 2.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658877647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12208706885576248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658891990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10825183987617493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57180160, "lr": 2.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658906661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12306025624275208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658906661, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784658906662, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784658989203, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6379542946815491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784658989203, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784658989204, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659004384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11502925306558609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57507840, "lr": 2.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659019205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11257729679346085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659033807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12699761986732483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57835520, "lr": 2.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659048760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12021943181753159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659063662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12229187786579132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58163200, "lr": 2.9579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659078151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1146773099899292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659092100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11813510209321976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58490880, "lr": 2.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659106627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12070698291063309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659120864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11855504661798477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58818560, "lr": 2.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659135074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11091650277376175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659149238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1183241605758667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59146240, "lr": 3.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659163383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1208546832203865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.0162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659177557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12334374338388443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59473920, "lr": 3.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659191791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11572325974702835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659191791, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659191792, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659274372, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6520432829856873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659274373, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659274373, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659288458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11744555830955505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59801600, "lr": 3.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659302521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10928478837013245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659316446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10844939947128296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60129280, "lr": 3.0579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659330327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11192812025547028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.0662499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659344247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10663960129022598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60456960, "lr": 3.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659358291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12116225808858871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659372666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11377149820327759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60784640, "lr": 3.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659387549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11914889514446259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659401893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10044839978218079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61112320, "lr": 3.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659416504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11946272850036621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.1162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659431109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11180496215820312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61440000, "lr": 3.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659445506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11170543730258942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659459685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10763774812221527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61767680, "lr": 3.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659473945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11583017557859421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659473946, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659473946, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659557432, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6697787642478943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659557433, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659557433, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659571753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11904582381248474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62095360, "lr": 3.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659586480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1189282014966011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.1662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659601162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12191496044397354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62423040, "lr": 3.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659615549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10360098630189896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659630102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11282636225223541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62750720, "lr": 3.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659644427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11593680828809738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659658832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11872338503599167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63078400, "lr": 3.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659673250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11904513835906982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659687430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10672628879547119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63406080, "lr": 3.2245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659701563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11773096024990082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659715879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.114909328520298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63733760, "lr": 3.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659729897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11470399051904678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659744028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11169275641441345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64061440, "lr": 3.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659758724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12516385316848755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659758724, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784659758725, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784659843297, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6870991587638855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784659843298, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784659843298, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784659858002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12403938919305801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64389120, "lr": 3.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659872695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11607769131660461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659887140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1163368746638298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64716800, "lr": 3.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659902093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12071496248245239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659916610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1086399108171463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65044480, "lr": 3.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659931310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11089086532592773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659945378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11261677742004395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65372160, "lr": 3.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659959892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084282398223877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659974446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11121903359889984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65699840, "lr": 3.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659989474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11608056724071503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660004182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12328866124153137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66027520, "lr": 3.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660018711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11485080420970917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660033724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1275276243686676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66355200, "lr": 3.3745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660048455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11234962195158005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660048456, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660048456, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660132153, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7157189846038818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660132154, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660132154, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660146985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12693756818771362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66682880, "lr": 3.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660161543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11385948956012726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.3995833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660176014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1162511557340622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67010560, "lr": 3.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660190935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11292923986911774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660205723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12001514434814453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67338240, "lr": 3.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660220760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11886285990476608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660236116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12130503356456757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67665920, "lr": 3.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660251390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12058158218860626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660266389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11504197865724564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67993600, "lr": 3.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660281878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11340747028589249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660296808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11622749269008636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68321280, "lr": 3.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660311453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11587648093700409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660326019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12607769668102264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68648960, "lr": 3.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660341085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11590025573968887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660341086, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660341086, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660424566, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7337918281555176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660424567, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660424567, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660439266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12271557748317719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68976640, "lr": 3.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660454336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11983563750982285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660469230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11392635852098465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69304320, "lr": 3.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660484355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.115829698741436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660499133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10927470028400421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69632000, "lr": 3.5412500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660513932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11598075181245804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660528747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11891163885593414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69959680, "lr": 3.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660543134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11213206499814987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660557813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10981077700853348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70287360, "lr": 3.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660572349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10865087807178497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660587396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12188133597373962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70615040, "lr": 3.5912500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660602502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10652673244476318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660617344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1217101439833641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70942720, "lr": 3.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660632738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12093881517648697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660632739, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784660632739, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784660715578, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7467658519744873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784660715579, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784660715579, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784660730615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11995065212249756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71270400, "lr": 3.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660745386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1197282075881958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660760128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11611475795507431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71598080, "lr": 3.6412499999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660774936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1170867457985878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660789563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10368441790342331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71925760, "lr": 3.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660804347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10520084947347641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660819074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12090286612510681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72253440, "lr": 3.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660833658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11510937660932541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660848229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1046375185251236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72581120, "lr": 3.6912499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660862667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039443388581276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660876763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10830876231193542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72908800, "lr": 3.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660891086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11325961351394653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660905352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11384862661361694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73236480, "lr": 3.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660919678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11696934700012207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660919719, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784660919720, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661002888, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.755730390548706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661002889, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661002890, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661017409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10551406443119049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73564160, "lr": 3.7412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661032242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10836630314588547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661047328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11690673232078552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73891840, "lr": 3.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661062122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11245717108249664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661077153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10829795151948929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74219520, "lr": 3.7745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661092350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11732516437768936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661107113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11403567343950272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74547200, "lr": 3.7912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661121653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11506544798612595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661136091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11743248254060745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74874880, "lr": 3.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661150547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11252182722091675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661164810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10169632732868195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75202560, "lr": 3.8245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661179299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11561254411935806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661193515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10819903761148453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75530240, "lr": 3.8412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661207864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10729663819074631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661207865, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661207866, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661292958, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7603189945220947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661292959, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661292959, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661307340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085595116019249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75857920, "lr": 3.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661321964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11449199914932251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661336604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09783316403627396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76185600, "lr": 3.8745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661351186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10482700169086456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661365663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11435598880052567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76513280, "lr": 3.8912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661379977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10656856000423431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661394532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09008070826530457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76840960, "lr": 3.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661408806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10966043919324875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661423082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1035551205277443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77168640, "lr": 3.9245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661437608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11816447973251343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661452188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10817869007587433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77496320, "lr": 3.9412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661466459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10096599161624908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661481254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10283384472131729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77824000, "lr": 3.9579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661495944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10850118100643158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661495945, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784661495945, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784661579210, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7637795209884644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784661579211, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784661579211, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784661593679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11639527976512909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78151680, "lr": 3.9745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661608093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11219251900911331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661622012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10443322360515594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78479360, "lr": 3.9912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661636708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10831297934055328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661650972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10032288730144501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78807040, "lr": 4.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661665563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10584508627653122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661680014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10877303779125214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79134720, "lr": 4.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661694637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10985640436410904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661709121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09949423372745514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79462400, "lr": 4.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661723761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11033382266759872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661738205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10567264258861542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79790080, "lr": 4.0579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661752675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10913330316543579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661766963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087067499756813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80117760, "lr": 4.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661781417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11363359540700912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661781418, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784661781418, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784661863496, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7643066644668579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784661863497, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784661863497, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784661878076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11039556562900543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80445440, "lr": 4.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661892286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10140808671712875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661906807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10431034117937088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80773120, "lr": 4.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661921548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10661078244447708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661936528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11290248483419418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81100800, "lr": 4.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661951580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11363186687231064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661966477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10108590871095657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81428480, "lr": 4.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661981310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10646922141313553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661996325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10823135077953339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81756160, "lr": 4.1579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662011204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10656394064426422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662026232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11282693594694138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82083840, "lr": 4.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662040510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0955958291888237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662055477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09870600700378418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82411520, "lr": 4.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662069803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10762746632099152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662069803, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662069804, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662153840, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7659448385238647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662153841, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662153841, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662168050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09175947308540344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82739200, "lr": 4.2079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662182386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10504128783941269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662196607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11022251099348068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83066880, "lr": 4.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662211058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.098680779337883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662225206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11074241995811462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83394560, "lr": 4.2412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662239462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11166910827159882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662253826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09815622121095657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83722240, "lr": 4.2579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662267794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11057554185390472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662281919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10549072176218033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84049920, "lr": 4.2745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662295986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1059456467628479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662310301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10871917754411697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84377600, "lr": 4.2912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662324290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10441464185714722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662338397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09744545817375183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84705280, "lr": 4.3079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662352680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10649677366018295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662352681, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662352681, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662435974, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.766512930393219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662435975, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662435975, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662450556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10458602011203766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85032960, "lr": 4.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662465284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10457268357276917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662479611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10569605976343155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85360640, "lr": 4.3412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662494159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10392570495605469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662508768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09837830066680908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85688320, "lr": 4.3579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662523168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10056566447019577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662537917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10682579129934311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86016000, "lr": 4.3745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662552065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09586073458194733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662566709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080840528011322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86343680, "lr": 4.3912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662581361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10911634564399719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662596073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11616100370883942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86671360, "lr": 4.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662611013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11440127342939377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662625711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11620751768350601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86999040, "lr": 4.4245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662640437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11507882922887802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662640438, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784662640438, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784662723721, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7678120136260986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784662723722, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784662723722, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784662738233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10131610184907913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87326720, "lr": 4.4412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662753037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11617577821016312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662767571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10872867703437805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87654400, "lr": 4.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662782404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1024293527007103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662797016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09116002917289734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87982080, "lr": 4.4745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662811703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09834719449281693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662826065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10446259379386902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88309760, "lr": 4.4912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662840840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10744815319776535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662855233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10802064090967178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88637440, "lr": 4.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662869710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10061800479888916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662884742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11306782811880112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88965120, "lr": 4.5245833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662899647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1000417172908783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662914552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10953826457262039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89292800, "lr": 4.5412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662929355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11205697804689407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.5495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662929355, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784662929356, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663011741, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7679907083511353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663011742, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663011743, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663026731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082443818449974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89620480, "lr": 4.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663041684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10165593773126602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663056529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11058223992586136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89948160, "lr": 4.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663071604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09161302447319031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663086208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11473284661769867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90275840, "lr": 4.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663100792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1068863794207573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663116025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11109202355146408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90603520, "lr": 4.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663130760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11179428547620773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663145859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12224256247282028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90931200, "lr": 4.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663160648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10556275397539139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663175522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11175458133220673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91258880, "lr": 4.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663190525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12328340113162994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663205332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10013885796070099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91586560, "lr": 4.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663220069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11126118153333664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663220069, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663220070, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663301973, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7687667012214661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663301974, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663301975, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663316737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10454314202070236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91914240, "lr": 4.6745833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663331637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11894288659095764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663346269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10904433578252792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92241920, "lr": 4.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663361482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10395601391792297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663376316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09701276570558548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92569600, "lr": 4.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663391087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11302808672189713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663406401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10665623098611832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92897280, "lr": 4.7245833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663421468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10609465837478638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663436405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11682891845703125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93224960, "lr": 4.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663450859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0994146317243576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663465744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12553928792476654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93552640, "lr": 4.7579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663480582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10818720608949661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663495116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11710181832313538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93880320, "lr": 4.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663509636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792446881532669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663509637, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784663509637, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784663593364, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7688585519790649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784663593365, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784663593365, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784663608165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11381227523088455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94208000, "lr": 4.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663623379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11524175852537155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663638567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11057521402835846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94535680, "lr": 4.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663654065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11779818683862686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663669248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11447061598300934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94863360, "lr": 4.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663684568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11923524737358093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663699900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.127125546336174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95191040, "lr": 4.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663714823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12121661007404327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663729854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11626705527305603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95518720, "lr": 4.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663745263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11053109169006348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663760242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10829618573188782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95846400, "lr": 4.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663775384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10873909294605255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663790509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11556925624608994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96174080, "lr": 4.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663805696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1142469123005867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663805697, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784663805698, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784663886576, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7698290944099426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784663886577, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784663886577, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784663901928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12054620683193207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96501760, "lr": 4.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663917285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10747353732585907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663932211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082199439406395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96829440, "lr": 4.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663947645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10765772312879562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663962983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11065156012773514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97157120, "lr": 4.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663978414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10963744670152664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663993467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11421643942594528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97484800, "lr": 4.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664008318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10784246772527695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664022962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10626571625471115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97812480, "lr": 4.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664037396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10047345608472824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664052336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10458473861217499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98140160, "lr": 4.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664066876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09693833440542221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664081393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10417400300502777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98467840, "lr": 5.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664095665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10106176882982254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664095666, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664095666, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664177383, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7707108855247498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664177384, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664177384, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664191745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10978298634290695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98795520, "lr": 5.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664206255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10111866146326065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664220674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0962742269039154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99123200, "lr": 5.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664235070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10864695906639099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664249469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11540719866752625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99450880, "lr": 5.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664263819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10521764308214188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664278234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10479560494422913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99778560, "lr": 5.074583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664292697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1049768403172493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664307005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10150916874408722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100106240, "lr": 5.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664321461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12067445367574692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664335978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11776181310415268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100433920, "lr": 5.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664350492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10127206146717072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664365326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10053417086601257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100761600, "lr": 5.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664379689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12034996598958969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664379690, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664379690, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664462078, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7708470225334167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664462078, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664462079, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664477050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11075176298618317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101089280, "lr": 5.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664491913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09679156541824341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664506653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10506570339202881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101416960, "lr": 5.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664521346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10336899757385254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664535705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10861444473266602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101744640, "lr": 5.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664550328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10515379905700684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.182916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664565234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10670946538448334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102072320, "lr": 5.191249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664579951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11206580698490143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664595117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10498671978712082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102400000, "lr": 5.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664610190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10767506808042526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664625070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10546337068080902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102727680, "lr": 5.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664639966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11473076045513153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664654574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10647991299629211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103055360, "lr": 5.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664669388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10216576606035233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664669389, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784664669389, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784664752170, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7713707089424133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784664752171, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784664752171, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784664767146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09928009659051895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103383040, "lr": 5.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664782203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09409144520759583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664797080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09622393548488617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103710720, "lr": 5.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664812125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09701599180698395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.282916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664826913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09711576253175735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104038400, "lr": 5.291249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664841892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0950046181678772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664856739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10021603852510452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104366080, "lr": 5.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664871529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09694884717464447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664886382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10295744240283966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104693760, "lr": 5.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664901071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10861501097679138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664916241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10483536869287491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105021440, "lr": 5.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664931044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10335119068622589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664945791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11423522979021072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105349120, "lr": 5.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664961064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11126511543989182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664961064, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784664961065, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665046756, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7717000842094421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665046757, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665046757, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665061491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09845655411481857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105676800, "lr": 5.374583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665076297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071082279086113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.382916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665091253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1101929172873497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106004480, "lr": 5.391249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665105832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10120666772127151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665120345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10042130202054977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106332160, "lr": 5.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665135210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10786592215299606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665149797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11078642308712006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106659840, "lr": 5.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665164434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10604935139417648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665179228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1026066467165947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106987520, "lr": 5.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665193650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1012057289481163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665207990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09995919466018677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107315200, "lr": 5.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665222436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10235434770584106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665236870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1092282310128212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107642880, "lr": 5.474583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665251170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11438706517219543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.482916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665251171, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665251171, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665337210, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7718310356140137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665337211, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665337211, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665351767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11022753268480301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107970560, "lr": 5.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665366452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10694991052150726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665381043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09113290905952454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108298240, "lr": 5.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665395430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10333118587732315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665410286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10677170753479004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108625920, "lr": 5.524583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665424805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1077955961227417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665439327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11063235253095627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108953600, "lr": 5.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665453902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10308889299631119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665468795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10019604116678238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109281280, "lr": 5.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665483646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10410460084676743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665498641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10281877219676971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109608960, "lr": 5.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665513497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081809401512146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665528768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10544139891862869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109936640, "lr": 5.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665543893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10880322754383087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665543895, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784665543895, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784665630340, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7722253799438477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784665630341, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784665630341, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784665645212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10368102043867111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110264320, "lr": 5.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665660106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10317471623420715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665674756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1038704514503479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110592000, "lr": 5.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665689049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1029072180390358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665704062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072206124663353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110919680, "lr": 5.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665718622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10145393759012222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665733316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09832171350717545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111247360, "lr": 5.657916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665748345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10223356634378433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.666250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665763186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11198436468839645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111575040, "lr": 5.674583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665778137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10523805022239685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665792786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10414654016494751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111902720, "lr": 5.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665807436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10258986055850983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665822323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10434655100107193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112230400, "lr": 5.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665837015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10640380531549454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665837016, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784665837017, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784665923402, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7726708650588989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784665923403, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784665923403, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784665938088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10863188654184341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112558080, "lr": 5.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665952763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1016484946012497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665967092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039639338850975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112885760, "lr": 5.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665981666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10117370635271072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665996339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10455064475536346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113213440, "lr": 5.757916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666010776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11332662403583527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.766250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666025382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10158169269561768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113541120, "lr": 5.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666039741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11052798479795456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666054510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1132238432765007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113868800, "lr": 5.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666069661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10611864924430847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666084383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10339532792568207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114196480, "lr": 5.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666098985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10461749136447906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666113895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10568413138389587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114524160, "lr": 5.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666128779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09875227510929108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666128780, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666128780, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666215114, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730655074119568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666215115, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666215115, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666230022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10219410806894302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114851840, "lr": 5.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666244941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09940332174301147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666259689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10602445900440216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115179520, "lr": 5.857916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666273903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10037476569414139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666288912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10828374326229095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115507200, "lr": 5.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666303646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10945728421211243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666318496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10752979665994644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115834880, "lr": 5.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666333607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11538208276033401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666348501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11378853023052216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116162560, "lr": 5.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666363289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1004980057477951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666378083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10392866283655167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116490240, "lr": 5.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666392795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1028134822845459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666407402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10561169683933258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116817920, "lr": 5.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666421971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10746919363737106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666421971, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666421972, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666504559, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7731917500495911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666504560, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666504560, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666519349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10198807716369629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117145600, "lr": 5.957916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666534013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10078465193510056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666548999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10938993096351624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117473280, "lr": 5.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666563600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10486166179180145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666578085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0943354144692421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117800960, "lr": 5.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666593113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11254645138978958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666607475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10125099122524261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118128640, "lr": 6.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666621923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10764870047569275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666636881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10311269015073776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118456320, "lr": 6.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666651860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10385473072528839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666666380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1093599870800972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118784000, "lr": 6.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666681206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.114145427942276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666696135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10645928233861923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119111680, "lr": 6.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666711236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10283980518579483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666711236, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784666711237, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784666793923, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773276686668396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784666793924, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784666793924, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784666809022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11264753341674805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119439360, "lr": 6.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666823833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10921858996152878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666838436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10961868613958359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119767040, "lr": 6.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666853434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10729439556598663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666867750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10105253010988235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120094720, "lr": 6.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666882044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10717029124498367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666897201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11791053414344788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120422400, "lr": 6.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666912209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10907493531703949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666927201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11639586836099625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120750080, "lr": 6.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666942513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12248367816209793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666957425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10714315623044968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121077760, "lr": 6.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666972434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11031551659107208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666987613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11500699818134308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121405440, "lr": 6.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667002611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11004745215177536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667002613, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667002613, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667085581, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7733807563781738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667085582, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667085582, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667100751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11324610561132431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121733120, "lr": 6.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667116054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10941973328590393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667131232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11083541065454483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122060800, "lr": 6.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667146410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10693589597940445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667161597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10393813252449036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122388480, "lr": 6.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667176753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11372072249650955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667192046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1068851575255394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122716160, "lr": 6.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667207219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11477333307266235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667222748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11199472844600677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123043840, "lr": 6.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667238225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12032651156187057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667253665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10629619657993317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123371520, "lr": 6.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667268985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11701486259698868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667284145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10730667412281036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123699200, "lr": 6.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667299687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11870327591896057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667299687, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667299688, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667384152, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7735241651535034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667384153, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667384154, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667399429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11937782913446426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124026880, "lr": 6.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667414620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10853338986635208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667429833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1194179505109787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124354560, "lr": 6.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667444750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1137010008096695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667459693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072809025645256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124682240, "lr": 6.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667475099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11511129140853882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667490061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11159142851829529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125009920, "lr": 6.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667505245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11678563803434372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667520452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1129024550318718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125337600, "lr": 6.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667535475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1058245375752449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667550468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11306238174438477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125665280, "lr": 6.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667565553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10685847699642181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667580500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11695557087659836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125992960, "lr": 6.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667595695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1052650734782219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667595695, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784667595696, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784667679770, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742419838905334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784667679772, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784667679772, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784667694766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10953886806964874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126320640, "lr": 6.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667710126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11892169713973999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667725500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11037251353263855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126648320, "lr": 6.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667740917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11349950730800629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667756218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11605124920606613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126976000, "lr": 6.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667771613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10883084684610367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667787124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10810153186321259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127303680, "lr": 6.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667802792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1095430999994278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.482916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667818542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11281441897153854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127631360, "lr": 6.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667833698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11261613667011261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667849217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11010776460170746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127959040, "lr": 6.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667864932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11032350361347198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667880573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1113646849989891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128286720, "lr": 6.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667896022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10937625914812088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667896022, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784667896023, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784667979731, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7743655443191528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784667979732, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784667979733, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784667994634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1181516945362091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128614400, "lr": 6.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668009615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11062642931938171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668024536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11038575321435928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128942080, "lr": 6.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668039292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10243108868598938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668054203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10600598901510239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129269760, "lr": 6.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668069444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11168890446424484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668084051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10776625573635101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129597440, "lr": 6.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668098692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10736948251724243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668113725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09793344885110855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129925120, "lr": 6.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668128447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10169947892427444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668143063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10549633204936981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130252800, "lr": 6.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668157713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10496149957180023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668172624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10926634073257446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130580480, "lr": 6.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668187604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11598134785890579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668187605, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668187605, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668269744, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746367454528809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668269745, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668269745, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668284415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11476525664329529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130908160, "lr": 6.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668299172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10530152916908264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668314192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10065707564353943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131235840, "lr": 6.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668328982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09263133257627487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668343673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11366003006696701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131563520, "lr": 6.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668358223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10303189605474472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668373007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10250444710254669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131891200, "lr": 6.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668388057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11247333884239197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668402881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10735206305980682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132218880, "lr": 6.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668417826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1034853607416153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668432990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10909824073314667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132546560, "lr": 6.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668448336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11063424497842789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668463611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10730907320976257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132874240, "lr": 6.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668478753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10676854848861694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668478754, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668478754, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668560419, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7747383117675781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668560420, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668560420, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668575343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11002103239297867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133201920, "lr": 6.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668590577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10331790894269943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668605481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11028443276882172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133529600, "lr": 6.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668620230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09888075292110443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668635346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11607608199119568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133857280, "lr": 6.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668650381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11011181771755219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668665144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1035495325922966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134184960, "lr": 6.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668680343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10801719129085541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668695404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10804269462823868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134512640, "lr": 6.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668710121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.108755923807621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668725115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11076835542917252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134840320, "lr": 6.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668739935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09885197877883911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668754438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071598008275032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135168000, "lr": 6.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668768873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0874006524682045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668768874, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784668768875, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784668851814, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7749252319335938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784668851815, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784668851816, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784668866546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09535553306341171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135495680, "lr": 6.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668881312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11166112124919891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668896060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10701604187488556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135823360, "lr": 6.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668910779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10127848386764526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668925582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10186677426099777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136151040, "lr": 6.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668940437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09036796540021896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668955213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10346736758947372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136478720, "lr": 6.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668969955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09756474196910858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668984619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10387992113828659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136806400, "lr": 6.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668999439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10616518557071686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669014454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11055491864681244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137134080, "lr": 6.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669029632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10837268084287643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669044491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10580842196941376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137461760, "lr": 6.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669059567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10393217951059341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669059568, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669059568, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669141469, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7749408483505249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669141470, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669141470, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669156678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09621662646532059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137789440, "lr": 7.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669172128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10526090115308762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669187416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09527690708637238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138117120, "lr": 7.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669202691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10145485401153564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669217831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09730442613363266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138444800, "lr": 7.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669232641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10082779079675674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669247342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10913863778114319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138772480, "lr": 7.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669262660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10112395882606506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.066249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669277632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10913747549057007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139100160, "lr": 7.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669292873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10973455011844635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669308304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10620740056037903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139427840, "lr": 7.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669323743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10433526337146759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669338760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10942414402961731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139755520, "lr": 7.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669353785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10876781493425369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669353786, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669353786, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669437400, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752856612205505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669437401, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669437401, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669452384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10696763545274734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140083200, "lr": 7.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669467198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1047927588224411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669481878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10322121530771255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140410880, "lr": 7.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669496728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09969180077314377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669511693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.106618732213974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140738560, "lr": 7.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669526626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09590014070272446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669542127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10526064783334732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141066240, "lr": 7.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669556946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10289575159549713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669571839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10995656251907349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141393920, "lr": 7.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669586849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09908945858478546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669601434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09741455316543579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141721600, "lr": 7.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669616277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10185326635837555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669631343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09088223427534103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142049280, "lr": 7.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669646550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10706773400306702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669646551, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784669646551, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784669729801, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.775657594203949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784669729802, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784669729803, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784669745148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09973179548978806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142376960, "lr": 7.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669760627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10033995658159256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669775420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09972897917032242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142704640, "lr": 7.257916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669790461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10960947722196579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669805591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1001359075307846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143032320, "lr": 7.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669820801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10218656808137894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669835851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10177962481975555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143360000, "lr": 7.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669850942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1054084300994873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669866132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10170314460992813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143687680, "lr": 7.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669880918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11221630126237869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669896017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10068412125110626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144015360, "lr": 7.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669910785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10798826068639755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669925615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10602875798940659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144343040, "lr": 7.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669940584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10616553574800491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669940585, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784669940585, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670023942, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7760641574859619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670023943, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670023943, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670038880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10572603344917297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144670720, "lr": 7.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670053688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09828606247901917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670068277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10655493289232254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144998400, "lr": 7.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670083210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10702754557132721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670098008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10265825688838959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145326080, "lr": 7.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670112470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10734546184539795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670127052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09741023182868958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145653760, "lr": 7.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670141845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09822960942983627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670156533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10496596992015839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145981440, "lr": 7.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670171333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.103910893201828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670186163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10272553563117981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146309120, "lr": 7.441250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670200961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10969291627407074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670215792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1069038063287735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146636800, "lr": 7.457916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670230608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1002931296825409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670230608, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670230609, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670313419, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7761626839637756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670313420, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670313420, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670328126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10293201357126236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146964480, "lr": 7.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670342883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10523776710033417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670357688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09540973603725433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147292160, "lr": 7.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670372902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09928587079048157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670387828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10194390267133713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147619840, "lr": 7.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670403096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10074706375598907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670418139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10722863674163818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147947520, "lr": 7.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670433350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10982834547758102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670448502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10883606970310211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148275200, "lr": 7.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670463537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10019560158252716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670478776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1020621508359909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148602880, "lr": 7.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670493425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09778377413749695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670508884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11191254109144211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148930560, "lr": 7.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670523680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11123016476631165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670523681, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670523681, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670605816, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7763913869857788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670605817, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670605817, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670620464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10602010041475296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149258240, "lr": 7.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670636097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10478255152702332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670651295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10588279366493225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149585920, "lr": 7.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670666927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11116191744804382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149749760, "lr": 7.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670682817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10672806203365326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149913600, "lr": 7.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670698502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10931304097175598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670713881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1075752004981041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150241280, "lr": 7.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670729460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0976814478635788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150405120, "lr": 7.649583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670744941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10647989064455032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150568960, "lr": 7.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670760242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10428845882415771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670775532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10854119807481766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150896640, "lr": 7.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670790554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10334066301584244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151060480, "lr": 7.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670806060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10617552697658539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151224320, "lr": 7.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670821282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09636261314153671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670821334, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784670821334, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784670902690, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7764504551887512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784670902691, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784670902691, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784670917985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09869398176670074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151552000, "lr": 7.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670933327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10553297400474548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151715840, "lr": 7.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670948757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10238863527774811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151879680, "lr": 7.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670963964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10431334376335144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670979034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10799982398748398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152207360, "lr": 7.741249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670993773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12293615937232971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152371200, "lr": 7.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671008713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10873296856880188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152535040, "lr": 7.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671023669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09900733083486557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671038672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11197234690189362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152862720, "lr": 7.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671053597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1115211471915245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153026560, "lr": 7.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671068714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11455177515745163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153190400, "lr": 7.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671083995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09913278371095657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671099422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10632834583520889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153518080, "lr": 7.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671114658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1037500724196434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153681920, "lr": 7.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671114707, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671114707, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671197233, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7770000100135803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671197234, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671197234, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671212403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09880984574556351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153845760, "lr": 7.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671227590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09879492968320847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.832916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671242737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11230605840682983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154173440, "lr": 7.841249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671258163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11787409335374832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154337280, "lr": 7.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671273236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11311928182840347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154501120, "lr": 7.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671287899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1106879711151123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671303345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10718660056591034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154828800, "lr": 7.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671318546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11990966647863388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154992640, "lr": 7.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671333675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10461977869272232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155156480, "lr": 7.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671349264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10859663784503937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671364773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1104474738240242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155484160, "lr": 7.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671380092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11030980944633484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155648000, "lr": 7.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671395425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11393123865127563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155811840, "lr": 7.924583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671410781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11197621375322342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.932916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671410782, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671410782, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671494206, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7774386405944824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671494208, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671494208, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671509575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10805604606866837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156139520, "lr": 7.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671524869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11032863706350327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156303360, "lr": 7.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671540312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1149178147315979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156467200, "lr": 7.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671555468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10993768274784088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671570459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10841679573059082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156794880, "lr": 7.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671585653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10844794660806656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156958720, "lr": 7.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671601045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10170120745897293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157122560, "lr": 7.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671616254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11275909096002579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671631558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12047013640403748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157450240, "lr": 8.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671646992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10930614173412323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157614080, "lr": 8.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671662102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10868102312088013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157777920, "lr": 8.024583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671677357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10479474067687988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.032916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671692844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10297378897666931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158105600, "lr": 8.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671708013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10880367457866669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158269440, "lr": 8.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671708014, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784671708015, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784671790596, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.777470588684082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784671790597, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784671790597, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784671805670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11448246240615845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158433280, "lr": 8.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671821030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11955557018518448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671836404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11375382542610168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158760960, "lr": 8.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671851903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11253856122493744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158924800, "lr": 8.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671867305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11077447980642319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159088640, "lr": 8.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671882402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1195726990699768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671897860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11117404699325562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159416320, "lr": 8.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671912638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10966477543115616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159580160, "lr": 8.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671928042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11020933091640472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159744000, "lr": 8.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671943443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09878554940223694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671958727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10969100892543793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160071680, "lr": 8.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671974427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10933828353881836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160235520, "lr": 8.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671989883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10450435429811478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160399360, "lr": 8.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672004947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11035916209220886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672004948, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672004949, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672089580, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7776263952255249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672089581, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672089581, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672104606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11305443942546844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160727040, "lr": 8.174583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672119750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12731507420539856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160890880, "lr": 8.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672134985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10872355103492737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161054720, "lr": 8.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672150107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11234673112630844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672165265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1127481609582901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161382400, "lr": 8.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672180615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11101729422807693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161546240, "lr": 8.216250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672195954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11409436166286469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161710080, "lr": 8.224583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672211253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1135326474905014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672226231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10684249550104141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162037760, "lr": 8.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672241440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10869015008211136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162201600, "lr": 8.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672256957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13033974170684814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162365440, "lr": 8.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672272145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11255691945552826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672287726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11339547485113144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162693120, "lr": 8.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672303245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11627250909805298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162856960, "lr": 8.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672303246, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672303246, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672386619, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7777079939842224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672386620, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672386621, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672401937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09635580331087112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163020800, "lr": 8.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672417477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11138806492090225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672432934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12173325568437576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163348480, "lr": 8.307916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672448441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10718898475170135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163512320, "lr": 8.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672463865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11043170839548111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163676160, "lr": 8.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672479009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10765890032052994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672494290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11277633160352707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164003840, "lr": 8.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672509336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1129363402724266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164167680, "lr": 8.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672524461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10334969311952591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164331520, "lr": 8.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672539059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11785952746868134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672553870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10705503076314926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164659200, "lr": 8.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672568944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11115894466638565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164823040, "lr": 8.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672584177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09255003929138184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164986880, "lr": 8.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672598863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10399321466684341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672598864, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784672598864, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784672684220, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7776152491569519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784672684221, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784672684221, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784672699215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10237597674131393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165314560, "lr": 8.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672714046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09628530591726303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165478400, "lr": 8.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672728633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11390165984630585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165642240, "lr": 8.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672743281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10850359499454498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672757853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10967843234539032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165969920, "lr": 8.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672772895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10426661372184753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166133760, "lr": 8.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672787720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10819854587316513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166297600, "lr": 8.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672802479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1130601242184639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672817239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09980181604623795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166625280, "lr": 8.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672832271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0956762284040451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166789120, "lr": 8.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672847224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09695913642644882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166952960, "lr": 8.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672861960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09747710078954697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672876444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10241592675447464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167280640, "lr": 8.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672891094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10077252238988876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167444480, "lr": 8.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672891094, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784672891095, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784672975805, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7775678038597107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784672975806, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784672975806, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784672990769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10293340682983398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167608320, "lr": 8.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673005875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10322614014148712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673020781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10339131206274033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167936000, "lr": 8.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673035650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10734470188617706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168099840, "lr": 8.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673050440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10235617309808731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168263680, "lr": 8.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673065597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09983355551958084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673080810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10913494974374771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168591360, "lr": 8.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673095755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10522083193063736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168755200, "lr": 8.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673110694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10654790699481964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168919040, "lr": 8.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673125441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10592575371265411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673140466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10478770732879639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169246720, "lr": 8.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673155064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10628117620944977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169410560, "lr": 8.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673169546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10585664957761765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169574400, "lr": 8.624583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673184705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11357520520687103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673184751, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673184752, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673268994, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7783006429672241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673268995, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673268995, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673283737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10829005390405655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169902080, "lr": 8.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673298474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11165773123502731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170065920, "lr": 8.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673313565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072721853852272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170229760, "lr": 8.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673328575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10748091340065002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673343148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09962110966444016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170557440, "lr": 8.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673358094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10214615613222122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170721280, "lr": 8.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673372988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09551379829645157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170885120, "lr": 8.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673387685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10691250115633011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673402371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09958139061927795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171212800, "lr": 8.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673417208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10150792449712753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171376640, "lr": 8.716249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673432195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10204659402370453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171540480, "lr": 8.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673446917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10287555307149887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673461790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09924736618995667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171868160, "lr": 8.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673476659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10364341735839844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172032000, "lr": 8.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673476704, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673476705, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673559456, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7783663868904114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673559457, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673559458, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673574360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09782955050468445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172195840, "lr": 8.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673589432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10844257473945618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673604251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09960614144802094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172523520, "lr": 8.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673619090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10229236632585526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172687360, "lr": 8.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673633886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0937352329492569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172851200, "lr": 8.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673649069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10686129331588745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673664389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10299418121576309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173178880, "lr": 8.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673679322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10315241664648056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173342720, "lr": 8.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673694527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10480692237615585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173506560, "lr": 8.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673709588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11211661249399185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673725082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09477433562278748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173834240, "lr": 8.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673740383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1112203598022461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173998080, "lr": 8.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673755882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11230438947677612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174161920, "lr": 8.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673770921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10060952603816986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673770970, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784673770971, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784673854167, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7786232233047485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784673854168, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784673854168, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784673869138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09776774793863297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174489600, "lr": 8.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673884580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10720999538898468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174653440, "lr": 8.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673900095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1022408977150917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174817280, "lr": 8.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673914995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10503406077623367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673930148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10064156353473663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175144960, "lr": 8.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673944975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10842219740152359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175308800, "lr": 8.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673959983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10032350569963455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175472640, "lr": 8.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673975263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09846900403499603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673990592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10861381143331528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175800320, "lr": 8.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674005822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10847962647676468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175964160, "lr": 8.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674020837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10085485875606537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176128000, "lr": 8.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674035990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09365560114383698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674050810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09577857702970505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176455680, "lr": 8.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674065749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10573173314332962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176619520, "lr": 8.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674065749, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674065750, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674148688, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.778984546661377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674148689, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674148689, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674163720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10300062596797943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176783360, "lr": 8.991249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674178652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11379949748516083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674193567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10164198279380798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177111040, "lr": 9.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674208491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10536274313926697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177274880, "lr": 9.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674223264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11128010600805283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177438720, "lr": 9.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674238166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10065484791994095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.032916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674253008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10856766253709793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177766400, "lr": 9.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674267742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10138743370771408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177930240, "lr": 9.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674282510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1069948598742485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178094080, "lr": 9.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674297139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1028190478682518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674311979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09798133373260498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178421760, "lr": 9.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674327207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11210732161998749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178585600, "lr": 9.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674342420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11963701993227005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178749440, "lr": 9.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674357388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10937993228435516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674357389, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674357390, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674440897, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7793816924095154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674440898, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674440898, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674456084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11558873951435089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179077120, "lr": 9.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674471053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10120101273059845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179240960, "lr": 9.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674486652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10077489912509918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179404800, "lr": 9.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674501913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10112103074789047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.132916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674517163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10626595467329025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179732480, "lr": 9.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674532379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09937550127506256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179896320, "lr": 9.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674547448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0998929888010025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180060160, "lr": 9.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674562779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10651737451553345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674577394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.105538509786129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180387840, "lr": 9.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674591911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09413986653089523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180551680, "lr": 9.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674607612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10402215272188187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180715520, "lr": 9.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674622821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10119221359491348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674638154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09340885281562805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181043200, "lr": 9.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674653663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10077707469463348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181207040, "lr": 9.216249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674653663, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784674653664, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784674736204, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.779661238193512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784674736206, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784674736206, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784674751753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11036303639411926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181370880, "lr": 9.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674767203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11382913589477539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674782610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10121094435453415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181698560, "lr": 9.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674797897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072535514831543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181862400, "lr": 9.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674813143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10301872342824936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182026240, "lr": 9.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674828397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10202836990356445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674843729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10688689351081848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182353920, "lr": 9.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674858849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10928075760602951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182517760, "lr": 9.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674873917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0971297174692154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182681600, "lr": 9.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674888853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09741850197315216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674904066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11158184707164764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183009280, "lr": 9.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674918967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09754068404436111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183173120, "lr": 9.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674934197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10274063795804977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183336960, "lr": 9.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674949030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10241048038005829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674949031, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784674949031, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675032018, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7797478437423706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675032019, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675032019, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675047071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.102015919983387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183664640, "lr": 9.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675061856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1074967160820961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183828480, "lr": 9.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675076638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11271261423826218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183992320, "lr": 9.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675092165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09419453889131546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675107440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10299596935510635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184320000, "lr": 9.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675122505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725047439336777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184483840, "lr": 9.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675137662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11519558727741241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184647680, "lr": 9.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675152788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10083010792732239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675168135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10556194186210632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184975360, "lr": 9.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675183722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10957034677267075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185139200, "lr": 9.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675198873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09706181287765503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185303040, "lr": 9.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675214081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10720019787549973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675229624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09882110357284546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185630720, "lr": 9.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675244594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1037924513220787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185794560, "lr": 9.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675244595, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675244595, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675327183, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7799726724624634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675327184, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675327184, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675342689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10980929434299469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185958400, "lr": 9.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675357831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10468385368585587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675372655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09256668388843536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186286080, "lr": 9.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675387814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10066402703523636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186449920, "lr": 9.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675402650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10883717238903046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186613760, "lr": 9.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675417576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11144675314426422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675432966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09656210243701935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186941440, "lr": 9.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675448275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10416623204946518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187105280, "lr": 9.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675463289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10634192824363708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187269120, "lr": 9.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675478220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09901802241802216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675493266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1041516363620758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187596800, "lr": 9.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675508141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10405829548835754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187760640, "lr": 9.549583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675522958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11093925684690475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187924480, "lr": 9.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675538086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11454667150974274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675538086, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675538087, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675620522, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7799948453903198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675620523, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675620524, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675635482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11003132164478302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188252160, "lr": 9.574583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675650701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10494028031826019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188416000, "lr": 9.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675665608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10510677844285965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188579840, "lr": 9.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675680878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10817872732877731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675695932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10131540894508362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188907520, "lr": 9.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675711129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09717411547899246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189071360, "lr": 9.616249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675726083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10258437693119049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189235200, "lr": 9.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675740864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1117018535733223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675761764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10098510980606079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189562880, "lr": 9.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675776745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09526977688074112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189726720, "lr": 9.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675791746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10540826618671417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189890560, "lr": 9.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675807082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11260036379098892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.666249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675822391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1020289957523346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190218240, "lr": 9.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675837516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11834818124771118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190382080, "lr": 9.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675837517, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784675837517, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784675921766, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7803231477737427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784675921767, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784675921767, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784675937107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10042553395032883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190545920, "lr": 9.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675952087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09739439934492111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675967327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10625086724758148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190873600, "lr": 9.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675982659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11214453727006912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191037440, "lr": 9.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675998090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11760076880455017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191201280, "lr": 9.724583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676013347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09638525545597076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676028656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1054893508553505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191528960, "lr": 9.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676043673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10170315951108932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191692800, "lr": 9.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676059091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10108616948127747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191856640, "lr": 9.757916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676074217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10470834374427795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676088923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1065049096941948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192184320, "lr": 9.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676104393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1098676547408104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192348160, "lr": 9.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676119719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09948199987411499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192512000, "lr": 9.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676135527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11499764770269394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.799583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676135527, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676135528, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676220058, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.781006395816803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676220059, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676220059, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676235824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11234048008918762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192839680, "lr": 9.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676251608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11133856326341629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193003520, "lr": 9.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676267392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10600941628217697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193167360, "lr": 9.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676283320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11987508088350296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676299163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11559200286865234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193495040, "lr": 9.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676315027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10181745141744614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193658880, "lr": 9.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676331068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11805438995361328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193822720, "lr": 9.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676346817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11319378018379211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.866249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676362682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10590420663356781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194150400, "lr": 9.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676378607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1077098622918129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194314240, "lr": 9.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676394654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10761167109012604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194478080, "lr": 9.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676410474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10667447745800018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676426440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10757973790168762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194805760, "lr": 9.907916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676441882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11685274541378021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194969600, "lr": 9.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676441883, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676441883, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676527645, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7811669707298279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676527645, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676527645, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676543646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1122468039393425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195133440, "lr": 9.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676559359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10396813601255417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676575002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10402611643075943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195461120, "lr": 9.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676590820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11610335111618042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195624960, "lr": 9.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676606728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10835517197847366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195788800, "lr": 9.957916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676622402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10570959001779556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676638345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10495207458734512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196116480, "lr": 9.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676654236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11431242525577545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196280320, "lr": 9.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676670455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11780320852994919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196444160, "lr": 9.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676686446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1045931801199913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.999583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676702518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11775758117437363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196771840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676718414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11135101318359375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196935680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676733922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10811511427164078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197099520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676749873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11596212536096573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676749874, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784676749875, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784676835932, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7816504836082458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784676835933, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784676835933, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784676851264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10040435194969177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197427200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676866886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11128810048103333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197591040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676882333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11408628523349762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197754880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676897590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10128842294216156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676913295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10143864154815674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198082560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676928978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10935956984758377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198246400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676944326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09869091212749481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198410240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676959615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10824012756347656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676974905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10159265995025635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198737920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676990037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10570728033781052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198901760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677005254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11449843645095825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199065600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677020512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11084305495023727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677035494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1179816722869873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199393280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677051006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11274842917919159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199557120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677051006, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677051007, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677137646, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.782129168510437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677137647, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677137648, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677152899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11251890659332275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199720960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677168244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10735400766134262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677183487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1106371358036995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200048640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677198790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11632662266492844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200212480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677213947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11171797662973404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200376320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677229277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11776918917894363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677244937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09984084963798523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200704000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677260415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10897643864154816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200867840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677275909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10734251141548157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201031680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677291432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10916750133037567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677307165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11703670769929886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201359360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677322984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11162159591913223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201523200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677338522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10220806300640106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201687040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677354309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11191898584365845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677354309, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677354310, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677439906, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7816577553749084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677439907, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677439907, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677455060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10513134300708771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202014720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677470538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11083178967237473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202178560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677485633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11894278973340988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202342400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677501652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10618061572313309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677516955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10602065920829773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202670080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677532392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10102405399084091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202833920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677548545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10633635520935059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202997760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677564033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10970672965049744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677579721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11359009146690369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203325440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677595223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09363829344511032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203489280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677610834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11716305464506149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203653120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677626238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10132802277803421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677641187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09899957478046417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203980800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677656301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10774485021829605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204144640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677656301, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784677656302, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784677739692, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7822102308273315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784677739693, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784677739693, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784677755157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10248102992773056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204308480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677770671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09373950213193893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677786163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09661147743463516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204636160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677801476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10842035710811615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204800000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677816869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10905250161886215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204963840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677832295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11246199160814285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677847623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0957522988319397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205291520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677862913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1060970276594162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205455360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677878230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10595804452896118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205619200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677893418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11034715920686722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677908622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09864529967308044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205946880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677924222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09951204061508179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206110720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677939519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11057724803686142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206274560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677954750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09875418245792389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677954751, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784677954751, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678036881, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7824049592018127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678036882, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678036882, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678052398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10584244877099991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206602240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678067750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09595076739788055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206766080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678083107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039179340004921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206929920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678098680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0956374853849411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678114235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10265258699655533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207257600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678129551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1115102618932724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207421440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678144849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10654888302087784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207585280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678159747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0959153100848198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678174783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11333131045103073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207912960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678189951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10399134457111359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208076800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678204624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10297806560993195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208240640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678219953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10166628658771515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678234904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10373738408088684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208568320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678250225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10107989609241486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208732160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678250225, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678250226, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678333380, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7829484939575195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678333382, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678333382, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678348797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10052739828824997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208896000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678364044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09922217577695847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678379274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10256920754909515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209223680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678394162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10224701464176178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209387520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678409507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09892712533473969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209551360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678424772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1014966368675232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678440039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09576165676116943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209879040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678455000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09918870031833649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210042880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678470263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10342995822429657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210206720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678485338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10877618938684464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678500401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10495041310787201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210534400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678515304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10745169222354889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210698240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678530520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10197096318006516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210862080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678545419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09570567309856415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678545420, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678545420, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678630506, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7833391427993774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678630508, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678630508, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678645614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09447591006755829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211189760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678660407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09956313669681549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211353600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678675254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09645996242761612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211517440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678689805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09983590245246887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678704719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10288690030574799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211845120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678720216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09543519467115402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212008960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678735237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10287076979875565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212172800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678750504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09824525564908981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678765473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09831804037094116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212500480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678780525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10421779006719589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212664320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678795542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09287778288125992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212828160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678810856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10705714672803879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678826128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10525722801685333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213155840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678841603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10961825400590897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213319680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678841604, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784678841604, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784678924740, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7840125560760498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784678924741, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784678924741, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784678939754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10473591834306717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213483520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678954770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10058774054050446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678969749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10528627038002014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213811200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678985041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10311265289783478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213975040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678999743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09606003016233444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214138880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679014432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10467848926782608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679029390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09524332731962204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214466560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679044266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1137469932436943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214630400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679059146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0982033759355545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214794240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679074701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09064000099897385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679089988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10000346601009369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215121920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679105001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10522542893886566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215285760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679119925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10359357297420502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215449600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679134891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10866481810808182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679134892, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679134892, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679217814, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7837282419204712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679217815, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679217815, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679232730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10054497420787811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215777280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679247375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10544989258050919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215941120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679262218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09761001914739609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216104960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679277072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09916829317808151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679292041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0996568575501442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216432640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679306743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0937655046582222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216596480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679321583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.103753000497818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216760320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679336568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10133164376020432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679351262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10027871280908585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217088000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679366456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11013643443584442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217251840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679381211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09734304249286652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217415680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679396237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09986454248428345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217579520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679411327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.099483922123909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217743360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679426545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10560328513383865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217907200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679426546, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679426547, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679508728, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7836151123046875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679508729, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679508729, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679523899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10118672996759415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218071040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679539181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09485578536987305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218234880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679554493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09799488633871078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218398720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679569505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10759411007165909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218562560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679584921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10321011394262314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218726400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679600222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08723301440477371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218890240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679615606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10617394745349884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219054080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679631040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09266380965709686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219217920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679646525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10364456474781036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219381760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679661734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09514205157756805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219545600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679676623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09434378892183304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219709440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679691323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10254760086536407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219873280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679706239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11220698803663254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220037120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679721163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09743020683526993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220200960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679721164, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784679721164, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784679805003, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7837023138999939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784679805004, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784679805004, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784679819448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09566503018140793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220364800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679834572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09491308033466339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220528640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679849782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10204130411148071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220692480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679864684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1020173728466034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220856320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679880109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10374946892261505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221020160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679895455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10514166951179504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221184000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679910684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10883693397045135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221347840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679925684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.115514375269413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221511680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679940893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10829773545265198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221675520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679956122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09895297139883041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221839360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679971417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1103682592511177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222003200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679986350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09635252505540848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222167040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680001677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09317448735237122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222330880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680016887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10847397148609161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222494720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680016887, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784680016888, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784680102068, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7826203107833862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784680102069, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784680102069, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784680117429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09900287538766861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222658560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680132742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1138279139995575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222822400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680147874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10543506592512131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222986240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680162995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09222284704446793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223150080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680178257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1036883145570755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223313920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680193592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09742192178964615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223477760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680208758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09478235244750977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223641600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680223829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10527403652667999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223805440, "lr": 1e-06}} diff --git a/recommendation_v4/rcp_logs/gbs_8192/seed552965065.log b/recommendation_v4/rcp_logs/gbs_8192/seed552965065.log new file mode 100644 index 000000000..5b4f6befc --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_8192/seed552965065.log @@ -0,0 +1,1815 @@ +:::MLLOG {"namespace": "", "time_ms": 1784658183613, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784658183613, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784658204806, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784658204808, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784658204808, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784658204808, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784658204808, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784658204808, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 8192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784658204808, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784658204808, "event_type": "POINT_IN_TIME", "key": "seed", "value": 552965065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784658204808, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784658204808, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784658204808, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784658204808, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784658204809, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784658204809, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784658987475, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784658987476, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784658987896, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784659258248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13799943029880524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840, "lr": 7.916666666666666e-10}} +:::MLLOG {"namespace": "", "time_ms": 1784659270346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13899414241313934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.6249999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659283258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13874348998069763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 491520, "lr": 2.458333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659296098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13925470411777496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659309155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13904842734336853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 819200, "lr": 4.125e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659322331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13886089622974396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659335671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13916324079036713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1146880, "lr": 5.7916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659348835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13850216567516327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.625e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659362308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13880807161331177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1474560, "lr": 7.458333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659375682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13887231051921844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.291666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659389295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13896244764328003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1802240, "lr": 9.124999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659402796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13890060782432556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659416637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13893844187259674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2129920, "lr": 1.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659430225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13825617730617523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659430226, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784659430226, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784659520036, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4999590218067169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784659520036, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784659520037, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784659532801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389658898115158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2457600, "lr": 1.2458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659545782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386823207139969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659559888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387050598859787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2785280, "lr": 1.4124999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659573336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388169527053833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4958333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659586970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13869225978851318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3112960, "lr": 1.5791666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659600497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13882645964622498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659613630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860014081001282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3440640, "lr": 1.7458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659626950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13858339190483093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659641444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13884922862052917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3768320, "lr": 1.9124999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659655206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13862979412078857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9958333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659669222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13867130875587463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4096000, "lr": 2.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659683073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387033760547638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.1624999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659696684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13851049542427063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4423680, "lr": 2.2458333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659710940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138844832777977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3291666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659710941, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784659710941, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784659795829, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5001230835914612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784659795830, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784659795830, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784659809498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13832128047943115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4751360, "lr": 2.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659823389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13853776454925537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659837269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381119191646576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5079040, "lr": 2.5791666666666668e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659851142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13785172998905182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659864534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860253989696503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5406720, "lr": 2.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659877977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13831062614917755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659891469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13799473643302917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5734400, "lr": 2.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659905539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13796545565128326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659919455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13742361962795258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6062080, "lr": 3.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659933204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13774333894252777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659947184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13799074292182922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6389760, "lr": 3.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659960893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13808418810367584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659975178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13804316520690918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6717440, "lr": 3.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659988804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378697156906128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659988805, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784659988805, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784660071834, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5003073811531067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784660071835, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784660071835, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784660085173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13737033307552338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7045120, "lr": 3.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660098871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377810686826706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660112434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13718874752521515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7372800, "lr": 3.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660126846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13748934864997864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.829166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660140823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13768510520458221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7700480, "lr": 3.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660155088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378512680530548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660169229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371222287416458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8028160, "lr": 4.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660183201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13757918775081635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660197628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13710801303386688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8355840, "lr": 4.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660211696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385217159986496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3291666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660225600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13662850856781006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8683520, "lr": 4.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660239458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378578245639801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.4958333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660253390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13573578000068665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9011200, "lr": 4.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660267484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13681861758232117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.6624999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660267485, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784660267485, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784660351246, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5005672574043274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784660351247, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784660351247, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784660365102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13639608025550842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9338880, "lr": 4.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660379095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13577444851398468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660392820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13758306205272675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9666560, "lr": 4.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660406832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13496892154216766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660420885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367558091878891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9994240, "lr": 5.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660434701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13387584686279297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1624999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660448875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13574974238872528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10321920, "lr": 5.2458333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660462568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13562487065792084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660476538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378091275691986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10649600, "lr": 5.4124999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660490404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13540299236774445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660504248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14615002274513245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10977280, "lr": 5.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660518486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13577800989151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660532288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13418659567832947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11304960, "lr": 5.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660546388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13771024346351624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660546388, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784660546389, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784660631430, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.500906229019165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784660631431, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784660631431, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784660645333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354517787694931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11632640, "lr": 5.9124999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660659161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13962006568908691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660673197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13615639507770538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11960320, "lr": 6.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660687115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13352316617965698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660701297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349768340587616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12288000, "lr": 6.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660715522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13919053971767426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.329166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660729472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335921436548233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12615680, "lr": 6.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660743421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13306336104869843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660757001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13324172794818878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12943360, "lr": 6.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660770427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13164928555488586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660784307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13124771416187286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13271040, "lr": 6.745833333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660797816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13508455455303192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660811457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13178212940692902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13598720, "lr": 6.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660825446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13874736428260803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660825446, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784660825447, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784660908517, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5014082789421082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784660908518, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784660908518, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784660922304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13377967476844788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13926400, "lr": 7.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660936008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13341419398784637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660950294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358257234096527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14254080, "lr": 7.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660964390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351969838142395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660978636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13791343569755554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14581760, "lr": 7.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660992931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12800505757331848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661006869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13549545407295227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14909440, "lr": 7.579166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661021037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1293003261089325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661035366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334085762500763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15237120, "lr": 7.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661049860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327323168516159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.829166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661064046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13373106718063354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15564800, "lr": 7.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661078338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13421498239040375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.995833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661092656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135316401720047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15892480, "lr": 8.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661106573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13355106115341187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661106574, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784661106574, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784661190483, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5022820234298706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784661190484, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784661190485, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784661204410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13253094255924225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16220160, "lr": 8.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661218260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13377249240875244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661232126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13589408993721008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16547840, "lr": 8.412500000000001e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661246353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355898231267929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661260303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1396108865737915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16875520, "lr": 8.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661274059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333117038011551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.662499999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661289297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135166734457016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17203200, "lr": 8.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661303728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13346953690052032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661317941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13289061188697815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17530880, "lr": 8.912499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661332386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318179965019226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661346316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13579308986663818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17858560, "lr": 9.079166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661360705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335703432559967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.162499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661374850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13355956971645355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18186240, "lr": 9.245833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661388850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13059742748737335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661388851, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784661388851, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784661473112, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5034599304199219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784661473113, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784661473114, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784661487881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13806092739105225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18513920, "lr": 9.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661502046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13500580191612244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.495833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661516315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354016661643982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18841600, "lr": 9.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661530618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13118544220924377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.662499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661544661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13364659249782562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19169280, "lr": 9.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661558829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343492567539215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661573216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13695314526557922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19496960, "lr": 9.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661587531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13398835062980652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661601713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13299560546875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19824640, "lr": 1.0079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661615731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294887512922287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0162499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661629374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1276707947254181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20152320, "lr": 1.0245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661643181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13311472535133362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661657054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12972883880138397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20480000, "lr": 1.0412499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661670665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12462257593870163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661670666, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784661670666, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784661753894, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5047202706336975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784661753895, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784661753895, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784661767886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13139000535011292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20807680, "lr": 1.0579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661781651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13588716089725494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661795725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13342653214931488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21135360, "lr": 1.0745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661809862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13063772022724152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661823671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1281563639640808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21463040, "lr": 1.0912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661837921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12640880048274994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661852186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12905199825763702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21790720, "lr": 1.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661866091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13243651390075684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661880066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13554884493350983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22118400, "lr": 1.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661894007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1250339150428772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661907687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12846575677394867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22446080, "lr": 1.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661921467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13435883820056915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661935552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13112753629684448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22773760, "lr": 1.1579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661949787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12920919060707092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661949787, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784661949788, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784662032866, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5063016414642334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784662032867, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784662032867, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784662046925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13044272363185883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23101440, "lr": 1.1745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662060755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12919020652770996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662074231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331424117088318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23429120, "lr": 1.1912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662088826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12385617196559906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662103265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1296423375606537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23756800, "lr": 1.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662117207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12588661909103394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662131187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12999510765075684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24084480, "lr": 1.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662145466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12189626693725586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662159255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13576778769493103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24412160, "lr": 1.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662173588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13706083595752716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662187565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12362799048423767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24739840, "lr": 1.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662202067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12330584973096848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662216329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12564173340797424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25067520, "lr": 1.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662230570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1290406435728073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662230571, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784662230571, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784662313135, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5081444978713989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784662313136, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784662313136, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784662326956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13665038347244263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25395200, "lr": 1.2912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662341380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13669784367084503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662355162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13560304045677185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25722880, "lr": 1.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662368815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14022551476955414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662382635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13665997982025146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26050560, "lr": 1.3245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662396513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384442001581192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662410378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12757378816604614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26378240, "lr": 1.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662424138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12620095908641815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662437880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14237329363822937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26705920, "lr": 1.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662452133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14098887145519257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662466258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1273718625307083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27033600, "lr": 1.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662480203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1283930540084839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662494110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13242369890213013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27361280, "lr": 1.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662507508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13152214884757996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662507509, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784662507509, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784662590272, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5102068781852722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784662590273, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784662590273, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784662605376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14137548208236694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27688960, "lr": 1.4079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662620196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14257048070430756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662634523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354004442691803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28016640, "lr": 1.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662649107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13933195173740387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662663999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1450531780719757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28344320, "lr": 1.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662678298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14101271331310272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662692804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12612766027450562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28672000, "lr": 1.4579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662707299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376419961452484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662722072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317634880542755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28999680, "lr": 1.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662736663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13615137338638306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662751222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13782374560832977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29327360, "lr": 1.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662765428, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331111192703247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4995833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662780248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13454820215702057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29655040, "lr": 1.5079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662794592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13660554587841034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662794592, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784662794593, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784662875526, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5123598575592041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784662875527, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784662875527, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784662889528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386594921350479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29982720, "lr": 1.5245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662903995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380886286497116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662918550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1316414326429367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30310400, "lr": 1.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662932981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329125612974167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662947436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337766796350479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30638080, "lr": 1.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662962411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13967552781105042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662977386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13275635242462158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30965760, "lr": 1.5745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662992348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14123030006885529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663006738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1406668871641159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31293440, "lr": 1.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663021574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13508111238479614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663036075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12762971222400665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31621120, "lr": 1.6079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663050261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13018545508384705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663064420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13411881029605865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31948800, "lr": 1.6245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663078902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12777194380760193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663078903, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784663078903, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784663163329, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5149442553520203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784663163329, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784663163330, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784663177552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322093904018402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32276480, "lr": 1.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663191911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13671553134918213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663206410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13343705236911774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32604160, "lr": 1.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663220893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13186843693256378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663235501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13394153118133545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32931840, "lr": 1.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663249896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334502398967743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663264527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13383246958255768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33259520, "lr": 1.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663278962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13258393108844757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663293173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13997139036655426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33587200, "lr": 1.7079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663308131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13166727125644684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663322573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.128742977976799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33914880, "lr": 1.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663336884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13121894001960754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663351262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13212114572525024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34242560, "lr": 1.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663365726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357986032962799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7495833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663365726, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784663365727, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784663449410, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5184434056282043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784663449411, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784663449412, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784663464036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335851103067398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34570240, "lr": 1.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663478674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12296156585216522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663493285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13054616749286652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34897920, "lr": 1.7745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663507977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13038809597492218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663522430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13508571684360504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35225600, "lr": 1.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663537527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13533343374729156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663552010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354304701089859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35553280, "lr": 1.8079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663566769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12398926913738251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8162500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663581248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13104626536369324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35880960, "lr": 1.8245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663595371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13104477524757385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663610112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303461343050003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36208640, "lr": 1.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663624308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13304094970226288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663638620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13107267022132874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36536320, "lr": 1.8579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663652730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13021479547023773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663652731, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784663652731, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784663735697, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5232583284378052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784663735698, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784663735698, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784663750116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13779330253601074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36864000, "lr": 1.8745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663764544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323932260274887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663778776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334737241268158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37191680, "lr": 1.8912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663793103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13027417659759521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663807236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12719541788101196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37519360, "lr": 1.9079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663821373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322733610868454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663835884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13575130701065063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37847040, "lr": 1.9245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663850269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329377144575119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663864578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13119536638259888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38174720, "lr": 1.9412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663879099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13887722790241241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663893624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337345391511917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38502400, "lr": 1.9579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663907539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13777250051498413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663922370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317223608493805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38830080, "lr": 1.9745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663936529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13325555622577667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663936529, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784663936530, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784664019013, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5301088690757751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784664019015, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784664019015, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784664033213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13112807273864746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39157760, "lr": 1.9912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664047409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350865364074707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664061744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13762417435646057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39485440, "lr": 2.0079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664075991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333334892988205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664090050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12729424238204956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39813120, "lr": 2.0245833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664104189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12927928566932678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0329166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664118383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12577621638774872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40140800, "lr": 2.0412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664132264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12398124486207962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664146615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358814239501953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40468480, "lr": 2.0579166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664161056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13717389106750488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664175364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13455830514431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40796160, "lr": 2.0745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664190200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12695352733135223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0829166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664204563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1305464506149292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41123840, "lr": 2.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664218451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12357718497514725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664218451, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784664218452, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784664301268, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5401998162269592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784664301269, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784664301269, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784664315804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13247530162334442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41451520, "lr": 2.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664330167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324150264263153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664344409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13364361226558685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41779200, "lr": 2.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664358640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13781920075416565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664372969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13622289896011353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42106880, "lr": 2.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664387365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13128021359443665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664401624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1271827667951584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42434560, "lr": 2.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664415897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350192427635193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664430015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12476976960897446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42762240, "lr": 2.1745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664444586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13038043677806854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664458965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321927160024643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43089920, "lr": 2.1912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664473558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13388708233833313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664488355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1404450535774231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43417600, "lr": 2.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664502700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13603733479976654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664502701, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784664502702, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784664583624, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5559850335121155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784664583625, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784664583625, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784664598485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13650840520858765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43745280, "lr": 2.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664613074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13111497461795807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664627362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13239818811416626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44072960, "lr": 2.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664641811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13881073892116547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664656898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12973974645137787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44400640, "lr": 2.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664672060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382676213979721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664686580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345081925392151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44728320, "lr": 2.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664701364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13686566054821014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664715997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344965249300003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45056000, "lr": 2.2912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664730830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.130527526140213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664745595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1399218738079071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45383680, "lr": 2.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664760337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13455846905708313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664775010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12854449450969696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45711360, "lr": 2.3245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664789772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333451122045517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664789773, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784664789774, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784664871946, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5728837847709656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784664871947, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784664871947, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784664886813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1413743495941162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46039040, "lr": 2.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664901339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322048008441925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664916320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13313816487789154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46366720, "lr": 2.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664930942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13019835948944092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664945959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13497790694236755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46694400, "lr": 2.3745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664960862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13318601250648499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664975595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335376799106598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47022080, "lr": 2.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664990276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13515476882457733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665004863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13077928125858307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47349760, "lr": 2.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665019748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323266625404358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665034568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13241222500801086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47677440, "lr": 2.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665049435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13669615983963013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665063909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358635276556015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48005120, "lr": 2.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665078735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13683247566223145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.4495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665078735, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784665078735, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784665163048, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5888171792030334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784665163048, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784665163049, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784665177608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13101112842559814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48332800, "lr": 2.4579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665192469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13461506366729736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665207144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13337691128253937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48660480, "lr": 2.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665222333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13159269094467163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665236903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13649587333202362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48988160, "lr": 2.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665251331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12881624698638916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665266414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13308638334274292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49315840, "lr": 2.507916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665280753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12234139442443848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.5162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665295309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12991908192634583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49643520, "lr": 2.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665309767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308651715517044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665324290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13019444048404694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49971200, "lr": 2.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665338480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13046270608901978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.5495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665352764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12806424498558044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50298880, "lr": 2.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665367265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12959638237953186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.5662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665367266, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784665367266, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784665452271, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6012089848518372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784665452272, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784665452272, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784665466355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12115747481584549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50626560, "lr": 2.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665480739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12562403082847595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665495090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1275976449251175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50954240, "lr": 2.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665509580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13216876983642578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665524300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13072320818901062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51281920, "lr": 2.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665538826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1206408143043518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665553626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12916713953018188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51609600, "lr": 2.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665568394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.130523681640625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665582988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357566863298416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51937280, "lr": 2.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665597193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12666645646095276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.6495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665612212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13168780505657196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52264960, "lr": 2.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665626413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11652933806180954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.6662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665640704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11997154355049133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52592640, "lr": 2.6745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665655066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12366432696580887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665655067, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784665655067, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784665739123, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6119140982627869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784665739124, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784665739124, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784665753246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12019892036914825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52920320, "lr": 2.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665767250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12546274065971375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.6995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665781473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1274472028017044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53248000, "lr": 2.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665795657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13040021061897278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665809782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11969975382089615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53575680, "lr": 2.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665824037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133904367685318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665838279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12760086357593536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53903360, "lr": 2.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665852420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12017550319433212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.7495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665866857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12554991245269775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54231040, "lr": 2.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665881454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12509562075138092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665895741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11682901531457901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54558720, "lr": 2.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665910541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13018769025802612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665925066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.126125305891037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54886400, "lr": 2.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665939472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12757688760757446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665939472, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784665939473, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784666025890, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.620954692363739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784666025891, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784666025891, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784666039855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1254407912492752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55214080, "lr": 2.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666054821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1168498620390892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666069581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12058831006288528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55541760, "lr": 2.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666084400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11953532695770264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666098945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12219134718179703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55869440, "lr": 2.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666113529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12452683597803116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666127980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11447031795978546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56197120, "lr": 2.8579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666142621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12158572673797607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666157203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12373241782188416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56524800, "lr": 2.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666171695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12193397432565689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666186277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1139625757932663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56852480, "lr": 2.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666200716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12083431333303452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666215042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10738841444253922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57180160, "lr": 2.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666229485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12303372472524643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666229485, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784666229486, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784666316188, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6297642588615417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784666316189, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784666316189, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784666330772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11469430476427078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57507840, "lr": 2.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666345297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11310458183288574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666359914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12785103917121887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57835520, "lr": 2.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666374952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11945724487304688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666389483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12211843580007553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58163200, "lr": 2.9579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666403922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11416764557361603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666418097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11758730560541153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58490880, "lr": 2.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666432930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1204940676689148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666447345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11945618689060211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58818560, "lr": 2.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666461750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1114073395729065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666476097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11857143044471741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59146240, "lr": 3.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666490393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1214645653963089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.0162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666504684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12401822209358215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59473920, "lr": 3.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666519120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11638699471950531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666519120, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784666519121, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784666605607, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6432831287384033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784666605608, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784666605609, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784666620001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11754223704338074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59801600, "lr": 3.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666634170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11094605922698975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666648244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10880561172962189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60129280, "lr": 3.0579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666662278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11260897666215897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.0662499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666676383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10790131241083145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60456960, "lr": 3.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666690591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12273216247558594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666705032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.114004947245121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60784640, "lr": 3.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666719719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12025664746761322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666733994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10052774101495743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61112320, "lr": 3.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666748680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1203790009021759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.1162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666763411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11204218119382858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61440000, "lr": 3.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666778029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11177803575992584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666792469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10890824347734451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61767680, "lr": 3.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666806903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11686602979898453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666806904, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784666806904, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784666891418, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6628134250640869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784666891419, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784666891419, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784666905895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12059533596038818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62095360, "lr": 3.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666920521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12058954685926437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.1662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666935231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12281739711761475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62423040, "lr": 3.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666949832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10503115504980087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666964421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11346498131752014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62750720, "lr": 3.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666978860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11680986732244492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666993588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11969521641731262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63078400, "lr": 3.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667008320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11896408349275589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667022873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10883548855781555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63406080, "lr": 3.2245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667037300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11796131730079651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667051666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11624391376972198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63733760, "lr": 3.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667065856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11689098924398422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667080234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11355000734329224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64061440, "lr": 3.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667094980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12568703293800354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667094981, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784667094981, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784667179442, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6876556873321533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784667179443, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784667179443, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784667194176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12357061356306076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64389120, "lr": 3.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667208902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11668358743190765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667223556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11678680032491684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64716800, "lr": 3.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667238800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12324497103691101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667253694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11042983829975128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65044480, "lr": 3.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667268472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11192025244235992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667282693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11352083832025528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65372160, "lr": 3.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667297709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10912134498357773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667312473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11115780472755432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65699840, "lr": 3.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667327682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11679071933031082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667342604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12447724491357803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66027520, "lr": 3.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667357352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11537674814462662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667372347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1278441995382309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66355200, "lr": 3.3745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667387195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11262689530849457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667387195, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784667387196, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784667470654, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7091096043586731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784667470655, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784667470655, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784667485581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12713424861431122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66682880, "lr": 3.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667500225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11452675610780716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.3995833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667514931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11777949333190918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67010560, "lr": 3.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667530024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11344048380851746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667544675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11951036751270294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67338240, "lr": 3.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667559646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11982087045907974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667574951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12213747203350067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67665920, "lr": 3.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667590141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12122014909982681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667605163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11611546576023102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67993600, "lr": 3.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667620671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11380874365568161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667635710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11679720133543015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68321280, "lr": 3.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667650493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1173253282904625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667665244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12641097605228424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68648960, "lr": 3.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667680669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11634532362222672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667680670, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784667680670, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784667764929, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7291173934936523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784667764930, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784667764930, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784667780167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12371691316366196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68976640, "lr": 3.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667795656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12096355110406876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667810770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11430855840444565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69304320, "lr": 3.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667825922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11629936099052429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667840844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10955443978309631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69632000, "lr": 3.5412500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667855876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11597130447626114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667870868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11884675920009613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69959680, "lr": 3.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667885693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11268441379070282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667900699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10992365330457687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70287360, "lr": 3.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667915594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10943824797868729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667930866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12353833764791489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70615040, "lr": 3.5912500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667946250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10787832736968994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667961292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12156340479850769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70942720, "lr": 3.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667976817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11976757645606995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667976818, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784667976818, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784668060889, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7434591054916382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784668060890, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784668060890, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784668076049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12144766747951508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71270400, "lr": 3.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668090944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1202075183391571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668105896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.115965835750103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71598080, "lr": 3.6412499999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668121261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11726990342140198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668136286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10428761690855026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71925760, "lr": 3.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668151340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10596517473459244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668166223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12063826620578766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72253440, "lr": 3.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668181134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11560311168432236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668195992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10565987229347229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72581120, "lr": 3.6912499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668210978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10434678196907043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668225668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1083378866314888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72908800, "lr": 3.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668240593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11311125010251999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668255469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11303956061601639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73236480, "lr": 3.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668270327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1163102388381958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668270373, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784668270374, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784668357095, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7527211308479309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784668357097, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784668357097, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784668371902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1049090176820755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73564160, "lr": 3.7412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668387103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10804679989814758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668402423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1168646588921547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73891840, "lr": 3.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668417330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11196686327457428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668432520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10903113335371017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74219520, "lr": 3.7745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668447894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11584555357694626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668463010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11360613256692886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74547200, "lr": 3.7912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668477803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11425746977329254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668492919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.117919921875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74874880, "lr": 3.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668507816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11229243874549866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668522530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10244238376617432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75202560, "lr": 3.8245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668537276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11459369957447052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668551786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10855696350336075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75530240, "lr": 3.8412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668566429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10802388936281204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668566430, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784668566430, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784668652507, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7561548352241516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784668652508, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784668652508, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784668667121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10776841640472412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75857920, "lr": 3.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668682032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11459986865520477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668696847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09808070212602615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76185600, "lr": 3.8745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668711460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.104339100420475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668726224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11396808922290802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76513280, "lr": 3.8912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668740817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1058289185166359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668755434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08985555917024612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76840960, "lr": 3.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668769970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11000413447618484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668784767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10409477353096008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77168640, "lr": 3.9245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668799626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11719410866498947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668814712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10837613791227341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77496320, "lr": 3.9412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668829616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10105680674314499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668844917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10145687311887741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77824000, "lr": 3.9579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668860082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1083582192659378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668860083, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784668860083, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784668944046, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7575886249542236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784668944046, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784668944047, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784668958922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11725829541683197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78151680, "lr": 3.9745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668973782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11249864846467972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668988127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10458160191774368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78479360, "lr": 3.9912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669003529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071750670671463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669018157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10041751712560654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78807040, "lr": 4.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669033115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10448572784662247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669047902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10781173408031464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79134720, "lr": 4.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669062954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10943431407213211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669077824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09833849966526031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79462400, "lr": 4.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669092917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10944432020187378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669107763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1054118201136589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79790080, "lr": 4.0579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669122685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10837013274431229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669137551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10842226445674896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80117760, "lr": 4.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669152464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1131020113825798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669152464, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784669152465, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784669234337, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7582312226295471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784669234338, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784669234338, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784669249250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11125095933675766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80445440, "lr": 4.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669263587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10024839639663696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669278317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10399632155895233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80773120, "lr": 4.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669293276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10748755186796188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669308188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11136133968830109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81100800, "lr": 4.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669323151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1140298843383789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669338083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10084249079227448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81428480, "lr": 4.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669353126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10602894425392151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669368150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10773903876543045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81756160, "lr": 4.1579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669382904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10656298696994781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669397719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11305981874465942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82083840, "lr": 4.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669411948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09556512534618378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669427331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09842762351036072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82411520, "lr": 4.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669442093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10843703895807266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669442094, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784669442095, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784669523565, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7607043981552124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784669523566, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784669523566, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784669537901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0926835909485817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82739200, "lr": 4.2079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669552569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10499082505702972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669567108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11043862253427505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83066880, "lr": 4.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669581840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09891252964735031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669596354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1110629215836525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83394560, "lr": 4.2412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669611000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11156409978866577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669625606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09795581549406052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83722240, "lr": 4.2579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669640038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11156366765499115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669654720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10636813193559647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84049920, "lr": 4.2745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669669213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10547517985105515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669683907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10938441008329391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84377600, "lr": 4.2912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669698093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1055293157696724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669712415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09825889766216278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84705280, "lr": 4.3079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669727074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10733496397733688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669727074, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784669727075, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784669808692, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7623792290687561, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784669808693, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784669808694, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784669823746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10510244220495224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85032960, "lr": 4.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669838801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10536855459213257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669853494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10598364472389221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85360640, "lr": 4.3412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669868376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10419239848852158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669883241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0989052876830101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85688320, "lr": 4.3579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669898024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10094214230775833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669913102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10651808977127075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86016000, "lr": 4.3745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669927407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09615535289049149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669942349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10823020339012146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86343680, "lr": 4.3912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669956926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10890167206525803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669971303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11711908131837845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86671360, "lr": 4.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669985820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11546462774276733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670000129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11623266339302063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86999040, "lr": 4.4245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670014535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11567256599664688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670014535, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784670014536, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784670096955, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7642495632171631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784670096956, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784670096956, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784670111148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10162676870822906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87326720, "lr": 4.4412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670125576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11619577556848526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670139987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082889661192894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87654400, "lr": 4.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670154763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10267677158117294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670169285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0912107601761818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87982080, "lr": 4.4745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670183915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09837912023067474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670198363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1045994758605957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88309760, "lr": 4.4912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670212983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10778253525495529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670227381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10833050310611725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88637440, "lr": 4.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670241906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10034714639186859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670256893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1132354587316513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88965120, "lr": 4.5245833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670272011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09982895851135254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670287188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11005252599716187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89292800, "lr": 4.5412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670302031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11303727328777313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.5495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670302032, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784670302033, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784670385090, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7652899026870728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784670385091, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784670385091, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784670400129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081085056066513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89620480, "lr": 4.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670415040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10190222412347794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670429680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11055015027523041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89948160, "lr": 4.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670444492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09185964614152908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670458768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11512662470340729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90275840, "lr": 4.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670473104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10722924768924713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670488726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11155542731285095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90603520, "lr": 4.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670503727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11109859496355057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670518943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12246204167604446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90931200, "lr": 4.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670533887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10555077344179153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670548947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11207500845193863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91258880, "lr": 4.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670564202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1233496367931366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670579117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09988876432180405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91586560, "lr": 4.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670593996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11175902932882309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670593997, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784670593998, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784670677499, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7659822106361389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784670677500, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784670677500, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784670692403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1041853278875351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91914240, "lr": 4.6745833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670707229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1188519150018692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670721978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10972927510738373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92241920, "lr": 4.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670737113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10551214963197708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670751924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09680968523025513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92569600, "lr": 4.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670766840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11308252066373825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670782117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10640640556812286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92897280, "lr": 4.7245833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670797243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10679514706134796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670812294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11681105941534042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93224960, "lr": 4.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670827043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09955725818872452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670842260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12470730394124985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93552640, "lr": 4.7579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670857373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10775639861822128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670872459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11724931001663208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93880320, "lr": 4.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670887416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10818365961313248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670887416, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784670887417, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784670972195, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7664766311645508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784670972196, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784670972196, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784670987319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11430183798074722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94208000, "lr": 4.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671002488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11581480503082275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671017659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11129315197467804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94535680, "lr": 4.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671032759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11803755164146423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671047761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11463473737239838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94863360, "lr": 4.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671062935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11893336474895477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671078378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12742270529270172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95191040, "lr": 4.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671093433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12101880460977554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671108491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11610987037420273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95518720, "lr": 4.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671123977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11090923845767975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671139177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10860344022512436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95846400, "lr": 4.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671154471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10804637521505356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671169621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1151478961110115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96174080, "lr": 4.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671184928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11478086560964584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671184929, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784671184929, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784671269494, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.768110990524292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784671269495, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784671269495, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784671284903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12000101059675217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96501760, "lr": 4.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671300290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10728660970926285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671315381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10903028398752213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96829440, "lr": 4.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671331009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10800377279520035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671346364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11121080070734024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97157120, "lr": 4.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671361509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10979808866977692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671376581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11363855749368668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97484800, "lr": 4.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671391490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10803906619548798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671406323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10671205073595047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97812480, "lr": 4.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671420829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10081226378679276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671435893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10422738641500473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98140160, "lr": 4.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671450616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09738171845674515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671465463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10440685600042343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98467840, "lr": 5.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671480272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10075122117996216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671480272, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784671480272, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784671563871, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7692722678184509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784671563872, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784671563872, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784671578557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10972961038351059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98795520, "lr": 5.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671593438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10126078873872757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671608293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0970630869269371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99123200, "lr": 5.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671623130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10866563767194748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671638196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11503998935222626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99450880, "lr": 5.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671653027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10500345379114151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671667828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10482362657785416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99778560, "lr": 5.074583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671682712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10499686747789383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671697500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10202785581350327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100106240, "lr": 5.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671712502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1200610026717186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671727533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1164766177535057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100433920, "lr": 5.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671742604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10167405754327774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671757880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10052410513162613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100761600, "lr": 5.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671772644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12051520496606827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671772646, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784671772646, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784671857530, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7696687579154968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784671857531, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784671857531, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784671872683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11068937182426453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101089280, "lr": 5.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671887679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09660215675830841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671902652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10506575554609299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101416960, "lr": 5.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671917711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10353858768939972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671932395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10871294885873795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101744640, "lr": 5.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671947337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10516069084405899, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.182916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671962072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10690950602293015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102072320, "lr": 5.191249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671976661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11220815777778625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671991603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10489332675933838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102400000, "lr": 5.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672006488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793318599462509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672021111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1053493320941925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102727680, "lr": 5.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672035775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11476066708564758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672050160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10678164660930634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103055360, "lr": 5.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672064753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10251594334840775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672064754, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784672064754, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784672149188, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7702942490577698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784672149189, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784672149189, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784672163786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09976334869861603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103383040, "lr": 5.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672178541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09443977475166321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672193090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09640350937843323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103710720, "lr": 5.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672207751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09660836309194565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.282916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672222396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09681497514247894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104038400, "lr": 5.291249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672237299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09481257945299149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672251980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10057759284973145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104366080, "lr": 5.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672266708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09689074009656906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672281779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10284414142370224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104693760, "lr": 5.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672296635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10890453308820724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672311729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10473132878541946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105021440, "lr": 5.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672326660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10343988984823227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672341540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11470083892345428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105349120, "lr": 5.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672356953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11166082322597504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672356954, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784672356955, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784672438824, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7708221077919006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784672438825, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784672438825, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784672453919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09870336204767227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105676800, "lr": 5.374583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672468977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10716582834720612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.382916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672484296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10977748036384583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106004480, "lr": 5.391249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672499121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10111355036497116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672514110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10075461119413376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106332160, "lr": 5.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672528958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10774217545986176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672543646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11068297922611237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106659840, "lr": 5.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672558408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10626283288002014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672573429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10247038304805756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106987520, "lr": 5.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672588123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10176991671323776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672602805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09983641654253006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107315200, "lr": 5.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672617423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10291752964258194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672632122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10973136872053146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107642880, "lr": 5.474583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672646733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11457201838493347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.482916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672646735, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784672646735, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784672727401, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7709139585494995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784672727402, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784672727402, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784672742045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11017722636461258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107970560, "lr": 5.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672756595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071045771241188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672771287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09126114100217819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108298240, "lr": 5.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672785923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10313673317432404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672800735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1067257672548294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108625920, "lr": 5.524583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672815167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10808424651622772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672829638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10991548001766205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108953600, "lr": 5.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672844333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10313700139522552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672859211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10033799707889557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109281280, "lr": 5.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672874171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10416644811630249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672889222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10264048725366592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109608960, "lr": 5.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672903938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10823312401771545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672918987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1050790399312973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109936640, "lr": 5.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672933992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1093808263540268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672933992, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784672933993, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784673015791, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7714128494262695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784673015792, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784673015792, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784673030514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10372438281774521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110264320, "lr": 5.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673045230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10383300483226776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673059743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039569079875946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110592000, "lr": 5.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673073966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1024923324584961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673089412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10730501264333725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110919680, "lr": 5.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673104099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10125790536403656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673118850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09889412671327591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111247360, "lr": 5.657916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673133853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10221643000841141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.666250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673148630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11163705587387085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111575040, "lr": 5.674583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673163369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10510937124490738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673178090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10386615991592407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111902720, "lr": 5.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673192729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10281877964735031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673207613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10434514284133911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112230400, "lr": 5.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673222328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10681722313165665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673222329, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784673222329, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784673307266, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7719566226005554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784673307267, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784673307267, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784673322207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10842598229646683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112558080, "lr": 5.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673337138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10217338055372238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673351896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10413184016942978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112885760, "lr": 5.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673366832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10138615220785141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673381881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10503864288330078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113213440, "lr": 5.757916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673396687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11246854811906815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.766250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673411918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10163788497447968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113541120, "lr": 5.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673427083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11032926291227341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673442617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11377698928117752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113868800, "lr": 5.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673458216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10611158609390259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673473494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10358814150094986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114196480, "lr": 5.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673488901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1048101857304573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673504637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10495594143867493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114524160, "lr": 5.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673520107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09945116192102432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673520107, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784673520108, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784673605239, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7724649310112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784673605240, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784673605240, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784673620687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10258351266384125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114851840, "lr": 5.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673635907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09946665912866592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673651186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10635492950677872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115179520, "lr": 5.857916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673665950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10072574764490128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673681333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10799048840999603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115507200, "lr": 5.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673696120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10942861437797546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673711122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10755161195993423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115834880, "lr": 5.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673726207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11526735872030258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673741177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11421159654855728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116162560, "lr": 5.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673756101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10090754181146622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673770885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.103833869099617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116490240, "lr": 5.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673785604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10311432927846909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673800301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10531531274318695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116817920, "lr": 5.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673814811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10762470960617065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673814812, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784673814812, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784673898097, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7723333835601807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784673898099, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784673898099, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784673913002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10236167907714844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117145600, "lr": 5.957916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673927860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10068614780902863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673942963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10920953750610352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117473280, "lr": 5.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673957896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10494718700647354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673972692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09432478249073029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117800960, "lr": 5.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673988026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11214663833379745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674002904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10120684653520584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118128640, "lr": 6.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674017763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1076372042298317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674032874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.103740394115448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118456320, "lr": 6.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674048123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10340366512537003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674063100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10963961482048035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118784000, "lr": 6.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674078319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11463237553834915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674093397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10606727749109268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119111680, "lr": 6.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674108620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10310696065425873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674108620, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784674108621, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784674192029, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.772475004196167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784674192030, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784674192030, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784674207354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11335679888725281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119439360, "lr": 6.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674222492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10921350866556168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674237551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10990176349878311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119767040, "lr": 6.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674252858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10751769691705704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674267666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10136682540178299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120094720, "lr": 6.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674282227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10810829699039459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674298093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11782412230968475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120422400, "lr": 6.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674313669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10903023928403854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674329121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11648276448249817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120750080, "lr": 6.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674344920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12224160879850388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674360411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10737156867980957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121077760, "lr": 6.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674375932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11086719483137131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674391564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11468353122472763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121405440, "lr": 6.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674406973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10974818468093872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674406974, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784674406974, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784674490114, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7725011706352234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784674490115, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784674490115, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784674505555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11369781941175461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121733120, "lr": 6.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674521171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1095953956246376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674536619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11111204326152802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122060800, "lr": 6.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674552136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1066138744354248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674567621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1045481488108635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122388480, "lr": 6.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674583124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11340418457984924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674598636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10667996108531952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122716160, "lr": 6.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674614224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11485638469457626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674630044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11223098635673523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123043840, "lr": 6.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674645795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12006304413080215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674661752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1060485988855362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123371520, "lr": 6.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674677521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11758163571357727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674693173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10743365436792374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123699200, "lr": 6.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674708868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11945103108882904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674708869, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784674708869, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784674791004, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7728175520896912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784674791005, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784674791006, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784674806601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11982090771198273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124026880, "lr": 6.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674822000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10802178829908371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674837140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11970112472772598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124354560, "lr": 6.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674852242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11422348022460938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674867537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10745739191770554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124682240, "lr": 6.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674883117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11521004885435104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674898403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11198542267084122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125009920, "lr": 6.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674913821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11656239628791809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674929299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11290138959884644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125337600, "lr": 6.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674944486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10586400330066681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674959635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11376823484897614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125665280, "lr": 6.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674974841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10608775913715363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674989934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11699650436639786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125992960, "lr": 6.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675005327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10553760826587677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675005328, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784675005328, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784675087926, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7733796834945679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784675087926, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784675087927, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784675103023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10985536128282547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126320640, "lr": 6.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675118360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11963549256324768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675133406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11065738648176193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126648320, "lr": 6.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675148524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11370307952165604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675163689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11601575464010239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126976000, "lr": 6.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675178810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10934940725564957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675194046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10823490470647812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127303680, "lr": 6.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675209527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11005768924951553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.482916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675225075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1124274879693985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127631360, "lr": 6.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675240255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11278194189071655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675255654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10992828756570816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127959040, "lr": 6.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675271142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11039676517248154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675286588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11148940026760101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128286720, "lr": 6.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675301907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1090904027223587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675301907, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784675301908, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784675386649, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7734882831573486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784675386650, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784675386650, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784675401587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11816207319498062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128614400, "lr": 6.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675416920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11108239740133286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675431795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11082352697849274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128942080, "lr": 6.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675446864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10315161198377609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675461597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10590894520282745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129269760, "lr": 6.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675476593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11198033392429352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675491428, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081482470035553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129597440, "lr": 6.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675506086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772236436605453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675520732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0972389280796051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129925120, "lr": 6.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675535382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1018366739153862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675550101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10551013797521591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130252800, "lr": 6.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675564622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1045004203915596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675579481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11025072634220123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130580480, "lr": 6.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675594480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11628006398677826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675594481, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784675594482, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784675679527, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7736825942993164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784675679528, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784675679529, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784675694222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11461648344993591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130908160, "lr": 6.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675708909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10559825599193573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675723729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10083957016468048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131235840, "lr": 6.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675738870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09228663146495819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675753594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11363030225038528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131563520, "lr": 6.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675768074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10408120602369308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675782819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10255050659179688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131891200, "lr": 6.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675797883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11254727095365524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675812814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10721845924854279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132218880, "lr": 6.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675827820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10362335294485092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675842744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10958860069513321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132546560, "lr": 6.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675857965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11101820319890976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675873303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772555321455002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132874240, "lr": 6.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675888398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10714070498943329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675888399, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784675888399, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784675970186, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7737115621566772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784675970187, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784675970187, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784675985049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10972150415182114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133201920, "lr": 6.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676000214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10336288064718246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676015201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11057031899690628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133529600, "lr": 6.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676029996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09952414035797119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676045610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11580689251422882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133857280, "lr": 6.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676060773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11035864800214767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676075785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10362912714481354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134184960, "lr": 6.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676091068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792698711156845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676106235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10840388387441635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134512640, "lr": 6.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676120993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087806299328804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676135958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1105596125125885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134840320, "lr": 6.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676150782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0992150530219078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676165605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10701523721218109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135168000, "lr": 6.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676180413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08735102415084839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676180414, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784676180414, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784676262471, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738859057426453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784676262472, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784676262473, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784676277537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09522689133882523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135495680, "lr": 6.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676292590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11109552532434464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676307686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10756984353065491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135823360, "lr": 6.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676322759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10154039412736893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676337763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10206974297761917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136151040, "lr": 6.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676353003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0902676060795784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676367973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10362037271261215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136478720, "lr": 6.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676382986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09716498851776123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676398031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039154902100563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136806400, "lr": 6.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676413204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10638890415430069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676428456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11056242138147354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137134080, "lr": 6.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676443509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10819084197282791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676458528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10648135095834732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137461760, "lr": 6.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676473732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10317320376634598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676473732, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784676473733, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784676557917, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773803174495697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784676557918, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784676557919, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784676573088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09651698917150497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137789440, "lr": 7.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676588604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10531379282474518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676604078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09497105330228806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138117120, "lr": 7.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676619434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10249438881874084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676634842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0979381799697876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138444800, "lr": 7.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676649867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10048148036003113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676664798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10922686755657196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138772480, "lr": 7.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676680344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10090120881795883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.066249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676695000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10900279134511948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139100160, "lr": 7.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676709851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10977187752723694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676725045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10658758133649826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139427840, "lr": 7.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676740170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10454875230789185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676755101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10964180529117584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139755520, "lr": 7.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676770074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10920406132936478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676770075, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784676770075, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784676852533, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742225527763367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784676852534, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784676852534, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784676867400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10668321698904037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140083200, "lr": 7.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676882238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1047973558306694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676897027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10382308810949326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140410880, "lr": 7.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676911880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0999973937869072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676926853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10667775571346283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140738560, "lr": 7.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676941777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09605084359645844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676957300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10548501461744308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141066240, "lr": 7.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676972207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10338583588600159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676986980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11048562079668045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141393920, "lr": 7.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677001847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09917221963405609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677016488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09752672165632248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141721600, "lr": 7.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677031284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10231642425060272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677046068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09088427573442459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142049280, "lr": 7.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677060955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10739145427942276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677060956, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784677060956, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784677142562, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742472290992737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784677142563, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784677142563, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784677157194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09944417327642441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142376960, "lr": 7.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677172155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10115378350019455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677186931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09906347095966339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142704640, "lr": 7.257916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677202053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11000346392393112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677217029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10043911635875702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143032320, "lr": 7.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677232178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10243477672338486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677246999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10215522348880768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143360000, "lr": 7.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677261689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10530959069728851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677276641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10203330218791962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143687680, "lr": 7.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677291198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11296837031841278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677306479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10116874426603317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144015360, "lr": 7.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677321998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10823141783475876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677337392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10547935962677002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144343040, "lr": 7.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677353079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1060882955789566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677353080, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784677353081, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784677434484, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7743372321128845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784677434485, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784677434485, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784677450101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1065770611166954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144670720, "lr": 7.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677465795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09834153950214386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677481228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10690996795892715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144998400, "lr": 7.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677496958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10739587992429733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677512642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10273580253124237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145326080, "lr": 7.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677528032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10731975734233856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677543601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09769302606582642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145653760, "lr": 7.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677559301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0985049307346344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677574834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1051524356007576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145981440, "lr": 7.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677590336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10466697812080383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677606009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10313636064529419, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146309120, "lr": 7.441250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677621753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1100674718618393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677637505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10679648816585541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146636800, "lr": 7.457916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677653055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10012850910425186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677653056, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784677653057, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784677735412, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7745670676231384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784677735413, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784677735413, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784677750913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10341231524944305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146964480, "lr": 7.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677766285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10544803738594055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677781810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09456998854875565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147292160, "lr": 7.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677797645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09875989705324173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677813118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10208892822265625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147619840, "lr": 7.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677828899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10032607614994049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677844601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10692936927080154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147947520, "lr": 7.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677860430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10973890125751495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677876073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10853261500597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148275200, "lr": 7.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677891688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10033433139324188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677907431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10270387679338455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148602880, "lr": 7.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677922625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0986272320151329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677938123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1124630942940712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148930560, "lr": 7.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677953143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.110997274518013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677953143, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784677953144, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784678038194, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7747896909713745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784678038195, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784678038196, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784678053043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10585133731365204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149258240, "lr": 7.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678068577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10483498871326447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678083286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10647380352020264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149585920, "lr": 7.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678098220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11115065217018127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149749760, "lr": 7.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678113511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10632440447807312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149913600, "lr": 7.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678128682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10900268703699112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678143345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10783589631319046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150241280, "lr": 7.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678158281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0977359265089035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150405120, "lr": 7.649583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678173183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10699149966239929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150568960, "lr": 7.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678187812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10376715660095215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678202744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082145944237709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150896640, "lr": 7.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678217273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10361424833536148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151060480, "lr": 7.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678232370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10634340345859528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151224320, "lr": 7.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678247296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09585977345705032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678247344, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784678247344, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784678329785, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7744455337524414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784678329786, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784678329786, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784678344626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09921760112047195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151552000, "lr": 7.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678359395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10545545071363449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151715840, "lr": 7.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678374355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10258479416370392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151879680, "lr": 7.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678389348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10407175868749619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678404339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10782988369464874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152207360, "lr": 7.741249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678419207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1234048455953598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152371200, "lr": 7.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678434332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10802687704563141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152535040, "lr": 7.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678449633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09885881841182709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678464934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11261860281229019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152862720, "lr": 7.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678480151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11222001910209656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153026560, "lr": 7.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678495494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11461423337459564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153190400, "lr": 7.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678510811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.099125936627388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678526272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10664304345846176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153518080, "lr": 7.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678541553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1047402024269104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153681920, "lr": 7.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678541601, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784678541601, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784678624448, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7749985456466675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784678624449, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784678624450, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784678639697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09798288345336914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153845760, "lr": 7.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678654898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09812422096729279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.832916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678669940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11301998049020767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154173440, "lr": 7.841249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678685186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11781749874353409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154337280, "lr": 7.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678700101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11432977020740509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154501120, "lr": 7.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678714701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11105909943580627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678730468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10679641366004944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154828800, "lr": 7.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678746023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12030281126499176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154992640, "lr": 7.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678761574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10520271956920624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155156480, "lr": 7.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678777406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10794291645288467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678793261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11125292629003525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155484160, "lr": 7.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678808852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11062360554933548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155648000, "lr": 7.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678824426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1130780577659607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155811840, "lr": 7.924583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678839953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11141259968280792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.932916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678839953, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784678839954, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784678924234, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752209901809692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784678924235, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784678924235, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784678939900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10720156133174896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156139520, "lr": 7.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678955608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11055469512939453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156303360, "lr": 7.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678971471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11452797800302505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156467200, "lr": 7.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678987124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10999633371829987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679002704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10821057111024857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156794880, "lr": 7.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679018516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10828284174203873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156958720, "lr": 7.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679034157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10174847394227982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157122560, "lr": 7.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679049727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11298618465662003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679065307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12004699558019638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157450240, "lr": 8.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679081034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10954868048429489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157614080, "lr": 8.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679096551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10886732488870621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157777920, "lr": 8.024583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679112500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10453587770462036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.032916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679128448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10349419713020325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158105600, "lr": 8.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679144283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10870437324047089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158269440, "lr": 8.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679144284, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784679144284, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784679229642, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7751529812812805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784679229642, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784679229643, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784679245329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11429392546415329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158433280, "lr": 8.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679260858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12012463808059692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679276413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11408296227455139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158760960, "lr": 8.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679292134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11227956414222717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158924800, "lr": 8.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679307911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11120575666427612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159088640, "lr": 8.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679323355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1203671544790268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679339040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1117851585149765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159416320, "lr": 8.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679354222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10903777927160263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159580160, "lr": 8.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679370023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11065398901700974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159744000, "lr": 8.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679385483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09889756143093109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679400742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11035359650850296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160071680, "lr": 8.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679416536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10935057699680328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160235520, "lr": 8.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679432245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10431171953678131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160399360, "lr": 8.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679447465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11110275238752365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679447466, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784679447466, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784679531755, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7751463651657104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784679531755, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784679531756, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784679546883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11320057511329651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160727040, "lr": 8.174583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679562289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1287248134613037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160890880, "lr": 8.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679577761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10823465138673782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161054720, "lr": 8.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679593159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1123003363609314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679608393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11308407783508301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161382400, "lr": 8.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679623823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11213038861751556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161546240, "lr": 8.216250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679639114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11454028636217117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161710080, "lr": 8.224583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679654684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11316046863794327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679670041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10698448121547699, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162037760, "lr": 8.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679685450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084374263882637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162201600, "lr": 8.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679701323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12970314919948578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162365440, "lr": 8.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679716794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1133008524775505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679732693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11337722837924957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162693120, "lr": 8.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679748640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11652883887290955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162856960, "lr": 8.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679748641, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784679748641, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784679831989, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7754664421081543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784679831989, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784679831990, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784679847237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09554106742143631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163020800, "lr": 8.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679862768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11126850545406342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679878432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12231703847646713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163348480, "lr": 8.307916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679893983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10698774456977844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163512320, "lr": 8.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679909800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10962364822626114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163676160, "lr": 8.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679925495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10747577250003815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679941001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11171978712081909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164003840, "lr": 8.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679956070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11260786652565002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164167680, "lr": 8.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679971607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10342445969581604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164331520, "lr": 8.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679986599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1184222623705864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680001707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084275171160698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164659200, "lr": 8.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680017008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11136496812105179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164823040, "lr": 8.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680032387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09249947220087051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164986880, "lr": 8.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680047405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10379166901111603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680047406, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784680047406, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784680130800, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7755693197250366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784680130801, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784680130801, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784680146077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10179179906845093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165314560, "lr": 8.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680161154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0969187542796135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165478400, "lr": 8.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680176094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11491759121417999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165642240, "lr": 8.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680190972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10774854570627213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680205708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10944300144910812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165969920, "lr": 8.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680220858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10445953160524368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166133760, "lr": 8.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680235865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10786265879869461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166297600, "lr": 8.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680250932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11337218433618546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680265946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09965506196022034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166625280, "lr": 8.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680281055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09585850685834885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166789120, "lr": 8.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680296068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09711847454309464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166952960, "lr": 8.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680310965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09679725766181946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680325795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10243436694145203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167280640, "lr": 8.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680340774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1008734181523323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167444480, "lr": 8.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680340775, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784680340775, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784680424550, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7755444049835205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784680424551, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784680424551, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784680439477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10375186800956726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167608320, "lr": 8.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680454735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1031138151884079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680469803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10438747704029083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167936000, "lr": 8.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680484923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10672374814748764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168099840, "lr": 8.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680500043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1025138795375824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168263680, "lr": 8.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680515396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09990967810153961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680530794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10955553501844406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168591360, "lr": 8.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680545930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10505404323339462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168755200, "lr": 8.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680561139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1067034900188446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168919040, "lr": 8.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680576158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1063026413321495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680591658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10431532561779022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169246720, "lr": 8.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680606668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082649901509285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169410560, "lr": 8.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680621565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10609164088964462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169574400, "lr": 8.624583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680637201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11336471140384674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680637239, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784680637240, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784680721083, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7757243514060974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784680721084, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784680721084, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784680735940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10813214629888535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169902080, "lr": 8.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680750968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11209828406572342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170065920, "lr": 8.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680766138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10728926956653595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170229760, "lr": 8.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680781469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10773249715566635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680796421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10058476030826569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170557440, "lr": 8.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680811587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10205656290054321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170721280, "lr": 8.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680826782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09628736972808838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170885120, "lr": 8.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680841637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080436035990715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680856396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09940848499536514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171212800, "lr": 8.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680871247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10275542736053467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171376640, "lr": 8.716249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680886464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10198643058538437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171540480, "lr": 8.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680901373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10316538065671921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680916208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09998275339603424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171868160, "lr": 8.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680931276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10382518917322159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172032000, "lr": 8.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680931323, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784680931323, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784681015712, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7757202982902527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784681015714, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784681015714, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784681030785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09797004610300064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172195840, "lr": 8.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681045943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10757656395435333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681060853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09886974096298218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172523520, "lr": 8.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681075801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10286851972341537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172687360, "lr": 8.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681090508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.094500832259655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172851200, "lr": 8.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681105448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10660538822412491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681120590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10397319495677948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173178880, "lr": 8.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681135566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1040806844830513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173342720, "lr": 8.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681150920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1051468625664711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173506560, "lr": 8.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681166001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11158427596092224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681181320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09640610963106155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173834240, "lr": 8.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681196747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11144086718559265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173998080, "lr": 8.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681211964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11225848644971848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174161920, "lr": 8.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681226967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1018434688448906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681227015, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784681227015, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784681313851, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7760013937950134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784681313852, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784681313852, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784681328612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09805140644311905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174489600, "lr": 8.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681343702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10714544355869293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174653440, "lr": 8.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681358867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10085044801235199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174817280, "lr": 8.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681373534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10652562230825424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681388656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10078267753124237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175144960, "lr": 8.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681403644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10904742777347565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175308800, "lr": 8.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681418719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1000368669629097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175472640, "lr": 8.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681433848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09902563691139221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681448952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10921299457550049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175800320, "lr": 8.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681464235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10948925465345383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175964160, "lr": 8.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681479503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10042422264814377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176128000, "lr": 8.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681494847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09451229870319366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681509894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09560400247573853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176455680, "lr": 8.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681524953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10597772151231766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176619520, "lr": 8.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681524954, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784681524954, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784681608550, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7762555480003357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784681608551, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784681608551, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784681623790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10305137187242508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176783360, "lr": 8.991249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681638979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11378656327724457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681654178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10162705183029175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177111040, "lr": 9.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681669489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10593324899673462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177274880, "lr": 9.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681684669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11052487045526505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177438720, "lr": 9.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681699773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10065224021673203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.032916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681715016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10977331548929214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177766400, "lr": 9.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681730107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10151757299900055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177930240, "lr": 9.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681745279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10764924436807632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178094080, "lr": 9.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681760398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10144416242837906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681775518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09777242690324783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178421760, "lr": 9.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681791023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11153887212276459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178585600, "lr": 9.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681806554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11908216774463654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178749440, "lr": 9.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681821869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10942085087299347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681821870, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784681821870, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784681904552, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7765254378318787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784681904552, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784681904553, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784681919916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11610374599695206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179077120, "lr": 9.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681935065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10081896185874939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179240960, "lr": 9.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681950681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09983973205089569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179404800, "lr": 9.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681966115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10080035030841827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.132916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681981360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10555675625801086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179732480, "lr": 9.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681996609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09896174818277359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179896320, "lr": 9.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682011832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10048213601112366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180060160, "lr": 9.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682027121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10615680366754532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682041972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10567493736743927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180387840, "lr": 9.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682056686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09437938034534454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180551680, "lr": 9.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682072372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10453102737665176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180715520, "lr": 9.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682087370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10177437961101532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682102476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09331221878528595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181043200, "lr": 9.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682117699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10050517320632935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181207040, "lr": 9.216249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682117700, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784682117700, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784682200153, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7765637040138245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784682200154, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784682200154, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784682215388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11073098331689835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181370880, "lr": 9.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682230630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11328841000795364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682245624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1006229817867279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181698560, "lr": 9.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682260765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10701052844524384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181862400, "lr": 9.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682275916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10350238531827927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182026240, "lr": 9.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682290941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10393205285072327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682306132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10643699020147324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182353920, "lr": 9.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682321190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10870818048715591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182517760, "lr": 9.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682336487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09823095053434372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182681600, "lr": 9.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682351534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09733237326145172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682366698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11196649819612503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183009280, "lr": 9.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682381539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09786434471607208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183173120, "lr": 9.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682396931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10187935084104538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183336960, "lr": 9.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682411874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10304316133260727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682411875, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784682411875, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784682496524, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7768917083740234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784682496524, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784682496525, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784682511681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10175113379955292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183664640, "lr": 9.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682526658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10848377645015717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183828480, "lr": 9.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682541716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11228736490011215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183992320, "lr": 9.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682557178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09397979080677032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682572660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10430128127336502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184320000, "lr": 9.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682588078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10764440149068832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184483840, "lr": 9.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682603529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11618561297655106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184647680, "lr": 9.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682618768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10104060173034668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682634224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10555383563041687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184975360, "lr": 9.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682649890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11004109680652618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185139200, "lr": 9.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682665532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09755495190620422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185303040, "lr": 9.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682680909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10849206894636154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682696345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09876315295696259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185630720, "lr": 9.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682711574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10654675215482712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185794560, "lr": 9.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682711575, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784682711576, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784682795647, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7771489024162292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784682795648, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784682795649, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784682811150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11029288172721863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185958400, "lr": 9.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682826104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10454930365085602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682840810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09227795153856277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186286080, "lr": 9.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682856382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1004406213760376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186449920, "lr": 9.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682871424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10887124389410019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186613760, "lr": 9.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682886542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1108269914984703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682902140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09618444740772247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186941440, "lr": 9.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682917590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10412478446960449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187105280, "lr": 9.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682932738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1058831512928009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187269120, "lr": 9.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682947881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09997168928384781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682962931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10440943390130997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187596800, "lr": 9.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682977938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10532116144895554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187760640, "lr": 9.549583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682992800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1104750782251358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187924480, "lr": 9.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683008017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11407165974378586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683008017, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784683008018, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784683089920, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7773520946502686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784683089921, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784683089921, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784683104745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10936039686203003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188252160, "lr": 9.574583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683119893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10452757775783539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188416000, "lr": 9.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683134770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10415691882371902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188579840, "lr": 9.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683149854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1088084951043129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683164744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10221856832504272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188907520, "lr": 9.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683180135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09768795967102051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189071360, "lr": 9.616249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683195335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10269258916378021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189235200, "lr": 9.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683210358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11251284182071686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683225468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10123918950557709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189562880, "lr": 9.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683240686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09529995918273926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189726720, "lr": 9.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683256153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1050400510430336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189890560, "lr": 9.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683271731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11248779296875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.666249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683287455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10162501037120819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190218240, "lr": 9.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683302936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11819541454315186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190382080, "lr": 9.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683302937, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784683302937, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784683384677, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7776286602020264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784683384677, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784683384678, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784683400362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10115775465965271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190545920, "lr": 9.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683415745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09717968851327896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683431327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10558876395225525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190873600, "lr": 9.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683447148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11213841289281845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191037440, "lr": 9.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683463000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11812715232372284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191201280, "lr": 9.724583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683478705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09662627428770065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683494536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10557949542999268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191528960, "lr": 9.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683509975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10139968991279602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191692800, "lr": 9.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683525963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10141575336456299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191856640, "lr": 9.757916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683541656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1042453944683075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683556761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10615433752536774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192184320, "lr": 9.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683572713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10991960018873215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192348160, "lr": 9.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683588081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09942445904016495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192512000, "lr": 9.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683603549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11498574167490005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.799583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683603550, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784683603550, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784683687466, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7775029540061951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784683687467, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784683687467, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784683702982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11254791915416718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192839680, "lr": 9.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683718626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1113620176911354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193003520, "lr": 9.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683734115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10688957571983337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193167360, "lr": 9.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683749700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12101852148771286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683765189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11606157571077347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193495040, "lr": 9.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683780726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10202674567699432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193658880, "lr": 9.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683796363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11864793300628662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193822720, "lr": 9.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683811671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11450738459825516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.866249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683827108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1056765615940094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194150400, "lr": 9.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683842562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10784120857715607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194314240, "lr": 9.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683858254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10796665400266647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194478080, "lr": 9.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683873691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10654494911432266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683889309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10739057511091232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194805760, "lr": 9.907916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683904552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11550680547952652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194969600, "lr": 9.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683904552, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784683904553, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784683989476, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7783324122428894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784683989476, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784683989476, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784684005008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11252786964178085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195133440, "lr": 9.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784684020262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10431145876646042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784684035655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10339304804801941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195461120, "lr": 9.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784684051243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11678243428468704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195624960, "lr": 9.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784684066853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10835223644971848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195788800, "lr": 9.957916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784684082180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1065356582403183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784684097835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10536456108093262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196116480, "lr": 9.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784684113358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11402176320552826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196280320, "lr": 9.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784684129273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.118105448782444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196444160, "lr": 9.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784684144854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10613447427749634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.999583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784684160354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11786451190710068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196771840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684175855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11286060512065887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196935680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684191093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10797473788261414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197099520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684206836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11533725261688232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684206837, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784684206837, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784684292913, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7784428596496582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784684292917, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784684292917, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784684308237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10092977434396744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197427200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684323955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11071209609508514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197591040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684339848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11596709489822388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197754880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684367122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1018984317779541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684383283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09974063187837601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198082560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684411423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10825282335281372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198246400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684427173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09818384796380997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198410240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684445650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10789960622787476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684461466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10110635310411453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198737920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684477211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10515380650758743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198901760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684492622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11442946642637253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199065600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684508140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11075793206691742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684523514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1172831580042839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199393280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684539333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11300824582576752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199557120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684539334, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784684539337, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784684624278, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7788059711456299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784684624285, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784684624286, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784684640010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11268700659275055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199720960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684656090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10788708180189133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684671577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11119088530540466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200048640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684691140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11675215512514114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200212480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684706895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11269690841436386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200376320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684722661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11816689372062683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684738488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09899529069662094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200704000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684754172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10830911248922348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200867840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684770027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10874839872121811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201031680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684785740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1089104562997818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684801566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11923922598361969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201359360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684817489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11282525956630707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201523200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684833026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10475925356149673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201687040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684848913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11158151924610138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684848914, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784684848914, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784684935090, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7790535092353821, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784684935091, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784684935091, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784684950571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10546159744262695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202014720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684966373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11102665960788727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202178560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684981738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11949467658996582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202342400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684997655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1064491793513298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685012756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10554107278585434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202670080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685028011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10192958265542984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202833920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685044109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10636454820632935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202997760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685059357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1100832149386406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685074650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11322292685508728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203325440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685089821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09375985711812973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203489280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685105082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11715856194496155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203653120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685120343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10084690898656845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685135173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09845791757106781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203980800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685150097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10868494212627411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204144640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685150098, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784685150098, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784685235946, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7789412140846252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784685235947, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784685235947, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784685251012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10250106453895569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204308480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685266319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09355605393648148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685281701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09633991122245789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204636160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685297168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10830835998058319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204800000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685312440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10888399183750153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204963840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685327700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11292658001184464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685342984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09563206136226654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205291520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685358303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10668633878231049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205455360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685373617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10588331520557404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205619200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685388764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11126109212636948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685403996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09855853021144867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205946880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685419605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09964843839406967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206110720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685435050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11074801534414291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206274560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685450263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09885291755199432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685450264, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784685450264, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784685537591, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7792759537696838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784685537592, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784685537592, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784685553108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10593144595623016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206602240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685568462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09601622819900513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206766080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685584078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10419458895921707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206929920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685599922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0971851795911789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685615696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10217641294002533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207257600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685631381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11237505078315735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207421440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685646883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1069425567984581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207585280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685661957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09574807435274124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685677126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11520587652921677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207912960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685692406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10442414879798889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208076800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685707071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1034182608127594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208240640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685722876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10167093575000763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685737904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10257232189178467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208568320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685753263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10173749178647995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208732160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685753264, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784685753264, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784685840151, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7793182730674744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784685840152, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784685840153, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784685855436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10109281539916992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208896000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685870763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09931676834821701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685886259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10254061967134476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209223680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685901336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1025238111615181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209387520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685916799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09837029874324799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209551360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685931992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10166099667549133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685947084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09697774797677994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209879040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685962251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09957364946603775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210042880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685977773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10356062650680542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210206720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685993303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10904625058174133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686008681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10432978719472885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210534400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686024106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10752524435520172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210698240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686039636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10176939517259598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210862080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686054889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0959811732172966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686054890, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784686054890, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784686140270, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7794694900512695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784686140271, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784686140271, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784686155823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09394524991512299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211189760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686171186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09922175854444504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211353600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686186574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09777156263589859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211517440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686201820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10010772198438644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686216979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10404306650161743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211845120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686232850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09538286179304123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212008960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686248319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10293464362621307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212172800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686263960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09848278015851974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686279234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09835398197174072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212500480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686294575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10439663380384445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212664320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686309969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09375689923763275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212828160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686325708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10786783695220947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686341442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10515213757753372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213155840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686357205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11028849333524704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213319680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686357205, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784686357206, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784686439664, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7793852090835571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784686439665, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784686439665, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784686454993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10513890534639359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213483520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686470453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10201021283864975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686485905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1052786186337471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213811200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686501830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10304904729127884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213975040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686516854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09672944992780685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214138880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686531754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10492707788944244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686547074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09564079344272614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214466560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686562237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11389676481485367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214630400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686577298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09790477901697159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214794240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686592768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09060725569725037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686608075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10038802027702332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215121920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686623363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10487190634012222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215285760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686638469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10388905555009842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215449600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686653768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10935771465301514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686653768, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784686653769, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784686736299, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7799687385559082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784686736300, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784686736300, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784686751442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1008588895201683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215777280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686766272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10497195273637772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215941120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686781314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09783347696065903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216104960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686796399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0999680832028389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686811705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10109575092792511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216432640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686826702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09447021037340164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216596480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686841885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10434375703334808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216760320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686857190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10312832146883011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686872196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10072551667690277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217088000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686887545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11011790484189987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217251840, "lr": 1e-06}} diff --git a/recommendation_v4/rcp_logs/gbs_8192/seed608892790.log b/recommendation_v4/rcp_logs/gbs_8192/seed608892790.log new file mode 100644 index 000000000..b61cc2aa3 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_8192/seed608892790.log @@ -0,0 +1,1862 @@ +:::MLLOG {"namespace": "", "time_ms": 1784651373285, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784651373285, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784651394802, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784651394808, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784651394808, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784651394808, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784651394808, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784651394808, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 8192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784651394808, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784651394808, "event_type": "POINT_IN_TIME", "key": "seed", "value": 608892790, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784651394808, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784651394808, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784651394808, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784651394808, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784651394818, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784651394818, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784652000394, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784652000395, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784652000849, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784652111742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14000114798545837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840, "lr": 7.916666666666666e-10}} +:::MLLOG {"namespace": "", "time_ms": 1784652123962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138068288564682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.6249999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652136899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13868677616119385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 491520, "lr": 2.458333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652149861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13840050995349884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652162999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13855326175689697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 819200, "lr": 4.125e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652176082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138470858335495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652189226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13829006254673004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1146880, "lr": 5.7916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652202465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13928113877773285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.625e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652215730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384502798318863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1474560, "lr": 7.458333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652228833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383925974369049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.291666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652242342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13861584663391113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1802240, "lr": 9.124999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652255698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13867270946502686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652269292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13832053542137146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2129920, "lr": 1.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652282602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13932497799396515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652282603, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652282604, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652367370, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5017393827438354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652367371, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652367371, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652380450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383504718542099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2457600, "lr": 1.2458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652393624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13823029398918152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652407471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383514404296875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2785280, "lr": 1.4124999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652420753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13814978301525116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4958333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652434319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376705765724182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3112960, "lr": 1.5791666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652447713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379237174987793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652460669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13778117299079895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3440640, "lr": 1.7458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652473929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13758443295955658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652488020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13785773515701294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3768320, "lr": 1.9124999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652501719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13799405097961426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9958333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652515426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13841718435287476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4096000, "lr": 2.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652528950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13839800655841827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.1624999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652542303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13780489563941956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4423680, "lr": 2.2458333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652556085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13814930617809296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3291666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652556086, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652556087, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652640917, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5019819140434265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652640918, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652640918, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652654821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13758760690689087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4751360, "lr": 2.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652668717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381624937057495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652682558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371433436870575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5079040, "lr": 2.5791666666666668e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652696250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13662269711494446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652709518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13778012990951538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5406720, "lr": 2.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652722802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13793230056762695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652736094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13734310865402222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5734400, "lr": 2.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652750003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13674136996269226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652763841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13654838502407074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6062080, "lr": 3.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652777488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13748766481876373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652791306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370123028755188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6389760, "lr": 3.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652804771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13735604286193848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652818734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375727653503418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6717440, "lr": 3.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652832325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13752198219299316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652832326, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784652832326, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784652916810, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5023037791252136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784652916812, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784652916812, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784652930056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13652504980564117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7045120, "lr": 3.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652943605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13730089366436005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652957044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365533024072647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7372800, "lr": 3.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652971104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1372009664773941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.829166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652984814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13725833594799042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7700480, "lr": 3.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652998881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13755150139331818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653012809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13640475273132324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8028160, "lr": 4.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653026686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373845338821411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653040793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364825814962387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8355840, "lr": 4.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653054860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13845239579677582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3291666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653068803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13611939549446106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8683520, "lr": 4.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653082623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375799924135208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.4958333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653096533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13493506610393524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9011200, "lr": 4.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653110532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367487758398056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.6624999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653110533, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653110533, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653193705, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5027639865875244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653193706, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653193706, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653207342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13603723049163818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9338880, "lr": 4.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653221085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13532456755638123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653234543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13762368261814117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9666560, "lr": 4.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653248251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13431593775749207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653262052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13675042986869812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9994240, "lr": 5.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653275737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312450051307678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1624999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653289557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13546667993068695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10321920, "lr": 5.2458333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653303369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13533969223499298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653317259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378367841243744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10649600, "lr": 5.4124999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653330913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351056843996048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653344797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.147623211145401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10977280, "lr": 5.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653358940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13549011945724487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653372703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13381673395633698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11304960, "lr": 5.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653386310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377456784248352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653386311, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653386311, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653468475, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5033531188964844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653468476, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653468477, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653482022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352279931306839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11632640, "lr": 5.9124999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653495501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13988736271858215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653509147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358812153339386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11960320, "lr": 6.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653522792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13314996659755707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653536585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13478609919548035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12288000, "lr": 6.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653550691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1391689032316208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.329166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653564465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332990974187851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12615680, "lr": 6.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653578254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13290834426879883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653592144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13320228457450867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12943360, "lr": 6.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653605797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315799206495285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653619811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13119730353355408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13271040, "lr": 6.745833333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653633401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13500797748565674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653647201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13191528618335724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13598720, "lr": 6.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653661252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13882099092006683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653661252, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784653661253, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784653742141, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5040301084518433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784653742142, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784653742143, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784653756066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13360385596752167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13926400, "lr": 7.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653769985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13373993337154388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653783775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13583062589168549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14254080, "lr": 7.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653797594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348603069782257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653811557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379128098487854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14581760, "lr": 7.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653825398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12779951095581055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653839059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13542883098125458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14909440, "lr": 7.579166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653853010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294671595096588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653867156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13360294699668884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15237120, "lr": 7.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653881491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13247844576835632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.829166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653895455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13375897705554962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15564800, "lr": 7.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653909396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13422346115112305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.995833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653924002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13539814949035645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15892480, "lr": 8.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653938140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13327687978744507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653938141, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784653938141, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654019396, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5049763917922974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654019397, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654019397, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654033692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13236111402511597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16220160, "lr": 8.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654047826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13390623033046722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654062142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13552922010421753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16547840, "lr": 8.412500000000001e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654076534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13543374836444855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654090828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13967913389205933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16875520, "lr": 8.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654104946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13307277858257294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.662499999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654119422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13514891266822815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17203200, "lr": 8.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654133527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13345253467559814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654147537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13328127562999725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17530880, "lr": 8.912499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654161832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13188818097114563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654175746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356024444103241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17858560, "lr": 9.079166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654190086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13364651799201965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.162499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654204071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333966702222824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18186240, "lr": 9.245833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654217921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1305338442325592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654217921, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654217922, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654300167, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5061982274055481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654300168, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654300168, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654314499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13783219456672668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18513920, "lr": 9.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654328659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349833905696869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.495833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654343196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353524625301361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18841600, "lr": 9.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654357961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1313028186559677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.662499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654372228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336992233991623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19169280, "lr": 9.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654386714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1340872347354889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654401394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13704881072044373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19496960, "lr": 9.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654415912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13411536812782288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654429996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331150233745575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19824640, "lr": 1.0079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654444243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12971250712871552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0162499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654457939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12786568701267242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20152320, "lr": 1.0245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654471804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13308677077293396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654485799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295679807662964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20480000, "lr": 1.0412499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654499405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1248839721083641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654499406, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654499406, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654580447, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5074137449264526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654580447, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654580448, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654594465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13124054670333862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20807680, "lr": 1.0579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654608181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13605035841464996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654622260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334376037120819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21135360, "lr": 1.0745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654636518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13026803731918335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654650330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1282372921705246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21463040, "lr": 1.0912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654664131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12679721415042877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654678477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12889611721038818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21790720, "lr": 1.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654692430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13239187002182007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654706572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13554134964942932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22118400, "lr": 1.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654720761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12507778406143188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654734510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1286332756280899, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22446080, "lr": 1.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654748327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13448426127433777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654762316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13098640739917755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22773760, "lr": 1.1579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654776762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12920625507831573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654776762, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784654776763, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784654859244, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5089985132217407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784654859245, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784654859245, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784654873569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13031615316867828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23101440, "lr": 1.1745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654887516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289985179901123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654900881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13307543098926544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23429120, "lr": 1.1912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654914847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12396544218063354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654929087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12942910194396973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23756800, "lr": 1.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654942639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12600994110107422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654956215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13001684844493866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24084480, "lr": 1.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654970021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12187302112579346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654983497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13587427139282227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24412160, "lr": 1.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654997456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13717910647392273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655011195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12370573729276657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24739840, "lr": 1.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655025355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12320242077112198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655039507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12573668360710144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25067520, "lr": 1.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655053574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288933902978897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655053575, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655053575, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655138221, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5107606649398804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655138222, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655138222, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655151830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13679595291614532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25395200, "lr": 1.2912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655165917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13655762374401093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655179734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357121616601944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25722880, "lr": 1.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655193518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1398375928401947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655207449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13654133677482605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26050560, "lr": 1.3245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655221514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13831061124801636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655235518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12717662751674652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26378240, "lr": 1.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655249477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1261729896068573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655263344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14218677580356598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26705920, "lr": 1.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655277663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14094887673854828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655292116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12719126045703888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27033600, "lr": 1.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655306306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12866869568824768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655320486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13252940773963928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27361280, "lr": 1.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655334111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315578818321228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655334111, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655334112, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655418286, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.512753427028656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655418287, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655418287, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655432870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14116446673870087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27688960, "lr": 1.4079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655447431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14241275191307068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655461477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13542652130126953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28016640, "lr": 1.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655475657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13945145905017853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655490357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14486132562160492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28344320, "lr": 1.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655504519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1410072296857834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655519017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12604902684688568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28672000, "lr": 1.4579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655533257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13775141537189484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655547703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13175959885120392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28999680, "lr": 1.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655562292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13623753190040588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655576748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13772843778133392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29327360, "lr": 1.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655590796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13293102383613586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4995833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655605266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13426880538463593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29655040, "lr": 1.5079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655619691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13643088936805725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655619692, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784655619692, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784655703501, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5148807764053345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784655703502, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784655703502, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784655717620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13859637081623077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29982720, "lr": 1.5245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655732006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13830192387104034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655746390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13176797330379486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30310400, "lr": 1.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655760682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330135017633438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655774890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13372744619846344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30638080, "lr": 1.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655789403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13965904712677002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655803759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13306811451911926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30965760, "lr": 1.5745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655818413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14125308394432068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655832325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1406189501285553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31293440, "lr": 1.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655846715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350088268518448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655860979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12764815986156464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31621120, "lr": 1.6079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655874919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1299593597650528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655888888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341792494058609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31948800, "lr": 1.6245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655903005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1276671439409256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655903005, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784655903006, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784655985341, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5173585414886475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784655985342, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784655985342, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784655999310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13221052289009094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32276480, "lr": 1.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656013280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368718147277832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656027332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336170881986618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32604160, "lr": 1.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656041367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13202744722366333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656055578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1339954137802124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32931840, "lr": 1.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656069592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13330070674419403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656084077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13376159965991974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33259520, "lr": 1.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656098323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13280895352363586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656112312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13978351652622223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33587200, "lr": 1.7079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656126789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13175027072429657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656141097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12880903482437134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33914880, "lr": 1.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656154958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312386840581894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656168819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13234595954418182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34242560, "lr": 1.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656182847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359529048204422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7495833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656182848, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656182848, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656266187, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5205321907997131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656266188, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656266188, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656280224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13357733190059662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34570240, "lr": 1.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656294119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12297489494085312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656308105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13067346811294556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34897920, "lr": 1.7745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656322239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303381621837616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656336200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13522467017173767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35225600, "lr": 1.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656350730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13519759476184845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656364938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13552874326705933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35553280, "lr": 1.8079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656379456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12401732802391052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8162500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656393720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13104309141635895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35880960, "lr": 1.8245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656407515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13114216923713684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656421759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304718255996704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36208640, "lr": 1.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656435878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13337352871894836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656450133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13088461756706238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36536320, "lr": 1.8579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656464122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13020633161067963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656464123, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656464123, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656548678, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5245648622512817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656548679, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656548680, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656562758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378382295370102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36864000, "lr": 1.8745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656576954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322588324546814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656591107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13343445956707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37191680, "lr": 1.8912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656605256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304611712694168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656619472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12712818384170532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37519360, "lr": 1.9079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656633484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13212689757347107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656647992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13570064306259155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37847040, "lr": 1.9245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656662271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13297870755195618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656676534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13140693306922913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38174720, "lr": 1.9412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656691004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13900849223136902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656705342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13387465476989746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38502400, "lr": 1.9579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656719054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13732844591140747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656733317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131922647356987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38830080, "lr": 1.9745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656747334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334579586982727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656747335, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784656747335, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784656832452, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5298720598220825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784656832453, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784656832453, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784656846482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13122856616973877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39157760, "lr": 1.9912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656860395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13503165543079376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656874506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13791635632514954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39485440, "lr": 2.0079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656888566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13332295417785645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656902573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12763948738574982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39813120, "lr": 2.0245833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656916615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294354498386383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0329166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656930632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12589934468269348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40140800, "lr": 2.0412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656944499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12393073737621307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656958640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13597019016742706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40468480, "lr": 2.0579166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656972890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13741081953048706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656987080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345970630645752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40796160, "lr": 2.0745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657001716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12715543806552887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0829166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657015795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13064968585968018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41123840, "lr": 2.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657029421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12378666549921036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657029422, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657029423, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657115523, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5369929075241089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657115525, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657115525, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657129618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324467808008194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41451520, "lr": 2.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657143820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13262085616588593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657158123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13367480039596558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41779200, "lr": 2.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657172348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13786937296390533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657186544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363953948020935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42106880, "lr": 2.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657200773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13160909712314606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657214818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12723268568515778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42434560, "lr": 2.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657228986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13511967658996582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657243014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12476570904254913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42762240, "lr": 2.1745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657257394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303790807723999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657271629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324956715106964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43089920, "lr": 2.1912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657286091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1342829465866089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657300578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1405745893716812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43417600, "lr": 2.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657314912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361006498336792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657314913, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657314913, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657399756, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5470276474952698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657399757, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657399757, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657414730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13672739267349243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43745280, "lr": 2.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657429202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315537691116333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657443392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13254567980766296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44072960, "lr": 2.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657457673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13917972147464752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657472397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13004785776138306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44400640, "lr": 2.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657487279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13835090398788452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657501651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13515137135982513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44728320, "lr": 2.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657516306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13751667737960815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657530810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13524393737316132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45056000, "lr": 2.2912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657545560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13074341416358948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657560049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14077730476856232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45383680, "lr": 2.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657574453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13465183973312378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657588886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1291329711675644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45711360, "lr": 2.3245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657603436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133405864238739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657603437, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657603437, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657688426, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5590032339096069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657688427, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657688427, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657703242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1422649472951889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46039040, "lr": 2.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657717787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13290730118751526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657732967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13341543078422546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46366720, "lr": 2.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657747442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308809220790863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657762440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13529038429260254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46694400, "lr": 2.3745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657777117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13394442200660706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657791934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13417409360408783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47022080, "lr": 2.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657806905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13588431477546692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657821581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1314810812473297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47349760, "lr": 2.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657836403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13324804604053497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657851221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330728530883789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47677440, "lr": 2.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657866121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13731858134269714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657880674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13622914254665375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48005120, "lr": 2.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657895329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377558559179306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.4495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657895329, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784657895330, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784657977404, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5730664134025574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784657977405, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784657977405, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784657991865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1319100558757782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48332800, "lr": 2.4579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658006644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13561823964118958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658021139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13454578816890717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48660480, "lr": 2.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658036140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13233207166194916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658050550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13748767971992493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48988160, "lr": 2.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658064930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12982645630836487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658079660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1340051144361496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49315840, "lr": 2.507916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658093794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12454209476709366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.5162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658108408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1313331127166748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49643520, "lr": 2.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658122845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317623108625412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658137345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13169869780540466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49971200, "lr": 2.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658151675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13208048045635223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.5495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658166076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12947766482830048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50298880, "lr": 2.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658180475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13138508796691895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.5662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658180476, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658180476, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658261446, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5855993628501892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658261447, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658261447, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658275522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12291616201400757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50626560, "lr": 2.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658290019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12751616537570953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658304369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13032039999961853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50954240, "lr": 2.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658318959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338053196668625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658333717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13275852799415588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51281920, "lr": 2.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658348344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12246491760015488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658363359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315556764602661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51609600, "lr": 2.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658378473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13277390599250793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658393260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13769230246543884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51937280, "lr": 2.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658407514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12886163592338562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.6495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658422246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13417582213878632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52264960, "lr": 2.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658436461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11876536160707474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.6662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658450664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12280888110399246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52592640, "lr": 2.6745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658464944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12649144232273102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658464945, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658464945, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658547256, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.598785936832428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658547257, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658547257, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658561390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1226564347743988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52920320, "lr": 2.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658575319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12915216386318207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.6995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658589466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297753006219864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53248000, "lr": 2.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658603493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13321612775325775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658617658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12301484495401382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53575680, "lr": 2.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658631783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135373055934906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658645822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13098646700382233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53903360, "lr": 2.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658659758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12312046438455582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.7495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658674035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12847277522087097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54231040, "lr": 2.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658688458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1287807822227478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658702591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11998475342988968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54558720, "lr": 2.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658717284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325565129518509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658731857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1284189075231552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54886400, "lr": 2.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658746255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12927192449569702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658746255, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784658746256, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784658829799, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6140835881233215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784658829800, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784658829801, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784658843824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13063745200634003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55214080, "lr": 2.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658858447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12066157162189484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658872939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1234724149107933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55541760, "lr": 2.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658887742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1232592985033989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658902306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12481098622083664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55869440, "lr": 2.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658916799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12786990404129028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658931250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11858420073986053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56197120, "lr": 2.8579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658945898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12400226294994354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658960426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12753300368785858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56524800, "lr": 2.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658974794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1261557638645172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658988890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11773308366537094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56852480, "lr": 2.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659003108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12707474827766418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659017218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11272622644901276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57180160, "lr": 2.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659031452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1264643669128418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659031453, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659031453, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659117632, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6280779838562012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659117633, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659117633, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659132383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11922008544206619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57507840, "lr": 2.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659147296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1173192486166954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659161910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13155175745487213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57835520, "lr": 2.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659176722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12397246807813644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659191215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12636758387088776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58163200, "lr": 2.9579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659205545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11910468339920044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659219713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1226826161146164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58490880, "lr": 2.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659234154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12379074096679688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659248481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12315414100885391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58818560, "lr": 2.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659263037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11528922617435455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659277294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1247343197464943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59146240, "lr": 3.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659291491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12295893579721451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.0162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659305833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1270066648721695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59473920, "lr": 3.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659320420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11883050948381424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659320420, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659320421, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659404458, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6490457057952881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659404458, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659404459, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659418828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12311500310897827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59801600, "lr": 3.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659433096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11420612037181854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659447235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11316808313131332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60129280, "lr": 3.0579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659461478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11514115333557129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.0662499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659475714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11160355806350708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60456960, "lr": 3.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659490050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12349960952997208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659504703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11842518299818039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60784640, "lr": 3.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659519588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12179208546876907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659534059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10656820237636566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61112320, "lr": 3.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659548927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12026790529489517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.1162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659563588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11453462392091751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61440000, "lr": 3.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659578250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11494127660989761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659592576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11213478446006775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61767680, "lr": 3.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659606907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11712580919265747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659606908, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659606908, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659690991, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6802730560302734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659690992, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659690992, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659705328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12144225835800171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62095360, "lr": 3.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659719856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12043830752372742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.1662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659734538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12450262904167175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62423040, "lr": 3.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659749073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10866102576255798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659763659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11520036309957504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62750720, "lr": 3.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659778039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11891253292560577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659792759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11995908617973328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63078400, "lr": 3.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659807389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11976273357868195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659821696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10997448861598969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63406080, "lr": 3.2245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659835971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11823613941669464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659850328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11587479710578918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63733760, "lr": 3.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659864458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11708100140094757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659878720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1141228899359703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64061440, "lr": 3.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659893327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12451628595590591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659893327, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784659893328, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784659977550, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7147897481918335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784659977551, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784659977552, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784659992323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12431229650974274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64389120, "lr": 3.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660007356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11736683547496796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660021889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11586586385965347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64716800, "lr": 3.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660036974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12172042578458786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660051910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10897259414196014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65044480, "lr": 3.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660066767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11187243461608887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660080858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11392423510551453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65372160, "lr": 3.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660095484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10828923434019089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660110190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11181662231683731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65699840, "lr": 3.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660125369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11627846956253052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660140162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1226077452301979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66027520, "lr": 3.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660154858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11495795100927353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660169761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12527374923229218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66355200, "lr": 3.3745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660184684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11274682730436325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660184685, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660184685, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660268285, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7546164989471436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660268286, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660268286, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660283176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12470842897891998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66682880, "lr": 3.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660297811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1131579652428627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.3995833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660312508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11499679833650589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67010560, "lr": 3.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660327582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1131376251578331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660342247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11769627034664154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67338240, "lr": 3.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660357048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11655879020690918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660372204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12022048234939575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67665920, "lr": 3.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660387319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11966336518526077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660402304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11390737444162369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67993600, "lr": 3.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660417654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11171571910381317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660432528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11423327773809433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68321280, "lr": 3.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660447264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11268696933984756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660462019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1232934296131134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68648960, "lr": 3.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660477031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11401788145303726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660477032, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660477032, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660561328, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7611082196235657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660561329, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660561329, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660576132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11984685808420181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68976640, "lr": 3.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660591011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11766360700130463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660605736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11281639337539673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69304320, "lr": 3.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660620684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11249017715454102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660635329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10715711116790771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69632000, "lr": 3.5412500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660650144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1134970560669899, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660664953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11642353981733322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69959680, "lr": 3.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660679434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10869534313678741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660694154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1089162677526474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70287360, "lr": 3.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660708692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1076248362660408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660723752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12176716327667236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70615040, "lr": 3.5912500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660738930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10492084175348282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660753812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11888004094362259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70942720, "lr": 3.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660769157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11917542666196823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660769158, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784660769158, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784660851596, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7644199728965759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784660851597, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784660851598, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784660866699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11719263345003128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71270400, "lr": 3.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660881338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11620736867189407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660895906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11513251066207886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71598080, "lr": 3.6412499999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660910867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11347945034503937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660925709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10158698260784149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71925760, "lr": 3.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660940411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.100437231361866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660954960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11755093932151794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72253440, "lr": 3.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660969400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11475787311792374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660984020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10288571566343307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72581120, "lr": 3.6912499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660998540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10083982348442078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661012676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10530228912830353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72908800, "lr": 3.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661027190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11142877489328384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661041725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10880748182535172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73236480, "lr": 3.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661056117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11360067129135132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661056155, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661056156, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661138375, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7666370272636414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661138376, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661138376, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661152927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10410186648368835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73564160, "lr": 3.7412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661167770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10686875134706497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661182716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11279323697090149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73891840, "lr": 3.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661197361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10895219445228577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661212379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10663411021232605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74219520, "lr": 3.7745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661227509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11324037611484528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661242342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1127328872680664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74547200, "lr": 3.7912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661256947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11012986302375793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661271521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.114463210105896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74874880, "lr": 3.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661286033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11047772318124771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661300395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09852208197116852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75202560, "lr": 3.8245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661314864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1128077581524849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661329109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10517806559801102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75530240, "lr": 3.8412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661343352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10429532080888748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661343352, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661343353, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661426111, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7683467268943787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661426112, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661426112, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661440494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10539735108613968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75857920, "lr": 3.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661454983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11198108643293381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661469472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09483291208744049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76185600, "lr": 3.8745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661483664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039741188287735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661498024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11181169748306274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76513280, "lr": 3.8912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661512161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10418003052473068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661526555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08836931735277176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76840960, "lr": 3.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661540766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10901370644569397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661555098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10006865113973618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77168640, "lr": 3.9245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661569587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11779609322547913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661584215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10711562633514404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77496320, "lr": 3.9412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661598564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10031800717115402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661613469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09854386746883392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77824000, "lr": 3.9579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661628175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10671648383140564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661628175, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784661628176, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784661710446, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7696629762649536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784661710446, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784661710447, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784661725045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11437379568815231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78151680, "lr": 3.9745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661739663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11137482523918152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661753675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10427194088697433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78479360, "lr": 3.9912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661768386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1048821285367012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661782703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0990864709019661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78807040, "lr": 4.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661797178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10014898329973221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661811387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10633654147386551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79134720, "lr": 4.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661825798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10660018771886826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661839931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09713813662528992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79462400, "lr": 4.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661854186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1086580827832222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661868512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10415586829185486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79790080, "lr": 4.0579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661882856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10629871487617493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661897233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.108119435608387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80117760, "lr": 4.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661911657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11258253455162048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661911658, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784661911658, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784661993606, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.77080237865448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784661993607, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784661993607, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784662008062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10967239737510681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80445440, "lr": 4.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662022257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09794043749570847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662036786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10246726870536804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80773120, "lr": 4.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662051451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10451114177703857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662066301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11069680750370026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81100800, "lr": 4.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662081016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11151731759309769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662095620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10104315727949142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81428480, "lr": 4.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662110277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10453389585018158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662125178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10760864615440369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81756160, "lr": 4.1579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662139499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10483434051275253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662154120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11210756748914719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82083840, "lr": 4.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662168100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09393900632858276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662182961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0973895937204361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82411520, "lr": 4.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662197620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10728548467159271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662197620, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662197621, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662280816, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7716514468193054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662280817, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662280817, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662295310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0916917473077774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82739200, "lr": 4.2079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662310111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10426215827465057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662324784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1098543033003807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83066880, "lr": 4.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662339671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09707118570804596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662354333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11027435213327408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83394560, "lr": 4.2412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662369040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10986372828483582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662383623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0970299169421196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83722240, "lr": 4.2579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662397952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11105860024690628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662412498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10462680459022522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84049920, "lr": 4.2745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662427085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1035844013094902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662441768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10626643896102905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84377600, "lr": 4.2912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662456066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10474835336208344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662470489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09693911671638489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84705280, "lr": 4.3079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662485152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10593495517969131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662485153, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662485153, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662568462, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7722427248954773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662568463, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662568463, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662583499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10348645597696304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85032960, "lr": 4.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662598485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1034751683473587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662612981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10499922931194305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85360640, "lr": 4.3412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662627788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10302161425352097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662642541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09805673360824585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85688320, "lr": 4.3579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662657196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09960448741912842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662672059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10473848879337311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86016000, "lr": 4.3745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662686332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09613465517759323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662700991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10754036903381348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86343680, "lr": 4.3912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662715758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10789915919303894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662730463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11687077581882477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86671360, "lr": 4.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662745419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11427178978919983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662760040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11648429930210114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86999040, "lr": 4.4245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662774563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11488109081983566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662774564, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784662774564, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784662856513, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7728497982025146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784662856514, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784662856514, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784662870999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09983842819929123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87326720, "lr": 4.4412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662885536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11583968997001648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662899929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10650186985731125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87654400, "lr": 4.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662914602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10148099064826965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662928960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09164933860301971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87982080, "lr": 4.4745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662943511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0970887839794159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662957675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10351342707872391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88309760, "lr": 4.4912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662972185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10714955627918243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662986375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10706751048564911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88637440, "lr": 4.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663000712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10037300735712051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663015579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11207718402147293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88965120, "lr": 4.5245833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663030380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10013123601675034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663045319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10946077853441238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89292800, "lr": 4.5412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663059900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1115405336022377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.5495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663059901, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663059901, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663143372, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7732407450675964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663143373, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663143373, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663158360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793280601501465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89620480, "lr": 4.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663173182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10192117840051651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663187860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1100916713476181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89948160, "lr": 4.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663202747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09164629876613617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663217020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11435072869062424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90275840, "lr": 4.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663231292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10645205527544022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663246323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10997629910707474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90603520, "lr": 4.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663260960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11080169677734375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663276031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12168574333190918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90931200, "lr": 4.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663290870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10521773993968964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663305763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1092955619096756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91258880, "lr": 4.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663320879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12242545187473297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663335733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09913983196020126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91586560, "lr": 4.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663350546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1105160042643547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663350547, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663350547, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663433388, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773664653301239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663433389, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663433389, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663448201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10426076501607895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91914240, "lr": 4.6745833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663463256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11798612028360367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663478149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10909633338451385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92241920, "lr": 4.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663493517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10283568501472473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663508416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09597435593605042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92569600, "lr": 4.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663523511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11316286027431488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663538861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10618574917316437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92897280, "lr": 4.7245833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663554106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10544896870851517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663569131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11450653523206711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93224960, "lr": 4.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663584056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09977230429649353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663599540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12502546608448029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93552640, "lr": 4.7579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663614983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10664557665586472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663630143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1139000803232193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93880320, "lr": 4.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663645069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10722939670085907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663645070, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784663645070, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784663730853, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738804221153259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784663730854, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784663730854, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784663745981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11365382373332977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94208000, "lr": 4.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663760781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11472412198781967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663775618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11139731854200363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94535680, "lr": 4.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663790672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11718259751796722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663805607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11385504901409149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94863360, "lr": 4.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663820474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11843079328536987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663835666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1266328990459442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95191040, "lr": 4.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663850468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12105007469654083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663865267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11600922048091888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95518720, "lr": 4.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663880328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11001784354448318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663895201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10790004581212997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95846400, "lr": 4.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663910131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10779757052659988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663924986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11442648619413376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96174080, "lr": 4.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663940059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1143011599779129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663940060, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784663940060, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664024497, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7741292119026184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664024498, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664024498, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664039705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1187518909573555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96501760, "lr": 4.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664054895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.107017382979393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664069849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10775497555732727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96829440, "lr": 4.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664085257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10848681628704071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664100386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11019229143857956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97157120, "lr": 4.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664115346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10948789864778519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664130326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11374341696500778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97484800, "lr": 4.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664145268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10842649638652802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664159969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10472045838832855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97812480, "lr": 4.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664174439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10012504458427429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664189368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10360241681337357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98140160, "lr": 4.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664204180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09592613577842712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664218767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10351555794477463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98467840, "lr": 5.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664233465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10088913142681122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664233465, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664233466, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664316750, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7744770646095276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664316751, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664316751, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664331325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10943292826414108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98795520, "lr": 5.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664345942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1006273627281189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664360411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09636594355106354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99123200, "lr": 5.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664374951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10834792256355286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664389486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11490774899721146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99450880, "lr": 5.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664404016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10390347987413406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664418539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10356065630912781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99778560, "lr": 5.074583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664433126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10428611189126968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664447775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10172651708126068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100106240, "lr": 5.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664462476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12005612254142761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664477235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11677751690149307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100433920, "lr": 5.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664491904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10150692611932755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664506947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09937343001365662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100761600, "lr": 5.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664521537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12041814625263214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664521538, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664521538, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664602804, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7747105956077576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664602805, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664602805, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664617994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11072000116109848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101089280, "lr": 5.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664632821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09583042562007904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664647699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10413941740989685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101416960, "lr": 5.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664662612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10287206619977951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664677283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10616306215524673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101744640, "lr": 5.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664691909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10448899865150452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.182916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664706803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10550379008054733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102072320, "lr": 5.191249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664721472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1119643896818161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664736469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10433974117040634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102400000, "lr": 5.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664751407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10765153914690018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664766160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10533269494771957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102727680, "lr": 5.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664780904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11407572031021118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664795305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10718140751123428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103055360, "lr": 5.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664809974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10201176255941391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664809975, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784664809975, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784664892244, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7747551798820496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784664892246, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784664892246, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784664907019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09901479631662369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103383040, "lr": 5.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664921983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09388746321201324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664936513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0961228683590889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103710720, "lr": 5.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664951393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09678340703248978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.282916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664966103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0962446853518486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104038400, "lr": 5.291249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664981129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09507126361131668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664995902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.100123330950737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104366080, "lr": 5.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665010682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09655866771936417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665025758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10274805128574371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104693760, "lr": 5.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665040914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10768607258796692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665056132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10498049855232239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105021440, "lr": 5.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665071059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10236633569002151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665085848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1141241118311882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105349120, "lr": 5.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665101329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11079597473144531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665101331, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665101331, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665184118, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7749427556991577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665184119, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665184119, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665199244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09765296429395676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105676800, "lr": 5.374583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665214365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10675755888223648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.382916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665229676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10971615463495255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106004480, "lr": 5.391249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665244403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10139184445142746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665259114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09935157001018524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106332160, "lr": 5.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665273910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10789445042610168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665288606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11014095693826675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106659840, "lr": 5.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665303431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10587553679943085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665318451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10263301432132721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106987520, "lr": 5.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665333070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1013115793466568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665347732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09975367039442062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107315200, "lr": 5.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665362304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10237877070903778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665376947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10848125070333481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107642880, "lr": 5.474583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665391341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1150880977511406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.482916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665391342, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665391342, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665474986, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752314209938049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665474987, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665474987, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665489571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10920076072216034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107970560, "lr": 5.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665504182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10685458779335022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665518768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09048513323068619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108298240, "lr": 5.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665533374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10106226056814194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665548244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10607195645570755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108625920, "lr": 5.524583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665562888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10778345912694931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665577483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10974394530057907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108953600, "lr": 5.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665592358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10150885581970215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665607368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10064822435379028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109281280, "lr": 5.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665622303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10389170050621033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665637372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10229598730802536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109608960, "lr": 5.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665652312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10688087344169617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665667512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10421838611364365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109936640, "lr": 5.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665682753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10881313681602478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665682755, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784665682755, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784665768363, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752134203910828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784665768364, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784665768364, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784665783457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1032046377658844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110264320, "lr": 5.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665798419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1025993674993515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665813013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039775162935257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110592000, "lr": 5.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665827438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10233592242002487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665842667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10668203979730606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110919680, "lr": 5.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665857258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10097745805978775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665872051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09803308546543121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111247360, "lr": 5.657916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665887126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10227040946483612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.666250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665902168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11160631477832794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111575040, "lr": 5.674583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665917040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10453545302152634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665931895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1040809378027916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111902720, "lr": 5.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665946652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10191112011671066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665961624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10299070179462433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112230400, "lr": 5.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665976735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10611464083194733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665976735, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784665976736, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666061552, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7755323648452759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666061553, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666061553, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666076545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10810758173465729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112558080, "lr": 5.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666091791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10162180662155151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666107010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10363411903381348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112885760, "lr": 5.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666122407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10084838420152664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666137916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.104619599878788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113213440, "lr": 5.757916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666152992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11244195699691772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.766250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666168314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10121044516563416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113541120, "lr": 5.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666183577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11044707894325256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666199086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11296727508306503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113868800, "lr": 5.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666214857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10596495121717453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666230319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10288488864898682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114196480, "lr": 5.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666245710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10581676661968231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666261474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10512429475784302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114524160, "lr": 5.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666277063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0981675386428833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666277063, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666277064, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666360160, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7756257057189941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666360161, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666360161, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666375493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10104544460773468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114851840, "lr": 5.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666390761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09926484525203705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666405895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10589398443698883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115179520, "lr": 5.857916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666420632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10034605860710144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666435913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10857467353343964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115507200, "lr": 5.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666450624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10874970257282257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666465475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10728070139884949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115834880, "lr": 5.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666480481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11516472697257996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666495276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11383868008852005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116162560, "lr": 5.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666509889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1001971885561943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666524429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10350283980369568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116490240, "lr": 5.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666539047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10208465158939362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666553548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10533906519412994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116817920, "lr": 5.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666567976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10652177780866623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666567977, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666567978, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666650229, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7758560180664062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666650230, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666650230, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666664887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10170295834541321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117145600, "lr": 5.957916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666679568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10046841949224472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666694561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1089654490351677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117473280, "lr": 5.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666709166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10421250015497208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666723606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09401123970746994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117800960, "lr": 5.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666738710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1119346022605896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666753214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10144788026809692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118128640, "lr": 6.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666767664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10751379281282425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666782534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1026025116443634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118456320, "lr": 6.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666797445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10334397852420807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666811891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10920944064855576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118784000, "lr": 6.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666826775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11428947746753693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666841560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.105608731508255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119111680, "lr": 6.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666856313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10261566191911697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666856313, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784666856314, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784666938543, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7757349610328674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784666938544, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784666938544, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784666953591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11287491768598557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119439360, "lr": 6.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666968338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10909225791692734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666983060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10969007760286331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119767040, "lr": 6.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666997974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10697819292545319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667012436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10144148021936417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120094720, "lr": 6.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667026820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10741734504699707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667042340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11800763756036758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120422400, "lr": 6.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667057700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10885107517242432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667073038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11599753051996231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120750080, "lr": 6.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667088630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1207118108868599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667103757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10704424232244492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121077760, "lr": 6.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667118857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1098903939127922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667134044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11495348066091537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121405440, "lr": 6.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667149233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10974089056253433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667149234, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667149234, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667231636, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7758291959762573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667231637, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667231637, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667246893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.113309845328331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121733120, "lr": 6.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667262147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10906170308589935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667277378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10998833924531937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122060800, "lr": 6.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667292651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10643799602985382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667307771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10431558638811111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122388480, "lr": 6.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667323003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11307273805141449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667337959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1068151518702507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122716160, "lr": 6.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667353100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11466419696807861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667368495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11245732009410858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123043840, "lr": 6.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667383979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1198427751660347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667399489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10596095025539398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123371520, "lr": 6.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667414707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11729647219181061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667430070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10708973556756973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123699200, "lr": 6.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667445686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11828083544969559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667445687, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667445687, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667531513, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7759235501289368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667531515, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667531515, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667546950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11929236352443695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124026880, "lr": 6.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667562324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1078702062368393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667577594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11929339170455933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124354560, "lr": 6.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667592736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11395562440156937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667608033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10735230892896652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124682240, "lr": 6.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667623648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11554884910583496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667638898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1110508069396019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125009920, "lr": 6.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667654478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11633240431547165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667669948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11137910932302475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125337600, "lr": 6.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667685132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10583265125751495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667700495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11245761066675186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125665280, "lr": 6.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667715912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10652391612529755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667731070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11692974716424942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125992960, "lr": 6.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667746660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10468790680170059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667746661, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784667746661, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784667830805, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7762800455093384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784667830806, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784667830806, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784667846153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10949083417654037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126320640, "lr": 6.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667861769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11827143281698227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667877204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11038926988840103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126648320, "lr": 6.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667892492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1134248748421669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667907651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11553449928760529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126976000, "lr": 6.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667922998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10886721312999725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667938502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10815735161304474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127303680, "lr": 6.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667954024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10931219160556793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.482916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667969628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1120849996805191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127631360, "lr": 6.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667984983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11193620413541794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668000607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10979701578617096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127959040, "lr": 6.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668016478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11043700575828552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668032177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11163162440061569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128286720, "lr": 6.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668047893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10894259810447693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668047893, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668047894, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668129914, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7766838669776917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668129916, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668129916, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668145100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11776703596115112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128614400, "lr": 6.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668160348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1099121943116188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668175550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11062590032815933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128942080, "lr": 6.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668190638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10270662605762482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668205837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10569436848163605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129269760, "lr": 6.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668221216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11121386289596558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668236134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1079118400812149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129597440, "lr": 6.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668250908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10687859356403351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668265812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09784430265426636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129925120, "lr": 6.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668280639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1015927791595459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668295614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10549277812242508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130252800, "lr": 6.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668310485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10442269593477249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668325517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10951066762208939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130580480, "lr": 6.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668340606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11544827371835709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668340606, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668340607, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668422550, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7768078446388245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668422551, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668422551, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668437458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11476413160562515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130908160, "lr": 6.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668452644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10489574074745178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668467686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10061895102262497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131235840, "lr": 6.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668482935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09234614670276642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668498069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11408361047506332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131563520, "lr": 6.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668512947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10241404175758362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668528068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10220953077077866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131891200, "lr": 6.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668543481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11223580688238144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668558632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1069001704454422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132218880, "lr": 6.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668573852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1031716912984848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668589111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10837776213884354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132546560, "lr": 6.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668604565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10936125367879868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668620023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725760459899902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132874240, "lr": 6.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668635164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10670141875743866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668635165, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668635165, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668717188, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7769597768783569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668717188, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668717189, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668732170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10882019251585007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133201920, "lr": 6.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668747188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10300895571708679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668762024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11091187596321106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133529600, "lr": 6.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668776600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09834402054548264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668791754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11541063338518143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133857280, "lr": 6.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668806770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10990656167268753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668821650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10307599604129791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134184960, "lr": 6.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668836888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10841881483793259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668852083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080934926867485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134512640, "lr": 6.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668866907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1083945631980896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668881864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1097526103258133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134840320, "lr": 6.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668896844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09858580678701401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668911468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10678296536207199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135168000, "lr": 6.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668926182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08692122250795364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668926183, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784668926183, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669008369, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7774516344070435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669008370, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669008370, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669023236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09561176598072052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135495680, "lr": 6.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669038225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11137309670448303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669053175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10662076622247696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135823360, "lr": 6.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669068057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10113725811243057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669082982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10184977948665619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136151040, "lr": 6.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669097953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09055839478969574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669112584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10326305776834488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136478720, "lr": 6.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669127366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09682448953390121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669142065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10328172147274017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136806400, "lr": 6.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669157135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10619624704122543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669172299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10996047407388687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137134080, "lr": 6.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669187426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081785187125206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669202434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10589540749788284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137461760, "lr": 6.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669217668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1030028834939003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669217669, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669217670, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669301680, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7775658369064331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669301681, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669301681, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669316763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09576897323131561, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137789440, "lr": 7.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669332069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10534138977527618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669346912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09471315145492554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138117120, "lr": 7.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669362029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10150184482336044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669376963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09733200818300247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138444800, "lr": 7.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669391395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10046831518411636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669405928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10910652577877045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138772480, "lr": 7.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669420868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10055288672447205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.066249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669435529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10871460288763046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139100160, "lr": 7.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669450281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10938157886266708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669465372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10613516718149185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139427840, "lr": 7.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669480386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10394731163978577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669495212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10911142081022263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139755520, "lr": 7.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669510080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10812763124704361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669510082, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669510082, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669593026, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7779987454414368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669593027, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669593027, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669607626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10609585791826248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140083200, "lr": 7.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669622319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10448279231786728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669637023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1028718426823616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140410880, "lr": 7.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669651697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09925448894500732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669666595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10712063312530518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140738560, "lr": 7.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669681562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09597675502300262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669697012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10432137548923492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141066240, "lr": 7.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669711690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10224746912717819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669726496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10970684140920639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141393920, "lr": 7.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669741384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.098420150578022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669756045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09706452488899231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141721600, "lr": 7.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669770884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10168980062007904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669785893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08993085473775864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142049280, "lr": 7.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669800945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10631591081619263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669800945, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784669800946, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784669884488, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7783759236335754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784669884488, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784669884489, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784669899230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09873980283737183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142376960, "lr": 7.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669914325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1000504344701767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669929343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09940285980701447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142704640, "lr": 7.257916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669944570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10851716250181198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669959861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09963851422071457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143032320, "lr": 7.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669975132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10221179574728012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669990171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1011556088924408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143360000, "lr": 7.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670005109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1052786260843277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670020404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10097033530473709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143687680, "lr": 7.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670035323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11074639856815338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670050536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10047364234924316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144015360, "lr": 7.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670065296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10745299607515335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670080225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10528364777565002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144343040, "lr": 7.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670095416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10574135929346085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670095417, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670095417, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670180213, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.778720498085022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670180214, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670180214, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670195562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10473089665174484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144670720, "lr": 7.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670210630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09772282838821411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670225482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10661906003952026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144998400, "lr": 7.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670240707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10616790503263474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670255969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10197277367115021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145326080, "lr": 7.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670270867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10730931907892227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670285731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.097060427069664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145653760, "lr": 7.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670300984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09769213199615479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670316255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10500429570674896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145981440, "lr": 7.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670331249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1032751053571701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670346462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10139202326536179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146309120, "lr": 7.441250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670361646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10935831069946289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670376653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10613537579774857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146636800, "lr": 7.457916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670391538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09957922250032425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670391539, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670391539, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670476896, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7791900038719177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670476897, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670476897, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670491450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10149786621332169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146964480, "lr": 7.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670505918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10522475093603134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670520564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09442780911922455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147292160, "lr": 7.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670535588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09845425188541412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670550324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1016613319516182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147619840, "lr": 7.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670565560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09941837936639786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670580712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10706344246864319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147947520, "lr": 7.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670595836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10937470942735672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670610977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10856827348470688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148275200, "lr": 7.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670626261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09873882681131363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670641625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1014343872666359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148602880, "lr": 7.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670656397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09698838740587234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670671632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11160343885421753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148930560, "lr": 7.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670686449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11042720079421997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670686450, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670686450, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670771560, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7795014381408691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670771561, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670771561, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670785981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10565131902694702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149258240, "lr": 7.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670801318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10450229793787003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670816234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10602576285600662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149585920, "lr": 7.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670831521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11123049259185791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149749760, "lr": 7.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670846961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10671916604042053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149913600, "lr": 7.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670862455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10781046003103256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670877528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1073506772518158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150241280, "lr": 7.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670892725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09717871248722076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150405120, "lr": 7.649583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670907959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10567782819271088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150568960, "lr": 7.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670922980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10364636033773422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670938154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10807977616786957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150896640, "lr": 7.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670952863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10366684943437576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151060480, "lr": 7.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670968156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10559242218732834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151224320, "lr": 7.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670983245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09509117901325226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670983292, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784670983293, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671067025, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7795891165733337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671067026, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671067026, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671082165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09883357584476471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151552000, "lr": 7.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671097203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10436942428350449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151715840, "lr": 7.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671112236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10245530307292938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151879680, "lr": 7.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671127439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10463204234838486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671142601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10770351439714432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152207360, "lr": 7.741249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671157628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12030281871557236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152371200, "lr": 7.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671172712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10778985172510147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152535040, "lr": 7.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671187637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09887620806694031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671202712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11017929762601852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152862720, "lr": 7.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671217667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11007815599441528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153026560, "lr": 7.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671232958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11504516005516052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153190400, "lr": 7.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671248259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0977713018655777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671263641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10603281110525131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153518080, "lr": 7.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671278807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10294102132320404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153681920, "lr": 7.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671278853, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671278854, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671362590, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7800170183181763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671362591, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671362591, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671377910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09589602053165436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153845760, "lr": 7.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671393274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09729047119617462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.832916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671408288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11212624609470367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154173440, "lr": 7.841249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671423811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11683442443609238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154337280, "lr": 7.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671438909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11253564059734344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154501120, "lr": 7.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671453692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11069117486476898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671469064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10653717070817947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154828800, "lr": 7.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671484008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1199997290968895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154992640, "lr": 7.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671498930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10442742705345154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155156480, "lr": 7.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671514525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10863355547189713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671529928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10929349809885025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155484160, "lr": 7.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671545100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11009372025728226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155648000, "lr": 7.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671560461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11298075318336487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155811840, "lr": 7.924583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671575805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11098053306341171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.932916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671575806, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671575807, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671659163, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7802456021308899, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671659164, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671659164, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671674413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10755036026239395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156139520, "lr": 7.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671689607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10978513211011887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156303360, "lr": 7.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671704948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11409157514572144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156467200, "lr": 7.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671720038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10980475693941116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671735116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1073610857129097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156794880, "lr": 7.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671750397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10713132470846176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156958720, "lr": 7.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671765490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10171766579151154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157122560, "lr": 7.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671780419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11218655109405518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671795442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11950506269931793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157450240, "lr": 8.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671810507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10845068097114563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157614080, "lr": 8.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671825496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1086277961730957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157777920, "lr": 8.024583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671840707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10383087396621704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.032916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671856068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10202944278717041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158105600, "lr": 8.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671871353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10875076055526733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158269440, "lr": 8.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671871353, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784671871354, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784671952637, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7807974219322205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784671952638, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784671952638, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784671967662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11422436684370041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158433280, "lr": 8.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671982964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11836467683315277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671998231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11278647929430008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158760960, "lr": 8.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672013743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11291701346635818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158924800, "lr": 8.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672029190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11000034213066101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159088640, "lr": 8.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672044299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11898383498191833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672059673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.110912024974823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159416320, "lr": 8.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672074436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1097634881734848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159580160, "lr": 8.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672089743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11004745960235596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159744000, "lr": 8.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672104941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09894969314336777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672120187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11001098901033401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160071680, "lr": 8.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672135947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10940300673246384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160235520, "lr": 8.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672151484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10342629253864288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160399360, "lr": 8.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672166674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11029630899429321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672166675, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672166675, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672248097, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.780972957611084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672248098, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672248098, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672263224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.112589530646801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160727040, "lr": 8.174583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672278707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12641653418540955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160890880, "lr": 8.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672294185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10781095921993256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161054720, "lr": 8.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672309588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1117968037724495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672324894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11212952435016632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161382400, "lr": 8.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672340478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10873544961214066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161546240, "lr": 8.216250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672356135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11390899866819382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161710080, "lr": 8.224583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672371756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11272364854812622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672387092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10660412907600403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162037760, "lr": 8.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672402499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10847050696611404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162201600, "lr": 8.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672418107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1296059638261795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162365440, "lr": 8.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672433417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11249697953462601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672449278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11187435686588287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162693120, "lr": 8.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672465139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11537877470254898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162856960, "lr": 8.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672465139, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672465140, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672548464, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7813300490379333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672548465, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672548465, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672564004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09598948806524277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163020800, "lr": 8.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672579802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11080239713191986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672595607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1213335171341896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163348480, "lr": 8.307916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672611303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10675707459449768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163512320, "lr": 8.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672627045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11079007387161255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163676160, "lr": 8.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672642738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10692290961742401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672658219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11268346756696701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164003840, "lr": 8.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672673291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11306016147136688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164167680, "lr": 8.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672688397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10307762026786804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164331520, "lr": 8.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672703099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11775368452072144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672717973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10658824443817139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164659200, "lr": 8.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672733068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11123857647180557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164823040, "lr": 8.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672748206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09172315895557404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164986880, "lr": 8.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672762856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10327904671430588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672762857, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784672762858, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784672844440, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7810871601104736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784672844441, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784672844441, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784672859443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09979330003261566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165314560, "lr": 8.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672874290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09597344696521759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165478400, "lr": 8.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672888970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11291813105344772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165642240, "lr": 8.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672903702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10745544731616974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672918312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10970255732536316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165969920, "lr": 8.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672933451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10513246059417725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166133760, "lr": 8.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672948361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10749775916337967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166297600, "lr": 8.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672963321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11196999251842499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672978196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0993768572807312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166625280, "lr": 8.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672993516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09518342465162277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166789120, "lr": 8.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673008619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09702128171920776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166952960, "lr": 8.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673023595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09680649638175964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673038606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10198073089122772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167280640, "lr": 8.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673053607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09980260580778122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167444480, "lr": 8.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673053608, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673053609, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673135605, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7813848853111267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673135607, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673135607, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673150669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.103104367852211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167608320, "lr": 8.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673166050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1019618883728981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673181163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10316160321235657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167936000, "lr": 8.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673196319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10677952319383621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168099840, "lr": 8.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673211604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10141456872224808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168263680, "lr": 8.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673227058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0981704592704773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673242700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10799796134233475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168591360, "lr": 8.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673257882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10541454702615738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168755200, "lr": 8.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673273087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10645154118537903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168919040, "lr": 8.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673288108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10579897463321686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673303496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10376352816820145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169246720, "lr": 8.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673318486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10604352504014969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169410560, "lr": 8.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673333228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1054924800992012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169574400, "lr": 8.624583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673348237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11207927018404007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673348286, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673348287, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673429722, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7814478874206543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673429723, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673429723, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673444274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10819416493177414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169902080, "lr": 8.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673459076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11155417561531067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170065920, "lr": 8.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673474061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10602088272571564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170229760, "lr": 8.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673489238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10645456612110138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673504021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09988641738891602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170557440, "lr": 8.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673518960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10049532353878021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170721280, "lr": 8.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673533818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09614469856023788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170885120, "lr": 8.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673548432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10593967139720917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673563032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0984976664185524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171212800, "lr": 8.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673577612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10085878521203995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171376640, "lr": 8.716249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673592565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10113655775785446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171540480, "lr": 8.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673607146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10207642614841461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673621864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09871628135442734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171868160, "lr": 8.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673636652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10416439920663834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172032000, "lr": 8.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673636695, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673636696, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673718592, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7815358638763428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673718593, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673718593, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673733451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09685229510068893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172195840, "lr": 8.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673748299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081019788980484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673763083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09958788752555847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172523520, "lr": 8.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673777608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10192176699638367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172687360, "lr": 8.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673792074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0941922515630722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172851200, "lr": 8.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673806863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10678271949291229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673821925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1026902124285698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173178880, "lr": 8.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673836738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10203984379768372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173342720, "lr": 8.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673851859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1037948951125145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173506560, "lr": 8.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673866692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11110074073076248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673881906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0944395437836647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173834240, "lr": 8.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673897166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11014951765537262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173998080, "lr": 8.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673912440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1111694946885109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174161920, "lr": 8.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673927313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10100254416465759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673927362, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784673927363, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784674009891, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7818500399589539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784674009891, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784674009892, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784674024744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09767629206180573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174489600, "lr": 8.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674039994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10726679861545563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174653440, "lr": 8.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674055260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10095824301242828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174817280, "lr": 8.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674069838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1047060415148735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674084716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10010268539190292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175144960, "lr": 8.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674099520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10825184732675552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175308800, "lr": 8.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674114498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10018213093280792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175472640, "lr": 8.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674129743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09755278378725052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674144795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085977852344513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175800320, "lr": 8.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674160060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10778588056564331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175964160, "lr": 8.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674175106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09846988320350647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176128000, "lr": 8.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674190213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09325054287910461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674205072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09436933696269989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176455680, "lr": 8.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674219930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10546888411045074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176619520, "lr": 8.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674219931, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674219931, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674304376, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7821173667907715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674304377, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674304377, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674319439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10244321823120117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176783360, "lr": 8.991249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674334490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11323723196983337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674349660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10012407600879669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177111040, "lr": 9.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674364839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10471468418836594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177274880, "lr": 9.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674379840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11002057045698166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177438720, "lr": 9.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674394858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10021598637104034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.032916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674409952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085430458188057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177766400, "lr": 9.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674424871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10048776119947433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177930240, "lr": 9.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674439984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10581637173891068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178094080, "lr": 9.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674454968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10032911598682404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674470093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09693190455436707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178421760, "lr": 9.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674485509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11120373010635376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178585600, "lr": 9.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674500896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1185288280248642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178749440, "lr": 9.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674516176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10812263190746307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674516177, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674516178, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674599768, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7823681831359863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674599769, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674599769, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674615252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11425869911909103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179077120, "lr": 9.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674630250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10147516429424286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179240960, "lr": 9.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674645750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09979044646024704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179404800, "lr": 9.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674661010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10073509812355042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.132916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674676231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1044510006904602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179732480, "lr": 9.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674691587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09864915907382965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179896320, "lr": 9.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674706648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10002578049898148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180060160, "lr": 9.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674721991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10581353306770325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674736831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10501938313245773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180387840, "lr": 9.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674751704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09308447688817978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180551680, "lr": 9.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674767193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1029311865568161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180715520, "lr": 9.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674782317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10081934928894043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674797634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09376794844865799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181043200, "lr": 9.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674813070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10017812252044678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181207040, "lr": 9.216249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674813071, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784674813072, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784674896911, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7823840379714966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784674896912, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784674896912, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784674912222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11063779890537262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181370880, "lr": 9.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674927432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11392972618341446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674942386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10057322680950165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181698560, "lr": 9.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674957599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10600695759057999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181862400, "lr": 9.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674972876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1018361747264862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182026240, "lr": 9.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674987920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1012246385216713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675003007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10628045350313187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182353920, "lr": 9.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675017924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10813786834478378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182517760, "lr": 9.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675033069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0971459150314331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182681600, "lr": 9.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675047952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09626905620098114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675062929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11006391793489456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183009280, "lr": 9.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675077751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09687604755163193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183173120, "lr": 9.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675092975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1021103709936142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183336960, "lr": 9.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675107900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10202256590127945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675107900, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675107901, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675192610, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7824775576591492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675192611, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675192611, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675207575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10181881487369537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183664640, "lr": 9.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675222499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10717082768678665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183828480, "lr": 9.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675237331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11194421350955963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183992320, "lr": 9.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675252767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09281563013792038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675268112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10169544070959091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184320000, "lr": 9.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675283435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10718944668769836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184483840, "lr": 9.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675298729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11567581444978714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184647680, "lr": 9.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675313947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10073819011449814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675329409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10546642541885376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184975360, "lr": 9.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675345024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1086815893650055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185139200, "lr": 9.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675360412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09659723192453384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185303040, "lr": 9.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675375463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1068684309720993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675390725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0983026772737503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185630720, "lr": 9.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675405689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10284562408924103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185794560, "lr": 9.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675405690, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675405690, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675490628, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7827094197273254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675490629, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675490629, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675506018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11007942259311676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185958400, "lr": 9.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675520887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1037072166800499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675535569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09212768822908401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186286080, "lr": 9.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675551009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10053717344999313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186449920, "lr": 9.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675566162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080736517906189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186613760, "lr": 9.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675581531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11107618361711502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675597423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09629315137863159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186941440, "lr": 9.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675613133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10320020467042923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187105280, "lr": 9.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675628647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10505464673042297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187269120, "lr": 9.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675644082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09788259863853455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675659468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10365903377532959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187596800, "lr": 9.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675674936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10452673584222794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187760640, "lr": 9.549583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675690450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11031609028577805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187924480, "lr": 9.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675706069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11329799890518188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675706069, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675706069, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675791018, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7829299569129944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675791019, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675791019, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675806201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10809475928544998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188252160, "lr": 9.574583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675821745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10502497851848602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188416000, "lr": 9.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675836867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10220322012901306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188579840, "lr": 9.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675852328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10778848081827164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675867738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10138515383005142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188907520, "lr": 9.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675883186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09666728973388672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189071360, "lr": 9.616249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675898338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10202895104885101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189235200, "lr": 9.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675913365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11135919392108917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675928334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10109741985797882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189562880, "lr": 9.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675943408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09435955435037613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189726720, "lr": 9.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675958550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.103609099984169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189890560, "lr": 9.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675973832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11178324371576309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.666249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675989191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10122798383235931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190218240, "lr": 9.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676004220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11737354844808578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190382080, "lr": 9.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676004221, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676004222, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676089245, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7830372452735901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676089246, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676089246, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676104576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09978458285331726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190545920, "lr": 9.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676119484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09688183665275574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676134458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10616863518953323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190873600, "lr": 9.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676149710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11240551620721817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191037440, "lr": 9.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676165048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11644410341978073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191201280, "lr": 9.724583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676180196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09594473242759705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676195493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10595989972352982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191528960, "lr": 9.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676210341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10070965439081192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191692800, "lr": 9.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676225608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10082750022411346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191856640, "lr": 9.757916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676240787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10434719920158386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676255438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10648022592067719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192184320, "lr": 9.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676270967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1092873066663742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192348160, "lr": 9.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676286305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09862904995679855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192512000, "lr": 9.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676301834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11340783536434174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.799583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676301835, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676301835, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676383933, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7833932042121887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676383935, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676383935, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676399531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11172718554735184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192839680, "lr": 9.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676415116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11066383868455887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193003520, "lr": 9.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676430437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10563471913337708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193167360, "lr": 9.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676445994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1187838762998581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676461299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1163952574133873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193495040, "lr": 9.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676476818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10037223994731903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193658880, "lr": 9.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676492394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11699657142162323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193822720, "lr": 9.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676507677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11345711350440979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.866249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676523118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10538486391305923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194150400, "lr": 9.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676538443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10678541660308838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194314240, "lr": 9.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676553878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10663831233978271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194478080, "lr": 9.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676569391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10560205578804016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676584977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10677821189165115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194805760, "lr": 9.907916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676600129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11544080078601837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194969600, "lr": 9.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676600130, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676600130, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676681671, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7835028171539307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676681672, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676681672, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676697055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11147087812423706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195133440, "lr": 9.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676712065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10348732769489288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676727408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10198625177145004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195461120, "lr": 9.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676742898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11654190719127655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195624960, "lr": 9.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676758563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792148858308792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195788800, "lr": 9.957916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676774010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10476168990135193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676789641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10354871302843094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196116480, "lr": 9.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676805095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11359565705060959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196280320, "lr": 9.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676820979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11788478493690491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196444160, "lr": 9.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676836527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10410354286432266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.999583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676851949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11610592156648636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196771840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676867680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11196054518222809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196935680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676882860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10748454928398132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197099520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676898481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11578895151615143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676898481, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784676898482, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784676980118, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7838961482048035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784676980119, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784676980119, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784676995416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1012599766254425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197427200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677011081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1109076589345932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197591040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677026635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11500103771686554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197754880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677042195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10020353645086288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677058149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09857619553804398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198082560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677073928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732414573431015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198246400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677089505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09853905439376831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198410240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677104973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10777215659618378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677120628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09962862730026245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198737920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677136002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10545747727155685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198901760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677151264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11455345898866653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199065600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677166811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11049023270606995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677182079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11717396229505539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199393280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677197906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11226601153612137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199557120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677197907, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677197907, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677280283, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7845732569694519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677280284, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677280284, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677295729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11156948655843735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199720960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677311392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10557401925325394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677326694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1106032058596611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200048640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677342100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11539960652589798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200212480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677357397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11199276894330978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200376320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677372889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11669138818979263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677388626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09926493465900421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200704000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677404016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10821489989757538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200867840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677419616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725107789039612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201031680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677435047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1078919842839241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677450647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11726675927639008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201359360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677465921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11015457659959793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201523200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677481072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10251963138580322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201687040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677496387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11070510745048523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677496388, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677496389, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677578320, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7837429046630859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677578321, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677578321, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677593306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10547448694705963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202014720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677608574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10983451455831528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202178560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677623704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11872343719005585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202342400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677639120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10679914802312851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677654220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1052824854850769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202670080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677669375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10045001655817032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202833920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677685084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10622108727693558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202997760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677700502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10905206203460693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677715716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11235379427671432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203325440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677730746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09367325901985168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203489280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677745979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11626666784286499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203653120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677761102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09954246878623962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677775958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09889503568410873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203980800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677790913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723663121461868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204144640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677790913, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784677790914, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784677872479, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7839300036430359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784677872480, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784677872480, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784677887549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10090027749538422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204308480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677902893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09317588061094284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677918606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09691544622182846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204636160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677934052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084933653473854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204800000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677949339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10771240293979645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204963840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677964713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11225724965333939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677979871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09513312578201294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205291520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677995301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10600251704454422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205455360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678010720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1051260381937027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205619200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678025966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10997620224952698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678041233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09760033339262009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205946880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678056956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09860437363386154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206110720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678072347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1110815778374672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206274560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678087574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09872838854789734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678087574, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678087575, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678168408, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7844783663749695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678168409, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678168409, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678184083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10571573674678802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206602240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678199507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09555906057357788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206766080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678215057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10292669385671616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206929920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678230830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09554927051067352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678246515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10231658071279526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207257600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678262142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1102536991238594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207421440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678277713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10654932260513306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207585280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678293166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09491526335477829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678308748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11352812498807907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207912960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678324240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10368206351995468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208076800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678339218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1027660220861435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208240640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678354695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10177940130233765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678369474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10282102227210999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208568320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678384302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1011575385928154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208732160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678384303, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678384303, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678466820, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7842319011688232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678466821, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678466821, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678482079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1002863273024559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208896000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678497271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09907042980194092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678512422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10127788782119751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209223680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678527154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10137521475553513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209387520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678542268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09873256832361221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209551360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678557192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1001386046409607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678572028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09680439531803131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209879040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678586714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10011087357997894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210042880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678601701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1029568687081337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210206720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678616799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10758139938116074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678631653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1036047413945198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210534400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678646417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10739327222108841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210698240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678661624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10137569904327393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210862080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678676332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09506594389677048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678676335, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678676335, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678759992, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7845577597618103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678759993, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678759993, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678775137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09424804151058197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211189760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678789654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09879379719495773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211353600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678804426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09608615934848785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211517440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678819346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09957832843065262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678834077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1037931814789772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211845120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678849423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09490721672773361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212008960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678864218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10168324410915375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212172800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678879368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09768480807542801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678894359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09778311103582382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212500480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678909260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10388895124197006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212664320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678924416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09305758029222488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212828160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678939875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10684406757354736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678955324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10458049923181534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213155840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678970746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725986212491989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213319680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678970747, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784678970747, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784679053349, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.784905731678009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784679053349, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784679053350, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784679068603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10490144789218903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213483520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679083701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10098166763782501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679098640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10476893931627274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213811200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679114057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10256075114011765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213975040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679128693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09610196948051453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214138880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679143411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10511057823896408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679158713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09461726993322372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214466560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679173813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11355739831924438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214630400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679189048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09801045805215836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214794240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679204887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08961890637874603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679220553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10048855096101761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215121920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679235906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10453606396913528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215285760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679251152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10418515652418137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215449600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679266266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087312400341034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679266266, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679266267, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679346989, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7849149107933044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679346989, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679346990, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679362208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09955613315105438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215777280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679376968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10518582165241241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215941120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679392083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09749330580234528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216104960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679407362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10019289702177048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679422817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10017166286706924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216432640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679437998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0943777933716774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216596480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679453385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10435215383768082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216760320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679468934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1024128794670105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679483941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09995562583208084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217088000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679499368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10946498811244965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217251840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679514260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09736167639493942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217415680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679529676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09951602667570114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217579520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679544875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09934127330780029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217743360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679559962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10532228648662567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217907200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679559963, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679559963, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679642452, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7846201658248901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679642453, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679642453, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679657784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1009301245212555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218071040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679673074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09493880718946457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218234880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679688102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09804070740938187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218398720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679702827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10825282335281372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218562560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679718112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1028633788228035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218726400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679733239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08705287426710129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218890240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679748475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10605185478925705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219054080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679763749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09289876371622086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219217920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679778998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1036633774638176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219381760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679794224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09509050101041794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219545600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679809177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09429726004600525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219709440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679823881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10288754105567932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219873280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679838945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11177383363246918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220037120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679853849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09754238277673721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220200960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679853850, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784679853850, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784679938056, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7854417562484741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784679938057, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784679938057, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784679952742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0951664075255394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220364800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679967910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09560933709144592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220528640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679982704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10040736198425293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220692480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679997425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10169914364814758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220856320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680012477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10303870588541031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221020160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680027662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1051115021109581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221184000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680042811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081317663192749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221347840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680057570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1155603677034378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221511680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680072373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10784298926591873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221675520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680087352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09909893572330475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221839360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680102298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11042483896017075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222003200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680117197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09602565318346024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222167040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680132160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09412126988172531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222330880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680147057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10758287459611893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222494720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680147057, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784680147058, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784680230745, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7854156494140625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784680230746, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784680230746, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} diff --git a/recommendation_v4/rcp_logs/gbs_8192/seed666410936.log b/recommendation_v4/rcp_logs/gbs_8192/seed666410936.log new file mode 100644 index 000000000..478963f17 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_8192/seed666410936.log @@ -0,0 +1,1818 @@ +:::MLLOG {"namespace": "", "time_ms": 1784651470907, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784651470908, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784651492049, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784651492052, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784651492052, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784651492052, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784651492052, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784651492052, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 8192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784651492052, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784651492052, "event_type": "POINT_IN_TIME", "key": "seed", "value": 666410936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784651492052, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784651492052, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784651492053, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784651492053, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784651492053, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784651492054, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784652395120, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784652395121, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784652395456, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784652571815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13779228925704956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840, "lr": 7.916666666666666e-10}} +:::MLLOG {"namespace": "", "time_ms": 1784652584317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13948743045330048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.6249999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652597193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13861797749996185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 491520, "lr": 2.458333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652610158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13914042711257935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652623332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13889621198177338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 819200, "lr": 4.125e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652636498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13873125612735748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652649661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13904644548892975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1146880, "lr": 5.7916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652662809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13885930180549622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.625e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652676207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13890239596366882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1474560, "lr": 7.458333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652689409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13903111219406128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.291666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652703014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13884574174880981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1802240, "lr": 9.124999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652716429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387496292591095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652730201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13901670277118683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2129920, "lr": 1.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652743562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13824324309825897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652743563, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652743563, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652928256, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49880778789520264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652928257, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652928257, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652941120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388450562953949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2457600, "lr": 1.2458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652954241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13882307708263397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652967663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386668086051941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2785280, "lr": 1.4124999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652981058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389031559228897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4958333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652994642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389792114496231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3112960, "lr": 1.5791666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653008478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387631893157959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653021582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387048214673996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3440640, "lr": 1.7458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653034945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387728452682495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653048944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13863505423069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3768320, "lr": 1.9124999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653062527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386292725801468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9958333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653076397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13875772058963776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4096000, "lr": 2.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653089857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13861820101737976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.1624999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653103364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13863421976566315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4423680, "lr": 2.2458333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653117252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13868071138858795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3291666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653117252, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784653117253, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784653202264, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49902045726776123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784653202265, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784653202266, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784653215990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13838881254196167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4751360, "lr": 2.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653229898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138384610414505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653243818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13822486996650696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5079040, "lr": 2.5791666666666668e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653257724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13787609338760376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653271072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13839542865753174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5406720, "lr": 2.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653284452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13854840397834778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653297947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13788393139839172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5734400, "lr": 2.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653312289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13787472248077393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653326278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13758111000061035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6062080, "lr": 3.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653340014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138236403465271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653353979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13785718381404877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6389760, "lr": 3.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653367661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381261646747589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653381740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380717158317566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6717440, "lr": 3.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653395242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13806062936782837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653395242, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784653395243, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784653479492, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4992886781692505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784653479492, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784653479493, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784653492790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13739073276519775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7045120, "lr": 3.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653506413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13777832686901093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653519990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13728058338165283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7372800, "lr": 3.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653534367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13757362961769104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.829166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653548334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378859281539917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7700480, "lr": 3.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653562370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13791893422603607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653576271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1372217833995819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8028160, "lr": 4.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653590060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13759775459766388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653604251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370912790298462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8355840, "lr": 4.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653618245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13867442309856415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3291666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653632018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13682374358177185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8683520, "lr": 4.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653645722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13803166151046753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.4958333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653659634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359868198633194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9011200, "lr": 4.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653673876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13735145330429077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.6624999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653673876, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653673877, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653757984, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49963000416755676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653757985, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653757985, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653771630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366024613380432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9338880, "lr": 4.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653785547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13615527749061584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653799250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13781486451625824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9666560, "lr": 4.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653813393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13519734144210815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653827712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368628293275833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9994240, "lr": 5.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653841618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13420552015304565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1624999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653855637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13592469692230225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10321920, "lr": 5.2458333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653869363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358049362897873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653883276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13790567219257355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10649600, "lr": 5.4124999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653897067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13553261756896973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653910790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1457153558731079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10977280, "lr": 5.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653924902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357528120279312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653938644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344693899154663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11304960, "lr": 5.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653952575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13795089721679688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653952575, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653952576, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784654036000, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5000606179237366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784654036001, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784654036002, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784654049468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13565371930599213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11632640, "lr": 5.9124999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654062989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13949376344680786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654076595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13622674345970154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11960320, "lr": 6.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654090092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13382376730442047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654103699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13532288372516632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12288000, "lr": 6.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654117411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13895002007484436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.329166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654131054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13370881974697113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12615680, "lr": 6.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654144818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13342763483524323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654158419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13365617394447327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12943360, "lr": 6.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654171832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13208380341529846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654185493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13173159956932068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13271040, "lr": 6.745833333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654198870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350904256105423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654212514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13260327279567719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13598720, "lr": 6.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654226537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387495994567871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654226538, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784654226538, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784654307975, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5006159543991089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784654307976, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784654307977, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784654321829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13397163152694702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13926400, "lr": 7.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654335558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13369858264923096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654349483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13594010472297668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14254080, "lr": 7.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654363176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356057971715927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654376805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13735611736774445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14581760, "lr": 7.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654390498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12852488458156586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654403887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13546152412891388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14909440, "lr": 7.579166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654417513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297309249639511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654431419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13366883993148804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15237120, "lr": 7.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654445462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1328895092010498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.829166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654459267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13370639085769653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15564800, "lr": 7.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654473363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13471965491771698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.995833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654487863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353222131729126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15892480, "lr": 8.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654501873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338682621717453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654501874, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654501874, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654584974, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5014109015464783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654584976, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654584976, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654599165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327541172504425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16220160, "lr": 8.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654613232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13395115733146667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654627478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13532766699790955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16547840, "lr": 8.412500000000001e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654641939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13541509211063385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654656229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13939908146858215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16875520, "lr": 8.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654670345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334255337715149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.662499999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654684950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13532161712646484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17203200, "lr": 8.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654698771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335832178592682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654712653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312476873397827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17530880, "lr": 8.912499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654726781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13199077546596527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654740597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13588209450244904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17858560, "lr": 9.079166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654754864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13362818956375122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.162499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654768740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334420144557953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18186240, "lr": 9.245833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654782497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13072247803211212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654782498, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654782498, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654867491, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5025701522827148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654867492, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654867492, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654882370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377616673707962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18513920, "lr": 9.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654896666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13482236862182617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.495833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654911120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13540126383304596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18841600, "lr": 9.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654925843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1314469277858734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.662499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654940241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13353198766708374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19169280, "lr": 9.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654954856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13431328535079956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654969596, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13686446845531464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19496960, "lr": 9.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654984343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13396470248699188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654998973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13306252658367157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19824640, "lr": 1.0079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655012978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13007725775241852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0162499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655026484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12765121459960938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20152320, "lr": 1.0245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655040318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13298562169075012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655054350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12970805168151855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20480000, "lr": 1.0412499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655068010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12482650578022003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655068010, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784655068011, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784655151843, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5037841200828552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784655151844, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784655151845, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784655165795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13116581737995148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20807680, "lr": 1.0579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655179652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13599827885627747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655193741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13310830295085907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21135360, "lr": 1.0745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655207969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13052485883235931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655221706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1283496618270874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21463040, "lr": 1.0912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655235846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1268254965543747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655250101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12902475893497467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21790720, "lr": 1.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655263823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324000060558319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655277782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13560549914836884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22118400, "lr": 1.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655291871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12516631186008453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655305680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12871238589286804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22446080, "lr": 1.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655319607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13457223773002625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655333832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13104213774204254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22773760, "lr": 1.1579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655348370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12935864925384521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655348371, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784655348371, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784655430030, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5053304433822632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784655430031, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784655430031, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784655444602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13067108392715454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23101440, "lr": 1.1745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655458656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12917710840702057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655472218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329631358385086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23429120, "lr": 1.1912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655486459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12404709309339523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655500739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12960034608840942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23756800, "lr": 1.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655514485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.125852569937706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655528285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13004086911678314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24084480, "lr": 1.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655542360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12191492319107056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655556018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13577960431575775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24412160, "lr": 1.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655570198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13711890578269958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655584186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12378627806901932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24739840, "lr": 1.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655598575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12339725345373154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655612804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12577466666698456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25067520, "lr": 1.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655627003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12898333370685577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655627003, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655627003, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655709433, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5071465373039246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655709434, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655709434, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655723179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13654941320419312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25395200, "lr": 1.2912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655737926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13655363023281097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655751940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13572977483272552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25722880, "lr": 1.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655765917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1399485170841217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655780026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13675682246685028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26050560, "lr": 1.3245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655794087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13833382725715637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655808088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12742003798484802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26378240, "lr": 1.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655822159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12632563710212708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655836345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14223116636276245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26705920, "lr": 1.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655850870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14084377884864807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655865497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12718668580055237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27033600, "lr": 1.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655880008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12829449772834778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655894416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325046420097351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27361280, "lr": 1.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655908190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13167470693588257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655908190, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655908191, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655991644, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5090919733047485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655991645, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655991645, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784656006005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1413619965314865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27688960, "lr": 1.4079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656020514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14226390421390533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656034550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13540548086166382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28016640, "lr": 1.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656048788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13947172462940216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656063294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14475145936012268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28344320, "lr": 1.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656077330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14104220271110535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656091684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12583285570144653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28672000, "lr": 1.4579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656105953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13760921359062195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656120399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13172399997711182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28999680, "lr": 1.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656134885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13613325357437134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656149317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13775093853473663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29327360, "lr": 1.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656163594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13310843706130981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4995833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656178794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13457806408405304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29655040, "lr": 1.5079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656193753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13654445111751556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656193753, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784656193754, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784656276563, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5111192464828491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784656276565, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784656276565, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784656290989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13857316970825195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29982720, "lr": 1.5245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656305862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13823062181472778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656320850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13173167407512665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30310400, "lr": 1.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656335798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13287745416164398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656350744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13383270800113678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30638080, "lr": 1.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656365887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13987033069133759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656380872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326596587896347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30965760, "lr": 1.5745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656395904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1412842720746994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656410163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1410370171070099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31293440, "lr": 1.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656424953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13499243557453156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656439468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12782010436058044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31621120, "lr": 1.6079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656453567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301051527261734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656467717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13420256972312927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31948800, "lr": 1.6245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656481956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12763676047325134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656481956, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784656481957, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784656565023, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5134171843528748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784656565025, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784656565025, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784656579227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323276311159134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32276480, "lr": 1.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656593285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368236392736435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656607364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13354766368865967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32604160, "lr": 1.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656621474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320073902606964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656635697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341448575258255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32931840, "lr": 1.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656649858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13341209292411804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656664368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13393601775169373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33259520, "lr": 1.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656678814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13289609551429749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656692970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13978737592697144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33587200, "lr": 1.7079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656707955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13195909559726715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656722477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12879972159862518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33914880, "lr": 1.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656736722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13121351599693298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656751101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323058307170868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34242560, "lr": 1.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656765513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13584312796592712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7495833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656765513, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656765514, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656849255, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5164732933044434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656849256, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656849256, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656863782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13348668813705444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34570240, "lr": 1.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656878133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12322060018777847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656892435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13071495294570923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34897920, "lr": 1.7745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656906949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13047625124454498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656921250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13525867462158203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35225600, "lr": 1.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656936177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135297492146492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656950637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13550104200839996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35553280, "lr": 1.8079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656965452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12408757209777832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8162500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656979861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13097259402275085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35880960, "lr": 1.8245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656993887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13118579983711243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657008618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304195374250412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36208640, "lr": 1.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657022797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133245050907135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657037103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13086554408073425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36536320, "lr": 1.8579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657051095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13035528361797333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657051096, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784657051097, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784657136617, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5203675627708435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784657136618, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784657136618, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784657150947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13779966533184052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36864000, "lr": 1.8745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657165218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13242554664611816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657179419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13365121185779572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37191680, "lr": 1.8912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657193669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13043734431266785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657207908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1271987110376358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37519360, "lr": 1.9079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657222044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13232997059822083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657236733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355055570602417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37847040, "lr": 1.9245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657251109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1328631043434143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657265527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1314629763364792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38174720, "lr": 1.9412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657280147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13906662166118622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657294628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13394209742546082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38502400, "lr": 1.9579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657308597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13751757144927979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657323480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13188093900680542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38830080, "lr": 1.9745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657337997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333867460489273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657337997, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784657337998, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784657422658, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5254380702972412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784657422659, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784657422659, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784657437080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13143497705459595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39157760, "lr": 1.9912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657451476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13510435819625854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657465921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377696692943573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39485440, "lr": 2.0079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657480494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13341769576072693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657494784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12741996347904205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39813120, "lr": 2.0245833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657509219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12941165268421173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0329166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657523626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12609319388866425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40140800, "lr": 2.0412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657537892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1242847815155983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657552582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13641062378883362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40468480, "lr": 2.0579166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657567298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373344510793686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657581979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13467484712600708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40796160, "lr": 2.0745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657597184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1270236372947693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0829166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657611938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13096272945404053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41123840, "lr": 2.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657626197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12364225089550018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657626198, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657626198, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657713072, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5326540470123291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657713072, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657713073, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657727795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13270393013954163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41451520, "lr": 2.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657742202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13289867341518402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657756337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338193714618683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41779200, "lr": 2.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657770471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13797368109226227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657784618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13640348613262177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42106880, "lr": 2.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657798750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13149426877498627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657812740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12723895907402039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42434560, "lr": 2.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657826788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352739930152893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657840709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12471026182174683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42762240, "lr": 2.1745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657854882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13052181899547577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657868939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324790120124817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43089920, "lr": 2.1912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657883335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13435068726539612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657897784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1405450701713562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43417600, "lr": 2.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657912082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362663358449936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657912082, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657912083, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657996358, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.543735682964325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657996359, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657996359, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784658011225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13665813207626343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43745280, "lr": 2.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658025650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13180691003799438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658039787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13278540968894958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44072960, "lr": 2.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658054352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13909024000167847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658069297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301417052745819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44400640, "lr": 2.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658084523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13859005272388458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658099072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13513141870498657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44728320, "lr": 2.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658113903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137381911277771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658128465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13507120311260223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45056000, "lr": 2.2912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658143429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13059453666210175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658158237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.140670046210289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45383680, "lr": 2.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658173046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13475002348423004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658187907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1292363554239273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45711360, "lr": 2.3245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658202849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13337688148021698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658202850, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784658202850, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784658287306, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.557884693145752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784658287307, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784658287308, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784658302481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14210940897464752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46039040, "lr": 2.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658317248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1328498274087906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658332734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334797739982605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46366720, "lr": 2.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658347661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13100557029247284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658362942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13539913296699524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46694400, "lr": 2.3745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658377956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13382643461227417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658392592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13422055542469025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47022080, "lr": 2.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658407252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13569436967372894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658421615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13138045370578766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47349760, "lr": 2.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658436207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13303789496421814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658450742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13302738964557648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47677440, "lr": 2.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658465359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371019333600998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658479712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360195428133011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48005120, "lr": 2.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658494457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13756565749645233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.4495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658494458, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784658494459, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784658578091, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5751428604125977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784658578092, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784658578092, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784658592467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13181172311306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48332800, "lr": 2.4579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658607028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13561171293258667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658621303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13432259857654572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48660480, "lr": 2.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658636094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322411745786667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658650360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374034434556961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48988160, "lr": 2.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658664614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297641545534134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658679849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13395215570926666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49315840, "lr": 2.507916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658694324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12427052110433578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.5162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658708960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13112297654151917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49643520, "lr": 2.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658723550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1316249519586563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658738149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13124796748161316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49971200, "lr": 2.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658752625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318431943655014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.5495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658767223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12922298908233643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50298880, "lr": 2.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658781658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13129569590091705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.5662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658781659, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658781659, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658864930, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5915071964263916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658864931, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658864931, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658879113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12284853309392929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50626560, "lr": 2.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658893568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1273595541715622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658907988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12971799075603485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50954240, "lr": 2.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658922571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13397392630577087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658937405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13171827793121338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51281920, "lr": 2.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658952026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1224258691072464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658967019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308584213256836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51609600, "lr": 2.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658981746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322462260723114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658996343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13686257600784302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51937280, "lr": 2.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659010577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12850356101989746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.6495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659025531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332244724035263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52264960, "lr": 2.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659039873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11819128692150116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.6662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659054234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12186399102210999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52592640, "lr": 2.6745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659068595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12564636766910553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659068596, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784659068596, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784659152534, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6068456768989563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784659152535, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784659152535, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784659166641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12183574587106705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52920320, "lr": 2.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659180798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1275227814912796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.6995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659194950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12874436378479004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53248000, "lr": 2.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659208978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13201232254505157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659223253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12169621884822845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53575680, "lr": 2.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659237527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13494938611984253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659251631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294204145669937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53903360, "lr": 2.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659265626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12129330635070801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.7495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659279981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12692409753799438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54231040, "lr": 2.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659294514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12725569307804108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659308826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11829659342765808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54558720, "lr": 2.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659323540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13121651113033295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659337955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12661416828632355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54886400, "lr": 2.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659352498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1283707171678543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659352498, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784659352499, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784659437443, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6218957901000977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784659437444, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784659437444, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784659451325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12806232273578644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55214080, "lr": 2.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659466135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11850620061159134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659480774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12207120656967163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55541760, "lr": 2.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659495681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12105769664049149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659510225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12296497076749802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55869440, "lr": 2.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659524587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12535502016544342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659538828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1158817857503891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56197120, "lr": 2.8579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659553137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12181994318962097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659567419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12445539981126785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56524800, "lr": 2.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659581777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1226082295179367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659596052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11479179561138153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56852480, "lr": 2.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659610457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12311818450689316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659624794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1093001589179039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57180160, "lr": 2.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659639431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12340936809778214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659639432, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659639432, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659721823, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6370042562484741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659721825, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659721825, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659736663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11570336669683456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57507840, "lr": 2.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659751444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11316894739866257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659766001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12788832187652588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57835520, "lr": 2.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659780922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12060708552598953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659795430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1223517656326294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58163200, "lr": 2.9579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659809832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11443130671977997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659823827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11812615394592285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58490880, "lr": 2.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659838898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12098999321460724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659853322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11907535791397095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58818560, "lr": 2.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659867886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1119735836982727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659882252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11921177804470062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59146240, "lr": 3.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659896630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1205078512430191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.0162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659910956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1224905475974083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59473920, "lr": 3.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659925581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11622431129217148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659925582, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659925582, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784660007440, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6873796582221985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784660007441, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784660007441, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784660021885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11849922686815262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59801600, "lr": 3.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660036337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1096729263663292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660050919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10837104171514511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60129280, "lr": 3.0579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660065283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11301089078187943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.0662499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660079704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10746709257364273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60456960, "lr": 3.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660094257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12077423930168152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660109048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11407039314508438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60784640, "lr": 3.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660124140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11886944621801376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660138772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10155009478330612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61112320, "lr": 3.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660153714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1192193552851677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.1162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660168490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11235448718070984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61440000, "lr": 3.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660183141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11139369755983353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660197678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10757995396852493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61767680, "lr": 3.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660212457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11607929319143295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660212458, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784660212458, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784660294724, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7329897880554199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784660294725, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784660294725, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784660309076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11784987896680832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62095360, "lr": 3.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660324048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11839611083269119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.1662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660338948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12128496915102005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62423040, "lr": 3.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660353704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10349538177251816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660368687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11206412315368652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62750720, "lr": 3.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660383221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11592396348714828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660398096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11814191937446594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63078400, "lr": 3.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660412868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11711037158966064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660427630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1070265993475914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63406080, "lr": 3.2245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660442276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1164742261171341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660456874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1131451204419136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63733760, "lr": 3.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660471416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11489156633615494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660485742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1111692562699318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64061440, "lr": 3.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660500566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12406039237976074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660500567, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784660500567, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784660582195, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7459519505500793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784660582196, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784660582196, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784660597082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12257744371891022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64389120, "lr": 3.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660611896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11508463323116302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660626484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11597990244626999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64716800, "lr": 3.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660641457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11944083869457245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660656178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10660254955291748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65044480, "lr": 3.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660670947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11021783947944641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660685193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11121246963739395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65372160, "lr": 3.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660700183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10762687027454376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660714887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10963212698698044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65699840, "lr": 3.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660730145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11526717990636826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660745047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12189017981290817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66027520, "lr": 3.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660759898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11434542387723923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660774993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1260741949081421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66355200, "lr": 3.3745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660789931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11102049052715302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660789932, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660789933, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660871767, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7539815306663513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660871768, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660871768, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660886781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1249026358127594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66682880, "lr": 3.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660901492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11215712130069733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.3995833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660916231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11536817252635956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67010560, "lr": 3.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660931266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11179704964160919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660945990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11770318448543549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67338240, "lr": 3.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660960909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11681848764419556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660976129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12079127132892609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67665920, "lr": 3.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660991308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11989577114582062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661006362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11472642421722412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67993600, "lr": 3.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661021851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1121939942240715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661036864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11582055687904358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68321280, "lr": 3.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661051634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11361641436815262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661066497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12377186864614487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68648960, "lr": 3.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661082182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11350269615650177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661082182, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784661082183, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784661163955, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.756658136844635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784661163956, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784661163956, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784661178967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1194806769490242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68976640, "lr": 3.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661194094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11828625202178955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661209046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11133471876382828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69304320, "lr": 3.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661224115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11339270323514938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661238846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10704877227544785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69632000, "lr": 3.5412500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661253686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11307384073734283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661268585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1165546104311943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69959680, "lr": 3.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661291934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11015934497117996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661306634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10926542431116104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70287360, "lr": 3.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661321113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10690488666296005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661335984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12241749465465546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70615040, "lr": 3.5912500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661351005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10509604960680008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661365847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12082305550575256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70942720, "lr": 3.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661381053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1197979599237442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661381053, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784661381054, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784661465214, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7581610083580017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784661465215, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784661465215, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784661480307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11795932054519653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71270400, "lr": 3.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661495088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11825006455183029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661509925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11518354713916779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71598080, "lr": 3.6412499999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661525432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11489858478307724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661540273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10279609262943268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71925760, "lr": 3.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661555171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10316548496484756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661569926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11815165728330612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72253440, "lr": 3.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661584647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11536318063735962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661599473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10273793339729309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72581120, "lr": 3.6912499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661614346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10218784958124161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661628920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10665574669837952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72908800, "lr": 3.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661643892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11298874020576477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661658771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11098573356866837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73236480, "lr": 3.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661673632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11400215327739716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661673679, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661673679, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661758175, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.758251428604126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661758176, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661758176, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661773090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10458837449550629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73564160, "lr": 3.7412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661788291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.106508769094944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661803530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11462503671646118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73891840, "lr": 3.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661818281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10983408987522125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661833487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10816118866205215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74219520, "lr": 3.7745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661848760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11433222889900208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661863681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11284468322992325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74547200, "lr": 3.7912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661878388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11281328648328781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661893259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11639093607664108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74874880, "lr": 3.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661907694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.110984668135643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661922115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1005333885550499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75202560, "lr": 3.8245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661936661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11345105618238449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661950913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10733485221862793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75530240, "lr": 3.8412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661965319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10565314441919327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661965320, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661965320, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784662052395, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7592753171920776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784662052396, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784662052396, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784662066686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10612494498491287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75857920, "lr": 3.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662081191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11296772956848145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662095844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09609023481607437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76185600, "lr": 3.8745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662110275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10414819419384003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662124696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11256776005029678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76513280, "lr": 3.8912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662139041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10465966165065765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662153549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0893137976527214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76840960, "lr": 3.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662167891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10903589427471161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662182343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10200699418783188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77168640, "lr": 3.9245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662196906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11659587174654007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662211523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1074209213256836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77496320, "lr": 3.9412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662225897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1007145568728447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662240830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10036127269268036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77824000, "lr": 3.9579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662255617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10783921182155609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662255618, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784662255618, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784662342076, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7607845067977905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784662342077, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784662342077, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784662356776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11738960444927216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78151680, "lr": 3.9745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662371549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1117834746837616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662385649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10455185174942017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78479360, "lr": 3.9912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662400916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725414752960205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662415475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09990319609642029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78807040, "lr": 4.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662430373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1027659997344017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662444866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10662689059972763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79134720, "lr": 4.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662459611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793192684650421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662474226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09811818599700928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79462400, "lr": 4.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662489117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.108811154961586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662503731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1059032529592514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79790080, "lr": 4.0579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662518249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10704021900892258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662532753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10859429836273193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80117760, "lr": 4.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662547452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11311278492212296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662547453, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784662547453, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784662632554, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7640342116355896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784662632555, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784662632555, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784662647193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11090485751628876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80445440, "lr": 4.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662661574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10008399188518524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662676408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10248935967683792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80773120, "lr": 4.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662691503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10679817944765091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662706546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11145040392875671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81100800, "lr": 4.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662721609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11408505588769913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662736460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10111231356859207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81428480, "lr": 4.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662751380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10509157925844193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662766419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10752663761377335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81756160, "lr": 4.1579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662781199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10605624318122864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662796071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11259784549474716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82083840, "lr": 4.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662810225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.095668725669384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662825285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09775999933481216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82411520, "lr": 4.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662839962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10704775899648666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662839962, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662839963, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662923798, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7651156187057495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662923799, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662923799, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662938105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09220807254314423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82739200, "lr": 4.2079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662952609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10501605272293091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662966972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1103169322013855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83066880, "lr": 4.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662981592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09801575541496277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662995938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11063049733638763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83394560, "lr": 4.2412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663010380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11110953241586685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663024691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09799806028604507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83722240, "lr": 4.2579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663038790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11105217039585114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663053175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10610555857419968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84049920, "lr": 4.2745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663067396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10494839400053024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663081834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10853607952594757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84377600, "lr": 4.2912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663095896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1055726557970047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663110159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0982198715209961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84705280, "lr": 4.3079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663124690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10674373805522919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663124690, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784663124691, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784663207440, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7653131484985352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784663207440, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784663207441, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784663222290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10523755848407745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85032960, "lr": 4.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663237224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039656326174736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663251767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10554816573858261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85360640, "lr": 4.3412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663266489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10416459292173386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663281180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09827622026205063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85688320, "lr": 4.3579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663295731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10086065530776978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663310598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10613524168729782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86016000, "lr": 4.3745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663324702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09606652706861496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663339740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10863800346851349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86343680, "lr": 4.3912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663354978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10940418392419815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663369737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1168668195605278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86671360, "lr": 4.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663384627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11454236507415771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663399290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11608192324638367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86999040, "lr": 4.4245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663414038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11496567726135254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663414039, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784663414039, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784663496813, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7665496468544006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784663496814, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784663496814, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784663511351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10126189142465591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87326720, "lr": 4.4412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663526218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11649172753095627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663540727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10841399431228638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87654400, "lr": 4.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663555594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10207924246788025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663570198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09170478582382202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87982080, "lr": 4.4745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663584820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09889429807662964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663599172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10425464063882828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88309760, "lr": 4.4912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663613954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10770843923091888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663628408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085488349199295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88637440, "lr": 4.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663642897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10132057964801788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663658042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11388225108385086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88965120, "lr": 4.5245833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663673167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10066469758749008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663688467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11066943407058716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89292800, "lr": 4.5412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663703355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111733578145504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.5495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663703356, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663703356, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663785928, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7667439579963684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663785929, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663785929, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663800944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10858535021543503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89620480, "lr": 4.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663815946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10190250724554062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663830722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11123606562614441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89948160, "lr": 4.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663845600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0921362116932869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663860071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11481524258852005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90275840, "lr": 4.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663874533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1075633093714714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663889960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1115265041589737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90603520, "lr": 4.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663904889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11190645396709442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663920327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12218473851680756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90931200, "lr": 4.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663935480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10525549948215485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663950509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11219148337841034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91258880, "lr": 4.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663965702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12333056330680847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663980857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09961844235658646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91586560, "lr": 4.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663996035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11099602282047272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663996036, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663996036, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784664078284, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.766491174697876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784664078285, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784664078285, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784664093392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10417351871728897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91914240, "lr": 4.6745833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664108580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11911449581384659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664123742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10952464491128922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92241920, "lr": 4.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664139218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10659032315015793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664154206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09748710691928864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92569600, "lr": 4.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664169499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11294085532426834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664184914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10665574669837952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92897280, "lr": 4.7245833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664200218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10766849666833878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664215344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11682768911123276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93224960, "lr": 4.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664230236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09973832219839096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664245538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1247454434633255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93552640, "lr": 4.7579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664260727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10898815095424652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664275671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11765473335981369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93880320, "lr": 4.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664290563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10810019820928574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664290563, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784664290564, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784664373764, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7671238780021667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784664373765, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784664373765, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784664388935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1142413467168808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94208000, "lr": 4.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664403846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1153440997004509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664418714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11091616004705429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94535680, "lr": 4.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664433754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11868502199649811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664448439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11474284529685974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94863360, "lr": 4.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664463106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11948350071907043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664477971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1273801028728485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95191040, "lr": 4.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664492299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12246906757354736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664506821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11650751531124115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95518720, "lr": 4.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664521756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11088525503873825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664536242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10897333920001984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95846400, "lr": 4.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664551026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10907216370105743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664565765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11605720967054367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96174080, "lr": 4.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664580571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11506200581789017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664580572, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664580572, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664662748, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7682990431785583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664662749, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664662749, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664677918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1200690045952797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96501760, "lr": 4.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664692940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792220383882523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664707612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087208241224289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96829440, "lr": 4.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664722952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10802231729030609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664737923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11188586056232452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97157120, "lr": 4.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664752908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1106669157743454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664767814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11463767290115356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97484800, "lr": 4.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664782771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10850857198238373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664797846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10708138346672058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97812480, "lr": 4.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664812371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10107002407312393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664827545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10524415224790573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98140160, "lr": 4.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664842384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09747777134180069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664857133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10489047318696976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98467840, "lr": 5.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664871820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10152829438447952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664871821, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664871822, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664955114, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7685767412185669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664955115, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664955115, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664969785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10993931442499161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98795520, "lr": 5.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664984570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10232682526111603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664999090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09663359820842743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99123200, "lr": 5.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665013575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10898790508508682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665028206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11565335839986801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99450880, "lr": 5.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665042826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10506856441497803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665057383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10597730427980423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99778560, "lr": 5.074583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665072010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10483748465776443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665086526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10244082659482956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100106240, "lr": 5.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665101201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1206861361861229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665115942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11750295758247375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100433920, "lr": 5.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665130630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10251644253730774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665145546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10096515715122223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100761600, "lr": 5.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665159964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12106253206729889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665159964, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784665159965, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784665244310, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7692006826400757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784665244311, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784665244311, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784665259321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11189648509025574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101089280, "lr": 5.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665274131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09741243720054626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665288820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10444527119398117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101416960, "lr": 5.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665303697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10449115186929703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665318335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10985074937343597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101744640, "lr": 5.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665333409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10631248354911804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.182916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665348017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071184054017067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102072320, "lr": 5.191249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665362729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11280123144388199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665377853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10528392344713211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102400000, "lr": 5.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665392839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084434762597084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665407638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10562460869550705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102727680, "lr": 5.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665422481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11581757664680481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665437131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10706445574760437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103055360, "lr": 5.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665451912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10283046215772629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665451913, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784665451913, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784665535011, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7696719765663147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784665535013, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784665535013, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784665549736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1004445031285286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103383040, "lr": 5.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665564735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09530457854270935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665579443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09644866734743118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103710720, "lr": 5.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665594376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09748273342847824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.282916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665608952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09694258123636246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104038400, "lr": 5.291249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665623732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09542427211999893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665638376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10048892349004745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104366080, "lr": 5.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665653189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09717489033937454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665668235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10324341058731079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104693760, "lr": 5.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665683053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10954219102859497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665698152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10517145693302155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105021440, "lr": 5.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665712897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1043163388967514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665727763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11453254520893097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105349120, "lr": 5.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665743236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11198519915342331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665743237, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665743237, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665825621, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7704042792320251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665825621, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665825622, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665840636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09912067651748657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105676800, "lr": 5.374583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665855599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10789385437965393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.382916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665870773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1102660521864891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106004480, "lr": 5.391249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665885496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10171958059072495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665900353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10143306106328964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106332160, "lr": 5.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665915234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10818132013082504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665930015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11146466434001923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106659840, "lr": 5.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665944840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10685279965400696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665960064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10302173346281052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106987520, "lr": 5.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665974947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1016290932893753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665989814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1003422737121582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107315200, "lr": 5.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666004574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10306209325790405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666019325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11039214581251144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107642880, "lr": 5.474583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666033989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11488932371139526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.482916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666033990, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784666033990, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784666117318, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.770160436630249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784666117319, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784666117320, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784666132304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11123161762952805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107970560, "lr": 5.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666146855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10750316083431244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666161730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09172645211219788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108298240, "lr": 5.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666176298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10357411950826645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666191097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10705558955669403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108625920, "lr": 5.524583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666205652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10848674923181534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666220515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11107546836137772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108953600, "lr": 5.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666235542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10421902686357498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666250459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10021824389696121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109281280, "lr": 5.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666265213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10492802411317825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666280290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10381156206130981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109608960, "lr": 5.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666294980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10919744521379471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666310018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10643143206834793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109936640, "lr": 5.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666325187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1094140037894249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666325187, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784666325188, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784666410700, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7711884379386902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784666410701, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784666410701, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784666425716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10323208570480347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110264320, "lr": 5.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666440774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1047891303896904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666455594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10452781617641449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110592000, "lr": 5.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666470148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10265380889177322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666485517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10835323482751846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110919680, "lr": 5.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666500263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10144633054733276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666515174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09906265884637833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111247360, "lr": 5.657916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666530444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10247892886400223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.666250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666545551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11229700595140457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111575040, "lr": 5.674583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666560652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10515496879816055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666575733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10410546511411667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111902720, "lr": 5.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666590589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10322032868862152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666605708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1051945835351944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112230400, "lr": 5.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666620638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.107624851167202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666620639, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666620640, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666703704, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7712177634239197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666703705, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666703705, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666718702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1089906394481659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112558080, "lr": 5.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666733821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10219824314117432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666748499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10451081395149231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112885760, "lr": 5.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666763199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10140541940927505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666778115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10544466972351074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113213440, "lr": 5.757916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666792634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11265996843576431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.766250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666807437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10172293335199356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113541120, "lr": 5.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666821878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11035771667957306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666836549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11346019059419632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113868800, "lr": 5.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666851600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10697108507156372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666866420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10340877622365952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114196480, "lr": 5.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666881139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10492032021284103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666896012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10589563846588135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114524160, "lr": 5.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666910938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09935091435909271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666910938, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666910939, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666995481, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7715105414390564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666995481, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666995482, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784667010125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10263223946094513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114851840, "lr": 5.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667024829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09992978721857071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667039565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10712939500808716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115179520, "lr": 5.857916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667053837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10117913037538528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667069372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10899879038333893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115507200, "lr": 5.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667084248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1102331131696701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667099437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1076979786157608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115834880, "lr": 5.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667114807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11592614650726318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667129802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1147426888346672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116162560, "lr": 5.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667144607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10102909058332443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667159510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10439550876617432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116490240, "lr": 5.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667174344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1035861000418663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667189014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10574014484882355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116817920, "lr": 5.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667203749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10740181058645248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667203750, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784667203750, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784667288200, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.771694004535675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784667288201, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784667288201, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784667303054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1023353636264801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117145600, "lr": 5.957916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667317820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10138408839702606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667333056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10956613719463348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117473280, "lr": 5.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667348038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10546864569187164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667362778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09475858509540558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117800960, "lr": 5.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667378049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1135205626487732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667392594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1020771935582161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118128640, "lr": 6.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667407370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10827904939651489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667422375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10343454778194427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118456320, "lr": 6.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667437486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10446037352085114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667452283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11026474088430405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118784000, "lr": 6.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667467336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11467533558607101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667482153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10686740279197693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119111680, "lr": 6.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667497326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10335944592952728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667497327, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784667497327, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784667580245, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7719758749008179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784667580246, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784667580246, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784667595373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11351846903562546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119439360, "lr": 6.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667610388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10971075296401978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667625584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11004337668418884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119767040, "lr": 6.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667641095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10785451531410217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667655750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10185644030570984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120094720, "lr": 6.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667670321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10828061401844025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667686136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11805924773216248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120422400, "lr": 6.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667701281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10935915261507034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667716314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11662132292985916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120750080, "lr": 6.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667731621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12275488674640656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667746548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792817920446396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121077760, "lr": 6.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667761453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11127591133117676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667776555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11624860018491745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121405440, "lr": 6.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667791460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1104358658194542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667791460, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667791461, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667874026, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7717100977897644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667874026, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667874026, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667889084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11402764171361923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121733120, "lr": 6.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667904190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10983198881149292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667919085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11203627288341522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122060800, "lr": 6.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667934001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10676795244216919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667948736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10446267575025558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122388480, "lr": 6.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667963750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11453354358673096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667978573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10743757337331772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122716160, "lr": 6.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667993406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11545392870903015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668008608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11222202330827713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123043840, "lr": 6.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668023815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12077335268259048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668039309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10585561394691467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123371520, "lr": 6.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668054449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11792152374982834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668069477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10870641469955444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123699200, "lr": 6.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668084723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1196366474032402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668084724, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784668084724, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784668167405, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7723491191864014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784668167406, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784668167406, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784668182471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12032903730869293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124026880, "lr": 6.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668197530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10848748683929443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668212339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11990504711866379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124354560, "lr": 6.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668226975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1140122041106224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668241797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10829805582761765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124682240, "lr": 6.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668257267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11564009636640549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668272122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11308622360229492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125009920, "lr": 6.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668287532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1176062822341919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668303087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11374222487211227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125337600, "lr": 6.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668318306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10619503259658813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668333589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11432678252458572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125665280, "lr": 6.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668348918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10629469156265259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668364159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11714635789394379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125992960, "lr": 6.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668379747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10595451295375824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668379748, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784668379748, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784668463861, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7725483775138855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784668463861, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784668463862, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784668479274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11029566079378128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126320640, "lr": 6.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668494947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12126728147268295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668510423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11037356406450272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126648320, "lr": 6.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668525774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11449411511421204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668540902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11647981405258179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126976000, "lr": 6.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668556371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11000800132751465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668572072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085960790514946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127303680, "lr": 6.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668587510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11040613800287247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.482916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668603236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1128082349896431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127631360, "lr": 6.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668618565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1132301390171051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668634041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11050301045179367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127959040, "lr": 6.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668649751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11148319393396378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668665147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11309029161930084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128286720, "lr": 6.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668680346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10938755422830582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668680347, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668680347, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668763777, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7727013230323792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668763778, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668763778, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668778895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11812707781791687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128614400, "lr": 6.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668794275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11081554740667343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668809004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11094754189252853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128942080, "lr": 6.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668823752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10323517769575119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668838767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10600966215133667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129269760, "lr": 6.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668853912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11199293285608292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668868675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10895495116710663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129597440, "lr": 6.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668883486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10901045054197311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668898340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0989181324839592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129925120, "lr": 6.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668912912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10228480398654938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668927616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10575974732637405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130252800, "lr": 6.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668942163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10509226471185684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668957121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11002559959888458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130580480, "lr": 6.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668971859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1168760359287262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668971860, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668971860, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784669055609, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7731391191482544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784669055610, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784669055611, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784669070455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11523551493883133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130908160, "lr": 6.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669085334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1063557043671608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669100022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10146715492010117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131235840, "lr": 6.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669114919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09283459931612015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669129612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1135353296995163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131563520, "lr": 6.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669144233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10571283102035522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669159139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10303918272256851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131891200, "lr": 6.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669174304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11288771778345108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669189247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725066065788269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132218880, "lr": 6.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669204253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10436973720788956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669219292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10946916788816452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132546560, "lr": 6.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669234525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11311660706996918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669249793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10849756002426147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132874240, "lr": 6.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669264846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10764004290103912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669264847, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784669264847, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784669348594, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7731704115867615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784669348595, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784669348595, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784669363907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11053189635276794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133201920, "lr": 6.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669378980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1043868437409401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669393626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11109109967947006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133529600, "lr": 6.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669408214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09964701533317566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669423471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11607228219509125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133857280, "lr": 6.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669438360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11070858687162399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669453150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.104305699467659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134184960, "lr": 6.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669468333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10817565023899078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669483261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10858042538166046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134512640, "lr": 6.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669497873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10912565886974335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669512738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11099584400653839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134840320, "lr": 6.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669527403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0994933694601059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669541833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10824538767337799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135168000, "lr": 6.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669556378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08761264383792877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669556378, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669556379, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669638197, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7733290791511536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669638198, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669638198, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669652983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09594782441854477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135495680, "lr": 6.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669667740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1110028326511383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669682484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10935529321432114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135823360, "lr": 6.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669697256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10209225118160248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669712138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10247208923101425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136151040, "lr": 6.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669727122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09080001711845398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669742008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10369621217250824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136478720, "lr": 6.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669756693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09844737499952316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669771167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10395567119121552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136806400, "lr": 6.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669786220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10727105289697647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669801365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11118700355291367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137134080, "lr": 6.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669816400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10895995795726776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669831317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10647457093000412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137461760, "lr": 6.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669846366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10429994761943817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669846367, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669846368, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669929637, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7735073566436768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669929638, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669929638, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669944687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09705324470996857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137789440, "lr": 7.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669959984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10535910725593567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669975042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09609873592853546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138117120, "lr": 7.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669990192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10340158641338348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670005386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0988369807600975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138444800, "lr": 7.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670020171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10156817734241486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670034868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10953794419765472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138772480, "lr": 7.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670050068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10200905054807663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.066249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670064787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10925847291946411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139100160, "lr": 7.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670079716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11028218269348145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670094913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10697904974222183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139427840, "lr": 7.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670110016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10529898852109909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670124927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11074420064687729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139755520, "lr": 7.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670139763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10896743088960648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670139764, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784670139764, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784670225760, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7736728191375732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784670225761, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784670225761, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784670240541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10763110220432281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140083200, "lr": 7.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670255236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10591668635606766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670270042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10413218289613724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140410880, "lr": 7.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670284989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10061998665332794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670300129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723122954368591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140738560, "lr": 7.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670315331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09678810834884644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670330998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10558079183101654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141066240, "lr": 7.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670346014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10433819890022278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670360988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11044807732105255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141393920, "lr": 7.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670376288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0994785726070404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670391057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0980367586016655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141721600, "lr": 7.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670406046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10306376963853836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670421210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09150764346122742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142049280, "lr": 7.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670436525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10811539739370346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670436526, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784670436526, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784670521828, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738646864891052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784670521829, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784670521829, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784670537001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10035301744937897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142376960, "lr": 7.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670552364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10173335671424866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670567400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09920316189527512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142704640, "lr": 7.257916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670582769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11027022451162338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670598101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10178069770336151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143032320, "lr": 7.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670613413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10309427231550217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670628554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10310270637273788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143360000, "lr": 7.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670643596, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10610523819923401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670658722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10174015909433365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143687680, "lr": 7.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670673404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11457791924476624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670688725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10157460719347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144015360, "lr": 7.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670704185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10866563767194748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670719341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10608558356761932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144343040, "lr": 7.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670734572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10739417374134064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670734573, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670734573, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670820010, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773925244808197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670820011, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670820012, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670835272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10739009827375412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144670720, "lr": 7.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670850478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09876921027898788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670865414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10708044469356537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144998400, "lr": 7.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670880705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10743719339370728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670895983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10348626226186752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145326080, "lr": 7.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670910625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10743443667888641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670925425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0975453108549118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145653760, "lr": 7.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670940458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09827112406492233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670955277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1057027131319046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145981440, "lr": 7.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670970065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10539416968822479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670985032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10318400710821152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146309120, "lr": 7.441250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670999815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11071106791496277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671014707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725880414247513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146636800, "lr": 7.457916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671029704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10074993222951889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671029705, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784671029706, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784671114776, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7741949558258057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784671114777, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784671114778, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784671129601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10440068691968918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146964480, "lr": 7.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671144415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10580072551965714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671159389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09506867825984955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147292160, "lr": 7.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671174661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09935220330953598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671189491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10336247086524963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147619840, "lr": 7.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671204866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10042334347963333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671219853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10714592784643173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147947520, "lr": 7.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671234914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11004868149757385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671249917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10821831226348877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148275200, "lr": 7.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671264812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10085000842809677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671279998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10349234193563461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148602880, "lr": 7.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671294740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09942512959241867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671309902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11228521913290024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148930560, "lr": 7.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671324673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11219429969787598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671324674, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784671324675, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784671411511, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742371559143066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784671411512, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784671411513, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784671426034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10622715204954147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149258240, "lr": 7.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671441622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10516694188117981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671456651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10634972900152206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149585920, "lr": 7.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671471910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11099711805582047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149749760, "lr": 7.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671487322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10687929391860962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149913600, "lr": 7.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671502676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10892698913812637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671517687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10868561267852783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150241280, "lr": 7.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671532877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09819488972425461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150405120, "lr": 7.649583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671548051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10761020332574844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150568960, "lr": 7.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671562983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1035892516374588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671578116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10896500945091248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150896640, "lr": 7.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671592958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10350484400987625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151060480, "lr": 7.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671608220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10674640536308289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151224320, "lr": 7.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671623249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09689601510763168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671623297, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671623298, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671709221, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774358868598938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671709222, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671709222, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671724310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09966647624969482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151552000, "lr": 7.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671739460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10561797767877579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151715840, "lr": 7.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671754635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1023624837398529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151879680, "lr": 7.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671769901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10472842305898666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671785117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10856954753398895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152207360, "lr": 7.741249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671800185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12469470500946045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152371200, "lr": 7.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671815374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10963567346334457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152535040, "lr": 7.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671830526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09943906217813492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671845859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11330919712781906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152862720, "lr": 7.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671861073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11253644526004791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153026560, "lr": 7.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671876404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11509773880243301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153190400, "lr": 7.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671891696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09941034764051437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671907204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10726025700569153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153518080, "lr": 7.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671922539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10501381754875183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153681920, "lr": 7.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671922584, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671922585, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784672008766, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774552583694458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784672008767, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784672008768, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784672024291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09907767921686172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153845760, "lr": 7.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672039608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09857383370399475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.832916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672054826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11298898607492447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154173440, "lr": 7.841249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672070295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11877603083848953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154337280, "lr": 7.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672085440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11427938938140869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154501120, "lr": 7.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672100246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11146964132785797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672115926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10758944600820541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154828800, "lr": 7.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672131314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12016630172729492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154992640, "lr": 7.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672146563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10598354786634445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155156480, "lr": 7.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672162139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10881493985652924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672177607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11234850436449051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155484160, "lr": 7.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672192863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11088250577449799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155648000, "lr": 7.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672208232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11366929858922958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155811840, "lr": 7.924583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672223534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11203683912754059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.932916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672223535, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784672223535, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784672307234, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746077179908752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784672307235, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784672307235, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784672322611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1074453815817833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156139520, "lr": 7.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672337984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11079395562410355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156303360, "lr": 7.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672353484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1148681640625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156467200, "lr": 7.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672368793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11083238571882248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672384225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10898153483867645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156794880, "lr": 7.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672399741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10844742506742477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156958720, "lr": 7.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672415169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10172275453805923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157122560, "lr": 7.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672430410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11249790340662003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672445810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12111125141382217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157450240, "lr": 8.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672461282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11046586185693741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157614080, "lr": 8.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672476568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10888776928186417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157777920, "lr": 8.024583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672492149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10503049194812775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.032916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672507624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10442622750997543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158105600, "lr": 8.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672523123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10936161875724792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158269440, "lr": 8.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672523124, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784672523124, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784672606403, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7748522162437439, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784672606404, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784672606404, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784672621685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1145123615860939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158433280, "lr": 8.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672637198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12009026110172272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672652705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.114539735019207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158760960, "lr": 8.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672668236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11232121288776398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158924800, "lr": 8.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672683736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11119571328163147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159088640, "lr": 8.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672698952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12166478484869003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672714460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11225560307502747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159416320, "lr": 8.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672729376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10928095132112503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159580160, "lr": 8.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672745393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11097585409879684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159744000, "lr": 8.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672760906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09902762621641159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672776312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11022423207759857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160071680, "lr": 8.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672792308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1098194345831871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160235520, "lr": 8.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672808202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10488779097795486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160399360, "lr": 8.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672823830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11127091944217682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672823830, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672823831, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672908186, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7749757766723633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672908187, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672908187, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672923931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11357761174440384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160727040, "lr": 8.174583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672939771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12933164834976196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160890880, "lr": 8.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672955277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10861459374427795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161054720, "lr": 8.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672970414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1128029078245163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672985344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11406569182872772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161382400, "lr": 8.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673000654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11128710955381393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161546240, "lr": 8.216250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673015984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11429277807474136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161710080, "lr": 8.224583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673031489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11405420303344727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673046704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10632790625095367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162037760, "lr": 8.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673061922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10871008783578873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162201600, "lr": 8.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673077403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12913395464420319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162365440, "lr": 8.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673092654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11328411102294922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673108455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1132471114397049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162693120, "lr": 8.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673124211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11656660586595535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162856960, "lr": 8.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673124212, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784673124212, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784673209086, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752427458763123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784673209087, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784673209087, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784673224349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09740118682384491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163020800, "lr": 8.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673240022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11206184327602386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673255530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1227216050028801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163348480, "lr": 8.307916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673271106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1065693199634552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163512320, "lr": 8.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673286605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11012135446071625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163676160, "lr": 8.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673302052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10804464668035507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673317284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1125197634100914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164003840, "lr": 8.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673332294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1123235747218132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164167680, "lr": 8.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673347665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10348530113697052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164331520, "lr": 8.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673362497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11794713884592056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673377400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10846291482448578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164659200, "lr": 8.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673392599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11153062433004379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164823040, "lr": 8.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673407934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0931161642074585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164986880, "lr": 8.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673422911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1042311042547226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673422912, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784673422912, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784673505961, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752667665481567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784673505961, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784673505962, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784673521075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10274146497249603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165314560, "lr": 8.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673536039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09702027589082718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165478400, "lr": 8.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673550853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11529821902513504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165642240, "lr": 8.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673565669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10854652523994446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673580437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10975764691829681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165969920, "lr": 8.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673595518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10441706329584122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166133760, "lr": 8.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673610448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10833382606506348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166297600, "lr": 8.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673625627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11400409787893295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673640643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1001153290271759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166625280, "lr": 8.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673655928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09661642462015152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166789120, "lr": 8.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673670836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09791809320449829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166952960, "lr": 8.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673685694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09755630791187286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673700468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10288963466882706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167280640, "lr": 8.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673715469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10149537771940231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167444480, "lr": 8.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673715470, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673715470, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673798467, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752794623374939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673798468, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673798468, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673813359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10354232043027878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167608320, "lr": 8.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673828526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10353801399469376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673843681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10430397093296051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167936000, "lr": 8.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673858862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10755624622106552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168099840, "lr": 8.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673874022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10299398750066757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168263680, "lr": 8.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673889346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.100792795419693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673904814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11013209074735641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168591360, "lr": 8.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673919915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10568073391914368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168755200, "lr": 8.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673935111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723371058702469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168919040, "lr": 8.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673950164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10602562874555588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673965461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10481980443000793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169246720, "lr": 8.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673980284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1089240163564682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169410560, "lr": 8.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673995055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10621747374534607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169574400, "lr": 8.624583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674010414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1137860044836998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674010468, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784674010468, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784674094435, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7754996418952942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784674094436, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784674094437, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784674109294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085377112030983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169902080, "lr": 8.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674124417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11214891821146011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170065920, "lr": 8.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674139767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10767322778701782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170229760, "lr": 8.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674155077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10777486115694046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674170079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10060925781726837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170557440, "lr": 8.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674185383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10285207629203796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170721280, "lr": 8.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674200646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09631936252117157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170885120, "lr": 8.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674215509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10877624899148941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674230239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10017919540405273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171212800, "lr": 8.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674245062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10278210043907166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171376640, "lr": 8.716249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674260222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10193649679422379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171540480, "lr": 8.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674274962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10344456881284714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674289698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09997213631868362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171868160, "lr": 8.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674304572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10426849126815796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172032000, "lr": 8.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674304624, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784674304625, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784674390085, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7754709720611572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784674390086, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784674390086, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784674405076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09839719533920288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172195840, "lr": 8.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674420006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10760672390460968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674435144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0998426228761673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172523520, "lr": 8.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674449998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10402244329452515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172687360, "lr": 8.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674464781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09421024471521378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172851200, "lr": 8.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674480076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10710486769676208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674495470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10435770452022552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173178880, "lr": 8.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674510670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10422546416521072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173342720, "lr": 8.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674526096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10530197620391846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173506560, "lr": 8.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674541235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11204942315816879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674556676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09690681099891663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173834240, "lr": 8.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674572152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11176567524671555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173998080, "lr": 8.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674587518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11242222040891647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174161920, "lr": 8.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674602509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1019306480884552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674602554, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784674602555, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784674688179, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7756123542785645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784674688180, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784674688180, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784674703140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09839025884866714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174489600, "lr": 8.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674718428, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10752486437559128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174653440, "lr": 8.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674733554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10198871046304703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174817280, "lr": 8.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674748141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1069134920835495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674763240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10089421272277832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175144960, "lr": 8.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674778101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10909280180931091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175308800, "lr": 8.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674793002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10080993920564651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175472640, "lr": 8.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674808257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09956123679876328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674823395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10931626707315445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175800320, "lr": 8.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674838664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10991813987493515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175964160, "lr": 8.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674853658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10061269253492355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176128000, "lr": 8.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674868699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09499377757310867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674883602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09677860885858536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176455680, "lr": 8.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674898745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10632886737585068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176619520, "lr": 8.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674898746, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674898746, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674984086, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7758821249008179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674984087, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674984088, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674999186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10338009893894196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176783360, "lr": 8.991249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675014323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11420866847038269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675029510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10152126848697662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177111040, "lr": 9.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675044546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10572750866413116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177274880, "lr": 9.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675059440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11149033159017563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177438720, "lr": 9.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675074369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10096921771764755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.032916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675089192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10990338772535324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177766400, "lr": 9.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675103830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10203257203102112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177930240, "lr": 9.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675118565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10790542513132095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178094080, "lr": 9.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675133216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10258839279413223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675148080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09866001456975937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178421760, "lr": 9.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675163477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11242654174566269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178585600, "lr": 9.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675178609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11975989490747452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178749440, "lr": 9.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675193708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10986080765724182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675193709, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784675193710, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784675277383, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7763038277626038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784675277384, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784675277384, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784675292597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11662010103464127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179077120, "lr": 9.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675307518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10126511752605438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179240960, "lr": 9.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675322925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1003381535410881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179404800, "lr": 9.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675338114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10146120935678482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.132916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675353212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10676675289869308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179732480, "lr": 9.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675368259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10003376752138138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179896320, "lr": 9.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675383271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10083489120006561, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180060160, "lr": 9.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675398396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10677052289247513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675413044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10619533807039261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180387840, "lr": 9.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675427556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09495513141155243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180551680, "lr": 9.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675443116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10488119721412659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180715520, "lr": 9.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675458171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10210099816322327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675473417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09441406279802322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181043200, "lr": 9.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675488802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10042594373226166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181207040, "lr": 9.216249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675488803, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784675488803, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784675571657, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7762972712516785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784675571657, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784675571658, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784675586884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11140605062246323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181370880, "lr": 9.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675602198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11401738226413727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675617248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1005912497639656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181698560, "lr": 9.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675632325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10774566233158112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181862400, "lr": 9.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675647203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10376131534576416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182026240, "lr": 9.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675661790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1041513979434967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675676452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10662417113780975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182353920, "lr": 9.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675691425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087929755449295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182517760, "lr": 9.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675706265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09903056919574738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182681600, "lr": 9.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675720816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09812669456005096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675735641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11242415010929108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183009280, "lr": 9.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675758683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0985153391957283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183173120, "lr": 9.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675773728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10283553600311279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183336960, "lr": 9.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675790980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1032082736492157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675790981, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675790981, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675873549, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7766724824905396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675873550, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675873550, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675888652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.102554552257061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183664640, "lr": 9.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675903398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10868321359157562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183828480, "lr": 9.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675918067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11250212043523788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183992320, "lr": 9.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675933410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09488393366336823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675948885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10475891083478928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184320000, "lr": 9.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675964122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10797204077243805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184483840, "lr": 9.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675979297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11639493703842163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184647680, "lr": 9.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675994292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10128704458475113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676009565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1057250127196312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184975360, "lr": 9.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676025274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11094112694263458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185139200, "lr": 9.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676040509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09819766879081726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185303040, "lr": 9.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676055688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10858774185180664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676070941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09876522421836853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185630720, "lr": 9.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676085784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1026102602481842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185794560, "lr": 9.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676085785, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784676085786, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784676166819, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7766553163528442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784676166820, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784676166820, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784676182492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11029969155788422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185958400, "lr": 9.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676197480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10494790226221085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676212259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09250514209270477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186286080, "lr": 9.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676227895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1006196066737175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186449920, "lr": 9.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676243220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10905735939741135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186613760, "lr": 9.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676258589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1110759899020195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676274384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0963239073753357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186941440, "lr": 9.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676289933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1049552857875824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187105280, "lr": 9.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676305418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10655958205461502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187269120, "lr": 9.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676320617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10007711499929428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676335888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10483012348413467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187596800, "lr": 9.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676351069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10516394674777985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187760640, "lr": 9.549583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676366236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11157233268022537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187924480, "lr": 9.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676381627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11552007496356964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676381628, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784676381629, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784676462689, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7764861583709717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784676462690, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784676462690, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784676477758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11011183261871338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188252160, "lr": 9.574583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676493180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10507632046937943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188416000, "lr": 9.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676508232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10512138903141022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188579840, "lr": 9.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676523514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10890284925699234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676538484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10251019150018692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188907520, "lr": 9.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676553781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0976414829492569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189071360, "lr": 9.616249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676568893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10386304557323456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189235200, "lr": 9.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676583858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11260077357292175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676598992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10133678466081619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189562880, "lr": 9.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676614107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09619663655757904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189726720, "lr": 9.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676629083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10542993992567062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189890560, "lr": 9.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676644418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11253558099269867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.666249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676659796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10255686193704605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190218240, "lr": 9.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676674785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11880583316087723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190382080, "lr": 9.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676674785, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676674786, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676758965, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7769604921340942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676758966, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676758966, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676774259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10123417526483536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190545920, "lr": 9.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676789159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09811395406723022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676804323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10602284967899323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190873600, "lr": 9.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676819710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11242923885583878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191037440, "lr": 9.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676835110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11838633567094803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191201280, "lr": 9.724583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676850318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09676345437765121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676865605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10574407875537872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191528960, "lr": 9.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676880486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10201616585254669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191692800, "lr": 9.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676895891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10206317156553268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191856640, "lr": 9.757916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676911137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10460135340690613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676925731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10665298253297806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192184320, "lr": 9.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676941450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10991105437278748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192348160, "lr": 9.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676956715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1002788320183754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192512000, "lr": 9.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676972415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11607842892408371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.799583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676972415, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676972416, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784677054888, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7771864533424377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784677054889, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784677054889, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784677070427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11255187541246414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192839680, "lr": 9.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677086159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11174723505973816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193003520, "lr": 9.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677101749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10698425769805908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193167360, "lr": 9.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677117533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1214948520064354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677133036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11576981842517853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193495040, "lr": 9.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677148731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10312933474779129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193658880, "lr": 9.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677164434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12004241347312927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193822720, "lr": 9.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677179970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11442195624113083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.866249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677195724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10565286874771118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194150400, "lr": 9.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677211523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10777423530817032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194314240, "lr": 9.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677227470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10727937519550323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194478080, "lr": 9.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677243176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10690651088953018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677259139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10776656866073608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194805760, "lr": 9.907916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677274563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11611852049827576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194969600, "lr": 9.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677274564, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784677274565, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784677356761, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7778541445732117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784677356761, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784677356762, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784677372603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11269893497228622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195133440, "lr": 9.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677388181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10534816235303879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677404000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10322686284780502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195461120, "lr": 9.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677419908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11689040064811707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195624960, "lr": 9.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677435730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10927785933017731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195788800, "lr": 9.957916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677451455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10664503276348114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677467342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10614991188049316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196116480, "lr": 9.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677483131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11420412361621857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196280320, "lr": 9.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677499370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1184493824839592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196444160, "lr": 9.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677515217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10584299266338348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.999583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677530835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1182144358754158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196771840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677546575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11299356073141098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196935680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677561865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10840726643800735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197099520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677577653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11581984907388687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677577654, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784677577654, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784677659945, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7777467966079712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784677659946, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784677659946, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784677675179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1013793870806694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197427200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677690728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11114591360092163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197591040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677706069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11519380658864975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197754880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677721446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10206017643213272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677737044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10061786323785782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198082560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677752718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10903353989124298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198246400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677768129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0988330990076065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198410240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677783310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10721451044082642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677798526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10208886861801147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198737920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677813589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10574379563331604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198901760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677828880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11494816839694977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199065600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677844080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1111842468380928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677859159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11758257448673248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199393280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677874500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11378920078277588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199557120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677874501, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677874501, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677956197, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7780810594558716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677956198, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677956198, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677971462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11359893530607224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199720960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677986943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10769675672054291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678002152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11175380647182465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200048640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678017515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11731504648923874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200212480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678032733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11247068643569946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200376320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678048204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11861444264650345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678063785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10027984529733658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200704000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678079140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10827679187059402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200867840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678094682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10870544612407684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201031680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678110201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1099226176738739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678125792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1196284368634224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201359360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678141279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11341071128845215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201523200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678156781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1053113266825676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201687040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678172490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11271516978740692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678172491, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784678172491, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784678256449, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7783223986625671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784678256451, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784678256451, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784678271383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10506192594766617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202014720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678286690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11106657981872559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202178560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678301686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11880674213171005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202342400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678317204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732235014438629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678331979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10612025111913681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202670080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678346983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10194278508424759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202833920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678362694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10703060030937195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202997760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678377671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11085426807403564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678392827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11422624439001083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203325440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678407786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09439211338758469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203489280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678422949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11763293296098709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203653120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678438052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10145939886569977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678452754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09916466474533081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203980800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678467509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10867501050233841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204144640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678467510, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784678467510, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784678551123, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7777716517448425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784678551124, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784678551124, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784678566057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10353630781173706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204308480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678581367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0944032371044159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678596573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09722398221492767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204636160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678611599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10829444974660873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204800000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678626752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10959458351135254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204963840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678641865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11292397230863571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678656849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09678759425878525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205291520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678672014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1067994087934494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205455360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678687022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10726537555456161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205619200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678701948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11145429313182831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678716895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09960593283176422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205946880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678732330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10030815005302429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206110720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678747496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11159928143024445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206274560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678762359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0985998809337616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678762360, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678762361, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678845616, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7781968712806702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678845617, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678845617, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678860864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10628467798233032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206602240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678875834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09645366668701172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206766080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678891016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10542475432157516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206929920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678906386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09636390209197998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678921953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10329315811395645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207257600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678937304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11270583420991898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207421440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678952702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081763282418251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207585280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678967622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09610337764024734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678982663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11605404317378998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207912960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678997769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10436414927244186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208076800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679012501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10332357883453369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208240640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679027985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10197797417640686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679042703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10278233885765076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208568320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679057681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1021430566906929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208732160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679057681, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784679057682, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784679142129, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7784578800201416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784679142130, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784679142130, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784679157321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10187067091464996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208896000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679172524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09984029084444046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679187804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10311678797006607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209223680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679202682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1028367206454277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209387520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679218142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09907148778438568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209551360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679233202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10262982547283173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679247987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09637533128261566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209879040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679262821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0991363674402237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210042880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679278111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10390663892030716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210206720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679293203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10911276191473007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679308208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10480337589979172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210534400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679323268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10770425200462341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210698240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679338552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10260921716690063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210862080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679353437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09656118601560593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679353438, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784679353438, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784679437449, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7784078121185303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784679437450, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784679437450, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784679452765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09580580145120621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211189760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679467600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10023661702871323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211353600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679482554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09785361588001251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211517440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679497221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09987707436084747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679512157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10433676838874817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211845120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679527776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09610987454652786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212008960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679542724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10323725640773773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212172800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679557945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09868337959051132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679572922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0983649417757988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212500480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679588147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10577677935361862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212664320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679603447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0930241197347641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212828160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679619010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10821833461523056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679634395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10570856928825378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213155840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679650022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11046188324689865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213319680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679650023, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784679650023, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784679732921, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7787220478057861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784679732922, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784679732922, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784679748156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1055678129196167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213483520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679763546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10207225382328033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679778802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10617764294147491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213811200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679794532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10363797098398209, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213975040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679809440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09720319509506226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214138880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679824397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10495387017726898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679839818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0961461290717125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214466560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679854837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11460188031196594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214630400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679869896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09856100380420685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214794240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679885457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0908965989947319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679900858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10059625655412674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215121920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679916051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10558567941188812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215285760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679931194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10363882035017014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215449600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679946363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10908914357423782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679946364, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679946364, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784680030020, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7789427638053894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784680030021, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784680030021, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784680045355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10184445977210999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215777280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680060244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10658824443817139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215941120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680075167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09809127449989319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216104960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680090233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10048490017652512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680105519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10089654475450516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216432640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680120393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09549888968467712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216596480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680135383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10453785955905914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216760320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680150562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1034712865948677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680165334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10090667754411697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217088000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680180494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11044096946716309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217251840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680195181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09823154658079147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217415680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680210398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10107409954071045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217579520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680225521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10064695030450821, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217743360, "lr": 1e-06}} diff --git a/recommendation_v4/rcp_logs/gbs_8192/seed681082816.log b/recommendation_v4/rcp_logs/gbs_8192/seed681082816.log new file mode 100644 index 000000000..ba32bfa4e --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_8192/seed681082816.log @@ -0,0 +1,1916 @@ +:::MLLOG {"namespace": "", "time_ms": 1784651299073, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784651299074, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784651319338, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784651319340, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784651319340, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784651319340, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784651319340, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784651319340, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 8192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784651319340, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784651319340, "event_type": "POINT_IN_TIME", "key": "seed", "value": 681082816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784651319340, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784651319340, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784651319340, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784651319341, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784651319341, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784651319341, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784651990338, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784651990341, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784651990655, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784652116205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13771222531795502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840, "lr": 7.916666666666666e-10}} +:::MLLOG {"namespace": "", "time_ms": 1784652128040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14031238853931427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.6249999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652140285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13862194120883942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 491520, "lr": 2.458333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652152734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13962367177009583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652165410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.139175146818161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 819200, "lr": 4.125e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652178015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13888496160507202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652190594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13970795273780823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1146880, "lr": 5.7916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652203332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13822603225708008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.625e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652216107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13930273056030273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1474560, "lr": 7.458333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652228841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13943462073802948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.291666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652241690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13887378573417664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1802240, "lr": 9.124999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652255483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13867473602294922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652268607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13935598731040955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2129920, "lr": 1.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652282910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137485072016716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652282911, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652282912, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652363758, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49788910150527954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652363759, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652363759, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652376241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13938696682453156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2457600, "lr": 1.2458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652388939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13919930160045624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652402121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13911646604537964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2785280, "lr": 1.4124999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652415602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13970261812210083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4958333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652428755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1395055055618286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3112960, "lr": 1.5791666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652441755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13960090279579163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652454311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1395157128572464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3440640, "lr": 1.7458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652467169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1395343840122223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652480530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1395704597234726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3768320, "lr": 1.9124999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652493719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1391899734735489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9958333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652506942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389874517917633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4096000, "lr": 2.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652520079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13899360597133636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.1624999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652533063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13915914297103882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4423680, "lr": 2.2458333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652546443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13897249102592468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3291666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652546443, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652546444, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652627704, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.498083233833313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652627705, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652627705, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652641039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13905125856399536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4751360, "lr": 2.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652654441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13898025453090668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652667796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1391158550977707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5079040, "lr": 2.5791666666666668e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652680962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1393146514892578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652693933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388390213251114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5406720, "lr": 2.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652706889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13885082304477692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652719908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13880310952663422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5734400, "lr": 2.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652733421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1390085071325302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652746693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138760045170784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6062080, "lr": 3.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652759910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13847298920154572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652773312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13863687217235565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6389760, "lr": 3.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652786342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13854920864105225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652799903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385686844587326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6717440, "lr": 3.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652813059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13870517909526825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652813060, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784652813060, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784652895118, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49833664298057556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784652895120, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784652895120, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784652908122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382170468568802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7045120, "lr": 3.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652921491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13828502595424652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652934749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13818231225013733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7372800, "lr": 3.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652948460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13822489976882935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.829166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652961711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13834209740161896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7700480, "lr": 3.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652975057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13845370709896088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652988358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380363553762436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8028160, "lr": 4.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653001585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13818314671516418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653015083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13778428733348846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8355840, "lr": 4.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653028615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13871639966964722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3291666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653041991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13768532872200012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8683520, "lr": 4.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653055310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13836461305618286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.4958333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653068833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13690422475337982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9011200, "lr": 4.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653082339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13776034116744995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.6624999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653082339, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653082340, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653163345, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49876242876052856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653163347, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653163347, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653176359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13759009540081024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9338880, "lr": 4.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653189464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13698963820934296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653202421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379309743642807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9666560, "lr": 4.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653215767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364665925502777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653229204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374896764755249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9994240, "lr": 5.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653242483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13583959639072418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1624999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653255704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368633210659027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10321920, "lr": 5.2458333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653268830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13656987249851227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653281888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13803702592849731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10649600, "lr": 5.4124999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653294790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13655391335487366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653307752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14393864572048187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10977280, "lr": 5.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653321078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13652613759040833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653334047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13569329679012299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11304960, "lr": 5.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653347051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379922777414322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653347052, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653347052, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653428831, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4993664026260376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653428832, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653428833, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653441892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362362802028656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11632640, "lr": 5.9124999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653455055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13900673389434814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653468298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366998255252838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11960320, "lr": 6.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653481324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347973644733429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653494531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358330249786377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12288000, "lr": 6.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653507980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13875670731067657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.329166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653521297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13481369614601135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12615680, "lr": 6.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653534525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13460899889469147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653547728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13471469283103943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12943360, "lr": 6.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653560525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334437131881714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653573541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312631845474243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13271040, "lr": 6.745833333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653586293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13595062494277954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653599321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13368016481399536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13598720, "lr": 6.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653612559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13846777379512787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653612560, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784653612560, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784653696254, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5001416802406311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784653696255, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784653696255, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784653709402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134684219956398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13926400, "lr": 7.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653722468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13439500331878662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653735650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13626544177532196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14254080, "lr": 7.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653748903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355787217617035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653762137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13735981285572052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14581760, "lr": 7.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653775429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13007204234600067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653788474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356765627861023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14909440, "lr": 7.579166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653801711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13098154962062836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653815248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13431890308856964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15237120, "lr": 7.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653829116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13336804509162903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.829166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653842518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13411881029605865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15564800, "lr": 7.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653856009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13486875593662262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.995833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653869968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13564422726631165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15892480, "lr": 8.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653883541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13445627689361572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653883542, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784653883542, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784653967756, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5012380480766296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784653967757, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784653967757, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784653981430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13326255977153778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16220160, "lr": 8.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653994952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13433200120925903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654008532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13558852672576904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16547840, "lr": 8.412500000000001e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654022469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13550186157226562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654036189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13869377970695496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16875520, "lr": 8.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654049782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13361084461212158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.662499999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654063885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13532856106758118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17203200, "lr": 8.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654077342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13367904722690582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654091029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336812824010849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17530880, "lr": 8.912499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654104949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13244874775409698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654118501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13586358726024628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17858560, "lr": 9.079166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654132419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13396228849887848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.162499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654146099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13365484774112701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18186240, "lr": 9.245833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654159655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13107936084270477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654159656, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654159656, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654243693, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5025910139083862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654243694, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654243694, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654257888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13757383823394775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18513920, "lr": 9.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654271850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13496515154838562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.495833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654285803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13533110916614532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18841600, "lr": 9.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654299848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13166607916355133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.662499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654313495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13383617997169495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19169280, "lr": 9.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654327403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13433513045310974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654341382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13675826787948608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19496960, "lr": 9.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654355279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13413068652153015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654368988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312189280986786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19824640, "lr": 1.0079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654382362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1299944967031479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0162499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654395430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12807582318782806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20152320, "lr": 1.0245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654408690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132929265499115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654422093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12988968193531036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20480000, "lr": 1.0412499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654435065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1253114640712738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654435065, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654435066, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654516140, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5040611028671265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654516141, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654516141, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654529569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131403848528862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20807680, "lr": 1.0579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654542748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13582263886928558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654556129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13337825238704681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21135360, "lr": 1.0745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654569546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.130586177110672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654582653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12853679060935974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21463040, "lr": 1.0912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654595983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.126909077167511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654609823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12924300134181976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21790720, "lr": 1.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654623193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13222657144069672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654636603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13544265925884247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22118400, "lr": 1.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654650044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12535491585731506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654663281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12881585955619812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22446080, "lr": 1.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654676491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341819167137146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654690050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13104036450386047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22773760, "lr": 1.1579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654703910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12948203086853027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654703910, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784654703911, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784654784707, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5059241652488708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784654784708, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784654784708, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784654798528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13038645684719086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23101440, "lr": 1.1745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654812020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1293793022632599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654825211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13296723365783691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23429120, "lr": 1.1912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654838900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1241694912314415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654852703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12957152724266052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23756800, "lr": 1.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654865991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.125825434923172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654879192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301743984222412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24084480, "lr": 1.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654892606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12203685194253922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654905734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358071118593216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24412160, "lr": 1.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654919381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13708838820457458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654932757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1237095519900322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24739840, "lr": 1.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654946436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12345212697982788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654960004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1256718933582306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25067520, "lr": 1.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654973505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12905652821063995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654973505, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784654973505, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655057144, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5080748796463013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655057145, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655057145, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655070370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13645760715007782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25395200, "lr": 1.2912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655084202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364504098892212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655097812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356598138809204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25722880, "lr": 1.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655111315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13995501399040222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655124892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13654816150665283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26050560, "lr": 1.3245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655138548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13822610676288605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655152158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12747043371200562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26378240, "lr": 1.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655165792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1261124163866043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655179357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14227652549743652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26705920, "lr": 1.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655193433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1411198377609253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655207423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.127297043800354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27033600, "lr": 1.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655221182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1282861828804016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655234875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13270065188407898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27361280, "lr": 1.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655248085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317131519317627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655248085, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655248086, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655333023, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5103504657745361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655333025, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655333025, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655346993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14135098457336426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27688960, "lr": 1.4079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655360970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14247478544712067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655374616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13530953228473663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28016640, "lr": 1.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655388430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13945241272449493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655402436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14491033554077148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28344320, "lr": 1.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655416180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14077487587928772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655430106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12583334743976593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28672000, "lr": 1.4579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655444049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13752244412899017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655458125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13185761868953705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28999680, "lr": 1.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655472140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13602018356323242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655486090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13788817822933197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29327360, "lr": 1.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655499783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329811066389084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4995833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655514266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13458189368247986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29655040, "lr": 1.5079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655528755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13663744926452637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655528756, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784655528756, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784655609965, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5127341747283936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784655609966, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784655609966, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784655623869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13872632384300232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29982720, "lr": 1.5245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655638121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381729692220688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655652128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13167187571525574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30310400, "lr": 1.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655666205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13295377790927887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655680317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13376925885677338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30638080, "lr": 1.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655694649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13961435854434967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655708835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13256706297397614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30965760, "lr": 1.5745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655723236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1411445587873459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655736910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14070191979408264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31293440, "lr": 1.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655750896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13514721393585205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655764653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12765295803546906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31621120, "lr": 1.6079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655778225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12988077104091644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655791763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13416153192520142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31948800, "lr": 1.6245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655805445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12771421670913696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655805445, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784655805446, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784655884246, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5158132314682007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784655884247, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784655884247, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784655897823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132221519947052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32276480, "lr": 1.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655911418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13687501847743988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655925203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13376812636852264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32604160, "lr": 1.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655938923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321364790201187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655952786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13392853736877441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32931840, "lr": 1.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655966470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334347426891327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655980520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13372154533863068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33259520, "lr": 1.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655994489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13282378017902374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656008169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13983415067195892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33587200, "lr": 1.7079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656022187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1319025307893753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656035861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12884311378002167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33914880, "lr": 1.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656049281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311187446117401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656062779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13216355443000793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34242560, "lr": 1.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656076519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13587895035743713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7495833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656076519, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656076520, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656156669, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5200971961021423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656156671, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656156671, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656170305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332913339138031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34570240, "lr": 1.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656183793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12287377566099167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656197258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1305890828371048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34897920, "lr": 1.7745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656210866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1306329071521759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656224468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13512910902500153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35225600, "lr": 1.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656238713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13532572984695435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656252462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13529640436172485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35553280, "lr": 1.8079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656266760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12388919293880463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8162500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656280803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308993101119995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35880960, "lr": 1.8245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656294397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13122868537902832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656308260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13023260235786438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36208640, "lr": 1.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656321824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13303622603416443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656335479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13091444969177246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36536320, "lr": 1.8579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656348856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13019375503063202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656348857, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656348858, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656429404, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5258970856666565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656429407, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656429407, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656443044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13777141273021698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36864000, "lr": 1.8745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656456731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13228681683540344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656470324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334223747253418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37191680, "lr": 1.8912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656483872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13044852018356323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656497434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1268361508846283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37519360, "lr": 1.9079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656510899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324801743030548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656524808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356820911169052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37847040, "lr": 1.9245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656538656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327984780073166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656552463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311853677034378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38174720, "lr": 1.9412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656566355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13906383514404297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656580175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13375268876552582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38502400, "lr": 1.9579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656593493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13757941126823425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656607712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13181786239147186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38830080, "lr": 1.9745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656621487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332755982875824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656621487, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784656621487, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784656700406, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5335649251937866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784656700406, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784656700407, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784656714012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13116003572940826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39157760, "lr": 1.9912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656727586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13496986031532288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656741235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376531720161438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39485440, "lr": 2.0079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656755057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331360936164856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656768780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12733836472034454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39813120, "lr": 2.0245833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656782524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12920856475830078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0329166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656796314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12585532665252686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40140800, "lr": 2.0412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656809946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12394177913665771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656823991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13600575923919678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40468480, "lr": 2.0579166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656837996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13716623187065125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656851659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345822811126709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40796160, "lr": 2.0745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656866077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12673257291316986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0829166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656880071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1305486261844635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41123840, "lr": 2.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656893494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12341387569904327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656893494, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784656893495, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784656973926, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.544227659702301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784656973927, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784656973927, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784656987789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13216707110404968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41451520, "lr": 2.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657001416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13263165950775146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657015007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13363540172576904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41779200, "lr": 2.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657028521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376463919878006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657042159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360497772693634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42106880, "lr": 2.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657055757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13114242255687714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657069280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12700693309307098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42434560, "lr": 2.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657082877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134910449385643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657096332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12443897128105164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42762240, "lr": 2.1745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657110040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300889402627945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657123722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13235530257225037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43089920, "lr": 2.1912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657137429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13375939428806305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657151177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14009296894073486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43417600, "lr": 2.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657164675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13594098389148712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657164676, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657164676, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657246803, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5590075254440308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657246804, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657246804, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657260834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363396942615509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43745280, "lr": 2.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657274447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1313738077878952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657287979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13237138092517853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44072960, "lr": 2.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657301671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13870389759540558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657316103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298091560602188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44400640, "lr": 2.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657330593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382077932357788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657344635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13470271229743958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44728320, "lr": 2.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657358860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13714686036109924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657372908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13433098793029785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45056000, "lr": 2.2912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657387101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13029754161834717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657401310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14035071432590485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45383680, "lr": 2.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657415523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344282478094101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657429624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12867119908332825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45711360, "lr": 2.3245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657443873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13322600722312927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657443873, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657443873, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657527143, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5742246508598328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657527144, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657527144, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657541585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14148971438407898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46039040, "lr": 2.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657555821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13248011469841003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657570524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13337033987045288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46366720, "lr": 2.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657584760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13023550808429718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657599394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349516659975052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46694400, "lr": 2.3745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657613675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13321468234062195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657627793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13334597647190094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47022080, "lr": 2.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657641979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13504600524902344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657655956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13060791790485382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47349760, "lr": 2.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657670035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13214240968227386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657684115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13264547288417816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47677440, "lr": 2.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657698177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366458535194397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657711967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13593672215938568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48005120, "lr": 2.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657726248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13703280687332153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.4495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657726248, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784657726249, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784657807018, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5902137160301208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784657807018, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784657807019, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784657821020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13129399716854095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48332800, "lr": 2.4579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657835262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13506154716014862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657849228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13378238677978516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48660480, "lr": 2.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657863611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13147683441638947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657877776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13669532537460327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48988160, "lr": 2.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657891744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12872272729873657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657905910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13323192298412323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49315840, "lr": 2.507916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657919718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12303484976291656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.5162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657933586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13013719022274017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49643520, "lr": 2.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657947394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13073085248470306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657961196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303349733352661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49971200, "lr": 2.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657974894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1305142045021057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.5495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657988683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12807978689670563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50298880, "lr": 2.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658002444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13004754483699799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.5662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658002445, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658002445, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658084872, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6024594306945801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658084873, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658084873, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658098464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12158553302288055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50626560, "lr": 2.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658112320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1259918510913849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658126028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12806397676467896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50954240, "lr": 2.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658139890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13228440284729004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658154102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308874487876892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51281920, "lr": 2.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658167968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12072835117578506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658182211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289505958557129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51609600, "lr": 2.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658196255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311330497264862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658210314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13529877364635468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51937280, "lr": 2.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658223925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12699222564697266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.6495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658238385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13166925311088562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52264960, "lr": 2.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658252573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11702980846166611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.6662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658266958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11962512135505676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52592640, "lr": 2.6745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658281447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12414347380399704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658281447, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658281448, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658364315, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6141406297683716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658364316, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658364316, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658378640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12031309306621552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52920320, "lr": 2.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658392771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1266314536333084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.6995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658407152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1268780529499054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53248000, "lr": 2.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658421326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1299523413181305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658435477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11986198276281357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53575680, "lr": 2.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658449661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330856829881668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658463664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1278616189956665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53903360, "lr": 2.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658477845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11958590894937515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.7495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658492104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12528091669082642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54231040, "lr": 2.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658506701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12477200478315353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658521198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11660575866699219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54558720, "lr": 2.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658535963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13023051619529724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658550433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12538494169712067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54886400, "lr": 2.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658564773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12739627063274384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658564774, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784658564774, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784658646993, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.625512957572937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784658646994, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784658646994, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784658660764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12582017481327057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55214080, "lr": 2.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658675259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11704692244529724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658689090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12046000361442566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55541760, "lr": 2.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658703060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11908119916915894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658716830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12203685194253922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55869440, "lr": 2.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658730629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12434214353561401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658744345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11421803385019302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56197120, "lr": 2.8579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658758044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12087755650281906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658771709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12316860258579254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56524800, "lr": 2.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658785262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12162988632917404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658798765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11407345533370972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56852480, "lr": 2.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658812475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12167395651340485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658825988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10727016627788544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57180160, "lr": 2.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658839809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1211378201842308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658839809, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784658839810, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784658919591, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6336947083473206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784658919592, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784658919592, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784658933665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11395083367824554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57507840, "lr": 2.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658947676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11114786565303802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658961759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12506119906902313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57835520, "lr": 2.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658975977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11942973732948303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658989830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12177443504333496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58163200, "lr": 2.9579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659003634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1131211370229721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659017142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11674564331769943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58490880, "lr": 2.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659031416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12032938748598099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659045491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11837728321552277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58818560, "lr": 2.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659059502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11018408834934235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659073810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11828451603651047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59146240, "lr": 3.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659087843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1195090040564537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.0162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659101809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12300071865320206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59473920, "lr": 3.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659116198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11555109173059464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659116199, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659116200, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659196400, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6453458666801453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659196401, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659196401, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659210604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11778213083744049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59801600, "lr": 3.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659224665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1088983565568924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659238628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10818643122911453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60129280, "lr": 3.0579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659252904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11244756728410721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.0662499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659266740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10765799880027771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60456960, "lr": 3.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659280897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11969335377216339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659295136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1131792888045311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60784640, "lr": 3.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659309678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11826705932617188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659323795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10036175698041916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61112320, "lr": 3.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659338445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12030018121004105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.1162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659352859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1126769408583641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61440000, "lr": 3.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659367200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11214502155780792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659381301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10801862925291061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61767680, "lr": 3.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659395363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11692067980766296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659395364, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659395364, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659478469, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6653547286987305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659478470, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659478471, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659492308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1198120042681694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62095360, "lr": 3.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659506316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11910925805568695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.1662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659520518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12182479351758957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62423040, "lr": 3.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659534503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10448430478572845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659548604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11235140264034271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62750720, "lr": 3.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659562479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11653943359851837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659576652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11874068528413773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63078400, "lr": 3.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659590610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11832110583782196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659604547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10699468106031418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63406080, "lr": 3.2245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659618386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11877104640007019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659632215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11320126801729202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63733760, "lr": 3.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659645817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1157936379313469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659659461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11228439956903458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64061440, "lr": 3.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659673547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12553605437278748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659673548, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784659673548, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784659757933, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6942802667617798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784659757933, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784659757934, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784659772123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1239108219742775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64389120, "lr": 3.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659786347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11669708043336868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659800480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11601518839597702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64716800, "lr": 3.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659815071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12078433483839035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659829482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10823052376508713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65044480, "lr": 3.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659843883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11103881150484085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659857834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11225128918886185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65372160, "lr": 3.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659872006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084471195936203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659886040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1104283332824707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65699840, "lr": 3.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659900540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11521189659833908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659914752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12328799813985825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66027520, "lr": 3.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659928986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11524202674627304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659943421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12723641097545624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66355200, "lr": 3.3745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659957772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11291384696960449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659957773, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784659957773, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660039846, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7334295511245728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660039847, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660039847, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660054279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1257617026567459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66682880, "lr": 3.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660068611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11334279924631119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.3995833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660082828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1147313341498375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67010560, "lr": 3.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660097474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11238206177949905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660111869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11894986778497696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67338240, "lr": 3.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660126484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11801910400390625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660141330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12169331312179565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67665920, "lr": 3.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660156006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12067268788814545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660170455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11387570947408676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67993600, "lr": 3.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660185536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11202207207679749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660200173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11507386714220047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68321280, "lr": 3.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660214597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11376737058162689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660228945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1256818026304245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68648960, "lr": 3.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660243628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1156059131026268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660243629, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660243629, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660324715, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7474913001060486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660324716, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660324716, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660339130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1218007504940033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68976640, "lr": 3.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660353734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1185261607170105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660367987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11328695714473724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69304320, "lr": 3.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660382551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11454910039901733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660396777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10802781581878662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69632000, "lr": 3.5412500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660411341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11497291922569275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660425649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11841320991516113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69959680, "lr": 3.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660439638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11123886704444885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660453915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11033175140619278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70287360, "lr": 3.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660468057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080699935555458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660482649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12190952897071838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70615040, "lr": 3.5912500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660497327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10547273606061935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660511799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12042888253927231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70942720, "lr": 3.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660526760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12023081630468369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660526760, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784660526760, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784660610335, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7542760372161865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784660610336, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784660610336, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784660624898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1185731291770935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71270400, "lr": 3.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660639246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11757932603359222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660653752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1162232980132103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71598080, "lr": 3.6412499999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660668304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11550933122634888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660682549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10328785330057144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71925760, "lr": 3.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660696821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10289867967367172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660711029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1186278760433197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72253440, "lr": 3.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660725113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1144903227686882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660739158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10407258570194244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72581120, "lr": 3.6912499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660753204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10236520320177078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660766962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10750637203454971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72908800, "lr": 3.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660781066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11270975321531296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660794995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11149738729000092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73236480, "lr": 3.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660808988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11458010226488113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660809032, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784660809032, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784660890204, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7592202425003052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784660890205, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784660890205, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784660904283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10486529767513275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73564160, "lr": 3.7412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660918569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10656307637691498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660932903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11427957564592361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73891840, "lr": 3.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660946909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11000615358352661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660961179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10694421827793121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74219520, "lr": 3.7745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660975600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11503966152667999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660989998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11311264336109161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74547200, "lr": 3.7912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661004152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11152707785367966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661018255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11597553640604019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74874880, "lr": 3.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661032705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11030151695013046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661046914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09860848635435104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75202560, "lr": 3.8245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661061315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11305255442857742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661075434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10752210021018982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75530240, "lr": 3.8412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661089721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1050916314125061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661089722, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661089722, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661170707, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7619774341583252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661170707, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661170708, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661184831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10658600181341171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75857920, "lr": 3.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661199034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11278734356164932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661213173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09554534405469894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76185600, "lr": 3.8745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661227319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10350877791643143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661241468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11240158975124359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76513280, "lr": 3.8912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661255517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10520248115062714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661269751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08772898465394974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76840960, "lr": 3.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661284807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.109085813164711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661298947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10101845115423203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77168640, "lr": 3.9245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661313186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11772070080041885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661327485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10781719535589218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77496320, "lr": 3.9412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661341302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09961709380149841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661355724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09852872788906097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77824000, "lr": 3.9579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661370060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10679809749126434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661370060, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784661370062, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784661451269, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7638276219367981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784661451270, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784661451270, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784661465651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11501955986022949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78151680, "lr": 3.9745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661479928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11042746156454086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661493578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10336939245462418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78479360, "lr": 3.9912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661507882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10514426976442337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661521944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09917531162500381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78807040, "lr": 4.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661536294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10130565613508224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661550603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1062275841832161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79134720, "lr": 4.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661564936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10616952180862427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661579174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09760165959596634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79462400, "lr": 4.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661593731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10836067795753479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661608058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10391271859407425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79790080, "lr": 4.0579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661622300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10671212524175644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661636561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10830670595169067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80117760, "lr": 4.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661650864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11267375946044922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661650864, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784661650864, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784661735179, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7647554278373718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784661735180, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784661735180, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784661749583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10994894057512283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80445440, "lr": 4.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661763574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09845544397830963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661777799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10259705036878586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80773120, "lr": 4.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661792089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10507544875144958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661806516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11057591438293457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81100800, "lr": 4.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661820985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11309661716222763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661835309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10046101361513138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81428480, "lr": 4.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661849702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10499198734760284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661864140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10836893320083618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81756160, "lr": 4.1579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661878276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10511566698551178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661892604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11263691633939743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82083840, "lr": 4.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661906492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09397896379232407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661920936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09784488379955292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82411520, "lr": 4.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661935002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10682113468647003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661935003, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784661935003, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662017544, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7661838531494141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662017545, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662017545, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662031456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09116104245185852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82739200, "lr": 4.2079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662045564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10436365753412247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662059600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11025796830654144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83066880, "lr": 4.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662073907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09783252328634262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662087919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11126011610031128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83394560, "lr": 4.2412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662102088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11118263006210327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662116208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09761209785938263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83722240, "lr": 4.2579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662130115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11190624535083771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662144197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10507293045520782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84049920, "lr": 4.2745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662158224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10448917746543884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662172474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072661355137825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84377600, "lr": 4.2912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662186385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1042269766330719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662200335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09774293005466461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84705280, "lr": 4.3079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662214584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10643064230680466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662214585, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662214585, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662296360, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7672280669212341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662296361, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662296361, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662310922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1035892590880394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85032960, "lr": 4.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662325452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10398008674383163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662339570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10613439232110977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85360640, "lr": 4.3412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662353909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10365023463964462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662368367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09767122566699982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85688320, "lr": 4.3579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662382635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09935953468084335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662397142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10643425583839417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86016000, "lr": 4.3745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662411044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09614503383636475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662425389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10790236294269562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86343680, "lr": 4.3912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662439635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1086350828409195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662453769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1163138672709465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86671360, "lr": 4.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662468001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11387752741575241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662482019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11638100445270538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86999040, "lr": 4.4245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662496360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11538861691951752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662496360, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784662496361, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784662577634, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7676960825920105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784662577635, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784662577635, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784662591566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10032930225133896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87326720, "lr": 4.4412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662605574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11560408771038055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662619294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793529450893402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87654400, "lr": 4.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662633349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10197356343269348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662647210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09172068536281586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87982080, "lr": 4.4745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662661266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09863091260194778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662675049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10431773215532303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88309760, "lr": 4.4912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662689199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10743609070777893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662703092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10765986889600754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88637440, "lr": 4.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662716974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10071826726198196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662731318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1127861738204956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88965120, "lr": 4.5245833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662745669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09931215643882751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662760066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1097998395562172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89292800, "lr": 4.5412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662774045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11187499761581421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.5495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662774046, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784662774046, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784662854482, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7682220339775085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784662854483, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784662854483, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784662868999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10757115483283997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89620480, "lr": 4.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662883397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10151541233062744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662897675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11104463785886765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89948160, "lr": 4.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662912193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09136421233415604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662926178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11455836892127991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90275840, "lr": 4.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662940143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10672984272241592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662955209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.110944002866745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90603520, "lr": 4.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662970081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11118093878030777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662984966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12281898409128189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90931200, "lr": 4.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662999772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10536635667085648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663014500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10910400748252869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91258880, "lr": 4.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663029277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12319110333919525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663043919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09918997436761856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91586560, "lr": 4.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663058547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11056973040103912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663058547, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663058548, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663137837, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7681711912155151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663137838, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663137838, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663152432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10391851514577866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91914240, "lr": 4.6745833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663167180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11840350925922394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663181860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11008424311876297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92241920, "lr": 4.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663196772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10294250398874283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663211535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09677263349294662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92569600, "lr": 4.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663226326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11262580752372742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663241465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10652157664299011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92897280, "lr": 4.7245833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663256398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10488476604223251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663271437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11651985347270966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93224960, "lr": 4.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663286178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0996176227927208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663301262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12480071187019348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93552640, "lr": 4.7579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663316325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10769788175821304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663331156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11442700773477554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93880320, "lr": 4.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663345834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10731793940067291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663345835, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784663345835, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784663427060, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7683420181274414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784663427062, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784663427062, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784663441905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11338063329458237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94208000, "lr": 4.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663456485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11600467562675476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663471051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11156890541315079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94535680, "lr": 4.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663485937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11822232604026794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663500462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11435253918170929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94863360, "lr": 4.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663515091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11838249117136002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663529853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.127475306391716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95191040, "lr": 4.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663544249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12065243721008301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663558682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1160629466176033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95518720, "lr": 4.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663573437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11046437174081802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663587853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10813063383102417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95846400, "lr": 4.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663602549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10762616991996765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663616936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11463689059019089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96174080, "lr": 4.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663631611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11375244706869125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663631611, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784663631612, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784663713301, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7691826224327087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784663713302, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784663713302, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784663728115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11989345401525497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96501760, "lr": 4.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663742786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723431408405304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663757155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10781346261501312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96829440, "lr": 4.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663772004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10841161012649536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663786746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11112009733915329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97157120, "lr": 4.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663801419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10864001512527466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663816023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11370762437582016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97484800, "lr": 4.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663830564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10880890488624573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663845159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10476534068584442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97812480, "lr": 4.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663859626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10006606578826904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663874351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10439606010913849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98140160, "lr": 4.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663888859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09606295078992844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663903258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1040777862071991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98467840, "lr": 5.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663917668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10090184956789017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663917668, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784663917669, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784663998901, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7701598405838013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784663998902, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784663998902, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664013133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10991422086954117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98795520, "lr": 5.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664027380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1006740853190422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664041352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09688588231801987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99123200, "lr": 5.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664055238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10810989141464233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664069300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11474083364009857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99450880, "lr": 5.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664083316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10493630915880203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664097237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10491269826889038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99778560, "lr": 5.074583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664111243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10462367534637451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664125224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10194014012813568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100106240, "lr": 5.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664139257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.120423324406147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664153480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11735742539167404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100433920, "lr": 5.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664167645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10154299437999725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664182001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10065121948719025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100761600, "lr": 5.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664195944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12107671797275543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664195944, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664195945, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664277077, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7704293131828308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664277078, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664277078, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664291507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11095532774925232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101089280, "lr": 5.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664305700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09657801687717438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664319863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10479463636875153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101416960, "lr": 5.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664334067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10370733588933945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664348145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10795273631811142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101744640, "lr": 5.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664362382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10529375821352005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.182916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664376754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10605141520500183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102072320, "lr": 5.191249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664391023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11178549379110336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664405402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10502991080284119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102400000, "lr": 5.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664419860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10774807631969452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664434130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10560641437768936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102727680, "lr": 5.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664448410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11427101492881775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664462302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10694784671068192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103055360, "lr": 5.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664476306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1021760106086731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664476306, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784664476307, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784664557184, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7712799906730652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784664557185, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784664557185, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784664571300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0991738885641098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103383040, "lr": 5.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664585594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09363225102424622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664599743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0962313860654831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103710720, "lr": 5.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664614017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09689575433731079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.282916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664628161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09671168774366379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104038400, "lr": 5.291249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664642576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09427580237388611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664656762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10013239830732346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104366080, "lr": 5.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664670837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0972953736782074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664685198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10295893996953964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104693760, "lr": 5.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664699325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10865352302789688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664713780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10449165105819702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105021440, "lr": 5.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664727990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1030883640050888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664742320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11396458745002747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105349120, "lr": 5.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664757255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11114034056663513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664757255, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784664757255, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784664840316, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7713361382484436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784664840317, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784664840317, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784664855008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09833411872386932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105676800, "lr": 5.374583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664869550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10716298222541809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.382916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664884346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10992131382226944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106004480, "lr": 5.391249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664898797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10148195177316666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664913108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09978289902210236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106332160, "lr": 5.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664927660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1074284166097641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664941775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11034887284040451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106659840, "lr": 5.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664956069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10621485114097595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664970601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10247448086738586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106987520, "lr": 5.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664984683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10079622268676758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664998794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0999419167637825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107315200, "lr": 5.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665012948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10230159014463425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665026970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10905510187149048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107642880, "lr": 5.474583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665040828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11467419564723969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.482916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665040829, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665040829, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665121201, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7716841101646423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665121202, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665121202, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665135253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10946009308099747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107970560, "lr": 5.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665149352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10666421055793762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665163349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09102733433246613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108298240, "lr": 5.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665177262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10151714831590652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665191469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10597078502178192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108625920, "lr": 5.524583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665205309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1077072024345398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665219200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10985107719898224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108953600, "lr": 5.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665233375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10264549404382706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665247710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09986155480146408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109281280, "lr": 5.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665262081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10374214500188828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665276659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10273449122905731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109608960, "lr": 5.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665290999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10747628659009933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665305571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10467890650033951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109936640, "lr": 5.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665320182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10982658714056015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665320182, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784665320183, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784665399759, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7720115184783936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784665399760, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784665399760, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784665414121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10321003943681717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110264320, "lr": 5.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665428435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10273704677820206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665442533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10391547530889511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110592000, "lr": 5.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665456389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10214047133922577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665471188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10672449320554733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110919680, "lr": 5.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665485445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10068007558584213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665499669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09809403866529465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111247360, "lr": 5.657916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665514228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10180290788412094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.666250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665528658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11123138666152954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111575040, "lr": 5.674583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665543171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10484782606363297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665557534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10367661714553833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111902720, "lr": 5.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665571840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10224922001361847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665586480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10311344265937805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112230400, "lr": 5.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665600807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10697071254253387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665600808, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784665600808, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784665680079, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7725686430931091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784665680080, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784665680080, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784665694457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10764146596193314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112558080, "lr": 5.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665708902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10158310830593109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665722972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10400755703449249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112885760, "lr": 5.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665737197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10131391882896423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665751565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10511216521263123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113213440, "lr": 5.757916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665765522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11246787756681442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.766250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665779859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10104936361312866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113541120, "lr": 5.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665793896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11021754890680313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665808379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11242365092039108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113868800, "lr": 5.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665823066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10687392950057983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665837378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10317914187908173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114196480, "lr": 5.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665851635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10506613552570343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665866282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10481679439544678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114524160, "lr": 5.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665880763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09817253798246384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665880763, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784665880764, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784665960225, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7731200456619263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784665960225, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784665960226, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784665974501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10094622522592545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114851840, "lr": 5.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665988829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0993785709142685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666003022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1058211550116539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115179520, "lr": 5.857916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666016779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1007244810461998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666031502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10729334503412247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115507200, "lr": 5.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666045917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10922595858573914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666060718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10751581192016602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115834880, "lr": 5.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666075660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11521542817354202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666090318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11377851665019989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116162560, "lr": 5.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666104884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09992086887359619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666119355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10395169258117676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116490240, "lr": 5.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666133762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10299262404441833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666148261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10521083325147629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116817920, "lr": 5.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666162621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10728824138641357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666162622, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666162622, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666241629, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7733516693115234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666241630, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666241630, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666256235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10121877491474152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117145600, "lr": 5.957916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666270717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10038679093122482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666285207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10865560173988342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117473280, "lr": 5.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666299476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10476656258106232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666313690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09417548030614853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117800960, "lr": 5.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666328334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11215829104185104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666342444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10109510272741318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118128640, "lr": 6.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666356626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10764233767986298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666371102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10304410010576248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118456320, "lr": 6.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666385590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10292252898216248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666399725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10971666872501373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118784000, "lr": 6.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666414159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11430055648088455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666428679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10567770898342133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119111680, "lr": 6.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666443064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10248652845621109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666443065, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784666443065, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784666522970, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7732779383659363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784666522971, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784666522971, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784666537899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11289908736944199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119439360, "lr": 6.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666552399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10876475274562836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666566816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10931092500686646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119767040, "lr": 6.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666581522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10687839984893799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666595781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10137956589460373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120094720, "lr": 6.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666610049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10763005167245865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666625044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11832745373249054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120422400, "lr": 6.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666639679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10899398475885391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666654393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11562903225421906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120750080, "lr": 6.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666669422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12212959676980972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666684150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071389690041542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121077760, "lr": 6.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666699022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10978469252586365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666713976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11482860893011093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121405440, "lr": 6.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666728691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11022135615348816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666728692, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784666728692, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784666807623, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773613691329956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784666807624, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784666807624, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784666822293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11314789950847626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121733120, "lr": 6.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666837248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10958286374807358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666852129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11034375429153442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122060800, "lr": 6.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666867089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10665416717529297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666881881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10428523272275925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122388480, "lr": 6.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666896739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11369027197360992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666911429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10663395375013351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122716160, "lr": 6.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666926103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11457222700119019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666941001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11278854310512543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123043840, "lr": 6.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666955848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12005262076854706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666970853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.105778269469738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123371520, "lr": 6.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666985579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11727311462163925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667000376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10682874917984009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123699200, "lr": 6.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667015513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11832724511623383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667015514, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667015514, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667095862, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7736829519271851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667095863, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667095863, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667110772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11913055926561356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124026880, "lr": 6.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667125531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10805072635412216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667140326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1195872575044632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124354560, "lr": 6.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667154808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11377354711294174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667169382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10747969150543213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124682240, "lr": 6.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667184244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11543317884206772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667198798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11095930635929108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125009920, "lr": 6.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667213606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11623881757259369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667228524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11166904121637344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125337600, "lr": 6.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667243325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10573405027389526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667258013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11234522610902786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125665280, "lr": 6.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667272831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10645903646945953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667287359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11657964438199997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125992960, "lr": 6.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667302118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10491914302110672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667302118, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784667302118, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784667384936, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746788263320923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784667384937, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784667384937, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784667399508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10979031026363373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126320640, "lr": 6.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667414403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11897087097167969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667429017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11021392047405243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126648320, "lr": 6.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667443913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11352507770061493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667458668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11545757204294205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126976000, "lr": 6.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667473316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10911735892295837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667488204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10820864140987396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127303680, "lr": 6.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667503281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10950326919555664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.482916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667518360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11237206310033798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127631360, "lr": 6.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667533178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11174982041120529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667548259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10991306602954865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127959040, "lr": 6.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667563206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11059160530567169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667578124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11170601099729538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128286720, "lr": 6.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667592919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10904745012521744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667592919, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784667592920, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784667675497, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746866345405579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784667675498, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784667675498, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784667690161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11775237321853638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128614400, "lr": 6.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667704686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11052800714969635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667719124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11099445819854736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128942080, "lr": 6.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667733500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10292735695838928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667747916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10547758638858795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129269760, "lr": 6.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667762462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11163219809532166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667776730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10764408856630325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129597440, "lr": 6.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667790838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10749578475952148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667805065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09757620841264725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129925120, "lr": 6.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667819250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1016073152422905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667833582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10584770888090134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130252800, "lr": 6.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667847595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10468987375497818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667861922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11014806479215622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130580480, "lr": 6.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667876380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11553658545017242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667876381, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784667876381, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784667958018, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7749338746070862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784667958019, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784667958019, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784667972235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11449310183525085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130908160, "lr": 6.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667986592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10529280453920364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668000828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10047170519828796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131235840, "lr": 6.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668015236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09215429425239563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668029504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11405976116657257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131563520, "lr": 6.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668043586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10259072482585907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668058104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10218916088342667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131891200, "lr": 6.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668072910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11219366639852524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668087427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10728734731674194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132218880, "lr": 6.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668102252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10301481932401657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668116981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10890769213438034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132546560, "lr": 6.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668131841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11001032590866089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668146787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072750836610794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132874240, "lr": 6.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668161425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10690552741289139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668161425, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668161426, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668242772, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7753920555114746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668242773, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668242773, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668257162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10897976905107498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133201920, "lr": 6.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668271689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10340233892202377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668285895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11074934154748917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133529600, "lr": 6.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668299966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0988878682255745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668314684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11603939533233643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133857280, "lr": 6.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668329012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11056118458509445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668343195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10334772616624832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134184960, "lr": 6.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668357849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10828372091054916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668372333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.108609639108181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134512640, "lr": 6.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668386346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10853592306375504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668400523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1099872887134552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134840320, "lr": 6.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668414828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09887070208787918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668428820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10614613443613052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135168000, "lr": 6.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668442870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08660341799259186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668442871, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784668442871, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784668523653, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7756896615028381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784668523654, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784668523654, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784668537890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09549378603696823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135495680, "lr": 6.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668552135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11164761334657669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668566404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10698570311069489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135823360, "lr": 6.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668580499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10125316679477692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668594760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10219017416238785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136151040, "lr": 6.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668609214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0903412327170372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668623393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10298417508602142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136478720, "lr": 6.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668637588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09710589051246643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668651624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10331118851900101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136806400, "lr": 6.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668665921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10611691325902939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668680587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1098966896533966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137134080, "lr": 6.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668695086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10785926878452301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668709417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10625440627336502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137461760, "lr": 6.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668723822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10316698998212814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668723823, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784668723824, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784668807418, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7759447693824768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784668807419, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784668807420, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784668821918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09548661857843399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137789440, "lr": 7.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668836700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10518413782119751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668851206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0949997678399086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138117120, "lr": 7.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668865655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10181041806936264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668880140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09792274981737137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138444800, "lr": 7.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668894131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10052938014268875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668908084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10928506404161453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138772480, "lr": 7.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668922604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10080427676439285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.066249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668936696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10868903994560242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139100160, "lr": 7.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668951029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1091141477227211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668965718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10633743554353714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139427840, "lr": 7.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668980323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10407593101263046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668994796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10901889204978943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139755520, "lr": 7.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669008995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10821304470300674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669008996, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669008996, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669091639, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7765141725540161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669091640, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669091640, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669105843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10649342089891434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140083200, "lr": 7.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669120129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10388552397489548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669134522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10322399437427521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140410880, "lr": 7.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669148840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09971053153276443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669163146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10695608705282211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140738560, "lr": 7.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669177429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09616955369710922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669192304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10483883321285248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141066240, "lr": 7.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669206594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10318107903003693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669220780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11057581752538681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141393920, "lr": 7.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669235266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09879183769226074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669249226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09726916253566742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141721600, "lr": 7.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669263277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10195410251617432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669277604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09050723165273666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142049280, "lr": 7.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669292090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10651710629463196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669292091, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784669292091, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784669374807, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7767730355262756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784669374808, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784669374808, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784669389173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09895555675029755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142376960, "lr": 7.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669403693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10061222314834595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669417825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09943683445453644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142704640, "lr": 7.257916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669432460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10912155359983444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669446835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10028618574142456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143032320, "lr": 7.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669461379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10238278657197952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669475606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10096438974142075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143360000, "lr": 7.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669489865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10557282716035843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669504323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10160498321056366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143687680, "lr": 7.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669518329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11171219497919083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669532769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10114249587059021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144015360, "lr": 7.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669547103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10778731107711792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669561333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10516048967838287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144343040, "lr": 7.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669575758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10561524331569672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669575758, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784669575759, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784669657761, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7772966623306274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784669657762, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784669657762, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784669672150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10551825910806656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144670720, "lr": 7.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669686399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09797800332307816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669700575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10685323923826218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144998400, "lr": 7.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669715047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1060190200805664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669729403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10199534147977829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145326080, "lr": 7.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669743477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10711319744586945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669757692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09762872755527496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145653760, "lr": 7.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669772248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0977715253829956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669786607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10483758896589279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145981440, "lr": 7.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669800766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10382764786481857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669815092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10214240103960037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146309120, "lr": 7.441250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669829408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10996648669242859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669843888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10634909570217133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146636800, "lr": 7.457916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669858257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0995849072933197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669858258, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784669858258, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784669941550, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7776336669921875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784669941552, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784669941552, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784669955743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10237082839012146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146964480, "lr": 7.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669969890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10527538508176804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669984317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09449677914381027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147292160, "lr": 7.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669998963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09801539778709412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670013295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10146176815032959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147619840, "lr": 7.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670028017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09982806444168091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670042298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1067717969417572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147947520, "lr": 7.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670056873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10996140539646149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670071510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10834111273288727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148275200, "lr": 7.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670086191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09857259690761566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670101006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10212171077728271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148602880, "lr": 7.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670115355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09788954257965088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670130044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11204443871974945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148930560, "lr": 7.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670144303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11042007803916931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670144303, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670144304, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670226572, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7780176401138306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670226573, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670226574, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670240618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10549969226121902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149258240, "lr": 7.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670255117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10435344278812408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670269104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10606437176465988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149585920, "lr": 7.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670283412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1109551265835762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149749760, "lr": 7.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670297898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10615909844636917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149913600, "lr": 7.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670312426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10777010768651962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670326506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10738123953342438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150241280, "lr": 7.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670340707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09729626774787903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150405120, "lr": 7.649583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670354869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10597806423902512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150568960, "lr": 7.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670368915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10325144976377487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670383107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772190243005753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150896640, "lr": 7.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670396967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1036251038312912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151060480, "lr": 7.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670411342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10608820617198944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151224320, "lr": 7.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670425425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09512599557638168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670425470, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784670425471, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784670505827, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7779695987701416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784670505828, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784670505829, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784670519966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09873918443918228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151552000, "lr": 7.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670533974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10496929287910461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151715840, "lr": 7.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670548092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10260479897260666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151879680, "lr": 7.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670562376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10453533381223679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670576498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10799968987703323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152207360, "lr": 7.741249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670590546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12163819372653961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152371200, "lr": 7.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670604667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10816188156604767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152535040, "lr": 7.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670618878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09836572408676147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670633274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11044707149267197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152862720, "lr": 7.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670647374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11062265932559967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153026560, "lr": 7.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670661931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11493205279111862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153190400, "lr": 7.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670676424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09785722196102142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670691132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10638938099145889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153518080, "lr": 7.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670705602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10351461172103882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153681920, "lr": 7.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670705650, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784670705650, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784670787037, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7784585952758789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784670787038, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784670787038, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784670801625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09608211368322372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153845760, "lr": 7.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670816135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09714801609516144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.832916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670830528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11279899626970291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154173440, "lr": 7.841249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670845153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11728516966104507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154337280, "lr": 7.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670859507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11336110532283783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154501120, "lr": 7.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670873706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1105274185538292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670888484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10694359242916107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154828800, "lr": 7.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670903113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1202782392501831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154992640, "lr": 7.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670917636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10365551710128784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155156480, "lr": 7.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670932568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10852912813425064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670947314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10975977033376694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155484160, "lr": 7.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670961799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1097491905093193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155648000, "lr": 7.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670976458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11322633177042007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155811840, "lr": 7.924583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670991143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11105816811323166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.932916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670991143, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784670991144, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671072744, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7789474725723267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671072745, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671072745, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671087471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10705175250768661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156139520, "lr": 7.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671102060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10980982333421707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156303360, "lr": 7.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671116927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11414626985788345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156467200, "lr": 7.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671131654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11013339459896088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671146382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10758160799741745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156794880, "lr": 7.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671161121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1076451987028122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156958720, "lr": 7.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671175966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10120748728513718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157122560, "lr": 7.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671190784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1118064820766449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671205505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11979889124631882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157450240, "lr": 8.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671220302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10937172919511795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157614080, "lr": 8.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671234932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10866264998912811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157777920, "lr": 8.024583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671250042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1033109799027443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.032916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671265199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10242220014333725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158105600, "lr": 8.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671280168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10883799940347672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158269440, "lr": 8.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671280168, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784671280169, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784671362379, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7792091965675354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784671362380, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784671362381, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784671377288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11447247117757797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158433280, "lr": 8.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671392283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11856947094202042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671407181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11325080692768097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158760960, "lr": 8.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671422229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11253128200769424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158924800, "lr": 8.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671437089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11014106869697571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159088640, "lr": 8.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671451645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11970248073339462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671466511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11164475232362747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159416320, "lr": 8.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671480876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10921857506036758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159580160, "lr": 8.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671495750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10999234765768051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159744000, "lr": 8.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671510315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09839978069067001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671525097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1099696159362793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160071680, "lr": 8.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671540259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10951043665409088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160235520, "lr": 8.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671555226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10386184602975845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160399360, "lr": 8.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671570037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10985489934682846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671570038, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784671570038, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784671651612, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7791255116462708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784671651613, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784671651613, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784671666437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1124267727136612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160727040, "lr": 8.174583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671681531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1273464560508728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160890880, "lr": 8.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671696657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10825219005346298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161054720, "lr": 8.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671711673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11175262928009033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671726527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11258808523416519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161382400, "lr": 8.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671741674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10782382637262344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161546240, "lr": 8.216250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671756800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11437998712062836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161710080, "lr": 8.224583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671772120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11302034556865692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671787126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10630546510219574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162037760, "lr": 8.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671802063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10829003155231476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162201600, "lr": 8.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671817284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12976817786693573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162365440, "lr": 8.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671832183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11279603093862534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671847412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11167463660240173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162693120, "lr": 8.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671862536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11610187590122223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162856960, "lr": 8.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671862537, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784671862537, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784671945064, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7794792056083679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784671945065, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784671945066, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784671959913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09501848369836807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163020800, "lr": 8.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671974869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11069482564926147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671989725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12108268588781357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163348480, "lr": 8.307916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672004624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10677757114171982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163512320, "lr": 8.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672019479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11058828979730606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163676160, "lr": 8.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672034301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10703586041927338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672049092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1123071014881134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164003840, "lr": 8.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672063744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11338335275650024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164167680, "lr": 8.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672078420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10303067415952682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164331520, "lr": 8.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672092777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11778765916824341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672107321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10674039274454117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164659200, "lr": 8.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672122016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11027026921510696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164823040, "lr": 8.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672136834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09202705323696136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164986880, "lr": 8.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672151107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10336937010288239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672151108, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784672151109, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784672233867, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7794865369796753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784672233868, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784672233868, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784672248487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10028187930583954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165314560, "lr": 8.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672262994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09613595902919769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165478400, "lr": 8.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672277283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1136987954378128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165642240, "lr": 8.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672291640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10696503520011902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672306011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10967548936605453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165969920, "lr": 8.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672320759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10425501316785812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166133760, "lr": 8.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672335073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732364654541016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166297600, "lr": 8.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672349495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11266323179006577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672363869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09954506903886795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166625280, "lr": 8.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672378602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09508813917636871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166789120, "lr": 8.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672393099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09675423800945282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166952960, "lr": 8.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672407470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09635862708091736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672421560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10269271582365036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167280640, "lr": 8.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672435914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10014795511960983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167444480, "lr": 8.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672435914, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784672435915, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784672519971, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.780205249786377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784672519972, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784672519972, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784672534380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1030035987496376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167608320, "lr": 8.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672549006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10239669680595398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672563449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10338165611028671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167936000, "lr": 8.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672577800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10664341598749161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168099840, "lr": 8.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672592443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10230427980422974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168263680, "lr": 8.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672607245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09818974882364273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672622005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10789645463228226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168591360, "lr": 8.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672636326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10540473461151123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168755200, "lr": 8.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672650645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10623770952224731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168919040, "lr": 8.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672664712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10532168298959732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672679347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10349265486001968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169246720, "lr": 8.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672693557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10648928582668304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169410560, "lr": 8.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672707597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10530990362167358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169574400, "lr": 8.624583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672722405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11279844492673874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672722450, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784672722450, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784672805703, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7805505990982056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784672805704, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784672805704, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784672820107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10832349210977554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169902080, "lr": 8.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672834709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11189933121204376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170065920, "lr": 8.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672849509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10619846731424332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170229760, "lr": 8.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672864321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10669924318790436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672878660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10034600645303726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170557440, "lr": 8.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672893380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10072776675224304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170721280, "lr": 8.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672908032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0955292358994484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170885120, "lr": 8.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672922391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10626262426376343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672936690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09848978370428085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171212800, "lr": 8.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672951053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10117946565151215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171376640, "lr": 8.716249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672965718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10085251927375793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171540480, "lr": 8.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672980004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10240506380796432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672994399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09863270819187164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171868160, "lr": 8.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673008954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10315186530351639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172032000, "lr": 8.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673009002, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673009003, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673090939, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7808977365493774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673090940, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673090940, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673105526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09698577970266342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172195840, "lr": 8.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673120187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10713594406843185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673134614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09836734086275101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172523520, "lr": 8.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673148966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10139473527669907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172687360, "lr": 8.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673163194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09389389306306839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172851200, "lr": 8.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673177672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10666466504335403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673192354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10248865187168121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173178880, "lr": 8.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673206737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1024690493941307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173342720, "lr": 8.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673221577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.104263074696064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173506560, "lr": 8.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673236206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11129125207662582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673250998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09459499269723892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173834240, "lr": 8.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673265665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10928961634635925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173998080, "lr": 8.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673280335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11097629368305206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174161920, "lr": 8.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673294705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10090658813714981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673294753, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784673294753, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784673375867, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7814043164253235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784673375868, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784673375868, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784673390218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09752576053142548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174489600, "lr": 8.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673404962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10692720115184784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174653440, "lr": 8.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673419795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10034731030464172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174817280, "lr": 8.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673434058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10506875813007355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673448686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10023622214794159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175144960, "lr": 8.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673462820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10820624977350235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175308800, "lr": 8.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673477068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10016319155693054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175472640, "lr": 8.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673491544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09826312959194183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673505988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10880913585424423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175800320, "lr": 8.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673520369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10818726569414139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175964160, "lr": 8.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673534728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09867673367261887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176128000, "lr": 8.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673549254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09377811849117279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673563434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09401117265224457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176455680, "lr": 8.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673577791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10566861927509308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176619520, "lr": 8.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673577791, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784673577792, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784673661044, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7819559574127197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784673661045, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784673661045, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784673675258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10174286365509033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176783360, "lr": 8.991249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673689610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11322490870952606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673703961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09949581325054169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177111040, "lr": 9.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673718270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10386195033788681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177274880, "lr": 9.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673732662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10960400104522705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177438720, "lr": 9.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673746875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09980147331953049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.032916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673760996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1093202456831932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177766400, "lr": 9.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673775148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10101031512022018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177930240, "lr": 9.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673789316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10673410445451736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178094080, "lr": 9.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673803474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1002781018614769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673817827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09667924791574478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178421760, "lr": 9.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673832461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1111208125948906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178585600, "lr": 9.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673847003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11825482547283173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178749440, "lr": 9.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673861418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082550659775734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673861418, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784673861419, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784673943680, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7817984223365784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784673943681, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784673943681, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784673958211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11394734680652618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179077120, "lr": 9.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673972515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10092375427484512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179240960, "lr": 9.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673987250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09976836293935776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179404800, "lr": 9.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674001622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10028862953186035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.132916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674016060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10482536256313324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179732480, "lr": 9.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674030463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09903731942176819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179896320, "lr": 9.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674044717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10026056319475174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180060160, "lr": 9.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674059378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1053149551153183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674073661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10534701496362686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180387840, "lr": 9.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674087765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09344318509101868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180551680, "lr": 9.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674102568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10288374871015549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180715520, "lr": 9.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674116899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10054522007703781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674131337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09389414638280869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181043200, "lr": 9.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674145982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09981614351272583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181207040, "lr": 9.216249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674145982, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784674145983, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784674229884, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7819409370422363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784674229886, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784674229886, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784674244474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11012573540210724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181370880, "lr": 9.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674259024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11340060085058212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674273377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09961215406656265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181698560, "lr": 9.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674287826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1070491224527359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181862400, "lr": 9.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674302158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10223103314638138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182026240, "lr": 9.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674316459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10032615810632706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674330768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10632577538490295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182353920, "lr": 9.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674345089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10818643867969513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182517760, "lr": 9.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674359546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09681913256645203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182681600, "lr": 9.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674374005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09644201397895813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674388397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1104249358177185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183009280, "lr": 9.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674402545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09639586508274078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183173120, "lr": 9.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674417123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10177382826805115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183336960, "lr": 9.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674431353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1023176908493042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674431353, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784674431354, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784674514907, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7823194861412048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784674514908, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784674514908, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784674529349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1013331338763237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183664640, "lr": 9.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674543711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10639609396457672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183828480, "lr": 9.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674557961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11116646230220795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183992320, "lr": 9.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674572773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09260784089565277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674587415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10154565423727036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184320000, "lr": 9.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674602115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10653949528932571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184483840, "lr": 9.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674616702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11590153723955154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184647680, "lr": 9.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674631075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10118746012449265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674645769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10553213953971863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184975360, "lr": 9.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674660640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10844563692808151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185139200, "lr": 9.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674675382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09607014805078506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185303040, "lr": 9.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674689958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10642936080694199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674704815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09777819365262985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185630720, "lr": 9.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674719210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10300293564796448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185794560, "lr": 9.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674719211, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784674719211, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784674802974, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7825934886932373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784674802975, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784674802975, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784674817984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11016173660755157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185958400, "lr": 9.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674832401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10348278284072876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674846485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09196615219116211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186286080, "lr": 9.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674861206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10082165896892548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186449920, "lr": 9.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674875618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10733262449502945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186613760, "lr": 9.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674890058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11075976490974426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674905072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0961211770772934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186941440, "lr": 9.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674919733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1034349650144577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187105280, "lr": 9.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674934431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10531030595302582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187269120, "lr": 9.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674948843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0979999303817749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674963110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10313671827316284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187596800, "lr": 9.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674977561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1040899008512497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187760640, "lr": 9.549583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674991822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11014974862337112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187924480, "lr": 9.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675006370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11230514943599701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675006370, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675006371, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675088059, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.783035933971405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675088060, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675088060, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675102439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10802280157804489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188252160, "lr": 9.574583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675116927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10505212098360062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188416000, "lr": 9.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675131165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10183759778738022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188579840, "lr": 9.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675145631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10815508663654327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675159979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10201860219240189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188907520, "lr": 9.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675174607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09684115648269653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189071360, "lr": 9.616249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675189174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10213243216276169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189235200, "lr": 9.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675203527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11086457222700119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675217874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10099928081035614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189562880, "lr": 9.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675232315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0944579616189003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189726720, "lr": 9.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675246751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10371987521648407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189890560, "lr": 9.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675261505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11181461811065674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.666249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675276077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10118744522333145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190218240, "lr": 9.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675290356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11778084188699722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190382080, "lr": 9.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675290356, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784675290357, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784675371461, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7825695276260376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784675371462, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784675371462, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784675386073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09979505091905594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190545920, "lr": 9.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675400539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09672880172729492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675414894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10611379891633987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190873600, "lr": 9.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675429663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1126936823129654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191037440, "lr": 9.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675444342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11648958176374435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191201280, "lr": 9.724583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675458847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09589564800262451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675473389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1058894544839859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191528960, "lr": 9.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675487786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10076646506786346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191692800, "lr": 9.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675502534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10012304037809372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191856640, "lr": 9.757916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675516860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10442431271076202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675530962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10661626607179642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192184320, "lr": 9.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675545874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10934396833181381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192348160, "lr": 9.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675560619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09836797416210175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192512000, "lr": 9.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675575516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11283436417579651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.799583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675575516, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784675575517, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784675658152, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7833617329597473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784675658153, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784675658154, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784675673084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11149994283914566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192839680, "lr": 9.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675688085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11027113348245621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193003520, "lr": 9.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675702892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10599249601364136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193167360, "lr": 9.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675717660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1182778999209404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675732393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.115727499127388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193495040, "lr": 9.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675756242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10038523375988007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193658880, "lr": 9.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675771336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11750467121601105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193822720, "lr": 9.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675785930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11383799463510513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.866249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675800830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1059081181883812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194150400, "lr": 9.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675815644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10678073018789291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194314240, "lr": 9.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675830498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10668173432350159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194478080, "lr": 9.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675845286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1050955057144165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675860221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10653004795312881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194805760, "lr": 9.907916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675874643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11583202332258224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194969600, "lr": 9.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675874644, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784675874644, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784675957933, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7833693623542786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784675957935, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784675957935, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784675972784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11201543360948563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195133440, "lr": 9.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675987373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10318779200315475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676002101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10204292088747025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195461120, "lr": 9.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676016909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11599981039762497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195624960, "lr": 9.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676031838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10704582929611206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195788800, "lr": 9.957916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676046600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10482965409755707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676061538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10326029360294342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196116480, "lr": 9.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676076326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1129300594329834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196280320, "lr": 9.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676091563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1177036240696907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196444160, "lr": 9.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676106421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10411347448825836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.999583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676121256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11454513669013977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196771840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676136183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1120765432715416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196935680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676150669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10767719894647598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197099520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676165525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11542575806379318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676165525, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784676165526, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784676246060, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7837756276130676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784676246061, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784676246061, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784676260568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10094389319419861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197427200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676275254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11035434901714325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197591040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676290156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11419522017240524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197754880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676304907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09949523210525513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676319969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09871844947338104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198082560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676334850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10700494050979614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198246400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676349604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09855116158723831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198410240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676364304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10880608856678009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676379141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0996505618095398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198737920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676393524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10563582926988602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198901760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676407961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11481830477714539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199065600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676422454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11003325134515762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676436850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11750580370426178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199393280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676451574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11199474334716797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199557120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676451574, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784676451575, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784676533901, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7843025922775269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784676533902, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784676533902, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784676548580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11099579185247421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199720960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676563265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10498686134815216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676577731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10973504185676575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200048640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676592461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.115827776491642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200212480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676607076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11214172095060349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200376320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676621986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11649005860090256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676636985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09867721050977707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200704000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676652030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1078985407948494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200867840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676667277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10743077099323273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201031680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676682346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1074950322508812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676697611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11785858869552612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201359360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676712655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1107008308172226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201523200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676727625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10132909566164017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201687040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676742808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11034766584634781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676742808, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784676742809, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784676823949, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.783912718296051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784676823950, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784676823950, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784676838405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10551365464925766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202014720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676853246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11098114401102066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202178560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676867979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11878921836614609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202342400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676883103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10597165673971176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676897830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10528165847063065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202670080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676912858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10118559002876282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202833920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676928405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10591786354780197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202997760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676943390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10915226489305496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676958412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11286942660808563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203325440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676973273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09277395159006119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203489280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676988238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11631309986114502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203653120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677003214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10137784481048584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677017750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09838394820690155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203980800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677032369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10780985653400421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204144640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677032370, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784677032370, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784677114375, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7837282419204712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784677114376, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784677114376, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784677129174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10102616250514984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204308480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677144174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09270666539669037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677159020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09539666771888733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204636160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677173944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10843439400196075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204800000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677188905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10734610259532928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204963840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677203849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1126076728105545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677218598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09500134736299515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205291520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677233508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10535935312509537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205455360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677248474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10541889071464539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205619200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677263253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11077417433261871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677278143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09797481447458267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205946880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677293483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09826315194368362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206110720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677308562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1088419035077095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206274560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677323389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09855497628450394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677323390, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784677323390, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784677407209, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7843371033668518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784677407210, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784677407211, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784677422520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1054578423500061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206602240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677437555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09549754112958908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206766080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677452669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10309574007987976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206929920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677468005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09557663649320602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677483337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10282792150974274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207257600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677498322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11206608265638351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207421440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677513424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10511045902967453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207585280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677528298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09467200189828873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677543377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11438558250665665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207912960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677558431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10366898775100708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208076800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677573020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10279970616102219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208240640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677587989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10085694491863251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677602499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10264488309621811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208568320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677617212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10105595737695694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208732160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677617213, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784677617213, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784677699385, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7836166620254517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784677699386, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784677699386, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784677714198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09952596575021744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208896000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677728989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.099375881254673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677743704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1007758229970932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209223680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677758132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1017361730337143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209387520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677772875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09889489412307739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209551360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677787489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09991870820522308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677801888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09653697907924652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209879040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677816306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09879887849092484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210042880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677830913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10349613428115845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210206720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677845498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.107472263276577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677859993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10305097699165344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210534400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677874544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10743246972560883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210698240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677889261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10183820873498917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210862080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677903608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09505340456962585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677903609, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784677903609, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784677986606, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7836195230484009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784677986607, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784677986607, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678001290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09403159469366074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211189760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678015829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09876471757888794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211353600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678030458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09642671048641205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211517440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678044936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10000067204236984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678059357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10370556265115738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211845120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678074351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09507767111063004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212008960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678088957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1018526628613472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212172800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678103666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09764290601015091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678117971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09755999594926834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212500480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678132426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10349329560995102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212664320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678146740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09364781528711319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212828160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678161552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10727233439683914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678176471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10350541770458221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213155840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678191663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084885522723198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213319680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678191664, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784678191664, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784678273662, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7843475341796875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784678273663, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784678273663, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784678288409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10573210567235947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213483520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678303053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10119477659463882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678317663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10464806109666824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213811200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678332646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10232799500226974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213975040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678346856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09606719762086868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214138880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678361202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10467879474163055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678376129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09481097757816315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214466560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678391047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11389254778623581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214630400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678406104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09792779386043549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214794240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678421649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09020402282476425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678436934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09984034299850464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215121920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678452009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10486840456724167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215285760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678466891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1043170616030693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215449600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678481832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10926855355501175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678481833, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784678481834, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784678563770, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7843266129493713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784678563771, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784678563771, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784678578550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10015271604061127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215777280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678593188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10508134961128235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215941120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678607836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09729056060314178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216104960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678622701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09956293553113937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678637675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10006140172481537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216432640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678652302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09359639137983322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216596480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678666998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10402557253837585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216760320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678682161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10212843865156174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678696739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09967504441738129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217088000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678711810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10967189073562622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217251840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678726421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09763708710670471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217415680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678741548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09983526170253754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217579520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678756512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09966250509023666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217743360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678771253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10509949922561646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217907200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678771254, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784678771254, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784678852115, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7843419909477234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784678852116, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784678852116, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784678866962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10071229934692383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218071040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678881835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09461238980293274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218234880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678896702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09786321222782135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218398720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678911337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10729039460420609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218562560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678926427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10293794423341751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218726400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678941248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08700121194124222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218890240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678956382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10442081838846207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219054080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678971449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09255798906087875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219217920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678986587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10467340797185898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219381760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679001822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09510897845029831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219545600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679016934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09473434090614319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219709440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679031900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10254229605197906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219873280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679047076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11097356677055359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220037120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679061957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09735143929719925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220200960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679061957, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784679061958, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784679146572, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7850779891014099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784679146573, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784679146573, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784679160943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09530860930681229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220364800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679175404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09564737230539322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220528640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679189743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10068672895431519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220692480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679204108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10249212384223938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220856320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679218811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10331883281469345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221020160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679233507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10497929155826569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221184000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679248460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10864897072315216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221347840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679262833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11488619446754456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221511680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679277406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10778601467609406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221675520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679292017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09840866178274155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221839360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679306585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11058306694030762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222003200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679321121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0961480662226677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222167040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679335644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09434182196855545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222330880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679350185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10896923393011093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222494720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679350185, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784679350186, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784679434550, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7847976684570312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784679434551, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784679434551, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784679449222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09868940711021423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222658560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679464029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11417931318283081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222822400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679478632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10562597215175629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222986240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679493307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09244155883789062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223150080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679508106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10344358533620834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223313920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679523004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0973079577088356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223477760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679537850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09469249099493027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223641600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679552719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10518466681241989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223805440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679567343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1036195456981659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223969280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679581904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09530769288539886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224133120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679596848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10602153837680817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224296960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679611633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09457076340913773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224460800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679626516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09760872274637222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224624640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679641000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09849347174167633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224788480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679641000, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784679641001, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784679723006, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7852804660797119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784679723007, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784679723007, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784679737975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10739230364561081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224952320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679752702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10219293087720871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225116160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679767479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10475054383277893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225280000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679782641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10053666681051254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225443840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679797525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09687643498182297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225607680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679812026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10623563826084137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225771520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679827020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10629212856292725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225935360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679841590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10676763206720352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226099200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679856459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10835833847522736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226263040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679871432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10426422953605652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226426880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679885922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11001455783843994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226590720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679900405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11179947108030319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226754560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679915692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09257249534130096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226918400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679930639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10408957302570343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227082240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679930640, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784679930641, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784680011847, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7855668663978577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784680011848, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784680011848, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784680026923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1009024977684021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227246080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680042436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10431569069623947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227409920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680057697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11286664754152298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227573760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680073260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10703492164611816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227737600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680088778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10584703087806702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227901440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680104050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10875824093818665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228065280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680119460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11104506999254227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228229120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680134911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10150852799415588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228392960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680150276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10973231494426727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228556800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680165814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10699834674596786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228720640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680180986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10129950195550919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228884480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680196153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11156036704778671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229048320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680211309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10697250813245773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229212160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680226530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10550966113805771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229376000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680226531, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784680226531, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} diff --git a/recommendation_v4/rcp_logs/gbs_8192/seed914924190.log b/recommendation_v4/rcp_logs/gbs_8192/seed914924190.log new file mode 100644 index 000000000..541b42617 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_8192/seed914924190.log @@ -0,0 +1,1840 @@ +:::MLLOG {"namespace": "", "time_ms": 1784651468375, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784651468375, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784651489971, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784651489973, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784651489974, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784651489974, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784651489974, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784651489974, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 8192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784651489974, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784651489974, "event_type": "POINT_IN_TIME", "key": "seed", "value": 914924190, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784651489974, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784651489974, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784651489974, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784651489974, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784651489974, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784651489975, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784652395978, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784652395979, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784652396394, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784652514109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13951851427555084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840, "lr": 7.916666666666666e-10}} +:::MLLOG {"namespace": "", "time_ms": 1784652526454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13819362223148346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.6249999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652539226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13855282962322235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 491520, "lr": 2.458333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652552094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13848203420639038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652565078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13853426277637482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 819200, "lr": 4.125e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652578220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13834579288959503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652591132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13838481903076172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1146880, "lr": 5.7916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652604455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13959240913391113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.625e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652617945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387408822774887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1474560, "lr": 7.458333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652630906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13853862881660461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.291666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652644464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138670414686203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1802240, "lr": 9.124999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652658115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13876578211784363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652671643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13840147852897644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2129920, "lr": 1.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652685001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13907094299793243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652685001, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652685002, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652767451, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49707460403442383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652767455, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652767455, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652780579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13830280303955078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2457600, "lr": 1.2458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652793902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382291615009308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652807796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13837756216526031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2785280, "lr": 1.4124999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652821085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382341980934143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4958333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652834776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13795702159404755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3112960, "lr": 1.5791666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652848652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13793408870697021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652861891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13796277344226837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3440640, "lr": 1.7458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652875274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13797645270824432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652889203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375451534986496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3768320, "lr": 1.9124999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652902713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13801603019237518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9958333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652916696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383693814277649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4096000, "lr": 2.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652930568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13861341774463654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.1624999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652944117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13783757388591766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4423680, "lr": 2.2458333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652958210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13842548429965973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3291666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652958210, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652958211, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784653039161, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4972328543663025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784653039162, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784653039162, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784653052639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376173496246338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4751360, "lr": 2.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653066356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13813334703445435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653080115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13730737566947937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5079040, "lr": 2.5791666666666668e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653094218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13645467162132263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653107775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376720815896988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5406720, "lr": 2.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653121155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13774965703487396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653134627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13702039420604706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5734400, "lr": 2.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653149106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13690844178199768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653162963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13653355836868286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6062080, "lr": 3.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653176566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13739821314811707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653190327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374027132987976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6389760, "lr": 3.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653203757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13751189410686493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653217962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13762672245502472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6717440, "lr": 3.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653231679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13745591044425964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653231680, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784653231680, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784653312642, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49742311239242554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784653312644, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784653312644, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784653325849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13662531971931458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7045120, "lr": 3.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653339468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13741935789585114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653352899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13649596273899078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7372800, "lr": 3.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653367178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13707399368286133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.829166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653381060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13736817240715027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7700480, "lr": 3.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653395027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375395506620407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653408988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13647836446762085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8028160, "lr": 4.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653422778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13736170530319214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653437044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13672098517417908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8355840, "lr": 4.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653450928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13872510194778442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3291666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653464682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361525058746338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8683520, "lr": 4.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653478306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378493309020996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.4958333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653492111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13510721921920776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9011200, "lr": 4.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653506038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363954097032547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.6624999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653506038, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653506039, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653591716, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49773159623146057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653591717, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653591717, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653605341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360086351633072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9338880, "lr": 4.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653618974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13525965809822083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653632317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13730329275131226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9666560, "lr": 4.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653646025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13433951139450073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653659954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13644443452358246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9994240, "lr": 5.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653673629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13319307565689087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1624999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653687562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135467991232872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10321920, "lr": 5.2458333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653701028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13519659638404846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653714861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13777241110801697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10649600, "lr": 5.4124999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653728318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13521729409694672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653741935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14829328656196594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10977280, "lr": 5.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653756072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13540995121002197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653769677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13372507691383362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11304960, "lr": 5.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653783567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13771967589855194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653783568, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653783569, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653869073, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4981797933578491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653869074, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653869074, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653882494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352488398551941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11632640, "lr": 5.9124999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653895990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13994455337524414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653909656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13592812418937683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11960320, "lr": 6.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653923120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13297617435455322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653936548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13474617898464203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12288000, "lr": 6.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653950194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13929271697998047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.329166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653963790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13330358266830444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12615680, "lr": 6.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653977706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13285589218139648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653991453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330314576625824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12943360, "lr": 6.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654004948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13133813440799713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654018632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308990865945816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13271040, "lr": 6.745833333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654032014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13493920862674713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654045648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13178613781929016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13598720, "lr": 6.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654059805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13907888531684875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654059806, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784654059807, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784654144641, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4987562894821167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784654144642, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784654144642, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784654158399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13369795680046082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13926400, "lr": 7.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654172080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13326720893383026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654185946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13571718335151672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14254080, "lr": 7.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654199677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13512013852596283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654213421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13770067691802979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14581760, "lr": 7.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654227258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12750589847564697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654240840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13557960093021393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14909440, "lr": 7.579166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654254691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12935368716716766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654268702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13356485962867737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15237120, "lr": 7.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654282890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13266897201538086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.829166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654296834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13361451029777527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15564800, "lr": 7.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654311160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13436509668827057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.995833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654325504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13534866273403168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15892480, "lr": 8.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654339540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133548304438591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654339541, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654339541, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654423829, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49961382150650024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654423830, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654423830, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654438007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13254708051681519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16220160, "lr": 8.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654452028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13381817936897278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654466209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13550618290901184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16547840, "lr": 8.412500000000001e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654480701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355738341808319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654494867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13960345089435577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16875520, "lr": 8.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654508747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332981437444687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.662499999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654523681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13525442779064178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17203200, "lr": 8.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654537895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13347357511520386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654552206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13290861248970032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17530880, "lr": 8.912499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654566691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13195425271987915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654580819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13596037030220032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17858560, "lr": 9.079166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654595420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13352897763252258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.162499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654609585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13348065316677094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18186240, "lr": 9.245833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654623572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13054174184799194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654623573, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654623573, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654708780, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5006323456764221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654708781, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654708781, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654723434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137940913438797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18513920, "lr": 9.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654737741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13502950966358185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.495833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654752062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13562911748886108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18841600, "lr": 9.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654766478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13117176294326782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.662499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654780779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13373509049415588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19169280, "lr": 9.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654795180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13419216871261597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654809700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1369687020778656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19496960, "lr": 9.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654824149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1340666115283966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654838717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329977959394455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19824640, "lr": 1.0079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654852839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12979090213775635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0162499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654866736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12769101560115814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20152320, "lr": 1.0245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654880730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13306543231010437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654894684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12949402630329132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20480000, "lr": 1.0412499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654908298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1247875988483429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654908299, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654908299, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654990145, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5017739534378052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654990146, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654990146, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784655004274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13116785883903503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20807680, "lr": 1.0579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655018152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13609525561332703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655032350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13326500356197357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21135360, "lr": 1.0745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655046541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13049113750457764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655060221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.128273606300354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21463040, "lr": 1.0912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655074275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12674881517887115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655088504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12892068922519684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21790720, "lr": 1.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655102185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13235504925251007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655115896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353323757648468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22118400, "lr": 1.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655129746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1250692903995514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655143427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12883499264717102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22446080, "lr": 1.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655157022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134393572807312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655170905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13106901943683624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22773760, "lr": 1.1579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655184939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1292581856250763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655184939, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784655184940, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784655268887, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5032806992530823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784655268889, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784655268889, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784655282870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13070078194141388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23101440, "lr": 1.1745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655296643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12924572825431824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655310101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13318179547786713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23429120, "lr": 1.1912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655324591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12379670888185501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655338945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12948788702487946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23756800, "lr": 1.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655352778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1259354203939438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655366725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13016308844089508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24084480, "lr": 1.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655380858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12196028232574463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655394742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13561828434467316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24412160, "lr": 1.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655409018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13723546266555786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655422964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12375228852033615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24739840, "lr": 1.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655437360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12330742925405502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655451557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12565001845359802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25067520, "lr": 1.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655465587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12902140617370605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655465588, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655465589, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655549166, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5050154328346252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655549167, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655549167, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655562695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13642561435699463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25395200, "lr": 1.2912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655577108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13656863570213318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655591100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356521099805832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25722880, "lr": 1.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655604911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13999302685260773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655618737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13662689924240112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26050560, "lr": 1.3245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655632576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13827961683273315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655646514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12740936875343323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26378240, "lr": 1.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655660365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12622682750225067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655674207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14209617674350739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26705920, "lr": 1.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655688467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1411171853542328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655702777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12726716697216034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27033600, "lr": 1.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655716877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12800811231136322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655730961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326916515827179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27361280, "lr": 1.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655744445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13158592581748962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655744445, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655744446, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655828456, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5067891478538513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655828457, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655828457, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655843270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1414201706647873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27688960, "lr": 1.4079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655857653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14239457249641418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655871807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13537077605724335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28016640, "lr": 1.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655886187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13938970863819122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655900675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1451673060655594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28344320, "lr": 1.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655914840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1409745216369629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655929371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1258518397808075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28672000, "lr": 1.4579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655943781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13755619525909424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655958409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13162946701049805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28999680, "lr": 1.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655972969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13603952527046204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655987419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13777753710746765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29327360, "lr": 1.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656001677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13304005563259125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4995833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656016506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13446283340454102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29655040, "lr": 1.5079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656030867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13638289272785187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656030868, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784656030869, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784656113001, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5086406469345093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784656113002, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784656113002, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784656127008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13863995671272278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29982720, "lr": 1.5245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656141460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13789090514183044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656155871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13182172179222107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30310400, "lr": 1.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656170265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13295626640319824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656184681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133761465549469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30638080, "lr": 1.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656199360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13982853293418884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656213929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13300755620002747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30965760, "lr": 1.5745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656228623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14127117395401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656242681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14079280197620392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31293440, "lr": 1.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656257362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13513807952404022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656271642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1277136653661728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31621120, "lr": 1.6079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656285700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13006626069545746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656299703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13412229716777802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31948800, "lr": 1.6245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656313855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12770019471645355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656313856, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784656313857, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784656395777, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5110849142074585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784656395779, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784656395779, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784656409891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13217739760875702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32276480, "lr": 1.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656423861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13691164553165436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656438177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13357456028461456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32604160, "lr": 1.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656452406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13202334940433502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656466868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1339401751756668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32931840, "lr": 1.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656481080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13358932733535767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656495703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13377182185649872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33259520, "lr": 1.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656510135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326957494020462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656524275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13982050120830536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33587200, "lr": 1.7079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656539148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131998211145401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656553361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12880538403987885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33914880, "lr": 1.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656567114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13108496367931366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656581002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13242831826210022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34242560, "lr": 1.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656595002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355794370174408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7495833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656595002, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656595003, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656676633, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5145903825759888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656676634, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656676634, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656690581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13351312279701233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34570240, "lr": 1.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656704456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12299859523773193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656718258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13058851659297943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34897920, "lr": 1.7745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656732131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304146647453308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656745992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13513275980949402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35225600, "lr": 1.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656760329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353587955236435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656774254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355082094669342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35553280, "lr": 1.8079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656788668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12419579923152924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8162500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656802778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1309286504983902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35880960, "lr": 1.8245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656816521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310889571905136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656831204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13035368919372559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36208640, "lr": 1.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656845288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332540512084961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656859450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308698207139969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36536320, "lr": 1.8579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656873385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13001573085784912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656873386, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656873387, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656955088, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5192397236824036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656955089, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656955090, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656969163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13802136480808258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36864000, "lr": 1.8745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656983279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323295682668686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656997482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335235834121704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37191680, "lr": 1.8912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657011877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13055014610290527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657026117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12699496746063232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37519360, "lr": 1.9079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657040456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320924162864685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657055320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13559795916080475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37847040, "lr": 1.9245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657070045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13317333161830902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657084593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13149167597293854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38174720, "lr": 1.9412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657099375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388409584760666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657113941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13381518423557281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38502400, "lr": 1.9579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657127887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375548392534256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657142653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13193529844284058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38830080, "lr": 1.9745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657156734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334007829427719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657156735, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784657156735, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784657239924, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5258198380470276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784657239925, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784657239925, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784657253853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13107047975063324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39157760, "lr": 1.9912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657267668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13486658036708832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657281593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377488523721695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39485440, "lr": 2.0079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657295636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13322030007839203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657309549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1275329887866974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39813120, "lr": 2.0245833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657323535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12918399274349213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0329166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657337450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12603016197681427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40140800, "lr": 2.0412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657351331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12419457733631134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657365457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13616912066936493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40468480, "lr": 2.0579166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657379712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13730286061763763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657393943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13463373482227325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40796160, "lr": 2.0745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657408781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1269654929637909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0829166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657423220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13101130723953247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41123840, "lr": 2.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657436859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1235862448811531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657436859, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657436860, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657520321, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5354706645011902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657520322, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657520323, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657534773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13242857158184052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41451520, "lr": 2.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657548875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13244673609733582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657563267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13374757766723633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41779200, "lr": 2.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657577625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377996802330017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657591737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13621146976947784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42106880, "lr": 2.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657606029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13132013380527496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657620366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1272916942834854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42434560, "lr": 2.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657634868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13484273850917816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657649065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12455197423696518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42762240, "lr": 2.1745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657663544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304454505443573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657677978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13234888017177582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43089920, "lr": 2.1912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657692485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13406121730804443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657706985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14014868438243866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43417600, "lr": 2.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657721214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13618884980678558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657721214, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657721215, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657801888, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5501266717910767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657801889, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657801889, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657816867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13665083050727844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43745280, "lr": 2.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657831220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1314096450805664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657845259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324666440486908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44072960, "lr": 2.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657859667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387152075767517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657874588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12996986508369446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44400640, "lr": 2.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657889684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382867395877838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657904303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347719132900238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44728320, "lr": 2.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657919053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1369338184595108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657933827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13458847999572754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45056000, "lr": 2.2912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657948682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13061945140361786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657963432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1402703821659088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45383680, "lr": 2.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657978150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13450050354003906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657993010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12880362570285797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45711360, "lr": 2.3245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658008014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13322190940380096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658008014, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784658008015, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784658089452, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5667417645454407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784658089454, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784658089454, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784658104334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14171868562698364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46039040, "lr": 2.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658119153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13259850442409515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658134488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13327069580554962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46366720, "lr": 2.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658149519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13065870106220245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658164609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13511024415493011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46694400, "lr": 2.3745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658179556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330936849117279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658194053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13383358716964722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47022080, "lr": 2.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658208687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351194828748703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658223218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310320347547531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47349760, "lr": 2.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658237915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324935406446457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658252590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326628178358078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47677440, "lr": 2.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658267176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366458386182785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658281387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359609216451645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48005120, "lr": 2.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658296024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13686227798461914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.4495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658296025, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784658296025, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784658377506, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5840389132499695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784658377507, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784658377507, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784658392056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13118776679039001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48332800, "lr": 2.4579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658406879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13469569385051727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658421402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13356541097164154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48660480, "lr": 2.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658436377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315343827009201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658450781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13651368021965027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48988160, "lr": 2.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658465109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12919391691684723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658480038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13319405913352966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49315840, "lr": 2.507916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658494328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12312334775924683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.5162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658508830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12993106245994568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49643520, "lr": 2.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658523335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13087566196918488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658537841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13006234169006348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49971200, "lr": 2.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658552281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13046598434448242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.5495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658566810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12808293104171753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50298880, "lr": 2.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658581272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12985174357891083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.5662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658581273, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658581273, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658661563, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.597716748714447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658661563, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658661564, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658675716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12123583257198334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50626560, "lr": 2.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658690165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1262061446905136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658704548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1277758926153183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50954240, "lr": 2.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658719012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13236358761787415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658733901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1306033879518509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51281920, "lr": 2.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658748566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12086113542318344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658763407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294410079717636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51609600, "lr": 2.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658778337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1306142508983612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658793228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13561289012432098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51937280, "lr": 2.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658807391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1269514560699463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.6495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658822520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131390780210495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52264960, "lr": 2.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658836842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11679404228925705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.6662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658851214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11983849853277206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52592640, "lr": 2.6745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658865656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.123848095536232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658865657, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658865657, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658945977, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6089483499526978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658945979, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658945979, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658960297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12016788125038147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52920320, "lr": 2.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658974545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1258222609758377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.6995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658989113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1270771324634552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53248000, "lr": 2.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659003700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13002334535121918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659017865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11932457983493805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53575680, "lr": 2.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659032074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13354304432868958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659046171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12764734029769897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53903360, "lr": 2.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659060260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11971744894981384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.7495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659074727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12553542852401733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54231040, "lr": 2.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659089289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12532325088977814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659103497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11615294963121414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54558720, "lr": 2.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659118254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298445761203766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659132817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12553173303604126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54886400, "lr": 2.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659147335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1274186074733734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659147336, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784659147336, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784659228632, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6194322109222412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784659228633, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784659228633, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784659242604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1253458857536316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55214080, "lr": 2.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659257606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11656416207551956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659272162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12034222483634949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55541760, "lr": 2.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659286461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11922803521156311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659300672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12289102375507355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55869440, "lr": 2.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659314888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12420984357595444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659329005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11406473815441132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56197120, "lr": 2.8579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659343062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12138283252716064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659357137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12331665307283401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56524800, "lr": 2.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659371186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1205926388502121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659385077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11343570053577423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56852480, "lr": 2.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659399102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11981081962585449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659413086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10712503641843796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57180160, "lr": 2.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659427320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1221344992518425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659427320, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659427321, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659510133, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.631496250629425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659510134, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659510134, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659524495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11436984688043594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57507840, "lr": 2.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659539004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11210490018129349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659553143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12659451365470886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57835520, "lr": 2.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659568058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11896269023418427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659582426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12106858938932419, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58163200, "lr": 2.9579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659596894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11364932358264923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659610835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11667665839195251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58490880, "lr": 2.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659625988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11942015588283539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659640408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1187303438782692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58818560, "lr": 2.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659654895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11067013442516327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659669396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11761705577373505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59146240, "lr": 3.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659683800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11983279138803482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.0162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659698265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12262414395809174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59473920, "lr": 3.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659712806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11638651043176651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659712806, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659712807, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659795585, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.654762864112854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659795586, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659795586, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659809874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11594486981630325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59801600, "lr": 3.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659824129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10968119651079178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659838316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10740665346384048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60129280, "lr": 3.0579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659852620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11154486238956451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.0662499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659867053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10652945190668106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60456960, "lr": 3.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659881723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12189936637878418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659896433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11418909579515457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60784640, "lr": 3.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659911409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11817489564418793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659926084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09919141232967377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61112320, "lr": 3.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659941150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11925877630710602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.1162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659956147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11110080033540726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61440000, "lr": 3.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659971228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11235302686691284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659986029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793550312519073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61767680, "lr": 3.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660000805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11593995243310928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660000805, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784660000806, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784660083148, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6963437795639038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784660083149, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784660083149, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784660097313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11905448138713837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62095360, "lr": 3.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660111788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1194983646273613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.1662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660126386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12191729247570038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62423040, "lr": 3.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660140880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10365217924118042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660155287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11272569000720978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62750720, "lr": 3.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660169684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11681342124938965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660184126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11955440044403076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63078400, "lr": 3.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660198513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11813142150640488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660212681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10781590640544891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63406080, "lr": 3.2245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660226799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11669033020734787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660240962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11498109996318817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63733760, "lr": 3.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660254917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11550657451152802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660268946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11147318035364151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64061440, "lr": 3.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660283392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1254209280014038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660283392, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784660283393, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784660366271, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.729306697845459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784660366272, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784660366272, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784660380797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12321529537439346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64389120, "lr": 3.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660395377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1152920350432396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660409669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11614107340574265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64716800, "lr": 3.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660424458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12055301666259766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660438953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10885833948850632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65044480, "lr": 3.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660453319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1100258082151413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660467379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111620232462883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65372160, "lr": 3.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660482424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10790517926216125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660497569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11062797904014587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65699840, "lr": 3.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660513219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11547921597957611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660528434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12294028699398041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66027520, "lr": 3.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660543492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11482632905244827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660558831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12679077684879303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66355200, "lr": 3.3745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660574182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11135312914848328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660574183, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660574183, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660656073, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.749374270439148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660656073, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660656074, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660671382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12592677772045135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66682880, "lr": 3.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660686470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11298660188913345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.3995833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660701445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11713558435440063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67010560, "lr": 3.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660716816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1122908815741539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660731949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11885841935873032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67338240, "lr": 3.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660747425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11802258342504501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660763118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12198573350906372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67665920, "lr": 3.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660778608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11951153725385666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660793992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11682331562042236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67993600, "lr": 3.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660809646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11346852779388428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660824834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11578354984521866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68321280, "lr": 3.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660839796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11607906222343445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660854665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1253659874200821, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68648960, "lr": 3.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660869748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11433738470077515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660869749, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660869749, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660952189, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7550464868545532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660952190, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660952190, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660966975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12052910029888153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68976640, "lr": 3.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660981924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11988215148448944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660996784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11354975402355194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69304320, "lr": 3.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661011890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11399007588624954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661026666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10768643021583557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69632000, "lr": 3.5412500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661041501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11408544331789017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661056313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11708945035934448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69959680, "lr": 3.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661070825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11100886762142181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661085739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1092962920665741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70287360, "lr": 3.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661100433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10756061971187592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661115519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12273620069026947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70615040, "lr": 3.5912500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661130637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10577965527772903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661145648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12197785824537277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70942720, "lr": 3.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661160940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12049926817417145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661160941, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784661160941, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784661244925, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7584042549133301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784661244927, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784661244927, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784661259928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11893458664417267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71270400, "lr": 3.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661274700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11993011087179184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661289450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1145906075835228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71598080, "lr": 3.6412499999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661305153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11546212434768677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661320243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10264112055301666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71925760, "lr": 3.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661335294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10448382794857025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661350243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11926914751529694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72253440, "lr": 3.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661365043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11485368758440018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661379848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10331116616725922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72581120, "lr": 3.6912499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661394578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10312729328870773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661409150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10695435851812363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72908800, "lr": 3.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661423893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11214709281921387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661438616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11181969940662384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73236480, "lr": 3.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661453328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11482857167720795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661453382, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661453382, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661538413, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7609966993331909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661538414, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661538414, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661553119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10475766658782959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73564160, "lr": 3.7412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661568142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10692141950130463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661583300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11610450595617294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73891840, "lr": 3.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661598090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11034782230854034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661613078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072847992181778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74219520, "lr": 3.7745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661628363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11457620561122894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661643268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11302147805690765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74547200, "lr": 3.7912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661657872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11192953586578369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661672839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11615899205207825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74874880, "lr": 3.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661687383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1111089289188385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661702015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10219135135412216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75202560, "lr": 3.8245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661716660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11339563131332397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661731169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1074664369225502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75530240, "lr": 3.8412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661745855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10706400126218796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661745856, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661745856, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661829611, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7616677284240723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661829612, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661829612, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661844178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10644214600324631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75857920, "lr": 3.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661858933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11413531005382538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661873659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09591548144817352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76185600, "lr": 3.8745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661888234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1037893146276474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661902730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11315039545297623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76513280, "lr": 3.8912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661917115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10508042573928833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661931784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08885429799556732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76840960, "lr": 3.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661946255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10938652604818344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661960837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10209403932094574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77168640, "lr": 3.9245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661975543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1162818968296051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661990446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10724795609712601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77496320, "lr": 3.9412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662004973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10102424025535583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662020036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10145585983991623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77824000, "lr": 3.9579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662034968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10765603929758072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662034969, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784662034969, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784662117821, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7627230286598206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784662117822, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784662117822, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784662132424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11666707694530487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78151680, "lr": 3.9745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662147085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11216597259044647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662161063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10321582108736038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78479360, "lr": 3.9912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662176181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10610475391149521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662190622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09960445016622543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78807040, "lr": 4.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662205390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10347115993499756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662219818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10669748485088348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79134720, "lr": 4.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662234500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1083209440112114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662249009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09805843234062195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79462400, "lr": 4.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662263676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10889076441526413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662278046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10622848570346832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79790080, "lr": 4.0579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662292467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10739195346832275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662306890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10868535190820694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80117760, "lr": 4.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662321338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11396551877260208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662321339, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784662321339, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784662404423, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7608395218849182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784662404424, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784662404424, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784662419015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11174838989973068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80445440, "lr": 4.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662433330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09973549842834473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662447853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10277803987264633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80773120, "lr": 4.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662462389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10618622601032257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662477322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11110015958547592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81100800, "lr": 4.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662492092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11402767151594162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662506651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09963998198509216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81428480, "lr": 4.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662521405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10695674270391464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662536329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10775955766439438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81756160, "lr": 4.1579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662550696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1078806146979332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662565359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11239707469940186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82083840, "lr": 4.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662579472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09402789920568466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662594599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.098151795566082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82411520, "lr": 4.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662609004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10761359333992004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662609004, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662609005, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662692292, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7633724212646484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662692293, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662692293, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662706667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09221290796995163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82739200, "lr": 4.2079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662721418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10456705093383789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662735916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11057783663272858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83066880, "lr": 4.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662750767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0968879833817482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662765195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11097344011068344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83394560, "lr": 4.2412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662779873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11074347048997879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662794515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09740129113197327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83722240, "lr": 4.2579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662809017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11151837557554245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662823659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1069561019539833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84049920, "lr": 4.2745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662838270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10577663034200668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662853128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1090664267539978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84377600, "lr": 4.2912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662867412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1051446869969368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662881720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09785065054893494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84705280, "lr": 4.3079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662896363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10728604346513748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662896364, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662896364, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662979777, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7627716064453125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662979778, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662979778, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662994597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10440170019865036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85032960, "lr": 4.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663009553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10572925955057144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663024034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1058088019490242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85360640, "lr": 4.3412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663038827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10500079393386841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663053775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09892479330301285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85688320, "lr": 4.3579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663068317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10069633275270462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663083522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10657774657011032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86016000, "lr": 4.3745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663097822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09618804603815079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663112765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10838782042264938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86343680, "lr": 4.3912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663127657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10851167142391205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663142680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11627639085054398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86671360, "lr": 4.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663157763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11476539820432663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663172636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11649604141712189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86999040, "lr": 4.4245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663187501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11593416333198547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663187502, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784663187502, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784663271311, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7649332284927368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784663271312, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784663271312, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784663285929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10165029019117355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87326720, "lr": 4.4412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663300531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11584054678678513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663314858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10806572437286377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87654400, "lr": 4.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663329451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10186579078435898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663343984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.091153085231781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87982080, "lr": 4.4745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663358682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09901586174964905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663373202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10478005558252335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88309760, "lr": 4.4912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663387920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10737283527851105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663402392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10701005160808563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88637440, "lr": 4.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663416943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10064917057752609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663432061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11469180881977081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88965120, "lr": 4.5245833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663447072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10082987695932388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663462044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1101938784122467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89292800, "lr": 4.5412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663476647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11212252080440521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.5495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663476647, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663476648, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663559025, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7635629773139954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663559026, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663559026, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663573713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10842915624380112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89620480, "lr": 4.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663588433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1023298054933548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663603044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11100520938634872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89948160, "lr": 4.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663618131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09234896302223206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663632700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1148853600025177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90275840, "lr": 4.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663647191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10696152597665787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663662407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11239062994718552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90603520, "lr": 4.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663677588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11207226663827896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663693192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12206222116947174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90931200, "lr": 4.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663708620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1055164709687233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663724024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1128450408577919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91258880, "lr": 4.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663739557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12319038063287735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663754967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0995035395026207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91586560, "lr": 4.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663770239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11088892817497253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663770240, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663770240, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663851712, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7636770606040955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663851713, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663851713, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663866775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10414852946996689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91914240, "lr": 4.6745833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663882109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11911071836948395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663897316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10918042808771133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92241920, "lr": 4.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663912828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1083282083272934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663928200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09692811965942383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92569600, "lr": 4.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663943652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11200051009654999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663959336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10653311759233475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92897280, "lr": 4.7245833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663974986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10634110867977142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663990274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11715427041053772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93224960, "lr": 4.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664005270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10033764690160751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664020727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1254924088716507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93552640, "lr": 4.7579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664035862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10875643044710159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664050662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11829700320959091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93880320, "lr": 4.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664065421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10808556526899338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664065422, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784664065422, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784664147277, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7634211182594299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784664147278, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784664147278, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784664162349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11417596787214279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94208000, "lr": 4.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664177453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11557183414697647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664192422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11184106767177582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94535680, "lr": 4.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664207578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11848663538694382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664222531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11471235752105713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94863360, "lr": 4.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664237579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11965975165367126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664252749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12721100449562073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95191040, "lr": 4.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664267439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1211310550570488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664282167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11690991371870041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95518720, "lr": 4.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664297153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11044881492853165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664311853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10819205641746521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95846400, "lr": 4.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664326964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10859448462724686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664342018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11575690656900406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96174080, "lr": 4.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664357196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11465172469615936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664357197, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664357197, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664437991, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7659510374069214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664437992, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664437992, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664453183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11958649009466171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96501760, "lr": 4.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664468302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10836153477430344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664483192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10899878293275833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96829440, "lr": 4.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664498560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10842183232307434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664513795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11150064319372177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97157120, "lr": 4.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664528946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11017884314060211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664544151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11389236152172089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97484800, "lr": 4.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664559309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10761227458715439, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664574057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10687749832868576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97812480, "lr": 4.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664588469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1006854921579361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664603414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10476640611886978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98140160, "lr": 4.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664618000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09717684239149094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664632611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10501432418823242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98467840, "lr": 5.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664647105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10089010000228882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664647106, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664647107, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664727540, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7678549885749817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664727541, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664727541, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664742041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10962226986885071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98795520, "lr": 5.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664756689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10251064598560333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664771015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09690315276384354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99123200, "lr": 5.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664785547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10866735130548477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664800271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1151350885629654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99450880, "lr": 5.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664814789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1048690602183342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664829236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10547244548797607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99778560, "lr": 5.074583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664843836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10513937473297119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664858350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10219498723745346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100106240, "lr": 5.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664873059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12082646787166595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664887808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11711021512746811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100433920, "lr": 5.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664902458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10148157924413681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664917338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10121981799602509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100761600, "lr": 5.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664931709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12046225368976593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664931710, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664931710, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784665012784, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7680723071098328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784665012785, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784665012785, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784665027811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11144648492336273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101089280, "lr": 5.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665042494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09666293859481812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665057265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10501262545585632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101416960, "lr": 5.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665072023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10360636562108994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665086466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11103497445583344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101744640, "lr": 5.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665101381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10557538270950317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.182916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665116324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10661054402589798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102072320, "lr": 5.191249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665131190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11239687353372574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665146305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10485448688268661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102400000, "lr": 5.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665161233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10847975313663483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665176036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10611219704151154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102727680, "lr": 5.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665190747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1151164323091507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665205229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10677815973758698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103055360, "lr": 5.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665219812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10274917632341385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665219813, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784665219813, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784665301367, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.768868625164032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784665301368, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784665301368, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784665316023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1005469560623169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103383040, "lr": 5.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665330889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09514143317937851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665345503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09628257900476456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103710720, "lr": 5.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665360277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0973416268825531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.282916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665374843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09738261997699738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104038400, "lr": 5.291249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665389849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0938575267791748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665404454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10039786994457245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104366080, "lr": 5.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665419100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09716975688934326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665434001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10314284265041351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104693760, "lr": 5.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665448839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10948928445577621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665463993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1049174889922142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105021440, "lr": 5.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665478745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10401095449924469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665493628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11403292417526245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105349120, "lr": 5.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665508851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11228300631046295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665508852, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665508853, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665590716, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7696759700775146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665590717, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665590717, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665605649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09893445670604706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105676800, "lr": 5.374583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665620448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10815548151731491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.382916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665635604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11002690345048904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106004480, "lr": 5.391249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665650301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10190070420503616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665665166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1011393815279007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106332160, "lr": 5.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665680100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10807599127292633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665694863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11114687472581863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106659840, "lr": 5.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665709823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10691943019628525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665724847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10288527607917786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106987520, "lr": 5.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665739274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10162779688835144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665753680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1003788635134697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107315200, "lr": 5.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665768209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10305988043546677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665782813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11081109941005707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107642880, "lr": 5.474583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665797327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11426202952861786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.482916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665797327, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665797328, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665879711, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7689056396484375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665879711, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665879712, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665894377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11046869307756424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107970560, "lr": 5.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665908884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10699737071990967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665923440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09228158742189407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108298240, "lr": 5.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665937946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10344073176383972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665952658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10649328678846359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108625920, "lr": 5.524583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665967166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10823854804039001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665981695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11077644675970078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108953600, "lr": 5.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665996276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10383147746324539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666011228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10022103041410446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109281280, "lr": 5.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666026051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10400830209255219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666041009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10341557115316391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109608960, "lr": 5.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666055773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10895240306854248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666070974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10655924677848816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109936640, "lr": 5.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666086143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10932396352291107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666086144, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784666086145, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784666168834, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7702418565750122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784666168834, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784666168835, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784666183897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10282351821660995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110264320, "lr": 5.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666198736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10402597486972809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666213300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10445063561201096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110592000, "lr": 5.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666227586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10308480262756348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666242920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10731486231088638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110919680, "lr": 5.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666257644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10131070762872696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666272380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09898210316896439, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111247360, "lr": 5.657916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666287525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10211844742298126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.666250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666302424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11229649931192398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111575040, "lr": 5.674583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666317212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1049850583076477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666331695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10358358919620514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111902720, "lr": 5.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666346243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10234571248292923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666361166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1047801822423935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112230400, "lr": 5.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666375803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10757316648960114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666375803, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666375804, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666459123, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7700169086456299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666459124, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666459125, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666474001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10842284560203552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112558080, "lr": 5.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666488899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10183333605527878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666503574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10413434356451035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112885760, "lr": 5.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666518377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10152316093444824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666533280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10563187301158905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113213440, "lr": 5.757916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666547838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11234840750694275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.766250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666562737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10137961804866791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113541120, "lr": 5.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666577513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11044442653656006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666592608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11361492425203323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113868800, "lr": 5.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666607842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10651081800460815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666622809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10347880423069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114196480, "lr": 5.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666637653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10480798035860062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666652730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1053089126944542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114524160, "lr": 5.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666667722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09939322620630264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666667723, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666667723, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666750761, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7707329392433167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666750762, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666750762, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666765509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10240296274423599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114851840, "lr": 5.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666780211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09992178529500961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666794921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10694962739944458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115179520, "lr": 5.857916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666809227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10115089267492294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666824574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10841885954141617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115507200, "lr": 5.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666839257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11003818362951279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666854158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10708703100681305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115834880, "lr": 5.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666869337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11601662635803223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666884249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11402375251054764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116162560, "lr": 5.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666899097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10093954205513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666913810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10443145036697388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116490240, "lr": 5.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666928516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10371007770299911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666943220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1052113026380539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116817920, "lr": 5.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666957726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10829927772283554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666957726, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666957727, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784667041553, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.770381510257721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784667041553, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784667041554, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784667056302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10228724032640457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117145600, "lr": 5.957916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667071093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10028024017810822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667086023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10912340134382248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117473280, "lr": 5.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667100668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10530008375644684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667115154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09444864094257355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117800960, "lr": 5.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667130408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11311542242765427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667145130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10190565139055252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118128640, "lr": 6.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667159836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10757552087306976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667174884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10355568677186966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118456320, "lr": 6.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667190082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10386643558740616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667204915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10963695496320724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118784000, "lr": 6.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667220041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11481548100709915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667235070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10711006075143814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119111680, "lr": 6.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667250231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10348254442214966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667250232, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784667250232, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784667331688, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7708614468574524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784667331689, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784667331689, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784667347215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11297699064016342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119439360, "lr": 6.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667362322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10944531857967377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667377277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11018601804971695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119767040, "lr": 6.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667392254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071934774518013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667406845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10158787667751312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120094720, "lr": 6.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667421388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10794094949960709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667436950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11872021853923798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120422400, "lr": 6.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667451943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10895240306854248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667466972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11659026890993118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120750080, "lr": 6.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667482399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1230560690164566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667497510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10768458247184753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121077760, "lr": 6.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667512487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11111745983362198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667527871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1156846284866333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121405440, "lr": 6.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667542959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11071041226387024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667542960, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667542961, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667625347, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7696937918663025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667625348, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667625348, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667640714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11376965045928955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121733120, "lr": 6.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667656081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10974347591400146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667671275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11160551011562347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122060800, "lr": 6.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667686529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10673508793115616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667701680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10439173132181168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122388480, "lr": 6.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667717044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1144833192229271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667732220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10747753828763962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122716160, "lr": 6.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667747401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1150098517537117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667763042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11219964176416397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123043840, "lr": 6.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667778510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12021785974502563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667793915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10593871027231216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123371520, "lr": 6.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667808977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11749924719333649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667824221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10760162025690079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123699200, "lr": 6.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667839732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11970734596252441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667839733, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667839734, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667922319, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7705992460250854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667922320, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667922320, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667937625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12023020535707474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124026880, "lr": 6.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667952729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10861573368310928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667967753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1200103834271431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124354560, "lr": 6.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667982485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11380978673696518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667997373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10809537768363953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124682240, "lr": 6.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668012719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11547377705574036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668027447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11263449490070343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125009920, "lr": 6.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668042560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11751175671815872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668057568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1141408234834671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125337600, "lr": 6.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668072352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10643111914396286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668087259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11340641230344772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125665280, "lr": 6.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668102322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10542149841785431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668116986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11699880659580231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125992960, "lr": 6.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668131876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10573676228523254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668131877, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784668131877, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784668215129, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7712219953536987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784668215130, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784668215131, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784668229875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11019653081893921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126320640, "lr": 6.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668245096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12196414172649384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668259779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11053057760000229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126648320, "lr": 6.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668274656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11400251090526581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668289547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11607448011636734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126976000, "lr": 6.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668304420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10958588868379593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668319453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1088619977235794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127303680, "lr": 6.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668334408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11036823689937592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.482916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668349501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11283598840236664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127631360, "lr": 6.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668364270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11281221359968185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668379249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11051535606384277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127959040, "lr": 6.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668394404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11122410744428635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668409397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11192701756954193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128286720, "lr": 6.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668424359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10933899879455566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668424360, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668424360, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668506172, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7713875770568848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668506173, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668506173, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668520952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1180403009057045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128614400, "lr": 6.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668536168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11082270741462708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668551243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11096109449863434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128942080, "lr": 6.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668566066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10347817093133926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668580941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10579667240381241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129269760, "lr": 6.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668596043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1119762435555458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668610897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10837054252624512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129597440, "lr": 6.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668625592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10862725228071213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668640448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0989568829536438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129925120, "lr": 6.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668655216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1023208498954773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668670020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10541413724422455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130252800, "lr": 6.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668684528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10507814586162567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668699374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11016174405813217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130580480, "lr": 6.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668714322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11673317849636078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668714323, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668714323, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668795402, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7719801068305969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668795403, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668795403, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668810139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11537590622901917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130908160, "lr": 6.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668825036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10590831935405731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668839978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10055369138717651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131235840, "lr": 6.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668855069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09261589497327805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668869916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11435283720493317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131563520, "lr": 6.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668884599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10578653216362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668899512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10269372910261154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131891200, "lr": 6.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668914676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11260104924440384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668929456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10744769871234894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132218880, "lr": 6.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668944405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10385572910308838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668959301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10999999195337296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132546560, "lr": 6.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668974276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11317694187164307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668989278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772264003753662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132874240, "lr": 6.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669004141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772299766540527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669004141, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784669004142, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784669087191, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7719723582267761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784669087192, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784669087192, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784669102023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11018897593021393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133201920, "lr": 6.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669117164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10327742993831635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669131954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11064589768648148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133529600, "lr": 6.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669146717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1002219170331955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669162028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11641030013561249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133857280, "lr": 6.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669176672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11038776487112045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669191243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10409452021121979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134184960, "lr": 6.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669206224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10834421217441559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669221056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10862310230731964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134512640, "lr": 6.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669235357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10871060937643051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669249793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11066107451915741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134840320, "lr": 6.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669264184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09973981231451035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669278588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10766025632619858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135168000, "lr": 6.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669292973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08807706832885742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669292973, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669292974, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669375911, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7718803286552429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669375912, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669375912, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669390460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09531928598880768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135495680, "lr": 6.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669405121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11135758459568024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669419727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10857824236154556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135823360, "lr": 6.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669434323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10211155563592911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669449022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1026914119720459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136151040, "lr": 6.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669463913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09057770669460297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669478546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1035737618803978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136478720, "lr": 6.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669492976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09845393896102905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669507506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10382619500160217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136806400, "lr": 6.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669522313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1066284030675888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669537027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11097100377082825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137134080, "lr": 6.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669551713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10829605907201767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669566480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10643890500068665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137461760, "lr": 6.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669581093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1037822738289833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669581093, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669581094, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669665904, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7719734311103821, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669665905, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669665905, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669680634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0971701592206955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137789440, "lr": 7.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669695413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1056327372789383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669710024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09511767327785492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138117120, "lr": 7.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669724691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10303428024053574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669739361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09866984188556671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138444800, "lr": 7.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669753717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10076683759689331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669767983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10913608223199844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138772480, "lr": 7.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669783349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10151934623718262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.066249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669797904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10965941101312637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139100160, "lr": 7.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669812720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11025281250476837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669827993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10625021159648895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139427840, "lr": 7.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669843063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10498633235692978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669857946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10944652557373047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139755520, "lr": 7.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669872833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10911870747804642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669872834, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669872834, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669956677, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7723056077957153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669956678, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669956678, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669971367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10736892372369766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140083200, "lr": 7.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669986129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10479751229286194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670000801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10368064045906067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140410880, "lr": 7.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670015586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10081726312637329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670030426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10706590116024017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140738560, "lr": 7.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670045068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09619719535112381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670060457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10576245933771133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141066240, "lr": 7.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670075160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10419678688049316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670089779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1107771173119545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141393920, "lr": 7.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670104819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09936597943305969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670119361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09826915711164474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141721600, "lr": 7.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670134131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10279584676027298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670148967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09161832183599472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142049280, "lr": 7.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670164018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.108053058385849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670164019, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784670164019, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784670248281, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7723597884178162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784670248282, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784670248282, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784670263238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10001012682914734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142376960, "lr": 7.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670278340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10192383825778961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670293100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09920132160186768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142704640, "lr": 7.257916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670308117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11041120439767838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670323072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10160984843969345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143032320, "lr": 7.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670338179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10275978595018387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670352969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10300175845623016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143360000, "lr": 7.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670367901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10545822978019714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670383137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10209396481513977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143687680, "lr": 7.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670397793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11479848623275757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670413121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10130627453327179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144015360, "lr": 7.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670427666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10923541337251663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670442208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10603763163089752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144343040, "lr": 7.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670457059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10683832317590714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670457060, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670457061, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670540742, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7723338007926941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670540743, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670540744, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670555538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072632223367691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144670720, "lr": 7.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670570061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09867840260267258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670584469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10728270560503006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144998400, "lr": 7.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670599221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10715340077877045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670613996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10306189954280853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145326080, "lr": 7.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670628286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1074538379907608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670642806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09747222065925598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145653760, "lr": 7.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670657577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09841570258140564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670672174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10588046908378601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145981440, "lr": 7.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670686793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10585835576057434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670701443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10364833474159241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146309120, "lr": 7.441250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670715925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11059852689504623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670730365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10672973841428757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146636800, "lr": 7.457916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670744797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1005413755774498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670744798, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670744798, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670828544, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7726564407348633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670828545, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670828545, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670843094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10412900894880295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146964480, "lr": 7.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670857504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10562823712825775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670871941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09473623335361481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147292160, "lr": 7.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670886853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09929139167070389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670901306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10275126248598099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147619840, "lr": 7.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670916160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10083019733428955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670930638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10688253492116928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147947520, "lr": 7.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670945571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10984687507152557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670960565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1094331368803978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148275200, "lr": 7.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670975424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10019391775131226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670990264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10356694459915161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148602880, "lr": 7.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671004616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09939266741275787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671019489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11213545501232147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148930560, "lr": 7.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671033861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11139510571956635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671033861, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784671033862, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784671118621, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.772913932800293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784671118622, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784671118622, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784671132999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10620784014463425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149258240, "lr": 7.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671148514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10502221435308456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671163473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10636048763990402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149585920, "lr": 7.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671178618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11071683466434479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149749760, "lr": 7.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671194197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10630536824464798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149913600, "lr": 7.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671209593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10865778475999832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671224426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085481196641922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150241280, "lr": 7.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671239195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09819760918617249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150405120, "lr": 7.649583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671254058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1068880558013916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150568960, "lr": 7.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671268751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10329602658748627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671283681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10880228877067566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150896640, "lr": 7.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671298379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10398140549659729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151060480, "lr": 7.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671313564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10685034096240997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151224320, "lr": 7.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671328621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09633220732212067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671328671, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671328672, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671413118, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7726266980171204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671413119, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671413119, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671428115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10019858926534653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151552000, "lr": 7.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671443099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10589537769556046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151715840, "lr": 7.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671458193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10192804038524628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151879680, "lr": 7.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671473377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10440424829721451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671488407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10855840146541595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152207360, "lr": 7.741249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671503250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12445464730262756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152371200, "lr": 7.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671518228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10846596211194992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152535040, "lr": 7.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671533294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09949006885290146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671548429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1126919761300087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152862720, "lr": 7.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671563321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11228539049625397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153026560, "lr": 7.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671578535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1148359403014183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153190400, "lr": 7.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671593789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09906880557537079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671609003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725883394479752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153518080, "lr": 7.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671623929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10529863089323044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153681920, "lr": 7.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671623973, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671623974, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671707505, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7731543183326721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671707506, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671707506, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671722573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09904955327510834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153845760, "lr": 7.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671737509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09827695786952972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.832916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671752352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11316066980361938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154173440, "lr": 7.841249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671767501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11808893084526062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154337280, "lr": 7.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671782229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11412468552589417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154501120, "lr": 7.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671796796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11114463210105896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671812273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10823369771242142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154828800, "lr": 7.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671827292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12048657238483429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154992640, "lr": 7.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671842234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10539118200540543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155156480, "lr": 7.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671857631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10855741798877716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671872841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11257130652666092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155484160, "lr": 7.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671887629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11081510037183762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155648000, "lr": 7.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671902639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11368443816900253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155811840, "lr": 7.924583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671917612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11170050501823425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.932916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671917612, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671917613, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784672000936, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730793356895447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784672000937, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784672000937, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784672015807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10737687349319458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156139520, "lr": 7.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672030732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11081668734550476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156303360, "lr": 7.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672045735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11447794735431671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156467200, "lr": 7.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672060775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11095447093248367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672075860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10894253849983215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156794880, "lr": 7.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672091132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10854391753673553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156958720, "lr": 7.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672106069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1014312207698822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157122560, "lr": 7.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672120993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11224207282066345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672136121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12022126466035843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157450240, "lr": 8.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672151247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11039496958255768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157614080, "lr": 8.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672166230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.108949676156044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157777920, "lr": 8.024583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672181595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10468072444200516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.032916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672196944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10430964082479477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158105600, "lr": 8.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672212215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10911089926958084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158269440, "lr": 8.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672212215, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784672212216, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784672295227, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7731490731239319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784672295228, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784672295228, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784672310278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11452697962522507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158433280, "lr": 8.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672325592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11933420598506927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672340891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11448374390602112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158760960, "lr": 8.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672356275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11250210553407669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158924800, "lr": 8.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672371601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11134026199579239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159088640, "lr": 8.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672386620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12185366451740265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672402002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11207068711519241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159416320, "lr": 8.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672416826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10939701646566391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159580160, "lr": 8.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672432505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11091728508472443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159744000, "lr": 8.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672447602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09911589324474335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672462790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10985645651817322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160071680, "lr": 8.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672478375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10954012721776962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160235520, "lr": 8.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672493739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1042756661772728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160399360, "lr": 8.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672508689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11041856557130814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672508689, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672508690, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672591700, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7734262943267822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672591701, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672591701, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672606793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11355361342430115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160727040, "lr": 8.174583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672622076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12918035686016083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160890880, "lr": 8.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672637390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10849718749523163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161054720, "lr": 8.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672652541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11290755867958069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672667620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1142139881849289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161382400, "lr": 8.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672683047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11118613183498383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161546240, "lr": 8.216250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672698446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11408234387636185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161710080, "lr": 8.224583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672713987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11306217312812805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672729038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10631493479013443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162037760, "lr": 8.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672744207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10844055563211441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162201600, "lr": 8.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672759720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12930433452129364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162365440, "lr": 8.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672775035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11329955607652664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672790719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11293356865644455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162693120, "lr": 8.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672806281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11664816737174988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162856960, "lr": 8.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672806281, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672806282, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672888916, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7737728357315063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672888917, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672888917, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672904149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09692110121250153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163020800, "lr": 8.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672919571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11154326796531677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672934850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12272371351718903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163348480, "lr": 8.307916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672950262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10642633587121964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163512320, "lr": 8.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672965654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11006096005439758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163676160, "lr": 8.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672980936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10729467123746872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672996109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1119188517332077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164003840, "lr": 8.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673011029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1124296709895134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164167680, "lr": 8.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673026435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10328531265258789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164331520, "lr": 8.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673041200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11816924065351486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673056080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10774576663970947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164659200, "lr": 8.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673071276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11155721545219421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164823040, "lr": 8.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673086362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09284503012895584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164986880, "lr": 8.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673100994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10362746566534042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673100994, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784673100995, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784673183151, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738905549049377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784673183152, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784673183152, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784673198209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10156386345624924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165314560, "lr": 8.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673213012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09747668355703354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165478400, "lr": 8.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673227694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1156916618347168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165642240, "lr": 8.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673242296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10817093402147293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673256783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10980986803770065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165969920, "lr": 8.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673271787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10421895235776901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166133760, "lr": 8.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673286619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10800737142562866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166297600, "lr": 8.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673301572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11363232135772705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673316472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09965946525335312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166625280, "lr": 8.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673331472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09645824879407883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166789120, "lr": 8.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673346330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09728090465068817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166952960, "lr": 8.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673361322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0972534716129303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673376181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10289257764816284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167280640, "lr": 8.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673390958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10120316594839096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167444480, "lr": 8.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673390959, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673390959, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673474329, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7739414572715759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673474330, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673474330, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673489240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10378031432628632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167608320, "lr": 8.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673504238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10348521918058395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673519053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1048920676112175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167936000, "lr": 8.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673533956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10734036564826965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168099840, "lr": 8.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673548660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10280933976173401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168263680, "lr": 8.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673563830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10027893632650375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673579138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10976262390613556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168591360, "lr": 8.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673593988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10550837218761444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168755200, "lr": 8.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673608837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10705248266458511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168919040, "lr": 8.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673623552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10619276762008667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673638514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1046048253774643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169246720, "lr": 8.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673653160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10929902642965317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169410560, "lr": 8.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673667626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10622316598892212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169574400, "lr": 8.624583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673682940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11324823647737503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673682990, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673682990, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673768183, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7740643620491028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673768184, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673768185, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673782842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10875066369771957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169902080, "lr": 8.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673797770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11202535033226013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170065920, "lr": 8.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673813114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10777626186609268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170229760, "lr": 8.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673828392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10745527595281601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673843312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10084418952465057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170557440, "lr": 8.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673858617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10247455537319183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170721280, "lr": 8.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673873940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09626781195402145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170885120, "lr": 8.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673888775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10867737978696823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673903593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09995903819799423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171212800, "lr": 8.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673918486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10315845906734467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171376640, "lr": 8.716249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673933675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10152362287044525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171540480, "lr": 8.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673948464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10338819772005081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673963418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0997919961810112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171868160, "lr": 8.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673978501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10473611205816269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172032000, "lr": 8.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673978548, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673978549, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784674063012, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774145781993866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784674063013, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784674063013, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784674078076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09750200808048248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172195840, "lr": 8.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674093070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10760372877120972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674107884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09966576099395752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172523520, "lr": 8.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674122458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10415935516357422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172687360, "lr": 8.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674137028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09436728805303574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172851200, "lr": 8.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674151960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1069345474243164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674167127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10365568101406097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173178880, "lr": 8.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674182225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10453972220420837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173342720, "lr": 8.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674197688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10515622049570084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173506560, "lr": 8.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674212744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11206692457199097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674228041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09691062569618225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173834240, "lr": 8.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674243608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1121593564748764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173998080, "lr": 8.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674259004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11210335791110992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174161920, "lr": 8.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674274219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10225799679756165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674274274, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784674274274, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784674358219, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7743735909461975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784674358220, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784674358221, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784674373144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09842416644096375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174489600, "lr": 8.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674388598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10717843472957611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174653440, "lr": 8.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674404044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10192105174064636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174817280, "lr": 8.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674418677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723217576742172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674433948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10092577338218689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175144960, "lr": 8.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674448630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10904238373041153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175308800, "lr": 8.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674463476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10068633407354355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175472640, "lr": 8.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674478551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09929390251636505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674493706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10929296165704727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175800320, "lr": 8.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674508988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10986706614494324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175964160, "lr": 8.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674524028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10038410127162933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176128000, "lr": 8.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674539106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09519659727811813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674554067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.096051424741745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176455680, "lr": 8.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674569128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10696622729301453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176619520, "lr": 8.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674569129, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674569130, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674651495, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746478915214539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674651496, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674651496, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674666629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10346151143312454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176783360, "lr": 8.991249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674681716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11374250799417496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674696680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10105893015861511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177111040, "lr": 9.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674711540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10592224448919296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177274880, "lr": 9.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674726415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11175098270177841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177438720, "lr": 9.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674741245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10104016214609146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.032916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674756197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11029704660177231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177766400, "lr": 9.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674771010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10159376263618469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177930240, "lr": 9.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674786023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10815075039863586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178094080, "lr": 9.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674801158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10193000733852386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674816039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09830248355865479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178421760, "lr": 9.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674831410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11199081689119339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178585600, "lr": 9.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674846661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11971462517976761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178749440, "lr": 9.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674861921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11004943400621414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674861922, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674861922, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674945460, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774811863899231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674945461, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674945461, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674960761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11644724756479263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179077120, "lr": 9.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674975547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1015472486615181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179240960, "lr": 9.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674990960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10003282874822617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179404800, "lr": 9.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675006183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10101208835840225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.132916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675021333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10628940165042877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179732480, "lr": 9.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675036429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10004270821809769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179896320, "lr": 9.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675051295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10095030069351196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180060160, "lr": 9.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675066316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1066991314291954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675080831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10635633766651154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180387840, "lr": 9.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675095293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09461767971515656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180551680, "lr": 9.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675110868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10478954017162323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180715520, "lr": 9.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675125575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10213448852300644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675140356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09415242075920105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181043200, "lr": 9.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675155257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10041484981775284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181207040, "lr": 9.216249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675155258, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784675155258, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784675236713, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7749334573745728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784675236714, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784675236715, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784675251777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11117034405469894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181370880, "lr": 9.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675266753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1132715567946434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675281703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10093260556459427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181698560, "lr": 9.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675296635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10707175731658936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181862400, "lr": 9.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675311642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10339802503585815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182026240, "lr": 9.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675326324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039164736866951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675341217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10651619732379913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182353920, "lr": 9.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675355932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10829200595617294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182517760, "lr": 9.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675370728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09820959717035294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182681600, "lr": 9.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675385477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09777504950761795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675400254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1118067279458046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183009280, "lr": 9.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675414684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09864991903305054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183173120, "lr": 9.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675429557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10220474749803543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183336960, "lr": 9.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675443938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10265675187110901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675443939, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675443939, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675525281, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752763628959656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675525282, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675525282, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675540029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1025301143527031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183664640, "lr": 9.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675554543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10853541642427444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183828480, "lr": 9.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675569148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11175630986690521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183992320, "lr": 9.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675584191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09420539438724518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675599056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1044461578130722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184320000, "lr": 9.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675613789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1078110933303833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184483840, "lr": 9.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675628579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11676914244890213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184647680, "lr": 9.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675643291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10121992975473404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675658077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10580836236476898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184975360, "lr": 9.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675673199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11069459468126297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185139200, "lr": 9.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675688109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09827464818954468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185303040, "lr": 9.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675702917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10823645442724228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675717891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09874998033046722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185630720, "lr": 9.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675732569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10217124968767166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185794560, "lr": 9.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675732569, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675732570, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675815063, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7754824161529541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675815064, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675815064, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675830258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11002310365438461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185958400, "lr": 9.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675844939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10475002229213715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675859387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09229744225740433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186286080, "lr": 9.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675874891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10066035389900208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186449920, "lr": 9.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675889681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10897195339202881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186613760, "lr": 9.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675904540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11109595000743866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675919824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09588682651519775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186941440, "lr": 9.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675935144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10536188632249832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187105280, "lr": 9.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675950311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10676463693380356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187269120, "lr": 9.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675965287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10001964122056961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675980201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10471190512180328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187596800, "lr": 9.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675995121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10504033416509628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187760640, "lr": 9.549583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676009886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11105620861053467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187924480, "lr": 9.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676024863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11544860154390335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676024864, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784676024865, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784676106938, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7756990194320679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784676106939, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784676106940, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784676121934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11016342043876648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188252160, "lr": 9.574583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676137126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10517815500497818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188416000, "lr": 9.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676152015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10453499108552933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188579840, "lr": 9.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676167115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10884460061788559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676181967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10266580432653427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188907520, "lr": 9.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676197168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09795045852661133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189071360, "lr": 9.616249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676211967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10357174277305603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189235200, "lr": 9.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676226699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11268553137779236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676241517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10129661113023758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189562880, "lr": 9.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676256174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09593787789344788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189726720, "lr": 9.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676270958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10494940727949142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189890560, "lr": 9.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676286236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11288374662399292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.666249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676301463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10246575623750687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190218240, "lr": 9.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676316456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11834380030632019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190382080, "lr": 9.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676316457, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676316457, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676400082, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7759143114089966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676400083, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676400083, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676415225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10103302448987961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190545920, "lr": 9.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676430054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09784837812185287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676444948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10537763684988022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190873600, "lr": 9.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676460053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11265068501234055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191037440, "lr": 9.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676475156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11852937936782837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191201280, "lr": 9.724583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676490178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09589050710201263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676505099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10592623800039291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191528960, "lr": 9.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676519768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10172299295663834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191692800, "lr": 9.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676534879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10240672528743744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191856640, "lr": 9.757916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676549890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1048348918557167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676564443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10683999955654144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192184320, "lr": 9.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676579987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1100415363907814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192348160, "lr": 9.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676594805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10022468119859695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192512000, "lr": 9.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676609887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11586572974920273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.799583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676609888, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676609888, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676693726, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7761202454566956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676693727, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676693728, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676709114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11282331496477127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192839680, "lr": 9.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676724529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11135397106409073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193003520, "lr": 9.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676739713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10701335966587067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193167360, "lr": 9.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676754900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12146719545125961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676769934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11605776101350784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193495040, "lr": 9.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676785235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10266154259443283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193658880, "lr": 9.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676800436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12019394338130951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193822720, "lr": 9.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676815271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1139361634850502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.866249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676830264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10587747395038605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194150400, "lr": 9.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676845317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10679730027914047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194314240, "lr": 9.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676860569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732841491699219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194478080, "lr": 9.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676875498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10593458265066147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676890754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10715819150209427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194805760, "lr": 9.907916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676905491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11526186019182205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194969600, "lr": 9.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676905491, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676905492, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676988028, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7766252756118774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676988029, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676988029, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784677003344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11269552260637283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195133440, "lr": 9.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677018228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10479175299406052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677033233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10255935788154602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195461120, "lr": 9.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677048385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11658257246017456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195624960, "lr": 9.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677063692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10928040742874146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195788800, "lr": 9.957916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677078919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10650421679019928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677094487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10585217922925949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196116480, "lr": 9.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677109789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11428233981132507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196280320, "lr": 9.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677125620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11865545064210892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196444160, "lr": 9.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677141271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10637788474559784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.999583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677156880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11728523671627045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196771840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677172393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11240235716104507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196935680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677187543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10801810771226883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197099520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677203201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11617656797170639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677203201, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784677203202, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784677284648, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7765882015228271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784677284650, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784677284650, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784677299827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10159323364496231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197427200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677315383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11134731024503708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197591040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677330543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11545737832784653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197754880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677345877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10166124999523163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677361604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09998909384012222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198082560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677377131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10852422565221786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198246400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677392555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0989702045917511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198410240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677407875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10774719715118408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677423285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10162489116191864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198737920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677438564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10572223365306854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198901760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677453824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11468837410211563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199065600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677469272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11083430051803589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677484520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11780449002981186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199393280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677500183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11328250914812088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199557120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677500183, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677500184, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677582495, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7768887877464294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677582496, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677582496, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677597868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11260362714529037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199720960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677613399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10710033029317856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677628647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11191307008266449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200048640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677644086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1168864294886589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200212480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677659509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11226239800453186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200376320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677675344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11720720678567886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677691272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09987843036651611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200704000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677707162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10805065184831619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200867840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677722982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10831151902675629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201031680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677738748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1096518412232399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677754712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1190633475780487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201359360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677770629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1128903180360794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201523200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677786283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10490678995847702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201687040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677802027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1112578809261322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677802027, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677802028, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677886434, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7772077918052673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677886435, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677886435, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677901691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10544899106025696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202014720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677917346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1106438934803009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202178560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677932763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11896926164627075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202342400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677948509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10666777193546295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677963402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10657111555337906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202670080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677978505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10218729823827744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202833920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677994340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1067143902182579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202997760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678009435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1106976792216301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678024604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11403150111436844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203325440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678039569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09465464949607849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203489280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678054879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11718034744262695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203653120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678070338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10139206796884537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678085380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09830401092767715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203980800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678100559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10825882107019424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204144640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678100560, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784678100560, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784678186352, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7771744728088379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784678186353, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784678186353, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784678201459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10277565568685532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204308480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678216644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09307795017957687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678231881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09718520194292068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204636160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678246980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10866881906986237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204800000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678262216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1089988574385643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204963840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678277419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11254843324422836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678292190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09643679112195969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205291520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678306999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10647602379322052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205455360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678321947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10722125321626663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205619200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678337030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11162400245666504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678352054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09887947887182236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205946880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678367862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09962727874517441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206110720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678383103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1116517186164856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206274560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678398192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09812911599874496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678398192, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678398193, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678483130, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7773910760879517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678483132, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678483132, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678498540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10626372694969177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206602240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678513726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09637433290481567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206766080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678529386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10539542138576508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206929920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678545304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09630221128463745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678561016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10296431928873062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207257600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678576602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11279483139514923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207421440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678592063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10827206820249557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207585280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678606849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09551886469125748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678621954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11608303338289261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207912960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678637131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10439218580722809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208076800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678651899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039082258939743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208240640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678668004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10151820629835129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678683159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10283852368593216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208568320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678698452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10214132070541382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208732160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678698453, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678698453, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678781362, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7774686813354492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678781363, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678781363, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678796897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10137844830751419, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208896000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678812308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09940645843744278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678827686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10260240733623505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209223680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678842834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10340050607919693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209387520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678858215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09877457469701767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209551360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678873407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10148677229881287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678888411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09691827744245529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209879040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678903281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09914998710155487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210042880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678918499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10362300276756287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210206720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678933612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10811429470777512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678948680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10432843863964081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210534400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678963653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10778222978115082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210698240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678978761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10219649225473404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210862080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678993675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09671175479888916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678993676, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678993676, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784679075320, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7775768041610718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784679075321, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784679075321, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784679090492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09491749107837677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211189760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679105302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09981236606836319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211353600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679120236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09770157933235168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211517440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679134890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0996452122926712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679149840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10483528673648834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211845120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679165191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09613760560750961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212008960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679180128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1027396023273468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212172800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679195245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09823426604270935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679210148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0983203873038292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212500480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679225094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1050875335931778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212664320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679240025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09324636310338974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212828160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679255325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10817120969295502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679270572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10574782639741898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213155840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679285830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1088656634092331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213319680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679285830, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784679285831, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784679367683, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7776129245758057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784679367684, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784679367684, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784679382653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10510575771331787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213483520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679397592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10221753269433975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679412677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10604342818260193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213811200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679428099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1031530424952507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213975040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679442877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09678778797388077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214138880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679457805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10528190433979034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679472851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0953587219119072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214466560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679487583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11445263773202896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214630400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679502487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09884084761142731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214794240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679517891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09036648273468018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679533129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10046559572219849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215121920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679548302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10545393079519272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215285760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679563287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10449913144111633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215449600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679578344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10897134989500046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679578344, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679578345, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679659412, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7777277231216431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679659412, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679659413, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679674497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10167291015386581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215777280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679689327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10626128315925598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215941120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679704434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09784206002950668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216104960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679719935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09998448193073273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679735463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10053589940071106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216432640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679750669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09521231055259705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216596480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679765902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10607171803712845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216760320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679781305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10298068076372147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679796407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1001158282160759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217088000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679812034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10958541184663773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217251840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679827117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09809023141860962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217415680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679842437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10129140317440033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217579520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679857390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1007080227136612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217743360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679872395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10551941394805908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217907200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679872396, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679872396, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679955352, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7779159545898438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679955353, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679955353, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679970397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10257373750209808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218071040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679985506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09525905549526215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218234880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680000670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09882068634033203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218398720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680015651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10839606076478958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218562560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680031031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10355225950479507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218726400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680046316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08801201730966568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218890240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680061924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10591001063585281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219054080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680077541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09257597476243973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219217920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680092999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10417633503675461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219381760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680108414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09565775841474533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219545600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680123793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09650707989931107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219709440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680138876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10291439294815063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219873280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680154209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11214839667081833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220037120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680169389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09995894134044647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220200960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680169389, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784680169390, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} From 5b1590d5d41641007e7a70ba7b16373a1a74de79 Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Sat, 25 Jul 2026 04:47:46 +0000 Subject: [PATCH 121/127] dlrmv3 train: SKIP_EVAL to skip first N periodic eval passes (default 0) Add $SKIP_EVAL (streaming_train_eval_loop.skip_eval, default 0 = OFF) to suppress the first N scheduled full-holdout eval passes, cutting eval overhead early in a run when the model is far from the AUC target and those points carry little signal. Works for both cadences, keyed off the same grid each already uses so the skip set is resume-safe (derived from checkpoint-restored global_step / absolute ts, never a per-process counter): * data-fraction: skip while ordinal k = global_step / eval_interval_steps <= N; eval begins at the (N+1)th data-pct point. * per-window: skip while (train_ts - eval_anchor_ts)/eval_every_n_windows < N. The end-of-run FINAL eval is never skipped; early-stop / MLPerf EVAL_ACCURACY still fire once eval starts. Wired through launch_slurm.sh's worker passthrough. Verified e2e (per-window, START_TS=150, NUM_TRAIN_TS=4, EVAL_EVERY_N_WINDOWS=1): SKIP_EVAL=0 eval'd ts=150,151,152,153,final; SKIP_EVAL=2 eval'd ts=152,153,final (first two points correctly dropped, final always run). Co-authored-by: Cursor --- .../dlrm_v3/train/gin/yambda_5b.gin | 43 ++++++++++ .../dlrm_v3/train/utils.py | 81 ++++++++++++++----- recommendation_v4/scripts/launch_slurm.sh | 1 + 3 files changed, 106 insertions(+), 19 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index 7ecd5d823..9212adbad 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -688,6 +688,49 @@ evn/env_int.default = 0 streaming_train_eval_loop.eval_every_data_pct = @edp/env_float() edp/env_float.key = "EVAL_EVERY_DATA_PCT" edp/env_float.default = 0.005 +# ---- SKIP_EVAL: warm-up eval skipping (early-run overhead cut) -------------- +# WHAT IT DOES +# Suppresses the FIRST N *scheduled/periodic* full-holdout eval passes. Early +# in a run the model sits far below the AUC target and those first eval points +# carry little signal, yet each full-holdout pass is a real cost (a forward +# sweep over the whole held-out set + its metric reduction). SKIP_EVAL lets you +# pay that cost only once training has advanced enough to matter. +# N = $SKIP_EVAL (default 0 = OFF -> eval from the very first scheduled +# point; behavior identical to before this knob existed.) +# +# HOW THE "FIRST N" ARE COUNTED (per cadence; only the active cadence applies) +# Eval boundaries are numbered on the SAME grid the cadence already uses, and +# a boundary is skipped while its index falls in the first N: +# * DATA-FRACTION cadence (EVAL_EVERY_DATA_PCT>0): a boundary fires when the +# global train step hits k*eval_interval_steps (k=1,2,3,...). Its 1-based +# ordinal is k = global_step / eval_interval_steps; the boundary is SKIPPED +# while k <= N. So eval begins at the (N+1)th data-fraction point, i.e. after +# ~ (N+1) * EVAL_EVERY_DATA_PCT of the data has been trained. +# * PER-WINDOW cadence (EVAL_EVERY_N_WINDOWS>0): the eval-eligible windows are +# numbered on the absolute-ts grid as ordinal = +# (train_ts - eval_anchor_ts) / eval_every_n_windows (0-based); a window is +# SKIPPED while ordinal < N. +# +# RESUME SAFETY +# The ordinal is derived purely from the checkpoint-restored global_step +# (data-fraction) or the absolute ts grid (per-window) -- never from a +# per-process counter -- so the skipped set is IDENTICAL on a cold start and on +# every mid-run resume (a resume cannot re-skip points it already trained past, +# nor start eval'ing points an earlier attempt skipped). +# +# WHAT IS NEVER SKIPPED +# The end-of-run FINAL eval always runs (it is the definitive convergence +# number), and early-stop / MLPerf EVAL_ACCURACY still fire normally once eval +# starts at the (N+1)th point. SKIP_EVAL only removes early *periodic* passes. +# +# EXAMPLE (verified e2e; per-window cadence, START_TS=150, NUM_TRAIN_TS=4, +# EVAL_EVERY_N_WINDOWS=1 -> eligible eval points ts=150,151,152,153 + final): +# SKIP_EVAL=0 -> eval at ts=150,151,152,153,final (all points) +# SKIP_EVAL=2 -> eval at ts=152,153,final (first two dropped) +# Override via $SKIP_EVAL. +streaming_train_eval_loop.skip_eval = @se/env_int() +se/env_int.key = "SKIP_EVAL" +se/env_int.default = 0 # Double-buffer windows: prepare the next window (index mask + first-batch # prefetch) in a background thread during the current window's compute, hiding # the per-window reset. Needs persistent_loader=1. Override via env. diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py index 2b85722a7..41811cc90 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -1990,6 +1990,13 @@ def streaming_train_eval_loop( # once to a global train-step interval. 0.0 = OFF (use the per-window # cadence). Wired to $EVAL_EVERY_DATA_PCT via gin. eval_every_data_pct: float = 0.0, + # SKIP_EVAL: skip the FIRST N scheduled (periodic) eval passes to cut eval + # overhead early in the run (model far from the AUC target). 0 (default) = + # OFF (eval from the first scheduled point). Keyed off the same + # global_step / absolute-ts eval grid as the cadence, so the skip set is + # identical across resume; the end-of-run final eval is never skipped. + # Wired to $SKIP_EVAL via gin. + skip_eval: int = 0, double_buffer: bool = False, # --- fixed user-holdout eval set --- # Window range the fixed eval set is drawn from. None -> default to @@ -2590,23 +2597,40 @@ def _run_train_window( and gstep > 0 and gstep % eval_interval_steps == 0 ): - if rank == 0: - logger.info( - "[data-pct-eval] trigger eval train_ts=%d global_step=%d " - "(interval=%d)", - train_ts, - gstep, - eval_interval_steps, - ) - do_eval(train_ts, gstep) - model.train() - dataset.dataset.is_eval = False # pyre-ignore [16] - # Data-fraction eval may hit the MLPerf target and emit RUN_STOP - # (via _do_eval_*). Stop the window immediately so we don't train - # past the convergence point; the outer window loop checks the - # same flag and breaks too. - if mlt.run_stopped: - break + # SKIP_EVAL: the 1-based ordinal of this data-pct eval boundary is + # gstep / eval_interval_steps. Skip it (keep training) while that + # ordinal is within the first `skip_eval` points. Ordinal is a pure + # function of the checkpoint-restored global_step, so the skip set + # is identical across resume. + _eval_ordinal = gstep // eval_interval_steps + if skip_eval > 0 and _eval_ordinal <= skip_eval: + if rank == 0: + logger.info( + "[data-pct-eval] SKIP eval train_ts=%d global_step=%d " + "ordinal=%d <= SKIP_EVAL=%d", + train_ts, + gstep, + _eval_ordinal, + skip_eval, + ) + else: + if rank == 0: + logger.info( + "[data-pct-eval] trigger eval train_ts=%d global_step=%d " + "(interval=%d)", + train_ts, + gstep, + eval_interval_steps, + ) + do_eval(train_ts, gstep) + model.train() + dataset.dataset.is_eval = False # pyre-ignore [16] + # Data-fraction eval may hit the MLPerf target and emit RUN_STOP + # (via _do_eval_*). Stop the window immediately so we don't train + # past the convergence point; the outer window loop checks the + # same flag and breaks too. + if mlt.run_stopped: + break # Test-only: deterministic crash for the failure-injection test. # Triggered AFTER the save above, so on resume we re-enter at # batch_idx_in_window=train_batch_idx and emit batches [K+1, end). @@ -2797,6 +2821,19 @@ def _should_eval(i: int) -> bool: return True return (train_ts_list[i] - eval_anchor_ts) % eval_every_n_windows == 0 or i == n_train - 1 + def _skip_early_window_eval(i: int) -> bool: + """SKIP_EVAL for the PER-WINDOW cadence: skip the first `skip_eval` + eval-eligible points on the absolute-ts grid so early (low-signal) eval + passes are not run. The ordinal is derived from the absolute ts + ((train_ts - eval_anchor_ts) / eval_every_n_windows), so the skip set is + stable across a mid-run resume. The final window's eval is never skipped + so the trajectory always ends with an eval point. + """ + if skip_eval <= 0 or i == n_train - 1: + return False + ordinal = (train_ts_list[i] - eval_anchor_ts) // max(1, eval_every_n_windows) + return ordinal < skip_eval + # Fixed eval set: held-out users' anchors over the resolved holdout window # range, computed ONCE and reused at every eval step. Same anchors every # step -> stable, comparable eval-AUC curve, and bounded eval time @@ -2984,7 +3021,7 @@ def _do_eval_db(train_ts: int, gstep: int) -> None: if _per_window_blocks: mlt.block_stop() should_stop = False - if _should_eval(i): + if _should_eval(i) and not _skip_early_window_eval(i): dataset.dataset.is_eval = True # pyre-ignore [16] assert eval_sampler is not None and eval_dl is not None mlt.eval_start() @@ -3075,7 +3112,7 @@ def _do_eval_nb(train_ts: int, gstep: int) -> None: if _per_window_blocks: mlt.block_stop() should_stop = False - if _should_eval(i): + if _should_eval(i) and not _skip_early_window_eval(i): dataset.dataset.is_eval = True # pyre-ignore [16] mlt.eval_start() if eval_global_indices is not None: @@ -3093,6 +3130,12 @@ def _do_eval_nb(train_ts: int, gstep: int) -> None: iter(make_streaming_dataloader(dataset=dataset, ts=train_ts + 1)) ) should_stop = mlt.eval_stop(eval_metrics) + elif _should_eval(i) and rank == 0: + logger.info( + "[per-window-eval] SKIP eval train_ts=%d (SKIP_EVAL=%d)", + train_ts, + skip_eval, + ) _maybe_checkpoint(train_ts) # should_stop: per-window convergence. mlt.run_stopped: # data-fraction convergence (RUN_STOP fired mid-window by _do_eval_nb). diff --git a/recommendation_v4/scripts/launch_slurm.sh b/recommendation_v4/scripts/launch_slurm.sh index 8fd21a519..b1289f144 100755 --- a/recommendation_v4/scripts/launch_slurm.sh +++ b/recommendation_v4/scripts/launch_slurm.sh @@ -292,6 +292,7 @@ orchestrate() { ${DIE_AT_STEP:+-e DIE_AT_STEP=$DIE_AT_STEP} \ -e METRIC_LOG_FREQ=$METRIC_LOG_FREQ \ ${EVAL_INTERIM_LOG:+-e EVAL_INTERIM_LOG=$EVAL_INTERIM_LOG} \ + ${SKIP_EVAL:+-e SKIP_EVAL=$SKIP_EVAL} \ ${MLPERF_LOGGING:+-e MLPERF_LOGGING=$MLPERF_LOGGING} \ ${MLPERF_TRAIN_LOSS_LOG_FREQ:+-e MLPERF_TRAIN_LOSS_LOG_FREQ=$MLPERF_TRAIN_LOSS_LOG_FREQ} \ ${STREAMING_SHUFFLE_FRACTION:+-e STREAMING_SHUFFLE_FRACTION=$STREAMING_SHUFFLE_FRACTION} \ From 23882d1ade8136035db9a686e707f84a7bbc7d3b Mon Sep 17 00:00:00 2001 From: suachong Date: Sun, 26 Jul 2026 10:41:32 -0500 Subject: [PATCH 122/127] dlrmv4: expand RCP reference logs to 20 runs per global batch size Add 10 more per-seed MLPerf logs to each gbs bucket (8192/16384/32768), bringing each to 20 runs. All 60 logs reach >=0.75 eval AUC; the three short single-node (gbs_8192) runs that stalled below 0.75 were replaced with full 8h runs. Co-authored-by: Cursor --- .../rcp_logs/gbs_16384/seed130327877.log | 679 ++++++ .../rcp_logs/gbs_16384/seed313553746.log | 672 ++++++ .../rcp_logs/gbs_16384/seed530691108.log | 669 ++++++ .../rcp_logs/gbs_16384/seed547461891.log | 620 ++++++ .../rcp_logs/gbs_16384/seed782855750.log | 676 ++++++ .../rcp_logs/gbs_16384/seed786517729.log | 585 ++++++ .../rcp_logs/gbs_16384/seed851663607.log | 676 ++++++ .../rcp_logs/gbs_16384/seed864725970.log | 670 ++++++ .../rcp_logs/gbs_16384/seed871175675.log | 661 ++++++ .../rcp_logs/gbs_16384/seed925729918.log | 672 ++++++ .../rcp_logs/gbs_32768/seed457724564.log | 687 ++++++ .../rcp_logs/gbs_32768/seed526341580.log | 666 ++++++ .../rcp_logs/gbs_32768/seed591489896.log | 684 ++++++ .../rcp_logs/gbs_32768/seed595408662.log | 641 ++++++ .../rcp_logs/gbs_32768/seed72228511.log | 684 ++++++ .../rcp_logs/gbs_32768/seed764588808.log | 675 ++++++ .../rcp_logs/gbs_32768/seed770053153.log | 634 ++++++ .../rcp_logs/gbs_32768/seed898059771.log | 701 +++++++ .../rcp_logs/gbs_32768/seed942014509.log | 675 ++++++ .../rcp_logs/gbs_32768/seed996913233.log | 617 ++++++ .../rcp_logs/gbs_8192/seed1023733178.log | 1779 ++++++++++++++++ .../rcp_logs/gbs_8192/seed1026364325.log | 639 ++++++ .../rcp_logs/gbs_8192/seed138251694.log | 681 ++++++ .../rcp_logs/gbs_8192/seed166896187.log | 1847 ++++++++++++++++ .../rcp_logs/gbs_8192/seed490920051.log | 681 ++++++ .../rcp_logs/gbs_8192/seed546672350.log | 678 ++++++ .../rcp_logs/gbs_8192/seed770796222.log | 684 ++++++ .../rcp_logs/gbs_8192/seed79275716.log | 662 ++++++ .../rcp_logs/gbs_8192/seed869764802.log | 1859 +++++++++++++++++ .../rcp_logs/gbs_8192/seed94090156.log | 632 ++++++ 30 files changed, 23386 insertions(+) create mode 100644 recommendation_v4/rcp_logs/gbs_16384/seed130327877.log create mode 100644 recommendation_v4/rcp_logs/gbs_16384/seed313553746.log create mode 100644 recommendation_v4/rcp_logs/gbs_16384/seed530691108.log create mode 100644 recommendation_v4/rcp_logs/gbs_16384/seed547461891.log create mode 100644 recommendation_v4/rcp_logs/gbs_16384/seed782855750.log create mode 100644 recommendation_v4/rcp_logs/gbs_16384/seed786517729.log create mode 100644 recommendation_v4/rcp_logs/gbs_16384/seed851663607.log create mode 100644 recommendation_v4/rcp_logs/gbs_16384/seed864725970.log create mode 100644 recommendation_v4/rcp_logs/gbs_16384/seed871175675.log create mode 100644 recommendation_v4/rcp_logs/gbs_16384/seed925729918.log create mode 100644 recommendation_v4/rcp_logs/gbs_32768/seed457724564.log create mode 100644 recommendation_v4/rcp_logs/gbs_32768/seed526341580.log create mode 100644 recommendation_v4/rcp_logs/gbs_32768/seed591489896.log create mode 100644 recommendation_v4/rcp_logs/gbs_32768/seed595408662.log create mode 100644 recommendation_v4/rcp_logs/gbs_32768/seed72228511.log create mode 100644 recommendation_v4/rcp_logs/gbs_32768/seed764588808.log create mode 100644 recommendation_v4/rcp_logs/gbs_32768/seed770053153.log create mode 100644 recommendation_v4/rcp_logs/gbs_32768/seed898059771.log create mode 100644 recommendation_v4/rcp_logs/gbs_32768/seed942014509.log create mode 100644 recommendation_v4/rcp_logs/gbs_32768/seed996913233.log create mode 100644 recommendation_v4/rcp_logs/gbs_8192/seed1023733178.log create mode 100644 recommendation_v4/rcp_logs/gbs_8192/seed1026364325.log create mode 100644 recommendation_v4/rcp_logs/gbs_8192/seed138251694.log create mode 100644 recommendation_v4/rcp_logs/gbs_8192/seed166896187.log create mode 100644 recommendation_v4/rcp_logs/gbs_8192/seed490920051.log create mode 100644 recommendation_v4/rcp_logs/gbs_8192/seed546672350.log create mode 100644 recommendation_v4/rcp_logs/gbs_8192/seed770796222.log create mode 100644 recommendation_v4/rcp_logs/gbs_8192/seed79275716.log create mode 100644 recommendation_v4/rcp_logs/gbs_8192/seed869764802.log create mode 100644 recommendation_v4/rcp_logs/gbs_8192/seed94090156.log diff --git a/recommendation_v4/rcp_logs/gbs_16384/seed130327877.log b/recommendation_v4/rcp_logs/gbs_16384/seed130327877.log new file mode 100644 index 000000000..a9d69a14c --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_16384/seed130327877.log @@ -0,0 +1,679 @@ +:::MLLOG {"namespace": "", "time_ms": 1784929469520, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784929469520, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784929490527, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784929490528, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784929490528, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784929490528, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784929490528, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784929490528, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 16384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784929490528, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784929490528, "event_type": "POINT_IN_TIME", "key": "seed", "value": 130327877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784929490529, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784929490529, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784929490529, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784929490529, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784929490532, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784929490533, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784930372702, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784930372703, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784930373031, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784930497305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13741058111190796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.5833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784930510850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13781070709228516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2499999999999997e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784930524438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13829509913921356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784930538163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13851675391197205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.583333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784930551938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13880226016044617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.25e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784930565816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387527883052826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784930579673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13852539658546448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784930579674, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784930579674, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784930659350, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5014888048171997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784930659351, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784930659351, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784930673060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13760338723659515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784930686909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13772650063037872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784930700676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381089687347412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784930714538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137943297624588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784930728544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379503756761551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9916666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784930742563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13722749054431915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784930756866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13821694254875183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784930756867, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784930756867, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784930800573, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5016469955444336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784930800574, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784930800574, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784930814480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13787244260311127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784930828603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13738864660263062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784930842411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13742533326148987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784930856829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13735651969909668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.9916666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784930870497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13757987320423126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784930884839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13771796226501465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784930898673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13721787929534912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784930898673, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784930898674, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784930942341, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5018129348754883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784930942342, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784930942342, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784930956242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13819126784801483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784930970563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13744735717773438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.8249999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784930984623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13632023334503174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784930998844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377808004617691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931013303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1369735300540924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3249999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931027294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13671943545341492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931041715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13664554059505463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931041716, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784931041716, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784931084394, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5020522475242615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784931084395, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784931084395, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784931098182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360839307308197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931112228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1372012495994568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931126395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13694654405117035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1583333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931140098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13646364212036133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931153648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346900761127472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931167548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13641348481178284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931181729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350817084312439, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931181730, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784931181730, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784931225260, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5023185014724731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784931225260, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784931225260, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784931239168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362655907869339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931253061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13568076491355896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931267284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13714197278022766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931281418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13608047366142273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931295057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1340285837650299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.658333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931308649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13476930558681488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931322494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13731053471565247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931322494, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784931322495, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784931365182, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5026360750198364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784931365183, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784931365183, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784931379536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364598423242569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931393838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13505887985229492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931407978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13520953059196472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931422095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341310441493988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931436448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13438838720321655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931451096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13387593626976013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931465211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13440167903900146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931465212, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784931465212, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784931507066, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5030504465103149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784931507067, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784931507067, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784931521396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13355876505374908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931535869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361204981803894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931550692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13822153210639954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931565313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13343259692192078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.824999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931580004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13582974672317505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931594402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13454318046569824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.158333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931609227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13653114438056946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.324999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931609228, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784931609228, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784931651009, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5036609768867493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784931651009, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784931651009, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784931665704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13650506734848022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931680340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366949826478958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931695045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13467320799827576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931709681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12969735264778137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931724383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13221287727355957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931738830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13352198898792267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931753143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13485737144947052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931753189, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784931753190, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784931794974, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5044026374816895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784931794974, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784931794975, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784931809471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349785029888153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931823936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12877029180526733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931839038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13098996877670288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931853236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368754506111145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.1158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931867403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13216570019721985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931881371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13128334283828735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931895951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325334906578064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931895951, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784931895952, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784931939435, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5052188038825989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784931939435, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784931939436, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784931953333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1316993087530136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931968311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341438740491867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931982440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12777823209762573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931996686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1291271150112152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932010878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13392679393291473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932025585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12824705243110657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932039873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297684907913208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932039874, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784932039874, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784932085320, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5063700079917908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784932085320, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784932085321, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784932100147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13472974300384521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932114219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330287605524063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.3158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932128489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13441359996795654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932142686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13486254215240479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932157419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13614408671855927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.3658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932171760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13350197672843933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932186348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13678127527236938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932186348, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784932186349, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784932229223, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5077407956123352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784932229223, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784932229224, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784932243925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1342841386795044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.4158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932258693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1305330991744995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932273444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13387757539749146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.4491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932288479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14049670100212097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.4658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932303322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13033631443977356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932318268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13240987062454224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932333119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13821211457252502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932333120, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784932333120, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784932376636, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5090627670288086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784932376637, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784932376637, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784932391750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12775777280330658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932406502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13108964264392853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932421443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370779573917389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932436284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13532330095767975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932451554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321917027235031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932465908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12864068150520325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.6158333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932480436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.127950519323349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932480437, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784932480437, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784932522820, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5104547739028931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784932522821, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784932522821, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784932537198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13011324405670166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932551620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134669691324234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.6658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932566335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14207105338573456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6825000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932580797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1340215504169464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932595808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1314902901649475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.7158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932610095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336403489112854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932624666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13590475916862488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932624666, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784932624667, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784932668645, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5120578408241272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784932668645, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784932668645, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784932682826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13468238711357117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932697281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13632658123970032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932712015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13551592826843262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932726831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13335202634334564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932741597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343533843755722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932756417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13292402029037476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932770947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13249844312667847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8658333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932770948, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784932770948, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784932813744, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5137181878089905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784932813746, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784932813746, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784932828301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13192260265350342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932842986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13430948555469513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8991666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932857932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14167660474777222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932873136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338476538658142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932888159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1342276930809021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932903079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323634386062622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932917875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13316604495048523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932917876, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784932917876, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784932966605, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5156402587890625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784932966606, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784932966606, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784932981255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137608140707016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932996085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13301944732666016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933010750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12875109910964966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933025604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13005080819129944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933040561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380065381526947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933055684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1316547393798828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933070551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13621389865875244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933070552, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784933070552, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784933113600, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5178365111351013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784933113601, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784933113601, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784933128523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367304027080536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.1158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933143122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13641595840454102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933157832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1313653290271759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1491666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933172457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12990793585777283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933187291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1314408779144287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933202465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13522374629974365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933217745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13259953260421753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933217746, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784933217746, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784933262076, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.520391047000885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784933262077, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784933262077, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784933276857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13598811626434326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933292191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13226543366909027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933307462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378733366727829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933322696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.136343315243721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933338014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385926604270935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933353037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13428834080696106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933368558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13920357823371887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933368559, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784933368559, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784933411612, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5229726433753967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784933411612, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784933411613, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784933427081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13508540391921997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933442453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14768365025520325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933458020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350589394569397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933473178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13308514654636383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933488527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13433343172073364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933503656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133161261677742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933518895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13662147521972656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933518895, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784933518896, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784933562185, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5260679125785828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784933562185, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784933562186, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784933577996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13611501455307007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933593421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13530835509300232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933608847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13374775648117065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933623690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13365629315376282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933638563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331365406513214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933653296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12933799624443054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933667955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12560515105724335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933667956, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784933667956, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784933710975, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5298409461975098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784933710976, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784933710976, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784933725678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13263115286827087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.5824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933740474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13244135677814484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933755361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13496941328048706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933770072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353374421596527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933784942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13662564754486084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933799960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12875956296920776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933814817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298283338546753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933814818, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784933814818, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784933859188, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5354326367378235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784933859189, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784933859190, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784933874057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327492594718933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.699166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933888909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12703540921211243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933903565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133143350481987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933918242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322110891342163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933933405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12697896361351013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933948660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323021501302719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933963292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13415291905403137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933963292, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784933963293, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784934009049, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5434387922286987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784934009050, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784934009050, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784934024175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294630765914917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934038828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13031145930290222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934053341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304529309272766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934067982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12977255880832672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.8658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934082320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12775647640228271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934096789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12317872792482376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934111688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.127197265625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934111689, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784934111689, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784934155480, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.554966151714325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784934155481, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784934155481, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784934170330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13577383756637573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934185310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332085132598877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934199779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13895773887634277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934215143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13328003883361816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934230068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12763072550296783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934244972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295953392982483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934260189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1268189549446106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934260190, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784934260190, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784934304744, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5694131255149841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784934304745, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784934304745, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784934319562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12884759902954102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934334413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12706129252910614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934349651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322837769985199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934364911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13229209184646606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934380264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13118326663970947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934395225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13331058621406555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934410435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1302037537097931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934410436, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784934410437, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784934453484, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5860764384269714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784934453485, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784934453485, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784934468442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1292733997106552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934483266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.129511296749115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934498144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12903490662574768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934513034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13209083676338196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934527714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13267509639263153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934542208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289127916097641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934557389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297374665737152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934557390, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784934557390, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784934600838, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.599346935749054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784934600838, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784934600839, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784934615786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12751734256744385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934631061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12995287775993347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934645518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12722760438919067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934661079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1265142858028412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934676600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12613072991371155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934692001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300499439239502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934707638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13094021379947662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934707639, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784934707639, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784934750436, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6118835806846619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784934750437, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784934750437, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784934765432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12840168178081512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934780760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12602829933166504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934796459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288432478904724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934812216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13079264760017395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934827842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317555457353592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934843034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12721608579158783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934858905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12741565704345703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.4991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934858905, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784934858906, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784934902955, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6247787475585938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784934902956, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784934902956, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784934918060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12746237218379974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.5158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934933136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1251930296421051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934948149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12620909512043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.5491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934962954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330440640449524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934978098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1287929266691208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934993453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12359479069709778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935008973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12615445256233215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935008973, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784935008973, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784935054359, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6358637809753418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784935054360, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784935054360, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784935069274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12257000803947449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6325000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935085173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12526625394821167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935100360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12452530860900879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935115392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1235341727733612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935130302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12219133973121643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935145389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11879190057516098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935160405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12145138531923294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935160406, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784935160406, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784935205388, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6496809124946594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784935205389, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784935205389, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784935220798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12360332161188126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935235989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1201949417591095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935251176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11310137808322906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.7824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935266374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11930988729000092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935281504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11460000276565552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935296561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11515715718269348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8324999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935311457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11312208324670792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.8491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935311457, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784935311457, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784935354747, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6717750430107117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784935354748, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784935354748, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784935369753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11890266835689545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935384511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1146303340792656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935399085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10977821052074432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.8991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935413853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11766226589679718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935429080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11679330468177795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935444572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11520588397979736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.9491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935459584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10744430124759674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.9658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935459585, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784935459585, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784935504495, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7100310325622559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784935504496, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784935504496, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784935519364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11425633728504181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935535232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11545336991548538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935550317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11790451407432556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.0158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935565221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11271555721759796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935580105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11145804822444916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935594916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11220148205757141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0658333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935609864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10820285975933075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935609907, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784935609908, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784935653402, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7336806058883667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784935653403, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784935653403, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784935668188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11309146881103516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935683610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11122581362724304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.1158333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935698758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11780869960784912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935714294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11128823459148407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935729189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11230617761611938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1658333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935744716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11813097447156906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935759774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10595645010471344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935759774, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784935759774, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784935802712, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7497020363807678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784935802713, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784935802713, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784935818036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11459602415561676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935833294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11259129643440247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935848655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1104150265455246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935863656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11031477153301239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935878626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10394148528575897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935893274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11772160232067108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935908559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11379587650299072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935908559, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784935908559, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784935951588, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7562284469604492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784935951589, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784935951590, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784935966687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11685779690742493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935982175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11278510838747025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.3491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935997358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11565368622541428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936013049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10143715143203735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936028272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11478415876626968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.3991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936043372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10841018706560135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.4158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936058478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10750117897987366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936058479, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784936058479, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784936102249, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7589854598045349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784936102250, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784936102250, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784936117451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10947012156248093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.4491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936132604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10869793593883514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936147693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11435183137655258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.4824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936162985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10937294363975525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936178490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11301885545253754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936193962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10363711416721344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936209587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11160542070865631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936209588, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784936209588, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784936253034, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7611272931098938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784936253035, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784936253036, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784936268387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10816161334514618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936283427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10957103967666626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.5824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936299681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11981431394815445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936315417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12345238029956818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936330862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11666906625032425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936346488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11183468997478485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936361979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11626444011926651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936361980, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784936361980, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784936404840, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7634645700454712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784936404841, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784936404841, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784936420269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11558946222066879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936435827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10910797864198685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.6991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936451608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11857803165912628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936467236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11146941035985947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936483286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11491192132234573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936498649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11437024176120758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936514729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10535179078578949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936514730, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784936514730, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784936558442, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7649858593940735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784936558443, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784936558443, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784936574085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1188303530216217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936589745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10751298815011978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936605274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10490494221448898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936620741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11476822197437286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936636291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11909498274326324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936651957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10816273093223572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936667981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11483326554298401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936667982, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784936667982, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784936712152, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7650060057640076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784936712152, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784936712152, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784936727885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11671522259712219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936743775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10888705402612686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936758838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10898895561695099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936774301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11372948437929153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936789626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10791391134262085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936804688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1041310653090477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936819664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11755736172199249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.015833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936819665, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784936819665, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784936863916, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7651113271713257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784936863917, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784936863917, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784936879111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11038492619991302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936894208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1148344874382019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936909157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10832541435956955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936924043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10596492886543274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936939512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10313886404037476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936954865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09733660519123077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936970406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10737061500549316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936970407, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784936970407, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784937014491, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7656068801879883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784937014492, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784937014493, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784937029634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10593140870332718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937044351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11028829216957092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937060023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11956215649843216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937075214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1090565174818039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937090058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10705958306789398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937104744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10178963840007782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937119763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10818703472614288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937119764, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784937119764, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784937163033, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.766601026058197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784937163034, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784937163034, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784937178204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10594727098941803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937192936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10602781176567078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937207946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11059771478176117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937223188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792025923728943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937238533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1076001524925232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937253796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11279842257499695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937269423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11060839146375656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937269424, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784937269424, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784937313566, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7676143646240234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784937313567, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784937313567, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784937328399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10563057661056519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937343740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10379929095506668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937359213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11115039885044098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937374679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10953591763973236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937390018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1053420677781105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937405510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10165087878704071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937420945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10525807738304138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937420945, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784937420945, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784937463902, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7685555815696716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784937463902, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784937463902, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784937479099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1050298810005188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937494528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10096332430839539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937509549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1055753231048584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937525075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1108141839504242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937540586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10684279352426529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937556260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10361512005329132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937571881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11075699329376221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937571882, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784937571882, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784937616050, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7685883045196533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784937616051, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784937616051, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784937630836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1007748693227768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937646377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10475396364927292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937661444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10770799219608307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937676752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10446397215127945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937691911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10302675515413284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937707044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10789551585912704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937722425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11043740063905716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937722426, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784937722426, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784937765773, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7698070406913757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784937765774, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784937765774, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784937780838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1073603481054306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937795814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1043315902352333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937810550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1095409169793129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937825774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10697400569915771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937841066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11123005300760269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937856384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10818073153495789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937871570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11174861341714859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937871571, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784937871571, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784937915439, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7695820331573486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784937915440, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784937915441, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784937930224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10668941587209702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937945748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1022358238697052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937961071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10548575222492218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937976468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10243897140026093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937991683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087808907032013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938006864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1134478896856308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938022251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10984211415052414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938022251, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784938022252, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784938065455, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.770494818687439, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784938065456, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784938065456, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784938080897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10604013502597809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938096225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11168843507766724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938111532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087055504322052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938127095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10099413245916367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938142613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11324885487556458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938158160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11370700597763062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938174077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10495191067457199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938174077, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784938174078, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784938217140, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7704154849052429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784938217141, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784938217141, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784938232437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10889706015586853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938247197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10062984377145767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938262925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11757730692625046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938278521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11731936037540436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938293863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11553119122982025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938309228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10564127564430237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938324697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10767528414726257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938324697, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784938324698, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784938368681, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7713645696640015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784938368681, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784938368681, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784938384239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11193837225437164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938399410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11445696651935577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938414814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11084121465682983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938430505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11152133345603943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938446115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1154436320066452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938461714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11301317811012268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938477646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11415009200572968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938477647, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784938477647, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784938522193, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7711153030395508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784938522194, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784938522194, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784938537326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11031399667263031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938552965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10778987407684326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938568770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10795535147190094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938584733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1152028888463974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938600217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1186620444059372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938615715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11073325574398041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938631599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11614029109477997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.415833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938631600, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784938631600, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784938676025, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7716622948646545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784938676026, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784938676026, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} diff --git a/recommendation_v4/rcp_logs/gbs_16384/seed313553746.log b/recommendation_v4/rcp_logs/gbs_16384/seed313553746.log new file mode 100644 index 000000000..0ad8d0f77 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_16384/seed313553746.log @@ -0,0 +1,672 @@ +:::MLLOG {"namespace": "", "time_ms": 1784922165375, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784922165375, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784922185840, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784922185842, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784922185842, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784922185842, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784922185842, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784922185842, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 16384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784922185842, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784922185842, "event_type": "POINT_IN_TIME", "key": "seed", "value": 313553746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784922185842, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784922185842, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784922185842, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784922185843, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784922185852, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784922185852, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784923110364, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784923110365, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784923110661, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784923286495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14034254848957062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.5833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784923300166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13995778560638428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2499999999999997e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784923313751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1391441524028778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784923327465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388830542564392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.583333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784923341103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13881923258304596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.25e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784923354726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13892051577568054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784923368201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13931384682655334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923368202, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784923368202, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784923451952, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49895021319389343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784923451953, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784923451954, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784923465376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13970088958740234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923478899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13943643867969513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923492607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13930463790893555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923506407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.139229416847229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923520243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13904637098312378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9916666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923534262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1395665407180786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923548556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13886556029319763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923548557, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784923548557, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784923591985, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49912527203559875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784923591985, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784923591985, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784923606034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13903075456619263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923620258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1392870545387268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923633874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13926281034946442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923648345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1391051709651947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.9916666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923662277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1390806883573532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923676783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1391146332025528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923690561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389055699110031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923690561, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784923690561, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784923734440, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49933359026908875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784923734441, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784923734441, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784923748360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13883185386657715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923762951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13883084058761597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.8249999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923777106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386522650718689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923791497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386401653289795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923806299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13842859864234924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3249999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923820306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382438987493515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923834936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384453922510147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923834937, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784923834937, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784923878386, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49958881735801697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784923878387, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784923878387, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784923892330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13820911943912506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923906577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13820016384124756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923921104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13824273645877838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1583333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923934965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138040691614151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923948865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13730236887931824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923962890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378091275691986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923977329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13726678490638733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923977330, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784923977330, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784924020486, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.499907910823822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784924020487, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784924020487, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784924034466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376279890537262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924048316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13725055754184723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924062539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379377245903015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924076958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13731792569160461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924090799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13614974915981293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.658333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924104553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13647881150245667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924118489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13761216402053833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924118490, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784924118490, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784924161782, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5002878308296204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784924161783, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784924161783, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784924176113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13731719553470612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924190443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365429162979126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924204701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365707516670227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924219012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13577911257743835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924233457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13585197925567627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924248282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13539224863052368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924262291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13565492630004883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924262292, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784924262292, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784924304838, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5007304549217224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784924304839, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784924304839, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784924318887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13491088151931763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924333135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367110013961792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924348115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13804218173027039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924362559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13464275002479553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.824999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924377187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13635891675949097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924391466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352977752685547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.158333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924406298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13675731420516968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.324999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924406299, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784924406300, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784924449939, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5013273358345032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784924449940, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784924449940, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784924464142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13666415214538574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924478380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367979794740677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924492770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352170705795288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924507218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131240114569664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924521664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331607699394226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924535704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13399943709373474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924549658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352946162223816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924549707, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784924549707, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784924593666, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.501997172832489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784924593666, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784924593667, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784924607818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13514569401741028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924621979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12990185618400574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924636910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13186797499656677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924650988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.136742502450943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.1158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924665102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13259582221508026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924679084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13188402354717255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924693693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13277778029441833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924693693, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784924693694, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784924737435, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5027119517326355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784924737436, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784924737436, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784924751385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13188621401786804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924766146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1342782974243164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924779911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1282966285943985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924793764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1302568018436432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924807830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338598132133484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924822289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12860895693302155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924836335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300169974565506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924836335, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784924836336, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784924880918, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5037481784820557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784924880919, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784924880919, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784924895845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13460445404052734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924909937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13298560678958893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.3158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924924389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13451936841011047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924938813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13502360880374908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924953661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13603031635284424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.3658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924968185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13355836272239685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924983029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13693177700042725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924983030, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784924983030, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784925027079, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5052387118339539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784925027080, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784925027080, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784925041693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13436082005500793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.4158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925056403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13080790638923645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925070991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133977472782135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.4491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925085813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14022064208984375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.4658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925100536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13045787811279297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925115576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13258662819862366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925130149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13813424110412598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925130149, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784925130149, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784925175066, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5068572759628296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784925175066, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784925175066, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784925189790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12536771595478058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925204254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1313553750514984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925219055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13710656762123108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925233742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13536225259304047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925249287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323932409286499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925263641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289791613817215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.6158333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925278270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12818694114685059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925278271, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784925278271, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784925321633, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5084006190299988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784925321634, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784925321634, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784925336056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13026157021522522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925350620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349189132452011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.6658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925365486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14223510026931763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6825000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925380102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13415119051933289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925395230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13157391548156738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.7158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925409479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13366323709487915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925424019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13610854744911194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925424019, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784925424020, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784925466800, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.510086178779602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784925466801, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784925466801, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784925481080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13473208248615265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925495579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13638761639595032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925510249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13559499382972717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925524903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332017183303833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925539681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13443133234977722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925554656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13303156197071075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925569322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13243705034255981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8658333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925569322, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784925569323, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784925612471, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5118792653083801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784925612472, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784925612472, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784925627090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13199833035469055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925641634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346013844013214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8991666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925656392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14185941219329834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925671398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13358628749847412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925686314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13441112637519836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925701460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132329061627388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925716076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331784874200821, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925716076, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784925716077, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784925758496, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5139707922935486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784925758497, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784925758497, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784925773026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13844731450080872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925787511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13297364115715027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925801691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12885211408138275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925816061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300225406885147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925830649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13795724511146545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925845486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13175609707832336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925860247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13621962070465088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925860247, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784925860248, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784925904075, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5163049697875977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784925904076, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784925904076, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784925918892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13684813678264618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.1158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925933507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13647854328155518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925948240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13141322135925293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1491666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925962822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13009263575077057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925977570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13164228200912476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925992593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13524477183818817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926007687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1328985095024109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926007688, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784926007688, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784926051469, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5189565420150757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784926051470, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784926051470, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784926066275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13590967655181885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926081563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13234004378318787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926096530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13787519931793213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926111335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363523304462433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926126252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385384351015091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926141144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344764232635498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926156362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13910910487174988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926156363, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784926156363, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784926199436, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5217757821083069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784926199437, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784926199437, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784926214408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349618285894394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926229444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14775991439819336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926244777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13516123592853546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926259587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13299715518951416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926274807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13411769270896912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926289745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331273317337036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926304696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366271823644638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926304696, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784926304697, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784926348447, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5247534513473511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784926348447, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784926348447, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784926363530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13618093729019165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926378739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352235972881317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926394186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13377463817596436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926408718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13364359736442566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926423113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331673562526703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926437558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12910330295562744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926451820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12563253939151764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926451821, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784926451821, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784926496316, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5282393097877502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784926496316, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784926496316, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784926510675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13274791836738586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.5824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926525353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13250254094600677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926540163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13504400849342346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926554872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13538005948066711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926569825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367051750421524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926584879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12885767221450806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926599878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12998144328594208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926599879, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784926599879, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784926643414, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5335071682929993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784926643414, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784926643414, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784926658189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13266557455062866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.699166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926672923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12723980844020844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926687597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13330388069152832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926702229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13212910294532776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926717201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12711921334266663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926732292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13227391242980957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926746754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13451914489269257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926746754, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784926746754, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784926791322, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5404473543167114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784926791323, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784926791323, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784926806598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12945058941841125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926821294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1305559277534485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926835801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13070455193519592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926850307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13000120222568512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.8658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926864565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1280267834663391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926878943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12341861426830292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926893687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12740525603294373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926893688, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784926893688, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784926936802, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.549720287322998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784926936803, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784926936803, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784926951476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360294222831726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926966244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13339826464653015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926980477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1392356902360916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926995752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13378536701202393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927010280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12799063324928284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927024828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300361007452011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927039633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12718912959098816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927039634, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784927039634, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784927084179, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5618278980255127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784927084179, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784927084179, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784927098457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12939715385437012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927112808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12770360708236694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927127553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13276824355125427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927142424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329708695411682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927157480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1316293329000473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927172314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13387885689735413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927187659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13072721660137177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927187660, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784927187660, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784927230713, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5784400701522827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784927230713, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784927230713, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784927245404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12987112998962402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927259979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300901174545288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927274670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13034231960773468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927289423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1328262984752655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927303935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13407903909683228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927318186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13005965948104858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927333147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13098470866680145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927333148, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784927333148, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784927376240, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5951531529426575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784927376240, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784927376240, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784927390903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12871117889881134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927405982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1314857006072998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927420384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1292611062526703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927435877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12829825282096863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927451072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1283256709575653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927466234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324734389781952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927481482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13308638334274292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927481483, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784927481483, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784927524683, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6155332922935486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784927524684, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784927524684, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784927539264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13089345395565033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927554333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12879741191864014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927569714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13104639947414398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927585055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13229236006736755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927600402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336984634399414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927615392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13010074198246002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927630936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12934017181396484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.4991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927630936, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784927630937, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784927674364, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.636498749256134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784927674365, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784927674365, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784927689186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13040310144424438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.5158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927704008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12836918234825134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927718905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13001194596290588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.5491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927733659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13572485744953156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927748541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312469094991684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927763598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12680183351039886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927778851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13066783547401428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927778851, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784927778852, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784927821778, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6608150601387024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784927821778, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784927821779, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784927836535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12723007798194885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6325000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927852160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12868568301200867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927867286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12825311720371246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927882151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12830281257629395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927896952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12603062391281128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927911841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1236049085855484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927926708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12471947073936462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927926709, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784927926709, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784927968937, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6877496838569641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784927968938, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784927968938, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784927984403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12765946984291077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927999746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12375127524137497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928015060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1174536943435669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.7824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928030458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1234794408082962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928045486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11978502571582794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928060122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12020723521709442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8324999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928074754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11847937107086182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.8491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928074755, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784928074755, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784928117044, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7112540006637573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784928117045, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784928117045, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784928131931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1232050359249115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928146462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11884751915931702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928161267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11539691686630249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.8991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928176217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12182827293872833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928191348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1200210228562355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928206815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11906507611274719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.9491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928221794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11040332913398743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.9658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928221794, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784928221795, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784928266079, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7377918362617493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784928266079, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784928266080, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784928280684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11677883565425873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928296139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11677147448062897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928311182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12041742354631424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.0158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928325999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11449004709720612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928340964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11423300206661224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928355640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11449122428894043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0658333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928370243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11026551574468613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928370291, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784928370291, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784928413801, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7480444312095642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784928413802, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784928413802, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784928428376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11544883251190186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928443699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11309882253408432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.1158333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928458985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11803142726421356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928474544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11305390298366547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928489321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11286622285842896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1658333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928504983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11972400546073914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928519584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10769321024417877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928519584, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784928519585, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784928563158, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7536473274230957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784928563159, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784928563159, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784928577797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1151939183473587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928592551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11331193149089813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928607556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11139173805713654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928622327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11008497327566147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928637082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1046566367149353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928651655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11711041629314423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928667083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11355789005756378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928667084, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784928667084, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784928710714, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.755949079990387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784928710715, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784928710715, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784928725773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11574973165988922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928741131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11294786632061005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.3491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928756078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11637299507856369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928771582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1018119752407074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928786755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11387968063354492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.3991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928801867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732918232679367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.4158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928816852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10672375559806824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928816852, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784928816853, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784928861016, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7592638731002808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784928861017, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784928861017, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784928875800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085132360458374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.4491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928890721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1083298996090889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928905267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11331772804260254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.4824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928920139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10863593220710754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928935369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11282055079936981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928950511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10203976929187775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928965743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11135314404964447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928965744, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784928965744, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784929009876, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7608283758163452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784929009876, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784929009877, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784929025015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10696162283420563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929039882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10901738703250885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.5824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929055866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11940069496631622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929071410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12111903727054596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929086677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11355902999639511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929101999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11053973436355591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929117193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11639192700386047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929117194, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784929117194, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784929161128, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7626438736915588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784929161129, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784929161129, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784929176322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1135602667927742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929191762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1068248599767685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.6991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929207366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11648652702569962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929222655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11025096476078033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929238404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11359985172748566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929253681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11399298906326294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929269410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10399416089057922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929269411, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784929269412, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784929314011, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7635855078697205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784929314012, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784929314012, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784929329430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11878705769777298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929344803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10670119524002075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929360183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10518242418766022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929375496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11346130073070526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929390692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11771883070468903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929406071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10621936619281769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929421908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11301160603761673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929421908, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784929421909, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784929465702, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7651747465133667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784929465703, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784929465703, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784929481185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11428344994783401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929497005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10678493231534958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929512112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10852670669555664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929527812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11407694220542908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929542972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10700219869613647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929557806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10356762260198593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929572680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11645941436290741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.015833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929572681, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784929572681, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784929616224, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7673921585083008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784929616224, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784929616225, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784929631150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11008653044700623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929646139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11468670517206192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929660943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10579819977283478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929675885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10507984459400177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929691220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10212881863117218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929706385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09604594111442566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929721956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10579203814268112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929721956, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784929721957, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784929765040, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7678667306900024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784929765041, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784929765041, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784929780078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10519352555274963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929794896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11026601493358612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929810706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1179376095533371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929826075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10988186299800873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929841303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10596532374620438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929856334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10101854801177979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929871559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10597026348114014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929871559, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784929871560, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784929914778, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7689653635025024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784929914778, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784929914778, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784929929937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10542990267276764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929944939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10517261922359467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929960257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10969395190477371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929975583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10743845254182816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929991036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10604487359523773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930006494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1116301566362381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930022278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10914334654808044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930022278, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784930022279, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784930065842, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7699422836303711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784930065843, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784930065843, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784930081006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10491664707660675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930096626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10199491679668427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930111984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10804000496864319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930127269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10832757502794266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930142406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10508449375629425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930157461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10103468596935272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930172675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10491721332073212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930172675, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784930172676, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784930216334, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7707123756408691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784930216336, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784930216336, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784930231441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10485590994358063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930246669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09984967112541199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930261678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10494448989629745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930277222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11019408702850342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930292697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10599091649055481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930308305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10302960127592087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930323887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10881300270557404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930323887, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784930323888, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784930368018, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7711547613143921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784930368019, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784930368019, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784930382946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09997622668743134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930398709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10297053307294846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930413814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10704267024993896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930428919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10318005084991455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930443944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10240603983402252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930458804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10692042112350464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930473929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10893847793340683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930473929, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784930473930, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784930517509, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7718929052352905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784930517510, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784930517510, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784930532602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10530628263950348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930547686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10350555181503296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930562768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10828660428524017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930578221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10603472590446472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930593793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10972472280263901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930609436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723234713077545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930624821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11071121692657471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930624822, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784930624822, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784930668826, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7728163003921509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784930668827, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784930668828, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784930683576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10672329366207123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930699246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1015467494726181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930714706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10422328114509583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930730253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10191339999437332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930745546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10852359235286713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930760758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11327914893627167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930776132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1076357364654541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930776133, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784930776134, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784930819969, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7728846073150635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784930819970, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784930819970, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784930835136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10362787544727325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930850344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11090055108070374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930865551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10763772577047348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930880966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09995588660240173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930896606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11171594262123108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930912239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11214211583137512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930928256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10559244453907013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930928256, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784930928257, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784930971926, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.772803783416748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784930971927, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784930971927, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784930987467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.107988640666008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931002548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09934523701667786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931018737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11590738594532013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931034661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11500571668148041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931050220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11469541490077972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931065716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10404683649539948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931081332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1061411052942276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931081332, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784931081333, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784931124520, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7732065320014954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784931124520, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784931124520, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784931140247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11094638705253601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931155631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11361470818519592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931171381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11001426726579666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931187261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11018174886703491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931203093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11399547010660172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931218774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11221614480018616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931234595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11243182420730591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931234595, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784931234595, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784931278493, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7736842036247253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784931278494, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784931278494, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784931293502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10898794233798981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931309538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10707272589206696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931325311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10646525025367737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931341377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11393536627292633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931356774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11769388616085052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.3825e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_16384/seed530691108.log b/recommendation_v4/rcp_logs/gbs_16384/seed530691108.log new file mode 100644 index 000000000..692721b49 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_16384/seed530691108.log @@ -0,0 +1,669 @@ +:::MLLOG {"namespace": "", "time_ms": 1784931742455, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784931742455, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784931763370, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784931763372, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784931763372, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784931763372, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784931763372, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784931763372, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 16384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784931763372, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784931763372, "event_type": "POINT_IN_TIME", "key": "seed", "value": 530691108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784931763372, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784931763372, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784931763373, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784931763373, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784931763499, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784931763500, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784932637675, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784932637676, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784932638012, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784932870493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13833817839622498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.5833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932884312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382971704006195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2499999999999997e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932897562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860516250133514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932911181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13874778151512146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.583333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932924637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386844664812088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.25e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932938165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13871726393699646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932951528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13812920451164246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932951529, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784932951530, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784933038073, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4985129237174988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784933038074, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784933038074, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784933051456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13815785944461823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933064981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13831965625286102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933078627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13855266571044922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933092153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13846394419670105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933105730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13856039941310883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9916666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933119871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13793516159057617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933134294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13845030963420868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933134294, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784933134295, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784933178092, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4987214505672455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784933178093, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784933178093, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784933191883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13823670148849487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933206044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380215436220169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933219805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13803039491176605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933234151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13789594173431396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.9916666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933247776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13809704780578613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933262169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13820835947990417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933275993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13781669735908508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933275994, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784933275994, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784933318820, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4990042448043823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784933318821, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784933318821, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784933332715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13834144175052643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933347138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13797345757484436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.8249999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933360948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13696660101413727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933375010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13808898627758026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933389770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13748608529567719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3249999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933403837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13711605966091156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933418431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13706600666046143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933418431, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784933418432, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784933461652, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49935898184776306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784933461653, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784933461653, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784933475097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13660281896591187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933488892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13740946352481842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933503091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13726723194122314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1583333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933517045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366814225912094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933530966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13541537523269653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933544978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13679899275302887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933559105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13573041558265686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933559106, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784933559106, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784933602077, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.499824583530426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784933602077, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784933602078, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784933615922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13657966256141663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933629597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13614654541015625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933643694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13741682469844818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933658009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13648560643196106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933671790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13464701175689697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.658333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933685432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13526782393455505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933699301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13718149065971375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933699302, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784933699302, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784933742526, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5003888010978699, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784933742526, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784933742527, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784933756848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13667887449264526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933770903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13547274470329285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933784673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13567276298999786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933798599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13463151454925537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933812787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13487458229064941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933827474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13444650173187256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933841559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13477271795272827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933841560, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933841560, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933884184, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5010992288589478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933884184, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933884185, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933898285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13384920358657837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933912556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13635583221912384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933927458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13827025890350342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933941695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338326334953308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.824999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933956053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360178291797638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933970246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13481006026268005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.158333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933984817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13670194149017334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.324999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933984818, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784933984819, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784934028253, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5019844770431519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784934028254, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784934028254, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784934042606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365453153848648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784934056942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365528702735901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784934071289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346653699874878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784934085770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12994441390037537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784934100269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324114203453064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934114328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335850954055786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934128241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13482940196990967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934128291, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784934128292, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784934171466, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5029578804969788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784934171467, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784934171467, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784934185643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349901556968689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934199748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12905532121658325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934214768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13111189007759094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934228617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13686397671699524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.1158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934242697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13231180608272552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934256655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13122248649597168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934271248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13240481913089752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934271248, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934271249, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934314213, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.503998339176178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934314214, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934314214, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934328081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13160935044288635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934342856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343594193458557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934356642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12794968485832214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934370487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12949582934379578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934384597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337938755750656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934398999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12805138528347015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934412894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12983697652816772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934412894, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934412894, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934456796, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5052961707115173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934456796, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934456797, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934471711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13462340831756592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934485513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13309475779533386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.3158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934499660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344483345746994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934513756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350967437028885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934528278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13606691360473633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.3658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934542483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13353794813156128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934556974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13697507977485657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934556974, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934556975, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934599394, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5068166255950928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934599395, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934599395, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934613963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13433733582496643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.4158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934628404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1305384784936905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934642772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13396002352237701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.4491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934657393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1403263360261917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.4658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934672025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13038834929466248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934686902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324405074119568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934701450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13833078742027283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934701450, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934701450, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934744501, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5082074999809265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934744502, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934744502, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934759401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1257702112197876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934773886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13114267587661743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934788755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13729867339134216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934803416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13545671105384827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934818470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13245537877082825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934832690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1286899447441101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.6158333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934847097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12803581357002258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934847097, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784934847098, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784934890989, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.50975102186203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784934890990, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784934890990, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784934905413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13018955290317535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934919836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13481810688972473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.6658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934934647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14229267835617065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6825000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934949216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13423112034797668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934964243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13152065873146057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.7158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934978382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13371416926383972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934992673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360093355178833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934992674, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784934992674, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784935036793, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5115908980369568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784935036793, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784935036794, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784935050926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346597969532013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935065074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13642728328704834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935079551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13566666841506958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935094075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13315343856811523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935108753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1342317759990692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935123257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13281211256980896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935137491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325078308582306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8658333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935137492, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784935137492, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784935180105, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5135055184364319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784935180105, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784935180105, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784935194406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13182413578033447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935208715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345825046300888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8991666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935223227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14183416962623596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935238043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338101029396057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935252760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13446632027626038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935267739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13227076828479767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935282328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13315705955028534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935282329, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784935282329, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784935324816, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5157426595687866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784935324817, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784935324817, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784935338903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13820959627628326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935353094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331394910812378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935367363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12883338332176208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935381730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12990377843379974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935396372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13795489072799683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935411124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131578266620636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935425798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13633495569229126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935425799, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784935425800, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784935469874, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5182860493659973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784935469875, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784935469875, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784935484546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13681994378566742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.1158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935498917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13632890582084656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935513558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13142946362495422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1491666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935527969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1299135833978653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935542550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13173484802246094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935557276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352209448814392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935572012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13266731798648834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935572013, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784935572013, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784935615473, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5211758613586426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784935615474, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784935615474, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784935630033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360083520412445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935645159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13217535614967346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935660032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379375457763672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935674665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13642239570617676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935689547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384769231081009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935704249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344021111726761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935719363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13901981711387634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935719364, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784935719364, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784935763234, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5240886211395264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784935763235, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784935763235, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784935778106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350283920764923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935793067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14776721596717834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935808395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13515688478946686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935823437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312587141990662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935838455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13415226340293884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935853216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13311800360679626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935868217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366090476512909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935868218, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784935868218, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784935912449, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5276654958724976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784935912449, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784935912449, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784935927432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13604553043842316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935942613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13532263040542603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935957928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337597817182541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935972375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13364535570144653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935986880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332501769065857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936001277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12951289117336273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936015497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12566375732421875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936015497, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784936015498, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784936058553, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5321872234344482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784936058554, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784936058554, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784936073040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13250431418418884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.5824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936087713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13262292742729187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936102497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13484594225883484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936117210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13542290031909943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936132373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366725116968155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936147137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288546323776245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936161825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12987348437309265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936161826, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784936161826, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784936204468, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5386106371879578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784936204469, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784936204469, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784936219007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13247206807136536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.699166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936233574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12700822949409485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936247940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331615447998047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936262372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13219450414180756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936277278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12705668807029724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936292175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323353350162506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936306534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13426847755908966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936306535, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784936306535, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784936350040, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5471780896186829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784936350042, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784936350042, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784936364982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12936198711395264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936379595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13020411133766174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936394054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13060204684734344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936408659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297989785671234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.8658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936422905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12783095240592957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936437300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12336324900388718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936452105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12712107598781586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936452106, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784936452106, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784936495927, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5584678649902344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784936495928, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784936495928, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784936510596, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13563251495361328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936525400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13323497772216797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936539642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13910728693008423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936554678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13339997828006744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936569323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12770771980285645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936583685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.129603311419487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936598279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1269536316394806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936598279, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784936598280, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784936642535, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.571601152420044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784936642536, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784936642536, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784936656724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12912680208683014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936670951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1275295913219452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936685599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13248306512832642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936700397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13269031047821045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936715252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13134746253490448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936729770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13370569050312042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936744691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13044774532318115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936744691, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784936744692, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784936789412, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5860704183578491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784936789412, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784936789412, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784936804375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12955200672149658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936819084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12997448444366455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936833854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12974104285240173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936848583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325576901435852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936863244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13346907496452332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936877770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12955273687839508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936893126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304420530796051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936893127, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784936893127, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784936938520, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5988540053367615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784936938520, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784936938520, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784936953342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12822359800338745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936968711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13081282377243042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936983169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1283353865146637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936998459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12766428291797638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937013709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1272287517786026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937028796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13191792368888855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937043953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13231343030929565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937043954, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784937043954, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784937089440, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6102829575538635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784937089440, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784937089441, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784937104094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13008743524551392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937119211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1279986947774887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937134614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303672194480896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937150081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13194531202316284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937165387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13336963951587677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937180220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12894777953624725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937195911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288244128227234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.4991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937195912, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784937195912, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784937241255, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6203109622001648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784937241256, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784937241256, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784937256301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12974829971790314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.5158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937271471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.127842515707016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937286617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12903723120689392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.5491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937301445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13526540994644165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937316442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13076025247573853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937331426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1261175274848938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937346708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12916278839111328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937346708, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784937346709, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784937391646, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6284033060073853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784937391647, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784937391647, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784937406419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12607446312904358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6325000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937421860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12815535068511963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937436743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12736114859580994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937451674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12713120877742767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937466377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12549060583114624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937481196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12260158360004425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937496154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12415672838687897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937496154, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784937496155, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784937540330, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6366043090820312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784937540331, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784937540331, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784937555819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1271584928035736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937570965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12324894219636917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937586085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11724844574928284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.7824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937601390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12294614315032959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937616331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11890138685703278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937631015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11959219723939896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8324999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937645692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11708557605743408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.8491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937645692, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784937645692, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784937690117, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.645308256149292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784937690118, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784937690118, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784937705042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1224801242351532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937719631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11849591135978699, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937734110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11483287066221237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.8991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937748856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.122162364423275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937763876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12030249834060669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937779067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11918897926807404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.9491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937793978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1105046272277832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.9658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937793979, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784937793979, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784937839427, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6589432954788208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784937839427, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784937839427, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784937854006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11725957691669464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937869560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11703576147556305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937884503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12072452902793884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.0158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937899267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11536988615989685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937914122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11358772963285446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937928870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11508932709693909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0658333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937943585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11137060821056366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937943634, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784937943634, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784937987832, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6728841662406921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784937987832, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784937987832, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784938002377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11662811785936356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938017730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11442252993583679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.1158333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938033010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11957229673862457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938048496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11457392573356628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938063278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11501116305589676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1658333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938078678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12154541909694672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938093393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10921785235404968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938093394, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784938093394, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784938136935, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6937421560287476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784938136936, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784938136936, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784938151777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11754634976387024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938166599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11568988859653473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938181639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1132710725069046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938196408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11143489927053452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938211297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10703767836093903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938226002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12032513320446014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938241442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11678753793239594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938241443, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784938241443, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784938285106, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7171506285667419, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784938285107, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784938285107, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784938300097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11951511353254318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938315368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11574643105268478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.3491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938330195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11852621287107468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938345597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10495689511299133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938360707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11652971804141998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.3991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938375761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11174336075782776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.4158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938390787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11014966666698456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938390788, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784938390788, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784938434071, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7374840974807739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784938434072, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784938434072, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784938448889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11253684759140015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.4491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938463795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11096355319023132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938478498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1165262758731842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.4824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938493446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11190599203109741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938508740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.116548553109169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938524092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10703637450933456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938539294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11460085213184357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938539294, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784938539295, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784938582620, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7497473955154419, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784938582621, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784938582621, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784938597659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10943181812763214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938612337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11231428384780884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.5824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938628256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12229709327220917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938643707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1243550032377243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938658815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11932452768087387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938674085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11345656216144562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938689175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11844301223754883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938689176, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784938689176, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784938731692, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.757445752620697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784938731692, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784938731693, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784938746765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11828060448169708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938762007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11048515141010284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.6991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938777510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.120303675532341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938792912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11310083419084549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938808588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11668585240840912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938823775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11484179645776749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938839517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10633064806461334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938839518, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784938839518, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784938883333, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7614441514015198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784938883335, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784938883335, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784938898638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11900435388088226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938914015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10925772786140442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938929332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10649004578590393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938944524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11595994979143143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938959706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11939727514982224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938975038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10858584940433502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938990738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11523958295583725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938990739, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784938990740, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784939034553, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7638744711875916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784939034553, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784939034553, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784939050079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11669066548347473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939065901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10912835597991943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939080893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10886730253696442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939096410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11474379897117615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939111346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10829667001962662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939126150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10423577576875687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939140986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11572988331317902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.015833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939140987, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784939140988, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784939185042, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7653255462646484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784939185043, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784939185043, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784939199845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11030484735965729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939214699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11517201364040375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939229296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10805819928646088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939243844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10565771162509918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939259032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10350878536701202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939274154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09713125228881836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939289573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1065792441368103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939289574, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784939289574, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784939334388, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7661648988723755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784939334389, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784939334389, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784939349323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10508207976818085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939363945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11016837507486343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939379932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11928801983594894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939395344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10892215371131897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939410223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10652267932891846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939424906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1010885089635849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939440085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10717801749706268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939440086, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784939440086, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784939482967, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7670717239379883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784939482968, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784939482968, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784939498067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10498560965061188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939512930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10579423606395721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939527907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10966964066028595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939543151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10798405855894089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939558448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10649394243955612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939573758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11197461187839508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939589383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10973761975765228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939589383, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784939589384, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784939633872, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7685182690620422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784939633873, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784939633873, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784939648813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10513243079185486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939664157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10232337564229965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939679314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11007751524448395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939694508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10945392400026321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939709583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1052856296300888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939724551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10048718750476837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939739622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10468053817749023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939739623, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784939739623, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784939783010, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7688003778457642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784939783011, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784939783011, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784939798050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10445070266723633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939813049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1000535637140274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939827742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10529014468193054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939842862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11039353162050247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939857944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10645131021738052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939873403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10325074195861816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939888748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10999242216348648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939888748, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784939888749, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784939932413, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.76991206407547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784939932414, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784939932414, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784939946984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09988299012184143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939962516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10336455702781677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939977673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072852611541748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939992851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10364783555269241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940007771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10255078971385956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940022578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10735450685024261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940037591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10918370634317398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940037592, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784940037592, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784940080872, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7699956297874451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784940080873, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784940080873, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784940095724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10557430982589722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940110551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10303381085395813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940125147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10863751173019409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940140264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10604734718799591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940155483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10999766737222672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940170817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1077997237443924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940185958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1109069436788559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940185958, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784940185959, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784940229455, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7713656425476074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784940229455, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784940229456, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784940244140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10682470351457596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940259530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10134953260421753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940274504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10495256632566452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940289671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10222934186458588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940304383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10814940929412842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940319210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11325640976428986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940333976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10837002843618393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940333977, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784940333977, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784940377334, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7711402773857117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784940377335, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784940377335, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784940392276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10455602407455444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940407243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1114589273929596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940422043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10750517249107361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940437244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10022680461406708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940452405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11245441436767578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940467608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1127045750617981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940483358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10529027879238129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940483359, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784940483359, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784940526626, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7710765600204468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784940526627, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784940526627, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784940541868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10821934044361115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940556739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09971412271261215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940572749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11643336713314056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940588404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11631868779659271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940603849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11490845680236816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940619397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10420700162649155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940634992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.106727734208107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940634993, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784940634994, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784940677882, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7708079814910889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784940677883, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784940677883, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784940693418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11149255931377411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940708792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1141812652349472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940724414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11002348363399506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940740109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11053971946239471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940755703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1146058663725853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940771129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11249101907014847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940786780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11261239647865295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940786781, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784940786781, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784940828983, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7715572714805603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784940828984, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784940828984, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784940843975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10953322798013687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940859676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10764065384864807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332499999999999e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_16384/seed547461891.log b/recommendation_v4/rcp_logs/gbs_16384/seed547461891.log new file mode 100644 index 000000000..55f3d204c --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_16384/seed547461891.log @@ -0,0 +1,620 @@ +:::MLLOG {"namespace": "", "time_ms": 1784931743669, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784931743670, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784931764687, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784931764689, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784931764689, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784931764689, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784931764689, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784931764689, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 16384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784931764689, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784931764689, "event_type": "POINT_IN_TIME", "key": "seed", "value": 547461891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784931764689, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784931764689, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784931764689, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784931764689, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784931764690, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784931764690, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784932551738, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784932551739, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784932552089, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784932843736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14011096954345703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.5833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932858259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1395670622587204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2499999999999997e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932872357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13922682404518127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932886851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389314979314804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.583333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932901303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13872897624969482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.25e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932915790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13898824155330658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932930006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13911840319633484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932930007, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784932930007, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784932977268, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5022149682044983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784932977268, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784932977268, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784932991620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13954810798168182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933009329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1393607258796692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933023724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13896581530570984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933038093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13927879929542542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933052137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13910746574401855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9916666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933066703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13925877213478088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933081756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13890935480594635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933081757, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784933081757, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784933131322, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5023825764656067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784933131323, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784933131323, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784933145489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389566957950592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933160319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13918301463127136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933174908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13894188404083252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933189849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13884998857975006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.9916666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933204142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138747900724411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933219703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13895055651664734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933234374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13872727751731873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933234375, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784933234375, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784933283392, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5025967955589294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784933283393, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784933283393, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784933297849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13870441913604736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933313041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384090781211853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.8249999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933327991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13840028643608093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933342650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386442929506302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933358506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13832679390907288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3249999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933373007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13827753067016602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933388725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13798464834690094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933388726, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784933388726, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784933439236, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5028647184371948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784933439237, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784933439238, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784933454264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137877956032753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933468794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13819561898708344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933483886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13790932297706604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1583333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933498814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13759195804595947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933513576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13691800832748413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933528447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13759098947048187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933543020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13695575296878815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933543021, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784933543021, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784933593351, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5032035112380981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784933593352, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784933593352, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784933608472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13739407062530518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933623112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13706108927726746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933637875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137693852186203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933652649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13723309338092804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933667771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13598591089248657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.658333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933682394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13623571395874023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933697044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375485062599182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933697044, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784933697045, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784933742548, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5036138296127319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784933742549, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784933742549, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784933757799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13711439073085785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933773142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13618260622024536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933788078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363830864429474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933803485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356608271598816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933818470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13563178479671478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933833977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352245807647705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933849620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13559122383594513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933849620, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933849621, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933899008, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5041259527206421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933899008, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933899009, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933914229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13490936160087585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933929450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367165446281433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933946992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138039231300354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933962917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13449686765670776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.824999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933978580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13622884452342987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933993901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13524246215820312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.158333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784934009386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13666486740112305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.324999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784934009387, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784934009387, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784934060104, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5048261880874634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784934060104, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784934060104, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784934075592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13659168779850006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784934091093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13674002885818481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784934106676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13522028923034668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784934124101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13065451383590698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784934140039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13305024802684784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934155457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338854283094406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934170929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351611614227295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934170976, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784934170977, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784934220282, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5055546164512634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784934220283, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784934220283, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784934235448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13515788316726685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934250441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12980356812477112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934265975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13162502646446228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934281508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13665707409381866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.1158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934296728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13255739212036133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934311696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1316598802804947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934327123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13265129923820496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934327123, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934327124, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934375426, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5063571929931641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934375427, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934375427, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934389906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318589150905609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934405797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13425494730472565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934420642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12819844484329224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934436106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298903524875641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934451483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13378405570983887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934466979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12847624719142914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934481707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13005979359149933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934481708, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934481708, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934528328, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5074077844619751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934528328, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934528329, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934544320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345539391040802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934559638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13301372528076172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.3158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934575189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13433894515037537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934590580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13487011194229126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934606547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13595494627952576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.3658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934621606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13351565599441528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934637072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13681206107139587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934637073, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934637073, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934688052, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5088557600975037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934688053, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934688053, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934703681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13436082005500793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.4158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934719353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13059073686599731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934735330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13370871543884277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.4491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934751138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14042264223098755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.4658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934766729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13028016686439514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934782750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13234582543373108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934798854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13805696368217468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934798854, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934798855, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934849002, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5103106498718262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934849002, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934849003, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934865018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12540830671787262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934880867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13105247914791107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934896856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.136959046125412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934912323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13542261719703674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934928325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13232268393039703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934943665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12871761620044708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.6158333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934959253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12800630927085876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934959253, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784934959254, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784935009213, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5117381811141968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784935009213, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784935009214, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784935024510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13008713722229004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935040325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13474318385124207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.6658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935055682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14225423336029053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6825000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935070883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341175138950348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935086890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13152994215488434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.7158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935102322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13356320559978485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935117985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13595551252365112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935117986, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784935117986, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784935168130, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5133688449859619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784935168130, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784935168131, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784935183643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13455727696418762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935199266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13647542893886566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935214944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354455202817917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935230502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13328252732753754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935246055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13436894118785858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935261620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13294774293899536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935276827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324637234210968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8658333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935276828, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784935276828, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784935326084, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5151175260543823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784935326085, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784935326085, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784935341649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1319682002067566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935357098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13443514704704285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8991666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935372668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14176282286643982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935388682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13357843458652496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935403883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13431233167648315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935419401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13231584429740906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935435083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13321059942245483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935435083, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784935435084, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784935484806, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5171326994895935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784935484807, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784935484807, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784935500140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13847440481185913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935515712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13296541571617126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935530831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288740336894989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935546314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12998417019844055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935562219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13778850436210632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935577623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13171282410621643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935593205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363215148448944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935593205, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784935593206, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784935644057, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5194522738456726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784935644058, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784935644058, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784935659525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13678738474845886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.1158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935675091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13653510808944702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935690917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13130921125411987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1491666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935705911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13012610375881195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935721707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1316889226436615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935737308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13527356088161469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935752874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13265028595924377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935752874, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784935752875, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784935800966, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5221157073974609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784935800967, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784935800967, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784935816365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13602040708065033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935832141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13227133452892303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935848465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13784542679786682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935864291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364675760269165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935880037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384248435497284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935896161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13431954383850098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935912178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13904334604740143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935912179, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784935912179, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784935961866, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5247279405593872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784935961867, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784935961867, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784935977998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13499850034713745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935993903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.15196719765663147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936010368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13506042957305908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936026354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13306938111782074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936042597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1342477798461914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936058657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133060485124588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936074841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365804672241211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936074842, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784936074842, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784936124355, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5278817415237427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784936124356, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784936124356, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784936140510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360626220703125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936156303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352592557668686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936172430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13383552432060242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936188403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336667835712433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936204158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13313280045986176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936219930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295199692249298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936235560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1256670504808426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936235561, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784936235561, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784936285062, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5318164229393005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784936285063, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784936285063, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784936300455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326139271259308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.5824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936316533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325017511844635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936332187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13501715660095215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936347483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13542543351650238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936363155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13669340312480927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936379094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288314163684845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936394783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13003814220428467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936394783, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784936394784, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784936443998, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5376914143562317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784936443999, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784936443999, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784936459522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13261525332927704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.699166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936475135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12701866030693054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936490886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13322436809539795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936506827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13215497136116028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936522828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1270776391029358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936538853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13231264054775238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936554143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13433502614498138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936554144, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784936554144, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784936603293, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.545604944229126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784936603294, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784936603294, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784936619480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12955603003501892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936635315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13045403361320496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936651014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13069327175617218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936666653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12992507219314575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.8658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936682297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1279032677412033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936697910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12345559149980545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936713913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12719964981079102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936713914, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784936713914, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784936763766, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5566655993461609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784936763767, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784936763767, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784936779871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13576695322990417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936795336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13321973383426666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936810239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389981359243393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936826854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13369101285934448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936842537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12783657014369965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936858514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297871470451355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936874264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12703537940979004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936874264, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784936874265, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784936924142, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5704361796379089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784936924143, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784936924143, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784936939847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12909197807312012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936955666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12758800387382507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936971581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13271518051624298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936987936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13282476365566254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937003981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13141891360282898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937019375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13376379013061523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937035271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13043105602264404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937035272, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784937035272, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784937085117, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.586930513381958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784937085118, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784937085118, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784937101297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12959027290344238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937117011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12985196709632874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937132708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298622190952301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937148703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13273832201957703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937164242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13362261652946472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937180120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12942543625831604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937196384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13045623898506165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937196384, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784937196385, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784937246648, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6019637584686279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784937246649, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784937246649, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784937262515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12830492854118347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937278733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13093438744544983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937293826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12854540348052979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937310255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1275492012500763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937326945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.126997172832489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937343383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13156616687774658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937359958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13219353556632996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937359958, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784937359959, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784937410172, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6191465854644775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784937410173, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784937410173, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784937426436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12977559864521027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937442812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1279548555612564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937459462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13019782304763794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937476283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13163168728351593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937492489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13328659534454346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937508196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289137601852417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937525157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1286933720111847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.4991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937525158, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784937525158, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784937575443, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.636939525604248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784937575444, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784937575444, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784937591693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12914124131202698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.5158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937608158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1275160312652588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937624303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12886005640029907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.5491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937640416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13477841019630432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937656904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13037139177322388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937673381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12538450956344604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937689718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12894853949546814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937689718, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784937689718, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784937738140, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.654011607170105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784937738141, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784937738141, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784937755076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12531189620494843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6325000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937772297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12726949155330658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937788323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1265944391489029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937804795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12632550299167633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937820717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12445471435785294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937836795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12175817787647247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937852999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12302751839160919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937852999, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784937852999, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784937902775, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6716793775558472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784937902775, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784937902776, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784937919341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12601493299007416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937935935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12231853604316711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937951880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1160428375005722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.7824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937967838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12207531929016113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937984041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11751396954059601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937999790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11861905455589294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8324999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938015941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11609408259391785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.8491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938015942, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784938015942, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784938066179, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6885344982147217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784938066180, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784938066180, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784938082228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12158489227294922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938098017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11686664074659348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938114214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11311318725347519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.8991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938130295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11962774395942688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938146689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11878795921802521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938163473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11748354136943817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.9491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938179314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10856171697378159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.9658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938179315, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784938179315, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784938227899, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7220892310142517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784938227899, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784938227900, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784938243537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11535663902759552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938260457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11568155884742737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938276735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11902016401290894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.0158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938292929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11351892352104187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938309028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11232701689004898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938324941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11385143548250198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0658333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938341061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10866038501262665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938341109, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784938341109, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784938390811, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7415403723716736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784938390812, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784938390812, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784938406776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1140742152929306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938423335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1124638020992279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.1158333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938439572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11700350791215897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938456024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11189227551221848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938471709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11213544011116028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1658333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938487730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11906982958316803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938503840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1059151142835617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938503840, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784938503840, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784938554273, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7525176405906677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784938554273, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784938554274, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784938570255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11491402238607407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938586461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1135338693857193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938602744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11127406358718872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938618756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11037218570709229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938634980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10466299951076508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938650739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11861427128314972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938667072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11337023228406906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938667072, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784938667073, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784938716960, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7597202658653259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784938716961, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784938716961, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784938733419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11608533561229706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938749773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11332957446575165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.3491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938765484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11649259179830551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938781785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10179827362298965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938798091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11411577463150024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.3991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938814191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10821951180696487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.4158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938830185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10684183239936829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938830186, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784938830186, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784938880076, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7633603811264038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784938880077, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784938880077, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784938896166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10872945189476013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.4491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938912127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10886016488075256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938928215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11370278149843216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.4824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938944299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10852843523025513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938960585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11228643357753754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938976986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10285025835037231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938993193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11140389740467072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938993194, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784938993194, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784939043000, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7662031054496765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784939043001, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784939043001, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784939059110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10729677975177765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939074752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10837195068597794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.5824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939091547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11915165185928345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939108245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12125395238399506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939124586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11431124806404114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939140951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11070214956998825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939157539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11595958471298218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939157539, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784939157540, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784939207661, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7678863406181335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784939207662, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784939207662, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784939223766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11453602463006973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939240092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732457786798477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.6991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939256836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11698345839977264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939273156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10988878458738327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939289768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11324897408485413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939305857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1133686825633049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939321929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10377815365791321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939321930, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784939321930, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784939371789, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7691175937652588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784939371790, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784939371790, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784939388538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11805646121501923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939404857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10651659965515137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939421479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10515613108873367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939437731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11358760297298431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939453905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11824697256088257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939470648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10629098117351532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939487277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11352603137493134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939487278, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784939487278, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784939536074, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7704080939292908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784939536074, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784939536075, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784939552689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11410464346408844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939569594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10681073367595673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939585336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10806068778038025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939602384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11327868700027466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939619053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10709598660469055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939635392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10391384363174438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939651455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11617778241634369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.015833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939651456, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784939651456, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784939700293, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7711290717124939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784939700294, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784939700294, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784939716616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10950686037540436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939732561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11449217796325684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939748458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1054808497428894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939764576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10441997647285461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939780760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10222119092941284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939797099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09526434540748596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939813705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10543382912874222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939813705, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784939813706, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784939862695, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7719566226005554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784939862696, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784939862696, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784939878628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10497322678565979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939894415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10969942808151245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939911343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11802917718887329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939927784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10957978665828705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939944199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10590498149394989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939960077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10056555271148682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939976540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10616771876811981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939976541, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784939976542, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784940026257, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7726616263389587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784940026258, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784940026258, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784940042465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10501104593276978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940058412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10557768493890762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940074905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10933056473731995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940091086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10659122467041016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940107459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10602819919586182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940124041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11118556559085846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940140553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10973838716745377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940140553, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784940140554, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784940189675, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773196280002594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784940189676, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784940189676, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784940205442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10467787832021713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940221554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10216784477233887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940238224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081535667181015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940254463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10811042785644531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940270787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.104738250374794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940286884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10065382719039917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940303022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10457825660705566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940303022, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784940303022, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784940352918, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7735177278518677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784940352919, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784940352919, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784940369160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10447078943252563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940385308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09955224394798279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940401505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10430006682872772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940418332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11000227183103561, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940434905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10579918324947357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940451679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1031409353017807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940468157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10918638110160828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940468157, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784940468158, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784940518492, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738419771194458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784940518493, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784940518493, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784940533798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09990628063678741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940550436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1029524952173233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940566828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10706661641597748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940583071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10325615108013153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940599182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10220102965831757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940615340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10599412024021149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940631347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1088932603597641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940631348, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784940631348, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784940680500, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7741097211837769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784940680501, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784940680501, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784940696408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10528606176376343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940712784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10319405794143677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940728727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10777462273836136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940745071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10552187263965607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940761541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10940141975879669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940778063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10755304992198944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940794420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11071759462356567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940794420, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784940794421, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784940844074, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7744320034980774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784940844075, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784940844075, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784940859643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10676389932632446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849166666666666e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_16384/seed782855750.log b/recommendation_v4/rcp_logs/gbs_16384/seed782855750.log new file mode 100644 index 000000000..fb0cfd6cf --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_16384/seed782855750.log @@ -0,0 +1,676 @@ +:::MLLOG {"namespace": "", "time_ms": 1784926157114, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784926157115, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784926177270, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784926177272, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784926177272, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784926177272, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784926177272, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784926177272, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 16384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784926177272, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784926177272, "event_type": "POINT_IN_TIME", "key": "seed", "value": 782855750, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784926177272, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784926177272, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784926177272, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784926177272, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784926177527, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784926177527, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784927081968, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784927081969, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784927082288, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784927239499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376626044511795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.5833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784927253440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137804314494133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2499999999999997e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784927266659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13822005689144135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784927280306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13868525624275208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.583333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784927293812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388828605413437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.25e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784927307185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384182721376419, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784927320462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13826918601989746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927320463, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784927320464, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784927364889, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5012149810791016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784927364890, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784927364890, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784927378231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137703537940979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927391658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13790380954742432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927405206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381080448627472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927419029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13797149062156677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927432618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384325921535492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9916666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927446785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375143676996231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927461028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385776549577713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927461029, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784927461030, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784927504050, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5014086961746216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784927504053, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784927504053, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784927517717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13808190822601318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927531897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13746735453605652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927545401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375974863767624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927559717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13722708821296692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.9916666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927573295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13769273459911346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927587528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376458704471588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927601134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13737203180789948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927601134, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784927601134, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784927642832, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5016529560089111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784927642833, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784927642833, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784927656517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13825973868370056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927670901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374104768037796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.8249999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927684752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363711953163147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927698893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379597783088684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927713456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370789110660553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3249999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927727281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367053985595703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927741792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13680648803710938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927741792, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784927741793, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784927784415, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5019574165344238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784927784415, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784927784416, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784927798091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13610446453094482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927811966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13710689544677734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927826151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1369991898536682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1583333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927840039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13620547950267792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927853878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13469505310058594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927867886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13642048835754395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927882172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13499832153320312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927882173, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784927882173, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784927925441, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5023584365844727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784927925441, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784927925442, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784927939175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13613854348659515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927952873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13567084074020386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927966950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13723990321159363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927981332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360432654619217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927995206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1340222805738449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.658333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784928009145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13459084928035736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784928023263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370900273323059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784928023263, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784928023264, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784928066540, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5028505921363831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784928066541, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784928066542, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784928080951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13639488816261292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784928095070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13497158885002136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784928108770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13538265228271484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784928122673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13421303033828735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784928136773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13440155982971191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784928151340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13413332402706146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784928165790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13433876633644104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784928165791, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784928165791, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784928207801, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.503443717956543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784928207802, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784928207802, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784928222409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13341385126113892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784928237040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13623614609241486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784928251915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13831913471221924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784928266044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13340383768081665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.824999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784928280343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13588359951972961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784928294380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13458402454853058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.158333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784928308962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13674728572368622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.324999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784928308963, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784928308963, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784928350687, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5041977763175964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784928350688, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784928350688, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784928365254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13652122020721436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784928379969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13669651746749878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784928394644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13440662622451782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784928409319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12955646216869354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784928423767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13209909200668335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928437761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334480196237564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928451739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346956193447113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928451794, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784928451794, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784928494526, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5050453543663025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784928494526, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784928494526, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784928508732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349819004535675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928522769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12853291630744934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928537830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308046281337738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928551814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13700565695762634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.1158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928566031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13203813135623932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928580125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13109256327152252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928594837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13233496248722076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928594837, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784928594837, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784928637113, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5059986710548401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784928637114, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784928637114, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784928650917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13155719637870789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928665712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13417267799377441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928679320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1276014745235443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928693137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12936876714229584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928707138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133804053068161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928721387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12798909842967987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928735114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12960490584373474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928735114, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784928735115, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784928778510, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5072160959243774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784928778510, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784928778510, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784928793508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13467201590538025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928807511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330549418926239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.3158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928821748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13445687294006348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928835842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13501986861228943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928850511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13617941737174988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.3658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928864756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13352349400520325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928879215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13708555698394775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928879215, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784928879216, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784928921815, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5086470246315002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784928921815, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784928921816, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784928936179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13435092568397522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.4158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928950650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13053834438323975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928964812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13390398025512695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.4491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928979322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14052040874958038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.4658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928993788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13035690784454346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929008544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132512629032135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929023185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13815763592720032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929023185, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784929023186, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784929066086, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5099921226501465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784929066087, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784929066087, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784929080917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13022896647453308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929095515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312026083469391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929110451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13701950013637543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929125044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13535574078559875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929140099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323889195919037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929154284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12880530953407288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.6158333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929168611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12796637415885925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929168611, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784929168611, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784929211498, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5114412903785706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784929211498, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784929211499, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784929225782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300477385520935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929240062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13475026190280914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.6658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929254726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14221519231796265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6825000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929269080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133914053440094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929284220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13159146904945374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.7158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929298372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337442398071289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929312844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13600462675094604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929312845, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784929312845, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784929355530, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5130937099456787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784929355531, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784929355531, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784929369740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13465572893619537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929384168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362747848033905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929398862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356944441795349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929413534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13321858644485474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929428207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13436144590377808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929442589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133013516664505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929456809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13259616494178772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8658333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929456809, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784929456810, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784929501299, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.514907956123352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784929501299, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784929501300, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784929515425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1319291591644287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929529478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13445520401000977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8991666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929543814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14180389046669006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929558337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13430514931678772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929572909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341383159160614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929587645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13237512111663818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929602206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331511288881302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929602207, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784929602207, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784929644887, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5170245170593262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784929644889, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784929644889, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784929659094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13832110166549683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929673495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13306251168251038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929687738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12889474630355835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929702193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13002699613571167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929716974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379118263721466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929731808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315631866455078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929746621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363416612148285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929746622, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784929746622, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784929790405, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5194810628890991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784929790406, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784929790407, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784929804767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13674049079418182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.1158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929818929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13645029067993164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929833241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13159671425819397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1491666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929847402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13002312183380127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929861803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315188705921173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929876416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13534659147262573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929891214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13259369134902954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929891215, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784929891215, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784929933638, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5224257707595825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784929933638, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784929933638, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784929948124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13593228161334991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929963152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132416233420372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929977944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13797831535339355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929992672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13644713163375854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930007563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13849592208862305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930022241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13448002934455872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930037309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1390131413936615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930037309, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784930037309, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784930079860, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.525466799736023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784930079861, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784930079861, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784930094774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13500447571277618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930109721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1478569507598877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930125050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350925713777542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930140053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13308417797088623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930155093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13424824178218842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930169738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13314279913902283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930184570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13650301098823547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930184570, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784930184570, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784930227292, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5292485952377319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784930227293, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784930227293, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784930242133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361599564552307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930257036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353200376033783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930272229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337677240371704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930286733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336369514465332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930301201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332125961780548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930315616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12918256223201752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930329924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12536895275115967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930329924, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784930329925, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784930372380, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5336562395095825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784930372381, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784930372381, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784930386856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324237883090973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.5824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930401526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13246741890907288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930416186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13493463397026062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930430859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13531288504600525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930445814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13675585389137268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930460494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288081705570221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930474999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297157108783722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930475000, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784930475000, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784930517472, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.54000324010849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784930517473, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784930517473, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784930531759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13229872286319733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.699166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930546021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1269683688879013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930560272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330803632736206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930574557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132076233625412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930589247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12699133157730103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930604174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13218840956687927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930618493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13396461308002472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930618494, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784930618494, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784930662169, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5488908290863037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784930662169, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784930662169, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784930677281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12938162684440613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930691818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1302758753299713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930706118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13044190406799316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930720467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12964442372322083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.8658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930734577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1274692863225937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930748901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1230480819940567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930763732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12688817083835602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930763733, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784930763733, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784930806729, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5613940358161926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784930806730, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784930806730, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784930821362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13532491028308868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930836067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13300873339176178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930850284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386261284351349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930865392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13296610116958618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930880047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1272779107093811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930894525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12903501093387604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930909110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12632277607917786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930909111, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784930909111, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784930951899, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.576807975769043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784930951900, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784930951900, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784930966078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1286076009273529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930980267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12686917185783386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930994881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13188570737838745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931009712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13193008303642273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931024584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13069120049476624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931039056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13290004432201385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931054191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12975597381591797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931054192, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784931054192, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784931097083, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5942441821098328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784931097083, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784931097083, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784931111906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289774477481842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931126710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12882760167121887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931141729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12852494418621063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931156557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13161206245422363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931171285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322091519832611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931185703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1281801164150238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931200993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12903060019016266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931200994, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784931200994, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784931244978, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6104823350906372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784931244979, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784931244979, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784931259862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12673602998256683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931275047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12916792929172516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931289556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12656454741954803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931305227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12568411231040955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931320296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1252669245004654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931335260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12958884239196777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931350428, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303296834230423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931350429, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784931350429, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784931393800, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6419984698295593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784931393800, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784931393800, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784931408382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12780433893203735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931423387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12548179924488068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931438574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.127826988697052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931453865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12956543266773224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931469189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308910846710205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931484086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1263713538646698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931500098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12667182087898254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.4991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931500099, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784931500099, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784931542732, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6867356300354004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784931542732, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784931542732, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784931557793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12676723301410675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.5158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931572965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12467384338378906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931588126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12563641369342804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.5491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931603212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13204491138458252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931618383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1272323727607727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931633715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12234009802341461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931649244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1249765157699585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931649244, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784931649245, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784931693019, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7143635153770447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784931693020, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784931693020, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784931707949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12200212478637695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6325000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931723579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12376454472541809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931738539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12325460463762283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931753395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.121815986931324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931768132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12027117609977722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931783039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11709046363830566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931797904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11992722749710083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931797904, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784931797905, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784931841193, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7354652881622314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784931841194, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784931841194, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784931856461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12160679697990417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931871565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11890718340873718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931886634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11067476868629456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.7824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931902059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11711336672306061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931917061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11246077716350555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931931749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11391161382198334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8324999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931946528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11132499575614929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.8491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931946529, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784931946529, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784931989880, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.746183454990387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784931989882, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784931989882, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784932004924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11642338335514069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932019490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11121250689029694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932033912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10799799859523773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.8991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932048558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11384722590446472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932063504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1130281537771225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932078558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11227796971797943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.9491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932093260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10344914346933365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.9658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932093260, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784932093260, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784932136267, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7545377612113953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784932136267, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784932136269, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784932150809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1096874475479126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932166147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1112702488899231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932181194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1162407249212265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.0158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932196050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10995270311832428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932210948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10958635807037354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932225692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10939804464578629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0658333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932240418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10362482815980911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932240464, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784932240464, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784932284505, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7571631669998169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784932284505, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784932284506, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784932299133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10810085386037827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932314377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085176020860672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.1158333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932329334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1147146075963974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932344655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10824751853942871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932359338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10979347676038742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1658333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932374674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11573080718517303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932389524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10305166244506836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932389525, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784932389525, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784932433381, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.762530505657196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784932433382, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784932433382, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784932448248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11135169118642807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932463105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11082189530134201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932478167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10825502127408981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932493000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10748501867055893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932507898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10183258354663849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932522548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11465819180011749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932538046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11073217540979385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932538047, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784932538047, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784932581305, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7649196982383728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784932581306, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784932581306, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784932596460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1143011748790741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932611775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11169569194316864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.3491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932626472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11422830820083618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932641953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10009552538394928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932656975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11371153593063354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.3991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932672050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10618472099304199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.4158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932687244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10506321489810944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932687245, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784932687245, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784932731591, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7665733098983765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784932731592, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784932731592, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784932746575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10807368159294128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.4491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932761621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10769647359848022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932776409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11227276921272278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.4824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932791389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10810402035713196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932806759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11206204444169998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932822086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10135897994041443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932837542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10944380611181259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932837542, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784932837543, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784932881266, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7685302495956421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784932881267, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784932881267, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784932896531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10689255595207214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932911276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10799267888069153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.5824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932926671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11822985112667084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932942062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12149415165185928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932957247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11408329010009766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932972553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11092279851436615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932987785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11496765911579132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932987785, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784932987786, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784933031416, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7699291110038757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784933031417, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784933031417, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784933046445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11392492055892944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933061697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10789236426353455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.6991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933077187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11697103083133698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933092518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11059319972991943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933108127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11342525482177734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933123267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11397464573383331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933138892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1043422520160675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933138893, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784933138894, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784933183258, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7705758213996887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784933183259, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784933183259, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784933198608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11812915652990341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933213982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10709266364574432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933229309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10444661974906921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933244471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11367730796337128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933259650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11804324388504028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933274933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10702274739742279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933290633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11424438655376434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933290633, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784933290634, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784933334633, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7708227038383484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784933334634, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784933334634, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784933349853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11514782905578613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933365434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792386531829834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933380513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10849332064390182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933396111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.113242968916893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933411083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10705579817295074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933425781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10460804402828217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933440507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11721976101398468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.015833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933440507, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784933440508, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784933484081, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7697142362594604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784933484082, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784933484082, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784933498864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10947300493717194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933513759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11463305354118347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933528656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10655726492404938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933543404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10523119568824768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933558624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10196909308433533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933573783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09681645035743713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933589149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10642533749341965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933589149, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784933589150, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784933633948, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7709494233131409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784933633949, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784933633949, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784933648994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10506248474121094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933663708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11052156984806061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933679280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11765065044164658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933694560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10927781462669373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933709487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10630400478839874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933724175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10094514489173889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933739140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10662305355072021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933739140, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784933739141, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784933782524, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7711796164512634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784933782524, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784933782524, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784933797484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10564565658569336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933812194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10519096255302429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933827100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10930485278367996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933842061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732044279575348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933857228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10633505880832672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933872392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11251448094844818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933887729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10939338058233261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933887729, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784933887730, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784933932363, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7718886137008667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784933932364, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784933932364, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784933947343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10510868579149246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933963031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10272912681102753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933978103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10919620096683502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933993229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10929697751998901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934008186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10481821000576019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934022972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10133959352970123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934038011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10486127436161041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934038012, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784934038012, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784934081730, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7722771167755127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784934081730, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784934081730, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784934096602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10471920669078827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934111429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10061214119195938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934126092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10508856922388077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934141370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11025913059711456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934156497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10636846721172333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934171861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10346599668264389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934187084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11022360622882843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934187084, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784934187084, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784934230846, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7725766897201538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784934230848, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784934230848, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784934245428, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09990881383419037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934260953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10338803380727768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934276029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723128914833069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934291084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10359860956668854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934306091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10336162149906158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934321152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10632757842540741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934336269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10970734059810638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934336269, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784934336270, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784934379929, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7729324698448181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784934379929, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784934379930, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784934394935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10621669888496399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934409935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10465019196271896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934424743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10823123157024384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934440164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10622413456439972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934455604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10965773463249207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934471266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10765855014324188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934486631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11103485524654388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934486632, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784934486632, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784934530039, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7727112770080566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784934530040, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784934530040, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784934544931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10715138167142868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934560574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10237397998571396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934575550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1045914888381958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934590684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10197058320045471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934605397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10873720049858093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934620426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11320950090885162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934635409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10830367356538773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934635409, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784934635410, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784934678414, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7731378674507141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784934678415, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784934678415, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784934693490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10378158092498779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934708513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11151833832263947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934723284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10794644057750702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934738645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10065540671348572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934754063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11221560835838318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934769208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11292291432619095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934784951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10513786971569061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934784952, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784934784952, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784934828054, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7735145092010498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784934828055, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784934828055, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784934843147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085008978843689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934857866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10012203454971313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934873834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1163325309753418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934889494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11565996706485748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934904839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11486536264419556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934920172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10462883114814758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934935764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10659930109977722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934935764, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784934935765, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784934981039, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7735799551010132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784934981040, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784934981040, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784934996418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11129443347454071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935011661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11391884088516235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935027075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10983163118362427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935042839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10994210839271545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935058494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1142735704779625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935073947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11201508343219757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935089720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11304529011249542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935089721, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784935089721, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784935133133, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7737796306610107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784935133134, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784935133134, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784935148311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10945701599121094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935164244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10779841244220734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935179688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10776568949222565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935195319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11430446803569794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935210673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11861900985240936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935225904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10985022783279419, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935241611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11575767397880554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.415833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935241612, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784935241612, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} diff --git a/recommendation_v4/rcp_logs/gbs_16384/seed786517729.log b/recommendation_v4/rcp_logs/gbs_16384/seed786517729.log new file mode 100644 index 000000000..af96e2d28 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_16384/seed786517729.log @@ -0,0 +1,585 @@ +:::MLLOG {"namespace": "", "time_ms": 1784931506857, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784931506857, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784931528138, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784931528140, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784931528140, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784931528140, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784931528140, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784931528140, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 16384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784931528140, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784931528140, "event_type": "POINT_IN_TIME", "key": "seed", "value": 786517729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784931528140, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784931528140, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784931528140, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784931528140, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784931528183, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784931528183, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784932164888, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784932164889, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784932165099, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784932666784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13898473978042603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.5833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932682231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13884752988815308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2499999999999997e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932697018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13877776265144348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932712363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13878664374351501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.583333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932727432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13872966170310974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.25e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932742697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13879486918449402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932758056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13889682292938232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932758058, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784932758058, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784932843367, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5015324354171753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784932843368, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784932843368, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784932858308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13886049389839172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932873638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13880683481693268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932889134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13867619633674622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932904062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860756158828735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932919025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386345624923706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9916666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932934991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13856877386569977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932950946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13793890178203583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932950947, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784932950947, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784933002248, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5017310976982117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784933002249, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784933002249, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784933017430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13858480751514435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933033491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13835549354553223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933048416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13838523626327515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933064892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13840821385383606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.9916666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933080002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13819941878318787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933096566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13854914903640747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933112071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13803303241729736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933112071, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784933112072, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784933161989, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5019874572753906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784933161989, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784933161990, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784933177084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384793072938919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933193793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13792727887630463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.8249999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933209322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374504268169403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933224853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382223665714264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933241925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378718614578247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3249999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933257055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13761654496192932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933273314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376309096813202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933273314, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784933273315, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784933324002, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5023024678230286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784933324002, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784933324002, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784933339724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13725417852401733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933354997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13787530362606049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933370835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13762032985687256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1583333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933386650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13715656101703644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933402164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13616037368774414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933417441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13721799850463867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933433799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365022212266922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933433800, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784933433800, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784933483976, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5026879906654358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784933483977, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784933483977, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784933499764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13693612813949585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933515465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13654008507728577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933531065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375579833984375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933547824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13684746623039246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933563592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354392021894455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.658333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933579201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13578388094902039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933594516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13756683468818665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933594517, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784933594517, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784933644831, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5031211972236633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784933644832, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784933644832, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784933661515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13693571090698242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933677496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13588222861289978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933693093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13609427213668823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933709010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13519829511642456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933724527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13540229201316833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933741041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13485506176948547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933756921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352388560771942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933756921, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933756921, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933805795, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5036556124687195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933805796, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933805796, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933821697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345565766096115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933837211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13649317622184753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933854032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13802382349967957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933870361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13444282114505768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.824999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933886725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13615509867668152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933902452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350935697555542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.158333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933919120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367815136909485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.324999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933919121, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784933919122, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784933969671, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5043526291847229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784933969672, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784933969672, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784933985956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13653649389743805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784934002305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.136667400598526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784934018283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13510417938232422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784934034238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308104693889618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784934050871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327565461397171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934067000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13396023213863373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934083065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13501469790935516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934083112, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784934083113, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784934134221, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5051441788673401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784934134222, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784934134222, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784934150292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13495121896266937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934165833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.129506915807724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934182847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13151252269744873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934198528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366274505853653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.1158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934214250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13251417875289917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934230003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13147050142288208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934245949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13260000944137573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934245950, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934245950, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934296418, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5060450434684753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934296419, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934296419, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934311595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13205093145370483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934328543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13419216871261597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934344433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12842640280723572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934360441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298254132270813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934376658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337822526693344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934392714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12839040160179138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934408109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12991291284561157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934408109, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934408110, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934459882, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5072081089019775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934459883, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934459883, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934477356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13471117615699768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934493019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13314950466156006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.3158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934508992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13434743881225586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934524893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348819136619568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934541146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360463798046112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.3658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934556553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336071789264679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934573226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13699758052825928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934573227, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934573227, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934624167, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5086670517921448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934624168, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934624168, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934640598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13442812860012054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.4158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934657052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13051560521125793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934673666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338217407464981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.4491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934690364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14031001925468445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.4658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934706460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13045236468315125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934723827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13243144750595093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934740216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13806472718715668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934740216, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934740217, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934790455, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5100842118263245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934790456, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934790456, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934807095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12697471678256989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934823474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13114649057388306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934839600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13695114850997925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934855513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13537457585334778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934872602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323358118534088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934888347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288674771785736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.6158333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934904255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12811116874217987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934904256, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784934904256, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784934953704, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5116221904754639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784934953705, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784934953705, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784934969661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300649791955948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934985716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13465812802314758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.6658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935001647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14210990071296692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6825000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935017117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13403496146202087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935034462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13163360953330994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.7158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935050366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133737251162529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935066319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360279619693756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935066319, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784935066320, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784935115914, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5132721662521362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784935115914, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784935115915, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784935131728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13466346263885498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935147763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13616493344306946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935164249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13559803366661072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935180146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331765353679657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935196141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13443078100681305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935212632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13290005922317505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935228796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13262605667114258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8658333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935228797, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784935228797, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784935280162, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5150278806686401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784935280162, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784935280163, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784935296356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1319141983985901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935312530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13442307710647583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8991666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935328787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14154626429080963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935345241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1339891105890274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935361055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13424262404441833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935377803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13243839144706726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935394253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329737901687622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935394253, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784935394254, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784935443293, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5170847177505493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784935443293, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784935443293, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784935459155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13824062049388885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935475103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13303279876708984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935491143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12881925702095032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935507376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300082802772522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935523778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378546804189682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935539857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1316823810338974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935556597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13637246191501617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935556598, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784935556598, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784935607369, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5193681120872498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784935607370, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784935607371, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784935623658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13678336143493652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.1158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935639698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362813413143158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935656206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131459578871727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1491666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935672310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13003166019916534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935688634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13165007531642914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935705155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13498163223266602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935721585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13264836370944977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935721585, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784935721585, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784935772871, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5219409465789795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784935772872, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784935772872, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784935788763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13588649034500122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935806232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13227200508117676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935823132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13798919320106506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935839715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13642480969429016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935856304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13851426541805267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935872695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13422739505767822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935889456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13909171521663666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935889456, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784935889457, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784935939746, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.524542510509491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784935939746, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784935939747, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784935956337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349862813949585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935972625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.15065997838974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935990458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13512060046195984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936007000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13311581313610077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936023704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13429217040538788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936040294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13299143314361572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936057106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13654303550720215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936057106, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784936057107, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784936107646, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5275810360908508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784936107647, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784936107647, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784936124383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362769901752472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936140649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13521543145179749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936158236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336124837398529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936174446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13358289003372192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936190758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13326311111450195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936207164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.129564106464386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936223597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1255851686000824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936223597, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784936223598, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784936273227, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5310937166213989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784936273228, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784936273228, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784936289497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13267964124679565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.5824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936306102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13252723217010498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936322615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348501443862915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936338649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352558135986328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936355557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366642713546753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936372179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12889301776885986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936388449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298687607049942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936388450, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784936388450, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784936438669, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.535902202129364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784936438670, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784936438670, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784936454851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13258537650108337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.699166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936471108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12707959115505219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936487375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133267343044281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936503668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13224291801452637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936520009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12712039053440094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936536250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13224922120571136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936551718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13428017497062683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936551719, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784936551719, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784936603182, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5423170924186707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784936603182, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784936603182, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784936620980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12940585613250732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936637445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1305665373802185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936653836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1307133287191391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936670174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1299382448196411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.8658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936686383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1279589831829071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936702659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1234017089009285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936719384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1273295283317566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936719385, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784936719386, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784936769132, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5506282448768616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784936769133, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784936769133, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784936785729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357661485671997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936802196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13333463668823242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936817808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13911879062652588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936835912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13362883031368256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936852365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12785381078720093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936868856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297951191663742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936885326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12712877988815308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936885326, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784936885327, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784936935387, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.561124324798584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784936935388, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784936935388, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784936951669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12911637127399445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936968114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12771821022033691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936984801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1328277587890625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937001495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329396665096283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937018089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315653920173645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937034029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13392771780490875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937051710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1307498812675476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937051711, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784937051711, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784937102289, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5725361704826355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784937102290, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784937102290, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784937119148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12972219288349152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937135819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12981605529785156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937152617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13017243146896362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937169310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329658031463623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937185785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338839828968048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937202199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12987929582595825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937219220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1307077705860138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937219221, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784937219221, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784937269020, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5840134024620056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784937269021, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784937269021, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784937285586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12858983874320984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937302255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13148248195648193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937317868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12921035289764404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937335677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12813618779182434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937352560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1277923583984375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937369403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13218730688095093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937386389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13307993113994598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937386390, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784937386390, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784937436456, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5943934321403503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784937436456, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784937436457, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784937453202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13064652681350708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937470258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12866172194480896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937487460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13109326362609863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937504526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324712336063385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937521299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1340145766735077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937537722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12974077463150024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937555782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12953686714172363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.4991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937555783, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784937555783, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784937605791, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6046862006187439, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784937605791, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784937605792, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784937622513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1306086927652359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.5158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937639361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12839025259017944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937656297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300029754638672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.5491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937673188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13580678403377533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937690241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13147014379501343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937707377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12693928182125092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937724167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13050317764282227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937724168, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784937724168, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784937775052, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6115285158157349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784937775053, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784937775053, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784937791340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12721037864685059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6325000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937809434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1286616027355194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937826165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1283898800611496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937842879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12862449884414673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937859511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12658122181892395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937876218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12366801500320435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937892889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12507563829421997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937892889, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784937892890, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784937943196, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6178302764892578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784937943198, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784937943198, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784937960442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1281648874282837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937977502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12408845871686935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937994026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11821454018354416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.7824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938011180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1240721195936203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938028049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12009420245885849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938044498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1207176148891449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8324999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938060938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11820967495441437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.8491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938060939, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784938060939, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784938111141, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6246562004089355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784938111141, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784938111142, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784938127829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1240130215883255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938144108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11959535628557205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938160527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11634513735771179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.8991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938176963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12332018464803696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938193555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12095258384943008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938210428, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12020768225193024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.9491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938226622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11162234842777252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.9658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938226622, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784938226623, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784938277064, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6342114210128784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784938277065, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784938277066, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784938292841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11782489717006683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938310579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11750458925962448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938327262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12126891314983368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.0158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938343818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1163719892501831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938360378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11517143249511719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938376814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11594276875257492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0658333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938393307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11241300404071808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938393358, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784938393359, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784938443928, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.642349123954773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784938443929, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784938443929, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784938460332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11757209151983261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938477284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11515772342681885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.1158333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938494058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11960060894489288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938511040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11569858342409134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938526826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1156340017914772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1658333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938544052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12299385666847229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938560672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11031246185302734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938560672, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784938560672, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784938611034, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6545653939247131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784938611035, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784938611035, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784938627704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11841578781604767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938644248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11599033325910568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938661018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11386166512966156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938677581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11233580112457275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938694212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10786126554012299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938710509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12134946882724762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938727509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11755618453025818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938727510, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784938727510, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784938777258, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6704023480415344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784938777259, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784938777259, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784938793892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11977432668209076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938810730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11733097583055496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.3491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938826770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11963599920272827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938844025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10657193511724472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938860812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1167156919836998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.3991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938877538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1119871437549591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.4158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938894368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11080241203308105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938894369, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784938894369, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784938944503, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6930586099624634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784938944503, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784938944503, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784938961153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11334621161222458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.4491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938977997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11251559108495712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938994649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11710500717163086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.4824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939011664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11273345351219177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939028811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11845217645168304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939045946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10806006193161011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939063026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1153797134757042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939063027, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784939063027, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784939114085, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7182535529136658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784939114086, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784939114086, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784939130856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1107901930809021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939147015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1131569892168045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.5824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939165149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12368859350681305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939182328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12536616623401642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939199391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12049229443073273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939216452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11390930414199829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939233480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11919798702001572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939233480, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784939233480, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784939283983, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7428810596466064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784939283984, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784939283984, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784939300815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11933280527591705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939317874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11197298765182495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.6991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939335276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12183603644371033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939352355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11406227946281433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939369641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11837802827358246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939386307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11625337600708008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939403493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10819266736507416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939403494, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784939403494, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784939455497, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7528859376907349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784939455498, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784939455498, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784939472727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11966514587402344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939489901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11117392033338547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939507019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1078840121626854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939523779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11623630672693253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939540647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1207611933350563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939557793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11039076745510101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939575154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1161867007613182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939575154, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784939575155, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784939624764, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7592704892158508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784939624765, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784939624765, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784939641756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11740115284919739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939658824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1094895750284195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939674861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11063157021999359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939692652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11500294506549835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939709271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10927718132734299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939725805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10579325258731842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939742270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11626521497964859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.015833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939742270, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784939742271, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784939791487, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7620676755905151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784939791488, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784939791488, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784939808138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11041397601366043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939824764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11547105759382248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939841378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10907407104969025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939858003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10659777373075485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939875070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10496875643730164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939892104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09689071774482727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939909246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10677805542945862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939909247, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784939909247, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784939959679, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7656545639038086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784939959680, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784939959680, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784939976523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10584641247987747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939992811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11035020649433136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940011037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1195593848824501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940028189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11028608679771423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940045122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10698390752077103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940061764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10153816640377045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940078640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10690774768590927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940078640, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784940078641, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784940128224, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7674455642700195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784940128224, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784940128224, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784940145120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10568457841873169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940161673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10576844215393066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940178525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10993097722530365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940195362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10764259099960327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940212467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1069626659154892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940229513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11176281422376633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940246794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11006252467632294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940246795, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784940246795, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784940296908, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7689560055732727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784940296909, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784940296909, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784940313105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10515223443508148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940329967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10270000994205475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940347033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10904045403003693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940364002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10916228592395782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940380809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10536027699708939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940397512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10078658163547516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940414310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10508742928504944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940414310, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784940414311, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784940464837, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7700018286705017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784940464837, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784940464838, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784940481319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10551631450653076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940497942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09986080229282379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.515833333333333e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_16384/seed851663607.log b/recommendation_v4/rcp_logs/gbs_16384/seed851663607.log new file mode 100644 index 000000000..f96593f96 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_16384/seed851663607.log @@ -0,0 +1,676 @@ +:::MLLOG {"namespace": "", "time_ms": 1784917043413, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784917043413, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784917062951, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784917062952, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784917062952, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784917062952, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784917062952, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784917062952, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 16384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784917062952, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784917062952, "event_type": "POINT_IN_TIME", "key": "seed", "value": 851663607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784917062952, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784917062952, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784917062952, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784917062952, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784917062953, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784917062953, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784917730633, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784917730634, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784917730919, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784917881157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1394873559474945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.5833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784917894810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13934031128883362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2499999999999997e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784917908407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389835774898529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784917922214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13865821063518524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.583333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784917935986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13873766362667084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.25e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784917949920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13882599771022797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784917963719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13867926597595215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917963720, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784917963720, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784918008028, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5000302791595459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784918008029, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784918008029, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784918021860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13917016983032227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918035738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13909979164600372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918049833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13894298672676086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918063954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13899695873260498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918078165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13874197006225586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9916666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918092549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13888561725616455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918107083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13871243596076965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918107083, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784918107084, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784918150405, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5001891851425171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784918150406, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784918150407, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784918164637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387714445590973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918179016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13865219056606293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918193230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13863739371299744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918207779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13865487277507782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.9916666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918221920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860197365283966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918236435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13802596926689148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918250777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13846050202846527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918250777, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784918250777, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784918294492, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5003810524940491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784918294492, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784918294493, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784918308854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13858389854431152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918323667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13831177353858948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.8249999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918338177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13805046677589417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918352846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13845577836036682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918367785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13797816634178162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3249999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918382260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13783526420593262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918397084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13769042491912842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918397085, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784918397085, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784918440773, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5006298422813416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784918440773, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784918440774, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784918455234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13741980493068695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918469760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13776931166648865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918484214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13775129616260529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1583333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918498694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13729546964168549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918513175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363067924976349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918527774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13721315562725067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918542317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13646860420703888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918542318, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784918542318, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784918586114, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5009397268295288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784918586115, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784918586115, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784918600518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13710501790046692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918614893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13675719499588013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918629519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13780446350574493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918644098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1369369477033615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918658487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135428786277771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.658333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918672809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13588789105415344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918687269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375715732574463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918687270, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784918687270, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784918731711, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5013230443000793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784918731712, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784918731712, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784918746287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13693250715732574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918761118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360560655593872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918775570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13625293970108032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918790262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13525302708148956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918805128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13532063364982605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918819978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13497918844223022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918834626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13511772453784943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918834626, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784918834626, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784918877790, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5018174648284912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784918877790, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784918877791, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784918892803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344814896583557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918907742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13644108176231384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918922851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13807934522628784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918937841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13417133688926697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.824999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918953096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13613376021385193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918968056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350381076335907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.158333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918983199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13671445846557617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.324999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918983200, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784918983200, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784919027520, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5024912357330322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784919027521, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784919027521, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784919042670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13645566999912262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784919057795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13652709126472473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784919072817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13494575023651123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784919087755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13060414791107178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784919102792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327623873949051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919117769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1339285671710968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919132591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13500714302062988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919132640, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784919132640, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784919176267, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5032829642295837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784919176268, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784919176268, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784919191342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13512451946735382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919206189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12952083349227905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919221191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13141974806785583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919235823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13664230704307556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.1158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919250673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13241037726402283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919265453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13140922784805298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919280721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132548987865448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919280722, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784919280722, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784919325136, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5041992664337158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784919325136, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784919325137, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784919339668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317163109779358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919354754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341826617717743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919369258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12813124060630798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919383874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12970700860023499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919398781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13375481963157654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919413840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1282307505607605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919428342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12986043095588684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919428343, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784919428343, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784919473706, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5054285526275635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784919473707, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784919473707, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784919488969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346060037612915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919503754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331506371498108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.3158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919518830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344548761844635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919533852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13494114577770233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919549288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13623285293579102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.3658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919564223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13358330726623535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919579153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13689161837100983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919579153, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784919579154, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784919623157, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5069516897201538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784919623158, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784919623158, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784919638187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13433921337127686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.4158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919653361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13057872653007507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919668453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13395926356315613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.4491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919683733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1404644101858139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.4658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919698781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13025525212287903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919714026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13256536424160004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919729365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13815239071846008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919729365, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784919729366, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784919773556, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5082917213439941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784919773556, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784919773557, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784919789143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12659141421318054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919804577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311493068933487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919820099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13710561394691467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919835351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13544154167175293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919850852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322908103466034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919865699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1287001222372055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.6158333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919880725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1280333250761032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919880726, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784919880726, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784919924214, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5096800923347473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784919924215, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784919924215, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784919939245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13003703951835632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919954317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13466036319732666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.6658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919969779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14213794469833374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6825000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919984879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13387247920036316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920000059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13151821494102478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.7158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920015017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335967481136322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920030069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359827220439911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920030069, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784920030070, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784920073817, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5114066004753113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784920073817, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784920073818, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784920088767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13459280133247375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920103890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363254338502884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920119248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13552916049957275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920134468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13328468799591064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920149564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13417071104049683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920164920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13296806812286377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920180082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13250118494033813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8658333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920180083, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784920180084, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784920224869, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5132560729980469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784920224870, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784920224870, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784920239919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131813183426857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920255144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13452652096748352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8991666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920270523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14166438579559326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920286118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13382403552532196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920301504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13416588306427002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920316635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323540359735489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920331889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13319306075572968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920331889, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784920331890, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784920376238, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5153238773345947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784920376238, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784920376239, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784920391323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382841020822525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920406452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13297414779663086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920421485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12876644730567932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920436698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12996092438697815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920452180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13799883425235748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920467691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315605640411377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920482609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13625012338161469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920482610, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784920482610, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784920528036, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.517638087272644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784920528037, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784920528037, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784920543312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368926465511322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.1158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920558476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364012062549591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920573915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13144710659980774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1491666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920589089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13010232150554657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920604530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13152003288269043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920620163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13526633381843567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920635804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13277718424797058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920635805, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784920635805, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784920679566, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5202612280845642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784920679567, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784920679567, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784920694876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13588733971118927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920710403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13229835033416748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920726454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378944218158722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920742128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13626648485660553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920757898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13854160904884338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920773453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344437599182129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920789401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13912193477153778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920789401, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784920789402, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784920833525, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5228205323219299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784920833526, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784920833526, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784920849506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13499140739440918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920865323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1463238000869751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920881232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13512015342712402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920897138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13289035856723785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920912978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13413697481155396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920928611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13298919796943665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920944402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366630643606186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920944402, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784920944403, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784920989017, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5259344577789307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784920989018, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784920989018, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784921004826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13610176742076874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921020499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13523831963539124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921036181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13378581404685974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921051646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13360755145549774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921067225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13316300511360168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921082666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1296989619731903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921097945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1255813091993332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921097945, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784921097946, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784921142728, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5298064947128296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784921142729, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784921142729, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784921158005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13260619342327118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.5824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921173596, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13258787989616394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921189325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13510340452194214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921204728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352960467338562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921220166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13657385110855103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921235685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1287747025489807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921251234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12997108697891235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921251235, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784921251235, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784921295305, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5352206230163574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784921295306, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784921295306, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784921310525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13245373964309692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.699166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921325740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12707102298736572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921341034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331184059381485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921356419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132249653339386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921372053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12699487805366516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921387856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13218563795089722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921403014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13413512706756592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921403015, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784921403015, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784921447885, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5422940850257874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784921447886, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784921447886, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784921463468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12942102551460266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921479053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13036374747753143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921494331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1306404173374176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921509870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12966175377368927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.8658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921525137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12779854238033295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921540548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12313179671764374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921556329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1269644945859909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921556330, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784921556330, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784921599470, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5521054267883301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784921599471, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784921599471, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784921615147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13583536446094513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921630771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13317465782165527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921645857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13913577795028687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921661356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13357873260974884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921676846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12777772545814514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921692177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297505646944046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921707683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1269921064376831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921707683, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784921707684, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784921752455, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5643366575241089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784921752455, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784921752456, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784921767498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12909889221191406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921782627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1273937076330185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921798104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13259825110435486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921813668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13275998830795288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921829305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13148801028728485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921844507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337626576423645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921860033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13037747144699097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921860034, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784921860034, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784921903507, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5791488289833069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784921903508, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784921903508, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784921919373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12977653741836548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921935227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12977471947669983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921951151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298542022705078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921967141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13259246945381165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921982890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13363569974899292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921998481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295236349105835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922014811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304255723953247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922014812, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784922014812, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784922057620, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5928864479064941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784922057621, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784922057621, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784922073436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12852585315704346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922089515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13111618161201477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922104809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12852555513381958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922120598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12791217863559723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922136644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12733086943626404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922152511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13182595372200012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922168536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13240772485733032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922168536, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784922168537, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784922211858, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6069369912147522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784922211859, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784922211859, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784922227516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12995675206184387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922243433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1279309093952179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922259658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304291933774948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922275842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13198265433311462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922291943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13313673436641693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922307635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12912610173225403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922323886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12921777367591858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.4991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922323887, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784922323888, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784922367644, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6194417476654053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784922367645, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784922367645, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784922383483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12956127524375916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.5158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922399606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12774479389190674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922415818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12893599271774292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.5491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922431780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352531909942627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922447961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13053521513938904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922464275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12600654363632202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922480728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12965598702430725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922480729, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784922480729, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784922524684, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6304243803024292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784922524685, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784922524686, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784922540375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12602877616882324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6325000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922556515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12810704112052917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922572316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12726643681526184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922588049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12704221904277802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922603723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1250847578048706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922619495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12222418189048767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922635251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12352356314659119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922635252, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784922635253, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784922678324, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6415353417396545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784922678325, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784922678325, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784922694603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12664708495140076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922710687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1228942796587944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922726639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11663462966680527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.7824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922742301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12276381254196167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922758113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11807802319526672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922773696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11864574998617172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8324999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922789319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11661645025014877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.8491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922789320, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784922789320, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784922833188, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6531572937965393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784922833189, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784922833189, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784922849138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12198789417743683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922864704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11810622364282608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922880457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11439875513315201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.8991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922896386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12071357667446136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922912515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11927647888660431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922928734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1183721125125885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.9491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922944498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10918429493904114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.9658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922944499, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784922944499, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784922988238, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6801076531410217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784922988239, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784922988239, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784923003472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11599129438400269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923019259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11682086437940598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923035061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11978936940431595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.0158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923050674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11455747485160828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923066388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11332206428050995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923081984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1144975945353508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0658333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923097641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11018805205821991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923097688, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784923097689, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784923141450, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.701538622379303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784923141451, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784923141451, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784923156926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1157451644539833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923173041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1134406104683876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.1158333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923188974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11801420897245407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923205082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11341647058725357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923220512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1136995255947113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1658333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923236288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12114256620407104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923251797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10798509418964386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923251797, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784923251798, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784923297091, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7277140021324158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784923297092, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784923297092, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784923312669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11636245250701904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923328253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11485715955495834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923344071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11233215034008026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923359800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11175940930843353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923375443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10591794550418854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923390927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11977137625217438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923407089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11571056395769119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923407090, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784923407090, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784923451583, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7456414103507996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784923451583, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784923451584, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784923467299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11769203841686249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923483374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11500446498394012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.3491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923498717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11808063089847565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923514316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10384100675582886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923530135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11528225988149643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.3991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923546050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1100723147392273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.4158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923561929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1088373214006424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923561930, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784923561930, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784923605575, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7565746307373047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784923605576, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784923605576, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784923621239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11069820821285248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.4491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923636918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11013772338628769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923652476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11508379876613617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.4824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923668277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11024945974349976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923684360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11469034105539322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923700342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10467571765184402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923716382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11360634863376617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923716383, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784923716383, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784923759640, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7613509297370911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784923759643, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784923759643, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784923775632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10846826434135437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923791162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11078247427940369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.5824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923807190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12186320126056671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923823392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12242364883422852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923839291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1170804500579834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923855292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11196745932102203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923871373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11680937558412552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923871373, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784923871373, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784923914406, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7648982405662537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784923914407, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784923914407, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784923930325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11644043028354645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923946441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10899905860424042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.6991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923962613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11859843134880066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923978668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11186729371547699, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923995002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11435550451278687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924010742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11438188701868057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924026651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10474897176027298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924026651, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784924026652, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784924071441, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7669612765312195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784924071442, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784924071442, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784924087669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11860683560371399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924103718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10795393586158752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924119905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10590022802352905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924136186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11406229436397552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924152413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11871536076068878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924168642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732609033584595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924185253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1145951896905899, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924185253, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784924185254, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784924229646, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7686635851860046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784924229647, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784924229647, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784924245895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11487111449241638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924262485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071426197886467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924278344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10861726105213165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924294173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11383840441703796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924310005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10756292939186096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924325548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10407228767871857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924341146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11634016036987305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.015833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924341147, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784924341147, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784924385251, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7699960470199585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784924385252, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784924385252, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784924400922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11021571606397629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924416564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11501757800579071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924432228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10649479925632477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924447768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10528801381587982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924463847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10276180505752563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924479680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0959303230047226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924495824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10567370057106018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924495825, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784924495825, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784924540567, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.770634651184082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784924540568, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784924540568, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784924556193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10526717454195023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924571503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10972138494253159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924587659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1184321790933609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924603770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10986176133155823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924619654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10637146234512329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924635293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10097648203372955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924651188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10583192110061646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924651188, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784924651189, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784924696477, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7713674902915955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784924696478, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784924696478, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784924712225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10535275936126709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924727882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10572896152734756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924743807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10934792459011078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924759783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10746745765209198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924775849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1055745780467987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924791862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11118713021278381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924808322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11013941466808319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924808323, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784924808323, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784924853542, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7720485329627991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784924853542, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784924853542, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784924869105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10450772196054459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924884767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10203459858894348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924900598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10839046537876129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924916487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10858448594808578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924932305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10473168641328812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924947896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10043920576572418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924963671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10470272600650787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924963671, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784924963672, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784925008923, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7724165320396423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784925008924, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784925008924, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784925024481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10486114025115967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925040155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09954991191625595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925055575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10436537861824036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925071512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11039449274539948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925087508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10604806244373322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925103815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10345354676246643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925119904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10956728458404541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925119904, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784925119904, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784925164544, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.77279132604599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784925164545, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784925164545, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784925179822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09985347092151642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925195874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10282643139362335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925211996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10722675919532776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925228052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10368628799915314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925244006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10251426696777344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925259940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10633211582899094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925275837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10910055041313171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925275838, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784925275838, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784925319635, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730819582939148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784925319636, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784925319636, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784925335238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.105671226978302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925350915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1030651181936264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925366428, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10800306499004364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925382255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10596199333667755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925398214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10967637598514557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925414270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10762940347194672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925430185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1104862242937088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925430186, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784925430186, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784925475756, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7733757495880127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784925475757, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784925475757, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784925491328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10679773986339569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925507153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10103645920753479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925522952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10454273968935013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925538768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10203014314174652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925554515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10862123966217041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925570248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11311658471822739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925586001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10817113518714905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925586002, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784925586002, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784925631483, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7737013697624207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784925631484, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784925631484, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784925647224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10389487445354462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925663025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11107322573661804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925678800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10773565620183945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925694764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09976738691329956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925710760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11146238446235657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925726797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11228717118501663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925743237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10520979762077332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925743238, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784925743238, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784925787222, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773806631565094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784925787223, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784925787223, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784925803209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10808892548084259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925818763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09964059293270111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925835184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11591681838035583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925851835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11539584398269653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925868209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11465522646903992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925884499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1036117672920227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925901025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10615789890289307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925901025, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784925901025, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784925945481, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773769736289978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784925945481, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784925945482, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784925961818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11144965887069702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925978067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11369216442108154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925994475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11004241555929184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926011295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11024743318557739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926028026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11437264084815979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926044682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11214926093816757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926061584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11304925382137299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926061585, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784926061585, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784926105671, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7739965915679932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784926105672, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784926105672, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784926121731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1090010553598404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926138013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1070762351155281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926154343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10688206553459167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926171027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11444613337516785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926187370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11821532249450684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926203763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10993129014968872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926220538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11537560820579529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.415833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926220539, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784926220539, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} diff --git a/recommendation_v4/rcp_logs/gbs_16384/seed864725970.log b/recommendation_v4/rcp_logs/gbs_16384/seed864725970.log new file mode 100644 index 000000000..b6bbee8c4 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_16384/seed864725970.log @@ -0,0 +1,670 @@ +:::MLLOG {"namespace": "", "time_ms": 1784941176105, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784941176105, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784941197141, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784941197144, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784941197144, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784941197144, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784941197144, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784941197144, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 16384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784941197144, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784941197144, "event_type": "POINT_IN_TIME", "key": "seed", "value": 864725970, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784941197144, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784941197144, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784941197144, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784941197144, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784941197151, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784941197151, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784942114713, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784942114714, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784942115049, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784942249871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1397874653339386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.5833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784942263588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13952040672302246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2499999999999997e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784942277174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13909995555877686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784942290951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13875611126422882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.583333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784942304850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13874804973602295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.25e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784942318548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387932002544403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784942332253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13906052708625793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942332254, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784942332254, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784942377030, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49946990609169006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784942377030, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784942377030, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784942390494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13936111330986023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942404195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13923616707324982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942418076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13909265398979187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942431908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13911877572536469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942445643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13902181386947632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9916666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942459607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1391967535018921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942473821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389692723751068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942473821, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784942473821, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784942518875, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4996529519557953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784942518875, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784942518875, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784942532606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13896194100379944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942546859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13891905546188354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942560717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13893800973892212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942575158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13876038789749146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.9916666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942588873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13868606090545654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942603202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13875794410705566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942617157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13846749067306519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942617158, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784942617158, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784942660175, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4998840093612671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784942660175, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784942660176, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784942674126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138668030500412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942688643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13868308067321777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.8249999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942702585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13820159435272217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942716746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13842512667179108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942731454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13828986883163452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3249999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942745431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13795867562294006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942759960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13801729679107666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942759960, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784942759961, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784942803129, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5001634359359741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784942803130, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784942803130, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784942816804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377868801355362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942830772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13805770874023438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942845123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13803532719612122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1583333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942859021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13774780929088593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942872923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368810385465622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942887024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375179886817932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942901323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368090659379959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942901324, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784942901324, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784942944324, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5005132555961609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784942944325, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784942944325, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784942958104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13735714554786682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942971999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13687273859977722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942986256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13780754804611206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943000578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13715724647045135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943014559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13590708374977112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.658333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943028435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362048089504242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943042395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13757574558258057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943042395, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784943042396, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784943084982, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5009409785270691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784943084982, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784943084982, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784943099421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13708442449569702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943113541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13624493777751923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943127349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.136351078748703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943141371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13550615310668945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943155570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13552266359329224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943170205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352074295282364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943184504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353975236415863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943184505, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784943184505, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784943227783, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5014675855636597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784943227784, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784943227784, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784943242122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13474789261817932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943256528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13657715916633606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943271247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380220502614975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943285421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13455989956855774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.824999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943299745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362118124961853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943313988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13534332811832428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.158333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943328730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13686725497245789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.324999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943328731, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784943328732, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784943371561, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.502174437046051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784943371561, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784943371561, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784943385804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13652342557907104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943400138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13682539761066437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943414583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13506728410720825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943429069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13081860542297363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943443644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329200565814972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943457693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13387849926948547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943471704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349833607673645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943471751, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784943471751, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784943515198, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5029847025871277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784943515198, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784943515198, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784943529386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13514147698879242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943543574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12980298697948456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943558487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13158684968948364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943572324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367282271385193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.1158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943586282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13253670930862427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943600190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13166362047195435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943614628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13269534707069397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943614628, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784943614629, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784943657634, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5038523077964783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784943657634, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784943657634, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784943671349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13198304176330566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943686488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13421162962913513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943700345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12840977311134338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943714521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12994785606861115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943728920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1339220106601715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943743551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12848405539989471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943757715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297726333141327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943757715, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784943757716, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784943800769, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5049543380737305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784943800769, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784943800769, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784943815640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13469362258911133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943829579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331290900707245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.3158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943843828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13447295129299164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943858049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134947270154953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943872704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13603749871253967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.3658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943886982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13344573974609375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943901522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13682377338409424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943901523, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784943901523, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784943944256, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5062965154647827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784943944257, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784943944257, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784943959064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13442906737327576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.4158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943974006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13051185011863708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943988803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1340462863445282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.4491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944003985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14034810662269592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.4658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944018872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13048279285430908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944033914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13243676722049713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944048531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138229638338089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944048532, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784944048532, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784944091153, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5075454711914062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784944091153, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784944091153, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784944105827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12685006856918335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944120365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1313363015651703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944135241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13708214461803436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944149813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13538818061351776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944165053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13234210014343262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944179418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12884622812271118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.6158333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944194043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12797924876213074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944194044, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784944194044, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784944237106, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5088125467300415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784944237107, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784944237107, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784944251486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13011112809181213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944266037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348382532596588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.6658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944280674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14216548204421997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6825000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944295047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13436931371688843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944310145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13143469393253326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.7158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944324524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13368099927902222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944339092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13611334562301636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944339093, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784944339093, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784944382860, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5103601813316345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784944382861, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784944382861, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784944397100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13476720452308655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944411467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13648277521133423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944426001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13564112782478333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944440646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13321977853775024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944455298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341719627380371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944470175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13306090235710144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944484784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13257813453674316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8658333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944484785, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784944484785, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784944529091, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5119689702987671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784944529092, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784944529092, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784944543598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13199666142463684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944558117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13447722792625427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8991666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944572877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14176645874977112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944587855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13443154096603394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944602666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13449326157569885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944617516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13245677947998047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944632075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13306507468223572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944632076, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784944632076, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784944674572, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5139074325561523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784944674574, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784944674574, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784944688712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380854994058609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944702974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331152319908142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944717097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12892237305641174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944731291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1299445629119873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944745818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381344199180603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944760459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13163211941719055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944775110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13631653785705566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944775110, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784944775111, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784944819593, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5159714818000793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784944819594, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784944819594, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784944834358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13688647747039795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.1158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944848812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13644367456436157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944863519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131405770778656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1491666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944877940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13015654683113098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944892662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13153545558452606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944907674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350896656513214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944922748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13278242945671082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944922748, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784944922749, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784944966706, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5182803869247437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784944966706, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784944966706, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784944981281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13610392808914185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944996361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13227123022079468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945011142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137957364320755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945025918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13636842370033264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945040671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13855871558189392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945055259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13445442914962769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945070299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13906404376029968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945070300, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784945070300, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784945113695, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5206590890884399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784945113695, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784945113695, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784945128692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13508188724517822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945143669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.15216492116451263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945158903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13515588641166687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945173888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330583542585373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945188958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1342441439628601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945203803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13323432207107544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945219014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13660134375095367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945219015, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784945219015, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784945261976, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5233317017555237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784945261977, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784945261977, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784945277071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13620872795581818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945292175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13534657657146454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945307289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13381671905517578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945321703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13371899724006653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945335979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13337525725364685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945350328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298483908176422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945364594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12567779421806335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945364595, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784945364595, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784945407717, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5265664458274841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784945407718, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784945407718, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784945422079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13271771371364594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.5824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945436752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325855255126953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945451483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13506320118904114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945466179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13542675971984863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945481270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13683456182479858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945496077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1290503889322281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945510742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1299334615468979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945510743, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784945510743, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784945554566, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5313462018966675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784945554567, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784945554567, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784945569085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13254761695861816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.699166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945583583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12722261250019073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945598169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13328896462917328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945612938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13246473670005798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945628159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12712794542312622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945643585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13246652483940125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945658299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13440266251564026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945658300, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784945658300, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784945702454, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5377703905105591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784945702455, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784945702455, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784945717595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294432282447815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945732474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13061505556106567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945747210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13085895776748657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945761907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12996526062488556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.8658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945776389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1282065063714981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945791213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12349121272563934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945806386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12756773829460144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945806386, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784945806387, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784945849317, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5464287996292114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784945849318, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784945849318, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784945864425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13599476218223572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945879402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13367964327335358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945893917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.139394611120224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945909032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13391107320785522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945923741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12809649109840393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945938251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13005471229553223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945952989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12729567289352417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945952989, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784945952990, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784945997183, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5576443076133728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784945997185, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784945997185, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784946011636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12935388088226318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946026067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1280178725719452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946040910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13303247094154358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946055768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13315528631210327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946070727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318482607603073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946085287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13411089777946472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946100411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13099901378154755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946100411, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784946100412, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784946144075, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5729410648345947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784946144076, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784946144076, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784946158952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12999296188354492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946173782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13042987883090973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946188806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13060805201530457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946204013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13317570090293884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946219087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13443955779075623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946233875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303776204586029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946249461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311403512954712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946249461, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784946249462, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784946292392, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5878069400787354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784946292392, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784946292393, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784946307691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12889480590820312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946323112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13190972805023193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946337740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12941960990428925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946353190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12869682908058167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946368362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1282958984375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946383529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326577365398407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946398899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13338316977024078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946398900, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784946398901, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784946443432, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6023815870285034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784946443433, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784946443433, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784946458291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13079842925071716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946473628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12925681471824646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946489337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13114801049232483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946505037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326189935207367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946520584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13419604301452637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946535683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13043132424354553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946551491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12949973344802856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.4991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946551491, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784946551492, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784946594310, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6154360771179199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784946594311, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784946594311, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784946609555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13063737750053406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.5158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946624942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12896248698234558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946640336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304023265838623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.5491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946655369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13624638319015503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946670773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317741572856903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946686295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12728184461593628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946701771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13105323910713196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946701771, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784946701772, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784946746551, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6267677545547485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784946746552, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784946746552, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784946761575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1278369277715683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6325000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946777121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288319230079651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946792206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12852126359939575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946807188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288486272096634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946821949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12670984864234924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946836812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1242666244506836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946851723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12474069744348526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946851724, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784946851724, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784946894373, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6390237212181091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784946894374, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784946894374, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784946909836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1285734921693802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946925171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12447385489940643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946940268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11835350841283798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.7824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946955438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12438046932220459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946970538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12030699104070663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946985470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12096985429525375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8324999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947000390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11878373473882675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.8491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947000391, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784947000392, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784947044413, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6516324877738953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784947044414, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784947044414, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784947059511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12384901195764542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947074322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11928658187389374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947089145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11676369607448578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.8991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947104022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12321335822343826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947119130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12112273275852203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947134447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.120366171002388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.9491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947149362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11124363541603088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.9658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947149363, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784947149363, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784947194483, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6766093373298645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784947194483, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784947194483, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784947209176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11769430339336395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947224344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11768792569637299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947239051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12129522860050201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.0158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947253477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11636258661746979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947268009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11479775607585907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947282319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1161797046661377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0658333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947296792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11213655769824982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947296838, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784947296839, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784947341068, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6977601647377014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784947341069, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784947341069, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784947355363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1168668195605278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947370434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11481854319572449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.1158333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947385139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11954870820045471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947400398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11503467708826065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947414970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11470947414636612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1658333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947430240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1220119446516037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947445047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10943911969661713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947445047, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784947445048, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784947488012, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7180039882659912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784947488012, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784947488012, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784947502951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11763903498649597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947517963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11577755212783813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947532977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11283320188522339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947547659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11247989535331726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947562478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10727857053279877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947576957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11975014209747314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947592383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1161745935678482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947592383, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784947592383, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784947636446, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7344949841499329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784947636446, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784947636446, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784947651355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11878872662782669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947666780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11587142944335938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.3491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947681797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11850404739379883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947697173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10494295507669449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947711979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11546510457992554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.3991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947726646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11109782755374908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.4158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947741370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10944682359695435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947741371, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784947741371, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784947784709, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7452856302261353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784947784710, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784947784710, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784947799122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11168606579303741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.4491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947813597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11097590625286102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947827922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11544740200042725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.4824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947842443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11061068624258041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947857364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1155254989862442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947872208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10494926571846008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947887256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11392386257648468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947887256, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784947887257, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784947930973, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7531366348266602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784947930974, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784947930974, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784947945687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10954242944717407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947960110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11087527871131897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.5824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947975875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12154411524534225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947991427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12230253219604492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948006572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11694938689470291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948021856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11223800480365753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948037193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11721522361040115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948037194, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784948037194, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784948082163, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7579126358032227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784948082164, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784948082164, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784948097239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11610789597034454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948112565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10898416489362717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.6991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948128049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11817934364080429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948143410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1117287129163742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948159135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11411139369010925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948174388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11452365666627884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948189814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1053793653845787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948189815, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784948189815, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784948236179, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7611420750617981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784948236180, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784948236180, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784948251278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11905965209007263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948266455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10774779319763184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948281753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10605702549219131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948296852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11390349268913269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948311876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11863984167575836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948327061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10734884440898895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948342882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11421683430671692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948342883, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784948342883, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784948387365, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7640437483787537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784948387366, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784948387366, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784948402805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11507615447044373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948418523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10754940658807755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948433548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10859168320894241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948449350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11360616236925125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948464536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1078757792711258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948479324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10417751967906952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948494136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11701902747154236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.015833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948494137, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784948494138, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784948538297, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7653827667236328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784948538298, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784948538298, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784948553174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11063875257968903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948567958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11497624963521957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948582707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10668044537305832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948597456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10534407943487167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948612675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10265389084815979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948627737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09651325643062592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948643005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10608121752738953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948643006, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784948643006, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784948687722, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7670707106590271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784948687723, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784948687723, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784948702609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.105240598320961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948717326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10985193401575089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948733066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11872665584087372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948748364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1098187267780304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948763321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10660284757614136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948778012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10125099867582321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948793066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10621906816959381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948793067, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784948793067, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784948837027, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7686648964881897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784948837028, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784948837029, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784948851815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10537247359752655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948866251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10559126734733582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948880873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10960381478071213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948895626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10776357352733612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948910564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10640833526849747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948925488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11187651753425598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948940779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11048026382923126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948940780, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784948940780, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784948984744, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7696915864944458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784948984745, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784948984745, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784948999481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10521098971366882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949014970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1023985892534256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949030089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10891799628734589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949045278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085682213306427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949060278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10495427250862122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949075196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1011374294757843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949090185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10490946471691132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949090186, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784949090186, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784949134455, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7704461216926575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784949134455, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784949134456, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784949149252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10484594851732254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949164220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10002215206623077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949179004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10514163225889206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949194266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1105571836233139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949209449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1062052920460701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949224958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10382558405399323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949240346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10981971025466919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949240346, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784949240347, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784949284379, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7713139653205872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784949284380, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784949284381, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784949299025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10000424087047577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949314197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10280404984951019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949329077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10736552625894547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949343936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10345062613487244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949358730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10224951803684235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949373445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10701732337474823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949388280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10905104875564575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949388281, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784949388281, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784949431823, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7716182470321655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784949431824, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784949431825, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784949446478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10584117472171783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949461133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10399790108203888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949475669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10831552743911743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949490581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10614696145057678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949505656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11015608161687851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949520765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1073777824640274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949535835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11093337833881378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949535835, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784949535836, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784949579882, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7719883918762207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784949579883, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784949579884, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784949594522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10704845190048218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949610209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10187636315822601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949625350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10459263622760773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949640510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1018192321062088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949655506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10865829885005951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949670484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11375370621681213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949685298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10837974399328232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949685299, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784949685299, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784949728425, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7724793553352356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784949728425, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784949728425, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784949743420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10362036526203156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949758483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11117930710315704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949773427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10797378420829773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949788550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09993715584278107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949803718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11231495440006256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949818900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1128617525100708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949834708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10554447025060654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949834709, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784949834709, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784949877771, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7729527354240417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784949877771, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784949877771, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784949892956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082712858915329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949907683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09993724524974823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949923528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11590301990509033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949939050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11566458642482758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949954349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11476646363735199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949969635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10383887588977814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949985150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10652823746204376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949985150, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784949985151, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784950028162, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7729090452194214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784950028163, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784950028163, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784950043607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1114928126335144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950058671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11375338584184647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950074041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11011599749326706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950089625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11038783192634583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950105277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11451004445552826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950120699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11241459846496582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950136462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11268934607505798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950136463, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784950136463, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784950180442, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7733930945396423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784950180443, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784950180443, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784950195534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10864710807800293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950211316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10744385421276093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950226708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10752913355827332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349166666666667e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_16384/seed871175675.log b/recommendation_v4/rcp_logs/gbs_16384/seed871175675.log new file mode 100644 index 000000000..fb83b8da6 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_16384/seed871175675.log @@ -0,0 +1,661 @@ +:::MLLOG {"namespace": "", "time_ms": 1784931508107, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784931508107, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784931528935, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784931528936, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784931528936, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784931528936, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784931528936, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784931528936, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 16384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784931528936, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784931528936, "event_type": "POINT_IN_TIME", "key": "seed", "value": 871175675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784931528936, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784931528937, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784931528937, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784931528937, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784931528939, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784931528940, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784932438963, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784932438964, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784932439276, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784932580766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13902899622917175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.5833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932594521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13881325721740723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2499999999999997e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932608139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387811303138733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932622210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13873691856861115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.583333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932636274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387946456670761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.25e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932650229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13857662677764893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932664221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13884514570236206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932664221, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784932664222, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784932709788, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4968470633029938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784932709789, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784932709789, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784932723519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13868482410907745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932737424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386837363243103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932751402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13874012231826782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932765370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860547542572021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932779449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13867872953414917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9916666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932793726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384655237197876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932808118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13836073875427246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932808120, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784932808120, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784932851396, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49698713421821594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784932851396, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784932851396, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784932865486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384759247303009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932879885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13833951950073242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932893745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13834773004055023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932908413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381358504295349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.9916666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932922389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13832315802574158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932937056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13766631484031677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932951326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13800638914108276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932951326, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784932951327, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784932994400, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4971446394920349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784932994400, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784932994400, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784933008420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13853543996810913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933023172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13807103037834167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.8249999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933037427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13744056224822998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933051770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13843324780464172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933066667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13768184185028076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3249999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933080942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13756471872329712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933095767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13750487565994263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933095768, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784933095768, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784933139271, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49734166264533997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784933139272, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784933139272, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784933153127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370665580034256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933167219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13770446181297302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933181583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13752980530261993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1583333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933195938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13701370358467102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933210267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13569214940071106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933224721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1369839310646057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933239150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13598297536373138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933239150, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784933239151, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784933282397, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49756094813346863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784933282397, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784933282398, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784933296335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13678669929504395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933310174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13631349802017212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933324387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13754230737686157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933338864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13665705919265747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933353099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13497915863990784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.658333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933367454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13548313081264496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933381843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13727280497550964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933381844, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784933381844, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784933425438, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49781399965286255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784933425438, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784933425438, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784933440070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1369035392999649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933454486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13577337563037872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933468541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13581126928329468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933482805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13494572043418884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933497131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13512113690376282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933512074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13459348678588867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933526552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13491909205913544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933526553, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933526553, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933569194, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4981750249862671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933569195, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933569195, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933583664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13416744768619537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933598229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363629251718521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933613585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13818703591823578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933627984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13387508690357208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.824999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933642693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13614973425865173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933657180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348433494567871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.158333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933671967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366879940032959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.324999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933671968, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784933671968, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784933715289, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49874094128608704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784933715290, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784933715291, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784933730010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13653454184532166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933744659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367771029472351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933759253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346583366394043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933773864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301383674144745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933788697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13247182965278625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933802978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13352563977241516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933817038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13496796786785126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933817085, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784933817085, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784933860286, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4994160234928131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784933860287, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784933860287, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784933874664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13500413298606873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933888841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1291210800409317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933903783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131157785654068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933917924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13687366247177124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.1158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933932209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323564201593399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933946281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1314118653535843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933960813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324690878391266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933960814, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784933960814, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934003108, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5002778768539429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934003108, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934003109, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934017012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1316462755203247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934031963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13418924808502197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934045954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12783677875995636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934060029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1296645700931549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934074356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13373741507530212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934088930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1282173991203308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934103066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12983134388923645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934103066, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934103067, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934147579, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5014894008636475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934147580, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934147580, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934162429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13466744124889374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934176506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13305309414863586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.3158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934190822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13442564010620117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934205069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13502296805381775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934219815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13618305325508118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.3658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934234225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13352656364440918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934248962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368906944990158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934248962, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934248963, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934293207, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5029221177101135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934293207, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934293207, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934308189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13419315218925476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.4158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934323155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13055714964866638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934337866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13381865620613098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.4491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934352898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14053812623023987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.4658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934367765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13053297996520996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934383013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13243404030799866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934397675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381128430366516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934397676, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934397677, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934441718, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5042513012886047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934441719, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934441719, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934456653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12407394498586655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934471279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13123857975006104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934486137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13723178207874298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934500828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13541635870933533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934516187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13238279521465302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934530547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12893913686275482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.6158333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934545087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12804856896400452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934545088, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784934545088, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784934588737, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5056440234184265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784934588738, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784934588738, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784934603203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13000616431236267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934617602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348503679037094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.6658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934632356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14225783944129944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6825000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934646832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134354367852211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934661993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13143879175186157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.7158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934676516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336519569158554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934691065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13617655634880066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934691066, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784934691066, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784934735677, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5072787404060364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784934735678, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784934735678, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784934749975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134632870554924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934764471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13634544610977173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934779240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13566583395004272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934793723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13322316110134125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934808447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13437673449516296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934822981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329214721918106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934837447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323450803756714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8658333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934837447, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784934837448, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784934881644, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5089629292488098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784934881645, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784934881645, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784934896032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13188812136650085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934910457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13444176316261292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8991666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934924971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14176449179649353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934939814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341807246208191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934954536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13426759839057922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934969414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13224947452545166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934984065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332228183746338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934984066, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784934984066, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784935027015, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5109360814094543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784935027015, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784935027015, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784935041465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13895739614963531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935056102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13302703201770782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935070569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288870871067047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935085179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13008329272270203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935099886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13807561993598938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935114777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13177338242530823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935129451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13640940189361572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935129452, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784935129453, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784935173469, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.513274610042572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784935173470, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784935173470, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784935188216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367449313402176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.1158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935202764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13632291555404663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935217616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13136141002178192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1491666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935232141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13000023365020752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935246841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13154500722885132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935261628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13515014946460724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935276576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13254210352897644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935276577, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784935276577, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784935320708, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5158866047859192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784935320708, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784935320708, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784935335299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13585209846496582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935350437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13227733969688416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935365335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13792279362678528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935380111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13643261790275574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935395029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13859879970550537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935409869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13436071574687958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935425109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1390305757522583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935425110, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784935425110, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784935469458, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5184902548789978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784935469459, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784935469459, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784935484637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350354701280594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935499813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.15146097540855408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935515493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13504885137081146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935530680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330661177635193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935545909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13417887687683105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935560870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13311198353767395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935576115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13647237420082092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935576115, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784935576116, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784935618733, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5217211842536926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784935618734, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784935618734, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784935633913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360282003879547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935648947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352231204509735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935664278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13376736640930176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935678911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13367503881454468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935693517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13314656913280487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935708092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12928389012813568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935722544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1256500631570816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935722544, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784935722545, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784935764581, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.525676429271698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784935764582, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784935764582, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784935779125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13266190886497498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.5824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935793947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13259001076221466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935808910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348743736743927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935823754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13539229333400726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935838597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13659392297267914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935853293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12879693508148193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935867699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12983573973178864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935867699, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784935867700, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784935910241, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.531480610370636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784935910242, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784935910242, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784935924636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13250958919525146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.699166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935938941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1270725429058075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935953315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13309501111507416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935967824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321968287229538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935982822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1269868016242981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935997984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13221295177936554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936012451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13412758708000183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936012451, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784936012452, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784936055993, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5391553640365601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784936055994, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784936055994, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784936071181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12944775819778442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936085971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13037657737731934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936100613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13065500557422638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936115335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12973718345165253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.8658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936130057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1276702582836151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936144823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12326689809560776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936160028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12722575664520264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936160029, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784936160029, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784936202978, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5498698949813843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784936202979, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784936202979, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784936218087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13551470637321472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936233070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13315194845199585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936247623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13873447477817535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936262849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13321971893310547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936277849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12778958678245544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936292774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294868290424347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936307992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12678344547748566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936307992, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784936307993, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784936351424, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5627915263175964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784936351425, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784936351425, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784936366146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288830041885376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936381021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12721475958824158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936396362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13230851292610168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936411584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132390558719635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936426776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13115954399108887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936441729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133550226688385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936457060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13020730018615723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936457061, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784936457062, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784936499633, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5773030519485474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784936499634, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784936499634, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784936514639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12935934960842133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936529509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12961652874946594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936544396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12919726967811584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936559258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13219064474105835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936573958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1328771412372589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936588535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1291702389717102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936603777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12988005578517914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936603778, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784936603778, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784936647274, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5895342230796814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784936647275, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784936647275, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784936662203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1277206838130951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936677368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13023991882801056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936691936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12754595279693604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936707215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12725073099136353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936722470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1267867386341095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936737689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13094647228717804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936752953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13157255947589874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936752953, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784936752954, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784936797534, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6002740263938904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784936797535, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784936797535, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784936812358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12923932075500488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936827536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1273198127746582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936843089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12953850626945496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936858656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13159576058387756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936874088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324934959411621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936889044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12808498740196228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936904948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12863972783088684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.4991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936904949, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784936904949, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784936947671, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6087287664413452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784936947672, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784936947672, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784936962757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.128921777009964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.5158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936977893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12706276774406433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936993013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12788823246955872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.5491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937007902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13439533114433289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937023149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13001874089241028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937038474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1248738020658493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937053883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12750157713890076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937053884, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784937053884, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784937098422, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6149845123291016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784937098423, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784937098423, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784937113425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12440183758735657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6325000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937129361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12714490294456482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937144287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12580382823944092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937159237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12584654986858368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937174037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12401483952999115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937188939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12103386223316193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937203816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12311579287052155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937203817, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784937203817, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784937246543, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.62086421251297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784937246544, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784937246544, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784937261951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12573757767677307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937277230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12202973663806915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937292425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1152971163392067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.7824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937307753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12169694900512695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937322790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11709412187337875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937337521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11765670776367188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8324999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937352294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11533275246620178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.8491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937352295, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784937352295, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784937395017, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.627290666103363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784937395018, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784937395018, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784937410021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12098344415426254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937424842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11706917732954025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937439562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11260509490966797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.8991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937454446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12092092633247375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937469591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11928682029247284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937484877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11806206405162811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.9491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937499783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10917207598686218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.9658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937499784, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784937499784, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784937543623, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6357048153877258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784937543624, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784937543624, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784937558260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11672553420066833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937573921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11701032519340515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937589152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11974721401929855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.0158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937604160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11451993882656097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937619242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1131005585193634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937634122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1145680621266365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0658333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937649204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11108524352312088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937649251, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784937649251, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784937692409, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6454594731330872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784937692410, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784937692410, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784937707136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11593388020992279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937722591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11378683149814606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.1158333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937737907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11965464055538177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937753561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11394363641738892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937768416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11483985930681229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1658333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937783676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12066234648227692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937798451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1093069463968277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937798452, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784937798452, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784937841744, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6573528051376343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784937841745, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784937841745, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784937856459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11776351928710938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937871325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11531385779380798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937886462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1134580671787262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937901352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11232081800699234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937916262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10803982615470886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937931036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12145970016717911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937946472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11717969179153442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937946473, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784937946473, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784937989957, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6732457876205444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784937989957, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784937989958, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784938005247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12015033513307571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938020802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11587564647197723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.3491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938035762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11932182312011719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938051199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10556544363498688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938066235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1168302595615387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.3991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938081293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11252988874912262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.4158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938096358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11087489873170853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938096359, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784938096359, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784938141123, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6929145455360413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784938141124, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784938141124, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784938156100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11307196319103241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.4491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938171077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11187836527824402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938185867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11729243397712708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.4824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938200931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11327570676803589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938216187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11740513145923615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938231442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10836631059646606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938246796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11597743630409241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938246796, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784938246797, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784938291561, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7115158438682556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784938291562, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784938291562, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784938306781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11061210930347443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938321706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11318899691104889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.5824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938337556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1239330917596817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938353013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.126948744058609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938368280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12213870882987976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938383654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11446896940469742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938398948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12013712525367737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938398949, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784938398949, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784938442734, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7280189394950867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784938442735, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784938442735, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784938457801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12021504342556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938473124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11256362497806549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.6991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938488692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12279421091079712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938504137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11463887989521027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938519869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11928413808345795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938535149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11714315414428711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938550913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10898706316947937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938550914, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784938550914, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784938595862, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7402312755584717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784938595863, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784938595863, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784938611523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1205558255314827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938627249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11196719110012054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938642990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793234407901764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938658672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11804059892892838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938674211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12145721912384033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938689921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11138493567705154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938706130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11779290437698364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938706131, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784938706131, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784938749832, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7494733333587646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784938749833, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784938749833, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784938765531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11879274994134903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938781489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11201075464487076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938796662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11172729730606079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938812189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1167951226234436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938827171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11052420735359192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938841971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10637503862380981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938856657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11653681099414825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.015833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938856657, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784938856658, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784938900249, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7553513646125793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784938900250, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784938900250, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784938915038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11291614919900894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938929873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11627399921417236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938944739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11194784194231033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938959424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10836437344551086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938974639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1062232255935669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938989824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09943190217018127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939005212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10854524374008179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939005212, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784939005213, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784939051279, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.759407103061676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784939051279, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784939051279, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784939066231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10752908885478973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939080934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11109967529773712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939096733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12195451557636261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939111925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11017598956823349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939126901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10819390416145325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939141759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10313780605792999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939156905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10905063152313232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939156906, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784939156906, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784939201116, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.762498676776886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784939201117, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784939201117, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784939216165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10610402375459671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939230947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10768327862024307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939246128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1106085404753685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939261350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10958807170391083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939276703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10816256701946259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939292083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11303859949111938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939307629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11122960597276688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939307630, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784939307630, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784939352825, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7644369006156921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784939352825, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784939352826, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784939367798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10644248127937317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939383340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10315839946269989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939398542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11153413355350494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939413867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1100628674030304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939428922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1061631590127945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939443879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10043776780366898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939458930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10503624379634857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939458930, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784939458931, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784939504173, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7652004361152649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784939504174, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784939504174, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784939519165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10527049005031586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939534315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10144820809364319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939549308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10594604164361954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939564769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11108587682247162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939580191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10657070577144623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939595879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10380889475345612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939611398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11052601784467697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939611398, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784939611398, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784939656769, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7668716907501221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784939656770, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784939656770, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784939671537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10029365122318268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939687185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10378184169530869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939702201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10794523358345032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939717354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10423065721988678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939732325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10238784551620483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939747325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10805638134479523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939762442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.109676294028759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939762442, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784939762443, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784939805709, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7671761512756348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784939805710, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784939805711, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784939820759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10675735771656036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939835718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10309891402721405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939850444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10969308763742447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939865678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10655219852924347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939880938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11067094653844833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939896343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10839985311031342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939911553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11130528151988983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939911554, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784939911554, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784939955149, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7687733769416809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784939955149, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784939955150, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784939969933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1065317839384079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939985662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10103076696395874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940000881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10556922852993011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940016143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10219375789165497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940031189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10860024392604828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940046183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11332394182682037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940061238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10903492569923401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940061239, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784940061239, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784940104754, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7688633799552917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784940104755, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784940104755, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784940119811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10525277256965637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940135027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11130712926387787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940150147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10794790089130402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940165430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09980109333992004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940180823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.112694650888443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940196173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11258135735988617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940211942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10543863475322723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940211942, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784940211943, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784940256363, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7695468068122864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784940256364, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784940256364, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784940271549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10846488177776337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940286492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.100139319896698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940302540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11606051027774811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940318062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11667993664741516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940333352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11489252001047134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940348574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10441732406616211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940363885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10691270977258682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940363886, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784940363886, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784940408055, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7695039510726929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784940408056, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784940408056, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784940423379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11205874383449554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940438482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11430022120475769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940453804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11033763736486435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940469355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11081426590681076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940484818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1145239770412445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940500267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11234401166439056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.2825e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_16384/seed925729918.log b/recommendation_v4/rcp_logs/gbs_16384/seed925729918.log new file mode 100644 index 000000000..4326618bf --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_16384/seed925729918.log @@ -0,0 +1,672 @@ +:::MLLOG {"namespace": "", "time_ms": 1784950494619, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784950494619, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784950515588, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784950515592, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784950515592, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784950515592, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784950515592, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784950515592, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 16384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784950515592, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784950515592, "event_type": "POINT_IN_TIME", "key": "seed", "value": 925729918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784950515592, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784950515592, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784950515592, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784950515593, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784950515668, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784950515668, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784951368593, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784951368594, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784951368898, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784951573283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13847559690475464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.5833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784951587247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13859286904335022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2499999999999997e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784951600834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13862386345863342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784951614700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13859890401363373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.583333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784951628630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386515498161316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.25e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784951642311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13872584700584412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784951655958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13831403851509094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951655958, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784951655959, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784951699419, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5002884864807129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784951699421, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784951699421, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784951713037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13828764855861664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951726830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13838374614715576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951740743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13840734958648682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951754643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13840578496456146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951768631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383359134197235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9916666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951782784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13809296488761902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951797180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13844585418701172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951797180, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784951797181, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784951839932, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5004202127456665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784951839932, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784951839933, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784951853968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13838502764701843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951868163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381653994321823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951881851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13798266649246216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951896411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13806968927383423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.9916666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951910321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13800083100795746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951924795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13811883330345154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951938642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379132866859436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951938642, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784951938643, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784951981158, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5005621314048767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784951981158, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784951981158, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784951994972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13831275701522827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952009572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378287971019745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.8249999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952023795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13719812035560608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952038073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13825343549251556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952052727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13766539096832275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3249999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952066593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1372944712638855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952081024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371997594833374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952081025, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784952081025, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784952124473, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5007615685462952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784952124475, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784952124475, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784952138172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13685496151447296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952152063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13755913078784943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952166426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13734744489192963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1583333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952180189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13693445920944214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952193906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13568298518657684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952207878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13677969574928284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952222038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13585889339447021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952222038, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784952222039, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784952264916, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5010018348693848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784952264916, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784952264916, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784952278905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13681122660636902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952292866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13627581298351288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952307066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374949961900711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952321398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365034580230713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952335110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350172609090805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.658333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952348824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13560235500335693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952362743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13738888502120972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952362743, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784952362743, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784952406715, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5012800693511963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784952406716, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784952406716, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784952420968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367274820804596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952435017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13566577434539795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952448994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13588997721672058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952463077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13490422070026398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952477200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13506457209587097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952491758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13451695442199707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952505886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349780261516571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952505887, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784952505887, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784952549558, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5016886591911316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784952549559, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784952549560, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784952563742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13412126898765564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952578081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13640056550502777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952592857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13809014856815338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952606994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1339997947216034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.824999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952621310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13589414954185486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952635445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13489577174186707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.158333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952649998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13660871982574463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.324999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952649999, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784952649999, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784952693119, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5022938251495361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784952693119, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784952693120, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784952707384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13645346462726593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952721712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13662844896316528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952736051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13480007648468018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952750412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13023634254932404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952764645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13234871625900269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952778629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337299346923828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952792534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13480296730995178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952792578, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784952792579, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784952836636, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5029956698417664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784952836638, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784952836638, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784952850879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13510215282440186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952865020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12932640314102173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952879769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13111761212348938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952893624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13683368265628815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.1158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952907568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13233676552772522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952921443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13136358559131622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952935801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13266521692276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952935802, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784952935802, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784952979086, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5037782788276672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784952979086, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784952979086, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784952992807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13163766264915466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953007769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13432399928569794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953021813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12795770168304443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953035822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12954191863536835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953050100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337074190378189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953064537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12821783125400543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953078527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1296144276857376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953078528, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784953078528, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784953123114, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5047837495803833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784953123115, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784953123115, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784953137951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13460713624954224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953151904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332055628299713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.3158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953166055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13447433710098267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953180125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347612738609314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953194680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13624250888824463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.3658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953208876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335715353488922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953223423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1369268149137497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953223424, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784953223425, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784953267224, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5060900449752808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784953267224, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784953267224, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784953281709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341484934091568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.4158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953296359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.130398228764534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953310921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13384300470352173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.4491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953325776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14038455486297607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.4658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953340377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304011195898056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953355301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323011815547943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953369696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13803283870220184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953369697, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784953369697, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784953411141, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5073729157447815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784953411143, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784953411143, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784953425862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1263532191514969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953440538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13100352883338928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953455601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13693298399448395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953470341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354098618030548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953485442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13233479857444763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953499546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12877774238586426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.6158333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953513865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12794113159179688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953513865, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784953513866, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784953555522, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5086999535560608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784953555522, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784953555523, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784953569773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13018035888671875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953584142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13471965491771698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.6658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953598842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1420166790485382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6825000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953613221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13411740958690643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953628402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13144239783287048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.7158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953642602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337738335132599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953657072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359492540359497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953657072, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784953657073, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784953698629, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5102251172065735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784953698629, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784953698629, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784953712681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347733587026596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953727055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363220363855362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953741667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13557443022727966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953756310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331101357936859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953770948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13425880670547485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953785175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13292795419692993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953799226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322193443775177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8658333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953799227, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784953799228, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784953841809, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5118816494941711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784953841809, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784953841810, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784953855798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13200098276138306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953869725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13428696990013123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8991666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953883872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.141882985830307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953898229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13373717665672302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953912669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13418996334075928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953927297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13225239515304565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953941902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13307349383831024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953941903, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784953941903, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784953983417, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5138294100761414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784953983417, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784953983418, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784953997741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13873139023780823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954012157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13295164704322815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954026401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1287592202425003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954040625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12986283004283905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954055207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13791675865650177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954069851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13165168464183807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954090242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361825168132782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954090243, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784954090249, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784954133086, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5160717368125916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784954133087, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784954133087, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784954147333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13681581616401672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.1158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954161326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363932490348816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954175632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13127340376377106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1491666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954189672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12997892498970032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954203951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13151150941848755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954218565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351618468761444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954233095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13263463973999023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954233095, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784954233095, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784954276257, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5186848044395447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784954276257, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784954276258, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784954290551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13593986630439758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954305757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13220757246017456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954320781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13790008425712585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954335634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13621971011161804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954350566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385124772787094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954365206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343858540058136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954380414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13894164562225342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954380415, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784954380416, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784954422897, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5212972164154053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784954422898, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784954422898, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784954438109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13498806953430176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954453333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1490492820739746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954468550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13497553765773773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954483354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13289058208465576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954498306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13418936729431152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954512936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13323372602462769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954527821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13638247549533844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954527821, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784954527822, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784954570771, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5244552493095398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784954570772, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784954570772, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784954585620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13621337711811066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954600515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13523226976394653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954620610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13373342156410217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954635357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13354156911373138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954650143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13309341669082642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954664803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12952929735183716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954679163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12560254335403442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954679164, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784954679164, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784954721240, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5284707546234131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784954721241, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784954721241, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784954735750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325659155845642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.5824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954750616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13244441151618958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954765537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350260078907013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954780327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354188472032547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954795240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13658851385116577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954809974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12865783274173737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954824600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12980452179908752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954824601, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784954824601, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784954867883, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5346848368644714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784954867883, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784954867883, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784954882202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325284242630005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.699166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954896696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1269327998161316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954911316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13303467631340027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954925963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13214609026908875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954941050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1268824338912964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954956099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13212671875953674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954970644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341215968132019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954970645, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784954970645, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784955015843, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5431119799613953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784955015844, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784955015844, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784955031153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12941491603851318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955045873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13030022382736206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955060362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13050046563148499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955075034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12969328463077545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.8658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955089418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12767964601516724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955104043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12320388853549957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955118981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12704813480377197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955118982, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784955118982, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784955163216, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5548246502876282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784955163217, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784955163217, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784955178000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13557949662208557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955192745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13288652896881104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955207142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860586285591125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955222465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331670731306076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955237370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12747061252593994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955252050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12940847873687744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955266939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1266246736049652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955266940, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784955266940, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784955309520, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5686448812484741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784955309521, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784955309521, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784955324053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288244128227234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955338621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12710420787334442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955353682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13218986988067627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955368727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324102282524109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955383967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13116693496704102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955398744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13325512409210205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955413933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13006539642810822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955413933, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784955413934, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784955457906, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5833144187927246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784955457906, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784955457906, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784955473013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289735585451126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955488025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12944483757019043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955503140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289118230342865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955518229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13192719221115112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955533082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13247233629226685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955547702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12871405482292175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955562891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12971818447113037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955562891, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784955562892, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784955606252, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5955064296722412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784955606253, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784955606253, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784955621295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12732404470443726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955636679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13014715909957886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955651426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12727046012878418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955666830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12690164148807526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955681926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12656167149543762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955697002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13045816123485565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955712335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13111813366413116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955712336, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784955712336, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784955755850, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6056730151176453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784955755851, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784955755851, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784955770583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12880948185920715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955785830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1265765130519867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955801258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12922745943069458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955816579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13146041333675385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955831926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13224807381629944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955853864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1277250051498413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955869675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12817862629890442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.4991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955869676, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784955869676, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784955912918, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6147444248199463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784955912919, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784955912919, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784955928019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1282023787498474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.5158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955943207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12604525685310364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955958465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1270100474357605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.5491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955973454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1339838206768036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955988692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12960472702980042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956004204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12424337863922119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956019779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1267642378807068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956019779, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784956019779, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784956063463, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6218673586845398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784956063463, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784956063464, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784956078456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12387694418430328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6325000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956093738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12686672806739807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956108465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1252410113811493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956123204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12427061796188354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956137896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12296458333730698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956152592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11964499950408936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956167318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12239247560501099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956167318, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784956167318, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784956209955, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6297379732131958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784956209956, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784956209956, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784956225141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12507547438144684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956240260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12105543166399002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956255214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11421870440244675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.7824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956270415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12104573845863342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956285534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11631909012794495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956300328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11723732948303223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8324999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956315095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11428992450237274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.8491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956315096, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784956315096, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784956357517, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6405841112136841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784956357518, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784956357518, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784956372600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12021856009960175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956387378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11612571775913239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956402245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11163477599620819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.8991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956417283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12114871293306351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956432636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11864139139652252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956448077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11774095892906189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.9491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956463040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10979671776294708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.9658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956463040, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784956463041, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784956507092, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6524690985679626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784956507093, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784956507093, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784956521770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11670201271772385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956536886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11585171520709991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956551533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11933852732181549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.0158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956566105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11389288306236267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956580643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11269092559814453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956595077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11418379843235016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0658333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956609599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11063864827156067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956609644, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784956609644, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784956653313, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6700317859649658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784956653314, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784956653314, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784956667564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11576317250728607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956682375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11296428740024567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.1158333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956697050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12011946737766266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956712175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11283627897500992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956726727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11465032398700714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1658333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956742064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11989132314920425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956756944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10872720181941986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956756944, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784956756945, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784956800783, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6912520527839661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784956800783, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784956800783, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784956815509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11708204448223114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956830220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11443966627120972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956845082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11248376220464706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956859890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11182686686515808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956874761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10690146684646606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956889417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12044252455234528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956904678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1171296238899231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956904678, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784956904678, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784956948647, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7089563012123108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784956948648, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784956948648, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784956963635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12039895355701447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956978841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11505624651908875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.3491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956998837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11887788772583008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957013939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1057792529463768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957028578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1172284185886383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.3991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957043369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1119542345404625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.4158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957058268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10959914326667786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957058269, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784957058269, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784957101889, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7291717529296875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784957101890, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784957101890, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784957116487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11338400840759277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.4491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957131347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11210057139396667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957145959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11701711267232895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.4824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957160818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11320964992046356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957175875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11636252701282501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957190950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723128914833069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957205985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11649778485298157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957205986, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784957205986, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784957249039, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7425472140312195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784957249040, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784957249040, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784957264118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11085571348667145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957283226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11377017199993134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.5824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957298832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12386348843574524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957314218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1269831657409668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957329312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12120848894119263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957344516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11491996794939041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957359618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11970505118370056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957359619, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784957359619, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784957402927, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7503325939178467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784957402928, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784957402928, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784957417872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11937803030014038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957433024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11233241856098175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.6991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957448377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12216776609420776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957463681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11416807770729065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957479201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11887681484222412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957494263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1168297603726387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957509854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10863080620765686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957509855, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784957509856, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784957552499, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7555879354476929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784957552500, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784957552500, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784957567746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12049564719200134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957582995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1108987033367157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957598429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10621588677167892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957613681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11755560338497162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957628947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12148471176624298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957644298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11050166189670563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957659996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11735906451940536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957659996, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784957659997, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784957702691, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7609198093414307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784957702692, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784957702692, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784957717909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11748023331165314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957733575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11127479374408722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957748815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11096462607383728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957767826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11630222201347351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957782787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11009502410888672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957797438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1058194488286972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957811976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11588724702596664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.015833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957811977, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784957811977, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784957854068, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7637112736701965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784957854068, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784957854068, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784957868875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11116243153810501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957883768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11618204414844513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957898542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11202012002468109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957913311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10667037963867188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957928645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10475722700357437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957943711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09893938153982162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957959044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10843673348426819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957959045, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784957959045, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784958000688, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7661231160163879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784958000689, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784958000689, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784958015590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10692629218101501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958030237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11090894043445587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958046337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1208338662981987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958061883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1099386215209961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958077137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10728442668914795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958092304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1027291864156723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958107714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10831934958696365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958107715, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784958107715, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784958150090, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7673848867416382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784958150090, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784958150091, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784958165497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10542112588882446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958180712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10735294222831726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958196130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11029212176799774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958211640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10858043283224106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958227255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10771728307008743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958242969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11305209994316101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958258905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11152664572000504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958258906, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784958258906, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784958301916, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7679041624069214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784958301917, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784958301917, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784958317365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10630768537521362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958335692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10298795998096466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958350823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11126118898391724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958366056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10992397367954254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958380912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10544025897979736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958395738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10037203133106232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958410788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1049066036939621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958410789, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784958410789, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784958453108, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7678709626197815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784958453108, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784958453109, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784958467946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10459521412849426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958482712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1007406935095787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958497290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10548203438520432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958512519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11100980639457703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958527768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10655676573514938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958543191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10387039929628372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958558532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11077690124511719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958558532, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784958558532, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784958601063, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7690240740776062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784958601063, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784958601064, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784958615704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09979033470153809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958631562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10421855002641678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958646964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10763227939605713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958662228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10498203337192535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958677387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10223498195409775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958692514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10759517550468445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958707838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10969722270965576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958707838, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784958707838, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784958750179, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7687154412269592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784958750180, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784958750180, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784958765432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10676732659339905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958780680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10297736525535583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958795763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10956752300262451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958811325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071600615978241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958826942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11036671698093414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958842563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10815511643886566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958858050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11138460040092468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958858051, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784958858051, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784958900695, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7700597643852234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784958900695, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784958900696, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784958915596, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10620564222335815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958931206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10121849924325943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958946123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10522536933422089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958961188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10255203396081924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958975995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10880730301141739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958990755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11287356913089752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959005669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087692379951477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959005670, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784959005670, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784959048076, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7694944739341736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784959048077, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784959048077, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784959063028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10513302683830261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959078065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11127573251724243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959093142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10836528986692429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959108572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1006607860326767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959124067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11277516186237335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959139483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1134573370218277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959155449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1054871678352356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959155449, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784959155449, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784959200113, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7703933715820312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784959200114, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784959200114, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784959215202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10844822227954865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959230112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10019782930612564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959246364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11582972854375839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959262381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1164027601480484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959277799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11531390994787216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959293229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1043301373720169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959308793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10686278343200684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959308793, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784959308794, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784959353124, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7697263360023499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784959353125, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784959353125, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784959368713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11182036995887756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959384137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11401896178722382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959399746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11060595512390137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959415516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11062347143888474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959431330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11459290981292725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959446802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11228188127279282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959462610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11314369738101959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959462610, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784959462611, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784959507395, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7707025408744812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784959507395, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784959507395, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784959522553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10966705530881882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959538644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10748502612113953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959554087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10705696791410446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959569714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11453980207443237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959584923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11833789199590683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.3825e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_32768/seed457724564.log b/recommendation_v4/rcp_logs/gbs_32768/seed457724564.log new file mode 100644 index 000000000..d55f07dcb --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_32768/seed457724564.log @@ -0,0 +1,687 @@ +:::MLLOG {"namespace": "", "time_ms": 1784956263534, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784956263534, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784956286929, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784956286935, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784956286935, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784956286935, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784956286935, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784956286935, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784956286935, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784956286935, "event_type": "POINT_IN_TIME", "key": "seed", "value": 457724564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784956286935, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784956286935, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784956286935, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784956286935, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784956286973, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784956286974, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784957130556, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784957130557, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784957130877, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784957340100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13891899585723877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.1666666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784957355386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13890013098716736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.4999999999999995e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784957369477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13882321119308472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784957376583, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784957376584, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784957400067, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5013392567634583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784957400068, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784957400068, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784957407107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13883662223815918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3166666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957421199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13874632120132446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957435788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13877946138381958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9833333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957450181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13863061368465424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957450182, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784957450182, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784957472940, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5014820694923401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784957472940, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784957472940, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784957487230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13863490521907806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957501517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13863971829414368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957516139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13858960568904877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957523258, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784957523259, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784957547206, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5016613602638245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784957547206, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784957547206, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784957554791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13859862089157104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957568964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138441264629364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957584002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384638100862503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957598577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382390856742859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957598577, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784957598578, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784957624212, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5018575191497803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784957624212, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784957624212, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784957638472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382429450750351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957653163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13790687918663025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957667694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13802769780158997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957674881, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784957674881, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784957700453, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5020803809165955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784957700454, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784957700454, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784957707385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13750620186328888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957722303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375921666622162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957736481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13723015785217285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957751114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1372499316930771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957751115, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784957751115, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784957776542, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5023305416107178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784957776543, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784957776543, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784957790497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13742339611053467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.316666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957804974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373400241136551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957820114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367240697145462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957827558, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784957827559, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784957851164, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5025795102119446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784957851165, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784957851166, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784957858688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13742467761039734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957874258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364569067955017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957889171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13678450882434845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957904793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1372275948524475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957904794, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784957904794, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784957930703, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5029360055923462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784957930703, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784957930704, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784957945547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13647456467151642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957960854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352510154247284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.983333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957975517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366303563117981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957982879, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784957982879, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784958005791, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5032640099525452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784958005792, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784958005792, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784958013195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335064321756363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958028275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13492649793624878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958042852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13522279262542725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958057434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13540643453598022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958057506, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784958057506, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784958083257, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5036149024963379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784958083258, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784958083258, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784958097924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13409343361854553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958112835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13473640382289886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958127568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13528616726398468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958135504, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784958135505, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784958161105, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5040175914764404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784958161106, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784958161106, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784958168326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13579820096492767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958183346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13493755459785461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958198393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13222503662109375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958213988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346449851989746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958213988, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784958213989, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784958239771, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5045293569564819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784958239772, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784958239772, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784958255058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13389813899993896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958270451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13518625497817993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958286148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13498398661613464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958293894, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784958293894, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784958319126, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5053638815879822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784958319127, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784958319127, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784958326632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13637349009513855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958341886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13354091346263885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958357778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1342584490776062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958372948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321282535791397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958373013, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784958373014, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784958397629, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.506300687789917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784958397630, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784958397630, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784958412607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13280968368053436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958428252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13125455379486084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958443345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13320094347000122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958450802, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784958450803, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784958476822, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5072986483573914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784958476823, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784958476823, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784958484218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13843291997909546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958499438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13110795617103577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958514931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12990978360176086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958530127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304004043340683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958530128, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784958530128, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784958555838, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5084630846977234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784958555839, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784958555839, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784958570734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13278508186340332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958585800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12975621223449707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958601202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1305902600288391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958608779, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784958608779, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784958633765, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5098000168800354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784958633766, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784958633766, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784958641208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13169744610786438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958656079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351403146982193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958671275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330956518650055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958686909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13567262887954712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958686909, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784958686910, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784958711653, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5112242698669434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784958711653, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784958711653, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784958726980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13522721827030182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958742238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13251903653144836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1649999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958757632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330910623073578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958765208, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784958765209, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784958788110, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5126243829727173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784958788110, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784958788110, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784958796641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13635340332984924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958812191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352698802947998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958827862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13620683550834656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958843547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13650059700012207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958843548, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784958843548, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784958867250, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5140380263328552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784958867250, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784958867251, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784958883485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13482026755809784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958899531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13083364069461823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958915082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13629522919654846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958923048, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784958923048, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784958947153, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5153489708900452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784958947154, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784958947154, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784958954841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13592126965522766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958970487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13248437643051147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958985948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326395571231842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959001311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13089875876903534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959001311, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784959001311, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784959026583, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5166118144989014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784959026584, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784959026584, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784959041744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13102656602859497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959057735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382453441619873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959073239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295485496520996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959080887, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784959080888, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784959106189, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5181728005409241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784959106190, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784959106190, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784959113698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12924614548683167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959129116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13474369049072266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959144537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345541775226593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959160561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13027578592300415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959160562, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784959160562, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784959186310, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5200087428092957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784959186310, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784959186310, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784959201878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12809327244758606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959217062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13179782032966614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959232596, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13561636209487915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959240321, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784959240322, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784959264294, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5218887329101562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784959264295, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784959264295, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784959272092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365503966808319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959288051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13365384936332703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959303731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13213074207305908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959319191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13095998764038086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959319192, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784959319192, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784959345176, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5240512490272522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784959345176, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784959345177, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784959360814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12961772084236145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959376260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325347125530243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959392286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13534465432167053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959400226, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784959400227, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784959426065, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5267432332038879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784959426066, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784959426066, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784959433865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13197660446166992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959449524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13339293003082275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959465242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349182426929474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959480841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346169263124466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959480842, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784959480842, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784959505435, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5296194553375244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784959505435, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784959505435, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784959520916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13604791462421417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959537480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13395604491233826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959553582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331167221069336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3650000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959561413, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784959561413, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784959586751, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5330262184143066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784959586752, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784959586752, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784959595000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1403738260269165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959610872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13527831435203552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959627128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13448156416416168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4649999999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959643823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13522355258464813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959643824, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784959643824, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784959669879, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5363401174545288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784959669880, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784959669880, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784959685838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13523399829864502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959701824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13320010900497437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.5649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959717614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13237221539020538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959726077, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784959726077, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784959752130, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5401883125305176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784959752131, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784959752131, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784959759923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325775682926178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959775729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13084883987903595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959791032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359279304742813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959806826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13239046931266785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959806827, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784959806827, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784959830566, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5445753931999207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784959830566, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784959830566, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784959846311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13478881120681763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959862773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13036870956420898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959878874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321103572845459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959886799, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784959886799, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784959912655, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5505914688110352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784959912656, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784959912656, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784959920487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13245789706707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959936347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13093248009681702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959952297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13320422172546387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959968230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13281871378421783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959968231, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784959968232, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784959994741, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.559337317943573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784959994742, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784959994742, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784960010214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320902705192566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960025793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.128410205245018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960041187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343182623386383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960049096, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784960049097, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784960074853, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5687724947929382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784960074854, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784960074855, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784960082657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13510572910308838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960098277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12704147398471832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960114323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12734690308570862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960130503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13327521085739136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960130504, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784960130504, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784960156613, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5813794136047363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784960156614, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784960156614, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784960172290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297808438539505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960188336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12890803813934326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960204653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1286291778087616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960212372, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784960212373, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784960235674, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5941627621650696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784960235675, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784960235675, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784960243585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13046184182167053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960259897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12824422121047974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960275578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13058169186115265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960291086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1290348470211029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960291087, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784960291087, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784960317364, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6092924475669861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784960317365, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784960317365, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784960333073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12745904922485352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960348954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13037744164466858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4983333333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960364618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13381421566009521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960372359, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784960372359, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784960396034, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6240498423576355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784960396035, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784960396035, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784960404510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321459859609604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960420896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1275506317615509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960437253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13186490535736084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960453405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12767748534679413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960453405, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784960453406, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784960479607, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6393148899078369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784960479608, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784960479608, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784960495863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13000720739364624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960511946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13117018342018127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960528457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1290844976902008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960536603, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784960536604, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784960562276, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6555496454238892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784960562277, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784960562277, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784960570451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1281774640083313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960586309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12809687852859497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960602502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12934815883636475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960618531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12835942208766937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.898333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960618531, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784960618532, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784960641658, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6733044981956482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784960641659, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784960641659, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784960657904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12256608158349991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960673661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12180937081575394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960689164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12054748088121414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960697057, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784960697058, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784960722730, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6898194551467896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784960722730, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784960722730, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784960730494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12175504863262177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960746368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11903546750545502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960762244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12296450883150101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960777866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12063363939523697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960777866, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784960777867, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784960801453, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7108113765716553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784960801454, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784960801454, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784960817928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12159843742847443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960833867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11331857740879059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.198333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960849386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11875748634338379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960857157, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784960857157, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784960882123, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7264090180397034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784960882124, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784960882124, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784960889736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11893691122531891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960905681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11922790110111237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.298333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960921398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1136632114648819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960936916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11717651039361954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960936982, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784960936982, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784960962666, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7344011664390564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784960962667, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784960962667, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784960978463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11555855721235275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.398333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960994236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11564940959215164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961010214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.109237439930439, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961017835, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784961017836, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784961043887, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7405363321304321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784961043888, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784961043888, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784961051559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1125059425830841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.498333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961067626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11316492408514023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.531666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961083433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111152283847332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961099598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11314146965742111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.598333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961099599, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784961099600, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784961125380, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7381808161735535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784961125380, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784961125380, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784961140978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11161806434392929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961156709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11055787652730942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961172798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11494477093219757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961180572, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784961180572, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784961206734, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7434138655662537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784961206735, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784961206735, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784961214511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10449215769767761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961230903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11264927685260773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961246815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10630682855844498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961262927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11036007106304169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961262928, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784961262928, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784961289932, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.742480456829071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784961289933, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784961289933, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784961305848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11359034478664398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961321675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11006511747837067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961337792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10881495475769043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961345340, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784961345340, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784961371217, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7468281388282776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784961371218, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784961371218, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784961378953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10915900021791458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961395232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11090788245201111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961411423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11166863888502121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961426672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10473746061325073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961426673, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784961426673, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784961451007, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7477802634239197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784961451008, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784961451008, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784961468201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11467345058917999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961484835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11881788074970245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961501216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11552406847476959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961509548, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784961509549, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784961535806, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.748195230960846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784961535807, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784961535807, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784961543821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1139366552233696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961560635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1142897978425026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961576780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11188821494579315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961592769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11302435398101807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961592770, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784961592770, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784961619231, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7512657642364502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784961619231, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784961619231, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784961635025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11752831935882568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961650984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11693165451288223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961667040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11069641262292862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961675049, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784961675050, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784961700913, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7540103197097778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784961700913, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784961700914, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784961708852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11562534421682358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961725131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11575483530759811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961741007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11129197478294373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961757698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10454580932855606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961757698, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784961757699, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784961783595, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7573225498199463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784961783595, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784961783596, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784961799442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10658053308725357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961815613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10557404905557632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961831600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10595542937517166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961839723, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784961839723, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784961865475, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7572221755981445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784961865476, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784961865476, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784961873535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10430456697940826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961889920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1118079423904419, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961905940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10761182755231857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961922081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11289849877357483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961922082, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784961922083, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784961949342, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7586656808853149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784961949342, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784961949343, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784961965435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10704407095909119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961981067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10643982887268066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961997277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10203249007463455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962005003, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784962005003, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784962031493, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7589748501777649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784962031494, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784962031494, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784962039469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10263306647539139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962055755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10184105485677719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962072202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1116897314786911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962087919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10408677160739899, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962087920, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784962087920, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784962112378, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7618294358253479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784962112379, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784962112379, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784962128574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1064106822013855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962145038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10635402053594589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962160672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1064569279551506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962169030, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784962169031, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784962194735, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7635859251022339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784962194736, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784962194736, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784962202817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10748255252838135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962219061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09976746886968613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962235387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793212056159973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962252064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1041644811630249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962252064, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784962252067, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784962276353, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7643488049507141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784962276354, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784962276354, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784962292161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10760503262281418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962308631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081416979432106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962324780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10375484079122543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962332746, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784962332747, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784962359071, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7631798982620239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784962359071, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784962359072, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784962366822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10883816331624985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962382939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10784405469894409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962398937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1114327535033226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962415254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10800297558307648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962415255, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784962415256, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784962441625, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7648054957389832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784962441626, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784962441626, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784962457929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10535947978496552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962473766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10167266428470612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962490644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10833010077476501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.564999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962499085, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784962499086, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784962525641, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7671501040458679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784962525641, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784962525641, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784962533640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10468652844429016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962549711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0994994044303894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962565985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11439765244722366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.664999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962582382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11106826364994049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962582382, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784962582383, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784962608562, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7673689126968384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784962608562, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784962608563, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784962624686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1062297374010086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962641299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084912046790123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962657486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10853336751461029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962665371, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784962665372, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784962689125, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7669713497161865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784962689125, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784962689126, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784962697563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11003214120864868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962714213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11473861336708069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962730663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11224702000617981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962747224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11283416301012039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962747225, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784962747225, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784962773027, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7673677206039429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784962773027, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784962773028, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784962789147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11006693542003632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962805778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11384376883506775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.998333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962822413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11086687445640564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962830502, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784962830503, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784962855479, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7677550911903381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784962855479, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784962855480, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784962863613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1103462278842926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962880829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10938966274261475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962897538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11231598258018494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.131666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962914075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11123602092266083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962914076, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784962914076, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784962940045, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7695180773735046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784962940046, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784962940046, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784962956755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11018175631761551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962973886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11275626718997955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962990451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11704202741384506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962998742, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784962998742, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784963022892, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7697660326957703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784963022893, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784963022893, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784963030864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11465989053249359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963047722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10761391371488571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.331666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963063965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10133599489927292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963080083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10671694576740265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963080084, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784963080084, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784963106485, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7706117630004883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784963106485, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784963106486, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784963122575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10475356131792068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963138606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10604254901409149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963154960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10554944723844528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963163434, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784963163435, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784963189052, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7703079581260681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784963189052, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784963189053, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784963197101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10837376117706299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963212969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10311063379049301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963229748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10487981140613556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963246352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10735005140304565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.631666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963246352, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784963246353, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784963272141, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7712681293487549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784963272141, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784963272142, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784963288128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10085343569517136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963304594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10374483466148376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963320886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10334064066410065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963328950, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784963328951, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784963355165, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7715346217155457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784963355166, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784963355166, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784963363332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10430413484573364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963380103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10278695821762085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963396210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1138230413198471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963412575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10300146788358688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963412575, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784963412576, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784963438079, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.771716833114624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784963438079, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784963438080, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784963454523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10726656019687653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963470647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10737602412700653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963487072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10160664469003677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963495005, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784963495006, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784963521465, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7712503671646118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784963521466, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784963521466, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784963529600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10101231187582016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963545751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10043717175722122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963562441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10611994564533234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963578975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1029839962720871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963578976, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784963578976, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784963603718, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7725918889045715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784963603718, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784963603719, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784963619633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10748043656349182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963636884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10369208455085754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963653407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10592159628868103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963661298, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784963661298, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} diff --git a/recommendation_v4/rcp_logs/gbs_32768/seed526341580.log b/recommendation_v4/rcp_logs/gbs_32768/seed526341580.log new file mode 100644 index 000000000..c1dff25c5 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_32768/seed526341580.log @@ -0,0 +1,666 @@ +:::MLLOG {"namespace": "", "time_ms": 1784909131269, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784909131269, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784909158466, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784909158469, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784909158469, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784909158469, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784909158469, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784909158469, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784909158469, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784909158469, "event_type": "POINT_IN_TIME", "key": "seed", "value": 526341580, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784909158469, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784909158469, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784909158469, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784909158469, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784909158479, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784909158479, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784909811615, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784909811617, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784909811821, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784910278396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13873390853405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.1666666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784910293682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13864240050315857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.4999999999999995e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784910308379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13876973092556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784910315667, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784910315667, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784910360129, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5013054013252258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784910360130, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784910360130, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784910367427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13871541619300842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3166666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910382023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13875891268253326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910396934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386837661266327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9833333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910411791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13873231410980225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910411791, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784910411792, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784910435353, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5014687776565552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784910435353, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784910435353, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784910450105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13853761553764343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910464939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13847529888153076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910479995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13836190104484558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910487336, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784910487336, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784910511106, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5016574859619141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784910511107, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784910511107, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784910518911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13857872784137726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910533895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383584886789322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910549106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138438418507576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910564119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13807858526706696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910564120, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784910564120, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784910586861, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5018885135650635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784910586861, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784910586862, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784910601772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382719874382019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910616612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13776469230651855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910631510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380549818277359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910638910, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784910638910, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784910661736, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5021727085113525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784910661737, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784910661737, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784910669033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13745126128196716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910684434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13741210103034973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910699326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13710175454616547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910714574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370881050825119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910714575, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784910714575, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784910738143, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5024818181991577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784910738144, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784910738144, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784910752651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371547132730484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.316666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910767623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13722294569015503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910783395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13646341860294342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910790942, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784910790943, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784910814530, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5028282999992371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784910814531, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784910814531, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784910822160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1372247040271759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910837998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13634926080703735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910853156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13661357760429382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910869177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370188593864441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910869177, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784910869178, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784910892172, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5032877326011658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784910892172, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784910892173, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784910907234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362975835800171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910923063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13502223789691925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.983333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910938038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13654518127441406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784910945576, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784910945577, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784910968780, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5037441849708557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784910968781, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784910968781, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784910976309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13310140371322632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784910991861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13452330231666565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911006772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134946808218956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911021737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13522250950336456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911021805, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784911021806, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784911045439, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5042166709899902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784911045440, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784911045440, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784911060305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13379254937171936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911075216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13462728261947632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911090166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352236568927765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911098221, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784911098222, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784911121186, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5047289133071899, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784911121187, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784911121187, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784911128492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13561874628067017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911143401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13480150699615479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911158637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318969875574112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911174305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13458195328712463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911174306, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784911174306, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784911197249, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5053913593292236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784911197250, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784911197250, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784911212674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13378125429153442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911228388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353323757648468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911244330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13509857654571533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911252140, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784911252140, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784911275212, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5064204931259155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784911275213, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784911275213, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784911282880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.136510968208313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911298349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334608793258667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911314318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134331613779068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911329783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13214366137981415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911329846, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784911329847, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784911352566, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5075273513793945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784911352567, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784911352567, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784911367957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13288283348083496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911384077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13125059008598328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911399168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13311518728733063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911406825, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784911406825, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784911429179, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5086913108825684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784911429179, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784911429179, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784911436722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13842891156673431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911452140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13109584152698517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911468071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12983228266239166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911483534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13036738336086273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911483534, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784911483535, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784911506609, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5100205540657043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784911506610, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784911506610, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784911522050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327946037054062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911537717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12975890934467316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911553525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1306544840335846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911560977, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784911560977, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784911583234, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5114631652832031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784911583235, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784911583235, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784911590805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13149988651275635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911605919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13521160185337067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911621624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13317516446113586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911637803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13566076755523682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911637804, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784911637804, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784911660285, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5130100250244141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784911660286, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784911660286, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784911675816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351589411497116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911691557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13262131810188293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1649999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911707499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332358568906784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911715377, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784911715377, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784911738506, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.514509379863739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784911738507, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784911738507, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784911746935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13635677099227905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911762768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352347582578659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911778554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361624300479889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911794622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364419013261795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911794622, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784911794623, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784911818247, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5160394906997681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784911818248, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784911818248, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784911834904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13478237390518188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911851267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13085666298866272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911867383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362246423959732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911875675, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784911875676, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784911899325, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5175673365592957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784911899325, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784911899325, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784911907274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13591983914375305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911923571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13252270221710205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911939287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13260261714458466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911954824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13087081909179688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911954824, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784911954825, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784911977477, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5189923644065857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784911977478, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784911977478, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784911993072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310516744852066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912009091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383226215839386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912025228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12954698503017426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912033226, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784912033226, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784912056246, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5206571221351624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784912056246, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784912056246, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784912064041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12922954559326172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912080049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13480716943740845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912096003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13462767004966736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912112568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13036905229091644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912112569, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784912112569, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784912135155, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5226723551750183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784912135156, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784912135156, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784912150760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1280834674835205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912166387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13184398412704468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912182152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13543814420700073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912190178, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784912190178, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784912213899, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5248515605926514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784912213900, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784912213900, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784912221871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365807056427002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912238058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13352788984775543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912253927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320158839225769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912269491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13091513514518738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912269492, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784912269492, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784912293036, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5273414850234985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784912293037, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784912293037, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784912308726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12954849004745483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912324386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324179768562317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912340717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13541127741336823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912348779, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784912348780, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784912371436, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5303677916526794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784912371437, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784912371437, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784912379355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318598985671997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912395452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13340581953525543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912411354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13496533036231995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912427399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13458910584449768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912427399, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784912427399, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784912450654, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5338490605354309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784912450654, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784912450655, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784912466974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135977104306221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912483361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13381677865982056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912499593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331564486026764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3650000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912507528, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784912507529, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784912531261, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5379123687744141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784912531262, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784912531262, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784912539512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14018258452415466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912555803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13521096110343933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912572134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13429218530654907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4649999999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912588936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13501933217048645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912588936, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784912588936, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784912612271, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5422119498252869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784912612272, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784912612272, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784912628450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351666897535324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912644858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133146271109581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.5649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912661365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13233748078346252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912669889, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784912669889, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784912693174, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.546717643737793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784912693174, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784912693174, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784912701314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13244116306304932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912717132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13075481355190277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912732887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357807219028473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912748977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13222664594650269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912748978, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784912748978, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784912772189, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5519312024116516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784912772190, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784912772190, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784912788232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13455882668495178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912804821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13010267913341522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912820954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13166996836662292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912829056, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784912829056, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784912851725, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5587932467460632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784912851726, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784912851726, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784912859639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13215193152427673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912875903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13051161170005798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912892276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13281315565109253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912908631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321818232536316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912908631, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784912908632, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784912932622, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5673744082450867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784912932623, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784912932623, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784912948369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13164040446281433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912964229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1277942657470703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912979817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336183249950409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912987862, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784912987863, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784913011241, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5762316584587097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784913011242, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784913011242, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784913019236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344124674797058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913035076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1263229250907898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913051367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12643089890480042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913067499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13243059813976288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913067500, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784913067500, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784913090582, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.587087094783783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784913090583, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784913090583, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784913106495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12888163328170776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913122361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12809208035469055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913138519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1273798793554306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913146373, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784913146374, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784913169693, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5938713550567627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784913169694, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784913169694, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784913177826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294785588979721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913194215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12691166996955872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913210253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12947508692741394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913226103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12781907618045807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913226104, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784913226104, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784913250252, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.603882908821106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784913250253, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784913250253, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784913266104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1261768639087677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913282255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12908634543418884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4983333333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913298232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321074664592743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913306238, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784913306239, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784913330897, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6131171584129333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784913330898, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784913330898, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784913339504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13052353262901306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913356205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12574172019958496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913372689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13047248125076294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913388884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12572461366653442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913388885, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784913388885, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784913412017, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6233498454093933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784913412018, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784913412018, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784913428581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12819552421569824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913444939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295987218618393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913461744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12720227241516113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913470086, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784913470086, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784913493793, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6340071558952332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784913493794, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784913493794, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784913501846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12659072875976562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913517895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12642589211463928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913534310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12686066329479218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913550629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12575215101242065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.898333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913550630, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784913550630, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784913574463, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.642676830291748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784913574464, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784913574464, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784913591017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12005706131458282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913607266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11964701861143112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913623248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11774566769599915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913631219, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784913631220, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784913654853, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6444427371025085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784913654853, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784913654854, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784913662774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11949822306632996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913678967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11646667867898941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913695326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12114882469177246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913711496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11870183795690536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913711497, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784913711497, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784913735711, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6650964617729187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784913735712, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784913735712, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784913752339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11983343213796616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913768385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11127642542123795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.198333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913784309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1172519326210022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913792229, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784913792229, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784913815337, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6855701208114624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784913815337, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784913815337, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784913823238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11791616678237915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913839494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11783672869205475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.298333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913855833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11245064437389374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913871616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11625031381845474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913871693, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784913871693, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784913895569, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7129432559013367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784913895570, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784913895570, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784913912122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11517942696809769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.398333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913928254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11510850489139557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913944568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10820861905813217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913952620, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784913952620, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784913976267, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7311861515045166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784913976268, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784913976268, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784913984355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1125912144780159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.498333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914000847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11333094537258148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.531666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914017140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11153963208198547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914033459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11347754299640656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.598333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914033460, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784914033460, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784914056984, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7551345825195312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784914056985, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784914056985, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784914073103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11223593354225159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914089215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11132904887199402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914105338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11550319194793701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914113341, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784914113342, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784914136847, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7616064548492432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784914136848, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784914136848, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784914144661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1047828197479248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914161009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11282044649124146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914177164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10666032135486603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914193456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11066770553588867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914193456, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784914193457, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784914217394, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.766037106513977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784914217395, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784914217395, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784914233588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11379291862249374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914249552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1105940192937851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914265715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1092788577079773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914273649, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784914273650, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784914296308, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7675557732582092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784914296309, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784914296309, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784914304461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10992272198200226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914320862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11108363419771194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914337543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11158545315265656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914353549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10519761592149734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914353550, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784914353550, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784914377073, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7692939043045044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784914377074, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784914377074, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784914394386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11460816115140915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914410610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11887967586517334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914427031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11556969583034515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914435158, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784914435158, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784914457883, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7704788446426392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784914457884, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784914457884, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784914465994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11488867551088333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914482652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11435891687870026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914499037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11175854504108429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914515310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11312663555145264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914515311, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784914515311, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784914539586, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7709818482398987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784914539587, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784914539587, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784914556470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11749798059463501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914573080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11741326749324799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914589907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11032170057296753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914598204, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784914598205, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784914621648, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7715117931365967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784914621648, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784914621649, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784914630084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11472757905721664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914646988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11493808031082153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914663664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11099553108215332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914680638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10390940308570862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914680638, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784914680639, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784914704359, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7717825770378113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784914704360, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784914704360, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784914720078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10623440146446228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914736153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10546213388442993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914752202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10546649992465973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914760226, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784914760227, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784914783881, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7723501920700073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784914783882, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784914783882, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784914792027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10362373292446136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914808695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11091048270463943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914825081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725167393684387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914841689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11227555572986603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914841689, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784914841690, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784914865480, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7726418375968933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784914865481, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784914865481, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784914881990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1064562052488327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914898205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10613948106765747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914914667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10156828910112381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914922706, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784914922706, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784914946843, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7729508280754089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784914946844, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784914946844, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784914955065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10236550867557526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914971553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10122734308242798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914988281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11083105206489563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915004500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10370521247386932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915004501, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784915004501, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784915029364, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730721235275269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784915029365, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784915029365, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784915045937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10554439574480057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915062427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10619352757930756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915078496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10640248656272888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915086878, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784915086878, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784915110160, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.77329021692276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784915110160, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784915110161, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784915118414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1068853810429573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915134594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09914456307888031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915151212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10752206295728683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915167797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10394388437271118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915167797, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784915167798, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784915191179, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7734025120735168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784915191181, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784915191181, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784915207260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732076317071915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915224108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10787960886955261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915240641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10399608314037323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915248813, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784915248814, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784915272107, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773574709892273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784915272108, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784915272108, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784915280147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10869725793600082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915296661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10702766478061676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915313044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11135011911392212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915329821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10786730796098709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915329821, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784915329822, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784915353177, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7737813591957092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784915353178, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784915353178, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784915370040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10482782870531082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915386455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10138830542564392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915403365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10789947956800461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.564999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915411743, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784915411744, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784915435416, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7737798690795898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784915435416, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784915435417, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784915443376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10406952351331711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915459406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09918810427188873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915475663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1137312799692154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.664999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915491903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11043062806129456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915491904, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784915491904, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784915515302, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7739419937133789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784915515303, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784915515303, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784915531722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10619118064641953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915548561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10836128145456314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915564994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10798825323581696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915573025, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784915573026, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784915597164, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7740586400032043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784915597165, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784915597165, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784915606069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10978671908378601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915623148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11438559740781784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915640171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11187699437141418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915657052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11274714022874832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915657052, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784915657053, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784915681633, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7741419076919556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784915681634, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784915681634, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784915698297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10975994169712067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915715279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11341701447963715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.998333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915732331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11037234961986542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915740751, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784915740752, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784915764718, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7743661999702454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784915764719, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784915764719, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784915772981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11038659512996674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915790168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1092933714389801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915806808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1124717965722084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.131666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915823496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11068357527256012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915823496, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784915823497, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784915847198, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774531900882721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784915847199, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784915847199, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784915863938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10979249328374863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915881064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11266990005970001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915898131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1167503148317337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915906599, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784915906599, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784915930927, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746140360832214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784915930928, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784915930928, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784915939121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11445701122283936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915956058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10762406140565872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.331666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915972405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1010226309299469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915988697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10659899562597275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915988698, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784915988698, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784916012810, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7744200825691223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784916012811, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784916012811, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784916029174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10469157248735428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916045565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10591776669025421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916062314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10500170290470123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916070751, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784916070751, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784916094582, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746195197105408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784916094583, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784916094583, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784916102840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10824399441480637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916118918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1031327024102211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916135787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10475759208202362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916152245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10707424581050873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.631666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916152245, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784916152246, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784916175614, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746298909187317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784916175615, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784916175615, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784916191744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10029705613851547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916208227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10350552201271057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916224560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10295145958662033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916232679, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784916232679, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784916256237, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7747148871421814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784916256238, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784916256238, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784916264521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10343869775533676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916281502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10277315229177475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916297994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11326272040605545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916314720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10286329686641693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916314721, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784916314721, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784916338758, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774863600730896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784916338759, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784916338759, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784916355336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10692591965198517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.898333333333333e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_32768/seed591489896.log b/recommendation_v4/rcp_logs/gbs_32768/seed591489896.log new file mode 100644 index 000000000..f96cfb128 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_32768/seed591489896.log @@ -0,0 +1,684 @@ +:::MLLOG {"namespace": "", "time_ms": 1784916346019, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784916346019, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784916369547, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784916369554, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784916369554, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784916369554, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784916369554, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784916369554, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784916369554, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784916369554, "event_type": "POINT_IN_TIME", "key": "seed", "value": 591489896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784916369554, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784916369554, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784916369554, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784916369554, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784916369574, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784916369574, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784917296632, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784917296632, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784917296999, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784917511961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13916730880737305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.1666666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784917527030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1390719711780548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.4999999999999995e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784917541111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13891232013702393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784917548258, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784917548259, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784917590838, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5022097826004028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784917590838, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784917590838, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784917597860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13898299634456635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3166666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917611828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13890250027179718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917626343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13879577815532684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9833333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917640713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13882529735565186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917640714, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784917640714, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784917662924, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5023571848869324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784917662924, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784917662924, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784917677367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13886931538581848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917691632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386762410402298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917706022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860508799552917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917713014, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784917713015, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784917735963, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.502550482749939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784917735963, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784917735963, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784917743247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385640799999237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917757575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13850364089012146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917772268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13854815065860748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917786710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13842178881168365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917786711, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784917786712, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784917809356, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5027703046798706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784917809357, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784917809357, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784917823675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13833904266357422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917838052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381373256444931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917852406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382337510585785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917859510, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784917859511, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784917882201, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5030468702316284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784917882201, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784917882202, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784917889196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13771390914916992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917903913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13776889443397522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917917881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13752096891403198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917932302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13749350607395172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917932303, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784917932303, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784917956428, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5033438801765442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784917956428, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784917956429, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784917970390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375231295824051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.316666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917984892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375395953655243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917999924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13705846667289734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918007170, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784918007170, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784918030184, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5036787986755371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784918030184, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784918030185, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784918037504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375596672296524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918052869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366693377494812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918067601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13694778084754944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918083043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13734754920005798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918083044, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784918083044, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784918106176, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5040976405143738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784918106177, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784918106177, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784918120814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13664434850215912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918136090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356474906206131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.983333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918150780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13680949807167053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918158122, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784918158123, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784918181718, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5045098066329956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784918181719, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784918181719, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784918189003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13398294150829315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918203935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352120190858841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918218190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13541993498802185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918232585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13567164540290833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918232650, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784918232651, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784918256556, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5049562454223633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784918256557, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784918256557, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784918271064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13435296714305878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918285624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13497979938983917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918300251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353917270898819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918308192, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784918308193, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784918331231, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5054174065589905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784918331232, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784918331232, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784918338316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13590773940086365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918352697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351233869791031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918367553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325761377811432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918382965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347222626209259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918382966, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784918382966, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784918405401, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5060067176818848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784918405402, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784918405402, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784918420152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13401328027248383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918435332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13529491424560547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918450936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13512276113033295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918458583, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784918458583, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784918481671, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5069608092308044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784918481672, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784918481672, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784918489035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13644595444202423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918504037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13353875279426575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918519247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343558132648468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918533836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13214752078056335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918533907, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784918533907, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784918557387, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.508042573928833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784918557388, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784918557388, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784918572022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132987380027771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918587497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312776803970337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918602030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331861913204193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918609322, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784918609322, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784918632823, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5092653632164001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784918632824, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784918632824, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784918640103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13826918601989746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918654858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13108892738819122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918670057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1299649178981781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918684827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13050684332847595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918684827, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784918684828, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784918708142, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5106553435325623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784918708143, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784918708143, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784918722908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13280485570430756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918737885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12974224984645844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918753316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1306486874818802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918760587, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784918760587, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784918783901, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5121439099311829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784918783902, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784918783902, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784918791270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13149522244930267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918805811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13512638211250305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918821007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331591159105301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918836756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13561254739761353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918836757, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784918836757, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784918859814, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5138207674026489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784918859815, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784918859815, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784918874687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135186105966568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918889652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13261359930038452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1649999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918904829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13313299417495728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918912305, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784918912305, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784918935337, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5155209898948669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784918935338, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784918935338, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784918943757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13638006150722504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918958948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13519078493118286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918974225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13622717559337616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918989866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364145129919052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918989866, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784918989867, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784919013414, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5172511339187622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784919013415, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784919013415, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784919029433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348608285188675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919044998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13085421919822693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919060383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13611820340156555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919068252, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784919068253, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784919091566, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5189443230628967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784919091567, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784919091567, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784919099261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13597983121871948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919115298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13242048025131226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919130280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325603574514389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919145155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13089323043823242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919145155, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784919145156, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784919167418, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5204777717590332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784919167419, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784919167419, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784919182475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13096502423286438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919198007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13813605904579163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919213126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294737607240677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919220616, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784919220617, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784919243437, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5223151445388794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784919243438, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784919243438, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784919250871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12932005524635315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919266101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134637251496315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919281457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13462108373641968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919297611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13032016158103943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919297611, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784919297612, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784919320416, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5245875120162964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784919320417, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784919320417, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784919335210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.128128781914711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919350110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317647099494934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919365154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13537120819091797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919372874, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784919372875, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784919396190, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5270369648933411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784919396190, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784919396191, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784919403809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365167200565338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919419507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13358516991138458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919434426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13195493817329407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919449239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13090765476226807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919449240, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784919449240, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784919472433, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5299233794212341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784919472434, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784919472434, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784919487521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12950873374938965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919502636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324438452720642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919518577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13522373139858246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919526227, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784919526228, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784919549298, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5334120392799377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784919549299, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784919549299, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784919556732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318831443786621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919571804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332702934741974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919586715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13485264778137207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919601909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344153732061386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919601910, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784919601910, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784919625603, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5373625755310059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784919625603, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784919625604, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784919640891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358959823846817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919656838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13387374579906464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919672429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331101357936859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3650000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919680051, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784919680052, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784919703528, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.54181307554245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784919703529, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784919703529, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784919711468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14008228480815887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919727244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13518056273460388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919743417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13429400324821472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4649999999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919759147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350385546684265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919759148, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784919759148, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784919782344, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5466248393058777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784919782345, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784919782345, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784919797702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351310759782791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919813132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312698900699615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.5649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919828622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13216888904571533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919836881, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784919836882, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784919860686, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5519147515296936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784919860687, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784919860687, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784919868437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322692334651947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919883703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13057881593704224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919898762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13558882474899292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919914241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321221888065338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919914241, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784919914242, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784919937972, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5582266449928284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784919937972, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784919937973, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784919953533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13432665169239044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919969416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12986387312412262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919984827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13163453340530396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919992328, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784919992328, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784920014804, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5661414861679077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784920014805, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784920014805, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784920022192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1319153606891632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920037468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13025036454200745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920052973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326451152563095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920068974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13215847313404083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920068975, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784920068975, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784920092223, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5756365656852722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784920092224, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784920092224, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784920107618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13151895999908447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920122819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12740473449230194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920137765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334744691848755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920145593, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784920145593, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784920168070, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5855426788330078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784920168071, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784920168071, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784920175884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13402381539344788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920191329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12617778778076172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920207247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12613697350025177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920222595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13202089071273804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920222595, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784920222596, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784920245140, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5954077243804932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784920245141, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784920245141, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784920260257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12844766676425934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920275282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12758806347846985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920290762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12691405415534973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920298270, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784920298271, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784920321209, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6034537553787231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784920321210, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784920321210, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784920328978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12926989793777466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920345022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12611024081707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920360463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289048194885254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920375749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12719744443893433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920375750, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784920375750, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784920398961, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6122161149978638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784920398961, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784920398962, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784920414113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12548762559890747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920429848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1285838931798935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4983333333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920445251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13126501441001892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920453002, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784920453002, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784920476420, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6197348237037659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784920476421, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784920476421, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784920484745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12970620393753052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920500631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1249862015247345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920516429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12976869940757751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920531928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12466605007648468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920531929, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784920531929, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784920555144, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6270090937614441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784920555144, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784920555144, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784920571009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12739863991737366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920586711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12871570885181427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920602973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12659290432929993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920610931, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784920610931, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784920633600, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6341928839683533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784920633600, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784920633600, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784920641444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12601496279239655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920656898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1261768341064453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920672675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12606967985630035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920688524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12542593479156494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.898333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920688524, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784920688524, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784920711448, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6398618817329407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784920711449, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784920711449, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784920727692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11918236315250397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920743344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11902447044849396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920758782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11699742078781128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920766504, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784920766504, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784920788868, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6426985263824463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784920788869, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784920788869, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784920796369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11934545636177063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920811872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11612818390130997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920827537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1207909807562828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920842945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11841994524002075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920842946, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784920842946, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784920866703, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6538046002388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784920866703, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784920866703, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784920882919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1199551448225975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920898007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11138162016868591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.198333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920913221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11733553558588028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920920703, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784920920703, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784920943664, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6651466488838196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784920943665, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784920943665, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784920951093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11793258041143417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920966526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11827854812145233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.298333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920981928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11251508444547653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920997029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11679022014141083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920997102, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784920997102, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784921021629, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6804744005203247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784921021629, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784921021630, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784921037091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11549985408782959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.398333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921052386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11576216667890549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921067823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087101474404335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921075431, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784921075431, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784921098336, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6998255252838135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784921098337, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784921098337, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784921105997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11272069811820984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.498333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921121589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11481953412294388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.531666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921137136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11294098198413849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921152923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11502503603696823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.598333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921152924, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784921152924, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784921176262, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7280968427658081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784921176262, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784921176262, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784921191569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11397753655910492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921206752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11320644617080688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921222020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11755678057670593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921229494, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784921229495, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784921252031, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7400047779083252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784921252032, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784921252032, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784921259569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10710929334163666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921275191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11489466577768326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921290625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10854531079530716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921306313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11345715820789337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921306314, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784921306314, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784921330087, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7513070106506348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784921330088, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784921330088, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784921345836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11650295555591583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921361171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11398591101169586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921376490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1118784099817276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921384022, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784921384023, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784921406910, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7560759782791138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784921406911, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784921406911, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784921414658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11220357567071915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921430191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11424423009157181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921446106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11386209726333618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921461380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10880579799413681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921461380, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784921461381, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784921485348, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7606480717658997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784921485349, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784921485349, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784921502029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11693304777145386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921517580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12085547298192978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921533425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11832462251186371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921541203, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784921541203, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784921563790, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7633547186851501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784921563791, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784921563791, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784921571474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11733946204185486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921587432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11697021126747131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921603340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11430242657661438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921619056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11469586193561554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921619057, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784921619057, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784921643140, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7650331854820251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784921643140, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784921643141, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784921659088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11975046992301941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921674702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11825934052467346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921690521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11169691383838654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921698459, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784921698459, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784921721675, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7662556171417236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784921721675, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784921721675, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784921729707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11698329448699951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921745768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11571358144283295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921761658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11207791417837143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921778067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1051856130361557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921778068, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784921778068, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784921800904, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.76688551902771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784921800905, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784921800905, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784921816047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723432898521423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921831600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10661903023719788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921846985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10676994919776917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921854748, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784921854749, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784921878789, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7676841616630554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784921878789, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784921878789, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784921886534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10464796423912048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921902403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11164618283510208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921918136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10827652364969254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921934425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11330695450305939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921934426, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784921934426, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784921958235, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7682861685752869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784921958236, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784921958236, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784921974080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10740956664085388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921989515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10716862976551056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922005258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10247275978326797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922012925, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784922012926, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784922036925, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7686084508895874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784922036926, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784922036926, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784922044852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10318586975336075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922060722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10197687149047852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922076942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1120435893535614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922092568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10458795726299286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922092568, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784922092569, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784922116526, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7694817185401917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784922116527, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784922116527, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784922132611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10648127645254135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922148310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10648699849843979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922163606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10718554258346558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922171549, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784922171550, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784922194568, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7698918581008911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784922194569, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784922194569, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784922202506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10743145644664764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922217847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09977541863918304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922233643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10758914053440094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922249562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10467082262039185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922249563, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784922249563, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784922273194, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.770348310470581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784922273194, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784922273195, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784922288576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10784374177455902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922304967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10868702828884125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922320995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10482455790042877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922328967, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784922328967, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784922351972, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7705785036087036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784922351972, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784922351972, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784922359745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10937333106994629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922375684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10798265039920807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922391419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11162185668945312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922407624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10797341167926788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922407625, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784922407625, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784922431158, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7710275053977966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784922431159, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784922431159, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784922447372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10532531142234802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922463044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10217393934726715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922479407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10852251946926117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.564999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922487466, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784922487466, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784922511187, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7711264491081238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784922511188, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784922511188, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784922518899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1045834869146347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922534240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09962071478366852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922550022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11424989998340607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.664999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922565595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11095310747623444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922565595, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784922565596, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784922588765, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7716414332389832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784922588766, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784922588766, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784922604546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1064886748790741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922620694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1089085042476654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922636527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082424521446228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922644308, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784922644309, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784922668061, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7717027068138123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784922668061, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784922668062, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784922676900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11023920029401779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922693126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11484254151582718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922709257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11245729774236679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922725401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11330684274435043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922725402, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784922725402, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784922748761, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7717416882514954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784922748762, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784922748762, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784922764628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1101658046245575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922780989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11392068862915039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.998333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922797257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11078357696533203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922805269, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784922805269, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784922829705, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7722246050834656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784922829706, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784922829706, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784922837711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11092657595872879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922854683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10943704843521118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922870658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11303018033504486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.131666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922886508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1114925816655159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922886508, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784922886509, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784922909796, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7726415395736694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784922909797, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784922909797, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784922925858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11046390980482101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922942252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11298200488090515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922958641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11685152351856232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922966783, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784922966784, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784922990622, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7726582884788513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784922990623, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784922990623, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784922998528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11480943858623505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923015053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10787256807088852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.331666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923030947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10150174051523209, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923046617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10727818310260773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923046617, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784923046618, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784923069605, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.772840678691864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784923069606, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784923069606, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784923085414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10539046674966812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923100951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10660900175571442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923117081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10558316111564636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923125172, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784923125173, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784923148507, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7731557488441467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784923148508, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784923148508, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784923156551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10858536511659622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923172266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10326175391674042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923188711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10502087324857712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923204582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1075931042432785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.631666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923204583, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784923204583, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784923227787, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7734609842300415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784923227788, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784923227788, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784923243143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10066185146570206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923259008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10379580408334732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923274652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10350850224494934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923282421, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784923282422, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784923305199, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7736172676086426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784923305200, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784923305200, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784923313146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10430043935775757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923329400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10306946188211441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923345130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.114463210105896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923361524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10337746143341064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923361525, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784923361525, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784923385171, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7737131118774414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784923385172, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784923385172, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784923401100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10762783139944077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923416809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10760640352964401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923432486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10177191346883774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923440384, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784923440384, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784923463979, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738653421401978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784923463979, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784923463980, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784923471770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10107596218585968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923487535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10081405937671661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923503609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10654085874557495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923519853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10296954959630966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923519853, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784923519854, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784923542992, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742135524749756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784923542993, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784923542993, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784923558562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1078573539853096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923575175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10387252271175385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.164999999999999e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_32768/seed595408662.log b/recommendation_v4/rcp_logs/gbs_32768/seed595408662.log new file mode 100644 index 000000000..81fa220a5 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_32768/seed595408662.log @@ -0,0 +1,641 @@ +:::MLLOG {"namespace": "", "time_ms": 1784923823966, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784923823966, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784923852552, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784923852555, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784923852555, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784923852555, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784923852556, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784923852556, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784923852556, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784923852556, "event_type": "POINT_IN_TIME", "key": "seed", "value": 595408662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784923852556, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784923852556, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784923852556, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784923852556, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784923852610, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784923852611, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784924774858, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784924774859, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784924775192, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784924948773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13836678862571716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.1666666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784924965581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383856236934662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.4999999999999995e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784924981566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13858231902122498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784924989482, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784924989483, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784925013821, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.500414252281189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784925013821, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784925013821, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784925021697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13837924599647522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3166666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925036557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383790671825409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925051930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384791135787964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9833333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925067015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383552998304367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925067015, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784925067016, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784925090444, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5005645751953125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784925090445, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784925090445, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784925105496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13796211779117584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925120489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13814091682434082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925135875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13802935183048248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925143241, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784925143241, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784925167377, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5007614493370056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784925167377, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784925167377, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784925175382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13828180730342865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925190514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13812199234962463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925206027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13821853697299957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925221352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13781917095184326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925221353, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784925221353, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784925247851, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5009873509407043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784925247851, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784925247852, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784925263086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13814330101013184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925278386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374904066324234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925297149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13781976699829102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925304640, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784925304641, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784925330423, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5012518763542175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784925330423, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784925330423, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784925337933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370255947113037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925353590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13720890879631042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925368733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13686257600784302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925384277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13694484531879425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925384278, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784925384278, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784925411601, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5015411376953125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784925411601, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784925411602, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784925426679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370881199836731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.316666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925446704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13706547021865845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925462799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13625629246234894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925470683, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784925470683, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784925495725, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5018681287765503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784925495726, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784925495726, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784925503589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13717231154441833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925519889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13622015714645386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925535547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13658837974071503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925551758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13698041439056396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925551759, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784925551760, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784925577871, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5022623538970947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784925577872, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784925577872, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784925593559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361859142780304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925609566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349833756685257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.983333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925625364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13668978214263916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925633055, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784925633056, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784925656880, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5026387572288513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784925656881, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784925656881, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784925665214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13316628336906433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925683266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13465803861618042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925698826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350046694278717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925714382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352211982011795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925714454, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784925714455, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784925742571, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5030246376991272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784925742572, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784925742572, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784925758028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13388320803642273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925773547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346368044614792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925789064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13521713018417358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925797519, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784925797520, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784925825159, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5034403800964355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784925825160, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784925825160, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784925833063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13576149940490723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925848944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13490794599056244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925864921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132160484790802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925881414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345817744731903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925881414, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784925881415, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784925908207, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5039543509483337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784925908207, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784925908207, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784925924389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13391070067882538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925940477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352497935295105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925957102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13489317893981934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925965427, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784925965428, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784925991891, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5048646926879883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784925991892, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784925991893, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784925999807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13636374473571777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926015948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13354897499084473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926032637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13435180485248566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926048605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321246474981308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926048679, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784926048679, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784926074317, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5057858228683472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784926074318, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784926074318, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784926090081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13293634355068207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926106798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13128437101840973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926123070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330903172492981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926131018, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784926131018, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784926157606, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5067757368087769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784926157607, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784926157607, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784926165533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13841784000396729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926181419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13113293051719666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926197864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12992313504219055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926213905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13045911490917206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926213906, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784926213906, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784926240722, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.507960319519043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784926240722, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784926240723, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784926256483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13281035423278809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926272291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1299067735671997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926289133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13069604337215424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926297296, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784926297297, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784926323957, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5093139410018921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784926323957, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784926323958, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784926332013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13169217109680176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926347868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13508354127407074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926363999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13315698504447937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926380628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13564418256282806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926380629, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784926380629, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784926406764, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5107851624488831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784926406765, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784926406765, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784926423017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13530486822128296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926438917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325453221797943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1649999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926454815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13323268294334412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926462762, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784926462762, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784926486098, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5122156143188477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784926486098, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784926486099, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784926494950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13639840483665466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926511856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13528470695018768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926528422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362430453300476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926545030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364486813545227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926545030, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784926545030, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784926569620, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5136656165122986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784926569621, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784926569621, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784926586392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13486212491989136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926603371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308765411376953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926619410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13623124361038208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926627737, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784926627737, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784926652818, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5151557922363281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784926652818, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784926652819, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784926660775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13591039180755615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926677465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13258016109466553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926693870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13252952694892883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926710151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13077950477600098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926710151, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784926710152, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784926736824, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5165654420852661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784926736825, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784926736825, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784926752965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13108789920806885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926769347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13819342851638794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926785760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1296387016773224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926793851, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784926793851, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784926819857, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5182176828384399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784926819857, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784926819858, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784926827832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12944668531417847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926844426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347389966249466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926860577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13460499048233032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926877459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304914504289627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926877460, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784926877460, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784926904013, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5202037692070007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784926904014, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784926904014, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784926920454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12826383113861084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926936607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13192737102508545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926953000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13562089204788208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926961144, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784926961145, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784926985666, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5223100781440735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784926985667, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784926985667, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784926993803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13665977120399475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927010504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13360287249088287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927027333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13207198679447174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927043612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310240477323532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927043612, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784927043613, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784927070538, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5247458219528198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784927070539, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784927070539, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784927086956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12958255410194397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927103244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13245639204978943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927121579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13541290163993835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927129969, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784927129970, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784927156923, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5276268720626831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784927156924, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784927156924, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784927165260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13188722729682922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927181975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334594488143921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927198493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13510122895240784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927214850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13468529284000397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927214851, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784927214851, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784927240358, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5308718681335449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784927240359, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784927240359, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784927256765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13602852821350098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927274324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13398832082748413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927291228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331438422203064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3650000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927299709, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784927299710, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784927326780, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5344003438949585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784927326781, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784927326781, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784927335475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14049780368804932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927352245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13521912693977356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927369157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13446122407913208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4649999999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927386448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13522741198539734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927386449, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784927386449, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784927413211, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5380398631095886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784927413211, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784927413211, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784927429971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13533861935138702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927446951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13333868980407715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.5649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927463458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13237205147743225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927472307, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784927472308, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784927499324, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5421988368034363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784927499325, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784927499325, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784927507687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13251972198486328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927524524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308952271938324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927540966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13599780201911926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927557630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323758214712143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927557630, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784927557631, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784927582202, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5473196506500244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784927582203, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784927582203, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784927598855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13462544977664948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927616186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13030727207660675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927632926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13187244534492493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927641274, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784927641275, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784927667758, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5539206266403198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784927667758, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784927667758, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784927676144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13235612213611603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927692704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13075610995292664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927709473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331050843000412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927726631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1328439563512802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927726632, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784927726632, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784927753949, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5627309083938599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784927753949, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784927753950, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784927770534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13196346163749695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927787068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12833121418952942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927803632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13418219983577728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927811877, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784927811877, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784927838140, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5725064277648926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784927838140, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784927838141, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784927846570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13494664430618286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927862957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12690362334251404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927884063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12712113559246063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927901031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13315246999263763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927901032, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784927901033, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784927926902, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5841225385665894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784927926902, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784927926902, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784927943419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294320821762085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927960072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12891286611557007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927977241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1281222701072693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927985244, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784927985244, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784928010476, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5945836901664734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784928010476, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784928010476, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784928018800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1302192062139511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928035906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12781557440757751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928052754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13022108376026154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928069264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12870031595230103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928069265, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784928069265, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784928095751, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6063676476478577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784928095751, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784928095751, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784928112326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1270788460969925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928129367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13009892404079437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4983333333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928145799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13331377506256104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928154060, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784928154061, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784928178286, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6173183917999268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784928178286, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784928178286, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784928187129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317720115184784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928204417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12695279717445374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928221642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13147717714309692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928238845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12690041959285736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928238846, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784928238846, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784928265610, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6277042627334595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784928265610, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784928265611, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784928282830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12935173511505127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928299543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13035161793231964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928316641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12832790613174438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928325434, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784928325435, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784928351644, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6385121941566467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784928351645, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784928351645, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784928360282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12764140963554382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928377167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12777121365070343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928394324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12847357988357544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928411108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12736982107162476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.898333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928411109, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784928411109, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784928437194, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6502321362495422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784928437195, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784928437195, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784928454293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12175190448760986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928471350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1209639385342598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928487891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11934148520231247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928496416, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784928496417, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784928522485, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6597985029220581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784928522486, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784928522486, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784928530758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12114790827035904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928547546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11810548603534698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928564275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12251409888267517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928580585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11966823786497116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928580586, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784928580586, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784928604821, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.679889440536499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784928604821, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784928604821, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784928622108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12079611420631409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928638965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11219915747642517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.198333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928655886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11792543530464172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928664160, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784928664161, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784928691170, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6984253525733948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784928691170, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784928691170, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784928699353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11854451894760132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928716564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11841905117034912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.298333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928733331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11319189518690109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928749698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11693151295185089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928749767, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784928749768, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784928777787, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.714821457862854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784928777788, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784928777788, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784928794439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11549095809459686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.398333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928811035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11547963321208954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928828050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10885325819253922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928836254, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784928836255, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784928862484, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7276089787483215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784928862485, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784928862485, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784928870730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11238370090723038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.498333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928887946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11412002891302109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.531666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928904647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.112364761531353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928921285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11443521082401276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.598333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928921286, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784928921286, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784928948806, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7318027019500732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784928948806, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784928948806, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784928965713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11352169513702393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928982326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11253584176301956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928999379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11623548716306686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929007510, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784929007510, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784929035160, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7387058734893799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784929035160, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784929035160, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784929043546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10595771670341492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929060567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11403662711381912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929077297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10755603015422821, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929093919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11216030269861221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929093920, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784929093920, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784929121537, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7406481504440308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784929121538, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784929121538, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784929138451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11508982628583908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929155074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11191394925117493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929171999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11068151891231537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929179883, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784929179883, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784929205836, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7440747618675232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784929205837, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784929205837, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784929214399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11080348491668701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929231286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11234578490257263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929248565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11279229819774628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929264614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10647105425596237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929264614, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784929264615, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784929289220, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7441914677619934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784929289220, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784929289221, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784929307340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11524942517280579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929324880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11904559284448624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929342297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11695726960897446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929350979, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784929350980, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784929377777, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7444848418235779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784929377777, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784929377778, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784929386229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1151493564248085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929403834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11530593782663345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929420750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11246289312839508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929437475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11369481682777405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929437476, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784929437477, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784929464601, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7453886270523071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784929464601, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784929464601, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784929481810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11815905570983887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929498933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11709735542535782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929516551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11120513826608658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929525059, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784929525059, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784929552552, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7482175230979919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784929552552, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784929552553, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784929561136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11552959680557251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929578522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11557095497846603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929595382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11111430823802948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929614103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10430765151977539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929614104, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784929614104, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784929640554, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7511080503463745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784929640554, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784929640554, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784929657331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10667753219604492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929674544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10543129593133926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929691425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10606537759304047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929699985, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784929699986, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784929727193, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7507250308990479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784929727193, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784929727194, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784929735750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10394252091646194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929753074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11155785620212555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929770017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10755014419555664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929787270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11337989568710327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929787271, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784929787271, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784929815178, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7519996762275696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784929815179, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784929815179, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784929832403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10704624652862549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929849114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10643665492534637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929866268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10208968073129654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929874373, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784929874374, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784929901089, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7519577741622925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784929901089, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784929901090, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784929909761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1028270274400711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929926782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10181627422571182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929944300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11211200058460236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929960698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10398860275745392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929960699, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784929960699, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784929985102, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7544057369232178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784929985103, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784929985103, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784930002446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10638546943664551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930020041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10623624920845032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930036877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10679753124713898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930045663, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784930045664, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784930071822, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7568017244338989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784930071822, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784930071822, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784930080293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1073092669248581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930097492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0996108129620552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930114666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10795137286186218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930132071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.104324571788311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930132072, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784930132072, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784930156624, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7576603889465332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784930156624, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784930156625, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784930173207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10774759203195572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930191095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10858386754989624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930208526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10420484095811844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930217029, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784930217030, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784930244178, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7562484741210938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784930244179, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784930244179, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784930252594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10911398380994797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930269826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084250956773758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930286971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11192867159843445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930304210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10804858058691025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930304210, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784930304211, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784930331038, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7573281526565552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784930331039, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784930331039, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784930348389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10581943392753601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930364974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10206581652164459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930382601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10840746760368347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.564999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930391490, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784930391491, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784930417895, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7596796154975891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784930417895, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784930417896, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784930426455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1048819050192833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930443317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09950181841850281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930460427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1146320179104805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.664999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930477681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11132492125034332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930477681, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784930477682, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784930504688, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7592162489891052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784930504689, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784930504689, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784930521743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10622525215148926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930539269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10866959393024445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930556302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10830457508563995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930564539, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784930564539, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784930589210, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7590803503990173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784930589210, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784930589211, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784930598309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11057723313570023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930616184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11478857696056366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930633830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11242997646331787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930651570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11310674250125885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930651571, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784930651571, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784930678254, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.759053647518158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784930678255, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784930678255, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784930695786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11011617630720139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930713628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11415384709835052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.998333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930731412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11109878122806549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930739923, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784930739924, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784930764997, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7590778470039368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784930764997, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784930764997, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784930773486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11090514063835144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930791683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10958805680274963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930809308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.112897127866745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.131666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930826782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11132475733757019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930826783, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784930826783, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784930853179, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7617993950843811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784930853180, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784930853180, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784930870660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11037082225084305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930888688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11312908679246902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930906142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11708196997642517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930914951, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784930914952, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784930940199, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7628907561302185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784930940199, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784930940199, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784930948542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11495848000049591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930966559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10766910761594772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.331666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930983686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10162195563316345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931001262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10715072602033615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931001263, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784931001263, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784931027788, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7634513974189758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784931027788, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784931027788, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784931044838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10494755953550339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931062032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10684455931186676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931079628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10541891306638718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931088436, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784931088437, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784931115308, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7635741829872131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784931115308, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784931115309, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784931123798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10853531956672668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931140772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10330960899591446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_32768/seed72228511.log b/recommendation_v4/rcp_logs/gbs_32768/seed72228511.log new file mode 100644 index 000000000..2c66b1cc9 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_32768/seed72228511.log @@ -0,0 +1,684 @@ +:::MLLOG {"namespace": "", "time_ms": 1784916345392, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784916345392, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784916368526, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784916368529, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784916368529, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784916368529, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784916368529, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784916368529, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784916368529, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784916368529, "event_type": "POINT_IN_TIME", "key": "seed", "value": 72228511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784916368530, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784916368530, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784916368530, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784916368530, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784916368566, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784916368566, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784917330598, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784917330598, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784917330921, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784917476758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13772447407245636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.1666666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784917491789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13826638460159302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.4999999999999995e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784917505944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138070210814476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784917513105, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784917513106, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784917556678, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4989660978317261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784917556678, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784917556679, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784917563805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137813538312912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3166666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917577861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13778658211231232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917592515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13798849284648895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9833333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917607002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13789497315883636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917607003, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784917607003, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784917629223, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4991390109062195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784917629224, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784917629224, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784917643705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13712993264198303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917658160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374618411064148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917672792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13732630014419556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917679847, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784917679848, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784917702530, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4993532598018646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784917702530, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784917702530, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784917709928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13775362074375153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917724565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13773253560066223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917739455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13780082762241364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917754227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13713935017585754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917754227, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784917754228, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784917776712, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4996105134487152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784917776713, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784917776713, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784917791280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13780613243579865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917805702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367093175649643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917820501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13741976022720337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917827756, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784917827757, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784917850474, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49990981817245483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784917850475, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784917850475, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784917857553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13609376549720764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917872479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13644516468048096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917886697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360141634941101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917901328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13613948225975037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917901329, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784917901329, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784917925376, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5002381205558777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784917925377, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784917925377, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784917939506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13656243681907654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.316666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917954054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13644900918006897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917969359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356075406074524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917976691, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784917976691, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784917999324, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5005657076835632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784917999325, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784917999325, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784918006715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13670508563518524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918022290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13542068004608154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918037169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13607509434223175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918052809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13670150935649872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918052810, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784918052810, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784918076149, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5010197758674622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784918076150, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784918076150, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784918090988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13572272658348083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918106509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13404619693756104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.983333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918121333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13622118532657623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918128684, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784918128685, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784918151660, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.501443088054657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784918151661, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784918151661, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784918158955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13195976614952087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918173897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13389018177986145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918188383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13442328572273254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918202868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134687602519989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918202938, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784918202938, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784918226872, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.501886248588562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784918226873, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784918226873, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784918241353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312333822250366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918255906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13417838513851166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918270579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13487321138381958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918278633, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784918278634, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784918300697, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5023907423019409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784918300698, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784918300698, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784918307864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355273425579071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918322300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13460753858089447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918337111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13135652244091034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918352684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13436147570610046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918352685, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784918352685, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784918374790, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5031273365020752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784918374790, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784918374791, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784918389672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13359442353248596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918405004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13528510928153992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918420757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13496118783950806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918428340, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784918428341, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784918451422, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5042551159858704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784918451423, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784918451423, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784918458857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13650265336036682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918473963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133332759141922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918489418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13431353867053986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918504364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13191646337509155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918504426, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784918504427, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784918527175, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5053201913833618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784918527176, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784918527176, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784918541930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327648162841797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918557673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311759650707245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918572439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330914944410324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918579964, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784918579964, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784918602349, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5064115524291992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784918602349, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784918602349, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784918609704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13836437463760376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918624675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13091865181922913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918639921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12979471683502197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918654472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13035453855991364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918654473, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784918654473, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784918677126, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5076655149459839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784918677128, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784918677128, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784918691845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13279159367084503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918706789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12981469929218292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918722352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13064764440059662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918729710, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784918729710, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784918752533, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5090273022651672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784918752534, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784918752534, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784918760063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13158594071865082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918774854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13513100147247314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918790209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13307851552963257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918805734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13560907542705536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918805735, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784918805735, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784918828956, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5104570388793945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784918828957, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784918828957, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784918843816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13531525433063507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918858579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325646936893463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1649999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918873595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13322141766548157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918881054, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784918881055, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784918904018, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5119597315788269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784918904019, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784918904019, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784918912366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13642945885658264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918927862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13524779677391052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918943282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13627271354198456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918958987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13638947904109955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918958988, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784918958988, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784918982063, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5134583115577698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784918982063, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784918982064, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784918998168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13469909131526947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919013478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308538019657135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919028636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363488882780075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919036438, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784919036438, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784919059590, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5149929523468018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784919059591, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784919059591, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784919067213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13597942888736725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919082982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132534459233284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919098170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13262024521827698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919113213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1309601366519928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919113213, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784919113214, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784919135932, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5164604187011719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784919135932, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784919135933, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784919151239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310616135597229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919166943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13828691840171814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919181802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12954318523406982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919189240, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784919189240, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784919211832, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5181271433830261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784919211832, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784919211832, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784919219120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12932106852531433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919233987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13465552031993866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919249066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13467355072498322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919265491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13043341040611267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919265492, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784919265493, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784919288129, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5201603174209595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784919288130, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784919288130, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784919303359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12808364629745483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919318813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13188719749450684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919334553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354047656059265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919342421, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784919342421, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784919365243, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5223453640937805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784919365243, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784919365244, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784919372967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13664932548999786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919388846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13352487981319427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919403813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320524513721466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919418710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13098083436489105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919418710, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784919418711, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784919441785, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5248721837997437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784919441786, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784919441786, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784919456821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12956860661506653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919471874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325283646583557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919487826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354128122329712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919495664, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784919495665, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784919518508, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5279187560081482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784919518509, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784919518509, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784919526154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13204112648963928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919541820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13342513144016266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919557178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13496290147304535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919572601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13464869558811188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919572601, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784919572602, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784919595894, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5313082337379456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784919595895, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784919595895, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784919611427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13605374097824097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919627289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1339590847492218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919642880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331298053264618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3650000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919650560, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784919650560, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784919673510, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5350656509399414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784919673511, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784919673511, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784919681576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14033368229866028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919697465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13530099391937256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919713609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134381502866745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4649999999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919729484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352352350950241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919729485, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784919729485, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784919752960, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5390918850898743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784919752960, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784919752960, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784919768398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13521337509155273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919783841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13314424455165863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.5649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919799271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13234010338783264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919807549, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784919807550, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784919830679, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5435341000556946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784919830679, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784919830679, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784919838388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13251668214797974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919853650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13090069591999054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919868707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359204351902008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919884172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13231590390205383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919884173, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784919884173, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784919907268, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5489684343338013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784919907268, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784919907269, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784919922746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346425712108612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919938457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301657259464264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919953737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13192710280418396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919961340, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784919961340, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784919984793, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5560663342475891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784919984794, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784919984794, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784919992182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13234712183475494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920007441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13055795431137085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920023028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13311515748500824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920039002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13258416950702667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920039003, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784920039003, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784920062206, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5655450820922852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784920062206, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784920062207, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784920077557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318531483411789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920092917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1280774474143982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920108075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13383027911186218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920115992, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784920115993, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784920138320, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5758188366889954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784920138321, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784920138321, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784920146135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13457328081130981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920161628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12660306692123413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920177661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12676239013671875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920193337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13279688358306885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920193338, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784920193338, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784920216248, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5875216722488403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784920216249, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784920216249, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784920231533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12916843593120575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920246889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1283700168132782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920262610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12777233123779297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920270244, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784920270245, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784920293064, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5971158146858215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784920293065, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784920293065, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784920300855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12987399101257324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920316709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12747779488563538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920332061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12991659343242645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920347329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1281830370426178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920347329, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784920347330, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784920370726, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6084883809089661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784920370727, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784920370727, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784920385732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1265394687652588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920401166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12956662476062775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4983333333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920416429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13265728950500488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920424064, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784920424065, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784920447766, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6188040971755981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784920447766, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784920447767, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784920456151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13095517456531525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920472321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12618733942508698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920488288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13087955117225647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920504103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12614627182483673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920504104, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784920504104, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784920527530, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6290419697761536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784920527531, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784920527531, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784920543695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12853315472602844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920559521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12974947690963745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920576008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12750142812728882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920583941, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784920583941, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784920606886, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6402062773704529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784920606887, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784920606887, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784920614609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1267077624797821, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920629993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12682293355464935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920645732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12737533450126648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920661415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12629444897174835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.898333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920661416, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784920661416, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784920684717, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6512894034385681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784920684718, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784920684718, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784920701093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12047374993562698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920716833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11978594213724136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920732188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11819213628768921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920739861, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784920739862, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784920762836, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6586418151855469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784920762836, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784920762836, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784920770412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12010522931814194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920785939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11710557341575623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920801770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1214291900396347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920817263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11884412169456482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920817263, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784920817264, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784920841357, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6771609783172607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784920841358, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784920841358, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784920857541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1201605275273323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920872761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1113944798707962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.198333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920888239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11734864860773087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920895876, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784920895876, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784920918471, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.697370171546936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784920918472, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784920918472, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784920926064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11801479756832123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920941743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11807899922132492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.298333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920957334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1131637841463089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920972628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11681164801120758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920972709, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784920972709, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784920997764, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7185215950012207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784920997764, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784920997765, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784921013951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11565908789634705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.398333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921029604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1153610423207283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921045367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10894449055194855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921053159, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784921053160, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784921076292, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7351769208908081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784921076292, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784921076293, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784921084080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11309948563575745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.498333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921099970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1140439510345459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.531666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921115868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11248067021369934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921131912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11430442333221436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.598333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921131913, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784921131913, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784921155453, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7477874755859375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784921155453, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784921155454, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784921170975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1131657138466835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921186262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11210916936397552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921201679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1165657564997673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921209267, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784921209267, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784921232642, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7576781511306763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784921232643, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784921232643, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784921240204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10571135580539703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921255921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11346253007650375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921271496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10686680674552917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921287515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11070648580789566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921287515, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784921287516, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784921312263, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7630941271781921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784921312264, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784921312264, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784921328343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11411026120185852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921344238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11047783493995667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921360068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10946878790855408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921367814, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784921367815, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784921390422, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7656477093696594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784921390422, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784921390423, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784921398342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10968473553657532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921414386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11102154850959778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921430689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11187690496444702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921446300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.105411097407341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921446300, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784921446301, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784921470592, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.765610933303833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784921470593, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784921470593, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784921487685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11434149742126465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921503738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11860518902540207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921519728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11523887515068054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921527627, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784921527628, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784921550967, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7662562131881714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784921550968, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784921550968, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784921558854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11470872163772583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921574958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1142851859331131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921590952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11175619810819626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921606704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11299964040517807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921606704, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784921606704, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784921631047, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7669826745986938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784921631048, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784921631048, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784921647194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11742375791072845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921663019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11694329977035522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921679285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11066903173923492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921687358, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784921687358, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784921710893, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7671718597412109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784921710894, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784921710894, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784921718918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11492278426885605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921735171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1154143288731575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921751254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11092615127563477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921767687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10386583209037781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921767687, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784921767687, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784921791880, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7686858773231506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784921791880, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784921791881, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784921807168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1063108891248703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921822786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10560918599367142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921838214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1058741882443428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921845901, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784921845902, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784921869590, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7689475417137146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784921869590, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784921869590, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784921877332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10374341160058975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921893181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11123684048652649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921908943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10735724121332169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921925070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1124037355184555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921925070, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784921925071, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784921948098, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7696838974952698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784921948099, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784921948099, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784921963870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10664421319961548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921979309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10608026385307312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921994962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10134241729974747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922002551, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784922002552, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784922025648, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.77006596326828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784922025649, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784922025649, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784922033563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1025160402059555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922049359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10120078921318054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922065470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11140459775924683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922081055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10363482683897018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922081056, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784922081056, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784922105242, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7706637382507324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784922105243, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784922105243, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784922121288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1060083732008934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922137130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10581669211387634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922152387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10647604614496231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922160466, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784922160466, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784922183848, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7712202072143555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784922183849, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784922183850, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784922191714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10683467984199524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922207067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09936770051717758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922222723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10763772577047348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922238472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10391654819250107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922238473, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784922238473, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784922261690, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.771274745464325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784922261691, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784922261691, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784922276986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10760411620140076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922293081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10803252458572388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922308922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1041300967335701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922316719, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784922316720, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784922339186, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7711797952651978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784922339187, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784922339187, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784922347000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10863196849822998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922363027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10730031877756119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922378795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11169744282960892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922394984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10815933346748352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922394984, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784922394985, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784922418283, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7718651294708252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784922418284, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784922418284, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784922434371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10496190190315247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922450061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.101616270840168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922466533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081785187125206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.564999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922474571, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784922474571, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784922497411, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7720791697502136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784922497412, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784922497412, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784922505204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10424646735191345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922520805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09959331899881363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922536713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1143416315317154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.664999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922552379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11066892743110657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922552380, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784922552380, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784922575153, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.772216260433197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784922575154, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784922575154, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784922590843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10598835349082947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922606933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084340512752533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922622829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10810564458370209, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922630629, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784922630629, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784922654062, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7722917795181274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784922654063, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784922654063, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784922662836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10954524576663971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922679249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1145743802189827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922695633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11200740933418274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922712017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11263970285654068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922712018, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784922712018, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784922735401, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7720116376876831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784922735402, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784922735402, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784922751541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10977865755558014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922768260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11354490369558334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.998333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922784831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11046762764453888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922793001, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784922793002, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784922816168, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7723965644836426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784922816168, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784922816168, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784922824232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11040705442428589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922841268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10952471941709518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922857450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1124858558177948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.131666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922873660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1108100414276123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922873661, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784922873661, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784922896654, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7727031707763672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784922896654, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784922896654, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784922912937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10993266850709915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922929404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11295507103204727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922945794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1169310137629509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922953891, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784922953892, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784922978128, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7727841138839722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784922978129, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784922978129, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784922986012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11471153050661087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923002883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10759150981903076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.331666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923018915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10122618079185486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923034864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1067088171839714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923034865, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784923034865, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784923058955, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7729902863502502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784923058956, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784923058956, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784923075046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10469332337379456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923091001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10629673302173615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923107361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10516456514596939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923115577, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784923115577, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784923139323, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7731150388717651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784923139324, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784923139324, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784923147351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10846081376075745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923163159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1033557653427124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923179803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10478110611438751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923195772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10726742446422577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.631666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923195773, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784923195773, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784923218929, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7732696533203125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784923218930, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784923218930, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784923234429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10038822889328003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923250541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10386776924133301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923266438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10296221822500229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923274390, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784923274390, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784923298171, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7734071016311646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784923298172, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784923298172, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784923306160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10381785035133362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923322565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1030702069401741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923338400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11330454796552658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923354830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10312199592590332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923354831, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784923354831, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784923378787, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7734457850456238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784923378788, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784923378788, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784923394912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10719934105873108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923411033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10701557993888855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923427171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10145726054906845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923435189, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784923435189, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784923458785, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7734838128089905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784923458786, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784923458786, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784923466933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10137020796537399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923483201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1004648357629776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923499703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.105995774269104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923516371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10333159565925598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923516371, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784923516372, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784923540528, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7736527323722839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784923540528, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784923540528, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784923556463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10731489211320877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923573340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10396728664636612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.164999999999999e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_32768/seed764588808.log b/recommendation_v4/rcp_logs/gbs_32768/seed764588808.log new file mode 100644 index 000000000..998fcffce --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_32768/seed764588808.log @@ -0,0 +1,675 @@ +:::MLLOG {"namespace": "", "time_ms": 1784909144545, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784909144545, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784909167735, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784909167741, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784909167741, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784909167741, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784909167741, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784909167741, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784909167741, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784909167742, "event_type": "POINT_IN_TIME", "key": "seed", "value": 764588808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784909167742, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784909167742, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784909167742, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784909167742, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784909167773, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784909167774, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784910117646, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784910117647, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784910117953, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784910312979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13820330798625946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.1666666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784910328476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382463276386261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.4999999999999995e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784910343211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13836143910884857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784910350642, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784910350643, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784910394067, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5034065842628479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784910394067, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784910394067, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784910401281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13823899626731873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3166666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910415729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13827399909496307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910430695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13836318254470825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9833333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910445523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13829629123210907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910445523, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784910445523, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784910467889, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5035640597343445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784910467889, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784910467889, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784910482903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13793504238128662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910497819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13802890479564667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910512847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13786938786506653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910520143, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784910520143, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784910542900, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5037497878074646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784910542900, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784910542900, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784910550474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13818907737731934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910565491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13805946707725525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910580739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13805735111236572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910595922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13759109377861023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910595923, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784910595923, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784910618535, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5039834976196289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784910618536, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784910618536, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784910633381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380254626274109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910648203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13731259107589722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910663051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13764618337154388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910670413, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784910670414, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784910693253, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5042589902877808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784910693254, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784910693254, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784910700427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13674083352088928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910715605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13695120811462402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910730017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13664567470550537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910745068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366850882768631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910745069, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784910745070, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784910768113, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5045554637908936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784910768114, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784910768114, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784910782627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13692858815193176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.316666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910797576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13685627281665802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910813197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13607747852802277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910820596, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784910820597, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784910843719, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5048900842666626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784910843719, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784910843719, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784910851158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1369481086730957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910867064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359652578830719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910882190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13634032011032104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910898092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368880271911621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910898093, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784910898093, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784910920394, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5053009390830994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784910920394, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784910920394, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784910935279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13586190342903137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910950820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13455606997013092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.983333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910965845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13633683323860168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784910973424, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784910973425, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784910996471, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5057012438774109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784910996472, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784910996472, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784911003976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325257271528244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911019335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13426047563552856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911033847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13469113409519196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911048457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13499926030635834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911048537, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784911048538, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784911072212, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5060967803001404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784911072213, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784911072213, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784911086940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13345035910606384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911101736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1342620998620987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911116625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350092887878418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911124699, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784911124700, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784911147932, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5065435171127319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784911147933, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784911147933, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784911155075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13563615083694458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911169532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346539855003357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911184406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13161532580852509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911199899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13440628349781036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911199900, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784911199900, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784911221919, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5071179866790771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784911221920, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784911221920, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784911236919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13367921113967896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911252188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13515013456344604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911267912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13489103317260742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911275530, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784911275531, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784911297677, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5080927014350891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784911297677, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784911297677, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784911305096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13639843463897705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911320213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13339300453662872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911335738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13423292338848114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911350650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13201023638248444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911350712, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784911350712, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784911373278, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5090498924255371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784911373279, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784911373279, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784911388164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13278095424175262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911403841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13120678067207336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911418819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13313302397727966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911426402, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784911426402, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784911448535, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5100327134132385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784911448535, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784911448535, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784911456000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13850805163383484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911471255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310124546289444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911486798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12981727719306946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911501607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303306519985199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911501608, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784911501609, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784911524400, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5112288594245911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784911524401, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784911524401, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784911539309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13276566565036774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911554481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297786682844162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911570384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13064062595367432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911577921, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784911577921, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784911600441, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5124815702438354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784911600441, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784911600442, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784911608087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13150522112846375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911623105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13520218431949615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911638668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13313645124435425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911654205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13569998741149902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911654205, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784911654206, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784911676643, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5138489007949829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784911676644, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784911676644, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784911691683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350744515657425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911706789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13259848952293396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1649999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911722132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13321472704410553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911729762, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784911729762, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784911753110, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5151954889297485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784911753111, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784911753111, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784911761856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13636158406734467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911777690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13526569306850433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911793264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362164318561554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911809148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13647061586380005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911809149, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784911809149, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784911832499, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5165829062461853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784911832499, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784911832500, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784911848917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347959041595459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911864546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13082635402679443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911880019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13620224595069885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911887946, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784911887946, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784911910777, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5179732441902161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784911910777, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784911910777, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784911918469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13588255643844604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911934727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324286013841629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911949813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13255886733531952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911964828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308419108390808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911964829, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784911964829, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784911987266, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5192909240722656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784911987267, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784911987267, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784912002570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13111421465873718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912018601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13828599452972412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912033951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12959140539169312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912041570, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784912041570, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784912064263, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5207582712173462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784912064263, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784912064264, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784912071661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12928323447704315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912086931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347729116678238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912102211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345614790916443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912118606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303723156452179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912118606, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784912118607, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784912141874, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5225046873092651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784912141875, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784912141875, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784912157031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12813451886177063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912172407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1319412887096405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912187763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353393793106079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912195559, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784912195560, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784912219454, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5244052410125732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784912219455, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784912219455, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784912227190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13662438094615936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912242823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13354535400867462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912257791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13206999003887177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912272716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13095015287399292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912272717, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784912272718, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784912295621, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.526627242565155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784912295622, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784912295622, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784912310873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12954393029212952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912326193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13255901634693146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912341872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353662610054016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912349692, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784912349692, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784912372448, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5293434262275696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784912372449, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784912372449, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784912380097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131979301571846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912395705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13333524763584137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912411199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13508671522140503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912426804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346048265695572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912426804, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784912426805, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784912450580, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5323871374130249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784912450581, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784912450581, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784912466256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13603250682353973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912482474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13398496806621552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912498114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331184357404709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3650000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912505771, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784912505772, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784912528338, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5358318090438843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784912528338, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784912528339, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784912536294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14031833410263062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912552246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13520775735378265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912568497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13450711965560913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4649999999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912584719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13525578379631042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912584719, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784912584720, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784912608445, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5395123362541199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784912608446, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784912608446, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784912623866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13524042069911957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912639409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13322031497955322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.5649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912655062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13225343823432922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912663525, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784912663526, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784912686861, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5439045429229736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784912686862, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784912686862, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784912694852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13241519033908844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912710621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13074944913387299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912726422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13580766320228577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912742414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13231085240840912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912742415, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784912742415, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784912765978, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5495175719261169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784912765979, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784912765979, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784912781861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134669691324234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912797982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13013997673988342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912813179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13187026977539062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912820767, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784912820767, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784912843655, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.557129442691803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784912843655, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784912843656, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784912851093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13227570056915283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912866314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13063809275627136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912881853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13298597931861877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912897685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13249635696411133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912897686, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784912897686, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784912921075, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5674247741699219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784912921076, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784912921076, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784912936911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13193079829216003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912952752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12797290086746216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912968287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338280290365219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912976288, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784912976288, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784912999447, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5787985920906067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784912999447, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784912999448, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784913007400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13448476791381836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913023141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1265079379081726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913039359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12664327025413513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913055042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13252101838588715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913055043, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784913055044, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784913078373, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5906544327735901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784913078373, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784913078373, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784913093462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12880684435367584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913108488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12813392281532288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913124006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1274450570344925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913131589, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784913131590, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784913155043, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6010954976081848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784913155044, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784913155044, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784913162866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295657455921173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913178716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12667112052440643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913194553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12931089103221893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913210344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12758558988571167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913210345, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784913210345, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784913233896, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6105836033821106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784913233897, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784913233897, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784913249248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12587103247642517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913265169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12902747094631195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4983333333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913280960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13170839846134186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913288783, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784913288784, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784913312293, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6180275082588196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784913312294, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784913312294, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784913320629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13030223548412323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913336577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1253012716770172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913352293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13013193011283875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913367960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12489432841539383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913367961, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784913367961, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784913391288, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6242179274559021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784913391289, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784913391289, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784913407283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1276513934135437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913423281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12887048721313477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913439851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12667803466320038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913448065, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784913448065, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784913471079, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6298779845237732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784913471079, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784913471079, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784913479103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12594611942768097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913494892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12597566843032837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913511032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12584879994392395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913527098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12546218931674957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.898333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913527099, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784913527099, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784913550413, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6346467137336731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784913550414, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784913550414, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784913566698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11883053928613663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913582485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11897464841604233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913597912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.116519495844841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913605658, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784913605659, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784913628656, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6400547623634338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784913628657, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784913628657, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784913636202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11873442679643631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913651777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11557237058877945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913667590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12053964287042618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913683125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11756888031959534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913683126, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784913683126, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784913707262, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6464189887046814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784913707263, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784913707263, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784913723558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11935362219810486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913739162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11056698113679886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.198333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913754864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11663470417261124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913762544, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784913762544, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784913785177, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.654391348361969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784913785177, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784913785178, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784913792911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11758242547512054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913808904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11752405017614365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.298333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913824788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1125873401761055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913840328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11626480519771576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913840402, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784913840403, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784913864413, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6627070307731628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784913864415, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784913864415, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784913880380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11534138023853302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.398333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913895938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11506126075983047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913911716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10811086744070053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913919479, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784913919479, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784913942763, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.672302782535553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784913942764, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784913942764, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784913950606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11250068992376328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.498333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913966556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11438177525997162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.531666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913982533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11181698739528656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913998582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11480274051427841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.598333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913998582, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784913998583, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784914022708, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6823498010635376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784914022709, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784914022709, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784914038306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11358526349067688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914053687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11322459578514099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914069283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11745114624500275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914076991, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784914076991, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784914099892, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6943596601486206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784914099893, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784914099893, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784914107448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10700191557407379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914123303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11552466452121735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914139051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10929881036281586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914154980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11374523490667343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914154980, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784914154981, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784914178892, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7104621529579163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784914178893, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784914178894, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784914195174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11716657876968384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914211122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11440394818782806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914226973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11276461184024811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914234667, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784914234667, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784914257521, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7282726168632507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784914257522, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784914257522, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784914265529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11341209709644318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914281646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11527660489082336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914297848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11471851170063019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914313426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10981239378452301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914313426, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784914313427, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784914337248, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7435504198074341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784914337249, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784914337249, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784914354208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1178157702088356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914370202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1215115562081337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914386481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11879980564117432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914394513, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784914394513, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784914416691, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7532715201377869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784914416692, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784914416692, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784914424663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11839203536510468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914441197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.118648000061512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914457535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11523827910423279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914473620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11589697003364563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914473621, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784914473622, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784914497261, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7585479617118835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784914497262, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784914497263, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784914513374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1215931698679924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914529183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11956530064344406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914545458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11273716390132904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914553505, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784914553506, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784914575986, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7611832618713379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784914575986, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784914575987, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784914584105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11815893650054932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914600473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11727094650268555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914616570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11346898972988129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914633372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10669398307800293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914633372, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784914633373, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784914655891, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.763435423374176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784914655892, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784914655892, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784914671582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10874494910240173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914687515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10779502987861633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914703279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10786817967891693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914711178, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784914711178, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784914734314, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7645410895347595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784914734315, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784914734315, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784914742213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10562983900308609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914758504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11220568418502808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914774687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10912683606147766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914791205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11402136087417603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914791206, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784914791207, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784914813696, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7654991149902344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784914813696, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784914813696, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784914829742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10830221325159073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914845456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10738680511713028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914861426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10274343937635422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914869100, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784914869100, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784914891986, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7661555409431458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784914891987, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784914891987, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784914899985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10388831049203873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914915984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10174589604139328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914932282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11280366778373718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914948096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10460726171731949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914948097, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784914948097, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784914971501, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7676534652709961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784914971502, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784914971502, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784914987757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10698585957288742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915003896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1066393107175827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915019501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10747850686311722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915027700, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784915027701, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784915050060, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7684279084205627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784915050061, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784915050061, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784915058082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10703179240226746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915073733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10050616413354874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915089838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10778223723173141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915106017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10449586063623428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915106017, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784915106018, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784915129228, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7692233920097351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784915129229, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784915129229, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784915144997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10752008855342865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915161878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10838459432125092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915178077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10487435758113861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915186013, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784915186013, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784915209107, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7694152593612671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784915209108, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784915209108, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784915216969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10950776189565659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915233127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10835957527160645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915248999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11222866922616959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915265250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10811607539653778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915265250, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784915265250, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784915288422, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7701581716537476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784915288423, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784915288423, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784915304648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10567232221364975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915320365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10209030658006668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915336668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10856326669454575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.564999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915344899, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784915344899, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784915368381, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7701243162155151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784915368382, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784915368382, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784915376321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10468561947345734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915392029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09985316544771194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915408039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11466952413320541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.664999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915423887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11091748625040054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915423888, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784915423888, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784915447367, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7708522081375122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784915447367, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784915447368, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784915463378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10678555816411972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915479747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10881776362657547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915495896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10844854265451431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915503778, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784915503779, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784915527643, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.771024763584137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784915527644, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784915527644, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784915536466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11013227701187134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915553062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11444812268018723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915569451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11287456750869751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915585789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11302164196968079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915585790, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784915585790, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784915609190, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7712953686714172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784915609191, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784915609191, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784915625231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11011148989200592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915641736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11393638700246811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.998333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915658117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11105769872665405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915666230, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784915666231, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784915690068, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7715229988098145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784915690068, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784915690069, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784915698112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11102274060249329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915714956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10982854664325714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915731340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11300218105316162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.131666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915747721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11100876331329346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915747721, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784915747722, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784915770815, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7719113230705261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784915770815, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784915770815, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784915787287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1104167178273201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915804211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11321032792329788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915821063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11703985929489136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915829406, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784915829406, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784915853342, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7720591425895691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784915853342, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784915853343, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784915861337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11476582288742065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915878217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10779406875371933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.331666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915894485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10157927870750427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915910637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10717672109603882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915910638, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784915910638, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784915933585, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7719971537590027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784915933586, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784915933587, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784915949638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.105145163834095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915965696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1066320538520813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915982151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10492795705795288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915990418, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784915990418, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784916013957, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7722120881080627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784916013958, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784916013958, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784916022066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10858382284641266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916037909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10344737023115158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916054827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10477666556835175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916071357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10741794109344482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.631666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916071357, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784916071357, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784916094943, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7724264860153198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784916094943, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784916094944, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784916110714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10054334998130798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916126925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039135605096817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916142896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10358244925737381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916150904, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784916150905, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784916173890, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7725723385810852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784916173891, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784916173891, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784916181957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1044023185968399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916198665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10315615683794022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916214817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11419813334941864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916231310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10320920497179031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916231310, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784916231311, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784916254463, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7729415893554688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784916254464, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784916254464, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784916270615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10739464312791824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916286624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10744218528270721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916302611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10184705257415771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916310536, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784916310537, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784916334463, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773004412651062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784916334464, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784916334464, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784916342420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10142387449741364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916358513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10057319700717926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.031666666666666e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_32768/seed770053153.log b/recommendation_v4/rcp_logs/gbs_32768/seed770053153.log new file mode 100644 index 000000000..dfffea1de --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_32768/seed770053153.log @@ -0,0 +1,634 @@ +:::MLLOG {"namespace": "", "time_ms": 1784909143635, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784909143635, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784909167457, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784909167459, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784909167459, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784909167459, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784909167459, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784909167459, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784909167459, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784909167459, "event_type": "POINT_IN_TIME", "key": "seed", "value": 770053153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784909167459, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784909167459, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784909167460, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784909167460, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784909167460, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784909167460, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784910140408, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784910140409, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784910140735, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784910324800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389344185590744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.1666666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784910340380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389683485031128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.4999999999999995e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784910355456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13872721791267395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784910362949, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784910362950, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784910405874, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5020070672035217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784910405875, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784910405875, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784910413330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13877063989639282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3166666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910428002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387576162815094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910443327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13874205946922302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9833333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910458663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860732316970825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910458664, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784910458664, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784910484483, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5021510720252991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784910484483, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784910484484, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784910499523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386704444885254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910514811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13855046033859253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910531115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385268270969391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910538575, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784910538575, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784910562561, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5023193359375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784910562562, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784910562562, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784910570332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13850875198841095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910585638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13839450478553772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910600945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13840480148792267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910616480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13813842833042145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910616481, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784910616481, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784910641708, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5025081634521484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784910641709, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784910641709, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784910659385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13829465210437775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910674476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13782820105552673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910689663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13814805448055267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910697270, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784910697270, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784910721505, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.502726137638092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784910721506, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784910721506, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784910728851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373191773891449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910744704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374582052230835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910759463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13714216649532318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910774857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13712553679943085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910774858, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784910774859, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784910799992, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5029904842376709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784910799992, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784910799993, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784910814967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13718166947364807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.316666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910830296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13720062375068665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910846231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13655930757522583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910853925, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784910853925, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784910876912, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.503267765045166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784910876913, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784910876913, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784910884635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137345552444458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910904931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13631121814250946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910920451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365870237350464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910936676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371016949415207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910936676, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784910936677, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784910960894, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5036372542381287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784910960895, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784910960895, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784910976184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13618811964988708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910992273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135063037276268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.983333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784911007603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13656127452850342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911015237, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784911015238, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784911038678, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5039888620376587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784911038679, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784911038679, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784911046292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13315671682357788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911061941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13461041450500488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911077153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349869668483734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911094729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13520447909832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911094805, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784911094806, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784911120492, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5043696165084839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784911120492, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784911120493, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784911135560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338566094636917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911150704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345287263393402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911165935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350940465927124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911174227, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784911174228, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784911200188, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5048134922981262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784911200189, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784911200189, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784911207633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13568013906478882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911222992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13484889268875122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911238549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13184264302253723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911254741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13453221321105957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911254742, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784911254742, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784911280018, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5054461359977722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784911280019, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784911280019, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784911295831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13365161418914795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911311539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13526003062725067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911327729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13498932123184204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911335692, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784911335692, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784911360717, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5064824819564819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784911360717, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784911360717, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784911368475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13653329014778137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911384268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13338954746723175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911400966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13423937559127808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911416507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13208194077014923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911416575, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784911416576, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784911441281, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.507524311542511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784911441281, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784911441282, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784911456690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13287732005119324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911472985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13111160695552826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911488636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329931914806366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911496393, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784911496394, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784911522118, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5086269378662109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784911522118, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784911522118, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784911529905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384134590625763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911545726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1309875249862671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911561904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12992608547210693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911577609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13035142421722412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911577610, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784911577610, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784911603005, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5098665356636047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784911603005, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784911603005, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784911618463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13278046250343323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911634123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297377496957779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911650161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13058561086654663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911658120, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784911658121, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784911683114, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.511155366897583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784911683115, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784911683115, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784911690831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131535142660141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911706357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13505303859710693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911722163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331595629453659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911738426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13565003871917725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911738427, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784911738427, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784911763911, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5125112533569336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784911763912, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784911763912, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784911779555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13518072664737701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911795360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13247103989124298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1649999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911811225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13323965668678284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911819050, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784911819051, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784911842550, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5138816237449646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784911842551, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784911842551, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784911851318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13627293705940247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911867780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13515758514404297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911883908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13621678948402405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911900146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13637763261795044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911900146, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784911900147, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784911924292, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.515277087688446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784911924293, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784911924293, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784911941049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347421407699585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911957420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13078640401363373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911973437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13619214296340942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911981696, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784911981696, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784912005788, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5167177319526672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784912005789, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784912005789, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784912013729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13590103387832642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912030143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13251669704914093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912046031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326831579208374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912061698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13082003593444824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912061699, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784912061699, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784912088139, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5180983543395996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784912088140, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784912088140, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784912103799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310250163078308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912120075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13818423449993134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912136126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294378936290741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912144112, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784912144113, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784912169212, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5196504592895508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784912169213, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784912169213, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784912176933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12926052510738373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912192994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13469800353050232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912208903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344725787639618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912225321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303904950618744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912225322, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784912225322, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784912251213, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5216056704521179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784912251214, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784912251214, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784912267091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12800781428813934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912282589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13200470805168152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912298817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13546030223369598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912306789, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784912306790, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784912330604, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5237603187561035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784912330605, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784912330605, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784912338663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13668221235275269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912355053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13352973759174347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912371026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321452260017395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912386701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13085110485553741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912386702, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784912386702, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784912412376, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5262863039970398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784912412377, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784912412377, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784912428409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12952736020088196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912444273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13245420157909393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912460639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13538897037506104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912468644, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784912468645, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784912494404, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5291929841041565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784912494405, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784912494405, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784912502338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13198266923427582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912518175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333407163619995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912534027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349261999130249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912549991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13458001613616943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912549992, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784912549992, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784912573620, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5327134728431702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784912573621, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784912573621, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784912589935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13601604104042053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912606981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13389244675636292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912623671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330435872077942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3650000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912631845, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784912631845, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784912658481, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5368900299072266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784912658481, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784912658481, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784912667046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14015386998653412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912683508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13516464829444885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912700191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13434410095214844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4649999999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912716877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13511916995048523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912716877, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784912716878, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784912743300, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5415626764297485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784912743301, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784912743301, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784912759847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351929008960724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912776476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13314247131347656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.5649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912792906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13221541047096252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912801754, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784912801755, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784912828014, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5467609167098999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784912828015, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784912828015, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784912836415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323441118001938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912853015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13068026304244995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912869174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13566631078720093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912885516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13210713863372803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912885516, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784912885517, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784912909337, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5530866384506226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784912909338, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784912909338, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784912925786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13448208570480347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912942798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1299229860305786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912959330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13158106803894043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912967453, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784912967454, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784912993652, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5618466138839722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784912993652, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784912993652, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784913001856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13184881210327148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913018182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13021688163280487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913034807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13253261148929596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913051441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13204127550125122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913051441, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784913051442, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784913077882, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5731747150421143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784913077883, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784913077883, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784913094148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13135573267936707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913110336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1272072196006775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913126776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13303112983703613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913134804, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784913134805, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784913160450, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5840501189231873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784913160451, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784913160451, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784913168717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13352680206298828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913184754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1255374252796173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913201297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12517598271369934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913218016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13113750517368317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913218016, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784913218017, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784913243771, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5993665456771851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784913243771, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784913243771, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784913260024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1274404525756836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913276419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12659335136413574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913293367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12566183507442474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913301407, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784913301408, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784913325724, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.609666645526886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784913325725, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784913325725, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784913334001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1277349293231964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913350544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12455026060342789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913367193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12745235860347748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913383641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12525179982185364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913383642, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784913383642, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784913409772, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6282863020896912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784913409773, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784913409773, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784913426244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12292923033237457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913443110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12631632387638092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4983333333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913459442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12848085165023804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913467681, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784913467681, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784913491727, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6530503034591675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784913491728, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784913491728, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784913500595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12677103281021118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913517608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12203839421272278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913534624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.126408189535141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913551644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12063780426979065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913551644, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784913551645, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784913577929, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6880220174789429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784913577929, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784913577930, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784913594922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12370394915342331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913611608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12477058172225952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913628813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12265840917825699, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913637371, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784913637371, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784913663204, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.732940137386322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784913663205, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784913663205, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784913671722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12131274491548538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913688469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12162134051322937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913705678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12069230526685715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913722484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11985331773757935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.898333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913722485, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784913722485, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784913746928, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7489625811576843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784913746928, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784913746929, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784913763984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11281489580869675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913780657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11181402951478958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913797122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11000147461891174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913805581, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784913805581, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784913831877, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7544142603874207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784913831877, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784913831877, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784913840059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11165246367454529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913856539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10867953300476074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913873192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11484955251216888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913889534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11228568851947784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913889534, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784913889534, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784913913547, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7560238242149353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784913913548, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784913913548, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784913930726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11401589214801788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913947243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10390228778123856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.198333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913963673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11089034378528595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913971927, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784913971928, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784913997933, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7606884837150574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784913997933, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784913997934, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784914005924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11353965103626251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914022928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11204653233289719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.298333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914039575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081051304936409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914055764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11138410866260529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914055845, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784914055845, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784914082491, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7630901336669922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784914082492, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784914082492, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784914098767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10912621021270752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.398333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914114992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11027466505765915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914131388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10346612334251404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914139331, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784914139331, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784914165198, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7654290199279785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784914165199, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784914165199, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784914173282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10806182026863098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.498333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914190173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10813065618276596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.531666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914206709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10555949807167053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914223634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10920141637325287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.598333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914223634, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784914223635, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784914249103, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7678111791610718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784914249104, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784914249104, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784914265844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10741252452135086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914282269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10641105473041534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914299137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11181633174419403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914307089, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784914307089, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784914333151, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7686997056007385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784914333152, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784914333152, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784914341481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10143506526947021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914358429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11073321849107742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914375123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1047273725271225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914391965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10783860087394714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914391965, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784914391966, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784914418501, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.769637405872345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784914418502, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784914418502, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784914435039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11174342781305313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914451498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10912062227725983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914468264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10765811800956726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914476201, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784914476201, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784914501495, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7702892422676086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784914501496, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784914501496, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784914509808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10880566388368607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914526638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10990476608276367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914543551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11143650114536285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914559688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1049978956580162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914559688, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784914559688, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784914584679, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7706796526908875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784914584680, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784914584680, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784914602620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11455186456441879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914619948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11868801712989807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914637473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11536802351474762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914646160, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784914646161, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784914672071, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.771597683429718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784914672071, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784914672071, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784914680433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11498531699180603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914698129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11425146460533142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914714957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1117720901966095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914731512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11374025046825409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914731513, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784914731513, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784914758204, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7716794013977051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784914758204, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784914758204, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784914775279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11744248867034912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914792164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11723889410495758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914809570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11063133925199509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914818115, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784914818116, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784914844412, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7718966603279114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784914844413, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784914844413, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784914852919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11529085040092468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914870213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11488215625286102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914887195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11108186841011047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914905071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10455073416233063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914905072, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784914905072, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784914931032, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.771872878074646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784914931033, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784914931033, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784914947948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1066843718290329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914965005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10609996318817139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914981883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10606445372104645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914990485, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784914990485, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784915016022, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.771892786026001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784915016022, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784915016022, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784915024588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10430027544498444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915041875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1117892861366272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915058690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793137550354004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915075995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11284301429986954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915075996, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784915075997, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784915103496, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7723210453987122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784915103496, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784915103497, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784915120423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732710361480713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915137025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10700968652963638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915154279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1021600067615509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915162448, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784915162449, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784915189185, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7725693583488464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784915189186, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784915189186, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784915197784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10330033302307129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915214827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10202473402023315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915232220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11149334907531738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915248761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10435442626476288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915248762, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784915248762, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784915273058, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.772398829460144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784915273059, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784915273059, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784915290268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10629518330097198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915307886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1068354994058609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915324795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1070491373538971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915333675, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784915333677, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784915359542, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7724577188491821, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784915359542, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784915359542, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784915368029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10785210877656937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915385216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10036914050579071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915402479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10839095711708069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915419637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10504861176013947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915419637, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784915419638, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784915443658, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7727499604225159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784915443658, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784915443658, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784915460240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10845385491847992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915477701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10887239873409271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915495185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10488373041152954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915503683, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784915503684, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784915530078, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730438113212585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784915530078, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784915530078, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784915538423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10973086208105087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915555529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10785815119743347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915572761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11215335130691528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915590095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10843516141176224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915590096, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784915590096, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784915616229, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7731454372406006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784915616230, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784915616230, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784915633541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10588090121746063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915650164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10237767547369003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915667809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10821081697940826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.564999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915676605, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784915676606, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784915702742, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7731905579566956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784915702742, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784915702743, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784915711262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10493486374616623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915727966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09976956248283386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915745148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11500424891710281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.664999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915762433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11137603968381882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915762433, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784915762433, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784915789066, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7732917666435242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784915789066, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784915789067, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784915806085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10720402002334595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915823454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10903333127498627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915840388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10877501219511032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915848639, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784915848639, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784915872539, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7735381722450256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784915872540, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784915872540, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784915881549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10999657958745956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915899318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1148197203874588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915916683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11251301318407059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915934112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11379208415746689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915934113, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784915934113, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784915960359, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773662805557251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784915960359, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784915960359, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784915977513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11030814051628113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915995054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11394002288579941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.998333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916012800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11086305230855942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916021285, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784916021286, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784916046014, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738317251205444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784916046014, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784916046015, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784916054543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11127790808677673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916072453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10982535779476166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916090136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11320339143276215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.131666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916107687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11133429408073425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916107688, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784916107688, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784916133483, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738375067710876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784916133483, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784916133484, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784916151079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11036783456802368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916169128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11322231590747833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916186564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11698813736438751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916195255, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784916195256, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784916220650, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738844156265259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784916220650, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784916220651, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784916228858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11539086699485779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916246709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10803443193435669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.331666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916263613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1018192395567894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916280666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071152538061142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916280666, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784916280667, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784916306749, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773594081401825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784916306749, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784916306749, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784916323461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10516075789928436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916340332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10674521327018738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916357740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10582875460386276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.498333333333333e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_32768/seed898059771.log b/recommendation_v4/rcp_logs/gbs_32768/seed898059771.log new file mode 100644 index 000000000..147c34969 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_32768/seed898059771.log @@ -0,0 +1,701 @@ +:::MLLOG {"namespace": "", "time_ms": 1784923825817, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784923825817, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784923849075, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784923849076, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784923849076, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784923849076, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784923849076, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784923849076, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784923849076, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784923849076, "event_type": "POINT_IN_TIME", "key": "seed", "value": 898059771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784923849076, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784923849076, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784923849076, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784923849076, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784923849124, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784923849125, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784924770411, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784924770412, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784924770759, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784924909414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13851618766784668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.1666666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784924924846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13849464058876038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.4999999999999995e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784924939347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13846668601036072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784924946461, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784924946462, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784924969730, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.500247597694397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784924969730, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784924969730, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784924976992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384524405002594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3166666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924991245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13845795392990112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925006229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13848111033439636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9833333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925021048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13845863938331604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925021048, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784925021049, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784925043771, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5004059672355652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784925043772, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784925043772, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784925058584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381346881389618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925073366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13822314143180847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925088101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13815061748027802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925095263, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784925095264, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784925118146, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5006042718887329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784925118147, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784925118147, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784925125753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13818307220935822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925140463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13828210532665253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925155392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138348788022995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925170364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13796043395996094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925170364, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784925170365, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784925192883, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5008304119110107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784925192884, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784925192884, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784925207279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13810153305530548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925221826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13741692900657654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925236451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13785940408706665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925243707, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784925243708, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784925266860, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.501090943813324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784925266861, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784925266861, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784925273885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13696201145648956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925288744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13716261088848114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925303145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13689057528972626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925317885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368727684020996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925317886, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784925317886, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784925341336, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5013877153396606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784925341337, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784925341337, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784925355548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13705477118492126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.316666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925370167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370309740304947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925385493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13629266619682312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925392856, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784925392857, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784925415822, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5016984939575195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784925415823, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784925415823, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784925423214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13708919286727905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925438932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361178159713745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925453637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13643459975719452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925469116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1369544416666031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925469117, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784925469117, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784925491803, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5020865797996521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784925491804, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784925491804, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784925506435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13611721992492676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925521699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13477958738803864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.983333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925536360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13654008507728577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925543663, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784925543663, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784925567017, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5024632215499878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784925567017, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784925567018, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784925574255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329861581325531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925589248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13454130291938782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925603787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13487952947616577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925618356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13505491614341736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925618425, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784925618426, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784925642345, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5028306245803833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784925642346, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784925642346, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784925656834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13368241488933563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925671359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13447807729244232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925685943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351320892572403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925694095, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784925694095, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784925716771, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5032281279563904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784925716772, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784925716772, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784925723908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13565343618392944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925738509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13489462435245514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925753488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13184131681919098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925769004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13453030586242676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925769005, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784925769006, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784925791416, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5037698149681091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784925791417, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784925791417, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784925806333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13374684751033783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925821647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352718472480774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925837296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348884552717209, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925844884, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784925844884, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784925868512, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5046960115432739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784925868513, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784925868513, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784925875932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13637982308864594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925891009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334618180990219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925906550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13425351679325104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925921397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320488601922989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925921462, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784925921462, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784925944864, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5056262016296387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784925944864, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784925944864, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784925959512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13287048041820526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925975028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13123157620429993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925989792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13307799398899078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925997251, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784925997252, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784926020101, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5066063404083252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784926020103, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784926020103, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784926027461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138504296541214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926042357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13108564913272858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926057835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12986746430397034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926072465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13029572367668152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926072465, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784926072466, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784926095511, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5077484250068665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784926095511, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784926095511, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784926110357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13278742134571075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926125445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12983572483062744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926141015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.130569726228714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926148446, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784926148447, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784926171268, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5089661478996277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784926171269, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784926171269, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784926178781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315373033285141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926193639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13515204191207886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926208963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331835687160492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926224942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13561052083969116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926224943, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784926224944, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784926247975, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5103926658630371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784926247975, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784926247976, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784926262779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13524380326271057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926277677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13255837559700012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1649999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926292651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13322246074676514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926300055, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784926300056, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784926323797, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5118592381477356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784926323797, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784926323798, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784926332195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13630619645118713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926347479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13517722487449646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926362791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362120509147644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926378297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13638201355934143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926378298, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784926378298, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784926401775, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5133873820304871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784926401776, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784926401776, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784926417776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13495218753814697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926433031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13088847696781158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926448046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13624048233032227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926455763, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784926455764, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784926479263, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5148209929466248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784926479263, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784926479263, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784926486803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13597680628299713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926502361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13243933022022247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926517387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13271787762641907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926532336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13082416355609894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926532337, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784926532337, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784926555160, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5160877108573914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784926555161, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784926555161, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784926570230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13105623424053192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926586059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13827309012413025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926601131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12948867678642273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926608548, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784926608549, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784926631493, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.517541229724884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784926631493, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784926631494, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784926638938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12939569354057312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926654115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13479766249656677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926669323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13460421562194824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926685626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304318904876709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926685626, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784926685627, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784926709251, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5193918347358704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784926709252, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784926709252, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784926724226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1281765252351761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926739284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13200750946998596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926754541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355375349521637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926762275, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784926762275, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784926786201, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5214159488677979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784926786201, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784926786202, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784926793796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13665568828582764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926809740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13355785608291626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926824897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13205572962760925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926839822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310458779335022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926839822, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784926839823, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784926863414, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5238081812858582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784926863415, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784926863415, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784926878498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12955807149410248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926893539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13248397409915924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926909207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353769600391388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926916807, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784926916808, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784926939784, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5268151760101318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784926939785, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784926939785, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784926947301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13197296857833862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926962652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13342124223709106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926977965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13493192195892334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926993375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13460201025009155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926993375, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784926993375, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784927017071, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5301899909973145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784927017071, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784927017072, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784927032353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13603439927101135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927048414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13396748900413513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927064054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13311733305454254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3650000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927071651, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784927071651, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784927095212, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5341899394989014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784927095213, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784927095213, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784927103197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14035245776176453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927119019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13525976240634918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927134868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13436466455459595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4649999999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927150836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351659744977951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927150836, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784927150837, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784927174644, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.538504421710968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784927174645, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784927174645, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784927190087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13520148396492004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927205622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13317759335041046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.5649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927221172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13235794007778168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927229396, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784927229397, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784927253410, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5434097647666931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784927253411, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784927253411, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784927261207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324964016675949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927276643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13084498047828674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927291868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13577058911323547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927307304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322665959596634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927307304, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784927307305, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784927331651, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.549701452255249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784927331651, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784927331652, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784927347133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13463589549064636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927363266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300952285528183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927378774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13174594938755035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927386479, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784927386480, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784927410139, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5583450198173523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784927410140, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784927410140, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784927417627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13205185532569885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927433035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1306046098470688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927448642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13270357251167297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927464476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13242322206497192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927464477, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784927464477, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784927488526, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.570210874080658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784927488527, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784927488527, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784927503891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13167372345924377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927519202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1277436912059784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927534278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336391419172287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927542121, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784927542121, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784927565908, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5821228623390198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784927565909, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784927565909, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784927573689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13421624898910522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927589116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12619149684906006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927605066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1262417584657669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927620744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13205039501190186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927620745, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784927620745, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784927644303, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5937678217887878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784927644304, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784927644304, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784927659547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12834595143795013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927674660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1274568885564804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927690267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12695084512233734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927697761, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784927697761, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784927721317, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6025128364562988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784927721317, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784927721318, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784927729098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.129136323928833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927745203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12584933638572693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927760668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12840625643730164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927776072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1266225427389145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927776072, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784927776073, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784927799374, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6101095676422119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784927799375, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784927799375, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784927814569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12500397861003876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927830277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12816780805587769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4983333333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927845720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303817480802536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927853440, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784927853440, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784927876641, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6161215305328369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784927876642, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784927876642, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784927884924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1290951669216156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927900893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12388870120048523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927916777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12900137901306152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927932450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12309715151786804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927932450, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784927932451, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784927955928, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6213796734809875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784927955929, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784927955929, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784927971865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12595142424106598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927987579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12714475393295288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928004075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.125588059425354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928012080, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784928012081, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784928035151, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6272590160369873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784928035153, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784928035153, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784928043009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12501442432403564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928058691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12498724460601807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928074571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12442117929458618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928090518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12441711127758026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.898333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928090519, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784928090519, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784928113871, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6306261420249939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784928113872, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784928113872, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784928130421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11677435040473938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928146120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11765661835670471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928161445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11538287252187729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928169255, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784928169256, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784928192643, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6335803270339966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784928192643, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784928192644, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784928200191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11792019754648209, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928215659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11475792527198792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928231299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11987549066543579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928246814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1172080934047699, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928246815, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784928246815, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784928270388, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6394566893577576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784928270388, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784928270388, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784928286940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1195121556520462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928302602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1107531189918518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.198333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928318168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1165068969130516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928325841, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784928325841, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784928349367, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6472029089927673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784928349368, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784928349368, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784928357026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1178789734840393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928372869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11788955330848694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.298333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928388631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11242390424013138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928403917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11633750051259995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928403987, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784928403988, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784928428317, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6570914387702942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784928428318, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784928428318, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784928444107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1159684956073761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.398333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928459719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11594419181346893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928475503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1105942353606224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928483234, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784928483234, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784928506695, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6646050214767456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784928506696, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784928506696, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784928514447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11288287490606308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.498333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928530242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11517022550106049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.531666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928545971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11341430991888046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928562005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11583206802606583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.598333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928562006, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784928562006, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784928585470, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6773422956466675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784928585471, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784928585471, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784928601059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1147501990199089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928616536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11441762745380402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928632283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11793440580368042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928639956, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784928639957, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784928662829, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6881502866744995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784928662830, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784928662830, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784928670551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10916250199079514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928686601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11594663560390472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928702307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10993434488773346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928718233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11579741537570953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928718233, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784928718234, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784928741871, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6994078159332275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784928741871, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784928741872, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784928757737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11797631531953812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928773178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11677858978509903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928788683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11446114629507065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928796175, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784928796176, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784928818999, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.705192506313324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784928818999, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784928819000, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784928826709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11421656608581543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928842331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11741218715906143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928858214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11570577323436737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928873472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11213439702987671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928873473, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784928873473, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784928897624, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7141785621643066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784928897625, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784928897625, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784928914641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11967889219522476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928930721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12333181500434875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928946908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12169323116540909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928954936, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784928954937, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784928977310, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7306665182113647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784928977311, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784928977311, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784928985174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12018358707427979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929001404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12000530958175659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929017383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11820365488529205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929033165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11773968487977982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929033166, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784929033166, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784929056965, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7390012145042419, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784929056966, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784929056966, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784929072857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12320350855588913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929088455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12032131850719452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929104270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11464203149080276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929112161, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784929112162, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784929134961, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.745743989944458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784929134962, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784929134962, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784929142852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12216303497552872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929158805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11865490674972534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929174666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11490596830844879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929191101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10936305671930313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929191102, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784929191102, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784929213789, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7517828941345215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784929213789, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784929213790, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784929229109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1118147075176239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929244771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11101673543453217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929260242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11061378568410873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929268020, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784929268021, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784929290907, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7550656795501709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784929290907, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784929290908, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784929298779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1096469834446907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929314887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11520106345415115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929330782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11192522943019867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929347046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11776698380708694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929347047, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784929347047, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784929369744, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7601944804191589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784929369745, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784929369745, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784929385474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11154051870107651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929400822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11128106713294983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929416427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10718280076980591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929424017, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784929424018, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784929447704, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7604387998580933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784929447704, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784929447705, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784929455506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10707204043865204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929471336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10470281541347504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929487452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11618468165397644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929503023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10947529971599579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929503023, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784929503024, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784929526573, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7623239755630493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784929526574, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784929526574, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784929542683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11038082838058472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929558546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11033276468515396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929573934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10976121574640274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929581920, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784929581921, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784929604719, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7642564177513123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784929604720, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784929604720, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784929612656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10947246104478836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929628136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1022871807217598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929644003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.110089972615242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929659940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725268721580505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929659940, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784929659940, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784929682734, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7650846838951111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784929682735, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784929682735, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784929698252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10897781699895859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929714649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11084393411874771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929730410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10860949009656906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929738100, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784929738100, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784929760821, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7661783695220947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784929760822, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784929760822, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784929768469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1129896491765976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929784090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10901700705289841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929799376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11256872862577438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929815017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10932882130146027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929815018, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784929815018, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784929837702, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7661504745483398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784929837703, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784929837703, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784929853509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10675202310085297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929868944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10424366593360901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929885349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1095624715089798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.564999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929893437, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784929893437, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784929915874, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7668023705482483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784929915875, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784929915875, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784929923816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10602317750453949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929939393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1007661446928978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929955230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11616943031549454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.664999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929970964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11212486028671265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929970965, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784929970965, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784929994388, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7674172520637512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784929994388, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784929994389, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784930010353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732746124267578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930026630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10996231436729431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930042582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10958119481801987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930050427, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784930050428, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784930074203, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7677519917488098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784930074204, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784930074204, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784930082921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11101046949625015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930099202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11526267975568771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930115365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11349549144506454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930131476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11626327782869339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930131477, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784930131477, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784930154052, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7685096263885498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784930154052, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784930154053, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784930169936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11139160394668579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930186367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1153145283460617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.998333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930202707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11158185452222824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930210826, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784930210826, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784930234328, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.769062340259552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784930234329, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784930234329, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784930242305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11214864253997803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930259221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11005795001983643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930275339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11377231776714325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.131666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930291532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11216118931770325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930291532, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784930291533, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784930314183, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7694895267486572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784930314184, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784930314184, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784930330391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11090710759162903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930346960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1131872832775116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930363417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11767882108688354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930371625, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784930371626, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784930395310, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.769891083240509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784930395311, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784930395311, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784930403171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11552394926548004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930419837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10812386125326157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.331666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930435778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10247902572154999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930451641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793086141347885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930451642, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784930451642, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784930474738, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7701855301856995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784930474738, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784930474739, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784930490509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10592330992221832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930506328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10686737298965454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930522502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10664002597332001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930530633, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784930530634, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784930554811, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7703453898429871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784930554812, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784930554812, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784930562762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10895249247550964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930578372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10373222082853317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930595251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10513504594564438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930611412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10790741443634033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.631666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930611413, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784930611414, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784930634533, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7709788084030151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784930634534, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784930634534, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784930650177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10105334222316742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930666221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10475845634937286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930682162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10409890860319138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930690095, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784930690096, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784930713930, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7714491486549377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784930713931, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784930713931, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784930721974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10509669035673141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930738475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1033371239900589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930754531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11474989354610443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930770983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10400542616844177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930770983, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784930770984, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784930794610, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7715696692466736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784930794611, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784930794611, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784930810603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10781586915254593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930826541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10874763876199722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930842660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10239779204130173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930850706, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784930850707, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784930873909, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7718785405158997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784930873910, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784930873910, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784930881905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1019010990858078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930897953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10103213042020798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930914213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10670678317546844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930930658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1034335270524025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930930658, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784930930659, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784930953811, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7722553014755249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784930953812, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784930953812, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784930969590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084270179271698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930986500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10429082065820694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931002442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10698859393596649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931010222, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784931010223, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784931034038, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7724200487136841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784931034039, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784931034039, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784931041945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10422153770923615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931057834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1090250164270401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931073684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10343965888023376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931090113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10761374235153198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931090114, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784931090114, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784931113659, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7727489471435547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784931113660, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784931113660, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784931130013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1026747077703476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931145866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10736606270074844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.398333333333333e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_32768/seed942014509.log b/recommendation_v4/rcp_logs/gbs_32768/seed942014509.log new file mode 100644 index 000000000..7f66183d6 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_32768/seed942014509.log @@ -0,0 +1,675 @@ +:::MLLOG {"namespace": "", "time_ms": 1784950456391, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784950456391, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784950479613, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784950479615, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784950479616, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784950479616, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784950479616, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784950479616, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784950479616, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784950479616, "event_type": "POINT_IN_TIME", "key": "seed", "value": 942014509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784950479616, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784950479616, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784950479616, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784950479616, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784950479631, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784950479632, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784951311027, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784951311027, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784951311354, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784951630015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13938525319099426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.1666666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784951646073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1392778754234314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.4999999999999995e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784951661234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1391793042421341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784951668638, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784951668639, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784951713176, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4997166693210602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784951713177, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784951713177, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784951720632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13905693590641022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3166666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951735373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1391364336013794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951750718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1390824317932129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9833333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951765761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13889843225479126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951765761, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784951765762, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784951788363, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49986180663108826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784951788364, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784951788364, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784951803493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1390383243560791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951818833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388559341430664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951834036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13886678218841553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951841392, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784951841393, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784951864659, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5000264048576355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784951864660, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784951864660, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784951872479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13878445327281952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951887776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13870707154273987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951903267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13863611221313477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951918497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13851848244667053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951918497, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784951918498, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784951941077, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5002193450927734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784951941078, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784951941078, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784951956045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13850121200084686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951971035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13821986317634583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951986216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13833360373973846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951993801, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784951993802, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784952016557, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.500444769859314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784952016558, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784952016558, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784952023916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13788744807243347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952039433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13774146139621735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952054279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13760662078857422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952069445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375463306903839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952069446, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784952069446, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784952092418, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5006869435310364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784952092419, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784952092419, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784952107219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13757741451263428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.316666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952122487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374356895685196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952138419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13696229457855225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952145937, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784952145937, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784952169152, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5009583234786987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784952169153, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784952169153, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784952176730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374688744544983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952192933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13668303191661835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952208359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13687802851200104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952224644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13727916777133942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952224644, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784952224645, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784952247092, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5013055205345154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784952247093, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784952247093, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784952262278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365201473236084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952278184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135393887758255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.983333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952293332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368338018655777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952300920, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784952300920, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784952323258, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5016603469848633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784952323259, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784952323259, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784952330860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13382910192012787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952346229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13510912656784058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952361052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13525255024433136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952376074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13543306291103363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952376144, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784952376144, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784952399907, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5020328164100647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784952399907, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784952399908, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784952415006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13424141705036163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952430058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13484102487564087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952445099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13535813987255096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952453469, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784952453470, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784952475854, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5024929642677307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784952475855, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784952475855, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784952483316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13578665256500244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952498412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13507810235023499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952513871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132317453622818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952529784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347334384918213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952529784, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784952529785, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784952552036, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5030809640884399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784952552037, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784952552038, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784952567335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13391870260238647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952582892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13525399565696716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952599251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13505718111991882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952607119, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784952607119, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784952630022, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5039742588996887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784952630022, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784952630023, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784952637844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363483965396881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952653670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13358965516090393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952669650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13420884311199188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952685151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13205666840076447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952685215, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784952685216, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784952708398, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5049456357955933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784952708399, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784952708399, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784952723682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1328895539045334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952739941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13133732974529266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952755373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13318605720996857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952763187, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784952763187, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784952786528, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5060123205184937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784952786529, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784952786529, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784952794188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13836802542209625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952809827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13105586171150208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952825859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12988857924938202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952841235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13041047751903534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952841235, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784952841236, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784952863870, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5072387456893921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784952863871, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784952863871, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784952879262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13275563716888428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952894889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12979170680046082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952910980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13059017062187195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952918634, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784952918635, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784952941714, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5086020231246948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784952941715, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784952941715, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784952949406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13157521188259125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952964788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13514301180839539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952980702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13311049342155457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952996877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356322169303894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952996877, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784952996878, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784953019816, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.510015070438385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784953019816, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784953019816, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784953035316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13523483276367188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953050857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13265900313854218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1649999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953066535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13322661817073822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953074316, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784953074316, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784953097536, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5113710165023804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784953097537, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784953097537, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784953106372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13635943830013275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953122415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135268896818161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953138342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13621237874031067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953154661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13634896278381348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953154661, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784953154661, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784953178065, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.512747049331665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784953178066, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784953178066, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784953194948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348572075366974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953211148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1307659149169922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953226920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362570822238922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953235011, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784953235012, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784953258596, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5141309499740601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784953258597, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784953258597, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784953266476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13595065474510193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953283022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13252303004264832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953298664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13269659876823425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953313893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13082928955554962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953313893, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784953313894, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784953336894, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5155054926872253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784953336894, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784953336895, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784953352506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13100923597812653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953368756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382891833782196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953384391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12956582009792328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953392205, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784953392206, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784953414884, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5170167684555054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784953414885, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784953414885, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784953422569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1293007880449295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953438291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347917914390564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953454128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345607042312622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953470649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13043645024299622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953470650, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784953470650, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784953493244, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5188102126121521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784953493245, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784953493245, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784953508636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12815743684768677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953524090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13201385736465454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953539771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356455236673355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953547756, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784953547756, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784953570396, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5206784009933472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784953570397, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784953570397, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784953578143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13665953278541565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953594278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13361874222755432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953609907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321638971567154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953625457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310407519340515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953625457, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784953625457, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784953648307, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5227980017662048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784953648308, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784953648308, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784953663997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12951189279556274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953679710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132567897439003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953696151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13545721769332886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953704054, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784953704055, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784953727219, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5253384709358215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784953727220, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784953727220, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784953734895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1319354623556137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953750611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333875209093094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953766157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13501572608947754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953781882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13464263081550598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953781882, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784953781883, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784953805800, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5281730890274048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784953805801, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784953805801, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784953821847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360304355621338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953838396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13395845890045166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953854433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330866515636444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3650000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953862292, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784953862292, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784953885880, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5313276648521423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784953885880, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784953885880, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784953893976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1403176188468933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953910119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352604180574417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953926426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13442742824554443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4649999999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953943055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352378875017166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953943055, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784953943056, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784953966157, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5347365736961365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784953966158, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784953966158, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784953982380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13526329398155212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953998613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13329443335533142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.5649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954014828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323557198047638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954023472, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784954023472, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784954047257, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5387015342712402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784954047258, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784954047258, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784954055228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325884312391281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954070931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13082589209079742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954086493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13591116666793823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954102399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323847472667694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954102400, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784954102400, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784954125331, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.543684720993042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784954125332, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784954125332, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784954141214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13471239805221558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954157906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1302986741065979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954173907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1319255828857422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954181878, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784954181879, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784954204931, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5503340363502502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784954204932, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784954204932, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784954212700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323852837085724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954228661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1307922601699829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954244731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13314031064510345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954261331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13281212747097015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954261332, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784954261332, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784954284865, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.559302806854248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784954284866, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784954284866, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784954300797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13204576075077057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954316599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1282271146774292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954332241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13405659794807434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954340324, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784954340324, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784954363609, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5698170065879822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784954363610, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784954363610, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784954371731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13464201986789703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954387729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12685194611549377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954404214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12686079740524292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954420502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13286302983760834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954420503, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784954420503, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784954443536, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5813202857971191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784954443537, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784954443537, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784954459398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1292615532875061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954475522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12856006622314453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954491867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12787222862243652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954499808, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784954499808, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784954523459, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5926159024238586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784954523460, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784954523461, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784954531615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13008570671081543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954548152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12742114067077637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954563989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300102174282074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954579720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12796533107757568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954579721, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784954579721, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784954602977, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6032059788703918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784954602978, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784954602978, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784954618680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12651316821575165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954634829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12944066524505615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4983333333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954650742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13251934945583344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954658690, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784954658691, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784954682769, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6120533347129822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784954682770, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784954682770, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784954691160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1309661865234375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954707920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12600508332252502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954724306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308455467224121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954740536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12590740621089935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954740537, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784954740537, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784954764399, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6195132732391357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784954764399, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784954764400, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784954781025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12831273674964905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954797413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295386403799057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954814479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12751233577728271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954822702, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784954822703, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784954846572, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6266418099403381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784954846572, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784954846572, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784954854688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12671127915382385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954870685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12670651078224182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954886922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12698805332183838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954903335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12633486092090607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.898333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954903336, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784954903336, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784954927105, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.632881760597229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784954927106, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784954927106, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784954943636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11993734538555145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954960017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11969032883644104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954976160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11773760616779327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954984250, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784954984251, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784955007423, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6400097608566284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784955007424, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784955007424, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784955015302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11992330849170685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955031561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11675269901752472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955047786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12138229608535767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955063771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11832931637763977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955063771, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784955063771, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784955088192, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6476168036460876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784955088193, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784955088193, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784955105000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1199447512626648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955121035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1111164316534996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.198333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955137004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11694562435150146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955144838, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784955144838, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784955168145, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6602447032928467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784955168146, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784955168146, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784955176007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11762797832489014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955192305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11814603954553604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.298333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955208525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1130245178937912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955224357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11668012291193008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955224431, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784955224431, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784955248898, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6784640550613403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784955248898, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784955248898, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784955265450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11540035158395767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.398333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955281633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11495864391326904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955298080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10799530148506165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955306170, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784955306171, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784955330169, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7081494927406311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784955330169, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784955330169, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784955338321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.112007275223732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.498333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955354782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11348062008619308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.531666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955371068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1113385260105133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955387572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11355845630168915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.598333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955387573, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784955387573, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784955411843, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7476333975791931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784955411843, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784955411844, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784955427886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11147540807723999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955443888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11088179796934128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955460175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11454156786203384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955468196, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784955468196, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784955491457, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7621630430221558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784955491458, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784955491458, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784955499564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10373278707265854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955516290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11199760437011719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955532584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10550068318843842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955549171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10976584255695343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955549172, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784955549172, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784955573253, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7619096040725708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784955573254, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784955573254, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784955589444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11299178749322891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955605439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11008986085653305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955621380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10864154994487762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955629171, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784955629171, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784955652326, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7626489400863647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784955652327, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784955652327, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784955660455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10915185511112213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955676764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11044737696647644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955693265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1118411049246788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955709273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10464973002672195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955709274, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784955709274, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784955733520, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7628002166748047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784955733521, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784955733521, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784955751033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11454451829195023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955767552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11876041442155838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955784200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11563084274530411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955792390, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784955792390, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784955816323, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7621578574180603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784955816323, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784955816324, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784955824471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11498970538377762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955841310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11427876353263855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955857803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11222153902053833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955874026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11336449533700943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955874027, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784955874028, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784955898166, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7634860277175903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784955898166, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784955898166, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784955914794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11792377382516861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955930983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11716579645872116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955947517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11058290302753448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955955700, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784955955700, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784955979158, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.76301509141922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784955979159, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784955979159, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784955987455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1151614785194397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956004022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11526508629322052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956020413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11156243830919266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956037390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10439623147249222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956037390, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784956037391, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784956061045, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7655073404312134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784956061046, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784956061046, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784956077172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10652606934309006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956093670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10597768425941467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956109993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10580595582723618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956118206, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784956118207, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784956141914, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7660907506942749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784956141914, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784956141915, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784956150078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10435639321804047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956166888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11129070073366165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956183441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10774901509284973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956200295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11314158886671066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956200295, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784956200296, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784956223138, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7670086026191711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784956223139, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784956223139, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784956239623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1069294661283493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956255508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10634833574295044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956271614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10200981050729752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956279471, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784956279472, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784956302217, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7670903205871582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784956302218, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784956302218, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784956310361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1026434674859047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956326735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10171990841627121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956343369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11174603551626205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956359580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1041506752371788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956359581, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784956359581, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784956383365, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7682419419288635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784956383365, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784956383365, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784956400014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10640476644039154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956416811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10669327527284622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956433041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10722540318965912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956441454, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784956441455, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784956464192, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7683331966400146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784956464192, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784956464192, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784956472479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10726846754550934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956488761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09986816346645355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956505338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772515088319778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956521840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10430725663900375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956521840, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784956521841, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784956545057, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7684311270713806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784956545057, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784956545057, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784956560979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10744550824165344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956577781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793745517730713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956594158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10435591638088226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956602269, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784956602269, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784956625374, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7673854827880859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784956625374, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784956625375, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784956633380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10898106545209885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956649821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10737067461013794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956666016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11189060658216476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956682606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10854028910398483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956682607, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784956682607, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784956705883, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7689808011054993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784956705883, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784956705884, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784956722492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10545680671930313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956738706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.101863332092762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956755552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1086369976401329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.564999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956764004, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784956764005, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784956786678, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7699291706085205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784956786679, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784956786679, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784956794694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10465662181377411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956810695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09986721724271774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956827148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1141803190112114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.664999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956843457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11111834645271301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956843457, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784956843458, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784956867292, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7698155641555786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784956867293, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784956867293, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784956883688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10646301507949829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956900430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10891763865947723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956916954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10868026316165924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956924944, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784956924945, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784956948582, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7693923115730286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784956948583, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784956948583, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784956957276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11033798009157181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956974214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11486578732728958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956991163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11282214522361755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957007969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11303339898586273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957007970, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784957007970, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784957031071, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7692562937736511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784957031071, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784957031071, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784957047694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11031508445739746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957064659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11396831274032593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.998333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957081639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11091317981481552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957090037, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784957090037, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784957113316, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7697288393974304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784957113317, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784957113317, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784957121591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11114037036895752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957139141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11027653515338898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957155920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11304263025522232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.131666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957172598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11125653982162476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957172598, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784957172598, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784957195029, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7704236507415771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784957195029, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784957195030, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784957211809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11016608029603958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957228793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11305087059736252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957245778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11720065027475357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957254223, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784957254223, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784957277321, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.770533561706543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784957277321, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784957277322, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784957285475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11493484675884247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957302742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080770194530487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.331666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957319230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1020095944404602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957335612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10734478384256363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957335612, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784957335613, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784957358972, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7709159255027771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784957358973, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784957358973, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784957375318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10520628839731216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957391484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10688610374927521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957408094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10560768842697144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957416453, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784957416453, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784957438785, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7710500955581665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784957438786, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784957438786, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784957447079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10872358083724976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957463281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10362883657217026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957480572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10498858988285065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957497020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793419182300568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.631666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957497021, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784957497021, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784957519267, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7714108824729919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784957519268, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784957519268, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784957535257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10092955082654953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957551615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039704754948616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957567789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10353685915470123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957575920, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784957575920, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784957598781, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7715893983840942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784957598782, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784957598782, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784957607108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10451418906450272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957623953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1032935231924057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957640179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1144774928689003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957656907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10395635664463043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957656907, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784957656908, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784957679842, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7715200185775757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784957679842, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784957679843, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784957696443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10757070779800415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957712793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10759414732456207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957729012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1020493358373642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957737143, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784957737144, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784957760134, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.771209716796875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784957760135, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784957760135, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784957768347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10225065052509308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957784731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10082433372735977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.031666666666666e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_32768/seed996913233.log b/recommendation_v4/rcp_logs/gbs_32768/seed996913233.log new file mode 100644 index 000000000..6b09c072a --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_32768/seed996913233.log @@ -0,0 +1,617 @@ +:::MLLOG {"namespace": "", "time_ms": 1784916338426, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784916338427, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784916362778, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784916362781, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784916362781, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784916362781, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784916362781, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784916362781, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784916362781, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784916362781, "event_type": "POINT_IN_TIME", "key": "seed", "value": 996913233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784916362781, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784916362781, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784916362781, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784916362781, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784916362803, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784916362804, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784917180038, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784917180039, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784917180377, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784917537545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13884864747524261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.1666666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784917553356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13883957266807556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.4999999999999995e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784917568349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13878655433654785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784917575814, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784917575814, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784917619029, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5015365481376648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784917619030, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784917619030, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784917626519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388218253850937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3166666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917641331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13867776095867157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917656732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13858988881111145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9833333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917671911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385660022497177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917671911, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784917671911, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784917696269, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5016977190971375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784917696270, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784917696270, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784917711464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13853825628757477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917726736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138581782579422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917742025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13844941556453705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917749458, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784917749458, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784917774195, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5018845200538635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784917774196, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784917774196, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784917782171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13844358921051025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917797553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13840222358703613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917813269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13832490146160126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917828779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13818082213401794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917828780, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784917828780, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784917854538, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5020955801010132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784917854539, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784917854539, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784917869617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13825136423110962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917884674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13789550960063934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917901553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13799332082271576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917909184, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784917909185, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784917934840, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5023759007453918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784917934841, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784917934841, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784917942279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13736926019191742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917958149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13754421472549438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917973296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13725632429122925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917988850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13726945221424103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917988851, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784917988851, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784918016296, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5026543140411377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784918016296, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784918016297, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784918031323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13735778629779816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.316666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918050170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13722500205039978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918066620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366882622241974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918074511, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784918074512, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784918100140, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5029589533805847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784918100141, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784918100141, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784918107970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13736268877983093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918124471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365812122821808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918140281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13675464689731598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918156848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13716955482959747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918156848, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784918156849, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784918183694, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5033390522003174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784918183694, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784918183694, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784918199483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13646015524864197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918215714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353917121887207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.983333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918231577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13674575090408325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918239279, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784918239280, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784918263031, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5037326216697693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784918263032, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784918263032, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784918270835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13384465873241425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918287441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13506847620010376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918303011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13531552255153656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918318559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13541756570339203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918318652, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784918318652, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784918346673, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5041405558586121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784918346674, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784918346674, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784918362252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13422803580760956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918379771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13494768738746643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918395538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354362815618515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918404432, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784918404432, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784918431742, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5045881867408752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784918431743, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784918431743, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784918439542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13588431477546692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918455292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13511045277118683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918471155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13251593708992004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918488022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13471272587776184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918488022, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784918488022, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784918514313, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5051766633987427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784918514314, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784918514314, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784918530613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13402649760246277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918546853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13525840640068054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918563795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135014146566391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918572110, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784918572110, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784918598343, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5061303973197937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784918598344, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784918598344, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784918606268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13634821772575378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918622292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13351459801197052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918639242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343356966972351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918655321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13228002190589905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918655388, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784918655388, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784918682392, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5071397423744202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784918682392, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784918682393, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784918698323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13299311697483063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918715058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1313813328742981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918731613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312095403671265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918739767, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784918739767, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784918766014, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5082727074623108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784918766015, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784918766015, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784918774029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13831080496311188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918790157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131221741437912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918807070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12989598512649536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918823340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13042806088924408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918823341, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784918823341, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784918850110, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5095977783203125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784918850111, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784918850111, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784918866002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327870786190033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918881903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12976966798305511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918898840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1305113136768341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918907015, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784918907016, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784918933428, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5110715627670288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784918933429, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784918933429, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784918941437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315292865037918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918957468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13503853976726532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918973651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331634521484375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918991009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13562193512916565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918991010, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784918991010, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784919017539, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5126305818557739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784919017539, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784919017540, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784919033929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13513460755348206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919050238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13236509263515472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1649999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919066328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13324055075645447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919074338, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784919074338, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784919098626, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.514167308807373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784919098626, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784919098626, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784919108105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363925188779831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919125123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13524962961673737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919141646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362456977367401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919158136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13642486929893494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919158137, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784919158137, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784919183971, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5156493782997131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784919183972, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784919183972, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784919202153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13478854298591614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919219312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13078776001930237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919235809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13611596822738647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919244249, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784919244249, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784919270000, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5171640515327454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784919270000, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784919270000, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784919278076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13594216108322144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919295137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13243165612220764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919311740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13255521655082703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919328303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13081327080726624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919328304, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784919328304, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784919354250, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5187360644340515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784919354250, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784919354250, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784919370335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310792863368988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919387917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13812652230262756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919404753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294945627450943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919412911, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784919412912, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784919438913, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.520534873008728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784919438914, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784919438914, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784919446980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12933844327926636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919463676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347765326499939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919479937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13462845981121063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919497170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13039666414260864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919497171, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784919497171, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784919523958, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5225886702537537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784919523959, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784919523959, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784919540560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12812545895576477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919556962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13198766112327576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919573583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13543885946273804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919581849, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784919581850, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784919607251, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5248034000396729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784919607252, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784919607252, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784919615458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13658864796161652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919633063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335182636976242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919649724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13201770186424255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919666249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1309453248977661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919666249, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784919666250, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784919693087, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5273799896240234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784919693088, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784919693088, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784919709703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294645518064499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919725895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13247615098953247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919743387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13541388511657715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919751721, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784919751722, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784919777528, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5302093625068665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784919777529, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784919777529, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784919785809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13185319304466248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919802664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13340872526168823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919819338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13496878743171692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919835743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13456059992313385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919835743, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784919835743, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784919862150, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.533427894115448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784919862150, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784919862151, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784919878422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358908861875534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919895887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13390816748142242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919912942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13305188715457916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3650000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919921390, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784919921390, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784919948036, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5368315577507019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784919948037, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784919948037, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784919956766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14043119549751282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919973693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13534612953662872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919991423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13442517817020416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4649999999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920008745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352335810661316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920008746, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784920008747, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784920034620, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5406181812286377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784920034620, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784920034620, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784920051586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13515880703926086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920068459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13323073089122772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.5649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920085129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13225868344306946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920094170, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784920094171, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784920122352, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5448633432388306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784920122353, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784920122353, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784920130972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13245484232902527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920147873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13078488409519196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920164761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13578557968139648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920181552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13219434022903442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920181553, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784920181553, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784920207698, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5499485731124878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784920207698, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784920207699, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784920224426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13458700478076935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920242283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301666647195816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920259270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13179247081279755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920267650, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784920267651, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784920293674, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5563074946403503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784920293675, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784920293675, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784920302004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13219869136810303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920318606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13060089945793152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920335259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329658478498459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920352439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13245044648647308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920352440, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784920352441, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784920379283, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5642351508140564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784920379283, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784920379284, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784920396235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13188835978507996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920413082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12799251079559326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920429970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13402059674263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920438486, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784920438487, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784920465170, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5733521580696106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784920465171, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784920465171, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784920473761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13461050391197205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920490384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12655183672904968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920507573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12683254480361938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920524731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13281089067459106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920524732, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784920524732, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784920551481, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5839505195617676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784920551482, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784920551482, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784920568246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12915225327014923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920585144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12850001454353333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920602463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12780198454856873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920610618, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784920610618, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784920636718, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5921359658241272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784920636719, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784920636719, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784920645169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1299773007631302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920663013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1273553967475891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920680261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12982575595378876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920697305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12809477746486664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920697306, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784920697306, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784920722951, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6033560633659363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784920722951, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784920722951, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784920739795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12674446403980255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920757194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12951405346393585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4983333333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920773816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13254496455192566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920782117, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784920782118, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784920808604, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6141943335533142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784920808604, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784920808604, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784920817739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13107900321483612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920835479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.126066654920578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920852951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308017373085022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920870327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12613433599472046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920870328, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784920870328, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784920896718, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.625633955001831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784920896719, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784920896719, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784920914155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12858355045318604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920931061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12972989678382874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920949105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12750297784805298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920958106, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784920958107, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784920984536, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.636995792388916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784920984536, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784920984536, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784920993291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12697193026542664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921010492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12659117579460144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921027923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12729662656784058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921044973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12621372938156128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.898333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921044974, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784921044974, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784921071876, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6481412649154663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784921071876, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784921071876, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784921089536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12040795385837555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921106941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11975416541099548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921123793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11814992129802704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921132281, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784921132281, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784921158898, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6511724591255188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784921158899, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784921158899, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784921167313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11975765228271484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921184261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11673430353403091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921201299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1214437410235405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921217819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11865557730197906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921217819, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784921217819, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784921244528, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6733489036560059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784921244529, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784921244529, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784921262512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12017570436000824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921279390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11151523888111115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.198333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921296469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11698942631483078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921304896, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784921304897, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784921331323, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6972360610961914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784921331323, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784921331323, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784921339663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11787327378988266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921356861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11780139058828354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.298333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921373645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11274323612451553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921389999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11612795293331146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921390067, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784921390067, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784921418591, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7243638634681702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784921418591, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784921418591, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784921435868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11472000926733017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.398333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921452936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11458882689476013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921470340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080269068479538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921478800, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784921478801, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784921505592, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7400403022766113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784921505593, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784921505593, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784921514169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11267617344856262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.498333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921531552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11278803646564484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.531666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921548601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11064855754375458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921566249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11254417151212692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.598333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921566250, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784921566250, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784921594170, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7591232061386108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784921594170, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784921594170, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784921611286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11153199523687363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921628258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11041388660669327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921645420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11519081890583038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921653829, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784921653830, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784921680547, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7623952031135559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784921680547, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784921680548, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784921688991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10415256768465042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921706444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11270906031131744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921723163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10637424886226654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921740541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10960125923156738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921740541, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784921740542, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784921768063, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7655860185623169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784921768064, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784921768064, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784921785412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11356334388256073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921802584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11064794659614563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921819820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10893280059099197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921828219, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784921828220, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784921854714, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7667389512062073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784921854714, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784921854714, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784921863459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10951267182826996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921880976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11073066294193268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921898418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11148597300052643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921914921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10527034103870392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921914922, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784921914922, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784921941272, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7682448029518127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784921941272, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784921941272, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784921959906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11472244560718536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921977475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11862578988075256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921995167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11560657620429993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922003965, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784922003965, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784922030282, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7696260213851929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784922030283, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784922030283, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784922038928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11496349424123764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922056682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11474613845348358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922074029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11205749213695526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922091053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11331313103437424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922091055, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784922091055, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784922120190, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7701637148857117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784922120190, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784922120191, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784922137930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11766868829727173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922155586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11797455698251724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922173369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11042679101228714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922182230, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784922182231, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784922209235, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7708371877670288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784922209235, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784922209236, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784922218107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11558092385530472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922235610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11510691791772842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922252728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11135758459568024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922271130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10404989123344421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922271131, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784922271131, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784922297984, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7710654139518738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784922297985, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784922297986, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784922315157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1067165657877922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922332568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1058078408241272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922349879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10575175285339355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922358582, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784922358583, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784922384515, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7712990045547485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784922384515, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784922384515, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784922393178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1038592979311943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922410841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11139846593141556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922427802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10748498141765594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922445701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11253955960273743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922445702, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784922445703, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784922472445, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7717676758766174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784922472445, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784922472445, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784922489885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10679377615451813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922506919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10647456347942352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922524074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10162212699651718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922532503, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784922532503, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784922559191, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7721009850502014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784922559192, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784922559192, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784922567923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10283152014017105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922585405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10180187225341797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922603220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11140365153551102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922619864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10402914136648178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922619865, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784922619865, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784922646575, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7721282839775085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784922646576, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784922646576, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784922664736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10614567995071411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922682460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10640344768762589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922699701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10669532418251038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922708602, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784922708603, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784922735094, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7724341154098511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784922735094, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784922735094, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784922743795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723066329956055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922761127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09976894408464432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922778777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1076837033033371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922796286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10464829951524734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922796287, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784922796287, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784922822944, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.772753119468689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784922822945, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784922822945, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784922839652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772513598203659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922858035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10805671662092209, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922875708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10436046123504639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922884403, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784922884404, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784922910988, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7729889154434204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784922910988, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784922910988, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784922919603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10914454609155655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922937080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772612690925598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922954468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11176042258739471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922972148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081547811627388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922972148, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784922972148, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784922998793, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7732656598091125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784922998794, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784922998794, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784923016340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1049158126115799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923033172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1019463837146759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923051573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.108269102871418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.564999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923060523, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784923060524, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784923086693, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7734629511833191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784923086694, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784923086694, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784923095339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10447770357131958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923112515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09965184330940247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923129914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11441800743341446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.664999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923147345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11088375747203827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923147345, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784923147346, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784923173947, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7736234068870544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784923173948, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784923173948, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784923191412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10647378116846085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923209210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10873275995254517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923226356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10804112255573273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923234734, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784923234735, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784923262475, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7739405632019043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784923262475, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784923262475, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784923272208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10994982719421387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923290333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1145445704460144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923308344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11243587732315063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923326210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11313311755657196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923326211, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784923326212, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784923353312, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7737875580787659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784923353312, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784923353313, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784923371189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11006898432970047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923389266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11387496441602707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.998333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923407042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11076583713293076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923415719, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784923415720, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784923442730, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7741102576255798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784923442730, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784923442731, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784923451347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11090462654829025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923470355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10953503847122192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923488079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11290520429611206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.131666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923505877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11129438877105713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923505878, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784923505878, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784923532350, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7744260430335999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784923532351, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784923532351, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784923550168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11031082272529602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923568182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1128053218126297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923585773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11729985475540161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.264999999999999e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_8192/seed1023733178.log b/recommendation_v4/rcp_logs/gbs_8192/seed1023733178.log new file mode 100644 index 000000000..ab0b3b9bd --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_8192/seed1023733178.log @@ -0,0 +1,1779 @@ +:::MLLOG {"namespace": "", "time_ms": 1785027753977, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1785027753977, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1785027775226, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1785027775228, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1785027775228, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1785027775228, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1785027775228, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1785027775228, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 8192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1785027775228, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1785027775228, "event_type": "POINT_IN_TIME", "key": "seed", "value": 1023733178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1785027775228, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1785027775228, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1785027775228, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1785027775228, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1785027775228, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1785027775229, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1785028699322, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1785028699323, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1785028699641, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1785028821175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13844019174575806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840, "lr": 7.916666666666666e-10}} +:::MLLOG {"namespace": "", "time_ms": 1785028833847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13924850523471832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.6249999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028846936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13864125311374664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 491520, "lr": 2.458333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028860229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13920073211193085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028873660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1391812264919281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 819200, "lr": 4.125e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028887007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13896214962005615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028900396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1392296403646469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1146880, "lr": 5.7916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028913955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13854539394378662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.625e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028927437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13886260986328125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1474560, "lr": 7.458333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028940753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1391156166791916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.291666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028954514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13876685500144958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1802240, "lr": 9.124999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028968074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13872379064559937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028981809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13899289071559906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2129920, "lr": 1.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785028995227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13867613673210144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785028995228, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1785028995228, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1785029081412, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5003319978713989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1785029081413, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1785029081414, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1785029094766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13919948041439056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2457600, "lr": 1.2458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029108425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13895992934703827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029122414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386914849281311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2785280, "lr": 1.4124999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029135997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13917577266693115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4958333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029149782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13906386494636536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3112960, "lr": 1.5791666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029163995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138967365026474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029177624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13893288373947144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3440640, "lr": 1.7458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029191471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13885538280010223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029205949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13895708322525024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3768320, "lr": 1.9124999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029219957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388533115386963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9958333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029234448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13871565461158752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4096000, "lr": 2.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029248821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13866043090820312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.1624999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029262881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388092339038849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4423680, "lr": 2.2458333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029277416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13876329362392426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3291666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029277417, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1785029277417, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1785029364601, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5004019141197205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1785029364602, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1785029364603, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1785029378844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13859286904335022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4751360, "lr": 2.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029393177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13870058953762054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029407514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13864335417747498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5079040, "lr": 2.5791666666666668e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029421989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13807161152362823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029436194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13833846151828766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5406720, "lr": 2.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029450397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860875368118286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029464595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382833570241928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5734400, "lr": 2.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029479330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13812696933746338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029493743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13762766122817993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6062080, "lr": 3.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029507992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13796959817409515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029522371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381940394639969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6389760, "lr": 3.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029536503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13811561465263367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029551703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13831716775894165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6717440, "lr": 3.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029566385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13811111450195312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029566386, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1785029566386, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1785029653076, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5004539489746094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1785029653077, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1785029653077, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1785029667586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13735169172286987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7045120, "lr": 3.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029682273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13796786963939667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029696747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137409046292305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7372800, "lr": 3.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029711844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13815976679325104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.829166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029726481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13775962591171265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7700480, "lr": 3.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029741237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382155418395996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029756033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13720513880252838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8028160, "lr": 4.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029770636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13767804205417633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029785633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13732370734214783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8355840, "lr": 4.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029800630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13847987353801727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3291666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029815581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1369217485189438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8683520, "lr": 4.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029830474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13819539546966553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.4958333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029845288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361556500196457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9011200, "lr": 4.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029860287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13715818524360657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.6624999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029860288, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1785029860288, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1785029946483, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5005810856819153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1785029946484, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1785029946484, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1785029960933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13683250546455383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9338880, "lr": 4.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029975413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13609425723552704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029989602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13746219873428345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9666560, "lr": 4.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030004164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13520598411560059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030018810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13705433905124664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9994240, "lr": 5.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030033300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13435417413711548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1624999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030048189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13620400428771973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10321920, "lr": 5.2458333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030062905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13581609725952148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030077879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379060298204422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10649600, "lr": 5.4124999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030092600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13553258776664734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030107461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14594043791294098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10977280, "lr": 5.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030122598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358208805322647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030137142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13441208004951477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11304960, "lr": 5.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030151716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378592848777771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030151717, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1785030151718, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1785030239222, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5007390379905701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1785030239223, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1785030239223, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1785030253633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13573387265205383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11632640, "lr": 5.9124999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030268059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13941703736782074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030282482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13645726442337036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11960320, "lr": 6.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030296868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337776929140091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030311559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13529525697231293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12288000, "lr": 6.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030326359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1390969455242157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.329166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030340900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1339222490787506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12615680, "lr": 6.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030355812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13353009521961212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030370642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13347071409225464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12943360, "lr": 6.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030385199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13195174932479858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030400041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13176904618740082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13271040, "lr": 6.745833333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030414610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13522541522979736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030429323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13221579790115356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13598720, "lr": 6.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030444369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389688104391098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030444370, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1785030444370, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1785030530931, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5010446906089783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1785030530932, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1785030530932, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1785030545769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13385866582393646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13926400, "lr": 7.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030560257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13351260125637054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030575009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13585223257541656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14254080, "lr": 7.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030589513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353573501110077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030604054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375717669725418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14581760, "lr": 7.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030618626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1285286843776703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030632818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13548016548156738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14909440, "lr": 7.579166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030647238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12945625185966492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030661895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13366350531578064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15237120, "lr": 7.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030676837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13276223838329315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.829166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030691476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13386204838752747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15564800, "lr": 7.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030706327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13437041640281677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.995833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030721670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353992372751236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15892480, "lr": 8.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030736323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13353446125984192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030736324, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785030736324, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785030822545, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5017223358154297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785030822545, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785030822546, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785030837532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13271965086460114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16220160, "lr": 8.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030852522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13408982753753662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030867534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355661004781723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16547840, "lr": 8.412500000000001e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030882708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13558000326156616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030897589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13933536410331726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16875520, "lr": 8.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030912118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13322687149047852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.662499999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030927917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13541437685489655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17203200, "lr": 8.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030943041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13343718647956848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030958206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13311582803726196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17530880, "lr": 8.912499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030973616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13200053572654724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030988536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357586532831192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17858560, "lr": 9.079166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031003905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336282640695572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.162499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031018857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13338139653205872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18186240, "lr": 9.245833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031033559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13097742199897766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031033560, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785031033560, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785031120479, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5028459429740906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785031120480, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785031120481, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785031136038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137786865234375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18513920, "lr": 9.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031151032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13501277565956116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.495833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031166040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354532092809677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18841600, "lr": 9.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031181157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13133224844932556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.662499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031196063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13373054563999176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19169280, "lr": 9.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031211173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13414597511291504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031226416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13700738549232483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19496960, "lr": 9.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031241471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13398508727550507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031256535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329403966665268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19824640, "lr": 1.0079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031271553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298399567604065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0162499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031286291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1277189999818802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20152320, "lr": 1.0245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031301107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13306131958961487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031315998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12976138293743134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20480000, "lr": 1.0412499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031330603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12492197751998901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031330604, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1785031330604, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1785031415825, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.504074215888977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1785031415826, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1785031415826, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1785031430863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13122062385082245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20807680, "lr": 1.0579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031445720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13597489893436432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031460734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133083775639534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21135360, "lr": 1.0745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031475790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13038215041160583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031490161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12857069075107574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21463040, "lr": 1.0912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031505000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12665905058383942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031520064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12912385165691376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21790720, "lr": 1.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031534703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13222643733024597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031549505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13534674048423767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22118400, "lr": 1.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031564426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12498313188552856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031578981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12880586087703705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22446080, "lr": 1.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031593605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13434022665023804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031608376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310667097568512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22773760, "lr": 1.1579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031623391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1291326880455017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031623392, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1785031623392, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1785031708004, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5055705904960632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1785031708005, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1785031708005, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1785031723061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13057023286819458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23101440, "lr": 1.1745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031737726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12917742133140564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031752038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13324013352394104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23429120, "lr": 1.1912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031767114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12407167255878448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031781956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295383870601654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23756800, "lr": 1.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031796186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12594732642173767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031810521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300540566444397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24084480, "lr": 1.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031825053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12170961499214172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031839142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357160359621048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24412160, "lr": 1.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031853786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13728217780590057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031868248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12363668531179428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24739840, "lr": 1.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031883127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12311547994613647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031897863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12568490207195282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25067520, "lr": 1.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031912438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12886250019073486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031912439, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1785031912439, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1785031996358, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5074251294136047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1785031996358, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1785031996359, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1785032010564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13678714632987976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25395200, "lr": 1.2912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032025741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366099715232849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032040423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13582664728164673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25722880, "lr": 1.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032055146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13987931609153748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032069894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365818828344345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26050560, "lr": 1.3245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032084728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138345867395401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032099511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1273384392261505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26378240, "lr": 1.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032114115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12607884407043457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032128777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14236125349998474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26705920, "lr": 1.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032143834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1409456878900528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032158825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1272560954093933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27033600, "lr": 1.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032173631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.128570094704628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032188467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13280849158763885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27361280, "lr": 1.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032202682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1314748227596283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032202683, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1785032202683, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1785032286291, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5096056461334229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1785032286292, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1785032286292, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1785032301642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14145684242248535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27688960, "lr": 1.4079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032316648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14235900342464447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032331305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13525785505771637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28016640, "lr": 1.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032346090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13951604068279266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032361125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14484068751335144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28344320, "lr": 1.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032375748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.140853613615036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032390783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12598732113838196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28672000, "lr": 1.4579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032405866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13755494356155396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032420980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317688226699829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28999680, "lr": 1.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032436145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13607503473758698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032451111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377616971731186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29327360, "lr": 1.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032465697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330672949552536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4995833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032481353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13457965850830078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29655040, "lr": 1.5079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032496794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365053504705429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032496795, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1785032496796, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1785032580026, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5118994116783142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1785032580027, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1785032580027, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1785032595111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386689841747284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29982720, "lr": 1.5245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032610683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13791711628437042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032626351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13175147771835327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30310400, "lr": 1.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032641662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13313131034374237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032656919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13383269309997559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30638080, "lr": 1.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032672470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1398388147354126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032687766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13287124037742615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30965760, "lr": 1.5745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032703087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14129826426506042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032717712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14063970744609833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31293440, "lr": 1.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032732820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13531534373760223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032747590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1276269257068634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31621120, "lr": 1.6079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032762049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12989234924316406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032776471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13407166302204132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31948800, "lr": 1.6245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032791133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12765942513942719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032791134, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1785032791134, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1785032871926, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5144737958908081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1785032871927, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1785032871927, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1785032886507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321432739496231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32276480, "lr": 1.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032901034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13678477704524994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032915742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335994005203247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32604160, "lr": 1.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032930408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1319030076265335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032945152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13400804996490479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32931840, "lr": 1.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032959863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13356629014015198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032975015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338094025850296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33259520, "lr": 1.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032989812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327255815267563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033004231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13982409238815308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33587200, "lr": 1.7079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033019398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318349391222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033034162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12891778349876404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33914880, "lr": 1.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033048725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13128729164600372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033063450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13227564096450806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34242560, "lr": 1.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033078327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13593457639217377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7495833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033078327, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1785033078327, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1785033161006, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5178228616714478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1785033161007, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1785033161008, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1785033175923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13349846005439758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34570240, "lr": 1.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033190656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12296018749475479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033205510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13061143457889557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34897920, "lr": 1.7745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033220420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304449737071991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033235174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13549520075321198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35225600, "lr": 1.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033250439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13539214432239532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033265254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13539369404315948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35553280, "lr": 1.8079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033280518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12430291622877121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8162500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033295457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311788558959961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35880960, "lr": 1.8245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033309801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13119295239448547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033324771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13045194745063782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36208640, "lr": 1.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033339381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333233267068863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033354135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1309577077627182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36536320, "lr": 1.8579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033368735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13020072877407074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033368736, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1785033368736, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1785033450874, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5224289894104004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1785033450874, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1785033450875, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1785033465673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378408670425415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36864000, "lr": 1.8745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033480413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13242915272712708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033495128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13361626863479614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37191680, "lr": 1.8912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033509947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13039182126522064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033524695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12702296674251556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37519360, "lr": 1.9079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033539434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320601999759674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033554730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13574156165122986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37847040, "lr": 1.9245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033569846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329241544008255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033584784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1313304752111435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38174720, "lr": 1.9412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033600198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13897982239723206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033615354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13392767310142517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38502400, "lr": 1.9579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033629880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13753411173820496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033645039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13201560080051422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38830080, "lr": 1.9745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033659833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13335324823856354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033659834, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1785033659834, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1785033742163, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.528555154800415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1785033742164, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1785033742164, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1785033757002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13131771981716156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39157760, "lr": 1.9912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033771769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13495539128780365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033786500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13797211647033691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39485440, "lr": 2.0079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033801415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13339857757091522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033815811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.127409428358078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39813120, "lr": 2.0245833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033830460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12955725193023682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0329166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033845077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1259702742099762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40140800, "lr": 2.0412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033859683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12410575896501541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033874684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13623842597007751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40468480, "lr": 2.0579166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033889608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13744521141052246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033904614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13457749783992767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40796160, "lr": 2.0745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033920012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12692728638648987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0829166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033934962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308896690607071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41123840, "lr": 2.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033949152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1235237643122673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033949153, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1785033949153, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1785034032820, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5375018119812012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1785034032821, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1785034032821, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1785034047845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13237468898296356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41451520, "lr": 2.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034062882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13264650106430054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034077876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13385668396949768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41779200, "lr": 2.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034092981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13801532983779907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034108010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13603994250297546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42106880, "lr": 2.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034122928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315443366765976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034137776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12732350826263428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42434560, "lr": 2.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034152783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350843906402588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034167570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12461381405591965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42762240, "lr": 2.1745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034182706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304798722267151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034197677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324165314435959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43089920, "lr": 2.1912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034212799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13409270346164703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034228035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14024177193641663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43417600, "lr": 2.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034243034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13629133999347687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034243035, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1785034243035, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1785034324754, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5504738092422485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1785034324755, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1785034324755, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1785034340206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.136746346950531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43745280, "lr": 2.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034355246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13165283203125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034369884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13286538422107697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44072960, "lr": 2.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034384773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13908126950263977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034400047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1299828737974167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44400640, "lr": 2.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034415625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13847024738788605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034430505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13504919409751892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44728320, "lr": 2.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034445543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13722334802150726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034460482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13486097753047943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45056000, "lr": 2.2912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034475561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13082918524742126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034490603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14060133695602417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45383680, "lr": 2.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034505784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13474735617637634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034520885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1290067583322525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45711360, "lr": 2.3245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034536084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13345292210578918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034536084, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1785034536085, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1785034619036, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5654403567314148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1785034619037, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1785034619037, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1785034634560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14187422394752502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46039040, "lr": 2.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034649722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13255751132965088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034665354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13355548679828644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46366720, "lr": 2.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034680465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13061444461345673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034695914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13546225428581238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46694400, "lr": 2.3745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034711194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13362394273281097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034726550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13403567671775818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47022080, "lr": 2.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034741922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13557080924510956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034757136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312486082315445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47349760, "lr": 2.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034772722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1328807771205902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034788255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1328924149274826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47677440, "lr": 2.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034803739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13701915740966797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034818865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13657203316688538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48005120, "lr": 2.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034834398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13739155232906342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.4495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034834398, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1785034834399, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1785034917361, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5814656019210815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1785034917362, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1785034917362, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1785034932631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13182333111763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48332800, "lr": 2.4579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034948239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13526400923728943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034963537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13415396213531494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48660480, "lr": 2.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034979412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13227973878383636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034994771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371677815914154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48988160, "lr": 2.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035009891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1296600103378296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035025225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133645161986351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49315840, "lr": 2.507916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035039896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1240004301071167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.5162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035054658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312023550271988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49643520, "lr": 2.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035069311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13148286938667297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035084033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13125433027744293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49971200, "lr": 2.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035098643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13142862915992737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.5495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035113383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12888780236244202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50298880, "lr": 2.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035128133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1314784437417984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.5662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035128133, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1785035128133, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1785035210565, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5955988168716431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1785035210566, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1785035210566, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1785035225182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12236940860748291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50626560, "lr": 2.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035240206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1270102858543396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035254833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12920676171779633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50954240, "lr": 2.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035269722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13318559527397156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035284843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131682887673378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51281920, "lr": 2.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035299590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12196451425552368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035314757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13060317933559418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51609600, "lr": 2.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035330031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320761889219284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035345258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1372046172618866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51937280, "lr": 2.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035359854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12813425064086914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.6495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035375330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332254260778427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52264960, "lr": 2.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035390614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11818988621234894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.6662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035405932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12163963913917542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52592640, "lr": 2.6745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035421132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12536531686782837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035421133, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1785035421134, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1785035505952, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6086992025375366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1785035505953, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1785035505953, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1785035521115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12168128788471222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52920320, "lr": 2.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035536123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1278877556324005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.6995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035551355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289099007844925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53248000, "lr": 2.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035566480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13237327337265015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035581585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1218152865767479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53575680, "lr": 2.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035596795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13530179858207703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035611849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12982019782066345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53903360, "lr": 2.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035626790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12187526375055313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.7495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035642102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1275699883699417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54231040, "lr": 2.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035657541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12771379947662354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035672703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11900296807289124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54558720, "lr": 2.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035688525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13176558911800385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035703989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12750427424907684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54886400, "lr": 2.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035719332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12912344932556152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035719333, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1785035719333, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1785035804515, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6209641098976135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1785035804515, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1785035804516, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1785035819133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12854643166065216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55214080, "lr": 2.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035834548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11914294213056564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035849909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12242341041564941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55541760, "lr": 2.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035865348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12146605551242828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035880517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12374625355005264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55869440, "lr": 2.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035895700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12659117579460144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035910859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1168239414691925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56197120, "lr": 2.8579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035926108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12282834947109222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035941032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12572936713695526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56524800, "lr": 2.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035956113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12396722286939621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035971019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11580073833465576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56852480, "lr": 2.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035986246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12398310750722885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036001261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11018551886081696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57180160, "lr": 2.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036016419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12440257519483566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036016420, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1785036016420, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1785036101017, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6342817544937134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1785036101018, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1785036101018, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1785036116319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1169544979929924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57507840, "lr": 2.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036131556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11475557088851929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036146619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13051199913024902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57835520, "lr": 2.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036161958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12209319323301315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036177364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1240401491522789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58163200, "lr": 2.9579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036192343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11670642346143723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036206800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1203446313738823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58490880, "lr": 2.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036222119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12198865413665771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036237213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12093774974346161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58818560, "lr": 2.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036252242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11371960490942001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036267234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12036240100860596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59146240, "lr": 3.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036282004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12192369997501373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.0162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036296896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12463582307100296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59473920, "lr": 3.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036312012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11731425672769547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036312013, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1785036312013, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1785036397457, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6630823016166687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1785036397458, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1785036397458, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1785036412376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11976276338100433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59801600, "lr": 3.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036427066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1120373085141182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036441906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10959010571241379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60129280, "lr": 3.0579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036456770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11322810500860214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.0662499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036471756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10892120003700256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60456960, "lr": 3.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036486802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12269294261932373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036502157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11589140444993973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60784640, "lr": 3.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036517698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1199980229139328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036532970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1032484918832779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61112320, "lr": 3.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036548727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12001372128725052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.1162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036564328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11300214380025864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61440000, "lr": 3.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036579694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11331827193498611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036594703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10906418412923813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61767680, "lr": 3.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036609835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11657495051622391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036609836, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1785036609836, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1785036694445, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7156702280044556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1785036694446, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1785036694446, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1785036709358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12012599408626556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62095360, "lr": 3.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036724409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11934267729520798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.1662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036739621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12281270325183868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62423040, "lr": 3.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036754985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10547930747270584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036770164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11430314928293228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62750720, "lr": 3.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036785141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1167711541056633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036800277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11920246481895447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63078400, "lr": 3.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036815286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11829275637865067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036830084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10815715044736862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63406080, "lr": 3.2245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036845056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11700911074876785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036859938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11476151645183563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63733760, "lr": 3.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036874595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11553287506103516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036889322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11197122931480408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64061440, "lr": 3.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036904606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12482064217329025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036904606, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1785036904607, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1785036989222, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.742989182472229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1785036989223, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1785036989223, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1785037004435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12384270131587982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64389120, "lr": 3.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037019490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11565243452787399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037034429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1165732741355896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64716800, "lr": 3.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037049861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12088752537965775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037065035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10800241678953171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65044480, "lr": 3.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037080208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11149990558624268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037094707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11228587478399277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65372160, "lr": 3.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037109846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10783343017101288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037124922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11113902181386948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65699840, "lr": 3.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037140459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11592024564743042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037155710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12137234210968018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66027520, "lr": 3.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037170918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11424882709980011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037186378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1261860579252243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66355200, "lr": 3.3745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037201726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1110435500741005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037201727, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1785037201727, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1785037286019, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7530915141105652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1785037286019, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1785037286020, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1785037301432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1258150041103363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66682880, "lr": 3.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037316897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11251317709684372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.3995833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037332293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11600583046674728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67010560, "lr": 3.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037347846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11313408613204956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037363269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11944250017404556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67338240, "lr": 3.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037378932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.117516428232193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037394687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12056851387023926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67665920, "lr": 3.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037410647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11993435770273209, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037426113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11492573469877243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67993600, "lr": 3.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037442296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11233106255531311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037457882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11487557739019394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68321280, "lr": 3.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037473314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1146988719701767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037488553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12369818985462189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68648960, "lr": 3.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037504269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11389931291341782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037504270, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1785037504270, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1785037587632, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7565168738365173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1785037587633, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1785037587633, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1785037602955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11974117159843445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68976640, "lr": 3.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037618451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11940973997116089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037633871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11152268201112747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69304320, "lr": 3.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037649374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11348595470190048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037664793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10769302397966385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69632000, "lr": 3.5412500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037680445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11312660574913025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037695853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11624450236558914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69959680, "lr": 3.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037710902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1100059226155281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037726285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10944774746894836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70287360, "lr": 3.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037741516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10691916197538376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037757175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12271939218044281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70615040, "lr": 3.5912500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037772870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10515876859426498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037788394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12026418745517731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70942720, "lr": 3.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037804345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11967593431472778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037804345, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1785037804345, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1785037887417, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7569957971572876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1785037887417, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1785037887418, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1785037902951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11813376843929291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71270400, "lr": 3.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037918102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11644461005926132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037933330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11523988097906113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71598080, "lr": 3.6412499999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037949027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11471188068389893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037964468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10267199575901031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71925760, "lr": 3.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037979965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10289730876684189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037995363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11764771491289139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72253440, "lr": 3.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038010635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11454169452190399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038025913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10259499400854111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72581120, "lr": 3.6912499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038041132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10183662176132202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038056123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10559078305959702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72908800, "lr": 3.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038071521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11180499941110611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038086816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11070932447910309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73236480, "lr": 3.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038102044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1133958101272583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038102082, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1785038102083, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1785038183065, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7583979368209839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1785038183066, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1785038183066, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1785038198424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10412256419658661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73564160, "lr": 3.7412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038214103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10592866688966751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038229708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11436646431684494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73891840, "lr": 3.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038245242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10882340371608734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038260909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10762213170528412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74219520, "lr": 3.7745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038276655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11313130706548691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038292024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.113291896879673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74547200, "lr": 3.7912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038307184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1119745671749115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038322459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11589708179235458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74874880, "lr": 3.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038337746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11036043614149094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038352848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10061279684305191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75202560, "lr": 3.8245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038368066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11275241523981094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038383227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10707775503396988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75530240, "lr": 3.8412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038398489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10608989745378494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038398490, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1785038398490, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1785038480985, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7597151398658752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1785038480985, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1785038480986, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1785038496136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10554756969213486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75857920, "lr": 3.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038511490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11347124725580215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038526524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09553229063749313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76185600, "lr": 3.8745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038541527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10432089865207672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038556580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11273107677698135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76513280, "lr": 3.8912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038571497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10478318482637405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038586666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08832807093858719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76840960, "lr": 3.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038601801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082974374294281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038617148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10111813247203827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77168640, "lr": 3.9245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038632537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11682692170143127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038648139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10649792850017548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77496320, "lr": 3.9412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038663446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10052849352359772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038679109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10034815967082977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77824000, "lr": 3.9579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038694628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10766758024692535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038694629, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1785038694629, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1785038778227, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7616530060768127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1785038778228, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1785038778228, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1785038793584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11566861718893051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78151680, "lr": 3.9745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038808840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11218993365764618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038823443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10297313332557678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78479360, "lr": 3.9912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038838967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10468433052301407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038853813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09950383752584457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78807040, "lr": 4.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038868988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10224831104278564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038884056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10613743960857391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79134720, "lr": 4.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038899172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10704206675291061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038914086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09773766249418259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79462400, "lr": 4.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038929321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1090969666838646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038944269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1045079156756401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79790080, "lr": 4.0579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038959275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10663264244794846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038974276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10843393951654434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80117760, "lr": 4.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038989410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11342356353998184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038989410, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1785038989411, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1785039072674, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7626348733901978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1785039072676, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1785039072676, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1785039087920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11190195381641388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80445440, "lr": 4.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039102781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09919563680887222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039118064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10202956944704056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80773120, "lr": 4.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039133407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10527824610471725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039148902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11163237690925598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81100800, "lr": 4.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039164388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11301141232252121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039179748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10002952814102173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81428480, "lr": 4.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039195283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1057560071349144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039210762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1070713922381401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81756160, "lr": 4.1579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039225974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10655330121517181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039241314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11259772628545761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82083840, "lr": 4.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039256043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09414374828338623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039271726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09809128940105438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82411520, "lr": 4.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039286948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10699663311243057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039286949, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1785039286949, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1785039370260, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7646677494049072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1785039370261, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1785039370261, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1785039385308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0924614667892456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82739200, "lr": 4.2079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039400561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10494410991668701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039415743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1090574711561203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83066880, "lr": 4.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039431218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09711456298828125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039446554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11121891438961029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83394560, "lr": 4.2412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039461886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11137281358242035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039477266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09709372371435165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83722240, "lr": 4.2579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039492569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11162955313920975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039508058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10578056424856186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84049920, "lr": 4.2745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039523358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10487069934606552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039538762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10756123065948486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84377600, "lr": 4.2912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039553676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10512350499629974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039568459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09750806540250778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84705280, "lr": 4.3079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039583532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10726872831583023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039583533, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1785039583533, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1785039668354, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.764612078666687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1785039668355, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1785039668355, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1785039683896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10409193485975266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85032960, "lr": 4.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039699394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1051609069108963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039714529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1054002046585083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85360640, "lr": 4.3412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039729860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10398494452238083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039745335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09818893671035767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85688320, "lr": 4.3579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039760605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09996767342090607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039776131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10632104426622391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86016000, "lr": 4.3745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039790857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09674769639968872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039806329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10817109793424606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86343680, "lr": 4.3912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039821824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10883419215679169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039837208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11632376909255981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86671360, "lr": 4.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039853002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11468056589365005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039868412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11670265346765518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86999040, "lr": 4.4245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039883867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11583961546421051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039883868, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1785039883868, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1785039967364, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7663647532463074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1785039967365, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1785039967365, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1785039982617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10141310840845108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87326720, "lr": 4.4412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039997984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11626933515071869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040013070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10784891992807388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87654400, "lr": 4.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040028490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10122011601924896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040043756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09124260395765305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87982080, "lr": 4.4745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040059136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0992942526936531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040074324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10385582596063614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88309760, "lr": 4.4912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040089717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10696138441562653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040104837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10726891458034515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88637440, "lr": 4.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040120037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10115097463130951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040135780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11379917711019516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88965120, "lr": 4.5245833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040151416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10032093524932861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040167068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10997994244098663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89292800, "lr": 4.5412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040182552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11157351732254028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.5495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040182552, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1785040182553, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1785040266615, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7659134864807129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1785040266615, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1785040266616, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1785040282224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10785884410142899, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89620480, "lr": 4.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040297758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10164423286914825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040313223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11136256158351898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89948160, "lr": 4.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040328877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09162995964288712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040344013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11502698063850403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90275840, "lr": 4.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040359070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10712441802024841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040374897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11149239540100098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90603520, "lr": 4.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040390343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11245548725128174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040406096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12205622345209122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90931200, "lr": 4.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040421667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10464732348918915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040437080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11098726093769073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91258880, "lr": 4.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040452603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1225229799747467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040468186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09935721009969711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91586560, "lr": 4.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040483841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11118056625127792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040483841, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1785040483842, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1785040567406, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7658413052558899, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1785040567407, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1785040567407, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1785040582911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10414224117994308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91914240, "lr": 4.6745833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040598279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11815498769283295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040613539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10992925614118576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92241920, "lr": 4.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040629201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10665899515151978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040644711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09693682938814163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92569600, "lr": 4.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040660163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11281440407037735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040675946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10660535842180252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92897280, "lr": 4.7245833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040691696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10532119870185852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040707267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11664056777954102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93224960, "lr": 4.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040722769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09977542608976364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040738591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12530894577503204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93552640, "lr": 4.7579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040754402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10734692960977554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040769828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11761261522769928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93880320, "lr": 4.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040785114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10835257917642593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040785115, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1785040785115, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1785040870806, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7661381959915161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1785040870807, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1785040870807, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1785040886364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1142486110329628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94208000, "lr": 4.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040902273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1152065247297287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040918366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1117226630449295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94535680, "lr": 4.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040934577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11786635220050812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040950667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1143484115600586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94863360, "lr": 4.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040966867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11924447864294052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040983154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12704825401306152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95191040, "lr": 4.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040999100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12067095190286636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041014993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11667440086603165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95518720, "lr": 4.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041031170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11126758903265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041046998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10857183486223221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95846400, "lr": 4.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041062963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10847475379705429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041078812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11529204994440079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96174080, "lr": 4.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041094802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11491795629262924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041094803, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1785041094803, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1785041181771, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7677146196365356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1785041181772, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1785041181772, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1785041197842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11963820457458496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96501760, "lr": 4.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041213872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10770595818758011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041229404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10828570276498795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96829440, "lr": 4.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041245505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10844992101192474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041261384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11090495437383652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97157120, "lr": 4.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041277277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11019238829612732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041292924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1142561212182045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97484800, "lr": 4.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041308315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10799622535705566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041323712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1060815304517746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97812480, "lr": 4.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041338735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10010023415088654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041354237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10466131567955017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98140160, "lr": 4.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041369304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09709160774946213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041384491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10482507199048996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98467840, "lr": 5.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041399617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10093088448047638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041399618, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1785041399618, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1785041485260, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.768900454044342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1785041485261, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1785041485262, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1785041500330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10912559181451797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98795520, "lr": 5.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041515552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1026235967874527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041530807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09685064107179642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99123200, "lr": 5.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041546052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10865399241447449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041561450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11548111587762833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99450880, "lr": 5.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041576579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1048249825835228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041591718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10577867180109024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99778560, "lr": 5.074583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041607042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10466279089450836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041622322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10194838792085648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100106240, "lr": 5.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041637687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12002065777778625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041653239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11742866039276123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100433920, "lr": 5.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041668731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10108470171689987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041684562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.100852832198143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100761600, "lr": 5.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041700052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12080330401659012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041700053, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1785041700054, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1785041784001, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7691178917884827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1785041784002, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1785041784002, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1785041800083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1114792674779892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101089280, "lr": 5.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041815745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0965057909488678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041831126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10489171743392944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101416960, "lr": 5.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041846624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10387007892131805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041861729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10938257724046707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101744640, "lr": 5.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041877116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10531437397003174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.182916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041892510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10671354085206985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102072320, "lr": 5.191249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041907796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1123092770576477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041923305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10489947348833084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102400000, "lr": 5.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041938777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10782451927661896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041954048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10554792732000351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102727680, "lr": 5.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041969292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1152544617652893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041984357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10630789399147034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103055360, "lr": 5.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041999613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10209311544895172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041999614, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1785041999614, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1785042082323, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7697376012802124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1785042082324, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1785042082324, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1785042097643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10000385344028473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103383040, "lr": 5.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042113178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09489002078771591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042128324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09593844413757324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103710720, "lr": 5.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042143584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09669487923383713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.282916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042158487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09691113978624344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104038400, "lr": 5.291249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042173769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09378800541162491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042188796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10033750534057617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104366080, "lr": 5.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042203869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09740402549505234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042219298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10344599187374115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104693760, "lr": 5.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042234684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10857328027486801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042250436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10479602962732315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105021440, "lr": 5.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042265969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10404420644044876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042281519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11429620534181595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105349120, "lr": 5.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042297611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11135905981063843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042297611, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1785042297612, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1785042381150, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7703872323036194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1785042381151, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1785042381151, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1785042396707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09856442362070084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105676800, "lr": 5.374583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042412288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1075512245297432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.382916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042428036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10996353626251221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106004480, "lr": 5.391249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042443339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1015990749001503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042458656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09998466819524765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106332160, "lr": 5.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042473820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10687122493982315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042488999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11067895591259003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106659840, "lr": 5.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042504358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10645583271980286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042520007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1027696281671524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106987520, "lr": 5.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042535226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10095632076263428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042550605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09978649020195007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107315200, "lr": 5.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042565893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1024080291390419, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042581125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10942089557647705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107642880, "lr": 5.474583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042596330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11433225125074387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.482916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042596330, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1785042596331, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1785042680302, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7702544331550598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1785042680302, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1785042680303, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1785042695607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10987614840269089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107970560, "lr": 5.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042710946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10667978227138519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042726254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0912221297621727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108298240, "lr": 5.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042741537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10345527529716492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042757065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10643618553876877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108625920, "lr": 5.524583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042772286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10820174962282181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042787534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11065710335969925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108953600, "lr": 5.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042803000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10387618839740753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042818561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09952940046787262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109281280, "lr": 5.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042834164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039847657084465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042849912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1026851087808609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109608960, "lr": 5.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042865200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10871800780296326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042880912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10567653924226761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109936640, "lr": 5.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042896546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10892289131879807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042896546, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1785042896547, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1785042979495, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7711983919143677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1785042979496, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1785042979496, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1785042995040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10281995683908463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110264320, "lr": 5.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043010581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1038026511669159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043025869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10418958216905594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110592000, "lr": 5.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043040791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10290750861167908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043056623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10713807493448257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110919680, "lr": 5.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043072007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1012154072523117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043087457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09878566116094589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111247360, "lr": 5.657916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043103071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10212603211402893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.666250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043118586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11201440542936325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111575040, "lr": 5.674583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043134082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10518346726894379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043149442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10387296229600906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111902720, "lr": 5.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043164618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10241600126028061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043180087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10441216826438904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112230400, "lr": 5.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043195337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10716909915208817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043195337, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1785043195338, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1785043279720, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7712945342063904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1785043279721, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1785043279722, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1785043295085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10854844003915787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112558080, "lr": 5.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043310659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10197083652019501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043326107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10375738888978958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112885760, "lr": 5.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043341596, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10109477490186691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043357105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10445991903543472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113213440, "lr": 5.757916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043372054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11219404637813568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.766250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043387370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10125410556793213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113541120, "lr": 5.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043402585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11075993627309799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043418106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11291006207466125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113868800, "lr": 5.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043433871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10658794641494751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043449425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10312892496585846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114196480, "lr": 5.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043464914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10451953113079071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043480687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10504025965929031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114524160, "lr": 5.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043496121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09920656681060791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043496121, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1785043496122, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1785043580420, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7718395590782166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1785043580421, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1785043580421, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1785043595912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10142170637845993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114851840, "lr": 5.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043611312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09944124519824982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043626690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10678368806838989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115179520, "lr": 5.857916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043641602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10089563578367233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043657302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085607036948204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115507200, "lr": 5.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043672379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11008895188570023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043687802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10802015662193298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115834880, "lr": 5.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043703363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.115870900452137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043718774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11404488235712051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116162560, "lr": 5.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043734023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10034260153770447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043749296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10393059253692627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116490240, "lr": 5.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043764648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10301709920167923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043779948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10538244992494583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116817920, "lr": 5.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043795165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10739403963088989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043795166, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1785043795166, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1785043878641, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7718482613563538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1785043878641, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1785043878642, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1785043894018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10208604484796524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117145600, "lr": 5.957916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043909369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10037367790937424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043924932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1091918796300888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117473280, "lr": 5.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043940223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10544835776090622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043955534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0943378359079361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117800960, "lr": 5.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043971248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11241808533668518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043986386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1014930009841919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118128640, "lr": 6.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044001645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793294757604599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044017187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10340537875890732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118456320, "lr": 6.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044032836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10361028462648392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044048170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10976092517375946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118784000, "lr": 6.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044063704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11415565013885498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044079245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1061646044254303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119111680, "lr": 6.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044094953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10294665396213531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044094953, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1785044094954, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1785044178324, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7721467018127441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1785044178325, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1785044178325, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1785044194136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11294101923704147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119439360, "lr": 6.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044209739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10966439545154572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044225193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1095907911658287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119767040, "lr": 6.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044240855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10676144063472748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044255983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10145042091608047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120094720, "lr": 6.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044270967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10785739868879318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044287143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11827811598777771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120422400, "lr": 6.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044303067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10922399163246155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044318842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11621065437793732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120750080, "lr": 6.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044334921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12265443801879883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044350693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732336342334747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121077760, "lr": 6.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044366478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11029677093029022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044382453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11588132381439209, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121405440, "lr": 6.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044398061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1101212427020073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044398062, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1785044398062, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1785044481220, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7717768549919128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1785044481221, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1785044481222, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1785044496925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1133842021226883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121733120, "lr": 6.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044512774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10957717150449753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044528488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11105159670114517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122060800, "lr": 6.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044544376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10667456686496735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044560158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10445009171962738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122388480, "lr": 6.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044575979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11387960612773895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044591711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10693113505840302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122716160, "lr": 6.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044607563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11471643298864365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044623624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11214888095855713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123043840, "lr": 6.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044639603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.119716577231884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044655676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10587132722139359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123371520, "lr": 6.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044671645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11747776716947556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044687523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10701770335435867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123699200, "lr": 6.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044703541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11889071762561798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044703541, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1785044703541, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1785044787078, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7721139192581177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1785044787079, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1785044787079, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1785044803112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11968272924423218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124026880, "lr": 6.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044818910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10831928998231888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044834699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11920900642871857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124354560, "lr": 6.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044850149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11388079077005386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044865726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10774396359920502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124682240, "lr": 6.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044881703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11546460539102554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044897508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11189161986112595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125009920, "lr": 6.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044913606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11724760383367538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044929562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11322631686925888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125337600, "lr": 6.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044945356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10649997740983963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044961057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1126059889793396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125665280, "lr": 6.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044976810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10545646399259567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044992458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11673485487699509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125992960, "lr": 6.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045008373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1055169478058815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045008374, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1785045008374, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1785045092782, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7728986740112305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1785045092783, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1785045092783, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1785045108608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11009503155946732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126320640, "lr": 6.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045124568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12148874998092651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045140257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10982910543680191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126648320, "lr": 6.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045155939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11336059123277664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045171691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11599365621805191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126976000, "lr": 6.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045187418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10925886780023575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045203327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1088276207447052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127303680, "lr": 6.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045219278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11018630862236023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.482916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045235384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11263208091259003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127631360, "lr": 6.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045251145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11225865036249161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045267127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11030004918575287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127959040, "lr": 6.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045283236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11095906049013138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045299167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11187516897916794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128286720, "lr": 6.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045315030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10932359844446182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045315031, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1785045315031, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1785045400494, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7732216715812683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1785045400495, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1785045400495, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1785045416006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11795496195554733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128614400, "lr": 6.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045431759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11051807552576065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045447298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11027785390615463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128942080, "lr": 6.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045462778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10331521928310394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045478305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10554412752389908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129269760, "lr": 6.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045494008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11147075891494751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045509378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10804431885480881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129597440, "lr": 6.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045524815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10819686204195023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045540382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09861279278993607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129925120, "lr": 6.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045555837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10166864842176437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045571448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10465064644813538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130252800, "lr": 6.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045586765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10494239628314972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045602389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11013364046812057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130580480, "lr": 6.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045618074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11627517640590668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045618075, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1785045618075, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1785045703307, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7737046480178833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1785045703308, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1785045703308, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1785045718703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1142604798078537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130908160, "lr": 6.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045734211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10487498342990875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045749727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10041959583759308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131235840, "lr": 6.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045765572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09229335933923721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045781224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11416031420230865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131563520, "lr": 6.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045796758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10470795631408691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045812642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10260246694087982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131891200, "lr": 6.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045828739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11250803619623184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045844444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10735689103603363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132218880, "lr": 6.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045860552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10308591276407242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045876581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1095443144440651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132546560, "lr": 6.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045892582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1123255267739296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045908724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10735747963190079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132874240, "lr": 6.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045924782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10714593529701233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045924782, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1785045924783, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1785046010823, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7739094495773315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1785046010824, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1785046010824, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1785046026382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10981611162424088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133201920, "lr": 6.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046042115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10307557880878448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046057530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11051390320062637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133529600, "lr": 6.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046072651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09922697395086288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046088807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11584432423114777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133857280, "lr": 6.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046104424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10990425944328308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046119912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10344468802213669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134184960, "lr": 6.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046135819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10867152363061905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046151580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10843443870544434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134512640, "lr": 6.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046167015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10797779262065887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046182593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11036918312311172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134840320, "lr": 6.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046198055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0993514284491539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046213289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10656377673149109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135168000, "lr": 6.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046228446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08717922866344452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046228447, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1785046228447, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1785046313603, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7739593386650085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1785046313604, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1785046313604, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1785046328975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.095250204205513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135495680, "lr": 6.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046344351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11129730939865112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046359761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10753466188907623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135823360, "lr": 6.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046375003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1014895811676979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046390542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10221555829048157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136151040, "lr": 6.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046405982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09035900980234146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046421207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10285398364067078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136478720, "lr": 6.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046436440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09753704071044922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046451737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10311660915613174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136806400, "lr": 6.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046467255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10641548782587051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046482791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11003240197896957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137134080, "lr": 6.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046498267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10797853767871857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046513530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10620857775211334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137461760, "lr": 6.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046528957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10338231176137924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046528958, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1785046528958, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1785046613331, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7741925120353699, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1785046613332, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1785046613333, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1785046629060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09621137380599976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137789440, "lr": 7.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046644888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10546307265758514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046660376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09478022903203964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138117120, "lr": 7.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046675889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1021609902381897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046691280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09826486557722092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138444800, "lr": 7.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046706357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10040194541215897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046721299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10934709012508392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138772480, "lr": 7.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046737176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10137784481048584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.066249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046752654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10926609486341476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139100160, "lr": 7.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046768261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10973326116800308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046784179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10665786266326904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139427840, "lr": 7.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046799956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10426990687847137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046815518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084439605474472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139755520, "lr": 7.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046830972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10861820727586746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046830972, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1785046830973, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1785046913498, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7745561599731445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1785046913499, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1785046913499, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1785046928878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10644318908452988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140083200, "lr": 7.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046944334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10405196994543076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046959833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10314298421144485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140410880, "lr": 7.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046975584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10012508928775787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046991284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10695026814937592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140738560, "lr": 7.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047007037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09588398039340973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047023324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10503676533699036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141066240, "lr": 7.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047038965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10339133441448212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047054292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11007937788963318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141393920, "lr": 7.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047070198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09890183806419373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047085532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09765660762786865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141721600, "lr": 7.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047100911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10233468562364578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047116559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09087993949651718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142049280, "lr": 7.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047132184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072629913687706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047132185, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1785047132185, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1785047216305, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7748035788536072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1785047216307, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1785047216307, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1785047232098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09924429655075073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142376960, "lr": 7.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047247977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10121461749076843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047263457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0986662358045578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142704640, "lr": 7.257916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047279427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10941512137651443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047295210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10145904123783112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143032320, "lr": 7.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047311153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10288138687610626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047326691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10235581547021866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143360000, "lr": 7.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047342232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10527245700359344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047358067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10142175108194351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143687680, "lr": 7.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047373225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11274166405200958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047388842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1010979413986206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144015360, "lr": 7.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047404773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10830564796924591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047420318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10580617189407349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144343040, "lr": 7.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047436067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10611627250909805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047436067, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1785047436068, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1785047520237, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7749396562576294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1785047520238, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1785047520239, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1785047535798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10621857643127441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144670720, "lr": 7.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047551445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09854690730571747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047566909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10691629350185394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144998400, "lr": 7.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047582699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10651107877492905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047598343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10228796303272247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145326080, "lr": 7.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047613793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10767363011837006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047629309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09720925986766815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145653760, "lr": 7.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047645119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09834668785333633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047660786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10572017729282379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145981440, "lr": 7.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047676527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1050526350736618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047692303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10224377363920212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146309120, "lr": 7.441250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047708175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10982493311166763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047723902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10666386038064957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146636800, "lr": 7.457916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047739705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0992799699306488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047739705, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1785047739706, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1785047824949, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752670645713806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1785047824950, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1785047824950, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1785047840523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10326206684112549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146964480, "lr": 7.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047856098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10568917542695999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047871775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09432324767112732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147292160, "lr": 7.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047887725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09890106320381165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047903314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10273885726928711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147619840, "lr": 7.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047919341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09997005015611649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047935159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10665712505578995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147947520, "lr": 7.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047951119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11026348173618317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047966943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10886025428771973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148275200, "lr": 7.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047982835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09928399324417114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047998799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10287144780158997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148602880, "lr": 7.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048014204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09859870374202728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048030175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11218224465847015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148930560, "lr": 7.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048045594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11099744588136673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048045595, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1785048045595, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1785048129492, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7755571603775024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1785048129493, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1785048129493, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1785048144550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10588846355676651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149258240, "lr": 7.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048160661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10488937050104141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048176163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10607078671455383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149585920, "lr": 7.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048191882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11132410913705826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149749760, "lr": 7.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048207790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10638770461082458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149913600, "lr": 7.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048223547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10791976004838943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048238961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10720154643058777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150241280, "lr": 7.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048254546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0978962630033493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150405120, "lr": 7.649583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048270234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10620579868555069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150568960, "lr": 7.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048285563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10330219566822052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048301105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10844404995441437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150896640, "lr": 7.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048316296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10340584069490433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151060480, "lr": 7.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048332033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1064678430557251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151224320, "lr": 7.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048347492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09575255215167999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048347532, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1785048347532, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1785048431070, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7754956483840942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1785048431071, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1785048431071, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1785048446444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09934557229280472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151552000, "lr": 7.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048461737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10528886318206787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151715840, "lr": 7.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048477153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10207775235176086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151879680, "lr": 7.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048492589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10445945709943771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048507901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10812358558177948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152207360, "lr": 7.741249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048522934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12312372028827667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152371200, "lr": 7.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048538228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10810960829257965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152535040, "lr": 7.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048553676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09897647053003311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048569295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1114044189453125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152862720, "lr": 7.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048584797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11121302843093872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153026560, "lr": 7.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048600573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11476586759090424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153190400, "lr": 7.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048616244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0983632281422615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048632089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10658685117959976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153518080, "lr": 7.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048647912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10458948463201523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153681920, "lr": 7.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048647950, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1785048647950, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1785048732347, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7759877443313599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1785048732348, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1785048732348, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1785048747908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0979757010936737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153845760, "lr": 7.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048763472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09727533906698227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.832916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048778980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1127978190779686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154173440, "lr": 7.841249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048794765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11795223504304886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154337280, "lr": 7.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048810206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11405187100172043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154501120, "lr": 7.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048825259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11192343384027481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048841279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10740824788808823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154828800, "lr": 7.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048857106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12058249115943909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154992640, "lr": 7.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048872903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10456175357103348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155156480, "lr": 7.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048889038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082591712474823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048905005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11155124008655548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155484160, "lr": 7.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048920734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11025093495845795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155648000, "lr": 7.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048936537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11361686140298843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155811840, "lr": 7.924583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048952236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11119285225868225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.932916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048952236, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1785048952236, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1785049035418, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.776131808757782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1785049035419, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1785049035419, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1785049051281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10773396492004395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156139520, "lr": 7.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049066907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11012009531259537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156303360, "lr": 7.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049082921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11389598250389099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156467200, "lr": 7.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049098800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11049704998731613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049114567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10825876891613007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156794880, "lr": 7.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049130546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10797278583049774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156958720, "lr": 7.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049146522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.101422980427742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157122560, "lr": 7.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049162290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11216869205236435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049178148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12057866901159286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157450240, "lr": 8.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049194020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10983136296272278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157614080, "lr": 8.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049209725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1088915467262268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157777920, "lr": 8.024583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049225703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10425311326980591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.032916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049241853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10366291552782059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158105600, "lr": 8.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049257847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10849743336439133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158269440, "lr": 8.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049257848, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1785049257848, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1785049339590, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7762653827667236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1785049339592, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1785049339592, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1785049355496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11427692323923111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158433280, "lr": 8.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049371558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11832380294799805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049387678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11309368163347244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158760960, "lr": 8.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049403821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11252378672361374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158924800, "lr": 8.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049419946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11072394996881485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159088640, "lr": 8.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049435765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12066096812486649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049451678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11175346374511719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159416320, "lr": 8.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049467063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10896798223257065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159580160, "lr": 8.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049483406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11021643131971359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159744000, "lr": 8.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049499145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09903962165117264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049514909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11011140793561935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160071680, "lr": 8.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049531137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.109600730240345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160235520, "lr": 8.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049547085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10399632155895233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160399360, "lr": 8.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049562914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11016716808080673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049562914, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785049562915, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785049645674, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7764533162117004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785049645675, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785049645675, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785049661439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11262299120426178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160727040, "lr": 8.174583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049677346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12907259166240692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160890880, "lr": 8.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049693387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10815557092428207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161054720, "lr": 8.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049709323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11208250373601913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049725354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1131477952003479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161382400, "lr": 8.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049741624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11056549847126007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161546240, "lr": 8.216250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049757948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1141301840543747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161710080, "lr": 8.224583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049774403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11344520002603531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049790515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10610675066709518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162037760, "lr": 8.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049806559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10802905261516571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162201600, "lr": 8.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049822861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12932397425174713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162365440, "lr": 8.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049838928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11273171752691269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049855460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11182940751314163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162693120, "lr": 8.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049871918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1163337305188179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162856960, "lr": 8.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049871919, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1785049871919, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1785049954881, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7767407298088074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1785049954882, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1785049954882, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1785049971008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09568304568529129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163020800, "lr": 8.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049987314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11148159205913544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050003773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1221008449792862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163348480, "lr": 8.307916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050020220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10639037936925888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163512320, "lr": 8.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050036730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10976195335388184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163676160, "lr": 8.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050053056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10717987269163132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050069265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11197929829359055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164003840, "lr": 8.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050085157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11330242455005646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164167680, "lr": 8.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050101299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10306557267904282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164331520, "lr": 8.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050116630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11797911673784256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050132197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10691560059785843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164659200, "lr": 8.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050147956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11095692217350006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164823040, "lr": 8.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050163842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09242291748523712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164986880, "lr": 8.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050179359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10338521003723145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050179359, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1785050179359, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1785050262893, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7769556641578674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1785050262893, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1785050262894, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1785050278570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10104546695947647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165314560, "lr": 8.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050294149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09669612348079681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165478400, "lr": 8.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050309539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11587943881750107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165642240, "lr": 8.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050325030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1073482558131218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050340434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10956810414791107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165969920, "lr": 8.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050356206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10387235134840012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166133760, "lr": 8.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050371817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10755886137485504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166297600, "lr": 8.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050387510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11266158521175385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050403123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09944037348031998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166625280, "lr": 8.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050418879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09590284526348114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166789120, "lr": 8.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050434544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09717573970556259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166952960, "lr": 8.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050450045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09677547961473465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050465435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1029542088508606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167280640, "lr": 8.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050481043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10055892914533615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167444480, "lr": 8.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050481044, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1785050481045, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1785050564021, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7772437334060669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1785050564022, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1785050564023, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1785050579612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10386199504137039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167608320, "lr": 8.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050595540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10240408778190613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050611353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10404208302497864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167936000, "lr": 8.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050627176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071949377655983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168099840, "lr": 8.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050643030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1019618883728981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168263680, "lr": 8.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050659171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0994638055562973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050675405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10911529511213303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168591360, "lr": 8.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050691268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10527073591947556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168755200, "lr": 8.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050707178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1063825711607933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168919040, "lr": 8.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050722978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10523383319377899, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050738926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1040949746966362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169246720, "lr": 8.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050754344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10853537917137146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169410560, "lr": 8.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050769488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10534870624542236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169574400, "lr": 8.624583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050785323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11230865865945816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050785361, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1785050785362, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1785050868966, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7773850560188293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1785050868968, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1785050868968, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1785050884564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10782922059297562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169902080, "lr": 8.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050900233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11150781810283661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170065920, "lr": 8.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050915942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.106721431016922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170229760, "lr": 8.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050931836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10721345245838165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050947342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10041217505931854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170557440, "lr": 8.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050962991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10228119790554047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170721280, "lr": 8.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050978607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09591623395681381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170885120, "lr": 8.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050994046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10774047672748566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051009372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09961051493883133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171212800, "lr": 8.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051024861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1024140864610672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171376640, "lr": 8.716249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051040618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10150760412216187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171540480, "lr": 8.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051056108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10286574810743332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051071657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09927225112915039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171868160, "lr": 8.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051087314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10324124991893768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172032000, "lr": 8.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051087356, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1785051087356, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1785051171055, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7776333689689636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1785051171057, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1785051171058, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1785051186540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09715550392866135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172195840, "lr": 8.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051202065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10718408226966858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051217776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0987834706902504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172523520, "lr": 8.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051233268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10260947793722153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172687360, "lr": 8.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051248647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09376011788845062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172851200, "lr": 8.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051264302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10659845173358917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051279990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10352923721075058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173178880, "lr": 8.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051295543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10380154848098755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173342720, "lr": 8.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051311656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10510361194610596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173506560, "lr": 8.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051327320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11197260767221451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051343237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09644870460033417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173834240, "lr": 8.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051359162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11094565689563751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173998080, "lr": 8.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051375012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11174077540636063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174161920, "lr": 8.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051390503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10197876393795013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051390542, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1785051390542, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1785051475037, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7778106331825256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1785051475038, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1785051475038, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1785051490565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.098127581179142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174489600, "lr": 8.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051506519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10637486726045609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174653440, "lr": 8.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051522343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10118655860424042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174817280, "lr": 8.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051537509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10633955895900726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051553323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10043023526668549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175144960, "lr": 8.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051568952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10924318432807922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175308800, "lr": 8.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051584640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10023977607488632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175472640, "lr": 8.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051600420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09948889911174774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051616466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10871562361717224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175800320, "lr": 8.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051632496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10918684303760529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175964160, "lr": 8.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051648396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09969040006399155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176128000, "lr": 8.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051664371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09458836913108826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051680002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09566080570220947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176455680, "lr": 8.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051695609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10588618367910385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176619520, "lr": 8.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051695609, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1785051695610, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1785051781569, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.778113603591919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1785051781571, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1785051781571, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1785051797266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10315796732902527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176783360, "lr": 8.991249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051812984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11348892748355865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051828908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10009859502315521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177111040, "lr": 9.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051844797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1046040803194046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177274880, "lr": 9.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051860596, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11059172451496124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177438720, "lr": 9.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051876135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1008690595626831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.032916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051891928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10907064378261566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177766400, "lr": 9.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051907583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10120120644569397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177930240, "lr": 9.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051923392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10784300416707993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178094080, "lr": 9.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051939189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10133907943964005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051954938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09704776108264923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178421760, "lr": 9.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051970972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1116553470492363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178585600, "lr": 9.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051987094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11915604770183563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178749440, "lr": 9.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052003058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10950537770986557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052003058, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1785052003059, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1785052088310, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7781916856765747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1785052088310, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1785052088311, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1785052104469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11581424623727798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179077120, "lr": 9.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052120396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10090004652738571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179240960, "lr": 9.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052136764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09989817440509796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179404800, "lr": 9.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052152732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10088163614273071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.132916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052168684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10494339466094971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179732480, "lr": 9.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052184749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09916020929813385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179896320, "lr": 9.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052200622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10047870129346848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180060160, "lr": 9.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052216741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10562727600336075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052232286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10590075701475143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180387840, "lr": 9.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052247679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09388544410467148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180551680, "lr": 9.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052263999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10420989990234375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180715520, "lr": 9.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052279858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1022205576300621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052295653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09330407530069351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181043200, "lr": 9.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052311558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09993059188127518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181207040, "lr": 9.216249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052311558, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1785052311558, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1785052395334, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7783814668655396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1785052395335, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1785052395335, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1785052411447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11062396317720413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181370880, "lr": 9.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052427438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11304132640361786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052443076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10080648213624954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181698560, "lr": 9.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052458877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1065312922000885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181862400, "lr": 9.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052474713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10300885140895844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182026240, "lr": 9.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052490231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1025804802775383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052505932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10610619187355042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182353920, "lr": 9.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052521482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10817042738199234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182517760, "lr": 9.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052537234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09806328266859055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182681600, "lr": 9.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052552877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09716376662254333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052568639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11045565456151962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183009280, "lr": 9.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052584104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09838316589593887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183173120, "lr": 9.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052599905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10184745490550995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183336960, "lr": 9.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052615355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10296864807605743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052615355, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785052615356, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785052698165, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7786149978637695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785052698166, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785052698166, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785052713849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10172569006681442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183664640, "lr": 9.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052729526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10784998536109924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183828480, "lr": 9.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052745050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11160578578710556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183992320, "lr": 9.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052761118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09362984448671341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052777139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10360120236873627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184320000, "lr": 9.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052793146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10770443081855774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184483840, "lr": 9.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052809097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11595381796360016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184647680, "lr": 9.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052824793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10080304741859436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052840734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10521402955055237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184975360, "lr": 9.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052856930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10924172401428223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185139200, "lr": 9.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052872980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09702498465776443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185303040, "lr": 9.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052888830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10766198486089706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052904865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09812960773706436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185630720, "lr": 9.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052920585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10494979470968246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185794560, "lr": 9.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052920586, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1785052920586, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1785053004675, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7788171768188477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1785053004676, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1785053004676, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1785053020770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11025577783584595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185958400, "lr": 9.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053036264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10494038462638855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053051365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09216279536485672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186286080, "lr": 9.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053067392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10010656714439392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186449920, "lr": 9.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053083117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082434207201004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186613760, "lr": 9.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053098602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.110938660800457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053114418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09611161798238754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186941440, "lr": 9.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053130130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10426657646894455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187105280, "lr": 9.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053145821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1060277447104454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187269120, "lr": 9.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053161467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09898915886878967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053177129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10380960255861282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187596800, "lr": 9.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053192808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10460717231035233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187760640, "lr": 9.549583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053208362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11103017628192902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187924480, "lr": 9.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053224065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11466465890407562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053224065, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1785053224066, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1785053307966, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7790400981903076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1785053307967, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1785053307967, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1785053323501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1094338446855545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188252160, "lr": 9.574583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053339309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10472080111503601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188416000, "lr": 9.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053354901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10320564359426498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188579840, "lr": 9.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053370609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087474450469017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053386144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10280735045671463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188907520, "lr": 9.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053401836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0971592515707016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189071360, "lr": 9.616249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053417468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10308779031038284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189235200, "lr": 9.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053432975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11221464723348618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053448466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10080191493034363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189562880, "lr": 9.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053463997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09507118910551071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189726720, "lr": 9.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053479538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10439208894968033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189890560, "lr": 9.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053495187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11226298660039902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.666249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053510973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10206388682126999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190218240, "lr": 9.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053526564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11821974813938141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190382080, "lr": 9.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053526564, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1785053526565, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1785053611378, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7792752385139465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1785053611379, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1785053611379, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1785053627254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10079661756753922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190545920, "lr": 9.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053642992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09697402268648148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053658799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10529815405607224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190873600, "lr": 9.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053674868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11282443255186081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191037440, "lr": 9.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053690795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11724307388067245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191201280, "lr": 9.724583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053706796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09552142024040222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053722992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10576630383729935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191528960, "lr": 9.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053738895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1014472246170044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191692800, "lr": 9.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053754952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10105352103710175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191856640, "lr": 9.757916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053770791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10411956906318665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053786181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10641738772392273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192184320, "lr": 9.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053802581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10976242274045944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192348160, "lr": 9.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053818453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09931863099336624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192512000, "lr": 9.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053834588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11498884111642838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.799583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053834588, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1785053834589, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1785053918624, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7791860103607178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1785053918625, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1785053918625, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1785053934898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11217022687196732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192839680, "lr": 9.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053951198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11073897778987885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193003520, "lr": 9.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053967338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10682467371225357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193167360, "lr": 9.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053983548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12002801895141602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053999684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1156967356801033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193495040, "lr": 9.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785054015969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1019643098115921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193658880, "lr": 9.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785054032320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11920247226953506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193822720, "lr": 9.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785054048396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11338551342487335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.866249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785054064618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1056523248553276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194150400, "lr": 9.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785054080961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10654320567846298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194314240, "lr": 9.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785054097288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071474701166153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194478080, "lr": 9.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785054113449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1054617315530777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785054129853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1065729483962059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194805760, "lr": 9.907916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785054145975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11548532545566559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194969600, "lr": 9.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785054145976, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1785054145977, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1785054230211, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7796122431755066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1785054230212, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1785054230212, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1785054246628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11222530901432037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195133440, "lr": 9.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785054262730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10433676093816757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785054279100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10240554064512253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195461120, "lr": 9.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785054295404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11670425534248352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195624960, "lr": 9.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785054311807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10833334177732468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195788800, "lr": 9.957916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785054327940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10568086057901382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785054344457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10502792149782181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196116480, "lr": 9.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785054360727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11355916410684586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196280320, "lr": 9.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785054377353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11875844746828079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196444160, "lr": 9.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785054393690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10527527332305908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.999583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785054410066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11618325859308243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196771840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054426224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1123441681265831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196935680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054442032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10760442167520523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197099520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054458410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11602023988962173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054458411, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1785054458411, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1785054542168, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7794124484062195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1785054542169, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1785054542169, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1785054558043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1012333557009697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197427200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054574225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1106337457895279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197591040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054590345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11512844264507294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197754880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054606403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10110724717378616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054622976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09913923591375351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198082560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054639449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1076960563659668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198246400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054655728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09897501021623611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198410240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054671971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10745072364807129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054688292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1004454493522644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198737920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054704406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10514990240335464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198901760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054720474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11447048932313919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199065600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054736706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11051114648580551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054752791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1170361265540123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199393280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054769176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11295599490404129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199557120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054769177, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1785054769178, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1785054853400, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7796792984008789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1785054853401, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1785054853401, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1785054869489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11242217570543289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199720960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054885813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10625597089529037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054901984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11105893552303314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200048640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054918296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1167774349451065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200212480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054934495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11210576444864273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200376320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054950783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11659279465675354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054967200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09976205974817276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200704000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054983404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10798609256744385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200867840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054999883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10764966160058975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201031680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055016209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085800975561142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055032573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11846086382865906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201359360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055048761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1124740019440651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201523200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055064997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1042240783572197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201687040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055081238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11084605008363724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055081239, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1785055081239, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1785055166226, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7798551917076111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1785055166227, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1785055166227, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1785055182089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10508094727993011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202014720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055198186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1103251501917839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202178560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055213988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1190112978219986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202342400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055230230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10627656430006027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055245991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10578213632106781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202670080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055262139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10163464397192001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202833920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055278720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10652968287467957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202997760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055294577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11041556298732758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055310386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1134968176484108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203325440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055325974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0944184884428978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203489280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055341849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11711808294057846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203653120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055357568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10093656927347183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055373019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09825185686349869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203980800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055388686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081087589263916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204144640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055388686, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1785055388687, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1785055474256, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7796524167060852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1785055474259, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1785055474259, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1785055489915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10224390029907227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204308480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055505705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09241347759962082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055521607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09721206873655319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204636160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055537338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10861765593290329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204800000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055553085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10794254392385483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204963840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055568801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11224592477083206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055584304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09554806351661682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205291520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055600027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10623259842395782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205455360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055615717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10656260699033737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205619200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055631245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11058173328638077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055646867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09823901951313019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205946880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055662937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0991285964846611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206110720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055678863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11091062426567078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206274560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055694569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09826585650444031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055694569, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1785055694570, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1785055780312, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7798555493354797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1785055780313, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1785055780313, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1785055796305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10574164986610413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206602240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055812110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09575555473566055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206766080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055828104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10471934080123901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206929920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055844256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09649120271205902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055860275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10253801941871643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207257600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055876114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11189506947994232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207421440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055892002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10689497739076614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207585280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055907623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09519552439451218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055923313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11527260392904282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207912960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055938958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10426713526248932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208076800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055954115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10331381857395172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208240640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055970066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.100717693567276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055985517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.102350153028965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208568320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056001237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10174252837896347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208732160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056001238, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1785056001238, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1785056086079, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7798238396644592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1785056086081, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1785056086081, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1785056101941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10068833827972412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208896000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056117746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09912340342998505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056133864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1019151508808136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209223680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056149503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10296670347452164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209387520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056165477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09889953583478928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209551360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056181143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10114128887653351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056196832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09642529487609863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209879040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056212503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09916957467794418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210042880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056228542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10364054888486862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210206720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056244517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10818169265985489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056260372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10358307510614395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210534400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056276074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10749109834432602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210698240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056291893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1014968678355217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210862080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056307383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09661363810300827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056307383, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1785056307384, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1785056390174, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7800012826919556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1785056390175, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1785056390175, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1785056406002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09455364942550659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211189760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056421475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09933817386627197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211353600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056436958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09738662838935852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211517440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056452469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09975464642047882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056467929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10403042286634445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211845120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056483928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0957406535744667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212008960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056499468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10182668268680573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212172800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056515067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09813594818115234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056530526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09788218140602112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212500480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056546250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10457142442464828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212664320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056561973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09304811805486679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212828160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056578109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10806923359632492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1e-06}} diff --git a/recommendation_v4/rcp_logs/gbs_8192/seed1026364325.log b/recommendation_v4/rcp_logs/gbs_8192/seed1026364325.log new file mode 100644 index 000000000..978bec1a2 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_8192/seed1026364325.log @@ -0,0 +1,639 @@ +:::MLLOG {"namespace": "", "time_ms": 1784941524458, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784941524458, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784941546250, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784941546252, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784941546252, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784941546252, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784941546252, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784941546252, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 8192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784941546252, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784941546252, "event_type": "POINT_IN_TIME", "key": "seed", "value": 1026364325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784941546252, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784941546252, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784941546252, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784941546252, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784941546253, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784941546253, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784942465782, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784942465783, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784942466213, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784942596928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13742241263389587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840, "lr": 7.916666666666666e-10}} +:::MLLOG {"namespace": "", "time_ms": 1784942609239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13938042521476746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.6249999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784942621763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13886286318302155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 491520, "lr": 2.458333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784942634396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1392468363046646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784942647277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389351785182953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 819200, "lr": 4.125e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784942660056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13904841244220734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784942672988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1391369104385376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1146880, "lr": 5.7916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784942686034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860578835010529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.625e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784942699163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389235556125641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1474560, "lr": 7.458333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784942712140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13910414278507233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.291666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784942725520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387670785188675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1802240, "lr": 9.124999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784942738733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13864858448505402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784942751991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13891690969467163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2129920, "lr": 1.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942765130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13906390964984894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942765131, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784942765131, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784942904084, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5000073313713074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784942904085, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784942904085, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784942916912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13908334076404572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2457600, "lr": 1.2458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942930103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13897421956062317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942943444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13897790014743805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2785280, "lr": 1.4124999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942956242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13888296484947205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4958333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942969452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13910463452339172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3112960, "lr": 1.5791666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942983282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13899990916252136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942996576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13909557461738586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3440640, "lr": 1.7458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943010011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13893473148345947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943023640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388172060251236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3768320, "lr": 1.9124999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943037101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13894455134868622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9958333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943050901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13878437876701355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4096000, "lr": 2.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943064640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13867396116256714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.1624999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943078236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13878978788852692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4423680, "lr": 2.2458333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943092845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386394053697586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3291666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943092846, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784943092846, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784943185796, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5002493858337402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784943185796, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784943185797, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784943199219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385965198278427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4751360, "lr": 2.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943212859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13864897191524506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943230233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13848280906677246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5079040, "lr": 2.5791666666666668e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943243821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13819153606891632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943257476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13839076459407806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5406720, "lr": 2.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943270926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13847528398036957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943284523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13817468285560608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5734400, "lr": 2.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943298172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13792595267295837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943311670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377793252468109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6062080, "lr": 3.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943325059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137987419962883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943338677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13808971643447876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6389760, "lr": 3.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943352174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13822215795516968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943366277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379290223121643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6717440, "lr": 3.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943380236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13809587061405182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943380237, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784943380237, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784943475051, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5005601644515991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784943475052, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784943475053, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784943488658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13756555318832397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7045120, "lr": 3.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943502565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13790832459926605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943516477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13744895160198212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7372800, "lr": 3.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943530806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376878172159195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.829166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943544682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378932148218155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7700480, "lr": 3.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943558782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13794592022895813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943572908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13719408214092255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8028160, "lr": 4.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943586844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13769276440143585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943601306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374053955078125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8355840, "lr": 4.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943615643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13863179087638855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3291666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943629762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13685248792171478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8683520, "lr": 4.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943644002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379208117723465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.4958333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943658107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13598769903182983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9011200, "lr": 4.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943672280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13695167005062103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.6624999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943672280, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784943672281, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784943766988, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5010226368904114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784943766989, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784943766990, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784943780375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13658630847930908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9338880, "lr": 4.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943794181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13593986630439758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943807758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375114619731903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9666560, "lr": 4.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943821490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13523073494434357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943835580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13682076334953308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9994240, "lr": 5.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943849373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13431355357170105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1624999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943863455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13587836921215057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10321920, "lr": 5.2458333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943877394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13580718636512756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943891589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13793492317199707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10649600, "lr": 5.4124999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943905701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13572774827480316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943920089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1461067497730255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10977280, "lr": 5.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943934341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13586664199829102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943948429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134622260928154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11304960, "lr": 5.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943962560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13771849870681763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943962561, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784943962561, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784944058716, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.501638650894165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784944058717, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784944058717, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784944072379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356346756219864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11632640, "lr": 5.9124999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944086208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13944940268993378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944099926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361686885356903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11960320, "lr": 6.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944113758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13356630504131317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944127647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13514244556427002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12288000, "lr": 6.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944141718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13898661732673645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.329166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944155612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13374638557434082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12615680, "lr": 6.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944169501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13359376788139343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944183948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13376428186893463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12943360, "lr": 6.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944197824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13200019299983978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944212505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13179898262023926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13271040, "lr": 6.745833333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944226326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351270079612732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944240396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13244356215000153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13598720, "lr": 6.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944255262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13870859146118164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944255262, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784944255262, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784944350243, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5024182200431824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784944350244, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784944350244, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784944364252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13410784304141998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13926400, "lr": 7.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944378484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13369950652122498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944392565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13588041067123413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14254080, "lr": 7.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944406415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13497388362884521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944420387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375962346792221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14581760, "lr": 7.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944434449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12844812870025635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944448132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13565988838672638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14909440, "lr": 7.579166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944462131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297820806503296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944476201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13350965082645416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15237120, "lr": 7.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944490733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13272623717784882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.829166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944504962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13369277119636536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15564800, "lr": 7.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944520111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13474500179290771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.995833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944535134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352992057800293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15892480, "lr": 8.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944549585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13359159231185913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944549586, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784944549586, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784944644439, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5034306049346924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784944644440, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784944644440, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784944659363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13281022012233734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16220160, "lr": 8.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944673831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13385939598083496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944688626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13551747798919678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16547840, "lr": 8.412500000000001e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944703732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355067640542984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944718096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1392674446105957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16875520, "lr": 8.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944732582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333032101392746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.662499999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944747703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353289783000946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17203200, "lr": 8.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944762424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13356488943099976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944776597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312554359436035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17530880, "lr": 8.912499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944791335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13200685381889343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944805889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13567785918712616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17858560, "lr": 9.079166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944820689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13394410908222198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.162499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944835296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13336537778377533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18186240, "lr": 9.245833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944849642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13066062331199646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944849642, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784944849643, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784944936301, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5046725273132324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784944936302, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784944936302, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784944951641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376209855079651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18513920, "lr": 9.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944966528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13486310839653015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.495833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944981225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354445070028305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18841600, "lr": 9.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944996365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13150626420974731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.662499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784945011166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13356754183769226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19169280, "lr": 9.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784945025646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13408879935741425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784945040813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13714149594306946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19496960, "lr": 9.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784945055518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13391271233558655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784945070261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13292187452316284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19824640, "lr": 1.0079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945084532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12974891066551208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0162499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945098626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12774540483951569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20152320, "lr": 1.0245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945113228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13301239907741547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945127782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12951353192329407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20480000, "lr": 1.0412499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945141980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12486007809638977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945141981, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784945141982, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784945236966, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.506056547164917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784945236968, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784945236968, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784945251538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311056762933731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20807680, "lr": 1.0579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945265725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357857882976532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945280578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331709772348404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21135360, "lr": 1.0745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945295183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303158402442932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945309298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12808731198310852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21463040, "lr": 1.0912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945324946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1267431080341339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945339920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12904247641563416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21790720, "lr": 1.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945354060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323758363723755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945368793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13549837470054626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22118400, "lr": 1.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945383549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12498108297586441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945397794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12865203619003296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22446080, "lr": 1.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945412229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13434208929538727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945426675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310819536447525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22773760, "lr": 1.1579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945441438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12939304113388062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945441439, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784945441439, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784945537323, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5077449679374695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784945537324, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784945537324, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784945552235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303529590368271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23101440, "lr": 1.1745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945566438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1292913854122162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945580331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13317173719406128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23429120, "lr": 1.1912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945595080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1238841786980629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945609972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12950190901756287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23756800, "lr": 1.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945624440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12572915852069855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945638809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13003243505954742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24084480, "lr": 1.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945653581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12198100239038467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945667744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13588206470012665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24412160, "lr": 1.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945682666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13707734644412994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945697394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12353332340717316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24739840, "lr": 1.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945712490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1231696754693985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945727302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12566415965557098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25067520, "lr": 1.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945742024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12894652783870697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945742024, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784945742025, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784945835358, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5095719695091248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784945835359, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784945835360, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784945849380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13652929663658142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25395200, "lr": 1.2912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945864761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13629518449306488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945879263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13568352162837982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25722880, "lr": 1.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945893730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1398308128118515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945907938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13655361533164978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26050560, "lr": 1.3245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945922313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13829608261585236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945936605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12736651301383972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26378240, "lr": 1.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945950969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12620896100997925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945965564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14217360317707062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26705920, "lr": 1.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945980189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1410769522190094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945994923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12739962339401245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27033600, "lr": 1.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946009472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1283000409603119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946023925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13273347914218903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27361280, "lr": 1.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946037798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13131961226463318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946037798, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784946037799, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784946124348, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5115414261817932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784946124349, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784946124350, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784946139227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.141310915350914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27688960, "lr": 1.4079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946154544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14236481487751007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946169148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353446990251541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28016640, "lr": 1.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946183777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13942871987819672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946199114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14488379657268524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28344320, "lr": 1.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946213496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14091186225414276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946228544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12586991488933563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28672000, "lr": 1.4579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946243770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13751104474067688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946258598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13181030750274658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28999680, "lr": 1.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946273813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13623905181884766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946288715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13773547112941742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29327360, "lr": 1.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946304124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330055594444275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4995833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946319490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345403641462326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29655040, "lr": 1.5079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946334617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13641314208507538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946334619, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784946334620, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784946429410, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5135456323623657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784946429411, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784946429412, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784946443823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13865207135677338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29982720, "lr": 1.5245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946458533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13816285133361816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946473553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13187214732170105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30310400, "lr": 1.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946488346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13297539949417114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946503182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13377198576927185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30638080, "lr": 1.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946518415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13988132774829865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946533369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329226791858673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30965760, "lr": 1.5745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946548716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14115020632743835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946563189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1407945603132248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31293440, "lr": 1.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946577992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13509155809879303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946593145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1277146190404892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31621120, "lr": 1.6079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946607828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12991692125797272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946622313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13432198762893677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31948800, "lr": 1.6245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946637474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12780366837978363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946637475, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784946637475, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784946732892, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5160844922065735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784946732894, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784946732894, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784946747414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13232871890068054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32276480, "lr": 1.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946761872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13697032630443573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946776972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13350440561771393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32604160, "lr": 1.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946791197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13187287747859955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946806122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13400965929031372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32931840, "lr": 1.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946821008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13334901630878448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946836083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13382023572921753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33259520, "lr": 1.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946851348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326749622821808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946865671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13972164690494537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33587200, "lr": 1.7079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946880739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13184455037117004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946895686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12878911197185516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33914880, "lr": 1.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946910045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310272514820099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946924655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13221269845962524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34242560, "lr": 1.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946939671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13597051799297333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7495833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946939671, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784946939671, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784947035664, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.519473135471344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784947035665, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784947035665, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784947050523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13361412286758423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34570240, "lr": 1.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947065505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12315766513347626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947080553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13062527775764465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34897920, "lr": 1.7745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947095418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13031430542469025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947110473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353178322315216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35225600, "lr": 1.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947125594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13552208244800568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947140469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13547323644161224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35553280, "lr": 1.8079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947155897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1240696832537651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8162500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947170735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13114574551582336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35880960, "lr": 1.8245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947185194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310739517211914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947199893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13032008707523346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36208640, "lr": 1.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947214681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13319939374923706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947229284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308547854423523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36536320, "lr": 1.8579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947243823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303940862417221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947243824, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784947243825, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784947339427, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5238083004951477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784947339428, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784947339428, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784947354585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13779179751873016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36864000, "lr": 1.8745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947369112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13241977989673615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947384091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13354238867759705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37191680, "lr": 1.8912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947398935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13034698367118835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947413572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12719006836414337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37519360, "lr": 1.9079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947428421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13207495212554932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947443349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356399655342102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37847040, "lr": 1.9245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947458604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13288187980651855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947473915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13129909336566925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38174720, "lr": 1.9412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947489065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13899220526218414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947504223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13379517197608948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38502400, "lr": 1.9579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947518536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13769134879112244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947533549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13189508020877838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38830080, "lr": 1.9745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947548427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333012580871582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947548428, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784947548430, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784947644480, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5296273827552795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784947644481, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784947644482, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784947659135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13133026659488678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39157760, "lr": 1.9912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947673756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350405365228653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947688766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377696841955185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39485440, "lr": 2.0079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947703460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13340483605861664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947717982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12751592695713043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39813120, "lr": 2.0245833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947732782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12940648198127747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0329166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947747401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12590952217578888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40140800, "lr": 2.0412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947761890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1239841878414154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947776803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13606929779052734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40468480, "lr": 2.0579166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947791820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13733412325382233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947806905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13471245765686035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40796160, "lr": 2.0745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947822215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12692523002624512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0829166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947837172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13095711171627045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41123840, "lr": 2.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947851514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12343057990074158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947851515, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784947851515, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784947943537, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.537490725517273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784947943538, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784947943538, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784947958765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13252437114715576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41451520, "lr": 2.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947973910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13267388939857483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947989129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13346615433692932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41779200, "lr": 2.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948004512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13781213760375977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948019708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13642406463623047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42106880, "lr": 2.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948034680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13142657279968262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948049920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12737233936786652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42434560, "lr": 2.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948064948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13503198325634003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948079879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12465999275445938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42762240, "lr": 2.1745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948095355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303391307592392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948110525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13256458938121796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43089920, "lr": 2.1912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948126097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13444629311561584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948141592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14036735892295837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43417600, "lr": 2.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948156632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13622508943080902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948156633, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784948156633, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784948252745, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5491786003112793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784948252746, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784948252747, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784948268508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13659654557704926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43745280, "lr": 2.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948283525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13157573342323303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948298165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13272322714328766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44072960, "lr": 2.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948313107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13894827663898468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948328393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300020068883896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44400640, "lr": 2.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948343868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13855160772800446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948358802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350577473640442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44728320, "lr": 2.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948373708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13736079633235931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948388933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348874568939209, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45056000, "lr": 2.2912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948404210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13085348904132843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948419237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14068524539470673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45383680, "lr": 2.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948434585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13476234674453735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948449709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12922582030296326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45711360, "lr": 2.3245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948465174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312572240829468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948465175, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784948465175, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784948559544, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5625792145729065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784948559545, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784948559545, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784948575354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14216236770153046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46039040, "lr": 2.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948590656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13271337747573853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948606651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13359954953193665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46366720, "lr": 2.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948622332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13097642362117767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948637957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13550373911857605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46694400, "lr": 2.3745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948653093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13356338441371918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948668561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1340518444776535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47022080, "lr": 2.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948683941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13571935892105103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948699379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13119418919086456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47349760, "lr": 2.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948714978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13302385807037354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948730454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13300170004367828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47677440, "lr": 2.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948746297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.136993870139122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948761574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13634224236011505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48005120, "lr": 2.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948777313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13754256069660187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.4495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948777314, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784948777314, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784948873107, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5775254368782043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784948873109, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784948873109, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784948888627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13178801536560059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48332800, "lr": 2.4579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948904023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13545621931552887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948919701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13424231112003326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48660480, "lr": 2.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948935698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13204637169837952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948950766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13733908534049988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48988160, "lr": 2.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948965762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12952473759651184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948981175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13391610980033875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49315840, "lr": 2.507916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948996213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12426531314849854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.5162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949011404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13124793767929077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49643520, "lr": 2.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949026096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13181839883327484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949041411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131117582321167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49971200, "lr": 2.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949056520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13167811930179596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.5495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949071799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12943990528583527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50298880, "lr": 2.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949087368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13098227977752686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.5662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949087368, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784949087369, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784949183663, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5897506475448608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784949183665, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784949183665, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784949198425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12252422422170639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50626560, "lr": 2.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949213929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12711501121520996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949229242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12969499826431274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50954240, "lr": 2.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949244502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13332955539226532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949260231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13221696019172668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51281920, "lr": 2.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949275414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12219476699829102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949291005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1306719183921814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51609600, "lr": 2.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949306493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13225732743740082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949321494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13704702258110046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51937280, "lr": 2.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949335997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12816287577152252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.6495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949351383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13364015519618988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52264960, "lr": 2.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949366426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11809934675693512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.6662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949381619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12171386927366257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52592640, "lr": 2.6745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949396707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1257641315460205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949396708, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784949396708, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784949492754, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6016207337379456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784949492755, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784949492755, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784949508018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12178269028663635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52920320, "lr": 2.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949523106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1279434859752655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.6995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949538514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1285974383354187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53248000, "lr": 2.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949553908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13231416046619415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949568864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12194996327161789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53575680, "lr": 2.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949584307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13524633646011353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949599301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1302611380815506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53903360, "lr": 2.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949614194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12192552536725998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.7495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949629657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12701445817947388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54231040, "lr": 2.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949645182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12760303914546967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949660664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11876402050256729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54558720, "lr": 2.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949676637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318582147359848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949692144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1275564432144165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54886400, "lr": 2.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949707593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12903186678886414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949707593, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784949707594, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784949803698, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6113557815551758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784949803699, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784949803699, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784949818167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1286386251449585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55214080, "lr": 2.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949833568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11937104165554047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949848921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12247850745916367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55541760, "lr": 2.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949864198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12202344089746475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949879252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12399055808782578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55869440, "lr": 2.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949894519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12643317878246307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949909383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11671560257673264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56197120, "lr": 2.8579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949924537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12305118143558502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949939781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12597112357616425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56524800, "lr": 2.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949954664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12408066540956497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949969801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11592482775449753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56852480, "lr": 2.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949984944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12464570999145508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949999873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11083588749170303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57180160, "lr": 2.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950015420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12498640269041061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950015420, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784950015420, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784950111917, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.619658350944519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784950111918, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784950111918, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784950127263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11749959737062454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57507840, "lr": 2.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950142646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11500047147274017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950158002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12997514009475708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57835520, "lr": 2.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950173558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12196364998817444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950188840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12418511509895325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58163200, "lr": 2.9579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950203921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11670851707458496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950218283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12029951810836792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58490880, "lr": 2.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950233528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12243445217609406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950248923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12153959274291992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58818560, "lr": 2.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950264173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11394033581018448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950279519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1218964084982872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59146240, "lr": 3.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950294860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12226888537406921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.0162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950309968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12539397180080414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59473920, "lr": 3.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950325335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11713549494743347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950325336, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784950325336, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784950420981, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.630948007106781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784950420982, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784950420982, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784950436524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12011384963989258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59801600, "lr": 3.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950451642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11251520365476608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950466922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11032222956418991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60129280, "lr": 3.0579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950482250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11392379552125931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.0662499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950497324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10934105515480042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60456960, "lr": 3.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950512788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12241443246603012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950528240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11632782965898514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60784640, "lr": 3.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950543947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12119251489639282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950559497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1034821942448616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61112320, "lr": 3.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950575109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11957269161939621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.1162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950590902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11296570301055908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61440000, "lr": 3.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950606719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11386486142873764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950621937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11051925271749496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61767680, "lr": 3.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950636968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1172223687171936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950636969, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784950636969, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784950733178, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6497419476509094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784950733180, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784950733180, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784950748039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1210394874215126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62095360, "lr": 3.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950763231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11975222826004028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.1662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950779059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12400878220796585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62423040, "lr": 3.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950794179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10657002031803131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950809653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1148744523525238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62750720, "lr": 3.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950825391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11794503033161163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950840658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11977992951869965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63078400, "lr": 3.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950856037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11919103562831879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950871314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10979210585355759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63406080, "lr": 3.2245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950886378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11819546669721603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950901868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11573594063520432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63733760, "lr": 3.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950916633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11757631599903107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950931532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11330569535493851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64061440, "lr": 3.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950947388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12723281979560852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950947388, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784950947389, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784951043343, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6745119094848633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784951043344, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784951043344, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784951058650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12514157593250275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64389120, "lr": 3.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951074155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11718439310789108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951089598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11683519184589386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64716800, "lr": 3.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951105190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12299320101737976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951120859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11087450385093689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65044480, "lr": 3.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951136215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11217683553695679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951150547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11414947360754013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65372160, "lr": 3.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951166009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10909619182348251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951181830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.112120121717453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65699840, "lr": 3.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951197750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11717221885919571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951213572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12420021742582321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66027520, "lr": 3.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951229498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11588915437459946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951245196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12796732783317566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66355200, "lr": 3.3745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951261170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11348115652799606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951261171, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784951261171, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784951356227, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7271044850349426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784951356228, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784951356228, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784951372141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12652361392974854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66682880, "lr": 3.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951387869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11449965089559555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.3995833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951403907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11831612139940262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67010560, "lr": 3.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951419720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11366499215364456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951435389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12049419432878494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67338240, "lr": 3.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951451625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11925993114709854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951467553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12167107313871384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67665920, "lr": 3.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951483663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12161265313625336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951499814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11572716385126114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67993600, "lr": 3.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951515982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11314544826745987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951532016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11547024548053741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68321280, "lr": 3.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951547215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11668617278337479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951562435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12501144409179688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68648960, "lr": 3.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951578448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11662144958972931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951578449, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784951578449, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784951674861, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7501415014266968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784951674862, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784951674862, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784951690728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12292882055044174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68976640, "lr": 3.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951707109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11950357258319855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951722749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11414574086666107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69304320, "lr": 3.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951738912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1151285395026207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951754881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10950080305337906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69632000, "lr": 3.5412500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951770665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11537306755781174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951786703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11799280345439911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69959680, "lr": 3.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951802114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11179741472005844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951817829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1102556586265564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70287360, "lr": 3.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951833908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1090901717543602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951849920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12332583218812943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70615040, "lr": 3.5912500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951865901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.107154481112957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951881852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12077640742063522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70942720, "lr": 3.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951897927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11940683424472809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951897927, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784951897927, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784951994276, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7597251534461975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784951994277, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784951994277, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784952010582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11943729221820831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71270400, "lr": 3.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952025847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11867070943117142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952041259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1163618192076683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71598080, "lr": 3.6412499999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952057114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11671170592308044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952072724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10324077308177948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71925760, "lr": 3.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952088762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10435380041599274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952104212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11993025243282318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72253440, "lr": 3.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952119844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11515036970376968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952135528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10372337698936462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72581120, "lr": 3.6912499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952150969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10297408699989319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952166465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071285530924797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72908800, "lr": 3.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952182112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11243951320648193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952197428, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11101038753986359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73236480, "lr": 3.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952213340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11491339653730392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952213377, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784952213377, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784952308227, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7645624876022339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784952308228, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784952308228, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784952323757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10432390868663788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73564160, "lr": 3.7412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952339925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10699755698442459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952356209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11534254997968674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73891840, "lr": 3.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952371809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11057397723197937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952387850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10768117755651474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74219520, "lr": 3.7745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952404025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11445451527833939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952419728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11313016712665558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74547200, "lr": 3.7912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952435232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11267698556184769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952450410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11623125523328781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74874880, "lr": 3.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952466071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11153814196586609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952481452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10092351585626602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75202560, "lr": 3.8245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952497223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11342328786849976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8329166666666665e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_8192/seed138251694.log b/recommendation_v4/rcp_logs/gbs_8192/seed138251694.log new file mode 100644 index 000000000..c9c4c74af --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_8192/seed138251694.log @@ -0,0 +1,681 @@ +:::MLLOG {"namespace": "", "time_ms": 1784940678141, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784940678141, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784940699724, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784940699726, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784940699726, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784940699726, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784940699726, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784940699726, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 8192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784940699726, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784940699726, "event_type": "POINT_IN_TIME", "key": "seed", "value": 138251694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784940699726, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784940699726, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784940699726, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784940699726, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784940699732, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784940699732, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784941584276, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784941584277, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784941584733, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784941737948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13866974413394928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840, "lr": 7.916666666666666e-10}} +:::MLLOG {"namespace": "", "time_ms": 1784941750230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13830728828907013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.6249999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784941763074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13849303126335144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 491520, "lr": 2.458333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784941776198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13847102224826813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784941789514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13856860995292664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 819200, "lr": 4.125e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784941802827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13847239315509796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784941816237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382628083229065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1146880, "lr": 5.7916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784941829655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13868878781795502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.625e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784941843175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13876457512378693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1474560, "lr": 7.458333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784941856619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13853897154331207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.291666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784941870440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13847948610782623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1802240, "lr": 9.124999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784941884012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388532519340515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784941897946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383674293756485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2129920, "lr": 1.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941911568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13907232880592346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941911568, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784941911569, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784941996629, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5013336539268494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784941996630, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784941996630, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784942009761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13839007914066315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2457600, "lr": 1.2458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942023058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13850615918636322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942037490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383703351020813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2785280, "lr": 1.4124999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942051110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13834752142429352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4958333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942065010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138162761926651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3112960, "lr": 1.5791666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942079381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380109041929245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942092654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138204887509346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3440640, "lr": 1.7458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942106289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13812881708145142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942120851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13791970908641815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3768320, "lr": 1.9124999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942134663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13827012479305267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9958333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942148899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385221630334854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4096000, "lr": 2.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942162917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13853959739208221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.1624999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942176670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13802139461040497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4423680, "lr": 2.2458333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942191015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13879024982452393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3291666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942191016, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784942191016, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784942272932, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5016124248504639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784942272933, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784942272933, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784942286821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13768979907035828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4751360, "lr": 2.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942300632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13828414678573608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942314516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373881846666336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5079040, "lr": 2.5791666666666668e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942328588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365291178226471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942342250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13776709139347076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5406720, "lr": 2.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942355836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13801166415214539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942369507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13741536438465118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5734400, "lr": 2.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942384037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13698731362819672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942398234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13650307059288025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6062080, "lr": 3.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942412287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1372741460800171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942426481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1372509002685547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6389760, "lr": 3.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942440362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375759094953537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942454768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378057599067688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6717440, "lr": 3.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942468359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13762608170509338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942468360, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784942468360, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784942549323, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5019965767860413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784942549324, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784942549324, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784942562779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367434412240982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7045120, "lr": 3.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942576547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373411864042282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942590259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13650484383106232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7372800, "lr": 3.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942604869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13719111680984497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.829166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942618934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13728244602680206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7700480, "lr": 3.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942633205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373216211795807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942647270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13651223480701447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8028160, "lr": 4.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942661255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13730812072753906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942675743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13681188225746155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8355840, "lr": 4.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942689971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386224925518036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3291666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942703988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13616842031478882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8683520, "lr": 4.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942717968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13799263536930084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.4958333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942731912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13497263193130493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9011200, "lr": 4.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942746295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13643650710582733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.6624999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942746296, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784942746296, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784942830866, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5024652481079102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784942830867, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784942830868, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784942844560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13609302043914795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9338880, "lr": 4.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942858304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13531087338924408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942871840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13744038343429565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9666560, "lr": 4.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942885631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13432861864566803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942899551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364355832338333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9994240, "lr": 5.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942913207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332699954509735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1624999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942927351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13523021340370178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10321920, "lr": 5.2458333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942941115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13519413769245148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942955045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13781386613845825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10649600, "lr": 5.4124999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942968849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13493362069129944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942982690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14763686060905457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10977280, "lr": 5.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942996926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354249268770218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943010640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13389833271503448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11304960, "lr": 5.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943024459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13753929734230042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943024460, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784943024460, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784943107290, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.503047525882721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784943107291, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784943107291, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784943121020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350109726190567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11632640, "lr": 5.9124999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943134737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13970838487148285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943148571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13584090769290924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11960320, "lr": 6.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943162301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13282990455627441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943176165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347324252128601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12288000, "lr": 6.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943190211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1392994523048401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.329166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943204108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331050544977188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12615680, "lr": 6.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943218340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13277648389339447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943232294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13289564847946167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12943360, "lr": 6.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943245869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13114900887012482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943259759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13087984919548035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13271040, "lr": 6.745833333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943273243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13471412658691406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943286951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13125014305114746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13598720, "lr": 6.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943301112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13911239802837372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943301113, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784943301113, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784943383125, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5036991834640503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784943383126, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784943383126, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784943397030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13331815600395203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13926400, "lr": 7.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943410750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13321655988693237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943425034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356871873140335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14254080, "lr": 7.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943438982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13478785753250122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943453067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13773882389068604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14581760, "lr": 7.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943467155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12733687460422516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943480811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135287344455719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14909440, "lr": 7.579166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943494705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1290522664785385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943508766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333162784576416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15237120, "lr": 7.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943523177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13234490156173706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.829166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943537210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13316906988620758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15564800, "lr": 7.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943551575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13430175185203552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.995833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943566461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13530348241329193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15892480, "lr": 8.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943580710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13329903781414032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943580711, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784943580712, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784943663040, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5046447515487671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784943663041, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784943663041, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784943677416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13240981101989746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16220160, "lr": 8.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943691554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13372398912906647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943705779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355431228876114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16547840, "lr": 8.412500000000001e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943720234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13557332754135132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943734499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1396748572587967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16875520, "lr": 8.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943748520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13302500545978546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.662499999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943763351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13524845242500305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17203200, "lr": 8.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943777374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332775205373764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943791397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330786645412445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17530880, "lr": 8.912499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943805474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13170461356639862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943819344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13570375740528107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17858560, "lr": 9.079166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943833664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13361454010009766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.162499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943847555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13311755657196045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18186240, "lr": 9.245833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943861303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13050971925258636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943861303, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784943861304, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784943944079, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5058592557907104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784943944080, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784943944081, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784943958822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13806255161762238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18513920, "lr": 9.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943973182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346038281917572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.495833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943987575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13541436195373535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18841600, "lr": 9.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944002138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13120798766613007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.662499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944016317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13353268802165985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19169280, "lr": 9.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944030761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1340671330690384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944045371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370886117219925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19496960, "lr": 9.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944059857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341196894645691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944074224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13277165591716766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19824640, "lr": 1.0079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944088242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12961351871490479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0162499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944101807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12749366462230682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20152320, "lr": 1.0245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944115630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132957324385643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944129620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12946678698062897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20480000, "lr": 1.0412499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944143246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1244872659444809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944143246, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784944143247, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784944223960, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5070699453353882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784944223961, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784944223961, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784944237928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13102023303508759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20807680, "lr": 1.0579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944251667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13583701848983765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944265676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13310423493385315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21135360, "lr": 1.0745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944279725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13025155663490295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944293463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1280190646648407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21463040, "lr": 1.0912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944307723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12647002935409546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944322250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12879173457622528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21790720, "lr": 1.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944336290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13233129680156708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944350534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13506591320037842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22118400, "lr": 1.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944364779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1248164251446724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944378710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1285477727651596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22446080, "lr": 1.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944392632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13433130085468292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944406876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13112208247184753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22773760, "lr": 1.1579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944421401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12914060056209564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944421402, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784944421402, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784944503601, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5086745023727417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784944503602, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784944503602, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784944518065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13053375482559204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23101440, "lr": 1.1745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944532253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12931300699710846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944546045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331900805234909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23429120, "lr": 1.1912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944560435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12387964874505997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944574633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12926073372364044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23756800, "lr": 1.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944588245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1256505697965622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944601944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1299981325864792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24084480, "lr": 1.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944615878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1216902956366539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944629531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13584913313388824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24412160, "lr": 1.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944643586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13705600798130035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944657431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1236468032002449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24739840, "lr": 1.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944671832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1231631264090538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944686023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12549695372581482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25067520, "lr": 1.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944700139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289748251438141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944700139, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784944700139, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784944783730, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5105555057525635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784944783731, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784944783731, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784944797443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13657806813716888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25395200, "lr": 1.2912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944812100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13643555343151093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944826280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13572748005390167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25722880, "lr": 1.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944840288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14000573754310608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944854251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13677887618541718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26050560, "lr": 1.3245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944868329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13840492069721222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944882228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12735222280025482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26378240, "lr": 1.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944896371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1261424720287323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944910553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14225342869758606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26705920, "lr": 1.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944925159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14094416797161102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944939678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12718594074249268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27033600, "lr": 1.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944954041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12830393016338348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944968316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13272882997989655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27361280, "lr": 1.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944982062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13135287165641785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944982063, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784944982063, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784945066630, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5125978589057922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784945066631, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784945066631, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784945081401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14137797057628632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27688960, "lr": 1.4079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945095893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14259056746959686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945110097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353149563074112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28016640, "lr": 1.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945124421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13934427499771118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945138981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14458467066287994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28344320, "lr": 1.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945153065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1406998336315155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945167542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12576410174369812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28672000, "lr": 1.4579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945182018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375160813331604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945196508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13174191117286682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28999680, "lr": 1.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945211149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13615646958351135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945225624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13762350380420685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29327360, "lr": 1.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945239917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13284137845039368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4995833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945254897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13467365503311157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29655040, "lr": 1.5079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945269758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13658714294433594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945269759, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784945269759, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784945353576, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5148406028747559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784945353577, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784945353577, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784945367893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384589821100235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29982720, "lr": 1.5245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945382698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13809290528297424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945397471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13156390190124512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30310400, "lr": 1.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945412103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1328871250152588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945426728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13372330367565155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30638080, "lr": 1.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945441648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1395696997642517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945456435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325719654560089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30965760, "lr": 1.5745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945471253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1411779224872589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945485425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1407012641429901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31293440, "lr": 1.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945500000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13517312705516815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945514218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12760290503501892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31621120, "lr": 1.6079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945528045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13009122014045715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945541865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13383178412914276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31948800, "lr": 1.6245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945555863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.127670556306839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945555863, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784945555864, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784945641513, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5176620483398438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784945641514, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784945641514, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784945655267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13205267488956451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32276480, "lr": 1.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945669007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13674403727054596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945683011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13360807299613953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32604160, "lr": 1.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945697093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317916065454483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945711142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341092437505722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32931840, "lr": 1.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945725198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333257555961609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945739586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13377153873443604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33259520, "lr": 1.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945753683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13246367871761322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945767595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13973018527030945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33587200, "lr": 1.7079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945782441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13191673159599304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945796726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12880949676036835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33914880, "lr": 1.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945810671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311231553554535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945824844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324755698442459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34242560, "lr": 1.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945839067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13573423027992249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7495833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945839068, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784945839068, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784945924775, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5217011570930481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784945924776, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784945924776, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784945939154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334647834300995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34570240, "lr": 1.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945953357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1228133961558342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945967570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13052617013454437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34897920, "lr": 1.7745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945981778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13032186031341553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945995776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13478568196296692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35225600, "lr": 1.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946010390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13540324568748474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946024522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352274864912033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35553280, "lr": 1.8079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946039083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12417808175086975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8162500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946053193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311016082763672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35880960, "lr": 1.8245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946067069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1309293955564499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946081752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13010276854038239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36208640, "lr": 1.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946095721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13305039703845978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946109720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13074073195457458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36536320, "lr": 1.8579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946123496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300451159477234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946123497, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784946123497, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784946208212, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5270775556564331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784946208213, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784946208213, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784946222200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13769391179084778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36864000, "lr": 1.8745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946236291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320735514163971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946250273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333255022764206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37191680, "lr": 1.8912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946264325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13042645156383514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946278336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12685903906822205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37519360, "lr": 1.9079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946292237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13179096579551697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946306645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355409473180771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37847040, "lr": 1.9245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946320959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326817125082016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946335155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13090728223323822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38174720, "lr": 1.9412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946349681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388615369796753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946364101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13365936279296875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38502400, "lr": 1.9579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946378049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13727208971977234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946392816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317090541124344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38830080, "lr": 1.9745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946406922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13308702409267426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946406922, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784946406923, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784946489509, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5344972014427185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784946489510, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784946489510, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784946503590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308101862668991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39157760, "lr": 1.9912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946517561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13476932048797607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946531662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13753171265125275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39485440, "lr": 2.0079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946545783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13270117342472076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946559980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12724114954471588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39813120, "lr": 2.0245833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946574063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12912632524967194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0329166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946588072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12560974061489105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40140800, "lr": 2.0412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946602294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12377482652664185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946616911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13574884831905365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40468480, "lr": 2.0579166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946631446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13698239624500275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946645762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1342490017414093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40796160, "lr": 2.0745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946660724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12684254348278046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0829166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946675281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13075877726078033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41123840, "lr": 2.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946689266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1231008842587471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946689266, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784946689267, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784946774385, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5446968078613281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784946774386, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784946774386, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784946789030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320635825395584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41451520, "lr": 2.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946803440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13230593502521515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946817865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333594173192978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41779200, "lr": 2.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946832310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137422576546669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946846698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13588282465934753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42106880, "lr": 2.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946861102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13113069534301758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946875427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.126657634973526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42434560, "lr": 2.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946889794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13469095528125763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946904018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12429992854595184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42762240, "lr": 2.1745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946918564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12987950444221497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946932998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318736970424652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43089920, "lr": 2.1912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946947622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335400491952896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946962315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1398126780986786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43417600, "lr": 2.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946976706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357780247926712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946976707, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784946976707, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784947060278, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5587624907493591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784947060279, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784947060279, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784947075177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13643589615821838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43745280, "lr": 2.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947089552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13097447156906128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947103658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320662647485733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44072960, "lr": 2.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947118191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385875642299652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947133161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1293112337589264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44400640, "lr": 2.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947148379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13779115676879883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947163065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13430531322956085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44728320, "lr": 2.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947177895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13663138449192047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947192745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13392706215381622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45056000, "lr": 2.2912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947207716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13004374504089355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947222685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13968510925769806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45383680, "lr": 2.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947237851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13399049639701843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947252971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1279892921447754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45711360, "lr": 2.3245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947268128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329597681760788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947268129, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784947268129, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784947352615, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5728825330734253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784947352616, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784947352616, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784947367992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14145693182945251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46039040, "lr": 2.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947382778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13205650448799133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947397991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13276298344135284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46366720, "lr": 2.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947412812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12964938580989838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947427975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346205472946167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46694400, "lr": 2.3745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947443065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13283702731132507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947457789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13279391825199127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47022080, "lr": 2.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947472595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13475990295410156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947487325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13012591004371643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47349760, "lr": 2.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947502327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13167813420295715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947517256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13245028257369995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47677440, "lr": 2.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947532393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13653019070625305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947547055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354488730430603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48005120, "lr": 2.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947561978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13656750321388245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.4495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947561979, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784947561979, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784947646699, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.586741030216217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784947646700, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784947646700, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784947661294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13072934746742249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48332800, "lr": 2.4579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947676264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343052089214325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947691162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13360817730426788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48660480, "lr": 2.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947706470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13118639588356018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947721160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13650785386562347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48988160, "lr": 2.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947735719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1284850537776947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947751051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13285906612873077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49315840, "lr": 2.507916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947765585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12209378182888031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.5162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947780497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294926255941391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49643520, "lr": 2.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947795400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13023445010185242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947810284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12974247336387634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49971200, "lr": 2.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947825039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12980705499649048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.5495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947839810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12780117988586426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50298880, "lr": 2.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947854505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12924855947494507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.5662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947854506, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784947854506, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784947936723, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5965870022773743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784947936724, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784947936725, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784947950945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12075558304786682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50626560, "lr": 2.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947965641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12517252564430237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947980296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12739957869052887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50954240, "lr": 2.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947995186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13214638829231262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948010239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13056445121765137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51281920, "lr": 2.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948025064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1199812963604927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948039998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12847089767456055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51609600, "lr": 2.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948054697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13058054447174072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948069370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13507765531539917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51937280, "lr": 2.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948083583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1262931078672409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.6495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948098689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1313568353652954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52264960, "lr": 2.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948113103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1159847304224968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.6662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948127368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11896755546331406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52592640, "lr": 2.6745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948141943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1237843856215477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948141944, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784948141945, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784948225096, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6090077757835388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784948225097, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784948225097, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784948239724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11973784118890762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52920320, "lr": 2.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948254243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12599775195121765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.6995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948268920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12604016065597534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53248000, "lr": 2.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948283585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1296379268169403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948298235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11898533254861832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53575680, "lr": 2.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948312960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13261960446834564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948327707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12786026298999786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53903360, "lr": 2.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948342558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11957090348005295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.7495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948357427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12527139484882355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54231040, "lr": 2.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948372659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12489907443523407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948387453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1164134219288826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54558720, "lr": 2.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948402853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298760026693344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948417853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12522506713867188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54886400, "lr": 2.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948432495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1269473284482956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948432496, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784948432496, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784948515497, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6226760745048523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784948515498, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784948515499, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784948529869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1256280392408371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55214080, "lr": 2.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948545032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11671819537878036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948559529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12013257294893265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55541760, "lr": 2.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948574154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11964146047830582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948588615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12251534312963486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55869440, "lr": 2.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948602805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12456847727298737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948617265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11383070796728134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56197120, "lr": 2.8579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948631617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12138435244560242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948645943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12364079058170319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56524800, "lr": 2.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948660053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1221560388803482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948673974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11431704461574554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56852480, "lr": 2.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948688309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12195175141096115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948702677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10777305066585541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57180160, "lr": 2.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948717178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12246658653020859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948717179, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784948717179, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784948803283, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6373053193092346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784948803284, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784948803284, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784948818187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11523382365703583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57507840, "lr": 2.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948832860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11183086037635803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948847285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12608110904693604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57835520, "lr": 2.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948862075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1203472912311554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948876479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12202437967061996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58163200, "lr": 2.9579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948890860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11440064013004303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948904969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11818862706422806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58490880, "lr": 2.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948919986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12070570886135101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948934554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11919202655553818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58818560, "lr": 2.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948949143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11047910898923874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948963827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11905796825885773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59146240, "lr": 3.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948978336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12052587419748306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.0162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948992758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1239781305193901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59473920, "lr": 3.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949007338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1156887635588646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949007338, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784949007339, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784949092772, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6641541123390198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784949092773, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784949092774, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784949107566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1181301549077034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59801600, "lr": 3.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949122221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1096779853105545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949136629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10854335129261017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60129280, "lr": 3.0579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949151027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11244995146989822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.0662499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949165472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10798350721597672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60456960, "lr": 3.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949179960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.120906762778759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949194590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11451924592256546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60784640, "lr": 3.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949209569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11901333928108215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949224096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10051814466714859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61112320, "lr": 3.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949239127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11897161602973938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.1162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949254082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11228842288255692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61440000, "lr": 3.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949268901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11203563213348389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949283418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10824114829301834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61767680, "lr": 3.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949298389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11552505195140839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949298389, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784949298390, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784949384916, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6992281079292297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784949384917, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784949384917, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784949399547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1192409098148346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62095360, "lr": 3.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949414242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11847119778394699, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.1662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949428997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12241227179765701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62423040, "lr": 3.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949443627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10418705642223358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949458414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11259254068136215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62750720, "lr": 3.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949473004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1170523464679718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949487848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11854133009910583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63078400, "lr": 3.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949502624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11707259714603424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949517200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10660119354724884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63406080, "lr": 3.2245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949531819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11730840802192688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949546340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11264733225107193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63733760, "lr": 3.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949560843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11545831710100174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949575279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111338309943676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64061440, "lr": 3.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949590248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12354318797588348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949590248, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784949590248, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784949676136, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7327044606208801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784949676137, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784949676137, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784949691025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12236051261425018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64389120, "lr": 3.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949705869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11580630391836166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949720381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11424748599529266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64716800, "lr": 3.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949735350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11931472271680832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949750041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10703637450933456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65044480, "lr": 3.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949764764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10920462757349014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949778939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11176209896802902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65372160, "lr": 3.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949794187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10713478922843933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949808941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10968312621116638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65699840, "lr": 3.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949824178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11507593095302582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949839158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12198494374752045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66027520, "lr": 3.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949853934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1136488988995552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949868980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.125734344124794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66355200, "lr": 3.3745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949883998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11135559529066086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949883999, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784949883999, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784949968707, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.75725257396698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784949968708, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784949968708, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784949983777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1243482381105423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66682880, "lr": 3.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949998607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11268525570631027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.3995833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950013448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11375816911458969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67010560, "lr": 3.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950028540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11235331743955612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950043370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11669475585222244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67338240, "lr": 3.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950058463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11400190740823746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950073794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11978667229413986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67665920, "lr": 3.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950089107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11866120994091034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950104303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11364184319972992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67993600, "lr": 3.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950119785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11078257858753204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950134980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11413022130727768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68321280, "lr": 3.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950149861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11154548078775406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950164671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12284570932388306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68648960, "lr": 3.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950179967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11325860768556595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950179968, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784950179968, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784950267056, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7601513862609863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784950267057, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784950267057, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784950281968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11944284290075302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68976640, "lr": 3.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950297117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11694103479385376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950312086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11163634061813354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69304320, "lr": 3.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950327314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11212333291769028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950342314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10642755031585693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69632000, "lr": 3.5412500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950357280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1123436689376831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950372233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11555012315511703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69959680, "lr": 3.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950386857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10808414965867996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950401786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10817396640777588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70287360, "lr": 3.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950416632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10685457289218903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950431907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12176334112882614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70615040, "lr": 3.5912500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950447152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10390472412109375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950462253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11827053129673004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70942720, "lr": 3.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950477870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1186804249882698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950477871, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784950477871, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784950563082, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7624872922897339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784950563083, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784950563083, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784950578382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11649609357118607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71270400, "lr": 3.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950593308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11549056321382523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950608236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11428650468587875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71598080, "lr": 3.6412499999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950623508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11278501152992249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950638644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1009308248758316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71925760, "lr": 3.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950653638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10090487450361252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950668734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11707717180252075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72253440, "lr": 3.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950683600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11457159370183945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950698529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10198068618774414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72581120, "lr": 3.6912499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950713648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10080414265394211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950728419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10455819219350815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72908800, "lr": 3.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950743462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11105462163686752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950758517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087915301322937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73236480, "lr": 3.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950773560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11318064481019974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950773597, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784950773597, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784950857242, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7641659379005432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784950857244, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784950857244, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784950871999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10381484776735306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73564160, "lr": 3.7412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950887238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.105070561170578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950902417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11300505697727203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73891840, "lr": 3.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950917241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10806339979171753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950932314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10639765858650208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74219520, "lr": 3.7745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950947783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11306784301996231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950962613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1126423180103302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74547200, "lr": 3.7912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950977230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10955392569303513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950992219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11506551504135132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74874880, "lr": 3.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951006784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10920751094818115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951021221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09853167086839676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75202560, "lr": 3.8245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951035764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11238931119441986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951049943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10630670189857483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75530240, "lr": 3.8412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951064331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10414193570613861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951064331, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784951064332, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784951146627, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7657396793365479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784951146627, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784951146628, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784951160895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10414861887693405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75857920, "lr": 3.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951175517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11262402683496475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951190061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09478321671485901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76185600, "lr": 3.8745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951204387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10265859216451645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951218863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11178240180015564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76513280, "lr": 3.8912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951233161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10367892682552338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951247725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08792971074581146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76840960, "lr": 3.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951262310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10837315768003464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951277034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10010869801044464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77168640, "lr": 3.9245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951291817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1178085133433342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951306767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1068267896771431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77496320, "lr": 3.9412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951321291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09979904443025589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951336316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09780265390872955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77824000, "lr": 3.9579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951351190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10607153922319412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951351190, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784951351191, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784951434428, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7666959166526794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784951434429, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784951434429, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784951449008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11482950299978256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78151680, "lr": 3.9745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951463612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11176272481679916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951477604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10402427613735199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78479360, "lr": 3.9912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951492882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10517571866512299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951507098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09859024733304977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78807040, "lr": 4.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951521721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10014737397432327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951536184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10556618869304657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79134720, "lr": 4.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951550766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10652752220630646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951565047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09644816070795059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79462400, "lr": 4.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951579654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10810810327529907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951594082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1043815165758133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79790080, "lr": 4.0579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951608545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10666074603796005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951622869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10856427252292633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80117760, "lr": 4.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951637307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11280310899019241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951637307, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784951637308, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} diff --git a/recommendation_v4/rcp_logs/gbs_8192/seed166896187.log b/recommendation_v4/rcp_logs/gbs_8192/seed166896187.log new file mode 100644 index 000000000..3207ad1e8 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_8192/seed166896187.log @@ -0,0 +1,1847 @@ +:::MLLOG {"namespace": "", "time_ms": 1785027752815, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1785027752815, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1785027773725, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1785027773730, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1785027773730, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1785027773730, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1785027773730, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1785027773730, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 8192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1785027773730, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1785027773730, "event_type": "POINT_IN_TIME", "key": "seed", "value": 166896187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1785027773731, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1785027773731, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1785027773731, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1785027773731, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1785027773751, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1785027773752, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1785028686126, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1785028686127, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1785028686499, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1785028846313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388864666223526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840, "lr": 7.916666666666666e-10}} +:::MLLOG {"namespace": "", "time_ms": 1785028858867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13892558217048645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.6249999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028871827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385093480348587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 491520, "lr": 2.458333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028884932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13865765929222107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028897950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13876421749591827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 819200, "lr": 4.125e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028911323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13883990049362183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028924430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13869628310203552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1146880, "lr": 5.7916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028937775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388900876045227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.625e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028951094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13881343603134155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1474560, "lr": 7.458333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028964198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13882973790168762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.291666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028978013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386541724205017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1802240, "lr": 9.124999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028991646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13876408338546753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785029005133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13861219584941864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2129920, "lr": 1.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029018486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13947561383247375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029018486, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1785029018488, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1785029156936, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4998452365398407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1785029156937, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1785029156937, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1785029170294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384577453136444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2457600, "lr": 1.2458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029183761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13843296468257904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029197485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383303701877594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2785280, "lr": 1.4124999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029210907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13846920430660248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4958333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029224573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13842083513736725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3112960, "lr": 1.5791666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029238224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13844294846057892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029251491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13826227188110352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3440640, "lr": 1.7458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029265006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13836286962032318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029279087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380576491355896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3768320, "lr": 1.9124999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029292793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13816821575164795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9958333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029306567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13842391967773438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4096000, "lr": 2.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029320152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13867121934890747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.1624999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029333612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13808469474315643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4423680, "lr": 2.2458333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029347585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13814450800418854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3291666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029347586, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1785029347586, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1785029433566, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5000662803649902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1785029433567, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1785029433567, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1785029447060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13791626691818237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4751360, "lr": 2.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029460554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13823148608207703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029474159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375342458486557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5079040, "lr": 2.5791666666666668e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029488060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370234489440918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029501680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13811320066452026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5406720, "lr": 2.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029515154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13811199367046356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029528743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373114138841629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5734400, "lr": 2.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029542574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13729971647262573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029555893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13677263259887695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6062080, "lr": 3.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029569014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378972977399826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029582320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13739019632339478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6389760, "lr": 3.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029595476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13751570880413055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029609725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13793060183525085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6717440, "lr": 3.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029623313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137541726231575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029623313, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1785029623313, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1785029709475, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5003789067268372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1785029709476, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1785029709476, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1785029722631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367262452840805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7045120, "lr": 3.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029736132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374284029006958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029749589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13663901388645172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7372800, "lr": 3.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029763702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13710007071495056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.829166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029777379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375618427991867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7700480, "lr": 3.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029791140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13768212497234344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029804826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366763859987259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8028160, "lr": 4.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029818460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13737890124320984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029832595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13659733533859253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8355840, "lr": 4.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029846508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13851653039455414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3291666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029860307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13634857535362244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8683520, "lr": 4.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029873926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375153660774231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.4958333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029887824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351761370897293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9011200, "lr": 4.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029901844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13665130734443665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.6624999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029901845, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1785029901845, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1785029984466, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5007866621017456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1785029984467, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1785029984467, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1785029997715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13612690567970276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9338880, "lr": 4.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030011203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13532063364982605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030024536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13777406513690948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9666560, "lr": 4.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030038257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13440048694610596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030052126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13638760149478912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9994240, "lr": 5.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030065600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13318558037281036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1624999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030079370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13549232482910156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10321920, "lr": 5.2458333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030092705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13509997725486755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030106145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378607600927353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10649600, "lr": 5.4124999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030119408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13502798974514008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030132689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14854854345321655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10977280, "lr": 5.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030146348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353890597820282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030159689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13364654779434204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11304960, "lr": 5.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030173321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13780459761619568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030173322, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1785030173322, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1785030256541, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5013368725776672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1785030256542, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1785030256542, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1785030270029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13489198684692383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11632640, "lr": 5.9124999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030283467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13976462185382843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030297040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358690708875656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11960320, "lr": 6.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030310558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330433189868927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030324199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13449501991271973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12288000, "lr": 6.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030338070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1392449587583542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.329166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030351830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330011636018753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12615680, "lr": 6.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030365792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13250704109668732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030379298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13284318149089813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12943360, "lr": 6.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030392480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308189481496811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030405851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1306505799293518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13271040, "lr": 6.745833333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030418981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13493871688842773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030432323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13120348751544952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13598720, "lr": 6.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030445970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13906274735927582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030445971, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1785030445971, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1785030526419, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.502023458480835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1785030526420, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1785030526421, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1785030539973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13346491754055023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13926400, "lr": 7.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030553401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330994963645935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030567363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356644630432129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14254080, "lr": 7.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030581109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13503660261631012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030594865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13783931732177734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14581760, "lr": 7.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030608601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12728546559810638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030622038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13529641926288605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14909440, "lr": 7.579166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030635709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12880831956863403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030649553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13338106870651245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15237120, "lr": 7.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030663688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323302984237671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.829166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030677564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13325008749961853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15564800, "lr": 7.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030691624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13422586023807526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.995833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030705693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13537657260894775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15892480, "lr": 8.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030719516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13352608680725098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030719517, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785030719518, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785030800419, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5030142068862915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785030800420, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785030800420, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785030814525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132379949092865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16220160, "lr": 8.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030828403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13383346796035767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030842322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13526636362075806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16547840, "lr": 8.412500000000001e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030856424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13534638285636902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030870360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13986501097679138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16875520, "lr": 8.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030884149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332627832889557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.662499999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030899084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352461725473404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17203200, "lr": 8.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030912886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335020363330841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030926832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325966864824295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17530880, "lr": 8.912499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030941311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13171479105949402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030955275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13602393865585327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17858560, "lr": 9.079166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030969720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335003823041916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.162499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030983605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13322541117668152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18186240, "lr": 9.245833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030997404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13060498237609863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030997405, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785030997405, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785031080435, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5042583346366882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785031080436, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785031080436, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785031095162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378413438796997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18513920, "lr": 9.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031109190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347648948431015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.495833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031123252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13531872630119324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18841600, "lr": 9.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031137506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13123908638954163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.662499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031151427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13362325727939606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19169280, "lr": 9.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031165420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13399560749530792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031179562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13720563054084778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19496960, "lr": 9.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031193755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13390101492404938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031207714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13292592763900757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19824640, "lr": 1.0079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031221423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12967421114444733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0162499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031234896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12745866179466248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20152320, "lr": 1.0245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031248534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13289101421833038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031262233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12943284213542938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20480000, "lr": 1.0412499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031275453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12437853962182999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031275454, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1785031275454, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1785031360127, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.505611002445221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1785031360128, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1785031360129, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1785031374089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312117725610733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20807680, "lr": 1.0579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031387801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359381377696991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031401749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13342425227165222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21135360, "lr": 1.0745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031415711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1302594244480133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031429264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12820588052272797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21463040, "lr": 1.0912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031443221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1265067458152771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031457161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12874653935432434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21790720, "lr": 1.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031470801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13245758414268494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031484517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13529863953590393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22118400, "lr": 1.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031498286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12488160282373428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031511929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1284051537513733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22446080, "lr": 1.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031525498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13440948724746704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031539304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13092860579490662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22773760, "lr": 1.1579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031553436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12922519445419312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031553436, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1785031553437, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1785031638534, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5072272419929504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1785031638535, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1785031638535, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1785031652442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303904801607132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23101440, "lr": 1.1745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031666177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12902472913265228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031679554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13295163214206696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23429120, "lr": 1.1912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031693874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1236753985285759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031707825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295754462480545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23756800, "lr": 1.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031721290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1256958395242691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031734875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298544704914093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24084480, "lr": 1.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031748754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12158294767141342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031762248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13562104105949402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24412160, "lr": 1.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031776265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13740584254264832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031790058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1235765665769577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24739840, "lr": 1.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031804223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12321179360151291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031818267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12558460235595703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25067520, "lr": 1.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031832217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12885522842407227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031832218, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1785031832218, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1785031915522, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5090755820274353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1785031915523, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1785031915523, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1785031928982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13660329580307007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25395200, "lr": 1.2912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031943167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.136553555727005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031956852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13559991121292114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25722880, "lr": 1.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031970386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13989143073558807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031984024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13675056397914886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26050560, "lr": 1.3245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031997658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13817130029201508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032011182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12735801935195923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26378240, "lr": 1.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032024645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12604360282421112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032038068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14188578724861145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26705920, "lr": 1.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032052029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14093023538589478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032066023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12741759419441223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27033600, "lr": 1.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032079888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1285352110862732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032093663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13278305530548096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27361280, "lr": 1.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032106875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13141947984695435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032106876, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1785032106876, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1785032192924, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5110593438148499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1785032192925, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1785032192926, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1785032207769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1414095163345337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27688960, "lr": 1.4079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032222424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14227867126464844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032236653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352609246969223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28016640, "lr": 1.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032250746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1393740177154541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032265297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14481516182422638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28344320, "lr": 1.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032279371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14106255769729614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032293868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12584784626960754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28672000, "lr": 1.4579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032308425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13753144443035126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032323164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317163109779358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28999680, "lr": 1.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032337918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13625216484069824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032352427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13767726719379425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29327360, "lr": 1.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032366604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13297978043556213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4995833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032381054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344914436340332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29655040, "lr": 1.5079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032395176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13649186491966248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032395177, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1785032395177, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1785032479297, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5132236480712891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1785032479298, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1785032479298, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1785032492975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387183517217636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29982720, "lr": 1.5245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032507080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13823886215686798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032521363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131539449095726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30310400, "lr": 1.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032535604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13299348950386047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032549879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13401761651039124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30638080, "lr": 1.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032564361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13983245193958282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032578526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327466070652008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30965760, "lr": 1.5745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032593117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1413416564464569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032607009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14063099026679993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31293440, "lr": 1.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032621560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13493460416793823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032635714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12785857915878296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31621120, "lr": 1.6079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032649590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13006015121936798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032663578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13409626483917236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31948800, "lr": 1.6245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032677746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12752772867679596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032677747, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1785032677748, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1785032761927, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5159296989440918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1785032761927, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1785032761928, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1785032775893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13195309042930603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32276480, "lr": 1.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032790006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13687734305858612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032804155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336178034543991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32604160, "lr": 1.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032818569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13187359273433685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032832895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13395707309246063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32931840, "lr": 1.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032847235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13342951238155365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032862130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133649080991745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33259520, "lr": 1.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032876744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13248786330223083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032891000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13976982235908508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33587200, "lr": 1.7079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032905523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13181471824645996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032919491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288854479789734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33914880, "lr": 1.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032933179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13098803162574768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032947066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13256309926509857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34242560, "lr": 1.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032961070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355534940958023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7495833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032961071, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1785032961071, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1785033044468, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5196309685707092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1785033044469, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1785033044469, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1785033058503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13340669870376587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34570240, "lr": 1.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033072338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1228279247879982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033086073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1305721551179886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34897920, "lr": 1.7745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033099894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13037805259227753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033113594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13511160016059875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35225600, "lr": 1.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033127855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13534590601921082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033141669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13543155789375305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35553280, "lr": 1.8079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033155853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12397385388612747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8162500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033169709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311330497264862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35880960, "lr": 1.8245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033183331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13104869425296783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033197896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13030482828617096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36208640, "lr": 1.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033211955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13328437507152557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033226006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1307576596736908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36536320, "lr": 1.8579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033239860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13011138141155243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033239860, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1785033239861, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1785033323850, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5245118737220764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1785033323851, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1785033323851, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1785033337760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13790582120418549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36864000, "lr": 1.8745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033351759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13235045969486237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033365587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334044337272644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37191680, "lr": 1.8912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033379388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13032440841197968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033393122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1270264834165573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37519360, "lr": 1.9079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033406910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13188713788986206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033421161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13559430837631226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37847040, "lr": 1.9245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033435305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13261345028877258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033449493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311928778886795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38174720, "lr": 1.9412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033463764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388966590166092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033477890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13357475399971008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38502400, "lr": 1.9579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033491559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373138576745987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033505915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1319921612739563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38830080, "lr": 1.9745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033519743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331401765346527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033519744, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1785033519744, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1785033604247, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5310262441635132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1785033604248, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1785033604248, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1785033618131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310853511095047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39157760, "lr": 1.9912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033631943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13493190705776215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033645735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13791489601135254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39485440, "lr": 2.0079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033659569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13339944183826447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033673252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12723788619041443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39813120, "lr": 2.0245833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033687049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1292969137430191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0329166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033700939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12600691616535187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40140800, "lr": 2.0412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033714689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12382451444864273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033728763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13596442341804504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40468480, "lr": 2.0579166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033742972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13713042438030243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033757070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344618946313858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40796160, "lr": 2.0745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033771535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12671038508415222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0829166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033785508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1306517869234085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41123840, "lr": 2.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033799100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12356328964233398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033799100, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1785033799100, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1785033883719, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5399622917175293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1785033883720, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1785033883720, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1785033898065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13221712410449982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41451520, "lr": 2.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033912302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326482594013214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033926415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338050663471222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41779200, "lr": 2.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033940488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375880092382431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033954524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13593263924121857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42106880, "lr": 2.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033968644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13102224469184875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033982708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1269039660692215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42434560, "lr": 2.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033996877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13502176105976105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034010837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12438275665044785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42762240, "lr": 2.1745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034025182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301429718732834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034039340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321643590927124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43089920, "lr": 2.1912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034053721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13398106396198273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034068178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14024177193641663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43417600, "lr": 2.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034082236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363077014684677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034082237, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1785034082237, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1785034165620, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5532662272453308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1785034165621, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1785034165621, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1785034180245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13643470406532288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43745280, "lr": 2.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034194450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13136112689971924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034208417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13234758377075195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44072960, "lr": 2.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034222611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13886532187461853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034237299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12989813089370728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44400640, "lr": 2.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034251806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13829660415649414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034265932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346624493598938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44728320, "lr": 2.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034280215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13702255487442017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034294226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13427820801734924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45056000, "lr": 2.2912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034308540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13044236600399017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034322732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14027146995067596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45383680, "lr": 2.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034336909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343926191329956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034351140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12857605516910553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45711360, "lr": 2.3245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034365461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13345178961753845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034365462, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1785034365462, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1785034448687, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5677496790885925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1785034448687, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1785034448688, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1785034463374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1419496387243271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46039040, "lr": 2.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034477592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323796510696411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034492792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332942694425583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46366720, "lr": 2.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034507359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1302345097064972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034522358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350250393152237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46694400, "lr": 2.3745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034537155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333354264497757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034551612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13351786136627197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47022080, "lr": 2.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034566123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351538598537445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034580440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13083414733409882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47349760, "lr": 2.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034595100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13213996589183807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034609494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13253526389598846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47677440, "lr": 2.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034623979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13675685226917267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034638157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135686457157135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48005120, "lr": 2.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034652836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13686583936214447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.4495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034652837, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1785034652837, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1785034734236, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5837122797966003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1785034734237, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1785034734237, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1785034748636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13142180442810059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48332800, "lr": 2.4579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034763337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13529041409492493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034777808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13434386253356934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48660480, "lr": 2.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034792768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13181617856025696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034807125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13693304359912872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48988160, "lr": 2.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034821476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12885531783103943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034836391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13314247131347656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49315840, "lr": 2.507916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034850378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12330485880374908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.5162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034864548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13044121861457825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49643520, "lr": 2.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034878711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13083410263061523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034892866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13063889741897583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49971200, "lr": 2.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034906893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13089467585086823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.5495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034921137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12826766073703766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50298880, "lr": 2.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034935321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13039995729923248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.5662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034935322, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1785034935322, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1785035017673, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.59700608253479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1785035017674, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1785035017674, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1785035031583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12179385125637054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50626560, "lr": 2.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035045718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12662896513938904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035059833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12851206958293915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50954240, "lr": 2.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035074199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13241654634475708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035088915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13131219148635864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51281920, "lr": 2.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035103297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12136378139257431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035118055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13010051846504211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51609600, "lr": 2.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035132607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13147668540477753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035147065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13652287423610687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51937280, "lr": 2.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035160960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1274699717760086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.6495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035175788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13261611759662628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52264960, "lr": 2.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035189883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11752640455961227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.6662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035204142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12085942178964615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52592640, "lr": 2.6745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035218549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12502416968345642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035218550, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1785035218550, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1785035301516, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6103338003158569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1785035301516, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1785035301517, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1785035315790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12097834050655365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52920320, "lr": 2.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035329840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1269187033176422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.6995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035344057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12796759605407715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53248000, "lr": 2.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035358377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13090679049491882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035372455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12049466371536255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53575680, "lr": 2.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035386661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343667060136795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035400543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12876570224761963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53903360, "lr": 2.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035414655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12120505422353745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.7495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035429099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12613391876220703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54231040, "lr": 2.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035443660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1263459473848343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035457979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11741597950458527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54558720, "lr": 2.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035472793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13083192706108093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035487204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12657064199447632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54886400, "lr": 2.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035501549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.128393292427063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035501550, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1785035501550, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1785035585581, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6246834993362427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1785035585582, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1785035585582, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1785035599471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12722909450531006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55214080, "lr": 2.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035614262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11822567135095596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035628564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12151923030614853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55541760, "lr": 2.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035642942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12054167687892914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035657031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12312858551740646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55869440, "lr": 2.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035671163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12529100477695465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035685178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1157054454088211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56197120, "lr": 2.8579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035699254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12250640988349915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035713291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12506625056266785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56524800, "lr": 2.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035727403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12357929348945618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035741228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11466113477945328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56852480, "lr": 2.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035755271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12310729920864105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035769281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10951048880815506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57180160, "lr": 2.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035783339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12387930601835251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035783340, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1785035783340, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1785035866962, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6396524906158447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1785035866962, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1785035866963, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1785035881343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11589287966489792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57507840, "lr": 2.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035895485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11373619735240936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035909642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12817609310150146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57835520, "lr": 2.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035924240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1212105005979538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035938476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12307200580835342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58163200, "lr": 2.9579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035952580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11589043587446213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035966347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11899782717227936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58490880, "lr": 2.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035981455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12143594771623611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035995678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12004859745502472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58818560, "lr": 2.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036009969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11155746132135391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036024192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11956745386123657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59146240, "lr": 3.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036038235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12089099735021591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.0162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036052316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12470508366823196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59473920, "lr": 3.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036066543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11645543575286865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036066544, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1785036066545, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1785036148509, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6650325655937195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1785036148510, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1785036148511, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1785036162802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11848466843366623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59801600, "lr": 3.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036176893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1113722175359726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036190870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10851946473121643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60129280, "lr": 3.0579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036205024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11282750964164734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.0662499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036219202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10815343260765076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60456960, "lr": 3.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036233375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12162543833255768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036247966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1158323585987091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60784640, "lr": 3.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036262740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1195920929312706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036277105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10076769441366196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61112320, "lr": 3.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036291799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11890680342912674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.1162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036306261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11219576001167297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61440000, "lr": 3.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036320415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11209841072559357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036334491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10881882160902023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61767680, "lr": 3.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036348719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11609241366386414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036348720, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1785036348720, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1785036432769, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7017551064491272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1785036432770, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1785036432770, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1785036446982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11922408640384674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62095360, "lr": 3.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036461691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11894481629133224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.1662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036476181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1225506067276001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62423040, "lr": 3.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036490357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10463158786296844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036504654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11251737922430038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62750720, "lr": 3.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036518818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11713359504938126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036533263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11862380057573318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63078400, "lr": 3.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036547652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11797159165143967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036561818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10774268954992294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63406080, "lr": 3.2245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036575941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11725763231515884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036590044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11350557208061218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63733760, "lr": 3.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036603968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11633169651031494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036617902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11205752193927765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64061440, "lr": 3.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036632366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12411196529865265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036632367, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1785036632367, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1785036714871, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7346967458724976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1785036714872, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1785036714872, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1785036729343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12281528860330582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64389120, "lr": 3.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036743682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11553246527910233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036757878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11479578912258148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64716800, "lr": 3.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036772623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1205926388502121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036786935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10835354775190353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65044480, "lr": 3.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036801180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10999445617198944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036815072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1113249734044075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65372160, "lr": 3.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036829701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10779767483472824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036844034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11016925424337387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65699840, "lr": 3.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036858735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11554615944623947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036873174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12252155691385269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66027520, "lr": 3.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036887473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1133807823061943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036902028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1269698292016983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66355200, "lr": 3.3745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036916574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1114434227347374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036916575, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1785036916575, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1785036999650, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7573849558830261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1785036999651, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1785036999651, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1785037014192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12441252171993256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66682880, "lr": 3.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037028607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1126139834523201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.3995833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037042822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11503225564956665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67010560, "lr": 3.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037057449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11239831149578094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037071748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11799058318138123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67338240, "lr": 3.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037086599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1154438927769661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037101568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11983969807624817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67665920, "lr": 3.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037116376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11922219395637512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037130930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11406566202640533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67993600, "lr": 3.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037146217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11065463721752167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037160783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11383412778377533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68321280, "lr": 3.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037175318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1138063371181488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037189760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12370728701353073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68648960, "lr": 3.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037204862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11375204473733902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037204862, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1785037204862, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1785037288474, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.763712465763092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1785037288475, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1785037288475, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1785037302971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11980598419904709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68976640, "lr": 3.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037317596, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11842509359121323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037332166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11181289702653885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69304320, "lr": 3.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037346941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11214258521795273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037361510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10624933987855911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69632000, "lr": 3.5412500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037376160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11336646229028702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037390893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11658724397420883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69959680, "lr": 3.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037405242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10910628736019135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037420128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10822586715221405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70287360, "lr": 3.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037434817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10724267363548279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037450014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12136687338352203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70615040, "lr": 3.5912500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037465212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10418597608804703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037480077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11849751323461533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70942720, "lr": 3.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037495535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11860337853431702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037495536, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1785037495536, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1785037579586, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7669507265090942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1785037579587, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1785037579587, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1785037594614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11609947681427002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71270400, "lr": 3.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037609288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11615807563066483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037624014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11506319791078568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71598080, "lr": 3.6412499999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037639087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11358412355184555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037653526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1011936143040657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71925760, "lr": 3.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037667963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09984087198972702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037682351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11712995171546936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72253440, "lr": 3.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037696718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11398350447416306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037711141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10170762240886688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72581120, "lr": 3.6912499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037725484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09951058030128479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037739517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10526330024003983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72908800, "lr": 3.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037753896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11117637157440186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037768166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10852327942848206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73236480, "lr": 3.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037782330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11294078826904297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037782371, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1785037782371, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1785037866155, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7687982320785522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1785037866156, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1785037866156, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1785037880565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10377296805381775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73564160, "lr": 3.7412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037895274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10598786920309067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037909858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11303692311048508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73891840, "lr": 3.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037924249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10846446454524994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037938710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1061740294098854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74219520, "lr": 3.7745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037953447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11181379854679108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037967710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11285907030105591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74547200, "lr": 3.7912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037981854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10955993086099625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037996317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11480589210987091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74874880, "lr": 3.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038010683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1090618446469307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038024811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09847968071699142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75202560, "lr": 3.8245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038038968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11241097748279572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038052997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10558854043483734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75530240, "lr": 3.8412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038067131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10447248816490173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038067131, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1785038067132, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1785038149046, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7696540355682373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1785038149047, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1785038149047, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1785038163205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10490820556879044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75857920, "lr": 3.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038177626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1125435084104538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038192044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09410307556390762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76185600, "lr": 3.8745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038206373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10315942764282227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038220882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11195038259029388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76513280, "lr": 3.8912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038235153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1035185307264328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038249461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08747789263725281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76840960, "lr": 3.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038263614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10911315679550171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038277819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09923788160085678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77168640, "lr": 3.9245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038292415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11750049144029617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038307214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1063026487827301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77496320, "lr": 3.9412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038321625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09940338134765625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038336516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0975513756275177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77824000, "lr": 3.9579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038351334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10598514974117279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038351334, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1785038351335, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1785038432525, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7705265879631042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1785038432526, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1785038432526, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1785038447076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11500313133001328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78151680, "lr": 3.9745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038461629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11094208806753159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038475534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1033710166811943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78479360, "lr": 3.9912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038490455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10445661097764969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038504638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09865637123584747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78807040, "lr": 4.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038519155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10002803057432175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038533442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10535430908203125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79134720, "lr": 4.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038547810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10561622679233551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038562011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09643135964870453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79462400, "lr": 4.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038576419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1078835055232048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038590821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10420869290828705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79790080, "lr": 4.0579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038605003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10585901141166687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038619106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10768641531467438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80117760, "lr": 4.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038633230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11245133727788925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038633231, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1785038633231, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1785038716650, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7709944248199463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1785038716651, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1785038716651, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1785038730916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10874354094266891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80445440, "lr": 4.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038744927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09785610437393188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038759354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10145630687475204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80773120, "lr": 4.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038773872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10405623912811279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038788616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11053438484668732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81100800, "lr": 4.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038803388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11243452876806259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038817971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1001901626586914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81428480, "lr": 4.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038832746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10480035841464996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038847568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10740005970001221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81756160, "lr": 4.1579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038862252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10504709929227829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038876906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11230747401714325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82083840, "lr": 4.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038890940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09330585598945618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038906050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09751816093921661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82411520, "lr": 4.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038920316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10704618692398071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038920316, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1785038920317, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1785039003087, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7715526819229126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1785039003087, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1785039003088, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1785039017226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09097284078598022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82739200, "lr": 4.2079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039031674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1045365184545517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039045886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11008644104003906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83066880, "lr": 4.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039060313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09698177874088287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039074491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11119028925895691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83394560, "lr": 4.2412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039088927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11007623374462128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039103182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09727931022644043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83722240, "lr": 4.2579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039117295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11124823987483978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039131597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10480500757694244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84049920, "lr": 4.2745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039145643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10387072712182999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039160009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10698047280311584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84377600, "lr": 4.2912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039174018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10476260632276535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039188086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09719084948301315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84705280, "lr": 4.3079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039202447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10591413080692291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039202447, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1785039202448, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1785039285180, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7721750140190125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1785039285180, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1785039285181, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1785039299756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10337474942207336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85032960, "lr": 4.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039314355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10345802456140518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039328602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10508565604686737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85360640, "lr": 4.3412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039342966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10303313285112381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039357539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09733211249113083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85688320, "lr": 4.3579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039371847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09974389523267746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039386544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10517069697380066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86016000, "lr": 4.3745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039400557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09591494500637054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039415442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10788498818874359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86343680, "lr": 4.3912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039430172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082276776432991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039444808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11716609448194504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86671360, "lr": 4.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039459686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11419276148080826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039474255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11627382785081863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86999040, "lr": 4.4245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039488923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11436602473258972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039488924, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1785039488924, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1785039571886, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7726191282272339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1785039571887, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1785039571887, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1785039586446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1001076027750969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87326720, "lr": 4.4412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039601061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11523124575614929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039615496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10661309957504272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87654400, "lr": 4.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039630245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10159241408109665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039644637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09075438231229782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87982080, "lr": 4.4745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039659113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09769691526889801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039673170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10339514166116714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88309760, "lr": 4.4912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039687418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10718962550163269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039701545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10705637186765671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88637440, "lr": 4.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039715692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10007268190383911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039730463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.112592414021492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88965120, "lr": 4.5245833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039745186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0992426723241806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039759914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10983019322156906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89292800, "lr": 4.5412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039774242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11163070797920227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.5495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039774242, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1785039774243, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1785039856538, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773040771484375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1785039856539, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1785039856539, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1785039871259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10803041607141495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89620480, "lr": 4.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039885834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10135054588317871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039900337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11009690910577774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89948160, "lr": 4.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039915109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0912795439362526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039929469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11445281654596329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90275840, "lr": 4.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039943914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10638377815485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039959011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11059219390153885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90603520, "lr": 4.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039973511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11041577160358429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039988367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1217619925737381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90931200, "lr": 4.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040002976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10453750938177109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040017638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1099841371178627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91258880, "lr": 4.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040032504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12213563919067383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040046974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09894898533821106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91586560, "lr": 4.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040061479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11042453348636627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040061480, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1785040061480, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1785040145023, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7726192474365234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1785040145024, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1785040145024, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1785040159431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10418334603309631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91914240, "lr": 4.6745833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040174109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1178889274597168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040188524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10915838181972504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92241920, "lr": 4.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040203488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10335852950811386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040218003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09629254788160324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92569600, "lr": 4.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040232604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11286794394254684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040247631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1057170033454895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92897280, "lr": 4.7245833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040262519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10529664903879166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040277504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11540088057518005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93224960, "lr": 4.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040292084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09960329532623291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040307228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12509165704250336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93552640, "lr": 4.7579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040322256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10645533353090286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040337039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11516932398080826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93880320, "lr": 4.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040351676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10731396824121475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040351677, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1785040351677, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1785040437162, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7723516821861267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1785040437163, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1785040437163, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1785040452386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11361824721097946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94208000, "lr": 4.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040467357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11511780321598053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040482142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11102855205535889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94535680, "lr": 4.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040496958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11731526255607605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040511718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11430994421243668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94863360, "lr": 4.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040526656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11884795874357224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040541578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12675431370735168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95191040, "lr": 4.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040556107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12040113657712936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040570652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11626604944467545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95518720, "lr": 4.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040585388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11036496609449387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040599785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1083454117178917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95846400, "lr": 4.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040614515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10766199231147766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040629169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11474879086017609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96174080, "lr": 4.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040643995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1139555424451828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040643995, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1785040643995, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1785040725494, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773385226726532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1785040725495, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1785040725495, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1785040740531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11934511363506317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96501760, "lr": 4.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040755475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10701271891593933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040770031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10811524838209152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96829440, "lr": 4.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040785054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.108159638941288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040799870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11054948717355728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97157120, "lr": 4.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040814812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1089172512292862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040829733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11316762864589691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97484800, "lr": 4.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040844429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080164909362793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040859082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10556953400373459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97812480, "lr": 4.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040873247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10013424605131149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040888040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10374774783849716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98140160, "lr": 4.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040902601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09621893614530563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040917231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10380618274211884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98467840, "lr": 5.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040931562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10061753541231155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040931563, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1785040931563, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1785041013653, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742182016372681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1785041013654, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1785041013654, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1785041028047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10962038487195969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98795520, "lr": 5.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041042749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10079829394817352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041057216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09666043519973755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99123200, "lr": 5.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041071704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10825692862272263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041086269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11515845358371735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99450880, "lr": 5.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041100747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10441232472658157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041115231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10422273725271225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99778560, "lr": 5.074583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041129945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10441062599420547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041144448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10177452117204666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100106240, "lr": 5.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041158985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12039065361022949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041173626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1167164221405983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100433920, "lr": 5.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041188248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10105864703655243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041202889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09979866445064545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100761600, "lr": 5.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041217215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12034054845571518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041217215, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1785041217216, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1785041298772, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7743598818778992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1785041298772, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1785041298773, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1785041313842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11066799610853195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101089280, "lr": 5.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041328552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09611137956380844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041343122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10485020279884338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101416960, "lr": 5.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041357699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10258878022432327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041372079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10685084015130997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101744640, "lr": 5.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041387050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10473919659852982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.182916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041401596, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10558085888624191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102072320, "lr": 5.191249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041416111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1121390089392662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041430902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10458870977163315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102400000, "lr": 5.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041445587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792158544063568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041460145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1053149625658989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102727680, "lr": 5.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041474613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11445918679237366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041488761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10702084004878998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103055360, "lr": 5.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041503060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10177350044250488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041503060, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1785041503061, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1785041586804, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746888399124146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1785041586805, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1785041586805, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1785041601295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09934697300195694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103383040, "lr": 5.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041615769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09413854777812958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041630005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09639673680067062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103710720, "lr": 5.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041644449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09632007032632828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.282916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041658605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09649157524108887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104038400, "lr": 5.291249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041673145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09524869173765182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041687388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1001824215054512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104366080, "lr": 5.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041701652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09707532078027725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041716187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10266857594251633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104693760, "lr": 5.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041730670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10813841223716736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041745555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10491398721933365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105021440, "lr": 5.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041760120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10307382792234421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041774551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1143871322274208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105349120, "lr": 5.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041789614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11112384498119354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041789615, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1785041789615, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1785041871251, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7747883796691895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1785041871252, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1785041871252, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1785041886012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09780242294073105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105676800, "lr": 5.374583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041900557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10720910876989365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.382916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041915296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10990086197853088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106004480, "lr": 5.391249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041929686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1013612225651741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041944604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10016927123069763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106332160, "lr": 5.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041959345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10787316411733627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041974172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11019742488861084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106659840, "lr": 5.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041988989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10548839718103409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042004174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1027488261461258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106987520, "lr": 5.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042018992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10089242458343506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042033614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10020284354686737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107315200, "lr": 5.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042048230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10223399847745895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042062836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10926788300275803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107642880, "lr": 5.474583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042077163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1147192046046257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.482916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042077164, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1785042077164, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1785042159986, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7751001715660095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1785042159987, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1785042159987, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1785042174493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10989899933338165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107970560, "lr": 5.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042189093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10648002475500107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042203680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09078040719032288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108298240, "lr": 5.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042217865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10132724791765213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042232456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10637447983026505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108625920, "lr": 5.524583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042246765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772231966257095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042261060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11014667898416519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108953600, "lr": 5.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042275611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10162898898124695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042290456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10006498545408249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109281280, "lr": 5.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042305336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1040065810084343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042320391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10227999091148376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109608960, "lr": 5.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042335378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10764668881893158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042350744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10471223294734955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109936640, "lr": 5.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042365943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10906568914651871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042365944, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1785042365945, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1785042450033, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752967476844788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1785042450034, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1785042450034, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1785042464893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10383394360542297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110264320, "lr": 5.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042479931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10242263227701187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042494696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10420577973127365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110592000, "lr": 5.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042509120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10229256749153137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042524583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10724786669015884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110919680, "lr": 5.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042539017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10065823793411255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042553429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09790955483913422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111247360, "lr": 5.657916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042568206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1019936203956604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.666250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042582913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11178267002105713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111575040, "lr": 5.674583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042597496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10447724163532257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042611834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10361360758543015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111902720, "lr": 5.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042626244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10210102051496506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042640842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1037134900689125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112230400, "lr": 5.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042655337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10619506984949112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042655337, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1785042655338, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1785042737295, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7757523655891418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1785042737295, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1785042737296, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1785042751924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10805083066225052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112558080, "lr": 5.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042766704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1016206368803978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042781329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10362795740365982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112885760, "lr": 5.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042795984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10091659426689148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042810581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10444502532482147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113213440, "lr": 5.757916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042824665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11294341087341309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.766250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042839093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1010277047753334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113541120, "lr": 5.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042853353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1103651151061058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042867966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11285652965307236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113868800, "lr": 5.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042882850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10640523582696915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042897438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10257551819086075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114196480, "lr": 5.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042912091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10544182360172272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042927029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1048317551612854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114524160, "lr": 5.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042941681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09817589074373245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042941682, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1785042941682, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1785043022725, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.77606201171875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1785043022726, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1785043022726, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1785043037305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10109666734933853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114851840, "lr": 5.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043051654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09932383894920349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043066085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10541767627000809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115179520, "lr": 5.857916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043080062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10084161162376404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043095279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10791386663913727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115507200, "lr": 5.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043109939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10880748182535172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043124860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10761411488056183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115834880, "lr": 5.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043139908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11518741399049759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043154782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1138082966208458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116162560, "lr": 5.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043169648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10005885362625122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043184371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10382141172885895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116490240, "lr": 5.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043199214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10259774327278137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043214032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10522965341806412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116817920, "lr": 5.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043228781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1068452000617981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043228782, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1785043228782, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1785043310305, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7762929201126099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1785043310306, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1785043310306, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1785043325150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10194870084524155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117145600, "lr": 5.957916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043339878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10045137256383896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043355089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10859809070825577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117473280, "lr": 5.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043369897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10397375375032425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043384549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09428758174180984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117800960, "lr": 5.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043399742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11233719438314438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043414437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10090341418981552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118128640, "lr": 6.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043429377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10779411345720291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043444549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10263834148645401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118456320, "lr": 6.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043459810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10384789109230042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043474810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087837740778923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118784000, "lr": 6.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043490116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11393953114748001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043505279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10589509457349777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119111680, "lr": 6.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043520514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1029118224978447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043520515, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1785043520515, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1785043602218, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7763927578926086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1785043602219, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1785043602219, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1785043617486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11200391501188278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119439360, "lr": 6.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043632395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10867103189229965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043646969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10924665629863739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119767040, "lr": 6.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043661821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10655295103788376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043676262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1012631207704544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120094720, "lr": 6.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043690585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10722406208515167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043706138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11799687147140503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120422400, "lr": 6.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043720928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10864812135696411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043735923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11614721268415451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120750080, "lr": 6.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043751336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1222354918718338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043766025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10720216482877731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121077760, "lr": 6.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043780995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11000112444162369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043795901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11468889564275742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121405440, "lr": 6.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043810754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10994555801153183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043810754, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1785043810755, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1785043893372, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7765450477600098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1785043893373, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1785043893373, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1785043908302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11300722509622574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121733120, "lr": 6.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043923500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10935705155134201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043938451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11053990572690964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122060800, "lr": 6.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043953679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10623964667320251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043968796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10416454821825027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122388480, "lr": 6.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043984068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11342424154281616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043999276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10702870041131973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122716160, "lr": 6.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044014316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11475138366222382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044029858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11224675923585892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123043840, "lr": 6.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044045202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11996138095855713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044060669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1061784103512764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123371520, "lr": 6.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044075898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.117475226521492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044091069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071518138051033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123699200, "lr": 6.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044106471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1181902289390564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044106471, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1785044106472, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1785044187475, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7769509553909302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1785044187476, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1785044187476, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1785044202596, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1194211095571518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124026880, "lr": 6.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044217714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793255269527435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044232678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1188896894454956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124354560, "lr": 6.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044247559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1134498193860054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044262527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10744419693946838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124682240, "lr": 6.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044278097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11522433906793594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044293074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11099894344806671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125009920, "lr": 6.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044308431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11639619618654251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044323695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1121782660484314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125337600, "lr": 6.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044338718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10542760044336319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044353735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11217182874679565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125665280, "lr": 6.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044368800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10619690269231796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044383620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11657953262329102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125992960, "lr": 6.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044398720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10497432202100754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044398720, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1785044398721, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1785044484042, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7774965167045593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1785044484043, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1785044484043, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1785044499091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11006797850131989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126320640, "lr": 6.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044514153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1186828538775444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044528890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10998338460922241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126648320, "lr": 6.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044543866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11331165581941605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044559029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11621180176734924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126976000, "lr": 6.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044573962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10909565538167953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044588937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080918163061142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127303680, "lr": 6.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044604058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10944168269634247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.482916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044619372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1123032495379448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127631360, "lr": 6.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044634270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11235200613737106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044649277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10979790985584259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127959040, "lr": 6.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044664436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11018021404743195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044679691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11151320487260818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128286720, "lr": 6.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044694874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10838433355093002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044694875, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1785044694875, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1785044777087, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7778357267379761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1785044777088, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1785044777088, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1785044791942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1180603876709938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128614400, "lr": 6.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044807269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10963930189609528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044822370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11043362319469452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128942080, "lr": 6.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044836935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10238341242074966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044851635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10556020587682724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129269760, "lr": 6.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044866409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111485056579113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044880829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10758668184280396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129597440, "lr": 6.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044895260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10704091191291809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044909906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09722104668617249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129925120, "lr": 6.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044924350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10169175267219543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044938859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10580556839704514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130252800, "lr": 6.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044953445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10434891283512115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044968100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1096138060092926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130580480, "lr": 6.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044982637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11518941074609756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044982638, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1785044982638, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1785045067176, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7777783870697021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1785045067177, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1785045067177, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1785045081755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11402033269405365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130908160, "lr": 6.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045096239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10443959385156631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045110691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10005877912044525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131235840, "lr": 6.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045125320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09219402074813843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045139797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11436480283737183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131563520, "lr": 6.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045154280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10245753824710846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045169001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10265417397022247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131891200, "lr": 6.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045183959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11206967383623123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045198662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10656146705150604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132218880, "lr": 6.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045213657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10303138196468353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045228564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10824110358953476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132546560, "lr": 6.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045243515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1095397025346756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045258720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10821478813886642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132874240, "lr": 6.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045273724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10660909116268158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045273725, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1785045273725, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1785045358129, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7782645225524902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1785045358130, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1785045358130, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1785045372921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10934755951166153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133201920, "lr": 6.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045387873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10291798412799835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045402529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11061583459377289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133529600, "lr": 6.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045417130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09812494367361069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045432401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11561024188995361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133857280, "lr": 6.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045447553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10984479635953903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045462415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10265295207500458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134184960, "lr": 6.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045477562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10812396556138992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045492571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10756035894155502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134512640, "lr": 6.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045507208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10803088545799255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045522004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10946612805128098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134840320, "lr": 6.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045536811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09867589920759201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045551268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10637107491493225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135168000, "lr": 6.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045565787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08721259981393814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045565789, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1785045565789, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1785045650211, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7787061333656311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1785045650212, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1785045650212, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1785045664935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09565211832523346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135495680, "lr": 6.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045679672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11130370199680328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045694299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10646052658557892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135823360, "lr": 6.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045708930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10070864856243134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045723684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10181304067373276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136151040, "lr": 6.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045738283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0902218371629715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045752824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10351769626140594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136478720, "lr": 6.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045767539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09671753644943237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045782075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10336560010910034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136806400, "lr": 6.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045796979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10638153553009033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045811898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11021101474761963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137134080, "lr": 6.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045826689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082724779844284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045841271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10615944862365723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137461760, "lr": 6.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045856118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10261591523885727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045856119, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1785045856119, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1785045939763, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.778816282749176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1785045939764, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1785045939764, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1785045954639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09503151476383209, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137789440, "lr": 7.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045969911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10507071018218994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045984970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09506333619356155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138117120, "lr": 7.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045999718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10065476596355438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046014443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09677587449550629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138444800, "lr": 7.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046028670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0998934656381607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046042988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1088888943195343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138772480, "lr": 7.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046058441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10102303326129913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.066249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046073193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10903295874595642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139100160, "lr": 7.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046087915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.109792560338974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046102933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10581669956445694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139427840, "lr": 7.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046117808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10372191667556763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046132523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10942765325307846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139755520, "lr": 7.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046147383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10871820896863937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046147383, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1785046147384, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1785046229216, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7790049910545349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1785046229217, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1785046229217, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1785046243868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10611841082572937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140083200, "lr": 7.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046258815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10398370772600174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046273604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10273410379886627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140410880, "lr": 7.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046288229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09928284585475922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046303016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10668706893920898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140738560, "lr": 7.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046317882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09593557566404343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046333341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10374660789966583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141066240, "lr": 7.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046348265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10271547734737396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046363073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10991611331701279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141393920, "lr": 7.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046378118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09826590120792389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046392773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09772329777479172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141721600, "lr": 7.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046407703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10115677118301392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046422684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08979713171720505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142049280, "lr": 7.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046437687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1059899851679802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046437688, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1785046437688, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1785046520345, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7793453335762024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1785046520345, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1785046520346, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1785046535250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09837256371974945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142376960, "lr": 7.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046550552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10008037090301514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046565315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09898253530263901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142704640, "lr": 7.257916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046580444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10855937004089355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046595535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10061171650886536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143032320, "lr": 7.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046610703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1020498126745224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046625680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10052390396595001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143360000, "lr": 7.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046640385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10531030595302582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046655420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10096576064825058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143687680, "lr": 7.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046669951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11090686917304993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046685027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10057704150676727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144015360, "lr": 7.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046699670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10777606815099716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046714426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10480818897485733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144343040, "lr": 7.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046729391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10551097244024277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046729393, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1785046729393, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1785046811560, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7798947095870972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1785046811561, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1785046811562, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1785046826480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10489483177661896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144670720, "lr": 7.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046841263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0977945402264595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046856000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10636701434850693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144998400, "lr": 7.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046870858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1055009514093399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046885708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10208287090063095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145326080, "lr": 7.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046900035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10696499049663544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046914558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09658635407686234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145653760, "lr": 7.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046929307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09815321862697601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046943651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10501468926668167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145981440, "lr": 7.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046958005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10256284475326538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046972532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10113965719938278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146309120, "lr": 7.441250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046987076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10914833843708038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047001713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10622553527355194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146636800, "lr": 7.457916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047016435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09945735335350037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047016435, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1785047016436, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1785047100206, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7800891399383545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1785047100207, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1785047100207, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1785047114682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10160353779792786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146964480, "lr": 7.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047129075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1054711863398552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047143601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09391480684280396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147292160, "lr": 7.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047158585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0982886329293251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047173019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10195581614971161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147619840, "lr": 7.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047188103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09982233494520187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047202807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1067667230963707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147947520, "lr": 7.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047217745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10901985317468643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047232847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10885415226221085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148275200, "lr": 7.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047247878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0979967787861824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047262965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10138889402151108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148602880, "lr": 7.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047277481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0967361256480217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047292670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11160892248153687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148930560, "lr": 7.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047307244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10969797521829605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047307245, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1785047307245, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1785047393314, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7803248167037964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1785047393314, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1785047393315, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1785047407628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10529115796089172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149258240, "lr": 7.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047422982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10412904620170593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047437599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10606518387794495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149585920, "lr": 7.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047452654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11136960983276367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149749760, "lr": 7.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047467878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10662868618965149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149913600, "lr": 7.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047483096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10731446743011475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047497954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071338877081871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150241280, "lr": 7.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047512889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09709842503070831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150405120, "lr": 7.649583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047527875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1053718850016594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150568960, "lr": 7.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047542538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10280416160821915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047557426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10742169618606567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150896640, "lr": 7.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047571989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10330182313919067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151060480, "lr": 7.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047587143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10593456774950027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151224320, "lr": 7.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047601942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0944879874587059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047601982, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1785047601982, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1785047686244, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7806148529052734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1785047686245, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1785047686245, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1785047701024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09852336347103119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151552000, "lr": 7.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047715760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10496671497821808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151715840, "lr": 7.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047730420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10205145180225372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151879680, "lr": 7.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047745325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10458611696958542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047760094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10814480483531952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152207360, "lr": 7.741249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047774744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11952785402536392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152371200, "lr": 7.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047789552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10797090828418732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152535040, "lr": 7.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047804481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09918372333049774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047819367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11008394509553909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152862720, "lr": 7.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047834155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10954318940639496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153026560, "lr": 7.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047849171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11461375653743744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153190400, "lr": 7.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047864243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09741746634244919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047879541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10644151270389557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153518080, "lr": 7.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047894482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10230959206819534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153681920, "lr": 7.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047894520, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1785047894520, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1785047979617, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7805980443954468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1785047979618, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1785047979618, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1785047994668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09477915614843369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153845760, "lr": 7.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048009517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0976942926645279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.832916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048024372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1108071431517601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154173440, "lr": 7.841249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048039484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11681555211544037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154337280, "lr": 7.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048054258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11222726851701736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154501120, "lr": 7.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048068902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1101522296667099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048084672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10645449906587601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154828800, "lr": 7.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048099775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1193791851401329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154992640, "lr": 7.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048114892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10477698594331741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155156480, "lr": 7.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048130299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10936909168958664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048145609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10975199937820435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155484160, "lr": 7.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048160988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10988134890794754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155648000, "lr": 7.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048176379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11349024623632431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155811840, "lr": 7.924583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048191730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11058986186981201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.932916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048191731, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1785048191731, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1785048275345, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7812168002128601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1785048275346, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1785048275346, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1785048290632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10630954802036285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156139520, "lr": 7.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048305979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10932041704654694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156303360, "lr": 7.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048321558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1141524463891983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156467200, "lr": 7.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048336929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10982565581798553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048352220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10766307264566422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156794880, "lr": 7.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048367678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1068699061870575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156958720, "lr": 7.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048382998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10140091180801392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157122560, "lr": 7.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048398048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11130692809820175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048413206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12031547725200653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157450240, "lr": 8.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048428619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10808955878019333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157614080, "lr": 8.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048443948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10839786380529404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157777920, "lr": 8.024583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048459498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10363755375146866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.032916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048474994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10309009253978729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158105600, "lr": 8.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048490347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10910321772098541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158269440, "lr": 8.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048490347, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1785048490348, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1785048575196, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7816405892372131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1785048575198, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1785048575198, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1785048590332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1138567179441452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158433280, "lr": 8.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048605695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11724001169204712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048620943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11261314898729324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158760960, "lr": 8.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048636352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11293256282806396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158924800, "lr": 8.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048651808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10963518917560577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159088640, "lr": 8.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048666913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11847423017024994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048682154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1106325089931488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159416320, "lr": 8.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048696836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10949695110321045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159580160, "lr": 8.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048712605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11000008136034012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159744000, "lr": 8.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048728033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09887130558490753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048743512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10954880714416504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160071680, "lr": 8.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048759372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10944122076034546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160235520, "lr": 8.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048775058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10378298908472061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160399360, "lr": 8.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048790575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11050572991371155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048790576, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785048790576, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785048875642, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7818273305892944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785048875643, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785048875643, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785048891159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11257854849100113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160727040, "lr": 8.174583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048906894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12726038694381714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160890880, "lr": 8.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048922471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072341799736023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161054720, "lr": 8.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048937927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1118563562631607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048953298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11129098385572433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161382400, "lr": 8.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048968808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11030767112970352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161546240, "lr": 8.216250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048984310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1138739213347435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161710080, "lr": 8.224583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048999931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11282762140035629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049015265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10641701519489288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162037760, "lr": 8.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049030457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10777216404676437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162201600, "lr": 8.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049045888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12861569225788116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162365440, "lr": 8.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049061158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11217280477285385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049076908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11210032552480698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162693120, "lr": 8.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049092445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11509743332862854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162856960, "lr": 8.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049092446, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1785049092446, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1785049177635, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.782070517539978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1785049177636, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1785049177636, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1785049192818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09532564133405685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163020800, "lr": 8.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049208231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11091098189353943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049223512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12072814255952835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163348480, "lr": 8.307916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049239046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10703427344560623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163512320, "lr": 8.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049254578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10987993329763412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163676160, "lr": 8.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049269882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10709808766841888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049285045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11255788803100586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164003840, "lr": 8.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049300143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11286423355340958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164167680, "lr": 8.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049315486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10259176045656204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164331520, "lr": 8.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049329962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1168028935790062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049344799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10610456019639969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164659200, "lr": 8.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049359831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11090979725122452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164823040, "lr": 8.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049374953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09197868406772614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164986880, "lr": 8.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049389432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10302180051803589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049389433, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1785049389433, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1785049473134, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7815589904785156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1785049473135, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1785049473135, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1785049487816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09935257583856583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165314560, "lr": 8.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049502346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09571467339992523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165478400, "lr": 8.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049516745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.112938791513443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165642240, "lr": 8.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049531276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10764879733324051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049545778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10967062413692474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165969920, "lr": 8.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049560497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10460568964481354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166133760, "lr": 8.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049574907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10766790807247162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166297600, "lr": 8.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049589439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11208930611610413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049603886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09959933906793594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166625280, "lr": 8.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049618562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0945219099521637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166789120, "lr": 8.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049633034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09684135019779205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166952960, "lr": 8.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049647424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09660609066486359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049661688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10167555510997772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167280640, "lr": 8.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049676042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09981278330087662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167444480, "lr": 8.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049676043, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1785049676044, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1785049759313, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7820755243301392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1785049759314, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1785049759314, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1785049773851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10271836072206497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167608320, "lr": 8.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049788691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10165663063526154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049803607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10216140002012253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167936000, "lr": 8.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049818409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10669827461242676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168099840, "lr": 8.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049833095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10215805470943451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168263680, "lr": 8.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049848132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09823344647884369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049863245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10795672982931137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168591360, "lr": 8.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049877958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10562825202941895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168755200, "lr": 8.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049892746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10635284334421158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168919040, "lr": 8.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049907543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10542457550764084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049922823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10352624207735062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169246720, "lr": 8.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049937593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10638923943042755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169410560, "lr": 8.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049952104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1048680916428566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169574400, "lr": 8.624583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049967439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1114523708820343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049967487, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1785049967487, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1785050050509, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7820145487785339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1785050050510, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1785050050510, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1785050065257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10794612020254135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169902080, "lr": 8.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050080062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1115005761384964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170065920, "lr": 8.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050095068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10590701550245285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170229760, "lr": 8.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050110162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10607380419969559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050124987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09952590614557266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170557440, "lr": 8.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050140246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10085979849100113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170721280, "lr": 8.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050155494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09572261571884155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170885120, "lr": 8.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050170377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10566414892673492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050185178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09818176180124283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171212800, "lr": 8.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050200016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10047449171543121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171376640, "lr": 8.716249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050215241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10081709921360016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171540480, "lr": 8.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050229971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10272485762834549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050244727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09836620837450027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171868160, "lr": 8.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050259656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10441988706588745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172032000, "lr": 8.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050259694, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1785050259694, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1785050342680, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7820056080818176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1785050342681, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1785050342681, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1785050357436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09618905931711197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172195840, "lr": 8.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050372203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10699281096458435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050387002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10024462640285492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172523520, "lr": 8.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050401733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10203883051872253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172687360, "lr": 8.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050416395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0939696729183197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172851200, "lr": 8.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050431357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10683303326368332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050446383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10253854840993881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173178880, "lr": 8.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050461107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10180618613958359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173342720, "lr": 8.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050476024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10339346528053284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173506560, "lr": 8.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050490753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11204454302787781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050505769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09479550272226334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173834240, "lr": 8.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050520882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10987336188554764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173998080, "lr": 8.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050535882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11132241040468216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174161920, "lr": 8.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050550574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10127319395542145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050550611, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1785050550612, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1785050633296, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7829100489616394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1785050633297, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1785050633297, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1785050648097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0978170782327652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174489600, "lr": 8.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050663141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10734256356954575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174653440, "lr": 8.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050678242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10192154347896576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174817280, "lr": 8.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050692823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10366273671388626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050707942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10007179528474808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175144960, "lr": 8.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050722471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792902112007141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175308800, "lr": 8.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050736955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09993590414524078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175472640, "lr": 8.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050751716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09769292175769806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050766519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10847590118646622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175800320, "lr": 8.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050781311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10759056359529495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175964160, "lr": 8.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050795907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09817501157522202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176128000, "lr": 8.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050810617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0936211571097374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050825141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09392853826284409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176455680, "lr": 8.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050839611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10551996529102325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176619520, "lr": 8.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050839611, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1785050839612, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1785050924498, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7831709980964661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1785050924499, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1785050924499, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1785050939362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10242902487516403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176783360, "lr": 8.991249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050954109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11316367983818054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050968872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10014183074235916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177111040, "lr": 9.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050983496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10500764101743698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177274880, "lr": 9.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050997962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10982763022184372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177438720, "lr": 9.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051012549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09953304380178452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.032916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051027119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10913630574941635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177766400, "lr": 9.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051041731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10028152912855148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177930240, "lr": 9.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051056529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10537119954824448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178094080, "lr": 9.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051071373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1001831442117691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051086152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0965910330414772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178421760, "lr": 9.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051101276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11083891987800598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178585600, "lr": 9.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051116325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11923499405384064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178749440, "lr": 9.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051131322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10790763050317764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051131322, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1785051131323, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1785051213746, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7832744717597961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1785051213747, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1785051213747, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1785051228940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11394920945167542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179077120, "lr": 9.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051243809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1013631671667099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179240960, "lr": 9.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051259028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09946119040250778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179404800, "lr": 9.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051274111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10063667595386505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.132916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051289236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10460114479064941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179732480, "lr": 9.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051304319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09854443371295929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179896320, "lr": 9.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051319134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09986018389463425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180060160, "lr": 9.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051334291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10556188225746155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051349056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10465335100889206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180387840, "lr": 9.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051363648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09278932213783264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180551680, "lr": 9.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051379249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10312089323997498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180715520, "lr": 9.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051394100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10029240697622299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051409027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09306734800338745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181043200, "lr": 9.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051424081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09981556236743927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181207040, "lr": 9.216249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051424082, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1785051424083, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1785051506114, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7831103801727295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1785051506115, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1785051506115, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1785051521289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11066681891679764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181370880, "lr": 9.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051536465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11362752318382263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051551546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10047091543674469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181698560, "lr": 9.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051566456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10584630072116852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181862400, "lr": 9.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051581068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10185506939888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182026240, "lr": 9.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051595909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10067830979824066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051610719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10648300498723984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182353920, "lr": 9.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051625595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10805953294038773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182517760, "lr": 9.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051640842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.097151018679142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182681600, "lr": 9.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051656187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09572865813970566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051671659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10909929126501083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183009280, "lr": 9.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051686876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09674452245235443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183173120, "lr": 9.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051702312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10145284235477448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183336960, "lr": 9.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051717192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.101251982152462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051717193, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785051717193, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785051800497, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7836289405822754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785051800499, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785051800499, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785051815441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10186857730150223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183664640, "lr": 9.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051830336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10781921446323395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183828480, "lr": 9.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051845187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11141395568847656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183992320, "lr": 9.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051860613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09264032542705536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051875770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10155698657035828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184320000, "lr": 9.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051890908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1066647469997406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184483840, "lr": 9.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051906104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11615733802318573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184647680, "lr": 9.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051921231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10103510320186615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051936291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10564160346984863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184975360, "lr": 9.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051951515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10753913968801498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185139200, "lr": 9.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051966640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0965888500213623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185303040, "lr": 9.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051981490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10715697705745697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051996678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09776138514280319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185630720, "lr": 9.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052011447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10201124101877213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185794560, "lr": 9.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052011448, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1785052011448, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1785052095805, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7839177250862122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1785052095806, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1785052095806, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1785052111215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10979041457176208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185958400, "lr": 9.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052125822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10366813093423843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052140180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09203482419252396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186286080, "lr": 9.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052155720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10046692192554474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186449920, "lr": 9.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052170654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10791515558958054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186613760, "lr": 9.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052185504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11074129492044449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052200738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09597989916801453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186941440, "lr": 9.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052215825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1040688082575798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187105280, "lr": 9.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052230738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10465965420007706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187269120, "lr": 9.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052245605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09697172045707703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052260391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10357512533664703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187596800, "lr": 9.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052275237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10370190441608429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187760640, "lr": 9.549583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052290058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10998889803886414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187924480, "lr": 9.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052305084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11342722177505493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052305085, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1785052305085, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1785052390674, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7843410968780518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1785052390675, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1785052390675, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1785052405457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10822075605392456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188252160, "lr": 9.574583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052420641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10414662212133408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188416000, "lr": 9.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052435565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10182289034128189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188579840, "lr": 9.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052450502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10768717527389526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052465271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10173480212688446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188907520, "lr": 9.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052480526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0970427468419075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189071360, "lr": 9.616249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052495531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10188174247741699, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189235200, "lr": 9.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052510385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11143983155488968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052525157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10057705640792847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189562880, "lr": 9.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052540275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09447313845157623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189726720, "lr": 9.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052555394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10397668182849884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189890560, "lr": 9.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052570712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11129438132047653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.666249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052586303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10152692347764969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190218240, "lr": 9.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052601548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1167202740907669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190382080, "lr": 9.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052601549, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1785052601549, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1785052686007, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7841959595680237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1785052686008, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1785052686008, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1785052701360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09948572516441345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190545920, "lr": 9.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052716222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09656713157892227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052731077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1056850329041481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190873600, "lr": 9.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052746224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11243949830532074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191037440, "lr": 9.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052761420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11627816408872604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191201280, "lr": 9.724583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052776549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09584492444992065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052791474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1060030460357666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191528960, "lr": 9.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052806262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10087740421295166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191692800, "lr": 9.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052821404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10028035938739777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191856640, "lr": 9.757916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052836293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10453366488218307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052850743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10677719116210938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192184320, "lr": 9.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052866307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10894504934549332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192348160, "lr": 9.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052881390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09847743064165115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192512000, "lr": 9.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052896552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11287738382816315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.799583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052896553, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1785052896553, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1785052982269, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.785286545753479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1785052982270, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1785052982270, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1785052997592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11150360107421875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192839680, "lr": 9.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053013032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11072318255901337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193003520, "lr": 9.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053028198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10576160252094269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193167360, "lr": 9.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053043455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11781877279281616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053058447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11646718531847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193495040, "lr": 9.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053073565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10036887973546982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193658880, "lr": 9.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053088783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11812970787286758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193822720, "lr": 9.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053103628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11374391615390778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.866249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053118549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1059577688574791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194150400, "lr": 9.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053133674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1068878248333931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194314240, "lr": 9.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053148992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10667425394058228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194478080, "lr": 9.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053164137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10549914836883545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053179574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1066567525267601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194805760, "lr": 9.907916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053194467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1157405897974968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194969600, "lr": 9.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053194468, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1785053194468, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1785053278906, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7840380668640137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1785053278907, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1785053278907, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1785053294309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1116146594285965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195133440, "lr": 9.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053309385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10269588977098465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053324627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10204021632671356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195461120, "lr": 9.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053339982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11720986664295197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195624960, "lr": 9.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053355356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10801978409290314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195788800, "lr": 9.957916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053370689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10469986498355865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053386255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10335058718919754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196116480, "lr": 9.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053401597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11336342245340347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196280320, "lr": 9.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053417219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11742918193340302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196444160, "lr": 9.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053432537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10330971330404282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.999583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053447913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11595282703638077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196771840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053463202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11159274727106094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196935680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053477980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10675453394651413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197099520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053493304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11615696549415588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053493304, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1785053493305, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1785053576819, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7847903370857239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1785053576820, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1785053576820, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1785053592015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1008848324418068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197427200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053607562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10997867584228516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197591040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053622621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1136992946267128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197754880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053637662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09916515648365021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053653173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09799695014953613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198082560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053668430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10657346993684769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198246400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053683665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0990438386797905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198410240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053698927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772090405225754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053714278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09999281913042068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198737920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053729388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10629232227802277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198901760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053744609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11432242393493652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199065600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053759936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11024311184883118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053774829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11747752130031586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199393280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053789785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11195269227027893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199557120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053789786, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1785053789786, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1785053873162, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7853400707244873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1785053873163, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1785053873163, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1785053888178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11119309067726135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199720960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053903141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1046438217163086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053918063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11011222004890442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200048640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053933009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11573071777820587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200212480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053948011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11202500760555267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200376320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053963367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11579592525959015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053978890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09949339181184769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200704000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053994154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10861620306968689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200867840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054009498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10670936852693558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201031680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054024928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10819900035858154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054040447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11709947139024734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201359360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054055897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10917288064956665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201523200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054071324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10209175944328308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201687040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054086691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10989396274089813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054086692, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1785054086692, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1785054169951, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7846912145614624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1785054169953, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1785054169953, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1785054184753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10505633056163788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202014720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054199917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10949629545211792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202178560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054214961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11779151111841202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202342400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054230481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10658399015665054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054245359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1048765778541565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202670080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054260579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1007520854473114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202833920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054276382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10627532750368118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202997760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054291494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10958320647478104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054306647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11279258131980896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203325440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054321700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0930999293923378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203489280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054336982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11567550152540207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203653120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054352022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09919948875904083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054366732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09896137565374374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203980800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054381556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10720133781433105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204144640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054381557, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1785054381557, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1785054466224, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7855264544487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1785054466224, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1785054466225, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1785054481352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10185138881206512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204308480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054496578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09281433373689651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054511702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0966859981417656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204636160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054526821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080508753657341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204800000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054541848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10750064253807068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204963840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054556966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11125741899013519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054571732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09483645111322403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205291520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054586805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10528049618005753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205455360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054601744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10431010276079178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205619200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054616638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10965075343847275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054631456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09798096865415573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205946880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054646809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09872226417064667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206110720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054661917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11091727018356323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206274560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054676634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09827237576246262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054676635, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1785054676635, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1785054760329, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7856441140174866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1785054760330, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1785054760330, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1785054775334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10573305189609528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206602240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054790308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09532858431339264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206766080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054805456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10267672687768936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206929920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054820904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0952346920967102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054836354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10213536769151688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207257600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054851516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10868731886148453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207421440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054866737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10640276223421097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207585280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054881603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09452205896377563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054896686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11383622139692307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207912960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054911955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10362818092107773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208076800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054926717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10304053127765656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208240640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054942131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10228630900382996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054956738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10236365348100662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208568320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054971481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10083948075771332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208732160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054971481, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1785054971482, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1785055055845, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.785584568977356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1785055055846, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1785055055846, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1785055070796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10055259615182877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208896000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055085636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09849420189857483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055100607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1020115464925766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209223680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055115300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10170906782150269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209387520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055130623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09886837750673294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209551360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055145559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10089602321386337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055160188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0959906056523323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209879040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055174896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09891350567340851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210042880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055189801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10259019583463669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210206720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055204680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10689833760261536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055219619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10378900170326233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210534400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055234441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10788504034280777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210698240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055249350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10111036151647568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210862080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055264104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09536255896091461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055264104, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1785055264105, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1785055346084, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7859992384910583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1785055346085, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1785055346085, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1785055361142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0936860516667366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211189760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055375782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0985625684261322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211353600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055390395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0959162712097168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211517440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055404814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09927826374769211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055419410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10288363695144653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211845120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055434561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09515611827373505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212008960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055449437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.101999431848526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212172800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055464259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09767604619264603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055478919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09818066656589508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212500480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055493776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10383246839046478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212664320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055508534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09236076474189758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212828160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055523547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10734912008047104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055538727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10456310957670212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213155840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055553885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10707563161849976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213319680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055553886, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1785055553886, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1785055637105, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7862826585769653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1785055637105, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1785055637106, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1785055651801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10443902015686035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213483520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055666555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10071565210819244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055681316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10447771102190018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213811200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055696563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1028033047914505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213975040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055711071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09573693573474884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214138880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055725716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1053655594587326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055740967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09461507201194763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214466560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055755887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11348536610603333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214630400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055770781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09809697419404984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214794240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055786290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08882388472557068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055801672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1007026880979538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215121920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055816873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10377722978591919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215285760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055831834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10320918262004852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215449600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055847043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10870666056871414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055847044, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1785055847045, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1785055931294, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7856706976890564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1785055931295, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1785055931295, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1785055946251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09861205518245697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215777280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055960888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10547354072332382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215941120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055975806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09718714654445648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216104960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055990939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0994492769241333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056006118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0996454730629921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216432640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056021091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09328864514827728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216596480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056035967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10432837903499603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216760320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056051014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10166945308446884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056065696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09930586814880371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217088000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056080754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10863876342773438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217251840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056095242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09705305099487305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217415680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056110183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09943512827157974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217579520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056125021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09911569207906723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217743360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056139725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10513070225715637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217907200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056139725, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1785056139726, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1785056222838, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7859638333320618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1785056222839, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1785056222840, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1785056237809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10115621238946915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218071040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056252605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09432638436555862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218234880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056267648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09805520623922348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218398720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056282386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1076393574476242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218562560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056297409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10269225388765335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218726400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056312292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0874122828245163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218890240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056327217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10598684847354889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219054080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056342247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09258448332548141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219217920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056357315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10371510684490204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219381760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056372188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0945233553647995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219545600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056387163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09337209910154343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219709440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056401748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10243560373783112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219873280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056416619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11144094914197922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220037120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056431390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09837399423122406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220200960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056431391, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1785056431391, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1785056515524, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7869479060173035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1785056515525, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1785056515526, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1785056529971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09523051232099533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220364800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056545195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09433571249246597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220528640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056559718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10040125995874405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220692480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056574390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10095041245222092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220856320, "lr": 1e-06}} diff --git a/recommendation_v4/rcp_logs/gbs_8192/seed490920051.log b/recommendation_v4/rcp_logs/gbs_8192/seed490920051.log new file mode 100644 index 000000000..87f14c2ee --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_8192/seed490920051.log @@ -0,0 +1,681 @@ +:::MLLOG {"namespace": "", "time_ms": 1784939253774, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784939253774, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784939275317, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784939275319, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784939275319, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784939275319, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784939275319, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784939275319, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 8192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784939275319, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784939275319, "event_type": "POINT_IN_TIME", "key": "seed", "value": 490920051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784939275319, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784939275319, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784939275319, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784939275319, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784939275372, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784939275373, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784940155789, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784940155790, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784940156195, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784940331615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13859927654266357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840, "lr": 7.916666666666666e-10}} +:::MLLOG {"namespace": "", "time_ms": 1784940343626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13922929763793945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.6249999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940356034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384446769952774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 491520, "lr": 2.458333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940368533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13919734954833984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940381248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388709843158722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 819200, "lr": 4.125e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940393804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13887163996696472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940406531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1390223354101181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1146880, "lr": 5.7916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940419279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387975513935089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.625e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940432166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13877074420452118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1474560, "lr": 7.458333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940444910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13875946402549744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.291666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940458059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13866257667541504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1802240, "lr": 9.124999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940471063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13863620162010193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940484055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387030929327011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2129920, "lr": 1.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940496968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381441056728363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940496968, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784940496969, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784940584073, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5020152926445007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784940584074, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784940584074, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784940596793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13923002779483795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2457600, "lr": 1.2458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940609706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13884514570236206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940623114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13862669467926025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2785280, "lr": 1.4124999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940635915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13878870010375977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4958333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940648964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13883745670318604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3112960, "lr": 1.5791666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940662721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13890577852725983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940675730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13871516287326813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3440640, "lr": 1.7458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940688970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387302130460739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940702532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13859783113002777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3768320, "lr": 1.9124999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940715797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13873599469661713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9958333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940729624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13869552314281464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4096000, "lr": 2.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940743168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385280191898346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.1624999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940756643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860180974006653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4423680, "lr": 2.2458333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940770691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13861210644245148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3291666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940770692, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784940770692, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784940855955, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.502264678478241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784940855956, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784940855957, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784940869292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382685750722885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4751360, "lr": 2.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940882771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384611427783966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940896256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13810458779335022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5079040, "lr": 2.5791666666666668e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940910044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13769923150539398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940923613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13849398493766785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5406720, "lr": 2.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940936937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13836857676506042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940950383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378992646932602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5734400, "lr": 2.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940964382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13773760199546814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940977986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13755056262016296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6062080, "lr": 3.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940991327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13797055184841156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941004841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13780245184898376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6389760, "lr": 3.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941018177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13800355792045593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941032306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379319578409195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6717440, "lr": 3.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941045726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379643976688385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941045727, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784941045727, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784941131207, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5026192665100098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784941131208, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784941131208, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784941144455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13731510937213898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7045120, "lr": 3.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941158113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13769763708114624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941171641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1372629851102829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7372800, "lr": 3.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941186028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376173049211502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.829166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941199846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376572996377945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7700480, "lr": 3.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941213715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379343718290329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941227463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13721320033073425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8028160, "lr": 4.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941240979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13766321539878845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941255126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1372668594121933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8355840, "lr": 4.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941269085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13854695856571198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3291666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941283014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1369173675775528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8683520, "lr": 4.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941296835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13803532719612122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.4958333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941310689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13581210374832153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9011200, "lr": 4.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941324866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371694803237915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.6624999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941324867, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784941324867, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784941409149, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5030995011329651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784941409150, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784941409150, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784941422561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13644614815711975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9338880, "lr": 4.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941436228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13602764904499054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941449495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376180648803711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9666560, "lr": 4.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941463298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13505932688713074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941477166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13682569563388824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9994240, "lr": 5.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941490705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13417525589466095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1624999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941504665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13585898280143738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10321920, "lr": 5.2458333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941518259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13557440042495728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941531937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377052217721939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10649600, "lr": 5.4124999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941545462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13552631437778473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941559061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14621542394161224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10977280, "lr": 5.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941573098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13581274449825287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941586723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13431517779827118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11304960, "lr": 5.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941600390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377854198217392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941600391, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784941600391, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784941683852, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.503790557384491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784941683853, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784941683853, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784941697303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355687826871872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11632640, "lr": 5.9124999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941710806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.139437735080719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941724397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13617487251758575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11960320, "lr": 6.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941737961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13373951613903046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941751726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13519012928009033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12288000, "lr": 6.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941765609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13884687423706055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.329166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941779288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13370142877101898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12615680, "lr": 6.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941793047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13326144218444824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941806926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13363991677761078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12943360, "lr": 6.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941820360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13196435570716858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941834207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13158074021339417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13271040, "lr": 6.745833333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941847778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13484467566013336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941861607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13224339485168457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13598720, "lr": 6.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941875752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385957896709442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941875753, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784941875753, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784941958477, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5046721696853638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784941958478, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784941958478, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784941972312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13387654721736908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13926400, "lr": 7.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941986088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13364115357398987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941999905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358756721019745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14254080, "lr": 7.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942013419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13515157997608185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942026955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13771654665470123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14581760, "lr": 7.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942040525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12823735177516937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942053757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13535527884960175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14909440, "lr": 7.579166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942067324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12978526949882507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942080996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13362035155296326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15237120, "lr": 7.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942095044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13295762240886688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.829166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942108751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13359582424163818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15564800, "lr": 7.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942122815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13445577025413513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.995833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942136897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13542531430721283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15892480, "lr": 8.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942150632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13351979851722717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942150632, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784942150632, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784942233858, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5058847665786743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784942233859, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784942233859, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784942247865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327456682920456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16220160, "lr": 8.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942261662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13378049433231354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942275637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356118768453598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16547840, "lr": 8.412500000000001e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942289961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356043666601181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942304076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13943403959274292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16875520, "lr": 8.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942317978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13336984813213348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.662499999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942332505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13529184460639954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17203200, "lr": 8.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942346266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13330376148223877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942360182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332497000694275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17530880, "lr": 8.912499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942374187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13193926215171814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942387903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13565626740455627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17858560, "lr": 9.079166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942402057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335052251815796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.162499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942415743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13321876525878906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18186240, "lr": 9.245833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942429363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13056045770645142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942429363, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784942429364, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784942514834, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5072068572044373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784942514835, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784942514835, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784942529390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376999467611313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18513920, "lr": 9.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942543421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347711682319641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.495833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942557632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13550856709480286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18841600, "lr": 9.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942572020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13138516247272491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.662499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942586115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13366322219371796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19169280, "lr": 9.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942600452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341938078403473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942614854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.136962890625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19496960, "lr": 9.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942629188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1339327096939087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942643424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13275788724422455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19824640, "lr": 1.0079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942657405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12963812053203583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0162499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942670917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12769916653633118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20152320, "lr": 1.0245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942684598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13295896351337433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942698397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12955772876739502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20480000, "lr": 1.0412499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942711937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12491875886917114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942711938, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784942711938, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784942795121, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5086705088615417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784942795122, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784942795122, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784942809231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13119040429592133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20807680, "lr": 1.0579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942823191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13582536578178406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942837210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13295860588550568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21135360, "lr": 1.0745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942851443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13037991523742676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942865239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.128158837556839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21463040, "lr": 1.0912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942879262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1267557442188263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942894058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12895750999450684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21790720, "lr": 1.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942908066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322495937347412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942922194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13557016849517822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22118400, "lr": 1.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942936363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1249459981918335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942950214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1286659687757492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22446080, "lr": 1.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942964116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13445454835891724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942978264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308278888463974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22773760, "lr": 1.1579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942992826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12906292080879211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942992826, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784942992827, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784943074925, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.510524332523346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784943074925, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784943074926, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784943089320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304185688495636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23101440, "lr": 1.1745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943103379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1291043609380722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943116944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13314157724380493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23429120, "lr": 1.1912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943131275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12368892878293991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943145451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12959475815296173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23756800, "lr": 1.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943159193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1255604326725006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943173178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13002286851406097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24084480, "lr": 1.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943187108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12163224816322327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943200663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357278823852539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24412160, "lr": 1.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943214764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13734115660190582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943228694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12365807592868805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24739840, "lr": 1.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943242920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12330617010593414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943257122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12560613453388214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25067520, "lr": 1.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943271242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12880557775497437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943271242, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784943271243, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784943354047, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5125288367271423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784943354048, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784943354048, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784943367845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13654133677482605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25395200, "lr": 1.2912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943382062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13656802475452423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943395685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13555662333965302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25722880, "lr": 1.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943409250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13999120891094208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943422974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13668790459632874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26050560, "lr": 1.3245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943436760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13823586702346802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943450505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12724298238754272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26378240, "lr": 1.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943464254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12601734697818756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943478036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1419983059167862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26705920, "lr": 1.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943492140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1410660445690155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943506087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12724047899246216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27033600, "lr": 1.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943520010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12857811152935028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943533737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13285255432128906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27361280, "lr": 1.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943546928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13149696588516235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943546928, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784943546928, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784943629434, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5145210027694702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784943629434, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784943629435, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784943644363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14137986302375793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27688960, "lr": 1.4079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943659011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14246876537799835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943673146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13514699041843414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28016640, "lr": 1.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943687470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13963519036769867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943702105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1447536051273346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28344320, "lr": 1.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943716149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14091549813747406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943730528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12596113979816437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28672000, "lr": 1.4579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943744866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375715434551239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943759381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317177712917328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28999680, "lr": 1.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943773865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13600246608257294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943788335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13771162927150726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29327360, "lr": 1.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943802530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13302433490753174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4995833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943817057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13440704345703125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29655040, "lr": 1.5079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943831383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13630349934101105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943831384, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784943831384, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784943913941, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5166200995445251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784943913942, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784943913942, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784943927753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13841576874256134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29982720, "lr": 1.5245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943941848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13820961117744446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943956155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13178957998752594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30310400, "lr": 1.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943970329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329849511384964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943984383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336788684129715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30638080, "lr": 1.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943998730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13968269526958466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944012964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323932260274887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30965760, "lr": 1.5745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944027526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1411522924900055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944041375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1407037079334259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31293440, "lr": 1.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944055848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13498209416866302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944069840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12766315042972565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31621120, "lr": 1.6079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944083622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13000017404556274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944097553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1339883804321289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31948800, "lr": 1.6245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944111663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.127712219953537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944111663, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784944111664, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784944194263, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5194805860519409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784944194264, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784944194264, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784944208239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13195665180683136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32276480, "lr": 1.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944222249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366698145866394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944236321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13327474892139435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32604160, "lr": 1.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944250328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13204774260520935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944264516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13399213552474976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32931840, "lr": 1.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944278591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13342995941638947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944293157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13384504616260529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33259520, "lr": 1.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944307357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13256260752677917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944321286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13976043462753296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33587200, "lr": 1.7079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944335776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13172395527362823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944349710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12870317697525024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33914880, "lr": 1.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944363321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13107652962207794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944376984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13222266733646393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34242560, "lr": 1.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944390782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13580107688903809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7495833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944390782, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784944390783, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784944472922, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5234965682029724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784944472922, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784944472923, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784944486926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13353800773620605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34570240, "lr": 1.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944500649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1228894367814064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944514306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13029327988624573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34897920, "lr": 1.7745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944528104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13008470833301544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944541790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350998729467392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35225600, "lr": 1.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944555962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13509775698184967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944569660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13555876910686493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35553280, "lr": 1.8079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944583884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12396443635225296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8162500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944597836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13099916279315948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35880960, "lr": 1.8245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944611454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1309293955564499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944626100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13011713325977325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36208640, "lr": 1.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944640129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329226791858673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944654284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13063709437847137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36536320, "lr": 1.8579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944668307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12995073199272156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944668308, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784944668310, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784944750968, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5288981795310974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784944750969, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784944750969, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784944765101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13757109642028809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36864000, "lr": 1.8745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944779316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13216263055801392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944793470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13321591913700104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37191680, "lr": 1.8912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944807637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1302211731672287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944821856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12677380442619324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37519360, "lr": 1.9079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944835941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318737119436264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944850424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356024146080017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37847040, "lr": 1.9245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944864778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13257235288619995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944879029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311214715242386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38174720, "lr": 1.9412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944893369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13867045938968658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944907595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335839331150055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38502400, "lr": 1.9579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944921299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13723935186862946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944935893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13156352937221527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38830080, "lr": 1.9745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944949822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13309980928897858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944949823, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784944949823, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784945033013, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5361767411231995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784945033014, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784945033014, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784945047042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1309022307395935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39157760, "lr": 1.9912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945060933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13489191234111786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945074844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376710832118988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39485440, "lr": 2.0079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945088777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13286380469799042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945102695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1269020438194275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39813120, "lr": 2.0245833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945116669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12915928661823273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0329166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945130576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12584395706653595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40140800, "lr": 2.0412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945144448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12361082434654236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945158599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13579653203487396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40468480, "lr": 2.0579166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945172835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13708479702472687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945186976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13436158001422882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40796160, "lr": 2.0745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945201777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12678682804107666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0829166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945215997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.130541130900383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41123840, "lr": 2.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945229679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12341818958520889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945229680, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784945229680, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784945312601, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5459666848182678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784945312602, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784945312602, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784945327112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13205109536647797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41451520, "lr": 2.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945341515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13232262432575226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945355967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333199441432953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41779200, "lr": 2.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945370310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13743942975997925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945384618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13594581186771393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42106880, "lr": 2.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945398945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13129186630249023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945413250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12691408395767212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42434560, "lr": 2.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945427573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13472893834114075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945441724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12408307939767838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42762240, "lr": 2.1745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945456310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12999001145362854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945470733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13204436004161835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43089920, "lr": 2.1912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945485352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13342787325382233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945499996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1397857964038849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43417600, "lr": 2.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945514460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13605241477489471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945514461, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784945514461, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784945595282, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.559440016746521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784945595283, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784945595283, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784945610251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360355019569397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43745280, "lr": 2.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945624837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13081102073192596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945639014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13203072547912598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44072960, "lr": 2.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945653580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13836769759655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945668178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12956948578357697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44400640, "lr": 2.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945682970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13782525062561035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945697052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1342705488204956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44728320, "lr": 2.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945711429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367308497428894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945725627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13440099358558655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45056000, "lr": 2.2912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945739981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13040991127490997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945754293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1397962123155594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45383680, "lr": 2.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945768522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13402299582958221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945782785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12824536859989166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45711360, "lr": 2.3245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945797242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13297873735427856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945797242, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784945797243, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784945878053, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.573367178440094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784945878054, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784945878055, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784945892637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.141582652926445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46039040, "lr": 2.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945906911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13215243816375732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945921747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331217736005783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46366720, "lr": 2.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945936077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13019666075706482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945950900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13500477373600006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46694400, "lr": 2.3745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945965799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13265404105186462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945980551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333765983581543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47022080, "lr": 2.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945995404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134675070643425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946010062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12997440993785858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47349760, "lr": 2.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946024928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1314457356929779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946039703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324947625398636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47677440, "lr": 2.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946054534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364191174507141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946069048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135491743683815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48005120, "lr": 2.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946083917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13666030764579773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.4495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946083917, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784946083918, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784946166819, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5862302184104919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784946166820, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784946166821, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784946181436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1307370811700821, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48332800, "lr": 2.4579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946196332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13439787924289703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946210970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13354723155498505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48660480, "lr": 2.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946226139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13121718168258667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946240798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13617612421512604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48988160, "lr": 2.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946255250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1284111589193344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946270212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327780932188034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49315840, "lr": 2.507916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946284359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12237422913312912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.5162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946298590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12946096062660217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49643520, "lr": 2.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946312812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13059017062187195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946327101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12992849946022034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49971200, "lr": 2.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946341114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.129959836602211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.5495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946355193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12777674198150635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50298880, "lr": 2.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946369226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1296989470720291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.5662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946369227, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784946369227, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784946452218, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5953759551048279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784946452219, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784946452219, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784946465929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12088421732187271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50626560, "lr": 2.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946479968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12525475025177002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946493915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12787780165672302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50954240, "lr": 2.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946508002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321086585521698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946522476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308131068944931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51281920, "lr": 2.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946536673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1203966736793518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946551268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1287183314561844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51609600, "lr": 2.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946565808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13077057898044586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946580225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13555549085140228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51937280, "lr": 2.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946594135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12672576308250427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.6495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946609057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317158341407776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52264960, "lr": 2.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946623544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11659713834524155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.6662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946638050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11965584009885788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52592640, "lr": 2.6745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946652506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12366268783807755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946652506, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784946652507, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784946734575, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6048654317855835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784946734576, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784946734576, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784946748952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11990294605493546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52920320, "lr": 2.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946762957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12603460252285004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.6995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946777231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1268959641456604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53248000, "lr": 2.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946791459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298450231552124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946805606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11968789994716644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53575680, "lr": 2.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946819921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13323359191417694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946834010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1276135891675949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53903360, "lr": 2.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946848086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11976123601198196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.7495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946862694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12544521689414978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54231040, "lr": 2.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946877130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12519942224025726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946891320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11692162603139877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54558720, "lr": 2.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946905976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13047480583190918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946920286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12569962441921234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54886400, "lr": 2.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946934580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12795472145080566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946934581, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784946934581, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784947018330, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6133107542991638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784947018331, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784947018331, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784947032101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12629376351833344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55214080, "lr": 2.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947047063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11721578985452652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947061378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12087432295084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55541760, "lr": 2.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947075799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12000866234302521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947090084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12277712672948837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55869440, "lr": 2.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947104568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12503793835639954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947118956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11431129276752472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56197120, "lr": 2.8579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947133636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12183865159749985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947148282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12372242659330368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56524800, "lr": 2.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947162928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12255614250898361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947177351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11453165858983994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56852480, "lr": 2.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947191916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12197306752204895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947206633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10804207623004913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57180160, "lr": 2.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947221166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12296830862760544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947221166, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784947221167, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784947304641, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6203591227531433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784947304642, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784947304642, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784947319444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11542637646198273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57507840, "lr": 2.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947334189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11296132206916809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947348673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1270565390586853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57835520, "lr": 2.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947363553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12081103771924973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947378138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12262531369924545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58163200, "lr": 2.9579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947392602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11516997963190079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947406584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11819484829902649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58490880, "lr": 2.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947421458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12068776041269302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947435499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11924299597740173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58818560, "lr": 2.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947449615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11149261146783829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947463791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11965543031692505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59146240, "lr": 3.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947477851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12075656652450562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.0162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947491873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12443070113658905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59473920, "lr": 3.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947506000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11601381003856659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947506001, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784947506001, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784947591600, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6333673000335693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784947591601, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784947591601, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784947605663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11816348135471344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59801600, "lr": 3.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947619591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11012353003025055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947633354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10986814647912979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60129280, "lr": 3.0579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947647219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11282025277614594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.0662499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947661396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10859400033950806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60456960, "lr": 3.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947675647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12146082520484924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947689942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11573119461536407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60784640, "lr": 3.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947704550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11992179602384567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947718744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10133904218673706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61112320, "lr": 3.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947733640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1199394166469574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.1162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947748183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11260378360748291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61440000, "lr": 3.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947762739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11269273608922958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947777098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10888693481683731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61767680, "lr": 3.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947791811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11569935083389282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947791812, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784947791812, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784947877458, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6575143933296204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784947877459, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784947877459, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784947891969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12043797969818115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62095360, "lr": 3.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947906793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1190786361694336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.1662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947921597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12286657094955444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62423040, "lr": 3.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947936240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10542796552181244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947950728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11407865583896637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62750720, "lr": 3.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947965032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1177515909075737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947979553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12003759294748306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63078400, "lr": 3.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947994258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1195526048541069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948008645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10830331593751907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63406080, "lr": 3.2245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948023197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1179245188832283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948037625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11510533839464188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63733760, "lr": 3.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948051902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11674662679433823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948066177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11328354477882385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64061440, "lr": 3.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948081048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12546315789222717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948081048, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784948081049, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784948164545, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6901081204414368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784948164546, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784948164546, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784948179165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.124729685485363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64389120, "lr": 3.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948193864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11758358031511307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948207989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11703549325466156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64716800, "lr": 3.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948222902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12277992814779282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948237353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10939943045377731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65044480, "lr": 3.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948251728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11176435649394989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948265642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11312877386808395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65372160, "lr": 3.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948280374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10894815623760223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948295003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11098593473434448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65699840, "lr": 3.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948310119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11595210433006287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948324903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12410631030797958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66027520, "lr": 3.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948339383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11527349054813385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948354178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12734445929527283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66355200, "lr": 3.3745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948368672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11270029097795486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948368672, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784948368673, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784948452622, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7417874932289124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784948452623, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784948452624, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784948467361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12607112526893616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66682880, "lr": 3.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948481789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11461179703474045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.3995833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948496231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1164291501045227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67010560, "lr": 3.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948511002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11352358758449554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948525376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11882325261831284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67338240, "lr": 3.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948540060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11846956610679626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948555007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12106776982545853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67665920, "lr": 3.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948569989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12064328044652939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948584840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11477416753768921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67993600, "lr": 3.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948600136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11284227669239044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948614974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11576158553361893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68321280, "lr": 3.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948629584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11597450822591782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948644112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12481151521205902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68648960, "lr": 3.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948659365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11584531515836716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948659366, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784948659366, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784948743632, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7530791759490967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784948743632, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784948743633, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784948758375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12181363254785538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68976640, "lr": 3.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948773241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11955328285694122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948787833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1133410632610321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69304320, "lr": 3.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948802453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11451572179794312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948816972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10882709175348282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69632000, "lr": 3.5412500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948831643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11549261957406998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948846353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11755172908306122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69959680, "lr": 3.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948860706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11164619028568268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948875353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10918166488409042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70287360, "lr": 3.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948889686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10851111263036728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948904615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1229882463812828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70615040, "lr": 3.5912500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948919509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1070166677236557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948934324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12040485441684723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70942720, "lr": 3.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948949563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11985700577497482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948949564, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784948949564, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784949033561, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7578440308570862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784949033562, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784949033562, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784949048565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11906713992357254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71270400, "lr": 3.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949063115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11820442229509354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949077778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11537687480449677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71598080, "lr": 3.6412499999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949092840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1158917099237442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949107400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10345382988452911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71925760, "lr": 3.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949122011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10392404347658157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949136641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11921677738428116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72253440, "lr": 3.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949151260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11503717303276062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949165862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10448352992534637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72581120, "lr": 3.6912499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949180393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10286813229322433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949194738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10653586685657501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72908800, "lr": 3.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949209285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11148856580257416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949223835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11125162243843079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73236480, "lr": 3.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949238284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11581775546073914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949238328, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784949238329, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784949320710, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7610298991203308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784949320711, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784949320711, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784949335278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.104971744120121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73564160, "lr": 3.7412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949350060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10709669440984726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949364826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11528096348047256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73891840, "lr": 3.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949379318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11000335961580276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949393985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10734222829341888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74219520, "lr": 3.7745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949408957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11500781774520874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949423629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11346275359392166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74547200, "lr": 3.7912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949438105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11294480413198471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949452883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11656227707862854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74874880, "lr": 3.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949467374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11087504029273987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949482053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10076527297496796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75202560, "lr": 3.8245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949496779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11396826803684235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949511180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10678533464670181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75530240, "lr": 3.8412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949525705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10587577521800995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949525706, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784949525706, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784949608626, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7637522220611572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784949608627, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784949608627, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784949623124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1061321422457695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75857920, "lr": 3.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949637904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11322446167469025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949652494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0960247740149498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76185600, "lr": 3.8745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949667006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10410737246274948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949681516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11261530965566635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76513280, "lr": 3.8912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949695817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10462532937526703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949710379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08855941891670227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76840960, "lr": 3.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949724761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10826919227838516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949739373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10135950148105621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77168640, "lr": 3.9245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949754272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11724327504634857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949769115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10705932229757309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77496320, "lr": 3.9412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949783807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09969587624073029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949799024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0995212197303772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77824000, "lr": 3.9579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949814165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10702421516180038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949814165, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784949814166, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784949897996, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7646704316139221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784949897997, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784949897997, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784949912908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11594580858945847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78151680, "lr": 3.9745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949927865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11218393594026566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949942117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10454435646533966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78479360, "lr": 3.9912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949957287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10636983066797256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949971418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09930862486362457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78807040, "lr": 4.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949985953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10233715921640396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950000309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10647007077932358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79134720, "lr": 4.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950014851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10825293511152267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950029121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09694813936948776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79462400, "lr": 4.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950043733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10844819247722626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950058093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1044066995382309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79790080, "lr": 4.0579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950072333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10710715502500534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950086854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10849037021398544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80117760, "lr": 4.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950101430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11279083788394928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950101431, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784950101431, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} diff --git a/recommendation_v4/rcp_logs/gbs_8192/seed546672350.log b/recommendation_v4/rcp_logs/gbs_8192/seed546672350.log new file mode 100644 index 000000000..f12a7a1f2 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_8192/seed546672350.log @@ -0,0 +1,678 @@ +:::MLLOG {"namespace": "", "time_ms": 1784930369192, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784930369192, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784930389693, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784930389698, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784930389698, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784930389698, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784930389698, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784930389698, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 8192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784930389698, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784930389698, "event_type": "POINT_IN_TIME", "key": "seed", "value": 546672350, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784930389698, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784930389698, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784930389698, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784930389698, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784930389707, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784930389707, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784930974695, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784930974697, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784930974995, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784931186696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1390204131603241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840, "lr": 7.916666666666666e-10}} +:::MLLOG {"namespace": "", "time_ms": 1784931199265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13786539435386658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.6249999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784931212192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13865627348423004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 491520, "lr": 2.458333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784931225397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13802677392959595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784931238622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383385956287384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 819200, "lr": 4.125e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784931252022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13846565783023834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784931265318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381155103445053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1146880, "lr": 5.7916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784931278824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13922083377838135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.625e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784931292341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13839635252952576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1474560, "lr": 7.458333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784931305842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381588876247406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.291666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784931319805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13873346149921417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1802240, "lr": 9.124999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784931333647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389465630054474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784931347407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13854007422924042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2129920, "lr": 1.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931360974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14015771448612213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931360974, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784931360975, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784931445741, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4994525909423828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784931445742, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784931445742, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784931459401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13784918189048767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2457600, "lr": 1.2458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931473266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380368173122406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931487254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13816162943840027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2785280, "lr": 1.4124999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931500859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377783715724945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4958333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931514803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13764257729053497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3112960, "lr": 1.5791666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931529104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13747434318065643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931542952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13779786229133606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3440640, "lr": 1.7458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931556871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13760286569595337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931571039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13718311488628387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3768320, "lr": 1.9124999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931584992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13758327066898346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9958333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931599134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13812220096588135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4096000, "lr": 2.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931613491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138294979929924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.1624999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931627613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13753624260425568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4423680, "lr": 2.2458333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931642090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13794384896755219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3291666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931642091, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784931642091, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784931725632, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49967291951179504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784931725633, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784931725633, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784931739887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13737691938877106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4751360, "lr": 2.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931754148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13809585571289062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931768324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13668009638786316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5079040, "lr": 2.5791666666666668e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931782680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357746720314026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931797053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13734029233455658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5406720, "lr": 2.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931811344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13803184032440186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931825531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13661524653434753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5734400, "lr": 2.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931839948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13619530200958252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931854205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13620692491531372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6062080, "lr": 3.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931868400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13744208216667175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931882835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13678397238254547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6389760, "lr": 3.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931896908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373472809791565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931911732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13752947747707367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6717440, "lr": 3.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931926219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13736192882061005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931926220, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784931926220, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784932010103, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49994543194770813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784932010104, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784932010104, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784932024540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13615094125270844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7045120, "lr": 3.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932039111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370156854391098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932053478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13623012602329254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7372800, "lr": 3.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932068345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13699166476726532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.829166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932082884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370358020067215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7700480, "lr": 3.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932097743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13763421773910522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932112460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360657811164856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8028160, "lr": 4.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932126920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13718916475772858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932141694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13652119040489197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8355840, "lr": 4.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932156596, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385268270969391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3291666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932171322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357440948486328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8683520, "lr": 4.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932185911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375763714313507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.4958333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932200429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13474738597869873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9011200, "lr": 4.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932215040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363845318555832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.6624999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932215041, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784932215041, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784932297744, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.50034099817276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784932297745, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784932297745, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784932312068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13581424951553345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9338880, "lr": 4.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932326564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350419968366623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932340806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13773706555366516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9666560, "lr": 4.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932355278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13401590287685394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932369935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13641871511936188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9994240, "lr": 5.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932384268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13275942206382751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1624999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932398633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352444589138031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10321920, "lr": 5.2458333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932412873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13495025038719177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932427314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13790039718151093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10649600, "lr": 5.4124999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932441667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347585767507553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932456113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14856696128845215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10977280, "lr": 5.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932470776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13532550632953644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932485141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13341832160949707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11304960, "lr": 5.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932499300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13791024684906006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932499300, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784932499301, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784932583794, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5008062124252319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784932583795, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784932583795, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784932598314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13518068194389343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11632640, "lr": 5.9124999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932612727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13986483216285706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932627311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357782781124115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11960320, "lr": 6.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932641650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13278305530548096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932656223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13478398323059082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12288000, "lr": 6.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932670959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13923342525959015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.329166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932685472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13299208879470825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12615680, "lr": 6.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932699997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13275980949401855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932714431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13306425511837006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12943360, "lr": 6.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932728663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13118581473827362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932743221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13073016703128815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13271040, "lr": 6.745833333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932757456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13487932085990906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932772000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13157713413238525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13598720, "lr": 6.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932786805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13899488747119904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932786805, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784932786806, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784932872638, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5014133453369141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784932872639, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784932872639, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784932887077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13349208235740662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13926400, "lr": 7.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932901369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13317444920539856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932915952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13556191325187683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14254080, "lr": 7.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932930580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13489803671836853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932945140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137803316116333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14581760, "lr": 7.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932959722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12744081020355225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932974037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13525868952274323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14909440, "lr": 7.579166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932988521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1292271614074707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933003181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13328766822814941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15237120, "lr": 7.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933017990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13244707882404327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.829166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933032525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13353879749774933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15564800, "lr": 7.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933046927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343860924243927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.995833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933062064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13549068570137024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15892480, "lr": 8.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933076606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13323698937892914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933076607, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933076607, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933162097, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.502288818359375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933162098, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933162098, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933176977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323731392621994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16220160, "lr": 8.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933191721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13368788361549377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933206479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356704980134964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16547840, "lr": 8.412500000000001e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933221462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13550212979316711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933236144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13968992233276367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16875520, "lr": 8.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933250607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331995725631714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.662499999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933265980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355317085981369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17203200, "lr": 8.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933280859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13340544700622559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933295849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312242925167084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17530880, "lr": 8.912499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933311188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13178585469722748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933326171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13584616780281067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17858560, "lr": 9.079166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933341306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13361775875091553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.162499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933356000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13339564204216003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18186240, "lr": 9.245833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933370584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1307116597890854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933370584, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784933370585, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784933456410, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5034345388412476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784933456411, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784933456411, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784933471555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13788090646266937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18513920, "lr": 9.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933486260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13487717509269714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.495833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933501125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354423463344574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18841600, "lr": 9.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933516136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13130652904510498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.662499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933530829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13361583650112152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19169280, "lr": 9.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933545580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13404113054275513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933560621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370977759361267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19496960, "lr": 9.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933575548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13383617997169495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933590190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13300740718841553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19824640, "lr": 1.0079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933605101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12970896065235138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0162499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933619489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1276036500930786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20152320, "lr": 1.0245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933633827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13289855420589447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933648483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12971150875091553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20480000, "lr": 1.0412499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933662827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1248345673084259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933662827, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784933662828, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784933747220, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5046277642250061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784933747220, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784933747221, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784933761978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13120490312576294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20807680, "lr": 1.0579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933776634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13594689965248108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933791555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133366197347641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21135360, "lr": 1.0745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933806396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13038285076618195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933820814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12821829319000244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21463040, "lr": 1.0912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933835273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12660683691501617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933849962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289212554693222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21790720, "lr": 1.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933864164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323247253894806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933878649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354707032442093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22118400, "lr": 1.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933893195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12501078844070435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933907433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12858030200004578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22446080, "lr": 1.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933921734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13447900116443634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933936223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13107462227344513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22773760, "lr": 1.1579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933951099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1291963756084442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933951100, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784933951100, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934034033, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5061578750610352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934034033, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934034034, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934048792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13058283925056458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23101440, "lr": 1.1745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934063250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1290827989578247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934077291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13303296267986298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23429120, "lr": 1.1912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934092017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1239861398935318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934106915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12961581349372864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23756800, "lr": 1.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934121344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12583142518997192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934135900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1302293986082077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24084480, "lr": 1.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934150587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1218237578868866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934164785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358364075422287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24412160, "lr": 1.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934179512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13729922473430634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934193973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12357193976640701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24739840, "lr": 1.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934208946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12321016937494278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934223667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12575945258140564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25067520, "lr": 1.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934238196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288527101278305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934238196, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934238196, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934323882, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5079554319381714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934323883, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934323883, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934337993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13646940886974335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25395200, "lr": 1.2912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934352737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13634203374385834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934367141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13571444153785706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25722880, "lr": 1.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934381300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13994985818862915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934395569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13656319677829742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26050560, "lr": 1.3245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934410027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13828469812870026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934424456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12738151848316193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26378240, "lr": 1.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934438807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12628492712974548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934453202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14212629199028015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26705920, "lr": 1.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934467988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14102551341056824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934482722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12733325362205505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27033600, "lr": 1.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934497296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12856151163578033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934511749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327178180217743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27361280, "lr": 1.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934525703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13142827153205872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934525703, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934525703, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934612474, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5100165009498596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934612475, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934612475, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934627367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14128318428993225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27688960, "lr": 1.4079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934642493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14238598942756653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934657333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13526351749897003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28016640, "lr": 1.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934672246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13955073058605194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934687486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14466935396194458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28344320, "lr": 1.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934702183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14097295701503754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934717353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12588156759738922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28672000, "lr": 1.4579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934732441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13749440014362335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934747599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13170044124126434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28999680, "lr": 1.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934762722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13611991703510284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934777769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137721985578537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29327360, "lr": 1.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934792340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329604983329773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4995833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934807316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343197524547577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29655040, "lr": 1.5079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934822334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13655684888362885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934822336, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934822336, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934907121, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5121243000030518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934907121, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934907122, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934921649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13866978883743286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29982720, "lr": 1.5245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934936514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13817159831523895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934951569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13193310797214508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30310400, "lr": 1.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934966816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1328168660402298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934982109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13371635973453522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30638080, "lr": 1.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934997452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13992708921432495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935012785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13275635242462158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30965760, "lr": 1.5745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935028246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14124181866645813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935043033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1408659815788269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31293440, "lr": 1.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935057832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13517534732818604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935072601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12779229879379272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31621120, "lr": 1.6079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935087236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12992382049560547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935101919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13420814275741577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31948800, "lr": 1.6245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935116776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1275714635848999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935116777, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784935116777, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784935201321, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5146987438201904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784935201322, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784935201322, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784935216042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13208593428134918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32276480, "lr": 1.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935230676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368894726037979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935245444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13347411155700684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32604160, "lr": 1.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935260045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13195888698101044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935274777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13392986357212067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32931840, "lr": 1.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935289466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13359524309635162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935304670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13393634557724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33259520, "lr": 1.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935319487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132762148976326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935334027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13983596861362457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33587200, "lr": 1.7079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935349041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13194821774959564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935364060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12879741191864014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33914880, "lr": 1.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935378908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312377154827118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935393839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13247929513454437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34242560, "lr": 1.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935408767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354905664920807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7495833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935408767, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784935408768, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784935492910, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5180987119674683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784935492910, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784935492910, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784935507916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13363142311573029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34570240, "lr": 1.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935522742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12303867191076279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935537418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13067308068275452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34897920, "lr": 1.7745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935552201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13043242692947388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935566906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351919323205948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35225600, "lr": 1.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935582060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13532361388206482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935596666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13542282581329346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35553280, "lr": 1.8079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935611683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12404372543096542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8162500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935626334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13107341527938843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35880960, "lr": 1.8245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935640696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13114838302135468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935655549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303464025259018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36208640, "lr": 1.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935670202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333281695842743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935685064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13084353506565094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36536320, "lr": 1.8579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935699700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13019680976867676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935699701, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784935699701, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784935784152, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5226544737815857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784935784153, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784935784154, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784935799072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13782382011413574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36864000, "lr": 1.8745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935813944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324256956577301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935828707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13338273763656616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37191680, "lr": 1.8912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935843321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303848773241043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935857924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12712883949279785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37519360, "lr": 1.9079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935872391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13192646205425262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935887346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13566800951957703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37847040, "lr": 1.9245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935902254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1328504979610443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935917144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13145823776721954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38174720, "lr": 1.9412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935932161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13914792239665985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935946999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13382640480995178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38502400, "lr": 1.9579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935961365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376328468322754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935976296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1319892704486847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38830080, "lr": 1.9745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935990913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13341636955738068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935990914, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784935990914, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784936075765, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5288664698600769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784936075766, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784936075766, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784936090513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1309298723936081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39157760, "lr": 1.9912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936105292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13484163582324982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936119949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380932331085205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39485440, "lr": 2.0079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936134616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333598792552948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936149141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1272531896829605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39813120, "lr": 2.0245833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936163822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1293255090713501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0329166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936178491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12600795924663544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40140800, "lr": 2.0412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936193183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12402349710464478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936208122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13630622625350952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40468480, "lr": 2.0579166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936223123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13720370829105377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936238073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347140222787857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40796160, "lr": 2.0745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936253592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12713760137557983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0829166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936268697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13087749481201172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41123840, "lr": 2.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936283147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1236271932721138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936283148, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784936283148, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784936369677, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5375683307647705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784936369678, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784936369678, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784936384321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323150396347046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41451520, "lr": 2.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936399243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325889378786087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936413992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13364316523075104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41779200, "lr": 2.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936428611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13792292773723602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936443200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13635781407356262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42106880, "lr": 2.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936457847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13113835453987122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936472416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1272830069065094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42434560, "lr": 2.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936487106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13494476675987244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936501640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12469716370105743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42762240, "lr": 2.1745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936516619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13046114146709442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936531370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326279640197754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43089920, "lr": 2.1912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936546270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13437965512275696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936561225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14056025445461273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43417600, "lr": 2.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936576003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362370252609253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936576004, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784936576004, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784936663025, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.550406813621521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784936663026, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784936663026, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784936678355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13666358590126038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43745280, "lr": 2.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936693237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13157066702842712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936707851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326279193162918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44072960, "lr": 2.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936722697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13910675048828125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936738121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13013368844985962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44400640, "lr": 2.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936753589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13843712210655212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936768753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13508574664592743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44728320, "lr": 2.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936783959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13712678849697113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936799152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13475729525089264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45056000, "lr": 2.2912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936814505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13069666922092438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936829782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1405787318944931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45383680, "lr": 2.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936844926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346164047718048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936859970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.129020094871521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45711360, "lr": 2.3245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936875189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333719938993454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936875189, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784936875190, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784936961483, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5654758810997009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784936961484, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784936961484, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784936976832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14211831986904144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46039040, "lr": 2.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936992219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13280479609966278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937008070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334669142961502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46366720, "lr": 2.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937023417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13067477941513062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937038861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13521920144557953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46694400, "lr": 2.3745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937053992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337593048810959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937069100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13426989316940308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47022080, "lr": 2.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937084434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13571999967098236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937099666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13116572797298431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47349760, "lr": 2.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937115173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1328655481338501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937130496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13289807736873627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47677440, "lr": 2.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937145851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13696204125881195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937160767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13613568246364594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48005120, "lr": 2.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937176240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13752590119838715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.4495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937176240, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784937176240, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784937260981, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5822179913520813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784937260982, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784937260982, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784937276140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1316874623298645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48332800, "lr": 2.4579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937291621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13524475693702698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937306827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13420315086841583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48660480, "lr": 2.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937322473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1319865882396698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937337463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13725003600120544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48988160, "lr": 2.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937352407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295015513896942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937367498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13361451029777527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49315840, "lr": 2.507916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937381992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12403292208909988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.5162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937396764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13087140023708344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49643520, "lr": 2.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937411393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1316005289554596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937426133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13102664053440094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49971200, "lr": 2.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937440735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315045952796936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.5495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937455442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12889645993709564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50298880, "lr": 2.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937470170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13078606128692627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.5662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937470171, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784937470171, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784937556080, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5962616205215454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784937556081, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784937556081, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784937570525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1223122775554657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50626560, "lr": 2.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937585255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12692785263061523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937599836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12960711121559143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50954240, "lr": 2.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937614599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132970929145813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937629705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13193121552467346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51281920, "lr": 2.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937644828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12166789919137955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937660069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13070692121982574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51609600, "lr": 2.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937675118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13206368684768677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937689958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370457410812378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51937280, "lr": 2.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937704425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1281294971704483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.6495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937719449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333521604537964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52264960, "lr": 2.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937734448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11786714941263199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.6662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937749426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12162790447473526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52592640, "lr": 2.6745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937764504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12541361153125763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937764505, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784937764505, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784937850776, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6104215979576111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784937850777, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784937850777, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784937865762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12144467234611511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52920320, "lr": 2.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937880533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12756207585334778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.6995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937895574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12882624566555023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53248000, "lr": 2.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937910434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13213808834552765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937925334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12153444439172745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53575680, "lr": 2.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937940242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13490822911262512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937954971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295929253101349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53903360, "lr": 2.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937969809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12177200615406036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.7495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937984893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12715664505958557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54231040, "lr": 2.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938000160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1272268444299698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938015109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11851995438337326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54558720, "lr": 2.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938030567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318323165178299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938045821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12717083096504211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54886400, "lr": 2.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938061024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12850505113601685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938061025, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784938061025, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784938149001, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6239259243011475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784938149002, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784938149002, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784938163685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12820583581924438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55214080, "lr": 2.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938178842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11873514950275421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938193908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12177902460098267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55541760, "lr": 2.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938209058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12143519520759583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938223996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12397589534521103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55869440, "lr": 2.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938238900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12616348266601562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938253834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11638250201940536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56197120, "lr": 2.8579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938268916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12266775220632553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938283889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12555143237113953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56524800, "lr": 2.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938298852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12399408966302872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938313568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.115031898021698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56852480, "lr": 2.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938328425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12398573011159897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938343198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11014449596405029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57180160, "lr": 2.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938358228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12482797354459763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938358230, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784938358230, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784938445140, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6372470855712891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784938445141, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784938445141, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784938460356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11695849895477295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57507840, "lr": 2.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938475445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11497151106595993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938490659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294596642255783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57835520, "lr": 2.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938505991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12181269377470016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938520827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12381323426961899, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58163200, "lr": 2.9579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938535643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11678199470043182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938549972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12024708837270737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58490880, "lr": 2.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938565465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1220075860619545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938580988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12060094624757767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58818560, "lr": 2.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938596615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11287448555231094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938612228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12081152200698853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59146240, "lr": 3.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938627587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12167034298181534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.0162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938642782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12474670261144638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59473920, "lr": 3.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938658110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11736506968736649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938658111, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784938658111, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784938744363, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6602479815483093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784938744363, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784938744364, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784938759589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11939649283885956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59801600, "lr": 3.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938774598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11208266019821167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938789715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1100817546248436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60129280, "lr": 3.0579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938805017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11294769495725632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.0662499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938820288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10849444568157196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60456960, "lr": 3.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938835788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12187722325325012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938851431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11633173376321793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60784640, "lr": 3.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938867149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12039190530776978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938882380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10293262451887131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61112320, "lr": 3.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938898004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11980141699314117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.1162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938913498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11317829042673111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61440000, "lr": 3.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938928932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11366777867078781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938944070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10928861051797867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61767680, "lr": 3.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938959160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11618045717477798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938959161, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784938959161, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784939044454, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6937454342842102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784939044455, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784939044456, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784939059311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11995279043912888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62095360, "lr": 3.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939074545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11940624564886093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.1662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939089751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12303195148706436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62423040, "lr": 3.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939104842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10529191792011261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939119994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11408238112926483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62750720, "lr": 3.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939134910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11694838851690292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939150050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11908195912837982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63078400, "lr": 3.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939165178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1182609498500824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939180224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087748259305954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63406080, "lr": 3.2245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939195163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11724633723497391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939210186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1156880185008049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63733760, "lr": 3.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939225014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11577170342206955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939239776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11275888979434967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64061440, "lr": 3.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939255061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12450011074542999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939255062, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784939255062, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784939339102, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7192524075508118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784939339103, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784939339103, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784939354423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12364639341831207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64389120, "lr": 3.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939369671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1161765530705452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939384663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11577105522155762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64716800, "lr": 3.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939400033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1215982735157013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939415203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10941796749830246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65044480, "lr": 3.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939430474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11106391996145248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939445135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11252942681312561, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65372160, "lr": 3.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939460386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10784813016653061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939475832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11083297431468964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65699840, "lr": 3.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939491735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11616229265928268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939507530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12299232184886932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66027520, "lr": 3.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939523213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1147681400179863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939539145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.126273974776268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66355200, "lr": 3.3745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939554958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11224939674139023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939554959, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784939554959, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784939637442, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7523471713066101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784939637443, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784939637443, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784939653285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12552717328071594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66682880, "lr": 3.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939668928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11391382664442062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.3995833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939684521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11569991707801819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67010560, "lr": 3.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939700253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11286569386720657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939715736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1188555508852005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67338240, "lr": 3.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939731436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1182413175702095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939747347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12112338095903397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67665920, "lr": 3.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939763283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1200919970870018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939779054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11501278728246689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67993600, "lr": 3.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939795166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11220811307430267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939810963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11479891836643219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68321280, "lr": 3.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939826402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11531636118888855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939841689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12500953674316406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68648960, "lr": 3.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939857310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11559991538524628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939857311, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784939857311, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784939943149, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7597017288208008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784939943150, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784939943150, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784939958565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1208360493183136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68976640, "lr": 3.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939974040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12020669132471085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939989533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11308158934116364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69304320, "lr": 3.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940005308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11444103717803955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940020907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1079978421330452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69632000, "lr": 3.5412500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940036511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11504068225622177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940052217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1168321818113327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69959680, "lr": 3.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940067521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11067923903465271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940083133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10910087823867798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70287360, "lr": 3.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940098528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10795875638723373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940114285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12209182977676392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70615040, "lr": 3.5912500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940129986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10602448135614395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940145775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12089183926582336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70942720, "lr": 3.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940161851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11972643435001373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940161852, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784940161852, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784940246766, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7638093829154968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784940246767, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784940246767, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784940262665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1185598075389862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71270400, "lr": 3.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940278071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1176624596118927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940293374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11544173210859299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71598080, "lr": 3.6412499999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940309173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11451605707406998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940324808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10258720815181732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71925760, "lr": 3.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940340341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1032528281211853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940355929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11881006509065628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72253440, "lr": 3.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940371386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11413559317588806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940386809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10359210520982742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72581120, "lr": 3.6912499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940402254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10223335772752762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940417300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1062302216887474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72908800, "lr": 3.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940432597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11153040826320648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940447870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11025074869394302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73236480, "lr": 3.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940462971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11479317396879196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940463012, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784940463012, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784940547628, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7664027810096741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784940547629, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784940547629, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784940562953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10448537021875381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73564160, "lr": 3.7412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940578612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10680893808603287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940594342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11492560803890228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73891840, "lr": 3.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940609609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11014033108949661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940625161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1063852310180664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74219520, "lr": 3.7745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940640904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11363869160413742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940656347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11286409199237823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74547200, "lr": 3.7912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940671634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11169115453958511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940686858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11657290160655975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74874880, "lr": 3.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940701856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11067714542150497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940716771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10031159967184067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75202560, "lr": 3.8245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940731669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11349481344223022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940746517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10605772584676743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75530240, "lr": 3.8412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940761512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10504526644945145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940761512, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784940761513, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784940845707, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7679443955421448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784940845708, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784940845708, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784940860556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10599172115325928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75857920, "lr": 3.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940875772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11353108286857605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940890892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09602254629135132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76185600, "lr": 3.8745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940905973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10330890119075775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940921050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11242339760065079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76513280, "lr": 3.8912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940935822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10443577170372009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940950970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0881919115781784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76840960, "lr": 3.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940965735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10869967937469482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940980857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10153903067111969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77168640, "lr": 3.9245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940996152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11773420870304108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784941011471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10696043819189072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77496320, "lr": 3.9412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784941026615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10016818344593048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784941042002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0996507927775383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77824000, "lr": 3.9579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784941057431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10751274228096008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784941057432, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784941057432, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784941141427, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7688356637954712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784941141428, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784941141429, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784941156493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11522859334945679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78151680, "lr": 3.9745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784941171566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1119188740849495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784941185975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10383007675409317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78479360, "lr": 3.9912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784941201389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10598888993263245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784941216515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09989817440509796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78807040, "lr": 4.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784941231886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10191570967435837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784941247171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10591349005699158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79134720, "lr": 4.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784941262580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10703720152378082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784941277663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09788559377193451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79462400, "lr": 4.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784941293012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084107756614685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784941308119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10487118363380432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79790080, "lr": 4.0579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784941323210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1061650738120079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784941338294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10820775479078293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80117760, "lr": 4.074583333333333e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_8192/seed770796222.log b/recommendation_v4/rcp_logs/gbs_8192/seed770796222.log new file mode 100644 index 000000000..ab7ae52d8 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_8192/seed770796222.log @@ -0,0 +1,684 @@ +:::MLLOG {"namespace": "", "time_ms": 1784939247303, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784939247303, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784939268454, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784939268455, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784939268455, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784939268456, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784939268456, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784939268456, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 8192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784939268456, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784939268456, "event_type": "POINT_IN_TIME", "key": "seed", "value": 770796222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784939268456, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784939268456, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784939268456, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784939268456, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784939268456, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784939268457, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784940110137, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784940110137, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784940110497, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784940225888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13908115029335022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840, "lr": 7.916666666666666e-10}} +:::MLLOG {"namespace": "", "time_ms": 1784940238689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13829296827316284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.6249999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940251849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384357213973999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 491520, "lr": 2.458333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940265251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382930427789688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940278733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386009007692337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 819200, "lr": 4.125e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940292201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13823385536670685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940305614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383131444454193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1146880, "lr": 5.7916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940319334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13871265947818756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.625e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940333013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860803842544556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1474560, "lr": 7.458333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940346429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13859035074710846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.291666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940360312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13851207494735718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1802240, "lr": 9.124999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940374023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13880814611911774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940387889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13882553577423096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2129920, "lr": 1.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940401489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13926050066947937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940401489, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784940401489, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784940483201, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4998165965080261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784940483202, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784940483202, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784940496432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13840055465698242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2457600, "lr": 1.2458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940509927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13842704892158508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940524066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13828259706497192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2785280, "lr": 1.4124999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940537480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13825000822544098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4958333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940551164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13795754313468933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3112960, "lr": 1.5791666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940565145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13807256519794464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940578514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13793312013149261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3440640, "lr": 1.7458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940592089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137899249792099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940605997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13773825764656067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3768320, "lr": 1.9124999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940619442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13816820085048676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9958333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940633402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382180154323578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4096000, "lr": 2.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940647369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384699046611786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.1624999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940661094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13784074783325195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4423680, "lr": 2.2458333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940675276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382867693901062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3291666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940675276, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784940675277, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784940758347, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.500006914138794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784940758349, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784940758349, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784940771860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13771642744541168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4751360, "lr": 2.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940785388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13823577761650085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940798897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13701128959655762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5079040, "lr": 2.5791666666666668e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940813016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366855502128601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940826669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379123330116272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5406720, "lr": 2.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940840257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13788297772407532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940853946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13718225061893463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5734400, "lr": 2.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940867818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368478238582611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940881412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365009993314743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6062080, "lr": 3.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940894857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13766390085220337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940908535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13720516860485077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6389760, "lr": 3.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940921873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13761404156684875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940936274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13777071237564087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6717440, "lr": 3.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940950145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373598873615265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940950146, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784940950146, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784941032018, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5002294778823853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784941032018, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784941032019, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784941045834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363908052444458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7045120, "lr": 3.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941059877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13721664249897003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941073729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13645799458026886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7372800, "lr": 3.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941088209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13699404895305634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.829166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941101923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13730719685554504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7700480, "lr": 3.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941115920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13769389688968658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941129816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13629816472530365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8028160, "lr": 4.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941143620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1372116059064865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941157914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13656985759735107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8355840, "lr": 4.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941171983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13863030076026917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3291666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941186138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13614261150360107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8683520, "lr": 4.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941200117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13786545395851135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.4958333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941214128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13506758213043213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9011200, "lr": 4.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941228298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13670232892036438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.6624999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941228299, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784941228300, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784941309336, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5005543828010559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784941309337, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784941309337, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784941322887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361103653907776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9338880, "lr": 4.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941336494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348491907119751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941349887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13771997392177582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9666560, "lr": 4.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941363597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13402718305587769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941377447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13645735383033752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9994240, "lr": 5.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941391026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13282491266727448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1624999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941404707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13524264097213745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10321920, "lr": 5.2458333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941418391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13512462377548218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941432133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13793335855007172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10649600, "lr": 5.4124999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941445769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347508728504181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941459546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1485426425933838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10977280, "lr": 5.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941473593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13532310724258423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941487229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13346447050571442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11304960, "lr": 5.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941500706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13783738017082214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941500707, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784941500707, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784941582026, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5009433031082153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784941582026, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784941582027, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784941595510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13505534827709198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11632640, "lr": 5.9124999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941609226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14002081751823425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941623116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13576224446296692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11960320, "lr": 6.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941637086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13282644748687744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941651148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346132606267929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12288000, "lr": 6.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941665316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13948161900043488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.329166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941679219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13303864002227783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12615680, "lr": 6.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941693210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13259045779705048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941707099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13295911252498627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12943360, "lr": 6.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941720661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1309451460838318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941734525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13067713379859924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13271040, "lr": 6.745833333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941748089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13475008308887482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941761995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131047785282135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13598720, "lr": 6.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941776259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13924817740917206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941776260, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784941776260, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784941857384, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5014552474021912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784941857385, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784941857385, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784941871401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13351845741271973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13926400, "lr": 7.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941885349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332426518201828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941899456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13579390943050385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14254080, "lr": 7.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941913126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349928081035614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941926882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381620168685913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14581760, "lr": 7.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941940766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12743262946605682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941954263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13524407148361206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14909440, "lr": 7.579166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941968086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12914790213108063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941982013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13317517936229706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15237120, "lr": 7.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941996085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13220351934432983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.829166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942009924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13336405158042908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15564800, "lr": 7.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942024032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13428744673728943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.995833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942038920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353127360343933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15892480, "lr": 8.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942053397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13355353474617004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942053397, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784942053398, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784942134869, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5022715330123901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784942134869, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784942134870, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784942149299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325080543756485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16220160, "lr": 8.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942163739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337248980998993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942178401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13541705906391144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16547840, "lr": 8.412500000000001e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942193174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13538599014282227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942207682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13978520035743713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16875520, "lr": 8.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942221879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331697404384613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.662499999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942236599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13533172011375427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17203200, "lr": 8.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942250622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13345962762832642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942264602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13326501846313477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17530880, "lr": 8.912499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942278844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13162453472614288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942292951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13565203547477722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17858560, "lr": 9.079166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942307515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13349440693855286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.162499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942321721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13321444392204285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18186240, "lr": 9.245833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942335734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13050124049186707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942335735, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784942335735, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784942417803, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5033178329467773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784942417804, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784942417805, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784942432708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13784071803092957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18513920, "lr": 9.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942447078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348913460969925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.495833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942461767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13533581793308258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18841600, "lr": 9.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942476524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310604214668274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.662499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942490763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335025131702423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19169280, "lr": 9.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942505345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13425405323505402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942520143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13724718987941742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19496960, "lr": 9.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942534791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13397210836410522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942549373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13295944035053253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19824640, "lr": 1.0079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942563642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12944483757019043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0162499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942577716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1275712549686432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20152320, "lr": 1.0245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942591881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13303858041763306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942606191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12945109605789185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20480000, "lr": 1.0412499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942620084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12461645156145096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942620085, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784942620085, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784942700644, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5043933987617493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784942700645, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784942700645, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784942714942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13101999461650848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20807680, "lr": 1.0579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942729149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360672414302826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942743571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333436369895935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21135360, "lr": 1.0745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942757950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13029563426971436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942771857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12814484536647797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21463040, "lr": 1.0912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942786038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12663990259170532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942800695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12885528802871704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21790720, "lr": 1.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942815050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13229918479919434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942829230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13521791994571686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22118400, "lr": 1.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942843412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12488709390163422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942857435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12857326865196228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22446080, "lr": 1.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942871497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344846487045288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942885665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13097885251045227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22773760, "lr": 1.1579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942900199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12945348024368286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942900199, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784942900200, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784942983060, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5058243274688721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784942983061, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784942983061, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784942997547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13077408075332642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23101440, "lr": 1.1745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943011599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12903596460819244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943025275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13308469951152802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23429120, "lr": 1.1912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943039622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12399835139513016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943054000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12955829501152039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23756800, "lr": 1.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943067881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12581558525562286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943081865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13009564578533173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24084480, "lr": 1.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943096108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12198678404092789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943109981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13594888150691986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24412160, "lr": 1.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943124243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13716500997543335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943138363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12368225306272507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24739840, "lr": 1.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943152968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12321016192436218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943167285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12568898499011993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25067520, "lr": 1.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943181493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12916119396686554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943181493, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784943181494, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784943264499, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5074609518051147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784943264500, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784943264500, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784943278425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364949494600296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25395200, "lr": 1.2912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943293110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13647565245628357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943306946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358993500471115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25722880, "lr": 1.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943320768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14011676609516144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943334810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13674508035182953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26050560, "lr": 1.3245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943348884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13819822669029236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943362862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12736250460147858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26378240, "lr": 1.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943376881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12607093155384064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943390884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14213509857654572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26705920, "lr": 1.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943405144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14093175530433655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943419395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12720386683940887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27033600, "lr": 1.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943433541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12792842090129852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943447610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13237667083740234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27361280, "lr": 1.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943461219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315924972295761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943461220, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784943461220, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784943545784, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5092635750770569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784943545785, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784943545785, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784943560660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1412469744682312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27688960, "lr": 1.4079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943575245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14236417412757874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943589635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353716105222702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28016640, "lr": 1.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943604159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13955093920230865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943618791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14500051736831665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28344320, "lr": 1.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943632969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1409694254398346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943647451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1261337846517563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28672000, "lr": 1.4579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943661912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13771438598632812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943676612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317090094089508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28999680, "lr": 1.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943691287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13627910614013672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943705855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13770975172519684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29327360, "lr": 1.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943720082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133291557431221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4995833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943735007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344006359577179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29655040, "lr": 1.5079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943749707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364137828350067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943749708, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784943749708, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784943832268, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5111699104309082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784943832269, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784943832269, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784943846438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13857853412628174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29982720, "lr": 1.5245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943861025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13825443387031555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943875630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13175275921821594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30310400, "lr": 1.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943890255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13292165100574493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943904739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13378921151161194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30638080, "lr": 1.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943919419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13955466449260712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943933949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326322704553604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30965760, "lr": 1.5745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943948624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14118103682994843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943962686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14083534479141235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31293440, "lr": 1.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943977467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13512617349624634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943991903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.127433180809021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31621120, "lr": 1.6079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944006186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300075799226761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944020459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13402236998081207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31948800, "lr": 1.6245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944034845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12760134041309357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944034846, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784944034846, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784944116588, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5134813785552979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784944116589, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784944116589, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784944130747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13232794404029846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32276480, "lr": 1.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944144926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13679839670658112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944159240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13366927206516266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32604160, "lr": 1.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944173646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13192607462406158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944188247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337735950946808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32931840, "lr": 1.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944202766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13347522914409637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944217590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13388365507125854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33259520, "lr": 1.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944232151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312214612960815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944246456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13986335694789886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33587200, "lr": 1.7079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944261249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13190999627113342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944275657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12872928380966187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33914880, "lr": 1.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944289740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13112932443618774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944303937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13251857459545135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34242560, "lr": 1.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944318304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13551901280879974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7495833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944318305, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784944318305, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784944400279, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5165881514549255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784944400280, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784944400280, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784944414766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13326072692871094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34570240, "lr": 1.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944429177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12302582710981369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944443619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13051612675189972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34897920, "lr": 1.7745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944458113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13037177920341492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944472537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13512372970581055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35225600, "lr": 1.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944487586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13540875911712646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944502131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13539008796215057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35553280, "lr": 1.8079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944516988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12418771535158157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8162500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944531546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13093052804470062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35880960, "lr": 1.8245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944545836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13119067251682281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944560699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1302916556596756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36208640, "lr": 1.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944575003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13333092629909515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944589373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13080580532550812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36536320, "lr": 1.8579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944603363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300952136516571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944603363, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784944603364, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784944685211, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5207803249359131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784944685213, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784944685213, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784944699586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377429962158203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36864000, "lr": 1.8745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944713904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13236793875694275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944728199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13350196182727814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37191680, "lr": 1.8912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944742516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1306360363960266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944756636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1270102858543396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37519360, "lr": 1.9079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944770694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13224314153194427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944785169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13574163615703583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37847040, "lr": 1.9245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944799740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13292191922664642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944814431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312902271747589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38174720, "lr": 1.9412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944829175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13902471959590912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944843712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13377511501312256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38502400, "lr": 1.9579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944857760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375884711742401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944872727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13193300366401672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38830080, "lr": 1.9745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944887360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13331569731235504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944887361, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784944887361, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784944970128, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5264279842376709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784944970129, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784944970129, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784944984744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13132791221141815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39157760, "lr": 1.9912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944999230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13487935066223145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945013764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13770917057991028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39485440, "lr": 2.0079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945028256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13341452181339264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945042647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12726476788520813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39813120, "lr": 2.0245833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945057148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1291874647140503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0329166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945071556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12599241733551025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40140800, "lr": 2.0412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945085933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12383176386356354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945100593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13615980744361877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40468480, "lr": 2.0579166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945115322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374301165342331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945129974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13457061350345612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40796160, "lr": 2.0745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945145123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12706312537193298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0829166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945159843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13096767663955688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41123840, "lr": 2.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945174030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12366443872451782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945174031, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784945174031, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784945258685, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5341439843177795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784945258686, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784945258686, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784945273434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327822059392929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41451520, "lr": 2.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945287694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13257776200771332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945302017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1339135766029358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41779200, "lr": 2.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945316282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13797128200531006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945330573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360999345779419, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42106880, "lr": 2.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945344852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13139860332012177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945358988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1272415816783905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42434560, "lr": 2.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945373196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13500526547431946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945387261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12464774399995804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42762240, "lr": 2.1745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945401761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13031405210494995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945416168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13240061700344086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43089920, "lr": 2.1912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945430695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13430683314800262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945445298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1404537558555603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43417600, "lr": 2.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945459732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362384408712387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945459732, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784945459733, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784945542880, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5457786917686462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784945542880, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784945542881, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784945557941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13642795383930206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43745280, "lr": 2.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945572715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13140136003494263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945586956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13258618116378784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44072960, "lr": 2.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945601573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1393444687128067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945617162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298486292362213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44400640, "lr": 2.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945632579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383754312992096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945647501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13501916825771332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44728320, "lr": 2.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945662559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13712294399738312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945677340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13494929671287537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45056000, "lr": 2.2912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945692095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1307007521390915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945706831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14053502678871155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45383680, "lr": 2.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945721433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13462436199188232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945735926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12921643257141113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45711360, "lr": 2.3245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945750556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13331720232963562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945750557, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784945750557, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784945833833, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5595343708992004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784945833834, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784945833834, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784945848607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1421358585357666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46039040, "lr": 2.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945863279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327095776796341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945878422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331631988286972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46366720, "lr": 2.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945893093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13091619312763214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945908132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13524457812309265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46694400, "lr": 2.3745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945923046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13378183543682098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945937807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13396644592285156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47022080, "lr": 2.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945952704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13559584319591522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945967374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312534064054489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47349760, "lr": 2.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945982284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13290619850158691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945996886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13314951956272125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47677440, "lr": 2.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946011628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13717882335186005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946026008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13646341860294342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48005120, "lr": 2.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946040811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375761181116104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.4495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946040811, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784946040812, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784946123536, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5758647322654724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784946123538, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784946123538, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784946138224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13154739141464233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48332800, "lr": 2.4579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946153130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355418711900711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946167885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13410288095474243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48660480, "lr": 2.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946183048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13237182796001434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946197675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371123194694519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48988160, "lr": 2.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946212355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1296699494123459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946227289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13369764387607574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49315840, "lr": 2.507916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946241392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12375826388597488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.5162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946255621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13104352355003357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49643520, "lr": 2.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946269739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13182342052459717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946283927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312585175037384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49971200, "lr": 2.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946298124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13183942437171936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.5495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946312428, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1290494203567505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50298880, "lr": 2.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946326839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13097837567329407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.5662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946326839, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784946326840, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784946410958, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5904187560081482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784946410959, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784946410959, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784946424899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1224183440208435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50626560, "lr": 2.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946439250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12711462378501892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946453378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12965339422225952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50954240, "lr": 2.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946467621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335100382566452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946482298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13196958601474762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51281920, "lr": 2.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946496572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1222563311457634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946511165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13090331852436066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51609600, "lr": 2.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946525564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13220496475696564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946539832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373385637998581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51937280, "lr": 2.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946553780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12830688059329987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.6495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946568717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13356652855873108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52264960, "lr": 2.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946583195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11812744289636612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.6662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946597530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12209746986627579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52592640, "lr": 2.6745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946611961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1257966160774231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946611961, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784946611963, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784946694449, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.605269730091095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784946694450, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784946694450, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784946708610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12196754664182663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52920320, "lr": 2.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946722652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12777233123779297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.6995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946736771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12911397218704224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53248000, "lr": 2.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946750945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322184056043625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946765025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12190928310155869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53575680, "lr": 2.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946779293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13521234691143036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946793449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12995512783527374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53903360, "lr": 2.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946807523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12219559401273727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.7495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946821873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1276315450668335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54231040, "lr": 2.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946836536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12750087678432465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946850985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11887017637491226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54558720, "lr": 2.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946865850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317559778690338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946880410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1276593953371048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54886400, "lr": 2.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946894901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289098858833313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946894901, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784946894901, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784946978259, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6186714768409729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784946978260, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784946978260, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784946992274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12892256677150726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55214080, "lr": 2.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947007256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11932592839002609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947021590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12278418987989426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55541760, "lr": 2.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947036334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12184914201498032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947050796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1239960715174675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55869440, "lr": 2.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947065631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12649443745613098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947080159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1172710582613945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56197120, "lr": 2.8579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947094686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12313637137413025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947109294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12562932074069977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56524800, "lr": 2.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947123993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12458573281764984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947138296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.116246797144413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56852480, "lr": 2.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947152847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12487556040287018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947167254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1108582466840744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57180160, "lr": 2.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947181637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12549887597560883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947181638, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784947181638, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784947265604, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6296008825302124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784947265605, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784947265605, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784947280180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11771519482135773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57507840, "lr": 2.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947294789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1153372973203659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947309806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13034747540950775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57835520, "lr": 2.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947324814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12259611487388611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947339421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12458934634923935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58163200, "lr": 2.9579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947353873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1172061562538147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947367892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12063441425561905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58490880, "lr": 2.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947382614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12274697422981262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947396900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12143584340810776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58818560, "lr": 2.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947411248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11318125575780869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947425563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12137096375226974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59146240, "lr": 3.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947439641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12246861308813095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.0162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947453809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12525615096092224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59473920, "lr": 3.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947468228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11755955219268799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947468229, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784947468229, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784947554067, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6432031989097595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784947554067, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784947554068, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784947568244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12023003399372101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59801600, "lr": 3.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947582227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11267684400081635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947596208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11055552959442139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60129280, "lr": 3.0579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947610270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11370252072811127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.0662499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947624308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10931577533483505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60456960, "lr": 3.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947638473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12218620628118515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947652921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11696108430624008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60784640, "lr": 3.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947667706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1210312694311142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947682126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10356951504945755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61112320, "lr": 3.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947697001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11947640031576157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.1162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947711675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11294088512659073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61440000, "lr": 3.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947726265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11368671804666519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947740666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11027511954307556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61767680, "lr": 3.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947755357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11599870771169662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947755358, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784947755358, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784947842120, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6611868739128113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784947842121, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784947842121, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784947856891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12149536609649658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62095360, "lr": 3.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947871385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11984942108392715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.1662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947886164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12389393150806427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62423040, "lr": 3.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947900840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10601567476987839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947915556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11471689492464066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62750720, "lr": 3.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947930027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11780353635549545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947944706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11966494470834732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63078400, "lr": 3.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947959228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11981038749217987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947973590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10898194462060928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63406080, "lr": 3.2245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947987953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11845295876264572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948002367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11586112529039383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63733760, "lr": 3.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948016333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11705241352319717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948030538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11378757655620575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64061440, "lr": 3.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948045264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12543413043022156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948045264, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784948045265, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784948130879, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6810621619224548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784948130880, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784948130881, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784948145466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.124947190284729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64389120, "lr": 3.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948159923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11705400049686432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948174062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11691385507583618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64716800, "lr": 3.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948188695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1222459003329277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948203004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.110455222427845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65044480, "lr": 3.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948217377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11143757402896881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948231350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11309821158647537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65372160, "lr": 3.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948245954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10866034775972366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948260371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11174283921718597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65699840, "lr": 3.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948275498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11624515056610107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948290180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12413394451141357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66027520, "lr": 3.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948304799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11545594781637192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948319659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12752750515937805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66355200, "lr": 3.3745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948334425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1136818528175354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948334426, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784948334427, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784948421598, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7033082246780396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784948421599, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784948421599, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784948436466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12726053595542908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66682880, "lr": 3.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948451087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11462090909481049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.3995833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948465809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1160336434841156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67010560, "lr": 3.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948480720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11386723816394806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948495167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1204114556312561, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67338240, "lr": 3.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948509917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11917835474014282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948524983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12184139341115952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67665920, "lr": 3.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948539910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12076765298843384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948554673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1155557855963707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67993600, "lr": 3.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948569907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11352156102657318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948584713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11603126674890518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68321280, "lr": 3.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948599415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11703617870807648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948614099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12658073008060455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68648960, "lr": 3.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948629217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11687831580638885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948629217, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784948629218, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784948715612, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7209222912788391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784948715613, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784948715613, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784948730405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12301886081695557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68976640, "lr": 3.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948745492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1208970695734024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948760305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11379323154687881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69304320, "lr": 3.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948775263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11600662022829056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948789959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11011282354593277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69632000, "lr": 3.5412500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948804731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11673440784215927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948819626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11925292760133743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69959680, "lr": 3.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948834144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11284158378839493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948849089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10967691987752914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70287360, "lr": 3.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948863895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10930924117565155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948879091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1235555037856102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70615040, "lr": 3.5912500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948894420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10707709938287735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948909471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12104865908622742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70942720, "lr": 3.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948924781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12014001607894897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948924782, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784948924783, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784949010384, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7403948903083801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784949010385, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784949010385, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784949025450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11940175294876099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71270400, "lr": 3.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949040347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1191188171505928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949055210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11645843833684921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71598080, "lr": 3.6412499999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949070225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11752833425998688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949084790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039857566356659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71925760, "lr": 3.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949099419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10515772551298141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949113999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12106933444738388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72253440, "lr": 3.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949128476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11533806473016739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949142982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10491164028644562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72581120, "lr": 3.6912499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949157496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10330644994974136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949171826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10755132883787155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72908800, "lr": 3.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949186354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1132061555981636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949200925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11391036957502365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73236480, "lr": 3.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949215342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11702322959899902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949215381, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784949215381, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784949298244, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7518444061279297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784949298245, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784949298245, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784949312665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10527181625366211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73564160, "lr": 3.7412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949327515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10760879516601562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949342497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11651502549648285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73891840, "lr": 3.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949357198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11225991696119308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949372060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10859246551990509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74219520, "lr": 3.7745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949386933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11640329658985138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949401602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11371051520109177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74547200, "lr": 3.7912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949416180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11442621052265167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949430903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11720287799835205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74874880, "lr": 3.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949445367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11219248175621033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949459783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.100554920732975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75202560, "lr": 3.8245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949474305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11457717418670654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949488659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10897984355688095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75530240, "lr": 3.8412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949503188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10640757530927658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949503189, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784949503189, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784949586816, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7571488618850708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784949586818, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784949586818, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784949601102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10836154967546463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75857920, "lr": 3.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949615773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11430785804986954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949630298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09742151945829391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76185600, "lr": 3.8745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949644852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10466013848781586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949659547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11370434612035751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76513280, "lr": 3.8912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949674089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10755450278520584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949688977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08868443965911865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76840960, "lr": 3.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949703696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10929553955793381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949718428, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1027444377541542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77168640, "lr": 3.9245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949733264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11751611530780792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949748313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10806339979171753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77496320, "lr": 3.9412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949763085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10070987790822983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949778423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10126803070306778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77824000, "lr": 3.9579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949793558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10778351128101349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949793559, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784949793559, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784949877089, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7598263025283813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784949877089, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784949877090, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784949891999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11708478629589081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78151680, "lr": 3.9745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949906812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1126629188656807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949921025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10418039560317993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78479360, "lr": 3.9912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949936074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1077382043004036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949950509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09984703361988068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78807040, "lr": 4.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949965105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10448721796274185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949979572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723063349723816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79134720, "lr": 4.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949994287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10949819535017014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950008736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09775824099779129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79462400, "lr": 4.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950023426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10901482403278351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950037940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10503822565078735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79790080, "lr": 4.0579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950052481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1086205244064331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950067116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10893835872411728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80117760, "lr": 4.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950081903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.113210529088974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950081904, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784950081904, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784950166267, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.760788083076477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784950166268, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784950166268, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} diff --git a/recommendation_v4/rcp_logs/gbs_8192/seed79275716.log b/recommendation_v4/rcp_logs/gbs_8192/seed79275716.log new file mode 100644 index 000000000..d6af0954f --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_8192/seed79275716.log @@ -0,0 +1,662 @@ +:::MLLOG {"namespace": "", "time_ms": 1784939249673, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784939249673, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784939271351, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784939271358, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784939271358, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784939271358, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784939271358, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784939271358, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 8192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784939271358, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784939271358, "event_type": "POINT_IN_TIME", "key": "seed", "value": 79275716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784939271359, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784939271359, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784939271359, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784939271359, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784939271498, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784939271498, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784940213661, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784940213661, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784940214093, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784940333193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384589970111847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840, "lr": 7.916666666666666e-10}} +:::MLLOG {"namespace": "", "time_ms": 1784940345653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13838879764080048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.6249999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940358548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13802246749401093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 491520, "lr": 2.458333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940371760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13874493539333344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940384891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138676717877388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 819200, "lr": 4.125e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940398255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13875487446784973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940411452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13855965435504913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1146880, "lr": 5.7916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940425725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382516622543335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.625e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940439088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386032998561859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1474560, "lr": 7.458333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940452247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13859611749649048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.291666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940466063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13872095942497253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1802240, "lr": 9.124999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940479680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13867543637752533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940493368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385490745306015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2129920, "lr": 1.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940506853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138570174574852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940506853, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784940506854, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784940591537, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5028355121612549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784940591539, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784940591539, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784940605183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13858914375305176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2457600, "lr": 1.2458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940618766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13861636817455292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940632555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13829097151756287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2785280, "lr": 1.4124999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940645816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13851116597652435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4958333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940659428, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13831153512001038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3112960, "lr": 1.5791666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940673574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13846313953399658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940687039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13834866881370544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3440640, "lr": 1.7458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940700839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138217955827713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940715140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13823214173316956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3768320, "lr": 1.9124999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940728763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13828836381435394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9958333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940742903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13845321536064148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4096000, "lr": 2.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940757110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13863115012645721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.1624999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940771007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13818883895874023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4423680, "lr": 2.2458333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940785417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13853207230567932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3291666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940785418, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784940785418, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784940872642, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5030320882797241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784940872643, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784940872643, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784940886525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13810506463050842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4751360, "lr": 2.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940900325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13845384120941162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940915002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374104619026184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5079040, "lr": 2.5791666666666668e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940929451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13723191618919373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940943298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13805048167705536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5406720, "lr": 2.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940957249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13794691860675812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940971145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376354694366455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5734400, "lr": 2.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940985409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13730959594249725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940999169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13682641088962555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6062080, "lr": 3.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941012839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13733188807964325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941026790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13734084367752075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6389760, "lr": 3.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941040421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13759243488311768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941055133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13772623240947723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6717440, "lr": 3.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941069223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375669538974762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941069224, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784941069224, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784941154052, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5032646656036377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784941154053, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784941154053, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784941167806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13681137561798096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7045120, "lr": 3.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941181920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13772065937519073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941195946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13677994906902313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7372800, "lr": 3.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941210421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13756728172302246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.829166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941224194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373857706785202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7700480, "lr": 3.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941238156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13764873147010803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941252121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366317719221115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8028160, "lr": 4.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941265933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13754764199256897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941280421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13690176606178284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8355840, "lr": 4.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941294905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860411942005157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3291666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941309230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363380253314972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8683520, "lr": 4.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941323368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13802608847618103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.4958333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941337564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13531439006328583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9011200, "lr": 4.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941351943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13651034235954285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.6624999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941351944, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784941351945, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784941436612, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5036002397537231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784941436613, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784941436613, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784941450297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13606128096580505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9338880, "lr": 4.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941464007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13549883663654327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941477639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1372385323047638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9666560, "lr": 4.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941491554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13444137573242188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941505530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13658683001995087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9994240, "lr": 5.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941519597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13344912230968475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1624999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941533631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13545337319374084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10321920, "lr": 5.2458333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941547471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13545702397823334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941561563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13766984641551971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10649600, "lr": 5.4124999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941575281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13505448400974274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941589181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14724452793598175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10977280, "lr": 5.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941603420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13540728390216827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941617388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337776631116867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11304960, "lr": 5.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941631293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13772110641002655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941631294, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784941631294, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784941715490, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5039758682250977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784941715492, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784941715492, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784941729290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135182186961174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11632640, "lr": 5.9124999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941743081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13948921859264374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941757040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359054446220398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11960320, "lr": 6.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941770960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332324743270874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941784894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13487255573272705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12288000, "lr": 6.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941799071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13909024000167847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.329166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941815770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13333703577518463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12615680, "lr": 6.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941829820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13276197016239166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941843930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13308610022068024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12943360, "lr": 6.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941857711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13144132494926453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941871719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13094298541545868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13271040, "lr": 6.745833333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941885442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13484132289886475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941899298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13164958357810974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13598720, "lr": 6.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941913612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13898012042045593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941913612, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784941913613, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784941997527, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5044925212860107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784941997528, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784941997528, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784942011538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13361403346061707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13926400, "lr": 7.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942027956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13313068449497223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942042276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13575048744678497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14254080, "lr": 7.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942056343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13490363955497742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942070431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374143660068512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14581760, "lr": 7.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942084639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1276223361492157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942098506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13536088168621063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14909440, "lr": 7.579166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942112658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12913143634796143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942126918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13326281309127808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15237120, "lr": 7.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942141452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13253575563430786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.829166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942155642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13355928659439087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15564800, "lr": 7.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942170239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13428160548210144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.995833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942185034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13530606031417847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15892480, "lr": 8.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942199289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13332223892211914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942199289, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784942199289, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784942287470, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5053205490112305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784942287471, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784942287471, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784942301942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13238157331943512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16220160, "lr": 8.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942316218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13367578387260437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942330721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13555413484573364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16547840, "lr": 8.412500000000001e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942345496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135372593998909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942360085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1396757960319519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16875520, "lr": 8.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942375686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332206279039383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.662499999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942391171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13534988462924957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17203200, "lr": 8.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942405900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13342468440532684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942420373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13305146992206573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17530880, "lr": 8.912499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942435124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318422257900238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942449589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13564135134220123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17858560, "lr": 9.079166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942464665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336050182580948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.162499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942479356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13352692127227783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18186240, "lr": 9.245833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942493752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13046236336231232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942493753, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784942493753, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784942580083, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5064427852630615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784942580084, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784942580084, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784942595159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380176842212677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18513920, "lr": 9.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942609582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13492722809314728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.495833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942624121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13533683121204376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18841600, "lr": 9.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942638657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13127614557743073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.662499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942653045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13355574011802673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19169280, "lr": 9.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942667530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13399839401245117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942682252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13710105419158936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19496960, "lr": 9.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942696760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13387475907802582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942711372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13266240060329437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19824640, "lr": 1.0079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942725898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12963411211967468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0162499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942740058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12744566798210144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20152320, "lr": 1.0245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942754362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330169141292572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942768738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12947812676429749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20480000, "lr": 1.0412499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942782829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12467687577009201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942782829, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784942782830, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784942870920, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5076387524604797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784942870921, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784942870921, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784942885397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13100849092006683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20807680, "lr": 1.0579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942899704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13599468767642975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942914319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13317906856536865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21135360, "lr": 1.0745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942928949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13033026456832886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942943174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1282762587070465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21463040, "lr": 1.0912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942957718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12640240788459778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942972481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289229393005371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21790720, "lr": 1.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942986680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13212737441062927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943001007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352493166923523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22118400, "lr": 1.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943015325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1248716488480568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943029480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12832552194595337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22446080, "lr": 1.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943043620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343757063150406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943058136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13081835210323334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22773760, "lr": 1.1579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943072903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12922723591327667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943072904, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784943072904, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784943160641, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5091302990913391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784943160642, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784943160642, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784943175357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13051624596118927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23101440, "lr": 1.1745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943189851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12906013429164886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943203871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13309800624847412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23429120, "lr": 1.1912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943218600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12377951294183731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943233192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295517385005951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23756800, "lr": 1.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943247433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12575110793113708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943261603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298164427280426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24084480, "lr": 1.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943275944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12179435789585114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943290055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13596394658088684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24412160, "lr": 1.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943304649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13724344968795776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943318981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12354634702205658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24739840, "lr": 1.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943333750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12318979948759079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943348398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12551932036876678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25067520, "lr": 1.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943362690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1291576474905014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943362691, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784943362691, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784943449734, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5108367204666138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784943449735, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784943449736, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784943465736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364586353302002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25395200, "lr": 1.2912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943480412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.136612206697464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943494376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13576073944568634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25722880, "lr": 1.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943508320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13996271789073944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943522513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13665401935577393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26050560, "lr": 1.3245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943536818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382865607738495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943550926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1273144632577896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26378240, "lr": 1.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943565154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12606240808963776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943579307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14227518439292908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26705920, "lr": 1.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943593938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14113564789295197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943608529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12727729976177216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27033600, "lr": 1.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943622948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12833034992218018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943637293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13250784575939178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27361280, "lr": 1.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943651003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13150382041931152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943651004, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784943651004, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784943735948, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5128982663154602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784943735949, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784943735949, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784943751221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14137209951877594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27688960, "lr": 1.4079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943766212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14223109185695648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943780787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13502879440784454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28016640, "lr": 1.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943795691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1395556777715683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943810722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14479726552963257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28344320, "lr": 1.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943825466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14097802340984344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943840575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12613138556480408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28672000, "lr": 1.4579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943855595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375386267900467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943870718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318052113056183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28999680, "lr": 1.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943885882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13602755963802338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943900876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13793987035751343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29327360, "lr": 1.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943915602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13287483155727386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4995833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943930712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13435012102127075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29655040, "lr": 1.5079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943945482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13635076582431793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943945483, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784943945483, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784944031929, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5150726437568665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784944031930, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784944031930, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784944046336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13835997879505157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29982720, "lr": 1.5245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944061079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13797283172607422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944075828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318012773990631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30310400, "lr": 1.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944090516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13286541402339935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944105060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13381525874137878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30638080, "lr": 1.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944120002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1396256536245346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944134881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13249988853931427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30965760, "lr": 1.5745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944149894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14113011956214905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944164216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14074121415615082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31293440, "lr": 1.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944182744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13505703210830688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944197581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12746065855026245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31621120, "lr": 1.6079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944212026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.129940003156662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944226719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13406094908714294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31948800, "lr": 1.6245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944241643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12767982482910156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944241644, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784944241644, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784944324695, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5177190899848938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784944324696, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784944324696, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784944339443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321127712726593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32276480, "lr": 1.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944354078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13691186904907227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944368907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13357096910476685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32604160, "lr": 1.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944383685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13191959261894226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944398618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13391011953353882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32931840, "lr": 1.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944413300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13334967195987701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944428500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13373778760433197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33259520, "lr": 1.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944443359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13275274634361267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944457766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.139910027384758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33587200, "lr": 1.7079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944472829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13178357481956482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944487481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12881991267204285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33914880, "lr": 1.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944502019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13080760836601257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944516531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13210564851760864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34242560, "lr": 1.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944531175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13587018847465515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7495833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944531176, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784944531176, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784944616697, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5211915969848633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784944616698, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784944616698, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784944631385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13332587480545044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34570240, "lr": 1.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944645905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12292779982089996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944660404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13043265044689178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34897920, "lr": 1.7745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944675073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303083300590515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944689569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13531696796417236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35225600, "lr": 1.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944704636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13530273735523224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944719303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13527308404445648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35553280, "lr": 1.8079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944734432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12395865470170975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8162500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944749172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1309892237186432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35880960, "lr": 1.8245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944763407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312272995710373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944778347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1302596479654312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36208640, "lr": 1.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944792862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13308760523796082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944807247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13091112673282623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36536320, "lr": 1.8579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944821353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1299605667591095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944821353, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784944821354, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784944907793, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5259149670600891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784944907794, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784944907794, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784944922182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13777679204940796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36864000, "lr": 1.8745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944936651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13245578110218048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944951042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13327381014823914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37191680, "lr": 1.8912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944965372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13034884631633759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944979791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1269235759973526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37519360, "lr": 1.9079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944994107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13202351331710815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945008945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354610174894333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37847040, "lr": 1.9245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945023770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327337622642517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945038628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13117243349552155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38174720, "lr": 1.9412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945053624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13894081115722656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945068535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133607417345047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38502400, "lr": 1.9579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945082749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1372377872467041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945099541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1319122612476349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38830080, "lr": 1.9745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945114030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331419050693512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945114031, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784945114031, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784945196805, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5322238206863403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784945196805, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784945196806, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784945211255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13105788826942444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39157760, "lr": 1.9912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945225488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349133402109146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945239654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13779981434345245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39485440, "lr": 2.0079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945253933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13318198919296265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945268066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1273028552532196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39813120, "lr": 2.0245833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945282243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1292535811662674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0329166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945296375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12582583725452423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40140800, "lr": 2.0412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945310422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12386692315340042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945324932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13596948981285095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40468480, "lr": 2.0579166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945339344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13711674511432648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945353804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13456317782402039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40796160, "lr": 2.0745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945368851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1268204152584076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0829166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945383448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13084052503108978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41123840, "lr": 2.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945397543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12340342253446579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945397544, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784945397544, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784945483268, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.540954053401947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784945483269, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784945483269, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784945498009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13226433098316193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41451520, "lr": 2.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945512462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326056867837906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945526925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335318386554718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41779200, "lr": 2.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945541465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13761144876480103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945555909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359250545501709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42106880, "lr": 2.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945570550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312418282032013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945585068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12699148058891296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42434560, "lr": 2.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945599601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13490253686904907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945613848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12461409717798233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42762240, "lr": 2.1745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945628585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301630586385727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945643176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13237299025058746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43089920, "lr": 2.1912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945657819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13398495316505432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945672409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14014899730682373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43417600, "lr": 2.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945686934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13623595237731934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945686934, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784945686935, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784945773331, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5536198019981384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784945773332, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784945773332, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784945788424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13654080033302307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43745280, "lr": 2.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945803140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13133756816387177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945817411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13215646147727966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44072960, "lr": 2.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945832047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13890017569065094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945847116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12981411814689636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44400640, "lr": 2.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945862237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13821111619472504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945876942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348700225353241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44728320, "lr": 2.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945891897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13717760145664215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945906685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13466227054595947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45056000, "lr": 2.2912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945921619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13045468926429749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945936534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14027011394500732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45383680, "lr": 2.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945951272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13464538753032684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945966164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12887316942214966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45711360, "lr": 2.3245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945981019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312436640262604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945981020, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784945981020, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784946066007, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5682582259178162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784946066008, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784946066008, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784946081192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14176234602928162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46039040, "lr": 2.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946096037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326344609260559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946111253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13341595232486725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46366720, "lr": 2.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946126110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13064800202846527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946141780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13522440195083618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46694400, "lr": 2.3745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946156902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13327041268348694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946171624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13374803960323334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47022080, "lr": 2.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946186386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354609578847885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946200985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13127972185611725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47349760, "lr": 2.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946215880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13240739703178406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946230751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13280631601810455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47677440, "lr": 2.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946245641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13703078031539917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946260241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13569779694080353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48005120, "lr": 2.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946275184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13731378316879272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.4495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946275184, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784946275185, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784946359929, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5837602615356445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784946359929, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784946359930, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784946374608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1314375400543213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48332800, "lr": 2.4579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946389688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13508403301239014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946404586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13416247069835663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48660480, "lr": 2.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946419964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13169914484024048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946434805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13702024519443512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48988160, "lr": 2.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946449579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12917961180210114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946465084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331995725631714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49315840, "lr": 2.507916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946479517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1233440712094307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.5162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946494137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13046760857105255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49643520, "lr": 2.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946508777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311265230178833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946523513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13097478449344635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49971200, "lr": 2.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946537974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310390681028366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.5495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946552447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12880928814411163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50298880, "lr": 2.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946566971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13055020570755005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.5662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946566972, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784946566973, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784946649275, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5961446762084961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784946649276, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784946649276, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784946663597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1219489574432373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50626560, "lr": 2.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946678069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12650297582149506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946692522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.128843754529953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50954240, "lr": 2.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946706995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13279326260089874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946721846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13121843338012695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51281920, "lr": 2.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946736457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1214793249964714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946751521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301001012325287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51609600, "lr": 2.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946766173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13194319605827332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946780826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13637301325798035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51937280, "lr": 2.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946795018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12805326282978058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.6495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946810258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324310600757599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52264960, "lr": 2.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946825065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1174347847700119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.6662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946839734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12090972065925598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52592640, "lr": 2.6745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946854598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12466387450695038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946854599, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784946854599, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784946940740, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6099399328231812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784946940741, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784946940741, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784946955294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12136825919151306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52920320, "lr": 2.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946969697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1272234171628952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.6995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946984341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12792085111141205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53248000, "lr": 2.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946998935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13125501573085785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947013472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12123528867959976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53575680, "lr": 2.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947028065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338879019021988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947042520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12944987416267395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53903360, "lr": 2.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947056934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12141864746809006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.7495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947072006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12693150341510773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54231040, "lr": 2.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947086919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12692146003246307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947101623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11813520640134811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54558720, "lr": 2.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947116776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1313779503107071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947131643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.126569926738739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54886400, "lr": 2.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947146733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12812970578670502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947146734, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784947146734, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784947230473, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6242638826370239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784947230474, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784947230474, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784947244648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12824563682079315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55214080, "lr": 2.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947259818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11872322857379913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947274445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1219017431139946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55541760, "lr": 2.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947289086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12122499942779541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947303437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12375984340906143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55869440, "lr": 2.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947317897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12556768953800201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947332415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1159924641251564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56197120, "lr": 2.8579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947346964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12250314652919769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947361746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12498848885297775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56524800, "lr": 2.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947376289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12439394742250443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947390572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11538880318403244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56852480, "lr": 2.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947405774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12472108751535416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947420340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10965210199356079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57180160, "lr": 2.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947435072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12401672452688217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947435072, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784947435073, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784947518671, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6381000280380249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784947518672, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784947518672, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784947533722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1167401671409607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57507840, "lr": 2.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947548647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11433231830596924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947563446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12830066680908203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57835520, "lr": 2.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947578594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12151102721691132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947593366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12402579933404922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58163200, "lr": 2.9579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947608100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11617665737867355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947622399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12001252174377441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58490880, "lr": 2.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947637552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12187697738409042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947652244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1210252195596695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58818560, "lr": 2.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947666895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11272695660591125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947681625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1215846836566925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59146240, "lr": 3.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947696145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12171158939599991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.0162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947710524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12547388672828674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59473920, "lr": 3.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947725221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11709696799516678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947725221, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784947725221, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784947808217, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.659315824508667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784947808218, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784947808218, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784947822826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12021778523921967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59801600, "lr": 3.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947837610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1118880957365036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947852172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11042608320713043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60129280, "lr": 3.0579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947866625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1136852577328682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.0662499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947881397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10926269739866257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60456960, "lr": 3.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947895978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12188941240310669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947910808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11590326577425003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60784640, "lr": 3.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947925989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12039399892091751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947940622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10360760241746902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61112320, "lr": 3.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947955716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11948325484991074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.1162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947970709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11349905282258987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61440000, "lr": 3.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947985534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11281759291887283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948000122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10897571593523026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61767680, "lr": 3.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948014747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11634759604930878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948014748, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784948014748, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784948100592, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6920140385627747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784948100593, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784948100593, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784948115358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12061599642038345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62095360, "lr": 3.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948130780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11872785538434982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.1662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948146018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12275169789791107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62423040, "lr": 3.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948160946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10612951964139938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948175909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11394484341144562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62750720, "lr": 3.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948190724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1181885227560997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948205950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11869572848081589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63078400, "lr": 3.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948221027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11922179907560349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948235875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10853414982557297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63406080, "lr": 3.2245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948250849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11821328848600388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948265884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11409124732017517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63733760, "lr": 3.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948280548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11646851897239685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948295295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11278320848941803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64061440, "lr": 3.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948310348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12488026916980743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948310348, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784948310349, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784948397052, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7236897349357605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784948397053, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784948397053, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784948412077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12312706559896469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64389120, "lr": 3.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948427100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11757326126098633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948441918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11546124517917633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64716800, "lr": 3.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948457054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12162691354751587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948471912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10825000703334808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65044480, "lr": 3.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948486799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11062552779912949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948501181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11277886480093002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65372160, "lr": 3.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948516232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10863063484430313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948531060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11075796186923981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65699840, "lr": 3.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948546425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11657357215881348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948561494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12307686358690262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66027520, "lr": 3.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948576413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11446765065193176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948591654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12574386596679688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66355200, "lr": 3.3745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948606818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11263883858919144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948606818, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784948606819, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784948694179, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.757096529006958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784948694181, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784948694181, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784948709233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12484127283096313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66682880, "lr": 3.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948723910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11313269287347794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.3995833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948738783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11485761404037476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67010560, "lr": 3.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948753921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11211565136909485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948768927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11784689873456955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67338240, "lr": 3.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948783977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1155950054526329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948799409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12123489379882812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67665920, "lr": 3.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948814729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11998263746500015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948829875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11394039541482925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67993600, "lr": 3.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948845598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11202368140220642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948860738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11479896306991577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68321280, "lr": 3.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948875837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11239942908287048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948890687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12354546785354614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68648960, "lr": 3.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948906151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11490156501531601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948906151, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784948906152, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784948997207, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7623125314712524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784948997208, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784948997208, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784949012319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12022266536951065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68976640, "lr": 3.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949027631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11714844405651093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949042638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1125977411866188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69304320, "lr": 3.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949058010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11314524710178375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949073083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10751591622829437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69632000, "lr": 3.5412500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949088031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11341077834367752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949103167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11698585748672485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69959680, "lr": 3.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949117993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10953201353549957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949133068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10871341824531555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70287360, "lr": 3.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949148043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732739418745041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949163360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12153635919094086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70615040, "lr": 3.5912500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949178802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10457732528448105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949194018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11951538920402527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70942720, "lr": 3.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949209545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11947057396173477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949209546, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784949209546, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784949297623, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7651932835578918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784949297624, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784949297624, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784949312913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11719481647014618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71270400, "lr": 3.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949327933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11590303480625153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949342974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11536448448896408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71598080, "lr": 3.6412499999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949358261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.112989142537117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949372949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10192976891994476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71925760, "lr": 3.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949387649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10093126446008682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949402473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11744915693998337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72253440, "lr": 3.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949417166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11478514224290848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949432066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10296855121850967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72581120, "lr": 3.6912499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949446785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10078190267086029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949461229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10594749450683594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72908800, "lr": 3.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949476140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11151231825351715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949490893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10971289873123169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73236480, "lr": 3.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949505752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11367005854845047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949505800, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784949505800, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784949592058, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7673619985580444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784949592059, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784949592059, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784949606781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10404139012098312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73564160, "lr": 3.7412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949621946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.105603888630867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949637382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1134587973356247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73891840, "lr": 3.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949652246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10962709039449692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949667547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10671421885490417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74219520, "lr": 3.7745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949682944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1134776696562767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949697926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1132398396730423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74547200, "lr": 3.7912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949712955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11081937700510025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949727983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11481109261512756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74874880, "lr": 3.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949742575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10988499224185944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949757169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0983113944530487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75202560, "lr": 3.8245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949771851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11265081912279129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949786390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1061093658208847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75530240, "lr": 3.8412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949801068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10491935163736343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949801069, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784949801069, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784949886222, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7688021659851074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784949886223, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784949886223, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784949900844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10519900918006897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75857920, "lr": 3.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949915528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1124427393078804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949930310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09566386044025421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76185600, "lr": 3.8745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949944924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10355310887098312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949959469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111878901720047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76513280, "lr": 3.8912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949973960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10456492751836777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949988556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08834969252347946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76840960, "lr": 3.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950002970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10854516178369522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950017612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09996579587459564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77168640, "lr": 3.9245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950032568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11731575429439545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950047636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072343960404396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77496320, "lr": 3.9412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950062462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09963557124137878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950077830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09840378910303116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77824000, "lr": 3.9579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950093004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10658617317676544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950093005, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784950093006, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} diff --git a/recommendation_v4/rcp_logs/gbs_8192/seed869764802.log b/recommendation_v4/rcp_logs/gbs_8192/seed869764802.log new file mode 100644 index 000000000..7995677c7 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_8192/seed869764802.log @@ -0,0 +1,1859 @@ +:::MLLOG {"namespace": "", "time_ms": 1785027754209, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1785027754209, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1785027774469, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1785027774470, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1785027774470, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1785027774470, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1785027774470, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1785027774470, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 8192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1785027774470, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1785027774470, "event_type": "POINT_IN_TIME", "key": "seed", "value": 869764802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1785027774470, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1785027774471, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1785027774471, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1785027774471, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1785027774471, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1785027774472, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1785028671186, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1785028671186, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1785028671541, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1785028788360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13898107409477234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840, "lr": 7.916666666666666e-10}} +:::MLLOG {"namespace": "", "time_ms": 1785028800588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13851670920848846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.6249999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028812930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13853199779987335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 491520, "lr": 2.458333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028825398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387976109981537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028838073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13885189592838287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 819200, "lr": 4.125e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028850540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13874991238117218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028863287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13886554539203644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1146880, "lr": 5.7916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028876076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138556569814682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.625e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028888952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13882212340831757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1474560, "lr": 7.458333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028901782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13869385421276093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.291666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028914732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13864915072917938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1802240, "lr": 9.124999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028927822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13859106600284576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028940892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385444849729538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2129920, "lr": 1.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785028953760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1394260972738266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785028953761, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1785028953761, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1785029038092, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49874141812324524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1785029038093, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1785029038093, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1785029050560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138471320271492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2457600, "lr": 1.2458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029063275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13882774114608765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029076588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13852284848690033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2785280, "lr": 1.4124999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029089500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13855212926864624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4958333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029102695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385209858417511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3112960, "lr": 1.5791666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029116292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13861756026744843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029129125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13855499029159546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3440640, "lr": 1.7458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029142227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13833901286125183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029155882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383262425661087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3768320, "lr": 1.9124999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029169224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384800225496292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9958333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029182836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383853405714035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4096000, "lr": 2.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029196292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860969245433807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.1624999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029209580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13826005160808563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4423680, "lr": 2.2458333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029223471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13876043260097504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3291666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029223471, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1785029223472, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1785029306262, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4989405572414398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1785029306263, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1785029306263, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1785029319795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381968855857849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4751360, "lr": 2.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029333430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13823062181472778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029347135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13768407702445984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5079040, "lr": 2.5791666666666668e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029361012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13715051114559174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029374298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13794060051441193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5406720, "lr": 2.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029387690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13816022872924805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029401122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13777537643909454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5734400, "lr": 2.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029415266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373656541109085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029428912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370459645986557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6062080, "lr": 3.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029442294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1372988522052765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029455920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373705118894577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6389760, "lr": 3.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029469286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13794144988059998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029483741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378132700920105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6717440, "lr": 3.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029497282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378411501646042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029497282, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1785029497283, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1785029580136, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4991629719734192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1785029580137, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1785029580137, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1785029593462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13712626695632935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7045120, "lr": 3.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029607064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13737013936042786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029620569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368318498134613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7372800, "lr": 3.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029634667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13735729455947876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.829166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029648186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374334692955017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7700480, "lr": 3.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029661888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13752532005310059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029675570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366948038339615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8028160, "lr": 4.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029689096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137497216463089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029703208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13693587481975555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8355840, "lr": 4.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029717043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13861161470413208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3291666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029730755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13650110363960266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8683520, "lr": 4.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029744282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13805820047855377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.4958333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029757968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13536369800567627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9011200, "lr": 4.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029772072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367400884628296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.6624999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029772072, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1785029772073, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1785029855801, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49948549270629883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1785029855802, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1785029855802, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1785029869226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362951695919037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9338880, "lr": 4.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029882744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13538837432861328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029896066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13716968894004822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9666560, "lr": 4.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029909724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13451749086380005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029923521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13659778237342834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9994240, "lr": 5.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029937129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13366007804870605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1624999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029950848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13562755286693573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10321920, "lr": 5.2458333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029964370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354185938835144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029978105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13792723417282104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10649600, "lr": 5.4124999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029991585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135382741689682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030005175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14739231765270233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10977280, "lr": 5.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030019062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13548900187015533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030032460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13397835195064545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11304960, "lr": 5.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030046169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379273235797882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030046170, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1785030046170, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1785030130391, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4998573958873749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1785030130392, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1785030130392, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1785030143869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13527265191078186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11632640, "lr": 5.9124999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030157344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13968053460121155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030171092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13600970804691315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11960320, "lr": 6.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030184848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13294439017772675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030198709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13494229316711426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12288000, "lr": 6.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030212877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13926194608211517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.329166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030226865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133244588971138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12615680, "lr": 6.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030241033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330481469631195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030254642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13317477703094482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12943360, "lr": 6.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030268115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13142292201519012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030281895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13113345205783844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13271040, "lr": 6.745833333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030295263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13506627082824707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030308862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13156355917453766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13598720, "lr": 6.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030322765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13879364728927612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030322765, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1785030322766, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1785030406187, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5003622770309448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1785030406188, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1785030406188, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1785030419986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338634043931961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13926400, "lr": 7.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030433626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13325437903404236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030447757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357172280550003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14254080, "lr": 7.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030461562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13471026718616486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030475375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137643963098526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14581760, "lr": 7.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030489294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1275871992111206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030502912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354709416627884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14909440, "lr": 7.579166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030516791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1292845904827118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030530781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13356344401836395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15237120, "lr": 7.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030545062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323835849761963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.829166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030559047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13364167511463165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15564800, "lr": 7.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030573168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13440929353237152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.995833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030587170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13537879288196564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15892480, "lr": 8.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030600992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13326020538806915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030600993, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785030600994, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785030686128, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5011882185935974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785030686129, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785030686129, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785030699994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13261064887046814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16220160, "lr": 8.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030713682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13370363414287567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030727478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13554790616035461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16547840, "lr": 8.412500000000001e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030741651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13523520529270172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030755630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1396695226430893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16875520, "lr": 8.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030769354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13314925134181976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.662499999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030784035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13557963073253632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17203200, "lr": 8.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030798124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13342292606830597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030812156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13327716290950775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17530880, "lr": 8.912499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030826508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317211538553238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030840566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13585814833641052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17858560, "lr": 9.079166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030855102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13378089666366577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.162499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030869289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13331890106201172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18186240, "lr": 9.245833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030883249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13061809539794922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030883250, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785030883250, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785030967365, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.502359926700592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785030967366, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785030967366, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785030982072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380193680524826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18513920, "lr": 9.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030995971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347745805978775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.495833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031009895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13524289429187775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18841600, "lr": 9.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031024091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13112135231494904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.662499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031037983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13362663984298706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19169280, "lr": 9.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031052003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1340661197900772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031066240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371760219335556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19496960, "lr": 9.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031080465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13395676016807556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031094850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13288359344005585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19824640, "lr": 1.0079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031108724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12952518463134766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0162499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031122251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12750637531280518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20152320, "lr": 1.0245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031136004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13282090425491333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031149938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12961708009243011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20480000, "lr": 1.0412499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031163518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12473078817129135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031163518, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1785031163519, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1785031246680, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5035377144813538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1785031246681, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1785031246681, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1785031260690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13110056519508362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20807680, "lr": 1.0579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031274516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13590505719184875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031288623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13328440487384796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21135360, "lr": 1.0745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031302675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13026708364486694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031316323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12829077243804932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21463040, "lr": 1.0912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031330233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12661612033843994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031344210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12900416553020477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21790720, "lr": 1.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031357738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321902722120285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031371303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352188140153885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22118400, "lr": 1.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031385013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12477665394544601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031398442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12856657803058624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22446080, "lr": 1.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031411910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13433955609798431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031425440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13098841905593872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22773760, "lr": 1.1579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031439313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12907272577285767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031439314, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1785031439314, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1785031521680, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.505046546459198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1785031521681, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1785031521681, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1785031535631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13043121993541718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23101440, "lr": 1.1745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031549298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1290569007396698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031562619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13336819410324097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23429120, "lr": 1.1912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031576961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12373275309801102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031591164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12954549491405487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23756800, "lr": 1.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031604852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1257096379995346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031618687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12989862263202667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24084480, "lr": 1.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031632647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12171173095703125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031646377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13574789464473724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24412160, "lr": 1.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031660560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13728168606758118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031674410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12346401065587997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24739840, "lr": 1.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031688627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12299896031618118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031702711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1254303902387619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25067520, "lr": 1.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031716662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12891677021980286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031716662, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1785031716662, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1785031799632, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.506938099861145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1785031799633, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1785031799633, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1785031813372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363847255706787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25395200, "lr": 1.2912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031827399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365797072649002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031840812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357940435409546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25722880, "lr": 1.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031854150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1398850977420807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031867618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13667358458042145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26050560, "lr": 1.3245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031881149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13833211362361908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031894497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12723608314990997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26378240, "lr": 1.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031907845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12606807053089142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031921186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14222481846809387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26705920, "lr": 1.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031935026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1410762518644333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031948882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12726807594299316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27033600, "lr": 1.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031962587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12848599255084991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031976135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13279594480991364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27361280, "lr": 1.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031989250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13146823644638062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031989250, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1785031989251, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1785032071995, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5090455412864685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1785032071996, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1785032071996, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1785032086664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14160659909248352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27688960, "lr": 1.4079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032100868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14239580929279327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032114751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352817565202713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28016640, "lr": 1.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032128855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13937541842460632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032143244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14471688866615295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28344320, "lr": 1.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032157155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14084112644195557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032171270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12568853795528412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28672000, "lr": 1.4579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032185296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376904547214508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032199477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13175557553768158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28999680, "lr": 1.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032213731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13604329526424408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032227906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137868732213974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29327360, "lr": 1.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032241725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13310232758522034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4995833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032256171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13434377312660217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29655040, "lr": 1.5079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032270381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363433599472046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032270382, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1785032270382, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1785032352131, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.511272668838501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1785032352132, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1785032352132, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1785032365796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13859963417053223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29982720, "lr": 1.5245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032380001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13814221322536469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032394221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131597101688385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30310400, "lr": 1.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032408336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13308095932006836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032422386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13376227021217346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30638080, "lr": 1.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032436697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13985638320446014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032450985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13291029632091522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30965760, "lr": 1.5745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032465307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14097607135772705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032479052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14078818261623383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31293440, "lr": 1.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032493471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13494953513145447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032507578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1276204138994217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31621120, "lr": 1.6079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032521497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12986157834529877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032535308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13401655852794647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31948800, "lr": 1.6245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032549291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12761567533016205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032549292, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1785032549292, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1785032631738, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5141140818595886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1785032631739, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1785032631739, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1785032645602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13225413858890533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32276480, "lr": 1.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032659353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13700087368488312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032673246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13351202011108398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32604160, "lr": 1.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032687157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13181568682193756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032701321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13393452763557434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32931840, "lr": 1.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032715348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13338357210159302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032729799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13382863998413086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33259520, "lr": 1.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032744090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13241194188594818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032757988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13991568982601166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33587200, "lr": 1.7079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032772543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13177715241909027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032786328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12870407104492188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33914880, "lr": 1.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032799817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13097204267978668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032813335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321365386247635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34242560, "lr": 1.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032827001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13538098335266113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7495833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032827002, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1785032827002, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1785032909136, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.51784747838974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1785032909137, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1785032909137, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1785032922736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13351058959960938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34570240, "lr": 1.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032936331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12295016646385193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032949960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13044534623622894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34897920, "lr": 1.7745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032963639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1302943378686905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032977212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13510408997535706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35225600, "lr": 1.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032991323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13551941514015198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033005009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13531766831874847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35553280, "lr": 1.8079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033019139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12393872439861298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8162500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033033042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13102102279663086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35880960, "lr": 1.8245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033046598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13113214075565338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033060960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301925778388977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36208640, "lr": 1.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033074996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13325785100460052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033089166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13087013363838196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36536320, "lr": 1.8579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033102969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13012462854385376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033102970, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1785033102970, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1785033184777, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5229683518409729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1785033184778, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1785033184778, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1785033198815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13776618242263794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36864000, "lr": 1.8745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033212787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132360577583313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033226659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335616260766983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37191680, "lr": 1.8912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033240607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13039442896842957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033254519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12672127783298492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37519360, "lr": 1.9079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033268358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322118192911148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033282608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13555727899074554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37847040, "lr": 1.9245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033296622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13279519975185394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033310752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13124214112758636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38174720, "lr": 1.9412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033325027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13885565102100372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033339199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338048279285431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38502400, "lr": 1.9579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033352868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13748043775558472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033367213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315261870622635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38830080, "lr": 1.9745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033381012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13315919041633606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033381013, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1785033381013, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1785033462589, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5298786163330078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1785033462591, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1785033462591, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1785033476244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.130939319729805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39157760, "lr": 1.9912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033489836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13484759628772736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033503496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13760262727737427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39485440, "lr": 2.0079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033517301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13309729099273682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033530943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1272576004266739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39813120, "lr": 2.0245833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033544651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12926635146141052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0329166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033558299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12584500014781952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40140800, "lr": 2.0412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033571819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1239573135972023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033585711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13623493909835815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40468480, "lr": 2.0579166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033599692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13704505562782288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033613573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13431251049041748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40796160, "lr": 2.0745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033628068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12679757177829742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0829166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033642047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13086266815662384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41123840, "lr": 2.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033655574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12338612973690033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033655574, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1785033655575, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1785033739613, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5392440557479858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1785033739614, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1785033739614, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1785033753876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322774440050125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41451520, "lr": 2.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033767856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13225281238555908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033782010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13344618678092957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41779200, "lr": 2.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033796173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377457231283188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033810187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13614621758460999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42106880, "lr": 2.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033824243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13110728561878204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033838282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1270628273487091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42434560, "lr": 2.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033852400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13477715849876404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033866332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12429249286651611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42762240, "lr": 2.1745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033880638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13005201518535614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033894782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13237686455249786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43089920, "lr": 2.1912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033909188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13374663889408112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033923556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1400359570980072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43417600, "lr": 2.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033937598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13601674139499664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033937598, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1785033937599, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1785034021138, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5526826977729797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1785034021139, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1785034021139, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1785034035814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13618646562099457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43745280, "lr": 2.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034050074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13101255893707275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034064113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13217957317829132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44072960, "lr": 2.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034078519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388949751853943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034092896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12984976172447205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44400640, "lr": 2.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034107629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13817612826824188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034121800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134648859500885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44728320, "lr": 2.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034136052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1369737982749939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034150057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13443537056446075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45056000, "lr": 2.2912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034164417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13044965267181396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034178733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14009904861450195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45383680, "lr": 2.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034193059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13442949950695038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034207198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12855258584022522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45711360, "lr": 2.3245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034221701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13308031857013702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034221701, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1785034221702, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1785034304616, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5675315260887146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1785034304617, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1785034304617, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1785034319321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14158780872821808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46039040, "lr": 2.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034333715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321834772825241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034348699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332286298274994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46366720, "lr": 2.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034363143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303015649318695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034377996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13516487181186676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46694400, "lr": 2.3745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034392695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13310500979423523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034407216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13339214026927948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47022080, "lr": 2.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034421699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13525016605854034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034436007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13079212605953217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47349760, "lr": 2.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034450594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321539580821991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034465078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13268084824085236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47677440, "lr": 2.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034479600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13668730854988098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034493882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13594378530979156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48005120, "lr": 2.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034508603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.136916846036911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.4495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034508604, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1785034508604, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1785034592429, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5827209949493408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1785034592430, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1785034592430, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1785034606695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13114377856254578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48332800, "lr": 2.4579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034621369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348288357257843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034635808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13396303355693817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48660480, "lr": 2.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034650687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1314782351255417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034665042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13652165234088898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48988160, "lr": 2.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034679332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12901028990745544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034694328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13328605890274048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49315840, "lr": 2.507916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034708388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12274497747421265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.5162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034722516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13001649081707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49643520, "lr": 2.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034736605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13078559935092926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034750607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13046371936798096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49971200, "lr": 2.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034764581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1306118667125702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.5495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034778590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12861885130405426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50298880, "lr": 2.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034792704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13027015328407288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.5662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034792705, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1785034792705, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1785034873709, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5937869548797607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1785034873710, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1785034873710, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1785034887804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12152598798274994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50626560, "lr": 2.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034901823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12583449482917786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034915778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12812523543834686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50954240, "lr": 2.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034929917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13280077278614044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034944400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13121534883975983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51281920, "lr": 2.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034958527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12114281952381134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034973059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12921574711799622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51609600, "lr": 2.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034987527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13119445741176605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035001778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13576669991016388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51937280, "lr": 2.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035015566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12715034186840057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.6495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035030191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322098970413208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52264960, "lr": 2.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035044388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11683426052331924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.6662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035058584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12006130069494247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52592640, "lr": 2.6745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035072837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12418857961893082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035072837, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1785035072838, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1785035154811, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6064016819000244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1785035154812, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1785035154812, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1785035168945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12065713852643967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52920320, "lr": 2.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035183084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1265815794467926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.6995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035197398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1271066963672638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53248000, "lr": 2.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035211534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13079816102981567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035225537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12055930495262146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53575680, "lr": 2.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035239677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13378246128559113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035253628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12800060212612152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53903360, "lr": 2.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035267680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12062317878007889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.7495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035281834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12601973116397858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54231040, "lr": 2.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035296105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1258808672428131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035310120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11778499186038971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54558720, "lr": 2.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035324635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13069500029087067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035338850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12595203518867493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54886400, "lr": 2.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035353062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1275722086429596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035353063, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1785035353063, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1785035436591, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6183522343635559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1785035436592, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1785035436592, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1785035450273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1274702101945877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55214080, "lr": 2.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035464960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11807235330343246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035479274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12086319178342819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55541760, "lr": 2.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035493783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12004774063825607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035508065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12274733185768127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55869440, "lr": 2.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035522226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12512077391147614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035536295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11555708944797516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56197120, "lr": 2.8579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035550609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1219552755355835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035564776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12396473437547684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56524800, "lr": 2.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035578858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1231248676776886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035592789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11453534662723541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56852480, "lr": 2.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035606922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12355378270149231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035621045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10957508534193039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57180160, "lr": 2.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035635037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12382210046052933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035635037, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1785035635038, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1785035719353, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6289666891098022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1785035719354, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1785035719354, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1785035733617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11582434177398682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57507840, "lr": 2.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035747919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11303749680519104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035762071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1269591748714447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57835520, "lr": 2.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035776875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12107805907726288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035791127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12289875745773315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58163200, "lr": 2.9579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035805321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11510120332241058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035819141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11939450353384018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58490880, "lr": 2.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035833718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12158135324716568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035847769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12019775807857513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58818560, "lr": 2.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035861952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1119416207075119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035876003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12077374011278152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59146240, "lr": 3.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035889971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12154814600944519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.0162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035904019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12477097660303116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59473920, "lr": 3.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035918226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11618328094482422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035918226, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1785035918227, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1785036002463, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6431187987327576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1785036002463, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1785036002464, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1785036016829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11952906847000122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59801600, "lr": 3.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036030728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11088480055332184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036044559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10934103280305862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60129280, "lr": 3.0579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036058585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1133686900138855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.0662499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036072591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10893559455871582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60456960, "lr": 3.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036086659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12189705669879913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036100999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11625505983829498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60784640, "lr": 3.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036115653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12042949348688126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036129748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10284296423196793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61112320, "lr": 3.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036144337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11977023631334305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.1162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036158868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11258519440889359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61440000, "lr": 3.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036173255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11313463002443314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036187515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10907308012247086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61767680, "lr": 3.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036202098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11680957674980164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036202099, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1785036202099, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1785036286442, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6625610589981079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1785036286443, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1785036286443, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1785036300644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1210140585899353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62095360, "lr": 3.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036314917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11913342773914337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.1662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036329372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12297184020280838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62423040, "lr": 3.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036343767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10683710128068924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036358197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11414726078510284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62750720, "lr": 3.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036372540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11779043078422546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036387193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11896362155675888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63078400, "lr": 3.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036401798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11898581683635712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036416127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082075983285904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63406080, "lr": 3.2245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036430496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11858968436717987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036444806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11498495191335678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63733760, "lr": 3.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036458873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11676366627216339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036472936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11300789564847946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64061440, "lr": 3.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036487513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12512464821338654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036487514, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1785036487514, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1785036572262, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6836523413658142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1785036572263, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1785036572263, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1785036586853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12365086376667023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64389120, "lr": 3.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036601465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11727097630500793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036615907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11643064022064209, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64716800, "lr": 3.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036630819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12141193449497223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036645556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1098303347826004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65044480, "lr": 3.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036660302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11116620898246765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036674385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11374668776988983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65372160, "lr": 3.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036689271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10948322713375092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036703661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11102768778800964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65699840, "lr": 3.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036718621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11561720818281174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036733095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12384845316410065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66027520, "lr": 3.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036747460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11477607488632202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036762137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12800820171833038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66355200, "lr": 3.3745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036776795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11388816684484482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036776796, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1785036776796, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1785036859466, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7200950384140015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1785036859467, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1785036859467, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1785036874115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1252536177635193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66682880, "lr": 3.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036888567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11389647424221039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.3995833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036903037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1161603182554245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67010560, "lr": 3.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036917636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11329453438520432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036932084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1192837581038475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67338240, "lr": 3.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036946709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11733483523130417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036961784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12156344950199127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67665920, "lr": 3.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036976739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12127597630023956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036991366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11543899029493332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67993600, "lr": 3.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037006572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11282321810722351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037021175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11626383662223816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68321280, "lr": 3.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037035699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1150059923529625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037050234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12567493319511414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68648960, "lr": 3.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037065416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11679857224225998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037065417, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1785037065417, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1785037147933, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7388628721237183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1785037147934, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1785037147934, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1785037162741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1227724626660347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68976640, "lr": 3.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037177631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11848834902048111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037192432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11405222862958908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69304320, "lr": 3.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037207484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.115668885409832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037222280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10863891243934631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69632000, "lr": 3.5412500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037237141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11607526242733002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037251848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1183989867568016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69959680, "lr": 3.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037266305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1117604523897171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037281037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11014046519994736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70287360, "lr": 3.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037295465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085827425122261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037310382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1229117140173912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70615040, "lr": 3.5912500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037325239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10740908235311508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037339942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12075953930616379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70942720, "lr": 3.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037355098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11953001469373703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037355099, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1785037355099, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1785037439261, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7505598068237305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1785037439262, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1785037439262, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1785037454106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11935276538133621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71270400, "lr": 3.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037468561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1184198185801506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037483103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11604278534650803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71598080, "lr": 3.6412499999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037498080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1168144941329956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037512576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10398884862661362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71925760, "lr": 3.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037527089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10413704812526703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037541463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12044068425893784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72253440, "lr": 3.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037555681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1147673949599266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037569920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10404534637928009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72581120, "lr": 3.6912499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037584125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1033811867237091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037598158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10694815218448639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72908800, "lr": 3.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037612553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.113070048391819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037626832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11259378492832184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73236480, "lr": 3.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037641210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11568903923034668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037641247, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1785037641247, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1785037726120, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7586737275123596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1785037726121, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1785037726121, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1785037740547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10541784763336182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73564160, "lr": 3.7412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037755210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10753197968006134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037770072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11426887661218643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73891840, "lr": 3.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037784475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11115315556526184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037799151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10844042152166367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74219520, "lr": 3.7745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037814093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11726313084363937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037828761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11389338970184326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74547200, "lr": 3.7912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037843208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11354943364858627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037857954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11670592427253723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74874880, "lr": 3.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037872285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11163027584552765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037886565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10045833885669708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75202560, "lr": 3.8245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037901087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11436381191015244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037915209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10805051773786545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75530240, "lr": 3.8412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037929568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10557027161121368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037929569, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1785037929569, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1785038012534, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7637120485305786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1785038012535, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1785038012535, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1785038026855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793829709291458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75857920, "lr": 3.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038041247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11359303444623947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038055750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09706210345029831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76185600, "lr": 3.8745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038070190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10461191087961197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038084697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11328978091478348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76513280, "lr": 3.8912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038098969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10563313215970993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038113550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08823878318071365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76840960, "lr": 3.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038127721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11005397140979767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038142070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10209009051322937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77168640, "lr": 3.9245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038156649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11787804961204529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038171560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10827374458312988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77496320, "lr": 3.9412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038186094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10040409862995148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038201212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10020065307617188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77824000, "lr": 3.9579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038216235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10712314397096634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038216235, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1785038216236, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1785038300111, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7669221758842468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1785038300112, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1785038300112, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1785038314683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11520344763994217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78151680, "lr": 3.9745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038329436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11216192692518234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038343576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1043621227145195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78479360, "lr": 3.9912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038359184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10522613674402237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038373873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09949195384979248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78807040, "lr": 4.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038388730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10346563905477524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038403497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071104034781456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79134720, "lr": 4.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038418411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10768695175647736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038433008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09817367047071457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79462400, "lr": 4.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038447975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10877540707588196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038462658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10477922856807709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79790080, "lr": 4.0579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038477239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1066729873418808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038491755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10874922573566437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80117760, "lr": 4.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038506564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11290159821510315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038506565, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1785038506566, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1785038589633, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7687166929244995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1785038589634, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1785038589634, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1785038604395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10938632488250732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80445440, "lr": 4.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038618623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09841214865446091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038632859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10312470048666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80773120, "lr": 4.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038647397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10572855174541473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038662084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11111455410718918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81100800, "lr": 4.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038676754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11241775006055832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038691140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1001686081290245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81428480, "lr": 4.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038705600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10447492450475693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038720147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10840605199337006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81756160, "lr": 4.1579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038734412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10497673600912094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038748785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11345429718494415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82083840, "lr": 4.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038762758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09524079412221909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038777886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09817755222320557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82411520, "lr": 4.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038792554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10733159631490707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038792555, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1785038792555, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1785038875469, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7698713541030884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1785038875470, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1785038875470, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1785038889658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09092394262552261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82739200, "lr": 4.2079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038904318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.103541798889637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038918886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11019019782543182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83066880, "lr": 4.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038933804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09815351665019989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038948576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11051246523857117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83394560, "lr": 4.2412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038963366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11098945885896683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038978193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.097931407392025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83722240, "lr": 4.2579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038992769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11069130897521973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039007483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10507486760616302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84049920, "lr": 4.2745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039022052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10432429611682892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039036815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10721137374639511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84377600, "lr": 4.2912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039051236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10439816117286682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039065484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09770926088094711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84705280, "lr": 4.3079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039080094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10600882768630981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039080094, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1785039080095, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1785039163708, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7707446813583374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1785039163709, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1785039163709, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1785039178747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10428828746080399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85032960, "lr": 4.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039193788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1040724590420723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039208545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10575000196695328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85360640, "lr": 4.3412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039223399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1034262627363205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039238420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09782432019710541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85688320, "lr": 4.3579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039253056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09978671371936798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039267883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10469523072242737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86016000, "lr": 4.3745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039281997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0963578000664711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039296827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10803906619548798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86343680, "lr": 4.3912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039311287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1079862117767334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039325699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11684411764144897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86671360, "lr": 4.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039340210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11420707404613495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039354520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11633242666721344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86999040, "lr": 4.4245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039368931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11494087427854538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039368932, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1785039368932, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1785039452301, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7713516354560852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1785039452302, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1785039452302, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1785039466468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10055563598871231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87326720, "lr": 4.4412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039480895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11578787863254547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039495018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.108212411403656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87654400, "lr": 4.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039509409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10161294043064117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039523542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09189607948064804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87982080, "lr": 4.4745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039538068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09851089864969254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039552240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1037716418504715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88309760, "lr": 4.4912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039566626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1077549010515213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039580815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10745874047279358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88637440, "lr": 4.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039595080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10009221732616425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039609758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1131066381931305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88965120, "lr": 4.5245833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039624348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09920955449342728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039639027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1096871867775917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89292800, "lr": 4.5412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039653400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11183418333530426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.5495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039653401, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1785039653401, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1785039737997, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7718645930290222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1785039737998, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1785039737998, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1785039752725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10770171880722046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89620480, "lr": 4.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039767340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10160303860902786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039781748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11046899110078812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89948160, "lr": 4.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039796354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.091947041451931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039810732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11448036134243011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90275840, "lr": 4.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039825056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10691892355680466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039840416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11098015308380127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90603520, "lr": 4.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039855207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11106141656637192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039870276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12190274149179459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90931200, "lr": 4.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039885196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1043306291103363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039900119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10962624847888947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91258880, "lr": 4.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039915164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12339212745428085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039930059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09922529757022858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91586560, "lr": 4.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039945101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11050239205360413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039945101, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1785039945102, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1785040029341, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7720608115196228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1785040029342, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1785040029343, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1785040044121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10364358872175217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91914240, "lr": 4.6745833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040059094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11886557191610336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040073918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10965602099895477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92241920, "lr": 4.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040089170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10241957753896713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040104126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09621185064315796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92569600, "lr": 4.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040119065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11243341863155365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040134407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10661821812391281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92897280, "lr": 4.7245833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040149683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10541021823883057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040165074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11554179340600967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93224960, "lr": 4.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040179949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09991499781608582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040195215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12512174248695374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93552640, "lr": 4.7579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040210221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10754769295454025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040224703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11401233077049255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93880320, "lr": 4.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040239263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10751233994960785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040239264, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1785040239264, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1785040322945, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7724158763885498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1785040322946, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1785040322946, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1785040337859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11352979391813278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94208000, "lr": 4.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040352815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11540025472640991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040367797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11075736582279205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94535680, "lr": 4.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040382936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11796253174543381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040397858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11442281305789948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94863360, "lr": 4.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040412931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11890258640050888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040427933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12722250819206238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95191040, "lr": 4.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040442525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12131829559803009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040457122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1159488707780838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95518720, "lr": 4.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040472058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11033650487661362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040486640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10867641121149063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95846400, "lr": 4.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040501577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1079489141702652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040516452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11459318548440933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96174080, "lr": 4.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040531380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11409846693277359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040531381, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1785040531381, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1785040613686, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7726709246635437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1785040613687, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1785040613687, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1785040628666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11992654949426651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96501760, "lr": 4.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040643611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10680367052555084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040658356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1075059026479721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96829440, "lr": 4.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040673635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10851582139730453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040688610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11087138950824738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97157120, "lr": 4.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040703335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10935825109481812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040717969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11362570524215698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97484800, "lr": 4.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040732507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085314229130745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040747052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10509642958641052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97812480, "lr": 4.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040761112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10050130635499954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040775632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10441181063652039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98140160, "lr": 4.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040789972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09565611928701401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040804285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10380551964044571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98467840, "lr": 5.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040818378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10040654987096786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040818379, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1785040818379, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1785040899265, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7728919982910156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1785040899266, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1785040899266, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1785040913302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10913611203432083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98795520, "lr": 5.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040927576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10093310475349426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040941759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09637162834405899, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99123200, "lr": 5.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040955934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10824834555387497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040970006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11498510837554932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99450880, "lr": 5.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040984071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10500358790159225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040998134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10393832623958588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99778560, "lr": 5.074583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041012460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10427729040384293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041026659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10126793384552002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100106240, "lr": 5.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041041061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11964753270149231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041055570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11688214540481567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100433920, "lr": 5.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041069832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10090741515159607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041084588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10002478212118149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100761600, "lr": 5.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041098830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12057120352983475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041098830, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1785041098831, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1785041180802, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773169994354248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1785041180803, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1785041180803, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1785041195484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1106390431523323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101089280, "lr": 5.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041210053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0965723991394043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041224620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10462447255849838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101416960, "lr": 5.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041239265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10304269939661026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041253530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10814250260591507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101744640, "lr": 5.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041268233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10429040342569351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.182916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041282613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10609500110149384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102072320, "lr": 5.191249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041296980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11241357028484344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041311684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.105167917907238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102400000, "lr": 5.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041326272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080334484577179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041340784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10570752620697021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102727680, "lr": 5.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041355146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11454000324010849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041369425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1068277508020401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103055360, "lr": 5.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041383911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10245957225561142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041383912, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1785041383912, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1785041466914, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7733433842658997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1785041466915, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1785041466915, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1785041481323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09952801465988159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103383040, "lr": 5.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041495857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09312146157026291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041510088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09609147161245346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103710720, "lr": 5.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041524579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09671977162361145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.282916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041538883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0965789407491684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104038400, "lr": 5.291249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041553396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09441717714071274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041567506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10035320371389389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104366080, "lr": 5.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041581719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0968392863869667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041596273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10285723209381104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104693760, "lr": 5.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041610709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10777643322944641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041625451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10465335100889206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105021440, "lr": 5.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041639740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10324513167142868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041653970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11409050226211548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105349120, "lr": 5.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041668768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11095251142978668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041668768, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1785041668768, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1785041751828, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7734858393669128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1785041751829, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1785041751829, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1785041766304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09801874309778214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105676800, "lr": 5.374583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041780711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10683964937925339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.382916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041795369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10954195261001587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106004480, "lr": 5.391249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041809586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10132191330194473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041824186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09924669563770294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106332160, "lr": 5.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041838791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10748124867677689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041853002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11046445369720459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106659840, "lr": 5.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041867304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10593511909246445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041881912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10268934816122055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106987520, "lr": 5.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041896055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10079678893089294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041910187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09995676577091217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107315200, "lr": 5.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041924312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1028614491224289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041938524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10907084494829178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107642880, "lr": 5.474583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041952610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11481061577796936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.482916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041952611, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1785041952612, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1785042035116, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773608922958374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1785042035117, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1785042035117, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1785042049369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10956225544214249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107970560, "lr": 5.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042063699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10667037963867188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042077953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09087102115154266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108298240, "lr": 5.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042092076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10102563351392746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042106455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10645650327205658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108625920, "lr": 5.524583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042120572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10736025124788284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042134669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1103699579834938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108953600, "lr": 5.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042148918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10228993743658066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042163359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10057242214679718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109281280, "lr": 5.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042177721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10384442657232285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042192325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10243874788284302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109608960, "lr": 5.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042206618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10720664262771606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042221127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10424961149692535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109936640, "lr": 5.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042235747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.108835868537426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042235748, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1785042235748, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1785042318631, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7737111449241638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1785042318632, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1785042318633, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1785042333110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10328163206577301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110264320, "lr": 5.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042347597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10285752266645432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042361699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10355991870164871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110592000, "lr": 5.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042375639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10260442644357681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042390657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725440829992294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110919680, "lr": 5.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042404935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10070332139730453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042419384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09810589253902435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111247360, "lr": 5.657916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042434044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10154598951339722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.666250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042448688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1119617372751236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111575040, "lr": 5.674583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042463262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10526324808597565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042477707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10395409166812897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111902720, "lr": 5.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042492115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10212095081806183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042506614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10362622141838074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112230400, "lr": 5.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042521025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10653871297836304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042521026, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1785042521026, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1785042604714, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738252282142639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1785042604715, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1785042604715, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1785042619183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10855220258235931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112558080, "lr": 5.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042633908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10169089585542679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042648409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10344725847244263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112885760, "lr": 5.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042663064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10100530833005905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042677847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10441668331623077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113213440, "lr": 5.757916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042692140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11320183426141739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.766250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042706722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.101235531270504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113541120, "lr": 5.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042721022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11033476144075394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042735755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11254958808422089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113868800, "lr": 5.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042750842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10692405700683594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042765522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1031406968832016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114196480, "lr": 5.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042780321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10557381063699722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042795312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1052471324801445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114524160, "lr": 5.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042810095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09806354343891144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042810095, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1785042810096, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1785042894983, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738267183303833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1785042894984, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1785042894984, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1785042909611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10102642327547073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114851840, "lr": 5.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042924249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09898138791322708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042938869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10635606199502945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115179520, "lr": 5.857916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042953059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10062753409147263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042968149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10842334479093552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115507200, "lr": 5.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042982401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1093006283044815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042996885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10780488699674606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115834880, "lr": 5.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043011505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1155586689710617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043025807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11373691260814667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116162560, "lr": 5.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043039976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10044945776462555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043054021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10368963330984116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116490240, "lr": 5.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043068242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10301890969276428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043082468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1053866595029831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116817920, "lr": 5.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043096638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10729727894067764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043096639, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1785043096639, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1785043179486, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7739631533622742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1785043179487, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1785043179487, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1785043193825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10133937746286392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117145600, "lr": 5.957916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043208044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10001853853464127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043222603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1092868372797966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117473280, "lr": 5.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043236946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10453467071056366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043251182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09393051266670227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117800960, "lr": 5.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043266023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11297674477100372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043280324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1011042594909668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118128640, "lr": 6.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043294737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772378742694855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043309389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10331428796052933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118456320, "lr": 6.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043324343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10351146757602692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043338793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10929463803768158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118784000, "lr": 6.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043353482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11420144140720367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043368268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10538181662559509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119111680, "lr": 6.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043383120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10278709977865219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043383120, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1785043383121, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1785043465312, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7740310430526733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1785043465313, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1785043465313, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1785043480449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11263345927000046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119439360, "lr": 6.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043495182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10881412774324417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043509817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10947076231241226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119767040, "lr": 6.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043524675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10666826367378235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043538999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10163544863462448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120094720, "lr": 6.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043553317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10737879574298859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043568652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11876723170280457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120422400, "lr": 6.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043583720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1094549223780632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043598611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11590006947517395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120750080, "lr": 6.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043613784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12140027433633804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043628604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10704666376113892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121077760, "lr": 6.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043643454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10985877364873886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043658436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11530294269323349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121405440, "lr": 6.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043673281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11024171113967896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043673282, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1785043673282, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1785043755753, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7740340232849121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1785043755754, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1785043755754, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1785043770727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11318147927522659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121733120, "lr": 6.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043785837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10940331965684891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043800821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11039263755083084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122060800, "lr": 6.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043815717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10643842071294785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043830476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10427824407815933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122388480, "lr": 6.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043845302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11373378336429596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043860154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10698573291301727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122716160, "lr": 6.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043875064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11472335457801819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043890233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11275634914636612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123043840, "lr": 6.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043905520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11935865879058838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043920742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10607051849365234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123371520, "lr": 6.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043935702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1175990030169487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043950765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10685086250305176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123699200, "lr": 6.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043966158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1184300109744072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043966159, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1785043966159, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1785044048683, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742920517921448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1785044048684, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1785044048684, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1785044063797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11878899484872818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124026880, "lr": 6.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044078688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10795008391141891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044093538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11934702098369598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124354560, "lr": 6.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044108280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11414128541946411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044123144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10783306509256363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124682240, "lr": 6.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044138438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11552131921052933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044153092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11097051203250885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125009920, "lr": 6.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044168019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11691984534263611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044182952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11196579039096832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125337600, "lr": 6.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044197547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10632389038801193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044212143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11237234622240067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125665280, "lr": 6.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044226908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1058461144566536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044241522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11700145900249481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125992960, "lr": 6.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044256443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10485583543777466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044256443, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1785044256444, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1785044338341, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742561101913452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1785044338342, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1785044338342, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1785044352936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10973937809467316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126320640, "lr": 6.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044368161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11991646885871887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044382913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10997551679611206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126648320, "lr": 6.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044397587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11347004771232605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044412189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11622529476881027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126976000, "lr": 6.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044426825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10892128199338913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044441649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10812753438949585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127303680, "lr": 6.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044456537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10983459651470184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.482916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044471494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11196254938840866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127631360, "lr": 6.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044486147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11197934299707413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044500980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11011625081300735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127959040, "lr": 6.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044516006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11085846275091171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044530800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11137691140174866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128286720, "lr": 6.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044545450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10916116833686829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044545451, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1785044545451, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1785044626476, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7743262648582458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1785044626476, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1785044626477, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1785044640993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11793509125709534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128614400, "lr": 6.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044656041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11053190380334854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044670918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11074178665876389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128942080, "lr": 6.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044685748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10360893607139587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044700737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1052609235048294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129269760, "lr": 6.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044715830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11131493747234344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044730594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10781487822532654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129597440, "lr": 6.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044745283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10707570612430573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044760167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09734514355659485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129925120, "lr": 6.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044774885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10167429596185684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044789545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10517582297325134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130252800, "lr": 6.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044804146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10424518585205078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044819022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10980686545372009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130580480, "lr": 6.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044833947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11589615046977997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044833948, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1785044833948, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1785044918022, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742754817008972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1785044918023, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1785044918023, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1785044933091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11438882350921631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130908160, "lr": 6.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044947706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10464029014110565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044962267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10026340186595917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131235840, "lr": 6.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044976868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0919642373919487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044991401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11498019844293594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131563520, "lr": 6.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045005858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10358390212059021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045020534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10232198983430862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131891200, "lr": 6.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045035407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11265844106674194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045049975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10731655359268188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132218880, "lr": 6.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045064623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10286223888397217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045079609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10907576233148575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132546560, "lr": 6.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045094584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11042637377977371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045109673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10766849666833878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132874240, "lr": 6.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045124433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10703017562627792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045124434, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1785045124434, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1785045209008, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7743895649909973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1785045209009, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1785045209009, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1785045223603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10918770730495453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133201920, "lr": 6.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045238310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10268652439117432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045252720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11081671714782715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133529600, "lr": 6.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045267134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09880869090557098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045281983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11674102395772934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133857280, "lr": 6.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045296521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10984823107719421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045310854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10337348282337189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134184960, "lr": 6.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045325651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1086292490363121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045340252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10812889784574509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134512640, "lr": 6.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045354489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10836725682020187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045368853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11010424792766571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134840320, "lr": 6.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045383177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09953165054321289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045397283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10587770491838455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135168000, "lr": 6.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045411464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08756517618894577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045411465, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1785045411465, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1785045495064, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7745414972305298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1785045495065, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1785045495065, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1785045509380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09551838040351868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135495680, "lr": 6.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045523694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11148380488157272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045538021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10702306032180786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135823360, "lr": 6.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045552280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10111048072576523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045566589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10223418474197388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136151040, "lr": 6.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045581024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09079927951097488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045595360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10354651510715485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136478720, "lr": 6.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045609644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09707368165254593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045623774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10351338982582092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136806400, "lr": 6.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045638200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10628857463598251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045652718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1100715771317482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137134080, "lr": 6.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045667181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10840624570846558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045681404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10603771358728409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137461760, "lr": 6.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045695880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10288653522729874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045695880, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1785045695880, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1785045780895, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7743710279464722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1785045780896, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1785045780896, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1785045795476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09605874121189117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137789440, "lr": 7.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045810345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10508861392736435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045824727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09490539133548737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138117120, "lr": 7.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045839168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10194901376962662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045853570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09806695580482483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138444800, "lr": 7.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045867598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10038144141435623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045881666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10933251678943634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138772480, "lr": 7.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045896884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10176768898963928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.066249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045911362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1088237389922142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139100160, "lr": 7.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045925900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10957042872905731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045940698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10645316541194916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139427840, "lr": 7.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045955499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10449289530515671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045970145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10913780331611633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139755520, "lr": 7.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045984884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10896828025579453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045984885, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1785045984885, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1785046068386, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7745290398597717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1785046068387, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1785046068388, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1785046082830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10608398169279099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140083200, "lr": 7.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046097319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10411304980516434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046111664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10280042886734009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140410880, "lr": 7.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046126234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10029977560043335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046140897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10656514018774033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140738560, "lr": 7.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046155619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09558530896902084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046170824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10484892129898071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141066240, "lr": 7.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046185425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10274410247802734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046200201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11041475832462311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141393920, "lr": 7.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046215210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09869467467069626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046229651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09782709926366806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141721600, "lr": 7.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046244044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10183434188365936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046258666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09032073616981506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142049280, "lr": 7.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046273350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10694222897291183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046273351, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1785046273352, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1785046356288, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7745841145515442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1785046356289, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1785046356289, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1785046370977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09841860085725784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142376960, "lr": 7.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046385911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10089950263500214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046400469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09906332939863205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142704640, "lr": 7.257916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046415414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10976993292570114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046430465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10107507556676865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143032320, "lr": 7.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046445327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10246255248785019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046459854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10209770500659943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143360000, "lr": 7.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046474434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10544847697019577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046489358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10150986909866333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143687680, "lr": 7.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046503812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11144793778657913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046518805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10118843615055084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144015360, "lr": 7.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046533136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10853132605552673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046547760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10508773475885391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144343040, "lr": 7.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046562501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1057133674621582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046562501, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1785046562502, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1785046645740, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7747053503990173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1785046645741, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1785046645742, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1785046660853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10628858953714371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144670720, "lr": 7.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046675798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09800248593091965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046690403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10716985166072845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144998400, "lr": 7.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046705673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10611335933208466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046720653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10201172530651093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145326080, "lr": 7.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046735298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10752817243337631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046750209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09738098084926605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145653760, "lr": 7.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046765345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09806769341230392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046780283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10529821366071701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145981440, "lr": 7.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046795081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10443245619535446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046809897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10255927592515945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146309120, "lr": 7.441250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046824814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10970936715602875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046839718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10648257285356522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146636800, "lr": 7.457916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046854627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09962116926908493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046854628, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1785046854628, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1785046940784, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7749870419502258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1785046940784, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1785046940785, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1785046955150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10288669914007187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146964480, "lr": 7.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046969588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10498549789190292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046983995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09434322267770767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147292160, "lr": 7.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046998877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09827584028244019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047013339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10238650441169739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147619840, "lr": 7.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047028547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09989573806524277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047043522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10662516206502914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147947520, "lr": 7.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047058532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11013785749673843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047073399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1095050573348999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148275200, "lr": 7.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047088402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09905343502759933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047103269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10219220072031021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148602880, "lr": 7.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047117718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0985604077577591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047132712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11241482198238373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148930560, "lr": 7.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047147323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11052330583333969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047147324, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1785047147324, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1785047231474, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7750868797302246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1785047231475, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1785047231475, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1785047245871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10570449382066727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149258240, "lr": 7.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047261192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10467128455638885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047275703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10590463876724243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149585920, "lr": 7.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047290558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11134190112352371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149749760, "lr": 7.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047305787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10662292689085007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149913600, "lr": 7.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047320873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.108212910592556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047335572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1077498346567154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150241280, "lr": 7.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047350371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09757579118013382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150405120, "lr": 7.649583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047365223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1063499003648758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150568960, "lr": 7.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047379749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10329656302928925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047394488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10844287276268005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150896640, "lr": 7.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047408702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10339990258216858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151060480, "lr": 7.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047423499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10662691295146942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151224320, "lr": 7.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047438148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09477879852056503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047438184, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1785047438185, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1785047522788, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7751519680023193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1785047522789, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1785047522789, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1785047537545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09955647587776184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151552000, "lr": 7.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047552255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10568113625049591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151715840, "lr": 7.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047566912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10313506424427032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151879680, "lr": 7.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047581688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10448441654443741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047596314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10826760530471802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152207360, "lr": 7.741249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047610886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1224060133099556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152371200, "lr": 7.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047625601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10769456624984741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152535040, "lr": 7.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047640295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10001680999994278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047655272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11101870238780975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152862720, "lr": 7.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047670141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1108303964138031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153026560, "lr": 7.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047685261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11509428173303604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153190400, "lr": 7.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047700353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09788370877504349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047715555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10700041800737381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153518080, "lr": 7.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047730651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10385849326848984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153681920, "lr": 7.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047730689, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1785047730689, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1785047814821, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7755433917045593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1785047814822, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1785047814822, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1785047829990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09667797386646271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153845760, "lr": 7.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047845100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09739083051681519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.832916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047859951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11255459487438202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154173440, "lr": 7.841249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047875048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11756789684295654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154337280, "lr": 7.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047889755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11354352533817291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154501120, "lr": 7.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047904198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11086272448301315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047919424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10753082484006882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154828800, "lr": 7.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047934185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11995879560709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154992640, "lr": 7.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047949062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10482539981603622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155156480, "lr": 7.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047964365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.108317069709301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047979484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11018221080303192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155484160, "lr": 7.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047994373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11068209260702133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155648000, "lr": 7.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048009275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11335147172212601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155811840, "lr": 7.924583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048024092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11114297062158585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.932916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048024093, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1785048024093, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1785048107482, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7758362889289856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1785048107483, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1785048107483, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1785048122274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10728780925273895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156139520, "lr": 7.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048137093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11035313457250595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156303360, "lr": 7.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048151993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11418145895004272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156467200, "lr": 7.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048166825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11079065501689911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048181632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10810791701078415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156794880, "lr": 7.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048196662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10789129137992859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156958720, "lr": 7.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048211467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10148188471794128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157122560, "lr": 7.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048226188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11229000985622406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048241065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12109246104955673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157450240, "lr": 8.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048255942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10938509553670883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157614080, "lr": 8.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048270729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10893795639276505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157777920, "lr": 8.024583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048285862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10383409261703491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.032916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048300965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1036749854683876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158105600, "lr": 8.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048316110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10910143703222275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158269440, "lr": 8.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048316111, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1785048316112, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1785048399056, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.776186466217041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1785048399057, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1785048399057, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1785048413972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11382982134819031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158433280, "lr": 8.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048429137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11842318624258041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048444245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11362925916910172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158760960, "lr": 8.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048459556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11238491535186768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158924800, "lr": 8.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048474854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11141747981309891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159088640, "lr": 8.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048489840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11946381628513336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048505049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11166662722826004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159416320, "lr": 8.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048519634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10956371575593948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159580160, "lr": 8.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048534975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.110596664249897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159744000, "lr": 8.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048550058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09923901408910751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048565246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10980060696601868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160071680, "lr": 8.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048580947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10990498960018158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160235520, "lr": 8.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048596387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10394967347383499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160399360, "lr": 8.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048611435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11023806780576706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048611435, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785048611436, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785048694846, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7764796614646912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785048694847, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785048694847, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785048709874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11327142268419266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160727040, "lr": 8.174583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048725288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12793390452861786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160890880, "lr": 8.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048740459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10809746384620667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161054720, "lr": 8.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048755500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11250756680965424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048770772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11272880434989929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161382400, "lr": 8.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048786102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111846923828125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161546240, "lr": 8.216250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048801562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11398662626743317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161710080, "lr": 8.224583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048817214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11274465173482895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048832502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10670453310012817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162037760, "lr": 8.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048847797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10796744376420975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162201600, "lr": 8.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048863265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12988406419754028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162365440, "lr": 8.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048878392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11296815425157547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048894167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111888587474823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162693120, "lr": 8.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048909955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11601226776838303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162856960, "lr": 8.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048909955, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1785048909955, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1785048995517, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7767296433448792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1785048995518, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1785048995518, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1785049010824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09579312801361084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163020800, "lr": 8.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049026249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11138460785150528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049041531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1219002902507782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163348480, "lr": 8.307916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049056886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10699749737977982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163512320, "lr": 8.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049072242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10955600440502167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163676160, "lr": 8.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049087453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10716854780912399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049102584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11220987886190414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164003840, "lr": 8.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049117456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11355932056903839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164167680, "lr": 8.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049132512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10331756621599197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164331520, "lr": 8.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049147012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11717823147773743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049161812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10688144713640213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164659200, "lr": 8.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049176818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11131212115287781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164823040, "lr": 8.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049191920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09220009297132492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164986880, "lr": 8.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049206624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10349095612764359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049206625, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1785049206625, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1785049292541, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7768514156341553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1785049292542, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1785049292542, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1785049307613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0998007282614708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165314560, "lr": 8.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049322437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09686541557312012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165478400, "lr": 8.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049337094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1140117198228836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165642240, "lr": 8.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049351661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10749848186969757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049366277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10963109880685806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165969920, "lr": 8.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049381395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10403084754943848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166133760, "lr": 8.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049396186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10762524604797363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166297600, "lr": 8.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049411026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11283259093761444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049425833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09997913241386414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166625280, "lr": 8.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049440868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09469850361347198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166789120, "lr": 8.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049455661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09707077592611313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166952960, "lr": 8.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049470305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09686420857906342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049484743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1026613637804985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167280640, "lr": 8.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049499533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10057331621646881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167444480, "lr": 8.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049499533, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1785049499534, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1785049586159, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7768312692642212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1785049586160, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1785049586160, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1785049601015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10324925929307938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167608320, "lr": 8.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049616162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10270736366510391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049631206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10393742471933365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167936000, "lr": 8.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049646099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10633566975593567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168099840, "lr": 8.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049660973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10189548134803772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168263680, "lr": 8.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049676266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09964390844106674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049691777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10855940729379654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168591360, "lr": 8.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049706798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10530519485473633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168755200, "lr": 8.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049721891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10673359781503677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168919040, "lr": 8.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049736881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10601437091827393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049752106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10386841744184494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169246720, "lr": 8.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049767012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10776780545711517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169410560, "lr": 8.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049781725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10615330934524536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169574400, "lr": 8.624583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049797260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1129695251584053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049797305, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1785049797306, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1785049882926, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7773341536521912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1785049882927, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1785049882927, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1785049897748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10889171063899994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169902080, "lr": 8.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049912741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11163238435983658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170065920, "lr": 8.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049928084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.106564961373806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170229760, "lr": 8.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049943607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10684159398078918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049958709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10035856813192368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170557440, "lr": 8.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049973825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10152994096279144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170721280, "lr": 8.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049988983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09582436829805374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170885120, "lr": 8.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050003783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10678065568208694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050018422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09937532246112823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171212800, "lr": 8.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050033065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10184117406606674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171376640, "lr": 8.716249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050047989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1010720282793045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171540480, "lr": 8.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050062550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10255014151334763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050077072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09922264516353607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171868160, "lr": 8.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050091967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10393715649843216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172032000, "lr": 8.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050092005, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1785050092006, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1785050176862, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7775543928146362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1785050176863, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1785050176863, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1785050191848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09687646478414536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172195840, "lr": 8.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050206822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10761061310768127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050221585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10046671330928802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172523520, "lr": 8.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050236348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10279783606529236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172687360, "lr": 8.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050251021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09440979361534119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172851200, "lr": 8.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050265964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10696648061275482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050281094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10319008678197861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173178880, "lr": 8.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050295919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10330111533403397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173342720, "lr": 8.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050310996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10446522384881973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173506560, "lr": 8.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050325820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11180733144283295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050340876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0953916683793068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173834240, "lr": 8.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050355937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11110544949769974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173998080, "lr": 8.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050370831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11172449588775635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174161920, "lr": 8.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050385419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1015208512544632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050385460, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1785050385461, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1785050469473, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7779313325881958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1785050469474, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1785050469474, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1785050484028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09827622771263123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174489600, "lr": 8.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050498807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10728023946285248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174653440, "lr": 8.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050513527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10151971131563187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174817280, "lr": 8.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050527913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10513811558485031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050542972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10058874636888504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175144960, "lr": 8.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050557658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084088534116745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175308800, "lr": 8.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050572282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09988905489444733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175472640, "lr": 8.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050587233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09811810404062271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050602150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10879916697740555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175800320, "lr": 8.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050616970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1088314801454544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175964160, "lr": 8.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050631715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0989164263010025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176128000, "lr": 8.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050646701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09392233192920685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050661309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09429013729095459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176455680, "lr": 8.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050676021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10553766787052155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176619520, "lr": 8.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050676022, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1785050676022, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1785050758008, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7785099148750305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1785050758009, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1785050758009, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1785050772772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1024998128414154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176783360, "lr": 8.991249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050787435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11365505307912827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050802149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10017678886651993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177111040, "lr": 9.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050816871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10524698346853256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177274880, "lr": 9.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050831551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1099468246102333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177438720, "lr": 9.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050846223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09997612982988358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.032916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050861020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10954024642705917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177766400, "lr": 9.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050875815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10045772045850754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177930240, "lr": 9.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050890647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10638388246297836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178094080, "lr": 9.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050905455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10018035024404526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050920444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09727338701486588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178421760, "lr": 9.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050935732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11207836121320724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178585600, "lr": 9.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050950952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11841566115617752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178749440, "lr": 9.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050966036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10894139111042023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050966037, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1785050966037, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1785051047997, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7788505554199219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1785051047998, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1785051047998, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1785051063075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11437927186489105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179077120, "lr": 9.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051077758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10107389092445374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179240960, "lr": 9.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051092873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09956853836774826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179404800, "lr": 9.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051107858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1007462665438652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.132916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051122937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10448732227087021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179732480, "lr": 9.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051137885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09894277155399323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179896320, "lr": 9.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051152716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1002778485417366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180060160, "lr": 9.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051167763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1058330088853836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051182441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1055213212966919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180387840, "lr": 9.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051197006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09330917149782181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180551680, "lr": 9.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051212481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10352843254804611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180715520, "lr": 9.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051227020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10142980515956879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051241694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09418658912181854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181043200, "lr": 9.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051256537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09950333833694458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181207040, "lr": 9.216249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051256538, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1785051256538, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1785051338575, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7790018320083618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1785051338576, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1785051338576, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1785051353446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11102540791034698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181370880, "lr": 9.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051368203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11338824033737183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051382735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10082482546567917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181698560, "lr": 9.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051397487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10664589703083038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181862400, "lr": 9.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051412189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10243093222379684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182026240, "lr": 9.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051426888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1017770767211914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051441395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10608028620481491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182353920, "lr": 9.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051455954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087418645620346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182517760, "lr": 9.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051470600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09756886214017868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182681600, "lr": 9.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051485123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09698090702295303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051499777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11002243310213089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183009280, "lr": 9.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051514097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09661427140235901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183173120, "lr": 9.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051528991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10210713744163513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183336960, "lr": 9.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051543515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10230713337659836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051543516, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785051543516, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785051625251, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7791846394538879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785051625252, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785051625252, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785051639811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10250681638717651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183664640, "lr": 9.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051654247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10810030996799469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183828480, "lr": 9.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051668812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11193227767944336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183992320, "lr": 9.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051684031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09294495731592178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051698905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1024547666311264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184320000, "lr": 9.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051713690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10744946449995041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184483840, "lr": 9.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051728570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1166396290063858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184647680, "lr": 9.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051743262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10088343173265457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051758158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10591216385364532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184975360, "lr": 9.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051773482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10895474255084991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185139200, "lr": 9.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051788374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09750571846961975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185303040, "lr": 9.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051803176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10714153945446014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051818121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09842141717672348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185630720, "lr": 9.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051832626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10456285625696182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185794560, "lr": 9.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051832627, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1785051832627, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1785051915149, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7796854972839355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1785051915149, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1785051915150, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1785051930445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11010333150625229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185958400, "lr": 9.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051945154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10358525067567825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051959533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09176407009363174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186286080, "lr": 9.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051974796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10055691003799438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186449920, "lr": 9.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051989550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792253166437149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186613760, "lr": 9.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052004285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1115415021777153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052019615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09655426442623138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186941440, "lr": 9.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052034734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10341567546129227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187105280, "lr": 9.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052049709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1050177812576294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187269120, "lr": 9.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052064570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0982932522892952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052079440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10344594717025757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187596800, "lr": 9.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052094305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10467391461133957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187760640, "lr": 9.549583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052109011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11002132296562195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187924480, "lr": 9.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052123948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11330369114875793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052123949, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1785052123949, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1785052206263, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7796527147293091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1785052206264, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1785052206264, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1785052221002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10859814286231995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188252160, "lr": 9.574583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052236206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10471327602863312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188416000, "lr": 9.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052251039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10199669003486633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188579840, "lr": 9.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052266068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10835345089435577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052280873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10208049416542053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188907520, "lr": 9.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052295933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0971112921833992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189071360, "lr": 9.616249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052310633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10241781175136566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189235200, "lr": 9.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052325284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11113369464874268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052340061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10125716030597687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189562880, "lr": 9.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052354979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09447497874498367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189726720, "lr": 9.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052369792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10378646105527878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189890560, "lr": 9.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052385002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11191640794277191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.666249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052400265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10137928277254105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190218240, "lr": 9.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052415184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11784270405769348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190382080, "lr": 9.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052415185, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1785052415185, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1785052500985, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7799276113510132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1785052500986, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1785052500986, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1785052516197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09981628507375717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190545920, "lr": 9.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052531111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09699826687574387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052546280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10601243376731873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190873600, "lr": 9.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052561547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11250147223472595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191037440, "lr": 9.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052576939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11696992069482803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191201280, "lr": 9.724583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052592157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09578622877597809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052607332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10636766254901886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191528960, "lr": 9.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052622162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10103476792573929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191692800, "lr": 9.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052637197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1011614054441452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191856640, "lr": 9.757916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052652161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10438644886016846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052666637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10658632963895798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192184320, "lr": 9.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052682105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10948285460472107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192348160, "lr": 9.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052696783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09938912093639374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192512000, "lr": 9.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052711605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11342119425535202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.799583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052711606, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1785052711606, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1785052797095, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7804022431373596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1785052797096, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1785052797096, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1785052812210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11216588318347931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192839680, "lr": 9.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052827499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11124176532030106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193003520, "lr": 9.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052842466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10635153204202652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193167360, "lr": 9.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052857372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11885037273168564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052872228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11619836091995239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193495040, "lr": 9.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052887336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10034791380167007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193658880, "lr": 9.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052902435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1174008771777153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193822720, "lr": 9.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052917299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11389193683862686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.866249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052932294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10618581622838974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194150400, "lr": 9.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052947274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10689610242843628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194314240, "lr": 9.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052962408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10721380263566971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194478080, "lr": 9.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052977401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10526325553655624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052992582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10710801929235458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194805760, "lr": 9.907916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053007266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11558616161346436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194969600, "lr": 9.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053007267, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1785053007267, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1785053088519, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7807079553604126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1785053088520, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1785053088520, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1785053103803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11172465234994888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195133440, "lr": 9.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053118720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10371177643537521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053134052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10229676961898804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195461120, "lr": 9.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053149245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11612074822187424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195624960, "lr": 9.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053164473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793616622686386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195788800, "lr": 9.957916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053179308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10489306598901749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053194335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10386357456445694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196116480, "lr": 9.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053209300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1132175549864769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196280320, "lr": 9.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053224686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11778215318918228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196444160, "lr": 9.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053239914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10482483357191086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.999583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053255242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11564325541257858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196771840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053270432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11180909723043442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196935680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053285057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10782753676176071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197099520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053300181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11619686335325241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053300182, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1785053300182, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1785053381695, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7810214161872864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1785053381696, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1785053381697, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1785053396502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10121721029281616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197427200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053411847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11133447289466858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197591040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053427057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11527463048696518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197754880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053442041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10014904290437698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053457445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09854444861412048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198082560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053472804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10742687433958054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198246400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053488133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09902872145175934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198410240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053503375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10819010436534882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053518678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10012488812208176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198737920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053533769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10582534223794937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198901760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053548823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11474596709012985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199065600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053563746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10989334434270859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053578552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11750618368387222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199393280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053593801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11215201020240784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199557120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053593802, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1785053593802, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1785053677228, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7814888954162598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1785053677229, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1785053677229, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1785053692146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11149968206882477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199720960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053707276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10562536865472794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053722119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11056513339281082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200048640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053737181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11595930904150009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200212480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053752343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11221322417259216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200376320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053767611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11652375012636185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053783214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09898607432842255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200704000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053798673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10843845456838608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200867840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053814186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10797138512134552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201031680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053829859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082003191113472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053845601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11769521236419678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201359360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053861051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11044465005397797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201523200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053876359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1030433177947998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201687040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053891831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11027389764785767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053891831, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1785053891832, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1785053972861, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7811171412467957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1785053972861, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1785053972862, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1785053987668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10567289590835571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202014720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054002908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11056801676750183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202178560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054017827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11880995333194733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202342400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054033191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10628723353147507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054047768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10490485280752182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202670080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054062527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10072874277830124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202833920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054077996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10620785504579544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202997760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054092760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10924496501684189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054107686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1120743602514267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203325440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054122362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09363242983818054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203489280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054137340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11679542809724808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203653120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054152163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09952055662870407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054166486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09829719364643097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203980800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054180880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10708057135343552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204144640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054180880, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1785054180881, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1785054263206, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7810669541358948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1785054263207, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1785054263207, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1785054277915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1007923111319542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204308480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054292961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09290392696857452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054307892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0963250994682312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204636160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054322630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10868389159440994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204800000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054337303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10776635259389877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204963840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054352004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11241219192743301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054366499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09500519931316376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205291520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054381266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10521692782640457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205455360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054395941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10602521896362305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205619200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054410504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10996668040752411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054424984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09757247567176819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205946880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054440011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09828417748212814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206110720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054454850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11128991842269897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206274560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054469351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09865450114011765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054469352, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1785054469352, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1785054551501, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7816809415817261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1785054551501, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1785054551502, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1785054566440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10561046004295349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206602240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054581169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0955706313252449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206766080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054596282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10295005887746811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206929920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054611344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09557194262742996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054626291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10189777612686157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207257600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054641048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11073702573776245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207421440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054655980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10646399855613708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207585280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054670576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09520167857408524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054685370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11325940489768982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207912960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054700106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10397934913635254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208076800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054714480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10344891250133514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208240640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054729876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10156407207250595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054744694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10304336994886398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208568320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054759937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1012105718255043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208732160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054759938, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1785054759938, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1785054842772, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7814342975616455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1785054842773, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1785054842773, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1785054858170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10042937844991684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208896000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054873489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09895973652601242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054888592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10099714249372482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209223680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054903288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10171964019536972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209387520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054918416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09881605207920074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209551360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054933213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10045759379863739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054947797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09683693945407867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209879040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054962390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10009517520666122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210042880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054977334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1032460406422615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210206720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054992097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10814335197210312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055007051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10348436236381531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210534400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055021986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1073966696858406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210698240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055036818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10138839483261108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210862080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055051340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0956558957695961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055051341, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1785055051341, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1785055135528, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7818154692649841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1785055135529, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1785055135529, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1785055150557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09400293231010437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211189760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055165098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0990832969546318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211353600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055179779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09640584886074066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211517440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055194485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09961166232824326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055209304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10393788665533066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211845120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055224846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09508147090673447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212008960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055239993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10174640268087387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212172800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055255321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0974823534488678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055270086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0976036936044693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212500480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055285195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1038234606385231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212664320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055300365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0934252068400383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212828160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055315917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723937302827835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055331332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10433796793222427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213155840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055346895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10809801518917084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213319680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055346896, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1785055346897, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1785055430313, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7819897532463074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1785055430313, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1785055430313, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1785055445533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1047096997499466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213483520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055460500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10112645477056503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055475444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10535863786935806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213811200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055491004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10291267186403275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213975040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055505816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09605276584625244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214138880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055520746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10469988733530045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055535743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09460142254829407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214466560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055550396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11387782543897629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214630400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055565000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09824220836162567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214794240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055580377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08957105875015259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055595422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10011179745197296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215121920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055610335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10408267378807068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215285760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055625129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10433867573738098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215449600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055639902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10841730237007141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055639903, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1785055639903, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1785055722196, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7822414040565491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1785055722197, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1785055722197, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1785055736908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09962751716375351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215777280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055751320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1051267609000206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215941120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055766119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09754883497953415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216104960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055781087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0999884307384491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055796002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10014534741640091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216432640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055810787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09425465017557144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216596480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055825540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10403574258089066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216760320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055840663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10226717591285706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055855222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10019170492887497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217088000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055870253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10997647792100906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217251840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055884608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09742698818445206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217415680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055899484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09936152398586273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217579520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055914249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09984339773654938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217743360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055929051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10510250926017761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217907200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055929051, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1785055929052, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1785056011426, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.782217800617218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1785056011427, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1785056011428, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1785056026461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10100886970758438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218071040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056041348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09520447254180908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218234880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056056327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09824647754430771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218398720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056071099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10760688781738281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218562560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056086216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10278007388114929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218726400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056101212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08729714900255203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218890240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056116296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10573576390743256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219054080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056131598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09288742393255234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219217920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056146768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10426854342222214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219381760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056161747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09526742249727249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219545600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056176661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09462322294712067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219709440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056191444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10253601521253586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219873280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056206270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11127454787492752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220037120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056220990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0977720245718956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220200960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056220991, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1785056220991, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1785056305251, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7828539609909058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1785056305252, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1785056305252, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1785056319591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09515446424484253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220364800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056334916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09562563896179199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220528640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056349621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10027037560939789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220692480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056364482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1021590456366539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220856320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056379795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10370168834924698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221020160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056395132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10564199835062027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221184000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056410322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10796640813350677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221347840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056425259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1155538260936737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221511680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056440312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10808967798948288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221675520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056455247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09902267158031464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221839360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056470201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11042363196611404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222003200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056485267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09637783467769623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222167040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056500393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09421882778406143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222330880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056515336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10838200896978378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222494720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056515337, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1785056515337, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} diff --git a/recommendation_v4/rcp_logs/gbs_8192/seed94090156.log b/recommendation_v4/rcp_logs/gbs_8192/seed94090156.log new file mode 100644 index 000000000..7b5fcc617 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_8192/seed94090156.log @@ -0,0 +1,632 @@ +:::MLLOG {"namespace": "", "time_ms": 1784940780263, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784940780263, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784940801844, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784940801845, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784940801845, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784940801845, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784940801845, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI355X", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784940801845, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 8192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784940801845, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784940801845, "event_type": "POINT_IN_TIME", "key": "seed", "value": 94090156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784940801846, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784940801846, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784940801846, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784940801846, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784940801850, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784940801850, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784941725157, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784941725158, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784941725528, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784941851421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13905055820941925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840, "lr": 7.916666666666666e-10}} +:::MLLOG {"namespace": "", "time_ms": 1784941864933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13886143267154694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.6249999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784941878903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13842469453811646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 491520, "lr": 2.458333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784941893021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13858996331691742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784941907049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13874505460262299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 819200, "lr": 4.125e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784941921129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13852688670158386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784941935144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13865657150745392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1146880, "lr": 5.7916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784941949290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13879381120204926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.625e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784941963308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385953724384308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1474560, "lr": 7.458333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784941977185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13871899247169495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.291666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784941991717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385965198278427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1802240, "lr": 9.124999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784942005949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13881181180477142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784942020528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13853195309638977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2129920, "lr": 1.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942034923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13886302709579468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942034923, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784942034924, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784942131914, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5030940771102905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784942131915, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784942131915, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784942146169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13853797316551208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2457600, "lr": 1.2458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942160316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13855013251304626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942174884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13834108412265778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2785280, "lr": 1.4124999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942188280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13857272267341614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4958333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942201981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381465345621109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3112960, "lr": 1.5791666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942216806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382952183485031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942230500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13827921450138092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3440640, "lr": 1.7458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942244412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381874680519104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942259083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381726861000061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3768320, "lr": 1.9124999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942273042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381683647632599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9958333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942287577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384214460849762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4096000, "lr": 2.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942302141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13864989578723907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.1624999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942316242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13804666697978973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4423680, "lr": 2.2458333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942331151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383395493030548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3291666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942331151, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784942331152, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784942426843, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5032354593276978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784942426844, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784942426844, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784942441188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13782553374767303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4751360, "lr": 2.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942455434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13823845982551575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942469866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137461319565773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5079040, "lr": 2.5791666666666668e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942486725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13684138655662537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942501325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13818742334842682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5406720, "lr": 2.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942515831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380784660577774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942529985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137678325176239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5734400, "lr": 2.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942544314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13726551830768585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942559062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13710835576057434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6062080, "lr": 3.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942573336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13789720833301544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942587614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13739560544490814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6389760, "lr": 3.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942601780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376650482416153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942616830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378381848335266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6717440, "lr": 3.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942631464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137699693441391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942631465, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784942631465, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784942726801, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5033757090568542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784942726802, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784942726802, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784942741327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13690634071826935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7045120, "lr": 3.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942755658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13739846646785736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942769912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367815136909485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7372800, "lr": 3.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942784692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13733865320682526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.829166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942799811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13746240735054016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7700480, "lr": 3.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942814616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13787606358528137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942829459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13661985099315643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8028160, "lr": 4.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942843987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374415159225464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942858793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1369202882051468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8355840, "lr": 4.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942873884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13843193650245667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3291666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942888712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13641183078289032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8683520, "lr": 4.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942903731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13775645196437836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.4958333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942918348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13548614084720612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9011200, "lr": 4.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942933357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13687552511692047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.6624999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942933358, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784942933358, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784943026642, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5035967826843262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784943026643, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784943026643, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784943040888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13635388016700745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9338880, "lr": 4.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943055154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13555485010147095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943069620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13825403153896332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9666560, "lr": 4.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943083883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347111165523529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943098385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13674448430538177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9994240, "lr": 5.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943112834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13342227041721344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1624999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943127285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355389952659607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10321920, "lr": 5.2458333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943142233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13544544577598572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943157174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378238946199417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10649600, "lr": 5.4124999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943171717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351134479045868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943186846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14766326546669006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10977280, "lr": 5.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943201657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355045735836029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943215931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13404406607151031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11304960, "lr": 5.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943230421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13772538304328918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943230422, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784943230422, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784943326481, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5038921236991882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784943326482, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784943326482, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784943340998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354065239429474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11632640, "lr": 5.9124999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943355272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13978120684623718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943370156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13596680760383606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11960320, "lr": 6.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943384496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13321910798549652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943399057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13496533036231995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12288000, "lr": 6.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943414003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13915176689624786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.329166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943428096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13343356549739838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12615680, "lr": 6.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943444224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13287730515003204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943459477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332816630601883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12943360, "lr": 6.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943473885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13154885172843933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943488927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13122256100177765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13271040, "lr": 6.745833333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943503683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13511055707931519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943518439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13219547271728516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13598720, "lr": 6.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943533838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13888314366340637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943533839, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784943533840, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784943628612, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5042330026626587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784943628612, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784943628613, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784943643251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13360434770584106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13926400, "lr": 7.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943657608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13340085744857788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943672091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357937902212143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14254080, "lr": 7.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943687106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13506732881069183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943701829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378706991672516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14581760, "lr": 7.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943716462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1278015524148941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943731196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13516132533550262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14909440, "lr": 7.579166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943745636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12918736040592194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943760633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13350629806518555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15237120, "lr": 7.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943776003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13253222405910492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.829166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943790396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13355326652526855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15564800, "lr": 7.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943805381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1342674195766449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.995833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943820861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353611797094345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15892480, "lr": 8.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943835873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13342984020709991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943835874, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784943835874, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784943930722, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5048770904541016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784943930723, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784943930723, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784943946062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13261187076568604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16220160, "lr": 8.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943961408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13372644782066345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943976590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356739103794098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16547840, "lr": 8.412500000000001e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943992186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13539868593215942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944007150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13965333998203278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16875520, "lr": 8.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944021727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13318023085594177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.662499999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944037720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353289783000946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17203200, "lr": 8.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944053058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13333433866500854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944068473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13285872340202332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17530880, "lr": 8.912499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944083687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13170048594474792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944098839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13588516414165497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17858560, "lr": 9.079166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944114121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13371813297271729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.162499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944129046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13335688412189484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18186240, "lr": 9.245833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944143799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13064339756965637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944143799, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784944143800, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784944236067, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5058125853538513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784944236067, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784944236068, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784944251383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13796916604042053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18513920, "lr": 9.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944267043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13479822874069214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.495833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944282388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135401651263237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18841600, "lr": 9.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944297880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13125328719615936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.662499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944313509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13364379107952118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19169280, "lr": 9.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944328800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13404783606529236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944344396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370343416929245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19496960, "lr": 9.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944359659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13384605944156647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944374639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329483687877655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19824640, "lr": 1.0079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944389826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1296517550945282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0162499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944404498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1276158094406128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20152320, "lr": 1.0245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944419645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329897940158844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944434511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12959618866443634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20480000, "lr": 1.0412499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944449245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12479651719331741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944449245, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784944449246, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784944543872, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5068331360816956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784944543874, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784944543874, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784944559326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311798244714737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20807680, "lr": 1.0579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944574296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13608810305595398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944589806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13323654234409332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21135360, "lr": 1.0745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944605024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13045592606067657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944619360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12830698490142822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21463040, "lr": 1.0912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944633949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12651613354682922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944649528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289021521806717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21790720, "lr": 1.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944664340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13242484629154205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944679357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13542760908603668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22118400, "lr": 1.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944694558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1248716413974762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944708958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1285681575536728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22446080, "lr": 1.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944723883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13418754935264587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944738789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13099823892116547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22773760, "lr": 1.1579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944754010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12918341159820557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944754012, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784944754012, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784944848288, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5081168413162231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784944848290, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784944848290, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784944863700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13040019571781158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23101440, "lr": 1.1745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944878437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12912043929100037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944892830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13308799266815186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23429120, "lr": 1.1912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944908413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12374423444271088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944923548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294095516204834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23756800, "lr": 1.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944938704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12574510276317596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944953305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12987980246543884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24084480, "lr": 1.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944968377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12185632437467575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944983135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13583222031593323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24412160, "lr": 1.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944998015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13715238869190216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945013178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12382373958826065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24739840, "lr": 1.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945028480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12314941734075546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945043382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12547567486763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25067520, "lr": 1.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945058332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12910687923431396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945058332, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784945058333, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784945152776, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5097717046737671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784945152777, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784945152777, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784945167064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13644269108772278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25395200, "lr": 1.2912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945182413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364903301000595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945197665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13566599786281586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25722880, "lr": 1.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945212728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1399025022983551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945227478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13666218519210815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26050560, "lr": 1.3245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945242733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13814234733581543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945257721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12723416090011597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26378240, "lr": 1.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945272462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12612593173980713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945287642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14221084117889404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26705920, "lr": 1.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945302836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1409851610660553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945318235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12714947760105133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27033600, "lr": 1.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945333352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12809398770332336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945347955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13263250887393951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27361280, "lr": 1.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945362053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131465345621109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945362054, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784945362054, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784945453486, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5118551850318909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784945453487, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784945453487, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784945468864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14121124148368835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27688960, "lr": 1.4079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945484544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1424025595188141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945499693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13532699644565582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28016640, "lr": 1.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945514750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13957887887954712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945530527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14494022727012634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28344320, "lr": 1.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945545474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14079736173152924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945560834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12582120299339294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28672000, "lr": 1.4579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945576106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13764746487140656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945591136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13184978067874908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28999680, "lr": 1.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945606644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13615870475769043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945621598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13783445954322815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29327360, "lr": 1.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945636308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13296347856521606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4995833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945652241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13444876670837402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29655040, "lr": 1.5079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945667938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13627639412879944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945667938, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784945667939, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784945762279, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5138501524925232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784945762280, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784945762280, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784945777791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13867592811584473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29982720, "lr": 1.5245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945793247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138139009475708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945808975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13163943588733673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30310400, "lr": 1.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945824871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13300028443336487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945840168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13378560543060303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30638080, "lr": 1.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945856048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14007599651813507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945871582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13294096291065216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30965760, "lr": 1.5745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945887022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14138725399971008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945901866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14082413911819458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31293440, "lr": 1.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945916997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350783109664917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945932199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1278083771467209, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31621120, "lr": 1.6079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945947058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12989988923072815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945961757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13409262895584106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31948800, "lr": 1.6245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945977090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12758849561214447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945977091, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784945977091, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784946071584, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5161877870559692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784946071585, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784946071585, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784946086356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13207630813121796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32276480, "lr": 1.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946101134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13695663213729858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946116433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13347512483596802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32604160, "lr": 1.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946131095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13190069794654846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946146225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13383010029792786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32931840, "lr": 1.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946161251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13345764577388763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946176542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13394534587860107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33259520, "lr": 1.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946191758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13280023634433746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946206162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13969773054122925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33587200, "lr": 1.7079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946222007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13179922103881836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946237581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12893040478229523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33914880, "lr": 1.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946252677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13093505799770355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946267822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13246136903762817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34242560, "lr": 1.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946283316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13608936965465546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7495833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946283317, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784946283318, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784946378420, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5193868279457092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784946378421, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784946378421, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784946393554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13348685204982758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34570240, "lr": 1.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946408569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12293566763401031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946423955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13050566613674164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34897920, "lr": 1.7745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946438856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1302546113729477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946454124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351708322763443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35225600, "lr": 1.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946469822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13543429970741272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946485017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354699581861496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35553280, "lr": 1.8079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946500865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.123927041888237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8162500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946516032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13094298541545868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35880960, "lr": 1.8245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946530452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312914490699768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946545856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13025623559951782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36208640, "lr": 1.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946561256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13316616415977478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946576612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1309370994567871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36536320, "lr": 1.8579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946591738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13027873635292053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946591738, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784946591739, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784946685496, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5237644910812378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784946685497, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784946685497, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784946700920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13770505785942078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36864000, "lr": 1.8745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946716131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132155179977417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946731584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335451304912567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37191680, "lr": 1.8912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946746972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13046012818813324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946762294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12713100016117096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37519360, "lr": 1.9079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946777792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13218577206134796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946793377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13563406467437744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37847040, "lr": 1.9245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946809088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13283993303775787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946824843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13121555745601654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38174720, "lr": 1.9412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946840422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13901394605636597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946855731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13389554619789124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38502400, "lr": 1.9579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946870240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375771015882492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946885588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13184355199337006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38830080, "lr": 1.9745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946900763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13340817391872406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946900764, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784946900764, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784946996512, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5296505093574524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784946996513, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784946996513, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784947011907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13108664751052856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39157760, "lr": 1.9912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947026899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13492120802402496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947042332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13779711723327637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39485440, "lr": 2.0079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947057421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13325388729572296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947072101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12746131420135498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39813120, "lr": 2.0245833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947087270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1292697638273239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0329166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947102342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12593616545200348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40140800, "lr": 2.0412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947117739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12394209951162338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947133324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13606664538383484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40468480, "lr": 2.0579166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947148734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1372380256652832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947164214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13472682237625122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40796160, "lr": 2.0745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947179956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12693139910697937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0829166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947195272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13070008158683777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41123840, "lr": 2.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947210061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12359321862459183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947210062, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784947210062, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784947302849, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5379031300544739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784947302850, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784947302850, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784947318650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13243316113948822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41451520, "lr": 2.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947334421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13261711597442627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947349858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13374851644039154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41779200, "lr": 2.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947365312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13802607357501984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947380793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13593114912509918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42106880, "lr": 2.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947396105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13134048879146576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947411815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12716495990753174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42434560, "lr": 2.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947427302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13512510061264038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947442317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1247188076376915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42762240, "lr": 2.1745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947458058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13033828139305115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947473393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13254795968532562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43089920, "lr": 2.1912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947489179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13412420451641083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947504897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14033126831054688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43417600, "lr": 2.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947520284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361795961856842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947520285, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784947520285, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784947614389, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5502026081085205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784947614391, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784947614391, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784947630404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13641151785850525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43745280, "lr": 2.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947645798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13149140775203705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947660530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13250961899757385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44072960, "lr": 2.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947675616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13895519077777863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947691756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12995222210884094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44400640, "lr": 2.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947707572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13836424052715302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947723248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13515114784240723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44728320, "lr": 2.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947738778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371249556541443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947754138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13468831777572632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45056000, "lr": 2.2912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947770175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13033072650432587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947785487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14035974442958832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45383680, "lr": 2.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947801112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13453394174575806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947816742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289331614971161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45711360, "lr": 2.3245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947832189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13330332934856415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947832189, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784947832190, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784947927494, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.564673900604248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784947927495, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784947927495, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784947943582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14185789227485657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46039040, "lr": 2.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947959070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13275563716888428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947975056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13347461819648743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46366720, "lr": 2.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947990999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13032373785972595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948006750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352643519639969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46694400, "lr": 2.3745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948022618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335311383008957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948038816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13399918377399445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47022080, "lr": 2.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948054587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13539493083953857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948070209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13099004328250885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47349760, "lr": 2.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948086277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13264256715774536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948102032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13283459842205048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47677440, "lr": 2.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948118044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13698606193065643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948133516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358991265296936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48005120, "lr": 2.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948149281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371888816356659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.4495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948149281, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784948149282, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784948243922, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5798835158348083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784948243923, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784948243923, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784948259784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315540224313736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48332800, "lr": 2.4579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948275771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13503558933734894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948291752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13371476531028748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48660480, "lr": 2.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948308063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13192249834537506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948323674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13691174983978271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48988160, "lr": 2.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948339042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12941157817840576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948355171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13350863754749298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49315840, "lr": 2.507916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948370397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12351956218481064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.5162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948385972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13092103600502014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49643520, "lr": 2.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948401041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131168931722641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948416530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13104663789272308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49971200, "lr": 2.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948431502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13129375874996185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.5495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948446616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1285214126110077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50298880, "lr": 2.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948462117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13054083287715912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.5662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948462118, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784948462118, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784948556196, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.591177225112915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784948556197, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784948556197, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784948571038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1221403181552887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50626560, "lr": 2.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948586385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12662363052368164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948601651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12924233078956604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50954240, "lr": 2.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948617051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329435557126999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948633015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13158658146858215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51281920, "lr": 2.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948648447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12167047709226608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948664334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303580403327942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51609600, "lr": 2.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948680318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13210472464561462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948695625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13658763468265533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51937280, "lr": 2.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948710553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12807725369930267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.6495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948726367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13305743038654327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52264960, "lr": 2.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948742352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11772496998310089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.6662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948757856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1212555319070816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52592640, "lr": 2.6745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948773452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1256648600101471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948773452, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784948773453, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784948868590, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6041976809501648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784948868591, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784948868591, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784948884280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12165308743715286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52920320, "lr": 2.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948899465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12752267718315125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.6995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948915081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1282796859741211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53248000, "lr": 2.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948930379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315780133008957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948945514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1216719001531601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53575680, "lr": 2.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948961118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13438677787780762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948976457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12958930432796478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53903360, "lr": 2.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948991574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1216166689991951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.7495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949007118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12687766551971436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54231040, "lr": 2.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949022658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1271541267633438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949038149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11860650777816772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54558720, "lr": 2.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949054053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13166259229183197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949069364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12672287225723267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54886400, "lr": 2.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949084774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12824372947216034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949084775, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784949084775, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784949178957, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6177582144737244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784949178958, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784949178959, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784949193669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12841536104679108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55214080, "lr": 2.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949209331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1189737468957901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949225088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12190018594264984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55541760, "lr": 2.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949240742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12161259353160858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949256238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12383794784545898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55869440, "lr": 2.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949272003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1265561580657959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949287375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11663893610239029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56197120, "lr": 2.8579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949303000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12298820167779922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949318759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12559285759925842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56524800, "lr": 2.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949334096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12388516217470169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949349609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11599884182214737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56852480, "lr": 2.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949364977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12472750246524811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949380571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11022903025150299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57180160, "lr": 2.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949396346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12448397278785706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949396346, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784949396347, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784949491195, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6297577619552612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784949491196, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784949491196, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784949506966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1172962561249733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57507840, "lr": 2.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949522743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11486797034740448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949538261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12929323315620422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57835520, "lr": 2.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949554201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12247244268655777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949569859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12469477951526642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58163200, "lr": 2.9579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949585259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11670318245887756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949600058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12050347775220871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58490880, "lr": 2.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949616391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1222861111164093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949631974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12163715064525604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58818560, "lr": 2.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949647538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11328402161598206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949662802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12176624685525894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59146240, "lr": 3.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949678302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12172585725784302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.0162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949693579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12507115304470062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59473920, "lr": 3.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949708932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11723426729440689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949708933, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784949708933, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784949803452, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6468431949615479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784949803453, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784949803454, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784949819060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12069575488567352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59801600, "lr": 3.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949834066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11184168606996536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949849036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11112887412309647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60129280, "lr": 3.0579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949864428, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1139146089553833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.0662499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949879572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10940233618021011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60456960, "lr": 3.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949895160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12187378108501434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949910646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11643083393573761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60784640, "lr": 3.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949926400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12039922177791595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949941968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10426948219537735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61112320, "lr": 3.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949957846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11994168162345886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.1162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949973553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11282042413949966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61440000, "lr": 3.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949989456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11340176314115524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950004592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10942406207323074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61767680, "lr": 3.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950020527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11699733138084412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950020528, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784950020528, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784950116062, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6734912395477295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784950116063, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784950116063, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784950131814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12016147375106812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62095360, "lr": 3.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950147429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11987997591495514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.1662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950163523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12381482869386673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62423040, "lr": 3.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950178996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10693004727363586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950194760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1140967383980751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62750720, "lr": 3.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950210566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11856260150671005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950226231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11955101042985916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63078400, "lr": 3.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950242270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11968287080526352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950257823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10903595387935638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63406080, "lr": 3.2245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950273018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11871703714132309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950288745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11451846361160278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63733760, "lr": 3.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950303695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11709967255592346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950318969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1134369894862175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64061440, "lr": 3.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950334602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1247100979089737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950334603, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784950334603, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784950429915, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7047087550163269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784950429916, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784950429916, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784950445625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12371472269296646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64389120, "lr": 3.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950461437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11717352271080017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950477120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11667104065418243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64716800, "lr": 3.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950493049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.121869757771492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950509065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10948721319437027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65044480, "lr": 3.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950524522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11145536601543427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950539605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1136884093284607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65372160, "lr": 3.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950555359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084410771727562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950571835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11108839511871338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65699840, "lr": 3.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950587940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1159980520606041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950603889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12329971045255661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66027520, "lr": 3.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950619935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11524996906518936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950635816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12680478394031525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66355200, "lr": 3.3745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950651903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11369699984788895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950651903, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784950651903, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784950746438, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7452806830406189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784950746439, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784950746439, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784950762515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12544560432434082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66682880, "lr": 3.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950778120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11340180039405823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.3995833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950794353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11583685874938965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67010560, "lr": 3.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950810310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1131121814250946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950825949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11889799684286118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67338240, "lr": 3.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950842512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11722265928983688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950858660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12064782530069351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67665920, "lr": 3.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950874918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12037031352519989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950891338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11503250151872635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67993600, "lr": 3.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950907785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11171238124370575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950923917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11563924700021744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68321280, "lr": 3.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950939651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11449366807937622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950955127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1247749850153923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68648960, "lr": 3.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950971569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11532419919967651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950971570, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784950971570, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784951065776, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7560091018676758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784951065776, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784951065777, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784951081685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12148363888263702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68976640, "lr": 3.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951097927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11821477860212326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951113567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11334174871444702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69304320, "lr": 3.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951129686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11423928290605545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951145572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10848351567983627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69632000, "lr": 3.5412500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951161389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11447238177061081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951177554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.117182157933712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69959680, "lr": 3.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951193118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11029712110757828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951209131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10885175317525864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70287360, "lr": 3.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951225208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10804615914821625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951241384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12229602038860321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70615040, "lr": 3.5912500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951257835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10594744980335236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951273931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1201845109462738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70942720, "lr": 3.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951290163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11923522502183914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951290164, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784951290164, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784951384220, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7621324062347412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784951384221, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784951384221, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784951400567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11783707141876221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71270400, "lr": 3.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951416053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11705782264471054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951431393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11529839038848877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71598080, "lr": 3.6412499999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951447770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11519453674554825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951463342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10257124900817871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71925760, "lr": 3.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951479405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10244587063789368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951494938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11874022334814072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72253440, "lr": 3.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951510603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11464724689722061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951526313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10299614816904068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72581120, "lr": 3.6912499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951541889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10176365822553635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951557531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10627223551273346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72908800, "lr": 3.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951573106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11212299019098282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951588667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11058098077774048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73236480, "lr": 3.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951604604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11456134170293808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951604643, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784951604643, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784951699547, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7663450837135315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784951699548, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784951699548, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784951715177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10499343276023865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73564160, "lr": 3.7412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951731306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10615099221467972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951747589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11413659155368805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73891840, "lr": 3.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951763353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10991502553224564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951779520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793928056955338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74219520, "lr": 3.7745833333333333e-07}} From 1c3c80845a32de22f1aa79696005eba1fb47e47f Mon Sep 17 00:00:00 2001 From: suachong Date: Sun, 26 Jul 2026 10:45:51 -0500 Subject: [PATCH 123/127] dlrmv4 gin: set CV-batch defaults (lr=1e-6, warmup=24k, eval=0.1%, skip_eval=10) Update yambda_5b.gin env-default values so the standard config matches the convergence/RCP batch: DENSE_LR/SPARSE_LR 1e-7->1e-6, LR_WARMUP_STEPS 0->24000, EVAL_EVERY_DATA_PCT 0.005->0.001, SKIP_EVAL 0->10. Env overrides still take precedence, so existing launch scripts are unaffected. Co-authored-by: Cursor --- .../dlrm_v3/train/gin/yambda_5b.gin | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin index 9212adbad..188d6cad6 100644 --- a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -93,10 +93,10 @@ drr/env_int.key = "DECORRELATE_DROPOUT" drr/env_int.default = 0 # dense model optimizer -# Learning rate is env-overridable via $DENSE_LR (default 1e-7). +# Learning rate is env-overridable via $DENSE_LR (default 1e-6). dense_optimizer_factory_and_class.learning_rate = @dlr/env_float() dlr/env_float.key = "DENSE_LR" -dlr/env_float.default = 0.0000001 +dlr/env_float.default = 0.000001 dense_optimizer_factory_and_class.optimizer_name = "Adam" dense_optimizer_factory_and_class.momentum = 0 dense_optimizer_factory_and_class.weight_decay = 0 @@ -104,10 +104,10 @@ dense_optimizer_factory_and_class.eps = 1e-8 dense_optimizer_factory_and_class.betas = (0.95, 0.999) # sparse model optimizer -# Learning rate is env-overridable via $SPARSE_LR (default 1e-7). +# Learning rate is env-overridable via $SPARSE_LR (default 1e-6). sparse_optimizer_factory_and_class.learning_rate = @slr/env_float() slr/env_float.key = "SPARSE_LR" -slr/env_float.default = 0.0000001 +slr/env_float.default = 0.000001 sparse_optimizer_factory_and_class.optimizer_name = "RowWiseAdagrad" sparse_optimizer_factory_and_class.momentum = 0 sparse_optimizer_factory_and_class.weight_decay = 0 @@ -167,7 +167,7 @@ gcn/env_float.default = 1.0 # via DENSE_LR / SPARSE_LR; warmup never overrides those base values. streaming_train_eval_loop.lr_warmup_steps = @lrw/env_int() lrw/env_int.key = "LR_WARMUP_STEPS" -lrw/env_int.default = 0 # 0 = OFF (disabled by default; baseline-safe) +lrw/env_int.default = 24000 # 24k-step LR warmup (CV-reduction default) # ---------------------------------------------------------------------------- # LR WARMUP FLOOR / STARTING LR (only meaningful when $LR_WARMUP_STEPS > 0) @@ -687,7 +687,7 @@ evn/env_int.default = 0 # trajectory can be plotted against data volume. Override via $EVAL_EVERY_DATA_PCT. streaming_train_eval_loop.eval_every_data_pct = @edp/env_float() edp/env_float.key = "EVAL_EVERY_DATA_PCT" -edp/env_float.default = 0.005 +edp/env_float.default = 0.001 # ---- SKIP_EVAL: warm-up eval skipping (early-run overhead cut) -------------- # WHAT IT DOES # Suppresses the FIRST N *scheduled/periodic* full-holdout eval passes. Early @@ -695,8 +695,8 @@ edp/env_float.default = 0.005 # carry little signal, yet each full-holdout pass is a real cost (a forward # sweep over the whole held-out set + its metric reduction). SKIP_EVAL lets you # pay that cost only once training has advanced enough to matter. -# N = $SKIP_EVAL (default 0 = OFF -> eval from the very first scheduled -# point; behavior identical to before this knob existed.) +# N = $SKIP_EVAL (default 10 -> skip the first 10 scheduled eval passes; +# set 0 to eval from the very first scheduled point.) # # HOW THE "FIRST N" ARE COUNTED (per cadence; only the active cadence applies) # Eval boundaries are numbered on the SAME grid the cadence already uses, and @@ -730,7 +730,7 @@ edp/env_float.default = 0.005 # Override via $SKIP_EVAL. streaming_train_eval_loop.skip_eval = @se/env_int() se/env_int.key = "SKIP_EVAL" -se/env_int.default = 0 +se/env_int.default = 10 # Double-buffer windows: prepare the next window (index mask + first-batch # prefetch) in a background thread during the current window's compute, hiding # the per-window reset. Needs persistent_loader=1. Override via env. From 28681cb52fd91c22a70e80adc3b830aef682114b Mon Sep 17 00:00:00 2001 From: suachong Date: Sun, 26 Jul 2026 10:54:08 -0500 Subject: [PATCH 124/127] dlrmv4: drop rcp_logs/.gitignore negation The 60 RCP reference logs are already tracked; remove the !*.log negation file now that the log set is final. Co-authored-by: Cursor --- recommendation_v4/rcp_logs/.gitignore | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 recommendation_v4/rcp_logs/.gitignore diff --git a/recommendation_v4/rcp_logs/.gitignore b/recommendation_v4/rcp_logs/.gitignore deleted file mode 100644 index 348996cfc..000000000 --- a/recommendation_v4/rcp_logs/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# RCP reference logs are intentionally tracked despite the parent *.log ignore -!*.log From de1be29a55235d644c1a2594ae0e9230fd34582e Mon Sep 17 00:00:00 2001 From: Chris Cai Date: Tue, 28 Jul 2026 00:01:39 +0000 Subject: [PATCH 125/127] dlrmv4 README: document RCP convergence sweep + 0.75 eval target Summarize the merged rcp_logs sweep (3 global batch sizes x 20 seeds), add samples- and wall-clock-to-converge at AUC>=0.75, switch the eval target to 0.75 throughout, clarify the fixed-start/warmup LR schedule and 0.1% eval frequency, and add a 10.1 tunable-hyperparameters subsection. Co-authored-by: Cursor --- recommendation_v4/README.MD | 60 +++++++++++++++++++++++++++++-------- 1 file changed, 48 insertions(+), 12 deletions(-) diff --git a/recommendation_v4/README.MD b/recommendation_v4/README.MD index e078bcf0a..fce13e049 100644 --- a/recommendation_v4/README.MD +++ b/recommendation_v4/README.MD @@ -33,7 +33,7 @@ fixed-duration windows and the model trains on window `T` then evaluates on the strictly-future window `T+1`, so every reported metric is genuine next-period generalization with no future leakage. The quality metric is **AUC** on the held-out future window, and the convergence target is -**AUC >= 0.80275** (matching the DLRM-DCNv2-style target). +**AUC >= 0.75**. The reference runs on 8 GPUs (validated on AMD Instinct MI350X / MI355X and NVIDIA B200; see [docs/training_recipe.md](docs/training_recipe.md)) and scales @@ -66,7 +66,7 @@ DLRM_DATA_PATH=/data/mlperf_dlrm_v4 ./run_and_time.sh [`md5sums_yambda_5b_processed.txt`](md5sums_yambda_5b_processed.txt) (falls back to a layout check until the canonical checksums are pinned). - [`run_and_time.sh`](run_and_time.sh) runs the full-reference streaming - train+eval sweep on a single 8-GPU host with `AUC_THRESHOLD=0.80275` and MLPerf + train+eval sweep on a single 8-GPU host with `AUC_THRESHOLD=0.75` and MLPerf compliance logging, printing the elapsed time of the timed region. ### 3.1 Multi-node (SLURM) @@ -84,7 +84,7 @@ sbatch --nodes=1 scripts/launch_slurm.sh START_TS=0 NUM_TRAIN_TS=299 \ NUM_TRAIN_BATCHES=0 NUM_EVAL_BATCHES=0 \ EVAL_EVERY_N_WINDOWS=0 EVAL_EVERY_DATA_PCT=0.005 \ -AUC_THRESHOLD=0.80275 \ +AUC_THRESHOLD=0.75 \ sbatch --nodes=2 scripts/launch_slurm.sh ``` @@ -285,7 +285,7 @@ the - **Metric**: AUC on the held-out future evaluation window (`window_auc` for the `listen_plus` task), computed by `MetricsLogger` in `generative_recommenders/dlrm_v3/utils.py`. -- **Target**: **eval AUC >= 0.80275**. Set via `AUC_THRESHOLD=0.80275` +- **Target**: **eval AUC >= 0.75**. Set via `AUC_THRESHOLD=0.75` (`MetricsLogger.auc_threshold`); the run logs `RUN_STOP` with `SUCCESS` and stops once the target is reached. The gin default of `1.0` is unreachable (trains all windows with no early stop) and is overridden by the reference @@ -304,15 +304,51 @@ next-period generalization. ## 10. Reference Convergence Points (RCP) -*Placeholder — to be generated.* - -RCPs have **not yet been generated** for this benchmark. Per the MLPerf +**Status: convergence logs generated for 3 batch sizes.** Per the MLPerf [CONTRIBUTING guidance](https://github.com/mlcommons/training_policies/blob/master/CONTRIBUTING.md), -RCPs must be generated for at least 3 reasonable batch sizes using at least 2N -seeds (N = number of submission runs), in FP32 or BF16, with the exact precision -recorded in the RCP JSON. The convergence curves (steps/samples to reach -AUC >= 0.80275) will be added under [`rcp/`](rcp/) once the convergence runs are -complete. This section is intentionally left blank for now. +RCPs must cover at least 3 reasonable batch sizes using at least 2N seeds +(N = number of submission runs). An initial sweep has now been produced and +merged under [`rcp_logs/`](rcp_logs/) — **3 global batch sizes × 20 seeds each +(60 runs total; 2N with N = 10)** in MLPerf MLLOG format +(`submission_org=AMD`, `submission_platform=MI350`, `submission_benchmark=hstu`, +`submission_division=closed`). All runs use `opt_name=Adam` (dense) + +`opt_sparse_name=RowWiseAdagrad`, `gradient_accumulation_steps=1`. The LR +schedule keeps the **starting LR fixed at `0`** and the **warmup steps fixed at +`24000`**; only the **target (post-warmup) LR scales with global batch size**, +linearly from `1e-6` (gbs 8192) to `4 * 1e-6` (gbs 32768) — i.e. `1e-6`, +`2e-6`, `4e-6` for the three batch sizes. Evaluation is run at a **fixed 0.1% +data eval frequency** (`EVAL_EVERY_DATA_PCT=0.001`) across all runs. + +**Eval target: `eval_accuracy` (AUC) `>= 0.75`.** All 60 runs cross this +threshold, so the sweep gives a complete samples-to-converge distribution at +every batch size: + +| Global batch size | Dir | Seeds | Seeds reaching AUC >= 0.75 | Samples-to-converge (min / mean / max) | Wall-clock to AUC 0.75 (min / mean / max) | +|---|---|---|---|---|---| +| 8192 | [`rcp_logs/gbs_8192/`](rcp_logs/gbs_8192/) | 20 | 20/20 | 61.9M / 69.3M / 80.3M | 2h22m / 2h39m / 2h59m | +| 16384 | [`rcp_logs/gbs_16384/`](rcp_logs/gbs_16384/) | 20 | 20/20 | 78.0M / 87.7M / 100.9M | 1h39m / 1h52m / 2h12m | +| 32768 | [`rcp_logs/gbs_32768/`](rcp_logs/gbs_32768/) | 20 | 20/20 | 98.6M / 113.0M / 128.5M | 1h17m / 1h23m / 1h36m | + +Each seed records the full `eval_accuracy` curve vs. `samples_count`; +"samples-to-converge" is the first eval crossing AUC `>= 0.75`. As expected, +samples-to-converge grows with global batch size. + +### 10.1 Tunable hyperparameters (benchmark submission) + +For a benchmark submission, only the following hyperparameters may be tuned: + +- **Target (post-warmup) learning rate** — the peak LR the warmup ramps up to. +- **Global batch size** — e.g. the 8192 / 16384 / 32768 points swept above. + +The rest of the LR schedule is **fixed and not tunable**: + +- **Starting LR = `0`** (the warmup always begins from zero). +- **Warmup steps = `24000`**. + + + + + ## 11. Streaming (temporal-order) training From 817e37927efaced0d38cc40adafcb0680ccdd1d7 Mon Sep 17 00:00:00 2001 From: Chris Cai Date: Tue, 28 Jul 2026 01:19:13 +0000 Subject: [PATCH 126/127] dlrmv4 README: add epoch-to-converge and CV columns to RCP table Co-authored-by: Cursor --- recommendation_v4/README.MD | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/recommendation_v4/README.MD b/recommendation_v4/README.MD index fce13e049..0e6557a93 100644 --- a/recommendation_v4/README.MD +++ b/recommendation_v4/README.MD @@ -323,15 +323,21 @@ data eval frequency** (`EVAL_EVERY_DATA_PCT=0.001`) across all runs. threshold, so the sweep gives a complete samples-to-converge distribution at every batch size: -| Global batch size | Dir | Seeds | Seeds reaching AUC >= 0.75 | Samples-to-converge (min / mean / max) | Wall-clock to AUC 0.75 (min / mean / max) | -|---|---|---|---|---|---| -| 8192 | [`rcp_logs/gbs_8192/`](rcp_logs/gbs_8192/) | 20 | 20/20 | 61.9M / 69.3M / 80.3M | 2h22m / 2h39m / 2h59m | -| 16384 | [`rcp_logs/gbs_16384/`](rcp_logs/gbs_16384/) | 20 | 20/20 | 78.0M / 87.7M / 100.9M | 1h39m / 1h52m / 2h12m | -| 32768 | [`rcp_logs/gbs_32768/`](rcp_logs/gbs_32768/) | 20 | 20/20 | 98.6M / 113.0M / 128.5M | 1h17m / 1h23m / 1h36m | +| Global batch size | Dir | Seeds | Seeds reaching AUC >= 0.75 | Samples-to-converge (min / mean / max) | CV (samples) | Epoch-to-converge (min / mean / max) | Wall-clock to AUC 0.75 (min / mean / max) | +|---|---|---|---|---|---|---|---| +| 8192 | [`rcp_logs/gbs_8192/`](rcp_logs/gbs_8192/) | 20 | 20/20 | 61.9M / 69.3M / 80.3M | 5.6% | 0.027 / 0.030 / 0.035 | 2h22m / 2h39m / 2h59m | +| 16384 | [`rcp_logs/gbs_16384/`](rcp_logs/gbs_16384/) | 20 | 20/20 | 78.0M / 87.7M / 100.9M | 7.2% | 0.034 / 0.038 / 0.044 | 1h39m / 1h52m / 2h12m | +| 32768 | [`rcp_logs/gbs_32768/`](rcp_logs/gbs_32768/) | 20 | 20/20 | 98.6M / 113.0M / 128.5M | 6.7% | 0.043 / 0.049 / 0.056 | 1h17m / 1h23m / 1h36m | Each seed records the full `eval_accuracy` curve vs. `samples_count`; -"samples-to-converge" is the first eval crossing AUC `>= 0.75`. As expected, -samples-to-converge grows with global batch size. +"samples-to-converge" is the first eval crossing AUC `>= 0.75`. Epoch-to-converge +is that sample count as a fraction of one full training epoch +(`train_samples = 2,290,835,423`, e.g. 50% of samples = 0.5 epoch), taken from +the log's `epoch_num`. As expected, samples- and epoch-to-converge grow with +global batch size (convergence is reached well within the first ~5% of an epoch). +"CV (samples)" is the coefficient of variation (sample stddev / mean) of +samples-to-converge across the 20 seeds — all under ~7%, i.e. low run-to-run +variance. ### 10.1 Tunable hyperparameters (benchmark submission) From aaa39e829a281069d1979819604737cb24222f11 Mon Sep 17 00:00:00 2001 From: Chris Cai Date: Tue, 28 Jul 2026 16:50:27 +0000 Subject: [PATCH 127/127] dlrmv4 README: note skip_eval=10 as a fixed submission hyperparameter Co-authored-by: Cursor --- recommendation_v4/README.MD | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/recommendation_v4/README.MD b/recommendation_v4/README.MD index 0e6557a93..fb6f65b7a 100644 --- a/recommendation_v4/README.MD +++ b/recommendation_v4/README.MD @@ -346,10 +346,12 @@ For a benchmark submission, only the following hyperparameters may be tuned: - **Target (post-warmup) learning rate** — the peak LR the warmup ramps up to. - **Global batch size** — e.g. the 8192 / 16384 / 32768 points swept above. -The rest of the LR schedule is **fixed and not tunable**: +The rest of the parameters, including the following, are **fixed and not tunable**: - **Starting LR = `0`** (the warmup always begins from zero). - **Warmup steps = `24000`**. +- **`skip_eval = 10`** — the first 10 eval rounds are skipped before convergence + is checked; fixed for submission.